arch/x86/kvm/mmu.c

   1 /*
   2  * Kernel-based Virtual Machine driver for Linux
   3  *
   4  * This module enables machines with Intel VT-x extensions to run virtual
   5  * machines without emulation or binary translation.
   6  *
   7  * MMU support
   8  *
   9  * Copyright (C) 2006 Qumranet, Inc.
  10  * Copyright 2010 Red Hat, Inc. and/or its affilates.
  11  *
  12  * Authors:
  13  *   Yaniv Kamay  <yaniv@qumranet.com>
  14  *   Avi Kivity   <avi@qumranet.com>
  15  *
  16  * This work is licensed under the terms of the GNU GPL, version 2.  See
  17  * the COPYING file in the top-level directory.
  18  *
  19  */
  20
  21 #include "mmu.h"
  22 #include "x86.h"
  23 #include "kvm_cache_regs.h"
  24
  25 #include <linux/kvm_host.h>
  26 #include <linux/types.h>
  27 #include <linux/string.h>
  28 #include <linux/mm.h>
  29 #include <linux/highmem.h>
  30 #include <linux/module.h>
  31 #include <linux/swap.h>
  32 #include <linux/hugetlb.h>
  33 #include <linux/compiler.h>
  34 #include <linux/srcu.h>
  35 #include <linux/slab.h>
  36 #include <linux/uaccess.h>
  37
  38 #include <asm/page.h>
  39 #include <asm/cmpxchg.h>
  40 #include <asm/io.h>
  41 #include <asm/vmx.h>
  42
  43 /*
  44  * When setting this variable to true it enables Two-Dimensional-Paging
  45  * where the hardware walks 2 page tables:
  46  * 1. the guest-virtual to guest-physical
  47  * 2. while doing 1. it walks guest-physical to host-physical
  48  * If the hardware supports that we don't need to do shadow paging.
  49  */
  50 bool tdp_enabled = false;
  51
  52 enum {
  53         AUDIT_PRE_PAGE_FAULT,
  54         AUDIT_POST_PAGE_FAULT,
  55         AUDIT_PRE_PTE_WRITE,
  56         AUDIT_POST_PTE_WRITE,
  57         AUDIT_PRE_SYNC,
  58         AUDIT_POST_SYNC
  59 };
  60
  61 char *audit_point_name[] = {
  62         "pre page fault",
  63         "post page fault",
  64         "pre pte write",
  65         "post pte write",
  66         "pre sync",
  67         "post sync"
  68 };
  69
  70 #undef MMU_DEBUG
  71
  72 #ifdef MMU_DEBUG
  73
  74 #define pgprintk(x...) do { if (dbg) printk(x); } while (0)
  75 #define rmap_printk(x...) do { if (dbg) printk(x); } while (0)
  76
  77 #else
  78
  79 #define pgprintk(x...) do { } while (0)
  80 #define rmap_printk(x...) do { } while (0)
  81
  82 #endif
  83
  84 #ifdef MMU_DEBUG
  85 static int dbg = 0;
  86 module_param(dbg, bool, 0644);
  87 #endif
  88
  89 static int oos_shadow = 1;
  90 module_param(oos_shadow, bool, 0644);
  91
  92 #ifndef MMU_DEBUG
  93 #define ASSERT(x) do { } while (0)
  94 #else
  95 #define ASSERT(x)                                                       \
  96         if (!(x)) {                                                     \
  97                 printk(KERN_WARNING "assertion failed %s:%d: %s\n",     \
  98                        __FILE__, __LINE__, #x);                         \
  99         }
 100 #endif
 101
 102 #define PTE_PREFETCH_NUM                8
 103
 104 #define PT_FIRST_AVAIL_BITS_SHIFT 9
 105 #define PT64_SECOND_AVAIL_BITS_SHIFT 52
 106
 107 #define PT64_LEVEL_BITS 9
 108
 109 #define PT64_LEVEL_SHIFT(level) \
 110                 (PAGE_SHIFT + (level - 1) * PT64_LEVEL_BITS)
 111
 112 #define PT64_LEVEL_MASK(level) \
 113                 (((1ULL << PT64_LEVEL_BITS) - 1) << PT64_LEVEL_SHIFT(level))
 114
 115 #define PT64_INDEX(address, level)\
 116         (((address) >> PT64_LEVEL_SHIFT(level)) & ((1 << PT64_LEVEL_BITS) - 1))
 117
 118
 119 #define PT32_LEVEL_BITS 10
 120
 121 #define PT32_LEVEL_SHIFT(level) \
 122                 (PAGE_SHIFT + (level - 1) * PT32_LEVEL_BITS)
 123
 124 #define PT32_LEVEL_MASK(level) \
 125                 (((1ULL << PT32_LEVEL_BITS) - 1) << PT32_LEVEL_SHIFT(level))
 126 #define PT32_LVL_OFFSET_MASK(level) \
 127         (PT32_BASE_ADDR_MASK & ((1ULL << (PAGE_SHIFT + (((level) - 1) \
 128                                                 * PT32_LEVEL_BITS))) - 1))
 129
 130 #define PT32_INDEX(address, level)\
 131         (((address) >> PT32_LEVEL_SHIFT(level)) & ((1 << PT32_LEVEL_BITS) - 1))
 132
 133
 134 #define PT64_BASE_ADDR_MASK (((1ULL << 52) - 1) & ~(u64)(PAGE_SIZE-1))
 135 #define PT64_DIR_BASE_ADDR_MASK \
 136         (PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + PT64_LEVEL_BITS)) - 1))
 137 #define PT64_LVL_ADDR_MASK(level) \
 138         (PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + (((level) - 1) \
 139                                                 * PT64_LEVEL_BITS))) - 1))
 140 #define PT64_LVL_OFFSET_MASK(level) \
 141         (PT64_BASE_ADDR_MASK & ((1ULL << (PAGE_SHIFT + (((level) - 1) \
 142                                                 * PT64_LEVEL_BITS))) - 1))
 143
 144 #define PT32_BASE_ADDR_MASK PAGE_MASK
 145 #define PT32_DIR_BASE_ADDR_MASK \
 146         (PAGE_MASK & ~((1ULL << (PAGE_SHIFT + PT32_LEVEL_BITS)) - 1))
 147 #define PT32_LVL_ADDR_MASK(level) \
 148         (PAGE_MASK & ~((1ULL << (PAGE_SHIFT + (((level) - 1) \
 149                                             * PT32_LEVEL_BITS))) - 1))
 150
 151 #define PT64_PERM_MASK (PT_PRESENT_MASK | PT_WRITABLE_MASK | PT_USER_MASK \
 152                         | PT64_NX_MASK)
 153
 154 #define RMAP_EXT 4
 155
 156 #define ACC_EXEC_MASK    1
 157 #define ACC_WRITE_MASK   PT_WRITABLE_MASK
 158 #define ACC_USER_MASK    PT_USER_MASK
 159 #define ACC_ALL          (ACC_EXEC_MASK | ACC_WRITE_MASK | ACC_USER_MASK)
 160
 161 #include <trace/events/kvm.h>
 162
 163 #define CREATE_TRACE_POINTS
 164 #include "mmutrace.h"
 165
 166 #define SPTE_HOST_WRITEABLE (1ULL << PT_FIRST_AVAIL_BITS_SHIFT)
 167
 168 #define SHADOW_PT_INDEX(addr, level) PT64_INDEX(addr, level)
 169
 170 struct kvm_rmap_desc {
 171         u64 *sptes[RMAP_EXT];
 172         struct kvm_rmap_desc *more;
 173 };
 174
 175 struct kvm_shadow_walk_iterator {
 176         u64 addr;
 177         hpa_t shadow_addr;
 178         int level;
 179         u64 *sptep;
 180         unsigned index;
 181 };
 182
 183 #define for_each_shadow_entry(_vcpu, _addr, _walker)    \
 184         for (shadow_walk_init(&(_walker), _vcpu, _addr);        \
 185              shadow_walk_okay(&(_walker));                      \
 186              shadow_walk_next(&(_walker)))
 187
 188 typedef void (*mmu_parent_walk_fn) (struct kvm_mmu_page *sp, u64 *spte);
 189
 190 static struct kmem_cache *pte_chain_cache;
 191 static struct kmem_cache *rmap_desc_cache;
 192 static struct kmem_cache *mmu_page_header_cache;
 193 static struct percpu_counter kvm_total_used_mmu_pages;
 194
 195 static u64 __read_mostly shadow_trap_nonpresent_pte;
 196 static u64 __read_mostly shadow_notrap_nonpresent_pte;
 197 static u64 __read_mostly shadow_base_present_pte;
 198 static u64 __read_mostly shadow_nx_mask;
 199 static u64 __read_mostly shadow_x_mask; /* mutual exclusive with nx_mask */
 200 static u64 __read_mostly shadow_user_mask;
 201 static u64 __read_mostly shadow_accessed_mask;
 202 static u64 __read_mostly shadow_dirty_mask;
 203
 204 static inline u64 rsvd_bits(int s, int e)
 205 {
 206         return ((1ULL << (e - s + 1)) - 1) << s;
 207 }
 208
 209 void kvm_mmu_set_nonpresent_ptes(u64 trap_pte, u64 notrap_pte)
 210 {
 211         shadow_trap_nonpresent_pte = trap_pte;
 212         shadow_notrap_nonpresent_pte = notrap_pte;
 213 }
 214 EXPORT_SYMBOL_GPL(kvm_mmu_set_nonpresent_ptes);
 215
 216 void kvm_mmu_set_base_ptes(u64 base_pte)
 217 {
 218         shadow_base_present_pte = base_pte;
 219 }
 220 EXPORT_SYMBOL_GPL(kvm_mmu_set_base_ptes);
 221
 222 void kvm_mmu_set_mask_ptes(u64 user_mask, u64 accessed_mask,
 223                 u64 dirty_mask, u64 nx_mask, u64 x_mask)
 224 {
 225         shadow_user_mask = user_mask;
 226         shadow_accessed_mask = accessed_mask;
 227         shadow_dirty_mask = dirty_mask;
 228         shadow_nx_mask = nx_mask;
 229         shadow_x_mask = x_mask;
 230 }
 231 EXPORT_SYMBOL_GPL(kvm_mmu_set_mask_ptes);
 232
 233 static bool is_write_protection(struct kvm_vcpu *vcpu)
 234 {
 235         return kvm_read_cr0_bits(vcpu, X86_CR0_WP);
 236 }
 237
 238 static int is_cpuid_PSE36(void)
 239 {
 240         return 1;
 241 }
 242
 243 static int is_nx(struct kvm_vcpu *vcpu)
 244 {
 245         return vcpu->arch.efer & EFER_NX;
 246 }
 247
 248 static int is_shadow_present_pte(u64 pte)
 249 {
 250         return pte != shadow_trap_nonpresent_pte
 251                 && pte != shadow_notrap_nonpresent_pte;
 252 }
 253
 254 static int is_large_pte(u64 pte)
 255 {
 256         return pte & PT_PAGE_SIZE_MASK;
 257 }
 258
 259 static int is_writable_pte(unsigned long pte)
 260 {
 261         return pte & PT_WRITABLE_MASK;
 262 }
 263
 264 static int is_dirty_gpte(unsigned long pte)
 265 {
 266         return pte & PT_DIRTY_MASK;
 267 }
 268
 269 static int is_rmap_spte(u64 pte)
 270 {
 271         return is_shadow_present_pte(pte);
 272 }
 273
 274 static int is_last_spte(u64 pte, int level)
 275 {
 276         if (level == PT_PAGE_TABLE_LEVEL)
 277                 return 1;
 278         if (is_large_pte(pte))
 279                 return 1;
 280         return 0;
 281 }
 282
 283 static pfn_t spte_to_pfn(u64 pte)
 284 {
 285         return (pte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT;
 286 }
 287
 288 static gfn_t pse36_gfn_delta(u32 gpte)
 289 {
 290         int shift = 32 - PT32_DIR_PSE36_SHIFT - PAGE_SHIFT;
 291
 292         return (gpte & PT32_DIR_PSE36_MASK) << shift;
 293 }
 294
 295 static void __set_spte(u64 *sptep, u64 spte)
 296 {
 297         set_64bit(sptep, spte);
 298 }
 299
 300 static u64 __xchg_spte(u64 *sptep, u64 new_spte)
 301 {
 302 #ifdef CONFIG_X86_64
 303         return xchg(sptep, new_spte);
 304 #else
 305         u64 old_spte;
 306
 307         do {
 308                 old_spte = *sptep;
 309         } while (cmpxchg64(sptep, old_spte, new_spte) != old_spte);
 310
 311         return old_spte;
 312 #endif
 313 }
 314
 315 static bool spte_has_volatile_bits(u64 spte)
 316 {
 317         if (!shadow_accessed_mask)
 318                 return false;
 319
 320         if (!is_shadow_present_pte(spte))
 321                 return false;
 322
 323         if ((spte & shadow_accessed_mask) &&
 324               (!is_writable_pte(spte) || (spte & shadow_dirty_mask)))
 325                 return false;
 326
 327         return true;
 328 }
 329
 330 static bool spte_is_bit_cleared(u64 old_spte, u64 new_spte, u64 bit_mask)
 331 {
 332         return (old_spte & bit_mask) && !(new_spte & bit_mask);
 333 }
 334
 335 static void update_spte(u64 *sptep, u64 new_spte)
 336 {
 337         u64 mask, old_spte = *sptep;
 338
 339         WARN_ON(!is_rmap_spte(new_spte));
 340
 341         new_spte |= old_spte & shadow_dirty_mask;
 342
 343         mask = shadow_accessed_mask;
 344         if (is_writable_pte(old_spte))
 345                 mask |= shadow_dirty_mask;
 346
 347         if (!spte_has_volatile_bits(old_spte) || (new_spte & mask) == mask)
 348                 __set_spte(sptep, new_spte);
 349         else
 350                 old_spte = __xchg_spte(sptep, new_spte);
 351
 352         if (!shadow_accessed_mask)
 353                 return;
 354
 355         if (spte_is_bit_cleared(old_spte, new_spte, shadow_accessed_mask))
 356                 kvm_set_pfn_accessed(spte_to_pfn(old_spte));
 357         if (spte_is_bit_cleared(old_spte, new_spte, shadow_dirty_mask))
 358                 kvm_set_pfn_dirty(spte_to_pfn(old_spte));
 359 }
 360
 361 static int mmu_topup_memory_cache(struct kvm_mmu_memory_cache *cache,
 362                                   struct kmem_cache *base_cache, int min)
 363 {
 364         void *obj;
 365
 366         if (cache->nobjs >= min)
 367                 return 0;
 368         while (cache->nobjs < ARRAY_SIZE(cache->objects)) {
 369                 obj = kmem_cache_zalloc(base_cache, GFP_KERNEL);
 370                 if (!obj)
 371                         return -ENOMEM;
 372                 cache->objects[cache->nobjs++] = obj;
 373         }
 374         return 0;
 375 }
 376
 377 static void mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc,
 378                                   struct kmem_cache *cache)
 379 {
 380         while (mc->nobjs)
 381                 kmem_cache_free(cache, mc->objects[--mc->nobjs]);
 382 }
 383
 384 static int mmu_topup_memory_cache_page(struct kvm_mmu_memory_cache *cache,
 385                                        int min)
 386 {
 387         struct page *page;
 388
 389         if (cache->nobjs >= min)
 390                 return 0;
 391         while (cache->nobjs < ARRAY_SIZE(cache->objects)) {
 392                 page = alloc_page(GFP_KERNEL);
 393                 if (!page)
 394                         return -ENOMEM;
 395                 cache->objects[cache->nobjs++] = page_address(page);
 396         }
 397         return 0;
 398 }
 399
 400 static void mmu_free_memory_cache_page(struct kvm_mmu_memory_cache *mc)
 401 {
 402         while (mc->nobjs)
 403                 free_page((unsigned long)mc->objects[--mc->nobjs]);
 404 }
 405
 406 static int mmu_topup_memory_caches(struct kvm_vcpu *vcpu)
 407 {
 408         int r;
 409
 410         r = mmu_topup_memory_cache(&vcpu->arch.mmu_pte_chain_cache,
 411                                    pte_chain_cache, 4);
 412         if (r)
 413                 goto out;
 414         r = mmu_topup_memory_cache(&vcpu->arch.mmu_rmap_desc_cache,
 415                                    rmap_desc_cache, 4 + PTE_PREFETCH_NUM);
 416         if (r)
 417                 goto out;
 418         r = mmu_topup_memory_cache_page(&vcpu->arch.mmu_page_cache, 8);
 419         if (r)
 420                 goto out;
 421         r = mmu_topup_memory_cache(&vcpu->arch.mmu_page_header_cache,
 422                                    mmu_page_header_cache, 4);
 423 out:
 424         return r;
 425 }
 426
 427 static void mmu_free_memory_caches(struct kvm_vcpu *vcpu)
 428 {
 429         mmu_free_memory_cache(&vcpu->arch.mmu_pte_chain_cache, pte_chain_cache);
 430         mmu_free_memory_cache(&vcpu->arch.mmu_rmap_desc_cache, rmap_desc_cache);
 431         mmu_free_memory_cache_page(&vcpu->arch.mmu_page_cache);
 432         mmu_free_memory_cache(&vcpu->arch.mmu_page_header_cache,
 433                                 mmu_page_header_cache);
 434 }
 435
 436 static void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc,
 437                                     size_t size)
 438 {
 439         void *p;
 440
 441         BUG_ON(!mc->nobjs);
 442         p = mc->objects[--mc->nobjs];
 443         return p;
 444 }
 445
 446 static struct kvm_pte_chain *mmu_alloc_pte_chain(struct kvm_vcpu *vcpu)
 447 {
 448         return mmu_memory_cache_alloc(&vcpu->arch.mmu_pte_chain_cache,
 449                                       sizeof(struct kvm_pte_chain));
 450 }
 451
 452 static void mmu_free_pte_chain(struct kvm_pte_chain *pc)
 453 {
 454         kmem_cache_free(pte_chain_cache, pc);
 455 }
 456
 457 static struct kvm_rmap_desc *mmu_alloc_rmap_desc(struct kvm_vcpu *vcpu)
 458 {
 459         return mmu_memory_cache_alloc(&vcpu->arch.mmu_rmap_desc_cache,
 460                                       sizeof(struct kvm_rmap_desc));
 461 }
 462
 463 static void mmu_free_rmap_desc(struct kvm_rmap_desc *rd)
 464 {
 465         kmem_cache_free(rmap_desc_cache, rd);
 466 }
 467
 468 static gfn_t kvm_mmu_page_get_gfn(struct kvm_mmu_page *sp, int index)
 469 {
 470         if (!sp->role.direct)
 471                 return sp->gfns[index];
 472
 473         return sp->gfn + (index << ((sp->role.level - 1) * PT64_LEVEL_BITS));
 474 }
 475
 476 static void kvm_mmu_page_set_gfn(struct kvm_mmu_page *sp, int index, gfn_t gfn)
 477 {
 478         if (sp->role.direct)
 479                 BUG_ON(gfn != kvm_mmu_page_get_gfn(sp, index));
 480         else
 481                 sp->gfns[index] = gfn;
 482 }
 483
 484 /*
 485  * Return the pointer to the largepage write count for a given
 486  * gfn, handling slots that are not large page aligned.
 487  */
 488 static int *slot_largepage_idx(gfn_t gfn,
 489                                struct kvm_memory_slot *slot,
 490                                int level)
 491 {
 492         unsigned long idx;
 493
 494         idx = (gfn >> KVM_HPAGE_GFN_SHIFT(level)) -
 495               (slot->base_gfn >> KVM_HPAGE_GFN_SHIFT(level));
 496         return &slot->lpage_info[level - 2][idx].write_count;
 497 }
 498
 499 static void account_shadowed(struct kvm *kvm, gfn_t gfn)
 500 {
 501         struct kvm_memory_slot *slot;
 502         int *write_count;
 503         int i;
 504
 505         slot = gfn_to_memslot(kvm, gfn);
 506         for (i = PT_DIRECTORY_LEVEL;
 507              i < PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES; ++i) {
 508                 write_count   = slot_largepage_idx(gfn, slot, i);
 509                 *write_count += 1;
 510         }
 511 }
 512
 513 static void unaccount_shadowed(struct kvm *kvm, gfn_t gfn)
 514 {
 515         struct kvm_memory_slot *slot;
 516         int *write_count;
 517         int i;
 518
 519         slot = gfn_to_memslot(kvm, gfn);
 520         for (i = PT_DIRECTORY_LEVEL;
 521              i < PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES; ++i) {
 522                 write_count   = slot_largepage_idx(gfn, slot, i);
 523                 *write_count -= 1;
 524                 WARN_ON(*write_count < 0);
 525         }
 526 }
 527
 528 static int has_wrprotected_page(struct kvm *kvm,
 529                                 gfn_t gfn,
 530                                 int level)
 531 {
 532         struct kvm_memory_slot *slot;
 533         int *largepage_idx;
 534
 535         slot = gfn_to_memslot(kvm, gfn);
 536         if (slot) {
 537                 largepage_idx = slot_largepage_idx(gfn, slot, level);
 538                 return *largepage_idx;
 539         }
 540
 541         return 1;
 542 }
 543
 544 static int host_mapping_level(struct kvm *kvm, gfn_t gfn)
 545 {
 546         unsigned long page_size;
 547         int i, ret = 0;
 548
 549         page_size = kvm_host_page_size(kvm, gfn);
 550
 551         for (i = PT_PAGE_TABLE_LEVEL;
 552              i < (PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES); ++i) {
 553                 if (page_size >= KVM_HPAGE_SIZE(i))
 554                         ret = i;
 555                 else
 556                         break;
 557         }
 558
 559         return ret;
 560 }
 561
 562 static int mapping_level(struct kvm_vcpu *vcpu, gfn_t large_gfn)
 563 {
 564         struct kvm_memory_slot *slot;
 565         int host_level, level, max_level;
 566
 567         slot = gfn_to_memslot(vcpu->kvm, large_gfn);
 568         if (slot && slot->dirty_bitmap)
 569                 return PT_PAGE_TABLE_LEVEL;
 570
 571         host_level = host_mapping_level(vcpu->kvm, large_gfn);
 572
 573         if (host_level == PT_PAGE_TABLE_LEVEL)
 574                 return host_level;
 575
 576         max_level = kvm_x86_ops->get_lpage_level() < host_level ?
 577                 kvm_x86_ops->get_lpage_level() : host_level;
 578
 579         for (level = PT_DIRECTORY_LEVEL; level <= max_level; ++level)
 580                 if (has_wrprotected_page(vcpu->kvm, large_gfn, level))
 581                         break;
 582
 583         return level - 1;
 584 }
 585
 586 /*
 587  * Take gfn and return the reverse mapping to it.
 588  */
 589
 590 static unsigned long *gfn_to_rmap(struct kvm *kvm, gfn_t gfn, int level)
 591 {
 592         struct kvm_memory_slot *slot;
 593         unsigned long idx;
 594
 595         slot = gfn_to_memslot(kvm, gfn);
 596         if (likely(level == PT_PAGE_TABLE_LEVEL))
 597                 return &slot->rmap[gfn - slot->base_gfn];
 598
 599         idx = (gfn >> KVM_HPAGE_GFN_SHIFT(level)) -
 600                 (slot->base_gfn >> KVM_HPAGE_GFN_SHIFT(level));
 601
 602         return &slot->lpage_info[level - 2][idx].rmap_pde;
 603 }
 604
 605 /*
 606  * Reverse mapping data structures:
 607  *
 608  * If rmapp bit zero is zero, then rmapp point to the shadw page table entry
 609  * that points to page_address(page).
 610  *
 611  * If rmapp bit zero is one, (then rmap & ~1) points to a struct kvm_rmap_desc
 612  * containing more mappings.
 613  *
 614  * Returns the number of rmap entries before the spte was added or zero if
 615  * the spte was not added.
 616  *
 617  */
 618 static int rmap_add(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn)
 619 {
 620         struct kvm_mmu_page *sp;
 621         struct kvm_rmap_desc *desc;
 622         unsigned long *rmapp;
 623         int i, count = 0;
 624
 625         if (!is_rmap_spte(*spte))
 626                 return count;
 627         sp = page_header(__pa(spte));
 628         kvm_mmu_page_set_gfn(sp, spte - sp->spt, gfn);
 629         rmapp = gfn_to_rmap(vcpu->kvm, gfn, sp->role.level);
 630         if (!*rmapp) {
 631                 rmap_printk("rmap_add: %p %llx 0->1\n", spte, *spte);
 632                 *rmapp = (unsigned long)spte;
 633         } else if (!(*rmapp & 1)) {
 634                 rmap_printk("rmap_add: %p %llx 1->many\n", spte, *spte);
 635                 desc = mmu_alloc_rmap_desc(vcpu);
 636                 desc->sptes[0] = (u64 *)*rmapp;
 637                 desc->sptes[1] = spte;
 638                 *rmapp = (unsigned long)desc | 1;
 639                 ++count;
 640         } else {
 641                 rmap_printk("rmap_add: %p %llx many->many\n", spte, *spte);
 642                 desc = (struct kvm_rmap_desc *)(*rmapp & ~1ul);
 643                 while (desc->sptes[RMAP_EXT-1] && desc->more) {
 644                         desc = desc->more;
 645                         count += RMAP_EXT;
 646                 }
 647                 if (desc->sptes[RMAP_EXT-1]) {
 648                         desc->more = mmu_alloc_rmap_desc(vcpu);
 649                         desc = desc->more;
 650                 }
 651                 for (i = 0; desc->sptes[i]; ++i)
 652                         ++count;
 653                 desc->sptes[i] = spte;
 654         }
 655         return count;
 656 }
 657
 658 static void rmap_desc_remove_entry(unsigned long *rmapp,
 659                                    struct kvm_rmap_desc *desc,
 660                                    int i,
 661                                    struct kvm_rmap_desc *prev_desc)
 662 {
 663         int j;
 664
 665         for (j = RMAP_EXT - 1; !desc->sptes[j] && j > i; --j)
 666                 ;
 667         desc->sptes[i] = desc->sptes[j];
 668         desc->sptes[j] = NULL;
 669         if (j != 0)
 670                 return;
 671         if (!prev_desc && !desc->more)
 672                 *rmapp = (unsigned long)desc->sptes[0];
 673         else
 674                 if (prev_desc)
 675                         prev_desc->more = desc->more;
 676                 else
 677                         *rmapp = (unsigned long)desc->more | 1;
 678         mmu_free_rmap_desc(desc);
 679 }
 680
 681 static void rmap_remove(struct kvm *kvm, u64 *spte)
 682 {
 683         struct kvm_rmap_desc *desc;
 684         struct kvm_rmap_desc *prev_desc;
 685         struct kvm_mmu_page *sp;
 686         gfn_t gfn;
 687         unsigned long *rmapp;
 688         int i;
 689
 690         sp = page_header(__pa(spte));
 691         gfn = kvm_mmu_page_get_gfn(sp, spte - sp->spt);
 692         rmapp = gfn_to_rmap(kvm, gfn, sp->role.level);
 693         if (!*rmapp) {
 694                 printk(KERN_ERR "rmap_remove: %p 0->BUG\n", spte);
 695                 BUG();
 696         } else if (!(*rmapp & 1)) {
 697                 rmap_printk("rmap_remove:  %p 1->0\n", spte);
 698                 if ((u64 *)*rmapp != spte) {
 699                         printk(KERN_ERR "rmap_remove:  %p 1->BUG\n", spte);
 700                         BUG();
 701                 }
 702                 *rmapp = 0;
 703         } else {
 704                 rmap_printk("rmap_remove:  %p many->many\n", spte);
 705                 desc = (struct kvm_rmap_desc *)(*rmapp & ~1ul);
 706                 prev_desc = NULL;
 707                 while (desc) {
 708                         for (i = 0; i < RMAP_EXT && desc->sptes[i]; ++i)
 709                                 if (desc->sptes[i] == spte) {
 710                                         rmap_desc_remove_entry(rmapp,
 711                                                                desc, i,
 712                                                                prev_desc);
 713                                         return;
 714                                 }
 715                         prev_desc = desc;
 716                         desc = desc->more;
 717                 }
 718                 pr_err("rmap_remove: %p many->many\n", spte);
 719                 BUG();
 720         }
 721 }
 722
 723 static void set_spte_track_bits(u64 *sptep, u64 new_spte)
 724 {
 725         pfn_t pfn;
 726         u64 old_spte = *sptep;
 727
 728         if (!spte_has_volatile_bits(old_spte))
 729                 __set_spte(sptep, new_spte);
 730         else
 731                 old_spte = __xchg_spte(sptep, new_spte);
 732
 733         if (!is_rmap_spte(old_spte))
 734                 return;
 735
 736         pfn = spte_to_pfn(old_spte);
 737         if (!shadow_accessed_mask || old_spte & shadow_accessed_mask)
 738                 kvm_set_pfn_accessed(pfn);
 739         if (!shadow_dirty_mask || (old_spte & shadow_dirty_mask))
 740                 kvm_set_pfn_dirty(pfn);
 741 }
 742
 743 static void drop_spte(struct kvm *kvm, u64 *sptep, u64 new_spte)
 744 {
 745         set_spte_track_bits(sptep, new_spte);
 746         rmap_remove(kvm, sptep);
 747 }
 748
 749 static u64 *rmap_next(struct kvm *kvm, unsigned long *rmapp, u64 *spte)
 750 {
 751         struct kvm_rmap_desc *desc;
 752         u64 *prev_spte;
 753         int i;
 754
 755         if (!*rmapp)
 756                 return NULL;
 757         else if (!(*rmapp & 1)) {
 758                 if (!spte)
 759                         return (u64 *)*rmapp;
 760                 return NULL;
 761         }
 762         desc = (struct kvm_rmap_desc *)(*rmapp & ~1ul);
 763         prev_spte = NULL;
 764         while (desc) {
 765                 for (i = 0; i < RMAP_EXT && desc->sptes[i]; ++i) {
 766                         if (prev_spte == spte)
 767                                 return desc->sptes[i];
 768                         prev_spte = desc->sptes[i];
 769                 }
 770                 desc = desc->more;
 771         }
 772         return NULL;
 773 }
 774
 775 static int rmap_write_protect(struct kvm *kvm, u64 gfn)
 776 {
 777         unsigned long *rmapp;
 778         u64 *spte;
 779         int i, write_protected = 0;
 780
 781         rmapp = gfn_to_rmap(kvm, gfn, PT_PAGE_TABLE_LEVEL);
 782
 783         spte = rmap_next(kvm, rmapp, NULL);
 784         while (spte) {
 785                 BUG_ON(!spte);
 786                 BUG_ON(!(*spte & PT_PRESENT_MASK));
 787                 rmap_printk("rmap_write_protect: spte %p %llx\n", spte, *spte);
 788                 if (is_writable_pte(*spte)) {
 789                         update_spte(spte, *spte & ~PT_WRITABLE_MASK);
 790                         write_protected = 1;
 791                 }
 792                 spte = rmap_next(kvm, rmapp, spte);
 793         }
 794
 795         /* check for huge page mappings */
 796         for (i = PT_DIRECTORY_LEVEL;
 797              i < PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES; ++i) {
 798                 rmapp = gfn_to_rmap(kvm, gfn, i);
 799                 spte = rmap_next(kvm, rmapp, NULL);
 800                 while (spte) {
 801                         BUG_ON(!spte);
 802                         BUG_ON(!(*spte & PT_PRESENT_MASK));
 803                         BUG_ON((*spte & (PT_PAGE_SIZE_MASK|PT_PRESENT_MASK)) != (PT_PAGE_SIZE_MASK|PT_PRESENT_MASK));
 804                         pgprintk("rmap_write_protect(large): spte %p %llx %lld\n", spte, *spte, gfn);
 805                         if (is_writable_pte(*spte)) {
 806                                 drop_spte(kvm, spte,
 807                                           shadow_trap_nonpresent_pte);
 808                                 --kvm->stat.lpages;
 809                                 spte = NULL;
 810                                 write_protected = 1;
 811                         }
 812                         spte = rmap_next(kvm, rmapp, spte);
 813                 }
 814         }
 815
 816         return write_protected;
 817 }
 818
 819 static int kvm_unmap_rmapp(struct kvm *kvm, unsigned long *rmapp,
 820                            unsigned long data)
 821 {
 822         u64 *spte;
 823         int need_tlb_flush = 0;
 824
 825         while ((spte = rmap_next(kvm, rmapp, NULL))) {
 826                 BUG_ON(!(*spte & PT_PRESENT_MASK));
 827                 rmap_printk("kvm_rmap_unmap_hva: spte %p %llx\n", spte, *spte);
 828                 drop_spte(kvm, spte, shadow_trap_nonpresent_pte);
 829                 need_tlb_flush = 1;
 830         }
 831         return need_tlb_flush;
 832 }
 833
 834 static int kvm_set_pte_rmapp(struct kvm *kvm, unsigned long *rmapp,
 835                              unsigned long data)
 836 {
 837         int need_flush = 0;
 838         u64 *spte, new_spte;
 839         pte_t *ptep = (pte_t *)data;
 840         pfn_t new_pfn;
 841
 842         WARN_ON(pte_huge(*ptep));
 843         new_pfn = pte_pfn(*ptep);
 844         spte = rmap_next(kvm, rmapp, NULL);
 845         while (spte) {
 846                 BUG_ON(!is_shadow_present_pte(*spte));
 847                 rmap_printk("kvm_set_pte_rmapp: spte %p %llx\n", spte, *spte);
 848                 need_flush = 1;
 849                 if (pte_write(*ptep)) {
 850                         drop_spte(kvm, spte, shadow_trap_nonpresent_pte);
 851                         spte = rmap_next(kvm, rmapp, NULL);
 852                 } else {
 853                         new_spte = *spte &~ (PT64_BASE_ADDR_MASK);
 854                         new_spte |= (u64)new_pfn << PAGE_SHIFT;
 855
 856                         new_spte &= ~PT_WRITABLE_MASK;
 857                         new_spte &= ~SPTE_HOST_WRITEABLE;
 858                         new_spte &= ~shadow_accessed_mask;
 859                         set_spte_track_bits(spte, new_spte);
 860                         spte = rmap_next(kvm, rmapp, spte);
 861                 }
 862         }
 863         if (need_flush)
 864                 kvm_flush_remote_tlbs(kvm);
 865
 866         return 0;
 867 }
 868
 869 static int kvm_handle_hva(struct kvm *kvm, unsigned long hva,
 870                           unsigned long data,
 871                           int (*handler)(struct kvm *kvm, unsigned long *rmapp,
 872                                          unsigned long data))
 873 {
 874         int i, j;
 875         int ret;
 876         int retval = 0;
 877         struct kvm_memslots *slots;
 878
 879         slots = kvm_memslots(kvm);
 880
 881         for (i = 0; i < slots->nmemslots; i++) {
 882                 struct kvm_memory_slot *memslot = &slots->memslots[i];
 883                 unsigned long start = memslot->userspace_addr;
 884                 unsigned long end;
 885
 886                 end = start + (memslot->npages << PAGE_SHIFT);
 887                 if (hva >= start && hva < end) {
 888                         gfn_t gfn_offset = (hva - start) >> PAGE_SHIFT;
 889
 890                         ret = handler(kvm, &memslot->rmap[gfn_offset], data);
 891
 892                         for (j = 0; j < KVM_NR_PAGE_SIZES - 1; ++j) {
 893                                 unsigned long idx;
 894                                 int sh;
 895
 896                                 sh = KVM_HPAGE_GFN_SHIFT(PT_DIRECTORY_LEVEL+j);
 897                                 idx = ((memslot->base_gfn+gfn_offset) >> sh) -
 898                                         (memslot->base_gfn >> sh);
 899                                 ret |= handler(kvm,
 900                                         &memslot->lpage_info[j][idx].rmap_pde,
 901                                         data);
 902                         }
 903                         trace_kvm_age_page(hva, memslot, ret);
 904                         retval |= ret;
 905                 }
 906         }
 907
 908         return retval;
 909 }
 910
 911 int kvm_unmap_hva(struct kvm *kvm, unsigned long hva)
 912 {
 913         return kvm_handle_hva(kvm, hva, 0, kvm_unmap_rmapp);
 914 }
 915
 916 void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte)
 917 {
 918         kvm_handle_hva(kvm, hva, (unsigned long)&pte, kvm_set_pte_rmapp);
 919 }
 920
 921 static int kvm_age_rmapp(struct kvm *kvm, unsigned long *rmapp,
 922                          unsigned long data)
 923 {
 924         u64 *spte;
 925         int young = 0;
 926
 927         /*
 928          * Emulate the accessed bit for EPT, by checking if this page has
 929          * an EPT mapping, and clearing it if it does. On the next access,
 930          * a new EPT mapping will be established.
 931          * This has some overhead, but not as much as the cost of swapping
 932          * out actively used pages or breaking up actively used hugepages.
 933          */
 934         if (!shadow_accessed_mask)
 935                 return kvm_unmap_rmapp(kvm, rmapp, data);
 936
 937         spte = rmap_next(kvm, rmapp, NULL);
 938         while (spte) {
 939                 int _young;
 940                 u64 _spte = *spte;
 941                 BUG_ON(!(_spte & PT_PRESENT_MASK));
 942                 _young = _spte & PT_ACCESSED_MASK;
 943                 if (_young) {
 944                         young = 1;
 945                         clear_bit(PT_ACCESSED_SHIFT, (unsigned long *)spte);
 946                 }
 947                 spte = rmap_next(kvm, rmapp, spte);
 948         }
 949         return young;
 950 }
 951
 952 #define RMAP_RECYCLE_THRESHOLD 1000
 953
 954 static void rmap_recycle(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn)
 955 {
 956         unsigned long *rmapp;
 957         struct kvm_mmu_page *sp;
 958
 959         sp = page_header(__pa(spte));
 960
 961         rmapp = gfn_to_rmap(vcpu->kvm, gfn, sp->role.level);
 962
 963         kvm_unmap_rmapp(vcpu->kvm, rmapp, 0);
 964         kvm_flush_remote_tlbs(vcpu->kvm);
 965 }
 966
 967 int kvm_age_hva(struct kvm *kvm, unsigned long hva)
 968 {
 969         return kvm_handle_hva(kvm, hva, 0, kvm_age_rmapp);
 970 }
 971
 972 #ifdef MMU_DEBUG
 973 static int is_empty_shadow_page(u64 *spt)
 974 {
 975         u64 *pos;
 976         u64 *end;
 977
 978         for (pos = spt, end = pos + PAGE_SIZE / sizeof(u64); pos != end; pos++)
 979                 if (is_shadow_present_pte(*pos)) {
 980                         printk(KERN_ERR "%s: %p %llx\n", __func__,
 981                                pos, *pos);
 982                         return 0;
 983                 }
 984         return 1;
 985 }
 986 #endif
 987
 988 /*
 989  * This value is the sum of all of the kvm instances's
 990  * kvm->arch.n_used_mmu_pages values.  We need a global,
 991  * aggregate version in order to make the slab shrinker
 992  * faster
 993  */
 994 static inline void kvm_mod_used_mmu_pages(struct kvm *kvm, int nr)
 995 {
 996         kvm->arch.n_used_mmu_pages += nr;
 997         percpu_counter_add(&kvm_total_used_mmu_pages, nr);
 998 }
 999
1000 static void kvm_mmu_free_page(struct kvm *kvm, struct kvm_mmu_page *sp)
1001 {
1002         ASSERT(is_empty_shadow_page(sp->spt));
1003         hlist_del(&sp->hash_link);
1004         list_del(&sp->link);
1005         __free_page(virt_to_page(sp->spt));
1006         if (!sp->role.direct)
1007                 __free_page(virt_to_page(sp->gfns));
1008         kmem_cache_free(mmu_page_header_cache, sp);
1009         kvm_mod_used_mmu_pages(kvm, -1);
1010 }
1011
1012 static unsigned kvm_page_table_hashfn(gfn_t gfn)
1013 {
1014         return gfn & ((1 << KVM_MMU_HASH_SHIFT) - 1);
1015 }
1016
1017 static struct kvm_mmu_page *kvm_mmu_alloc_page(struct kvm_vcpu *vcpu,
1018                                                u64 *parent_pte, int direct)
1019 {
1020         struct kvm_mmu_page *sp;
1021
1022         sp = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_header_cache, sizeof *sp);
1023         sp->spt = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache, PAGE_SIZE);
1024         if (!direct)
1025                 sp->gfns = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache,
1026                                                   PAGE_SIZE);
1027         set_page_private(virt_to_page(sp->spt), (unsigned long)sp);
1028         list_add(&sp->link, &vcpu->kvm->arch.active_mmu_pages);
1029         bitmap_zero(sp->slot_bitmap, KVM_MEMORY_SLOTS + KVM_PRIVATE_MEM_SLOTS);
1030         sp->multimapped = 0;
1031         sp->parent_pte = parent_pte;
1032         kvm_mod_used_mmu_pages(vcpu->kvm, +1);
1033         return sp;
1034 }
1035
1036 static void mmu_page_add_parent_pte(struct kvm_vcpu *vcpu,
1037                                     struct kvm_mmu_page *sp, u64 *parent_pte)
1038 {
1039         struct kvm_pte_chain *pte_chain;
1040         struct hlist_node *node;
1041         int i;
1042
1043         if (!parent_pte)
1044                 return;
1045         if (!sp->multimapped) {
1046                 u64 *old = sp->parent_pte;
1047
1048                 if (!old) {
1049                         sp->parent_pte = parent_pte;
1050                         return;
1051                 }
1052                 sp->multimapped = 1;
1053                 pte_chain = mmu_alloc_pte_chain(vcpu);
1054                 INIT_HLIST_HEAD(&sp->parent_ptes);
1055                 hlist_add_head(&pte_chain->link, &sp->parent_ptes);
1056                 pte_chain->parent_ptes[0] = old;
1057         }
1058         hlist_for_each_entry(pte_chain, node, &sp->parent_ptes, link) {
1059                 if (pte_chain->parent_ptes[NR_PTE_CHAIN_ENTRIES-1])
1060                         continue;
1061                 for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i)
1062                         if (!pte_chain->parent_ptes[i]) {
1063                                 pte_chain->parent_ptes[i] = parent_pte;
1064                                 return;
1065                         }
1066         }
1067         pte_chain = mmu_alloc_pte_chain(vcpu);
1068         BUG_ON(!pte_chain);
1069         hlist_add_head(&pte_chain->link, &sp->parent_ptes);
1070         pte_chain->parent_ptes[0] = parent_pte;
1071 }
1072
1073 static void mmu_page_remove_parent_pte(struct kvm_mmu_page *sp,
1074                                        u64 *parent_pte)
1075 {
1076         struct kvm_pte_chain *pte_chain;
1077         struct hlist_node *node;
1078         int i;
1079
1080         if (!sp->multimapped) {
1081                 BUG_ON(sp->parent_pte != parent_pte);
1082                 sp->parent_pte = NULL;
1083                 return;
1084         }
1085         hlist_for_each_entry(pte_chain, node, &sp->parent_ptes, link)
1086                 for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i) {
1087                         if (!pte_chain->parent_ptes[i])
1088                                 break;
1089                         if (pte_chain->parent_ptes[i] != parent_pte)
1090                                 continue;
1091                         while (i + 1 < NR_PTE_CHAIN_ENTRIES
1092                                 && pte_chain->parent_ptes[i + 1]) {
1093                                 pte_chain->parent_ptes[i]
1094                                         = pte_chain->parent_ptes[i + 1];
1095                                 ++i;
1096                         }
1097                         pte_chain->parent_ptes[i] = NULL;
1098                         if (i == 0) {
1099                                 hlist_del(&pte_chain->link);
1100                                 mmu_free_pte_chain(pte_chain);
1101                                 if (hlist_empty(&sp->parent_ptes)) {
1102                                         sp->multimapped = 0;
1103                                         sp->parent_pte = NULL;
1104                                 }
1105                         }
1106                         return;
1107                 }
1108         BUG();
1109 }
1110
1111 static void mmu_parent_walk(struct kvm_mmu_page *sp, mmu_parent_walk_fn fn)
1112 {
1113         struct kvm_pte_chain *pte_chain;
1114         struct hlist_node *node;
1115         struct kvm_mmu_page *parent_sp;
1116         int i;
1117
1118         if (!sp->multimapped && sp->parent_pte) {
1119                 parent_sp = page_header(__pa(sp->parent_pte));
1120                 fn(parent_sp, sp->parent_pte);
1121                 return;
1122         }
1123
1124         hlist_for_each_entry(pte_chain, node, &sp->parent_ptes, link)
1125                 for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i) {
1126                         u64 *spte = pte_chain->parent_ptes[i];
1127
1128                         if (!spte)
1129                                 break;
1130                         parent_sp = page_header(__pa(spte));
1131                         fn(parent_sp, spte);
1132                 }
1133 }
1134
1135 static void mark_unsync(struct kvm_mmu_page *sp, u64 *spte);
1136 static void kvm_mmu_mark_parents_unsync(struct kvm_mmu_page *sp)
1137 {
1138         mmu_parent_walk(sp, mark_unsync);
1139 }
1140
1141 static void mark_unsync(struct kvm_mmu_page *sp, u64 *spte)
1142 {
1143         unsigned int index;
1144
1145         index = spte - sp->spt;
1146         if (__test_and_set_bit(index, sp->unsync_child_bitmap))
1147                 return;
1148         if (sp->unsync_children++)
1149                 return;
1150         kvm_mmu_mark_parents_unsync(sp);
1151 }
1152
1153 static void nonpaging_prefetch_page(struct kvm_vcpu *vcpu,
1154                                     struct kvm_mmu_page *sp)
1155 {
1156         int i;
1157
1158         for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
1159                 sp->spt[i] = shadow_trap_nonpresent_pte;
1160 }
1161
1162 static int nonpaging_sync_page(struct kvm_vcpu *vcpu,
1163                                struct kvm_mmu_page *sp, bool clear_unsync)
1164 {
1165         return 1;
1166 }
1167
1168 static void nonpaging_invlpg(struct kvm_vcpu *vcpu, gva_t gva)
1169 {
1170 }
1171
1172 #define KVM_PAGE_ARRAY_NR 16
1173
1174 struct kvm_mmu_pages {
1175         struct mmu_page_and_offset {
1176                 struct kvm_mmu_page *sp;
1177                 unsigned int idx;
1178         } page[KVM_PAGE_ARRAY_NR];
1179         unsigned int nr;
1180 };
1181
1182 #define for_each_unsync_children(bitmap, idx)           \
1183         for (idx = find_first_bit(bitmap, 512);         \
1184              idx < 512;                                 \
1185              idx = find_next_bit(bitmap, 512, idx+1))
1186
1187 static int mmu_pages_add(struct kvm_mmu_pages *pvec, struct kvm_mmu_page *sp,
1188                          int idx)
1189 {
1190         int i;
1191
1192         if (sp->unsync)
1193                 for (i=0; i < pvec->nr; i++)
1194                         if (pvec->page[i].sp == sp)
1195                                 return 0;
1196
1197         pvec->page[pvec->nr].sp = sp;
1198         pvec->page[pvec->nr].idx = idx;
1199         pvec->nr++;
1200         return (pvec->nr == KVM_PAGE_ARRAY_NR);
1201 }
1202
1203 static int __mmu_unsync_walk(struct kvm_mmu_page *sp,
1204                            struct kvm_mmu_pages *pvec)
1205 {
1206         int i, ret, nr_unsync_leaf = 0;
1207
1208         for_each_unsync_children(sp->unsync_child_bitmap, i) {
1209                 struct kvm_mmu_page *child;
1210                 u64 ent = sp->spt[i];
1211
1212                 if (!is_shadow_present_pte(ent) || is_large_pte(ent))
1213                         goto clear_child_bitmap;
1214
1215                 child = page_header(ent & PT64_BASE_ADDR_MASK);
1216
1217                 if (child->unsync_children) {
1218                         if (mmu_pages_add(pvec, child, i))
1219                                 return -ENOSPC;
1220
1221                         ret = __mmu_unsync_walk(child, pvec);
1222                         if (!ret)
1223                                 goto clear_child_bitmap;
1224                         else if (ret > 0)
1225                                 nr_unsync_leaf += ret;
1226                         else
1227                                 return ret;
1228                 } else if (child->unsync) {
1229                         nr_unsync_leaf++;
1230                         if (mmu_pages_add(pvec, child, i))
1231                                 return -ENOSPC;
1232                 } else
1233                          goto clear_child_bitmap;
1234
1235                 continue;
1236
1237 clear_child_bitmap:
1238                 __clear_bit(i, sp->unsync_child_bitmap);
1239                 sp->unsync_children--;
1240                 WARN_ON((int)sp->unsync_children < 0);
1241         }
1242
1243
1244         return nr_unsync_leaf;
1245 }
1246
1247 static int mmu_unsync_walk(struct kvm_mmu_page *sp,
1248                            struct kvm_mmu_pages *pvec)
1249 {
1250         if (!sp->unsync_children)
1251                 return 0;
1252
1253         mmu_pages_add(pvec, sp, 0);
1254         return __mmu_unsync_walk(sp, pvec);
1255 }
1256
1257 static void kvm_unlink_unsync_page(struct kvm *kvm, struct kvm_mmu_page *sp)
1258 {
1259         WARN_ON(!sp->unsync);
1260         trace_kvm_mmu_sync_page(sp);
1261         sp->unsync = 0;
1262         --kvm->stat.mmu_unsync;
1263 }
1264
1265 static int kvm_mmu_prepare_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp,
1266                                     struct list_head *invalid_list);
1267 static void kvm_mmu_commit_zap_page(struct kvm *kvm,
1268                                     struct list_head *invalid_list);
1269
1270 #define for_each_gfn_sp(kvm, sp, gfn, pos)                              \
1271   hlist_for_each_entry(sp, pos,                                         \
1272    &(kvm)->arch.mmu_page_hash[kvm_page_table_hashfn(gfn)], hash_link)   \
1273         if ((sp)->gfn != (gfn)) {} else
1274
1275 #define for_each_gfn_indirect_valid_sp(kvm, sp, gfn, pos)               \
1276   hlist_for_each_entry(sp, pos,                                         \
1277    &(kvm)->arch.mmu_page_hash[kvm_page_table_hashfn(gfn)], hash_link)   \
1278                 if ((sp)->gfn != (gfn) || (sp)->role.direct ||          \
1279                         (sp)->role.invalid) {} else
1280
1281 /* @sp->gfn should be write-protected at the call site */
1282 static int __kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
1283                            struct list_head *invalid_list, bool clear_unsync)
1284 {
1285         if (sp->role.cr4_pae != !!is_pae(vcpu)) {
1286                 kvm_mmu_prepare_zap_page(vcpu->kvm, sp, invalid_list);
1287                 return 1;
1288         }
1289
1290         if (clear_unsync)
1291                 kvm_unlink_unsync_page(vcpu->kvm, sp);
1292
1293         if (vcpu->arch.mmu.sync_page(vcpu, sp, clear_unsync)) {
1294                 kvm_mmu_prepare_zap_page(vcpu->kvm, sp, invalid_list);
1295                 return 1;
1296         }
1297
1298         kvm_mmu_flush_tlb(vcpu);
1299         return 0;
1300 }
1301
1302 static int kvm_sync_page_transient(struct kvm_vcpu *vcpu,
1303                                    struct kvm_mmu_page *sp)
1304 {
1305         LIST_HEAD(invalid_list);
1306         int ret;
1307
1308         ret = __kvm_sync_page(vcpu, sp, &invalid_list, false);
1309         if (ret)
1310                 kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
1311
1312         return ret;
1313 }
1314
1315 static int kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
1316                          struct list_head *invalid_list)
1317 {
1318         return __kvm_sync_page(vcpu, sp, invalid_list, true);
1319 }
1320
1321 /* @gfn should be write-protected at the call site */
1322 static void kvm_sync_pages(struct kvm_vcpu *vcpu,  gfn_t gfn)
1323 {
1324         struct kvm_mmu_page *s;
1325         struct hlist_node *node;
1326         LIST_HEAD(invalid_list);
1327         bool flush = false;
1328
1329         for_each_gfn_indirect_valid_sp(vcpu->kvm, s, gfn, node) {
1330                 if (!s->unsync)
1331                         continue;
1332
1333                 WARN_ON(s->role.level != PT_PAGE_TABLE_LEVEL);
1334                 if ((s->role.cr4_pae != !!is_pae(vcpu)) ||
1335                         (vcpu->arch.mmu.sync_page(vcpu, s, true))) {
1336                         kvm_mmu_prepare_zap_page(vcpu->kvm, s, &invalid_list);
1337                         continue;
1338                 }
1339                 kvm_unlink_unsync_page(vcpu->kvm, s);
1340                 flush = true;
1341         }
1342
1343         kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
1344         if (flush)
1345                 kvm_mmu_flush_tlb(vcpu);
1346 }
1347
1348 struct mmu_page_path {
1349         struct kvm_mmu_page *parent[PT64_ROOT_LEVEL-1];
1350         unsigned int idx[PT64_ROOT_LEVEL-1];
1351 };
1352
1353 #define for_each_sp(pvec, sp, parents, i)                       \
1354                 for (i = mmu_pages_next(&pvec, &parents, -1),   \
1355                         sp = pvec.page[i].sp;                   \
1356                         i < pvec.nr && ({ sp = pvec.page[i].sp; 1;});   \
1357                         i = mmu_pages_next(&pvec, &parents, i))
1358
1359 static int mmu_pages_next(struct kvm_mmu_pages *pvec,
1360                           struct mmu_page_path *parents,
1361                           int i)
1362 {
1363         int n;
1364
1365         for (n = i+1; n < pvec->nr; n++) {
1366                 struct kvm_mmu_page *sp = pvec->page[n].sp;
1367
1368                 if (sp->role.level == PT_PAGE_TABLE_LEVEL) {
1369                         parents->idx[0] = pvec->page[n].idx;
1370                         return n;
1371                 }
1372
1373                 parents->parent[sp->role.level-2] = sp;
1374                 parents->idx[sp->role.level-1] = pvec->page[n].idx;
1375         }
1376
1377         return n;
1378 }
1379
1380 static void mmu_pages_clear_parents(struct mmu_page_path *parents)
1381 {
1382         struct kvm_mmu_page *sp;
1383         unsigned int level = 0;
1384
1385         do {
1386                 unsigned int idx = parents->idx[level];
1387
1388                 sp = parents->parent[level];
1389                 if (!sp)
1390                         return;
1391
1392                 --sp->unsync_children;
1393                 WARN_ON((int)sp->unsync_children < 0);
1394                 __clear_bit(idx, sp->unsync_child_bitmap);
1395                 level++;
1396         } while (level < PT64_ROOT_LEVEL-1 && !sp->unsync_children);
1397 }
1398
1399 static void kvm_mmu_pages_init(struct kvm_mmu_page *parent,
1400                                struct mmu_page_path *parents,
1401                                struct kvm_mmu_pages *pvec)
1402 {
1403         parents->parent[parent->role.level-1] = NULL;
1404         pvec->nr = 0;
1405 }
1406
1407 static void mmu_sync_children(struct kvm_vcpu *vcpu,
1408                               struct kvm_mmu_page *parent)
1409 {
1410         int i;
1411         struct kvm_mmu_page *sp;
1412         struct mmu_page_path parents;
1413         struct kvm_mmu_pages pages;
1414         LIST_HEAD(invalid_list);
1415
1416         kvm_mmu_pages_init(parent, &parents, &pages);
1417         while (mmu_unsync_walk(parent, &pages)) {
1418                 int protected = 0;
1419
1420                 for_each_sp(pages, sp, parents, i)
1421                         protected |= rmap_write_protect(vcpu->kvm, sp->gfn);
1422
1423                 if (protected)
1424                         kvm_flush_remote_tlbs(vcpu->kvm);
1425
1426                 for_each_sp(pages, sp, parents, i) {
1427                         kvm_sync_page(vcpu, sp, &invalid_list);
1428                         mmu_pages_clear_parents(&parents);
1429                 }
1430                 kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
1431                 cond_resched_lock(&vcpu->kvm->mmu_lock);
1432                 kvm_mmu_pages_init(parent, &parents, &pages);
1433         }
1434 }
1435
1436 static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu,
1437                                              gfn_t gfn,
1438                                              gva_t gaddr,
1439                                              unsigned level,
1440                                              int direct,
1441                                              unsigned access,
1442                                              u64 *parent_pte)
1443 {
1444         union kvm_mmu_page_role role;
1445         unsigned quadrant;
1446         struct kvm_mmu_page *sp;
1447         struct hlist_node *node;
1448         bool need_sync = false;
1449
1450         role = vcpu->arch.mmu.base_role;
1451         role.level = level;
1452         role.direct = direct;
1453         if (role.direct)
1454                 role.cr4_pae = 0;
1455         role.access = access;
1456         if (!vcpu->arch.mmu.direct_map
1457             && vcpu->arch.mmu.root_level <= PT32_ROOT_LEVEL) {
1458                 quadrant = gaddr >> (PAGE_SHIFT + (PT64_PT_BITS * level));
1459                 quadrant &= (1 << ((PT32_PT_BITS - PT64_PT_BITS) * level)) - 1;
1460                 role.quadrant = quadrant;
1461         }
1462         for_each_gfn_sp(vcpu->kvm, sp, gfn, node) {
1463                 if (!need_sync && sp->unsync)
1464                         need_sync = true;
1465
1466                 if (sp->role.word != role.word)
1467                         continue;
1468
1469                 if (sp->unsync && kvm_sync_page_transient(vcpu, sp))
1470                         break;
1471
1472                 mmu_page_add_parent_pte(vcpu, sp, parent_pte);
1473                 if (sp->unsync_children) {
1474                         kvm_make_request(KVM_REQ_MMU_SYNC, vcpu);
1475                         kvm_mmu_mark_parents_unsync(sp);
1476                 } else if (sp->unsync)
1477                         kvm_mmu_mark_parents_unsync(sp);
1478
1479                 trace_kvm_mmu_get_page(sp, false);
1480                 return sp;
1481         }
1482         ++vcpu->kvm->stat.mmu_cache_miss;
1483         sp = kvm_mmu_alloc_page(vcpu, parent_pte, direct);
1484         if (!sp)
1485                 return sp;
1486         sp->gfn = gfn;
1487         sp->role = role;
1488         hlist_add_head(&sp->hash_link,
1489                 &vcpu->kvm->arch.mmu_page_hash[kvm_page_table_hashfn(gfn)]);
1490         if (!direct) {
1491                 if (rmap_write_protect(vcpu->kvm, gfn))
1492                         kvm_flush_remote_tlbs(vcpu->kvm);
1493                 if (level > PT_PAGE_TABLE_LEVEL && need_sync)
1494                         kvm_sync_pages(vcpu, gfn);
1495
1496                 account_shadowed(vcpu->kvm, gfn);
1497         }
1498         if (shadow_trap_nonpresent_pte != shadow_notrap_nonpresent_pte)
1499                 vcpu->arch.mmu.prefetch_page(vcpu, sp);
1500         else
1501                 nonpaging_prefetch_page(vcpu, sp);
1502         trace_kvm_mmu_get_page(sp, true);
1503         return sp;
1504 }
1505
1506 static void shadow_walk_init(struct kvm_shadow_walk_iterator *iterator,
1507                              struct kvm_vcpu *vcpu, u64 addr)
1508 {
1509         iterator->addr = addr;
1510         iterator->shadow_addr = vcpu->arch.mmu.root_hpa;
1511         iterator->level = vcpu->arch.mmu.shadow_root_level;
1512
1513         if (iterator->level == PT64_ROOT_LEVEL &&
1514             vcpu->arch.mmu.root_level < PT64_ROOT_LEVEL &&
1515             !vcpu->arch.mmu.direct_map)
1516                 --iterator->level;
1517
1518         if (iterator->level == PT32E_ROOT_LEVEL) {
1519                 iterator->shadow_addr
1520                         = vcpu->arch.mmu.pae_root[(addr >> 30) & 3];
1521                 iterator->shadow_addr &= PT64_BASE_ADDR_MASK;
1522                 --iterator->level;
1523                 if (!iterator->shadow_addr)
1524                         iterator->level = 0;
1525         }
1526 }
1527
1528 static bool shadow_walk_okay(struct kvm_shadow_walk_iterator *iterator)
1529 {
1530         if (iterator->level < PT_PAGE_TABLE_LEVEL)
1531                 return false;
1532
1533         if (iterator->level == PT_PAGE_TABLE_LEVEL)
1534                 if (is_large_pte(*iterator->sptep))
1535                         return false;
1536
1537         iterator->index = SHADOW_PT_INDEX(iterator->addr, iterator->level);
1538         iterator->sptep = ((u64 *)__va(iterator->shadow_addr)) + iterator->index;
1539         return true;
1540 }
1541
1542 static void shadow_walk_next(struct kvm_shadow_walk_iterator *iterator)
1543 {
1544         iterator->shadow_addr = *iterator->sptep & PT64_BASE_ADDR_MASK;
1545         --iterator->level;
1546 }
1547
1548 static void link_shadow_page(u64 *sptep, struct kvm_mmu_page *sp)
1549 {
1550         u64 spte;
1551
1552         spte = __pa(sp->spt)
1553                 | PT_PRESENT_MASK | PT_ACCESSED_MASK
1554                 | PT_WRITABLE_MASK | PT_USER_MASK;
1555         __set_spte(sptep, spte);
1556 }
1557
1558 static void drop_large_spte(struct kvm_vcpu *vcpu, u64 *sptep)
1559 {
1560         if (is_large_pte(*sptep)) {
1561                 drop_spte(vcpu->kvm, sptep, shadow_trap_nonpresent_pte);
1562                 kvm_flush_remote_tlbs(vcpu->kvm);
1563         }
1564 }
1565
1566 static void validate_direct_spte(struct kvm_vcpu *vcpu, u64 *sptep,
1567                                    unsigned direct_access)
1568 {
1569         if (is_shadow_present_pte(*sptep) && !is_large_pte(*sptep)) {
1570                 struct kvm_mmu_page *child;
1571
1572                 /*
1573                  * For the direct sp, if the guest pte's dirty bit
1574                  * changed form clean to dirty, it will corrupt the
1575                  * sp's access: allow writable in the read-only sp,
1576                  * so we should update the spte at this point to get
1577                  * a new sp with the correct access.
1578                  */
1579                 child = page_header(*sptep & PT64_BASE_ADDR_MASK);
1580                 if (child->role.access == direct_access)
1581                         return;
1582
1583                 mmu_page_remove_parent_pte(child, sptep);
1584                 __set_spte(sptep, shadow_trap_nonpresent_pte);
1585                 kvm_flush_remote_tlbs(vcpu->kvm);
1586         }
1587 }
1588
1589 static void kvm_mmu_page_unlink_children(struct kvm *kvm,
1590                                          struct kvm_mmu_page *sp)
1591 {
1592         unsigned i;
1593         u64 *pt;
1594         u64 ent;
1595
1596         pt = sp->spt;
1597
1598         for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
1599                 ent = pt[i];
1600
1601                 if (is_shadow_present_pte(ent)) {
1602                         if (!is_last_spte(ent, sp->role.level)) {
1603                                 ent &= PT64_BASE_ADDR_MASK;
1604                                 mmu_page_remove_parent_pte(page_header(ent),
1605                                                            &pt[i]);
1606                         } else {
1607                                 if (is_large_pte(ent))
1608                                         --kvm->stat.lpages;
1609                                 drop_spte(kvm, &pt[i],
1610                                           shadow_trap_nonpresent_pte);
1611                         }
1612                 }
1613                 pt[i] = shadow_trap_nonpresent_pte;
1614         }
1615 }
1616
1617 static void kvm_mmu_put_page(struct kvm_mmu_page *sp, u64 *parent_pte)
1618 {
1619         mmu_page_remove_parent_pte(sp, parent_pte);
1620 }
1621
1622 static void kvm_mmu_reset_last_pte_updated(struct kvm *kvm)
1623 {
1624         int i;
1625         struct kvm_vcpu *vcpu;
1626
1627         kvm_for_each_vcpu(i, vcpu, kvm)
1628                 vcpu->arch.last_pte_updated = NULL;
1629 }
1630
1631 static void kvm_mmu_unlink_parents(struct kvm *kvm, struct kvm_mmu_page *sp)
1632 {
1633         u64 *parent_pte;
1634
1635         while (sp->multimapped || sp->parent_pte) {
1636                 if (!sp->multimapped)
1637                         parent_pte = sp->parent_pte;
1638                 else {
1639                         struct kvm_pte_chain *chain;
1640
1641                         chain = container_of(sp->parent_ptes.first,
1642                                              struct kvm_pte_chain, link);
1643                         parent_pte = chain->parent_ptes[0];
1644                 }
1645                 BUG_ON(!parent_pte);
1646                 kvm_mmu_put_page(sp, parent_pte);
1647                 __set_spte(parent_pte, shadow_trap_nonpresent_pte);
1648         }
1649 }
1650
1651 static int mmu_zap_unsync_children(struct kvm *kvm,
1652                                    struct kvm_mmu_page *parent,
1653                                    struct list_head *invalid_list)
1654 {
1655         int i, zapped = 0;
1656         struct mmu_page_path parents;
1657         struct kvm_mmu_pages pages;
1658
1659         if (parent->role.level == PT_PAGE_TABLE_LEVEL)
1660                 return 0;
1661
1662         kvm_mmu_pages_init(parent, &parents, &pages);
1663         while (mmu_unsync_walk(parent, &pages)) {
1664                 struct kvm_mmu_page *sp;
1665
1666                 for_each_sp(pages, sp, parents, i) {
1667                         kvm_mmu_prepare_zap_page(kvm, sp, invalid_list);
1668                         mmu_pages_clear_parents(&parents);
1669                         zapped++;
1670                 }
1671                 kvm_mmu_pages_init(parent, &parents, &pages);
1672         }
1673
1674         return zapped;
1675 }
1676
1677 static int kvm_mmu_prepare_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp,
1678                                     struct list_head *invalid_list)
1679 {
1680         int ret;
1681
1682         trace_kvm_mmu_prepare_zap_page(sp);
1683         ++kvm->stat.mmu_shadow_zapped;
1684         ret = mmu_zap_unsync_children(kvm, sp, invalid_list);
1685         kvm_mmu_page_unlink_children(kvm, sp);
1686         kvm_mmu_unlink_parents(kvm, sp);
1687         if (!sp->role.invalid && !sp->role.direct)
1688                 unaccount_shadowed(kvm, sp->gfn);
1689         if (sp->unsync)
1690                 kvm_unlink_unsync_page(kvm, sp);
1691         if (!sp->root_count) {
1692                 /* Count self */
1693                 ret++;
1694                 list_move(&sp->link, invalid_list);
1695         } else {
1696                 list_move(&sp->link, &kvm->arch.active_mmu_pages);
1697                 kvm_reload_remote_mmus(kvm);
1698         }
1699
1700         sp->role.invalid = 1;
1701         kvm_mmu_reset_last_pte_updated(kvm);
1702         return ret;
1703 }
1704
1705 static void kvm_mmu_commit_zap_page(struct kvm *kvm,
1706                                     struct list_head *invalid_list)
1707 {
1708         struct kvm_mmu_page *sp;
1709
1710         if (list_empty(invalid_list))
1711                 return;
1712
1713         kvm_flush_remote_tlbs(kvm);
1714
1715         do {
1716                 sp = list_first_entry(invalid_list, struct kvm_mmu_page, link);
1717                 WARN_ON(!sp->role.invalid || sp->root_count);
1718                 kvm_mmu_free_page(kvm, sp);
1719         } while (!list_empty(invalid_list));
1720
1721 }
1722
1723 /*
1724  * Changing the number of mmu pages allocated to the vm
1725  * Note: if goal_nr_mmu_pages is too small, you will get dead lock
1726  */
1727 void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned int goal_nr_mmu_pages)
1728 {
1729         LIST_HEAD(invalid_list);
1730         /*
1731          * If we set the number of mmu pages to be smaller be than the
1732          * number of actived pages , we must to free some mmu pages before we
1733          * change the value
1734          */
1735
1736         if (kvm->arch.n_used_mmu_pages > goal_nr_mmu_pages) {
1737                 while (kvm->arch.n_used_mmu_pages > goal_nr_mmu_pages &&
1738                         !list_empty(&kvm->arch.active_mmu_pages)) {
1739                         struct kvm_mmu_page *page;
1740
1741                         page = container_of(kvm->arch.active_mmu_pages.prev,
1742                                             struct kvm_mmu_page, link);
1743                         kvm_mmu_prepare_zap_page(kvm, page, &invalid_list);
1744                         kvm_mmu_commit_zap_page(kvm, &invalid_list);
1745                 }
1746                 goal_nr_mmu_pages = kvm->arch.n_used_mmu_pages;
1747         }
1748
1749         kvm->arch.n_max_mmu_pages = goal_nr_mmu_pages;
1750 }
1751
1752 static int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn)
1753 {
1754         struct kvm_mmu_page *sp;
1755         struct hlist_node *node;
1756         LIST_HEAD(invalid_list);
1757         int r;
1758
1759         pgprintk("%s: looking for gfn %llx\n", __func__, gfn);
1760         r = 0;
1761
1762         for_each_gfn_indirect_valid_sp(kvm, sp, gfn, node) {
1763                 pgprintk("%s: gfn %llx role %x\n", __func__, gfn,
1764                          sp->role.word);
1765                 r = 1;
1766                 kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list);
1767         }
1768         kvm_mmu_commit_zap_page(kvm, &invalid_list);
1769         return r;
1770 }
1771
1772 static void mmu_unshadow(struct kvm *kvm, gfn_t gfn)
1773 {
1774         struct kvm_mmu_page *sp;
1775         struct hlist_node *node;
1776         LIST_HEAD(invalid_list);
1777
1778         for_each_gfn_indirect_valid_sp(kvm, sp, gfn, node) {
1779                 pgprintk("%s: zap %llx %x\n",
1780                          __func__, gfn, sp->role.word);
1781                 kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list);
1782         }
1783         kvm_mmu_commit_zap_page(kvm, &invalid_list);
1784 }
1785
1786 static void page_header_update_slot(struct kvm *kvm, void *pte, gfn_t gfn)
1787 {
1788         int slot = memslot_id(kvm, gfn);
1789         struct kvm_mmu_page *sp = page_header(__pa(pte));
1790
1791         __set_bit(slot, sp->slot_bitmap);
1792 }
1793
1794 static void mmu_convert_notrap(struct kvm_mmu_page *sp)
1795 {
1796         int i;
1797         u64 *pt = sp->spt;
1798
1799         if (shadow_trap_nonpresent_pte == shadow_notrap_nonpresent_pte)
1800                 return;
1801
1802         for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
1803                 if (pt[i] == shadow_notrap_nonpresent_pte)
1804                         __set_spte(&pt[i], shadow_trap_nonpresent_pte);
1805         }
1806 }
1807
1808 /*
1809  * The function is based on mtrr_type_lookup() in
1810  * arch/x86/kernel/cpu/mtrr/generic.c
1811  */
1812 static int get_mtrr_type(struct mtrr_state_type *mtrr_state,
1813                          u64 start, u64 end)
1814 {
1815         int i;
1816         u64 base, mask;
1817         u8 prev_match, curr_match;
1818         int num_var_ranges = KVM_NR_VAR_MTRR;
1819
1820         if (!mtrr_state->enabled)
1821                 return 0xFF;
1822
1823         /* Make end inclusive end, instead of exclusive */
1824         end--;
1825
1826         /* Look in fixed ranges. Just return the type as per start */
1827         if (mtrr_state->have_fixed && (start < 0x100000)) {
1828                 int idx;
1829
1830                 if (start < 0x80000) {
1831                         idx = 0;
1832                         idx += (start >> 16);
1833                         return mtrr_state->fixed_ranges[idx];
1834                 } else if (start < 0xC0000) {
1835                         idx = 1 * 8;
1836                         idx += ((start - 0x80000) >> 14);
1837                         return mtrr_state->fixed_ranges[idx];
1838                 } else if (start < 0x1000000) {
1839                         idx = 3 * 8;
1840                         idx += ((start - 0xC0000) >> 12);
1841                         return mtrr_state->fixed_ranges[idx];
1842                 }
1843         }
1844
1845         /*
1846          * Look in variable ranges
1847          * Look of multiple ranges matching this address and pick type
1848          * as per MTRR precedence
1849          */
1850         if (!(mtrr_state->enabled & 2))
1851                 return mtrr_state->def_type;
1852
1853         prev_match = 0xFF;
1854         for (i = 0; i < num_var_ranges; ++i) {
1855                 unsigned short start_state, end_state;
1856
1857                 if (!(mtrr_state->var_ranges[i].mask_lo & (1 << 11)))
1858                         continue;
1859
1860                 base = (((u64)mtrr_state->var_ranges[i].base_hi) << 32) +
1861                        (mtrr_state->var_ranges[i].base_lo & PAGE_MASK);
1862                 mask = (((u64)mtrr_state->var_ranges[i].mask_hi) << 32) +
1863                        (mtrr_state->var_ranges[i].mask_lo & PAGE_MASK);
1864
1865                 start_state = ((start & mask) == (base & mask));
1866                 end_state = ((end & mask) == (base & mask));
1867                 if (start_state != end_state)
1868                         return 0xFE;
1869
1870                 if ((start & mask) != (base & mask))
1871                         continue;
1872
1873                 curr_match = mtrr_state->var_ranges[i].base_lo & 0xff;
1874                 if (prev_match == 0xFF) {
1875                         prev_match = curr_match;
1876                         continue;
1877                 }
1878
1879                 if (prev_match == MTRR_TYPE_UNCACHABLE ||
1880                     curr_match == MTRR_TYPE_UNCACHABLE)
1881                         return MTRR_TYPE_UNCACHABLE;
1882
1883                 if ((prev_match == MTRR_TYPE_WRBACK &&
1884                      curr_match == MTRR_TYPE_WRTHROUGH) ||
1885                     (prev_match == MTRR_TYPE_WRTHROUGH &&
1886                      curr_match == MTRR_TYPE_WRBACK)) {
1887                         prev_match = MTRR_TYPE_WRTHROUGH;
1888                         curr_match = MTRR_TYPE_WRTHROUGH;
1889                 }
1890
1891                 if (prev_match != curr_match)
1892                         return MTRR_TYPE_UNCACHABLE;
1893         }
1894
1895         if (prev_match != 0xFF)
1896                 return prev_match;
1897
1898         return mtrr_state->def_type;
1899 }
1900
1901 u8 kvm_get_guest_memory_type(struct kvm_vcpu *vcpu, gfn_t gfn)
1902 {
1903         u8 mtrr;
1904
1905         mtrr = get_mtrr_type(&vcpu->arch.mtrr_state, gfn << PAGE_SHIFT,
1906                              (gfn << PAGE_SHIFT) + PAGE_SIZE);
1907         if (mtrr == 0xfe || mtrr == 0xff)
1908                 mtrr = MTRR_TYPE_WRBACK;
1909         return mtrr;
1910 }
1911 EXPORT_SYMBOL_GPL(kvm_get_guest_memory_type);
1912
1913 static void __kvm_unsync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp)
1914 {
1915         trace_kvm_mmu_unsync_page(sp);
1916         ++vcpu->kvm->stat.mmu_unsync;
1917         sp->unsync = 1;
1918
1919         kvm_mmu_mark_parents_unsync(sp);
1920         mmu_convert_notrap(sp);
1921 }
1922
1923 static void kvm_unsync_pages(struct kvm_vcpu *vcpu,  gfn_t gfn)
1924 {
1925         struct kvm_mmu_page *s;
1926         struct hlist_node *node;
1927
1928         for_each_gfn_indirect_valid_sp(vcpu->kvm, s, gfn, node) {
1929                 if (s->unsync)
1930                         continue;
1931                 WARN_ON(s->role.level != PT_PAGE_TABLE_LEVEL);
1932                 __kvm_unsync_page(vcpu, s);
1933         }
1934 }
1935
1936 static int mmu_need_write_protect(struct kvm_vcpu *vcpu, gfn_t gfn,
1937                                   bool can_unsync)
1938 {
1939         struct kvm_mmu_page *s;
1940         struct hlist_node *node;
1941         bool need_unsync = false;
1942
1943         for_each_gfn_indirect_valid_sp(vcpu->kvm, s, gfn, node) {
1944                 if (!can_unsync)
1945                         return 1;
1946
1947                 if (s->role.level != PT_PAGE_TABLE_LEVEL)
1948                         return 1;
1949
1950                 if (!need_unsync && !s->unsync) {
1951                         if (!oos_shadow)
1952                                 return 1;
1953                         need_unsync = true;
1954                 }
1955         }
1956         if (need_unsync)
1957                 kvm_unsync_pages(vcpu, gfn);
1958         return 0;
1959 }
1960
1961 static int set_spte(struct kvm_vcpu *vcpu, u64 *sptep,
1962                     unsigned pte_access, int user_fault,
1963                     int write_fault, int dirty, int level,
1964                     gfn_t gfn, pfn_t pfn, bool speculative,
1965                     bool can_unsync, bool reset_host_protection)
1966 {
1967         u64 spte;
1968         int ret = 0;
1969
1970         /*
1971          * We don't set the accessed bit, since we sometimes want to see
1972          * whether the guest actually used the pte (in order to detect
1973          * demand paging).
1974          */
1975         spte = shadow_base_present_pte;
1976         if (!speculative)
1977                 spte |= shadow_accessed_mask;
1978         if (!dirty)
1979                 pte_access &= ~ACC_WRITE_MASK;
1980         if (pte_access & ACC_EXEC_MASK)
1981                 spte |= shadow_x_mask;
1982         else
1983                 spte |= shadow_nx_mask;
1984         if (pte_access & ACC_USER_MASK)
1985                 spte |= shadow_user_mask;
1986         if (level > PT_PAGE_TABLE_LEVEL)
1987                 spte |= PT_PAGE_SIZE_MASK;
1988         if (tdp_enabled)
1989                 spte |= kvm_x86_ops->get_mt_mask(vcpu, gfn,
1990                         kvm_is_mmio_pfn(pfn));
1991
1992         if (reset_host_protection)
1993                 spte |= SPTE_HOST_WRITEABLE;
1994
1995         spte |= (u64)pfn << PAGE_SHIFT;
1996
1997         if ((pte_access & ACC_WRITE_MASK)
1998             || (!vcpu->arch.mmu.direct_map && write_fault
1999                 && !is_write_protection(vcpu) && !user_fault)) {
2000
2001                 if (level > PT_PAGE_TABLE_LEVEL &&
2002                     has_wrprotected_page(vcpu->kvm, gfn, level)) {
2003                         ret = 1;
2004                         drop_spte(vcpu->kvm, sptep, shadow_trap_nonpresent_pte);
2005                         goto done;
2006                 }
2007
2008                 spte |= PT_WRITABLE_MASK;
2009
2010                 if (!vcpu->arch.mmu.direct_map
2011                     && !(pte_access & ACC_WRITE_MASK))
2012                         spte &= ~PT_USER_MASK;
2013
2014                 /*
2015                  * Optimization: for pte sync, if spte was writable the hash
2016                  * lookup is unnecessary (and expensive). Write protection
2017                  * is responsibility of mmu_get_page / kvm_sync_page.
2018                  * Same reasoning can be applied to dirty page accounting.
2019                  */
2020                 if (!can_unsync && is_writable_pte(*sptep))
2021                         goto set_pte;
2022
2023                 if (mmu_need_write_protect(vcpu, gfn, can_unsync)) {
2024                         pgprintk("%s: found shadow page for %llx, marking ro\n",
2025                                  __func__, gfn);
2026                         ret = 1;
2027                         pte_access &= ~ACC_WRITE_MASK;
2028                         if (is_writable_pte(spte))
2029                                 spte &= ~PT_WRITABLE_MASK;
2030                 }
2031         }
2032
2033         if (pte_access & ACC_WRITE_MASK)
2034                 mark_page_dirty(vcpu->kvm, gfn);
2035
2036 set_pte:
2037         update_spte(sptep, spte);
2038 done:
2039         return ret;
2040 }
2041
2042 static void mmu_set_spte(struct kvm_vcpu *vcpu, u64 *sptep,
2043                          unsigned pt_access, unsigned pte_access,
2044                          int user_fault, int write_fault, int dirty,
2045                          int *ptwrite, int level, gfn_t gfn,
2046                          pfn_t pfn, bool speculative,
2047                          bool reset_host_protection)
2048 {
2049         int was_rmapped = 0;
2050         int rmap_count;
2051
2052         pgprintk("%s: spte %llx access %x write_fault %d"
2053                  " user_fault %d gfn %llx\n",
2054                  __func__, *sptep, pt_access,
2055                  write_fault, user_fault, gfn);
2056
2057         if (is_rmap_spte(*sptep)) {
2058                 /*
2059                  * If we overwrite a PTE page pointer with a 2MB PMD, unlink
2060                  * the parent of the now unreachable PTE.
2061                  */
2062                 if (level > PT_PAGE_TABLE_LEVEL &&
2063                     !is_large_pte(*sptep)) {
2064                         struct kvm_mmu_page *child;
2065                         u64 pte = *sptep;
2066
2067                         child = page_header(pte & PT64_BASE_ADDR_MASK);
2068                         mmu_page_remove_parent_pte(child, sptep);
2069                         __set_spte(sptep, shadow_trap_nonpresent_pte);
2070                         kvm_flush_remote_tlbs(vcpu->kvm);
2071                 } else if (pfn != spte_to_pfn(*sptep)) {
2072                         pgprintk("hfn old %llx new %llx\n",
2073                                  spte_to_pfn(*sptep), pfn);
2074                         drop_spte(vcpu->kvm, sptep, shadow_trap_nonpresent_pte);
2075                         kvm_flush_remote_tlbs(vcpu->kvm);
2076                 } else
2077                         was_rmapped = 1;
2078         }
2079
2080         if (set_spte(vcpu, sptep, pte_access, user_fault, write_fault,
2081                       dirty, level, gfn, pfn, speculative, true,
2082                       reset_host_protection)) {
2083                 if (write_fault)
2084                         *ptwrite = 1;
2085                 kvm_mmu_flush_tlb(vcpu);
2086         }
2087
2088         pgprintk("%s: setting spte %llx\n", __func__, *sptep);
2089         pgprintk("instantiating %s PTE (%s) at %llx (%llx) addr %p\n",
2090                  is_large_pte(*sptep)? "2MB" : "4kB",
2091                  *sptep & PT_PRESENT_MASK ?"RW":"R", gfn,
2092                  *sptep, sptep);
2093         if (!was_rmapped && is_large_pte(*sptep))
2094                 ++vcpu->kvm->stat.lpages;
2095
2096         page_header_update_slot(vcpu->kvm, sptep, gfn);
2097         if (!was_rmapped) {
2098                 rmap_count = rmap_add(vcpu, sptep, gfn);
2099                 if (rmap_count > RMAP_RECYCLE_THRESHOLD)
2100                         rmap_recycle(vcpu, sptep, gfn);
2101         }
2102         kvm_release_pfn_clean(pfn);
2103         if (speculative) {
2104                 vcpu->arch.last_pte_updated = sptep;
2105                 vcpu->arch.last_pte_gfn = gfn;
2106         }
2107 }
2108
2109 static void nonpaging_new_cr3(struct kvm_vcpu *vcpu)
2110 {
2111 }
2112
2113 static struct kvm_memory_slot *
2114 pte_prefetch_gfn_to_memslot(struct kvm_vcpu *vcpu, gfn_t gfn, bool no_dirty_log)
2115 {
2116         struct kvm_memory_slot *slot;
2117
2118         slot = gfn_to_memslot(vcpu->kvm, gfn);
2119         if (!slot || slot->flags & KVM_MEMSLOT_INVALID ||
2120               (no_dirty_log && slot->dirty_bitmap))
2121                 slot = NULL;
2122
2123         return slot;
2124 }
2125
2126 static pfn_t pte_prefetch_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn,
2127                                      bool no_dirty_log)
2128 {
2129         struct kvm_memory_slot *slot;
2130         unsigned long hva;
2131
2132         slot = pte_prefetch_gfn_to_memslot(vcpu, gfn, no_dirty_log);
2133         if (!slot) {
2134                 get_page(bad_page);
2135                 return page_to_pfn(bad_page);
2136         }
2137
2138         hva = gfn_to_hva_memslot(slot, gfn);
2139
2140         return hva_to_pfn_atomic(vcpu->kvm, hva);
2141 }
2142
2143 static int direct_pte_prefetch_many(struct kvm_vcpu *vcpu,
2144                                     struct kvm_mmu_page *sp,
2145                                     u64 *start, u64 *end)
2146 {
2147         struct page *pages[PTE_PREFETCH_NUM];
2148         unsigned access = sp->role.access;
2149         int i, ret;
2150         gfn_t gfn;
2151
2152         gfn = kvm_mmu_page_get_gfn(sp, start - sp->spt);
2153         if (!pte_prefetch_gfn_to_memslot(vcpu, gfn, access & ACC_WRITE_MASK))
2154                 return -1;
2155
2156         ret = gfn_to_page_many_atomic(vcpu->kvm, gfn, pages, end - start);
2157         if (ret <= 0)
2158                 return -1;
2159
2160         for (i = 0; i < ret; i++, gfn++, start++)
2161                 mmu_set_spte(vcpu, start, ACC_ALL,
2162                              access, 0, 0, 1, NULL,
2163                              sp->role.level, gfn,
2164                              page_to_pfn(pages[i]), true, true);
2165
2166         return 0;
2167 }
2168
2169 static void __direct_pte_prefetch(struct kvm_vcpu *vcpu,
2170                                   struct kvm_mmu_page *sp, u64 *sptep)
2171 {
2172         u64 *spte, *start = NULL;
2173         int i;
2174
2175         WARN_ON(!sp->role.direct);
2176
2177         i = (sptep - sp->spt) & ~(PTE_PREFETCH_NUM - 1);
2178         spte = sp->spt + i;
2179
2180         for (i = 0; i < PTE_PREFETCH_NUM; i++, spte++) {
2181                 if (*spte != shadow_trap_nonpresent_pte || spte == sptep) {
2182                         if (!start)
2183                                 continue;
2184                         if (direct_pte_prefetch_many(vcpu, sp, start, spte) < 0)
2185                                 break;
2186                         start = NULL;
2187                 } else if (!start)
2188                         start = spte;
2189         }
2190 }
2191
2192 static void direct_pte_prefetch(struct kvm_vcpu *vcpu, u64 *sptep)
2193 {
2194         struct kvm_mmu_page *sp;
2195
2196         /*
2197          * Since it's no accessed bit on EPT, it's no way to
2198          * distinguish between actually accessed translations
2199          * and prefetched, so disable pte prefetch if EPT is
2200          * enabled.
2201          */
2202         if (!shadow_accessed_mask)
2203                 return;
2204
2205         sp = page_header(__pa(sptep));
2206         if (sp->role.level > PT_PAGE_TABLE_LEVEL)
2207                 return;
2208
2209         __direct_pte_prefetch(vcpu, sp, sptep);
2210 }
2211
2212 static int __direct_map(struct kvm_vcpu *vcpu, gpa_t v, int write,
2213                         int level, gfn_t gfn, pfn_t pfn)
2214 {
2215         struct kvm_shadow_walk_iterator iterator;
2216         struct kvm_mmu_page *sp;
2217         int pt_write = 0;
2218         gfn_t pseudo_gfn;
2219
2220         for_each_shadow_entry(vcpu, (u64)gfn << PAGE_SHIFT, iterator) {
2221                 if (iterator.level == level) {
2222                         mmu_set_spte(vcpu, iterator.sptep, ACC_ALL, ACC_ALL,
2223                                      0, write, 1, &pt_write,
2224                                      level, gfn, pfn, false, true);
2225                         direct_pte_prefetch(vcpu, iterator.sptep);
2226                         ++vcpu->stat.pf_fixed;
2227                         break;
2228                 }
2229
2230                 if (*iterator.sptep == shadow_trap_nonpresent_pte) {
2231                         u64 base_addr = iterator.addr;
2232
2233                         base_addr &= PT64_LVL_ADDR_MASK(iterator.level);
2234                         pseudo_gfn = base_addr >> PAGE_SHIFT;
2235                         sp = kvm_mmu_get_page(vcpu, pseudo_gfn, iterator.addr,
2236                                               iterator.level - 1,
2237                                               1, ACC_ALL, iterator.sptep);
2238                         if (!sp) {
2239                                 pgprintk("nonpaging_map: ENOMEM\n");
2240                                 kvm_release_pfn_clean(pfn);
2241                                 return -ENOMEM;
2242                         }
2243
2244                         __set_spte(iterator.sptep,
2245                                    __pa(sp->spt)
2246                                    | PT_PRESENT_MASK | PT_WRITABLE_MASK
2247                                    | shadow_user_mask | shadow_x_mask
2248                                    | shadow_accessed_mask);
2249                 }
2250         }
2251         return pt_write;
2252 }
2253
2254 static void kvm_send_hwpoison_signal(struct kvm *kvm, gfn_t gfn)
2255 {
2256         char buf[1];
2257         void __user *hva;
2258         int r;
2259
2260         /* Touch the page, so send SIGBUS */
2261         hva = (void __user *)gfn_to_hva(kvm, gfn);
2262         r = copy_from_user(buf, hva, 1);
2263 }
2264
2265 static int kvm_handle_bad_page(struct kvm *kvm, gfn_t gfn, pfn_t pfn)
2266 {
2267         kvm_release_pfn_clean(pfn);
2268         if (is_hwpoison_pfn(pfn)) {
2269                 kvm_send_hwpoison_signal(kvm, gfn);
2270                 return 0;
2271         } else if (is_fault_pfn(pfn))
2272                 return -EFAULT;
2273
2274         return 1;
2275 }
2276
2277 static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, int write, gfn_t gfn)
2278 {
2279         int r;
2280         int level;
2281         pfn_t pfn;
2282         unsigned long mmu_seq;
2283
2284         level = mapping_level(vcpu, gfn);
2285
2286         /*
2287          * This path builds a PAE pagetable - so we can map 2mb pages at
2288          * maximum. Therefore check if the level is larger than that.
2289          */
2290         if (level > PT_DIRECTORY_LEVEL)
2291                 level = PT_DIRECTORY_LEVEL;
2292
2293         gfn &= ~(KVM_PAGES_PER_HPAGE(level) - 1);
2294
2295         mmu_seq = vcpu->kvm->mmu_notifier_seq;
2296         smp_rmb();
2297         pfn = gfn_to_pfn(vcpu->kvm, gfn);
2298
2299         /* mmio */
2300         if (is_error_pfn(pfn))
2301                 return kvm_handle_bad_page(vcpu->kvm, gfn, pfn);
2302
2303         spin_lock(&vcpu->kvm->mmu_lock);
2304         if (mmu_notifier_retry(vcpu, mmu_seq))
2305                 goto out_unlock;
2306         kvm_mmu_free_some_pages(vcpu);
2307         r = __direct_map(vcpu, v, write, level, gfn, pfn);
2308         spin_unlock(&vcpu->kvm->mmu_lock);
2309
2310
2311         return r;
2312
2313 out_unlock:
2314         spin_unlock(&vcpu->kvm->mmu_lock);
2315         kvm_release_pfn_clean(pfn);
2316         return 0;
2317 }
2318
2319
2320 static void mmu_free_roots(struct kvm_vcpu *vcpu)
2321 {
2322         int i;
2323         struct kvm_mmu_page *sp;
2324         LIST_HEAD(invalid_list);
2325
2326         if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
2327                 return;
2328         spin_lock(&vcpu->kvm->mmu_lock);
2329         if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL &&
2330             (vcpu->arch.mmu.root_level == PT64_ROOT_LEVEL ||
2331              vcpu->arch.mmu.direct_map)) {
2332                 hpa_t root = vcpu->arch.mmu.root_hpa;
2333
2334                 sp = page_header(root);
2335                 --sp->root_count;
2336                 if (!sp->root_count && sp->role.invalid) {
2337                         kvm_mmu_prepare_zap_page(vcpu->kvm, sp, &invalid_list);
2338                         kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
2339                 }
2340                 vcpu->arch.mmu.root_hpa = INVALID_PAGE;
2341                 spin_unlock(&vcpu->kvm->mmu_lock);
2342                 return;
2343         }
2344         for (i = 0; i < 4; ++i) {
2345                 hpa_t root = vcpu->arch.mmu.pae_root[i];
2346
2347                 if (root) {
2348                         root &= PT64_BASE_ADDR_MASK;
2349                         sp = page_header(root);
2350                         --sp->root_count;
2351                         if (!sp->root_count && sp->role.invalid)
2352                                 kvm_mmu_prepare_zap_page(vcpu->kvm, sp,
2353                                                          &invalid_list);
2354                 }
2355                 vcpu->arch.mmu.pae_root[i] = INVALID_PAGE;
2356         }
2357         kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
2358         spin_unlock(&vcpu->kvm->mmu_lock);
2359         vcpu->arch.mmu.root_hpa = INVALID_PAGE;
2360 }
2361
2362 static int mmu_check_root(struct kvm_vcpu *vcpu, gfn_t root_gfn)
2363 {
2364         int ret = 0;
2365
2366         if (!kvm_is_visible_gfn(vcpu->kvm, root_gfn)) {
2367                 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
2368                 ret = 1;
2369         }
2370
2371         return ret;
2372 }
2373
2374 static int mmu_alloc_direct_roots(struct kvm_vcpu *vcpu)
2375 {
2376         struct kvm_mmu_page *sp;
2377         unsigned i;
2378
2379         if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) {
2380                 spin_lock(&vcpu->kvm->mmu_lock);
2381                 kvm_mmu_free_some_pages(vcpu);
2382                 sp = kvm_mmu_get_page(vcpu, 0, 0, PT64_ROOT_LEVEL,
2383                                       1, ACC_ALL, NULL);
2384                 ++sp->root_count;
2385                 spin_unlock(&vcpu->kvm->mmu_lock);
2386                 vcpu->arch.mmu.root_hpa = __pa(sp->spt);
2387         } else if (vcpu->arch.mmu.shadow_root_level == PT32E_ROOT_LEVEL) {
2388                 for (i = 0; i < 4; ++i) {
2389                         hpa_t root = vcpu->arch.mmu.pae_root[i];
2390
2391                         ASSERT(!VALID_PAGE(root));
2392                         spin_lock(&vcpu->kvm->mmu_lock);
2393                         kvm_mmu_free_some_pages(vcpu);
2394                         sp = kvm_mmu_get_page(vcpu, i << 30, i << 30,
2395                                               PT32_ROOT_LEVEL, 1, ACC_ALL,
2396                                               NULL);
2397                         root = __pa(sp->spt);
2398                         ++sp->root_count;
2399                         spin_unlock(&vcpu->kvm->mmu_lock);
2400                         vcpu->arch.mmu.pae_root[i] = root | PT_PRESENT_MASK;
2401                 }
2402                 vcpu->arch.mmu.root_hpa = __pa(vcpu->arch.mmu.pae_root);
2403         } else
2404                 BUG();
2405
2406         return 0;
2407 }
2408
2409 static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu)
2410 {
2411         struct kvm_mmu_page *sp;
2412         u64 pdptr, pm_mask;
2413         gfn_t root_gfn;
2414         int i;
2415
2416         root_gfn = vcpu->arch.mmu.get_cr3(vcpu) >> PAGE_SHIFT;
2417
2418         if (mmu_check_root(vcpu, root_gfn))
2419                 return 1;
2420
2421         /*
2422          * Do we shadow a long mode page table? If so we need to
2423          * write-protect the guests page table root.
2424          */
2425         if (vcpu->arch.mmu.root_level == PT64_ROOT_LEVEL) {
2426                 hpa_t root = vcpu->arch.mmu.root_hpa;
2427
2428                 ASSERT(!VALID_PAGE(root));
2429
2430                 spin_lock(&vcpu->kvm->mmu_lock);
2431                 kvm_mmu_free_some_pages(vcpu);
2432                 sp = kvm_mmu_get_page(vcpu, root_gfn, 0, PT64_ROOT_LEVEL,
2433                                       0, ACC_ALL, NULL);
2434                 root = __pa(sp->spt);
2435                 ++sp->root_count;
2436                 spin_unlock(&vcpu->kvm->mmu_lock);
2437                 vcpu->arch.mmu.root_hpa = root;
2438                 return 0;
2439         }
2440
2441         /*
2442          * We shadow a 32 bit page table. This may be a legacy 2-level
2443          * or a PAE 3-level page table. In either case we need to be aware that
2444          * the shadow page table may be a PAE or a long mode page table.
2445          */
2446         pm_mask = PT_PRESENT_MASK;
2447         if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL)
2448                 pm_mask |= PT_ACCESSED_MASK | PT_WRITABLE_MASK | PT_USER_MASK;
2449
2450         for (i = 0; i < 4; ++i) {
2451                 hpa_t root = vcpu->arch.mmu.pae_root[i];
2452
2453                 ASSERT(!VALID_PAGE(root));
2454                 if (vcpu->arch.mmu.root_level == PT32E_ROOT_LEVEL) {
2455                         pdptr = kvm_pdptr_read_mmu(vcpu, &vcpu->arch.mmu, i);
2456                         if (!is_present_gpte(pdptr)) {
2457                                 vcpu->arch.mmu.pae_root[i] = 0;
2458                                 continue;
2459                         }
2460                         root_gfn = pdptr >> PAGE_SHIFT;
2461                         if (mmu_check_root(vcpu, root_gfn))
2462                                 return 1;
2463                 }
2464                 spin_lock(&vcpu->kvm->mmu_lock);
2465                 kvm_mmu_free_some_pages(vcpu);
2466                 sp = kvm_mmu_get_page(vcpu, root_gfn, i << 30,
2467                                       PT32_ROOT_LEVEL, 0,
2468                                       ACC_ALL, NULL);
2469                 root = __pa(sp->spt);
2470                 ++sp->root_count;
2471                 spin_unlock(&vcpu->kvm->mmu_lock);
2472
2473                 vcpu->arch.mmu.pae_root[i] = root | pm_mask;
2474         }
2475         vcpu->arch.mmu.root_hpa = __pa(vcpu->arch.mmu.pae_root);
2476
2477         /*
2478          * If we shadow a 32 bit page table with a long mode page
2479          * table we enter this path.
2480          */
2481         if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) {
2482                 if (vcpu->arch.mmu.lm_root == NULL) {
2483                         /*
2484                          * The additional page necessary for this is only
2485                          * allocated on demand.
2486                          */
2487
2488                         u64 *lm_root;
2489
2490                         lm_root = (void*)get_zeroed_page(GFP_KERNEL);
2491                         if (lm_root == NULL)
2492                                 return 1;
2493
2494                         lm_root[0] = __pa(vcpu->arch.mmu.pae_root) | pm_mask;
2495
2496                         vcpu->arch.mmu.lm_root = lm_root;
2497                 }
2498
2499                 vcpu->arch.mmu.root_hpa = __pa(vcpu->arch.mmu.lm_root);
2500         }
2501
2502         return 0;
2503 }
2504
2505 static int mmu_alloc_roots(struct kvm_vcpu *vcpu)
2506 {
2507         if (vcpu->arch.mmu.direct_map)
2508                 return mmu_alloc_direct_roots(vcpu);
2509         else
2510                 return mmu_alloc_shadow_roots(vcpu);
2511 }
2512
2513 static void mmu_sync_roots(struct kvm_vcpu *vcpu)
2514 {
2515         int i;
2516         struct kvm_mmu_page *sp;
2517
2518         if (vcpu->arch.mmu.direct_map)
2519                 return;
2520
2521         if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
2522                 return;
2523
2524         trace_kvm_mmu_audit(vcpu, AUDIT_PRE_SYNC);
2525         if (vcpu->arch.mmu.root_level == PT64_ROOT_LEVEL) {
2526                 hpa_t root = vcpu->arch.mmu.root_hpa;
2527                 sp = page_header(root);
2528                 mmu_sync_children(vcpu, sp);
2529                 return;
2530         }
2531         for (i = 0; i < 4; ++i) {
2532                 hpa_t root = vcpu->arch.mmu.pae_root[i];
2533
2534                 if (root && VALID_PAGE(root)) {
2535                         root &= PT64_BASE_ADDR_MASK;
2536                         sp = page_header(root);
2537                         mmu_sync_children(vcpu, sp);
2538                 }
2539         }
2540         trace_kvm_mmu_audit(vcpu, AUDIT_POST_SYNC);
2541 }
2542
2543 void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu)
2544 {
2545         spin_lock(&vcpu->kvm->mmu_lock);
2546         mmu_sync_roots(vcpu);
2547         spin_unlock(&vcpu->kvm->mmu_lock);
2548 }
2549
2550 static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, gva_t vaddr,
2551                                   u32 access, u32 *error)
2552 {
2553         if (error)
2554                 *error = 0;
2555         return vaddr;
2556 }
2557
2558 static gpa_t nonpaging_gva_to_gpa_nested(struct kvm_vcpu *vcpu, gva_t vaddr,
2559                                          u32 access, u32 *error)
2560 {
2561         if (error)
2562                 *error = 0;
2563         return vcpu->arch.nested_mmu.translate_gpa(vcpu, vaddr, access);
2564 }
2565
2566 static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva,
2567                                 u32 error_code)
2568 {
2569         gfn_t gfn;
2570         int r;
2571
2572         pgprintk("%s: gva %lx error %x\n", __func__, gva, error_code);
2573         r = mmu_topup_memory_caches(vcpu);
2574         if (r)
2575                 return r;
2576
2577         ASSERT(vcpu);
2578         ASSERT(VALID_PAGE(vcpu->arch.mmu.root_hpa));
2579
2580         gfn = gva >> PAGE_SHIFT;
2581
2582         return nonpaging_map(vcpu, gva & PAGE_MASK,
2583                              error_code & PFERR_WRITE_MASK, gfn);
2584 }
2585
2586 static int tdp_page_fault(struct kvm_vcpu *vcpu, gva_t gpa,
2587                                 u32 error_code)
2588 {
2589         pfn_t pfn;
2590         int r;
2591         int level;
2592         gfn_t gfn = gpa >> PAGE_SHIFT;
2593         unsigned long mmu_seq;
2594
2595         ASSERT(vcpu);
2596         ASSERT(VALID_PAGE(vcpu->arch.mmu.root_hpa));
2597
2598         r = mmu_topup_memory_caches(vcpu);
2599         if (r)
2600                 return r;
2601
2602         level = mapping_level(vcpu, gfn);
2603
2604         gfn &= ~(KVM_PAGES_PER_HPAGE(level) - 1);
2605
2606         mmu_seq = vcpu->kvm->mmu_notifier_seq;
2607         smp_rmb();
2608         pfn = gfn_to_pfn(vcpu->kvm, gfn);
2609         if (is_error_pfn(pfn))
2610                 return kvm_handle_bad_page(vcpu->kvm, gfn, pfn);
2611         spin_lock(&vcpu->kvm->mmu_lock);
2612         if (mmu_notifier_retry(vcpu, mmu_seq))
2613                 goto out_unlock;
2614         kvm_mmu_free_some_pages(vcpu);
2615         r = __direct_map(vcpu, gpa, error_code & PFERR_WRITE_MASK,
2616                          level, gfn, pfn);
2617         spin_unlock(&vcpu->kvm->mmu_lock);
2618
2619         return r;
2620
2621 out_unlock:
2622         spin_unlock(&vcpu->kvm->mmu_lock);
2623         kvm_release_pfn_clean(pfn);
2624         return 0;
2625 }
2626
2627 static void nonpaging_free(struct kvm_vcpu *vcpu)
2628 {
2629         mmu_free_roots(vcpu);
2630 }
2631
2632 static int nonpaging_init_context(struct kvm_vcpu *vcpu,
2633                                   struct kvm_mmu *context)
2634 {
2635         context->new_cr3 = nonpaging_new_cr3;
2636         context->page_fault = nonpaging_page_fault;
2637         context->gva_to_gpa = nonpaging_gva_to_gpa;
2638         context->free = nonpaging_free;
2639         context->prefetch_page = nonpaging_prefetch_page;
2640         context->sync_page = nonpaging_sync_page;
2641         context->invlpg = nonpaging_invlpg;
2642         context->root_level = 0;
2643         context->shadow_root_level = PT32E_ROOT_LEVEL;
2644         context->root_hpa = INVALID_PAGE;
2645         context->direct_map = true;
2646         context->nx = false;
2647         return 0;
2648 }
2649
2650 void kvm_mmu_flush_tlb(struct kvm_vcpu *vcpu)
2651 {
2652         ++vcpu->stat.tlb_flush;
2653         kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2654 }
2655
2656 static void paging_new_cr3(struct kvm_vcpu *vcpu)
2657 {
2658         pgprintk("%s: cr3 %lx\n", __func__, vcpu->arch.cr3);
2659         mmu_free_roots(vcpu);
2660 }
2661
2662 static unsigned long get_cr3(struct kvm_vcpu *vcpu)
2663 {
2664         return vcpu->arch.cr3;
2665 }
2666
2667 static void inject_page_fault(struct kvm_vcpu *vcpu)
2668 {
2669         vcpu->arch.mmu.inject_page_fault(vcpu);
2670 }
2671
2672 static void paging_free(struct kvm_vcpu *vcpu)
2673 {
2674         nonpaging_free(vcpu);
2675 }
2676
2677 static bool is_rsvd_bits_set(struct kvm_mmu *mmu, u64 gpte, int level)
2678 {
2679         int bit7;
2680
2681         bit7 = (gpte >> 7) & 1;
2682         return (gpte & mmu->rsvd_bits_mask[bit7][level-1]) != 0;
2683 }
2684
2685 #define PTTYPE 64
2686 #include "paging_tmpl.h"
2687 #undef PTTYPE
2688
2689 #define PTTYPE 32
2690 #include "paging_tmpl.h"
2691 #undef PTTYPE
2692
2693 static void reset_rsvds_bits_mask(struct kvm_vcpu *vcpu,
2694                                   struct kvm_mmu *context,
2695                                   int level)
2696 {
2697         int maxphyaddr = cpuid_maxphyaddr(vcpu);
2698         u64 exb_bit_rsvd = 0;
2699
2700         if (!context->nx)
2701                 exb_bit_rsvd = rsvd_bits(63, 63);
2702         switch (level) {
2703         case PT32_ROOT_LEVEL:
2704                 /* no rsvd bits for 2 level 4K page table entries */
2705                 context->rsvd_bits_mask[0][1] = 0;
2706                 context->rsvd_bits_mask[0][0] = 0;
2707                 context->rsvd_bits_mask[1][0] = context->rsvd_bits_mask[0][0];
2708
2709                 if (!is_pse(vcpu)) {
2710                         context->rsvd_bits_mask[1][1] = 0;
2711                         break;
2712                 }
2713
2714                 if (is_cpuid_PSE36())
2715                         /* 36bits PSE 4MB page */
2716                         context->rsvd_bits_mask[1][1] = rsvd_bits(17, 21);
2717                 else
2718                         /* 32 bits PSE 4MB page */
2719                         context->rsvd_bits_mask[1][1] = rsvd_bits(13, 21);
2720                 break;
2721         case PT32E_ROOT_LEVEL:
2722                 context->rsvd_bits_mask[0][2] =
2723                         rsvd_bits(maxphyaddr, 63) |
2724                         rsvd_bits(7, 8) | rsvd_bits(1, 2);      /* PDPTE */
2725                 context->rsvd_bits_mask[0][1] = exb_bit_rsvd |
2726                         rsvd_bits(maxphyaddr, 62);      /* PDE */
2727                 context->rsvd_bits_mask[0][0] = exb_bit_rsvd |
2728                         rsvd_bits(maxphyaddr, 62);      /* PTE */
2729                 context->rsvd_bits_mask[1][1] = exb_bit_rsvd |
2730                         rsvd_bits(maxphyaddr, 62) |
2731                         rsvd_bits(13, 20);              /* large page */
2732                 context->rsvd_bits_mask[1][0] = context->rsvd_bits_mask[0][0];
2733                 break;
2734         case PT64_ROOT_LEVEL:
2735                 context->rsvd_bits_mask[0][3] = exb_bit_rsvd |
2736                         rsvd_bits(maxphyaddr, 51) | rsvd_bits(7, 8);
2737                 context->rsvd_bits_mask[0][2] = exb_bit_rsvd |
2738                         rsvd_bits(maxphyaddr, 51) | rsvd_bits(7, 8);
2739                 context->rsvd_bits_mask[0][1] = exb_bit_rsvd |
2740                         rsvd_bits(maxphyaddr, 51);
2741                 context->rsvd_bits_mask[0][0] = exb_bit_rsvd |
2742                         rsvd_bits(maxphyaddr, 51);
2743                 context->rsvd_bits_mask[1][3] = context->rsvd_bits_mask[0][3];
2744                 context->rsvd_bits_mask[1][2] = exb_bit_rsvd |
2745                         rsvd_bits(maxphyaddr, 51) |
2746                         rsvd_bits(13, 29);
2747                 context->rsvd_bits_mask[1][1] = exb_bit_rsvd |
2748                         rsvd_bits(maxphyaddr, 51) |
2749                         rsvd_bits(13, 20);              /* large page */
2750                 context->rsvd_bits_mask[1][0] = context->rsvd_bits_mask[0][0];
2751                 break;
2752         }
2753 }
2754
2755 static int paging64_init_context_common(struct kvm_vcpu *vcpu,
2756                                         struct kvm_mmu *context,
2757                                         int level)
2758 {
2759         context->nx = is_nx(vcpu);
2760
2761         reset_rsvds_bits_mask(vcpu, context, level);
2762
2763         ASSERT(is_pae(vcpu));
2764         context->new_cr3 = paging_new_cr3;
2765         context->page_fault = paging64_page_fault;
2766         context->gva_to_gpa = paging64_gva_to_gpa;
2767         context->prefetch_page = paging64_prefetch_page;
2768         context->sync_page = paging64_sync_page;
2769         context->invlpg = paging64_invlpg;
2770         context->free = paging_free;
2771         context->root_level = level;
2772         context->shadow_root_level = level;
2773         context->root_hpa = INVALID_PAGE;
2774         context->direct_map = false;
2775         return 0;
2776 }
2777
2778 static int paging64_init_context(struct kvm_vcpu *vcpu,
2779                                  struct kvm_mmu *context)
2780 {
2781         return paging64_init_context_common(vcpu, context, PT64_ROOT_LEVEL);
2782 }
2783
2784 static int paging32_init_context(struct kvm_vcpu *vcpu,
2785                                  struct kvm_mmu *context)
2786 {
2787         context->nx = false;
2788
2789         reset_rsvds_bits_mask(vcpu, context, PT32_ROOT_LEVEL);
2790
2791         context->new_cr3 = paging_new_cr3;
2792         context->page_fault = paging32_page_fault;
2793         context->gva_to_gpa = paging32_gva_to_gpa;
2794         context->free = paging_free;
2795         context->prefetch_page = paging32_prefetch_page;
2796         context->sync_page = paging32_sync_page;
2797         context->invlpg = paging32_invlpg;
2798         context->root_level = PT32_ROOT_LEVEL;
2799         context->shadow_root_level = PT32E_ROOT_LEVEL;
2800         context->root_hpa = INVALID_PAGE;
2801         context->direct_map = false;
2802         return 0;
2803 }
2804
2805 static int paging32E_init_context(struct kvm_vcpu *vcpu,
2806                                   struct kvm_mmu *context)
2807 {
2808         return paging64_init_context_common(vcpu, context, PT32E_ROOT_LEVEL);
2809 }
2810
2811 static int init_kvm_tdp_mmu(struct kvm_vcpu *vcpu)
2812 {
2813         struct kvm_mmu *context = vcpu->arch.walk_mmu;
2814
2815         context->new_cr3 = nonpaging_new_cr3;
2816         context->page_fault = tdp_page_fault;
2817         context->free = nonpaging_free;
2818         context->prefetch_page = nonpaging_prefetch_page;
2819         context->sync_page = nonpaging_sync_page;
2820         context->invlpg = nonpaging_invlpg;
2821         context->shadow_root_level = kvm_x86_ops->get_tdp_level();
2822         context->root_hpa = INVALID_PAGE;
2823         context->direct_map = true;
2824         context->set_cr3 = kvm_x86_ops->set_tdp_cr3;
2825         context->get_cr3 = get_cr3;
2826         context->inject_page_fault = kvm_inject_page_fault;
2827         context->nx = is_nx(vcpu);
2828
2829         if (!is_paging(vcpu)) {
2830                 context->nx = false;
2831                 context->gva_to_gpa = nonpaging_gva_to_gpa;
2832                 context->root_level = 0;
2833         } else if (is_long_mode(vcpu)) {
2834                 context->nx = is_nx(vcpu);
2835                 reset_rsvds_bits_mask(vcpu, context, PT64_ROOT_LEVEL);
2836                 context->gva_to_gpa = paging64_gva_to_gpa;
2837                 context->root_level = PT64_ROOT_LEVEL;
2838         } else if (is_pae(vcpu)) {
2839                 context->nx = is_nx(vcpu);
2840                 reset_rsvds_bits_mask(vcpu, context, PT32E_ROOT_LEVEL);
2841                 context->gva_to_gpa = paging64_gva_to_gpa;
2842                 context->root_level = PT32E_ROOT_LEVEL;
2843         } else {
2844                 context->nx = false;
2845                 reset_rsvds_bits_mask(vcpu, context, PT32_ROOT_LEVEL);
2846                 context->gva_to_gpa = paging32_gva_to_gpa;
2847                 context->root_level = PT32_ROOT_LEVEL;
2848         }
2849
2850         return 0;
2851 }
2852
2853 int kvm_init_shadow_mmu(struct kvm_vcpu *vcpu, struct kvm_mmu *context)
2854 {
2855         int r;
2856         ASSERT(vcpu);
2857         ASSERT(!VALID_PAGE(vcpu->arch.mmu.root_hpa));
2858
2859         if (!is_paging(vcpu))
2860                 r = nonpaging_init_context(vcpu, context);
2861         else if (is_long_mode(vcpu))
2862                 r = paging64_init_context(vcpu, context);
2863         else if (is_pae(vcpu))
2864                 r = paging32E_init_context(vcpu, context);
2865         else
2866                 r = paging32_init_context(vcpu, context);
2867
2868         vcpu->arch.mmu.base_role.cr4_pae = !!is_pae(vcpu);
2869         vcpu->arch.mmu.base_role.cr0_wp  = is_write_protection(vcpu);
2870
2871         return r;
2872 }
2873 EXPORT_SYMBOL_GPL(kvm_init_shadow_mmu);
2874
2875 static int init_kvm_softmmu(struct kvm_vcpu *vcpu)
2876 {
2877         int r = kvm_init_shadow_mmu(vcpu, vcpu->arch.walk_mmu);
2878
2879         vcpu->arch.walk_mmu->set_cr3           = kvm_x86_ops->set_cr3;
2880         vcpu->arch.walk_mmu->get_cr3           = get_cr3;
2881         vcpu->arch.walk_mmu->inject_page_fault = kvm_inject_page_fault;
2882
2883         return r;
2884 }
2885
2886 static int init_kvm_nested_mmu(struct kvm_vcpu *vcpu)
2887 {
2888         struct kvm_mmu *g_context = &vcpu->arch.nested_mmu;
2889
2890         g_context->get_cr3           = get_cr3;
2891         g_context->inject_page_fault = kvm_inject_page_fault;
2892
2893         /*
2894          * Note that arch.mmu.gva_to_gpa translates l2_gva to l1_gpa. The
2895          * translation of l2_gpa to l1_gpa addresses is done using the
2896          * arch.nested_mmu.gva_to_gpa function. Basically the gva_to_gpa
2897          * functions between mmu and nested_mmu are swapped.
2898          */
2899         if (!is_paging(vcpu)) {
2900                 g_context->nx = false;
2901                 g_context->root_level = 0;
2902                 g_context->gva_to_gpa = nonpaging_gva_to_gpa_nested;
2903         } else if (is_long_mode(vcpu)) {
2904                 g_context->nx = is_nx(vcpu);
2905                 reset_rsvds_bits_mask(vcpu, g_context, PT64_ROOT_LEVEL);
2906                 g_context->root_level = PT64_ROOT_LEVEL;
2907                 g_context->gva_to_gpa = paging64_gva_to_gpa_nested;
2908         } else if (is_pae(vcpu)) {
2909                 g_context->nx = is_nx(vcpu);
2910                 reset_rsvds_bits_mask(vcpu, g_context, PT32E_ROOT_LEVEL);
2911                 g_context->root_level = PT32E_ROOT_LEVEL;
2912                 g_context->gva_to_gpa = paging64_gva_to_gpa_nested;
2913         } else {
2914                 g_context->nx = false;
2915                 reset_rsvds_bits_mask(vcpu, g_context, PT32_ROOT_LEVEL);
2916                 g_context->root_level = PT32_ROOT_LEVEL;
2917                 g_context->gva_to_gpa = paging32_gva_to_gpa_nested;
2918         }
2919
2920         return 0;
2921 }
2922
2923 static int init_kvm_mmu(struct kvm_vcpu *vcpu)
2924 {
2925         vcpu->arch.update_pte.pfn = bad_pfn;
2926
2927         if (mmu_is_nested(vcpu))
2928                 return init_kvm_nested_mmu(vcpu);
2929         else if (tdp_enabled)
2930                 return init_kvm_tdp_mmu(vcpu);
2931         else
2932                 return init_kvm_softmmu(vcpu);
2933 }
2934
2935 static void destroy_kvm_mmu(struct kvm_vcpu *vcpu)
2936 {
2937         ASSERT(vcpu);
2938         if (VALID_PAGE(vcpu->arch.mmu.root_hpa))
2939                 /* mmu.free() should set root_hpa = INVALID_PAGE */
2940                 vcpu->arch.mmu.free(vcpu);
2941 }
2942
2943 int kvm_mmu_reset_context(struct kvm_vcpu *vcpu)
2944 {
2945         destroy_kvm_mmu(vcpu);
2946         return init_kvm_mmu(vcpu);
2947 }
2948 EXPORT_SYMBOL_GPL(kvm_mmu_reset_context);
2949
2950 int kvm_mmu_load(struct kvm_vcpu *vcpu)
2951 {
2952         int r;
2953
2954         r = mmu_topup_memory_caches(vcpu);
2955         if (r)
2956                 goto out;
2957         r = mmu_alloc_roots(vcpu);
2958         spin_lock(&vcpu->kvm->mmu_lock);
2959         mmu_sync_roots(vcpu);
2960         spin_unlock(&vcpu->kvm->mmu_lock);
2961         if (r)
2962                 goto out;
2963         /* set_cr3() should ensure TLB has been flushed */
2964         vcpu->arch.mmu.set_cr3(vcpu, vcpu->arch.mmu.root_hpa);
2965 out:
2966         return r;
2967 }
2968 EXPORT_SYMBOL_GPL(kvm_mmu_load);
2969
2970 void kvm_mmu_unload(struct kvm_vcpu *vcpu)
2971 {
2972         mmu_free_roots(vcpu);
2973 }
2974 EXPORT_SYMBOL_GPL(kvm_mmu_unload);
2975
2976 static void mmu_pte_write_zap_pte(struct kvm_vcpu *vcpu,
2977                                   struct kvm_mmu_page *sp,
2978                                   u64 *spte)
2979 {
2980         u64 pte;
2981         struct kvm_mmu_page *child;
2982
2983         pte = *spte;
2984         if (is_shadow_present_pte(pte)) {
2985                 if (is_last_spte(pte, sp->role.level))
2986                         drop_spte(vcpu->kvm, spte, shadow_trap_nonpresent_pte);
2987                 else {
2988                         child = page_header(pte & PT64_BASE_ADDR_MASK);
2989                         mmu_page_remove_parent_pte(child, spte);
2990                 }
2991         }
2992         __set_spte(spte, shadow_trap_nonpresent_pte);
2993         if (is_large_pte(pte))
2994                 --vcpu->kvm->stat.lpages;
2995 }
2996
2997 static void mmu_pte_write_new_pte(struct kvm_vcpu *vcpu,
2998                                   struct kvm_mmu_page *sp,
2999                                   u64 *spte,
3000                                   const void *new)
3001 {
3002         if (sp->role.level != PT_PAGE_TABLE_LEVEL) {
3003                 ++vcpu->kvm->stat.mmu_pde_zapped;
3004                 return;
3005         }
3006
3007         if (is_rsvd_bits_set(&vcpu->arch.mmu, *(u64 *)new, PT_PAGE_TABLE_LEVEL))
3008                 return;
3009
3010         ++vcpu->kvm->stat.mmu_pte_updated;
3011         if (!sp->role.cr4_pae)
3012                 paging32_update_pte(vcpu, sp, spte, new);
3013         else
3014                 paging64_update_pte(vcpu, sp, spte, new);
3015 }
3016
3017 static bool need_remote_flush(u64 old, u64 new)
3018 {
3019         if (!is_shadow_present_pte(old))
3020                 return false;
3021         if (!is_shadow_present_pte(new))
3022                 return true;
3023         if ((old ^ new) & PT64_BASE_ADDR_MASK)
3024                 return true;
3025         old ^= PT64_NX_MASK;
3026         new ^= PT64_NX_MASK;
3027         return (old & ~new & PT64_PERM_MASK) != 0;
3028 }
3029
3030 static void mmu_pte_write_flush_tlb(struct kvm_vcpu *vcpu, bool zap_page,
3031                                     bool remote_flush, bool local_flush)
3032 {
3033         if (zap_page)
3034                 return;
3035
3036         if (remote_flush)
3037                 kvm_flush_remote_tlbs(vcpu->kvm);
3038         else if (local_flush)
3039                 kvm_mmu_flush_tlb(vcpu);
3040 }
3041
3042 static bool last_updated_pte_accessed(struct kvm_vcpu *vcpu)
3043 {
3044         u64 *spte = vcpu->arch.last_pte_updated;
3045
3046         return !!(spte && (*spte & shadow_accessed_mask));
3047 }
3048
3049 static void mmu_guess_page_from_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,
3050                                           u64 gpte)
3051 {
3052         gfn_t gfn;
3053         pfn_t pfn;
3054
3055         if (!is_present_gpte(gpte))
3056                 return;
3057         gfn = (gpte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT;
3058
3059         vcpu->arch.update_pte.mmu_seq = vcpu->kvm->mmu_notifier_seq;
3060         smp_rmb();
3061         pfn = gfn_to_pfn(vcpu->kvm, gfn);
3062
3063         if (is_error_pfn(pfn)) {
3064                 kvm_release_pfn_clean(pfn);
3065                 return;
3066         }
3067         vcpu->arch.update_pte.gfn = gfn;
3068         vcpu->arch.update_pte.pfn = pfn;
3069 }
3070
3071 static void kvm_mmu_access_page(struct kvm_vcpu *vcpu, gfn_t gfn)
3072 {
3073         u64 *spte = vcpu->arch.last_pte_updated;
3074
3075         if (spte
3076             && vcpu->arch.last_pte_gfn == gfn
3077             && shadow_accessed_mask
3078             && !(*spte & shadow_accessed_mask)
3079             && is_shadow_present_pte(*spte))
3080                 set_bit(PT_ACCESSED_SHIFT, (unsigned long *)spte);
3081 }
3082
3083 void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,
3084                        const u8 *new, int bytes,
3085                        bool guest_initiated)
3086 {
3087         gfn_t gfn = gpa >> PAGE_SHIFT;
3088         union kvm_mmu_page_role mask = { .word = 0 };
3089         struct kvm_mmu_page *sp;
3090         struct hlist_node *node;
3091         LIST_HEAD(invalid_list);
3092         u64 entry, gentry;
3093         u64 *spte;
3094         unsigned offset = offset_in_page(gpa);
3095         unsigned pte_size;
3096         unsigned page_offset;
3097         unsigned misaligned;
3098         unsigned quadrant;
3099         int level;
3100         int flooded = 0;
3101         int npte;
3102         int r;
3103         int invlpg_counter;
3104         bool remote_flush, local_flush, zap_page;
3105
3106         zap_page = remote_flush = local_flush = false;
3107
3108         pgprintk("%s: gpa %llx bytes %d\n", __func__, gpa, bytes);
3109
3110         invlpg_counter = atomic_read(&vcpu->kvm->arch.invlpg_counter);
3111
3112         /*
3113          * Assume that the pte write on a page table of the same type
3114          * as the current vcpu paging mode.  This is nearly always true
3115          * (might be false while changing modes).  Note it is verified later
3116          * by update_pte().
3117          */
3118         if ((is_pae(vcpu) && bytes == 4) || !new) {
3119                 /* Handle a 32-bit guest writing two halves of a 64-bit gpte */
3120                 if (is_pae(vcpu)) {
3121                         gpa &= ~(gpa_t)7;
3122                         bytes = 8;
3123                 }
3124                 r = kvm_read_guest(vcpu->kvm, gpa, &gentry, min(bytes, 8));
3125                 if (r)
3126                         gentry = 0;
3127                 new = (const u8 *)&gentry;
3128         }
3129
3130         switch (bytes) {
3131         case 4:
3132                 gentry = *(const u32 *)new;
3133                 break;
3134         case 8:
3135                 gentry = *(const u64 *)new;
3136                 break;
3137         default:
3138                 gentry = 0;
3139                 break;
3140         }
3141
3142         mmu_guess_page_from_pte_write(vcpu, gpa, gentry);
3143         spin_lock(&vcpu->kvm->mmu_lock);
3144         if (atomic_read(&vcpu->kvm->arch.invlpg_counter) != invlpg_counter)
3145                 gentry = 0;
3146         kvm_mmu_access_page(vcpu, gfn);
3147         kvm_mmu_free_some_pages(vcpu);
3148         ++vcpu->kvm->stat.mmu_pte_write;
3149         trace_kvm_mmu_audit(vcpu, AUDIT_PRE_PTE_WRITE);
3150         if (guest_initiated) {
3151                 if (gfn == vcpu->arch.last_pt_write_gfn
3152                     && !last_updated_pte_accessed(vcpu)) {
3153                         ++vcpu->arch.last_pt_write_count;
3154                         if (vcpu->arch.last_pt_write_count >= 3)
3155                                 flooded = 1;
3156                 } else {
3157                         vcpu->arch.last_pt_write_gfn = gfn;
3158                         vcpu->arch.last_pt_write_count = 1;
3159                         vcpu->arch.last_pte_updated = NULL;
3160                 }
3161         }
3162
3163         mask.cr0_wp = mask.cr4_pae = mask.nxe = 1;
3164         for_each_gfn_indirect_valid_sp(vcpu->kvm, sp, gfn, node) {
3165                 pte_size = sp->role.cr4_pae ? 8 : 4;
3166                 misaligned = (offset ^ (offset + bytes - 1)) & ~(pte_size - 1);
3167                 misaligned |= bytes < 4;
3168                 if (misaligned || flooded) {
3169                         /*
3170                          * Misaligned accesses are too much trouble to fix
3171                          * up; also, they usually indicate a page is not used
3172                          * as a page table.
3173                          *
3174                          * If we're seeing too many writes to a page,
3175                          * it may no longer be a page table, or we may be
3176                          * forking, in which case it is better to unmap the
3177                          * page.
3178                          */
3179                         pgprintk("misaligned: gpa %llx bytes %d role %x\n",
3180                                  gpa, bytes, sp->role.word);
3181                         zap_page |= !!kvm_mmu_prepare_zap_page(vcpu->kvm, sp,
3182                                                      &invalid_list);
3183                         ++vcpu->kvm->stat.mmu_flooded;
3184                         continue;
3185                 }
3186                 page_offset = offset;
3187                 level = sp->role.level;
3188                 npte = 1;
3189                 if (!sp->role.cr4_pae) {
3190                         page_offset <<= 1;      /* 32->64 */
3191                         /*
3192                          * A 32-bit pde maps 4MB while the shadow pdes map
3193                          * only 2MB.  So we need to double the offset again
3194                          * and zap two pdes instead of one.
3195                          */
3196                         if (level == PT32_ROOT_LEVEL) {
3197                                 page_offset &= ~7; /* kill rounding error */
3198                                 page_offset <<= 1;
3199                                 npte = 2;
3200                         }
3201                         quadrant = page_offset >> PAGE_SHIFT;
3202                         page_offset &= ~PAGE_MASK;
3203                         if (quadrant != sp->role.quadrant)
3204                                 continue;
3205                 }
3206                 local_flush = true;
3207                 spte = &sp->spt[page_offset / sizeof(*spte)];
3208                 while (npte--) {
3209                         entry = *spte;
3210                         mmu_pte_write_zap_pte(vcpu, sp, spte);
3211                         if (gentry &&
3212                               !((sp->role.word ^ vcpu->arch.mmu.base_role.word)
3213                               & mask.word))
3214                                 mmu_pte_write_new_pte(vcpu, sp, spte, &gentry);
3215                         if (!remote_flush && need_remote_flush(entry, *spte))
3216                                 remote_flush = true;
3217                         ++spte;
3218                 }
3219         }
3220         mmu_pte_write_flush_tlb(vcpu, zap_page, remote_flush, local_flush);
3221         kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
3222         trace_kvm_mmu_audit(vcpu, AUDIT_POST_PTE_WRITE);
3223         spin_unlock(&vcpu->kvm->mmu_lock);
3224         if (!is_error_pfn(vcpu->arch.update_pte.pfn)) {
3225                 kvm_release_pfn_clean(vcpu->arch.update_pte.pfn);
3226                 vcpu->arch.update_pte.pfn = bad_pfn;
3227         }
3228 }
3229
3230 int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva)
3231 {
3232         gpa_t gpa;
3233         int r;
3234
3235         if (vcpu->arch.mmu.direct_map)
3236                 return 0;
3237
3238         gpa = kvm_mmu_gva_to_gpa_read(vcpu, gva, NULL);
3239
3240         spin_lock(&vcpu->kvm->mmu_lock);
3241         r = kvm_mmu_unprotect_page(vcpu->kvm, gpa >> PAGE_SHIFT);
3242         spin_unlock(&vcpu->kvm->mmu_lock);
3243         return r;
3244 }
3245 EXPORT_SYMBOL_GPL(kvm_mmu_unprotect_page_virt);
3246
3247 void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu)
3248 {
3249         LIST_HEAD(invalid_list);
3250
3251         while (kvm_mmu_available_pages(vcpu->kvm) < KVM_REFILL_PAGES &&
3252                !list_empty(&vcpu->kvm->arch.active_mmu_pages)) {
3253                 struct kvm_mmu_page *sp;
3254
3255                 sp = container_of(vcpu->kvm->arch.active_mmu_pages.prev,
3256                                   struct kvm_mmu_page, link);
3257                 kvm_mmu_prepare_zap_page(vcpu->kvm, sp, &invalid_list);
3258                 kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
3259                 ++vcpu->kvm->stat.mmu_recycled;
3260         }
3261 }
3262
3263 int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u32 error_code)
3264 {
3265         int r;
3266         enum emulation_result er;
3267
3268         r = vcpu->arch.mmu.page_fault(vcpu, cr2, error_code);
3269         if (r < 0)
3270                 goto out;
3271
3272         if (!r) {
3273                 r = 1;
3274                 goto out;
3275         }
3276
3277         r = mmu_topup_memory_caches(vcpu);
3278         if (r)
3279                 goto out;
3280
3281         er = emulate_instruction(vcpu, cr2, error_code, 0);
3282
3283         switch (er) {
3284         case EMULATE_DONE:
3285                 return 1;
3286         case EMULATE_DO_MMIO:
3287                 ++vcpu->stat.mmio_exits;
3288                 /* fall through */
3289         case EMULATE_FAIL:
3290                 return 0;
3291         default:
3292                 BUG();
3293         }
3294 out:
3295         return r;
3296 }
3297 EXPORT_SYMBOL_GPL(kvm_mmu_page_fault);
3298
3299 void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva)
3300 {
3301         vcpu->arch.mmu.invlpg(vcpu, gva);
3302         kvm_mmu_flush_tlb(vcpu);
3303         ++vcpu->stat.invlpg;
3304 }
3305 EXPORT_SYMBOL_GPL(kvm_mmu_invlpg);
3306
3307 void kvm_enable_tdp(void)
3308 {
3309         tdp_enabled = true;
3310 }
3311 EXPORT_SYMBOL_GPL(kvm_enable_tdp);
3312
3313 void kvm_disable_tdp(void)
3314 {
3315         tdp_enabled = false;
3316 }
3317 EXPORT_SYMBOL_GPL(kvm_disable_tdp);
3318
3319 static void free_mmu_pages(struct kvm_vcpu *vcpu)
3320 {
3321         free_page((unsigned long)vcpu->arch.mmu.pae_root);
3322         if (vcpu->arch.mmu.lm_root != NULL)
3323                 free_page((unsigned long)vcpu->arch.mmu.lm_root);
3324 }
3325
3326 static int alloc_mmu_pages(struct kvm_vcpu *vcpu)
3327 {
3328         struct page *page;
3329         int i;
3330
3331         ASSERT(vcpu);
3332
3333         /*
3334          * When emulating 32-bit mode, cr3 is only 32 bits even on x86_64.
3335          * Therefore we need to allocate shadow page tables in the first
3336          * 4GB of memory, which happens to fit the DMA32 zone.
3337          */
3338         page = alloc_page(GFP_KERNEL | __GFP_DMA32);
3339         if (!page)
3340                 return -ENOMEM;
3341
3342         vcpu->arch.mmu.pae_root = page_address(page);
3343         for (i = 0; i < 4; ++i)
3344                 vcpu->arch.mmu.pae_root[i] = INVALID_PAGE;
3345
3346         return 0;
3347 }
3348
3349 int kvm_mmu_create(struct kvm_vcpu *vcpu)
3350 {
3351         ASSERT(vcpu);
3352         ASSERT(!VALID_PAGE(vcpu->arch.mmu.root_hpa));
3353
3354         return alloc_mmu_pages(vcpu);
3355 }
3356
3357 int kvm_mmu_setup(struct kvm_vcpu *vcpu)
3358 {
3359         ASSERT(vcpu);
3360         ASSERT(!VALID_PAGE(vcpu->arch.mmu.root_hpa));
3361
3362         return init_kvm_mmu(vcpu);
3363 }
3364
3365 void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot)
3366 {
3367         struct kvm_mmu_page *sp;
3368
3369         list_for_each_entry(sp, &kvm->arch.active_mmu_pages, link) {
3370                 int i;
3371                 u64 *pt;
3372
3373                 if (!test_bit(slot, sp->slot_bitmap))
3374                         continue;
3375
3376                 pt = sp->spt;
3377                 for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
3378                         /* avoid RMW */
3379                         if (is_writable_pte(pt[i]))
3380                                 pt[i] &= ~PT_WRITABLE_MASK;
3381         }
3382         kvm_flush_remote_tlbs(kvm);
3383 }
3384
3385 void kvm_mmu_zap_all(struct kvm *kvm)
3386 {
3387         struct kvm_mmu_page *sp, *node;
3388         LIST_HEAD(invalid_list);
3389
3390         spin_lock(&kvm->mmu_lock);
3391 restart:
3392         list_for_each_entry_safe(sp, node, &kvm->arch.active_mmu_pages, link)
3393                 if (kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list))
3394                         goto restart;
3395
3396         kvm_mmu_commit_zap_page(kvm, &invalid_list);
3397         spin_unlock(&kvm->mmu_lock);
3398 }
3399
3400 static int kvm_mmu_remove_some_alloc_mmu_pages(struct kvm *kvm,
3401                                                struct list_head *invalid_list)
3402 {
3403         struct kvm_mmu_page *page;
3404
3405         page = container_of(kvm->arch.active_mmu_pages.prev,
3406                             struct kvm_mmu_page, link);
3407         return kvm_mmu_prepare_zap_page(kvm, page, invalid_list);
3408 }
3409
3410 static int mmu_shrink(struct shrinker *shrink, int nr_to_scan, gfp_t gfp_mask)
3411 {
3412         struct kvm *kvm;
3413         struct kvm *kvm_freed = NULL;
3414
3415         if (nr_to_scan == 0)
3416                 goto out;
3417
3418         spin_lock(&kvm_lock);
3419
3420         list_for_each_entry(kvm, &vm_list, vm_list) {
3421                 int idx, freed_pages;
3422                 LIST_HEAD(invalid_list);
3423
3424                 idx = srcu_read_lock(&kvm->srcu);
3425                 spin_lock(&kvm->mmu_lock);
3426                 if (!kvm_freed && nr_to_scan > 0 &&
3427                     kvm->arch.n_used_mmu_pages > 0) {
3428                         freed_pages = kvm_mmu_remove_some_alloc_mmu_pages(kvm,
3429                                                           &invalid_list);
3430                         kvm_freed = kvm;
3431                 }
3432                 nr_to_scan--;
3433
3434                 kvm_mmu_commit_zap_page(kvm, &invalid_list);
3435                 spin_unlock(&kvm->mmu_lock);
3436                 srcu_read_unlock(&kvm->srcu, idx);
3437         }
3438         if (kvm_freed)
3439                 list_move_tail(&kvm_freed->vm_list, &vm_list);
3440
3441         spin_unlock(&kvm_lock);
3442
3443 out:
3444         return percpu_counter_read_positive(&kvm_total_used_mmu_pages);
3445 }
3446
3447 static struct shrinker mmu_shrinker = {
3448         .shrink = mmu_shrink,
3449         .seeks = DEFAULT_SEEKS * 10,
3450 };
3451
3452 static void mmu_destroy_caches(void)
3453 {
3454         if (pte_chain_cache)
3455                 kmem_cache_destroy(pte_chain_cache);
3456         if (rmap_desc_cache)
3457                 kmem_cache_destroy(rmap_desc_cache);
3458         if (mmu_page_header_cache)
3459                 kmem_cache_destroy(mmu_page_header_cache);
3460 }
3461
3462 void kvm_mmu_module_exit(void)
3463 {
3464         mmu_destroy_caches();
3465         percpu_counter_destroy(&kvm_total_used_mmu_pages);
3466         unregister_shrinker(&mmu_shrinker);
3467 }
3468
3469 int kvm_mmu_module_init(void)
3470 {
3471         pte_chain_cache = kmem_cache_create("kvm_pte_chain",
3472                                             sizeof(struct kvm_pte_chain),
3473                                             0, 0, NULL);
3474         if (!pte_chain_cache)
3475                 goto nomem;
3476         rmap_desc_cache = kmem_cache_create("kvm_rmap_desc",
3477                                             sizeof(struct kvm_rmap_desc),
3478                                             0, 0, NULL);
3479         if (!rmap_desc_cache)
3480                 goto nomem;
3481
3482         mmu_page_header_cache = kmem_cache_create("kvm_mmu_page_header",
3483                                                   sizeof(struct kvm_mmu_page),
3484                                                   0, 0, NULL);
3485         if (!mmu_page_header_cache)
3486                 goto nomem;
3487
3488         if (percpu_counter_init(&kvm_total_used_mmu_pages, 0))
3489                 goto nomem;
3490
3491         register_shrinker(&mmu_shrinker);
3492
3493         return 0;
3494
3495 nomem:
3496         mmu_destroy_caches();
3497         return -ENOMEM;
3498 }
3499
3500 /*
3501  * Caculate mmu pages needed for kvm.
3502  */
3503 unsigned int kvm_mmu_calculate_mmu_pages(struct kvm *kvm)
3504 {
3505         int i;
3506         unsigned int nr_mmu_pages;
3507         unsigned int  nr_pages = 0;
3508         struct kvm_memslots *slots;
3509
3510         slots = kvm_memslots(kvm);
3511
3512         for (i = 0; i < slots->nmemslots; i++)
3513                 nr_pages += slots->memslots[i].npages;
3514
3515         nr_mmu_pages = nr_pages * KVM_PERMILLE_MMU_PAGES / 1000;
3516         nr_mmu_pages = max(nr_mmu_pages,
3517                         (unsigned int) KVM_MIN_ALLOC_MMU_PAGES);
3518
3519         return nr_mmu_pages;
3520 }
3521
3522 static void *pv_mmu_peek_buffer(struct kvm_pv_mmu_op_buffer *buffer,
3523                                 unsigned len)
3524 {
3525         if (len > buffer->len)
3526                 return NULL;
3527         return buffer->ptr;
3528 }
3529
3530 static void *pv_mmu_read_buffer(struct kvm_pv_mmu_op_buffer *buffer,
3531                                 unsigned len)
3532 {
3533         void *ret;
3534
3535         ret = pv_mmu_peek_buffer(buffer, len);
3536         if (!ret)
3537                 return ret;
3538         buffer->ptr += len;
3539         buffer->len -= len;
3540         buffer->processed += len;
3541         return ret;
3542 }
3543
3544 static int kvm_pv_mmu_write(struct kvm_vcpu *vcpu,
3545                              gpa_t addr, gpa_t value)
3546 {
3547         int bytes = 8;
3548         int r;
3549
3550         if (!is_long_mode(vcpu) && !is_pae(vcpu))
3551                 bytes = 4;
3552
3553         r = mmu_topup_memory_caches(vcpu);
3554         if (r)
3555                 return r;
3556
3557         if (!emulator_write_phys(vcpu, addr, &value, bytes))
3558                 return -EFAULT;
3559
3560         return 1;
3561 }
3562
3563 static int kvm_pv_mmu_flush_tlb(struct kvm_vcpu *vcpu)
3564 {
3565         (void)kvm_set_cr3(vcpu, vcpu->arch.cr3);
3566         return 1;
3567 }
3568
3569 static int kvm_pv_mmu_release_pt(struct kvm_vcpu *vcpu, gpa_t addr)
3570 {
3571         spin_lock(&vcpu->kvm->mmu_lock);
3572         mmu_unshadow(vcpu->kvm, addr >> PAGE_SHIFT);
3573         spin_unlock(&vcpu->kvm->mmu_lock);
3574         return 1;
3575 }
3576
3577 static int kvm_pv_mmu_op_one(struct kvm_vcpu *vcpu,
3578                              struct kvm_pv_mmu_op_buffer *buffer)
3579 {
3580         struct kvm_mmu_op_header *header;
3581
3582         header = pv_mmu_peek_buffer(buffer, sizeof *header);
3583         if (!header)
3584                 return 0;
3585         switch (header->op) {
3586         case KVM_MMU_OP_WRITE_PTE: {
3587                 struct kvm_mmu_op_write_pte *wpte;
3588
3589                 wpte = pv_mmu_read_buffer(buffer, sizeof *wpte);
3590                 if (!wpte)
3591                         return 0;
3592                 return kvm_pv_mmu_write(vcpu, wpte->pte_phys,
3593                                         wpte->pte_val);
3594         }
3595         case KVM_MMU_OP_FLUSH_TLB: {
3596                 struct kvm_mmu_op_flush_tlb *ftlb;
3597
3598                 ftlb = pv_mmu_read_buffer(buffer, sizeof *ftlb);
3599                 if (!ftlb)
3600                         return 0;
3601                 return kvm_pv_mmu_flush_tlb(vcpu);
3602         }
3603         case KVM_MMU_OP_RELEASE_PT: {
3604                 struct kvm_mmu_op_release_pt *rpt;
3605
3606                 rpt = pv_mmu_read_buffer(buffer, sizeof *rpt);
3607                 if (!rpt)
3608                         return 0;
3609                 return kvm_pv_mmu_release_pt(vcpu, rpt->pt_phys);
3610         }
3611         default: return 0;
3612         }
3613 }
3614
3615 int kvm_pv_mmu_op(struct kvm_vcpu *vcpu, unsigned long bytes,
3616                   gpa_t addr, unsigned long *ret)
3617 {
3618         int r;
3619         struct kvm_pv_mmu_op_buffer *buffer = &vcpu->arch.mmu_op_buffer;
3620
3621         buffer->ptr = buffer->buf;
3622         buffer->len = min_t(unsigned long, bytes, sizeof buffer->buf);
3623         buffer->processed = 0;
3624
3625         r = kvm_read_guest(vcpu->kvm, addr, buffer->buf, buffer->len);
3626         if (r)
3627                 goto out;
3628
3629         while (buffer->len) {
3630                 r = kvm_pv_mmu_op_one(vcpu, buffer);
3631                 if (r < 0)
3632                         goto out;
3633                 if (r == 0)
3634                         break;
3635         }
3636
3637         r = 1;
3638 out:
3639         *ret = buffer->processed;
3640         return r;
3641 }
3642
3643 int kvm_mmu_get_spte_hierarchy(struct kvm_vcpu *vcpu, u64 addr, u64 sptes[4])
3644 {
3645         struct kvm_shadow_walk_iterator iterator;
3646         int nr_sptes = 0;
3647
3648         spin_lock(&vcpu->kvm->mmu_lock);
3649         for_each_shadow_entry(vcpu, addr, iterator) {
3650                 sptes[iterator.level-1] = *iterator.sptep;
3651                 nr_sptes++;
3652                 if (!is_shadow_present_pte(*iterator.sptep))
3653                         break;
3654         }
3655         spin_unlock(&vcpu->kvm->mmu_lock);
3656
3657         return nr_sptes;
3658 }
3659 EXPORT_SYMBOL_GPL(kvm_mmu_get_spte_hierarchy);
3660
3661 #ifdef CONFIG_KVM_MMU_AUDIT
3662 #include "mmu_audit.c"
3663 #else
3664 static void mmu_audit_disable(void) { }
3665 #endif
3666
3667 void kvm_mmu_destroy(struct kvm_vcpu *vcpu)
3668 {
3669         ASSERT(vcpu);
3670
3671         destroy_kvm_mmu(vcpu);
3672         free_mmu_pages(vcpu);
3673         mmu_free_memory_caches(vcpu);
3674         mmu_audit_disable();
3675 }