]> bbs.cooldavid.org Git - net-next-2.6.git/blame - drivers/kvm/kvm_main.c
git://bbs.cooldavid.org / net-next-2.6.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
KVM: Correctly handle writes crossing a page boundary
[net-next-2.6.git] / drivers / kvm / kvm_main.c
CommitLineData
6aa8b732
AK		 1/*
2 * Kernel-based Virtual Machine driver for Linux
3 *
4 * This module enables machines with Intel VT-x extensions to run virtual
5 * machines without emulation or binary translation.
6 *
7 * Copyright (C) 2006 Qumranet, Inc.
8 *
9 * Authors:
10 * Avi Kivity <avi@qumranet.com>
11 * Yaniv Kamay <yaniv@qumranet.com>
12 *
13 * This work is licensed under the terms of the GNU GPL, version 2. See
14 * the COPYING file in the top-level directory.
15 *
16 */
17
18#include "kvm.h"
e495606d
AK		 19#include "x86_emulate.h"
20#include "segment_descriptor.h"
6aa8b732
AK		 21
22#include <linux/kvm.h>
23#include <linux/module.h>
24#include <linux/errno.h>
6aa8b732
AK		 25#include <linux/percpu.h>
26#include <linux/gfp.h>
6aa8b732
AK		 27#include <linux/mm.h>
28#include <linux/miscdevice.h>
29#include <linux/vmalloc.h>
6aa8b732 30#include <linux/reboot.h>
6aa8b732
AK		 31#include <linux/debugfs.h>
32#include <linux/highmem.h>
33#include <linux/file.h>
59ae6c6b 34#include <linux/sysdev.h>
774c47f1 35#include <linux/cpu.h>
e8edc6e0 36#include <linux/sched.h>
d9e368d6
AK		 37#include <linux/cpumask.h>
38#include <linux/smp.h>
d6d28168 39#include <linux/anon_inodes.h>
6aa8b732 40
e495606d
AK		 41#include <asm/processor.h>
42#include <asm/msr.h>
43#include <asm/io.h>
44#include <asm/uaccess.h>
45#include <asm/desc.h>
6aa8b732
AK		 46
47MODULE_AUTHOR("Qumranet");
48MODULE_LICENSE("GPL");
49
133de902
AK		 50static DEFINE_SPINLOCK(kvm_lock);
51static LIST_HEAD(vm_list);
52
1b6c0168
AK		 53static cpumask_t cpus_hardware_enabled;
54
6aa8b732 55struct kvm_arch_ops *kvm_arch_ops;
1165f5fe 56
1b6c0168
AK		 57static void hardware_disable(void *ignored);
58
1165f5fe 59#define STAT_OFFSET(x) offsetof(struct kvm_vcpu, stat.x)
6aa8b732
AK		 60
61static struct kvm_stats_debugfs_item {
62 const char *name;
1165f5fe 63 int offset;
6aa8b732
AK		 64 struct dentry *dentry;
65} debugfs_entries[] = {
1165f5fe
AK		 66 { "pf_fixed", STAT_OFFSET(pf_fixed) },
67 { "pf_guest", STAT_OFFSET(pf_guest) },
68 { "tlb_flush", STAT_OFFSET(tlb_flush) },
69 { "invlpg", STAT_OFFSET(invlpg) },
70 { "exits", STAT_OFFSET(exits) },
71 { "io_exits", STAT_OFFSET(io_exits) },
72 { "mmio_exits", STAT_OFFSET(mmio_exits) },
73 { "signal_exits", STAT_OFFSET(signal_exits) },
74 { "irq_window", STAT_OFFSET(irq_window_exits) },
75 { "halt_exits", STAT_OFFSET(halt_exits) },
76 { "request_irq", STAT_OFFSET(request_irq_exits) },
77 { "irq_exits", STAT_OFFSET(irq_exits) },
e6adf283 78 { "light_exits", STAT_OFFSET(light_exits) },
2cc51560 79 { "efer_reload", STAT_OFFSET(efer_reload) },
1165f5fe 80 { NULL }
6aa8b732
AK		 81};
82
83static struct dentry *debugfs_dir;
84
85#define MAX_IO_MSRS 256
86
87#define CR0_RESEVED_BITS 0xffffffff1ffaffc0ULL
88#define LMSW_GUEST_MASK 0x0eULL
89#define CR4_RESEVED_BITS (~((1ULL << 11) - 1))
90#define CR8_RESEVED_BITS (~0x0fULL)
91#define EFER_RESERVED_BITS 0xfffffffffffff2fe
92
05b3e0c2 93#ifdef CONFIG_X86_64
6aa8b732
AK		 94// LDT or TSS descriptor in the GDT. 16 bytes.
95struct segment_descriptor_64 {
96 struct segment_descriptor s;
97 u32 base_higher;
98 u32 pad_zero;
99};
100
101#endif
102
bccf2150
AK		 103static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl,
104 unsigned long arg);
105
6aa8b732
AK		 106unsigned long segment_base(u16 selector)
107{
108 struct descriptor_table gdt;
109 struct segment_descriptor *d;
110 unsigned long table_base;
111 typedef unsigned long ul;
112 unsigned long v;
113
114 if (selector == 0)
115 return 0;
116
117 asm ("sgdt %0" : "=m"(gdt));
118 table_base = gdt.base;
119
120 if (selector & 4) { /* from ldt */
121 u16 ldt_selector;
122
123 asm ("sldt %0" : "=g"(ldt_selector));
124 table_base = segment_base(ldt_selector);
125 }
126 d = (struct segment_descriptor *)(table_base + (selector & ~7));
127 v = d->base_low | ((ul)d->base_mid << 16) | ((ul)d->base_high << 24);
05b3e0c2 128#ifdef CONFIG_X86_64
6aa8b732
AK		 129 if (d->system == 0
130 && (d->type == 2 || d->type == 9 || d->type == 11))
131 v |= ((ul)((struct segment_descriptor_64 *)d)->base_higher) << 32;
132#endif
133 return v;
134}
135EXPORT_SYMBOL_GPL(segment_base);
136
5aacf0ca
JM		 137static inline int valid_vcpu(int n)
138{
139 return likely(n >= 0 && n < KVM_MAX_VCPUS);
140}
141
d27d4aca
AK		 142int kvm_read_guest(struct kvm_vcpu *vcpu, gva_t addr, unsigned long size,
143 void *dest)
6aa8b732
AK		 144{
145 unsigned char *host_buf = dest;
146 unsigned long req_size = size;
147
148 while (size) {
149 hpa_t paddr;
150 unsigned now;
151 unsigned offset;
152 hva_t guest_buf;
153
154 paddr = gva_to_hpa(vcpu, addr);
155
156 if (is_error_hpa(paddr))
157 break;
158
159 guest_buf = (hva_t)kmap_atomic(
160 pfn_to_page(paddr >> PAGE_SHIFT),
161 KM_USER0);
162 offset = addr & ~PAGE_MASK;
163 guest_buf |= offset;
164 now = min(size, PAGE_SIZE - offset);
165 memcpy(host_buf, (void*)guest_buf, now);
166 host_buf += now;
167 addr += now;
168 size -= now;
169 kunmap_atomic((void *)(guest_buf & PAGE_MASK), KM_USER0);
170 }
171 return req_size - size;
172}
173EXPORT_SYMBOL_GPL(kvm_read_guest);
174
d27d4aca
AK		 175int kvm_write_guest(struct kvm_vcpu *vcpu, gva_t addr, unsigned long size,
176 void *data)
6aa8b732
AK		 177{
178 unsigned char *host_buf = data;
179 unsigned long req_size = size;
180
181 while (size) {
182 hpa_t paddr;
183 unsigned now;
184 unsigned offset;
185 hva_t guest_buf;
ab51a434 186 gfn_t gfn;
6aa8b732
AK		 187
188 paddr = gva_to_hpa(vcpu, addr);
189
190 if (is_error_hpa(paddr))
191 break;
192
ab51a434
UL		 193 gfn = vcpu->mmu.gva_to_gpa(vcpu, addr) >> PAGE_SHIFT;
194 mark_page_dirty(vcpu->kvm, gfn);
6aa8b732
AK		 195 guest_buf = (hva_t)kmap_atomic(
196 pfn_to_page(paddr >> PAGE_SHIFT), KM_USER0);
197 offset = addr & ~PAGE_MASK;
198 guest_buf |= offset;
199 now = min(size, PAGE_SIZE - offset);
200 memcpy((void*)guest_buf, host_buf, now);
201 host_buf += now;
202 addr += now;
203 size -= now;
204 kunmap_atomic((void *)(guest_buf & PAGE_MASK), KM_USER0);
205 }
206 return req_size - size;
207}
208EXPORT_SYMBOL_GPL(kvm_write_guest);
209
7702fd1f
AK		 210void kvm_load_guest_fpu(struct kvm_vcpu *vcpu)
211{
212 if (!vcpu->fpu_active || vcpu->guest_fpu_loaded)
213 return;
214
215 vcpu->guest_fpu_loaded = 1;
216 fx_save(vcpu->host_fx_image);
217 fx_restore(vcpu->guest_fx_image);
218}
219EXPORT_SYMBOL_GPL(kvm_load_guest_fpu);
220
221void kvm_put_guest_fpu(struct kvm_vcpu *vcpu)
222{
223 if (!vcpu->guest_fpu_loaded)
224 return;
225
226 vcpu->guest_fpu_loaded = 0;
227 fx_save(vcpu->guest_fx_image);
228 fx_restore(vcpu->host_fx_image);
229}
230EXPORT_SYMBOL_GPL(kvm_put_guest_fpu);
231
bccf2150
AK		 232/*
233 * Switches to specified vcpu, until a matching vcpu_put()
234 */
235static void vcpu_load(struct kvm_vcpu *vcpu)
6aa8b732 236{
bccf2150
AK		 237 mutex_lock(&vcpu->mutex);
238 kvm_arch_ops->vcpu_load(vcpu);
6aa8b732
AK		 239}
240
6aa8b732
AK		 241static void vcpu_put(struct kvm_vcpu *vcpu)
242{
243 kvm_arch_ops->vcpu_put(vcpu);
6aa8b732
AK		 244 mutex_unlock(&vcpu->mutex);
245}
246
d9e368d6
AK		 247static void ack_flush(void *_completed)
248{
249 atomic_t *completed = _completed;
250
251 atomic_inc(completed);
252}
253
254void kvm_flush_remote_tlbs(struct kvm *kvm)
255{
256 int i, cpu, needed;
257 cpumask_t cpus;
258 struct kvm_vcpu *vcpu;
259 atomic_t completed;
260
261 atomic_set(&completed, 0);
262 cpus_clear(cpus);
263 needed = 0;
264 for (i = 0; i < kvm->nvcpus; ++i) {
265 vcpu = &kvm->vcpus[i];
266 if (test_and_set_bit(KVM_TLB_FLUSH, &vcpu->requests))
267 continue;
268 cpu = vcpu->cpu;
269 if (cpu != -1 && cpu != raw_smp_processor_id())
270 if (!cpu_isset(cpu, cpus)) {
271 cpu_set(cpu, cpus);
272 ++needed;
273 }
274 }
275
276 /*
277 * We really want smp_call_function_mask() here. But that's not
278 * available, so ipi all cpus in parallel and wait for them
279 * to complete.
280 */
281 for (cpu = first_cpu(cpus); cpu != NR_CPUS; cpu = next_cpu(cpu, cpus))
282 smp_call_function_single(cpu, ack_flush, &completed, 1, 0);
283 while (atomic_read(&completed) != needed) {
284 cpu_relax();
285 barrier();
286 }
287}
288
f17abe9a 289static struct kvm *kvm_create_vm(void)
6aa8b732
AK		 290{
291 struct kvm *kvm = kzalloc(sizeof(struct kvm), GFP_KERNEL);
292 int i;
293
294 if (!kvm)
f17abe9a 295 return ERR_PTR(-ENOMEM);
6aa8b732 296
74906345 297 kvm_io_bus_init(&kvm->pio_bus);
6aa8b732
AK		 298 spin_lock_init(&kvm->lock);
299 INIT_LIST_HEAD(&kvm->active_mmu_pages);
120e9a45
AK		 300 spin_lock(&kvm_lock);
301 list_add(&kvm->vm_list, &vm_list);
302 spin_unlock(&kvm_lock);
2eeb2e94 303 kvm_io_bus_init(&kvm->mmio_bus);
6aa8b732
AK		 304 for (i = 0; i < KVM_MAX_VCPUS; ++i) {
305 struct kvm_vcpu *vcpu = &kvm->vcpus[i];
306
307 mutex_init(&vcpu->mutex);
133de902 308 vcpu->cpu = -1;
86a2b42e 309 vcpu->kvm = kvm;
6aa8b732 310 vcpu->mmu.root_hpa = INVALID_PAGE;
6aa8b732 311 }
f17abe9a
AK		 312 return kvm;
313}
314
315static int kvm_dev_open(struct inode *inode, struct file *filp)
316{
6aa8b732
AK		 317 return 0;
318}
319
320/*
321 * Free any memory in @free but not in @dont.
322 */
323static void kvm_free_physmem_slot(struct kvm_memory_slot *free,
324 struct kvm_memory_slot *dont)
325{
326 int i;
327
328 if (!dont || free->phys_mem != dont->phys_mem)
329 if (free->phys_mem) {
330 for (i = 0; i < free->npages; ++i)
55a54f79
AK		 331 if (free->phys_mem[i])
332 __free_page(free->phys_mem[i]);
6aa8b732
AK		 333 vfree(free->phys_mem);
334 }
335
336 if (!dont || free->dirty_bitmap != dont->dirty_bitmap)
337 vfree(free->dirty_bitmap);
338
8b6d44c7 339 free->phys_mem = NULL;
6aa8b732 340 free->npages = 0;
8b6d44c7 341 free->dirty_bitmap = NULL;
6aa8b732
AK		 342}
343
344static void kvm_free_physmem(struct kvm *kvm)
345{
346 int i;
347
348 for (i = 0; i < kvm->nmemslots; ++i)
8b6d44c7 349 kvm_free_physmem_slot(&kvm->memslots[i], NULL);
6aa8b732
AK		 350}
351
039576c0
AK		 352static void free_pio_guest_pages(struct kvm_vcpu *vcpu)
353{
354 int i;
355
356 for (i = 0; i < 2; ++i)
357 if (vcpu->pio.guest_pages[i]) {
358 __free_page(vcpu->pio.guest_pages[i]);
359 vcpu->pio.guest_pages[i] = NULL;
360 }
361}
362
7b53aa56
AK		 363static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu)
364{
365 if (!vcpu->vmcs)
366 return;
367
368 vcpu_load(vcpu);
369 kvm_mmu_unload(vcpu);
370 vcpu_put(vcpu);
371}
372
6aa8b732
AK		 373static void kvm_free_vcpu(struct kvm_vcpu *vcpu)
374{
bccf2150 375 if (!vcpu->vmcs)
1e8ba6fb
IM		 376 return;
377
bccf2150 378 vcpu_load(vcpu);
6aa8b732 379 kvm_mmu_destroy(vcpu);
08438475 380 vcpu_put(vcpu);
9ede74e0 381 kvm_arch_ops->vcpu_free(vcpu);
9a2bb7f4
AK		 382 free_page((unsigned long)vcpu->run);
383 vcpu->run = NULL;
039576c0
AK		 384 free_page((unsigned long)vcpu->pio_data);
385 vcpu->pio_data = NULL;
386 free_pio_guest_pages(vcpu);
6aa8b732
AK		 387}
388
389static void kvm_free_vcpus(struct kvm *kvm)
390{
391 unsigned int i;
392
7b53aa56
AK		 393 /*
394 * Unpin any mmu pages first.
395 */
396 for (i = 0; i < KVM_MAX_VCPUS; ++i)
397 kvm_unload_vcpu_mmu(&kvm->vcpus[i]);
6aa8b732
AK		 398 for (i = 0; i < KVM_MAX_VCPUS; ++i)
399 kvm_free_vcpu(&kvm->vcpus[i]);
400}
401
402static int kvm_dev_release(struct inode *inode, struct file *filp)
403{
f17abe9a
AK		 404 return 0;
405}
6aa8b732 406
f17abe9a
AK		 407static void kvm_destroy_vm(struct kvm *kvm)
408{
133de902
AK		 409 spin_lock(&kvm_lock);
410 list_del(&kvm->vm_list);
411 spin_unlock(&kvm_lock);
74906345 412 kvm_io_bus_destroy(&kvm->pio_bus);
2eeb2e94 413 kvm_io_bus_destroy(&kvm->mmio_bus);
6aa8b732
AK		 414 kvm_free_vcpus(kvm);
415 kvm_free_physmem(kvm);
416 kfree(kvm);
f17abe9a
AK		 417}
418
419static int kvm_vm_release(struct inode *inode, struct file *filp)
420{
421 struct kvm *kvm = filp->private_data;
422
423 kvm_destroy_vm(kvm);
6aa8b732
AK		 424 return 0;
425}
426
427static void inject_gp(struct kvm_vcpu *vcpu)
428{
429 kvm_arch_ops->inject_gp(vcpu, 0);
430}
431
1342d353
AK		 432/*
433 * Load the pae pdptrs. Return true is they are all valid.
434 */
435static int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3)
6aa8b732
AK		 436{
437 gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT;
1342d353 438 unsigned offset = ((cr3 & (PAGE_SIZE-1)) >> 5) << 2;
6aa8b732
AK		 439 int i;
440 u64 pdpte;
441 u64 *pdpt;
1342d353 442 int ret;
954bbbc2 443 struct page *page;
6aa8b732
AK		 444
445 spin_lock(&vcpu->kvm->lock);
954bbbc2
AK		 446 page = gfn_to_page(vcpu->kvm, pdpt_gfn);
447 /* FIXME: !page - emulate? 0xff? */
448 pdpt = kmap_atomic(page, KM_USER0);
6aa8b732 449
1342d353 450 ret = 1;
6aa8b732
AK		 451 for (i = 0; i < 4; ++i) {
452 pdpte = pdpt[offset + i];
1342d353
AK		 453 if ((pdpte & 1) && (pdpte & 0xfffffff0000001e6ull)) {
454 ret = 0;
455 goto out;
456 }
6aa8b732
AK		 457 }
458
1342d353
AK		 459 for (i = 0; i < 4; ++i)
460 vcpu->pdptrs[i] = pdpt[offset + i];
461
462out:
6aa8b732
AK		 463 kunmap_atomic(pdpt, KM_USER0);
464 spin_unlock(&vcpu->kvm->lock);
465
1342d353 466 return ret;
6aa8b732
AK		 467}
468
469void set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
470{
471 if (cr0 & CR0_RESEVED_BITS) {
472 printk(KERN_DEBUG "set_cr0: 0x%lx #GP, reserved bits 0x%lx\n",
473 cr0, vcpu->cr0);
474 inject_gp(vcpu);
475 return;
476 }
477
478 if ((cr0 & CR0_NW_MASK) && !(cr0 & CR0_CD_MASK)) {
479 printk(KERN_DEBUG "set_cr0: #GP, CD == 0 && NW == 1\n");
480 inject_gp(vcpu);
481 return;
482 }
483
484 if ((cr0 & CR0_PG_MASK) && !(cr0 & CR0_PE_MASK)) {
485 printk(KERN_DEBUG "set_cr0: #GP, set PG flag "
486 "and a clear PE flag\n");
487 inject_gp(vcpu);
488 return;
489 }
490
491 if (!is_paging(vcpu) && (cr0 & CR0_PG_MASK)) {
05b3e0c2 492#ifdef CONFIG_X86_64
6aa8b732
AK		 493 if ((vcpu->shadow_efer & EFER_LME)) {
494 int cs_db, cs_l;
495
496 if (!is_pae(vcpu)) {
497 printk(KERN_DEBUG "set_cr0: #GP, start paging "
498 "in long mode while PAE is disabled\n");
499 inject_gp(vcpu);
500 return;
501 }
502 kvm_arch_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
503 if (cs_l) {
504 printk(KERN_DEBUG "set_cr0: #GP, start paging "
505 "in long mode while CS.L == 1\n");
506 inject_gp(vcpu);
507 return;
508
509 }
510 } else
511#endif
1342d353 512 if (is_pae(vcpu) && !load_pdptrs(vcpu, vcpu->cr3)) {
6aa8b732
AK		 513 printk(KERN_DEBUG "set_cr0: #GP, pdptrs "
514 "reserved bits\n");
515 inject_gp(vcpu);
516 return;
517 }
518
519 }
520
521 kvm_arch_ops->set_cr0(vcpu, cr0);
522 vcpu->cr0 = cr0;
523
524 spin_lock(&vcpu->kvm->lock);
525 kvm_mmu_reset_context(vcpu);
526 spin_unlock(&vcpu->kvm->lock);
527 return;
528}
529EXPORT_SYMBOL_GPL(set_cr0);
530
531void lmsw(struct kvm_vcpu *vcpu, unsigned long msw)
532{
533 set_cr0(vcpu, (vcpu->cr0 & ~0x0ful) | (msw & 0x0f));
534}
535EXPORT_SYMBOL_GPL(lmsw);
536
537void set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
538{
539 if (cr4 & CR4_RESEVED_BITS) {
540 printk(KERN_DEBUG "set_cr4: #GP, reserved bits\n");
541 inject_gp(vcpu);
542 return;
543 }
544
a9058ecd 545 if (is_long_mode(vcpu)) {
6aa8b732
AK		 546 if (!(cr4 & CR4_PAE_MASK)) {
547 printk(KERN_DEBUG "set_cr4: #GP, clearing PAE while "
548 "in long mode\n");
549 inject_gp(vcpu);
550 return;
551 }
552 } else if (is_paging(vcpu) && !is_pae(vcpu) && (cr4 & CR4_PAE_MASK)
1342d353 553 && !load_pdptrs(vcpu, vcpu->cr3)) {
6aa8b732
AK		 554 printk(KERN_DEBUG "set_cr4: #GP, pdptrs reserved bits\n");
555 inject_gp(vcpu);
556 }
557
558 if (cr4 & CR4_VMXE_MASK) {
559 printk(KERN_DEBUG "set_cr4: #GP, setting VMXE\n");
560 inject_gp(vcpu);
561 return;
562 }
563 kvm_arch_ops->set_cr4(vcpu, cr4);
564 spin_lock(&vcpu->kvm->lock);
565 kvm_mmu_reset_context(vcpu);
566 spin_unlock(&vcpu->kvm->lock);
567}
568EXPORT_SYMBOL_GPL(set_cr4);
569
570void set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
571{
a9058ecd 572 if (is_long_mode(vcpu)) {
d27d4aca 573 if (cr3 & CR3_L_MODE_RESEVED_BITS) {
6aa8b732
AK		 574 printk(KERN_DEBUG "set_cr3: #GP, reserved bits\n");
575 inject_gp(vcpu);
576 return;
577 }
578 } else {
579 if (cr3 & CR3_RESEVED_BITS) {
580 printk(KERN_DEBUG "set_cr3: #GP, reserved bits\n");
581 inject_gp(vcpu);
582 return;
583 }
584 if (is_paging(vcpu) && is_pae(vcpu) &&
1342d353 585 !load_pdptrs(vcpu, cr3)) {
6aa8b732
AK		 586 printk(KERN_DEBUG "set_cr3: #GP, pdptrs "
587 "reserved bits\n");
588 inject_gp(vcpu);
589 return;
590 }
591 }
592
593 vcpu->cr3 = cr3;
594 spin_lock(&vcpu->kvm->lock);
d21225ee
IM		 595 /*
596 * Does the new cr3 value map to physical memory? (Note, we
597 * catch an invalid cr3 even in real-mode, because it would
598 * cause trouble later on when we turn on paging anyway.)
599 *
600 * A real CPU would silently accept an invalid cr3 and would
601 * attempt to use it - with largely undefined (and often hard
602 * to debug) behavior on the guest side.
603 */
604 if (unlikely(!gfn_to_memslot(vcpu->kvm, cr3 >> PAGE_SHIFT)))
605 inject_gp(vcpu);
606 else
607 vcpu->mmu.new_cr3(vcpu);
6aa8b732
AK		 608 spin_unlock(&vcpu->kvm->lock);
609}
610EXPORT_SYMBOL_GPL(set_cr3);
611
612void set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8)
613{
614 if ( cr8 & CR8_RESEVED_BITS) {
615 printk(KERN_DEBUG "set_cr8: #GP, reserved bits 0x%lx\n", cr8);
616 inject_gp(vcpu);
617 return;
618 }
619 vcpu->cr8 = cr8;
620}
621EXPORT_SYMBOL_GPL(set_cr8);
622
623void fx_init(struct kvm_vcpu *vcpu)
624{
625 struct __attribute__ ((__packed__)) fx_image_s {
626 u16 control; //fcw
627 u16 status; //fsw
628 u16 tag; // ftw
629 u16 opcode; //fop
630 u64 ip; // fpu ip
631 u64 operand;// fpu dp
632 u32 mxcsr;
633 u32 mxcsr_mask;
634
635 } *fx_image;
636
637 fx_save(vcpu->host_fx_image);
638 fpu_init();
639 fx_save(vcpu->guest_fx_image);
640 fx_restore(vcpu->host_fx_image);
641
642 fx_image = (struct fx_image_s *)vcpu->guest_fx_image;
643 fx_image->mxcsr = 0x1f80;
644 memset(vcpu->guest_fx_image + sizeof(struct fx_image_s),
645 0, FX_IMAGE_SIZE - sizeof(struct fx_image_s));
646}
647EXPORT_SYMBOL_GPL(fx_init);
648
6aa8b732
AK		 649/*
650 * Allocate some memory and give it an address in the guest physical address
651 * space.
652 *
653 * Discontiguous memory is allowed, mostly for framebuffers.
654 */
2c6f5df9
AK		 655static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,
656 struct kvm_memory_region *mem)
6aa8b732
AK		 657{
658 int r;
659 gfn_t base_gfn;
660 unsigned long npages;
661 unsigned long i;
662 struct kvm_memory_slot *memslot;
663 struct kvm_memory_slot old, new;
664 int memory_config_version;
665
666 r = -EINVAL;
667 /* General sanity checks */
668 if (mem->memory_size & (PAGE_SIZE - 1))
669 goto out;
670 if (mem->guest_phys_addr & (PAGE_SIZE - 1))
671 goto out;
672 if (mem->slot >= KVM_MEMORY_SLOTS)
673 goto out;
674 if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
675 goto out;
676
677 memslot = &kvm->memslots[mem->slot];
678 base_gfn = mem->guest_phys_addr >> PAGE_SHIFT;
679 npages = mem->memory_size >> PAGE_SHIFT;
680
681 if (!npages)
682 mem->flags &= ~KVM_MEM_LOG_DIRTY_PAGES;
683
684raced:
685 spin_lock(&kvm->lock);
686
687 memory_config_version = kvm->memory_config_version;
688 new = old = *memslot;
689
690 new.base_gfn = base_gfn;
691 new.npages = npages;
692 new.flags = mem->flags;
693
694 /* Disallow changing a memory slot's size. */
695 r = -EINVAL;
696 if (npages && old.npages && npages != old.npages)
697 goto out_unlock;
698
699 /* Check for overlaps */
700 r = -EEXIST;
701 for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
702 struct kvm_memory_slot *s = &kvm->memslots[i];
703
704 if (s == memslot)
705 continue;
706 if (!((base_gfn + npages <= s->base_gfn) ||
707 (base_gfn >= s->base_gfn + s->npages)))
708 goto out_unlock;
709 }
710 /*
711 * Do memory allocations outside lock. memory_config_version will
712 * detect any races.
713 */
714 spin_unlock(&kvm->lock);
715
716 /* Deallocate if slot is being removed */
717 if (!npages)
8b6d44c7 718 new.phys_mem = NULL;
6aa8b732
AK		 719
720 /* Free page dirty bitmap if unneeded */
721 if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES))
8b6d44c7 722 new.dirty_bitmap = NULL;
6aa8b732
AK		 723
724 r = -ENOMEM;
725
726 /* Allocate if a slot is being created */
727 if (npages && !new.phys_mem) {
728 new.phys_mem = vmalloc(npages * sizeof(struct page *));
729
730 if (!new.phys_mem)
731 goto out_free;
732
733 memset(new.phys_mem, 0, npages * sizeof(struct page *));
734 for (i = 0; i < npages; ++i) {
735 new.phys_mem[i] = alloc_page(GFP_HIGHUSER
736 | __GFP_ZERO);
737 if (!new.phys_mem[i])
738 goto out_free;
5972e953 739 set_page_private(new.phys_mem[i],0);
6aa8b732
AK		 740 }
741 }
742
743 /* Allocate page dirty bitmap if needed */
744 if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) {
745 unsigned dirty_bytes = ALIGN(npages, BITS_PER_LONG) / 8;
746
747 new.dirty_bitmap = vmalloc(dirty_bytes);
748 if (!new.dirty_bitmap)
749 goto out_free;
750 memset(new.dirty_bitmap, 0, dirty_bytes);
751 }
752
753 spin_lock(&kvm->lock);
754
755 if (memory_config_version != kvm->memory_config_version) {
756 spin_unlock(&kvm->lock);
757 kvm_free_physmem_slot(&new, &old);
758 goto raced;
759 }
760
761 r = -EAGAIN;
762 if (kvm->busy)
763 goto out_unlock;
764
765 if (mem->slot >= kvm->nmemslots)
766 kvm->nmemslots = mem->slot + 1;
767
768 *memslot = new;
769 ++kvm->memory_config_version;
770
90cb0529
AK		 771 kvm_mmu_slot_remove_write_access(kvm, mem->slot);
772 kvm_flush_remote_tlbs(kvm);
6aa8b732 773
90cb0529 774 spin_unlock(&kvm->lock);
6aa8b732
AK		 775
776 kvm_free_physmem_slot(&old, &new);
777 return 0;
778
779out_unlock:
780 spin_unlock(&kvm->lock);
781out_free:
782 kvm_free_physmem_slot(&new, &old);
783out:
784 return r;
785}
786
787/*
788 * Get (and clear) the dirty memory log for a memory slot.
789 */
2c6f5df9
AK		 790static int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
791 struct kvm_dirty_log *log)
6aa8b732
AK		 792{
793 struct kvm_memory_slot *memslot;
794 int r, i;
795 int n;
796 unsigned long any = 0;
797
798 spin_lock(&kvm->lock);
799
800 /*
801 * Prevent changes to guest memory configuration even while the lock
802 * is not taken.
803 */
804 ++kvm->busy;
805 spin_unlock(&kvm->lock);
806 r = -EINVAL;
807 if (log->slot >= KVM_MEMORY_SLOTS)
808 goto out;
809
810 memslot = &kvm->memslots[log->slot];
811 r = -ENOENT;
812 if (!memslot->dirty_bitmap)
813 goto out;
814
cd1a4a98 815 n = ALIGN(memslot->npages, BITS_PER_LONG) / 8;
6aa8b732 816
cd1a4a98 817 for (i = 0; !any && i < n/sizeof(long); ++i)
6aa8b732
AK		 818 any = memslot->dirty_bitmap[i];
819
820 r = -EFAULT;
821 if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n))
822 goto out;
823
90cb0529
AK		 824 spin_lock(&kvm->lock);
825 kvm_mmu_slot_remove_write_access(kvm, log->slot);
826 kvm_flush_remote_tlbs(kvm);
827 memset(memslot->dirty_bitmap, 0, n);
828 spin_unlock(&kvm->lock);
6aa8b732
AK		 829
830 r = 0;
831
832out:
833 spin_lock(&kvm->lock);
834 --kvm->busy;
835 spin_unlock(&kvm->lock);
836 return r;
837}
838
e8207547
AK		 839/*
840 * Set a new alias region. Aliases map a portion of physical memory into
841 * another portion. This is useful for memory windows, for example the PC
842 * VGA region.
843 */
844static int kvm_vm_ioctl_set_memory_alias(struct kvm *kvm,
845 struct kvm_memory_alias *alias)
846{
847 int r, n;
848 struct kvm_mem_alias *p;
849
850 r = -EINVAL;
851 /* General sanity checks */
852 if (alias->memory_size & (PAGE_SIZE - 1))
853 goto out;
854 if (alias->guest_phys_addr & (PAGE_SIZE - 1))
855 goto out;
856 if (alias->slot >= KVM_ALIAS_SLOTS)
857 goto out;
858 if (alias->guest_phys_addr + alias->memory_size
859 < alias->guest_phys_addr)
860 goto out;
861 if (alias->target_phys_addr + alias->memory_size
862 < alias->target_phys_addr)
863 goto out;
864
865 spin_lock(&kvm->lock);
866
867 p = &kvm->aliases[alias->slot];
868 p->base_gfn = alias->guest_phys_addr >> PAGE_SHIFT;
869 p->npages = alias->memory_size >> PAGE_SHIFT;
870 p->target_gfn = alias->target_phys_addr >> PAGE_SHIFT;
871
872 for (n = KVM_ALIAS_SLOTS; n > 0; --n)
873 if (kvm->aliases[n - 1].npages)
874 break;
875 kvm->naliases = n;
876
90cb0529 877 kvm_mmu_zap_all(kvm);
e8207547 878
e8207547 879 spin_unlock(&kvm->lock);
e8207547
AK		 880
881 return 0;
882
883out:
884 return r;
885}
886
887static gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn)
888{
889 int i;
890 struct kvm_mem_alias *alias;
891
892 for (i = 0; i < kvm->naliases; ++i) {
893 alias = &kvm->aliases[i];
894 if (gfn >= alias->base_gfn
895 && gfn < alias->base_gfn + alias->npages)
896 return alias->target_gfn + gfn - alias->base_gfn;
897 }
898 return gfn;
899}
900
901static struct kvm_memory_slot *__gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
6aa8b732
AK		 902{
903 int i;
904
905 for (i = 0; i < kvm->nmemslots; ++i) {
906 struct kvm_memory_slot *memslot = &kvm->memslots[i];
907
908 if (gfn >= memslot->base_gfn
909 && gfn < memslot->base_gfn + memslot->npages)
910 return memslot;
911 }
8b6d44c7 912 return NULL;
6aa8b732 913}
e8207547
AK		 914
915struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
916{
917 gfn = unalias_gfn(kvm, gfn);
918 return __gfn_to_memslot(kvm, gfn);
919}
6aa8b732 920
954bbbc2
AK		 921struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn)
922{
923 struct kvm_memory_slot *slot;
924
e8207547
AK		 925 gfn = unalias_gfn(kvm, gfn);
926 slot = __gfn_to_memslot(kvm, gfn);
954bbbc2
AK		 927 if (!slot)
928 return NULL;
929 return slot->phys_mem[gfn - slot->base_gfn];
930}
931EXPORT_SYMBOL_GPL(gfn_to_page);
932
6aa8b732
AK		 933void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
934{
935 int i;
31389947 936 struct kvm_memory_slot *memslot;
6aa8b732
AK		 937 unsigned long rel_gfn;
938
939 for (i = 0; i < kvm->nmemslots; ++i) {
940 memslot = &kvm->memslots[i];
941
942 if (gfn >= memslot->base_gfn
943 && gfn < memslot->base_gfn + memslot->npages) {
944
31389947 945 if (!memslot->dirty_bitmap)
6aa8b732
AK		 946 return;
947
948 rel_gfn = gfn - memslot->base_gfn;
949
950 /* avoid RMW */
951 if (!test_bit(rel_gfn, memslot->dirty_bitmap))
952 set_bit(rel_gfn, memslot->dirty_bitmap);
953 return;
954 }
955 }
956}
957
958static int emulator_read_std(unsigned long addr,
4c690a1e 959 void *val,
6aa8b732
AK		 960 unsigned int bytes,
961 struct x86_emulate_ctxt *ctxt)
962{
963 struct kvm_vcpu *vcpu = ctxt->vcpu;
964 void *data = val;
965
966 while (bytes) {
967 gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, addr);
968 unsigned offset = addr & (PAGE_SIZE-1);
969 unsigned tocopy = min(bytes, (unsigned)PAGE_SIZE - offset);
970 unsigned long pfn;
954bbbc2
AK		 971 struct page *page;
972 void *page_virt;
6aa8b732
AK		 973
974 if (gpa == UNMAPPED_GVA)
975 return X86EMUL_PROPAGATE_FAULT;
976 pfn = gpa >> PAGE_SHIFT;
954bbbc2
AK		 977 page = gfn_to_page(vcpu->kvm, pfn);
978 if (!page)
6aa8b732 979 return X86EMUL_UNHANDLEABLE;
954bbbc2 980 page_virt = kmap_atomic(page, KM_USER0);
6aa8b732 981
954bbbc2 982 memcpy(data, page_virt + offset, tocopy);
6aa8b732 983
954bbbc2 984 kunmap_atomic(page_virt, KM_USER0);
6aa8b732
AK		 985
986 bytes -= tocopy;
987 data += tocopy;
988 addr += tocopy;
989 }
990
991 return X86EMUL_CONTINUE;
992}
993
994static int emulator_write_std(unsigned long addr,
4c690a1e 995 const void *val,
6aa8b732
AK		 996 unsigned int bytes,
997 struct x86_emulate_ctxt *ctxt)
998{
999 printk(KERN_ERR "emulator_write_std: addr %lx n %d\n",
1000 addr, bytes);
1001 return X86EMUL_UNHANDLEABLE;
1002}
1003
2eeb2e94
GH		 1004static struct kvm_io_device *vcpu_find_mmio_dev(struct kvm_vcpu *vcpu,
1005 gpa_t addr)
1006{
1007 /*
1008 * Note that its important to have this wrapper function because
1009 * in the very near future we will be checking for MMIOs against
1010 * the LAPIC as well as the general MMIO bus
1011 */
1012 return kvm_io_bus_find_dev(&vcpu->kvm->mmio_bus, addr);
1013}
1014
74906345
ED		 1015static struct kvm_io_device *vcpu_find_pio_dev(struct kvm_vcpu *vcpu,
1016 gpa_t addr)
1017{
1018 return kvm_io_bus_find_dev(&vcpu->kvm->pio_bus, addr);
1019}
1020
6aa8b732 1021static int emulator_read_emulated(unsigned long addr,
4c690a1e 1022 void *val,
6aa8b732
AK		 1023 unsigned int bytes,
1024 struct x86_emulate_ctxt *ctxt)
1025{
2eeb2e94
GH		 1026 struct kvm_vcpu *vcpu = ctxt->vcpu;
1027 struct kvm_io_device *mmio_dev;
1028 gpa_t gpa;
6aa8b732
AK		 1029
1030 if (vcpu->mmio_read_completed) {
1031 memcpy(val, vcpu->mmio_data, bytes);
1032 vcpu->mmio_read_completed = 0;
1033 return X86EMUL_CONTINUE;
1034 } else if (emulator_read_std(addr, val, bytes, ctxt)
1035 == X86EMUL_CONTINUE)
1036 return X86EMUL_CONTINUE;
d27d4aca 1037
2eeb2e94
GH		 1038 gpa = vcpu->mmu.gva_to_gpa(vcpu, addr);
1039 if (gpa == UNMAPPED_GVA)
1040 return X86EMUL_PROPAGATE_FAULT;
6aa8b732 1041
2eeb2e94
GH		 1042 /*
1043 * Is this MMIO handled locally?
1044 */
1045 mmio_dev = vcpu_find_mmio_dev(vcpu, gpa);
1046 if (mmio_dev) {
1047 kvm_iodevice_read(mmio_dev, gpa, bytes, val);
1048 return X86EMUL_CONTINUE;
6aa8b732 1049 }
2eeb2e94
GH		 1050
1051 vcpu->mmio_needed = 1;
1052 vcpu->mmio_phys_addr = gpa;
1053 vcpu->mmio_size = bytes;
1054 vcpu->mmio_is_write = 0;
1055
1056 return X86EMUL_UNHANDLEABLE;
6aa8b732
AK		 1057}
1058
da4a00f0 1059static int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,
4c690a1e 1060 const void *val, int bytes)
da4a00f0 1061{
da4a00f0
AK		 1062 struct page *page;
1063 void *virt;
09072daf 1064 unsigned offset = offset_in_page(gpa);
da4a00f0
AK		 1065
1066 if (((gpa + bytes - 1) >> PAGE_SHIFT) != (gpa >> PAGE_SHIFT))
1067 return 0;
954bbbc2
AK		 1068 page = gfn_to_page(vcpu->kvm, gpa >> PAGE_SHIFT);
1069 if (!page)
da4a00f0 1070 return 0;
ab51a434 1071 mark_page_dirty(vcpu->kvm, gpa >> PAGE_SHIFT);
da4a00f0 1072 virt = kmap_atomic(page, KM_USER0);
a3c870bd
LT		 1073 if (memcmp(virt + offset_in_page(gpa), val, bytes)) {
1074 kvm_mmu_pte_write(vcpu, gpa, virt + offset, val, bytes);
1075 memcpy(virt + offset_in_page(gpa), val, bytes);
1076 }
da4a00f0 1077 kunmap_atomic(virt, KM_USER0);
da4a00f0
AK		 1078 return 1;
1079}
1080
b0fcd903
AK		 1081static int emulator_write_emulated_onepage(unsigned long addr,
1082 const void *val,
1083 unsigned int bytes,
1084 struct x86_emulate_ctxt *ctxt)
6aa8b732 1085{
2eeb2e94
GH		 1086 struct kvm_vcpu *vcpu = ctxt->vcpu;
1087 struct kvm_io_device *mmio_dev;
1088 gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, addr);
6aa8b732 1089
c9047f53
AK		 1090 if (gpa == UNMAPPED_GVA) {
1091 kvm_arch_ops->inject_page_fault(vcpu, addr, 2);
6aa8b732 1092 return X86EMUL_PROPAGATE_FAULT;
c9047f53 1093 }
6aa8b732 1094
da4a00f0
AK		 1095 if (emulator_write_phys(vcpu, gpa, val, bytes))
1096 return X86EMUL_CONTINUE;
1097
2eeb2e94
GH		 1098 /*
1099 * Is this MMIO handled locally?
1100 */
1101 mmio_dev = vcpu_find_mmio_dev(vcpu, gpa);
1102 if (mmio_dev) {
1103 kvm_iodevice_write(mmio_dev, gpa, bytes, val);
1104 return X86EMUL_CONTINUE;
1105 }
1106
6aa8b732
AK		 1107 vcpu->mmio_needed = 1;
1108 vcpu->mmio_phys_addr = gpa;
1109 vcpu->mmio_size = bytes;
1110 vcpu->mmio_is_write = 1;
4c690a1e 1111 memcpy(vcpu->mmio_data, val, bytes);
6aa8b732
AK		 1112
1113 return X86EMUL_CONTINUE;
1114}
1115
b0fcd903
AK		 1116static int emulator_write_emulated(unsigned long addr,
1117 const void *val,
1118 unsigned int bytes,
1119 struct x86_emulate_ctxt *ctxt)
1120{
1121 /* Crossing a page boundary? */
1122 if (((addr + bytes - 1) ^ addr) & PAGE_MASK) {
1123 int rc, now;
1124
1125 now = -addr & ~PAGE_MASK;
1126 rc = emulator_write_emulated_onepage(addr, val, now, ctxt);
1127 if (rc != X86EMUL_CONTINUE)
1128 return rc;
1129 addr += now;
1130 val += now;
1131 bytes -= now;
1132 }
1133 return emulator_write_emulated_onepage(addr, val, bytes, ctxt);
1134}
1135
6aa8b732 1136static int emulator_cmpxchg_emulated(unsigned long addr,
4c690a1e
AK		 1137 const void *old,
1138 const void *new,
6aa8b732
AK		 1139 unsigned int bytes,
1140 struct x86_emulate_ctxt *ctxt)
1141{
1142 static int reported;
1143
1144 if (!reported) {
1145 reported = 1;
1146 printk(KERN_WARNING "kvm: emulating exchange as write\n");
1147 }
1148 return emulator_write_emulated(addr, new, bytes, ctxt);
1149}
1150
1151static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg)
1152{
1153 return kvm_arch_ops->get_segment_base(vcpu, seg);
1154}
1155
1156int emulate_invlpg(struct kvm_vcpu *vcpu, gva_t address)
1157{
6aa8b732
AK		 1158 return X86EMUL_CONTINUE;
1159}
1160
1161int emulate_clts(struct kvm_vcpu *vcpu)
1162{
399badf3 1163 unsigned long cr0;
6aa8b732 1164
399badf3 1165 cr0 = vcpu->cr0 & ~CR0_TS_MASK;
6aa8b732
AK		 1166 kvm_arch_ops->set_cr0(vcpu, cr0);
1167 return X86EMUL_CONTINUE;
1168}
1169
1170int emulator_get_dr(struct x86_emulate_ctxt* ctxt, int dr, unsigned long *dest)
1171{
1172 struct kvm_vcpu *vcpu = ctxt->vcpu;
1173
1174 switch (dr) {
1175 case 0 ... 3:
1176 *dest = kvm_arch_ops->get_dr(vcpu, dr);
1177 return X86EMUL_CONTINUE;
1178 default:
1179 printk(KERN_DEBUG "%s: unexpected dr %u\n",
1180 __FUNCTION__, dr);
1181 return X86EMUL_UNHANDLEABLE;
1182 }
1183}
1184
1185int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long value)
1186{
1187 unsigned long mask = (ctxt->mode == X86EMUL_MODE_PROT64) ? ~0ULL : ~0U;
1188 int exception;
1189
1190 kvm_arch_ops->set_dr(ctxt->vcpu, dr, value & mask, &exception);
1191 if (exception) {
1192 /* FIXME: better handling */
1193 return X86EMUL_UNHANDLEABLE;
1194 }
1195 return X86EMUL_CONTINUE;
1196}
1197
1198static void report_emulation_failure(struct x86_emulate_ctxt *ctxt)
1199{
1200 static int reported;
1201 u8 opcodes[4];
1202 unsigned long rip = ctxt->vcpu->rip;
1203 unsigned long rip_linear;
1204
1205 rip_linear = rip + get_segment_base(ctxt->vcpu, VCPU_SREG_CS);
1206
1207 if (reported)
1208 return;
1209
1210 emulator_read_std(rip_linear, (void *)opcodes, 4, ctxt);
1211
1212 printk(KERN_ERR "emulation failed but !mmio_needed?"
1213 " rip %lx %02x %02x %02x %02x\n",
1214 rip, opcodes[0], opcodes[1], opcodes[2], opcodes[3]);
1215 reported = 1;
1216}
1217
1218struct x86_emulate_ops emulate_ops = {
1219 .read_std = emulator_read_std,
1220 .write_std = emulator_write_std,
1221 .read_emulated = emulator_read_emulated,
1222 .write_emulated = emulator_write_emulated,
1223 .cmpxchg_emulated = emulator_cmpxchg_emulated,
1224};
1225
1226int emulate_instruction(struct kvm_vcpu *vcpu,
1227 struct kvm_run *run,
1228 unsigned long cr2,
1229 u16 error_code)
1230{
1231 struct x86_emulate_ctxt emulate_ctxt;
1232 int r;
1233 int cs_db, cs_l;
1234
e7df56e4 1235 vcpu->mmio_fault_cr2 = cr2;
6aa8b732
AK		 1236 kvm_arch_ops->cache_regs(vcpu);
1237
1238 kvm_arch_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
1239
1240 emulate_ctxt.vcpu = vcpu;
1241 emulate_ctxt.eflags = kvm_arch_ops->get_rflags(vcpu);
1242 emulate_ctxt.cr2 = cr2;
1243 emulate_ctxt.mode = (emulate_ctxt.eflags & X86_EFLAGS_VM)
1244 ? X86EMUL_MODE_REAL : cs_l
1245 ? X86EMUL_MODE_PROT64 : cs_db
1246 ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;
1247
1248 if (emulate_ctxt.mode == X86EMUL_MODE_PROT64) {
1249 emulate_ctxt.cs_base = 0;
1250 emulate_ctxt.ds_base = 0;
1251 emulate_ctxt.es_base = 0;
1252 emulate_ctxt.ss_base = 0;
1253 } else {
1254 emulate_ctxt.cs_base = get_segment_base(vcpu, VCPU_SREG_CS);
1255 emulate_ctxt.ds_base = get_segment_base(vcpu, VCPU_SREG_DS);
1256 emulate_ctxt.es_base = get_segment_base(vcpu, VCPU_SREG_ES);
1257 emulate_ctxt.ss_base = get_segment_base(vcpu, VCPU_SREG_SS);
1258 }
1259
1260 emulate_ctxt.gs_base = get_segment_base(vcpu, VCPU_SREG_GS);
1261 emulate_ctxt.fs_base = get_segment_base(vcpu, VCPU_SREG_FS);
1262
1263 vcpu->mmio_is_write = 0;
1264 r = x86_emulate_memop(&emulate_ctxt, &emulate_ops);
1265
1266 if ((r || vcpu->mmio_is_write) && run) {
1267 run->mmio.phys_addr = vcpu->mmio_phys_addr;
1268 memcpy(run->mmio.data, vcpu->mmio_data, 8);
1269 run->mmio.len = vcpu->mmio_size;
1270 run->mmio.is_write = vcpu->mmio_is_write;
1271 }
1272
1273 if (r) {
a436036b
AK		 1274 if (kvm_mmu_unprotect_page_virt(vcpu, cr2))
1275 return EMULATE_DONE;
6aa8b732
AK		 1276 if (!vcpu->mmio_needed) {
1277 report_emulation_failure(&emulate_ctxt);
1278 return EMULATE_FAIL;
1279 }
1280 return EMULATE_DO_MMIO;
1281 }
1282
1283 kvm_arch_ops->decache_regs(vcpu);
1284 kvm_arch_ops->set_rflags(vcpu, emulate_ctxt.eflags);
1285
02c83209
AK		 1286 if (vcpu->mmio_is_write) {
1287 vcpu->mmio_needed = 0;
6aa8b732 1288 return EMULATE_DO_MMIO;
02c83209 1289 }
6aa8b732
AK		 1290
1291 return EMULATE_DONE;
1292}
1293EXPORT_SYMBOL_GPL(emulate_instruction);
1294
d3bef15f
AK		 1295int kvm_emulate_halt(struct kvm_vcpu *vcpu)
1296{
1297 if (vcpu->irq_summary)
1298 return 1;
1299
1300 vcpu->run->exit_reason = KVM_EXIT_HLT;
1301 ++vcpu->stat.halt_exits;
1302 return 0;
1303}
1304EXPORT_SYMBOL_GPL(kvm_emulate_halt);
1305
270fd9b9
AK		 1306int kvm_hypercall(struct kvm_vcpu *vcpu, struct kvm_run *run)
1307{
1308 unsigned long nr, a0, a1, a2, a3, a4, a5, ret;
1309
9b22bf57 1310 kvm_arch_ops->cache_regs(vcpu);
270fd9b9
AK		 1311 ret = -KVM_EINVAL;
1312#ifdef CONFIG_X86_64
1313 if (is_long_mode(vcpu)) {
1314 nr = vcpu->regs[VCPU_REGS_RAX];
1315 a0 = vcpu->regs[VCPU_REGS_RDI];
1316 a1 = vcpu->regs[VCPU_REGS_RSI];
1317 a2 = vcpu->regs[VCPU_REGS_RDX];
1318 a3 = vcpu->regs[VCPU_REGS_RCX];
1319 a4 = vcpu->regs[VCPU_REGS_R8];
1320 a5 = vcpu->regs[VCPU_REGS_R9];
1321 } else
1322#endif
1323 {
1324 nr = vcpu->regs[VCPU_REGS_RBX] & -1u;
1325 a0 = vcpu->regs[VCPU_REGS_RAX] & -1u;
1326 a1 = vcpu->regs[VCPU_REGS_RCX] & -1u;
1327 a2 = vcpu->regs[VCPU_REGS_RDX] & -1u;
1328 a3 = vcpu->regs[VCPU_REGS_RSI] & -1u;
1329 a4 = vcpu->regs[VCPU_REGS_RDI] & -1u;
1330 a5 = vcpu->regs[VCPU_REGS_RBP] & -1u;
1331 }
1332 switch (nr) {
1333 default:
b4e63f56
AK		 1334 run->hypercall.args[0] = a0;
1335 run->hypercall.args[1] = a1;
1336 run->hypercall.args[2] = a2;
1337 run->hypercall.args[3] = a3;
1338 run->hypercall.args[4] = a4;
1339 run->hypercall.args[5] = a5;
1340 run->hypercall.ret = ret;
1341 run->hypercall.longmode = is_long_mode(vcpu);
1342 kvm_arch_ops->decache_regs(vcpu);
1343 return 0;
270fd9b9
AK		 1344 }
1345 vcpu->regs[VCPU_REGS_RAX] = ret;
9b22bf57 1346 kvm_arch_ops->decache_regs(vcpu);
270fd9b9
AK		 1347 return 1;
1348}
1349EXPORT_SYMBOL_GPL(kvm_hypercall);
1350
6aa8b732
AK		 1351static u64 mk_cr_64(u64 curr_cr, u32 new_val)
1352{
1353 return (curr_cr & ~((1ULL << 32) - 1)) | new_val;
1354}
1355
1356void realmode_lgdt(struct kvm_vcpu *vcpu, u16 limit, unsigned long base)
1357{
1358 struct descriptor_table dt = { limit, base };
1359
1360 kvm_arch_ops->set_gdt(vcpu, &dt);
1361}
1362
1363void realmode_lidt(struct kvm_vcpu *vcpu, u16 limit, unsigned long base)
1364{
1365 struct descriptor_table dt = { limit, base };
1366
1367 kvm_arch_ops->set_idt(vcpu, &dt);
1368}
1369
1370void realmode_lmsw(struct kvm_vcpu *vcpu, unsigned long msw,
1371 unsigned long *rflags)
1372{
1373 lmsw(vcpu, msw);
1374 *rflags = kvm_arch_ops->get_rflags(vcpu);
1375}
1376
1377unsigned long realmode_get_cr(struct kvm_vcpu *vcpu, int cr)
1378{
25c4c276 1379 kvm_arch_ops->decache_cr4_guest_bits(vcpu);
6aa8b732
AK		 1380 switch (cr) {
1381 case 0:
1382 return vcpu->cr0;
1383 case 2:
1384 return vcpu->cr2;
1385 case 3:
1386 return vcpu->cr3;
1387 case 4:
1388 return vcpu->cr4;
1389 default:
1390 vcpu_printf(vcpu, "%s: unexpected cr %u\n", __FUNCTION__, cr);
1391 return 0;
1392 }
1393}
1394
1395void realmode_set_cr(struct kvm_vcpu *vcpu, int cr, unsigned long val,
1396 unsigned long *rflags)
1397{
1398 switch (cr) {
1399 case 0:
1400 set_cr0(vcpu, mk_cr_64(vcpu->cr0, val));
1401 *rflags = kvm_arch_ops->get_rflags(vcpu);
1402 break;
1403 case 2:
1404 vcpu->cr2 = val;
1405 break;
1406 case 3:
1407 set_cr3(vcpu, val);
1408 break;
1409 case 4:
1410 set_cr4(vcpu, mk_cr_64(vcpu->cr4, val));
1411 break;
1412 default:
1413 vcpu_printf(vcpu, "%s: unexpected cr %u\n", __FUNCTION__, cr);
1414 }
1415}
1416
102d8325
IM		 1417/*
1418 * Register the para guest with the host:
1419 */
1420static int vcpu_register_para(struct kvm_vcpu *vcpu, gpa_t para_state_gpa)
1421{
1422 struct kvm_vcpu_para_state *para_state;
1423 hpa_t para_state_hpa, hypercall_hpa;
1424 struct page *para_state_page;
1425 unsigned char *hypercall;
1426 gpa_t hypercall_gpa;
1427
1428 printk(KERN_DEBUG "kvm: guest trying to enter paravirtual mode\n");
1429 printk(KERN_DEBUG ".... para_state_gpa: %08Lx\n", para_state_gpa);
1430
1431 /*
1432 * Needs to be page aligned:
1433 */
1434 if (para_state_gpa != PAGE_ALIGN(para_state_gpa))
1435 goto err_gp;
1436
1437 para_state_hpa = gpa_to_hpa(vcpu, para_state_gpa);
1438 printk(KERN_DEBUG ".... para_state_hpa: %08Lx\n", para_state_hpa);
1439 if (is_error_hpa(para_state_hpa))
1440 goto err_gp;
1441
ab51a434 1442 mark_page_dirty(vcpu->kvm, para_state_gpa >> PAGE_SHIFT);
102d8325
IM		 1443 para_state_page = pfn_to_page(para_state_hpa >> PAGE_SHIFT);
1444 para_state = kmap_atomic(para_state_page, KM_USER0);
1445
1446 printk(KERN_DEBUG ".... guest version: %d\n", para_state->guest_version);
1447 printk(KERN_DEBUG ".... size: %d\n", para_state->size);
1448
1449 para_state->host_version = KVM_PARA_API_VERSION;
1450 /*
1451 * We cannot support guests that try to register themselves
1452 * with a newer API version than the host supports:
1453 */
1454 if (para_state->guest_version > KVM_PARA_API_VERSION) {
1455 para_state->ret = -KVM_EINVAL;
1456 goto err_kunmap_skip;
1457 }
1458
1459 hypercall_gpa = para_state->hypercall_gpa;
1460 hypercall_hpa = gpa_to_hpa(vcpu, hypercall_gpa);
1461 printk(KERN_DEBUG ".... hypercall_hpa: %08Lx\n", hypercall_hpa);
1462 if (is_error_hpa(hypercall_hpa)) {
1463 para_state->ret = -KVM_EINVAL;
1464 goto err_kunmap_skip;
1465 }
1466
1467 printk(KERN_DEBUG "kvm: para guest successfully registered.\n");
1468 vcpu->para_state_page = para_state_page;
1469 vcpu->para_state_gpa = para_state_gpa;
1470 vcpu->hypercall_gpa = hypercall_gpa;
1471
ab51a434 1472 mark_page_dirty(vcpu->kvm, hypercall_gpa >> PAGE_SHIFT);
102d8325
IM		 1473 hypercall = kmap_atomic(pfn_to_page(hypercall_hpa >> PAGE_SHIFT),
1474 KM_USER1) + (hypercall_hpa & ~PAGE_MASK);
1475 kvm_arch_ops->patch_hypercall(vcpu, hypercall);
1476 kunmap_atomic(hypercall, KM_USER1);
1477
1478 para_state->ret = 0;
1479err_kunmap_skip:
1480 kunmap_atomic(para_state, KM_USER0);
1481 return 0;
1482err_gp:
1483 return 1;
1484}
1485
3bab1f5d
AK		 1486int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
1487{
1488 u64 data;
1489
1490 switch (msr) {
1491 case 0xc0010010: /* SYSCFG */
1492 case 0xc0010015: /* HWCR */
1493 case MSR_IA32_PLATFORM_ID:
1494 case MSR_IA32_P5_MC_ADDR:
1495 case MSR_IA32_P5_MC_TYPE:
1496 case MSR_IA32_MC0_CTL:
1497 case MSR_IA32_MCG_STATUS:
1498 case MSR_IA32_MCG_CAP:
1499 case MSR_IA32_MC0_MISC:
1500 case MSR_IA32_MC0_MISC+4:
1501 case MSR_IA32_MC0_MISC+8:
1502 case MSR_IA32_MC0_MISC+12:
1503 case MSR_IA32_MC0_MISC+16:
1504 case MSR_IA32_UCODE_REV:
a8d13ea2 1505 case MSR_IA32_PERF_STATUS:
2dc7094b 1506 case MSR_IA32_EBL_CR_POWERON:
3bab1f5d
AK		 1507 /* MTRR registers */
1508 case 0xfe:
1509 case 0x200 ... 0x2ff:
1510 data = 0;
1511 break;
a8d13ea2
AK		 1512 case 0xcd: /* fsb frequency */
1513 data = 3;
1514 break;
3bab1f5d
AK		 1515 case MSR_IA32_APICBASE:
1516 data = vcpu->apic_base;
1517 break;
6f00e68f
AK		 1518 case MSR_IA32_MISC_ENABLE:
1519 data = vcpu->ia32_misc_enable_msr;
1520 break;
3bab1f5d
AK		 1521#ifdef CONFIG_X86_64
1522 case MSR_EFER:
1523 data = vcpu->shadow_efer;
1524 break;
1525#endif
1526 default:
1527 printk(KERN_ERR "kvm: unhandled rdmsr: 0x%x\n", msr);
1528 return 1;
1529 }
1530 *pdata = data;
1531 return 0;
1532}
1533EXPORT_SYMBOL_GPL(kvm_get_msr_common);
1534
6aa8b732
AK		 1535/*
1536 * Reads an msr value (of 'msr_index') into 'pdata'.
1537 * Returns 0 on success, non-0 otherwise.
1538 * Assumes vcpu_load() was already called.
1539 */
35f3f286 1540int kvm_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata)
6aa8b732
AK		 1541{
1542 return kvm_arch_ops->get_msr(vcpu, msr_index, pdata);
1543}
1544
05b3e0c2 1545#ifdef CONFIG_X86_64
6aa8b732 1546
3bab1f5d 1547static void set_efer(struct kvm_vcpu *vcpu, u64 efer)
6aa8b732 1548{
6aa8b732
AK		 1549 if (efer & EFER_RESERVED_BITS) {
1550 printk(KERN_DEBUG "set_efer: 0x%llx #GP, reserved bits\n",
1551 efer);
1552 inject_gp(vcpu);
1553 return;
1554 }
1555
1556 if (is_paging(vcpu)
1557 && (vcpu->shadow_efer & EFER_LME) != (efer & EFER_LME)) {
1558 printk(KERN_DEBUG "set_efer: #GP, change LME while paging\n");
1559 inject_gp(vcpu);
1560 return;
1561 }
1562
7725f0ba
AK		 1563 kvm_arch_ops->set_efer(vcpu, efer);
1564
6aa8b732
AK		 1565 efer &= ~EFER_LMA;
1566 efer |= vcpu->shadow_efer & EFER_LMA;
1567
1568 vcpu->shadow_efer = efer;
6aa8b732 1569}
6aa8b732
AK		 1570
1571#endif
1572
3bab1f5d
AK		 1573int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data)
1574{
1575 switch (msr) {
1576#ifdef CONFIG_X86_64
1577 case MSR_EFER:
1578 set_efer(vcpu, data);
1579 break;
1580#endif
1581 case MSR_IA32_MC0_STATUS:
1582 printk(KERN_WARNING "%s: MSR_IA32_MC0_STATUS 0x%llx, nop\n",
1583 __FUNCTION__, data);
1584 break;
0e5bf0d0
SK		 1585 case MSR_IA32_MCG_STATUS:
1586 printk(KERN_WARNING "%s: MSR_IA32_MCG_STATUS 0x%llx, nop\n",
1587 __FUNCTION__, data);
1588 break;
3bab1f5d
AK		 1589 case MSR_IA32_UCODE_REV:
1590 case MSR_IA32_UCODE_WRITE:
1591 case 0x200 ... 0x2ff: /* MTRRs */
1592 break;
1593 case MSR_IA32_APICBASE:
1594 vcpu->apic_base = data;
1595 break;
6f00e68f
AK		 1596 case MSR_IA32_MISC_ENABLE:
1597 vcpu->ia32_misc_enable_msr = data;
1598 break;
102d8325
IM		 1599 /*
1600 * This is the 'probe whether the host is KVM' logic:
1601 */
1602 case MSR_KVM_API_MAGIC:
1603 return vcpu_register_para(vcpu, data);
1604
3bab1f5d
AK		 1605 default:
1606 printk(KERN_ERR "kvm: unhandled wrmsr: 0x%x\n", msr);
1607 return 1;
1608 }
1609 return 0;
1610}
1611EXPORT_SYMBOL_GPL(kvm_set_msr_common);
1612
6aa8b732
AK		 1613/*
1614 * Writes msr value into into the appropriate "register".
1615 * Returns 0 on success, non-0 otherwise.
1616 * Assumes vcpu_load() was already called.
1617 */
35f3f286 1618int kvm_set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data)
6aa8b732
AK		 1619{
1620 return kvm_arch_ops->set_msr(vcpu, msr_index, data);
1621}
1622
1623void kvm_resched(struct kvm_vcpu *vcpu)
1624{
3fca0365
YD		 1625 if (!need_resched())
1626 return;
6aa8b732
AK		 1627 vcpu_put(vcpu);
1628 cond_resched();
bccf2150 1629 vcpu_load(vcpu);
6aa8b732
AK		 1630}
1631EXPORT_SYMBOL_GPL(kvm_resched);
1632
1633void load_msrs(struct vmx_msr_entry *e, int n)
1634{
1635 int i;
1636
1637 for (i = 0; i < n; ++i)
1638 wrmsrl(e[i].index, e[i].data);
1639}
1640EXPORT_SYMBOL_GPL(load_msrs);
1641
1642void save_msrs(struct vmx_msr_entry *e, int n)
1643{
1644 int i;
1645
1646 for (i = 0; i < n; ++i)
1647 rdmsrl(e[i].index, e[i].data);
1648}
1649EXPORT_SYMBOL_GPL(save_msrs);
1650
06465c5a
AK		 1651void kvm_emulate_cpuid(struct kvm_vcpu *vcpu)
1652{
1653 int i;
1654 u32 function;
1655 struct kvm_cpuid_entry *e, *best;
1656
1657 kvm_arch_ops->cache_regs(vcpu);
1658 function = vcpu->regs[VCPU_REGS_RAX];
1659 vcpu->regs[VCPU_REGS_RAX] = 0;
1660 vcpu->regs[VCPU_REGS_RBX] = 0;
1661 vcpu->regs[VCPU_REGS_RCX] = 0;
1662 vcpu->regs[VCPU_REGS_RDX] = 0;
1663 best = NULL;
1664 for (i = 0; i < vcpu->cpuid_nent; ++i) {
1665 e = &vcpu->cpuid_entries[i];
1666 if (e->function == function) {
1667 best = e;
1668 break;
1669 }
1670 /*
1671 * Both basic or both extended?
1672 */
1673 if (((e->function ^ function) & 0x80000000) == 0)
1674 if (!best || e->function > best->function)
1675 best = e;
1676 }
1677 if (best) {
1678 vcpu->regs[VCPU_REGS_RAX] = best->eax;
1679 vcpu->regs[VCPU_REGS_RBX] = best->ebx;
1680 vcpu->regs[VCPU_REGS_RCX] = best->ecx;
1681 vcpu->regs[VCPU_REGS_RDX] = best->edx;
1682 }
1683 kvm_arch_ops->decache_regs(vcpu);
1684 kvm_arch_ops->skip_emulated_instruction(vcpu);
1685}
1686EXPORT_SYMBOL_GPL(kvm_emulate_cpuid);
1687
039576c0 1688static int pio_copy_data(struct kvm_vcpu *vcpu)
46fc1477 1689{
039576c0
AK		 1690 void *p = vcpu->pio_data;
1691 void *q;
1692 unsigned bytes;
1693 int nr_pages = vcpu->pio.guest_pages[1] ? 2 : 1;
1694
1695 kvm_arch_ops->vcpu_put(vcpu);
1696 q = vmap(vcpu->pio.guest_pages, nr_pages, VM_READ|VM_WRITE,
1697 PAGE_KERNEL);
1698 if (!q) {
1699 kvm_arch_ops->vcpu_load(vcpu);
1700 free_pio_guest_pages(vcpu);
1701 return -ENOMEM;
1702 }
1703 q += vcpu->pio.guest_page_offset;
1704 bytes = vcpu->pio.size * vcpu->pio.cur_count;
1705 if (vcpu->pio.in)
1706 memcpy(q, p, bytes);
1707 else
1708 memcpy(p, q, bytes);
1709 q -= vcpu->pio.guest_page_offset;
1710 vunmap(q);
1711 kvm_arch_ops->vcpu_load(vcpu);
1712 free_pio_guest_pages(vcpu);
1713 return 0;
1714}
1715
1716static int complete_pio(struct kvm_vcpu *vcpu)
1717{
1718 struct kvm_pio_request *io = &vcpu->pio;
46fc1477 1719 long delta;
039576c0 1720 int r;
46fc1477
AK		 1721
1722 kvm_arch_ops->cache_regs(vcpu);
1723
1724 if (!io->string) {
039576c0
AK		 1725 if (io->in)
1726 memcpy(&vcpu->regs[VCPU_REGS_RAX], vcpu->pio_data,
46fc1477
AK		 1727 io->size);
1728 } else {
039576c0
AK		 1729 if (io->in) {
1730 r = pio_copy_data(vcpu);
1731 if (r) {
1732 kvm_arch_ops->cache_regs(vcpu);
1733 return r;
1734 }
1735 }
1736
46fc1477
AK		 1737 delta = 1;
1738 if (io->rep) {
039576c0 1739 delta *= io->cur_count;
46fc1477
AK		 1740 /*
1741 * The size of the register should really depend on
1742 * current address size.
1743 */
1744 vcpu->regs[VCPU_REGS_RCX] -= delta;
1745 }
039576c0 1746 if (io->down)
46fc1477
AK		 1747 delta = -delta;
1748 delta *= io->size;
039576c0 1749 if (io->in)
46fc1477
AK		 1750 vcpu->regs[VCPU_REGS_RDI] += delta;
1751 else
1752 vcpu->regs[VCPU_REGS_RSI] += delta;
1753 }
1754
46fc1477
AK		 1755 kvm_arch_ops->decache_regs(vcpu);
1756
039576c0
AK		 1757 io->count -= io->cur_count;
1758 io->cur_count = 0;
1759
1760 if (!io->count)
1761 kvm_arch_ops->skip_emulated_instruction(vcpu);
1762 return 0;
46fc1477
AK		 1763}
1764
74906345
ED		 1765void kernel_pio(struct kvm_io_device *pio_dev, struct kvm_vcpu *vcpu)
1766{
1767 /* TODO: String I/O for in kernel device */
1768
1769 if (vcpu->pio.in)
1770 kvm_iodevice_read(pio_dev, vcpu->pio.port,
1771 vcpu->pio.size,
1772 vcpu->pio_data);
1773 else
1774 kvm_iodevice_write(pio_dev, vcpu->pio.port,
1775 vcpu->pio.size,
1776 vcpu->pio_data);
1777}
1778
039576c0
AK		 1779int kvm_setup_pio(struct kvm_vcpu *vcpu, struct kvm_run *run, int in,
1780 int size, unsigned long count, int string, int down,
1781 gva_t address, int rep, unsigned port)
1782{
1783 unsigned now, in_page;
1784 int i;
1785 int nr_pages = 1;
1786 struct page *page;
74906345 1787 struct kvm_io_device *pio_dev;
039576c0
AK		 1788
1789 vcpu->run->exit_reason = KVM_EXIT_IO;
1790 vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT;
1791 vcpu->run->io.size = size;
1792 vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE;
1793 vcpu->run->io.count = count;
1794 vcpu->run->io.port = port;
1795 vcpu->pio.count = count;
1796 vcpu->pio.cur_count = count;
1797 vcpu->pio.size = size;
1798 vcpu->pio.in = in;
74906345 1799 vcpu->pio.port = port;
039576c0
AK		 1800 vcpu->pio.string = string;
1801 vcpu->pio.down = down;
1802 vcpu->pio.guest_page_offset = offset_in_page(address);
1803 vcpu->pio.rep = rep;
1804
74906345 1805 pio_dev = vcpu_find_pio_dev(vcpu, port);
039576c0
AK		 1806 if (!string) {
1807 kvm_arch_ops->cache_regs(vcpu);
1808 memcpy(vcpu->pio_data, &vcpu->regs[VCPU_REGS_RAX], 4);
1809 kvm_arch_ops->decache_regs(vcpu);
74906345
ED		 1810 if (pio_dev) {
1811 kernel_pio(pio_dev, vcpu);
1812 complete_pio(vcpu);
1813 return 1;
1814 }
039576c0
AK		 1815 return 0;
1816 }
74906345
ED		 1817 /* TODO: String I/O for in kernel device */
1818 if (pio_dev)
1819 printk(KERN_ERR "kvm_setup_pio: no string io support\n");
039576c0
AK		 1820
1821 if (!count) {
1822 kvm_arch_ops->skip_emulated_instruction(vcpu);
1823 return 1;
1824 }
1825
1826 now = min(count, PAGE_SIZE / size);
1827
1828 if (!down)
1829 in_page = PAGE_SIZE - offset_in_page(address);
1830 else
1831 in_page = offset_in_page(address) + size;
1832 now = min(count, (unsigned long)in_page / size);
1833 if (!now) {
1834 /*
1835 * String I/O straddles page boundary. Pin two guest pages
1836 * so that we satisfy atomicity constraints. Do just one
1837 * transaction to avoid complexity.
1838 */
1839 nr_pages = 2;
1840 now = 1;
1841 }
1842 if (down) {
1843 /*
1844 * String I/O in reverse. Yuck. Kill the guest, fix later.
1845 */
1846 printk(KERN_ERR "kvm: guest string pio down\n");
1847 inject_gp(vcpu);
1848 return 1;
1849 }
1850 vcpu->run->io.count = now;
1851 vcpu->pio.cur_count = now;
1852
1853 for (i = 0; i < nr_pages; ++i) {
1854 spin_lock(&vcpu->kvm->lock);
1855 page = gva_to_page(vcpu, address + i * PAGE_SIZE);
1856 if (page)
1857 get_page(page);
1858 vcpu->pio.guest_pages[i] = page;
1859 spin_unlock(&vcpu->kvm->lock);
1860 if (!page) {
1861 inject_gp(vcpu);
1862 free_pio_guest_pages(vcpu);
1863 return 1;
1864 }
1865 }
1866
1867 if (!vcpu->pio.in)
1868 return pio_copy_data(vcpu);
1869 return 0;
1870}
1871EXPORT_SYMBOL_GPL(kvm_setup_pio);
1872
bccf2150 1873static int kvm_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
6aa8b732 1874{
6aa8b732 1875 int r;
1961d276 1876 sigset_t sigsaved;
6aa8b732 1877
bccf2150 1878 vcpu_load(vcpu);
6aa8b732 1879
1961d276
AK		 1880 if (vcpu->sigset_active)
1881 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
1882
54810342
DL		 1883 /* re-sync apic's tpr */
1884 vcpu->cr8 = kvm_run->cr8;
1885
02c83209
AK		 1886 if (vcpu->pio.cur_count) {
1887 r = complete_pio(vcpu);
1888 if (r)
1889 goto out;
1890 }
1891
1892 if (vcpu->mmio_needed) {
1893 memcpy(vcpu->mmio_data, kvm_run->mmio.data, 8);
1894 vcpu->mmio_read_completed = 1;
1895 vcpu->mmio_needed = 0;
1896 r = emulate_instruction(vcpu, kvm_run,
1897 vcpu->mmio_fault_cr2, 0);
1898 if (r == EMULATE_DO_MMIO) {
1899 /*
1900 * Read-modify-write. Back to userspace.
1901 */
1902 kvm_run->exit_reason = KVM_EXIT_MMIO;
1903 r = 0;
1904 goto out;
46fc1477 1905 }
6aa8b732
AK		 1906 }
1907
8eb7d334 1908 if (kvm_run->exit_reason == KVM_EXIT_HYPERCALL) {
b4e63f56
AK		 1909 kvm_arch_ops->cache_regs(vcpu);
1910 vcpu->regs[VCPU_REGS_RAX] = kvm_run->hypercall.ret;
1911 kvm_arch_ops->decache_regs(vcpu);
1912 }
1913
6aa8b732
AK		 1914 r = kvm_arch_ops->run(vcpu, kvm_run);
1915
039576c0 1916out:
1961d276
AK		 1917 if (vcpu->sigset_active)
1918 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
1919
6aa8b732
AK		 1920 vcpu_put(vcpu);
1921 return r;
1922}
1923
bccf2150
AK		 1924static int kvm_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu,
1925 struct kvm_regs *regs)
6aa8b732 1926{
bccf2150 1927 vcpu_load(vcpu);
6aa8b732
AK		 1928
1929 kvm_arch_ops->cache_regs(vcpu);
1930
1931 regs->rax = vcpu->regs[VCPU_REGS_RAX];
1932 regs->rbx = vcpu->regs[VCPU_REGS_RBX];
1933 regs->rcx = vcpu->regs[VCPU_REGS_RCX];
1934 regs->rdx = vcpu->regs[VCPU_REGS_RDX];
1935 regs->rsi = vcpu->regs[VCPU_REGS_RSI];
1936 regs->rdi = vcpu->regs[VCPU_REGS_RDI];
1937 regs->rsp = vcpu->regs[VCPU_REGS_RSP];
1938 regs->rbp = vcpu->regs[VCPU_REGS_RBP];
05b3e0c2 1939#ifdef CONFIG_X86_64
6aa8b732
AK		 1940 regs->r8 = vcpu->regs[VCPU_REGS_R8];
1941 regs->r9 = vcpu->regs[VCPU_REGS_R9];
1942 regs->r10 = vcpu->regs[VCPU_REGS_R10];
1943 regs->r11 = vcpu->regs[VCPU_REGS_R11];
1944 regs->r12 = vcpu->regs[VCPU_REGS_R12];
1945 regs->r13 = vcpu->regs[VCPU_REGS_R13];
1946 regs->r14 = vcpu->regs[VCPU_REGS_R14];
1947 regs->r15 = vcpu->regs[VCPU_REGS_R15];
1948#endif
1949
1950 regs->rip = vcpu->rip;
1951 regs->rflags = kvm_arch_ops->get_rflags(vcpu);
1952
1953 /*
1954 * Don't leak debug flags in case they were set for guest debugging
1955 */
1956 if (vcpu->guest_debug.enabled && vcpu->guest_debug.singlestep)
1957 regs->rflags &= ~(X86_EFLAGS_TF | X86_EFLAGS_RF);
1958
1959 vcpu_put(vcpu);
1960
1961 return 0;
1962}
1963
bccf2150
AK		 1964static int kvm_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu,
1965 struct kvm_regs *regs)
6aa8b732 1966{
bccf2150 1967 vcpu_load(vcpu);
6aa8b732
AK		 1968
1969 vcpu->regs[VCPU_REGS_RAX] = regs->rax;
1970 vcpu->regs[VCPU_REGS_RBX] = regs->rbx;
1971 vcpu->regs[VCPU_REGS_RCX] = regs->rcx;
1972 vcpu->regs[VCPU_REGS_RDX] = regs->rdx;
1973 vcpu->regs[VCPU_REGS_RSI] = regs->rsi;
1974 vcpu->regs[VCPU_REGS_RDI] = regs->rdi;
1975 vcpu->regs[VCPU_REGS_RSP] = regs->rsp;
1976 vcpu->regs[VCPU_REGS_RBP] = regs->rbp;
05b3e0c2 1977#ifdef CONFIG_X86_64
6aa8b732
AK		 1978 vcpu->regs[VCPU_REGS_R8] = regs->r8;
1979 vcpu->regs[VCPU_REGS_R9] = regs->r9;
1980 vcpu->regs[VCPU_REGS_R10] = regs->r10;
1981 vcpu->regs[VCPU_REGS_R11] = regs->r11;
1982 vcpu->regs[VCPU_REGS_R12] = regs->r12;
1983 vcpu->regs[VCPU_REGS_R13] = regs->r13;
1984 vcpu->regs[VCPU_REGS_R14] = regs->r14;
1985 vcpu->regs[VCPU_REGS_R15] = regs->r15;
1986#endif
1987
1988 vcpu->rip = regs->rip;
1989 kvm_arch_ops->set_rflags(vcpu, regs->rflags);
1990
1991 kvm_arch_ops->decache_regs(vcpu);
1992
1993 vcpu_put(vcpu);
1994
1995 return 0;
1996}
1997
1998static void get_segment(struct kvm_vcpu *vcpu,
1999 struct kvm_segment *var, int seg)
2000{
2001 return kvm_arch_ops->get_segment(vcpu, var, seg);
2002}
2003
bccf2150
AK		 2004static int kvm_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
2005 struct kvm_sregs *sregs)
6aa8b732 2006{
6aa8b732
AK		 2007 struct descriptor_table dt;
2008
bccf2150 2009 vcpu_load(vcpu);
6aa8b732
AK		 2010
2011 get_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
2012 get_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
2013 get_segment(vcpu, &sregs->es, VCPU_SREG_ES);
2014 get_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
2015 get_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
2016 get_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
2017
2018 get_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
2019 get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
2020
2021 kvm_arch_ops->get_idt(vcpu, &dt);
2022 sregs->idt.limit = dt.limit;
2023 sregs->idt.base = dt.base;
2024 kvm_arch_ops->get_gdt(vcpu, &dt);
2025 sregs->gdt.limit = dt.limit;
2026 sregs->gdt.base = dt.base;
2027
25c4c276 2028 kvm_arch_ops->decache_cr4_guest_bits(vcpu);
6aa8b732
AK		 2029 sregs->cr0 = vcpu->cr0;
2030 sregs->cr2 = vcpu->cr2;
2031 sregs->cr3 = vcpu->cr3;
2032 sregs->cr4 = vcpu->cr4;
2033 sregs->cr8 = vcpu->cr8;
2034 sregs->efer = vcpu->shadow_efer;
2035 sregs->apic_base = vcpu->apic_base;
2036
2037 memcpy(sregs->interrupt_bitmap, vcpu->irq_pending,
2038 sizeof sregs->interrupt_bitmap);
2039
2040 vcpu_put(vcpu);
2041
2042 return 0;
2043}
2044
2045static void set_segment(struct kvm_vcpu *vcpu,
2046 struct kvm_segment *var, int seg)
2047{
2048 return kvm_arch_ops->set_segment(vcpu, var, seg);
2049}
2050
bccf2150
AK		 2051static int kvm_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
2052 struct kvm_sregs *sregs)
6aa8b732 2053{
6aa8b732
AK		 2054 int mmu_reset_needed = 0;
2055 int i;
2056 struct descriptor_table dt;
2057
bccf2150 2058 vcpu_load(vcpu);
6aa8b732 2059
6aa8b732
AK		 2060 dt.limit = sregs->idt.limit;
2061 dt.base = sregs->idt.base;
2062 kvm_arch_ops->set_idt(vcpu, &dt);
2063 dt.limit = sregs->gdt.limit;
2064 dt.base = sregs->gdt.base;
2065 kvm_arch_ops->set_gdt(vcpu, &dt);
2066
2067 vcpu->cr2 = sregs->cr2;
2068 mmu_reset_needed |= vcpu->cr3 != sregs->cr3;
2069 vcpu->cr3 = sregs->cr3;
2070
2071 vcpu->cr8 = sregs->cr8;
2072
2073 mmu_reset_needed |= vcpu->shadow_efer != sregs->efer;
05b3e0c2 2074#ifdef CONFIG_X86_64
6aa8b732
AK		 2075 kvm_arch_ops->set_efer(vcpu, sregs->efer);
2076#endif
2077 vcpu->apic_base = sregs->apic_base;
2078
25c4c276 2079 kvm_arch_ops->decache_cr4_guest_bits(vcpu);
399badf3 2080
6aa8b732 2081 mmu_reset_needed |= vcpu->cr0 != sregs->cr0;
f6528b03 2082 kvm_arch_ops->set_cr0(vcpu, sregs->cr0);
6aa8b732
AK		 2083
2084 mmu_reset_needed |= vcpu->cr4 != sregs->cr4;
2085 kvm_arch_ops->set_cr4(vcpu, sregs->cr4);
1b0973bd
AK		 2086 if (!is_long_mode(vcpu) && is_pae(vcpu))
2087 load_pdptrs(vcpu, vcpu->cr3);
6aa8b732
AK		 2088
2089 if (mmu_reset_needed)
2090 kvm_mmu_reset_context(vcpu);
2091
2092 memcpy(vcpu->irq_pending, sregs->interrupt_bitmap,
2093 sizeof vcpu->irq_pending);
2094 vcpu->irq_summary = 0;
2095 for (i = 0; i < NR_IRQ_WORDS; ++i)
2096 if (vcpu->irq_pending[i])
2097 __set_bit(i, &vcpu->irq_summary);
2098
024aa1c0
AK		 2099 set_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
2100 set_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
2101 set_segment(vcpu, &sregs->es, VCPU_SREG_ES);
2102 set_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
2103 set_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
2104 set_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
2105
2106 set_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
2107 set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
2108
6aa8b732
AK		 2109 vcpu_put(vcpu);
2110
2111 return 0;
2112}
2113
2114/*
2115 * List of msr numbers which we expose to userspace through KVM_GET_MSRS
2116 * and KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST.
bf591b24
MR		 2117 *
2118 * This list is modified at module load time to reflect the
2119 * capabilities of the host cpu.
6aa8b732
AK		 2120 */
2121static u32 msrs_to_save[] = {
2122 MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
2123 MSR_K6_STAR,
05b3e0c2 2124#ifdef CONFIG_X86_64
6aa8b732
AK		 2125 MSR_CSTAR, MSR_KERNEL_GS_BASE, MSR_SYSCALL_MASK, MSR_LSTAR,
2126#endif
2127 MSR_IA32_TIME_STAMP_COUNTER,
2128};
2129
bf591b24
MR		 2130static unsigned num_msrs_to_save;
2131
6f00e68f
AK		 2132static u32 emulated_msrs[] = {
2133 MSR_IA32_MISC_ENABLE,
2134};
2135
bf591b24
MR		 2136static __init void kvm_init_msr_list(void)
2137{
2138 u32 dummy[2];
2139 unsigned i, j;
2140
2141 for (i = j = 0; i < ARRAY_SIZE(msrs_to_save); i++) {
2142 if (rdmsr_safe(msrs_to_save[i], &dummy[0], &dummy[1]) < 0)
2143 continue;
2144 if (j < i)
2145 msrs_to_save[j] = msrs_to_save[i];
2146 j++;
2147 }
2148 num_msrs_to_save = j;
2149}
6aa8b732
AK		 2150
2151/*
2152 * Adapt set_msr() to msr_io()'s calling convention
2153 */
2154static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
2155{
35f3f286 2156 return kvm_set_msr(vcpu, index, *data);
6aa8b732
AK		 2157}
2158
2159/*
2160 * Read or write a bunch of msrs. All parameters are kernel addresses.
2161 *
2162 * @return number of msrs set successfully.
2163 */
bccf2150 2164static int __msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs,
6aa8b732
AK		 2165 struct kvm_msr_entry *entries,
2166 int (*do_msr)(struct kvm_vcpu *vcpu,
2167 unsigned index, u64 *data))
2168{
6aa8b732
AK		 2169 int i;
2170
bccf2150 2171 vcpu_load(vcpu);
6aa8b732
AK		 2172
2173 for (i = 0; i < msrs->nmsrs; ++i)
2174 if (do_msr(vcpu, entries[i].index, &entries[i].data))
2175 break;
2176
2177 vcpu_put(vcpu);
2178
2179 return i;
2180}
2181
2182/*
2183 * Read or write a bunch of msrs. Parameters are user addresses.
2184 *
2185 * @return number of msrs set successfully.
2186 */
bccf2150 2187static int msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs,
6aa8b732
AK		 2188 int (*do_msr)(struct kvm_vcpu *vcpu,
2189 unsigned index, u64 *data),
2190 int writeback)
2191{
2192 struct kvm_msrs msrs;
2193 struct kvm_msr_entry *entries;
2194 int r, n;
2195 unsigned size;
2196
2197 r = -EFAULT;
2198 if (copy_from_user(&msrs, user_msrs, sizeof msrs))
2199 goto out;
2200
2201 r = -E2BIG;
2202 if (msrs.nmsrs >= MAX_IO_MSRS)
2203 goto out;
2204
2205 r = -ENOMEM;
2206 size = sizeof(struct kvm_msr_entry) * msrs.nmsrs;
2207 entries = vmalloc(size);
2208 if (!entries)
2209 goto out;
2210
2211 r = -EFAULT;
2212 if (copy_from_user(entries, user_msrs->entries, size))
2213 goto out_free;
2214
bccf2150 2215 r = n = __msr_io(vcpu, &msrs, entries, do_msr);
6aa8b732
AK		 2216 if (r < 0)
2217 goto out_free;
2218
2219 r = -EFAULT;
2220 if (writeback && copy_to_user(user_msrs->entries, entries, size))
2221 goto out_free;
2222
2223 r = n;
2224
2225out_free:
2226 vfree(entries);
2227out:
2228 return r;
2229}
2230
2231/*
2232 * Translate a guest virtual address to a guest physical address.
2233 */
bccf2150
AK		 2234static int kvm_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
2235 struct kvm_translation *tr)
6aa8b732
AK		 2236{
2237 unsigned long vaddr = tr->linear_address;
6aa8b732
AK		 2238 gpa_t gpa;
2239
bccf2150
AK		 2240 vcpu_load(vcpu);
2241 spin_lock(&vcpu->kvm->lock);
6aa8b732
AK		 2242 gpa = vcpu->mmu.gva_to_gpa(vcpu, vaddr);
2243 tr->physical_address = gpa;
2244 tr->valid = gpa != UNMAPPED_GVA;
2245 tr->writeable = 1;
2246 tr->usermode = 0;
bccf2150 2247 spin_unlock(&vcpu->kvm->lock);
6aa8b732
AK		 2248 vcpu_put(vcpu);
2249
2250 return 0;
2251}
2252
bccf2150
AK		 2253static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu,
2254 struct kvm_interrupt *irq)
6aa8b732 2255{
6aa8b732
AK		 2256 if (irq->irq < 0 || irq->irq >= 256)
2257 return -EINVAL;
bccf2150 2258 vcpu_load(vcpu);
6aa8b732
AK		 2259
2260 set_bit(irq->irq, vcpu->irq_pending);
2261 set_bit(irq->irq / BITS_PER_LONG, &vcpu->irq_summary);
2262
2263 vcpu_put(vcpu);
2264
2265 return 0;
2266}
2267
bccf2150
AK		 2268static int kvm_vcpu_ioctl_debug_guest(struct kvm_vcpu *vcpu,
2269 struct kvm_debug_guest *dbg)
6aa8b732 2270{
6aa8b732
AK		 2271 int r;
2272
bccf2150 2273 vcpu_load(vcpu);
6aa8b732
AK		 2274
2275 r = kvm_arch_ops->set_guest_debug(vcpu, dbg);
2276
2277 vcpu_put(vcpu);
2278
2279 return r;
2280}
2281
9a2bb7f4
AK		 2282static struct page *kvm_vcpu_nopage(struct vm_area_struct *vma,
2283 unsigned long address,
2284 int *type)
2285{
2286 struct kvm_vcpu *vcpu = vma->vm_file->private_data;
2287 unsigned long pgoff;
2288 struct page *page;
2289
2290 *type = VM_FAULT_MINOR;
2291 pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
039576c0
AK		 2292 if (pgoff == 0)
2293 page = virt_to_page(vcpu->run);
2294 else if (pgoff == KVM_PIO_PAGE_OFFSET)
2295 page = virt_to_page(vcpu->pio_data);
2296 else
9a2bb7f4 2297 return NOPAGE_SIGBUS;
9a2bb7f4
AK		 2298 get_page(page);
2299 return page;
2300}
2301
2302static struct vm_operations_struct kvm_vcpu_vm_ops = {
2303 .nopage = kvm_vcpu_nopage,
2304};
2305
2306static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma)
2307{
2308 vma->vm_ops = &kvm_vcpu_vm_ops;
2309 return 0;
2310}
2311
bccf2150
AK		 2312static int kvm_vcpu_release(struct inode *inode, struct file *filp)
2313{
2314 struct kvm_vcpu *vcpu = filp->private_data;
2315
2316 fput(vcpu->kvm->filp);
2317 return 0;
2318}
2319
2320static struct file_operations kvm_vcpu_fops = {
2321 .release = kvm_vcpu_release,
2322 .unlocked_ioctl = kvm_vcpu_ioctl,
2323 .compat_ioctl = kvm_vcpu_ioctl,
9a2bb7f4 2324 .mmap = kvm_vcpu_mmap,
bccf2150
AK		 2325};
2326
2327/*
2328 * Allocates an inode for the vcpu.
2329 */
2330static int create_vcpu_fd(struct kvm_vcpu *vcpu)
2331{
2332 int fd, r;
2333 struct inode *inode;
2334 struct file *file;
2335
d6d28168
AK		 2336 r = anon_inode_getfd(&fd, &inode, &file,
2337 "kvm-vcpu", &kvm_vcpu_fops, vcpu);
2338 if (r)
2339 return r;
bccf2150 2340 atomic_inc(&vcpu->kvm->filp->f_count);
bccf2150 2341 return fd;
bccf2150
AK		 2342}
2343
c5ea7660
AK		 2344/*
2345 * Creates some virtual cpus. Good luck creating more than one.
2346 */
2347static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, int n)
2348{
2349 int r;
2350 struct kvm_vcpu *vcpu;
9a2bb7f4 2351 struct page *page;
c5ea7660
AK		 2352
2353 r = -EINVAL;
2354 if (!valid_vcpu(n))
2355 goto out;
2356
2357 vcpu = &kvm->vcpus[n];
2358
2359 mutex_lock(&vcpu->mutex);
2360
2361 if (vcpu->vmcs) {
2362 mutex_unlock(&vcpu->mutex);
2363 return -EEXIST;
2364 }
2365
9a2bb7f4
AK		 2366 page = alloc_page(GFP_KERNEL | __GFP_ZERO);
2367 r = -ENOMEM;
2368 if (!page)
2369 goto out_unlock;
2370 vcpu->run = page_address(page);
2371
039576c0
AK		 2372 page = alloc_page(GFP_KERNEL | __GFP_ZERO);
2373 r = -ENOMEM;
2374 if (!page)
2375 goto out_free_run;
2376 vcpu->pio_data = page_address(page);
2377
c5ea7660
AK		 2378 vcpu->host_fx_image = (char*)ALIGN((hva_t)vcpu->fx_buf,
2379 FX_IMAGE_ALIGN);
2380 vcpu->guest_fx_image = vcpu->host_fx_image + FX_IMAGE_SIZE;
d917a6b9 2381 vcpu->cr0 = 0x10;
c5ea7660
AK		 2382
2383 r = kvm_arch_ops->vcpu_create(vcpu);
2384 if (r < 0)
2385 goto out_free_vcpus;
2386
2387 r = kvm_mmu_create(vcpu);
2388 if (r < 0)
2389 goto out_free_vcpus;
2390
2391 kvm_arch_ops->vcpu_load(vcpu);
2392 r = kvm_mmu_setup(vcpu);
2393 if (r >= 0)
2394 r = kvm_arch_ops->vcpu_setup(vcpu);
2395 vcpu_put(vcpu);
2396
2397 if (r < 0)
2398 goto out_free_vcpus;
2399
bccf2150
AK		 2400 r = create_vcpu_fd(vcpu);
2401 if (r < 0)
2402 goto out_free_vcpus;
2403
39c3b86e
AK		 2404 spin_lock(&kvm_lock);
2405 if (n >= kvm->nvcpus)
2406 kvm->nvcpus = n + 1;
2407 spin_unlock(&kvm_lock);
2408
bccf2150 2409 return r;
c5ea7660
AK		 2410
2411out_free_vcpus:
2412 kvm_free_vcpu(vcpu);
039576c0
AK		 2413out_free_run:
2414 free_page((unsigned long)vcpu->run);
2415 vcpu->run = NULL;
9a2bb7f4 2416out_unlock:
c5ea7660
AK		 2417 mutex_unlock(&vcpu->mutex);
2418out:
2419 return r;
2420}
2421
2cc51560
ED		 2422static void cpuid_fix_nx_cap(struct kvm_vcpu *vcpu)
2423{
2424 u64 efer;
2425 int i;
2426 struct kvm_cpuid_entry *e, *entry;
2427
2428 rdmsrl(MSR_EFER, efer);
2429 entry = NULL;
2430 for (i = 0; i < vcpu->cpuid_nent; ++i) {
2431 e = &vcpu->cpuid_entries[i];
2432 if (e->function == 0x80000001) {
2433 entry = e;
2434 break;
2435 }
2436 }
2437 if (entry && (entry->edx & EFER_NX) && !(efer & EFER_NX)) {
2438 entry->edx &= ~(1 << 20);
2439 printk(KERN_INFO ": guest NX capability removed\n");
2440 }
2441}
2442
06465c5a
AK		 2443static int kvm_vcpu_ioctl_set_cpuid(struct kvm_vcpu *vcpu,
2444 struct kvm_cpuid *cpuid,
2445 struct kvm_cpuid_entry __user *entries)
2446{
2447 int r;
2448
2449 r = -E2BIG;
2450 if (cpuid->nent > KVM_MAX_CPUID_ENTRIES)
2451 goto out;
2452 r = -EFAULT;
2453 if (copy_from_user(&vcpu->cpuid_entries, entries,
2454 cpuid->nent * sizeof(struct kvm_cpuid_entry)))
2455 goto out;
2456 vcpu->cpuid_nent = cpuid->nent;
2cc51560 2457 cpuid_fix_nx_cap(vcpu);
06465c5a
AK		 2458 return 0;
2459
2460out:
2461 return r;
2462}
2463
1961d276
AK		 2464static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset)
2465{
2466 if (sigset) {
2467 sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP));
2468 vcpu->sigset_active = 1;
2469 vcpu->sigset = *sigset;
2470 } else
2471 vcpu->sigset_active = 0;
2472 return 0;
2473}
2474
b8836737
AK		 2475/*
2476 * fxsave fpu state. Taken from x86_64/processor.h. To be killed when
2477 * we have asm/x86/processor.h
2478 */
2479struct fxsave {
2480 u16 cwd;
2481 u16 swd;
2482 u16 twd;
2483 u16 fop;
2484 u64 rip;
2485 u64 rdp;
2486 u32 mxcsr;
2487 u32 mxcsr_mask;
2488 u32 st_space[32]; /* 8*16 bytes for each FP-reg = 128 bytes */
2489#ifdef CONFIG_X86_64
2490 u32 xmm_space[64]; /* 16*16 bytes for each XMM-reg = 256 bytes */
2491#else
2492 u32 xmm_space[32]; /* 8*16 bytes for each XMM-reg = 128 bytes */
2493#endif
2494};
2495
2496static int kvm_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
2497{
2498 struct fxsave *fxsave = (struct fxsave *)vcpu->guest_fx_image;
2499
2500 vcpu_load(vcpu);
2501
2502 memcpy(fpu->fpr, fxsave->st_space, 128);
2503 fpu->fcw = fxsave->cwd;
2504 fpu->fsw = fxsave->swd;
2505 fpu->ftwx = fxsave->twd;
2506 fpu->last_opcode = fxsave->fop;
2507 fpu->last_ip = fxsave->rip;
2508 fpu->last_dp = fxsave->rdp;
2509 memcpy(fpu->xmm, fxsave->xmm_space, sizeof fxsave->xmm_space);
2510
2511 vcpu_put(vcpu);
2512
2513 return 0;
2514}
2515
2516static int kvm_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
2517{
2518 struct fxsave *fxsave = (struct fxsave *)vcpu->guest_fx_image;
2519
2520 vcpu_load(vcpu);
2521
2522 memcpy(fxsave->st_space, fpu->fpr, 128);
2523 fxsave->cwd = fpu->fcw;
2524 fxsave->swd = fpu->fsw;
2525 fxsave->twd = fpu->ftwx;
2526 fxsave->fop = fpu->last_opcode;
2527 fxsave->rip = fpu->last_ip;
2528 fxsave->rdp = fpu->last_dp;
2529 memcpy(fxsave->xmm_space, fpu->xmm, sizeof fxsave->xmm_space);
2530
2531 vcpu_put(vcpu);
2532
2533 return 0;
2534}
2535
bccf2150
AK		 2536static long kvm_vcpu_ioctl(struct file *filp,
2537 unsigned int ioctl, unsigned long arg)
6aa8b732 2538{
bccf2150 2539 struct kvm_vcpu *vcpu = filp->private_data;
2f366987 2540 void __user *argp = (void __user *)arg;
6aa8b732
AK		 2541 int r = -EINVAL;
2542
2543 switch (ioctl) {
9a2bb7f4 2544 case KVM_RUN:
f0fe5108
AK		 2545 r = -EINVAL;
2546 if (arg)
2547 goto out;
9a2bb7f4 2548 r = kvm_vcpu_ioctl_run(vcpu, vcpu->run);
6aa8b732 2549 break;
6aa8b732
AK		 2550 case KVM_GET_REGS: {
2551 struct kvm_regs kvm_regs;
2552
bccf2150
AK		 2553 memset(&kvm_regs, 0, sizeof kvm_regs);
2554 r = kvm_vcpu_ioctl_get_regs(vcpu, &kvm_regs);
6aa8b732
AK		 2555 if (r)
2556 goto out;
2557 r = -EFAULT;
2f366987 2558 if (copy_to_user(argp, &kvm_regs, sizeof kvm_regs))
6aa8b732
AK		 2559 goto out;
2560 r = 0;
2561 break;
2562 }
2563 case KVM_SET_REGS: {
2564 struct kvm_regs kvm_regs;
2565
2566 r = -EFAULT;
2f366987 2567 if (copy_from_user(&kvm_regs, argp, sizeof kvm_regs))
6aa8b732 2568 goto out;
bccf2150 2569 r = kvm_vcpu_ioctl_set_regs(vcpu, &kvm_regs);
6aa8b732
AK		 2570 if (r)
2571 goto out;
2572 r = 0;
2573 break;
2574 }
2575 case KVM_GET_SREGS: {
2576 struct kvm_sregs kvm_sregs;
2577
bccf2150
AK		 2578 memset(&kvm_sregs, 0, sizeof kvm_sregs);
2579 r = kvm_vcpu_ioctl_get_sregs(vcpu, &kvm_sregs);
6aa8b732
AK		 2580 if (r)
2581 goto out;
2582 r = -EFAULT;
2f366987 2583 if (copy_to_user(argp, &kvm_sregs, sizeof kvm_sregs))
6aa8b732
AK		 2584 goto out;
2585 r = 0;
2586 break;
2587 }
2588 case KVM_SET_SREGS: {
2589 struct kvm_sregs kvm_sregs;
2590
2591 r = -EFAULT;
2f366987 2592 if (copy_from_user(&kvm_sregs, argp, sizeof kvm_sregs))
6aa8b732 2593 goto out;
bccf2150 2594 r = kvm_vcpu_ioctl_set_sregs(vcpu, &kvm_sregs);
6aa8b732
AK		 2595 if (r)
2596 goto out;
2597 r = 0;
2598 break;
2599 }
2600 case KVM_TRANSLATE: {
2601 struct kvm_translation tr;
2602
2603 r = -EFAULT;
2f366987 2604 if (copy_from_user(&tr, argp, sizeof tr))
6aa8b732 2605 goto out;
bccf2150 2606 r = kvm_vcpu_ioctl_translate(vcpu, &tr);
6aa8b732
AK		 2607 if (r)
2608 goto out;
2609 r = -EFAULT;
2f366987 2610 if (copy_to_user(argp, &tr, sizeof tr))
6aa8b732
AK		 2611 goto out;
2612 r = 0;
2613 break;
2614 }
2615 case KVM_INTERRUPT: {
2616 struct kvm_interrupt irq;
2617
2618 r = -EFAULT;
2f366987 2619 if (copy_from_user(&irq, argp, sizeof irq))
6aa8b732 2620 goto out;
bccf2150 2621 r = kvm_vcpu_ioctl_interrupt(vcpu, &irq);
6aa8b732
AK		 2622 if (r)
2623 goto out;
2624 r = 0;
2625 break;
2626 }
2627 case KVM_DEBUG_GUEST: {
2628 struct kvm_debug_guest dbg;
2629
2630 r = -EFAULT;
2f366987 2631 if (copy_from_user(&dbg, argp, sizeof dbg))
6aa8b732 2632 goto out;
bccf2150 2633 r = kvm_vcpu_ioctl_debug_guest(vcpu, &dbg);
6aa8b732
AK		 2634 if (r)
2635 goto out;
2636 r = 0;
2637 break;
2638 }
bccf2150 2639 case KVM_GET_MSRS:
35f3f286 2640 r = msr_io(vcpu, argp, kvm_get_msr, 1);
bccf2150
AK		 2641 break;
2642 case KVM_SET_MSRS:
2643 r = msr_io(vcpu, argp, do_set_msr, 0);
2644 break;
06465c5a
AK		 2645 case KVM_SET_CPUID: {
2646 struct kvm_cpuid __user *cpuid_arg = argp;
2647 struct kvm_cpuid cpuid;
2648
2649 r = -EFAULT;
2650 if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid))
2651 goto out;
2652 r = kvm_vcpu_ioctl_set_cpuid(vcpu, &cpuid, cpuid_arg->entries);
2653 if (r)
2654 goto out;
2655 break;
2656 }
1961d276
AK		 2657 case KVM_SET_SIGNAL_MASK: {
2658 struct kvm_signal_mask __user *sigmask_arg = argp;
2659 struct kvm_signal_mask kvm_sigmask;
2660 sigset_t sigset, *p;
2661
2662 p = NULL;
2663 if (argp) {
2664 r = -EFAULT;
2665 if (copy_from_user(&kvm_sigmask, argp,
2666 sizeof kvm_sigmask))
2667 goto out;
2668 r = -EINVAL;
2669 if (kvm_sigmask.len != sizeof sigset)
2670 goto out;
2671 r = -EFAULT;
2672 if (copy_from_user(&sigset, sigmask_arg->sigset,
2673 sizeof sigset))
2674 goto out;
2675 p = &sigset;
2676 }
2677 r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset);
2678 break;
2679 }
b8836737
AK		 2680 case KVM_GET_FPU: {
2681 struct kvm_fpu fpu;
2682
2683 memset(&fpu, 0, sizeof fpu);
2684 r = kvm_vcpu_ioctl_get_fpu(vcpu, &fpu);
2685 if (r)
2686 goto out;
2687 r = -EFAULT;
2688 if (copy_to_user(argp, &fpu, sizeof fpu))
2689 goto out;
2690 r = 0;
2691 break;
2692 }
2693 case KVM_SET_FPU: {
2694 struct kvm_fpu fpu;
2695
2696 r = -EFAULT;
2697 if (copy_from_user(&fpu, argp, sizeof fpu))
2698 goto out;
2699 r = kvm_vcpu_ioctl_set_fpu(vcpu, &fpu);
2700 if (r)
2701 goto out;
2702 r = 0;
2703 break;
2704 }
bccf2150
AK		 2705 default:
2706 ;
2707 }
2708out:
2709 return r;
2710}
2711
2712static long kvm_vm_ioctl(struct file *filp,
2713 unsigned int ioctl, unsigned long arg)
2714{
2715 struct kvm *kvm = filp->private_data;
2716 void __user *argp = (void __user *)arg;
2717 int r = -EINVAL;
2718
2719 switch (ioctl) {
2720 case KVM_CREATE_VCPU:
2721 r = kvm_vm_ioctl_create_vcpu(kvm, arg);
2722 if (r < 0)
2723 goto out;
2724 break;
6aa8b732
AK		 2725 case KVM_SET_MEMORY_REGION: {
2726 struct kvm_memory_region kvm_mem;
2727
2728 r = -EFAULT;
2f366987 2729 if (copy_from_user(&kvm_mem, argp, sizeof kvm_mem))
6aa8b732 2730 goto out;
2c6f5df9 2731 r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_mem);
6aa8b732
AK		 2732 if (r)
2733 goto out;
2734 break;
2735 }
2736 case KVM_GET_DIRTY_LOG: {
2737 struct kvm_dirty_log log;
2738
2739 r = -EFAULT;
2f366987 2740 if (copy_from_user(&log, argp, sizeof log))
6aa8b732 2741 goto out;
2c6f5df9 2742 r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
6aa8b732
AK		 2743 if (r)
2744 goto out;
2745 break;
2746 }
e8207547
AK		 2747 case KVM_SET_MEMORY_ALIAS: {
2748 struct kvm_memory_alias alias;
2749
2750 r = -EFAULT;
2751 if (copy_from_user(&alias, argp, sizeof alias))
2752 goto out;
2753 r = kvm_vm_ioctl_set_memory_alias(kvm, &alias);
2754 if (r)
2755 goto out;
2756 break;
2757 }
f17abe9a
AK		 2758 default:
2759 ;
2760 }
2761out:
2762 return r;
2763}
2764
2765static struct page *kvm_vm_nopage(struct vm_area_struct *vma,
2766 unsigned long address,
2767 int *type)
2768{
2769 struct kvm *kvm = vma->vm_file->private_data;
2770 unsigned long pgoff;
f17abe9a
AK		 2771 struct page *page;
2772
2773 *type = VM_FAULT_MINOR;
2774 pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
954bbbc2 2775 page = gfn_to_page(kvm, pgoff);
f17abe9a
AK		 2776 if (!page)
2777 return NOPAGE_SIGBUS;
2778 get_page(page);
2779 return page;
2780}
2781
2782static struct vm_operations_struct kvm_vm_vm_ops = {
2783 .nopage = kvm_vm_nopage,
2784};
2785
2786static int kvm_vm_mmap(struct file *file, struct vm_area_struct *vma)
2787{
2788 vma->vm_ops = &kvm_vm_vm_ops;
2789 return 0;
2790}
2791
2792static struct file_operations kvm_vm_fops = {
2793 .release = kvm_vm_release,
2794 .unlocked_ioctl = kvm_vm_ioctl,
2795 .compat_ioctl = kvm_vm_ioctl,
2796 .mmap = kvm_vm_mmap,
2797};
2798
2799static int kvm_dev_ioctl_create_vm(void)
2800{
2801 int fd, r;
2802 struct inode *inode;
2803 struct file *file;
2804 struct kvm *kvm;
2805
f17abe9a 2806 kvm = kvm_create_vm();
d6d28168
AK		 2807 if (IS_ERR(kvm))
2808 return PTR_ERR(kvm);
2809 r = anon_inode_getfd(&fd, &inode, &file, "kvm-vm", &kvm_vm_fops, kvm);
2810 if (r) {
2811 kvm_destroy_vm(kvm);
2812 return r;
f17abe9a
AK		 2813 }
2814
bccf2150 2815 kvm->filp = file;
f17abe9a 2816
f17abe9a 2817 return fd;
f17abe9a
AK		 2818}
2819
2820static long kvm_dev_ioctl(struct file *filp,
2821 unsigned int ioctl, unsigned long arg)
2822{
2823 void __user *argp = (void __user *)arg;
07c45a36 2824 long r = -EINVAL;
f17abe9a
AK		 2825
2826 switch (ioctl) {
2827 case KVM_GET_API_VERSION:
f0fe5108
AK		 2828 r = -EINVAL;
2829 if (arg)
2830 goto out;
f17abe9a
AK		 2831 r = KVM_API_VERSION;
2832 break;
2833 case KVM_CREATE_VM:
f0fe5108
AK		 2834 r = -EINVAL;
2835 if (arg)
2836 goto out;
f17abe9a
AK		 2837 r = kvm_dev_ioctl_create_vm();
2838 break;
6aa8b732 2839 case KVM_GET_MSR_INDEX_LIST: {
2f366987 2840 struct kvm_msr_list __user *user_msr_list = argp;
6aa8b732
AK		 2841 struct kvm_msr_list msr_list;
2842 unsigned n;
2843
2844 r = -EFAULT;
2845 if (copy_from_user(&msr_list, user_msr_list, sizeof msr_list))
2846 goto out;
2847 n = msr_list.nmsrs;
6f00e68f 2848 msr_list.nmsrs = num_msrs_to_save + ARRAY_SIZE(emulated_msrs);
6aa8b732
AK		 2849 if (copy_to_user(user_msr_list, &msr_list, sizeof msr_list))
2850 goto out;
2851 r = -E2BIG;
bf591b24 2852 if (n < num_msrs_to_save)
6aa8b732
AK		 2853 goto out;
2854 r = -EFAULT;
2855 if (copy_to_user(user_msr_list->indices, &msrs_to_save,
bf591b24 2856 num_msrs_to_save * sizeof(u32)))
6aa8b732 2857 goto out;
6f00e68f
AK		 2858 if (copy_to_user(user_msr_list->indices
2859 + num_msrs_to_save * sizeof(u32),
2860 &emulated_msrs,
2861 ARRAY_SIZE(emulated_msrs) * sizeof(u32)))
2862 goto out;
6aa8b732 2863 r = 0;
cc1d8955 2864 break;
6aa8b732 2865 }
5d308f45
AK		 2866 case KVM_CHECK_EXTENSION:
2867 /*
2868 * No extensions defined at present.
2869 */
2870 r = 0;
2871 break;
07c45a36
AK		 2872 case KVM_GET_VCPU_MMAP_SIZE:
2873 r = -EINVAL;
2874 if (arg)
2875 goto out;
039576c0 2876 r = 2 * PAGE_SIZE;
07c45a36 2877 break;
6aa8b732
AK		 2878 default:
2879 ;
2880 }
2881out:
2882 return r;
2883}
2884
6aa8b732
AK		 2885static struct file_operations kvm_chardev_ops = {
2886 .open = kvm_dev_open,
2887 .release = kvm_dev_release,
2888 .unlocked_ioctl = kvm_dev_ioctl,
2889 .compat_ioctl = kvm_dev_ioctl,
6aa8b732
AK		 2890};
2891
2892static struct miscdevice kvm_dev = {
bbe4432e 2893 KVM_MINOR,
6aa8b732
AK		 2894 "kvm",
2895 &kvm_chardev_ops,
2896};
2897
2898static int kvm_reboot(struct notifier_block *notifier, unsigned long val,
2899 void *v)
2900{
2901 if (val == SYS_RESTART) {
2902 /*
2903 * Some (well, at least mine) BIOSes hang on reboot if
2904 * in vmx root mode.
2905 */
2906 printk(KERN_INFO "kvm: exiting hardware virtualization\n");
1b6c0168 2907 on_each_cpu(hardware_disable, NULL, 0, 1);
6aa8b732
AK		 2908 }
2909 return NOTIFY_OK;
2910}
2911
2912static struct notifier_block kvm_reboot_notifier = {
2913 .notifier_call = kvm_reboot,
2914 .priority = 0,
2915};
2916
774c47f1
AK		 2917/*
2918 * Make sure that a cpu that is being hot-unplugged does not have any vcpus
2919 * cached on it.
2920 */
2921static void decache_vcpus_on_cpu(int cpu)
2922{
2923 struct kvm *vm;
2924 struct kvm_vcpu *vcpu;
2925 int i;
2926
2927 spin_lock(&kvm_lock);
2928 list_for_each_entry(vm, &vm_list, vm_list)
2929 for (i = 0; i < KVM_MAX_VCPUS; ++i) {
2930 vcpu = &vm->vcpus[i];
2931 /*
2932 * If the vcpu is locked, then it is running on some
2933 * other cpu and therefore it is not cached on the
2934 * cpu in question.
2935 *
2936 * If it's not locked, check the last cpu it executed
2937 * on.
2938 */
2939 if (mutex_trylock(&vcpu->mutex)) {
2940 if (vcpu->cpu == cpu) {
2941 kvm_arch_ops->vcpu_decache(vcpu);
2942 vcpu->cpu = -1;
2943 }
2944 mutex_unlock(&vcpu->mutex);
2945 }
2946 }
2947 spin_unlock(&kvm_lock);
2948}
2949
1b6c0168
AK		 2950static void hardware_enable(void *junk)
2951{
2952 int cpu = raw_smp_processor_id();
2953
2954 if (cpu_isset(cpu, cpus_hardware_enabled))
2955 return;
2956 cpu_set(cpu, cpus_hardware_enabled);
2957 kvm_arch_ops->hardware_enable(NULL);
2958}
2959
2960static void hardware_disable(void *junk)
2961{
2962 int cpu = raw_smp_processor_id();
2963
2964 if (!cpu_isset(cpu, cpus_hardware_enabled))
2965 return;
2966 cpu_clear(cpu, cpus_hardware_enabled);
2967 decache_vcpus_on_cpu(cpu);
2968 kvm_arch_ops->hardware_disable(NULL);
2969}
2970
774c47f1
AK		 2971static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val,
2972 void *v)
2973{
2974 int cpu = (long)v;
2975
2976 switch (val) {
cec9ad27
AK		 2977 case CPU_DYING:
2978 case CPU_DYING_FROZEN:
774c47f1 2979 case CPU_UP_CANCELED:
8bb78442 2980 case CPU_UP_CANCELED_FROZEN:
43934a38
JK		 2981 printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n",
2982 cpu);
1b6c0168 2983 smp_call_function_single(cpu, hardware_disable, NULL, 0, 1);
774c47f1 2984 break;
43934a38 2985 case CPU_ONLINE:
8bb78442 2986 case CPU_ONLINE_FROZEN:
43934a38
JK		 2987 printk(KERN_INFO "kvm: enabling virtualization on CPU%d\n",
2988 cpu);
1b6c0168 2989 smp_call_function_single(cpu, hardware_enable, NULL, 0, 1);
774c47f1
AK		 2990 break;
2991 }
2992 return NOTIFY_OK;
2993}
2994
2eeb2e94
GH		 2995void kvm_io_bus_init(struct kvm_io_bus *bus)
2996{
2997 memset(bus, 0, sizeof(*bus));
2998}
2999
3000void kvm_io_bus_destroy(struct kvm_io_bus *bus)
3001{
3002 int i;
3003
3004 for (i = 0; i < bus->dev_count; i++) {
3005 struct kvm_io_device *pos = bus->devs[i];
3006
3007 kvm_iodevice_destructor(pos);
3008 }
3009}
3010
3011struct kvm_io_device *kvm_io_bus_find_dev(struct kvm_io_bus *bus, gpa_t addr)
3012{
3013 int i;
3014
3015 for (i = 0; i < bus->dev_count; i++) {
3016 struct kvm_io_device *pos = bus->devs[i];
3017
3018 if (pos->in_range(pos, addr))
3019 return pos;
3020 }
3021
3022 return NULL;
3023}
3024
3025void kvm_io_bus_register_dev(struct kvm_io_bus *bus, struct kvm_io_device *dev)
3026{
3027 BUG_ON(bus->dev_count > (NR_IOBUS_DEVS-1));
3028
3029 bus->devs[bus->dev_count++] = dev;
3030}
3031
774c47f1
AK		 3032static struct notifier_block kvm_cpu_notifier = {
3033 .notifier_call = kvm_cpu_hotplug,
3034 .priority = 20, /* must be > scheduler priority */
3035};
3036
1165f5fe
AK		 3037static u64 stat_get(void *_offset)
3038{
3039 unsigned offset = (long)_offset;
3040 u64 total = 0;
3041 struct kvm *kvm;
3042 struct kvm_vcpu *vcpu;
3043 int i;
3044
3045 spin_lock(&kvm_lock);
3046 list_for_each_entry(kvm, &vm_list, vm_list)
3047 for (i = 0; i < KVM_MAX_VCPUS; ++i) {
3048 vcpu = &kvm->vcpus[i];
3049 total += *(u32 *)((void *)vcpu + offset);
3050 }
3051 spin_unlock(&kvm_lock);
3052 return total;
3053}
3054
3055static void stat_set(void *offset, u64 val)
3056{
3057}
3058
3059DEFINE_SIMPLE_ATTRIBUTE(stat_fops, stat_get, stat_set, "%llu\n");
3060
6aa8b732
AK		 3061static __init void kvm_init_debug(void)
3062{
3063 struct kvm_stats_debugfs_item *p;
3064
8b6d44c7 3065 debugfs_dir = debugfs_create_dir("kvm", NULL);
6aa8b732 3066 for (p = debugfs_entries; p->name; ++p)
1165f5fe
AK		 3067 p->dentry = debugfs_create_file(p->name, 0444, debugfs_dir,
3068 (void *)(long)p->offset,
3069 &stat_fops);
6aa8b732
AK		 3070}
3071
3072static void kvm_exit_debug(void)
3073{
3074 struct kvm_stats_debugfs_item *p;
3075
3076 for (p = debugfs_entries; p->name; ++p)
3077 debugfs_remove(p->dentry);
3078 debugfs_remove(debugfs_dir);
3079}
3080
59ae6c6b
AK		 3081static int kvm_suspend(struct sys_device *dev, pm_message_t state)
3082{
4267c41a 3083 hardware_disable(NULL);
59ae6c6b
AK		 3084 return 0;
3085}
3086
3087static int kvm_resume(struct sys_device *dev)
3088{
4267c41a 3089 hardware_enable(NULL);
59ae6c6b
AK		 3090 return 0;
3091}
3092
3093static struct sysdev_class kvm_sysdev_class = {
3094 set_kset_name("kvm"),
3095 .suspend = kvm_suspend,
3096 .resume = kvm_resume,
3097};
3098
3099static struct sys_device kvm_sysdev = {
3100 .id = 0,
3101 .cls = &kvm_sysdev_class,
3102};
3103
6aa8b732
AK		 3104hpa_t bad_page_address;
3105
3106int kvm_init_arch(struct kvm_arch_ops *ops, struct module *module)
3107{
3108 int r;
3109
09db28b8
YI		 3110 if (kvm_arch_ops) {
3111 printk(KERN_ERR "kvm: already loaded the other module\n");
3112 return -EEXIST;
3113 }
3114
e097f35c 3115 if (!ops->cpu_has_kvm_support()) {
6aa8b732
AK		 3116 printk(KERN_ERR "kvm: no hardware support\n");
3117 return -EOPNOTSUPP;
3118 }
e097f35c 3119 if (ops->disabled_by_bios()) {
6aa8b732
AK		 3120 printk(KERN_ERR "kvm: disabled by bios\n");
3121 return -EOPNOTSUPP;
3122 }
3123
e097f35c
YI		 3124 kvm_arch_ops = ops;
3125
6aa8b732
AK		 3126 r = kvm_arch_ops->hardware_setup();
3127 if (r < 0)
ca45aaae 3128 goto out;
6aa8b732 3129
1b6c0168 3130 on_each_cpu(hardware_enable, NULL, 0, 1);
774c47f1
AK		 3131 r = register_cpu_notifier(&kvm_cpu_notifier);
3132 if (r)
3133 goto out_free_1;
6aa8b732
AK		 3134 register_reboot_notifier(&kvm_reboot_notifier);
3135
59ae6c6b
AK		 3136 r = sysdev_class_register(&kvm_sysdev_class);
3137 if (r)
3138 goto out_free_2;
3139
3140 r = sysdev_register(&kvm_sysdev);
3141 if (r)
3142 goto out_free_3;
3143
6aa8b732
AK		 3144 kvm_chardev_ops.owner = module;
3145
3146 r = misc_register(&kvm_dev);
3147 if (r) {
3148 printk (KERN_ERR "kvm: misc device register failed\n");
3149 goto out_free;
3150 }
3151
3152 return r;
3153
3154out_free:
59ae6c6b
AK		 3155 sysdev_unregister(&kvm_sysdev);
3156out_free_3:
3157 sysdev_class_unregister(&kvm_sysdev_class);
3158out_free_2:
6aa8b732 3159 unregister_reboot_notifier(&kvm_reboot_notifier);
774c47f1
AK		 3160 unregister_cpu_notifier(&kvm_cpu_notifier);
3161out_free_1:
1b6c0168 3162 on_each_cpu(hardware_disable, NULL, 0, 1);
6aa8b732 3163 kvm_arch_ops->hardware_unsetup();
ca45aaae
AK		 3164out:
3165 kvm_arch_ops = NULL;
6aa8b732
AK		 3166 return r;
3167}
3168
3169void kvm_exit_arch(void)
3170{
3171 misc_deregister(&kvm_dev);
59ae6c6b
AK		 3172 sysdev_unregister(&kvm_sysdev);
3173 sysdev_class_unregister(&kvm_sysdev_class);
6aa8b732 3174 unregister_reboot_notifier(&kvm_reboot_notifier);
59ae6c6b 3175 unregister_cpu_notifier(&kvm_cpu_notifier);
1b6c0168 3176 on_each_cpu(hardware_disable, NULL, 0, 1);
6aa8b732 3177 kvm_arch_ops->hardware_unsetup();
09db28b8 3178 kvm_arch_ops = NULL;
6aa8b732
AK		 3179}
3180
3181static __init int kvm_init(void)
3182{
3183 static struct page *bad_page;
37e29d90
AK		 3184 int r;
3185
b5a33a75
AK		 3186 r = kvm_mmu_module_init();
3187 if (r)
3188 goto out4;
3189
6aa8b732
AK		 3190 kvm_init_debug();
3191
bf591b24
MR		 3192 kvm_init_msr_list();
3193
6aa8b732
AK		 3194 if ((bad_page = alloc_page(GFP_KERNEL)) == NULL) {
3195 r = -ENOMEM;
3196 goto out;
3197 }
3198
3199 bad_page_address = page_to_pfn(bad_page) << PAGE_SHIFT;
3200 memset(__va(bad_page_address), 0, PAGE_SIZE);
3201
58e690e6 3202 return 0;
6aa8b732
AK		 3203
3204out:
3205 kvm_exit_debug();
b5a33a75
AK		 3206 kvm_mmu_module_exit();
3207out4:
6aa8b732
AK		 3208 return r;
3209}
3210
3211static __exit void kvm_exit(void)
3212{
3213 kvm_exit_debug();
3214 __free_page(pfn_to_page(bad_page_address >> PAGE_SHIFT));
b5a33a75 3215 kvm_mmu_module_exit();
6aa8b732
AK		 3216}
3217
3218module_init(kvm_init)
3219module_exit(kvm_exit)
3220
3221EXPORT_SYMBOL_GPL(kvm_init_arch);
3222EXPORT_SYMBOL_GPL(kvm_exit_arch);
Clone of davem's net-next-2.6 tree