]> bbs.cooldavid.org Git - net-next-2.6.git/blame - arch/x86/kernel/cpu/mtrr/generic.c
include cleanup: Update gfp.h and slab.h includes to prepare for breaking implicit...
[net-next-2.6.git] / arch / x86 / kernel / cpu / mtrr / generic.c
CommitLineData
a1a499a3
JSR
1/*
2 * This only handles 32bit MTRR on 32bit hosts. This is strictly wrong
3 * because MTRRs can span upto 40 bits (36bits on most modern x86)
4 */
5#define DEBUG
6
7#include <linux/module.h>
1da177e4 8#include <linux/init.h>
a1a499a3 9#include <linux/io.h>
1da177e4 10#include <linux/mm.h>
a1a499a3 11
7ebad705 12#include <asm/processor-flags.h>
a1a499a3 13#include <asm/cpufeature.h>
1da177e4 14#include <asm/tlbflush.h>
a1a499a3
JSR
15#include <asm/system.h>
16#include <asm/mtrr.h>
17#include <asm/msr.h>
2e5d9c85 18#include <asm/pat.h>
a1a499a3 19
1da177e4
LT
20#include "mtrr.h"
21
de938c51 22struct fixed_range_block {
a1a499a3
JSR
23 int base_msr; /* start address of an MTRR block */
24 int ranges; /* number of MTRRs in this block */
de938c51
BK
25};
26
27static struct fixed_range_block fixed_range_blocks[] = {
a1a499a3
JSR
28 { MSR_MTRRfix64K_00000, 1 }, /* one 64k MTRR */
29 { MSR_MTRRfix16K_80000, 2 }, /* two 16k MTRRs */
30 { MSR_MTRRfix4K_C0000, 8 }, /* eight 4k MTRRs */
de938c51
BK
31 {}
32};
33
1da177e4 34static unsigned long smp_changes_mask;
2e5d9c85 35static int mtrr_state_set;
95ffa243 36u64 mtrr_tom2;
1da177e4 37
a1a499a3 38struct mtrr_state_type mtrr_state;
932d27a7
SY
39EXPORT_SYMBOL_GPL(mtrr_state);
40
a1a499a3 41/*
3ff42da5
AH
42 * BIOS is expected to clear MtrrFixDramModEn bit, see for example
43 * "BIOS and Kernel Developer's Guide for the AMD Athlon 64 and AMD
44 * Opteron Processors" (26094 Rev. 3.30 February 2006), section
45 * "13.2.1.2 SYSCFG Register": "The MtrrFixDramModEn bit should be set
46 * to 1 during BIOS initalization of the fixed MTRRs, then cleared to
47 * 0 for operation."
48 */
49static inline void k8_check_syscfg_dram_mod_en(void)
50{
51 u32 lo, hi;
52
53 if (!((boot_cpu_data.x86_vendor == X86_VENDOR_AMD) &&
54 (boot_cpu_data.x86 >= 0x0f)))
55 return;
56
57 rdmsr(MSR_K8_SYSCFG, lo, hi);
58 if (lo & K8_MTRRFIXRANGE_DRAM_MODIFY) {
59 printk(KERN_ERR FW_WARN "MTRR: CPU %u: SYSCFG[MtrrFixDramModEn]"
60 " not cleared by BIOS, clearing this bit\n",
61 smp_processor_id());
62 lo &= ~K8_MTRRFIXRANGE_DRAM_MODIFY;
63 mtrr_wrmsr(MSR_K8_SYSCFG, lo, hi);
64 }
65}
66
2e5d9c85 67/*
68 * Returns the effective MTRR type for the region
69 * Error returns:
70 * - 0xFE - when the range is "not entirely covered" by _any_ var range MTRR
71 * - 0xFF - when MTRR is not enabled
72 */
73u8 mtrr_type_lookup(u64 start, u64 end)
74{
75 int i;
76 u64 base, mask;
77 u8 prev_match, curr_match;
78
79 if (!mtrr_state_set)
80 return 0xFF;
81
82 if (!mtrr_state.enabled)
83 return 0xFF;
84
85 /* Make end inclusive end, instead of exclusive */
86 end--;
87
88 /* Look in fixed ranges. Just return the type as per start */
89 if (mtrr_state.have_fixed && (start < 0x100000)) {
90 int idx;
91
92 if (start < 0x80000) {
93 idx = 0;
94 idx += (start >> 16);
95 return mtrr_state.fixed_ranges[idx];
96 } else if (start < 0xC0000) {
97 idx = 1 * 8;
98 idx += ((start - 0x80000) >> 14);
99 return mtrr_state.fixed_ranges[idx];
100 } else if (start < 0x1000000) {
101 idx = 3 * 8;
102 idx += ((start - 0xC0000) >> 12);
103 return mtrr_state.fixed_ranges[idx];
104 }
105 }
106
107 /*
108 * Look in variable ranges
109 * Look of multiple ranges matching this address and pick type
110 * as per MTRR precedence
111 */
a1a499a3 112 if (!(mtrr_state.enabled & 2))
2e5d9c85 113 return mtrr_state.def_type;
2e5d9c85 114
115 prev_match = 0xFF;
116 for (i = 0; i < num_var_ranges; ++i) {
117 unsigned short start_state, end_state;
118
119 if (!(mtrr_state.var_ranges[i].mask_lo & (1 << 11)))
120 continue;
121
122 base = (((u64)mtrr_state.var_ranges[i].base_hi) << 32) +
123 (mtrr_state.var_ranges[i].base_lo & PAGE_MASK);
124 mask = (((u64)mtrr_state.var_ranges[i].mask_hi) << 32) +
125 (mtrr_state.var_ranges[i].mask_lo & PAGE_MASK);
126
127 start_state = ((start & mask) == (base & mask));
128 end_state = ((end & mask) == (base & mask));
129 if (start_state != end_state)
130 return 0xFE;
131
a1a499a3 132 if ((start & mask) != (base & mask))
2e5d9c85 133 continue;
2e5d9c85 134
135 curr_match = mtrr_state.var_ranges[i].base_lo & 0xff;
136 if (prev_match == 0xFF) {
137 prev_match = curr_match;
138 continue;
139 }
140
141 if (prev_match == MTRR_TYPE_UNCACHABLE ||
142 curr_match == MTRR_TYPE_UNCACHABLE) {
143 return MTRR_TYPE_UNCACHABLE;
144 }
145
146 if ((prev_match == MTRR_TYPE_WRBACK &&
147 curr_match == MTRR_TYPE_WRTHROUGH) ||
148 (prev_match == MTRR_TYPE_WRTHROUGH &&
149 curr_match == MTRR_TYPE_WRBACK)) {
150 prev_match = MTRR_TYPE_WRTHROUGH;
151 curr_match = MTRR_TYPE_WRTHROUGH;
152 }
153
a1a499a3 154 if (prev_match != curr_match)
2e5d9c85 155 return MTRR_TYPE_UNCACHABLE;
2e5d9c85 156 }
157
95ffa243
YL
158 if (mtrr_tom2) {
159 if (start >= (1ULL<<32) && (end < mtrr_tom2))
35605a10
YL
160 return MTRR_TYPE_WRBACK;
161 }
162
2e5d9c85 163 if (prev_match != 0xFF)
164 return prev_match;
165
166 return mtrr_state.def_type;
167}
168
a1a499a3 169/* Get the MSR pair relating to a var range */
bf8c4817 170static void
1da177e4
LT
171get_mtrr_var_range(unsigned int index, struct mtrr_var_range *vr)
172{
173 rdmsr(MTRRphysBase_MSR(index), vr->base_lo, vr->base_hi);
174 rdmsr(MTRRphysMask_MSR(index), vr->mask_lo, vr->mask_hi);
175}
176
a1a499a3 177/* Fill the MSR pair relating to a var range */
95ffa243
YL
178void fill_mtrr_var_range(unsigned int index,
179 u32 base_lo, u32 base_hi, u32 mask_lo, u32 mask_hi)
180{
181 struct mtrr_var_range *vr;
182
183 vr = mtrr_state.var_ranges;
184
185 vr[index].base_lo = base_lo;
186 vr[index].base_hi = base_hi;
187 vr[index].mask_lo = mask_lo;
188 vr[index].mask_hi = mask_hi;
189}
190
a1a499a3 191static void get_fixed_ranges(mtrr_type *frs)
1da177e4 192{
a1a499a3 193 unsigned int *p = (unsigned int *)frs;
1da177e4
LT
194 int i;
195
3ff42da5
AH
196 k8_check_syscfg_dram_mod_en();
197
a036c7a3 198 rdmsr(MSR_MTRRfix64K_00000, p[0], p[1]);
1da177e4
LT
199
200 for (i = 0; i < 2; i++)
7d9d55e4 201 rdmsr(MSR_MTRRfix16K_80000 + i, p[2 + i * 2], p[3 + i * 2]);
1da177e4 202 for (i = 0; i < 8; i++)
ba5673ff 203 rdmsr(MSR_MTRRfix4K_C0000 + i, p[6 + i * 2], p[7 + i * 2]);
1da177e4
LT
204}
205
2b3b4835
BK
206void mtrr_save_fixed_ranges(void *info)
207{
84288ad8
AM
208 if (cpu_has_mtrr)
209 get_fixed_ranges(mtrr_state.fixed_ranges);
2b3b4835
BK
210}
211
d4c90e37
YL
212static unsigned __initdata last_fixed_start;
213static unsigned __initdata last_fixed_end;
214static mtrr_type __initdata last_fixed_type;
215
216static void __init print_fixed_last(void)
217{
218 if (!last_fixed_end)
219 return;
220
a1a499a3
JSR
221 pr_debug(" %05X-%05X %s\n", last_fixed_start,
222 last_fixed_end - 1, mtrr_attrib_to_str(last_fixed_type));
d4c90e37
YL
223
224 last_fixed_end = 0;
225}
226
227static void __init update_fixed_last(unsigned base, unsigned end,
a1a499a3 228 mtrr_type type)
d4c90e37
YL
229{
230 last_fixed_start = base;
231 last_fixed_end = end;
232 last_fixed_type = type;
233}
234
a1a499a3
JSR
235static void __init
236print_fixed(unsigned base, unsigned step, const mtrr_type *types)
365bff80
JB
237{
238 unsigned i;
239
d4c90e37
YL
240 for (i = 0; i < 8; ++i, ++types, base += step) {
241 if (last_fixed_end == 0) {
242 update_fixed_last(base, base + step, *types);
243 continue;
244 }
245 if (last_fixed_end == base && last_fixed_type == *types) {
246 last_fixed_end = base + step;
247 continue;
248 }
249 /* new segments: gap or different type */
250 print_fixed_last();
251 update_fixed_last(base, base + step, *types);
252 }
365bff80
JB
253}
254
2e5d9c85 255static void prepare_set(void);
256static void post_set(void);
257
8ad97905
YL
258static void __init print_mtrr_state(void)
259{
260 unsigned int i;
261 int high_width;
262
a1a499a3
JSR
263 pr_debug("MTRR default type: %s\n",
264 mtrr_attrib_to_str(mtrr_state.def_type));
8ad97905 265 if (mtrr_state.have_fixed) {
a1a499a3
JSR
266 pr_debug("MTRR fixed ranges %sabled:\n",
267 mtrr_state.enabled & 1 ? "en" : "dis");
8ad97905
YL
268 print_fixed(0x00000, 0x10000, mtrr_state.fixed_ranges + 0);
269 for (i = 0; i < 2; ++i)
a1a499a3
JSR
270 print_fixed(0x80000 + i * 0x20000, 0x04000,
271 mtrr_state.fixed_ranges + (i + 1) * 8);
8ad97905 272 for (i = 0; i < 8; ++i)
a1a499a3
JSR
273 print_fixed(0xC0000 + i * 0x08000, 0x01000,
274 mtrr_state.fixed_ranges + (i + 3) * 8);
d4c90e37
YL
275
276 /* tail */
277 print_fixed_last();
8ad97905 278 }
a1a499a3
JSR
279 pr_debug("MTRR variable ranges %sabled:\n",
280 mtrr_state.enabled & 2 ? "en" : "dis");
917a0153
YL
281 if (size_or_mask & 0xffffffffUL)
282 high_width = ffs(size_or_mask & 0xffffffffUL) - 1;
283 else
284 high_width = ffs(size_or_mask>>32) + 32 - 1;
285 high_width = (high_width - (32 - PAGE_SHIFT) + 3) / 4;
a1a499a3 286
8ad97905
YL
287 for (i = 0; i < num_var_ranges; ++i) {
288 if (mtrr_state.var_ranges[i].mask_lo & (1 << 11))
a1a499a3
JSR
289 pr_debug(" %u base %0*X%05X000 mask %0*X%05X000 %s\n",
290 i,
291 high_width,
292 mtrr_state.var_ranges[i].base_hi,
293 mtrr_state.var_ranges[i].base_lo >> 12,
294 high_width,
295 mtrr_state.var_ranges[i].mask_hi,
296 mtrr_state.var_ranges[i].mask_lo >> 12,
297 mtrr_attrib_to_str(mtrr_state.var_ranges[i].base_lo & 0xff));
8ad97905 298 else
a1a499a3 299 pr_debug(" %u disabled\n", i);
8ad97905 300 }
a1a499a3
JSR
301 if (mtrr_tom2)
302 pr_debug("TOM2: %016llx aka %lldM\n", mtrr_tom2, mtrr_tom2>>20);
8ad97905
YL
303}
304
a1a499a3 305/* Grab all of the MTRR state for this CPU into *state */
9ef231a4 306void __init get_mtrr_state(void)
1da177e4 307{
1da177e4 308 struct mtrr_var_range *vrs;
2e5d9c85 309 unsigned long flags;
a1a499a3
JSR
310 unsigned lo, dummy;
311 unsigned int i;
1da177e4 312
1da177e4
LT
313 vrs = mtrr_state.var_ranges;
314
d9bcc01d 315 rdmsr(MSR_MTRRcap, lo, dummy);
365bff80
JB
316 mtrr_state.have_fixed = (lo >> 8) & 1;
317
1da177e4
LT
318 for (i = 0; i < num_var_ranges; i++)
319 get_mtrr_var_range(i, &vrs[i]);
365bff80
JB
320 if (mtrr_state.have_fixed)
321 get_fixed_ranges(mtrr_state.fixed_ranges);
1da177e4 322
52650257 323 rdmsr(MSR_MTRRdefType, lo, dummy);
1da177e4
LT
324 mtrr_state.def_type = (lo & 0xff);
325 mtrr_state.enabled = (lo & 0xc00) >> 10;
365bff80 326
35605a10 327 if (amd_special_default_mtrr()) {
0da72a4a 328 unsigned low, high;
a1a499a3 329
35605a10 330 /* TOP_MEM2 */
0da72a4a 331 rdmsr(MSR_K8_TOP_MEM2, low, high);
95ffa243
YL
332 mtrr_tom2 = high;
333 mtrr_tom2 <<= 32;
334 mtrr_tom2 |= low;
8004dd96 335 mtrr_tom2 &= 0xffffff800000ULL;
35605a10 336 }
8ad97905
YL
337
338 print_mtrr_state();
339
2e5d9c85 340 mtrr_state_set = 1;
341
342 /* PAT setup for BP. We need to go through sync steps here */
343 local_irq_save(flags);
344 prepare_set();
345
346 pat_init();
347
348 post_set();
349 local_irq_restore(flags);
1da177e4
LT
350}
351
a1a499a3 352/* Some BIOS's are messed up and don't set all MTRRs the same! */
1da177e4
LT
353void __init mtrr_state_warn(void)
354{
355 unsigned long mask = smp_changes_mask;
356
357 if (!mask)
358 return;
359 if (mask & MTRR_CHANGE_MASK_FIXED)
a1a499a3 360 pr_warning("mtrr: your CPUs had inconsistent fixed MTRR settings\n");
1da177e4 361 if (mask & MTRR_CHANGE_MASK_VARIABLE)
a1a499a3 362 pr_warning("mtrr: your CPUs had inconsistent variable MTRR settings\n");
1da177e4 363 if (mask & MTRR_CHANGE_MASK_DEFTYPE)
a1a499a3
JSR
364 pr_warning("mtrr: your CPUs had inconsistent MTRRdefType settings\n");
365
1da177e4
LT
366 printk(KERN_INFO "mtrr: probably your BIOS does not setup all CPUs.\n");
367 printk(KERN_INFO "mtrr: corrected configuration.\n");
368}
369
a1a499a3
JSR
370/*
371 * Doesn't attempt to pass an error out to MTRR users
372 * because it's quite complicated in some cases and probably not
373 * worth it because the best error handling is to ignore it.
374 */
1da177e4
LT
375void mtrr_wrmsr(unsigned msr, unsigned a, unsigned b)
376{
a1a499a3 377 if (wrmsr_safe(msr, a, b) < 0) {
1da177e4
LT
378 printk(KERN_ERR
379 "MTRR: CPU %u: Writing MSR %x to %x:%x failed\n",
380 smp_processor_id(), msr, a, b);
a1a499a3 381 }
1da177e4
LT
382}
383
de938c51 384/**
a1a499a3
JSR
385 * set_fixed_range - checks & updates a fixed-range MTRR if it
386 * differs from the value it should have
1d3381eb
RD
387 * @msr: MSR address of the MTTR which should be checked and updated
388 * @changed: pointer which indicates whether the MTRR needed to be changed
389 * @msrwords: pointer to the MSR values which the MSR should have
de938c51 390 */
2d2ee8de 391static void set_fixed_range(int msr, bool *changed, unsigned int *msrwords)
de938c51
BK
392{
393 unsigned lo, hi;
394
395 rdmsr(msr, lo, hi);
396
397 if (lo != msrwords[0] || hi != msrwords[1]) {
de938c51 398 mtrr_wrmsr(msr, msrwords[0], msrwords[1]);
2d2ee8de 399 *changed = true;
de938c51
BK
400 }
401}
402
1d3381eb
RD
403/**
404 * generic_get_free_region - Get a free MTRR.
405 * @base: The starting (base) address of the region.
406 * @size: The size (in bytes) of the region.
407 * @replace_reg: mtrr index to be replaced; set to invalid value if none.
408 *
409 * Returns: The index of the region on success, else negative on error.
410 */
a1a499a3
JSR
411int
412generic_get_free_region(unsigned long base, unsigned long size, int replace_reg)
1da177e4 413{
365bff80 414 unsigned long lbase, lsize;
a1a499a3
JSR
415 mtrr_type ltype;
416 int i, max;
1da177e4
LT
417
418 max = num_var_ranges;
365bff80
JB
419 if (replace_reg >= 0 && replace_reg < max)
420 return replace_reg;
a1a499a3 421
1da177e4
LT
422 for (i = 0; i < max; ++i) {
423 mtrr_if->get(i, &lbase, &lsize, &ltype);
424 if (lsize == 0)
425 return i;
426 }
a1a499a3 427
1da177e4
LT
428 return -ENOSPC;
429}
430
408b664a 431static void generic_get_mtrr(unsigned int reg, unsigned long *base,
365bff80 432 unsigned long *size, mtrr_type *type)
1da177e4
LT
433{
434 unsigned int mask_lo, mask_hi, base_lo, base_hi;
38cc1c3d 435 unsigned int tmp, hi;
63516ef6 436 int cpu;
1da177e4 437
8ad97905
YL
438 /*
439 * get_mtrr doesn't need to update mtrr_state, also it could be called
440 * from any cpu, so try to print it out directly.
441 */
63516ef6
YL
442 cpu = get_cpu();
443
1da177e4 444 rdmsr(MTRRphysMask_MSR(reg), mask_lo, mask_hi);
8ad97905 445
1da177e4 446 if ((mask_lo & 0x800) == 0) {
a1a499a3 447 /* Invalid (i.e. free) range */
1da177e4
LT
448 *base = 0;
449 *size = 0;
450 *type = 0;
63516ef6 451 goto out_put_cpu;
1da177e4
LT
452 }
453
454 rdmsr(MTRRphysBase_MSR(reg), base_lo, base_hi);
455
63516ef6 456 /* Work out the shifted address mask: */
38cc1c3d
YL
457 tmp = mask_hi << (32 - PAGE_SHIFT) | mask_lo >> PAGE_SHIFT;
458 mask_lo = size_or_mask | tmp;
63516ef6
YL
459
460 /* Expand tmp with high bits to all 1s: */
38cc1c3d
YL
461 hi = fls(tmp);
462 if (hi > 0) {
463 tmp |= ~((1<<(hi - 1)) - 1);
464
465 if (tmp != mask_lo) {
942fa3b6 466 printk(KERN_WARNING "mtrr: your BIOS has configured an incorrect mask, fixing it.\n");
38cc1c3d
YL
467 mask_lo = tmp;
468 }
469 }
1da177e4 470
63516ef6
YL
471 /*
472 * This works correctly if size is a power of two, i.e. a
473 * contiguous range:
474 */
1da177e4
LT
475 *size = -mask_lo;
476 *base = base_hi << (32 - PAGE_SHIFT) | base_lo >> PAGE_SHIFT;
477 *type = base_lo & 0xff;
8ad97905 478
63516ef6
YL
479out_put_cpu:
480 put_cpu();
1da177e4
LT
481}
482
de938c51 483/**
a1a499a3
JSR
484 * set_fixed_ranges - checks & updates the fixed-range MTRRs if they
485 * differ from the saved set
1d3381eb 486 * @frs: pointer to fixed-range MTRR values, saved by get_fixed_ranges()
de938c51 487 */
a1a499a3 488static int set_fixed_ranges(mtrr_type *frs)
1da177e4 489{
a1a499a3 490 unsigned long long *saved = (unsigned long long *)frs;
2d2ee8de 491 bool changed = false;
a1a499a3 492 int block = -1, range;
1da177e4 493
3ff42da5
AH
494 k8_check_syscfg_dram_mod_en();
495
a1a499a3
JSR
496 while (fixed_range_blocks[++block].ranges) {
497 for (range = 0; range < fixed_range_blocks[block].ranges; range++)
498 set_fixed_range(fixed_range_blocks[block].base_msr + range,
499 &changed, (unsigned int *)saved++);
500 }
1da177e4 501
1da177e4
LT
502 return changed;
503}
504
a1a499a3
JSR
505/*
506 * Set the MSR pair relating to a var range.
507 * Returns true if changes are made.
508 */
2d2ee8de 509static bool set_mtrr_var_ranges(unsigned int index, struct mtrr_var_range *vr)
1da177e4
LT
510{
511 unsigned int lo, hi;
2d2ee8de 512 bool changed = false;
1da177e4
LT
513
514 rdmsr(MTRRphysBase_MSR(index), lo, hi);
515 if ((vr->base_lo & 0xfffff0ffUL) != (lo & 0xfffff0ffUL)
cf94b62f
SS
516 || (vr->base_hi & (size_and_mask >> (32 - PAGE_SHIFT))) !=
517 (hi & (size_and_mask >> (32 - PAGE_SHIFT)))) {
a1a499a3 518
1da177e4 519 mtrr_wrmsr(MTRRphysBase_MSR(index), vr->base_lo, vr->base_hi);
2d2ee8de 520 changed = true;
1da177e4
LT
521 }
522
523 rdmsr(MTRRphysMask_MSR(index), lo, hi);
524
525 if ((vr->mask_lo & 0xfffff800UL) != (lo & 0xfffff800UL)
cf94b62f
SS
526 || (vr->mask_hi & (size_and_mask >> (32 - PAGE_SHIFT))) !=
527 (hi & (size_and_mask >> (32 - PAGE_SHIFT)))) {
1da177e4 528 mtrr_wrmsr(MTRRphysMask_MSR(index), vr->mask_lo, vr->mask_hi);
2d2ee8de 529 changed = true;
1da177e4
LT
530 }
531 return changed;
532}
533
365bff80
JB
534static u32 deftype_lo, deftype_hi;
535
1d3381eb
RD
536/**
537 * set_mtrr_state - Set the MTRR state for this CPU.
538 *
539 * NOTE: The CPU must already be in a safe state for MTRR changes.
540 * RETURNS: 0 if no changes made, else a mask indicating what was changed.
541 */
365bff80 542static unsigned long set_mtrr_state(void)
1da177e4 543{
1da177e4 544 unsigned long change_mask = 0;
a1a499a3 545 unsigned int i;
1da177e4 546
a1a499a3 547 for (i = 0; i < num_var_ranges; i++) {
1da177e4
LT
548 if (set_mtrr_var_ranges(i, &mtrr_state.var_ranges[i]))
549 change_mask |= MTRR_CHANGE_MASK_VARIABLE;
a1a499a3 550 }
1da177e4 551
365bff80 552 if (mtrr_state.have_fixed && set_fixed_ranges(mtrr_state.fixed_ranges))
1da177e4
LT
553 change_mask |= MTRR_CHANGE_MASK_FIXED;
554
a1a499a3
JSR
555 /*
556 * Set_mtrr_restore restores the old value of MTRRdefType,
557 * so to set it we fiddle with the saved value:
558 */
1da177e4
LT
559 if ((deftype_lo & 0xff) != mtrr_state.def_type
560 || ((deftype_lo & 0xc00) >> 10) != mtrr_state.enabled) {
a1a499a3
JSR
561
562 deftype_lo = (deftype_lo & ~0xcff) | mtrr_state.def_type |
563 (mtrr_state.enabled << 10);
1da177e4
LT
564 change_mask |= MTRR_CHANGE_MASK_DEFTYPE;
565 }
566
567 return change_mask;
568}
569
570
a1a499a3 571static unsigned long cr4;
40d6753e 572static DEFINE_RAW_SPINLOCK(set_atomicity_lock);
1da177e4
LT
573
574/*
a1a499a3
JSR
575 * Since we are disabling the cache don't allow any interrupts,
576 * they would run extremely slow and would only increase the pain.
577 *
578 * The caller must ensure that local interrupts are disabled and
579 * are reenabled after post_set() has been called.
1da177e4 580 */
182daa55 581static void prepare_set(void) __acquires(set_atomicity_lock)
1da177e4
LT
582{
583 unsigned long cr0;
584
a1a499a3
JSR
585 /*
586 * Note that this is not ideal
587 * since the cache is only flushed/disabled for this CPU while the
588 * MTRRs are changed, but changing this requires more invasive
589 * changes to the way the kernel boots
590 */
1da177e4 591
40d6753e 592 raw_spin_lock(&set_atomicity_lock);
1da177e4 593
a1a499a3 594 /* Enter the no-fill (CD=1, NW=0) cache mode and flush caches. */
7ebad705 595 cr0 = read_cr0() | X86_CR0_CD;
1da177e4
LT
596 write_cr0(cr0);
597 wbinvd();
598
a1a499a3
JSR
599 /* Save value of CR4 and clear Page Global Enable (bit 7) */
600 if (cpu_has_pge) {
1da177e4
LT
601 cr4 = read_cr4();
602 write_cr4(cr4 & ~X86_CR4_PGE);
603 }
604
605 /* Flush all TLBs via a mov %cr3, %reg; mov %reg, %cr3 */
606 __flush_tlb();
607
a1a499a3 608 /* Save MTRR state */
52650257 609 rdmsr(MSR_MTRRdefType, deftype_lo, deftype_hi);
1da177e4 610
a1a499a3 611 /* Disable MTRRs, and set the default type to uncached */
52650257 612 mtrr_wrmsr(MSR_MTRRdefType, deftype_lo & ~0xcff, deftype_hi);
1da177e4
LT
613}
614
182daa55 615static void post_set(void) __releases(set_atomicity_lock)
1da177e4 616{
a1a499a3 617 /* Flush TLBs (no need to flush caches - they are disabled) */
1da177e4
LT
618 __flush_tlb();
619
620 /* Intel (P6) standard MTRRs */
52650257 621 mtrr_wrmsr(MSR_MTRRdefType, deftype_lo, deftype_hi);
a1a499a3
JSR
622
623 /* Enable caches */
1da177e4
LT
624 write_cr0(read_cr0() & 0xbfffffff);
625
a1a499a3
JSR
626 /* Restore value of CR4 */
627 if (cpu_has_pge)
1da177e4 628 write_cr4(cr4);
40d6753e 629 raw_spin_unlock(&set_atomicity_lock);
1da177e4
LT
630}
631
632static void generic_set_all(void)
633{
634 unsigned long mask, count;
635 unsigned long flags;
636
637 local_irq_save(flags);
638 prepare_set();
639
640 /* Actually set the state */
365bff80 641 mask = set_mtrr_state();
1da177e4 642
2e5d9c85 643 /* also set PAT */
644 pat_init();
645
1da177e4
LT
646 post_set();
647 local_irq_restore(flags);
648
a1a499a3 649 /* Use the atomic bitops to update the global mask */
1da177e4
LT
650 for (count = 0; count < sizeof mask * 8; ++count) {
651 if (mask & 0x01)
652 set_bit(count, &smp_changes_mask);
653 mask >>= 1;
654 }
a1a499a3 655
1da177e4
LT
656}
657
a1a499a3
JSR
658/**
659 * generic_set_mtrr - set variable MTRR register on the local CPU.
660 *
661 * @reg: The register to set.
662 * @base: The base address of the region.
663 * @size: The size of the region. If this is 0 the region is disabled.
664 * @type: The type of the region.
665 *
666 * Returns nothing.
667 */
1da177e4
LT
668static void generic_set_mtrr(unsigned int reg, unsigned long base,
669 unsigned long size, mtrr_type type)
1da177e4
LT
670{
671 unsigned long flags;
3b520b23
SL
672 struct mtrr_var_range *vr;
673
674 vr = &mtrr_state.var_ranges[reg];
1da177e4
LT
675
676 local_irq_save(flags);
677 prepare_set();
678
679 if (size == 0) {
a1a499a3
JSR
680 /*
681 * The invalid bit is kept in the mask, so we simply
682 * clear the relevant mask register to disable a range.
683 */
1da177e4 684 mtrr_wrmsr(MTRRphysMask_MSR(reg), 0, 0);
3b520b23 685 memset(vr, 0, sizeof(struct mtrr_var_range));
1da177e4 686 } else {
3b520b23
SL
687 vr->base_lo = base << PAGE_SHIFT | type;
688 vr->base_hi = (base & size_and_mask) >> (32 - PAGE_SHIFT);
689 vr->mask_lo = -size << PAGE_SHIFT | 0x800;
690 vr->mask_hi = (-size & size_and_mask) >> (32 - PAGE_SHIFT);
691
692 mtrr_wrmsr(MTRRphysBase_MSR(reg), vr->base_lo, vr->base_hi);
693 mtrr_wrmsr(MTRRphysMask_MSR(reg), vr->mask_lo, vr->mask_hi);
1da177e4
LT
694 }
695
696 post_set();
697 local_irq_restore(flags);
698}
699
a1a499a3
JSR
700int generic_validate_add_page(unsigned long base, unsigned long size,
701 unsigned int type)
1da177e4
LT
702{
703 unsigned long lbase, last;
704
a1a499a3
JSR
705 /*
706 * For Intel PPro stepping <= 7
707 * must be 4 MiB aligned and not touch 0x70000000 -> 0x7003FFFF
708 */
1da177e4
LT
709 if (is_cpu(INTEL) && boot_cpu_data.x86 == 6 &&
710 boot_cpu_data.x86_model == 1 &&
711 boot_cpu_data.x86_mask <= 7) {
712 if (base & ((1 << (22 - PAGE_SHIFT)) - 1)) {
a1a499a3 713 pr_warning("mtrr: base(0x%lx000) is not 4 MiB aligned\n", base);
1da177e4
LT
714 return -EINVAL;
715 }
9b483417 716 if (!(base + size < 0x70000 || base > 0x7003F) &&
1da177e4
LT
717 (type == MTRR_TYPE_WRCOMB
718 || type == MTRR_TYPE_WRBACK)) {
a1a499a3 719 pr_warning("mtrr: writable mtrr between 0x70000000 and 0x7003FFFF may hang the CPU.\n");
1da177e4
LT
720 return -EINVAL;
721 }
722 }
723
a1a499a3
JSR
724 /*
725 * Check upper bits of base and last are equal and lower bits are 0
726 * for base and 1 for last
727 */
1da177e4
LT
728 last = base + size - 1;
729 for (lbase = base; !(lbase & 1) && (last & 1);
a1a499a3
JSR
730 lbase = lbase >> 1, last = last >> 1)
731 ;
1da177e4 732 if (lbase != last) {
a1a499a3 733 pr_warning("mtrr: base(0x%lx000) is not aligned on a size(0x%lx000) boundary\n", base, size);
1da177e4
LT
734 return -EINVAL;
735 }
736 return 0;
737}
738
1da177e4
LT
739static int generic_have_wrcomb(void)
740{
741 unsigned long config, dummy;
d9bcc01d 742 rdmsr(MSR_MTRRcap, config, dummy);
a1a499a3 743 return config & (1 << 10);
1da177e4
LT
744}
745
746int positive_have_wrcomb(void)
747{
748 return 1;
749}
750
a1a499a3
JSR
751/*
752 * Generic structure...
1da177e4 753 */
3b9cfc0a 754const struct mtrr_ops generic_mtrr_ops = {
a1a499a3
JSR
755 .use_intel_if = 1,
756 .set_all = generic_set_all,
757 .get = generic_get_mtrr,
758 .get_free_region = generic_get_free_region,
759 .set = generic_set_mtrr,
760 .validate_add_page = generic_validate_add_page,
761 .have_wrcomb = generic_have_wrcomb,
1da177e4 762};