]> bbs.cooldavid.org Git - net-next-2.6.git/blame - arch/x86/mm/init_64.c
git://bbs.cooldavid.org / net-next-2.6.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
x86: add stub init_gbpages() for 32-bit init_memory_mapping()
[net-next-2.6.git] / arch / x86 / mm / init_64.c
CommitLineData
1da177e4
LT		 1/*
2 * linux/arch/x86_64/mm/init.c
3 *
4 * Copyright (C) 1995 Linus Torvalds
5 * Copyright (C) 2000 Pavel Machek <pavel@suse.cz>
6 * Copyright (C) 2002,2003 Andi Kleen <ak@suse.de>
7 */
8
1da177e4
LT		 9#include <linux/signal.h>
10#include <linux/sched.h>
11#include <linux/kernel.h>
12#include <linux/errno.h>
13#include <linux/string.h>
14#include <linux/types.h>
15#include <linux/ptrace.h>
16#include <linux/mman.h>
17#include <linux/mm.h>
18#include <linux/swap.h>
19#include <linux/smp.h>
20#include <linux/init.h>
11034d55 21#include <linux/initrd.h>
1da177e4
LT		 22#include <linux/pagemap.h>
23#include <linux/bootmem.h>
24#include <linux/proc_fs.h>
59170891 25#include <linux/pci.h>
6fb14755 26#include <linux/pfn.h>
c9cf5528 27#include <linux/poison.h>
17a941d8 28#include <linux/dma-mapping.h>
44df75e6
MT		 29#include <linux/module.h>
30#include <linux/memory_hotplug.h>
ae32b129 31#include <linux/nmi.h>
1da177e4
LT		 32
33#include <asm/processor.h>
46eaa670 34#include <asm/bios_ebda.h>
1da177e4
LT		 35#include <asm/system.h>
36#include <asm/uaccess.h>
37#include <asm/pgtable.h>
38#include <asm/pgalloc.h>
39#include <asm/dma.h>
40#include <asm/fixmap.h>
41#include <asm/e820.h>
42#include <asm/apic.h>
43#include <asm/tlb.h>
44#include <asm/mmu_context.h>
45#include <asm/proto.h>
46#include <asm/smp.h>
2bc0414e 47#include <asm/sections.h>
718fc13b 48#include <asm/kdebug.h>
aaa64e04 49#include <asm/numa.h>
7bfeab9a 50#include <asm/cacheflush.h>
1da177e4 51
064d25f1
YL		 52/*
53 * end_pfn only includes RAM, while max_pfn_mapped includes all e820 entries.
54 * The direct mapping extends to max_pfn_mapped, so that we can directly access
55 * apertures, ACPI and other tables without having to play with fixmaps.
56 */
f361a450 57unsigned long max_low_pfn_mapped;
064d25f1
YL		 58unsigned long max_pfn_mapped;
59
e18c6874
AK		 60static unsigned long dma_reserve __initdata;
61
1da177e4
LT		 62DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);
63
a06de630 64int direct_gbpages
00d1c5e0
IM		 65#ifdef CONFIG_DIRECT_GBPAGES
66 = 1
67#endif
68;
69
70static int __init parse_direct_gbpages_off(char *arg)
71{
72 direct_gbpages = 0;
73 return 0;
74}
75early_param("nogbpages", parse_direct_gbpages_off);
76
77static int __init parse_direct_gbpages_on(char *arg)
78{
79 direct_gbpages = 1;
80 return 0;
81}
82early_param("gbpages", parse_direct_gbpages_on);
83
1da177e4
LT		 84/*
85 * NOTE: pagetable_init alloc all the fixmap pagetables contiguous on the
86 * physical space so we can cache the place of the first one and move
87 * around without checking the pgd every time.
88 */
89
1da177e4
LT		 90int after_bootmem;
91
be43d728 92pteval_t __supported_pte_mask __read_mostly = ~_PAGE_IOMAP;
bd220a24
YL		 93EXPORT_SYMBOL_GPL(__supported_pte_mask);
94
95static int do_not_nx __cpuinitdata;
96
deed05b7
IM		 97/*
98 * noexec=on|off
99 * Control non-executable mappings for 64-bit processes.
100 *
101 * on Enable (default)
102 * off Disable
103 */
bd220a24
YL		 104static int __init nonx_setup(char *str)
105{
106 if (!str)
107 return -EINVAL;
108 if (!strncmp(str, "on", 2)) {
109 __supported_pte_mask |= _PAGE_NX;
110 do_not_nx = 0;
111 } else if (!strncmp(str, "off", 3)) {
112 do_not_nx = 1;
113 __supported_pte_mask &= ~_PAGE_NX;
114 }
115 return 0;
116}
117early_param("noexec", nonx_setup);
118
119void __cpuinit check_efer(void)
120{
121 unsigned long efer;
122
123 rdmsrl(MSR_EFER, efer);
124 if (!(efer & EFER_NX) || do_not_nx)
125 __supported_pte_mask &= ~_PAGE_NX;
126}
127
128int force_personality32;
129
deed05b7
IM		 130/*
131 * noexec32=on|off
132 * Control non executable heap for 32bit processes.
133 * To control the stack too use noexec=off
134 *
135 * on PROT_READ does not imply PROT_EXEC for 32-bit processes (default)
136 * off PROT_READ implies PROT_EXEC
137 */
bd220a24
YL		 138static int __init nonx32_setup(char *str)
139{
140 if (!strcmp(str, "on"))
141 force_personality32 &= ~READ_IMPLIES_EXEC;
142 else if (!strcmp(str, "off"))
143 force_personality32 |= READ_IMPLIES_EXEC;
144 return 1;
145}
146__setup("noexec32=", nonx32_setup);
147
8d6ea967
MS		 148/*
149 * NOTE: This function is marked __ref because it calls __init function
150 * (alloc_bootmem_pages). It's safe to do it ONLY when after_bootmem == 0.
151 */
152static __ref void *spp_getpage(void)
14a62c34 153{
1da177e4 154 void *ptr;
14a62c34 155
1da177e4 156 if (after_bootmem)
14a62c34 157 ptr = (void *) get_zeroed_page(GFP_ATOMIC);
1da177e4
LT		 158 else
159 ptr = alloc_bootmem_pages(PAGE_SIZE);
14a62c34
TG		 160
161 if (!ptr || ((unsigned long)ptr & ~PAGE_MASK)) {
162 panic("set_pte_phys: cannot allocate page data %s\n",
163 after_bootmem ? "after bootmem" : "");
164 }
1da177e4 165
10f22dde 166 pr_debug("spp_getpage %p\n", ptr);
14a62c34 167
1da177e4 168 return ptr;
14a62c34 169}
1da177e4 170
d494a961 171void
0814e0ba 172set_pte_vaddr_pud(pud_t *pud_page, unsigned long vaddr, pte_t new_pte)
1da177e4 173{
1da177e4
LT		 174 pud_t *pud;
175 pmd_t *pmd;
d494a961 176 pte_t *pte;
1da177e4 177
0814e0ba 178 pud = pud_page + pud_index(vaddr);
1da177e4 179 if (pud_none(*pud)) {
14a62c34 180 pmd = (pmd_t *) spp_getpage();
bb23e403 181 pud_populate(&init_mm, pud, pmd);
1da177e4 182 if (pmd != pmd_offset(pud, 0)) {
10f22dde 183 printk(KERN_ERR "PAGETABLE BUG #01! %p <-> %p\n",
14a62c34 184 pmd, pmd_offset(pud, 0));
1da177e4
LT		 185 return;
186 }
187 }
188 pmd = pmd_offset(pud, vaddr);
189 if (pmd_none(*pmd)) {
190 pte = (pte_t *) spp_getpage();
bb23e403 191 pmd_populate_kernel(&init_mm, pmd, pte);
1da177e4 192 if (pte != pte_offset_kernel(pmd, 0)) {
10f22dde 193 printk(KERN_ERR "PAGETABLE BUG #02!\n");
1da177e4
LT		 194 return;
195 }
196 }
1da177e4
LT		 197
198 pte = pte_offset_kernel(pmd, vaddr);
1da177e4
LT		 199 set_pte(pte, new_pte);
200
201 /*
202 * It's enough to flush this one mapping.
203 * (PGE mappings get flushed as well)
204 */
205 __flush_tlb_one(vaddr);
206}
207
0814e0ba
EH		 208void
209set_pte_vaddr(unsigned long vaddr, pte_t pteval)
210{
211 pgd_t *pgd;
212 pud_t *pud_page;
213
214 pr_debug("set_pte_vaddr %lx to %lx\n", vaddr, native_pte_val(pteval));
215
216 pgd = pgd_offset_k(vaddr);
217 if (pgd_none(*pgd)) {
218 printk(KERN_ERR
219 "PGD FIXMAP MISSING, it should be setup in head.S!\n");
220 return;
221 }
222 pud_page = (pud_t*)pgd_page_vaddr(*pgd);
223 set_pte_vaddr_pud(pud_page, vaddr, pteval);
224}
225
3a9e189d
JS		 226/*
227 * Create large page table mappings for a range of physical addresses.
228 */
229static void __init __init_extra_mapping(unsigned long phys, unsigned long size,
230 pgprot_t prot)
231{
232 pgd_t *pgd;
233 pud_t *pud;
234 pmd_t *pmd;
235
236 BUG_ON((phys & ~PMD_MASK) || (size & ~PMD_MASK));
237 for (; size; phys += PMD_SIZE, size -= PMD_SIZE) {
238 pgd = pgd_offset_k((unsigned long)__va(phys));
239 if (pgd_none(*pgd)) {
240 pud = (pud_t *) spp_getpage();
241 set_pgd(pgd, __pgd(__pa(pud) | _KERNPG_TABLE |
242 _PAGE_USER));
243 }
244 pud = pud_offset(pgd, (unsigned long)__va(phys));
245 if (pud_none(*pud)) {
246 pmd = (pmd_t *) spp_getpage();
247 set_pud(pud, __pud(__pa(pmd) | _KERNPG_TABLE |
248 _PAGE_USER));
249 }
250 pmd = pmd_offset(pud, phys);
251 BUG_ON(!pmd_none(*pmd));
252 set_pmd(pmd, __pmd(phys | pgprot_val(prot)));
253 }
254}
255
256void __init init_extra_mapping_wb(unsigned long phys, unsigned long size)
257{
258 __init_extra_mapping(phys, size, PAGE_KERNEL_LARGE);
259}
260
261void __init init_extra_mapping_uc(unsigned long phys, unsigned long size)
262{
263 __init_extra_mapping(phys, size, PAGE_KERNEL_LARGE_NOCACHE);
264}
265
31eedd82 266/*
88f3aec7
IM		 267 * The head.S code sets up the kernel high mapping:
268 *
269 * from __START_KERNEL_map to __START_KERNEL_map + size (== _end-_text)
31eedd82
TG		 270 *
271 * phys_addr holds the negative offset to the kernel, which is added
272 * to the compile time generated pmds. This results in invalid pmds up
273 * to the point where we hit the physaddr 0 mapping.
274 *
275 * We limit the mappings to the region from _text to _end. _end is
276 * rounded up to the 2MB boundary. This catches the invalid pmds as
277 * well, as they are located before _text:
278 */
279void __init cleanup_highmap(void)
280{
281 unsigned long vaddr = __START_KERNEL_map;
d86bb0da 282 unsigned long end = roundup((unsigned long)_end, PMD_SIZE) - 1;
31eedd82
TG		 283 pmd_t *pmd = level2_kernel_pgt;
284 pmd_t *last_pmd = pmd + PTRS_PER_PMD;
285
286 for (; pmd < last_pmd; pmd++, vaddr += PMD_SIZE) {
2884f110 287 if (pmd_none(*pmd))
31eedd82
TG		 288 continue;
289 if (vaddr < (unsigned long) _text || vaddr > end)
290 set_pmd(pmd, __pmd(0));
291 }
292}
293
75175278
AK		 294static unsigned long __initdata table_start;
295static unsigned long __meminitdata table_end;
d86623a0 296static unsigned long __meminitdata table_top;
1da177e4 297
9482ac6e 298static __ref void *alloc_low_page(unsigned long *phys)
14a62c34 299{
dafe41ee 300 unsigned long pfn = table_end++;
1da177e4
LT		 301 void *adr;
302
44df75e6
MT		 303 if (after_bootmem) {
304 adr = (void *)get_zeroed_page(GFP_ATOMIC);
305 *phys = __pa(adr);
14a62c34 306
44df75e6
MT		 307 return adr;
308 }
309
d86623a0 310 if (pfn >= table_top)
14a62c34 311 panic("alloc_low_page: ran out of memory");
dafe41ee 312
14941779 313 adr = early_memremap(pfn * PAGE_SIZE, PAGE_SIZE);
44df75e6 314 memset(adr, 0, PAGE_SIZE);
dafe41ee
VG		 315 *phys = pfn * PAGE_SIZE;
316 return adr;
317}
1da177e4 318
9482ac6e 319static __ref void unmap_low_page(void *adr)
14a62c34 320{
44df75e6
MT		 321 if (after_bootmem)
322 return;
323
dafe41ee 324 early_iounmap(adr, PAGE_SIZE);
14a62c34 325}
1da177e4 326
7b16eb89 327static unsigned long __meminit
b27a43c1
SS		 328phys_pte_init(pte_t *pte_page, unsigned long addr, unsigned long end,
329 pgprot_t prot)
4f9c11dd
JF		 330{
331 unsigned pages = 0;
7b16eb89 332 unsigned long last_map_addr = end;
4f9c11dd 333 int i;
7b16eb89 334
4f9c11dd
JF		 335 pte_t *pte = pte_page + pte_index(addr);
336
337 for(i = pte_index(addr); i < PTRS_PER_PTE; i++, addr += PAGE_SIZE, pte++) {
338
339 if (addr >= end) {
340 if (!after_bootmem) {
341 for(; i < PTRS_PER_PTE; i++, pte++)
342 set_pte(pte, __pte(0));
343 }
344 break;
345 }
346
b27a43c1
SS		 347 /*
348 * We will re-use the existing mapping.
349 * Xen for example has some special requirements, like mapping
350 * pagetable pages as RO. So assume someone who pre-setup
351 * these mappings are more intelligent.
352 */
3afa3949
YL		 353 if (pte_val(*pte)) {
354 pages++;
4f9c11dd 355 continue;
3afa3949 356 }
4f9c11dd
JF		 357
358 if (0)
359 printk(" pte=%p addr=%lx pte=%016lx\n",
360 pte, addr, pfn_pte(addr >> PAGE_SHIFT, PAGE_KERNEL).pte);
4f9c11dd 361 pages++;
b27a43c1 362 set_pte(pte, pfn_pte(addr >> PAGE_SHIFT, prot));
7b16eb89 363 last_map_addr = (addr & PAGE_MASK) + PAGE_SIZE;
4f9c11dd 364 }
a2699e47 365
4f9c11dd 366 update_page_count(PG_LEVEL_4K, pages);
7b16eb89
YL		 367
368 return last_map_addr;
4f9c11dd
JF		 369}
370
7b16eb89 371static unsigned long __meminit
b27a43c1
SS		 372phys_pte_update(pmd_t *pmd, unsigned long address, unsigned long end,
373 pgprot_t prot)
4f9c11dd
JF		 374{
375 pte_t *pte = (pte_t *)pmd_page_vaddr(*pmd);
376
b27a43c1 377 return phys_pte_init(pte, address, end, prot);
4f9c11dd
JF		 378}
379
cc615032 380static unsigned long __meminit
b50efd2a 381phys_pmd_init(pmd_t *pmd_page, unsigned long address, unsigned long end,
b27a43c1 382 unsigned long page_size_mask, pgprot_t prot)
44df75e6 383{
ce0c0e50 384 unsigned long pages = 0;
7b16eb89 385 unsigned long last_map_addr = end;
ce0c0e50 386
6ad91658 387 int i = pmd_index(address);
44df75e6 388
6ad91658 389 for (; i < PTRS_PER_PMD; i++, address += PMD_SIZE) {
4f9c11dd 390 unsigned long pte_phys;
6ad91658 391 pmd_t *pmd = pmd_page + pmd_index(address);
4f9c11dd 392 pte_t *pte;
b27a43c1 393 pgprot_t new_prot = prot;
44df75e6 394
5f51e139 395 if (address >= end) {
14a62c34 396 if (!after_bootmem) {
5f51e139
JB		 397 for (; i < PTRS_PER_PMD; i++, pmd++)
398 set_pmd(pmd, __pmd(0));
14a62c34 399 }
44df75e6
MT		 400 break;
401 }
6ad91658 402
4f9c11dd 403 if (pmd_val(*pmd)) {
8ae3a5a8
JB		 404 if (!pmd_large(*pmd)) {
405 spin_lock(&init_mm.page_table_lock);
7b16eb89 406 last_map_addr = phys_pte_update(pmd, address,
b27a43c1 407 end, prot);
8ae3a5a8 408 spin_unlock(&init_mm.page_table_lock);
a2699e47 409 continue;
8ae3a5a8 410 }
b27a43c1
SS		 411 /*
412 * If we are ok with PG_LEVEL_2M mapping, then we will
413 * use the existing mapping,
414 *
415 * Otherwise, we will split the large page mapping but
416 * use the same existing protection bits except for
417 * large page, so that we don't violate Intel's TLB
418 * Application note (317080) which says, while changing
419 * the page sizes, new and old translations should
420 * not differ with respect to page frame and
421 * attributes.
422 */
3afa3949
YL		 423 if (page_size_mask & (1 << PG_LEVEL_2M)) {
424 pages++;
b27a43c1 425 continue;
3afa3949 426 }
b27a43c1 427 new_prot = pte_pgprot(pte_clrhuge(*(pte_t *)pmd));
4f9c11dd
JF		 428 }
429
b50efd2a 430 if (page_size_mask & (1<<PG_LEVEL_2M)) {
4f9c11dd 431 pages++;
8ae3a5a8 432 spin_lock(&init_mm.page_table_lock);
4f9c11dd 433 set_pte((pte_t *)pmd,
b27a43c1
SS		 434 pfn_pte(address >> PAGE_SHIFT,
435 __pgprot(pgprot_val(prot) | _PAGE_PSE)));
8ae3a5a8 436 spin_unlock(&init_mm.page_table_lock);
7b16eb89 437 last_map_addr = (address & PMD_MASK) + PMD_SIZE;
6ad91658 438 continue;
4f9c11dd 439 }
6ad91658 440
4f9c11dd 441 pte = alloc_low_page(&pte_phys);
b27a43c1 442 last_map_addr = phys_pte_init(pte, address, end, new_prot);
4f9c11dd
JF		 443 unmap_low_page(pte);
444
8ae3a5a8 445 spin_lock(&init_mm.page_table_lock);
4f9c11dd 446 pmd_populate_kernel(&init_mm, pmd, __va(pte_phys));
8ae3a5a8 447 spin_unlock(&init_mm.page_table_lock);
44df75e6 448 }
ce0c0e50 449 update_page_count(PG_LEVEL_2M, pages);
7b16eb89 450 return last_map_addr;
44df75e6
MT		 451}
452
cc615032 453static unsigned long __meminit
b50efd2a 454phys_pmd_update(pud_t *pud, unsigned long address, unsigned long end,
b27a43c1 455 unsigned long page_size_mask, pgprot_t prot)
44df75e6 456{
14a62c34 457 pmd_t *pmd = pmd_offset(pud, 0);
cc615032
AK		 458 unsigned long last_map_addr;
459
b27a43c1 460 last_map_addr = phys_pmd_init(pmd, address, end, page_size_mask, prot);
6ad91658 461 __flush_tlb_all();
cc615032 462 return last_map_addr;
44df75e6
MT		 463}
464
cc615032 465static unsigned long __meminit
b50efd2a
YL		 466phys_pud_init(pud_t *pud_page, unsigned long addr, unsigned long end,
467 unsigned long page_size_mask)
14a62c34 468{
ce0c0e50 469 unsigned long pages = 0;
cc615032 470 unsigned long last_map_addr = end;
6ad91658 471 int i = pud_index(addr);
44df75e6 472
14a62c34 473 for (; i < PTRS_PER_PUD; i++, addr = (addr & PUD_MASK) + PUD_SIZE) {
6ad91658
KM		 474 unsigned long pmd_phys;
475 pud_t *pud = pud_page + pud_index(addr);
1da177e4 476 pmd_t *pmd;
b27a43c1 477 pgprot_t prot = PAGE_KERNEL;
1da177e4 478
6ad91658 479 if (addr >= end)
1da177e4 480 break;
1da177e4 481
14a62c34
TG		 482 if (!after_bootmem &&
483 !e820_any_mapped(addr, addr+PUD_SIZE, 0)) {
484 set_pud(pud, __pud(0));
1da177e4 485 continue;
14a62c34 486 }
1da177e4 487
6ad91658 488 if (pud_val(*pud)) {
a2699e47 489 if (!pud_large(*pud)) {
b50efd2a 490 last_map_addr = phys_pmd_update(pud, addr, end,
b27a43c1 491 page_size_mask, prot);
a2699e47
SS		 492 continue;
493 }
b27a43c1
SS		 494 /*
495 * If we are ok with PG_LEVEL_1G mapping, then we will
496 * use the existing mapping.
497 *
498 * Otherwise, we will split the gbpage mapping but use
499 * the same existing protection bits except for large
500 * page, so that we don't violate Intel's TLB
501 * Application note (317080) which says, while changing
502 * the page sizes, new and old translations should
503 * not differ with respect to page frame and
504 * attributes.
505 */
3afa3949
YL		 506 if (page_size_mask & (1 << PG_LEVEL_1G)) {
507 pages++;
b27a43c1 508 continue;
3afa3949 509 }
b27a43c1 510 prot = pte_pgprot(pte_clrhuge(*(pte_t *)pud));
ef925766
AK		 511 }
512
b50efd2a 513 if (page_size_mask & (1<<PG_LEVEL_1G)) {
ce0c0e50 514 pages++;
8ae3a5a8 515 spin_lock(&init_mm.page_table_lock);
ef925766
AK		 516 set_pte((pte_t *)pud,
517 pfn_pte(addr >> PAGE_SHIFT, PAGE_KERNEL_LARGE));
8ae3a5a8 518 spin_unlock(&init_mm.page_table_lock);
cc615032 519 last_map_addr = (addr & PUD_MASK) + PUD_SIZE;
6ad91658
KM		 520 continue;
521 }
522
dafe41ee 523 pmd = alloc_low_page(&pmd_phys);
b27a43c1
SS		 524 last_map_addr = phys_pmd_init(pmd, addr, end, page_size_mask,
525 prot);
4f9c11dd 526 unmap_low_page(pmd);
8ae3a5a8
JB		 527
528 spin_lock(&init_mm.page_table_lock);
4f9c11dd 529 pud_populate(&init_mm, pud, __va(pmd_phys));
44df75e6 530 spin_unlock(&init_mm.page_table_lock);
1da177e4 531 }
1a2b4412 532 __flush_tlb_all();
a2699e47 533
ce0c0e50 534 update_page_count(PG_LEVEL_1G, pages);
cc615032 535
1a0db38e 536 return last_map_addr;
14a62c34 537}
1da177e4 538
4f9c11dd 539static unsigned long __meminit
b50efd2a
YL		 540phys_pud_update(pgd_t *pgd, unsigned long addr, unsigned long end,
541 unsigned long page_size_mask)
4f9c11dd
JF		 542{
543 pud_t *pud;
544
545 pud = (pud_t *)pgd_page_vaddr(*pgd);
546
b50efd2a 547 return phys_pud_init(pud, addr, end, page_size_mask);
4f9c11dd
JF		 548}
549
0b8fdcbc
SS		 550static void __init find_early_table_space(unsigned long end, int use_pse,
551 int use_gbpages)
1da177e4 552{
c2e6d65b 553 unsigned long puds, pmds, ptes, tables, start;
1da177e4
LT		 554
555 puds = (end + PUD_SIZE - 1) >> PUD_SHIFT;
d86bb0da 556 tables = roundup(puds * sizeof(pud_t), PAGE_SIZE);
c3f5d2d8 557
0b8fdcbc 558 if (use_gbpages) {
c2e6d65b 559 unsigned long extra;
c3f5d2d8 560
c2e6d65b
YL		 561 extra = end - ((end>>PUD_SHIFT) << PUD_SHIFT);
562 pmds = (extra + PMD_SIZE - 1) >> PMD_SHIFT;
563 } else
564 pmds = (end + PMD_SIZE - 1) >> PMD_SHIFT;
c3f5d2d8 565
d86bb0da 566 tables += roundup(pmds * sizeof(pmd_t), PAGE_SIZE);
c2e6d65b 567
0b8fdcbc 568 if (use_pse) {
c2e6d65b 569 unsigned long extra;
c3f5d2d8 570
c2e6d65b 571 extra = end - ((end>>PMD_SHIFT) << PMD_SHIFT);
49a2bf73
PE		 572#ifdef CONFIG_X86_32
573 extra += PMD_SIZE;
574#endif
c2e6d65b
YL		 575 ptes = (extra + PAGE_SIZE - 1) >> PAGE_SHIFT;
576 } else
577 ptes = (end + PAGE_SIZE - 1) >> PAGE_SHIFT;
c3f5d2d8 578
d86bb0da 579 tables += roundup(ptes * sizeof(pte_t), PAGE_SIZE);
1da177e4 580
49a2bf73
PE		 581#ifdef CONFIG_X86_32
582 /* for fixmap */
583 tables += roundup(__end_of_fixed_addresses * sizeof(pte_t), PAGE_SIZE);
584#endif
585
14a62c34
TG		 586 /*
587 * RED-PEN putting page tables only on node 0 could
588 * cause a hotspot and fill up ZONE_DMA. The page tables
589 * need roughly 0.5KB per GB.
590 */
49a2bf73
PE		 591#ifdef CONFIG_X86_32
592 start = 0x7000;
593 table_start = find_e820_area(start, max_pfn_mapped<<PAGE_SHIFT,
594 tables, PAGE_SIZE);
595#else /* CONFIG_X86_64 */
14a62c34 596 start = 0x8000;
24a5da73 597 table_start = find_e820_area(start, end, tables, PAGE_SIZE);
49a2bf73 598#endif
1da177e4
LT		 599 if (table_start == -1UL)
600 panic("Cannot find space for the kernel page tables");
601
602 table_start >>= PAGE_SHIFT;
603 table_end = table_start;
d86623a0 604 table_top = table_start + (tables >> PAGE_SHIFT);
44df75e6 605
d86623a0
YL		 606 printk(KERN_DEBUG "kernel direct mapping tables up to %lx @ %lx-%lx\n",
607 end, table_start << PAGE_SHIFT, table_top << PAGE_SHIFT);
1da177e4
LT		 608}
609
ef925766
AK		 610static void __init init_gbpages(void)
611{
612 if (direct_gbpages && cpu_has_gbpages)
613 printk(KERN_INFO "Using GB pages for direct mapping\n");
614 else
615 direct_gbpages = 0;
616}
617
f5495506 618static unsigned long __meminit kernel_physical_mapping_init(unsigned long start,
b50efd2a
YL		 619 unsigned long end,
620 unsigned long page_size_mask)
14a62c34 621{
1da177e4 622
b50efd2a 623 unsigned long next, last_map_addr = end;
1da177e4
LT		 624
625 start = (unsigned long)__va(start);
626 end = (unsigned long)__va(end);
627
628 for (; start < end; start = next) {
44df75e6 629 pgd_t *pgd = pgd_offset_k(start);
14a62c34 630 unsigned long pud_phys;
44df75e6
MT		 631 pud_t *pud;
632
e22146e6 633 next = (start + PGDIR_SIZE) & PGDIR_MASK;
4f9c11dd
JF		 634 if (next > end)
635 next = end;
636
637 if (pgd_val(*pgd)) {
b50efd2a
YL		 638 last_map_addr = phys_pud_update(pgd, __pa(start),
639 __pa(end), page_size_mask);
4f9c11dd
JF		 640 continue;
641 }
642
8ae3a5a8 643 pud = alloc_low_page(&pud_phys);
b50efd2a
YL		 644 last_map_addr = phys_pud_init(pud, __pa(start), __pa(next),
645 page_size_mask);
4f9c11dd 646 unmap_low_page(pud);
8ae3a5a8
JB		 647
648 spin_lock(&init_mm.page_table_lock);
649 pgd_populate(&init_mm, pgd, __va(pud_phys));
650 spin_unlock(&init_mm.page_table_lock);
14a62c34 651 }
a2699e47 652 __flush_tlb_all();
1da177e4 653
b50efd2a
YL		 654 return last_map_addr;
655}
7b16eb89
YL		 656
657struct map_range {
658 unsigned long start;
659 unsigned long end;
660 unsigned page_size_mask;
661};
662
b47e3418
PE		 663#ifdef CONFIG_X86_32
664#define NR_RANGE_MR 3
665#else /* CONFIG_X86_64 */
7b16eb89 666#define NR_RANGE_MR 5
b47e3418 667#endif
7b16eb89
YL		 668
669static int save_mr(struct map_range *mr, int nr_range,
670 unsigned long start_pfn, unsigned long end_pfn,
671 unsigned long page_size_mask)
672{
7b16eb89
YL		 673 if (start_pfn < end_pfn) {
674 if (nr_range >= NR_RANGE_MR)
675 panic("run out of range for init_memory_mapping\n");
676 mr[nr_range].start = start_pfn<<PAGE_SHIFT;
677 mr[nr_range].end = end_pfn<<PAGE_SHIFT;
678 mr[nr_range].page_size_mask = page_size_mask;
679 nr_range++;
680 }
681
682 return nr_range;
683}
684
b50efd2a
YL		 685/*
686 * Setup the direct mapping of the physical memory at PAGE_OFFSET.
687 * This runs before bootmem is initialized and gets pages directly from
688 * the physical memory. To access them they are temporarily mapped.
689 */
690unsigned long __init_refok init_memory_mapping(unsigned long start,
691 unsigned long end)
692{
b50efd2a 693 unsigned long page_size_mask = 0;
c2e6d65b 694 unsigned long start_pfn, end_pfn;
f96f57d9 695 unsigned long pos;
cbba6579 696 unsigned long ret;
b50efd2a 697
7b16eb89
YL		 698 struct map_range mr[NR_RANGE_MR];
699 int nr_range, i;
0b8fdcbc 700 int use_pse, use_gbpages;
7b16eb89 701
f96f57d9 702 printk(KERN_INFO "init_memory_mapping: %016lx-%016lx\n", start, end);
b50efd2a 703
7b16eb89 704 if (!after_bootmem)
b50efd2a 705 init_gbpages();
b50efd2a 706
0b8fdcbc
SS		 707#ifdef CONFIG_DEBUG_PAGEALLOC
708 /*
709 * For CONFIG_DEBUG_PAGEALLOC, identity mapping will use small pages.
710 * This will simplify cpa(), which otherwise needs to support splitting
711 * large pages into small in interrupt context, etc.
712 */
713 use_pse = use_gbpages = 0;
714#else
715 use_pse = cpu_has_pse;
716 use_gbpages = direct_gbpages;
717#endif
718
54e63f3a
PE		 719#ifdef CONFIG_X86_32
720#ifdef CONFIG_X86_PAE
721 set_nx();
722 if (nx_enabled)
723 printk(KERN_INFO "NX (Execute Disable) protection: active\n");
724#endif
725
726 /* Enable PSE if available */
727 if (cpu_has_pse)
728 set_in_cr4(X86_CR4_PSE);
729
730 /* Enable PGE if available */
731 if (cpu_has_pge) {
732 set_in_cr4(X86_CR4_PGE);
733 __supported_pte_mask |= _PAGE_GLOBAL;
734 }
735#endif
736
0b8fdcbc 737 if (use_gbpages)
b50efd2a 738 page_size_mask |= 1 << PG_LEVEL_1G;
0b8fdcbc 739 if (use_pse)
b50efd2a
YL		 740 page_size_mask |= 1 << PG_LEVEL_2M;
741
7b16eb89
YL		 742 memset(mr, 0, sizeof(mr));
743 nr_range = 0;
744
c3f5d2d8 745 /* head if not big page alignment ? */
c2e6d65b 746 start_pfn = start >> PAGE_SHIFT;
f96f57d9 747 pos = start_pfn << PAGE_SHIFT;
c338d6f6
PE		 748#ifdef CONFIG_X86_32
749 /*
750 * Don't use a large page for the first 2/4MB of memory
751 * because there are often fixed size MTRRs in there
752 * and overlapping MTRRs into large pages can cause
753 * slowdowns.
754 */
755 if (pos == 0)
756 end_pfn = 1<<(PMD_SHIFT - PAGE_SHIFT);
757 else
758 end_pfn = ((pos + (PMD_SIZE - 1))>>PMD_SHIFT)
759 << (PMD_SHIFT - PAGE_SHIFT);
760#else /* CONFIG_X86_64 */
f96f57d9 761 end_pfn = ((pos + (PMD_SIZE - 1)) >> PMD_SHIFT)
c2e6d65b 762 << (PMD_SHIFT - PAGE_SHIFT);
c338d6f6 763#endif
0fc59d3a
YL		 764 if (end_pfn > (end >> PAGE_SHIFT))
765 end_pfn = end >> PAGE_SHIFT;
f96f57d9
YL		 766 if (start_pfn < end_pfn) {
767 nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 0);
768 pos = end_pfn << PAGE_SHIFT;
769 }
c2e6d65b 770
c3f5d2d8 771 /* big page (2M) range */
f96f57d9 772 start_pfn = ((pos + (PMD_SIZE - 1))>>PMD_SHIFT)
c2e6d65b 773 << (PMD_SHIFT - PAGE_SHIFT);
c338d6f6
PE		 774#ifdef CONFIG_X86_32
775 end_pfn = (end>>PMD_SHIFT) << (PMD_SHIFT - PAGE_SHIFT);
776#else /* CONFIG_X86_64 */
f96f57d9 777 end_pfn = ((pos + (PUD_SIZE - 1))>>PUD_SHIFT)
c2e6d65b 778 << (PUD_SHIFT - PAGE_SHIFT);
f96f57d9
YL		 779 if (end_pfn > ((end>>PMD_SHIFT)<<(PMD_SHIFT - PAGE_SHIFT)))
780 end_pfn = ((end>>PMD_SHIFT)<<(PMD_SHIFT - PAGE_SHIFT));
c338d6f6
PE		 781#endif
782
f96f57d9
YL		 783 if (start_pfn < end_pfn) {
784 nr_range = save_mr(mr, nr_range, start_pfn, end_pfn,
785 page_size_mask & (1<<PG_LEVEL_2M));
786 pos = end_pfn << PAGE_SHIFT;
787 }
c2e6d65b 788
c338d6f6 789#ifdef CONFIG_X86_64
c2e6d65b 790 /* big page (1G) range */
f96f57d9
YL		 791 start_pfn = ((pos + (PUD_SIZE - 1))>>PUD_SHIFT)
792 << (PUD_SHIFT - PAGE_SHIFT);
793 end_pfn = (end >> PUD_SHIFT) << (PUD_SHIFT - PAGE_SHIFT);
794 if (start_pfn < end_pfn) {
795 nr_range = save_mr(mr, nr_range, start_pfn, end_pfn,
7b16eb89
YL		 796 page_size_mask &
797 ((1<<PG_LEVEL_2M)|(1<<PG_LEVEL_1G)));
f96f57d9
YL		 798 pos = end_pfn << PAGE_SHIFT;
799 }
c2e6d65b
YL		 800
801 /* tail is not big page (1G) alignment */
f96f57d9
YL		 802 start_pfn = ((pos + (PMD_SIZE - 1))>>PMD_SHIFT)
803 << (PMD_SHIFT - PAGE_SHIFT);
804 end_pfn = (end >> PMD_SHIFT) << (PMD_SHIFT - PAGE_SHIFT);
805 if (start_pfn < end_pfn) {
806 nr_range = save_mr(mr, nr_range, start_pfn, end_pfn,
807 page_size_mask & (1<<PG_LEVEL_2M));
808 pos = end_pfn << PAGE_SHIFT;
809 }
c338d6f6 810#endif
7b16eb89 811
c2e6d65b 812 /* tail is not big page (2M) alignment */
f96f57d9 813 start_pfn = pos>>PAGE_SHIFT;
c2e6d65b 814 end_pfn = end>>PAGE_SHIFT;
7b16eb89
YL		 815 nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 0);
816
9958e810
YL		 817 /* try to merge same page size and continuous */
818 for (i = 0; nr_range > 1 && i < nr_range - 1; i++) {
819 unsigned long old_start;
820 if (mr[i].end != mr[i+1].start ||
821 mr[i].page_size_mask != mr[i+1].page_size_mask)
822 continue;
823 /* move it */
824 old_start = mr[i].start;
825 memmove(&mr[i], &mr[i+1],
c3f5d2d8 826 (nr_range - 1 - i) * sizeof(struct map_range));
5e72d9e4 827 mr[i--].start = old_start;
9958e810
YL		 828 nr_range--;
829 }
830
7b16eb89
YL		 831 for (i = 0; i < nr_range; i++)
832 printk(KERN_DEBUG " %010lx - %010lx page %s\n",
833 mr[i].start, mr[i].end,
834 (mr[i].page_size_mask & (1<<PG_LEVEL_1G))?"1G":(
835 (mr[i].page_size_mask & (1<<PG_LEVEL_2M))?"2M":"4k"));
836
c3f5d2d8
PE		 837 /*
838 * Find space for the kernel direct mapping tables.
839 *
840 * Later we should allocate these tables in the local node of the
841 * memory mapped. Unfortunately this is done currently before the
842 * nodes are discovered.
843 */
7b16eb89 844 if (!after_bootmem)
0b8fdcbc 845 find_early_table_space(end, use_pse, use_gbpages);
7b16eb89 846
cbba6579
PE		 847#ifdef CONFIG_X86_32
848 for (i = 0; i < nr_range; i++)
849 kernel_physical_mapping_init(
850 mr[i].start >> PAGE_SHIFT,
851 mr[i].end >> PAGE_SHIFT,
852 mr[i].page_size_mask == (1<<PG_LEVEL_2M));
853 ret = end;
854#else /* CONFIG_X86_64 */
7b16eb89 855 for (i = 0; i < nr_range; i++)
cbba6579
PE		 856 ret = kernel_physical_mapping_init(mr[i].start, mr[i].end,
857 mr[i].page_size_mask);
858#endif
b50efd2a 859
01ced9ec
PE		 860#ifdef CONFIG_X86_32
861 early_ioremap_page_table_range_init();
862
863 load_cr3(swapper_pg_dir);
864#endif
865
866#ifdef CONFIG_X86_64
44df75e6 867 if (!after_bootmem)
f51c9452 868 mmu_cr4_features = read_cr4();
01ced9ec 869#endif
1da177e4 870 __flush_tlb_all();
75175278 871
b50efd2a 872 if (!after_bootmem && table_end > table_start)
24a5da73
YL		 873 reserve_early(table_start << PAGE_SHIFT,
874 table_end << PAGE_SHIFT, "PGTABLE");
272b9cad
YL		 875
876 if (!after_bootmem)
b50efd2a 877 early_memtest(start, end);
cc615032 878
cbba6579 879 return ret >> PAGE_SHIFT;
1da177e4
LT		 880}
881
2b97690f 882#ifndef CONFIG_NUMA
1f75d7e3
YL		 883void __init initmem_init(unsigned long start_pfn, unsigned long end_pfn)
884{
885 unsigned long bootmap_size, bootmap;
886
887 bootmap_size = bootmem_bootmap_pages(end_pfn)<<PAGE_SHIFT;
888 bootmap = find_e820_area(0, end_pfn<<PAGE_SHIFT, bootmap_size,
889 PAGE_SIZE);
890 if (bootmap == -1L)
891 panic("Cannot find bootmem map of size %ld\n", bootmap_size);
346cafec
YL		 892 /* don't touch min_low_pfn */
893 bootmap_size = init_bootmem_node(NODE_DATA(0), bootmap >> PAGE_SHIFT,
894 0, end_pfn);
1f75d7e3
YL		 895 e820_register_active_regions(0, start_pfn, end_pfn);
896 free_bootmem_with_active_regions(0, end_pfn);
897 early_res_to_bootmem(0, end_pfn<<PAGE_SHIFT);
898 reserve_bootmem(bootmap, bootmap_size, BOOTMEM_DEFAULT);
899}
900
1da177e4
LT		 901void __init paging_init(void)
902{
6391af17 903 unsigned long max_zone_pfns[MAX_NR_ZONES];
14a62c34 904
6391af17
MG		 905 memset(max_zone_pfns, 0, sizeof(max_zone_pfns));
906 max_zone_pfns[ZONE_DMA] = MAX_DMA_PFN;
907 max_zone_pfns[ZONE_DMA32] = MAX_DMA32_PFN;
c987d12f 908 max_zone_pfns[ZONE_NORMAL] = max_pfn;
6391af17 909
c987d12f 910 memory_present(0, 0, max_pfn);
44df75e6 911 sparse_init();
5cb248ab 912 free_area_init_nodes(max_zone_pfns);
1da177e4
LT		 913}
914#endif
915
44df75e6
MT		 916/*
917 * Memory hotplug specific functions
44df75e6 918 */
bc02af93 919#ifdef CONFIG_MEMORY_HOTPLUG
9d99aaa3
AK		 920/*
921 * Memory is added always to NORMAL zone. This means you will never get
922 * additional DMA/DMA32 memory.
923 */
bc02af93 924int arch_add_memory(int nid, u64 start, u64 size)
44df75e6 925{
bc02af93 926 struct pglist_data *pgdat = NODE_DATA(nid);
776ed98b 927 struct zone *zone = pgdat->node_zones + ZONE_NORMAL;
cc615032 928 unsigned long last_mapped_pfn, start_pfn = start >> PAGE_SHIFT;
44df75e6
MT		 929 unsigned long nr_pages = size >> PAGE_SHIFT;
930 int ret;
931
60817c9b 932 last_mapped_pfn = init_memory_mapping(start, start + size);
cc615032
AK		 933 if (last_mapped_pfn > max_pfn_mapped)
934 max_pfn_mapped = last_mapped_pfn;
45e0b78b 935
c04fc586 936 ret = __add_pages(nid, zone, start_pfn, nr_pages);
fe8b868e 937 WARN_ON_ONCE(ret);
44df75e6 938
44df75e6 939 return ret;
44df75e6 940}
bc02af93 941EXPORT_SYMBOL_GPL(arch_add_memory);
44df75e6 942
8243229f 943#if !defined(CONFIG_ACPI_NUMA) && defined(CONFIG_NUMA)
4942e998
KM		 944int memory_add_physaddr_to_nid(u64 start)
945{
946 return 0;
947}
8c2676a5 948EXPORT_SYMBOL_GPL(memory_add_physaddr_to_nid);
4942e998
KM		 949#endif
950
45e0b78b
KM		 951#endif /* CONFIG_MEMORY_HOTPLUG */
952
14a62c34
TG		 953static struct kcore_list kcore_mem, kcore_vmalloc, kcore_kernel,
954 kcore_modules, kcore_vsyscall;
1da177e4
LT		 955
956void __init mem_init(void)
957{
0a43e4bf 958 long codesize, reservedpages, datasize, initsize;
11a6b0c9 959 unsigned long absent_pages;
1da177e4 960
0dc243ae 961 pci_iommu_alloc();
1da177e4 962
48ddb154 963 /* clear_bss() already clear the empty_zero_page */
1da177e4
LT		 964
965 reservedpages = 0;
966
967 /* this will put all low memory onto the freelists */
2b97690f 968#ifdef CONFIG_NUMA
0a43e4bf 969 totalram_pages = numa_free_all_bootmem();
1da177e4 970#else
0a43e4bf 971 totalram_pages = free_all_bootmem();
1da177e4 972#endif
11a6b0c9
YL		 973
974 absent_pages = absent_pages_in_range(0, max_pfn);
975 reservedpages = max_pfn - totalram_pages - absent_pages;
1da177e4
LT		 976 after_bootmem = 1;
977
978 codesize = (unsigned long) &_etext - (unsigned long) &_text;
979 datasize = (unsigned long) &_edata - (unsigned long) &_etext;
980 initsize = (unsigned long) &__init_end - (unsigned long) &__init_begin;
981
982 /* Register memory areas for /proc/kcore */
14a62c34
TG		 983 kclist_add(&kcore_mem, __va(0), max_low_pfn << PAGE_SHIFT);
984 kclist_add(&kcore_vmalloc, (void *)VMALLOC_START,
1da177e4
LT		 985 VMALLOC_END-VMALLOC_START);
986 kclist_add(&kcore_kernel, &_stext, _end - _stext);
987 kclist_add(&kcore_modules, (void *)MODULES_VADDR, MODULES_LEN);
14a62c34 988 kclist_add(&kcore_vsyscall, (void *)VSYSCALL_START,
1da177e4
LT		 989 VSYSCALL_END - VSYSCALL_START);
990
10f22dde 991 printk(KERN_INFO "Memory: %luk/%luk available (%ldk kernel code, "
11a6b0c9 992 "%ldk absent, %ldk reserved, %ldk data, %ldk init)\n",
1da177e4 993 (unsigned long) nr_free_pages() << (PAGE_SHIFT-10),
c987d12f 994 max_pfn << (PAGE_SHIFT-10),
1da177e4 995 codesize >> 10,
11a6b0c9 996 absent_pages << (PAGE_SHIFT-10),
1da177e4
LT		 997 reservedpages << (PAGE_SHIFT-10),
998 datasize >> 10,
999 initsize >> 10);
1da177e4
LT		 1000}
1001
67df197b 1002#ifdef CONFIG_DEBUG_RODATA
edeed305
AV		 1003const int rodata_test_data = 0xC3;
1004EXPORT_SYMBOL_GPL(rodata_test_data);
67df197b 1005
67df197b
AV		 1006void mark_rodata_ro(void)
1007{
4e4eee0e 1008 unsigned long start = PFN_ALIGN(_stext), end = PFN_ALIGN(__end_rodata);
8f0f996e
SR		 1009 unsigned long rodata_start =
1010 ((unsigned long)__start_rodata + PAGE_SIZE - 1) & PAGE_MASK;
1011
1012#ifdef CONFIG_DYNAMIC_FTRACE
1013 /* Dynamic tracing modifies the kernel text section */
1014 start = rodata_start;
1015#endif
67df197b 1016
6fb14755 1017 printk(KERN_INFO "Write protecting the kernel read-only data: %luk\n",
e3ebadd9 1018 (end - start) >> 10);
984bb80d
AV		 1019 set_memory_ro(start, (end - start) >> PAGE_SHIFT);
1020
1021 /*
1022 * The rodata section (but not the kernel text!) should also be
1023 * not-executable.
1024 */
72b59d67 1025 set_memory_nx(rodata_start, (end - rodata_start) >> PAGE_SHIFT);
67df197b 1026
1a487252
AV		 1027 rodata_test();
1028
0c42f392 1029#ifdef CONFIG_CPA_DEBUG
10f22dde 1030 printk(KERN_INFO "Testing CPA: undo %lx-%lx\n", start, end);
6d238cc4 1031 set_memory_rw(start, (end-start) >> PAGE_SHIFT);
0c42f392 1032
10f22dde 1033 printk(KERN_INFO "Testing CPA: again\n");
6d238cc4 1034 set_memory_ro(start, (end-start) >> PAGE_SHIFT);
0c42f392 1035#endif
67df197b 1036}
4e4eee0e 1037
67df197b
AV		 1038#endif
1039
d2dbf343
YL		 1040int __init reserve_bootmem_generic(unsigned long phys, unsigned long len,
1041 int flags)
14a62c34 1042{
2b97690f 1043#ifdef CONFIG_NUMA
8b3cd09e 1044 int nid, next_nid;
6a07a0ed 1045 int ret;
5e58a02a
AK		 1046#endif
1047 unsigned long pfn = phys >> PAGE_SHIFT;
14a62c34 1048
c987d12f 1049 if (pfn >= max_pfn) {
14a62c34
TG		 1050 /*
1051 * This can happen with kdump kernels when accessing
1052 * firmware tables:
1053 */
67794292 1054 if (pfn < max_pfn_mapped)
8b2ef1d7 1055 return -EFAULT;
14a62c34 1056
6a07a0ed 1057 printk(KERN_ERR "reserve_bootmem: illegal reserve %lx %lu\n",
5e58a02a 1058 phys, len);
8b2ef1d7 1059 return -EFAULT;
5e58a02a
AK		 1060 }
1061
1062 /* Should check here against the e820 map to avoid double free */
1063#ifdef CONFIG_NUMA
8b3cd09e
YL		 1064 nid = phys_to_nid(phys);
1065 next_nid = phys_to_nid(phys + len - 1);
1066 if (nid == next_nid)
8b2ef1d7 1067 ret = reserve_bootmem_node(NODE_DATA(nid), phys, len, flags);
8b3cd09e 1068 else
8b2ef1d7
BW		 1069 ret = reserve_bootmem(phys, len, flags);
1070
1071 if (ret != 0)
1072 return ret;
1073
14a62c34 1074#else
72a7fe39 1075 reserve_bootmem(phys, len, BOOTMEM_DEFAULT);
1da177e4 1076#endif
8b3cd09e 1077
0e0b864e 1078 if (phys+len <= MAX_DMA_PFN*PAGE_SIZE) {
e18c6874 1079 dma_reserve += len / PAGE_SIZE;
0e0b864e
MG		 1080 set_dma_reserve(dma_reserve);
1081 }
8b2ef1d7
BW		 1082
1083 return 0;
1da177e4
LT		 1084}
1085
14a62c34
TG		 1086int kern_addr_valid(unsigned long addr)
1087{
1da177e4 1088 unsigned long above = ((long)addr) >> __VIRTUAL_MASK_SHIFT;
14a62c34
TG		 1089 pgd_t *pgd;
1090 pud_t *pud;
1091 pmd_t *pmd;
1092 pte_t *pte;
1da177e4
LT		 1093
1094 if (above != 0 && above != -1UL)
14a62c34
TG		 1095 return 0;
1096
1da177e4
LT		 1097 pgd = pgd_offset_k(addr);
1098 if (pgd_none(*pgd))
1099 return 0;
1100
1101 pud = pud_offset(pgd, addr);
1102 if (pud_none(*pud))
14a62c34 1103 return 0;
1da177e4
LT		 1104
1105 pmd = pmd_offset(pud, addr);
1106 if (pmd_none(*pmd))
1107 return 0;
14a62c34 1108
1da177e4
LT		 1109 if (pmd_large(*pmd))
1110 return pfn_valid(pmd_pfn(*pmd));
1111
1112 pte = pte_offset_kernel(pmd, addr);
1113 if (pte_none(*pte))
1114 return 0;
14a62c34 1115
1da177e4
LT		 1116 return pfn_valid(pte_pfn(*pte));
1117}
1118
14a62c34
TG		 1119/*
1120 * A pseudo VMA to allow ptrace access for the vsyscall page. This only
1121 * covers the 64bit vsyscall page now. 32bit has a real VMA now and does
1122 * not need special handling anymore:
1123 */
1da177e4 1124static struct vm_area_struct gate_vma = {
14a62c34
TG		 1125 .vm_start = VSYSCALL_START,
1126 .vm_end = VSYSCALL_START + (VSYSCALL_MAPPED_PAGES * PAGE_SIZE),
1127 .vm_page_prot = PAGE_READONLY_EXEC,
1128 .vm_flags = VM_READ | VM_EXEC
1da177e4
LT		 1129};
1130
1da177e4
LT		 1131struct vm_area_struct *get_gate_vma(struct task_struct *tsk)
1132{
1133#ifdef CONFIG_IA32_EMULATION
1e014410
AK		 1134 if (test_tsk_thread_flag(tsk, TIF_IA32))
1135 return NULL;
1da177e4
LT		 1136#endif
1137 return &gate_vma;
1138}
1139
1140int in_gate_area(struct task_struct *task, unsigned long addr)
1141{
1142 struct vm_area_struct *vma = get_gate_vma(task);
14a62c34 1143
1e014410
AK		 1144 if (!vma)
1145 return 0;
14a62c34 1146
1da177e4
LT		 1147 return (addr >= vma->vm_start) && (addr < vma->vm_end);
1148}
1149
14a62c34
TG		 1150/*
1151 * Use this when you have no reliable task/vma, typically from interrupt
1152 * context. It is less reliable than using the task's vma and may give
1153 * false positives:
1da177e4
LT		 1154 */
1155int in_gate_area_no_task(unsigned long addr)
1156{
1e014410 1157 return (addr >= VSYSCALL_START) && (addr < VSYSCALL_END);
1da177e4 1158}
2e1c49db 1159
2aae950b
AK		 1160const char *arch_vma_name(struct vm_area_struct *vma)
1161{
1162 if (vma->vm_mm && vma->vm_start == (long)vma->vm_mm->context.vdso)
1163 return "[vdso]";
1164 if (vma == &gate_vma)
1165 return "[vsyscall]";
1166 return NULL;
1167}
0889eba5
CL		 1168
1169#ifdef CONFIG_SPARSEMEM_VMEMMAP
1170/*
1171 * Initialise the sparsemem vmemmap using huge-pages at the PMD level.
1172 */
c2b91e2e
YL		 1173static long __meminitdata addr_start, addr_end;
1174static void __meminitdata *p_start, *p_end;
1175static int __meminitdata node_start;
1176
14a62c34
TG		 1177int __meminit
1178vmemmap_populate(struct page *start_page, unsigned long size, int node)
0889eba5
CL		 1179{
1180 unsigned long addr = (unsigned long)start_page;
1181 unsigned long end = (unsigned long)(start_page + size);
1182 unsigned long next;
1183 pgd_t *pgd;
1184 pud_t *pud;
1185 pmd_t *pmd;
1186
1187 for (; addr < end; addr = next) {
7c934d39 1188 void *p = NULL;
0889eba5
CL		 1189
1190 pgd = vmemmap_pgd_populate(addr, node);
1191 if (!pgd)
1192 return -ENOMEM;
14a62c34 1193
0889eba5
CL		 1194 pud = vmemmap_pud_populate(pgd, addr, node);
1195 if (!pud)
1196 return -ENOMEM;
1197
7c934d39
JF		 1198 if (!cpu_has_pse) {
1199 next = (addr + PAGE_SIZE) & PAGE_MASK;
1200 pmd = vmemmap_pmd_populate(pud, addr, node);
1201
1202 if (!pmd)
1203 return -ENOMEM;
1204
1205 p = vmemmap_pte_populate(pmd, addr, node);
14a62c34 1206
0889eba5
CL		 1207 if (!p)
1208 return -ENOMEM;
1209
7c934d39
JF		 1210 addr_end = addr + PAGE_SIZE;
1211 p_end = p + PAGE_SIZE;
14a62c34 1212 } else {
7c934d39
JF		 1213 next = pmd_addr_end(addr, end);
1214
1215 pmd = pmd_offset(pud, addr);
1216 if (pmd_none(*pmd)) {
1217 pte_t entry;
1218
1219 p = vmemmap_alloc_block(PMD_SIZE, node);
1220 if (!p)
1221 return -ENOMEM;
1222
1223 entry = pfn_pte(__pa(p) >> PAGE_SHIFT,
1224 PAGE_KERNEL_LARGE);
1225 set_pmd(pmd, __pmd(pte_val(entry)));
1226
7c934d39
JF		 1227 /* check to see if we have contiguous blocks */
1228 if (p_end != p || node_start != node) {
1229 if (p_start)
1230 printk(KERN_DEBUG " [%lx-%lx] PMD -> [%p-%p] on node %d\n",
1231 addr_start, addr_end-1, p_start, p_end-1, node_start);
1232 addr_start = addr;
1233 node_start = node;
1234 p_start = p;
1235 }
49c980df
YL		 1236
1237 addr_end = addr + PMD_SIZE;
1238 p_end = p + PMD_SIZE;
7c934d39
JF		 1239 } else
1240 vmemmap_verify((pte_t *)pmd, node, addr, next);
14a62c34 1241 }
7c934d39 1242
0889eba5 1243 }
0889eba5
CL		 1244 return 0;
1245}
c2b91e2e
YL		 1246
1247void __meminit vmemmap_populate_print_last(void)
1248{
1249 if (p_start) {
1250 printk(KERN_DEBUG " [%lx-%lx] PMD -> [%p-%p] on node %d\n",
1251 addr_start, addr_end-1, p_start, p_end-1, node_start);
1252 p_start = NULL;
1253 p_end = NULL;
1254 node_start = 0;
1255 }
1256}
0889eba5 1257#endif
Clone of davem's net-next-2.6 tree