[net-next-2.6.git] / arch / powerpc / mm / mem.c

/*
 *  PowerPC version
 *    Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
 *
 *  Modifications by Paul Mackerras (PowerMac) (paulus@cs.anu.edu.au)
 *  and Cort Dougan (PReP) (cort@cs.nmt.edu)
 *    Copyright (C) 1996 Paul Mackerras
 *  PPC44x/36-bit changes by Matt Porter (mporter@mvista.com)
 *
 *  Derived from "arch/i386/mm/init.c"
 *    Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
 *
 *  This program is free software; you can redistribute it and/or
 *  modify it under the terms of the GNU General Public License
 *  as published by the Free Software Foundation; either version
 *  2 of the License, or (at your option) any later version.
 *
 */

#include <linux/module.h>
#include <linux/sched.h>
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/string.h>
#include <linux/types.h>
#include <linux/mm.h>
#include <linux/stddef.h>
#include <linux/init.h>
#include <linux/bootmem.h>
#include <linux/highmem.h>
#include <linux/initrd.h>
#include <linux/pagemap.h>
#include <linux/suspend.h>
#include <linux/lmb.h>

#include <asm/pgalloc.h>
#include <asm/prom.h>
#include <asm/io.h>
#include <asm/mmu_context.h>
#include <asm/pgtable.h>
#include <asm/mmu.h>
#include <asm/smp.h>
#include <asm/machdep.h>
#include <asm/btext.h>
#include <asm/tlb.h>
#include <asm/sections.h>
#include <asm/sparsemem.h>
#include <asm/vdso.h>
#include <asm/fixmap.h>

#include "mmu_decl.h"

#ifndef CPU_FTR_COHERENT_ICACHE
#define CPU_FTR_COHERENT_ICACHE	0	/* XXX for now */
#define CPU_FTR_NOEXECUTE	0
#endif

int init_bootmem_done;
int mem_init_done;
phys_addr_t memory_limit;

#ifdef CONFIG_HIGHMEM
pte_t *kmap_pte;
pgprot_t kmap_prot;

EXPORT_SYMBOL(kmap_prot);
EXPORT_SYMBOL(kmap_pte);

static inline pte_t *virt_to_kpte(unsigned long vaddr)
{
	return pte_offset_kernel(pmd_offset(pud_offset(pgd_offset_k(vaddr),
			vaddr), vaddr), vaddr);
}
#endif

int page_is_ram(unsigned long pfn)
{
#ifndef CONFIG_PPC64	/* XXX for now */
	return pfn < max_pfn;
#else
	unsigned long paddr = (pfn << PAGE_SHIFT);
	int i;
	for (i=0; i < lmb.memory.cnt; i++) {
		unsigned long base;

		base = lmb.memory.region[i].base;

		if ((paddr >= base) &&
			(paddr < (base + lmb.memory.region[i].size))) {
			return 1;
		}
	}

	return 0;
#endif
}

pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
			      unsigned long size, pgprot_t vma_prot)
{
	if (ppc_md.phys_mem_access_prot)
		return ppc_md.phys_mem_access_prot(file, pfn, size, vma_prot);

	if (!page_is_ram(pfn))
		vma_prot = pgprot_noncached(vma_prot);

	return vma_prot;
}
EXPORT_SYMBOL(phys_mem_access_prot);

#ifdef CONFIG_MEMORY_HOTPLUG

#ifdef CONFIG_NUMA
int memory_add_physaddr_to_nid(u64 start)
{
	return hot_add_scn_to_nid(start);
}
#endif

int arch_add_memory(int nid, u64 start, u64 size)
{
	struct pglist_data *pgdata;
	struct zone *zone;
	unsigned long start_pfn = start >> PAGE_SHIFT;
	unsigned long nr_pages = size >> PAGE_SHIFT;

	pgdata = NODE_DATA(nid);

	start = (unsigned long)__va(start);
	create_section_mapping(start, start + size);

	/* this should work for most non-highmem platforms */
	zone = pgdata->node_zones;

	return __add_pages(nid, zone, start_pfn, nr_pages);
}
#endif /* CONFIG_MEMORY_HOTPLUG */

/*
 * walk_memory_resource() needs to make sure there is no holes in a given
 * memory range.  PPC64 does not maintain the memory layout in /proc/iomem.
 * Instead it maintains it in lmb.memory structures.  Walk through the
 * memory regions, find holes and callback for contiguous regions.
 */
int
walk_memory_resource(unsigned long start_pfn, unsigned long nr_pages, void *arg,
			int (*func)(unsigned long, unsigned long, void *))
{
	struct lmb_property res;
	unsigned long pfn, len;
	u64 end;
	int ret = -1;

	res.base = (u64) start_pfn << PAGE_SHIFT;
	res.size = (u64) nr_pages << PAGE_SHIFT;

	end = res.base + res.size - 1;
	while ((res.base < end) && (lmb_find(&res) >= 0)) {
		pfn = (unsigned long)(res.base >> PAGE_SHIFT);
		len = (unsigned long)(res.size >> PAGE_SHIFT);
		ret = (*func)(pfn, len, arg);
		if (ret)
			break;
		res.base += (res.size + 1);
		res.size = (end - res.base + 1);
	}
	return ret;
}
EXPORT_SYMBOL_GPL(walk_memory_resource);

/*
 * Initialize the bootmem system and give it all the memory we
 * have available.  If we are using highmem, we only put the
 * lowmem into the bootmem system.
 */
#ifndef CONFIG_NEED_MULTIPLE_NODES
void __init do_init_bootmem(void)
{
	unsigned long i;
	unsigned long start, bootmap_pages;
	unsigned long total_pages;
	int boot_mapsize;

	max_low_pfn = max_pfn = lmb_end_of_DRAM() >> PAGE_SHIFT;
	total_pages = (lmb_end_of_DRAM() - memstart_addr) >> PAGE_SHIFT;
#ifdef CONFIG_HIGHMEM
	total_pages = total_lowmem >> PAGE_SHIFT;
	max_low_pfn = lowmem_end_addr >> PAGE_SHIFT;
#endif

	/*
	 * Find an area to use for the bootmem bitmap.  Calculate the size of
	 * bitmap required as (Total Memory) / PAGE_SIZE / BITS_PER_BYTE.
	 * Add 1 additional page in case the address isn't page-aligned.
	 */
	bootmap_pages = bootmem_bootmap_pages(total_pages);

	start = lmb_alloc(bootmap_pages << PAGE_SHIFT, PAGE_SIZE);

	min_low_pfn = MEMORY_START >> PAGE_SHIFT;
	boot_mapsize = init_bootmem_node(NODE_DATA(0), start >> PAGE_SHIFT, min_low_pfn, max_low_pfn);

	/* Add active regions with valid PFNs */
	for (i = 0; i < lmb.memory.cnt; i++) {
		unsigned long start_pfn, end_pfn;
		start_pfn = lmb.memory.region[i].base >> PAGE_SHIFT;
		end_pfn = start_pfn + lmb_size_pages(&lmb.memory, i);
		add_active_range(0, start_pfn, end_pfn);
	}

	/* Add all physical memory to the bootmem map, mark each area
	 * present.
	 */
#ifdef CONFIG_HIGHMEM
	free_bootmem_with_active_regions(0, lowmem_end_addr >> PAGE_SHIFT);

	/* reserve the sections we're already using */
	for (i = 0; i < lmb.reserved.cnt; i++) {
		unsigned long addr = lmb.reserved.region[i].base +
				     lmb_size_bytes(&lmb.reserved, i) - 1;
		if (addr < lowmem_end_addr)
			reserve_bootmem(lmb.reserved.region[i].base,
					lmb_size_bytes(&lmb.reserved, i),
					BOOTMEM_DEFAULT);
		else if (lmb.reserved.region[i].base < lowmem_end_addr) {
			unsigned long adjusted_size = lowmem_end_addr -
				      lmb.reserved.region[i].base;
			reserve_bootmem(lmb.reserved.region[i].base,
					adjusted_size, BOOTMEM_DEFAULT);
		}
	}
#else
	free_bootmem_with_active_regions(0, max_pfn);

	/* reserve the sections we're already using */
	for (i = 0; i < lmb.reserved.cnt; i++)
		reserve_bootmem(lmb.reserved.region[i].base,
				lmb_size_bytes(&lmb.reserved, i),
				BOOTMEM_DEFAULT);

#endif
	/* XXX need to clip this if using highmem? */
	sparse_memory_present_with_active_regions(0);

	init_bootmem_done = 1;
}

/* mark pages that don't exist as nosave */
static int __init mark_nonram_nosave(void)
{
	unsigned long lmb_next_region_start_pfn,
		      lmb_region_max_pfn;
	int i;

	for (i = 0; i < lmb.memory.cnt - 1; i++) {
		lmb_region_max_pfn =
			(lmb.memory.region[i].base >> PAGE_SHIFT) +
			(lmb.memory.region[i].size >> PAGE_SHIFT);
		lmb_next_region_start_pfn =
			lmb.memory.region[i+1].base >> PAGE_SHIFT;

		if (lmb_region_max_pfn < lmb_next_region_start_pfn)
			register_nosave_region(lmb_region_max_pfn,
					       lmb_next_region_start_pfn);
	}

	return 0;
}

/*
 * paging_init() sets up the page tables - in fact we've already done this.
 */
void __init paging_init(void)
{
	unsigned long total_ram = lmb_phys_mem_size();
	phys_addr_t top_of_ram = lmb_end_of_DRAM();
	unsigned long max_zone_pfns[MAX_NR_ZONES];

#ifdef CONFIG_PPC32
	unsigned long v = __fix_to_virt(__end_of_fixed_addresses - 1);
	unsigned long end = __fix_to_virt(FIX_HOLE);

	for (; v < end; v += PAGE_SIZE)
		map_page(v, 0, 0); /* XXX gross */
#endif

#ifdef CONFIG_HIGHMEM
	map_page(PKMAP_BASE, 0, 0);	/* XXX gross */
	pkmap_page_table = virt_to_kpte(PKMAP_BASE);

	kmap_pte = virt_to_kpte(__fix_to_virt(FIX_KMAP_BEGIN));
	kmap_prot = PAGE_KERNEL;
#endif /* CONFIG_HIGHMEM */

	printk(KERN_DEBUG "Top of RAM: 0x%llx, Total RAM: 0x%lx\n",
	       (unsigned long long)top_of_ram, total_ram);
	printk(KERN_DEBUG "Memory hole size: %ldMB\n",
	       (long int)((top_of_ram - total_ram) >> 20));
	memset(max_zone_pfns, 0, sizeof(max_zone_pfns));
#ifdef CONFIG_HIGHMEM
	max_zone_pfns[ZONE_DMA] = lowmem_end_addr >> PAGE_SHIFT;
	max_zone_pfns[ZONE_HIGHMEM] = top_of_ram >> PAGE_SHIFT;
#else
	max_zone_pfns[ZONE_DMA] = top_of_ram >> PAGE_SHIFT;
#endif
	free_area_init_nodes(max_zone_pfns);

	mark_nonram_nosave();
}
#endif /* ! CONFIG_NEED_MULTIPLE_NODES */

void __init mem_init(void)
{
#ifdef CONFIG_NEED_MULTIPLE_NODES
	int nid;
#endif
	pg_data_t *pgdat;
	unsigned long i;
	struct page *page;
	unsigned long reservedpages = 0, codesize, initsize, datasize, bsssize;

	num_physpages = lmb.memory.size >> PAGE_SHIFT;
	high_memory = (void *) __va(max_low_pfn * PAGE_SIZE);

#ifdef CONFIG_NEED_MULTIPLE_NODES
        for_each_online_node(nid) {
		if (NODE_DATA(nid)->node_spanned_pages != 0) {
			printk("freeing bootmem node %d\n", nid);
			totalram_pages +=
				free_all_bootmem_node(NODE_DATA(nid));
		}
	}
#else
	max_mapnr = max_pfn;
	totalram_pages += free_all_bootmem();
#endif
	for_each_online_pgdat(pgdat) {
		for (i = 0; i < pgdat->node_spanned_pages; i++) {
			if (!pfn_valid(pgdat->node_start_pfn + i))
				continue;
			page = pgdat_page_nr(pgdat, i);
			if (PageReserved(page))
				reservedpages++;
		}
	}

	codesize = (unsigned long)&_sdata - (unsigned long)&_stext;
	datasize = (unsigned long)&_edata - (unsigned long)&_sdata;
	initsize = (unsigned long)&__init_end - (unsigned long)&__init_begin;
	bsssize = (unsigned long)&__bss_stop - (unsigned long)&__bss_start;

#ifdef CONFIG_HIGHMEM
	{
		unsigned long pfn, highmem_mapnr;

		highmem_mapnr = lowmem_end_addr >> PAGE_SHIFT;
		for (pfn = highmem_mapnr; pfn < max_mapnr; ++pfn) {
			struct page *page = pfn_to_page(pfn);
			if (lmb_is_reserved(pfn << PAGE_SHIFT))
				continue;
			ClearPageReserved(page);
			init_page_count(page);
			__free_page(page);
			totalhigh_pages++;
			reservedpages--;
		}
		totalram_pages += totalhigh_pages;
		printk(KERN_DEBUG "High memory: %luk\n",
		       totalhigh_pages << (PAGE_SHIFT-10));
	}
#endif /* CONFIG_HIGHMEM */

	printk(KERN_INFO "Memory: %luk/%luk available (%luk kernel code, "
	       "%luk reserved, %luk data, %luk bss, %luk init)\n",
		nr_free_pages() << (PAGE_SHIFT-10),
		num_physpages << (PAGE_SHIFT-10),
		codesize >> 10,
		reservedpages << (PAGE_SHIFT-10),
		datasize >> 10,
		bsssize >> 10,
		initsize >> 10);

#ifdef CONFIG_PPC32
	pr_info("Kernel virtual memory layout:\n");
	pr_info("  * 0x%08lx..0x%08lx  : fixmap\n", FIXADDR_START, FIXADDR_TOP);
#ifdef CONFIG_HIGHMEM
	pr_info("  * 0x%08lx..0x%08lx  : highmem PTEs\n",
		PKMAP_BASE, PKMAP_ADDR(LAST_PKMAP));
#endif /* CONFIG_HIGHMEM */
#ifdef CONFIG_NOT_COHERENT_CACHE
	pr_info("  * 0x%08lx..0x%08lx  : consistent mem\n",
		IOREMAP_TOP, IOREMAP_TOP + CONFIG_CONSISTENT_SIZE);
#endif /* CONFIG_NOT_COHERENT_CACHE */
	pr_info("  * 0x%08lx..0x%08lx  : early ioremap\n",
		ioremap_bot, IOREMAP_TOP);
	pr_info("  * 0x%08lx..0x%08lx  : vmalloc & ioremap\n",
		VMALLOC_START, VMALLOC_END);
#endif /* CONFIG_PPC32 */

	mem_init_done = 1;
}

/*
 * This is called when a page has been modified by the kernel.
 * It just marks the page as not i-cache clean.  We do the i-cache
 * flush later when the page is given to a user process, if necessary.
 */
void flush_dcache_page(struct page *page)
{
	if (cpu_has_feature(CPU_FTR_COHERENT_ICACHE))
		return;
	/* avoid an atomic op if possible */
	if (test_bit(PG_arch_1, &page->flags))
		clear_bit(PG_arch_1, &page->flags);
}
EXPORT_SYMBOL(flush_dcache_page);

void flush_dcache_icache_page(struct page *page)
{
#ifdef CONFIG_BOOKE
	void *start = kmap_atomic(page, KM_PPC_SYNC_ICACHE);
	__flush_dcache_icache(start);
	kunmap_atomic(start, KM_PPC_SYNC_ICACHE);
#elif defined(CONFIG_8xx) || defined(CONFIG_PPC64)
	/* On 8xx there is no need to kmap since highmem is not supported */
	__flush_dcache_icache(page_address(page)); 
#else
	__flush_dcache_icache_phys(page_to_pfn(page) << PAGE_SHIFT);
#endif

}
void clear_user_page(void *page, unsigned long vaddr, struct page *pg)
{
	clear_page(page);

	/*
	 * We shouldnt have to do this, but some versions of glibc
	 * require it (ld.so assumes zero filled pages are icache clean)
	 * - Anton
	 */
	flush_dcache_page(pg);
}
EXPORT_SYMBOL(clear_user_page);

void copy_user_page(void *vto, void *vfrom, unsigned long vaddr,
		    struct page *pg)
{
	copy_page(vto, vfrom);

	/*
	 * We should be able to use the following optimisation, however
	 * there are two problems.
	 * Firstly a bug in some versions of binutils meant PLT sections
	 * were not marked executable.
	 * Secondly the first word in the GOT section is blrl, used
	 * to establish the GOT address. Until recently the GOT was
	 * not marked executable.
	 * - Anton
	 */
#if 0
	if (!vma->vm_file && ((vma->vm_flags & VM_EXEC) == 0))
		return;
#endif

	flush_dcache_page(pg);
}

void flush_icache_user_range(struct vm_area_struct *vma, struct page *page,
			     unsigned long addr, int len)
{
	unsigned long maddr;

	maddr = (unsigned long) kmap(page) + (addr & ~PAGE_MASK);
	flush_icache_range(maddr, maddr + len);
	kunmap(page);
}
EXPORT_SYMBOL(flush_icache_user_range);

/*
 * This is called at the end of handling a user page fault, when the
 * fault has been handled by updating a PTE in the linux page tables.
 * We use it to preload an HPTE into the hash table corresponding to
 * the updated linux PTE.
 * 
 * This must always be called with the pte lock held.
 */
void update_mmu_cache(struct vm_area_struct *vma, unsigned long address,
		      pte_t pte)
{
#ifdef CONFIG_PPC_STD_MMU
	unsigned long access = 0, trap;

	/* We only want HPTEs for linux PTEs that have _PAGE_ACCESSED set */
	if (!pte_young(pte) || address >= TASK_SIZE)
		return;

	/* We try to figure out if we are coming from an instruction
	 * access fault and pass that down to __hash_page so we avoid
	 * double-faulting on execution of fresh text. We have to test
	 * for regs NULL since init will get here first thing at boot
	 *
	 * We also avoid filling the hash if not coming from a fault
	 */
	if (current->thread.regs == NULL)
		return;
	trap = TRAP(current->thread.regs);
	if (trap == 0x400)
		access |= _PAGE_EXEC;
	else if (trap != 0x300)
		return;
	hash_preload(vma->vm_mm, address, access, trap);
#endif /* CONFIG_PPC_STD_MMU */
}
Commit	Line	Data
14cf11af PM	1	/*
	2	* PowerPC version
	3	* Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
	4	*
	5	* Modifications by Paul Mackerras (PowerMac) (paulus@cs.anu.edu.au)
	6	* and Cort Dougan (PReP) (cort@cs.nmt.edu)
	7	* Copyright (C) 1996 Paul Mackerras
14cf11af PM	8	* PPC44x/36-bit changes by Matt Porter (mporter@mvista.com)
	9	*
	10	* Derived from "arch/i386/mm/init.c"
	11	* Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
	12	*
	13	* This program is free software; you can redistribute it and/or
	14	* modify it under the terms of the GNU General Public License
	15	* as published by the Free Software Foundation; either version
	16	* 2 of the License, or (at your option) any later version.
	17	*
	18	*/
	19
14cf11af PM	20	#include <linux/module.h>
	21	#include <linux/sched.h>
	22	#include <linux/kernel.h>
	23	#include <linux/errno.h>
	24	#include <linux/string.h>
	25	#include <linux/types.h>
	26	#include <linux/mm.h>
	27	#include <linux/stddef.h>
	28	#include <linux/init.h>
	29	#include <linux/bootmem.h>
	30	#include <linux/highmem.h>
	31	#include <linux/initrd.h>
	32	#include <linux/pagemap.h>
4e8ad3e8	33	#include <linux/suspend.h>
d9b2b2a2	34	#include <linux/lmb.h>
14cf11af PM	35
	36	#include <asm/pgalloc.h>
	37	#include <asm/prom.h>
	38	#include <asm/io.h>
	39	#include <asm/mmu_context.h>
	40	#include <asm/pgtable.h>
	41	#include <asm/mmu.h>
	42	#include <asm/smp.h>
	43	#include <asm/machdep.h>
	44	#include <asm/btext.h>
	45	#include <asm/tlb.h>
7c8c6b97	46	#include <asm/sections.h>
db7f37de	47	#include <asm/sparsemem.h>
ab1f9dac	48	#include <asm/vdso.h>
2c419bde	49	#include <asm/fixmap.h>
14cf11af	50
14cf11af PM	51	#include "mmu_decl.h"
	52
	53	#ifndef CPU_FTR_COHERENT_ICACHE
	54	#define CPU_FTR_COHERENT_ICACHE 0 /* XXX for now */
	55	#define CPU_FTR_NOEXECUTE 0
	56	#endif
	57
7c8c6b97 PM	58	int init_bootmem_done;
7c8c6b97 PM	59	int mem_init_done;
49a84965	60	phys_addr_t memory_limit;
7c8c6b97	61
2c419bde KG	62	#ifdef CONFIG_HIGHMEM
	63	pte_t *kmap_pte;
	64	pgprot_t kmap_prot;
	65
	66	EXPORT_SYMBOL(kmap_prot);
	67	EXPORT_SYMBOL(kmap_pte);
	68
	69	static inline pte_t *virt_to_kpte(unsigned long vaddr)
	70	{
	71	return pte_offset_kernel(pmd_offset(pud_offset(pgd_offset_k(vaddr),
	72	vaddr), vaddr), vaddr);
	73	}
	74	#endif
	75
14cf11af PM	76	int page_is_ram(unsigned long pfn)
14cf11af PM	77	{
14cf11af	78	#ifndef CONFIG_PPC64 /* XXX for now */
a880e762	79	return pfn < max_pfn;
14cf11af	80	#else
a880e762	81	unsigned long paddr = (pfn << PAGE_SHIFT);
14cf11af PM	82	int i;
	83	for (i=0; i < lmb.memory.cnt; i++) {
	84	unsigned long base;
	85
	86	base = lmb.memory.region[i].base;
	87
	88	if ((paddr >= base) &&
	89	(paddr < (base + lmb.memory.region[i].size))) {
	90	return 1;
	91	}
	92	}
	93
	94	return 0;
	95	#endif
	96	}
14cf11af	97
8b150478	98	pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
14cf11af PM	99	unsigned long size, pgprot_t vma_prot)
	100	{
	101	if (ppc_md.phys_mem_access_prot)
8b150478	102	return ppc_md.phys_mem_access_prot(file, pfn, size, vma_prot);
14cf11af	103
8b150478	104	if (!page_is_ram(pfn))
64b3d0e8 BH	105	vma_prot = pgprot_noncached(vma_prot);
64b3d0e8 BH	106
14cf11af PM	107	return vma_prot;
	108	}
	109	EXPORT_SYMBOL(phys_mem_access_prot);
	110
23fd0775 PM	111	#ifdef CONFIG_MEMORY_HOTPLUG
23fd0775 PM	112
bc02af93 YG	113	#ifdef CONFIG_NUMA
	114	int memory_add_physaddr_to_nid(u64 start)
	115	{
	116	return hot_add_scn_to_nid(start);
	117	}
	118	#endif
	119
fa90f70a	120	int arch_add_memory(int nid, u64 start, u64 size)
23fd0775	121	{
237a0989	122	struct pglist_data *pgdata;
23fd0775 PM	123	struct zone *zone;
	124	unsigned long start_pfn = start >> PAGE_SHIFT;
	125	unsigned long nr_pages = size >> PAGE_SHIFT;
	126
237a0989 MK	127	pgdata = NODE_DATA(nid);
237a0989 MK	128
2d0eee14	129	start = (unsigned long)__va(start);
54b79248 MK	130	create_section_mapping(start, start + size);
54b79248 MK	131
23fd0775 PM	132	/* this should work for most non-highmem platforms */
	133	zone = pgdata->node_zones;
	134
c04fc586	135	return __add_pages(nid, zone, start_pfn, nr_pages);
23fd0775	136	}
0d579944	137	#endif /* CONFIG_MEMORY_HOTPLUG */
a99824f3 BP	138
	139	/*
	140	* walk_memory_resource() needs to make sure there is no holes in a given
9d88a2eb BP	141	* memory range. PPC64 does not maintain the memory layout in /proc/iomem.
	142	* Instead it maintains it in lmb.memory structures. Walk through the
	143	* memory regions, find holes and callback for contiguous regions.
a99824f3 BP	144	*/
	145	int
	146	walk_memory_resource(unsigned long start_pfn, unsigned long nr_pages, void *arg,
	147	int (func)(unsigned long, unsigned long, void ))
	148	{
9d88a2eb BP	149	struct lmb_property res;
	150	unsigned long pfn, len;
	151	u64 end;
	152	int ret = -1;
	153
	154	res.base = (u64) start_pfn << PAGE_SHIFT;
	155	res.size = (u64) nr_pages << PAGE_SHIFT;
	156
	157	end = res.base + res.size - 1;
	158	while ((res.base < end) && (lmb_find(&res) >= 0)) {
	159	pfn = (unsigned long)(res.base >> PAGE_SHIFT);
	160	len = (unsigned long)(res.size >> PAGE_SHIFT);
	161	ret = (*func)(pfn, len, arg);
	162	if (ret)
	163	break;
	164	res.base += (res.size + 1);
	165	res.size = (end - res.base + 1);
	166	}
	167	return ret;
a99824f3	168	}
9d88a2eb	169	EXPORT_SYMBOL_GPL(walk_memory_resource);
a99824f3	170
7c8c6b97 PM	171	/*
	172	* Initialize the bootmem system and give it all the memory we
	173	* have available. If we are using highmem, we only put the
	174	* lowmem into the bootmem system.
	175	*/
	176	#ifndef CONFIG_NEED_MULTIPLE_NODES
	177	void __init do_init_bootmem(void)
	178	{
	179	unsigned long i;
	180	unsigned long start, bootmap_pages;
	181	unsigned long total_pages;
	182	int boot_mapsize;
	183
37dd2bad	184	max_low_pfn = max_pfn = lmb_end_of_DRAM() >> PAGE_SHIFT;
d7917ba7	185	total_pages = (lmb_end_of_DRAM() - memstart_addr) >> PAGE_SHIFT;
7c8c6b97 PM	186	#ifdef CONFIG_HIGHMEM
7c8c6b97 PM	187	total_pages = total_lowmem >> PAGE_SHIFT;
d7917ba7	188	max_low_pfn = lowmem_end_addr >> PAGE_SHIFT;
7c8c6b97 PM	189	#endif
	190
	191	/*
	192	* Find an area to use for the bootmem bitmap. Calculate the size of
	193	* bitmap required as (Total Memory) / PAGE_SIZE / BITS_PER_BYTE.
	194	* Add 1 additional page in case the address isn't page-aligned.
	195	*/
	196	bootmap_pages = bootmem_bootmap_pages(total_pages);
	197
	198	start = lmb_alloc(bootmap_pages << PAGE_SHIFT, PAGE_SIZE);
7c8c6b97	199
37dd2bad KG	200	min_low_pfn = MEMORY_START >> PAGE_SHIFT;
37dd2bad KG	201	boot_mapsize = init_bootmem_node(NODE_DATA(0), start >> PAGE_SHIFT, min_low_pfn, max_low_pfn);
7c8c6b97	202
c67c3cb4 MG	203	/* Add active regions with valid PFNs */
	204	for (i = 0; i < lmb.memory.cnt; i++) {
	205	unsigned long start_pfn, end_pfn;
	206	start_pfn = lmb.memory.region[i].base >> PAGE_SHIFT;
	207	end_pfn = start_pfn + lmb_size_pages(&lmb.memory, i);
	208	add_active_range(0, start_pfn, end_pfn);
	209	}
	210
7c8c6b97 PM	211	/* Add all physical memory to the bootmem map, mark each area
	212	* present.
	213	*/
7c8c6b97	214	#ifdef CONFIG_HIGHMEM
d7917ba7	215	free_bootmem_with_active_regions(0, lowmem_end_addr >> PAGE_SHIFT);
f98eeb4e KG	216
	217	/* reserve the sections we're already using */
	218	for (i = 0; i < lmb.reserved.cnt; i++) {
	219	unsigned long addr = lmb.reserved.region[i].base +
	220	lmb_size_bytes(&lmb.reserved, i) - 1;
d7917ba7	221	if (addr < lowmem_end_addr)
f98eeb4e	222	reserve_bootmem(lmb.reserved.region[i].base,
72a7fe39 BW	223	lmb_size_bytes(&lmb.reserved, i),
72a7fe39 BW	224	BOOTMEM_DEFAULT);
d7917ba7 KG	225	else if (lmb.reserved.region[i].base < lowmem_end_addr) {
d7917ba7 KG	226	unsigned long adjusted_size = lowmem_end_addr -
f98eeb4e KG	227	lmb.reserved.region[i].base;
f98eeb4e KG	228	reserve_bootmem(lmb.reserved.region[i].base,
72a7fe39	229	adjusted_size, BOOTMEM_DEFAULT);
f98eeb4e KG	230	}
f98eeb4e KG	231	}
c67c3cb4 MG	232	#else
c67c3cb4 MG	233	free_bootmem_with_active_regions(0, max_pfn);
7c8c6b97 PM	234
	235	/* reserve the sections we're already using */
	236	for (i = 0; i < lmb.reserved.cnt; i++)
	237	reserve_bootmem(lmb.reserved.region[i].base,
72a7fe39 BW	238	lmb_size_bytes(&lmb.reserved, i),
72a7fe39 BW	239	BOOTMEM_DEFAULT);
7c8c6b97	240
f98eeb4e	241	#endif
7c8c6b97	242	/* XXX need to clip this if using highmem? */
c67c3cb4 MG	243	sparse_memory_present_with_active_regions(0);
c67c3cb4 MG	244
7c8c6b97 PM	245	init_bootmem_done = 1;
	246	}
	247
4e8ad3e8 JB	248	/* mark pages that don't exist as nosave */
	249	static int __init mark_nonram_nosave(void)
	250	{
	251	unsigned long lmb_next_region_start_pfn,
	252	lmb_region_max_pfn;
	253	int i;
	254
	255	for (i = 0; i < lmb.memory.cnt - 1; i++) {
	256	lmb_region_max_pfn =
	257	(lmb.memory.region[i].base >> PAGE_SHIFT) +
	258	(lmb.memory.region[i].size >> PAGE_SHIFT);
	259	lmb_next_region_start_pfn =
	260	lmb.memory.region[i+1].base >> PAGE_SHIFT;
	261
	262	if (lmb_region_max_pfn < lmb_next_region_start_pfn)
	263	register_nosave_region(lmb_region_max_pfn,
	264	lmb_next_region_start_pfn);
	265	}
	266
	267	return 0;
	268	}
	269
7c8c6b97 PM	270	/*
	271	* paging_init() sets up the page tables - in fact we've already done this.
	272	*/
	273	void __init paging_init(void)
	274	{
7c8c6b97	275	unsigned long total_ram = lmb_phys_mem_size();
2bf3016f	276	phys_addr_t top_of_ram = lmb_end_of_DRAM();
c67c3cb4	277	unsigned long max_zone_pfns[MAX_NR_ZONES];
7c8c6b97	278
2c419bde KG	279	#ifdef CONFIG_PPC32
	280	unsigned long v = __fix_to_virt(__end_of_fixed_addresses - 1);
	281	unsigned long end = __fix_to_virt(FIX_HOLE);
	282
	283	for (; v < end; v += PAGE_SIZE)
	284	map_page(v, 0, 0); /* XXX gross */
	285	#endif
	286
7c8c6b97 PM	287	#ifdef CONFIG_HIGHMEM
7c8c6b97 PM	288	map_page(PKMAP_BASE, 0, 0); /* XXX gross */
2c419bde KG	289	pkmap_page_table = virt_to_kpte(PKMAP_BASE);
	290
	291	kmap_pte = virt_to_kpte(__fix_to_virt(FIX_KMAP_BEGIN));
7c8c6b97 PM	292	kmap_prot = PAGE_KERNEL;
	293	#endif /* CONFIG_HIGHMEM */
	294
2bf3016f	295	printk(KERN_DEBUG "Top of RAM: 0x%llx, Total RAM: 0x%lx\n",
fb610635	296	(unsigned long long)top_of_ram, total_ram);
e110b281	297	printk(KERN_DEBUG "Memory hole size: %ldMB\n",
2bf3016f	298	(long int)((top_of_ram - total_ram) >> 20));
6391af17	299	memset(max_zone_pfns, 0, sizeof(max_zone_pfns));
7c8c6b97	300	#ifdef CONFIG_HIGHMEM
d7917ba7	301	max_zone_pfns[ZONE_DMA] = lowmem_end_addr >> PAGE_SHIFT;
6391af17	302	max_zone_pfns[ZONE_HIGHMEM] = top_of_ram >> PAGE_SHIFT;
7c8c6b97	303	#else
6391af17	304	max_zone_pfns[ZONE_DMA] = top_of_ram >> PAGE_SHIFT;
c67c3cb4 MG	305	#endif
c67c3cb4 MG	306	free_area_init_nodes(max_zone_pfns);
4e8ad3e8 JB	307
4e8ad3e8 JB	308	mark_nonram_nosave();
7c8c6b97 PM	309	}
	310	#endif /* ! CONFIG_NEED_MULTIPLE_NODES */
	311
	312	void __init mem_init(void)
	313	{
	314	#ifdef CONFIG_NEED_MULTIPLE_NODES
	315	int nid;
	316	#endif
	317	pg_data_t *pgdat;
	318	unsigned long i;
	319	struct page *page;
	320	unsigned long reservedpages = 0, codesize, initsize, datasize, bsssize;
	321
fb6d73d3	322	num_physpages = lmb.memory.size >> PAGE_SHIFT;
7c8c6b97 PM	323	high_memory = (void ) __va(max_low_pfn PAGE_SIZE);
	324
	325	#ifdef CONFIG_NEED_MULTIPLE_NODES
	326	for_each_online_node(nid) {
	327	if (NODE_DATA(nid)->node_spanned_pages != 0) {
c258dd40	328	printk("freeing bootmem node %d\n", nid);
7c8c6b97 PM	329	totalram_pages +=
	330	free_all_bootmem_node(NODE_DATA(nid));
	331	}
	332	}
	333	#else
fb6d73d3	334	max_mapnr = max_pfn;
7c8c6b97 PM	335	totalram_pages += free_all_bootmem();
7c8c6b97 PM	336	#endif
ec936fc5	337	for_each_online_pgdat(pgdat) {
7c8c6b97	338	for (i = 0; i < pgdat->node_spanned_pages; i++) {
fb6d73d3 PM	339	if (!pfn_valid(pgdat->node_start_pfn + i))
fb6d73d3 PM	340	continue;
7c8c6b97 PM	341	page = pgdat_page_nr(pgdat, i);
	342	if (PageReserved(page))
	343	reservedpages++;
	344	}
	345	}
	346
	347	codesize = (unsigned long)&_sdata - (unsigned long)&_stext;
bcb35576	348	datasize = (unsigned long)&_edata - (unsigned long)&_sdata;
7c8c6b97 PM	349	initsize = (unsigned long)&__init_end - (unsigned long)&__init_begin;
	350	bsssize = (unsigned long)&__bss_stop - (unsigned long)&__bss_start;
	351
	352	#ifdef CONFIG_HIGHMEM
	353	{
	354	unsigned long pfn, highmem_mapnr;
	355
d7917ba7	356	highmem_mapnr = lowmem_end_addr >> PAGE_SHIFT;
7c8c6b97 PM	357	for (pfn = highmem_mapnr; pfn < max_mapnr; ++pfn) {
7c8c6b97 PM	358	struct page *page = pfn_to_page(pfn);
f98eeb4e KG	359	if (lmb_is_reserved(pfn << PAGE_SHIFT))
f98eeb4e KG	360	continue;
7c8c6b97	361	ClearPageReserved(page);
7835e98b	362	init_page_count(page);
7c8c6b97 PM	363	__free_page(page);
7c8c6b97 PM	364	totalhigh_pages++;
f98eeb4e	365	reservedpages--;
7c8c6b97 PM	366	}
7c8c6b97 PM	367	totalram_pages += totalhigh_pages;
e110b281	368	printk(KERN_DEBUG "High memory: %luk\n",
7c8c6b97 PM	369	totalhigh_pages << (PAGE_SHIFT-10));
	370	}
	371	#endif /* CONFIG_HIGHMEM */
	372
	373	printk(KERN_INFO "Memory: %luk/%luk available (%luk kernel code, "
	374	"%luk reserved, %luk data, %luk bss, %luk init)\n",
cc013a88	375	nr_free_pages() << (PAGE_SHIFT-10),
7c8c6b97 PM	376	num_physpages << (PAGE_SHIFT-10),
	377	codesize >> 10,
	378	reservedpages << (PAGE_SHIFT-10),
	379	datasize >> 10,
	380	bsssize >> 10,
	381	initsize >> 10);
	382
f637a49e BH	383	#ifdef CONFIG_PPC32
	384	pr_info("Kernel virtual memory layout:\n");
	385	pr_info(" * 0x%08lx..0x%08lx : fixmap\n", FIXADDR_START, FIXADDR_TOP);
	386	#ifdef CONFIG_HIGHMEM
	387	pr_info(" * 0x%08lx..0x%08lx : highmem PTEs\n",
	388	PKMAP_BASE, PKMAP_ADDR(LAST_PKMAP));
	389	#endif /* CONFIG_HIGHMEM */
8b31e49d BH	390	#ifdef CONFIG_NOT_COHERENT_CACHE
	391	pr_info(" * 0x%08lx..0x%08lx : consistent mem\n",
	392	IOREMAP_TOP, IOREMAP_TOP + CONFIG_CONSISTENT_SIZE);
	393	#endif /* CONFIG_NOT_COHERENT_CACHE */
f637a49e BH	394	pr_info(" * 0x%08lx..0x%08lx : early ioremap\n",
	395	ioremap_bot, IOREMAP_TOP);
	396	pr_info(" * 0x%08lx..0x%08lx : vmalloc & ioremap\n",
	397	VMALLOC_START, VMALLOC_END);
	398	#endif /* CONFIG_PPC32 */
	399
7c8c6b97	400	mem_init_done = 1;
7c8c6b97 PM	401	}
7c8c6b97 PM	402
14cf11af PM	403	/*
	404	* This is called when a page has been modified by the kernel.
	405	* It just marks the page as not i-cache clean. We do the i-cache
	406	* flush later when the page is given to a user process, if necessary.
	407	*/
	408	void flush_dcache_page(struct page *page)
	409	{
	410	if (cpu_has_feature(CPU_FTR_COHERENT_ICACHE))
	411	return;
	412	/* avoid an atomic op if possible */
	413	if (test_bit(PG_arch_1, &page->flags))
	414	clear_bit(PG_arch_1, &page->flags);
	415	}
	416	EXPORT_SYMBOL(flush_dcache_page);
	417
	418	void flush_dcache_icache_page(struct page *page)
	419	{
	420	#ifdef CONFIG_BOOKE
	421	void *start = kmap_atomic(page, KM_PPC_SYNC_ICACHE);
	422	__flush_dcache_icache(start);
	423	kunmap_atomic(start, KM_PPC_SYNC_ICACHE);
ab1f9dac	424	#elif defined(CONFIG_8xx) \|\| defined(CONFIG_PPC64)
14cf11af PM	425	/* On 8xx there is no need to kmap since highmem is not supported */
	426	__flush_dcache_icache(page_address(page));
	427	#else
	428	__flush_dcache_icache_phys(page_to_pfn(page) << PAGE_SHIFT);
	429	#endif
	430
	431	}
	432	void clear_user_page(void page, unsigned long vaddr, struct page pg)
	433	{
	434	clear_page(page);
	435
14cf11af PM	436	/*
	437	* We shouldnt have to do this, but some versions of glibc
	438	* require it (ld.so assumes zero filled pages are icache clean)
	439	* - Anton
	440	*/
09f5dc44	441	flush_dcache_page(pg);
14cf11af PM	442	}
	443	EXPORT_SYMBOL(clear_user_page);
	444
	445	void copy_user_page(void vto, void vfrom, unsigned long vaddr,
	446	struct page *pg)
	447	{
	448	copy_page(vto, vfrom);
	449
	450	/*
	451	* We should be able to use the following optimisation, however
	452	* there are two problems.
	453	* Firstly a bug in some versions of binutils meant PLT sections
	454	* were not marked executable.
	455	* Secondly the first word in the GOT section is blrl, used
	456	* to establish the GOT address. Until recently the GOT was
	457	* not marked executable.
	458	* - Anton
	459	*/
	460	#if 0
	461	if (!vma->vm_file && ((vma->vm_flags & VM_EXEC) == 0))
	462	return;
	463	#endif
	464
09f5dc44	465	flush_dcache_page(pg);
14cf11af PM	466	}
	467
	468	void flush_icache_user_range(struct vm_area_struct vma, struct page page,
	469	unsigned long addr, int len)
	470	{
	471	unsigned long maddr;
	472
	473	maddr = (unsigned long) kmap(page) + (addr & ~PAGE_MASK);
	474	flush_icache_range(maddr, maddr + len);
	475	kunmap(page);
	476	}
	477	EXPORT_SYMBOL(flush_icache_user_range);
	478
	479	/*
	480	* This is called at the end of handling a user page fault, when the
	481	* fault has been handled by updating a PTE in the linux page tables.
	482	* We use it to preload an HPTE into the hash table corresponding to
	483	* the updated linux PTE.
	484	*
01edcd89	485	* This must always be called with the pte lock held.
14cf11af PM	486	*/
	487	void update_mmu_cache(struct vm_area_struct *vma, unsigned long address,
	488	pte_t pte)
	489	{
3c726f8d BH	490	#ifdef CONFIG_PPC_STD_MMU
3c726f8d BH	491	unsigned long access = 0, trap;
14cf11af	492
14cf11af PM	493	/* We only want HPTEs for linux PTEs that have _PAGE_ACCESSED set */
	494	if (!pte_young(pte) \|\| address >= TASK_SIZE)
	495	return;
14cf11af	496
3c726f8d BH	497	/* We try to figure out if we are coming from an instruction
	498	* access fault and pass that down to __hash_page so we avoid
	499	* double-faulting on execution of fresh text. We have to test
	500	* for regs NULL since init will get here first thing at boot
	501	*
	502	* We also avoid filling the hash if not coming from a fault
	503	*/
	504	if (current->thread.regs == NULL)
14cf11af	505	return;
3c726f8d BH	506	trap = TRAP(current->thread.regs);
	507	if (trap == 0x400)
	508	access \|= _PAGE_EXEC;
	509	else if (trap != 0x300)
	510	return;
	511	hash_preload(vma->vm_mm, address, access, trap);
	512	#endif /* CONFIG_PPC_STD_MMU */
14cf11af	513	}