[net-next-2.6.git] / arch / x86 / mm / memblock.c

#include <linux/kernel.h>
#include <linux/types.h>
#include <linux/init.h>
#include <linux/bitops.h>
#include <linux/memblock.h>
#include <linux/bootmem.h>
#include <linux/mm.h>
#include <linux/range.h>

/* Check for already reserved areas */
static inline bool __init bad_addr_size(u64 *addrp, u64 *sizep, u64 align)
{
	struct memblock_region *r;
	u64 addr = *addrp, last;
	u64 size = *sizep;
	bool changed = false;

again:
	last = addr + size;
	for_each_memblock(reserved, r) {
		if (last > r->base && addr < r->base) {
			size = r->base - addr;
			changed = true;
			goto again;
		}
		if (last > (r->base + r->size) && addr < (r->base + r->size)) {
			addr = round_up(r->base + r->size, align);
			size = last - addr;
			changed = true;
			goto again;
		}
		if (last <= (r->base + r->size) && addr >= r->base) {
			(*sizep)++;
			return false;
		}
	}
	if (changed) {
		*addrp = addr;
		*sizep = size;
	}
	return changed;
}

static u64 __init __memblock_x86_find_in_range_size(u64 ei_start, u64 ei_last, u64 start,
			 u64 *sizep, u64 align)
{
	u64 addr, last;

	addr = round_up(ei_start, align);
	if (addr < start)
		addr = round_up(start, align);
	if (addr >= ei_last)
		goto out;
	*sizep = ei_last - addr;
	while (bad_addr_size(&addr, sizep, align) && addr + *sizep <= ei_last)
		;
	last = addr + *sizep;
	if (last > ei_last)
		goto out;

	return addr;

out:
	return MEMBLOCK_ERROR;
}

/*
 * Find next free range after start, and size is returned in *sizep
 */
u64 __init memblock_x86_find_in_range_size(u64 start, u64 *sizep, u64 align)
{
	struct memblock_region *r;

	for_each_memblock(memory, r) {
		u64 ei_start = r->base;
		u64 ei_last = ei_start + r->size;
		u64 addr;

		addr = __memblock_x86_find_in_range_size(ei_start, ei_last, start,
					 sizep, align);

		if (addr != MEMBLOCK_ERROR)
			return addr;
	}

	return MEMBLOCK_ERROR;
}

static __init struct range *find_range_array(int count)
{
	u64 end, size, mem;
	struct range *range;

	size = sizeof(struct range) * count;
	end = memblock.current_limit;

	mem = memblock_find_in_range(0, end, size, sizeof(struct range));
	if (mem == MEMBLOCK_ERROR)
		panic("can not find more space for range array");

	/*
	 * This range is tempoaray, so don't reserve it, it will not be
	 * overlapped because We will not alloccate new buffer before
	 * We discard this one
	 */
	range = __va(mem);
	memset(range, 0, size);

	return range;
}

#ifdef CONFIG_NO_BOOTMEM
static void __init memblock_x86_subtract_reserved(struct range *range, int az)
{
	u64 final_start, final_end;
	struct memblock_region *r;

	/* Take out region array itself at first*/
	memblock_free_reserved_regions();

	pr_info("Subtract (%ld early reservations)\n", memblock.reserved.cnt);

	for_each_memblock(reserved, r) {
		pr_info("  [%010llx-%010llx]\n", (u64)r->base, (u64)r->base + r->size - 1);
		final_start = PFN_DOWN(r->base);
		final_end = PFN_UP(r->base + r->size);
		if (final_start >= final_end)
			continue;
		subtract_range(range, az, final_start, final_end);
	}

	/* Put region array back ? */
	memblock_reserve_reserved_regions();
}

struct count_data {
	int nr;
};

static int __init count_work_fn(unsigned long start_pfn,
				unsigned long end_pfn, void *datax)
{
	struct count_data *data = datax;

	data->nr++;

	return 0;
}

static int __init count_early_node_map(int nodeid)
{
	struct count_data data;

	data.nr = 0;
	work_with_active_regions(nodeid, count_work_fn, &data);

	return data.nr;
}

int __init get_free_all_memory_range(struct range **rangep, int nodeid)
{
	int count;
	struct range *range;
	int nr_range;

	count = (memblock.reserved.cnt + count_early_node_map(nodeid)) * 2;

	range = find_range_array(count);
	nr_range = 0;

	/*
	 * Use early_node_map[] and memblock.reserved.region to get range array
	 * at first
	 */
	nr_range = add_from_early_node_map(range, count, nr_range, nodeid);
#ifdef CONFIG_X86_32
	subtract_range(range, count, max_low_pfn, -1ULL);
#endif
	memblock_x86_subtract_reserved(range, count);
	nr_range = clean_sort_range(range, count);

	*rangep = range;
	return nr_range;
}
#else
void __init memblock_x86_to_bootmem(u64 start, u64 end)
{
	int count;
	u64 final_start, final_end;
	struct memblock_region *r;

	/* Take out region array itself */
	memblock_free_reserved_regions();

	count  = memblock.reserved.cnt;
	pr_info("(%d early reservations) ==> bootmem [%010llx-%010llx]\n", count, start, end - 1);
	for_each_memblock(reserved, r) {
		pr_info("  [%010llx-%010llx] ", (u64)r->base, (u64)r->base + r->size - 1);
		final_start = max(start, r->base);
		final_end = min(end, r->base + r->size);
		if (final_start >= final_end) {
			pr_cont("\n");
			continue;
		}
		pr_cont(" ==> [%010llx-%010llx]\n", final_start, final_end - 1);
		reserve_bootmem_generic(final_start, final_end - final_start, BOOTMEM_DEFAULT);
	}

	/* Put region array back ? */
	memblock_reserve_reserved_regions();
}
#endif

void __init memblock_x86_reserve_range(u64 start, u64 end, char *name)
{
	if (start == end)
		return;

	if (WARN_ONCE(start > end, "memblock_x86_reserve_range: wrong range [%#llx, %#llx]\n", start, end))
		return;

	memblock_reserve(start, end - start);
}

void __init memblock_x86_free_range(u64 start, u64 end)
{
	if (start == end)
		return;

	if (WARN_ONCE(start > end, "memblock_x86_free_range: wrong range [%#llx, %#llx]\n", start, end))
		return;

	memblock_free(start, end - start);
}

/*
 * Need to call this function after memblock_x86_register_active_regions,
 * so early_node_map[] is filled already.
 */
u64 __init memblock_x86_find_in_range_node(int nid, u64 start, u64 end, u64 size, u64 align)
{
	u64 addr;
	addr = find_memory_core_early(nid, size, align, start, end);
	if (addr != MEMBLOCK_ERROR)
		return addr;

	/* Fallback, should already have start end within node range */
	return memblock_find_in_range(start, end, size, align);
}

/*
 * Finds an active region in the address range from start_pfn to last_pfn and
 * returns its range in ei_startpfn and ei_endpfn for the memblock entry.
 */
static int __init memblock_x86_find_active_region(const struct memblock_region *ei,
				  unsigned long start_pfn,
				  unsigned long last_pfn,
				  unsigned long *ei_startpfn,
				  unsigned long *ei_endpfn)
{
	u64 align = PAGE_SIZE;

	*ei_startpfn = round_up(ei->base, align) >> PAGE_SHIFT;
	*ei_endpfn = round_down(ei->base + ei->size, align) >> PAGE_SHIFT;

	/* Skip map entries smaller than a page */
	if (*ei_startpfn >= *ei_endpfn)
		return 0;

	/* Skip if map is outside the node */
	if (*ei_endpfn <= start_pfn || *ei_startpfn >= last_pfn)
		return 0;

	/* Check for overlaps */
	if (*ei_startpfn < start_pfn)
		*ei_startpfn = start_pfn;
	if (*ei_endpfn > last_pfn)
		*ei_endpfn = last_pfn;

	return 1;
}

/* Walk the memblock.memory map and register active regions within a node */
void __init memblock_x86_register_active_regions(int nid, unsigned long start_pfn,
					 unsigned long last_pfn)
{
	unsigned long ei_startpfn;
	unsigned long ei_endpfn;
	struct memblock_region *r;

	for_each_memblock(memory, r)
		if (memblock_x86_find_active_region(r, start_pfn, last_pfn,
					   &ei_startpfn, &ei_endpfn))
			add_active_range(nid, ei_startpfn, ei_endpfn);
}

/*
 * Find the hole size (in bytes) in the memory range.
 * @start: starting address of the memory range to scan
 * @end: ending address of the memory range to scan
 */
u64 __init memblock_x86_hole_size(u64 start, u64 end)
{
	unsigned long start_pfn = start >> PAGE_SHIFT;
	unsigned long last_pfn = end >> PAGE_SHIFT;
	unsigned long ei_startpfn, ei_endpfn, ram = 0;
	struct memblock_region *r;

	for_each_memblock(memory, r)
		if (memblock_x86_find_active_region(r, start_pfn, last_pfn,
					   &ei_startpfn, &ei_endpfn))
			ram += ei_endpfn - ei_startpfn;

	return end - start - ((u64)ram << PAGE_SHIFT);
}
Commit	Line	Data
fb74fb6d YL	1	#include <linux/kernel.h>
	2	#include <linux/types.h>
	3	#include <linux/init.h>
	4	#include <linux/bitops.h>
	5	#include <linux/memblock.h>
	6	#include <linux/bootmem.h>
	7	#include <linux/mm.h>
	8	#include <linux/range.h>
	9
	10	/* Check for already reserved areas */
	11	static inline bool __init bad_addr_size(u64 addrp, u64 sizep, u64 align)
	12	{
	13	struct memblock_region *r;
	14	u64 addr = *addrp, last;
	15	u64 size = *sizep;
	16	bool changed = false;
	17
	18	again:
	19	last = addr + size;
	20	for_each_memblock(reserved, r) {
	21	if (last > r->base && addr < r->base) {
	22	size = r->base - addr;
	23	changed = true;
	24	goto again;
	25	}
	26	if (last > (r->base + r->size) && addr < (r->base + r->size)) {
	27	addr = round_up(r->base + r->size, align);
	28	size = last - addr;
	29	changed = true;
	30	goto again;
	31	}
	32	if (last <= (r->base + r->size) && addr >= r->base) {
	33	(*sizep)++;
	34	return false;
	35	}
	36	}
	37	if (changed) {
	38	*addrp = addr;
	39	*sizep = size;
	40	}
	41	return changed;
	42	}
	43
	44	static u64 __init __memblock_x86_find_in_range_size(u64 ei_start, u64 ei_last, u64 start,
	45	u64 *sizep, u64 align)
	46	{
	47	u64 addr, last;
	48
	49	addr = round_up(ei_start, align);
	50	if (addr < start)
	51	addr = round_up(start, align);
	52	if (addr >= ei_last)
	53	goto out;
	54	*sizep = ei_last - addr;
	55	while (bad_addr_size(&addr, sizep, align) && addr + *sizep <= ei_last)
	56	;
	57	last = addr + *sizep;
	58	if (last > ei_last)
	59	goto out;
	60
	61	return addr;
	62
	63	out:
	64	return MEMBLOCK_ERROR;
65	}
66
67	/*
68	* Find next free range after start, and size is returned in *sizep
69	*/
70	u64 __init memblock_x86_find_in_range_size(u64 start, u64 *sizep, u64 align)
71	{
72	struct memblock_region *r;
73
74	for_each_memblock(memory, r) {
75	u64 ei_start = r->base;
76	u64 ei_last = ei_start + r->size;
77	u64 addr;
78
79	addr = __memblock_x86_find_in_range_size(ei_start, ei_last, start,
80	sizep, align);
81
82	if (addr != MEMBLOCK_ERROR)
83	return addr;
84	}
85
86	return MEMBLOCK_ERROR;
87	}
27de7943	88
4d5cf86c YL	89	static __init struct range *find_range_array(int count)
	90	{
	91	u64 end, size, mem;
	92	struct range *range;
	93
	94	size = sizeof(struct range) * count;
	95	end = memblock.current_limit;
	96
	97	mem = memblock_find_in_range(0, end, size, sizeof(struct range));
	98	if (mem == MEMBLOCK_ERROR)
	99	panic("can not find more space for range array");
	100
	101	/*
	102	* This range is tempoaray, so don't reserve it, it will not be
	103	* overlapped because We will not alloccate new buffer before
	104	* We discard this one
	105	*/
	106	range = __va(mem);
	107	memset(range, 0, size);
	108
	109	return range;
	110	}
	111
	112	#ifdef CONFIG_NO_BOOTMEM
	113	static void __init memblock_x86_subtract_reserved(struct range *range, int az)
	114	{
	115	u64 final_start, final_end;
	116	struct memblock_region *r;
	117
	118	/* Take out region array itself at first*/
	119	memblock_free_reserved_regions();
	120
	121	pr_info("Subtract (%ld early reservations)\n", memblock.reserved.cnt);
	122
	123	for_each_memblock(reserved, r) {
	124	pr_info(" [%010llx-%010llx]\n", (u64)r->base, (u64)r->base + r->size - 1);
	125	final_start = PFN_DOWN(r->base);
	126	final_end = PFN_UP(r->base + r->size);
	127	if (final_start >= final_end)
	128	continue;
	129	subtract_range(range, az, final_start, final_end);
	130	}
	131
	132	/* Put region array back ? */
	133	memblock_reserve_reserved_regions();
	134	}
	135
	136	struct count_data {
	137	int nr;
	138	};
	139
	140	static int __init count_work_fn(unsigned long start_pfn,
	141	unsigned long end_pfn, void *datax)
	142	{
	143	struct count_data *data = datax;
	144
	145	data->nr++;
	146
	147	return 0;
	148	}
	149
	150	static int __init count_early_node_map(int nodeid)
	151	{
	152	struct count_data data;
153
154	data.nr = 0;
155	work_with_active_regions(nodeid, count_work_fn, &data);
156
157	return data.nr;
158	}
159
160	int __init get_free_all_memory_range(struct range **rangep, int nodeid)
161	{
162	int count;
163	struct range *range;
164	int nr_range;
165
166	count = (memblock.reserved.cnt + count_early_node_map(nodeid)) * 2;
167
168	range = find_range_array(count);
169	nr_range = 0;
170
171	/*
172	* Use early_node_map[] and memblock.reserved.region to get range array
173	* at first
174	*/
175	nr_range = add_from_early_node_map(range, count, nr_range, nodeid);
176	#ifdef CONFIG_X86_32
177	subtract_range(range, count, max_low_pfn, -1ULL);
178	#endif
179	memblock_x86_subtract_reserved(range, count);
180	nr_range = clean_sort_range(range, count);
181
182	*rangep = range;
183	return nr_range;
184	}
185	#else
27de7943 YL	186	void __init memblock_x86_to_bootmem(u64 start, u64 end)
	187	{
	188	int count;
	189	u64 final_start, final_end;
	190	struct memblock_region *r;
	191
	192	/* Take out region array itself */
	193	memblock_free_reserved_regions();
	194
	195	count = memblock.reserved.cnt;
	196	pr_info("(%d early reservations) ==> bootmem [%010llx-%010llx]\n", count, start, end - 1);
	197	for_each_memblock(reserved, r) {
	198	pr_info(" [%010llx-%010llx] ", (u64)r->base, (u64)r->base + r->size - 1);
	199	final_start = max(start, r->base);
	200	final_end = min(end, r->base + r->size);
	201	if (final_start >= final_end) {
	202	pr_cont("\n");
	203	continue;
	204	}
	205	pr_cont(" ==> [%010llx-%010llx]\n", final_start, final_end - 1);
	206	reserve_bootmem_generic(final_start, final_end - final_start, BOOTMEM_DEFAULT);
	207	}
	208
	209	/* Put region array back ? */
	210	memblock_reserve_reserved_regions();
	211	}
	212	#endif
9dc5d569 YL	213
	214	void __init memblock_x86_reserve_range(u64 start, u64 end, char *name)
	215	{
	216	if (start == end)
	217	return;
	218
	219	if (WARN_ONCE(start > end, "memblock_x86_reserve_range: wrong range [%#llx, %#llx]\n", start, end))
	220	return;
	221
	222	memblock_reserve(start, end - start);
	223	}
	224
	225	void __init memblock_x86_free_range(u64 start, u64 end)
	226	{
	227	if (start == end)
	228	return;
	229
	230	if (WARN_ONCE(start > end, "memblock_x86_free_range: wrong range [%#llx, %#llx]\n", start, end))
	231	return;
	232
	233	memblock_free(start, end - start);
	234	}
88ba088c	235
6bcc8176 YL	236	/*
	237	* Need to call this function after memblock_x86_register_active_regions,
	238	* so early_node_map[] is filled already.
	239	*/
	240	u64 __init memblock_x86_find_in_range_node(int nid, u64 start, u64 end, u64 size, u64 align)
	241	{
	242	u64 addr;
	243	addr = find_memory_core_early(nid, size, align, start, end);
	244	if (addr != MEMBLOCK_ERROR)
	245	return addr;
	246
	247	/* Fallback, should already have start end within node range */
	248	return memblock_find_in_range(start, end, size, align);
	249	}
	250
88ba088c YL	251	/*
	252	* Finds an active region in the address range from start_pfn to last_pfn and
	253	* returns its range in ei_startpfn and ei_endpfn for the memblock entry.
	254	*/
	255	static int __init memblock_x86_find_active_region(const struct memblock_region *ei,
	256	unsigned long start_pfn,
	257	unsigned long last_pfn,
	258	unsigned long *ei_startpfn,
	259	unsigned long *ei_endpfn)
	260	{
	261	u64 align = PAGE_SIZE;
	262
	263	*ei_startpfn = round_up(ei->base, align) >> PAGE_SHIFT;
	264	*ei_endpfn = round_down(ei->base + ei->size, align) >> PAGE_SHIFT;
	265
	266	/* Skip map entries smaller than a page */
	267	if (ei_startpfn >= ei_endpfn)
	268	return 0;
	269
	270	/* Skip if map is outside the node */
	271	if (ei_endpfn <= start_pfn \|\| ei_startpfn >= last_pfn)
	272	return 0;
	273
	274	/* Check for overlaps */
	275	if (*ei_startpfn < start_pfn)
	276	*ei_startpfn = start_pfn;
	277	if (*ei_endpfn > last_pfn)
	278	*ei_endpfn = last_pfn;
	279
	280	return 1;
	281	}
	282
	283	/* Walk the memblock.memory map and register active regions within a node */
	284	void __init memblock_x86_register_active_regions(int nid, unsigned long start_pfn,
	285	unsigned long last_pfn)
	286	{
	287	unsigned long ei_startpfn;
	288	unsigned long ei_endpfn;
	289	struct memblock_region *r;
	290
	291	for_each_memblock(memory, r)
	292	if (memblock_x86_find_active_region(r, start_pfn, last_pfn,
	293	&ei_startpfn, &ei_endpfn))
	294	add_active_range(nid, ei_startpfn, ei_endpfn);
	295	}
	296
	297	/*
	298	* Find the hole size (in bytes) in the memory range.
	299	* @start: starting address of the memory range to scan
	300	* @end: ending address of the memory range to scan
	301	*/
	302	u64 __init memblock_x86_hole_size(u64 start, u64 end)
	303	{
	304	unsigned long start_pfn = start >> PAGE_SHIFT;
	305	unsigned long last_pfn = end >> PAGE_SHIFT;
	306	unsigned long ei_startpfn, ei_endpfn, ram = 0;
	307	struct memblock_region *r;
	308
	309	for_each_memblock(memory, r)
	310	if (memblock_x86_find_active_region(r, start_pfn, last_pfn,
	311	&ei_startpfn, &ei_endpfn))
	312	ram += ei_endpfn - ei_startpfn;
	313
	314	return end - start - ((u64)ram << PAGE_SHIFT);
315	}