[net-next-2.6.git] / arch / x86 / mm / srat_64.c

/*
 * ACPI 3.0 based NUMA setup
 * Copyright 2004 Andi Kleen, SuSE Labs.
 *
 * Reads the ACPI SRAT table to figure out what memory belongs to which CPUs.
 *
 * Called from acpi_numa_init while reading the SRAT and SLIT tables.
 * Assumes all memory regions belonging to a single proximity domain
 * are in one chunk. Holes between them will be included in the node.
 */

#include <linux/kernel.h>
#include <linux/acpi.h>
#include <linux/mmzone.h>
#include <linux/bitmap.h>
#include <linux/module.h>
#include <linux/topology.h>
#include <linux/bootmem.h>
#include <linux/mm.h>
#include <asm/proto.h>
#include <asm/numa.h>
#include <asm/e820.h>
#include <asm/apic.h>
#include <asm/uv/uv.h>

int acpi_numa __initdata;

static struct acpi_table_slit *acpi_slit;

static nodemask_t nodes_parsed __initdata;
static nodemask_t cpu_nodes_parsed __initdata;
static struct bootnode nodes[MAX_NUMNODES] __initdata;
static struct bootnode nodes_add[MAX_NUMNODES];

static int num_node_memblks __initdata;
static struct bootnode node_memblk_range[NR_NODE_MEMBLKS] __initdata;
static int memblk_nodeid[NR_NODE_MEMBLKS] __initdata;

static __init int setup_node(int pxm)
{
	return acpi_map_pxm_to_node(pxm);
}

static __init int conflicting_memblks(unsigned long start, unsigned long end)
{
	int i;
	for (i = 0; i < num_node_memblks; i++) {
		struct bootnode *nd = &node_memblk_range[i];
		if (nd->start == nd->end)
			continue;
		if (nd->end > start && nd->start < end)
			return memblk_nodeid[i];
		if (nd->end == end && nd->start == start)
			return memblk_nodeid[i];
	}
	return -1;
}

static __init void cutoff_node(int i, unsigned long start, unsigned long end)
{
	struct bootnode *nd = &nodes[i];

	if (nd->start < start) {
		nd->start = start;
		if (nd->end < nd->start)
			nd->start = nd->end;
	}
	if (nd->end > end) {
		nd->end = end;
		if (nd->start > nd->end)
			nd->start = nd->end;
	}
}

static __init void bad_srat(void)
{
	int i;
	printk(KERN_ERR "SRAT: SRAT not used.\n");
	acpi_numa = -1;
	for (i = 0; i < MAX_LOCAL_APIC; i++)
		apicid_to_node[i] = NUMA_NO_NODE;
	for (i = 0; i < MAX_NUMNODES; i++) {
		nodes[i].start = nodes[i].end = 0;
		nodes_add[i].start = nodes_add[i].end = 0;
	}
	remove_all_active_ranges();
}

static __init inline int srat_disabled(void)
{
	return numa_off || acpi_numa < 0;
}

/* Callback for SLIT parsing */
void __init acpi_numa_slit_init(struct acpi_table_slit *slit)
{
	unsigned length;
	unsigned long phys;

	length = slit->header.length;
	phys = find_e820_area(0, max_pfn_mapped<<PAGE_SHIFT, length,
		 PAGE_SIZE);

	if (phys == -1L)
		panic(" Can not save slit!\n");

	acpi_slit = __va(phys);
	memcpy(acpi_slit, slit, length);
	reserve_early(phys, phys + length, "ACPI SLIT");
}

/* Callback for Proximity Domain -> x2APIC mapping */
void __init
acpi_numa_x2apic_affinity_init(struct acpi_srat_x2apic_cpu_affinity *pa)
{
	int pxm, node;
	int apic_id;

	if (srat_disabled())
		return;
	if (pa->header.length < sizeof(struct acpi_srat_x2apic_cpu_affinity)) {
		bad_srat();
		return;
	}
	if ((pa->flags & ACPI_SRAT_CPU_ENABLED) == 0)
		return;
	pxm = pa->proximity_domain;
	node = setup_node(pxm);
	if (node < 0) {
		printk(KERN_ERR "SRAT: Too many proximity domains %x\n", pxm);
		bad_srat();
		return;
	}

	apic_id = pa->apic_id;
	apicid_to_node[apic_id] = node;
	node_set(node, cpu_nodes_parsed);
	acpi_numa = 1;
	printk(KERN_INFO "SRAT: PXM %u -> APIC 0x%04x -> Node %u\n",
	       pxm, apic_id, node);
}

/* Callback for Proximity Domain -> LAPIC mapping */
void __init
acpi_numa_processor_affinity_init(struct acpi_srat_cpu_affinity *pa)
{
	int pxm, node;
	int apic_id;

	if (srat_disabled())
		return;
	if (pa->header.length != sizeof(struct acpi_srat_cpu_affinity)) {
		bad_srat();
		return;
	}
	if ((pa->flags & ACPI_SRAT_CPU_ENABLED) == 0)
		return;
	pxm = pa->proximity_domain_lo;
	node = setup_node(pxm);
	if (node < 0) {
		printk(KERN_ERR "SRAT: Too many proximity domains %x\n", pxm);
		bad_srat();
		return;
	}

	if (get_uv_system_type() >= UV_X2APIC)
		apic_id = (pa->apic_id << 8) | pa->local_sapic_eid;
	else
		apic_id = pa->apic_id;
	apicid_to_node[apic_id] = node;
	node_set(node, cpu_nodes_parsed);
	acpi_numa = 1;
	printk(KERN_INFO "SRAT: PXM %u -> APIC 0x%02x -> Node %u\n",
	       pxm, apic_id, node);
}

#ifdef CONFIG_MEMORY_HOTPLUG_SPARSE
static inline int save_add_info(void) {return 1;}
#else
static inline int save_add_info(void) {return 0;}
#endif
/*
 * Update nodes_add[]
 * This code supports one contiguous hot add area per node
 */
static void __init
update_nodes_add(int node, unsigned long start, unsigned long end)
{
	unsigned long s_pfn = start >> PAGE_SHIFT;
	unsigned long e_pfn = end >> PAGE_SHIFT;
	int changed = 0;
	struct bootnode *nd = &nodes_add[node];

	/* I had some trouble with strange memory hotadd regions breaking
	   the boot. Be very strict here and reject anything unexpected.
	   If you want working memory hotadd write correct SRATs.

	   The node size check is a basic sanity check to guard against
	   mistakes */
	if ((signed long)(end - start) < NODE_MIN_SIZE) {
		printk(KERN_ERR "SRAT: Hotplug area too small\n");
		return;
	}

	/* This check might be a bit too strict, but I'm keeping it for now. */
	if (absent_pages_in_range(s_pfn, e_pfn) != e_pfn - s_pfn) {
		printk(KERN_ERR
			"SRAT: Hotplug area %lu -> %lu has existing memory\n",
			s_pfn, e_pfn);
		return;
	}

	/* Looks good */

	if (nd->start == nd->end) {
		nd->start = start;
		nd->end = end;
		changed = 1;
	} else {
		if (nd->start == end) {
			nd->start = start;
			changed = 1;
		}
		if (nd->end == start) {
			nd->end = end;
			changed = 1;
		}
		if (!changed)
			printk(KERN_ERR "SRAT: Hotplug zone not continuous. Partly ignored\n");
	}

	if (changed)
		printk(KERN_INFO "SRAT: hot plug zone found %Lx - %Lx\n",
				 nd->start, nd->end);
}

/* Callback for parsing of the Proximity Domain <-> Memory Area mappings */
void __init
acpi_numa_memory_affinity_init(struct acpi_srat_mem_affinity *ma)
{
	struct bootnode *nd, oldnode;
	unsigned long start, end;
	int node, pxm;
	int i;

	if (srat_disabled())
		return;
	if (ma->header.length != sizeof(struct acpi_srat_mem_affinity)) {
		bad_srat();
		return;
	}
	if ((ma->flags & ACPI_SRAT_MEM_ENABLED) == 0)
		return;

	if ((ma->flags & ACPI_SRAT_MEM_HOT_PLUGGABLE) && !save_add_info())
		return;
	start = ma->base_address;
	end = start + ma->length;
	pxm = ma->proximity_domain;
	node = setup_node(pxm);
	if (node < 0) {
		printk(KERN_ERR "SRAT: Too many proximity domains.\n");
		bad_srat();
		return;
	}
	i = conflicting_memblks(start, end);
	if (i == node) {
		printk(KERN_WARNING
		"SRAT: Warning: PXM %d (%lx-%lx) overlaps with itself (%Lx-%Lx)\n",
			pxm, start, end, nodes[i].start, nodes[i].end);
	} else if (i >= 0) {
		printk(KERN_ERR
		       "SRAT: PXM %d (%lx-%lx) overlaps with PXM %d (%Lx-%Lx)\n",
		       pxm, start, end, node_to_pxm(i),
			nodes[i].start, nodes[i].end);
		bad_srat();
		return;
	}
	nd = &nodes[node];
	oldnode = *nd;
	if (!node_test_and_set(node, nodes_parsed)) {
		nd->start = start;
		nd->end = end;
	} else {
		if (start < nd->start)
			nd->start = start;
		if (nd->end < end)
			nd->end = end;
	}

	printk(KERN_INFO "SRAT: Node %u PXM %u %lx-%lx\n", node, pxm,
	       start, end);

	if (ma->flags & ACPI_SRAT_MEM_HOT_PLUGGABLE) {
		update_nodes_add(node, start, end);
		/* restore nodes[node] */
		*nd = oldnode;
		if ((nd->start | nd->end) == 0)
			node_clear(node, nodes_parsed);
	}

	node_memblk_range[num_node_memblks].start = start;
	node_memblk_range[num_node_memblks].end = end;
	memblk_nodeid[num_node_memblks] = node;
	num_node_memblks++;
}

/* Sanity check to catch more bad SRATs (they are amazingly common).
   Make sure the PXMs cover all memory. */
static int __init nodes_cover_memory(const struct bootnode *nodes)
{
	int i;
	unsigned long pxmram, e820ram;

	pxmram = 0;
	for_each_node_mask(i, nodes_parsed) {
		unsigned long s = nodes[i].start >> PAGE_SHIFT;
		unsigned long e = nodes[i].end >> PAGE_SHIFT;
		pxmram += e - s;
		pxmram -= __absent_pages_in_range(i, s, e);
		if ((long)pxmram < 0)
			pxmram = 0;
	}

	e820ram = max_pfn - (e820_hole_size(0, max_pfn<<PAGE_SHIFT)>>PAGE_SHIFT);
	/* We seem to lose 3 pages somewhere. Allow 1M of slack. */
	if ((long)(e820ram - pxmram) >= (1<<(20 - PAGE_SHIFT))) {
		printk(KERN_ERR
	"SRAT: PXMs only cover %luMB of your %luMB e820 RAM. Not used.\n",
			(pxmram << PAGE_SHIFT) >> 20,
			(e820ram << PAGE_SHIFT) >> 20);
		return 0;
	}
	return 1;
}

void __init acpi_numa_arch_fixup(void) {}

int __init acpi_get_nodes(struct bootnode *physnodes)
{
	int i;
	int ret = 0;

	for_each_node_mask(i, nodes_parsed) {
		physnodes[ret].start = nodes[i].start;
		physnodes[ret].end = nodes[i].end;
		ret++;
	}
	return ret;
}

/* Use the information discovered above to actually set up the nodes. */
int __init acpi_scan_nodes(unsigned long start, unsigned long end)
{
	int i;

	if (acpi_numa <= 0)
		return -1;

	/* First clean up the node list */
	for (i = 0; i < MAX_NUMNODES; i++)
		cutoff_node(i, start, end);

	memnode_shift = compute_hash_shift(node_memblk_range, num_node_memblks,
					   memblk_nodeid);
	if (memnode_shift < 0) {
		printk(KERN_ERR
		     "SRAT: No NUMA node hash function found. Contact maintainer\n");
		bad_srat();
		return -1;
	}

	for_each_node_mask(i, nodes_parsed)
		e820_register_active_regions(i, nodes[i].start >> PAGE_SHIFT,
						nodes[i].end >> PAGE_SHIFT);
	/* for out of order entries in SRAT */
	sort_node_map();
	if (!nodes_cover_memory(nodes)) {
		bad_srat();
		return -1;
	}

	/* Account for nodes with cpus and no memory */
	nodes_or(node_possible_map, nodes_parsed, cpu_nodes_parsed);

	/* Finally register nodes */
	for_each_node_mask(i, node_possible_map)
		setup_node_bootmem(i, nodes[i].start, nodes[i].end);
	/* Try again in case setup_node_bootmem missed one due
	   to missing bootmem */
	for_each_node_mask(i, node_possible_map)
		if (!node_online(i))
			setup_node_bootmem(i, nodes[i].start, nodes[i].end);

	for (i = 0; i < nr_cpu_ids; i++) {
		int node = early_cpu_to_node(i);

		if (node == NUMA_NO_NODE)
			continue;
		if (!node_online(node))
			numa_clear_node(i);
	}
	numa_init_array();
	return 0;
}

#ifdef CONFIG_NUMA_EMU
static int fake_node_to_pxm_map[MAX_NUMNODES] __initdata = {
	[0 ... MAX_NUMNODES-1] = PXM_INVAL
};
static s16 fake_apicid_to_node[MAX_LOCAL_APIC] __initdata = {
	[0 ... MAX_LOCAL_APIC-1] = NUMA_NO_NODE
};
static int __init find_node_by_addr(unsigned long addr)
{
	int ret = NUMA_NO_NODE;
	int i;

	for_each_node_mask(i, nodes_parsed) {
		/*
		 * Find the real node that this emulated node appears on.  For
		 * the sake of simplicity, we only use a real node's starting
		 * address to determine which emulated node it appears on.
		 */
		if (addr >= nodes[i].start && addr < nodes[i].end) {
			ret = i;
			break;
		}
	}
	return ret;
}

/*
 * In NUMA emulation, we need to setup proximity domain (_PXM) to node ID
 * mappings that respect the real ACPI topology but reflect our emulated
 * environment.  For each emulated node, we find which real node it appears on
 * and create PXM to NID mappings for those fake nodes which mirror that
 * locality.  SLIT will now represent the correct distances between emulated
 * nodes as a result of the real topology.
 */
void __init acpi_fake_nodes(const struct bootnode *fake_nodes, int num_nodes)
{
	int i, j;

	printk(KERN_INFO "Faking PXM affinity for fake nodes on real "
			 "topology.\n");
	for (i = 0; i < num_nodes; i++) {
		int nid, pxm;

		nid = find_node_by_addr(fake_nodes[i].start);
		if (nid == NUMA_NO_NODE)
			continue;
		pxm = node_to_pxm(nid);
		if (pxm == PXM_INVAL)
			continue;
		fake_node_to_pxm_map[i] = pxm;
		/*
		 * For each apicid_to_node mapping that exists for this real
		 * node, it must now point to the fake node ID.
		 */
		for (j = 0; j < MAX_LOCAL_APIC; j++)
			if (apicid_to_node[j] == nid)
				fake_apicid_to_node[j] = i;
	}
	for (i = 0; i < num_nodes; i++)
		__acpi_map_pxm_to_node(fake_node_to_pxm_map[i], i);
	memcpy(apicid_to_node, fake_apicid_to_node, sizeof(apicid_to_node));

	nodes_clear(nodes_parsed);
	for (i = 0; i < num_nodes; i++)
		if (fake_nodes[i].start != fake_nodes[i].end)
			node_set(i, nodes_parsed);
}

static int null_slit_node_compare(int a, int b)
{
	return node_to_pxm(a) == node_to_pxm(b);
}
#else
static int null_slit_node_compare(int a, int b)
{
	return a == b;
}
#endif /* CONFIG_NUMA_EMU */

int __node_distance(int a, int b)
{
	int index;

	if (!acpi_slit)
		return null_slit_node_compare(a, b) ? LOCAL_DISTANCE :
						      REMOTE_DISTANCE;
	index = acpi_slit->locality_count * node_to_pxm(a);
	return acpi_slit->entry[index + node_to_pxm(b)];
}

EXPORT_SYMBOL(__node_distance);

#if defined(CONFIG_MEMORY_HOTPLUG_SPARSE) || defined(CONFIG_ACPI_HOTPLUG_MEMORY)
int memory_add_physaddr_to_nid(u64 start)
{
	int i, ret = 0;

	for_each_node(i)
		if (nodes_add[i].start <= start && nodes_add[i].end > start)
			ret = i;

	return ret;
}
EXPORT_SYMBOL_GPL(memory_add_physaddr_to_nid);
#endif
Commit	Line	Data
1da177e4 LT	1	/*
	2	* ACPI 3.0 based NUMA setup
	3	* Copyright 2004 Andi Kleen, SuSE Labs.
	4	*
	5	* Reads the ACPI SRAT table to figure out what memory belongs to which CPUs.
	6	*
	7	* Called from acpi_numa_init while reading the SRAT and SLIT tables.
	8	* Assumes all memory regions belonging to a single proximity domain
	9	* are in one chunk. Holes between them will be included in the node.
	10	*/
	11
	12	#include <linux/kernel.h>
	13	#include <linux/acpi.h>
	14	#include <linux/mmzone.h>
	15	#include <linux/bitmap.h>
	16	#include <linux/module.h>
	17	#include <linux/topology.h>
68a3a7fe AK	18	#include <linux/bootmem.h>
68a3a7fe AK	19	#include <linux/mm.h>
1da177e4 LT	20	#include <asm/proto.h>
1da177e4 LT	21	#include <asm/numa.h>
8a6fdd3e	22	#include <asm/e820.h>
7b6aa335	23	#include <asm/apic.h>
4ec71fa2	24	#include <asm/uv/uv.h>
1da177e4	25
c31fbb1a AK	26	int acpi_numa __initdata;
c31fbb1a AK	27
1da177e4 LT	28	static struct acpi_table_slit *acpi_slit;
	29
	30	static nodemask_t nodes_parsed __initdata;
dc098551	31	static nodemask_t cpu_nodes_parsed __initdata;
abe059e7	32	static struct bootnode nodes[MAX_NUMNODES] __initdata;
4942e998	33	static struct bootnode nodes_add[MAX_NUMNODES];
1da177e4	34
6ec6e0d9 SS	35	static int num_node_memblks __initdata;
	36	static struct bootnode node_memblk_range[NR_NODE_MEMBLKS] __initdata;
	37	static int memblk_nodeid[NR_NODE_MEMBLKS] __initdata;
	38
1da177e4 LT	39	static __init int setup_node(int pxm)
1da177e4 LT	40	{
762834e8	41	return acpi_map_pxm_to_node(pxm);
1da177e4 LT	42	}
1da177e4 LT	43
6ec6e0d9	44	static __init int conflicting_memblks(unsigned long start, unsigned long end)
1da177e4 LT	45	{
1da177e4 LT	46	int i;
6ec6e0d9 SS	47	for (i = 0; i < num_node_memblks; i++) {
6ec6e0d9 SS	48	struct bootnode *nd = &node_memblk_range[i];
1da177e4 LT	49	if (nd->start == nd->end)
	50	continue;
	51	if (nd->end > start && nd->start < end)
6ec6e0d9	52	return memblk_nodeid[i];
1da177e4	53	if (nd->end == end && nd->start == start)
6ec6e0d9	54	return memblk_nodeid[i];
1da177e4 LT	55	}
	56	return -1;
	57	}
	58
	59	static __init void cutoff_node(int i, unsigned long start, unsigned long end)
	60	{
abe059e7	61	struct bootnode *nd = &nodes[i];
68a3a7fe	62
1da177e4 LT	63	if (nd->start < start) {
	64	nd->start = start;
	65	if (nd->end < nd->start)
	66	nd->start = nd->end;
	67	}
	68	if (nd->end > end) {
1da177e4 LT	69	nd->end = end;
	70	if (nd->start > nd->end)
	71	nd->start = nd->end;
	72	}
	73	}
	74
	75	static __init void bad_srat(void)
	76	{
2bce2b54	77	int i;
1da177e4 LT	78	printk(KERN_ERR "SRAT: SRAT not used.\n");
1da177e4 LT	79	acpi_numa = -1;
2bce2b54 AK	80	for (i = 0; i < MAX_LOCAL_APIC; i++)
2bce2b54 AK	81	apicid_to_node[i] = NUMA_NO_NODE;
429b2b31 AK	82	for (i = 0; i < MAX_NUMNODES; i++) {
	83	nodes[i].start = nodes[i].end = 0;
	84	nodes_add[i].start = nodes_add[i].end = 0;
	85	}
5cb248ab	86	remove_all_active_ranges();
1da177e4 LT	87	}
	88
	89	static __init inline int srat_disabled(void)
	90	{
	91	return numa_off \|\| acpi_numa < 0;
	92	}
	93
	94	/* Callback for SLIT parsing */
	95	void __init acpi_numa_slit_init(struct acpi_table_slit *slit)
	96	{
f302a5bb YL	97	unsigned length;
	98	unsigned long phys;
	99
	100	length = slit->header.length;
	101	phys = find_e820_area(0, max_pfn_mapped<<PAGE_SHIFT, length,
	102	PAGE_SIZE);
	103
	104	if (phys == -1L)
	105	panic(" Can not save slit!\n");
	106
	107	acpi_slit = __va(phys);
	108	memcpy(acpi_slit, slit, length);
	109	reserve_early(phys, phys + length, "ACPI SLIT");
1da177e4 LT	110	}
1da177e4 LT	111
7237d3de SS	112	/* Callback for Proximity Domain -> x2APIC mapping */
	113	void __init
	114	acpi_numa_x2apic_affinity_init(struct acpi_srat_x2apic_cpu_affinity *pa)
	115	{
	116	int pxm, node;
	117	int apic_id;
	118
	119	if (srat_disabled())
	120	return;
	121	if (pa->header.length < sizeof(struct acpi_srat_x2apic_cpu_affinity)) {
	122	bad_srat();
	123	return;
	124	}
	125	if ((pa->flags & ACPI_SRAT_CPU_ENABLED) == 0)
	126	return;
	127	pxm = pa->proximity_domain;
	128	node = setup_node(pxm);
	129	if (node < 0) {
	130	printk(KERN_ERR "SRAT: Too many proximity domains %x\n", pxm);
	131	bad_srat();
	132	return;
	133	}
	134
	135	apic_id = pa->apic_id;
	136	apicid_to_node[apic_id] = node;
dc098551	137	node_set(node, cpu_nodes_parsed);
7237d3de	138	acpi_numa = 1;
163d3866	139	printk(KERN_INFO "SRAT: PXM %u -> APIC 0x%04x -> Node %u\n",
7237d3de SS	140	pxm, apic_id, node);
	141	}
	142
1da177e4 LT	143	/* Callback for Proximity Domain -> LAPIC mapping */
1da177e4 LT	144	void __init
15a58ed1	145	acpi_numa_processor_affinity_init(struct acpi_srat_cpu_affinity *pa)
1da177e4 LT	146	{
1da177e4 LT	147	int pxm, node;
ef97001f	148	int apic_id;
ef97001f	149
d22fe808 AK	150	if (srat_disabled())
d22fe808 AK	151	return;
15a58ed1	152	if (pa->header.length != sizeof(struct acpi_srat_cpu_affinity)) {
fad7906d	153	bad_srat();
d22fe808 AK	154	return;
d22fe808 AK	155	}
15a58ed1	156	if ((pa->flags & ACPI_SRAT_CPU_ENABLED) == 0)
1da177e4	157	return;
15a58ed1	158	pxm = pa->proximity_domain_lo;
1da177e4 LT	159	node = setup_node(pxm);
	160	if (node < 0) {
	161	printk(KERN_ERR "SRAT: Too many proximity domains %x\n", pxm);
	162	bad_srat();
	163	return;
	164	}
beafe91f	165
2e42060c	166	if (get_uv_system_type() >= UV_X2APIC)
a65d1d64 JS	167	apic_id = (pa->apic_id << 8) \| pa->local_sapic_eid;
	168	else
	169	apic_id = pa->apic_id;
ef97001f	170	apicid_to_node[apic_id] = node;
dc098551	171	node_set(node, cpu_nodes_parsed);
1da177e4	172	acpi_numa = 1;
163d3866	173	printk(KERN_INFO "SRAT: PXM %u -> APIC 0x%02x -> Node %u\n",
ef97001f	174	pxm, apic_id, node);
1da177e4 LT	175	}
1da177e4 LT	176
71efa8fd KM	177	#ifdef CONFIG_MEMORY_HOTPLUG_SPARSE
	178	static inline int save_add_info(void) {return 1;}
	179	#else
	180	static inline int save_add_info(void) {return 0;}
	181	#endif
68a3a7fe	182	/*
888a589f YL	183	* Update nodes_add[]
888a589f YL	184	* This code supports one contiguous hot add area per node
68a3a7fe	185	*/
888a589f YL	186	static void __init
888a589f YL	187	update_nodes_add(int node, unsigned long start, unsigned long end)
68a3a7fe AK	188	{
	189	unsigned long s_pfn = start >> PAGE_SHIFT;
	190	unsigned long e_pfn = end >> PAGE_SHIFT;
888a589f	191	int changed = 0;
68a3a7fe AK	192	struct bootnode *nd = &nodes_add[node];
	193
	194	/* I had some trouble with strange memory hotadd regions breaking
	195	the boot. Be very strict here and reject anything unexpected.
	196	If you want working memory hotadd write correct SRATs.
	197
	198	The node size check is a basic sanity check to guard against
	199	mistakes */
	200	if ((signed long)(end - start) < NODE_MIN_SIZE) {
	201	printk(KERN_ERR "SRAT: Hotplug area too small\n");
888a589f	202	return;
68a3a7fe AK	203	}
	204
	205	/* This check might be a bit too strict, but I'm keeping it for now. */
5cb248ab	206	if (absent_pages_in_range(s_pfn, e_pfn) != e_pfn - s_pfn) {
9c7cd687 MG	207	printk(KERN_ERR
	208	"SRAT: Hotplug area %lu -> %lu has existing memory\n",
	209	s_pfn, e_pfn);
888a589f	210	return;
68a3a7fe AK	211	}
	212
	213	/* Looks good */
	214
68a3a7fe	215	if (nd->start == nd->end) {
15a58ed1 AS	216	nd->start = start;
15a58ed1 AS	217	nd->end = end;
68a3a7fe	218	changed = 1;
15a58ed1 AS	219	} else {
	220	if (nd->start == end) {
	221	nd->start = start;
68a3a7fe AK	222	changed = 1;
68a3a7fe AK	223	}
15a58ed1 AS	224	if (nd->end == start) {
15a58ed1 AS	225	nd->end = end;
68a3a7fe AK	226	changed = 1;
	227	}
	228	if (!changed)
	229	printk(KERN_ERR "SRAT: Hotplug zone not continuous. Partly ignored\n");
15a58ed1	230	}
68a3a7fe	231
68a3a7fe	232	if (changed)
888a589f YL	233	printk(KERN_INFO "SRAT: hot plug zone found %Lx - %Lx\n",
888a589f YL	234	nd->start, nd->end);
68a3a7fe	235	}
68a3a7fe	236
1da177e4 LT	237	/* Callback for parsing of the Proximity Domain <-> Memory Area mappings */
1da177e4 LT	238	void __init
15a58ed1	239	acpi_numa_memory_affinity_init(struct acpi_srat_mem_affinity *ma)
1da177e4	240	{
68a3a7fe	241	struct bootnode *nd, oldnode;
1da177e4 LT	242	unsigned long start, end;
	243	int node, pxm;
	244	int i;
	245
d22fe808	246	if (srat_disabled())
1da177e4	247	return;
15a58ed1	248	if (ma->header.length != sizeof(struct acpi_srat_mem_affinity)) {
d22fe808 AK	249	bad_srat();
	250	return;
	251	}
15a58ed1	252	if ((ma->flags & ACPI_SRAT_MEM_ENABLED) == 0)
d22fe808	253	return;
15a58ed1 AS	254
15a58ed1 AS	255	if ((ma->flags & ACPI_SRAT_MEM_HOT_PLUGGABLE) && !save_add_info())
68a3a7fe	256	return;
15a58ed1 AS	257	start = ma->base_address;
15a58ed1 AS	258	end = start + ma->length;
1da177e4 LT	259	pxm = ma->proximity_domain;
	260	node = setup_node(pxm);
	261	if (node < 0) {
	262	printk(KERN_ERR "SRAT: Too many proximity domains.\n");
	263	bad_srat();
	264	return;
	265	}
6ec6e0d9	266	i = conflicting_memblks(start, end);
05d1fa4b AK	267	if (i == node) {
	268	printk(KERN_WARNING
	269	"SRAT: Warning: PXM %d (%lx-%lx) overlaps with itself (%Lx-%Lx)\n",
	270	pxm, start, end, nodes[i].start, nodes[i].end);
	271	} else if (i >= 0) {
1da177e4	272	printk(KERN_ERR
05d1fa4b AK	273	"SRAT: PXM %d (%lx-%lx) overlaps with PXM %d (%Lx-%Lx)\n",
	274	pxm, start, end, node_to_pxm(i),
	275	nodes[i].start, nodes[i].end);
1da177e4 LT	276	bad_srat();
	277	return;
	278	}
	279	nd = &nodes[node];
68a3a7fe	280	oldnode = *nd;
1da177e4 LT	281	if (!node_test_and_set(node, nodes_parsed)) {
	282	nd->start = start;
	283	nd->end = end;
	284	} else {
	285	if (start < nd->start)
	286	nd->start = start;
	287	if (nd->end < end)
	288	nd->end = end;
	289	}
68a3a7fe	290
6ec6e0d9 SS	291	printk(KERN_INFO "SRAT: Node %u PXM %u %lx-%lx\n", node, pxm,
6ec6e0d9 SS	292	start, end);
68a3a7fe	293
888a589f YL	294	if (ma->flags & ACPI_SRAT_MEM_HOT_PLUGGABLE) {
	295	update_nodes_add(node, start, end);
	296	/* restore nodes[node] */
68a3a7fe AK	297	*nd = oldnode;
	298	if ((nd->start \| nd->end) == 0)
	299	node_clear(node, nodes_parsed);
	300	}
6ec6e0d9 SS	301
	302	node_memblk_range[num_node_memblks].start = start;
	303	node_memblk_range[num_node_memblks].end = end;
	304	memblk_nodeid[num_node_memblks] = node;
	305	num_node_memblks++;
1da177e4 LT	306	}
1da177e4 LT	307
8a6fdd3e AK	308	/* Sanity check to catch more bad SRATs (they are amazingly common).
8a6fdd3e AK	309	Make sure the PXMs cover all memory. */
3484d798	310	static int __init nodes_cover_memory(const struct bootnode *nodes)
8a6fdd3e AK	311	{
	312	int i;
	313	unsigned long pxmram, e820ram;
	314
	315	pxmram = 0;
	316	for_each_node_mask(i, nodes_parsed) {
	317	unsigned long s = nodes[i].start >> PAGE_SHIFT;
	318	unsigned long e = nodes[i].end >> PAGE_SHIFT;
	319	pxmram += e - s;
32996250	320	pxmram -= __absent_pages_in_range(i, s, e);
68a3a7fe AK	321	if ((long)pxmram < 0)
68a3a7fe AK	322	pxmram = 0;
8a6fdd3e AK	323	}
8a6fdd3e AK	324
b37ab919	325	e820ram = max_pfn - (e820_hole_size(0, max_pfn<<PAGE_SHIFT)>>PAGE_SHIFT);
0964b056 YL	326	/* We seem to lose 3 pages somewhere. Allow 1M of slack. */
0964b056 YL	327	if ((long)(e820ram - pxmram) >= (1<<(20 - PAGE_SHIFT))) {
8a6fdd3e AK	328	printk(KERN_ERR
	329	"SRAT: PXMs only cover %luMB of your %luMB e820 RAM. Not used.\n",
	330	(pxmram << PAGE_SHIFT) >> 20,
	331	(e820ram << PAGE_SHIFT) >> 20);
	332	return 0;
	333	}
	334	return 1;
	335	}
	336
1da177e4 LT	337	void __init acpi_numa_arch_fixup(void) {}
1da177e4 LT	338
8716273c DR	339	int __init acpi_get_nodes(struct bootnode *physnodes)
	340	{
	341	int i;
	342	int ret = 0;
	343
	344	for_each_node_mask(i, nodes_parsed) {
	345	physnodes[ret].start = nodes[i].start;
	346	physnodes[ret].end = nodes[i].end;
	347	ret++;
	348	}
	349	return ret;
	350	}
	351
1da177e4 LT	352	/* Use the information discovered above to actually set up the nodes. */
	353	int __init acpi_scan_nodes(unsigned long start, unsigned long end)
	354	{
	355	int i;
8a6fdd3e	356
ae2c6dcf DR	357	if (acpi_numa <= 0)
	358	return -1;
	359
e58e0d03	360	/* First clean up the node list */
7c43769a	361	for (i = 0; i < MAX_NUMNODES; i++)
15a58ed1	362	cutoff_node(i, start, end);
e58e0d03	363
6ec6e0d9 SS	364	memnode_shift = compute_hash_shift(node_memblk_range, num_node_memblks,
6ec6e0d9 SS	365	memblk_nodeid);
1da177e4 LT	366	if (memnode_shift < 0) {
	367	printk(KERN_ERR
	368	"SRAT: No NUMA node hash function found. Contact maintainer\n");
	369	bad_srat();
	370	return -1;
	371	}
e58e0d03	372
8716273c DR	373	for_each_node_mask(i, nodes_parsed)
	374	e820_register_active_regions(i, nodes[i].start >> PAGE_SHIFT,
	375	nodes[i].end >> PAGE_SHIFT);
32996250 YL	376	/* for out of order entries in SRAT */
32996250 YL	377	sort_node_map();
8716273c DR	378	if (!nodes_cover_memory(nodes)) {
	379	bad_srat();
	380	return -1;
	381	}
	382
dc098551 JS	383	/* Account for nodes with cpus and no memory */
dc098551 JS	384	nodes_or(node_possible_map, nodes_parsed, cpu_nodes_parsed);
e3f1caee	385
e58e0d03	386	/* Finally register nodes */
e3f1caee	387	for_each_node_mask(i, node_possible_map)
1da177e4	388	setup_node_bootmem(i, nodes[i].start, nodes[i].end);
a8062231 AK	389	/* Try again in case setup_node_bootmem missed one due
a8062231 AK	390	to missing bootmem */
e3f1caee	391	for_each_node_mask(i, node_possible_map)
a8062231 AK	392	if (!node_online(i))
	393	setup_node_bootmem(i, nodes[i].start, nodes[i].end);
	394
168ef543	395	for (i = 0; i < nr_cpu_ids; i++) {
0164fe16 MT	396	int node = early_cpu_to_node(i);
0164fe16 MT	397
834beda1	398	if (node == NUMA_NO_NODE)
1da177e4	399	continue;
7c43769a	400	if (!node_online(node))
23ca4bba	401	numa_clear_node(i);
1da177e4 LT	402	}
	403	numa_init_array();
	404	return 0;
	405	}
	406
3484d798	407	#ifdef CONFIG_NUMA_EMU
ef97001f	408	static int fake_node_to_pxm_map[MAX_NUMNODES] __initdata = {
	409	[0 ... MAX_NUMNODES-1] = PXM_INVAL
	410	};
602a54a8	411	static s16 fake_apicid_to_node[MAX_LOCAL_APIC] __initdata = {
ef97001f	412	[0 ... MAX_LOCAL_APIC-1] = NUMA_NO_NODE
ef97001f	413	};
3484d798 DR	414	static int __init find_node_by_addr(unsigned long addr)
	415	{
	416	int ret = NUMA_NO_NODE;
	417	int i;
	418
	419	for_each_node_mask(i, nodes_parsed) {
	420	/*
	421	* Find the real node that this emulated node appears on. For
	422	* the sake of simplicity, we only use a real node's starting
	423	* address to determine which emulated node it appears on.
	424	*/
	425	if (addr >= nodes[i].start && addr < nodes[i].end) {
	426	ret = i;
	427	break;
	428	}
	429	}
9a1b62fe	430	return ret;
3484d798 DR	431	}
	432
	433	/*
	434	* In NUMA emulation, we need to setup proximity domain (_PXM) to node ID
	435	* mappings that respect the real ACPI topology but reflect our emulated
	436	* environment. For each emulated node, we find which real node it appears on
	437	* and create PXM to NID mappings for those fake nodes which mirror that
	438	* locality. SLIT will now represent the correct distances between emulated
	439	* nodes as a result of the real topology.
	440	*/
	441	void __init acpi_fake_nodes(const struct bootnode *fake_nodes, int num_nodes)
	442	{
08705b89	443	int i, j;
3484d798 DR	444
	445	printk(KERN_INFO "Faking PXM affinity for fake nodes on real "
	446	"topology.\n");
	447	for (i = 0; i < num_nodes; i++) {
	448	int nid, pxm;
	449
	450	nid = find_node_by_addr(fake_nodes[i].start);
	451	if (nid == NUMA_NO_NODE)
	452	continue;
	453	pxm = node_to_pxm(nid);
	454	if (pxm == PXM_INVAL)
	455	continue;
	456	fake_node_to_pxm_map[i] = pxm;
08705b89 DR	457	/*
	458	* For each apicid_to_node mapping that exists for this real
	459	* node, it must now point to the fake node ID.
	460	*/
	461	for (j = 0; j < MAX_LOCAL_APIC; j++)
	462	if (apicid_to_node[j] == nid)
	463	fake_apicid_to_node[j] = i;
3484d798 DR	464	}
	465	for (i = 0; i < num_nodes; i++)
	466	__acpi_map_pxm_to_node(fake_node_to_pxm_map[i], i);
08705b89	467	memcpy(apicid_to_node, fake_apicid_to_node, sizeof(apicid_to_node));
3484d798 DR	468
	469	nodes_clear(nodes_parsed);
	470	for (i = 0; i < num_nodes; i++)
	471	if (fake_nodes[i].start != fake_nodes[i].end)
	472	node_set(i, nodes_parsed);
3484d798 DR	473	}
	474
	475	static int null_slit_node_compare(int a, int b)
	476	{
	477	return node_to_pxm(a) == node_to_pxm(b);
	478	}
	479	#else
	480	static int null_slit_node_compare(int a, int b)
	481	{
	482	return a == b;
	483	}
	484	#endif /* CONFIG_NUMA_EMU */
	485
1da177e4 LT	486	int __node_distance(int a, int b)
	487	{
	488	int index;
	489
	490	if (!acpi_slit)
3484d798 DR	491	return null_slit_node_compare(a, b) ? LOCAL_DISTANCE :
3484d798 DR	492	REMOTE_DISTANCE;
15a58ed1	493	index = acpi_slit->locality_count * node_to_pxm(a);
1da177e4 LT	494	return acpi_slit->entry[index + node_to_pxm(b)];
	495	}
	496
	497	EXPORT_SYMBOL(__node_distance);
4942e998	498
6a1673ae	499	#if defined(CONFIG_MEMORY_HOTPLUG_SPARSE) \|\| defined(CONFIG_ACPI_HOTPLUG_MEMORY)
4942e998 KM	500	int memory_add_physaddr_to_nid(u64 start)
	501	{
	502	int i, ret = 0;
	503
	504	for_each_node(i)
	505	if (nodes_add[i].start <= start && nodes_add[i].end > start)
	506	ret = i;
	507
	508	return ret;
	509	}
8c2676a5	510	EXPORT_SYMBOL_GPL(memory_add_physaddr_to_nid);
6a1673ae	511	#endif