[net-next-2.6.git] / arch / x86 / mm / srat_64.c

/*
 * ACPI 3.0 based NUMA setup
 * Copyright 2004 Andi Kleen, SuSE Labs.
 *
 * Reads the ACPI SRAT table to figure out what memory belongs to which CPUs.
 *
 * Called from acpi_numa_init while reading the SRAT and SLIT tables.
 * Assumes all memory regions belonging to a single proximity domain
 * are in one chunk. Holes between them will be included in the node.
 */

#include <linux/kernel.h>
#include <linux/acpi.h>
#include <linux/mmzone.h>
#include <linux/bitmap.h>
#include <linux/module.h>
#include <linux/topology.h>
#include <linux/bootmem.h>
#include <linux/mm.h>
#include <asm/proto.h>
#include <asm/numa.h>
#include <asm/e820.h>
#include <asm/apic.h>
#include <asm/uv/uv.h>

int acpi_numa __initdata;

static struct acpi_table_slit *acpi_slit;

static nodemask_t nodes_parsed __initdata;
static nodemask_t cpu_nodes_parsed __initdata;
static struct bootnode nodes[MAX_NUMNODES] __initdata;
static struct bootnode nodes_add[MAX_NUMNODES];

static int num_node_memblks __initdata;
static struct bootnode node_memblk_range[NR_NODE_MEMBLKS] __initdata;
static int memblk_nodeid[NR_NODE_MEMBLKS] __initdata;

static __init int setup_node(int pxm)
{
	return acpi_map_pxm_to_node(pxm);
}

static __init int conflicting_memblks(unsigned long start, unsigned long end)
{
	int i;
	for (i = 0; i < num_node_memblks; i++) {
		struct bootnode *nd = &node_memblk_range[i];
		if (nd->start == nd->end)
			continue;
		if (nd->end > start && nd->start < end)
			return memblk_nodeid[i];
		if (nd->end == end && nd->start == start)
			return memblk_nodeid[i];
	}
	return -1;
}

static __init void cutoff_node(int i, unsigned long start, unsigned long end)
{
	struct bootnode *nd = &nodes[i];

	if (nd->start < start) {
		nd->start = start;
		if (nd->end < nd->start)
			nd->start = nd->end;
	}
	if (nd->end > end) {
		nd->end = end;
		if (nd->start > nd->end)
			nd->start = nd->end;
	}
}

static __init void bad_srat(void)
{
	int i;
	printk(KERN_ERR "SRAT: SRAT not used.\n");
	acpi_numa = -1;
	for (i = 0; i < MAX_LOCAL_APIC; i++)
		apicid_to_node[i] = NUMA_NO_NODE;
	for (i = 0; i < MAX_NUMNODES; i++) {
		nodes[i].start = nodes[i].end = 0;
		nodes_add[i].start = nodes_add[i].end = 0;
	}
	remove_all_active_ranges();
}

static __init inline int srat_disabled(void)
{
	return numa_off || acpi_numa < 0;
}

/* Callback for SLIT parsing */
void __init acpi_numa_slit_init(struct acpi_table_slit *slit)
{
	unsigned length;
	unsigned long phys;

	length = slit->header.length;
	phys = find_e820_area(0, max_pfn_mapped<<PAGE_SHIFT, length,
		 PAGE_SIZE);

	if (phys == -1L)
		panic(" Can not save slit!\n");

	acpi_slit = __va(phys);
	memcpy(acpi_slit, slit, length);
	reserve_early(phys, phys + length, "ACPI SLIT");
}

/* Callback for Proximity Domain -> x2APIC mapping */
void __init
acpi_numa_x2apic_affinity_init(struct acpi_srat_x2apic_cpu_affinity *pa)
{
	int pxm, node;
	int apic_id;

	if (srat_disabled())
		return;
	if (pa->header.length < sizeof(struct acpi_srat_x2apic_cpu_affinity)) {
		bad_srat();
		return;
	}
	if ((pa->flags & ACPI_SRAT_CPU_ENABLED) == 0)
		return;
	pxm = pa->proximity_domain;
	node = setup_node(pxm);
	if (node < 0) {
		printk(KERN_ERR "SRAT: Too many proximity domains %x\n", pxm);
		bad_srat();
		return;
	}

	apic_id = pa->apic_id;
	apicid_to_node[apic_id] = node;
	node_set(node, cpu_nodes_parsed);
	acpi_numa = 1;
	printk(KERN_INFO "SRAT: PXM %u -> APIC 0x%04x -> Node %u\n",
	       pxm, apic_id, node);
}

/* Callback for Proximity Domain -> LAPIC mapping */
void __init
acpi_numa_processor_affinity_init(struct acpi_srat_cpu_affinity *pa)
{
	int pxm, node;
	int apic_id;

	if (srat_disabled())
		return;
	if (pa->header.length != sizeof(struct acpi_srat_cpu_affinity)) {
		bad_srat();
		return;
	}
	if ((pa->flags & ACPI_SRAT_CPU_ENABLED) == 0)
		return;
	pxm = pa->proximity_domain_lo;
	node = setup_node(pxm);
	if (node < 0) {
		printk(KERN_ERR "SRAT: Too many proximity domains %x\n", pxm);
		bad_srat();
		return;
	}

	if (get_uv_system_type() >= UV_X2APIC)
		apic_id = (pa->apic_id << 8) | pa->local_sapic_eid;
	else
		apic_id = pa->apic_id;
	apicid_to_node[apic_id] = node;
	node_set(node, cpu_nodes_parsed);
	acpi_numa = 1;
	printk(KERN_INFO "SRAT: PXM %u -> APIC 0x%02x -> Node %u\n",
	       pxm, apic_id, node);
}

#ifdef CONFIG_MEMORY_HOTPLUG_SPARSE
static inline int save_add_info(void) {return 1;}
#else
static inline int save_add_info(void) {return 0;}
#endif
/*
 * Update nodes_add[]
 * This code supports one contiguous hot add area per node
 */
static void __init
update_nodes_add(int node, unsigned long start, unsigned long end)
{
	unsigned long s_pfn = start >> PAGE_SHIFT;
	unsigned long e_pfn = end >> PAGE_SHIFT;
	int changed = 0;
	struct bootnode *nd = &nodes_add[node];

	/* I had some trouble with strange memory hotadd regions breaking
	   the boot. Be very strict here and reject anything unexpected.
	   If you want working memory hotadd write correct SRATs.

	   The node size check is a basic sanity check to guard against
	   mistakes */
	if ((signed long)(end - start) < NODE_MIN_SIZE) {
		printk(KERN_ERR "SRAT: Hotplug area too small\n");
		return;
	}

	/* This check might be a bit too strict, but I'm keeping it for now. */
	if (absent_pages_in_range(s_pfn, e_pfn) != e_pfn - s_pfn) {
		printk(KERN_ERR
			"SRAT: Hotplug area %lu -> %lu has existing memory\n",
			s_pfn, e_pfn);
		return;
	}

	/* Looks good */

	if (nd->start == nd->end) {
		nd->start = start;
		nd->end = end;
		changed = 1;
	} else {
		if (nd->start == end) {
			nd->start = start;
			changed = 1;
		}
		if (nd->end == start) {
			nd->end = end;
			changed = 1;
		}
		if (!changed)
			printk(KERN_ERR "SRAT: Hotplug zone not continuous. Partly ignored\n");
	}

	if (changed) {
		node_set(node, cpu_nodes_parsed);
		printk(KERN_INFO "SRAT: hot plug zone found %Lx - %Lx\n",
				 nd->start, nd->end);
	}
}

/* Callback for parsing of the Proximity Domain <-> Memory Area mappings */
void __init
acpi_numa_memory_affinity_init(struct acpi_srat_mem_affinity *ma)
{
	struct bootnode *nd, oldnode;
	unsigned long start, end;
	int node, pxm;
	int i;

	if (srat_disabled())
		return;
	if (ma->header.length != sizeof(struct acpi_srat_mem_affinity)) {
		bad_srat();
		return;
	}
	if ((ma->flags & ACPI_SRAT_MEM_ENABLED) == 0)
		return;

	if ((ma->flags & ACPI_SRAT_MEM_HOT_PLUGGABLE) && !save_add_info())
		return;
	start = ma->base_address;
	end = start + ma->length;
	pxm = ma->proximity_domain;
	node = setup_node(pxm);
	if (node < 0) {
		printk(KERN_ERR "SRAT: Too many proximity domains.\n");
		bad_srat();
		return;
	}
	i = conflicting_memblks(start, end);
	if (i == node) {
		printk(KERN_WARNING
		"SRAT: Warning: PXM %d (%lx-%lx) overlaps with itself (%Lx-%Lx)\n",
			pxm, start, end, nodes[i].start, nodes[i].end);
	} else if (i >= 0) {
		printk(KERN_ERR
		       "SRAT: PXM %d (%lx-%lx) overlaps with PXM %d (%Lx-%Lx)\n",
		       pxm, start, end, node_to_pxm(i),
			nodes[i].start, nodes[i].end);
		bad_srat();
		return;
	}
	nd = &nodes[node];
	oldnode = *nd;
	if (!node_test_and_set(node, nodes_parsed)) {
		nd->start = start;
		nd->end = end;
	} else {
		if (start < nd->start)
			nd->start = start;
		if (nd->end < end)
			nd->end = end;
	}

	printk(KERN_INFO "SRAT: Node %u PXM %u %lx-%lx\n", node, pxm,
	       start, end);

	if (ma->flags & ACPI_SRAT_MEM_HOT_PLUGGABLE) {
		update_nodes_add(node, start, end);
		/* restore nodes[node] */
		*nd = oldnode;
		if ((nd->start | nd->end) == 0)
			node_clear(node, nodes_parsed);
	}

	node_memblk_range[num_node_memblks].start = start;
	node_memblk_range[num_node_memblks].end = end;
	memblk_nodeid[num_node_memblks] = node;
	num_node_memblks++;
}

/* Sanity check to catch more bad SRATs (they are amazingly common).
   Make sure the PXMs cover all memory. */
static int __init nodes_cover_memory(const struct bootnode *nodes)
{
	int i;
	unsigned long pxmram, e820ram;

	pxmram = 0;
	for_each_node_mask(i, nodes_parsed) {
		unsigned long s = nodes[i].start >> PAGE_SHIFT;
		unsigned long e = nodes[i].end >> PAGE_SHIFT;
		pxmram += e - s;
		pxmram -= __absent_pages_in_range(i, s, e);
		if ((long)pxmram < 0)
			pxmram = 0;
	}

	e820ram = max_pfn - (e820_hole_size(0, max_pfn<<PAGE_SHIFT)>>PAGE_SHIFT);
	/* We seem to lose 3 pages somewhere. Allow 1M of slack. */
	if ((long)(e820ram - pxmram) >= (1<<(20 - PAGE_SHIFT))) {
		printk(KERN_ERR
	"SRAT: PXMs only cover %luMB of your %luMB e820 RAM. Not used.\n",
			(pxmram << PAGE_SHIFT) >> 20,
			(e820ram << PAGE_SHIFT) >> 20);
		return 0;
	}
	return 1;
}

void __init acpi_numa_arch_fixup(void) {}

int __init acpi_get_nodes(struct bootnode *physnodes)
{
	int i;
	int ret = 0;

	for_each_node_mask(i, nodes_parsed) {
		physnodes[ret].start = nodes[i].start;
		physnodes[ret].end = nodes[i].end;
		ret++;
	}
	return ret;
}

/* Use the information discovered above to actually set up the nodes. */
int __init acpi_scan_nodes(unsigned long start, unsigned long end)
{
	int i;

	if (acpi_numa <= 0)
		return -1;

	/* First clean up the node list */
	for (i = 0; i < MAX_NUMNODES; i++)
		cutoff_node(i, start, end);

	/*
	 * Join together blocks on the same node, holes between
	 * which don't overlap with memory on other nodes.
	 */
	for (i = 0; i < num_node_memblks; ++i) {
		int j, k;

		for (j = i + 1; j < num_node_memblks; ++j) {
			unsigned long start, end;

			if (memblk_nodeid[i] != memblk_nodeid[j])
				continue;
			start = min(node_memblk_range[i].end,
			            node_memblk_range[j].end);
			end = max(node_memblk_range[i].start,
			          node_memblk_range[j].start);
			for (k = 0; k < num_node_memblks; ++k) {
				if (memblk_nodeid[i] == memblk_nodeid[k])
					continue;
				if (start < node_memblk_range[k].end &&
				    end > node_memblk_range[k].start)
					break;
			}
			if (k < num_node_memblks)
				continue;
			start = min(node_memblk_range[i].start,
			            node_memblk_range[j].start);
			end = max(node_memblk_range[i].end,
			          node_memblk_range[j].end);
			printk(KERN_INFO "SRAT: Node %d "
			       "[%Lx,%Lx) + [%Lx,%Lx) -> [%lx,%lx)\n",
			       memblk_nodeid[i],
			       node_memblk_range[i].start,
			       node_memblk_range[i].end,
			       node_memblk_range[j].start,
			       node_memblk_range[j].end,
			       start, end);
			node_memblk_range[i].start = start;
			node_memblk_range[i].end = end;
			k = --num_node_memblks - j;
			memmove(memblk_nodeid + j, memblk_nodeid + j+1,
				k * sizeof(*memblk_nodeid));
			memmove(node_memblk_range + j, node_memblk_range + j+1,
				k * sizeof(*node_memblk_range));
			--j;
		}
	}

	memnode_shift = compute_hash_shift(node_memblk_range, num_node_memblks,
					   memblk_nodeid);
	if (memnode_shift < 0) {
		printk(KERN_ERR
		     "SRAT: No NUMA node hash function found. Contact maintainer\n");
		bad_srat();
		return -1;
	}

	for (i = 0; i < num_node_memblks; i++)
		e820_register_active_regions(memblk_nodeid[i],
				node_memblk_range[i].start >> PAGE_SHIFT,
				node_memblk_range[i].end >> PAGE_SHIFT);

	/* for out of order entries in SRAT */
	sort_node_map();
	if (!nodes_cover_memory(nodes)) {
		bad_srat();
		return -1;
	}

	/* Account for nodes with cpus and no memory */
	nodes_or(node_possible_map, nodes_parsed, cpu_nodes_parsed);

	/* Finally register nodes */
	for_each_node_mask(i, node_possible_map)
		setup_node_bootmem(i, nodes[i].start, nodes[i].end);
	/* Try again in case setup_node_bootmem missed one due
	   to missing bootmem */
	for_each_node_mask(i, node_possible_map)
		if (!node_online(i))
			setup_node_bootmem(i, nodes[i].start, nodes[i].end);

	for (i = 0; i < nr_cpu_ids; i++) {
		int node = early_cpu_to_node(i);

		if (node == NUMA_NO_NODE)
			continue;
		if (!node_online(node))
			numa_clear_node(i);
	}
	numa_init_array();
	return 0;
}

#ifdef CONFIG_NUMA_EMU
static int fake_node_to_pxm_map[MAX_NUMNODES] __initdata = {
	[0 ... MAX_NUMNODES-1] = PXM_INVAL
};
static s16 fake_apicid_to_node[MAX_LOCAL_APIC] __initdata = {
	[0 ... MAX_LOCAL_APIC-1] = NUMA_NO_NODE
};
static int __init find_node_by_addr(unsigned long addr)
{
	int ret = NUMA_NO_NODE;
	int i;

	for_each_node_mask(i, nodes_parsed) {
		/*
		 * Find the real node that this emulated node appears on.  For
		 * the sake of simplicity, we only use a real node's starting
		 * address to determine which emulated node it appears on.
		 */
		if (addr >= nodes[i].start && addr < nodes[i].end) {
			ret = i;
			break;
		}
	}
	return ret;
}

/*
 * In NUMA emulation, we need to setup proximity domain (_PXM) to node ID
 * mappings that respect the real ACPI topology but reflect our emulated
 * environment.  For each emulated node, we find which real node it appears on
 * and create PXM to NID mappings for those fake nodes which mirror that
 * locality.  SLIT will now represent the correct distances between emulated
 * nodes as a result of the real topology.
 */
void __init acpi_fake_nodes(const struct bootnode *fake_nodes, int num_nodes)
{
	int i, j;

	printk(KERN_INFO "Faking PXM affinity for fake nodes on real "
			 "topology.\n");
	for (i = 0; i < num_nodes; i++) {
		int nid, pxm;

		nid = find_node_by_addr(fake_nodes[i].start);
		if (nid == NUMA_NO_NODE)
			continue;
		pxm = node_to_pxm(nid);
		if (pxm == PXM_INVAL)
			continue;
		fake_node_to_pxm_map[i] = pxm;
		/*
		 * For each apicid_to_node mapping that exists for this real
		 * node, it must now point to the fake node ID.
		 */
		for (j = 0; j < MAX_LOCAL_APIC; j++)
			if (apicid_to_node[j] == nid &&
			    fake_apicid_to_node[j] == NUMA_NO_NODE)
				fake_apicid_to_node[j] = i;
	}
	for (i = 0; i < num_nodes; i++)
		__acpi_map_pxm_to_node(fake_node_to_pxm_map[i], i);
	memcpy(apicid_to_node, fake_apicid_to_node, sizeof(apicid_to_node));

	nodes_clear(nodes_parsed);
	for (i = 0; i < num_nodes; i++)
		if (fake_nodes[i].start != fake_nodes[i].end)
			node_set(i, nodes_parsed);
}

static int null_slit_node_compare(int a, int b)
{
	return node_to_pxm(a) == node_to_pxm(b);
}
#else
static int null_slit_node_compare(int a, int b)
{
	return a == b;
}
#endif /* CONFIG_NUMA_EMU */

int __node_distance(int a, int b)
{
	int index;

	if (!acpi_slit)
		return null_slit_node_compare(a, b) ? LOCAL_DISTANCE :
						      REMOTE_DISTANCE;
	index = acpi_slit->locality_count * node_to_pxm(a);
	return acpi_slit->entry[index + node_to_pxm(b)];
}

EXPORT_SYMBOL(__node_distance);

#if defined(CONFIG_MEMORY_HOTPLUG_SPARSE) || defined(CONFIG_ACPI_HOTPLUG_MEMORY)
int memory_add_physaddr_to_nid(u64 start)
{
	int i, ret = 0;

	for_each_node(i)
		if (nodes_add[i].start <= start && nodes_add[i].end > start)
			ret = i;

	return ret;
}
EXPORT_SYMBOL_GPL(memory_add_physaddr_to_nid);
#endif
Commit	Line	Data
	1	/*
	2	* ACPI 3.0 based NUMA setup
	3	* Copyright 2004 Andi Kleen, SuSE Labs.
	4	*
	5	* Reads the ACPI SRAT table to figure out what memory belongs to which CPUs.
	6	*
	7	* Called from acpi_numa_init while reading the SRAT and SLIT tables.
	8	* Assumes all memory regions belonging to a single proximity domain
	9	* are in one chunk. Holes between them will be included in the node.
	10	*/
	11
	12	#include <linux/kernel.h>
	13	#include <linux/acpi.h>
	14	#include <linux/mmzone.h>
	15	#include <linux/bitmap.h>
	16	#include <linux/module.h>
	17	#include <linux/topology.h>
	18	#include <linux/bootmem.h>
	19	#include <linux/mm.h>
	20	#include <asm/proto.h>
	21	#include <asm/numa.h>
	22	#include <asm/e820.h>
	23	#include <asm/apic.h>
	24	#include <asm/uv/uv.h>
	25
	26	int acpi_numa __initdata;
	27
	28	static struct acpi_table_slit *acpi_slit;
	29
	30	static nodemask_t nodes_parsed __initdata;
	31	static nodemask_t cpu_nodes_parsed __initdata;
	32	static struct bootnode nodes[MAX_NUMNODES] __initdata;
	33	static struct bootnode nodes_add[MAX_NUMNODES];
	34
	35	static int num_node_memblks __initdata;
	36	static struct bootnode node_memblk_range[NR_NODE_MEMBLKS] __initdata;
	37	static int memblk_nodeid[NR_NODE_MEMBLKS] __initdata;
	38
	39	static __init int setup_node(int pxm)
	40	{
	41	return acpi_map_pxm_to_node(pxm);
	42	}
	43
	44	static __init int conflicting_memblks(unsigned long start, unsigned long end)
	45	{
	46	int i;
	47	for (i = 0; i < num_node_memblks; i++) {
	48	struct bootnode *nd = &node_memblk_range[i];
	49	if (nd->start == nd->end)
	50	continue;
	51	if (nd->end > start && nd->start < end)
	52	return memblk_nodeid[i];
	53	if (nd->end == end && nd->start == start)
	54	return memblk_nodeid[i];
	55	}
	56	return -1;
	57	}
	58
	59	static __init void cutoff_node(int i, unsigned long start, unsigned long end)
	60	{
	61	struct bootnode *nd = &nodes[i];
	62
	63	if (nd->start < start) {
	64	nd->start = start;
	65	if (nd->end < nd->start)
	66	nd->start = nd->end;
	67	}
	68	if (nd->end > end) {
	69	nd->end = end;
	70	if (nd->start > nd->end)
	71	nd->start = nd->end;
	72	}
	73	}
	74
	75	static __init void bad_srat(void)
	76	{
	77	int i;
	78	printk(KERN_ERR "SRAT: SRAT not used.\n");
	79	acpi_numa = -1;
	80	for (i = 0; i < MAX_LOCAL_APIC; i++)
	81	apicid_to_node[i] = NUMA_NO_NODE;
	82	for (i = 0; i < MAX_NUMNODES; i++) {
	83	nodes[i].start = nodes[i].end = 0;
	84	nodes_add[i].start = nodes_add[i].end = 0;
	85	}
	86	remove_all_active_ranges();
	87	}
	88
	89	static __init inline int srat_disabled(void)
	90	{
	91	return numa_off \|\| acpi_numa < 0;
	92	}
	93
	94	/* Callback for SLIT parsing */
	95	void __init acpi_numa_slit_init(struct acpi_table_slit *slit)
	96	{
	97	unsigned length;
	98	unsigned long phys;
	99
	100	length = slit->header.length;
	101	phys = find_e820_area(0, max_pfn_mapped<<PAGE_SHIFT, length,
	102	PAGE_SIZE);
	103
	104	if (phys == -1L)
	105	panic(" Can not save slit!\n");
	106
	107	acpi_slit = __va(phys);
	108	memcpy(acpi_slit, slit, length);
	109	reserve_early(phys, phys + length, "ACPI SLIT");
	110	}
	111
	112	/* Callback for Proximity Domain -> x2APIC mapping */
	113	void __init
	114	acpi_numa_x2apic_affinity_init(struct acpi_srat_x2apic_cpu_affinity *pa)
	115	{
	116	int pxm, node;
	117	int apic_id;
	118
	119	if (srat_disabled())
	120	return;
	121	if (pa->header.length < sizeof(struct acpi_srat_x2apic_cpu_affinity)) {
	122	bad_srat();
	123	return;
	124	}
	125	if ((pa->flags & ACPI_SRAT_CPU_ENABLED) == 0)
	126	return;
	127	pxm = pa->proximity_domain;
	128	node = setup_node(pxm);
	129	if (node < 0) {
	130	printk(KERN_ERR "SRAT: Too many proximity domains %x\n", pxm);
	131	bad_srat();
	132	return;
	133	}
	134
	135	apic_id = pa->apic_id;
	136	apicid_to_node[apic_id] = node;
	137	node_set(node, cpu_nodes_parsed);
	138	acpi_numa = 1;
	139	printk(KERN_INFO "SRAT: PXM %u -> APIC 0x%04x -> Node %u\n",
	140	pxm, apic_id, node);
	141	}
	142
	143	/* Callback for Proximity Domain -> LAPIC mapping */
	144	void __init
	145	acpi_numa_processor_affinity_init(struct acpi_srat_cpu_affinity *pa)
	146	{
	147	int pxm, node;
	148	int apic_id;
	149
	150	if (srat_disabled())
	151	return;
	152	if (pa->header.length != sizeof(struct acpi_srat_cpu_affinity)) {
	153	bad_srat();
	154	return;
	155	}
	156	if ((pa->flags & ACPI_SRAT_CPU_ENABLED) == 0)
	157	return;
	158	pxm = pa->proximity_domain_lo;
	159	node = setup_node(pxm);
	160	if (node < 0) {
	161	printk(KERN_ERR "SRAT: Too many proximity domains %x\n", pxm);
	162	bad_srat();
	163	return;
	164	}
	165
	166	if (get_uv_system_type() >= UV_X2APIC)
	167	apic_id = (pa->apic_id << 8) \| pa->local_sapic_eid;
	168	else
	169	apic_id = pa->apic_id;
	170	apicid_to_node[apic_id] = node;
	171	node_set(node, cpu_nodes_parsed);
	172	acpi_numa = 1;
	173	printk(KERN_INFO "SRAT: PXM %u -> APIC 0x%02x -> Node %u\n",
	174	pxm, apic_id, node);
	175	}
	176
	177	#ifdef CONFIG_MEMORY_HOTPLUG_SPARSE
	178	static inline int save_add_info(void) {return 1;}
	179	#else
	180	static inline int save_add_info(void) {return 0;}
	181	#endif
	182	/*
	183	* Update nodes_add[]
	184	* This code supports one contiguous hot add area per node
	185	*/
	186	static void __init
	187	update_nodes_add(int node, unsigned long start, unsigned long end)
	188	{
	189	unsigned long s_pfn = start >> PAGE_SHIFT;
	190	unsigned long e_pfn = end >> PAGE_SHIFT;
	191	int changed = 0;
	192	struct bootnode *nd = &nodes_add[node];
	193
	194	/* I had some trouble with strange memory hotadd regions breaking
	195	the boot. Be very strict here and reject anything unexpected.
	196	If you want working memory hotadd write correct SRATs.
	197
	198	The node size check is a basic sanity check to guard against
	199	mistakes */
	200	if ((signed long)(end - start) < NODE_MIN_SIZE) {
	201	printk(KERN_ERR "SRAT: Hotplug area too small\n");
	202	return;
	203	}
	204
	205	/* This check might be a bit too strict, but I'm keeping it for now. */
	206	if (absent_pages_in_range(s_pfn, e_pfn) != e_pfn - s_pfn) {
	207	printk(KERN_ERR
	208	"SRAT: Hotplug area %lu -> %lu has existing memory\n",
	209	s_pfn, e_pfn);
	210	return;
	211	}
	212
	213	/* Looks good */
	214
	215	if (nd->start == nd->end) {
	216	nd->start = start;
	217	nd->end = end;
	218	changed = 1;
	219	} else {
	220	if (nd->start == end) {
	221	nd->start = start;
	222	changed = 1;
	223	}
	224	if (nd->end == start) {
	225	nd->end = end;
	226	changed = 1;
	227	}
	228	if (!changed)
	229	printk(KERN_ERR "SRAT: Hotplug zone not continuous. Partly ignored\n");
	230	}
	231
	232	if (changed) {
	233	node_set(node, cpu_nodes_parsed);
	234	printk(KERN_INFO "SRAT: hot plug zone found %Lx - %Lx\n",
	235	nd->start, nd->end);
	236	}
	237	}
	238
	239	/* Callback for parsing of the Proximity Domain <-> Memory Area mappings */
	240	void __init
	241	acpi_numa_memory_affinity_init(struct acpi_srat_mem_affinity *ma)
	242	{
	243	struct bootnode *nd, oldnode;
	244	unsigned long start, end;
	245	int node, pxm;
	246	int i;
	247
	248	if (srat_disabled())
	249	return;
	250	if (ma->header.length != sizeof(struct acpi_srat_mem_affinity)) {
	251	bad_srat();
	252	return;
	253	}
	254	if ((ma->flags & ACPI_SRAT_MEM_ENABLED) == 0)
	255	return;
	256
	257	if ((ma->flags & ACPI_SRAT_MEM_HOT_PLUGGABLE) && !save_add_info())
	258	return;
	259	start = ma->base_address;
	260	end = start + ma->length;
	261	pxm = ma->proximity_domain;
	262	node = setup_node(pxm);
	263	if (node < 0) {
	264	printk(KERN_ERR "SRAT: Too many proximity domains.\n");
	265	bad_srat();
	266	return;
	267	}
	268	i = conflicting_memblks(start, end);
	269	if (i == node) {
	270	printk(KERN_WARNING
	271	"SRAT: Warning: PXM %d (%lx-%lx) overlaps with itself (%Lx-%Lx)\n",
	272	pxm, start, end, nodes[i].start, nodes[i].end);
	273	} else if (i >= 0) {
	274	printk(KERN_ERR
	275	"SRAT: PXM %d (%lx-%lx) overlaps with PXM %d (%Lx-%Lx)\n",
	276	pxm, start, end, node_to_pxm(i),
	277	nodes[i].start, nodes[i].end);
	278	bad_srat();
	279	return;
	280	}
	281	nd = &nodes[node];
	282	oldnode = *nd;
	283	if (!node_test_and_set(node, nodes_parsed)) {
	284	nd->start = start;
	285	nd->end = end;
	286	} else {
	287	if (start < nd->start)
	288	nd->start = start;
	289	if (nd->end < end)
	290	nd->end = end;
	291	}
	292
	293	printk(KERN_INFO "SRAT: Node %u PXM %u %lx-%lx\n", node, pxm,
	294	start, end);
	295
	296	if (ma->flags & ACPI_SRAT_MEM_HOT_PLUGGABLE) {
	297	update_nodes_add(node, start, end);
	298	/* restore nodes[node] */
	299	*nd = oldnode;
	300	if ((nd->start \| nd->end) == 0)
	301	node_clear(node, nodes_parsed);
	302	}
	303
	304	node_memblk_range[num_node_memblks].start = start;
	305	node_memblk_range[num_node_memblks].end = end;
	306	memblk_nodeid[num_node_memblks] = node;
	307	num_node_memblks++;
	308	}
	309
	310	/* Sanity check to catch more bad SRATs (they are amazingly common).
	311	Make sure the PXMs cover all memory. */
	312	static int __init nodes_cover_memory(const struct bootnode *nodes)
	313	{
	314	int i;
	315	unsigned long pxmram, e820ram;
	316
	317	pxmram = 0;
	318	for_each_node_mask(i, nodes_parsed) {
	319	unsigned long s = nodes[i].start >> PAGE_SHIFT;
	320	unsigned long e = nodes[i].end >> PAGE_SHIFT;
	321	pxmram += e - s;
	322	pxmram -= __absent_pages_in_range(i, s, e);
	323	if ((long)pxmram < 0)
	324	pxmram = 0;
	325	}
	326
	327	e820ram = max_pfn - (e820_hole_size(0, max_pfn<<PAGE_SHIFT)>>PAGE_SHIFT);
	328	/* We seem to lose 3 pages somewhere. Allow 1M of slack. */
	329	if ((long)(e820ram - pxmram) >= (1<<(20 - PAGE_SHIFT))) {
	330	printk(KERN_ERR
	331	"SRAT: PXMs only cover %luMB of your %luMB e820 RAM. Not used.\n",
	332	(pxmram << PAGE_SHIFT) >> 20,
	333	(e820ram << PAGE_SHIFT) >> 20);
	334	return 0;
	335	}
	336	return 1;
	337	}
	338
	339	void __init acpi_numa_arch_fixup(void) {}
	340
	341	int __init acpi_get_nodes(struct bootnode *physnodes)
	342	{
	343	int i;
	344	int ret = 0;
	345
	346	for_each_node_mask(i, nodes_parsed) {
	347	physnodes[ret].start = nodes[i].start;
	348	physnodes[ret].end = nodes[i].end;
	349	ret++;
	350	}
	351	return ret;
	352	}
	353
	354	/* Use the information discovered above to actually set up the nodes. */
	355	int __init acpi_scan_nodes(unsigned long start, unsigned long end)
	356	{
	357	int i;
	358
	359	if (acpi_numa <= 0)
	360	return -1;
	361
	362	/* First clean up the node list */
	363	for (i = 0; i < MAX_NUMNODES; i++)
	364	cutoff_node(i, start, end);
	365
	366	/*
	367	* Join together blocks on the same node, holes between
	368	* which don't overlap with memory on other nodes.
	369	*/
	370	for (i = 0; i < num_node_memblks; ++i) {
	371	int j, k;
	372
	373	for (j = i + 1; j < num_node_memblks; ++j) {
	374	unsigned long start, end;
	375
	376	if (memblk_nodeid[i] != memblk_nodeid[j])
	377	continue;
	378	start = min(node_memblk_range[i].end,
	379	node_memblk_range[j].end);
	380	end = max(node_memblk_range[i].start,
	381	node_memblk_range[j].start);
	382	for (k = 0; k < num_node_memblks; ++k) {
	383	if (memblk_nodeid[i] == memblk_nodeid[k])
	384	continue;
	385	if (start < node_memblk_range[k].end &&
	386	end > node_memblk_range[k].start)
	387	break;
	388	}
	389	if (k < num_node_memblks)
	390	continue;
	391	start = min(node_memblk_range[i].start,
	392	node_memblk_range[j].start);
	393	end = max(node_memblk_range[i].end,
	394	node_memblk_range[j].end);
	395	printk(KERN_INFO "SRAT: Node %d "
	396	"[%Lx,%Lx) + [%Lx,%Lx) -> [%lx,%lx)\n",
	397	memblk_nodeid[i],
	398	node_memblk_range[i].start,
	399	node_memblk_range[i].end,
	400	node_memblk_range[j].start,
	401	node_memblk_range[j].end,
	402	start, end);
	403	node_memblk_range[i].start = start;
	404	node_memblk_range[i].end = end;
	405	k = --num_node_memblks - j;
	406	memmove(memblk_nodeid + j, memblk_nodeid + j+1,
	407	k * sizeof(*memblk_nodeid));
	408	memmove(node_memblk_range + j, node_memblk_range + j+1,
	409	k * sizeof(*node_memblk_range));
	410	--j;
	411	}
	412	}
	413
	414	memnode_shift = compute_hash_shift(node_memblk_range, num_node_memblks,
	415	memblk_nodeid);
	416	if (memnode_shift < 0) {
	417	printk(KERN_ERR
	418	"SRAT: No NUMA node hash function found. Contact maintainer\n");
	419	bad_srat();
	420	return -1;
	421	}
	422
	423	for (i = 0; i < num_node_memblks; i++)
	424	e820_register_active_regions(memblk_nodeid[i],
	425	node_memblk_range[i].start >> PAGE_SHIFT,
	426	node_memblk_range[i].end >> PAGE_SHIFT);
	427
	428	/* for out of order entries in SRAT */
	429	sort_node_map();
	430	if (!nodes_cover_memory(nodes)) {
	431	bad_srat();
	432	return -1;
	433	}
	434
	435	/* Account for nodes with cpus and no memory */
	436	nodes_or(node_possible_map, nodes_parsed, cpu_nodes_parsed);
	437
	438	/* Finally register nodes */
	439	for_each_node_mask(i, node_possible_map)
	440	setup_node_bootmem(i, nodes[i].start, nodes[i].end);
	441	/* Try again in case setup_node_bootmem missed one due
	442	to missing bootmem */
	443	for_each_node_mask(i, node_possible_map)
	444	if (!node_online(i))
	445	setup_node_bootmem(i, nodes[i].start, nodes[i].end);
	446
	447	for (i = 0; i < nr_cpu_ids; i++) {
	448	int node = early_cpu_to_node(i);
	449
	450	if (node == NUMA_NO_NODE)
	451	continue;
	452	if (!node_online(node))
	453	numa_clear_node(i);
	454	}
	455	numa_init_array();
	456	return 0;
	457	}
	458
	459	#ifdef CONFIG_NUMA_EMU
	460	static int fake_node_to_pxm_map[MAX_NUMNODES] __initdata = {
	461	[0 ... MAX_NUMNODES-1] = PXM_INVAL
	462	};
	463	static s16 fake_apicid_to_node[MAX_LOCAL_APIC] __initdata = {
	464	[0 ... MAX_LOCAL_APIC-1] = NUMA_NO_NODE
	465	};
	466	static int __init find_node_by_addr(unsigned long addr)
	467	{
	468	int ret = NUMA_NO_NODE;
	469	int i;
	470
	471	for_each_node_mask(i, nodes_parsed) {
	472	/*
	473	* Find the real node that this emulated node appears on. For
	474	* the sake of simplicity, we only use a real node's starting
	475	* address to determine which emulated node it appears on.
	476	*/
	477	if (addr >= nodes[i].start && addr < nodes[i].end) {
	478	ret = i;
	479	break;
	480	}
	481	}
	482	return ret;
	483	}
	484
	485	/*
	486	* In NUMA emulation, we need to setup proximity domain (_PXM) to node ID
	487	* mappings that respect the real ACPI topology but reflect our emulated
	488	* environment. For each emulated node, we find which real node it appears on
	489	* and create PXM to NID mappings for those fake nodes which mirror that
	490	* locality. SLIT will now represent the correct distances between emulated
	491	* nodes as a result of the real topology.
	492	*/
	493	void __init acpi_fake_nodes(const struct bootnode *fake_nodes, int num_nodes)
	494	{
	495	int i, j;
	496
	497	printk(KERN_INFO "Faking PXM affinity for fake nodes on real "
	498	"topology.\n");
	499	for (i = 0; i < num_nodes; i++) {
	500	int nid, pxm;
	501
	502	nid = find_node_by_addr(fake_nodes[i].start);
	503	if (nid == NUMA_NO_NODE)
	504	continue;
	505	pxm = node_to_pxm(nid);
	506	if (pxm == PXM_INVAL)
	507	continue;
	508	fake_node_to_pxm_map[i] = pxm;
	509	/*
	510	* For each apicid_to_node mapping that exists for this real
	511	* node, it must now point to the fake node ID.
	512	*/
	513	for (j = 0; j < MAX_LOCAL_APIC; j++)
	514	if (apicid_to_node[j] == nid &&
	515	fake_apicid_to_node[j] == NUMA_NO_NODE)
	516	fake_apicid_to_node[j] = i;
	517	}
	518	for (i = 0; i < num_nodes; i++)
	519	__acpi_map_pxm_to_node(fake_node_to_pxm_map[i], i);
	520	memcpy(apicid_to_node, fake_apicid_to_node, sizeof(apicid_to_node));
	521
	522	nodes_clear(nodes_parsed);
	523	for (i = 0; i < num_nodes; i++)
	524	if (fake_nodes[i].start != fake_nodes[i].end)
	525	node_set(i, nodes_parsed);
	526	}
	527
	528	static int null_slit_node_compare(int a, int b)
	529	{
	530	return node_to_pxm(a) == node_to_pxm(b);
	531	}
	532	#else
	533	static int null_slit_node_compare(int a, int b)
	534	{
	535	return a == b;
	536	}
	537	#endif /* CONFIG_NUMA_EMU */
	538
	539	int __node_distance(int a, int b)
	540	{
	541	int index;
	542
	543	if (!acpi_slit)
	544	return null_slit_node_compare(a, b) ? LOCAL_DISTANCE :
	545	REMOTE_DISTANCE;
	546	index = acpi_slit->locality_count * node_to_pxm(a);
	547	return acpi_slit->entry[index + node_to_pxm(b)];
	548	}
	549
	550	EXPORT_SYMBOL(__node_distance);
	551
	552	#if defined(CONFIG_MEMORY_HOTPLUG_SPARSE) \|\| defined(CONFIG_ACPI_HOTPLUG_MEMORY)
	553	int memory_add_physaddr_to_nid(u64 start)
	554	{
	555	int i, ret = 0;
	556
	557	for_each_node(i)
	558	if (nodes_add[i].start <= start && nodes_add[i].end > start)
	559	ret = i;
	560
	561	return ret;
	562	}
	563	EXPORT_SYMBOL_GPL(memory_add_physaddr_to_nid);
	564	#endif