[net-next-2.6.git] / arch / x86 / kernel / setup.c

#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/bootmem.h>
#include <linux/percpu.h>
#include <asm/smp.h>
#include <asm/percpu.h>
#include <asm/sections.h>
#include <asm/processor.h>
#include <asm/setup.h>
#include <asm/topology.h>
#include <asm/mpspec.h>
#include <asm/apicdef.h>

#ifdef CONFIG_X86_LOCAL_APIC
unsigned int num_processors;
unsigned disabled_cpus __cpuinitdata;
/* Processor that is doing the boot up */
unsigned int boot_cpu_physical_apicid = -1U;
unsigned int max_physical_apicid;
EXPORT_SYMBOL(boot_cpu_physical_apicid);

/* Bitmask of physically existing CPUs */
physid_mask_t phys_cpu_present_map;
#endif

/* map cpu index to physical APIC ID */
DEFINE_EARLY_PER_CPU(u16, x86_cpu_to_apicid, BAD_APICID);
DEFINE_EARLY_PER_CPU(u16, x86_bios_cpu_apicid, BAD_APICID);
EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_apicid);
EXPORT_EARLY_PER_CPU_SYMBOL(x86_bios_cpu_apicid);

#if defined(CONFIG_NUMA) && defined(CONFIG_X86_64)
#define	X86_64_NUMA	1

/* map cpu index to node index */
DEFINE_EARLY_PER_CPU(int, x86_cpu_to_node_map, NUMA_NO_NODE);
EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_node_map);

/* which logical CPUs are on which nodes */
cpumask_t *node_to_cpumask_map;
EXPORT_SYMBOL(node_to_cpumask_map);

/* setup node_to_cpumask_map */
static void __init setup_node_to_cpumask_map(void);

#else
static inline void setup_node_to_cpumask_map(void) { }
#endif

#if defined(CONFIG_HAVE_SETUP_PER_CPU_AREA) && defined(CONFIG_X86_SMP)
/*
 * Copy data used in early init routines from the initial arrays to the
 * per cpu data areas.  These arrays then become expendable and the
 * *_early_ptr's are zeroed indicating that the static arrays are gone.
 */
static void __init setup_per_cpu_maps(void)
{
	int cpu;

	for_each_possible_cpu(cpu) {
		per_cpu(x86_cpu_to_apicid, cpu) =
				early_per_cpu_map(x86_cpu_to_apicid, cpu);
		per_cpu(x86_bios_cpu_apicid, cpu) =
				early_per_cpu_map(x86_bios_cpu_apicid, cpu);
#ifdef X86_64_NUMA
		per_cpu(x86_cpu_to_node_map, cpu) =
				early_per_cpu_map(x86_cpu_to_node_map, cpu);
#endif
	}

	/* indicate the early static arrays will soon be gone */
	early_per_cpu_ptr(x86_cpu_to_apicid) = NULL;
	early_per_cpu_ptr(x86_bios_cpu_apicid) = NULL;
#ifdef X86_64_NUMA
	early_per_cpu_ptr(x86_cpu_to_node_map) = NULL;
#endif
}

#ifdef CONFIG_HAVE_CPUMASK_OF_CPU_MAP
cpumask_t *cpumask_of_cpu_map __read_mostly;
EXPORT_SYMBOL(cpumask_of_cpu_map);

/* requires nr_cpu_ids to be initialized */
static void __init setup_cpumask_of_cpu(void)
{
	int i;

	/* alloc_bootmem zeroes memory */
	cpumask_of_cpu_map = alloc_bootmem_low(sizeof(cpumask_t) * nr_cpu_ids);
	for (i = 0; i < nr_cpu_ids; i++)
		cpu_set(i, cpumask_of_cpu_map[i]);
}
#else
static inline void setup_cpumask_of_cpu(void) { }
#endif

#ifdef CONFIG_X86_32
/*
 * Great future not-so-futuristic plan: make i386 and x86_64 do it
 * the same way
 */
unsigned long __per_cpu_offset[NR_CPUS] __read_mostly;
EXPORT_SYMBOL(__per_cpu_offset);
static inline void setup_cpu_pda_map(void) { }

#elif !defined(CONFIG_SMP)
static inline void setup_cpu_pda_map(void) { }

#else /* CONFIG_SMP && CONFIG_X86_64 */

/*
 * Allocate cpu_pda pointer table and array via alloc_bootmem.
 */
static void __init setup_cpu_pda_map(void)
{
	char *pda;
	struct x8664_pda **new_cpu_pda;
	unsigned long size;
	int cpu;

	size = roundup(sizeof(struct x8664_pda), cache_line_size());

	/* allocate cpu_pda array and pointer table */
	{
		unsigned long tsize = nr_cpu_ids * sizeof(void *);
		unsigned long asize = size * (nr_cpu_ids - 1);

		tsize = roundup(tsize, cache_line_size());
		new_cpu_pda = alloc_bootmem(tsize + asize);
		pda = (char *)new_cpu_pda + tsize;
	}

	/* initialize pointer table to static pda's */
	for_each_possible_cpu(cpu) {
		if (cpu == 0) {
			/* leave boot cpu pda in place */
			new_cpu_pda[0] = cpu_pda(0);
			continue;
		}
		new_cpu_pda[cpu] = (struct x8664_pda *)pda;
		new_cpu_pda[cpu]->in_bootmem = 1;
		pda += size;
	}

	/* point to new pointer table */
	_cpu_pda = new_cpu_pda;
}
#endif

/*
 * Great future plan:
 * Declare PDA itself and support (irqstack,tss,pgd) as per cpu data.
 * Always point %gs to its beginning
 */
void __init setup_per_cpu_areas(void)
{
	ssize_t size = PERCPU_ENOUGH_ROOM;
	char *ptr;
	int cpu;

#ifdef CONFIG_HOTPLUG_CPU
	prefill_possible_map();
#else
	nr_cpu_ids = num_processors;
#endif

	/* Setup cpu_pda map */
	setup_cpu_pda_map();

	/* Copy section for each CPU (we discard the original) */
	size = PERCPU_ENOUGH_ROOM;
	printk(KERN_INFO "PERCPU: Allocating %zd bytes of per cpu data\n",
			  size);

	for_each_possible_cpu(cpu) {
#ifndef CONFIG_NEED_MULTIPLE_NODES
		ptr = alloc_bootmem_pages(size);
#else
		int node = early_cpu_to_node(cpu);
		if (!node_online(node) || !NODE_DATA(node)) {
			ptr = alloc_bootmem_pages(size);
			printk(KERN_INFO
			       "cpu %d has no node %d or node-local memory\n",
				cpu, node);
		}
		else
			ptr = alloc_bootmem_pages_node(NODE_DATA(node), size);
#endif
		per_cpu_offset(cpu) = ptr - __per_cpu_start;
		memcpy(ptr, __per_cpu_start, __per_cpu_end - __per_cpu_start);

	}

	printk(KERN_DEBUG "NR_CPUS: %d, nr_cpu_ids: %d, nr_node_ids %d\n",
		NR_CPUS, nr_cpu_ids, nr_node_ids);

	/* Setup percpu data maps */
	setup_per_cpu_maps();

	/* Setup node to cpumask map */
	setup_node_to_cpumask_map();

	/* Setup cpumask_of_cpu map */
	setup_cpumask_of_cpu();
}

#endif

void __init parse_setup_data(void)
{
	struct setup_data *data;
	u64 pa_data;

	if (boot_params.hdr.version < 0x0209)
		return;
	pa_data = boot_params.hdr.setup_data;
	while (pa_data) {
		data = early_ioremap(pa_data, PAGE_SIZE);
		switch (data->type) {
		case SETUP_E820_EXT:
			parse_e820_ext(data, pa_data);
			break;
		default:
			break;
		}
#ifndef CONFIG_DEBUG_BOOT_PARAMS
		free_early(pa_data, pa_data+sizeof(*data)+data->len);
#endif
		pa_data = data->next;
		early_iounmap(data, PAGE_SIZE);
	}
}

#ifdef X86_64_NUMA

/*
 * Allocate node_to_cpumask_map based on number of available nodes
 * Requires node_possible_map to be valid.
 *
 * Note: node_to_cpumask() is not valid until after this is done.
 */
static void __init setup_node_to_cpumask_map(void)
{
	unsigned int node, num = 0;
	cpumask_t *map;

	/* setup nr_node_ids if not done yet */
	if (nr_node_ids == MAX_NUMNODES) {
		for_each_node_mask(node, node_possible_map)
			num = node;
		nr_node_ids = num + 1;
	}

	/* allocate the map */
	map = alloc_bootmem_low(nr_node_ids * sizeof(cpumask_t));

	Dprintk(KERN_DEBUG "Node to cpumask map at %p for %d nodes\n",
		map, nr_node_ids);

	/* node_to_cpumask() will now work */
	node_to_cpumask_map = map;
}

void __cpuinit numa_set_node(int cpu, int node)
{
	int *cpu_to_node_map = early_per_cpu_ptr(x86_cpu_to_node_map);

	if (cpu_pda(cpu) && node != NUMA_NO_NODE)
		cpu_pda(cpu)->nodenumber = node;

	if (cpu_to_node_map)
		cpu_to_node_map[cpu] = node;

	else if (per_cpu_offset(cpu))
		per_cpu(x86_cpu_to_node_map, cpu) = node;

	else
		Dprintk(KERN_INFO "Setting node for non-present cpu %d\n", cpu);
}

void __cpuinit numa_clear_node(int cpu)
{
	numa_set_node(cpu, NUMA_NO_NODE);
}

#ifndef CONFIG_DEBUG_PER_CPU_MAPS

void __cpuinit numa_add_cpu(int cpu)
{
	cpu_set(cpu, node_to_cpumask_map[early_cpu_to_node(cpu)]);
}

void __cpuinit numa_remove_cpu(int cpu)
{
	cpu_clear(cpu, node_to_cpumask_map[cpu_to_node(cpu)]);
}

#else /* CONFIG_DEBUG_PER_CPU_MAPS */

/*
 * --------- debug versions of the numa functions ---------
 */
static void __cpuinit numa_set_cpumask(int cpu, int enable)
{
	int node = cpu_to_node(cpu);
	cpumask_t *mask;
	char buf[64];

	if (node_to_cpumask_map == NULL) {
		printk(KERN_ERR "node_to_cpumask_map NULL\n");
		dump_stack();
		return;
	}

	mask = &node_to_cpumask_map[node];
	if (enable)
		cpu_set(cpu, *mask);
	else
		cpu_clear(cpu, *mask);

	cpulist_scnprintf(buf, sizeof(buf), *mask);
	printk(KERN_DEBUG "%s cpu %d node %d: mask now %s\n",
		enable? "numa_add_cpu":"numa_remove_cpu", cpu, node, buf);
 }

void __cpuinit numa_add_cpu(int cpu)
{
	numa_set_cpumask(cpu, 1);
}

void __cpuinit numa_remove_cpu(int cpu)
{
	numa_set_cpumask(cpu, 0);
}

int cpu_to_node(int cpu)
{
	if (early_per_cpu_ptr(x86_cpu_to_node_map)) {
		printk(KERN_WARNING
			"cpu_to_node(%d): usage too early!\n", cpu);
		dump_stack();
		return early_per_cpu_ptr(x86_cpu_to_node_map)[cpu];
	}
	return per_cpu(x86_cpu_to_node_map, cpu);
}
EXPORT_SYMBOL(cpu_to_node);

/*
 * Same function as cpu_to_node() but used if called before the
 * per_cpu areas are setup.
 */
int early_cpu_to_node(int cpu)
{
	if (early_per_cpu_ptr(x86_cpu_to_node_map))
		return early_per_cpu_ptr(x86_cpu_to_node_map)[cpu];

	if (!per_cpu_offset(cpu)) {
		printk(KERN_WARNING
			"early_cpu_to_node(%d): no per_cpu area!\n", cpu);
		dump_stack();
		return NUMA_NO_NODE;
	}
	return per_cpu(x86_cpu_to_node_map, cpu);
}

/*
 * Returns a pointer to the bitmask of CPUs on Node 'node'.
 */
cpumask_t *_node_to_cpumask_ptr(int node)
{
	if (node_to_cpumask_map == NULL) {
		printk(KERN_WARNING
			"_node_to_cpumask_ptr(%d): no node_to_cpumask_map!\n",
			node);
		dump_stack();
		return &cpu_online_map;
	}
	BUG_ON(node >= nr_node_ids);
	return &node_to_cpumask_map[node];
}
EXPORT_SYMBOL(_node_to_cpumask_ptr);

/*
 * Returns a bitmask of CPUs on Node 'node'.
 */
cpumask_t node_to_cpumask(int node)
{
	if (node_to_cpumask_map == NULL) {
		printk(KERN_WARNING
			"node_to_cpumask(%d): no node_to_cpumask_map!\n", node);
		dump_stack();
		return cpu_online_map;
	}
	BUG_ON(node >= nr_node_ids);
	return node_to_cpumask_map[node];
}
EXPORT_SYMBOL(node_to_cpumask);

/*
 * --------- end of debug versions of the numa functions ---------
 */

#endif /* CONFIG_DEBUG_PER_CPU_MAPS */

#endif /* X86_64_NUMA */
Commit	Line	Data
4fe29a85 GOC	1	#include <linux/kernel.h>
	2	#include <linux/module.h>
	3	#include <linux/init.h>
	4	#include <linux/bootmem.h>
	5	#include <linux/percpu.h>
	6	#include <asm/smp.h>
	7	#include <asm/percpu.h>
	8	#include <asm/sections.h>
	9	#include <asm/processor.h>
	10	#include <asm/setup.h>
	11	#include <asm/topology.h>
0fc0906e	12	#include <asm/mpspec.h>
76eb4131 AS	13	#include <asm/apicdef.h>
76eb4131 AS	14
f8955ebe	15	#ifdef CONFIG_X86_LOCAL_APIC
2fe60147 AS	16	unsigned int num_processors;
	17	unsigned disabled_cpus __cpuinitdata;
	18	/* Processor that is doing the boot up */
	19	unsigned int boot_cpu_physical_apicid = -1U;
e0da3364	20	unsigned int max_physical_apicid;
2fe60147 AS	21	EXPORT_SYMBOL(boot_cpu_physical_apicid);
2fe60147 AS	22
0fc0906e AS	23	/* Bitmask of physically existing CPUs */
0fc0906e AS	24	physid_mask_t phys_cpu_present_map;
f8955ebe	25	#endif
0fc0906e	26
23ca4bba MT	27	/* map cpu index to physical APIC ID */
	28	DEFINE_EARLY_PER_CPU(u16, x86_cpu_to_apicid, BAD_APICID);
	29	DEFINE_EARLY_PER_CPU(u16, x86_bios_cpu_apicid, BAD_APICID);
	30	EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_apicid);
	31	EXPORT_EARLY_PER_CPU_SYMBOL(x86_bios_cpu_apicid);
	32
	33	#if defined(CONFIG_NUMA) && defined(CONFIG_X86_64)
	34	#define X86_64_NUMA 1
	35
7891a24e	36	/* map cpu index to node index */
23ca4bba MT	37	DEFINE_EARLY_PER_CPU(int, x86_cpu_to_node_map, NUMA_NO_NODE);
23ca4bba MT	38	EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_node_map);
9f248bde MT	39
	40	/* which logical CPUs are on which nodes */
	41	cpumask_t *node_to_cpumask_map;
	42	EXPORT_SYMBOL(node_to_cpumask_map);
	43
	44	/* setup node_to_cpumask_map */
	45	static void __init setup_node_to_cpumask_map(void);
	46
	47	#else
	48	static inline void setup_node_to_cpumask_map(void) { }
23ca4bba MT	49	#endif
23ca4bba MT	50
f8955ebe	51	#if defined(CONFIG_HAVE_SETUP_PER_CPU_AREA) && defined(CONFIG_X86_SMP)
4fe29a85 GOC	52	/*
	53	* Copy data used in early init routines from the initial arrays to the
	54	* per cpu data areas. These arrays then become expendable and the
	55	* *_early_ptr's are zeroed indicating that the static arrays are gone.
	56	*/
	57	static void __init setup_per_cpu_maps(void)
	58	{
	59	int cpu;
	60
	61	for_each_possible_cpu(cpu) {
23ca4bba MT	62	per_cpu(x86_cpu_to_apicid, cpu) =
23ca4bba MT	63	early_per_cpu_map(x86_cpu_to_apicid, cpu);
b447a468	64	per_cpu(x86_bios_cpu_apicid, cpu) =
23ca4bba MT	65	early_per_cpu_map(x86_bios_cpu_apicid, cpu);
23ca4bba MT	66	#ifdef X86_64_NUMA
b447a468	67	per_cpu(x86_cpu_to_node_map, cpu) =
23ca4bba	68	early_per_cpu_map(x86_cpu_to_node_map, cpu);
4fe29a85 GOC	69	#endif
	70	}
	71
	72	/* indicate the early static arrays will soon be gone */
23ca4bba MT	73	early_per_cpu_ptr(x86_cpu_to_apicid) = NULL;
	74	early_per_cpu_ptr(x86_bios_cpu_apicid) = NULL;
	75	#ifdef X86_64_NUMA
	76	early_per_cpu_ptr(x86_cpu_to_node_map) = NULL;
4fe29a85 GOC	77	#endif
	78	}
	79
9f0e8d04 MT	80	#ifdef CONFIG_HAVE_CPUMASK_OF_CPU_MAP
	81	cpumask_t *cpumask_of_cpu_map __read_mostly;
	82	EXPORT_SYMBOL(cpumask_of_cpu_map);
	83
	84	/* requires nr_cpu_ids to be initialized */
	85	static void __init setup_cpumask_of_cpu(void)
	86	{
	87	int i;
	88
	89	/* alloc_bootmem zeroes memory */
	90	cpumask_of_cpu_map = alloc_bootmem_low(sizeof(cpumask_t) * nr_cpu_ids);
	91	for (i = 0; i < nr_cpu_ids; i++)
	92	cpu_set(i, cpumask_of_cpu_map[i]);
	93	}
	94	#else
	95	static inline void setup_cpumask_of_cpu(void) { }
	96	#endif
	97
4fe29a85 GOC	98	#ifdef CONFIG_X86_32
	99	/*
	100	* Great future not-so-futuristic plan: make i386 and x86_64 do it
	101	* the same way
	102	*/
	103	unsigned long __per_cpu_offset[NR_CPUS] __read_mostly;
	104	EXPORT_SYMBOL(__per_cpu_offset);
3461b0af MT	105	static inline void setup_cpu_pda_map(void) { }
	106
	107	#elif !defined(CONFIG_SMP)
	108	static inline void setup_cpu_pda_map(void) { }
	109
	110	#else /* CONFIG_SMP && CONFIG_X86_64 */
	111
	112	/*
	113	* Allocate cpu_pda pointer table and array via alloc_bootmem.
	114	*/
	115	static void __init setup_cpu_pda_map(void)
	116	{
	117	char *pda;
	118	struct x8664_pda **new_cpu_pda;
	119	unsigned long size;
	120	int cpu;
	121
	122	size = roundup(sizeof(struct x8664_pda), cache_line_size());
	123
	124	/* allocate cpu_pda array and pointer table */
	125	{
	126	unsigned long tsize = nr_cpu_ids * sizeof(void *);
	127	unsigned long asize = size * (nr_cpu_ids - 1);
	128
	129	tsize = roundup(tsize, cache_line_size());
	130	new_cpu_pda = alloc_bootmem(tsize + asize);
	131	pda = (char *)new_cpu_pda + tsize;
	132	}
	133
	134	/* initialize pointer table to static pda's */
	135	for_each_possible_cpu(cpu) {
	136	if (cpu == 0) {
	137	/* leave boot cpu pda in place */
	138	new_cpu_pda[0] = cpu_pda(0);
	139	continue;
	140	}
	141	new_cpu_pda[cpu] = (struct x8664_pda *)pda;
	142	new_cpu_pda[cpu]->in_bootmem = 1;
	143	pda += size;
	144	}
	145
	146	/* point to new pointer table */
	147	_cpu_pda = new_cpu_pda;
	148	}
4fe29a85 GOC	149	#endif
	150
	151	/*
	152	* Great future plan:
	153	* Declare PDA itself and support (irqstack,tss,pgd) as per cpu data.
	154	* Always point %gs to its beginning
	155	*/
	156	void __init setup_per_cpu_areas(void)
	157	{
3461b0af MT	158	ssize_t size = PERCPU_ENOUGH_ROOM;
	159	char *ptr;
	160	int cpu;
4fe29a85 GOC	161
	162	#ifdef CONFIG_HOTPLUG_CPU
	163	prefill_possible_map();
3461b0af MT	164	#else
3461b0af MT	165	nr_cpu_ids = num_processors;
4fe29a85 GOC	166	#endif
4fe29a85 GOC	167
3461b0af MT	168	/* Setup cpu_pda map */
	169	setup_cpu_pda_map();
	170
4fe29a85 GOC	171	/* Copy section for each CPU (we discard the original) */
4fe29a85 GOC	172	size = PERCPU_ENOUGH_ROOM;
053713f5	173	printk(KERN_INFO "PERCPU: Allocating %zd bytes of per cpu data\n",
4fe29a85	174	size);
b447a468	175
3461b0af	176	for_each_possible_cpu(cpu) {
4fe29a85 GOC	177	#ifndef CONFIG_NEED_MULTIPLE_NODES
	178	ptr = alloc_bootmem_pages(size);
	179	#else
3461b0af	180	int node = early_cpu_to_node(cpu);
b447a468	181	if (!node_online(node) \|\| !NODE_DATA(node)) {
4fe29a85	182	ptr = alloc_bootmem_pages(size);
b447a468	183	printk(KERN_INFO
23ca4bba	184	"cpu %d has no node %d or node-local memory\n",
3461b0af	185	cpu, node);
b447a468	186	}
4fe29a85 GOC	187	else
	188	ptr = alloc_bootmem_pages_node(NODE_DATA(node), size);
	189	#endif
3461b0af	190	per_cpu_offset(cpu) = ptr - __per_cpu_start;
4fe29a85	191	memcpy(ptr, __per_cpu_start, __per_cpu_end - __per_cpu_start);
9f0e8d04	192
4fe29a85 GOC	193	}
4fe29a85 GOC	194
9f248bde MT	195	printk(KERN_DEBUG "NR_CPUS: %d, nr_cpu_ids: %d, nr_node_ids %d\n",
9f248bde MT	196	NR_CPUS, nr_cpu_ids, nr_node_ids);
9f0e8d04	197
b447a468	198	/* Setup percpu data maps */
4fe29a85	199	setup_per_cpu_maps();
9f0e8d04	200
9f248bde MT	201	/* Setup node to cpumask map */
	202	setup_node_to_cpumask_map();
	203
9f0e8d04 MT	204	/* Setup cpumask_of_cpu map */
9f0e8d04 MT	205	setup_cpumask_of_cpu();
4fe29a85 GOC	206	}
	207
	208	#endif
c45a707d HY	209
	210	void __init parse_setup_data(void)
	211	{
	212	struct setup_data *data;
	213	u64 pa_data;
	214
	215	if (boot_params.hdr.version < 0x0209)
	216	return;
	217	pa_data = boot_params.hdr.setup_data;
	218	while (pa_data) {
	219	data = early_ioremap(pa_data, PAGE_SIZE);
	220	switch (data->type) {
8c5beb50 HY	221	case SETUP_E820_EXT:
	222	parse_e820_ext(data, pa_data);
	223	break;
c45a707d HY	224	default:
	225	break;
	226	}
	227	#ifndef CONFIG_DEBUG_BOOT_PARAMS
	228	free_early(pa_data, pa_data+sizeof(*data)+data->len);
	229	#endif
	230	pa_data = data->next;
	231	early_iounmap(data, PAGE_SIZE);
	232	}
	233	}
2b4fa851	234
23ca4bba	235	#ifdef X86_64_NUMA
9f248bde MT	236
	237	/*
	238	* Allocate node_to_cpumask_map based on number of available nodes
	239	* Requires node_possible_map to be valid.
	240	*
	241	* Note: node_to_cpumask() is not valid until after this is done.
	242	*/
	243	static void __init setup_node_to_cpumask_map(void)
	244	{
	245	unsigned int node, num = 0;
	246	cpumask_t *map;
	247
	248	/* setup nr_node_ids if not done yet */
	249	if (nr_node_ids == MAX_NUMNODES) {
	250	for_each_node_mask(node, node_possible_map)
	251	num = node;
	252	nr_node_ids = num + 1;
	253	}
	254
	255	/* allocate the map */
	256	map = alloc_bootmem_low(nr_node_ids * sizeof(cpumask_t));
	257
	258	Dprintk(KERN_DEBUG "Node to cpumask map at %p for %d nodes\n",
	259	map, nr_node_ids);
	260
	261	/* node_to_cpumask() will now work */
	262	node_to_cpumask_map = map;
	263	}
	264
23ca4bba MT	265	void __cpuinit numa_set_node(int cpu, int node)
	266	{
	267	int *cpu_to_node_map = early_per_cpu_ptr(x86_cpu_to_node_map);
	268
3461b0af	269	if (cpu_pda(cpu) && node != NUMA_NO_NODE)
7891a24e MT	270	cpu_pda(cpu)->nodenumber = node;
7891a24e MT	271
23ca4bba MT	272	if (cpu_to_node_map)
	273	cpu_to_node_map[cpu] = node;
	274
	275	else if (per_cpu_offset(cpu))
	276	per_cpu(x86_cpu_to_node_map, cpu) = node;
	277
	278	else
	279	Dprintk(KERN_INFO "Setting node for non-present cpu %d\n", cpu);
	280	}
	281
	282	void __cpuinit numa_clear_node(int cpu)
	283	{
	284	numa_set_node(cpu, NUMA_NO_NODE);
	285	}
	286
9f248bde MT	287	#ifndef CONFIG_DEBUG_PER_CPU_MAPS
9f248bde MT	288
23ca4bba MT	289	void __cpuinit numa_add_cpu(int cpu)
	290	{
	291	cpu_set(cpu, node_to_cpumask_map[early_cpu_to_node(cpu)]);
	292	}
	293
	294	void __cpuinit numa_remove_cpu(int cpu)
	295	{
	296	cpu_clear(cpu, node_to_cpumask_map[cpu_to_node(cpu)]);
	297	}
23ca4bba	298
9f248bde MT	299	#else /* CONFIG_DEBUG_PER_CPU_MAPS */
	300
	301	/*
	302	* --------- debug versions of the numa functions ---------
	303	*/
	304	static void __cpuinit numa_set_cpumask(int cpu, int enable)
	305	{
	306	int node = cpu_to_node(cpu);
	307	cpumask_t *mask;
	308	char buf[64];
	309
	310	if (node_to_cpumask_map == NULL) {
	311	printk(KERN_ERR "node_to_cpumask_map NULL\n");
	312	dump_stack();
	313	return;
	314	}
	315
	316	mask = &node_to_cpumask_map[node];
	317	if (enable)
	318	cpu_set(cpu, *mask);
	319	else
	320	cpu_clear(cpu, *mask);
	321
	322	cpulist_scnprintf(buf, sizeof(buf), *mask);
	323	printk(KERN_DEBUG "%s cpu %d node %d: mask now %s\n",
	324	enable? "numa_add_cpu":"numa_remove_cpu", cpu, node, buf);
	325	}
	326
	327	void __cpuinit numa_add_cpu(int cpu)
	328	{
	329	numa_set_cpumask(cpu, 1);
	330	}
	331
	332	void __cpuinit numa_remove_cpu(int cpu)
	333	{
	334	numa_set_cpumask(cpu, 0);
	335	}
23ca4bba MT	336
	337	int cpu_to_node(int cpu)
	338	{
	339	if (early_per_cpu_ptr(x86_cpu_to_node_map)) {
	340	printk(KERN_WARNING
	341	"cpu_to_node(%d): usage too early!\n", cpu);
	342	dump_stack();
	343	return early_per_cpu_ptr(x86_cpu_to_node_map)[cpu];
	344	}
	345	return per_cpu(x86_cpu_to_node_map, cpu);
	346	}
	347	EXPORT_SYMBOL(cpu_to_node);
	348
9f248bde MT	349	/*
	350	* Same function as cpu_to_node() but used if called before the
	351	* per_cpu areas are setup.
	352	*/
23ca4bba MT	353	int early_cpu_to_node(int cpu)
	354	{
	355	if (early_per_cpu_ptr(x86_cpu_to_node_map))
	356	return early_per_cpu_ptr(x86_cpu_to_node_map)[cpu];
	357
	358	if (!per_cpu_offset(cpu)) {
	359	printk(KERN_WARNING
	360	"early_cpu_to_node(%d): no per_cpu area!\n", cpu);
9f248bde	361	dump_stack();
23ca4bba MT	362	return NUMA_NO_NODE;
	363	}
	364	return per_cpu(x86_cpu_to_node_map, cpu);
	365	}
9f248bde MT	366
	367	/*
	368	* Returns a pointer to the bitmask of CPUs on Node 'node'.
	369	*/
	370	cpumask_t *_node_to_cpumask_ptr(int node)
	371	{
	372	if (node_to_cpumask_map == NULL) {
	373	printk(KERN_WARNING
	374	"_node_to_cpumask_ptr(%d): no node_to_cpumask_map!\n",
	375	node);
	376	dump_stack();
	377	return &cpu_online_map;
	378	}
03db1f74	379	BUG_ON(node >= nr_node_ids);
9f248bde MT	380	return &node_to_cpumask_map[node];
	381	}
	382	EXPORT_SYMBOL(_node_to_cpumask_ptr);
	383
	384	/*
	385	* Returns a bitmask of CPUs on Node 'node'.
	386	*/
	387	cpumask_t node_to_cpumask(int node)
	388	{
	389	if (node_to_cpumask_map == NULL) {
	390	printk(KERN_WARNING
	391	"node_to_cpumask(%d): no node_to_cpumask_map!\n", node);
	392	dump_stack();
	393	return cpu_online_map;
	394	}
03db1f74	395	BUG_ON(node >= nr_node_ids);
9f248bde MT	396	return node_to_cpumask_map[node];
	397	}
	398	EXPORT_SYMBOL(node_to_cpumask);
	399
	400	/*
	401	* --------- end of debug versions of the numa functions ---------
	402	*/
	403
	404	#endif /* CONFIG_DEBUG_PER_CPU_MAPS */
	405
	406	#endif /* X86_64_NUMA */