[net-next-2.6.git] / arch / x86 / kernel / setup_percpu.c

#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/bootmem.h>
#include <linux/percpu.h>
#include <linux/kexec.h>
#include <linux/crash_dump.h>
#include <linux/smp.h>
#include <linux/topology.h>
#include <linux/pfn.h>
#include <asm/sections.h>
#include <asm/processor.h>
#include <asm/setup.h>
#include <asm/mpspec.h>
#include <asm/apicdef.h>
#include <asm/highmem.h>
#include <asm/proto.h>
#include <asm/cpumask.h>
#include <asm/cpu.h>
#include <asm/stackprotector.h>

#ifdef CONFIG_DEBUG_PER_CPU_MAPS
# define DBG(x...) printk(KERN_DEBUG x)
#else
# define DBG(x...)
#endif

DEFINE_PER_CPU(int, cpu_number);
EXPORT_PER_CPU_SYMBOL(cpu_number);

#ifdef CONFIG_X86_64
#define BOOT_PERCPU_OFFSET ((unsigned long)__per_cpu_load)
#else
#define BOOT_PERCPU_OFFSET 0
#endif

DEFINE_PER_CPU(unsigned long, this_cpu_off) = BOOT_PERCPU_OFFSET;
EXPORT_PER_CPU_SYMBOL(this_cpu_off);

unsigned long __per_cpu_offset[NR_CPUS] __read_mostly = {
	[0 ... NR_CPUS-1] = BOOT_PERCPU_OFFSET,
};
EXPORT_SYMBOL(__per_cpu_offset);

/**
 * pcpu_need_numa - determine percpu allocation needs to consider NUMA
 *
 * If NUMA is not configured or there is only one NUMA node available,
 * there is no reason to consider NUMA.  This function determines
 * whether percpu allocation should consider NUMA or not.
 *
 * RETURNS:
 * true if NUMA should be considered; otherwise, false.
 */
static bool __init pcpu_need_numa(void)
{
#ifdef CONFIG_NEED_MULTIPLE_NODES
	pg_data_t *last = NULL;
	unsigned int cpu;

	for_each_possible_cpu(cpu) {
		int node = early_cpu_to_node(cpu);

		if (node_online(node) && NODE_DATA(node) &&
		    last && last != NODE_DATA(node))
			return true;

		last = NODE_DATA(node);
	}
#endif
	return false;
}

/**
 * pcpu_alloc_bootmem - NUMA friendly alloc_bootmem wrapper for percpu
 * @cpu: cpu to allocate for
 * @size: size allocation in bytes
 * @align: alignment
 *
 * Allocate @size bytes aligned at @align for cpu @cpu.  This wrapper
 * does the right thing for NUMA regardless of the current
 * configuration.
 *
 * RETURNS:
 * Pointer to the allocated area on success, NULL on failure.
 */
static void * __init pcpu_alloc_bootmem(unsigned int cpu, unsigned long size,
					unsigned long align)
{
	const unsigned long goal = __pa(MAX_DMA_ADDRESS);
#ifdef CONFIG_NEED_MULTIPLE_NODES
	int node = early_cpu_to_node(cpu);
	void *ptr;

	if (!node_online(node) || !NODE_DATA(node)) {
		ptr = __alloc_bootmem_nopanic(size, align, goal);
		pr_info("cpu %d has no node %d or node-local memory\n",
			cpu, node);
		pr_debug("per cpu data for cpu%d %lu bytes at %016lx\n",
			 cpu, size, __pa(ptr));
	} else {
		ptr = __alloc_bootmem_node_nopanic(NODE_DATA(node),
						   size, align, goal);
		pr_debug("per cpu data for cpu%d %lu bytes on node%d at "
			 "%016lx\n", cpu, size, node, __pa(ptr));
	}
	return ptr;
#else
	return __alloc_bootmem_nopanic(size, align, goal);
#endif
}

/*
 * Embedding allocator
 *
 * The first chunk is sized to just contain the static area plus
 * PERCPU_DYNAMIC_RESERVE and allocated as a contiguous area using
 * bootmem allocator and used as-is without being mapped into vmalloc
 * area.  This enables the first chunk to piggy back on the linear
 * physical PMD mapping and doesn't add any additional pressure to
 * TLB.
 */
static void *pcpue_ptr __initdata;
static size_t pcpue_unit_size __initdata;

static struct page * __init pcpue_get_page(unsigned int cpu, int pageno)
{
	return virt_to_page(pcpue_ptr + cpu * pcpue_unit_size
			    + ((size_t)pageno << PAGE_SHIFT));
}

static ssize_t __init setup_pcpu_embed(size_t static_size)
{
	unsigned int cpu;

	/*
	 * If large page isn't supported, there's no benefit in doing
	 * this.  Also, embedding allocation doesn't play well with
	 * NUMA.
	 */
	if (!cpu_has_pse || pcpu_need_numa())
		return -EINVAL;

	/* allocate and copy */
	pcpue_unit_size = PFN_ALIGN(static_size + PERCPU_DYNAMIC_RESERVE);
	pcpue_unit_size = max(pcpue_unit_size, PCPU_MIN_UNIT_SIZE);
	pcpue_ptr = pcpu_alloc_bootmem(0, num_possible_cpus() * pcpue_unit_size,
				       PAGE_SIZE);
	if (!pcpue_ptr)
		return -ENOMEM;

	for_each_possible_cpu(cpu)
		memcpy(pcpue_ptr + cpu * pcpue_unit_size, __per_cpu_load,
		       static_size);

	/* we're ready, commit */
	pr_info("PERCPU: Embedded %zu pages at %p, static data %zu bytes\n",
		pcpue_unit_size >> PAGE_SHIFT, pcpue_ptr, static_size);

	return pcpu_setup_first_chunk(pcpue_get_page, static_size,
				      pcpue_unit_size,
				      pcpue_unit_size - static_size, pcpue_ptr,
				      NULL);
}

/*
 * 4k page allocator
 *
 * This is the basic allocator.  Static percpu area is allocated
 * page-by-page and most of initialization is done by the generic
 * setup function.
 */
static struct page **pcpu4k_pages __initdata;
static int pcpu4k_nr_static_pages __initdata;

static struct page * __init pcpu4k_get_page(unsigned int cpu, int pageno)
{
	if (pageno < pcpu4k_nr_static_pages)
		return pcpu4k_pages[cpu * pcpu4k_nr_static_pages + pageno];
	return NULL;
}

static void __init pcpu4k_populate_pte(unsigned long addr)
{
	populate_extra_pte(addr);
}

static ssize_t __init setup_pcpu_4k(size_t static_size)
{
	size_t pages_size;
	unsigned int cpu;
	int i, j;
	ssize_t ret;

	pcpu4k_nr_static_pages = PFN_UP(static_size);

	/* unaligned allocations can't be freed, round up to page size */
	pages_size = PFN_ALIGN(pcpu4k_nr_static_pages * num_possible_cpus()
			       * sizeof(pcpu4k_pages[0]));
	pcpu4k_pages = alloc_bootmem(pages_size);

	/* allocate and copy */
	j = 0;
	for_each_possible_cpu(cpu)
		for (i = 0; i < pcpu4k_nr_static_pages; i++) {
			void *ptr;

			ptr = pcpu_alloc_bootmem(cpu, PAGE_SIZE, PAGE_SIZE);
			if (!ptr)
				goto enomem;

			memcpy(ptr, __per_cpu_load + i * PAGE_SIZE, PAGE_SIZE);
			pcpu4k_pages[j++] = virt_to_page(ptr);
		}

	/* we're ready, commit */
	pr_info("PERCPU: Allocated %d 4k pages, static data %zu bytes\n",
		pcpu4k_nr_static_pages, static_size);

	ret = pcpu_setup_first_chunk(pcpu4k_get_page, static_size, 0, 0, NULL,
				     pcpu4k_populate_pte);
	goto out_free_ar;

enomem:
	while (--j >= 0)
		free_bootmem(__pa(page_address(pcpu4k_pages[j])), PAGE_SIZE);
	ret = -ENOMEM;
out_free_ar:
	free_bootmem(__pa(pcpu4k_pages), pages_size);
	return ret;
}

static inline void setup_percpu_segment(int cpu)
{
#ifdef CONFIG_X86_32
	struct desc_struct gdt;

	pack_descriptor(&gdt, per_cpu_offset(cpu), 0xFFFFF,
			0x2 | DESCTYPE_S, 0x8);
	gdt.s = 1;
	write_gdt_entry(get_cpu_gdt_table(cpu),
			GDT_ENTRY_PERCPU, &gdt, DESCTYPE_S);
#endif
}

/*
 * Great future plan:
 * Declare PDA itself and support (irqstack,tss,pgd) as per cpu data.
 * Always point %gs to its beginning
 */
void __init setup_per_cpu_areas(void)
{
	size_t static_size = __per_cpu_end - __per_cpu_start;
	unsigned int cpu;
	unsigned long delta;
	size_t pcpu_unit_size;
	ssize_t ret;

	pr_info("NR_CPUS:%d nr_cpumask_bits:%d nr_cpu_ids:%d nr_node_ids:%d\n",
		NR_CPUS, nr_cpumask_bits, nr_cpu_ids, nr_node_ids);

	/* allocate percpu area */
	ret = setup_pcpu_embed(static_size);
	if (ret < 0)
		ret = setup_pcpu_4k(static_size);
	if (ret < 0)
		panic("cannot allocate static percpu area (%zu bytes, err=%zd)",
		      static_size, ret);

	pcpu_unit_size = ret;

	/* alrighty, percpu areas up and running */
	delta = (unsigned long)pcpu_base_addr - (unsigned long)__per_cpu_start;
	for_each_possible_cpu(cpu) {
		per_cpu_offset(cpu) = delta + cpu * pcpu_unit_size;
		per_cpu(this_cpu_off, cpu) = per_cpu_offset(cpu);
		per_cpu(cpu_number, cpu) = cpu;
		setup_percpu_segment(cpu);
		setup_stack_canary_segment(cpu);
		/*
		 * Copy data used in early init routines from the
		 * initial arrays to the per cpu data areas.  These
		 * arrays then become expendable and the *_early_ptr's
		 * are zeroed indicating that the static arrays are
		 * gone.
		 */
#ifdef CONFIG_X86_LOCAL_APIC
		per_cpu(x86_cpu_to_apicid, cpu) =
			early_per_cpu_map(x86_cpu_to_apicid, cpu);
		per_cpu(x86_bios_cpu_apicid, cpu) =
			early_per_cpu_map(x86_bios_cpu_apicid, cpu);
#endif
#ifdef CONFIG_X86_64
		per_cpu(irq_stack_ptr, cpu) =
			per_cpu(irq_stack_union.irq_stack, cpu) +
			IRQ_STACK_SIZE - 64;
#ifdef CONFIG_NUMA
		per_cpu(x86_cpu_to_node_map, cpu) =
			early_per_cpu_map(x86_cpu_to_node_map, cpu);
#endif
#endif
		/*
		 * Up to this point, the boot CPU has been using .data.init
		 * area.  Reload any changed state for the boot CPU.
		 */
		if (cpu == boot_cpu_id)
			switch_to_new_gdt(cpu);

		DBG("PERCPU: cpu %4d %p\n", cpu, ptr);
	}

	/* indicate the early static arrays will soon be gone */
#ifdef CONFIG_X86_LOCAL_APIC
	early_per_cpu_ptr(x86_cpu_to_apicid) = NULL;
	early_per_cpu_ptr(x86_bios_cpu_apicid) = NULL;
#endif
#if defined(CONFIG_X86_64) && defined(CONFIG_NUMA)
	early_per_cpu_ptr(x86_cpu_to_node_map) = NULL;
#endif

	/* Setup node to cpumask map */
	setup_node_to_cpumask_map();

	/* Setup cpu initialized, callin, callout masks */
	setup_cpu_local_masks();
}
Commit	Line	Data
4fe29a85 GOC	1	#include <linux/kernel.h>
	2	#include <linux/module.h>
	3	#include <linux/init.h>
	4	#include <linux/bootmem.h>
	5	#include <linux/percpu.h>
1ecd2765	6	#include <linux/kexec.h>
17b4cceb	7	#include <linux/crash_dump.h>
8a87dd9a JSR	8	#include <linux/smp.h>
8a87dd9a JSR	9	#include <linux/topology.h>
5f5d8405	10	#include <linux/pfn.h>
4fe29a85 GOC	11	#include <asm/sections.h>
	12	#include <asm/processor.h>
	13	#include <asm/setup.h>
0fc0906e	14	#include <asm/mpspec.h>
76eb4131	15	#include <asm/apicdef.h>
1ecd2765	16	#include <asm/highmem.h>
1a51e3a0	17	#include <asm/proto.h>
06879033	18	#include <asm/cpumask.h>
34019be1	19	#include <asm/cpu.h>
60a5317f	20	#include <asm/stackprotector.h>
76eb4131	21
c90aa894 MT	22	#ifdef CONFIG_DEBUG_PER_CPU_MAPS
	23	# define DBG(x...) printk(KERN_DEBUG x)
	24	#else
	25	# define DBG(x...)
	26	#endif
	27
ea927906 BG	28	DEFINE_PER_CPU(int, cpu_number);
ea927906 BG	29	EXPORT_PER_CPU_SYMBOL(cpu_number);
ea927906	30
1688401a BG	31	#ifdef CONFIG_X86_64
	32	#define BOOT_PERCPU_OFFSET ((unsigned long)__per_cpu_load)
	33	#else
	34	#define BOOT_PERCPU_OFFSET 0
	35	#endif
	36
	37	DEFINE_PER_CPU(unsigned long, this_cpu_off) = BOOT_PERCPU_OFFSET;
	38	EXPORT_PER_CPU_SYMBOL(this_cpu_off);
	39
9939ddaf	40	unsigned long __per_cpu_offset[NR_CPUS] __read_mostly = {
34019be1	41	[0 ... NR_CPUS-1] = BOOT_PERCPU_OFFSET,
9939ddaf	42	};
9939ddaf	43	EXPORT_SYMBOL(__per_cpu_offset);
4fe29a85	44
89c92151 TH	45	/**
	46	* pcpu_need_numa - determine percpu allocation needs to consider NUMA
	47	*
	48	* If NUMA is not configured or there is only one NUMA node available,
	49	* there is no reason to consider NUMA. This function determines
	50	* whether percpu allocation should consider NUMA or not.
	51	*
	52	* RETURNS:
	53	* true if NUMA should be considered; otherwise, false.
	54	*/
	55	static bool __init pcpu_need_numa(void)
	56	{
	57	#ifdef CONFIG_NEED_MULTIPLE_NODES
	58	pg_data_t *last = NULL;
	59	unsigned int cpu;
	60
	61	for_each_possible_cpu(cpu) {
	62	int node = early_cpu_to_node(cpu);
	63
	64	if (node_online(node) && NODE_DATA(node) &&
	65	last && last != NODE_DATA(node))
	66	return true;
	67
	68	last = NODE_DATA(node);
	69	}
	70	#endif
	71	return false;
	72	}
	73
5f5d8405 TH	74	/**
	75	* pcpu_alloc_bootmem - NUMA friendly alloc_bootmem wrapper for percpu
	76	* @cpu: cpu to allocate for
	77	* @size: size allocation in bytes
	78	* @align: alignment
	79	*
	80	* Allocate @size bytes aligned at @align for cpu @cpu. This wrapper
	81	* does the right thing for NUMA regardless of the current
	82	* configuration.
	83	*
	84	* RETURNS:
	85	* Pointer to the allocated area on success, NULL on failure.
	86	*/
	87	static void * __init pcpu_alloc_bootmem(unsigned int cpu, unsigned long size,
	88	unsigned long align)
	89	{
	90	const unsigned long goal = __pa(MAX_DMA_ADDRESS);
	91	#ifdef CONFIG_NEED_MULTIPLE_NODES
	92	int node = early_cpu_to_node(cpu);
	93	void *ptr;
	94
	95	if (!node_online(node) \|\| !NODE_DATA(node)) {
	96	ptr = __alloc_bootmem_nopanic(size, align, goal);
	97	pr_info("cpu %d has no node %d or node-local memory\n",
	98	cpu, node);
	99	pr_debug("per cpu data for cpu%d %lu bytes at %016lx\n",
	100	cpu, size, __pa(ptr));
	101	} else {
	102	ptr = __alloc_bootmem_node_nopanic(NODE_DATA(node),
	103	size, align, goal);
	104	pr_debug("per cpu data for cpu%d %lu bytes on node%d at "
	105	"%016lx\n", cpu, size, node, __pa(ptr));
	106	}
	107	return ptr;
	108	#else
	109	return __alloc_bootmem_nopanic(size, align, goal);
	110	#endif
	111	}
	112
89c92151 TH	113	/*
	114	* Embedding allocator
	115	*
	116	* The first chunk is sized to just contain the static area plus
	117	* PERCPU_DYNAMIC_RESERVE and allocated as a contiguous area using
	118	* bootmem allocator and used as-is without being mapped into vmalloc
	119	* area. This enables the first chunk to piggy back on the linear
	120	* physical PMD mapping and doesn't add any additional pressure to
	121	* TLB.
	122	*/
	123	static void *pcpue_ptr __initdata;
	124	static size_t pcpue_unit_size __initdata;
	125
	126	static struct page * __init pcpue_get_page(unsigned int cpu, int pageno)
	127	{
	128	return virt_to_page(pcpue_ptr + cpu * pcpue_unit_size
	129	+ ((size_t)pageno << PAGE_SHIFT));
	130	}
	131
	132	static ssize_t __init setup_pcpu_embed(size_t static_size)
	133	{
	134	unsigned int cpu;
	135
	136	/*
	137	* If large page isn't supported, there's no benefit in doing
	138	* this. Also, embedding allocation doesn't play well with
	139	* NUMA.
	140	*/
	141	if (!cpu_has_pse \|\| pcpu_need_numa())
	142	return -EINVAL;
	143
	144	/* allocate and copy */
	145	pcpue_unit_size = PFN_ALIGN(static_size + PERCPU_DYNAMIC_RESERVE);
	146	pcpue_unit_size = max(pcpue_unit_size, PCPU_MIN_UNIT_SIZE);
	147	pcpue_ptr = pcpu_alloc_bootmem(0, num_possible_cpus() * pcpue_unit_size,
	148	PAGE_SIZE);
	149	if (!pcpue_ptr)
	150	return -ENOMEM;
	151
	152	for_each_possible_cpu(cpu)
	153	memcpy(pcpue_ptr + cpu * pcpue_unit_size, __per_cpu_load,
	154	static_size);
	155
	156	/* we're ready, commit */
	157	pr_info("PERCPU: Embedded %zu pages at %p, static data %zu bytes\n",
	158	pcpue_unit_size >> PAGE_SHIFT, pcpue_ptr, static_size);
	159
	160	return pcpu_setup_first_chunk(pcpue_get_page, static_size,
	161	pcpue_unit_size,
	162	pcpue_unit_size - static_size, pcpue_ptr,
	163	NULL);
	164	}
	165
5f5d8405 TH	166	/*
	167	* 4k page allocator
	168	*
	169	* This is the basic allocator. Static percpu area is allocated
	170	* page-by-page and most of initialization is done by the generic
	171	* setup function.
	172	*/
8d408b4b TH	173	static struct page **pcpu4k_pages __initdata;
	174	static int pcpu4k_nr_static_pages __initdata;
	175
	176	static struct page * __init pcpu4k_get_page(unsigned int cpu, int pageno)
	177	{
	178	if (pageno < pcpu4k_nr_static_pages)
	179	return pcpu4k_pages[cpu * pcpu4k_nr_static_pages + pageno];
	180	return NULL;
	181	}
	182
458a3e64 TH	183	static void __init pcpu4k_populate_pte(unsigned long addr)
	184	{
	185	populate_extra_pte(addr);
	186	}
	187
5f5d8405 TH	188	static ssize_t __init setup_pcpu_4k(size_t static_size)
	189	{
	190	size_t pages_size;
	191	unsigned int cpu;
	192	int i, j;
	193	ssize_t ret;
	194
	195	pcpu4k_nr_static_pages = PFN_UP(static_size);
	196
	197	/* unaligned allocations can't be freed, round up to page size */
	198	pages_size = PFN_ALIGN(pcpu4k_nr_static_pages * num_possible_cpus()
	199	* sizeof(pcpu4k_pages[0]));
	200	pcpu4k_pages = alloc_bootmem(pages_size);
	201
	202	/* allocate and copy */
	203	j = 0;
	204	for_each_possible_cpu(cpu)
	205	for (i = 0; i < pcpu4k_nr_static_pages; i++) {
	206	void *ptr;
	207
	208	ptr = pcpu_alloc_bootmem(cpu, PAGE_SIZE, PAGE_SIZE);
	209	if (!ptr)
	210	goto enomem;
	211
	212	memcpy(ptr, __per_cpu_load + i * PAGE_SIZE, PAGE_SIZE);
	213	pcpu4k_pages[j++] = virt_to_page(ptr);
	214	}
	215
	216	/* we're ready, commit */
	217	pr_info("PERCPU: Allocated %d 4k pages, static data %zu bytes\n",
	218	pcpu4k_nr_static_pages, static_size);
	219
	220	ret = pcpu_setup_first_chunk(pcpu4k_get_page, static_size, 0, 0, NULL,
	221	pcpu4k_populate_pte);
	222	goto out_free_ar;
	223
	224	enomem:
	225	while (--j >= 0)
	226	free_bootmem(__pa(page_address(pcpu4k_pages[j])), PAGE_SIZE);
	227	ret = -ENOMEM;
	228	out_free_ar:
	229	free_bootmem(__pa(pcpu4k_pages), pages_size);
	230	return ret;
	231	}
	232
b2d2f431 BG	233	static inline void setup_percpu_segment(int cpu)
	234	{
	235	#ifdef CONFIG_X86_32
	236	struct desc_struct gdt;
	237
	238	pack_descriptor(&gdt, per_cpu_offset(cpu), 0xFFFFF,
	239	0x2 \| DESCTYPE_S, 0x8);
	240	gdt.s = 1;
	241	write_gdt_entry(get_cpu_gdt_table(cpu),
	242	GDT_ENTRY_PERCPU, &gdt, DESCTYPE_S);
	243	#endif
	244	}
	245
4fe29a85 GOC	246	/*
	247	* Great future plan:
	248	* Declare PDA itself and support (irqstack,tss,pgd) as per cpu data.
	249	* Always point %gs to its beginning
	250	*/
	251	void __init setup_per_cpu_areas(void)
	252	{
5f5d8405 TH	253	size_t static_size = __per_cpu_end - __per_cpu_start;
5f5d8405 TH	254	unsigned int cpu;
11124411 TH	255	unsigned long delta;
11124411 TH	256	size_t pcpu_unit_size;
5f5d8405	257	ssize_t ret;
a1681965	258
ab14398a	259	pr_info("NR_CPUS:%d nr_cpumask_bits:%d nr_cpu_ids:%d nr_node_ids:%d\n",
a1681965	260	NR_CPUS, nr_cpumask_bits, nr_cpu_ids, nr_node_ids);
11124411	261
5f5d8405	262	/* allocate percpu area */
89c92151 TH	263	ret = setup_pcpu_embed(static_size);
	264	if (ret < 0)
	265	ret = setup_pcpu_4k(static_size);
5f5d8405 TH	266	if (ret < 0)
	267	panic("cannot allocate static percpu area (%zu bytes, err=%zd)",
	268	static_size, ret);
1a51e3a0	269
5f5d8405	270	pcpu_unit_size = ret;
11124411	271
5f5d8405	272	/* alrighty, percpu areas up and running */
11124411 TH	273	delta = (unsigned long)pcpu_base_addr - (unsigned long)__per_cpu_start;
	274	for_each_possible_cpu(cpu) {
	275	per_cpu_offset(cpu) = delta + cpu * pcpu_unit_size;
26f80bd6	276	per_cpu(this_cpu_off, cpu) = per_cpu_offset(cpu);
ea927906	277	per_cpu(cpu_number, cpu) = cpu;
b2d2f431	278	setup_percpu_segment(cpu);
60a5317f	279	setup_stack_canary_segment(cpu);
0d77e7f0	280	/*
cf3997f5 TH	281	* Copy data used in early init routines from the
	282	* initial arrays to the per cpu data areas. These
	283	* arrays then become expendable and the *_early_ptr's
	284	* are zeroed indicating that the static arrays are
	285	* gone.
0d77e7f0	286	*/
ec70de8b	287	#ifdef CONFIG_X86_LOCAL_APIC
0d77e7f0	288	per_cpu(x86_cpu_to_apicid, cpu) =
cf3997f5	289	early_per_cpu_map(x86_cpu_to_apicid, cpu);
0d77e7f0	290	per_cpu(x86_bios_cpu_apicid, cpu) =
cf3997f5	291	early_per_cpu_map(x86_bios_cpu_apicid, cpu);
ec70de8b	292	#endif
1a51e3a0	293	#ifdef CONFIG_X86_64
26f80bd6	294	per_cpu(irq_stack_ptr, cpu) =
cf3997f5 TH	295	per_cpu(irq_stack_union.irq_stack, cpu) +
cf3997f5 TH	296	IRQ_STACK_SIZE - 64;
6470aff6 BG	297	#ifdef CONFIG_NUMA
6470aff6 BG	298	per_cpu(x86_cpu_to_node_map, cpu) =
cf3997f5	299	early_per_cpu_map(x86_cpu_to_node_map, cpu);
2697fbd5	300	#endif
6470aff6	301	#endif
1a51e3a0	302	/*
34019be1	303	* Up to this point, the boot CPU has been using .data.init
2697fbd5	304	* area. Reload any changed state for the boot CPU.
1a51e3a0	305	*/
34019be1	306	if (cpu == boot_cpu_id)
552be871	307	switch_to_new_gdt(cpu);
c90aa894 MT	308
c90aa894 MT	309	DBG("PERCPU: cpu %4d %p\n", cpu, ptr);
4fe29a85 GOC	310	}
4fe29a85 GOC	311
0d77e7f0	312	/* indicate the early static arrays will soon be gone */
22f25138	313	#ifdef CONFIG_X86_LOCAL_APIC
0d77e7f0 BG	314	early_per_cpu_ptr(x86_cpu_to_apicid) = NULL;
0d77e7f0 BG	315	early_per_cpu_ptr(x86_bios_cpu_apicid) = NULL;
22f25138	316	#endif
6470aff6	317	#if defined(CONFIG_X86_64) && defined(CONFIG_NUMA)
0d77e7f0 BG	318	early_per_cpu_ptr(x86_cpu_to_node_map) = NULL;
0d77e7f0 BG	319	#endif
9f0e8d04	320
9f248bde MT	321	/* Setup node to cpumask map */
9f248bde MT	322	setup_node_to_cpumask_map();
c2d1cec1 MT	323
	324	/* Setup cpu initialized, callin, callout masks */
	325	setup_cpu_local_masks();
4fe29a85	326	}