[net-next-2.6.git] / arch / powerpc / kernel / smp.c

/*
 * SMP support for ppc.
 *
 * Written by Cort Dougan (cort@cs.nmt.edu) borrowing a great
 * deal of code from the sparc and intel versions.
 *
 * Copyright (C) 1999 Cort Dougan <cort@cs.nmt.edu>
 *
 * PowerPC-64 Support added by Dave Engebretsen, Peter Bergner, and
 * Mike Corrigan {engebret|bergner|mikec}@us.ibm.com
 *
 *      This program is free software; you can redistribute it and/or
 *      modify it under the terms of the GNU General Public License
 *      as published by the Free Software Foundation; either version
 *      2 of the License, or (at your option) any later version.
 */

#undef DEBUG

#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/sched.h>
#include <linux/smp.h>
#include <linux/interrupt.h>
#include <linux/delay.h>
#include <linux/init.h>
#include <linux/spinlock.h>
#include <linux/cache.h>
#include <linux/err.h>
#include <linux/sysdev.h>
#include <linux/cpu.h>
#include <linux/notifier.h>
#include <linux/topology.h>

#include <asm/ptrace.h>
#include <asm/atomic.h>
#include <asm/irq.h>
#include <asm/page.h>
#include <asm/pgtable.h>
#include <asm/prom.h>
#include <asm/smp.h>
#include <asm/time.h>
#include <asm/machdep.h>
#include <asm/cputable.h>
#include <asm/system.h>
#include <asm/mpic.h>
#include <asm/vdso_datapage.h>
#ifdef CONFIG_PPC64
#include <asm/paca.h>
#endif

#ifdef DEBUG
#include <asm/udbg.h>
#define DBG(fmt...) udbg_printf(fmt)
#else
#define DBG(fmt...)
#endif

int smp_hw_index[NR_CPUS];
struct thread_info *secondary_ti;

cpumask_t cpu_possible_map = CPU_MASK_NONE;
cpumask_t cpu_online_map = CPU_MASK_NONE;
cpumask_t cpu_sibling_map[NR_CPUS] = { [0 ... NR_CPUS-1] = CPU_MASK_NONE };

EXPORT_SYMBOL(cpu_online_map);
EXPORT_SYMBOL(cpu_possible_map);
EXPORT_SYMBOL(cpu_sibling_map);

/* SMP operations for this machine */
struct smp_ops_t *smp_ops;

static volatile unsigned int cpu_callin_map[NR_CPUS];

void smp_call_function_interrupt(void);

int smt_enabled_at_boot = 1;

static void (*crash_ipi_function_ptr)(struct pt_regs *) = NULL;

#ifdef CONFIG_PPC64
void __devinit smp_generic_kick_cpu(int nr)
{
	BUG_ON(nr < 0 || nr >= NR_CPUS);

	/*
	 * The processor is currently spinning, waiting for the
	 * cpu_start field to become non-zero After we set cpu_start,
	 * the processor will continue on to secondary_start
	 */
	paca[nr].cpu_start = 1;
	smp_mb();
}
#endif

void smp_message_recv(int msg)
{
	switch(msg) {
	case PPC_MSG_CALL_FUNCTION:
		smp_call_function_interrupt();
		break;
	case PPC_MSG_RESCHEDULE:
		/* XXX Do we have to do this? */
		set_need_resched();
		break;
	case PPC_MSG_DEBUGGER_BREAK:
		if (crash_ipi_function_ptr) {
			crash_ipi_function_ptr(get_irq_regs());
			break;
		}
#ifdef CONFIG_DEBUGGER
		debugger_ipi(get_irq_regs());
		break;
#endif /* CONFIG_DEBUGGER */
		/* FALLTHROUGH */
	default:
		printk("SMP %d: smp_message_recv(): unknown msg %d\n",
		       smp_processor_id(), msg);
		break;
	}
}

void smp_send_reschedule(int cpu)
{
	if (likely(smp_ops))
		smp_ops->message_pass(cpu, PPC_MSG_RESCHEDULE);
}

#ifdef CONFIG_DEBUGGER
void smp_send_debugger_break(int cpu)
{
	if (likely(smp_ops))
		smp_ops->message_pass(cpu, PPC_MSG_DEBUGGER_BREAK);
}
#endif

#ifdef CONFIG_KEXEC
void crash_send_ipi(void (*crash_ipi_callback)(struct pt_regs *))
{
	crash_ipi_function_ptr = crash_ipi_callback;
	if (crash_ipi_callback && smp_ops) {
		mb();
		smp_ops->message_pass(MSG_ALL_BUT_SELF, PPC_MSG_DEBUGGER_BREAK);
	}
}
#endif

static void stop_this_cpu(void *dummy)
{
	local_irq_disable();
	while (1)
		;
}

/*
 * Structure and data for smp_call_function(). This is designed to minimise
 * static memory requirements. It also looks cleaner.
 * Stolen from the i386 version.
 */
static  __cacheline_aligned_in_smp DEFINE_SPINLOCK(call_lock);

static struct call_data_struct {
	void (*func) (void *info);
	void *info;
	atomic_t started;
	atomic_t finished;
	int wait;
} *call_data;

/* delay of at least 8 seconds */
#define SMP_CALL_TIMEOUT	8

/*
 * These functions send a 'generic call function' IPI to other online
 * CPUS in the system.
 *
 * [SUMMARY] Run a function on other CPUs.
 * <func> The function to run. This must be fast and non-blocking.
 * <info> An arbitrary pointer to pass to the function.
 * <nonatomic> currently unused.
 * <wait> If true, wait (atomically) until function has completed on other CPUs.
 * [RETURNS] 0 on success, else a negative status code. Does not return until
 * remote CPUs are nearly ready to execute <<func>> or are or have executed.
 *
 * You must not call this function with disabled interrupts or from a
 * hardware interrupt handler or from a bottom half handler.
 */
int smp_call_function_map(void (*func) (void *info), void *info, int nonatomic,
			int wait, cpumask_t map)
{
	struct call_data_struct data;
	int ret = -1, num_cpus;
	int cpu;
	u64 timeout;

	if (unlikely(smp_ops == NULL))
		return ret;

	data.func = func;
	data.info = info;
	atomic_set(&data.started, 0);
	data.wait = wait;
	if (wait)
		atomic_set(&data.finished, 0);

	spin_lock(&call_lock);

	/* remove 'self' from the map */
	if (cpu_isset(smp_processor_id(), map))
		cpu_clear(smp_processor_id(), map);

	/* sanity check the map, remove any non-online processors. */
	cpus_and(map, map, cpu_online_map);

	num_cpus = cpus_weight(map);
	if (!num_cpus)
		goto done;

	call_data = &data;
	smp_wmb();
	/* Send a message to all CPUs in the map */
	for_each_cpu_mask(cpu, map)
		smp_ops->message_pass(cpu, PPC_MSG_CALL_FUNCTION);

	timeout = get_tb() + (u64) SMP_CALL_TIMEOUT * tb_ticks_per_sec;

	/* Wait for indication that they have received the message */
	while (atomic_read(&data.started) != num_cpus) {
		HMT_low();
		if (get_tb() >= timeout) {
			printk("smp_call_function on cpu %d: other cpus not "
				"responding (%d)\n", smp_processor_id(),
				atomic_read(&data.started));
			debugger(NULL);
			goto out;
		}
	}

	/* optionally wait for the CPUs to complete */
	if (wait) {
		while (atomic_read(&data.finished) != num_cpus) {
			HMT_low();
			if (get_tb() >= timeout) {
				printk("smp_call_function on cpu %d: other "
					"cpus not finishing (%d/%d)\n",
					smp_processor_id(),
					atomic_read(&data.finished),
					atomic_read(&data.started));
				debugger(NULL);
				goto out;
			}
		}
	}

 done:
	ret = 0;

 out:
	call_data = NULL;
	HMT_medium();
	spin_unlock(&call_lock);
	return ret;
}

static int __smp_call_function(void (*func)(void *info), void *info,
			       int nonatomic, int wait)
{
	return smp_call_function_map(func,info,nonatomic,wait,cpu_online_map);
}

int smp_call_function(void (*func) (void *info), void *info, int nonatomic,
			int wait)
{
	/* Can deadlock when called with interrupts disabled */
	WARN_ON(irqs_disabled());

	return __smp_call_function(func, info, nonatomic, wait);
}
EXPORT_SYMBOL(smp_call_function);

int smp_call_function_single(int cpu, void (*func) (void *info), void *info, int nonatomic,
			int wait)
{
	cpumask_t map = CPU_MASK_NONE;
	int ret = 0;

	/* Can deadlock when called with interrupts disabled */
	WARN_ON(irqs_disabled());

	if (!cpu_online(cpu))
		return -EINVAL;

	cpu_set(cpu, map);
	if (cpu != get_cpu())
		ret = smp_call_function_map(func,info,nonatomic,wait,map);
	else {
		local_irq_disable();
		func(info);
		local_irq_enable();
	}
	put_cpu();
	return ret;
}
EXPORT_SYMBOL(smp_call_function_single);

void smp_send_stop(void)
{
	__smp_call_function(stop_this_cpu, NULL, 1, 0);
}

void smp_call_function_interrupt(void)
{
	void (*func) (void *info);
	void *info;
	int wait;

	/* call_data will be NULL if the sender timed out while
	 * waiting on us to receive the call.
	 */
	if (!call_data)
		return;

	func = call_data->func;
	info = call_data->info;
	wait = call_data->wait;

	if (!wait)
		smp_mb__before_atomic_inc();

	/*
	 * Notify initiating CPU that I've grabbed the data and am
	 * about to execute the function
	 */
	atomic_inc(&call_data->started);
	/*
	 * At this point the info structure may be out of scope unless wait==1
	 */
	(*func)(info);
	if (wait) {
		smp_mb__before_atomic_inc();
		atomic_inc(&call_data->finished);
	}
}

extern struct gettimeofday_struct do_gtod;

struct thread_info *current_set[NR_CPUS];

DECLARE_PER_CPU(unsigned int, pvr);

static void __devinit smp_store_cpu_info(int id)
{
	per_cpu(pvr, id) = mfspr(SPRN_PVR);
}

static void __init smp_create_idle(unsigned int cpu)
{
	struct task_struct *p;

	/* create a process for the processor */
	p = fork_idle(cpu);
	if (IS_ERR(p))
		panic("failed fork for CPU %u: %li", cpu, PTR_ERR(p));
#ifdef CONFIG_PPC64
	paca[cpu].__current = p;
#endif
	current_set[cpu] = task_thread_info(p);
	task_thread_info(p)->cpu = cpu;
}

void __init smp_prepare_cpus(unsigned int max_cpus)
{
	unsigned int cpu;

	DBG("smp_prepare_cpus\n");

	/* 
	 * setup_cpu may need to be called on the boot cpu. We havent
	 * spun any cpus up but lets be paranoid.
	 */
	BUG_ON(boot_cpuid != smp_processor_id());

	/* Fixup boot cpu */
	smp_store_cpu_info(boot_cpuid);
	cpu_callin_map[boot_cpuid] = 1;

	if (smp_ops)
		max_cpus = smp_ops->probe();
	else
		max_cpus = 1;
 
	smp_space_timers(max_cpus);

	for_each_possible_cpu(cpu)
		if (cpu != boot_cpuid)
			smp_create_idle(cpu);
}

void __devinit smp_prepare_boot_cpu(void)
{
	BUG_ON(smp_processor_id() != boot_cpuid);

	cpu_set(boot_cpuid, cpu_online_map);
#ifdef CONFIG_PPC64
	paca[boot_cpuid].__current = current;
#endif
	current_set[boot_cpuid] = task_thread_info(current);
}

#ifdef CONFIG_HOTPLUG_CPU
/* State of each CPU during hotplug phases */
DEFINE_PER_CPU(int, cpu_state) = { 0 };

int generic_cpu_disable(void)
{
	unsigned int cpu = smp_processor_id();

	if (cpu == boot_cpuid)
		return -EBUSY;

	cpu_clear(cpu, cpu_online_map);
#ifdef CONFIG_PPC64
	vdso_data->processorCount--;
	fixup_irqs(cpu_online_map);
#endif
	return 0;
}

int generic_cpu_enable(unsigned int cpu)
{
	/* Do the normal bootup if we haven't
	 * already bootstrapped. */
	if (system_state != SYSTEM_RUNNING)
		return -ENOSYS;

	/* get the target out of it's holding state */
	per_cpu(cpu_state, cpu) = CPU_UP_PREPARE;
	smp_wmb();

	while (!cpu_online(cpu))
		cpu_relax();

#ifdef CONFIG_PPC64
	fixup_irqs(cpu_online_map);
	/* counter the irq disable in fixup_irqs */
	local_irq_enable();
#endif
	return 0;
}

void generic_cpu_die(unsigned int cpu)
{
	int i;

	for (i = 0; i < 100; i++) {
		smp_rmb();
		if (per_cpu(cpu_state, cpu) == CPU_DEAD)
			return;
		msleep(100);
	}
	printk(KERN_ERR "CPU%d didn't die...\n", cpu);
}

void generic_mach_cpu_die(void)
{
	unsigned int cpu;

	local_irq_disable();
	cpu = smp_processor_id();
	printk(KERN_DEBUG "CPU%d offline\n", cpu);
	__get_cpu_var(cpu_state) = CPU_DEAD;
	smp_wmb();
	while (__get_cpu_var(cpu_state) != CPU_UP_PREPARE)
		cpu_relax();
	cpu_set(cpu, cpu_online_map);
	local_irq_enable();
}
#endif

static int __devinit cpu_enable(unsigned int cpu)
{
	if (smp_ops && smp_ops->cpu_enable)
		return smp_ops->cpu_enable(cpu);

	return -ENOSYS;
}

int __cpuinit __cpu_up(unsigned int cpu)
{
	int c;

	secondary_ti = current_set[cpu];
	if (!cpu_enable(cpu))
		return 0;

	if (smp_ops == NULL ||
	    (smp_ops->cpu_bootable && !smp_ops->cpu_bootable(cpu)))
		return -EINVAL;

	/* Make sure callin-map entry is 0 (can be leftover a CPU
	 * hotplug
	 */
	cpu_callin_map[cpu] = 0;

	/* The information for processor bringup must
	 * be written out to main store before we release
	 * the processor.
	 */
	smp_mb();

	/* wake up cpus */
	DBG("smp: kicking cpu %d\n", cpu);
	smp_ops->kick_cpu(cpu);

	/*
	 * wait to see if the cpu made a callin (is actually up).
	 * use this value that I found through experimentation.
	 * -- Cort
	 */
	if (system_state < SYSTEM_RUNNING)
		for (c = 50000; c && !cpu_callin_map[cpu]; c--)
			udelay(100);
#ifdef CONFIG_HOTPLUG_CPU
	else
		/*
		 * CPUs can take much longer to come up in the
		 * hotplug case.  Wait five seconds.
		 */
		for (c = 25; c && !cpu_callin_map[cpu]; c--) {
			msleep(200);
		}
#endif

	if (!cpu_callin_map[cpu]) {
		printk("Processor %u is stuck.\n", cpu);
		return -ENOENT;
	}

	printk("Processor %u found.\n", cpu);

	if (smp_ops->give_timebase)
		smp_ops->give_timebase();

	/* Wait until cpu puts itself in the online map */
	while (!cpu_online(cpu))
		cpu_relax();

	return 0;
}


/* Activate a secondary processor. */
int __devinit start_secondary(void *unused)
{
	unsigned int cpu = smp_processor_id();

	atomic_inc(&init_mm.mm_count);
	current->active_mm = &init_mm;

	smp_store_cpu_info(cpu);
	set_dec(tb_ticks_per_jiffy);
	preempt_disable();
	cpu_callin_map[cpu] = 1;

	smp_ops->setup_cpu(cpu);
	if (smp_ops->take_timebase)
		smp_ops->take_timebase();

	if (system_state > SYSTEM_BOOTING)
		snapshot_timebase();

	spin_lock(&call_lock);
	cpu_set(cpu, cpu_online_map);
	spin_unlock(&call_lock);

	local_irq_enable();

	cpu_idle();
	return 0;
}

int setup_profiling_timer(unsigned int multiplier)
{
	return 0;
}

void __init smp_cpus_done(unsigned int max_cpus)
{
	cpumask_t old_mask;

	/* We want the setup_cpu() here to be called from CPU 0, but our
	 * init thread may have been "borrowed" by another CPU in the meantime
	 * se we pin us down to CPU 0 for a short while
	 */
	old_mask = current->cpus_allowed;
	set_cpus_allowed(current, cpumask_of_cpu(boot_cpuid));
	
	if (smp_ops)
		smp_ops->setup_cpu(boot_cpuid);

	set_cpus_allowed(current, old_mask);

	snapshot_timebases();

	dump_numa_cpu_topology();
}

#ifdef CONFIG_HOTPLUG_CPU
int __cpu_disable(void)
{
	if (smp_ops->cpu_disable)
		return smp_ops->cpu_disable();

	return -ENOSYS;
}

void __cpu_die(unsigned int cpu)
{
	if (smp_ops->cpu_die)
		smp_ops->cpu_die(cpu);
}
#endif
Commit	Line	Data
1da177e4 LT	1	/*
	2	* SMP support for ppc.
	3	*
	4	* Written by Cort Dougan (cort@cs.nmt.edu) borrowing a great
	5	* deal of code from the sparc and intel versions.
	6	*
	7	* Copyright (C) 1999 Cort Dougan <cort@cs.nmt.edu>
	8	*
	9	* PowerPC-64 Support added by Dave Engebretsen, Peter Bergner, and
	10	* Mike Corrigan {engebret\|bergner\|mikec}@us.ibm.com
	11	*
	12	* This program is free software; you can redistribute it and/or
	13	* modify it under the terms of the GNU General Public License
	14	* as published by the Free Software Foundation; either version
	15	* 2 of the License, or (at your option) any later version.
	16	*/
	17
	18	#undef DEBUG
	19
1da177e4 LT	20	#include <linux/kernel.h>
	21	#include <linux/module.h>
	22	#include <linux/sched.h>
	23	#include <linux/smp.h>
	24	#include <linux/interrupt.h>
	25	#include <linux/delay.h>
	26	#include <linux/init.h>
	27	#include <linux/spinlock.h>
	28	#include <linux/cache.h>
	29	#include <linux/err.h>
	30	#include <linux/sysdev.h>
	31	#include <linux/cpu.h>
	32	#include <linux/notifier.h>
4b703a23	33	#include <linux/topology.h>
1da177e4 LT	34
	35	#include <asm/ptrace.h>
	36	#include <asm/atomic.h>
	37	#include <asm/irq.h>
	38	#include <asm/page.h>
	39	#include <asm/pgtable.h>
	40	#include <asm/prom.h>
	41	#include <asm/smp.h>
1da177e4 LT	42	#include <asm/time.h>
	43	#include <asm/machdep.h>
	44	#include <asm/cputable.h>
	45	#include <asm/system.h>
bbeb3f4c	46	#include <asm/mpic.h>
a7f290da	47	#include <asm/vdso_datapage.h>
5ad57078 PM	48	#ifdef CONFIG_PPC64
	49	#include <asm/paca.h>
	50	#endif
	51
1da177e4	52	#ifdef DEBUG
f9e4ec57	53	#include <asm/udbg.h>
1da177e4 LT	54	#define DBG(fmt...) udbg_printf(fmt)
	55	#else
	56	#define DBG(fmt...)
	57	#endif
	58
f9e4ec57 ME	59	int smp_hw_index[NR_CPUS];
	60	struct thread_info *secondary_ti;
	61
1da177e4 LT	62	cpumask_t cpu_possible_map = CPU_MASK_NONE;
	63	cpumask_t cpu_online_map = CPU_MASK_NONE;
	64	cpumask_t cpu_sibling_map[NR_CPUS] = { [0 ... NR_CPUS-1] = CPU_MASK_NONE };
	65
	66	EXPORT_SYMBOL(cpu_online_map);
	67	EXPORT_SYMBOL(cpu_possible_map);
36ca4ba4	68	EXPORT_SYMBOL(cpu_sibling_map);
1da177e4	69
5ad57078	70	/* SMP operations for this machine */
1da177e4 LT	71	struct smp_ops_t *smp_ops;
	72
	73	static volatile unsigned int cpu_callin_map[NR_CPUS];
	74
1da177e4 LT	75	void smp_call_function_interrupt(void);
	76
	77	int smt_enabled_at_boot = 1;
	78
cc532915 ME	79	static void (crash_ipi_function_ptr)(struct pt_regs ) = NULL;
cc532915 ME	80
5ad57078	81	#ifdef CONFIG_PPC64
1da177e4 LT	82	void __devinit smp_generic_kick_cpu(int nr)
	83	{
	84	BUG_ON(nr < 0 \|\| nr >= NR_CPUS);
	85
	86	/*
	87	* The processor is currently spinning, waiting for the
	88	* cpu_start field to become non-zero After we set cpu_start,
	89	* the processor will continue on to secondary_start
	90	*/
	91	paca[nr].cpu_start = 1;
0d8d4d42	92	smp_mb();
1da177e4	93	}
5ad57078	94	#endif
1da177e4	95
7d12e780	96	void smp_message_recv(int msg)
1da177e4 LT	97	{
	98	switch(msg) {
	99	case PPC_MSG_CALL_FUNCTION:
	100	smp_call_function_interrupt();
	101	break;
5ad57078	102	case PPC_MSG_RESCHEDULE:
1da177e4 LT	103	/* XXX Do we have to do this? */
	104	set_need_resched();
	105	break;
1da177e4	106	case PPC_MSG_DEBUGGER_BREAK:
cc532915	107	if (crash_ipi_function_ptr) {
7d12e780	108	crash_ipi_function_ptr(get_irq_regs());
cc532915 ME	109	break;
	110	}
	111	#ifdef CONFIG_DEBUGGER
7d12e780	112	debugger_ipi(get_irq_regs());
1da177e4	113	break;
cc532915 ME	114	#endif /* CONFIG_DEBUGGER */
cc532915 ME	115	/* FALLTHROUGH */
1da177e4 LT	116	default:
	117	printk("SMP %d: smp_message_recv(): unknown msg %d\n",
	118	smp_processor_id(), msg);
	119	break;
	120	}
	121	}
	122
	123	void smp_send_reschedule(int cpu)
	124	{
8cffc6ac BH	125	if (likely(smp_ops))
8cffc6ac BH	126	smp_ops->message_pass(cpu, PPC_MSG_RESCHEDULE);
1da177e4 LT	127	}
	128
	129	#ifdef CONFIG_DEBUGGER
	130	void smp_send_debugger_break(int cpu)
	131	{
8cffc6ac BH	132	if (likely(smp_ops))
8cffc6ac BH	133	smp_ops->message_pass(cpu, PPC_MSG_DEBUGGER_BREAK);
1da177e4 LT	134	}
	135	#endif
	136
cc532915 ME	137	#ifdef CONFIG_KEXEC
	138	void crash_send_ipi(void (crash_ipi_callback)(struct pt_regs ))
	139	{
	140	crash_ipi_function_ptr = crash_ipi_callback;
8cffc6ac	141	if (crash_ipi_callback && smp_ops) {
cc532915 ME	142	mb();
	143	smp_ops->message_pass(MSG_ALL_BUT_SELF, PPC_MSG_DEBUGGER_BREAK);
	144	}
	145	}
	146	#endif
	147
1da177e4 LT	148	static void stop_this_cpu(void *dummy)
	149	{
	150	local_irq_disable();
	151	while (1)
	152	;
	153	}
	154
1da177e4 LT	155	/*
	156	* Structure and data for smp_call_function(). This is designed to minimise
	157	* static memory requirements. It also looks cleaner.
	158	* Stolen from the i386 version.
	159	*/
	160	static __cacheline_aligned_in_smp DEFINE_SPINLOCK(call_lock);
	161
	162	static struct call_data_struct {
	163	void (func) (void info);
	164	void *info;
	165	atomic_t started;
	166	atomic_t finished;
	167	int wait;
	168	} *call_data;
	169
5ad57078 PM	170	/* delay of at least 8 seconds */
5ad57078 PM	171	#define SMP_CALL_TIMEOUT 8
1da177e4 LT	172
1da177e4 LT	173	/*
44755d11	174	* These functions send a 'generic call function' IPI to other online
44755d11	175	* CPUS in the system.
1da177e4	176	*
44755d11	177	* [SUMMARY] Run a function on other CPUs.
1da177e4 LT	178	* <func> The function to run. This must be fast and non-blocking.
	179	* <info> An arbitrary pointer to pass to the function.
	180	* <nonatomic> currently unused.
	181	* <wait> If true, wait (atomically) until function has completed on other CPUs.
	182	* [RETURNS] 0 on success, else a negative status code. Does not return until
	183	* remote CPUs are nearly ready to execute <<func>> or are or have executed.
	184	*
	185	* You must not call this function with disabled interrupts or from a
	186	* hardware interrupt handler or from a bottom half handler.
	187	*/
44755d11	188	int smp_call_function_map(void (func) (void info), void *info, int nonatomic,
	189	int wait, cpumask_t map)
	190	{
1da177e4	191	struct call_data_struct data;
44755d11	192	int ret = -1, num_cpus;
44755d11	193	int cpu;
5ad57078	194	u64 timeout;
1da177e4	195
8cffc6ac	196	if (unlikely(smp_ops == NULL))
44755d11	197	return ret;
8cffc6ac	198
1da177e4 LT	199	data.func = func;
	200	data.info = info;
	201	atomic_set(&data.started, 0);
	202	data.wait = wait;
	203	if (wait)
	204	atomic_set(&data.finished, 0);
	205
	206	spin_lock(&call_lock);
d3fdaed9 HD	207
	208	/* remove 'self' from the map */
	209	if (cpu_isset(smp_processor_id(), map))
	210	cpu_clear(smp_processor_id(), map);
	211
	212	/* sanity check the map, remove any non-online processors. */
	213	cpus_and(map, map, cpu_online_map);
17aa3a82 KC	214
	215	num_cpus = cpus_weight(map);
	216	if (!num_cpus)
d3fdaed9	217	goto done;
1da177e4 LT	218
1da177e4 LT	219	call_data = &data;
0d8d4d42	220	smp_wmb();
44755d11	221	/* Send a message to all CPUs in the map */
	222	for_each_cpu_mask(cpu, map)
	223	smp_ops->message_pass(cpu, PPC_MSG_CALL_FUNCTION);
1da177e4	224
5ad57078 PM	225	timeout = get_tb() + (u64) SMP_CALL_TIMEOUT * tb_ticks_per_sec;
5ad57078 PM	226
44755d11	227	/* Wait for indication that they have received the message */
44755d11	228	while (atomic_read(&data.started) != num_cpus) {
1da177e4	229	HMT_low();
5ad57078	230	if (get_tb() >= timeout) {
1da177e4	231	printk("smp_call_function on cpu %d: other cpus not "
44755d11	232	"responding (%d)\n", smp_processor_id(),
44755d11	233	atomic_read(&data.started));
1da177e4 LT	234	debugger(NULL);
	235	goto out;
	236	}
	237	}
	238
44755d11	239	/* optionally wait for the CPUs to complete */
1da177e4	240	if (wait) {
44755d11	241	while (atomic_read(&data.finished) != num_cpus) {
1da177e4	242	HMT_low();
5ad57078	243	if (get_tb() >= timeout) {
1da177e4	244	printk("smp_call_function on cpu %d: other "
44755d11	245	"cpus not finishing (%d/%d)\n",
	246	smp_processor_id(),
	247	atomic_read(&data.finished),
	248	atomic_read(&data.started));
1da177e4 LT	249	debugger(NULL);
	250	goto out;
	251	}
	252	}
	253	}
	254
d3fdaed9	255	done:
1da177e4 LT	256	ret = 0;
1da177e4 LT	257
5ad57078	258	out:
1da177e4 LT	259	call_data = NULL;
	260	HMT_medium();
	261	spin_unlock(&call_lock);
	262	return ret;
	263	}
	264
8fd7675c SS	265	static int __smp_call_function(void (func)(void info), void *info,
	266	int nonatomic, int wait)
	267	{
	268	return smp_call_function_map(func,info,nonatomic,wait,cpu_online_map);
	269	}
	270
44755d11	271	int smp_call_function(void (func) (void info), void *info, int nonatomic,
	272	int wait)
	273	{
8fd7675c SS	274	/* Can deadlock when called with interrupts disabled */
	275	WARN_ON(irqs_disabled());
	276
	277	return __smp_call_function(func, info, nonatomic, wait);
44755d11	278	}
1da177e4 LT	279	EXPORT_SYMBOL(smp_call_function);
1da177e4 LT	280
44755d11	281	int smp_call_function_single(int cpu, void (func) (void info), void *info, int nonatomic,
	282	int wait)
	283	{
d3fdaed9	284	cpumask_t map = CPU_MASK_NONE;
adff093d	285	int ret = 0;
44755d11	286
8fd7675c SS	287	/* Can deadlock when called with interrupts disabled */
	288	WARN_ON(irqs_disabled());
	289
44755d11	290	if (!cpu_online(cpu))
	291	return -EINVAL;
	292
44755d11	293	cpu_set(cpu, map);
d3fdaed9 HD	294	if (cpu != get_cpu())
d3fdaed9 HD	295	ret = smp_call_function_map(func,info,nonatomic,wait,map);
adff093d AK	296	else {
	297	local_irq_disable();
	298	func(info);
	299	local_irq_enable();
	300	}
d3fdaed9 HD	301	put_cpu();
d3fdaed9 HD	302	return ret;
44755d11	303	}
	304	EXPORT_SYMBOL(smp_call_function_single);
	305
8fd7675c SS	306	void smp_send_stop(void)
	307	{
	308	__smp_call_function(stop_this_cpu, NULL, 1, 0);
	309	}
	310
1da177e4 LT	311	void smp_call_function_interrupt(void)
	312	{
	313	void (func) (void info);
	314	void *info;
	315	int wait;
	316
	317	/* call_data will be NULL if the sender timed out while
	318	* waiting on us to receive the call.
	319	*/
	320	if (!call_data)
	321	return;
	322
	323	func = call_data->func;
	324	info = call_data->info;
	325	wait = call_data->wait;
	326
	327	if (!wait)
	328	smp_mb__before_atomic_inc();
	329
	330	/*
	331	* Notify initiating CPU that I've grabbed the data and am
	332	* about to execute the function
	333	*/
	334	atomic_inc(&call_data->started);
	335	/*
	336	* At this point the info structure may be out of scope unless wait==1
	337	*/
	338	(*func)(info);
	339	if (wait) {
	340	smp_mb__before_atomic_inc();
	341	atomic_inc(&call_data->finished);
	342	}
	343	}
	344
1da177e4 LT	345	extern struct gettimeofday_struct do_gtod;
	346
	347	struct thread_info *current_set[NR_CPUS];
	348
	349	DECLARE_PER_CPU(unsigned int, pvr);
	350
	351	static void __devinit smp_store_cpu_info(int id)
	352	{
	353	per_cpu(pvr, id) = mfspr(SPRN_PVR);
	354	}
	355
	356	static void __init smp_create_idle(unsigned int cpu)
	357	{
	358	struct task_struct *p;
	359
	360	/* create a process for the processor */
	361	p = fork_idle(cpu);
	362	if (IS_ERR(p))
	363	panic("failed fork for CPU %u: %li", cpu, PTR_ERR(p));
5ad57078	364	#ifdef CONFIG_PPC64
1da177e4	365	paca[cpu].__current = p;
5ad57078	366	#endif
b5e2fc1c AV	367	current_set[cpu] = task_thread_info(p);
b5e2fc1c AV	368	task_thread_info(p)->cpu = cpu;
1da177e4 LT	369	}
	370
	371	void __init smp_prepare_cpus(unsigned int max_cpus)
	372	{
	373	unsigned int cpu;
	374
	375	DBG("smp_prepare_cpus\n");
	376
	377	/*
	378	* setup_cpu may need to be called on the boot cpu. We havent
	379	* spun any cpus up but lets be paranoid.
	380	*/
	381	BUG_ON(boot_cpuid != smp_processor_id());
	382
	383	/* Fixup boot cpu */
	384	smp_store_cpu_info(boot_cpuid);
	385	cpu_callin_map[boot_cpuid] = 1;
	386
8cffc6ac BH	387	if (smp_ops)
	388	max_cpus = smp_ops->probe();
	389	else
	390	max_cpus = 1;
1da177e4 LT	391
	392	smp_space_timers(max_cpus);
	393
0e551954	394	for_each_possible_cpu(cpu)
1da177e4 LT	395	if (cpu != boot_cpuid)
	396	smp_create_idle(cpu);
	397	}
	398
	399	void __devinit smp_prepare_boot_cpu(void)
	400	{
	401	BUG_ON(smp_processor_id() != boot_cpuid);
	402
	403	cpu_set(boot_cpuid, cpu_online_map);
5ad57078	404	#ifdef CONFIG_PPC64
1da177e4	405	paca[boot_cpuid].__current = current;
5ad57078	406	#endif
b5e2fc1c	407	current_set[boot_cpuid] = task_thread_info(current);
1da177e4 LT	408	}
	409
	410	#ifdef CONFIG_HOTPLUG_CPU
	411	/* State of each CPU during hotplug phases */
	412	DEFINE_PER_CPU(int, cpu_state) = { 0 };
	413
	414	int generic_cpu_disable(void)
	415	{
	416	unsigned int cpu = smp_processor_id();
	417
	418	if (cpu == boot_cpuid)
	419	return -EBUSY;
	420
1da177e4	421	cpu_clear(cpu, cpu_online_map);
799d6046	422	#ifdef CONFIG_PPC64
a7f290da	423	vdso_data->processorCount--;
1da177e4	424	fixup_irqs(cpu_online_map);
094fe2e7	425	#endif
1da177e4 LT	426	return 0;
	427	}
	428
	429	int generic_cpu_enable(unsigned int cpu)
	430	{
	431	/* Do the normal bootup if we haven't
	432	* already bootstrapped. */
	433	if (system_state != SYSTEM_RUNNING)
	434	return -ENOSYS;
	435
	436	/* get the target out of it's holding state */
	437	per_cpu(cpu_state, cpu) = CPU_UP_PREPARE;
0d8d4d42	438	smp_wmb();
1da177e4 LT	439
	440	while (!cpu_online(cpu))
	441	cpu_relax();
	442
094fe2e7	443	#ifdef CONFIG_PPC64
1da177e4 LT	444	fixup_irqs(cpu_online_map);
	445	/* counter the irq disable in fixup_irqs */
	446	local_irq_enable();
094fe2e7	447	#endif
1da177e4 LT	448	return 0;
	449	}
	450
	451	void generic_cpu_die(unsigned int cpu)
	452	{
	453	int i;
	454
	455	for (i = 0; i < 100; i++) {
0d8d4d42	456	smp_rmb();
1da177e4 LT	457	if (per_cpu(cpu_state, cpu) == CPU_DEAD)
	458	return;
	459	msleep(100);
	460	}
	461	printk(KERN_ERR "CPU%d didn't die...\n", cpu);
	462	}
	463
	464	void generic_mach_cpu_die(void)
	465	{
	466	unsigned int cpu;
	467
	468	local_irq_disable();
	469	cpu = smp_processor_id();
	470	printk(KERN_DEBUG "CPU%d offline\n", cpu);
	471	__get_cpu_var(cpu_state) = CPU_DEAD;
0d8d4d42	472	smp_wmb();
1da177e4 LT	473	while (__get_cpu_var(cpu_state) != CPU_UP_PREPARE)
1da177e4 LT	474	cpu_relax();
1da177e4 LT	475	cpu_set(cpu, cpu_online_map);
	476	local_irq_enable();
	477	}
	478	#endif
	479
	480	static int __devinit cpu_enable(unsigned int cpu)
	481	{
8cffc6ac	482	if (smp_ops && smp_ops->cpu_enable)
1da177e4 LT	483	return smp_ops->cpu_enable(cpu);
	484
	485	return -ENOSYS;
	486	}
	487
b282b6f8	488	int __cpuinit __cpu_up(unsigned int cpu)
1da177e4 LT	489	{
	490	int c;
	491
5ad57078	492	secondary_ti = current_set[cpu];
1da177e4 LT	493	if (!cpu_enable(cpu))
	494	return 0;
	495
8cffc6ac BH	496	if (smp_ops == NULL \|\|
8cffc6ac BH	497	(smp_ops->cpu_bootable && !smp_ops->cpu_bootable(cpu)))
1da177e4 LT	498	return -EINVAL;
1da177e4 LT	499
1da177e4 LT	500	/* Make sure callin-map entry is 0 (can be leftover a CPU
	501	* hotplug
	502	*/
	503	cpu_callin_map[cpu] = 0;
	504
	505	/* The information for processor bringup must
	506	* be written out to main store before we release
	507	* the processor.
	508	*/
0d8d4d42	509	smp_mb();
1da177e4 LT	510
	511	/* wake up cpus */
	512	DBG("smp: kicking cpu %d\n", cpu);
	513	smp_ops->kick_cpu(cpu);
	514
	515	/*
	516	* wait to see if the cpu made a callin (is actually up).
	517	* use this value that I found through experimentation.
	518	* -- Cort
	519	*/
	520	if (system_state < SYSTEM_RUNNING)
ee0339f2	521	for (c = 50000; c && !cpu_callin_map[cpu]; c--)
1da177e4 LT	522	udelay(100);
	523	#ifdef CONFIG_HOTPLUG_CPU
	524	else
	525	/*
	526	* CPUs can take much longer to come up in the
	527	* hotplug case. Wait five seconds.
	528	*/
	529	for (c = 25; c && !cpu_callin_map[cpu]; c--) {
	530	msleep(200);
	531	}
	532	#endif
	533
	534	if (!cpu_callin_map[cpu]) {
	535	printk("Processor %u is stuck.\n", cpu);
	536	return -ENOENT;
	537	}
	538
	539	printk("Processor %u found.\n", cpu);
	540
	541	if (smp_ops->give_timebase)
	542	smp_ops->give_timebase();
	543
	544	/* Wait until cpu puts itself in the online map */
	545	while (!cpu_online(cpu))
	546	cpu_relax();
	547
	548	return 0;
	549	}
	550
	551
	552	/* Activate a secondary processor. */
	553	int __devinit start_secondary(void *unused)
	554	{
	555	unsigned int cpu = smp_processor_id();
	556
	557	atomic_inc(&init_mm.mm_count);
	558	current->active_mm = &init_mm;
	559
	560	smp_store_cpu_info(cpu);
5ad57078	561	set_dec(tb_ticks_per_jiffy);
e4d76e1c	562	preempt_disable();
1da177e4 LT	563	cpu_callin_map[cpu] = 1;
	564
	565	smp_ops->setup_cpu(cpu);
	566	if (smp_ops->take_timebase)
	567	smp_ops->take_timebase();
	568
7d4d6154	569	if (system_state > SYSTEM_BOOTING)
c6622f63	570	snapshot_timebase();
7d4d6154	571
1da177e4 LT	572	spin_lock(&call_lock);
	573	cpu_set(cpu, cpu_online_map);
	574	spin_unlock(&call_lock);
	575
	576	local_irq_enable();
	577
	578	cpu_idle();
	579	return 0;
	580	}
	581
	582	int setup_profiling_timer(unsigned int multiplier)
	583	{
	584	return 0;
	585	}
	586
	587	void __init smp_cpus_done(unsigned int max_cpus)
	588	{
	589	cpumask_t old_mask;
	590
	591	/* We want the setup_cpu() here to be called from CPU 0, but our
	592	* init thread may have been "borrowed" by another CPU in the meantime
	593	* se we pin us down to CPU 0 for a short while
	594	*/
	595	old_mask = current->cpus_allowed;
	596	set_cpus_allowed(current, cpumask_of_cpu(boot_cpuid));
	597
8cffc6ac BH	598	if (smp_ops)
8cffc6ac BH	599	smp_ops->setup_cpu(boot_cpuid);
1da177e4 LT	600
1da177e4 LT	601	set_cpus_allowed(current, old_mask);
4b703a23	602
c6622f63 PM	603	snapshot_timebases();
c6622f63 PM	604
4b703a23	605	dump_numa_cpu_topology();
1da177e4 LT	606	}
	607
	608	#ifdef CONFIG_HOTPLUG_CPU
	609	int __cpu_disable(void)
	610	{
	611	if (smp_ops->cpu_disable)
	612	return smp_ops->cpu_disable();
	613
	614	return -ENOSYS;
	615	}
	616
	617	void __cpu_die(unsigned int cpu)
	618	{
	619	if (smp_ops->cpu_die)
	620	smp_ops->cpu_die(cpu);
	621	}
	622	#endif