[net-next-2.6.git] / kernel / softirq.c

/*
 *	linux/kernel/softirq.c
 *
 *	Copyright (C) 1992 Linus Torvalds
 *
 *	Distribute under GPLv2.
 *
 *	Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
 */

#include <linux/module.h>
#include <linux/kernel_stat.h>
#include <linux/interrupt.h>
#include <linux/init.h>
#include <linux/mm.h>
#include <linux/notifier.h>
#include <linux/percpu.h>
#include <linux/cpu.h>
#include <linux/freezer.h>
#include <linux/kthread.h>
#include <linux/rcupdate.h>
#include <linux/smp.h>
#include <linux/tick.h>

#include <asm/irq.h>
/*
   - No shared variables, all the data are CPU local.
   - If a softirq needs serialization, let it serialize itself
     by its own spinlocks.
   - Even if softirq is serialized, only local cpu is marked for
     execution. Hence, we get something sort of weak cpu binding.
     Though it is still not clear, will it result in better locality
     or will not.

   Examples:
   - NET RX softirq. It is multithreaded and does not require
     any global serialization.
   - NET TX softirq. It kicks software netdevice queues, hence
     it is logically serialized per device, but this serialization
     is invisible to common code.
   - Tasklets: serialized wrt itself.
 */

#ifndef __ARCH_IRQ_STAT
irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
EXPORT_SYMBOL(irq_stat);
#endif

static struct softirq_action softirq_vec[32] __cacheline_aligned_in_smp;

static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);

/*
 * we cannot loop indefinitely here to avoid userspace starvation,
 * but we also don't want to introduce a worst case 1/HZ latency
 * to the pending events, so lets the scheduler to balance
 * the softirq load for us.
 */
static inline void wakeup_softirqd(void)
{
	/* Interrupts are disabled: no need to stop preemption */
	struct task_struct *tsk = __get_cpu_var(ksoftirqd);

	if (tsk && tsk->state != TASK_RUNNING)
		wake_up_process(tsk);
}

/*
 * This one is for softirq.c-internal use,
 * where hardirqs are disabled legitimately:
 */
#ifdef CONFIG_TRACE_IRQFLAGS
static void __local_bh_disable(unsigned long ip)
{
	unsigned long flags;

	WARN_ON_ONCE(in_irq());

	raw_local_irq_save(flags);
	add_preempt_count(SOFTIRQ_OFFSET);
	/*
	 * Were softirqs turned off above:
	 */
	if (softirq_count() == SOFTIRQ_OFFSET)
		trace_softirqs_off(ip);
	raw_local_irq_restore(flags);
}
#else /* !CONFIG_TRACE_IRQFLAGS */
static inline void __local_bh_disable(unsigned long ip)
{
	add_preempt_count(SOFTIRQ_OFFSET);
	barrier();
}
#endif /* CONFIG_TRACE_IRQFLAGS */

void local_bh_disable(void)
{
	__local_bh_disable((unsigned long)__builtin_return_address(0));
}

EXPORT_SYMBOL(local_bh_disable);

void __local_bh_enable(void)
{
	WARN_ON_ONCE(in_irq());

	/*
	 * softirqs should never be enabled by __local_bh_enable(),
	 * it always nests inside local_bh_enable() sections:
	 */
	WARN_ON_ONCE(softirq_count() == SOFTIRQ_OFFSET);

	sub_preempt_count(SOFTIRQ_OFFSET);
}
EXPORT_SYMBOL_GPL(__local_bh_enable);

/*
 * Special-case - softirqs can safely be enabled in
 * cond_resched_softirq(), or by __do_softirq(),
 * without processing still-pending softirqs:
 */
void _local_bh_enable(void)
{
	WARN_ON_ONCE(in_irq());
	WARN_ON_ONCE(!irqs_disabled());

	if (softirq_count() == SOFTIRQ_OFFSET)
		trace_softirqs_on((unsigned long)__builtin_return_address(0));
	sub_preempt_count(SOFTIRQ_OFFSET);
}

EXPORT_SYMBOL(_local_bh_enable);

void local_bh_enable(void)
{
#ifdef CONFIG_TRACE_IRQFLAGS
	unsigned long flags;

	WARN_ON_ONCE(in_irq());
#endif
	WARN_ON_ONCE(irqs_disabled());

#ifdef CONFIG_TRACE_IRQFLAGS
	local_irq_save(flags);
#endif
	/*
	 * Are softirqs going to be turned on now:
	 */
	if (softirq_count() == SOFTIRQ_OFFSET)
		trace_softirqs_on((unsigned long)__builtin_return_address(0));
	/*
	 * Keep preemption disabled until we are done with
	 * softirq processing:
 	 */
 	sub_preempt_count(SOFTIRQ_OFFSET - 1);

	if (unlikely(!in_interrupt() && local_softirq_pending()))
		do_softirq();

	dec_preempt_count();
#ifdef CONFIG_TRACE_IRQFLAGS
	local_irq_restore(flags);
#endif
	preempt_check_resched();
}
EXPORT_SYMBOL(local_bh_enable);

void local_bh_enable_ip(unsigned long ip)
{
#ifdef CONFIG_TRACE_IRQFLAGS
	unsigned long flags;

	WARN_ON_ONCE(in_irq());

	local_irq_save(flags);
#endif
	/*
	 * Are softirqs going to be turned on now:
	 */
	if (softirq_count() == SOFTIRQ_OFFSET)
		trace_softirqs_on(ip);
	/*
	 * Keep preemption disabled until we are done with
	 * softirq processing:
 	 */
 	sub_preempt_count(SOFTIRQ_OFFSET - 1);

	if (unlikely(!in_interrupt() && local_softirq_pending()))
		do_softirq();

	dec_preempt_count();
#ifdef CONFIG_TRACE_IRQFLAGS
	local_irq_restore(flags);
#endif
	preempt_check_resched();
}
EXPORT_SYMBOL(local_bh_enable_ip);

/*
 * We restart softirq processing MAX_SOFTIRQ_RESTART times,
 * and we fall back to softirqd after that.
 *
 * This number has been established via experimentation.
 * The two things to balance is latency against fairness -
 * we want to handle softirqs as soon as possible, but they
 * should not be able to lock up the box.
 */
#define MAX_SOFTIRQ_RESTART 10

asmlinkage void __do_softirq(void)
{
	struct softirq_action *h;
	__u32 pending;
	int max_restart = MAX_SOFTIRQ_RESTART;
	int cpu;

	pending = local_softirq_pending();
	account_system_vtime(current);

	__local_bh_disable((unsigned long)__builtin_return_address(0));
	trace_softirq_enter();

	cpu = smp_processor_id();
restart:
	/* Reset the pending bitmask before enabling irqs */
	set_softirq_pending(0);

	local_irq_enable();

	h = softirq_vec;

	do {
		if (pending & 1) {
			h->action(h);
			rcu_bh_qsctr_inc(cpu);
		}
		h++;
		pending >>= 1;
	} while (pending);

	local_irq_disable();

	pending = local_softirq_pending();
	if (pending && --max_restart)
		goto restart;

	if (pending)
		wakeup_softirqd();

	trace_softirq_exit();

	account_system_vtime(current);
	_local_bh_enable();
}

#ifndef __ARCH_HAS_DO_SOFTIRQ

asmlinkage void do_softirq(void)
{
	__u32 pending;
	unsigned long flags;

	if (in_interrupt())
		return;

	local_irq_save(flags);

	pending = local_softirq_pending();

	if (pending)
		__do_softirq();

	local_irq_restore(flags);
}

#endif

/*
 * Enter an interrupt context.
 */
void irq_enter(void)
{
#ifdef CONFIG_NO_HZ
	int cpu = smp_processor_id();
	if (idle_cpu(cpu) && !in_interrupt())
		tick_nohz_stop_idle(cpu);
#endif
	__irq_enter();
#ifdef CONFIG_NO_HZ
	if (idle_cpu(cpu))
		tick_nohz_update_jiffies();
#endif
}

#ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED
# define invoke_softirq()	__do_softirq()
#else
# define invoke_softirq()	do_softirq()
#endif

/*
 * Exit an interrupt context. Process softirqs if needed and possible:
 */
void irq_exit(void)
{
	account_system_vtime(current);
	trace_hardirq_exit();
	sub_preempt_count(IRQ_EXIT_OFFSET);
	if (!in_interrupt() && local_softirq_pending())
		invoke_softirq();

#ifdef CONFIG_NO_HZ
	/* Make sure that timer wheel updates are propagated */
	if (!in_interrupt() && idle_cpu(smp_processor_id()) && !need_resched())
		tick_nohz_stop_sched_tick();
	rcu_irq_exit();
#endif
	preempt_enable_no_resched();
}

/*
 * This function must run with irqs disabled!
 */
inline void raise_softirq_irqoff(unsigned int nr)
{
	__raise_softirq_irqoff(nr);

	/*
	 * If we're in an interrupt or softirq, we're done
	 * (this also catches softirq-disabled code). We will
	 * actually run the softirq once we return from
	 * the irq or softirq.
	 *
	 * Otherwise we wake up ksoftirqd to make sure we
	 * schedule the softirq soon.
	 */
	if (!in_interrupt())
		wakeup_softirqd();
}

void raise_softirq(unsigned int nr)
{
	unsigned long flags;

	local_irq_save(flags);
	raise_softirq_irqoff(nr);
	local_irq_restore(flags);
}

void open_softirq(int nr, void (*action)(struct softirq_action *))
{
	softirq_vec[nr].action = action;
}

/* Tasklets */
struct tasklet_head
{
	struct tasklet_struct *head;
	struct tasklet_struct **tail;
};

/* Some compilers disobey section attribute on statics when not
   initialized -- RR */
static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec) = { NULL };
static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec) = { NULL };

void __tasklet_schedule(struct tasklet_struct *t)
{
	unsigned long flags;

	local_irq_save(flags);
	t->next = NULL;
	*__get_cpu_var(tasklet_vec).tail = t;
	__get_cpu_var(tasklet_vec).tail = &(t->next);
	raise_softirq_irqoff(TASKLET_SOFTIRQ);
	local_irq_restore(flags);
}

EXPORT_SYMBOL(__tasklet_schedule);

void __tasklet_hi_schedule(struct tasklet_struct *t)
{
	unsigned long flags;

	local_irq_save(flags);
	t->next = NULL;
	*__get_cpu_var(tasklet_hi_vec).tail = t;
	__get_cpu_var(tasklet_hi_vec).tail = &(t->next);
	raise_softirq_irqoff(HI_SOFTIRQ);
	local_irq_restore(flags);
}

EXPORT_SYMBOL(__tasklet_hi_schedule);

static void tasklet_action(struct softirq_action *a)
{
	struct tasklet_struct *list;

	local_irq_disable();
	list = __get_cpu_var(tasklet_vec).head;
	__get_cpu_var(tasklet_vec).head = NULL;
	__get_cpu_var(tasklet_vec).tail = &__get_cpu_var(tasklet_vec).head;
	local_irq_enable();

	while (list) {
		struct tasklet_struct *t = list;

		list = list->next;

		if (tasklet_trylock(t)) {
			if (!atomic_read(&t->count)) {
				if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
					BUG();
				t->func(t->data);
				tasklet_unlock(t);
				continue;
			}
			tasklet_unlock(t);
		}

		local_irq_disable();
		t->next = NULL;
		*__get_cpu_var(tasklet_vec).tail = t;
		__get_cpu_var(tasklet_vec).tail = &(t->next);
		__raise_softirq_irqoff(TASKLET_SOFTIRQ);
		local_irq_enable();
	}
}

static void tasklet_hi_action(struct softirq_action *a)
{
	struct tasklet_struct *list;

	local_irq_disable();
	list = __get_cpu_var(tasklet_hi_vec).head;
	__get_cpu_var(tasklet_hi_vec).head = NULL;
	__get_cpu_var(tasklet_hi_vec).tail = &__get_cpu_var(tasklet_hi_vec).head;
	local_irq_enable();

	while (list) {
		struct tasklet_struct *t = list;

		list = list->next;

		if (tasklet_trylock(t)) {
			if (!atomic_read(&t->count)) {
				if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
					BUG();
				t->func(t->data);
				tasklet_unlock(t);
				continue;
			}
			tasklet_unlock(t);
		}

		local_irq_disable();
		t->next = NULL;
		*__get_cpu_var(tasklet_hi_vec).tail = t;
		__get_cpu_var(tasklet_hi_vec).tail = &(t->next);
		__raise_softirq_irqoff(HI_SOFTIRQ);
		local_irq_enable();
	}
}


void tasklet_init(struct tasklet_struct *t,
		  void (*func)(unsigned long), unsigned long data)
{
	t->next = NULL;
	t->state = 0;
	atomic_set(&t->count, 0);
	t->func = func;
	t->data = data;
}

EXPORT_SYMBOL(tasklet_init);

void tasklet_kill(struct tasklet_struct *t)
{
	if (in_interrupt())
		printk("Attempt to kill tasklet from interrupt\n");

	while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
		do
			yield();
		while (test_bit(TASKLET_STATE_SCHED, &t->state));
	}
	tasklet_unlock_wait(t);
	clear_bit(TASKLET_STATE_SCHED, &t->state);
}

EXPORT_SYMBOL(tasklet_kill);

void __init softirq_init(void)
{
	int cpu;

	for_each_possible_cpu(cpu) {
		per_cpu(tasklet_vec, cpu).tail =
			&per_cpu(tasklet_vec, cpu).head;
		per_cpu(tasklet_hi_vec, cpu).tail =
			&per_cpu(tasklet_hi_vec, cpu).head;
	}

	open_softirq(TASKLET_SOFTIRQ, tasklet_action);
	open_softirq(HI_SOFTIRQ, tasklet_hi_action);
}

static int ksoftirqd(void * __bind_cpu)
{
	set_current_state(TASK_INTERRUPTIBLE);

	while (!kthread_should_stop()) {
		preempt_disable();
		if (!local_softirq_pending()) {
			preempt_enable_no_resched();
			schedule();
			preempt_disable();
		}

		__set_current_state(TASK_RUNNING);

		while (local_softirq_pending()) {
			/* Preempt disable stops cpu going offline.
			   If already offline, we'll be on wrong CPU:
			   don't process */
			if (cpu_is_offline((long)__bind_cpu))
				goto wait_to_die;
			do_softirq();
			preempt_enable_no_resched();
			cond_resched();
			preempt_disable();
		}
		preempt_enable();
		set_current_state(TASK_INTERRUPTIBLE);
	}
	__set_current_state(TASK_RUNNING);
	return 0;

wait_to_die:
	preempt_enable();
	/* Wait for kthread_stop */
	set_current_state(TASK_INTERRUPTIBLE);
	while (!kthread_should_stop()) {
		schedule();
		set_current_state(TASK_INTERRUPTIBLE);
	}
	__set_current_state(TASK_RUNNING);
	return 0;
}

#ifdef CONFIG_HOTPLUG_CPU
/*
 * tasklet_kill_immediate is called to remove a tasklet which can already be
 * scheduled for execution on @cpu.
 *
 * Unlike tasklet_kill, this function removes the tasklet
 * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
 *
 * When this function is called, @cpu must be in the CPU_DEAD state.
 */
void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
{
	struct tasklet_struct **i;

	BUG_ON(cpu_online(cpu));
	BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));

	if (!test_bit(TASKLET_STATE_SCHED, &t->state))
		return;

	/* CPU is dead, so no lock needed. */
	for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
		if (*i == t) {
			*i = t->next;
			/* If this was the tail element, move the tail ptr */
			if (*i == NULL)
				per_cpu(tasklet_vec, cpu).tail = i;
			return;
		}
	}
	BUG();
}

static void takeover_tasklets(unsigned int cpu)
{
	/* CPU is dead, so no lock needed. */
	local_irq_disable();

	/* Find end, append list for that CPU. */
	if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
		*(__get_cpu_var(tasklet_vec).tail) = per_cpu(tasklet_vec, cpu).head;
		__get_cpu_var(tasklet_vec).tail = per_cpu(tasklet_vec, cpu).tail;
		per_cpu(tasklet_vec, cpu).head = NULL;
		per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
	}
	raise_softirq_irqoff(TASKLET_SOFTIRQ);

	if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
		*__get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).head;
		__get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).tail;
		per_cpu(tasklet_hi_vec, cpu).head = NULL;
		per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
	}
	raise_softirq_irqoff(HI_SOFTIRQ);

	local_irq_enable();
}
#endif /* CONFIG_HOTPLUG_CPU */

static int __cpuinit cpu_callback(struct notifier_block *nfb,
				  unsigned long action,
				  void *hcpu)
{
	int hotcpu = (unsigned long)hcpu;
	struct task_struct *p;

	switch (action) {
	case CPU_UP_PREPARE:
	case CPU_UP_PREPARE_FROZEN:
		p = kthread_create(ksoftirqd, hcpu, "ksoftirqd/%d", hotcpu);
		if (IS_ERR(p)) {
			printk("ksoftirqd for %i failed\n", hotcpu);
			return NOTIFY_BAD;
		}
		kthread_bind(p, hotcpu);
  		per_cpu(ksoftirqd, hotcpu) = p;
 		break;
	case CPU_ONLINE:
	case CPU_ONLINE_FROZEN:
		wake_up_process(per_cpu(ksoftirqd, hotcpu));
		break;
#ifdef CONFIG_HOTPLUG_CPU
	case CPU_UP_CANCELED:
	case CPU_UP_CANCELED_FROZEN:
		if (!per_cpu(ksoftirqd, hotcpu))
			break;
		/* Unbind so it can run.  Fall thru. */
		kthread_bind(per_cpu(ksoftirqd, hotcpu),
			     any_online_cpu(cpu_online_map));
	case CPU_DEAD:
	case CPU_DEAD_FROZEN: {
		struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };

		p = per_cpu(ksoftirqd, hotcpu);
		per_cpu(ksoftirqd, hotcpu) = NULL;
		sched_setscheduler(p, SCHED_FIFO, &param);
		kthread_stop(p);
		takeover_tasklets(hotcpu);
		break;
	}
#endif /* CONFIG_HOTPLUG_CPU */
 	}
	return NOTIFY_OK;
}

static struct notifier_block __cpuinitdata cpu_nfb = {
	.notifier_call = cpu_callback
};

__init int spawn_ksoftirqd(void)
{
	void *cpu = (void *)(long)smp_processor_id();
	int err = cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu);

	BUG_ON(err == NOTIFY_BAD);
	cpu_callback(&cpu_nfb, CPU_ONLINE, cpu);
	register_cpu_notifier(&cpu_nfb);
	return 0;
}

#ifdef CONFIG_SMP
/*
 * Call a function on all processors
 */
int on_each_cpu(void (*func) (void *info), void *info, int retry, int wait)
{
	int ret = 0;

	preempt_disable();
	ret = smp_call_function(func, info, retry, wait);
	local_irq_disable();
	func(info);
	local_irq_enable();
	preempt_enable();
	return ret;
}
EXPORT_SYMBOL(on_each_cpu);
#endif
Commit	Line	Data
1da177e4 LT	1	/*
	2	* linux/kernel/softirq.c
	3	*
	4	* Copyright (C) 1992 Linus Torvalds
	5	*
b10db7f0 PM	6	* Distribute under GPLv2.
	7	*
	8	* Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
1da177e4 LT	9	*/
	10
	11	#include <linux/module.h>
	12	#include <linux/kernel_stat.h>
	13	#include <linux/interrupt.h>
	14	#include <linux/init.h>
	15	#include <linux/mm.h>
	16	#include <linux/notifier.h>
	17	#include <linux/percpu.h>
	18	#include <linux/cpu.h>
83144186	19	#include <linux/freezer.h>
1da177e4 LT	20	#include <linux/kthread.h>
1da177e4 LT	21	#include <linux/rcupdate.h>
78eef01b	22	#include <linux/smp.h>
79bf2bb3	23	#include <linux/tick.h>
1da177e4 LT	24
	25	#include <asm/irq.h>
	26	/*
	27	- No shared variables, all the data are CPU local.
	28	- If a softirq needs serialization, let it serialize itself
	29	by its own spinlocks.
	30	- Even if softirq is serialized, only local cpu is marked for
	31	execution. Hence, we get something sort of weak cpu binding.
	32	Though it is still not clear, will it result in better locality
	33	or will not.
	34
	35	Examples:
	36	- NET RX softirq. It is multithreaded and does not require
	37	any global serialization.
	38	- NET TX softirq. It kicks software netdevice queues, hence
	39	it is logically serialized per device, but this serialization
	40	is invisible to common code.
	41	- Tasklets: serialized wrt itself.
	42	*/
	43
	44	#ifndef __ARCH_IRQ_STAT
	45	irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
	46	EXPORT_SYMBOL(irq_stat);
	47	#endif
	48
	49	static struct softirq_action softirq_vec[32] __cacheline_aligned_in_smp;
	50
	51	static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
	52
	53	/*
	54	* we cannot loop indefinitely here to avoid userspace starvation,
	55	* but we also don't want to introduce a worst case 1/HZ latency
	56	* to the pending events, so lets the scheduler to balance
	57	* the softirq load for us.
	58	*/
	59	static inline void wakeup_softirqd(void)
	60	{
	61	/* Interrupts are disabled: no need to stop preemption */
	62	struct task_struct *tsk = __get_cpu_var(ksoftirqd);
	63
	64	if (tsk && tsk->state != TASK_RUNNING)
	65	wake_up_process(tsk);
	66	}
	67
de30a2b3 IM	68	/*
	69	* This one is for softirq.c-internal use,
	70	* where hardirqs are disabled legitimately:
	71	*/
3c829c36	72	#ifdef CONFIG_TRACE_IRQFLAGS
de30a2b3 IM	73	static void __local_bh_disable(unsigned long ip)
	74	{
	75	unsigned long flags;
	76
	77	WARN_ON_ONCE(in_irq());
	78
	79	raw_local_irq_save(flags);
	80	add_preempt_count(SOFTIRQ_OFFSET);
	81	/*
	82	* Were softirqs turned off above:
	83	*/
	84	if (softirq_count() == SOFTIRQ_OFFSET)
	85	trace_softirqs_off(ip);
	86	raw_local_irq_restore(flags);
	87	}
3c829c36 TC	88	#else /* !CONFIG_TRACE_IRQFLAGS */
	89	static inline void __local_bh_disable(unsigned long ip)
	90	{
	91	add_preempt_count(SOFTIRQ_OFFSET);
	92	barrier();
	93	}
	94	#endif /* CONFIG_TRACE_IRQFLAGS */
de30a2b3 IM	95
	96	void local_bh_disable(void)
	97	{
	98	__local_bh_disable((unsigned long)__builtin_return_address(0));
	99	}
	100
	101	EXPORT_SYMBOL(local_bh_disable);
	102
	103	void __local_bh_enable(void)
	104	{
	105	WARN_ON_ONCE(in_irq());
	106
	107	/*
	108	* softirqs should never be enabled by __local_bh_enable(),
	109	* it always nests inside local_bh_enable() sections:
	110	*/
	111	WARN_ON_ONCE(softirq_count() == SOFTIRQ_OFFSET);
	112
	113	sub_preempt_count(SOFTIRQ_OFFSET);
	114	}
	115	EXPORT_SYMBOL_GPL(__local_bh_enable);
	116
	117	/*
	118	* Special-case - softirqs can safely be enabled in
	119	* cond_resched_softirq(), or by __do_softirq(),
	120	* without processing still-pending softirqs:
	121	*/
	122	void _local_bh_enable(void)
	123	{
	124	WARN_ON_ONCE(in_irq());
	125	WARN_ON_ONCE(!irqs_disabled());
	126
	127	if (softirq_count() == SOFTIRQ_OFFSET)
	128	trace_softirqs_on((unsigned long)__builtin_return_address(0));
	129	sub_preempt_count(SOFTIRQ_OFFSET);
	130	}
	131
	132	EXPORT_SYMBOL(_local_bh_enable);
	133
	134	void local_bh_enable(void)
	135	{
3c829c36	136	#ifdef CONFIG_TRACE_IRQFLAGS
de30a2b3 IM	137	unsigned long flags;
	138
	139	WARN_ON_ONCE(in_irq());
3c829c36	140	#endif
de30a2b3 IM	141	WARN_ON_ONCE(irqs_disabled());
de30a2b3 IM	142
3c829c36	143	#ifdef CONFIG_TRACE_IRQFLAGS
de30a2b3	144	local_irq_save(flags);
3c829c36	145	#endif
de30a2b3 IM	146	/*
	147	* Are softirqs going to be turned on now:
	148	*/
	149	if (softirq_count() == SOFTIRQ_OFFSET)
	150	trace_softirqs_on((unsigned long)__builtin_return_address(0));
	151	/*
	152	* Keep preemption disabled until we are done with
	153	* softirq processing:
	154	*/
	155	sub_preempt_count(SOFTIRQ_OFFSET - 1);
	156
	157	if (unlikely(!in_interrupt() && local_softirq_pending()))
	158	do_softirq();
	159
	160	dec_preempt_count();
3c829c36	161	#ifdef CONFIG_TRACE_IRQFLAGS
de30a2b3	162	local_irq_restore(flags);
3c829c36	163	#endif
de30a2b3 IM	164	preempt_check_resched();
	165	}
	166	EXPORT_SYMBOL(local_bh_enable);
	167
	168	void local_bh_enable_ip(unsigned long ip)
	169	{
3c829c36	170	#ifdef CONFIG_TRACE_IRQFLAGS
de30a2b3 IM	171	unsigned long flags;
	172
	173	WARN_ON_ONCE(in_irq());
	174
	175	local_irq_save(flags);
3c829c36	176	#endif
de30a2b3 IM	177	/*
	178	* Are softirqs going to be turned on now:
	179	*/
	180	if (softirq_count() == SOFTIRQ_OFFSET)
	181	trace_softirqs_on(ip);
	182	/*
	183	* Keep preemption disabled until we are done with
	184	* softirq processing:
	185	*/
	186	sub_preempt_count(SOFTIRQ_OFFSET - 1);
	187
	188	if (unlikely(!in_interrupt() && local_softirq_pending()))
	189	do_softirq();
	190
	191	dec_preempt_count();
3c829c36	192	#ifdef CONFIG_TRACE_IRQFLAGS
de30a2b3	193	local_irq_restore(flags);
3c829c36	194	#endif
de30a2b3 IM	195	preempt_check_resched();
	196	}
	197	EXPORT_SYMBOL(local_bh_enable_ip);
	198
1da177e4 LT	199	/*
	200	* We restart softirq processing MAX_SOFTIRQ_RESTART times,
	201	* and we fall back to softirqd after that.
	202	*
	203	* This number has been established via experimentation.
	204	* The two things to balance is latency against fairness -
	205	* we want to handle softirqs as soon as possible, but they
	206	* should not be able to lock up the box.
	207	*/
	208	#define MAX_SOFTIRQ_RESTART 10
	209
	210	asmlinkage void __do_softirq(void)
	211	{
	212	struct softirq_action *h;
	213	__u32 pending;
	214	int max_restart = MAX_SOFTIRQ_RESTART;
	215	int cpu;
	216
	217	pending = local_softirq_pending();
829035fd PM	218	account_system_vtime(current);
829035fd PM	219
de30a2b3 IM	220	__local_bh_disable((unsigned long)__builtin_return_address(0));
de30a2b3 IM	221	trace_softirq_enter();
1da177e4	222
1da177e4 LT	223	cpu = smp_processor_id();
	224	restart:
	225	/* Reset the pending bitmask before enabling irqs */
3f74478b	226	set_softirq_pending(0);
1da177e4	227
c70f5d66	228	local_irq_enable();
1da177e4 LT	229
	230	h = softirq_vec;
	231
	232	do {
	233	if (pending & 1) {
	234	h->action(h);
	235	rcu_bh_qsctr_inc(cpu);
	236	}
	237	h++;
	238	pending >>= 1;
	239	} while (pending);
	240
c70f5d66	241	local_irq_disable();
1da177e4 LT	242
	243	pending = local_softirq_pending();
	244	if (pending && --max_restart)
	245	goto restart;
	246
	247	if (pending)
	248	wakeup_softirqd();
	249
de30a2b3	250	trace_softirq_exit();
829035fd PM	251
829035fd PM	252	account_system_vtime(current);
de30a2b3	253	_local_bh_enable();
1da177e4 LT	254	}
	255
	256	#ifndef __ARCH_HAS_DO_SOFTIRQ
	257
	258	asmlinkage void do_softirq(void)
	259	{
	260	__u32 pending;
	261	unsigned long flags;
	262
	263	if (in_interrupt())
	264	return;
	265
	266	local_irq_save(flags);
	267
	268	pending = local_softirq_pending();
	269
	270	if (pending)
	271	__do_softirq();
	272
	273	local_irq_restore(flags);
	274	}
	275
1da177e4 LT	276	#endif
1da177e4 LT	277
dde4b2b5 IM	278	/*
	279	* Enter an interrupt context.
	280	*/
	281	void irq_enter(void)
	282	{
6378ddb5 VP	283	#ifdef CONFIG_NO_HZ
	284	int cpu = smp_processor_id();
	285	if (idle_cpu(cpu) && !in_interrupt())
	286	tick_nohz_stop_idle(cpu);
	287	#endif
79bf2bb3 TG	288	__irq_enter();
79bf2bb3 TG	289	#ifdef CONFIG_NO_HZ
6378ddb5	290	if (idle_cpu(cpu))
79bf2bb3 TG	291	tick_nohz_update_jiffies();
79bf2bb3 TG	292	#endif
dde4b2b5 IM	293	}
dde4b2b5 IM	294
1da177e4 LT	295	#ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED
	296	# define invoke_softirq() __do_softirq()
	297	#else
	298	# define invoke_softirq() do_softirq()
	299	#endif
	300
	301	/*
	302	* Exit an interrupt context. Process softirqs if needed and possible:
	303	*/
	304	void irq_exit(void)
	305	{
	306	account_system_vtime(current);
de30a2b3	307	trace_hardirq_exit();
1da177e4 LT	308	sub_preempt_count(IRQ_EXIT_OFFSET);
	309	if (!in_interrupt() && local_softirq_pending())
	310	invoke_softirq();
79bf2bb3 TG	311
	312	#ifdef CONFIG_NO_HZ
	313	/* Make sure that timer wheel updates are propagated */
	314	if (!in_interrupt() && idle_cpu(smp_processor_id()) && !need_resched())
	315	tick_nohz_stop_sched_tick();
2232c2d8	316	rcu_irq_exit();
79bf2bb3	317	#endif
1da177e4 LT	318	preempt_enable_no_resched();
	319	}
	320
	321	/*
	322	* This function must run with irqs disabled!
	323	*/
7ad5b3a5	324	inline void raise_softirq_irqoff(unsigned int nr)
1da177e4 LT	325	{
	326	__raise_softirq_irqoff(nr);
	327
	328	/*
	329	* If we're in an interrupt or softirq, we're done
	330	* (this also catches softirq-disabled code). We will
	331	* actually run the softirq once we return from
	332	* the irq or softirq.
	333	*
	334	* Otherwise we wake up ksoftirqd to make sure we
	335	* schedule the softirq soon.
	336	*/
	337	if (!in_interrupt())
	338	wakeup_softirqd();
	339	}
	340
7ad5b3a5	341	void raise_softirq(unsigned int nr)
1da177e4 LT	342	{
	343	unsigned long flags;
	344
	345	local_irq_save(flags);
	346	raise_softirq_irqoff(nr);
	347	local_irq_restore(flags);
	348	}
	349
962cf36c	350	void open_softirq(int nr, void (action)(struct softirq_action ))
1da177e4	351	{
1da177e4 LT	352	softirq_vec[nr].action = action;
	353	}
	354
1da177e4 LT	355	/* Tasklets */
	356	struct tasklet_head
	357	{
48f20a9a OJ	358	struct tasklet_struct *head;
48f20a9a OJ	359	struct tasklet_struct **tail;
1da177e4 LT	360	};
	361
	362	/* Some compilers disobey section attribute on statics when not
	363	initialized -- RR */
	364	static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec) = { NULL };
	365	static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec) = { NULL };
	366
7ad5b3a5	367	void __tasklet_schedule(struct tasklet_struct *t)
1da177e4 LT	368	{
	369	unsigned long flags;
	370
	371	local_irq_save(flags);
48f20a9a OJ	372	t->next = NULL;
	373	*__get_cpu_var(tasklet_vec).tail = t;
	374	__get_cpu_var(tasklet_vec).tail = &(t->next);
1da177e4 LT	375	raise_softirq_irqoff(TASKLET_SOFTIRQ);
	376	local_irq_restore(flags);
	377	}
	378
	379	EXPORT_SYMBOL(__tasklet_schedule);
	380
7ad5b3a5	381	void __tasklet_hi_schedule(struct tasklet_struct *t)
1da177e4 LT	382	{
	383	unsigned long flags;
	384
	385	local_irq_save(flags);
48f20a9a OJ	386	t->next = NULL;
	387	*__get_cpu_var(tasklet_hi_vec).tail = t;
	388	__get_cpu_var(tasklet_hi_vec).tail = &(t->next);
1da177e4 LT	389	raise_softirq_irqoff(HI_SOFTIRQ);
	390	local_irq_restore(flags);
	391	}
	392
	393	EXPORT_SYMBOL(__tasklet_hi_schedule);
	394
	395	static void tasklet_action(struct softirq_action *a)
	396	{
	397	struct tasklet_struct *list;
	398
	399	local_irq_disable();
48f20a9a OJ	400	list = __get_cpu_var(tasklet_vec).head;
	401	__get_cpu_var(tasklet_vec).head = NULL;
	402	__get_cpu_var(tasklet_vec).tail = &__get_cpu_var(tasklet_vec).head;
1da177e4 LT	403	local_irq_enable();
	404
	405	while (list) {
	406	struct tasklet_struct *t = list;
	407
	408	list = list->next;
	409
	410	if (tasklet_trylock(t)) {
	411	if (!atomic_read(&t->count)) {
	412	if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
	413	BUG();
	414	t->func(t->data);
	415	tasklet_unlock(t);
	416	continue;
	417	}
	418	tasklet_unlock(t);
	419	}
	420
	421	local_irq_disable();
48f20a9a OJ	422	t->next = NULL;
	423	*__get_cpu_var(tasklet_vec).tail = t;
	424	__get_cpu_var(tasklet_vec).tail = &(t->next);
1da177e4 LT	425	__raise_softirq_irqoff(TASKLET_SOFTIRQ);
	426	local_irq_enable();
	427	}
	428	}
	429
	430	static void tasklet_hi_action(struct softirq_action *a)
	431	{
	432	struct tasklet_struct *list;
	433
	434	local_irq_disable();
48f20a9a OJ	435	list = __get_cpu_var(tasklet_hi_vec).head;
	436	__get_cpu_var(tasklet_hi_vec).head = NULL;
	437	__get_cpu_var(tasklet_hi_vec).tail = &__get_cpu_var(tasklet_hi_vec).head;
1da177e4 LT	438	local_irq_enable();
	439
	440	while (list) {
	441	struct tasklet_struct *t = list;
	442
	443	list = list->next;
	444
	445	if (tasklet_trylock(t)) {
	446	if (!atomic_read(&t->count)) {
	447	if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
	448	BUG();
	449	t->func(t->data);
	450	tasklet_unlock(t);
	451	continue;
	452	}
	453	tasklet_unlock(t);
	454	}
	455
	456	local_irq_disable();
48f20a9a OJ	457	t->next = NULL;
	458	*__get_cpu_var(tasklet_hi_vec).tail = t;
	459	__get_cpu_var(tasklet_hi_vec).tail = &(t->next);
1da177e4 LT	460	__raise_softirq_irqoff(HI_SOFTIRQ);
	461	local_irq_enable();
	462	}
	463	}
	464
	465
	466	void tasklet_init(struct tasklet_struct *t,
	467	void (*func)(unsigned long), unsigned long data)
	468	{
	469	t->next = NULL;
	470	t->state = 0;
	471	atomic_set(&t->count, 0);
	472	t->func = func;
	473	t->data = data;
	474	}
	475
	476	EXPORT_SYMBOL(tasklet_init);
	477
	478	void tasklet_kill(struct tasklet_struct *t)
	479	{
	480	if (in_interrupt())
	481	printk("Attempt to kill tasklet from interrupt\n");
	482
	483	while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
	484	do
	485	yield();
	486	while (test_bit(TASKLET_STATE_SCHED, &t->state));
	487	}
	488	tasklet_unlock_wait(t);
	489	clear_bit(TASKLET_STATE_SCHED, &t->state);
	490	}
	491
	492	EXPORT_SYMBOL(tasklet_kill);
	493
	494	void __init softirq_init(void)
	495	{
48f20a9a OJ	496	int cpu;
	497
	498	for_each_possible_cpu(cpu) {
	499	per_cpu(tasklet_vec, cpu).tail =
	500	&per_cpu(tasklet_vec, cpu).head;
	501	per_cpu(tasklet_hi_vec, cpu).tail =
	502	&per_cpu(tasklet_hi_vec, cpu).head;
	503	}
	504
962cf36c CM	505	open_softirq(TASKLET_SOFTIRQ, tasklet_action);
962cf36c CM	506	open_softirq(HI_SOFTIRQ, tasklet_hi_action);
1da177e4 LT	507	}
	508
	509	static int ksoftirqd(void * __bind_cpu)
	510	{
1da177e4 LT	511	set_current_state(TASK_INTERRUPTIBLE);
	512
	513	while (!kthread_should_stop()) {
	514	preempt_disable();
	515	if (!local_softirq_pending()) {
	516	preempt_enable_no_resched();
	517	schedule();
	518	preempt_disable();
	519	}
	520
	521	__set_current_state(TASK_RUNNING);
	522
	523	while (local_softirq_pending()) {
	524	/* Preempt disable stops cpu going offline.
	525	If already offline, we'll be on wrong CPU:
	526	don't process */
	527	if (cpu_is_offline((long)__bind_cpu))
	528	goto wait_to_die;
	529	do_softirq();
	530	preempt_enable_no_resched();
	531	cond_resched();
	532	preempt_disable();
	533	}
	534	preempt_enable();
	535	set_current_state(TASK_INTERRUPTIBLE);
	536	}
	537	__set_current_state(TASK_RUNNING);
	538	return 0;
	539
	540	wait_to_die:
	541	preempt_enable();
	542	/* Wait for kthread_stop */
	543	set_current_state(TASK_INTERRUPTIBLE);
	544	while (!kthread_should_stop()) {
	545	schedule();
	546	set_current_state(TASK_INTERRUPTIBLE);
	547	}
	548	__set_current_state(TASK_RUNNING);
	549	return 0;
	550	}
	551
	552	#ifdef CONFIG_HOTPLUG_CPU
	553	/*
	554	* tasklet_kill_immediate is called to remove a tasklet which can already be
	555	* scheduled for execution on @cpu.
	556	*
	557	* Unlike tasklet_kill, this function removes the tasklet
	558	* _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
	559	*
	560	* When this function is called, @cpu must be in the CPU_DEAD state.
	561	*/
	562	void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
	563	{
	564	struct tasklet_struct **i;
	565
	566	BUG_ON(cpu_online(cpu));
	567	BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
	568
	569	if (!test_bit(TASKLET_STATE_SCHED, &t->state))
	570	return;
	571
	572	/* CPU is dead, so no lock needed. */
48f20a9a	573	for (i = &per_cpu(tasklet_vec, cpu).head; i; i = &(i)->next) {
1da177e4 LT	574	if (*i == t) {
1da177e4 LT	575	*i = t->next;
48f20a9a OJ	576	/* If this was the tail element, move the tail ptr */
	577	if (*i == NULL)
	578	per_cpu(tasklet_vec, cpu).tail = i;
1da177e4 LT	579	return;
	580	}
	581	}
	582	BUG();
	583	}
	584
	585	static void takeover_tasklets(unsigned int cpu)
	586	{
1da177e4 LT	587	/* CPU is dead, so no lock needed. */
	588	local_irq_disable();
	589
	590	/* Find end, append list for that CPU. */
e5e41723 CB	591	if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
	592	*(__get_cpu_var(tasklet_vec).tail) = per_cpu(tasklet_vec, cpu).head;
	593	__get_cpu_var(tasklet_vec).tail = per_cpu(tasklet_vec, cpu).tail;
	594	per_cpu(tasklet_vec, cpu).head = NULL;
	595	per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
	596	}
1da177e4 LT	597	raise_softirq_irqoff(TASKLET_SOFTIRQ);
1da177e4 LT	598
e5e41723 CB	599	if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
	600	*__get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).head;
	601	__get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).tail;
	602	per_cpu(tasklet_hi_vec, cpu).head = NULL;
	603	per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
	604	}
1da177e4 LT	605	raise_softirq_irqoff(HI_SOFTIRQ);
	606
	607	local_irq_enable();
	608	}
	609	#endif /* CONFIG_HOTPLUG_CPU */
	610
8c78f307	611	static int __cpuinit cpu_callback(struct notifier_block *nfb,
1da177e4 LT	612	unsigned long action,
	613	void *hcpu)
	614	{
	615	int hotcpu = (unsigned long)hcpu;
	616	struct task_struct *p;
	617
	618	switch (action) {
	619	case CPU_UP_PREPARE:
8bb78442	620	case CPU_UP_PREPARE_FROZEN:
1da177e4 LT	621	p = kthread_create(ksoftirqd, hcpu, "ksoftirqd/%d", hotcpu);
	622	if (IS_ERR(p)) {
	623	printk("ksoftirqd for %i failed\n", hotcpu);
	624	return NOTIFY_BAD;
	625	}
	626	kthread_bind(p, hotcpu);
	627	per_cpu(ksoftirqd, hotcpu) = p;
	628	break;
	629	case CPU_ONLINE:
8bb78442	630	case CPU_ONLINE_FROZEN:
1da177e4 LT	631	wake_up_process(per_cpu(ksoftirqd, hotcpu));
	632	break;
	633	#ifdef CONFIG_HOTPLUG_CPU
	634	case CPU_UP_CANCELED:
8bb78442	635	case CPU_UP_CANCELED_FROZEN:
fc75cdfa HC	636	if (!per_cpu(ksoftirqd, hotcpu))
fc75cdfa HC	637	break;
1da177e4	638	/* Unbind so it can run. Fall thru. */
a4c4af7c HC	639	kthread_bind(per_cpu(ksoftirqd, hotcpu),
a4c4af7c HC	640	any_online_cpu(cpu_online_map));
1da177e4	641	case CPU_DEAD:
1c6b4aa9 ST	642	case CPU_DEAD_FROZEN: {
	643	struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
	644
1da177e4 LT	645	p = per_cpu(ksoftirqd, hotcpu);
1da177e4 LT	646	per_cpu(ksoftirqd, hotcpu) = NULL;
1c6b4aa9	647	sched_setscheduler(p, SCHED_FIFO, &param);
1da177e4 LT	648	kthread_stop(p);
	649	takeover_tasklets(hotcpu);
	650	break;
1c6b4aa9	651	}
1da177e4 LT	652	#endif /* CONFIG_HOTPLUG_CPU */
	653	}
	654	return NOTIFY_OK;
	655	}
	656
8c78f307	657	static struct notifier_block __cpuinitdata cpu_nfb = {
1da177e4 LT	658	.notifier_call = cpu_callback
	659	};
	660
	661	__init int spawn_ksoftirqd(void)
	662	{
	663	void cpu = (void )(long)smp_processor_id();
07dccf33 AM	664	int err = cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu);
	665
	666	BUG_ON(err == NOTIFY_BAD);
1da177e4 LT	667	cpu_callback(&cpu_nfb, CPU_ONLINE, cpu);
	668	register_cpu_notifier(&cpu_nfb);
	669	return 0;
	670	}
78eef01b AM	671
	672	#ifdef CONFIG_SMP
	673	/*
	674	* Call a function on all processors
	675	*/
	676	int on_each_cpu(void (func) (void info), void *info, int retry, int wait)
	677	{
	678	int ret = 0;
	679
	680	preempt_disable();
	681	ret = smp_call_function(func, info, retry, wait);
	682	local_irq_disable();
	683	func(info);
	684	local_irq_enable();
	685	preempt_enable();
	686	return ret;
	687	}
	688	EXPORT_SYMBOL(on_each_cpu);
	689	#endif