[net-next-2.6.git] / lib / percpu_counter.c

/*
 * Fast batching percpu counters.
 */

#include <linux/percpu_counter.h>
#include <linux/notifier.h>
#include <linux/mutex.h>
#include <linux/init.h>
#include <linux/cpu.h>
#include <linux/module.h>

static LIST_HEAD(percpu_counters);
static DEFINE_MUTEX(percpu_counters_lock);

void percpu_counter_set(struct percpu_counter *fbc, s64 amount)
{
	int cpu;

	spin_lock(&fbc->lock);
	for_each_possible_cpu(cpu) {
		s32 *pcount = per_cpu_ptr(fbc->counters, cpu);
		*pcount = 0;
	}
	fbc->count = amount;
	spin_unlock(&fbc->lock);
}
EXPORT_SYMBOL(percpu_counter_set);

void __percpu_counter_add(struct percpu_counter *fbc, s64 amount, s32 batch)
{
	s64 count;
	s32 *pcount;
	int cpu = get_cpu();

	pcount = per_cpu_ptr(fbc->counters, cpu);
	count = *pcount + amount;
	if (count >= batch || count <= -batch) {
		spin_lock(&fbc->lock);
		fbc->count += count;
		*pcount = 0;
		spin_unlock(&fbc->lock);
	} else {
		*pcount = count;
	}
	put_cpu();
}
EXPORT_SYMBOL(__percpu_counter_add);

/*
 * Add up all the per-cpu counts, return the result.  This is a more accurate
 * but much slower version of percpu_counter_read_positive()
 */
s64 __percpu_counter_sum(struct percpu_counter *fbc)
{
	s64 ret;
	int cpu;

	spin_lock(&fbc->lock);
	ret = fbc->count;
	for_each_online_cpu(cpu) {
		s32 *pcount = per_cpu_ptr(fbc->counters, cpu);
		ret += *pcount;
	}
	spin_unlock(&fbc->lock);
	return ret;
}
EXPORT_SYMBOL(__percpu_counter_sum);

static struct lock_class_key percpu_counter_irqsafe;

int percpu_counter_init(struct percpu_counter *fbc, s64 amount)
{
	spin_lock_init(&fbc->lock);
	fbc->count = amount;
	fbc->counters = alloc_percpu(s32);
	if (!fbc->counters)
		return -ENOMEM;
#ifdef CONFIG_HOTPLUG_CPU
	mutex_lock(&percpu_counters_lock);
	list_add(&fbc->list, &percpu_counters);
	mutex_unlock(&percpu_counters_lock);
#endif
	return 0;
}
EXPORT_SYMBOL(percpu_counter_init);

int percpu_counter_init_irq(struct percpu_counter *fbc, s64 amount)
{
	int err;

	err = percpu_counter_init(fbc, amount);
	if (!err)
		lockdep_set_class(&fbc->lock, &percpu_counter_irqsafe);
	return err;
}

void percpu_counter_destroy(struct percpu_counter *fbc)
{
	if (!fbc->counters)
		return;

#ifdef CONFIG_HOTPLUG_CPU
	mutex_lock(&percpu_counters_lock);
	list_del(&fbc->list);
	mutex_unlock(&percpu_counters_lock);
#endif
	free_percpu(fbc->counters);
	fbc->counters = NULL;
}
EXPORT_SYMBOL(percpu_counter_destroy);

int percpu_counter_batch __read_mostly = 32;
EXPORT_SYMBOL(percpu_counter_batch);

static void compute_batch_value(void)
{
	int nr = num_online_cpus();

	percpu_counter_batch = max(32, nr*2);
}

static int __cpuinit percpu_counter_hotcpu_callback(struct notifier_block *nb,
					unsigned long action, void *hcpu)
{
#ifdef CONFIG_HOTPLUG_CPU
	unsigned int cpu;
	struct percpu_counter *fbc;

	compute_batch_value();
	if (action != CPU_DEAD)
		return NOTIFY_OK;

	cpu = (unsigned long)hcpu;
	mutex_lock(&percpu_counters_lock);
	list_for_each_entry(fbc, &percpu_counters, list) {
		s32 *pcount;
		unsigned long flags;

		spin_lock_irqsave(&fbc->lock, flags);
		pcount = per_cpu_ptr(fbc->counters, cpu);
		fbc->count += *pcount;
		*pcount = 0;
		spin_unlock_irqrestore(&fbc->lock, flags);
	}
	mutex_unlock(&percpu_counters_lock);
#endif
	return NOTIFY_OK;
}

static int __init percpu_counter_startup(void)
{
	compute_batch_value();
	hotcpu_notifier(percpu_counter_hotcpu_callback, 0);
	return 0;
}
module_init(percpu_counter_startup);
Commit	Line	Data
3cbc5640 RT	1	/*
	2	* Fast batching percpu counters.
	3	*/
	4
	5	#include <linux/percpu_counter.h>
c67ad917 AM	6	#include <linux/notifier.h>
	7	#include <linux/mutex.h>
	8	#include <linux/init.h>
	9	#include <linux/cpu.h>
3cbc5640 RT	10	#include <linux/module.h>
3cbc5640 RT	11
c67ad917 AM	12	static LIST_HEAD(percpu_counters);
c67ad917 AM	13	static DEFINE_MUTEX(percpu_counters_lock);
c67ad917	14
3a587f47 PZ	15	void percpu_counter_set(struct percpu_counter *fbc, s64 amount)
	16	{
	17	int cpu;
	18
	19	spin_lock(&fbc->lock);
	20	for_each_possible_cpu(cpu) {
	21	s32 *pcount = per_cpu_ptr(fbc->counters, cpu);
	22	*pcount = 0;
	23	}
	24	fbc->count = amount;
	25	spin_unlock(&fbc->lock);
	26	}
	27	EXPORT_SYMBOL(percpu_counter_set);
	28
20e89767	29	void __percpu_counter_add(struct percpu_counter *fbc, s64 amount, s32 batch)
3cbc5640	30	{
20e89767	31	s64 count;
0216bfcf	32	s32 *pcount;
3cbc5640 RT	33	int cpu = get_cpu();
	34
	35	pcount = per_cpu_ptr(fbc->counters, cpu);
	36	count = *pcount + amount;
252e0ba6	37	if (count >= batch \|\| count <= -batch) {
3cbc5640 RT	38	spin_lock(&fbc->lock);
	39	fbc->count += count;
	40	*pcount = 0;
	41	spin_unlock(&fbc->lock);
	42	} else {
	43	*pcount = count;
	44	}
	45	put_cpu();
	46	}
252e0ba6	47	EXPORT_SYMBOL(__percpu_counter_add);
3cbc5640 RT	48
	49	/*
	50	* Add up all the per-cpu counts, return the result. This is a more accurate
	51	* but much slower version of percpu_counter_read_positive()
	52	*/
02d21168	53	s64 __percpu_counter_sum(struct percpu_counter *fbc)
3cbc5640	54	{
0216bfcf	55	s64 ret;
3cbc5640 RT	56	int cpu;
	57
	58	spin_lock(&fbc->lock);
	59	ret = fbc->count;
b4ef0296	60	for_each_online_cpu(cpu) {
0216bfcf	61	s32 *pcount = per_cpu_ptr(fbc->counters, cpu);
3cbc5640 RT	62	ret += *pcount;
	63	}
	64	spin_unlock(&fbc->lock);
bf1d89c8	65	return ret;
3cbc5640	66	}
bf1d89c8	67	EXPORT_SYMBOL(__percpu_counter_sum);
c67ad917	68
dc62a30e PZ	69	static struct lock_class_key percpu_counter_irqsafe;
dc62a30e PZ	70
833f4077	71	int percpu_counter_init(struct percpu_counter *fbc, s64 amount)
c67ad917 AM	72	{
	73	spin_lock_init(&fbc->lock);
	74	fbc->count = amount;
	75	fbc->counters = alloc_percpu(s32);
833f4077 PZ	76	if (!fbc->counters)
833f4077 PZ	77	return -ENOMEM;
c67ad917 AM	78	#ifdef CONFIG_HOTPLUG_CPU
	79	mutex_lock(&percpu_counters_lock);
	80	list_add(&fbc->list, &percpu_counters);
	81	mutex_unlock(&percpu_counters_lock);
	82	#endif
833f4077	83	return 0;
c67ad917 AM	84	}
c67ad917 AM	85	EXPORT_SYMBOL(percpu_counter_init);
dc62a30e PZ	86
	87	int percpu_counter_init_irq(struct percpu_counter *fbc, s64 amount)
	88	{
	89	int err;
	90
	91	err = percpu_counter_init(fbc, amount);
	92	if (!err)
	93	lockdep_set_class(&fbc->lock, &percpu_counter_irqsafe);
	94	return err;
	95	}
c67ad917 AM	96
	97	void percpu_counter_destroy(struct percpu_counter *fbc)
	98	{
833f4077 PZ	99	if (!fbc->counters)
	100	return;
	101
c67ad917 AM	102	#ifdef CONFIG_HOTPLUG_CPU
	103	mutex_lock(&percpu_counters_lock);
	104	list_del(&fbc->list);
	105	mutex_unlock(&percpu_counters_lock);
	106	#endif
fd3d664f ED	107	free_percpu(fbc->counters);
fd3d664f ED	108	fbc->counters = NULL;
c67ad917 AM	109	}
	110	EXPORT_SYMBOL(percpu_counter_destroy);
	111
179f7ebf ED	112	int percpu_counter_batch __read_mostly = 32;
	113	EXPORT_SYMBOL(percpu_counter_batch);
	114
	115	static void compute_batch_value(void)
	116	{
	117	int nr = num_online_cpus();
	118
	119	percpu_counter_batch = max(32, nr*2);
	120	}
	121
c67ad917 AM	122	static int __cpuinit percpu_counter_hotcpu_callback(struct notifier_block *nb,
	123	unsigned long action, void *hcpu)
	124	{
179f7ebf	125	#ifdef CONFIG_HOTPLUG_CPU
c67ad917 AM	126	unsigned int cpu;
	127	struct percpu_counter *fbc;
	128
179f7ebf	129	compute_batch_value();
c67ad917 AM	130	if (action != CPU_DEAD)
	131	return NOTIFY_OK;
	132
	133	cpu = (unsigned long)hcpu;
	134	mutex_lock(&percpu_counters_lock);
	135	list_for_each_entry(fbc, &percpu_counters, list) {
	136	s32 *pcount;
d2b20b11	137	unsigned long flags;
c67ad917	138
d2b20b11	139	spin_lock_irqsave(&fbc->lock, flags);
c67ad917 AM	140	pcount = per_cpu_ptr(fbc->counters, cpu);
	141	fbc->count += *pcount;
	142	*pcount = 0;
d2b20b11	143	spin_unlock_irqrestore(&fbc->lock, flags);
c67ad917 AM	144	}
c67ad917 AM	145	mutex_unlock(&percpu_counters_lock);
179f7ebf	146	#endif
c67ad917 AM	147	return NOTIFY_OK;
	148	}
	149
	150	static int __init percpu_counter_startup(void)
	151	{
179f7ebf	152	compute_batch_value();
c67ad917 AM	153	hotcpu_notifier(percpu_counter_hotcpu_callback, 0);
	154	return 0;
	155	}
	156	module_init(percpu_counter_startup);