[net-next-2.6.git] / lib / percpu_counter.c

/*
 * Fast batching percpu counters.
 */

#include <linux/percpu_counter.h>
#include <linux/notifier.h>
#include <linux/mutex.h>
#include <linux/init.h>
#include <linux/cpu.h>
#include <linux/module.h>

#ifdef CONFIG_HOTPLUG_CPU
static LIST_HEAD(percpu_counters);
static DEFINE_MUTEX(percpu_counters_lock);
#endif

void __percpu_counter_add(struct percpu_counter *fbc, s64 amount, s32 batch)
{
	s64 count;
	s32 *pcount;
	int cpu = get_cpu();

	pcount = per_cpu_ptr(fbc->counters, cpu);
	count = *pcount + amount;
	if (count >= batch || count <= -batch) {
		spin_lock(&fbc->lock);
		fbc->count += count;
		*pcount = 0;
		spin_unlock(&fbc->lock);
	} else {
		*pcount = count;
	}
	put_cpu();
}
EXPORT_SYMBOL(__percpu_counter_add);

/*
 * Add up all the per-cpu counts, return the result.  This is a more accurate
 * but much slower version of percpu_counter_read_positive()
 */
s64 percpu_counter_sum(struct percpu_counter *fbc)
{
	s64 ret;
	int cpu;

	spin_lock(&fbc->lock);
	ret = fbc->count;
	for_each_online_cpu(cpu) {
		s32 *pcount = per_cpu_ptr(fbc->counters, cpu);
		ret += *pcount;
	}
	spin_unlock(&fbc->lock);
	return ret < 0 ? 0 : ret;
}
EXPORT_SYMBOL(percpu_counter_sum);

void percpu_counter_init(struct percpu_counter *fbc, s64 amount)
{
	spin_lock_init(&fbc->lock);
	fbc->count = amount;
	fbc->counters = alloc_percpu(s32);
#ifdef CONFIG_HOTPLUG_CPU
	mutex_lock(&percpu_counters_lock);
	list_add(&fbc->list, &percpu_counters);
	mutex_unlock(&percpu_counters_lock);
#endif
}
EXPORT_SYMBOL(percpu_counter_init);

void percpu_counter_destroy(struct percpu_counter *fbc)
{
	free_percpu(fbc->counters);
#ifdef CONFIG_HOTPLUG_CPU
	mutex_lock(&percpu_counters_lock);
	list_del(&fbc->list);
	mutex_unlock(&percpu_counters_lock);
#endif
}
EXPORT_SYMBOL(percpu_counter_destroy);

#ifdef CONFIG_HOTPLUG_CPU
static int __cpuinit percpu_counter_hotcpu_callback(struct notifier_block *nb,
					unsigned long action, void *hcpu)
{
	unsigned int cpu;
	struct percpu_counter *fbc;

	if (action != CPU_DEAD)
		return NOTIFY_OK;

	cpu = (unsigned long)hcpu;
	mutex_lock(&percpu_counters_lock);
	list_for_each_entry(fbc, &percpu_counters, list) {
		s32 *pcount;

		spin_lock(&fbc->lock);
		pcount = per_cpu_ptr(fbc->counters, cpu);
		fbc->count += *pcount;
		*pcount = 0;
		spin_unlock(&fbc->lock);
	}
	mutex_unlock(&percpu_counters_lock);
	return NOTIFY_OK;
}

static int __init percpu_counter_startup(void)
{
	hotcpu_notifier(percpu_counter_hotcpu_callback, 0);
	return 0;
}
module_init(percpu_counter_startup);
#endif
Commit	Line	Data
3cbc5640 RT	1	/*
	2	* Fast batching percpu counters.
	3	*/
	4
	5	#include <linux/percpu_counter.h>
c67ad917 AM	6	#include <linux/notifier.h>
	7	#include <linux/mutex.h>
	8	#include <linux/init.h>
	9	#include <linux/cpu.h>
3cbc5640 RT	10	#include <linux/module.h>
3cbc5640 RT	11
c67ad917 AM	12	#ifdef CONFIG_HOTPLUG_CPU
	13	static LIST_HEAD(percpu_counters);
	14	static DEFINE_MUTEX(percpu_counters_lock);
	15	#endif
	16
20e89767	17	void __percpu_counter_add(struct percpu_counter *fbc, s64 amount, s32 batch)
3cbc5640	18	{
20e89767	19	s64 count;
0216bfcf	20	s32 *pcount;
3cbc5640 RT	21	int cpu = get_cpu();
	22
	23	pcount = per_cpu_ptr(fbc->counters, cpu);
	24	count = *pcount + amount;
252e0ba6	25	if (count >= batch \|\| count <= -batch) {
3cbc5640 RT	26	spin_lock(&fbc->lock);
	27	fbc->count += count;
	28	*pcount = 0;
	29	spin_unlock(&fbc->lock);
	30	} else {
	31	*pcount = count;
	32	}
	33	put_cpu();
	34	}
252e0ba6	35	EXPORT_SYMBOL(__percpu_counter_add);
3cbc5640 RT	36
	37	/*
	38	* Add up all the per-cpu counts, return the result. This is a more accurate
	39	* but much slower version of percpu_counter_read_positive()
	40	*/
0216bfcf	41	s64 percpu_counter_sum(struct percpu_counter *fbc)
3cbc5640	42	{
0216bfcf	43	s64 ret;
3cbc5640 RT	44	int cpu;
	45
	46	spin_lock(&fbc->lock);
	47	ret = fbc->count;
b4ef0296	48	for_each_online_cpu(cpu) {
0216bfcf	49	s32 *pcount = per_cpu_ptr(fbc->counters, cpu);
3cbc5640 RT	50	ret += *pcount;
	51	}
	52	spin_unlock(&fbc->lock);
	53	return ret < 0 ? 0 : ret;
	54	}
	55	EXPORT_SYMBOL(percpu_counter_sum);
c67ad917 AM	56
	57	void percpu_counter_init(struct percpu_counter *fbc, s64 amount)
	58	{
	59	spin_lock_init(&fbc->lock);
	60	fbc->count = amount;
	61	fbc->counters = alloc_percpu(s32);
	62	#ifdef CONFIG_HOTPLUG_CPU
	63	mutex_lock(&percpu_counters_lock);
	64	list_add(&fbc->list, &percpu_counters);
	65	mutex_unlock(&percpu_counters_lock);
	66	#endif
	67	}
	68	EXPORT_SYMBOL(percpu_counter_init);
	69
	70	void percpu_counter_destroy(struct percpu_counter *fbc)
	71	{
	72	free_percpu(fbc->counters);
	73	#ifdef CONFIG_HOTPLUG_CPU
	74	mutex_lock(&percpu_counters_lock);
	75	list_del(&fbc->list);
	76	mutex_unlock(&percpu_counters_lock);
	77	#endif
	78	}
	79	EXPORT_SYMBOL(percpu_counter_destroy);
	80
	81	#ifdef CONFIG_HOTPLUG_CPU
	82	static int __cpuinit percpu_counter_hotcpu_callback(struct notifier_block *nb,
	83	unsigned long action, void *hcpu)
	84	{
	85	unsigned int cpu;
	86	struct percpu_counter *fbc;
	87
	88	if (action != CPU_DEAD)
	89	return NOTIFY_OK;
	90
	91	cpu = (unsigned long)hcpu;
	92	mutex_lock(&percpu_counters_lock);
	93	list_for_each_entry(fbc, &percpu_counters, list) {
	94	s32 *pcount;
	95
	96	spin_lock(&fbc->lock);
	97	pcount = per_cpu_ptr(fbc->counters, cpu);
	98	fbc->count += *pcount;
	99	*pcount = 0;
	100	spin_unlock(&fbc->lock);
	101	}
	102	mutex_unlock(&percpu_counters_lock);
	103	return NOTIFY_OK;
	104	}
	105
	106	static int __init percpu_counter_startup(void)
	107	{
	108	hotcpu_notifier(percpu_counter_hotcpu_callback, 0);
	109	return 0;
	110	}
	111	module_init(percpu_counter_startup);
	112	#endif