[net-next-2.6.git] / lib / percpu_counter.c

/*
 * Fast batching percpu counters.
 */

#include <linux/percpu_counter.h>
#include <linux/notifier.h>
#include <linux/mutex.h>
#include <linux/init.h>
#include <linux/cpu.h>
#include <linux/module.h>

static LIST_HEAD(percpu_counters);
static DEFINE_MUTEX(percpu_counters_lock);

void percpu_counter_set(struct percpu_counter *fbc, s64 amount)
{
	int cpu;

	spin_lock(&fbc->lock);
	for_each_possible_cpu(cpu) {
		s32 *pcount = per_cpu_ptr(fbc->counters, cpu);
		*pcount = 0;
	}
	fbc->count = amount;
	spin_unlock(&fbc->lock);
}
EXPORT_SYMBOL(percpu_counter_set);

void __percpu_counter_add(struct percpu_counter *fbc, s64 amount, s32 batch)
{
	s64 count;
	s32 *pcount;
	int cpu = get_cpu();

	pcount = per_cpu_ptr(fbc->counters, cpu);
	count = *pcount + amount;
	if (count >= batch || count <= -batch) {
		spin_lock(&fbc->lock);
		fbc->count += count;
		*pcount = 0;
		spin_unlock(&fbc->lock);
	} else {
		*pcount = count;
	}
	put_cpu();
}
EXPORT_SYMBOL(__percpu_counter_add);

/*
 * Add up all the per-cpu counts, return the result.  This is a more accurate
 * but much slower version of percpu_counter_read_positive()
 */
s64 __percpu_counter_sum(struct percpu_counter *fbc)
{
	s64 ret;
	int cpu;

	spin_lock(&fbc->lock);
	ret = fbc->count;
	for_each_online_cpu(cpu) {
		s32 *pcount = per_cpu_ptr(fbc->counters, cpu);
		ret += *pcount;
	}
	spin_unlock(&fbc->lock);
	return ret;
}
EXPORT_SYMBOL(__percpu_counter_sum);

int __percpu_counter_init(struct percpu_counter *fbc, s64 amount,
			  struct lock_class_key *key)
{
	spin_lock_init(&fbc->lock);
	lockdep_set_class(&fbc->lock, key);
	fbc->count = amount;
	fbc->counters = alloc_percpu(s32);
	if (!fbc->counters)
		return -ENOMEM;
#ifdef CONFIG_HOTPLUG_CPU
	INIT_LIST_HEAD(&fbc->list);
	mutex_lock(&percpu_counters_lock);
	list_add(&fbc->list, &percpu_counters);
	mutex_unlock(&percpu_counters_lock);
#endif
	return 0;
}
EXPORT_SYMBOL(__percpu_counter_init);

void percpu_counter_destroy(struct percpu_counter *fbc)
{
	if (!fbc->counters)
		return;

#ifdef CONFIG_HOTPLUG_CPU
	mutex_lock(&percpu_counters_lock);
	list_del(&fbc->list);
	mutex_unlock(&percpu_counters_lock);
#endif
	free_percpu(fbc->counters);
	fbc->counters = NULL;
}
EXPORT_SYMBOL(percpu_counter_destroy);

int percpu_counter_batch __read_mostly = 32;
EXPORT_SYMBOL(percpu_counter_batch);

static void compute_batch_value(void)
{
	int nr = num_online_cpus();

	percpu_counter_batch = max(32, nr*2);
}

static int __cpuinit percpu_counter_hotcpu_callback(struct notifier_block *nb,
					unsigned long action, void *hcpu)
{
#ifdef CONFIG_HOTPLUG_CPU
	unsigned int cpu;
	struct percpu_counter *fbc;

	compute_batch_value();
	if (action != CPU_DEAD)
		return NOTIFY_OK;

	cpu = (unsigned long)hcpu;
	mutex_lock(&percpu_counters_lock);
	list_for_each_entry(fbc, &percpu_counters, list) {
		s32 *pcount;
		unsigned long flags;

		spin_lock_irqsave(&fbc->lock, flags);
		pcount = per_cpu_ptr(fbc->counters, cpu);
		fbc->count += *pcount;
		*pcount = 0;
		spin_unlock_irqrestore(&fbc->lock, flags);
	}
	mutex_unlock(&percpu_counters_lock);
#endif
	return NOTIFY_OK;
}

/*
 * Compare counter against given value.
 * Return 1 if greater, 0 if equal and -1 if less
 */
int percpu_counter_compare(struct percpu_counter *fbc, s64 rhs)
{
	s64	count;

	count = percpu_counter_read(fbc);
	/* Check to see if rough count will be sufficient for comparison */
	if (abs(count - rhs) > (percpu_counter_batch*num_online_cpus())) {
		if (count > rhs)
			return 1;
		else
			return -1;
	}
	/* Need to use precise count */
	count = percpu_counter_sum(fbc);
	if (count > rhs)
		return 1;
	else if (count < rhs)
		return -1;
	else
		return 0;
}
EXPORT_SYMBOL(percpu_counter_compare);

static int __init percpu_counter_startup(void)
{
	compute_batch_value();
	hotcpu_notifier(percpu_counter_hotcpu_callback, 0);
	return 0;
}
module_init(percpu_counter_startup);
Commit	Line	Data
3cbc5640 RT	1	/*
	2	* Fast batching percpu counters.
	3	*/
	4
	5	#include <linux/percpu_counter.h>
c67ad917 AM	6	#include <linux/notifier.h>
	7	#include <linux/mutex.h>
	8	#include <linux/init.h>
	9	#include <linux/cpu.h>
3cbc5640 RT	10	#include <linux/module.h>
3cbc5640 RT	11
c67ad917 AM	12	static LIST_HEAD(percpu_counters);
c67ad917 AM	13	static DEFINE_MUTEX(percpu_counters_lock);
c67ad917	14
3a587f47 PZ	15	void percpu_counter_set(struct percpu_counter *fbc, s64 amount)
	16	{
	17	int cpu;
	18
	19	spin_lock(&fbc->lock);
	20	for_each_possible_cpu(cpu) {
	21	s32 *pcount = per_cpu_ptr(fbc->counters, cpu);
	22	*pcount = 0;
	23	}
	24	fbc->count = amount;
	25	spin_unlock(&fbc->lock);
	26	}
	27	EXPORT_SYMBOL(percpu_counter_set);
	28
20e89767	29	void __percpu_counter_add(struct percpu_counter *fbc, s64 amount, s32 batch)
3cbc5640	30	{
20e89767	31	s64 count;
0216bfcf	32	s32 *pcount;
3cbc5640 RT	33	int cpu = get_cpu();
	34
	35	pcount = per_cpu_ptr(fbc->counters, cpu);
	36	count = *pcount + amount;
252e0ba6	37	if (count >= batch \|\| count <= -batch) {
3cbc5640 RT	38	spin_lock(&fbc->lock);
	39	fbc->count += count;
	40	*pcount = 0;
	41	spin_unlock(&fbc->lock);
	42	} else {
	43	*pcount = count;
	44	}
	45	put_cpu();
	46	}
252e0ba6	47	EXPORT_SYMBOL(__percpu_counter_add);
3cbc5640 RT	48
	49	/*
	50	* Add up all the per-cpu counts, return the result. This is a more accurate
	51	* but much slower version of percpu_counter_read_positive()
	52	*/
02d21168	53	s64 __percpu_counter_sum(struct percpu_counter *fbc)
3cbc5640	54	{
0216bfcf	55	s64 ret;
3cbc5640 RT	56	int cpu;
	57
	58	spin_lock(&fbc->lock);
	59	ret = fbc->count;
b4ef0296	60	for_each_online_cpu(cpu) {
0216bfcf	61	s32 *pcount = per_cpu_ptr(fbc->counters, cpu);
3cbc5640 RT	62	ret += *pcount;
	63	}
	64	spin_unlock(&fbc->lock);
bf1d89c8	65	return ret;
3cbc5640	66	}
bf1d89c8	67	EXPORT_SYMBOL(__percpu_counter_sum);
c67ad917	68
ea319518 PZ	69	int __percpu_counter_init(struct percpu_counter *fbc, s64 amount,
ea319518 PZ	70	struct lock_class_key *key)
c67ad917 AM	71	{
c67ad917 AM	72	spin_lock_init(&fbc->lock);
ea319518	73	lockdep_set_class(&fbc->lock, key);
c67ad917 AM	74	fbc->count = amount;
c67ad917 AM	75	fbc->counters = alloc_percpu(s32);
833f4077 PZ	76	if (!fbc->counters)
833f4077 PZ	77	return -ENOMEM;
c67ad917	78	#ifdef CONFIG_HOTPLUG_CPU
8474b591	79	INIT_LIST_HEAD(&fbc->list);
c67ad917 AM	80	mutex_lock(&percpu_counters_lock);
	81	list_add(&fbc->list, &percpu_counters);
	82	mutex_unlock(&percpu_counters_lock);
	83	#endif
833f4077	84	return 0;
c67ad917	85	}
ea319518	86	EXPORT_SYMBOL(__percpu_counter_init);
c67ad917 AM	87
	88	void percpu_counter_destroy(struct percpu_counter *fbc)
	89	{
833f4077 PZ	90	if (!fbc->counters)
	91	return;
	92
c67ad917 AM	93	#ifdef CONFIG_HOTPLUG_CPU
	94	mutex_lock(&percpu_counters_lock);
	95	list_del(&fbc->list);
	96	mutex_unlock(&percpu_counters_lock);
	97	#endif
fd3d664f ED	98	free_percpu(fbc->counters);
fd3d664f ED	99	fbc->counters = NULL;
c67ad917 AM	100	}
	101	EXPORT_SYMBOL(percpu_counter_destroy);
	102
179f7ebf ED	103	int percpu_counter_batch __read_mostly = 32;
	104	EXPORT_SYMBOL(percpu_counter_batch);
	105
	106	static void compute_batch_value(void)
	107	{
	108	int nr = num_online_cpus();
	109
	110	percpu_counter_batch = max(32, nr*2);
	111	}
	112
c67ad917 AM	113	static int __cpuinit percpu_counter_hotcpu_callback(struct notifier_block *nb,
	114	unsigned long action, void *hcpu)
	115	{
179f7ebf	116	#ifdef CONFIG_HOTPLUG_CPU
c67ad917 AM	117	unsigned int cpu;
	118	struct percpu_counter *fbc;
	119
179f7ebf	120	compute_batch_value();
c67ad917 AM	121	if (action != CPU_DEAD)
	122	return NOTIFY_OK;
	123
	124	cpu = (unsigned long)hcpu;
	125	mutex_lock(&percpu_counters_lock);
	126	list_for_each_entry(fbc, &percpu_counters, list) {
	127	s32 *pcount;
d2b20b11	128	unsigned long flags;
c67ad917	129
d2b20b11	130	spin_lock_irqsave(&fbc->lock, flags);
c67ad917 AM	131	pcount = per_cpu_ptr(fbc->counters, cpu);
	132	fbc->count += *pcount;
	133	*pcount = 0;
d2b20b11	134	spin_unlock_irqrestore(&fbc->lock, flags);
c67ad917 AM	135	}
c67ad917 AM	136	mutex_unlock(&percpu_counters_lock);
179f7ebf	137	#endif
c67ad917 AM	138	return NOTIFY_OK;
	139	}
	140
27f5e0f6 TC	141	/*
	142	* Compare counter against given value.
	143	* Return 1 if greater, 0 if equal and -1 if less
	144	*/
	145	int percpu_counter_compare(struct percpu_counter *fbc, s64 rhs)
	146	{
	147	s64 count;
	148
	149	count = percpu_counter_read(fbc);
	150	/* Check to see if rough count will be sufficient for comparison */
	151	if (abs(count - rhs) > (percpu_counter_batch*num_online_cpus())) {
	152	if (count > rhs)
	153	return 1;
	154	else
	155	return -1;
	156	}
	157	/* Need to use precise count */
	158	count = percpu_counter_sum(fbc);
	159	if (count > rhs)
	160	return 1;
	161	else if (count < rhs)
	162	return -1;
	163	else
	164	return 0;
	165	}
	166	EXPORT_SYMBOL(percpu_counter_compare);
	167
c67ad917 AM	168	static int __init percpu_counter_startup(void)
c67ad917 AM	169	{
179f7ebf	170	compute_batch_value();
c67ad917 AM	171	hotcpu_notifier(percpu_counter_hotcpu_callback, 0);
	172	return 0;
	173	}
	174	module_init(percpu_counter_startup);