[net-next-2.6.git] / include / linux / slab_def.h

#ifndef _LINUX_SLAB_DEF_H
#define	_LINUX_SLAB_DEF_H

/*
 * Definitions unique to the original Linux SLAB allocator.
 *
 * What we provide here is a way to optimize the frequent kmalloc
 * calls in the kernel by selecting the appropriate general cache
 * if kmalloc was called with a size that can be established at
 * compile time.
 */

#include <linux/init.h>
#include <asm/page.h>		/* kmalloc_sizes.h needs PAGE_SIZE */
#include <asm/cache.h>		/* kmalloc_sizes.h needs L1_CACHE_BYTES */
#include <linux/compiler.h>
#include <linux/kmemtrace.h>

/*
 * struct kmem_cache
 *
 * manages a cache.
 */

struct kmem_cache {
/* 1) per-cpu data, touched during every alloc/free */
	struct array_cache *array[NR_CPUS];
/* 2) Cache tunables. Protected by cache_chain_mutex */
	unsigned int batchcount;
	unsigned int limit;
	unsigned int shared;

	unsigned int buffer_size;
	u32 reciprocal_buffer_size;
/* 3) touched by every alloc & free from the backend */

	unsigned int flags;		/* constant flags */
	unsigned int num;		/* # of objs per slab */

/* 4) cache_grow/shrink */
	/* order of pgs per slab (2^n) */
	unsigned int gfporder;

	/* force GFP flags, e.g. GFP_DMA */
	gfp_t gfpflags;

	size_t colour;			/* cache colouring range */
	unsigned int colour_off;	/* colour offset */
	struct kmem_cache *slabp_cache;
	unsigned int slab_size;
	unsigned int dflags;		/* dynamic flags */

	/* constructor func */
	void (*ctor)(void *obj);

/* 5) cache creation/removal */
	const char *name;
	struct list_head next;

/* 6) statistics */
#ifdef CONFIG_DEBUG_SLAB
	unsigned long num_active;
	unsigned long num_allocations;
	unsigned long high_mark;
	unsigned long grown;
	unsigned long reaped;
	unsigned long errors;
	unsigned long max_freeable;
	unsigned long node_allocs;
	unsigned long node_frees;
	unsigned long node_overflow;
	atomic_t allochit;
	atomic_t allocmiss;
	atomic_t freehit;
	atomic_t freemiss;

	/*
	 * If debugging is enabled, then the allocator can add additional
	 * fields and/or padding to every object. buffer_size contains the total
	 * object size including these internal fields, the following two
	 * variables contain the offset to the user object and its size.
	 */
	int obj_offset;
	int obj_size;
#endif /* CONFIG_DEBUG_SLAB */

	/*
	 * We put nodelists[] at the end of kmem_cache, because we want to size
	 * this array to nr_node_ids slots instead of MAX_NUMNODES
	 * (see kmem_cache_init())
	 * We still use [MAX_NUMNODES] and not [1] or [0] because cache_cache
	 * is statically defined, so we reserve the max number of nodes.
	 */
	struct kmem_list3 *nodelists[MAX_NUMNODES];
	/*
	 * Do not add fields after nodelists[]
	 */
};

/* Size description struct for general caches. */
struct cache_sizes {
	size_t		 	cs_size;
	struct kmem_cache	*cs_cachep;
#ifdef CONFIG_ZONE_DMA
	struct kmem_cache	*cs_dmacachep;
#endif
};
extern struct cache_sizes malloc_sizes[];

void *kmem_cache_alloc(struct kmem_cache *, gfp_t);
void *__kmalloc(size_t size, gfp_t flags);

#ifdef CONFIG_TRACING
extern void *kmem_cache_alloc_notrace(struct kmem_cache *cachep, gfp_t flags);
extern size_t slab_buffer_size(struct kmem_cache *cachep);
#else
static __always_inline void *
kmem_cache_alloc_notrace(struct kmem_cache *cachep, gfp_t flags)
{
	return kmem_cache_alloc(cachep, flags);
}
static inline size_t slab_buffer_size(struct kmem_cache *cachep)
{
	return 0;
}
#endif

static __always_inline void *kmalloc(size_t size, gfp_t flags)
{
	struct kmem_cache *cachep;
	void *ret;

	if (__builtin_constant_p(size)) {
		int i = 0;

		if (!size)
			return ZERO_SIZE_PTR;

#define CACHE(x) \
		if (size <= x) \
			goto found; \
		else \
			i++;
#include <linux/kmalloc_sizes.h>
#undef CACHE
		return NULL;
found:
#ifdef CONFIG_ZONE_DMA
		if (flags & GFP_DMA)
			cachep = malloc_sizes[i].cs_dmacachep;
		else
#endif
			cachep = malloc_sizes[i].cs_cachep;

		ret = kmem_cache_alloc_notrace(cachep, flags);

		trace_kmalloc(_THIS_IP_, ret,
			      size, slab_buffer_size(cachep), flags);

		return ret;
	}
	return __kmalloc(size, flags);
}

#ifdef CONFIG_NUMA
extern void *__kmalloc_node(size_t size, gfp_t flags, int node);
extern void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node);

#ifdef CONFIG_TRACING
extern void *kmem_cache_alloc_node_notrace(struct kmem_cache *cachep,
					   gfp_t flags,
					   int nodeid);
#else
static __always_inline void *
kmem_cache_alloc_node_notrace(struct kmem_cache *cachep,
			      gfp_t flags,
			      int nodeid)
{
	return kmem_cache_alloc_node(cachep, flags, nodeid);
}
#endif

static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node)
{
	struct kmem_cache *cachep;
	void *ret;

	if (__builtin_constant_p(size)) {
		int i = 0;

		if (!size)
			return ZERO_SIZE_PTR;

#define CACHE(x) \
		if (size <= x) \
			goto found; \
		else \
			i++;
#include <linux/kmalloc_sizes.h>
#undef CACHE
		return NULL;
found:
#ifdef CONFIG_ZONE_DMA
		if (flags & GFP_DMA)
			cachep = malloc_sizes[i].cs_dmacachep;
		else
#endif
			cachep = malloc_sizes[i].cs_cachep;

		ret = kmem_cache_alloc_node_notrace(cachep, flags, node);

		trace_kmalloc_node(_THIS_IP_, ret,
				   size, slab_buffer_size(cachep),
				   flags, node);

		return ret;
	}
	return __kmalloc_node(size, flags, node);
}

#endif	/* CONFIG_NUMA */

#endif	/* _LINUX_SLAB_DEF_H */
Commit	Line	Data
2e892f43 CL	1	#ifndef _LINUX_SLAB_DEF_H
	2	#define _LINUX_SLAB_DEF_H
	3
	4	/*
	5	* Definitions unique to the original Linux SLAB allocator.
	6	*
	7	* What we provide here is a way to optimize the frequent kmalloc
	8	* calls in the kernel by selecting the appropriate general cache
	9	* if kmalloc was called with a size that can be established at
	10	* compile time.
	11	*/
	12
	13	#include <linux/init.h>
	14	#include <asm/page.h> /* kmalloc_sizes.h needs PAGE_SIZE */
	15	#include <asm/cache.h> /* kmalloc_sizes.h needs L1_CACHE_BYTES */
	16	#include <linux/compiler.h>
02af61bb	17	#include <linux/kmemtrace.h>
2e892f43	18
8eae985f PE	19	/*
	20	* struct kmem_cache
	21	*
	22	* manages a cache.
	23	*/
	24
	25	struct kmem_cache {
	26	/* 1) per-cpu data, touched during every alloc/free */
	27	struct array_cache *array[NR_CPUS];
	28	/* 2) Cache tunables. Protected by cache_chain_mutex */
	29	unsigned int batchcount;
	30	unsigned int limit;
	31	unsigned int shared;
	32
	33	unsigned int buffer_size;
	34	u32 reciprocal_buffer_size;
	35	/* 3) touched by every alloc & free from the backend */
	36
	37	unsigned int flags; /* constant flags */
	38	unsigned int num; /* # of objs per slab */
	39
	40	/* 4) cache_grow/shrink */
	41	/* order of pgs per slab (2^n) */
	42	unsigned int gfporder;
	43
	44	/* force GFP flags, e.g. GFP_DMA */
	45	gfp_t gfpflags;
	46
	47	size_t colour; /* cache colouring range */
	48	unsigned int colour_off; /* colour offset */
	49	struct kmem_cache *slabp_cache;
	50	unsigned int slab_size;
	51	unsigned int dflags; /* dynamic flags */
	52
	53	/* constructor func */
	54	void (ctor)(void obj);
	55
	56	/* 5) cache creation/removal */
	57	const char *name;
	58	struct list_head next;
	59
	60	/* 6) statistics */
	61	#ifdef CONFIG_DEBUG_SLAB
	62	unsigned long num_active;
	63	unsigned long num_allocations;
	64	unsigned long high_mark;
	65	unsigned long grown;
	66	unsigned long reaped;
	67	unsigned long errors;
	68	unsigned long max_freeable;
	69	unsigned long node_allocs;
	70	unsigned long node_frees;
	71	unsigned long node_overflow;
	72	atomic_t allochit;
	73	atomic_t allocmiss;
	74	atomic_t freehit;
	75	atomic_t freemiss;
	76
	77	/*
	78	* If debugging is enabled, then the allocator can add additional
	79	* fields and/or padding to every object. buffer_size contains the total
	80	* object size including these internal fields, the following two
	81	* variables contain the offset to the user object and its size.
	82	*/
83	int obj_offset;
84	int obj_size;
85	#endif /* CONFIG_DEBUG_SLAB */
86
87	/*
88	* We put nodelists[] at the end of kmem_cache, because we want to size
89	* this array to nr_node_ids slots instead of MAX_NUMNODES
90	* (see kmem_cache_init())
91	* We still use [MAX_NUMNODES] and not [1] or [0] because cache_cache
92	* is statically defined, so we reserve the max number of nodes.
93	*/
94	struct kmem_list3 *nodelists[MAX_NUMNODES];
95	/*
96	* Do not add fields after nodelists[]
97	*/
98	};
99
2e892f43 CL	100	/* Size description struct for general caches. */
	101	struct cache_sizes {
	102	size_t cs_size;
	103	struct kmem_cache *cs_cachep;
4b51d669	104	#ifdef CONFIG_ZONE_DMA
2e892f43	105	struct kmem_cache *cs_dmacachep;
4b51d669	106	#endif
2e892f43 CL	107	};
	108	extern struct cache_sizes malloc_sizes[];
	109
6193a2ff PM	110	void kmem_cache_alloc(struct kmem_cache , gfp_t);
	111	void *__kmalloc(size_t size, gfp_t flags);
	112
0f24f128	113	#ifdef CONFIG_TRACING
36555751 EGM	114	extern void kmem_cache_alloc_notrace(struct kmem_cache cachep, gfp_t flags);
	115	extern size_t slab_buffer_size(struct kmem_cache *cachep);
	116	#else
	117	static __always_inline void *
	118	kmem_cache_alloc_notrace(struct kmem_cache *cachep, gfp_t flags)
2e892f43	119	{
36555751 EGM	120	return kmem_cache_alloc(cachep, flags);
	121	}
	122	static inline size_t slab_buffer_size(struct kmem_cache *cachep)
	123	{
	124	return 0;
	125	}
	126	#endif
	127
	128	static __always_inline void *kmalloc(size_t size, gfp_t flags)
	129	{
	130	struct kmem_cache *cachep;
	131	void *ret;
	132
2e892f43 CL	133	if (__builtin_constant_p(size)) {
2e892f43 CL	134	int i = 0;
6cb8f913 CL	135
	136	if (!size)
	137	return ZERO_SIZE_PTR;
	138
2e892f43 CL	139	#define CACHE(x) \
	140	if (size <= x) \
	141	goto found; \
	142	else \
	143	i++;
1c61fc40	144	#include <linux/kmalloc_sizes.h>
2e892f43	145	#undef CACHE
1cf3eb2f	146	return NULL;
2e892f43	147	found:
4b51d669 CL	148	#ifdef CONFIG_ZONE_DMA
4b51d669 CL	149	if (flags & GFP_DMA)
36555751 EGM	150	cachep = malloc_sizes[i].cs_dmacachep;
36555751 EGM	151	else
4b51d669	152	#endif
36555751 EGM	153	cachep = malloc_sizes[i].cs_cachep;
	154
	155	ret = kmem_cache_alloc_notrace(cachep, flags);
	156
ca2b84cb EGM	157	trace_kmalloc(_THIS_IP_, ret,
ca2b84cb EGM	158	size, slab_buffer_size(cachep), flags);
36555751 EGM	159
36555751 EGM	160	return ret;
2e892f43 CL	161	}
	162	return __kmalloc(size, flags);
	163	}
	164
2e892f43 CL	165	#ifdef CONFIG_NUMA
2e892f43 CL	166	extern void *__kmalloc_node(size_t size, gfp_t flags, int node);
6193a2ff	167	extern void kmem_cache_alloc_node(struct kmem_cache , gfp_t flags, int node);
2e892f43	168
0f24f128	169	#ifdef CONFIG_TRACING
36555751 EGM	170	extern void kmem_cache_alloc_node_notrace(struct kmem_cache cachep,
	171	gfp_t flags,
	172	int nodeid);
	173	#else
	174	static __always_inline void *
	175	kmem_cache_alloc_node_notrace(struct kmem_cache *cachep,
	176	gfp_t flags,
	177	int nodeid)
	178	{
	179	return kmem_cache_alloc_node(cachep, flags, nodeid);
	180	}
	181	#endif
	182
	183	static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node)
2e892f43	184	{
36555751 EGM	185	struct kmem_cache *cachep;
	186	void *ret;
	187
2e892f43 CL	188	if (__builtin_constant_p(size)) {
2e892f43 CL	189	int i = 0;
6cb8f913 CL	190
	191	if (!size)
	192	return ZERO_SIZE_PTR;
	193
2e892f43 CL	194	#define CACHE(x) \
	195	if (size <= x) \
	196	goto found; \
	197	else \
	198	i++;
1c61fc40	199	#include <linux/kmalloc_sizes.h>
2e892f43	200	#undef CACHE
1cf3eb2f	201	return NULL;
2e892f43	202	found:
4b51d669 CL	203	#ifdef CONFIG_ZONE_DMA
4b51d669 CL	204	if (flags & GFP_DMA)
36555751 EGM	205	cachep = malloc_sizes[i].cs_dmacachep;
36555751 EGM	206	else
4b51d669	207	#endif
36555751 EGM	208	cachep = malloc_sizes[i].cs_cachep;
	209
	210	ret = kmem_cache_alloc_node_notrace(cachep, flags, node);
	211
ca2b84cb EGM	212	trace_kmalloc_node(_THIS_IP_, ret,
	213	size, slab_buffer_size(cachep),
	214	flags, node);
36555751 EGM	215
36555751 EGM	216	return ret;
2e892f43 CL	217	}
	218	return __kmalloc_node(size, flags, node);
	219	}
	220
	221	#endif /* CONFIG_NUMA */
	222
	223	#endif /* _LINUX_SLAB_DEF_H */