[net-next-2.6.git] / include / linux / slab_def.h

#ifndef _LINUX_SLAB_DEF_H
#define	_LINUX_SLAB_DEF_H

/*
 * Definitions unique to the original Linux SLAB allocator.
 *
 * What we provide here is a way to optimize the frequent kmalloc
 * calls in the kernel by selecting the appropriate general cache
 * if kmalloc was called with a size that can be established at
 * compile time.
 */

#include <linux/init.h>
#include <asm/page.h>		/* kmalloc_sizes.h needs PAGE_SIZE */
#include <asm/cache.h>		/* kmalloc_sizes.h needs L1_CACHE_BYTES */
#include <linux/compiler.h>

#include <trace/events/kmem.h>

/*
 * Enforce a minimum alignment for the kmalloc caches.
 * Usually, the kmalloc caches are cache_line_size() aligned, except when
 * DEBUG and FORCED_DEBUG are enabled, then they are BYTES_PER_WORD aligned.
 * Some archs want to perform DMA into kmalloc caches and need a guaranteed
 * alignment larger than the alignment of a 64-bit integer.
 * ARCH_KMALLOC_MINALIGN allows that.
 * Note that increasing this value may disable some debug features.
 */
#ifdef ARCH_DMA_MINALIGN
#define ARCH_KMALLOC_MINALIGN ARCH_DMA_MINALIGN
#else
#define ARCH_KMALLOC_MINALIGN __alignof__(unsigned long long)
#endif

#ifndef ARCH_SLAB_MINALIGN
/*
 * Enforce a minimum alignment for all caches.
 * Intended for archs that get misalignment faults even for BYTES_PER_WORD
 * aligned buffers. Includes ARCH_KMALLOC_MINALIGN.
 * If possible: Do not enable this flag for CONFIG_DEBUG_SLAB, it disables
 * some debug features.
 */
#define ARCH_SLAB_MINALIGN 0
#endif

/*
 * struct kmem_cache
 *
 * manages a cache.
 */

struct kmem_cache {
/* 1) per-cpu data, touched during every alloc/free */
	struct array_cache *array[NR_CPUS];
/* 2) Cache tunables. Protected by cache_chain_mutex */
	unsigned int batchcount;
	unsigned int limit;
	unsigned int shared;

	unsigned int buffer_size;
	u32 reciprocal_buffer_size;
/* 3) touched by every alloc & free from the backend */

	unsigned int flags;		/* constant flags */
	unsigned int num;		/* # of objs per slab */

/* 4) cache_grow/shrink */
	/* order of pgs per slab (2^n) */
	unsigned int gfporder;

	/* force GFP flags, e.g. GFP_DMA */
	gfp_t gfpflags;

	size_t colour;			/* cache colouring range */
	unsigned int colour_off;	/* colour offset */
	struct kmem_cache *slabp_cache;
	unsigned int slab_size;
	unsigned int dflags;		/* dynamic flags */

	/* constructor func */
	void (*ctor)(void *obj);

/* 5) cache creation/removal */
	const char *name;
	struct list_head next;

/* 6) statistics */
#ifdef CONFIG_DEBUG_SLAB
	unsigned long num_active;
	unsigned long num_allocations;
	unsigned long high_mark;
	unsigned long grown;
	unsigned long reaped;
	unsigned long errors;
	unsigned long max_freeable;
	unsigned long node_allocs;
	unsigned long node_frees;
	unsigned long node_overflow;
	atomic_t allochit;
	atomic_t allocmiss;
	atomic_t freehit;
	atomic_t freemiss;

	/*
	 * If debugging is enabled, then the allocator can add additional
	 * fields and/or padding to every object. buffer_size contains the total
	 * object size including these internal fields, the following two
	 * variables contain the offset to the user object and its size.
	 */
	int obj_offset;
	int obj_size;
#endif /* CONFIG_DEBUG_SLAB */

	/*
	 * We put nodelists[] at the end of kmem_cache, because we want to size
	 * this array to nr_node_ids slots instead of MAX_NUMNODES
	 * (see kmem_cache_init())
	 * We still use [MAX_NUMNODES] and not [1] or [0] because cache_cache
	 * is statically defined, so we reserve the max number of nodes.
	 */
	struct kmem_list3 *nodelists[MAX_NUMNODES];
	/*
	 * Do not add fields after nodelists[]
	 */
};

/* Size description struct for general caches. */
struct cache_sizes {
	size_t		 	cs_size;
	struct kmem_cache	*cs_cachep;
#ifdef CONFIG_ZONE_DMA
	struct kmem_cache	*cs_dmacachep;
#endif
};
extern struct cache_sizes malloc_sizes[];

void *kmem_cache_alloc(struct kmem_cache *, gfp_t);
void *__kmalloc(size_t size, gfp_t flags);

#ifdef CONFIG_TRACING
extern void *kmem_cache_alloc_notrace(struct kmem_cache *cachep, gfp_t flags);
extern size_t slab_buffer_size(struct kmem_cache *cachep);
#else
static __always_inline void *
kmem_cache_alloc_notrace(struct kmem_cache *cachep, gfp_t flags)
{
	return kmem_cache_alloc(cachep, flags);
}
static inline size_t slab_buffer_size(struct kmem_cache *cachep)
{
	return 0;
}
#endif

static __always_inline void *kmalloc(size_t size, gfp_t flags)
{
	struct kmem_cache *cachep;
	void *ret;

	if (__builtin_constant_p(size)) {
		int i = 0;

		if (!size)
			return ZERO_SIZE_PTR;

#define CACHE(x) \
		if (size <= x) \
			goto found; \
		else \
			i++;
#include <linux/kmalloc_sizes.h>
#undef CACHE
		return NULL;
found:
#ifdef CONFIG_ZONE_DMA
		if (flags & GFP_DMA)
			cachep = malloc_sizes[i].cs_dmacachep;
		else
#endif
			cachep = malloc_sizes[i].cs_cachep;

		ret = kmem_cache_alloc_notrace(cachep, flags);

		trace_kmalloc(_THIS_IP_, ret,
			      size, slab_buffer_size(cachep), flags);

		return ret;
	}
	return __kmalloc(size, flags);
}

#ifdef CONFIG_NUMA
extern void *__kmalloc_node(size_t size, gfp_t flags, int node);
extern void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node);

#ifdef CONFIG_TRACING
extern void *kmem_cache_alloc_node_notrace(struct kmem_cache *cachep,
					   gfp_t flags,
					   int nodeid);
#else
static __always_inline void *
kmem_cache_alloc_node_notrace(struct kmem_cache *cachep,
			      gfp_t flags,
			      int nodeid)
{
	return kmem_cache_alloc_node(cachep, flags, nodeid);
}
#endif

static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node)
{
	struct kmem_cache *cachep;
	void *ret;

	if (__builtin_constant_p(size)) {
		int i = 0;

		if (!size)
			return ZERO_SIZE_PTR;

#define CACHE(x) \
		if (size <= x) \
			goto found; \
		else \
			i++;
#include <linux/kmalloc_sizes.h>
#undef CACHE
		return NULL;
found:
#ifdef CONFIG_ZONE_DMA
		if (flags & GFP_DMA)
			cachep = malloc_sizes[i].cs_dmacachep;
		else
#endif
			cachep = malloc_sizes[i].cs_cachep;

		ret = kmem_cache_alloc_node_notrace(cachep, flags, node);

		trace_kmalloc_node(_THIS_IP_, ret,
				   size, slab_buffer_size(cachep),
				   flags, node);

		return ret;
	}
	return __kmalloc_node(size, flags, node);
}

#endif	/* CONFIG_NUMA */

#endif	/* _LINUX_SLAB_DEF_H */
Commit	Line	Data
2e892f43 CL	1	#ifndef _LINUX_SLAB_DEF_H
	2	#define _LINUX_SLAB_DEF_H
	3
	4	/*
	5	* Definitions unique to the original Linux SLAB allocator.
	6	*
	7	* What we provide here is a way to optimize the frequent kmalloc
	8	* calls in the kernel by selecting the appropriate general cache
	9	* if kmalloc was called with a size that can be established at
	10	* compile time.
	11	*/
	12
	13	#include <linux/init.h>
	14	#include <asm/page.h> /* kmalloc_sizes.h needs PAGE_SIZE */
	15	#include <asm/cache.h> /* kmalloc_sizes.h needs L1_CACHE_BYTES */
	16	#include <linux/compiler.h>
039ca4e7 LZ	17
039ca4e7 LZ	18	#include <trace/events/kmem.h>
2e892f43	19
1f0ce8b3 DW	20	/*
	21	* Enforce a minimum alignment for the kmalloc caches.
	22	* Usually, the kmalloc caches are cache_line_size() aligned, except when
	23	* DEBUG and FORCED_DEBUG are enabled, then they are BYTES_PER_WORD aligned.
	24	* Some archs want to perform DMA into kmalloc caches and need a guaranteed
	25	* alignment larger than the alignment of a 64-bit integer.
	26	* ARCH_KMALLOC_MINALIGN allows that.
	27	* Note that increasing this value may disable some debug features.
	28	*/
a6eb9fe1 FT	29	#ifdef ARCH_DMA_MINALIGN
	30	#define ARCH_KMALLOC_MINALIGN ARCH_DMA_MINALIGN
	31	#else
1f0ce8b3 DW	32	#define ARCH_KMALLOC_MINALIGN __alignof__(unsigned long long)
	33	#endif
	34
	35	#ifndef ARCH_SLAB_MINALIGN
	36	/*
	37	* Enforce a minimum alignment for all caches.
	38	* Intended for archs that get misalignment faults even for BYTES_PER_WORD
	39	* aligned buffers. Includes ARCH_KMALLOC_MINALIGN.
	40	* If possible: Do not enable this flag for CONFIG_DEBUG_SLAB, it disables
	41	* some debug features.
	42	*/
	43	#define ARCH_SLAB_MINALIGN 0
	44	#endif
	45
8eae985f PE	46	/*
	47	* struct kmem_cache
	48	*
	49	* manages a cache.
	50	*/
	51
	52	struct kmem_cache {
	53	/* 1) per-cpu data, touched during every alloc/free */
	54	struct array_cache *array[NR_CPUS];
	55	/* 2) Cache tunables. Protected by cache_chain_mutex */
	56	unsigned int batchcount;
	57	unsigned int limit;
	58	unsigned int shared;
	59
	60	unsigned int buffer_size;
	61	u32 reciprocal_buffer_size;
	62	/* 3) touched by every alloc & free from the backend */
	63
	64	unsigned int flags; /* constant flags */
	65	unsigned int num; /* # of objs per slab */
	66
	67	/* 4) cache_grow/shrink */
	68	/* order of pgs per slab (2^n) */
	69	unsigned int gfporder;
	70
	71	/* force GFP flags, e.g. GFP_DMA */
	72	gfp_t gfpflags;
	73
	74	size_t colour; /* cache colouring range */
	75	unsigned int colour_off; /* colour offset */
	76	struct kmem_cache *slabp_cache;
	77	unsigned int slab_size;
	78	unsigned int dflags; /* dynamic flags */
	79
	80	/* constructor func */
	81	void (ctor)(void obj);
	82
	83	/* 5) cache creation/removal */
	84	const char *name;
	85	struct list_head next;
	86
	87	/* 6) statistics */
	88	#ifdef CONFIG_DEBUG_SLAB
	89	unsigned long num_active;
	90	unsigned long num_allocations;
	91	unsigned long high_mark;
	92	unsigned long grown;
	93	unsigned long reaped;
	94	unsigned long errors;
	95	unsigned long max_freeable;
	96	unsigned long node_allocs;
	97	unsigned long node_frees;
	98	unsigned long node_overflow;
	99	atomic_t allochit;
	100	atomic_t allocmiss;
	101	atomic_t freehit;
	102	atomic_t freemiss;
	103
	104	/*
	105	* If debugging is enabled, then the allocator can add additional
	106	* fields and/or padding to every object. buffer_size contains the total
	107	* object size including these internal fields, the following two
	108	* variables contain the offset to the user object and its size.
	109	*/
110	int obj_offset;
111	int obj_size;
112	#endif /* CONFIG_DEBUG_SLAB */
113
114	/*
115	* We put nodelists[] at the end of kmem_cache, because we want to size
116	* this array to nr_node_ids slots instead of MAX_NUMNODES
117	* (see kmem_cache_init())
118	* We still use [MAX_NUMNODES] and not [1] or [0] because cache_cache
119	* is statically defined, so we reserve the max number of nodes.
120	*/
121	struct kmem_list3 *nodelists[MAX_NUMNODES];
122	/*
123	* Do not add fields after nodelists[]
124	*/
125	};
126
2e892f43 CL	127	/* Size description struct for general caches. */
	128	struct cache_sizes {
	129	size_t cs_size;
	130	struct kmem_cache *cs_cachep;
4b51d669	131	#ifdef CONFIG_ZONE_DMA
2e892f43	132	struct kmem_cache *cs_dmacachep;
4b51d669	133	#endif
2e892f43 CL	134	};
	135	extern struct cache_sizes malloc_sizes[];
	136
6193a2ff PM	137	void kmem_cache_alloc(struct kmem_cache , gfp_t);
	138	void *__kmalloc(size_t size, gfp_t flags);
	139
0f24f128	140	#ifdef CONFIG_TRACING
36555751 EGM	141	extern void kmem_cache_alloc_notrace(struct kmem_cache cachep, gfp_t flags);
	142	extern size_t slab_buffer_size(struct kmem_cache *cachep);
	143	#else
	144	static __always_inline void *
	145	kmem_cache_alloc_notrace(struct kmem_cache *cachep, gfp_t flags)
2e892f43	146	{
36555751 EGM	147	return kmem_cache_alloc(cachep, flags);
	148	}
	149	static inline size_t slab_buffer_size(struct kmem_cache *cachep)
	150	{
	151	return 0;
	152	}
	153	#endif
	154
	155	static __always_inline void *kmalloc(size_t size, gfp_t flags)
	156	{
	157	struct kmem_cache *cachep;
	158	void *ret;
	159
2e892f43 CL	160	if (__builtin_constant_p(size)) {
2e892f43 CL	161	int i = 0;
6cb8f913 CL	162
	163	if (!size)
	164	return ZERO_SIZE_PTR;
	165
2e892f43 CL	166	#define CACHE(x) \
	167	if (size <= x) \
	168	goto found; \
	169	else \
	170	i++;
1c61fc40	171	#include <linux/kmalloc_sizes.h>
2e892f43	172	#undef CACHE
1cf3eb2f	173	return NULL;
2e892f43	174	found:
4b51d669 CL	175	#ifdef CONFIG_ZONE_DMA
4b51d669 CL	176	if (flags & GFP_DMA)
36555751 EGM	177	cachep = malloc_sizes[i].cs_dmacachep;
36555751 EGM	178	else
4b51d669	179	#endif
36555751 EGM	180	cachep = malloc_sizes[i].cs_cachep;
	181
	182	ret = kmem_cache_alloc_notrace(cachep, flags);
	183
ca2b84cb EGM	184	trace_kmalloc(_THIS_IP_, ret,
ca2b84cb EGM	185	size, slab_buffer_size(cachep), flags);
36555751 EGM	186
36555751 EGM	187	return ret;
2e892f43 CL	188	}
	189	return __kmalloc(size, flags);
	190	}
	191
2e892f43 CL	192	#ifdef CONFIG_NUMA
2e892f43 CL	193	extern void *__kmalloc_node(size_t size, gfp_t flags, int node);
6193a2ff	194	extern void kmem_cache_alloc_node(struct kmem_cache , gfp_t flags, int node);
2e892f43	195
0f24f128	196	#ifdef CONFIG_TRACING
36555751 EGM	197	extern void kmem_cache_alloc_node_notrace(struct kmem_cache cachep,
	198	gfp_t flags,
	199	int nodeid);
	200	#else
	201	static __always_inline void *
	202	kmem_cache_alloc_node_notrace(struct kmem_cache *cachep,
	203	gfp_t flags,
	204	int nodeid)
	205	{
	206	return kmem_cache_alloc_node(cachep, flags, nodeid);
	207	}
	208	#endif
	209
	210	static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node)
2e892f43	211	{
36555751 EGM	212	struct kmem_cache *cachep;
	213	void *ret;
	214
2e892f43 CL	215	if (__builtin_constant_p(size)) {
2e892f43 CL	216	int i = 0;
6cb8f913 CL	217
	218	if (!size)
	219	return ZERO_SIZE_PTR;
	220
2e892f43 CL	221	#define CACHE(x) \
	222	if (size <= x) \
	223	goto found; \
	224	else \
	225	i++;
1c61fc40	226	#include <linux/kmalloc_sizes.h>
2e892f43	227	#undef CACHE
1cf3eb2f	228	return NULL;
2e892f43	229	found:
4b51d669 CL	230	#ifdef CONFIG_ZONE_DMA
4b51d669 CL	231	if (flags & GFP_DMA)
36555751 EGM	232	cachep = malloc_sizes[i].cs_dmacachep;
36555751 EGM	233	else
4b51d669	234	#endif
36555751 EGM	235	cachep = malloc_sizes[i].cs_cachep;
	236
	237	ret = kmem_cache_alloc_node_notrace(cachep, flags, node);
	238
ca2b84cb EGM	239	trace_kmalloc_node(_THIS_IP_, ret,
	240	size, slab_buffer_size(cachep),
	241	flags, node);
36555751 EGM	242
36555751 EGM	243	return ret;
2e892f43 CL	244	}
	245	return __kmalloc_node(size, flags, node);
	246	}
	247
	248	#endif /* CONFIG_NUMA */
	249
	250	#endif /* _LINUX_SLAB_DEF_H */