[net-next-2.6.git] / net / core / skbuff.c

/*
 *	Routines having to do with the 'struct sk_buff' memory handlers.
 *
 *	Authors:	Alan Cox <alan@lxorguk.ukuu.org.uk>
 *			Florian La Roche <rzsfl@rz.uni-sb.de>
 *
 *	Fixes:
 *		Alan Cox	:	Fixed the worst of the load
 *					balancer bugs.
 *		Dave Platt	:	Interrupt stacking fix.
 *	Richard Kooijman	:	Timestamp fixes.
 *		Alan Cox	:	Changed buffer format.
 *		Alan Cox	:	destructor hook for AF_UNIX etc.
 *		Linus Torvalds	:	Better skb_clone.
 *		Alan Cox	:	Added skb_copy.
 *		Alan Cox	:	Added all the changed routines Linus
 *					only put in the headers
 *		Ray VanTassle	:	Fixed --skb->lock in free
 *		Alan Cox	:	skb_copy copy arp field
 *		Andi Kleen	:	slabified it.
 *		Robert Olsson	:	Removed skb_head_pool
 *
 *	NOTE:
 *		The __skb_ routines should be called with interrupts
 *	disabled, or you better be *real* sure that the operation is atomic
 *	with respect to whatever list is being frobbed (e.g. via lock_sock()
 *	or via disabling bottom half handlers, etc).
 *
 *	This program is free software; you can redistribute it and/or
 *	modify it under the terms of the GNU General Public License
 *	as published by the Free Software Foundation; either version
 *	2 of the License, or (at your option) any later version.
 */

/*
 *	The functions in this file will not compile correctly with gcc 2.4.x
 */

#include <linux/module.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/interrupt.h>
#include <linux/in.h>
#include <linux/inet.h>
#include <linux/slab.h>
#include <linux/netdevice.h>
#ifdef CONFIG_NET_CLS_ACT
#include <net/pkt_sched.h>
#endif
#include <linux/string.h>
#include <linux/skbuff.h>
#include <linux/splice.h>
#include <linux/cache.h>
#include <linux/rtnetlink.h>
#include <linux/init.h>
#include <linux/scatterlist.h>

#include <net/protocol.h>
#include <net/dst.h>
#include <net/sock.h>
#include <net/checksum.h>
#include <net/xfrm.h>

#include <asm/uaccess.h>
#include <asm/system.h>

#include "kmap_skb.h"

static struct kmem_cache *skbuff_head_cache __read_mostly;
static struct kmem_cache *skbuff_fclone_cache __read_mostly;

static void sock_pipe_buf_release(struct pipe_inode_info *pipe,
				  struct pipe_buffer *buf)
{
	struct sk_buff *skb = (struct sk_buff *) buf->private;

	kfree_skb(skb);
}

static void sock_pipe_buf_get(struct pipe_inode_info *pipe,
				struct pipe_buffer *buf)
{
	struct sk_buff *skb = (struct sk_buff *) buf->private;

	skb_get(skb);
}

static int sock_pipe_buf_steal(struct pipe_inode_info *pipe,
			       struct pipe_buffer *buf)
{
	return 1;
}


/* Pipe buffer operations for a socket. */
static struct pipe_buf_operations sock_pipe_buf_ops = {
	.can_merge = 0,
	.map = generic_pipe_buf_map,
	.unmap = generic_pipe_buf_unmap,
	.confirm = generic_pipe_buf_confirm,
	.release = sock_pipe_buf_release,
	.steal = sock_pipe_buf_steal,
	.get = sock_pipe_buf_get,
};

/*
 *	Keep out-of-line to prevent kernel bloat.
 *	__builtin_return_address is not used because it is not always
 *	reliable.
 */

/**
 *	skb_over_panic	- 	private function
 *	@skb: buffer
 *	@sz: size
 *	@here: address
 *
 *	Out of line support code for skb_put(). Not user callable.
 */
void skb_over_panic(struct sk_buff *skb, int sz, void *here)
{
	printk(KERN_EMERG "skb_over_panic: text:%p len:%d put:%d head:%p "
			  "data:%p tail:%#lx end:%#lx dev:%s\n",
	       here, skb->len, sz, skb->head, skb->data,
	       (unsigned long)skb->tail, (unsigned long)skb->end,
	       skb->dev ? skb->dev->name : "<NULL>");
	BUG();
}

/**
 *	skb_under_panic	- 	private function
 *	@skb: buffer
 *	@sz: size
 *	@here: address
 *
 *	Out of line support code for skb_push(). Not user callable.
 */

void skb_under_panic(struct sk_buff *skb, int sz, void *here)
{
	printk(KERN_EMERG "skb_under_panic: text:%p len:%d put:%d head:%p "
			  "data:%p tail:%#lx end:%#lx dev:%s\n",
	       here, skb->len, sz, skb->head, skb->data,
	       (unsigned long)skb->tail, (unsigned long)skb->end,
	       skb->dev ? skb->dev->name : "<NULL>");
	BUG();
}

void skb_truesize_bug(struct sk_buff *skb)
{
	printk(KERN_ERR "SKB BUG: Invalid truesize (%u) "
	       "len=%u, sizeof(sk_buff)=%Zd\n",
	       skb->truesize, skb->len, sizeof(struct sk_buff));
}
EXPORT_SYMBOL(skb_truesize_bug);

/* 	Allocate a new skbuff. We do this ourselves so we can fill in a few
 *	'private' fields and also do memory statistics to find all the
 *	[BEEP] leaks.
 *
 */

/**
 *	__alloc_skb	-	allocate a network buffer
 *	@size: size to allocate
 *	@gfp_mask: allocation mask
 *	@fclone: allocate from fclone cache instead of head cache
 *		and allocate a cloned (child) skb
 *	@node: numa node to allocate memory on
 *
 *	Allocate a new &sk_buff. The returned buffer has no headroom and a
 *	tail room of size bytes. The object has a reference count of one.
 *	The return is the buffer. On a failure the return is %NULL.
 *
 *	Buffers may only be allocated from interrupts using a @gfp_mask of
 *	%GFP_ATOMIC.
 */
struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask,
			    int fclone, int node)
{
	struct kmem_cache *cache;
	struct skb_shared_info *shinfo;
	struct sk_buff *skb;
	u8 *data;

	cache = fclone ? skbuff_fclone_cache : skbuff_head_cache;

	/* Get the HEAD */
	skb = kmem_cache_alloc_node(cache, gfp_mask & ~__GFP_DMA, node);
	if (!skb)
		goto out;

	size = SKB_DATA_ALIGN(size);
	data = kmalloc_node_track_caller(size + sizeof(struct skb_shared_info),
			gfp_mask, node);
	if (!data)
		goto nodata;

	/*
	 * Only clear those fields we need to clear, not those that we will
	 * actually initialise below. Hence, don't put any more fields after
	 * the tail pointer in struct sk_buff!
	 */
	memset(skb, 0, offsetof(struct sk_buff, tail));
	skb->truesize = size + sizeof(struct sk_buff);
	atomic_set(&skb->users, 1);
	skb->head = data;
	skb->data = data;
	skb_reset_tail_pointer(skb);
	skb->end = skb->tail + size;
	/* make sure we initialize shinfo sequentially */
	shinfo = skb_shinfo(skb);
	atomic_set(&shinfo->dataref, 1);
	shinfo->nr_frags  = 0;
	shinfo->gso_size = 0;
	shinfo->gso_segs = 0;
	shinfo->gso_type = 0;
	shinfo->ip6_frag_id = 0;
	shinfo->frag_list = NULL;

	if (fclone) {
		struct sk_buff *child = skb + 1;
		atomic_t *fclone_ref = (atomic_t *) (child + 1);

		skb->fclone = SKB_FCLONE_ORIG;
		atomic_set(fclone_ref, 1);

		child->fclone = SKB_FCLONE_UNAVAILABLE;
	}
out:
	return skb;
nodata:
	kmem_cache_free(cache, skb);
	skb = NULL;
	goto out;
}

/**
 *	__netdev_alloc_skb - allocate an skbuff for rx on a specific device
 *	@dev: network device to receive on
 *	@length: length to allocate
 *	@gfp_mask: get_free_pages mask, passed to alloc_skb
 *
 *	Allocate a new &sk_buff and assign it a usage count of one. The
 *	buffer has unspecified headroom built in. Users should allocate
 *	the headroom they think they need without accounting for the
 *	built in space. The built in space is used for optimisations.
 *
 *	%NULL is returned if there is no free memory.
 */
struct sk_buff *__netdev_alloc_skb(struct net_device *dev,
		unsigned int length, gfp_t gfp_mask)
{
	int node = dev->dev.parent ? dev_to_node(dev->dev.parent) : -1;
	struct sk_buff *skb;

	skb = __alloc_skb(length + NET_SKB_PAD, gfp_mask, 0, node);
	if (likely(skb)) {
		skb_reserve(skb, NET_SKB_PAD);
		skb->dev = dev;
	}
	return skb;
}

struct page *__netdev_alloc_page(struct net_device *dev, gfp_t gfp_mask)
{
	int node = dev->dev.parent ? dev_to_node(dev->dev.parent) : -1;
	struct page *page;

	page = alloc_pages_node(node, gfp_mask, 0);
	return page;
}
EXPORT_SYMBOL(__netdev_alloc_page);

void skb_add_rx_frag(struct sk_buff *skb, int i, struct page *page, int off,
		int size)
{
	skb_fill_page_desc(skb, i, page, off, size);
	skb->len += size;
	skb->data_len += size;
	skb->truesize += size;
}
EXPORT_SYMBOL(skb_add_rx_frag);

/**
 *	dev_alloc_skb - allocate an skbuff for receiving
 *	@length: length to allocate
 *
 *	Allocate a new &sk_buff and assign it a usage count of one. The
 *	buffer has unspecified headroom built in. Users should allocate
 *	the headroom they think they need without accounting for the
 *	built in space. The built in space is used for optimisations.
 *
 *	%NULL is returned if there is no free memory. Although this function
 *	allocates memory it can be called from an interrupt.
 */
struct sk_buff *dev_alloc_skb(unsigned int length)
{
	/*
	 * There is more code here than it seems:
	 * __dev_alloc_skb is an inline
	 */
	return __dev_alloc_skb(length, GFP_ATOMIC);
}
EXPORT_SYMBOL(dev_alloc_skb);

static void skb_drop_list(struct sk_buff **listp)
{
	struct sk_buff *list = *listp;

	*listp = NULL;

	do {
		struct sk_buff *this = list;
		list = list->next;
		kfree_skb(this);
	} while (list);
}

static inline void skb_drop_fraglist(struct sk_buff *skb)
{
	skb_drop_list(&skb_shinfo(skb)->frag_list);
}

static void skb_clone_fraglist(struct sk_buff *skb)
{
	struct sk_buff *list;

	for (list = skb_shinfo(skb)->frag_list; list; list = list->next)
		skb_get(list);
}

static void skb_release_data(struct sk_buff *skb)
{
	if (!skb->cloned ||
	    !atomic_sub_return(skb->nohdr ? (1 << SKB_DATAREF_SHIFT) + 1 : 1,
			       &skb_shinfo(skb)->dataref)) {
		if (skb_shinfo(skb)->nr_frags) {
			int i;
			for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
				put_page(skb_shinfo(skb)->frags[i].page);
		}

		if (skb_shinfo(skb)->frag_list)
			skb_drop_fraglist(skb);

		kfree(skb->head);
	}
}

/*
 *	Free an skbuff by memory without cleaning the state.
 */
static void kfree_skbmem(struct sk_buff *skb)
{
	struct sk_buff *other;
	atomic_t *fclone_ref;

	switch (skb->fclone) {
	case SKB_FCLONE_UNAVAILABLE:
		kmem_cache_free(skbuff_head_cache, skb);
		break;

	case SKB_FCLONE_ORIG:
		fclone_ref = (atomic_t *) (skb + 2);
		if (atomic_dec_and_test(fclone_ref))
			kmem_cache_free(skbuff_fclone_cache, skb);
		break;

	case SKB_FCLONE_CLONE:
		fclone_ref = (atomic_t *) (skb + 1);
		other = skb - 1;

		/* The clone portion is available for
		 * fast-cloning again.
		 */
		skb->fclone = SKB_FCLONE_UNAVAILABLE;

		if (atomic_dec_and_test(fclone_ref))
			kmem_cache_free(skbuff_fclone_cache, other);
		break;
	}
}

static void skb_release_head_state(struct sk_buff *skb)
{
	dst_release(skb->dst);
#ifdef CONFIG_XFRM
	secpath_put(skb->sp);
#endif
	if (skb->destructor) {
		WARN_ON(in_irq());
		skb->destructor(skb);
	}
#if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
	nf_conntrack_put(skb->nfct);
	nf_conntrack_put_reasm(skb->nfct_reasm);
#endif
#ifdef CONFIG_BRIDGE_NETFILTER
	nf_bridge_put(skb->nf_bridge);
#endif
/* XXX: IS this still necessary? - JHS */
#ifdef CONFIG_NET_SCHED
	skb->tc_index = 0;
#ifdef CONFIG_NET_CLS_ACT
	skb->tc_verd = 0;
#endif
#endif
}

/* Free everything but the sk_buff shell. */
static void skb_release_all(struct sk_buff *skb)
{
	skb_release_head_state(skb);
	skb_release_data(skb);
}

/**
 *	__kfree_skb - private function
 *	@skb: buffer
 *
 *	Free an sk_buff. Release anything attached to the buffer.
 *	Clean the state. This is an internal helper function. Users should
 *	always call kfree_skb
 */

void __kfree_skb(struct sk_buff *skb)
{
	skb_release_all(skb);
	kfree_skbmem(skb);
}
Commit	Line	Data
1da177e4 LT	1	/*
	2	* Routines having to do with the 'struct sk_buff' memory handlers.
	3	*
113aa838	4	* Authors: Alan Cox <alan@lxorguk.ukuu.org.uk>
1da177e4 LT	5	* Florian La Roche <rzsfl@rz.uni-sb.de>
1da177e4 LT	6	*
1da177e4 LT	7	* Fixes:
	8	* Alan Cox : Fixed the worst of the load
	9	* balancer bugs.
	10	* Dave Platt : Interrupt stacking fix.
	11	* Richard Kooijman : Timestamp fixes.
	12	* Alan Cox : Changed buffer format.
	13	* Alan Cox : destructor hook for AF_UNIX etc.
	14	* Linus Torvalds : Better skb_clone.
	15	* Alan Cox : Added skb_copy.
	16	* Alan Cox : Added all the changed routines Linus
	17	* only put in the headers
	18	* Ray VanTassle : Fixed --skb->lock in free
	19	* Alan Cox : skb_copy copy arp field
	20	* Andi Kleen : slabified it.
	21	* Robert Olsson : Removed skb_head_pool
	22	*
	23	* NOTE:
	24	* The __skb_ routines should be called with interrupts
	25	* disabled, or you better be real sure that the operation is atomic
	26	* with respect to whatever list is being frobbed (e.g. via lock_sock()
	27	* or via disabling bottom half handlers, etc).
	28	*
	29	* This program is free software; you can redistribute it and/or
	30	* modify it under the terms of the GNU General Public License
	31	* as published by the Free Software Foundation; either version
	32	* 2 of the License, or (at your option) any later version.
	33	*/
	34
	35	/*
	36	* The functions in this file will not compile correctly with gcc 2.4.x
	37	*/
	38
1da177e4 LT	39	#include <linux/module.h>
	40	#include <linux/types.h>
	41	#include <linux/kernel.h>
1da177e4 LT	42	#include <linux/mm.h>
	43	#include <linux/interrupt.h>
	44	#include <linux/in.h>
	45	#include <linux/inet.h>
	46	#include <linux/slab.h>
	47	#include <linux/netdevice.h>
	48	#ifdef CONFIG_NET_CLS_ACT
	49	#include <net/pkt_sched.h>
	50	#endif
	51	#include <linux/string.h>
	52	#include <linux/skbuff.h>
9c55e01c	53	#include <linux/splice.h>
1da177e4 LT	54	#include <linux/cache.h>
	55	#include <linux/rtnetlink.h>
	56	#include <linux/init.h>
716ea3a7	57	#include <linux/scatterlist.h>
1da177e4 LT	58
	59	#include <net/protocol.h>
	60	#include <net/dst.h>
	61	#include <net/sock.h>
	62	#include <net/checksum.h>
	63	#include <net/xfrm.h>
	64
	65	#include <asm/uaccess.h>
	66	#include <asm/system.h>
	67
a1f8e7f7 AV	68	#include "kmap_skb.h"
a1f8e7f7 AV	69
e18b890b CL	70	static struct kmem_cache *skbuff_head_cache __read_mostly;
e18b890b CL	71	static struct kmem_cache *skbuff_fclone_cache __read_mostly;
1da177e4	72
9c55e01c JA	73	static void sock_pipe_buf_release(struct pipe_inode_info *pipe,
	74	struct pipe_buffer *buf)
	75	{
	76	struct sk_buff skb = (struct sk_buff ) buf->private;
	77
	78	kfree_skb(skb);
	79	}
	80
	81	static void sock_pipe_buf_get(struct pipe_inode_info *pipe,
	82	struct pipe_buffer *buf)
	83	{
	84	struct sk_buff skb = (struct sk_buff ) buf->private;
	85
	86	skb_get(skb);
	87	}
	88
	89	static int sock_pipe_buf_steal(struct pipe_inode_info *pipe,
	90	struct pipe_buffer *buf)
	91	{
	92	return 1;
	93	}
	94
	95
	96	/* Pipe buffer operations for a socket. */
	97	static struct pipe_buf_operations sock_pipe_buf_ops = {
	98	.can_merge = 0,
	99	.map = generic_pipe_buf_map,
	100	.unmap = generic_pipe_buf_unmap,
	101	.confirm = generic_pipe_buf_confirm,
	102	.release = sock_pipe_buf_release,
	103	.steal = sock_pipe_buf_steal,
	104	.get = sock_pipe_buf_get,
	105	};
	106
1da177e4 LT	107	/*
	108	* Keep out-of-line to prevent kernel bloat.
	109	* __builtin_return_address is not used because it is not always
	110	* reliable.
	111	*/
	112
	113	/**
	114	* skb_over_panic - private function
	115	* @skb: buffer
	116	* @sz: size
	117	* @here: address
	118	*
	119	* Out of line support code for skb_put(). Not user callable.
	120	*/
	121	void skb_over_panic(struct sk_buff skb, int sz, void here)
	122	{
26095455	123	printk(KERN_EMERG "skb_over_panic: text:%p len:%d put:%d head:%p "
4305b541	124	"data:%p tail:%#lx end:%#lx dev:%s\n",
27a884dc	125	here, skb->len, sz, skb->head, skb->data,
4305b541	126	(unsigned long)skb->tail, (unsigned long)skb->end,
26095455	127	skb->dev ? skb->dev->name : "<NULL>");
1da177e4 LT	128	BUG();
	129	}
	130
	131	/**
	132	* skb_under_panic - private function
	133	* @skb: buffer
	134	* @sz: size
	135	* @here: address
	136	*
	137	* Out of line support code for skb_push(). Not user callable.
	138	*/
	139
	140	void skb_under_panic(struct sk_buff skb, int sz, void here)
	141	{
26095455	142	printk(KERN_EMERG "skb_under_panic: text:%p len:%d put:%d head:%p "
4305b541	143	"data:%p tail:%#lx end:%#lx dev:%s\n",
27a884dc	144	here, skb->len, sz, skb->head, skb->data,
4305b541	145	(unsigned long)skb->tail, (unsigned long)skb->end,
26095455	146	skb->dev ? skb->dev->name : "<NULL>");
1da177e4 LT	147	BUG();
	148	}
	149
dc6de336 DM	150	void skb_truesize_bug(struct sk_buff *skb)
	151	{
	152	printk(KERN_ERR "SKB BUG: Invalid truesize (%u) "
	153	"len=%u, sizeof(sk_buff)=%Zd\n",
	154	skb->truesize, skb->len, sizeof(struct sk_buff));
	155	}
	156	EXPORT_SYMBOL(skb_truesize_bug);
	157
1da177e4 LT	158	/* Allocate a new skbuff. We do this ourselves so we can fill in a few
	159	* 'private' fields and also do memory statistics to find all the
	160	* [BEEP] leaks.
	161	*
	162	*/
	163
	164	/**
d179cd12	165	* __alloc_skb - allocate a network buffer
1da177e4 LT	166	* @size: size to allocate
1da177e4 LT	167	* @gfp_mask: allocation mask
c83c2486 RD	168	* @fclone: allocate from fclone cache instead of head cache
c83c2486 RD	169	* and allocate a cloned (child) skb
b30973f8	170	* @node: numa node to allocate memory on
1da177e4 LT	171	*
	172	* Allocate a new &sk_buff. The returned buffer has no headroom and a
	173	* tail room of size bytes. The object has a reference count of one.
	174	* The return is the buffer. On a failure the return is %NULL.
	175	*
	176	* Buffers may only be allocated from interrupts using a @gfp_mask of
	177	* %GFP_ATOMIC.
	178	*/
dd0fc66f	179	struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask,
b30973f8	180	int fclone, int node)
1da177e4	181	{
e18b890b	182	struct kmem_cache *cache;
4947d3ef	183	struct skb_shared_info *shinfo;
1da177e4 LT	184	struct sk_buff *skb;
	185	u8 *data;
	186
8798b3fb HX	187	cache = fclone ? skbuff_fclone_cache : skbuff_head_cache;
8798b3fb HX	188
1da177e4	189	/* Get the HEAD */
b30973f8	190	skb = kmem_cache_alloc_node(cache, gfp_mask & ~__GFP_DMA, node);
1da177e4 LT	191	if (!skb)
	192	goto out;
	193
1da177e4	194	size = SKB_DATA_ALIGN(size);
b30973f8 CH	195	data = kmalloc_node_track_caller(size + sizeof(struct skb_shared_info),
b30973f8 CH	196	gfp_mask, node);
1da177e4 LT	197	if (!data)
	198	goto nodata;
	199
ca0605a7	200	/*
c8005785 JB	201	* Only clear those fields we need to clear, not those that we will
	202	* actually initialise below. Hence, don't put any more fields after
	203	* the tail pointer in struct sk_buff!
ca0605a7 ACM	204	*/
ca0605a7 ACM	205	memset(skb, 0, offsetof(struct sk_buff, tail));
1da177e4 LT	206	skb->truesize = size + sizeof(struct sk_buff);
	207	atomic_set(&skb->users, 1);
	208	skb->head = data;
	209	skb->data = data;
27a884dc	210	skb_reset_tail_pointer(skb);
4305b541	211	skb->end = skb->tail + size;
4947d3ef BL	212	/* make sure we initialize shinfo sequentially */
	213	shinfo = skb_shinfo(skb);
	214	atomic_set(&shinfo->dataref, 1);
	215	shinfo->nr_frags = 0;
7967168c HX	216	shinfo->gso_size = 0;
	217	shinfo->gso_segs = 0;
	218	shinfo->gso_type = 0;
4947d3ef BL	219	shinfo->ip6_frag_id = 0;
	220	shinfo->frag_list = NULL;
	221
d179cd12 DM	222	if (fclone) {
	223	struct sk_buff *child = skb + 1;
	224	atomic_t fclone_ref = (atomic_t ) (child + 1);
1da177e4	225
d179cd12 DM	226	skb->fclone = SKB_FCLONE_ORIG;
	227	atomic_set(fclone_ref, 1);
	228
	229	child->fclone = SKB_FCLONE_UNAVAILABLE;
	230	}
1da177e4 LT	231	out:
	232	return skb;
	233	nodata:
8798b3fb	234	kmem_cache_free(cache, skb);
1da177e4 LT	235	skb = NULL;
1da177e4 LT	236	goto out;
1da177e4 LT	237	}
1da177e4 LT	238
8af27456 CH	239	/**
	240	* __netdev_alloc_skb - allocate an skbuff for rx on a specific device
	241	* @dev: network device to receive on
	242	* @length: length to allocate
	243	* @gfp_mask: get_free_pages mask, passed to alloc_skb
	244	*
	245	* Allocate a new &sk_buff and assign it a usage count of one. The
	246	* buffer has unspecified headroom built in. Users should allocate
	247	* the headroom they think they need without accounting for the
	248	* built in space. The built in space is used for optimisations.
	249	*
	250	* %NULL is returned if there is no free memory.
	251	*/
	252	struct sk_buff __netdev_alloc_skb(struct net_device dev,
	253	unsigned int length, gfp_t gfp_mask)
	254	{
43cb76d9	255	int node = dev->dev.parent ? dev_to_node(dev->dev.parent) : -1;
8af27456 CH	256	struct sk_buff *skb;
8af27456 CH	257
4ec93edb	258	skb = __alloc_skb(length + NET_SKB_PAD, gfp_mask, 0, node);
7b2e497a	259	if (likely(skb)) {
8af27456	260	skb_reserve(skb, NET_SKB_PAD);
7b2e497a CH	261	skb->dev = dev;
7b2e497a CH	262	}
8af27456 CH	263	return skb;
8af27456 CH	264	}
1da177e4	265
654bed16 PZ	266	struct page __netdev_alloc_page(struct net_device dev, gfp_t gfp_mask)
	267	{
	268	int node = dev->dev.parent ? dev_to_node(dev->dev.parent) : -1;
	269	struct page *page;
	270
	271	page = alloc_pages_node(node, gfp_mask, 0);
	272	return page;
	273	}
	274	EXPORT_SYMBOL(__netdev_alloc_page);
	275
	276	void skb_add_rx_frag(struct sk_buff skb, int i, struct page page, int off,
	277	int size)
	278	{
	279	skb_fill_page_desc(skb, i, page, off, size);
	280	skb->len += size;
	281	skb->data_len += size;
	282	skb->truesize += size;
	283	}
	284	EXPORT_SYMBOL(skb_add_rx_frag);
	285
f58518e6 IJ	286	/**
	287	* dev_alloc_skb - allocate an skbuff for receiving
	288	* @length: length to allocate
	289	*
	290	* Allocate a new &sk_buff and assign it a usage count of one. The
	291	* buffer has unspecified headroom built in. Users should allocate
	292	* the headroom they think they need without accounting for the
	293	* built in space. The built in space is used for optimisations.
	294	*
	295	* %NULL is returned if there is no free memory. Although this function
	296	* allocates memory it can be called from an interrupt.
	297	*/
	298	struct sk_buff *dev_alloc_skb(unsigned int length)
	299	{
1483b874 DV	300	/*
1483b874 DV	301	* There is more code here than it seems:
a0f55e0e	302	* __dev_alloc_skb is an inline
1483b874	303	*/
f58518e6 IJ	304	return __dev_alloc_skb(length, GFP_ATOMIC);
	305	}
	306	EXPORT_SYMBOL(dev_alloc_skb);
	307
27b437c8	308	static void skb_drop_list(struct sk_buff **listp)
1da177e4	309	{
27b437c8	310	struct sk_buff list = listp;
1da177e4	311
27b437c8	312	*listp = NULL;
1da177e4 LT	313
	314	do {
	315	struct sk_buff *this = list;
	316	list = list->next;
	317	kfree_skb(this);
	318	} while (list);
	319	}
	320
27b437c8 HX	321	static inline void skb_drop_fraglist(struct sk_buff *skb)
	322	{
	323	skb_drop_list(&skb_shinfo(skb)->frag_list);
	324	}
	325
1da177e4 LT	326	static void skb_clone_fraglist(struct sk_buff *skb)
	327	{
	328	struct sk_buff *list;
	329
	330	for (list = skb_shinfo(skb)->frag_list; list; list = list->next)
	331	skb_get(list);
	332	}
	333
5bba1712	334	static void skb_release_data(struct sk_buff *skb)
1da177e4 LT	335	{
	336	if (!skb->cloned \|\|
	337	!atomic_sub_return(skb->nohdr ? (1 << SKB_DATAREF_SHIFT) + 1 : 1,
	338	&skb_shinfo(skb)->dataref)) {
	339	if (skb_shinfo(skb)->nr_frags) {
	340	int i;
	341	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
	342	put_page(skb_shinfo(skb)->frags[i].page);
	343	}
	344
	345	if (skb_shinfo(skb)->frag_list)
	346	skb_drop_fraglist(skb);
	347
	348	kfree(skb->head);
	349	}
	350	}
	351
	352	/*
	353	* Free an skbuff by memory without cleaning the state.
	354	*/
2d4baff8	355	static void kfree_skbmem(struct sk_buff *skb)
1da177e4	356	{
d179cd12 DM	357	struct sk_buff *other;
	358	atomic_t *fclone_ref;
	359
d179cd12 DM	360	switch (skb->fclone) {
	361	case SKB_FCLONE_UNAVAILABLE:
	362	kmem_cache_free(skbuff_head_cache, skb);
	363	break;
	364
	365	case SKB_FCLONE_ORIG:
	366	fclone_ref = (atomic_t *) (skb + 2);
	367	if (atomic_dec_and_test(fclone_ref))
	368	kmem_cache_free(skbuff_fclone_cache, skb);
	369	break;
	370
	371	case SKB_FCLONE_CLONE:
	372	fclone_ref = (atomic_t *) (skb + 1);
	373	other = skb - 1;
	374
	375	/* The clone portion is available for
	376	* fast-cloning again.
	377	*/
	378	skb->fclone = SKB_FCLONE_UNAVAILABLE;
	379
	380	if (atomic_dec_and_test(fclone_ref))
	381	kmem_cache_free(skbuff_fclone_cache, other);
	382	break;
3ff50b79	383	}
1da177e4 LT	384	}
1da177e4 LT	385
04a4bb55	386	static void skb_release_head_state(struct sk_buff *skb)
1da177e4	387	{
1da177e4 LT	388	dst_release(skb->dst);
	389	#ifdef CONFIG_XFRM
	390	secpath_put(skb->sp);
	391	#endif
9c2b3328 SH	392	if (skb->destructor) {
9c2b3328 SH	393	WARN_ON(in_irq());
1da177e4 LT	394	skb->destructor(skb);
1da177e4 LT	395	}
9fb9cbb1	396	#if defined(CONFIG_NF_CONNTRACK) \|\| defined(CONFIG_NF_CONNTRACK_MODULE)
5f79e0f9	397	nf_conntrack_put(skb->nfct);
9fb9cbb1 YK	398	nf_conntrack_put_reasm(skb->nfct_reasm);
9fb9cbb1 YK	399	#endif
1da177e4 LT	400	#ifdef CONFIG_BRIDGE_NETFILTER
	401	nf_bridge_put(skb->nf_bridge);
	402	#endif
1da177e4 LT	403	/* XXX: IS this still necessary? - JHS */
	404	#ifdef CONFIG_NET_SCHED
	405	skb->tc_index = 0;
	406	#ifdef CONFIG_NET_CLS_ACT
	407	skb->tc_verd = 0;
1da177e4 LT	408	#endif
1da177e4 LT	409	#endif
04a4bb55 LB	410	}
	411
	412	/* Free everything but the sk_buff shell. */
	413	static void skb_release_all(struct sk_buff *skb)
	414	{
	415	skb_release_head_state(skb);
2d4baff8 HX	416	skb_release_data(skb);
	417	}
	418
	419	/**
	420	* __kfree_skb - private function
	421	* @skb: buffer
	422	*
	423	* Free an sk_buff. Release anything attached to the buffer.
	424	* Clean the state. This is an internal helper function. Users should
	425	* always call kfree_skb
	426	*/
1da177e4	427
2d4baff8 HX	428	void __kfree_skb(struct sk_buff *skb)
	429	{
	430	skb_release_all(skb);
1da177e4 LT	431	kfree_skbmem(skb);
	432	}
	433