[net-next-2.6.git] / mm / page_cgroup.c

#include <linux/mm.h>
#include <linux/mmzone.h>
#include <linux/bootmem.h>
#include <linux/bit_spinlock.h>
#include <linux/page_cgroup.h>
#include <linux/hash.h>
#include <linux/slab.h>
#include <linux/memory.h>
#include <linux/vmalloc.h>
#include <linux/cgroup.h>
#include <linux/swapops.h>

static void __meminit
__init_page_cgroup(struct page_cgroup *pc, unsigned long pfn)
{
	pc->flags = 0;
	pc->mem_cgroup = NULL;
	pc->page = pfn_to_page(pfn);
	INIT_LIST_HEAD(&pc->lru);
}
static unsigned long total_usage;

#if !defined(CONFIG_SPARSEMEM)


void __meminit pgdat_page_cgroup_init(struct pglist_data *pgdat)
{
	pgdat->node_page_cgroup = NULL;
}

struct page_cgroup *lookup_page_cgroup(struct page *page)
{
	unsigned long pfn = page_to_pfn(page);
	unsigned long offset;
	struct page_cgroup *base;

	base = NODE_DATA(page_to_nid(page))->node_page_cgroup;
	if (unlikely(!base))
		return NULL;

	offset = pfn - NODE_DATA(page_to_nid(page))->node_start_pfn;
	return base + offset;
}

static int __init alloc_node_page_cgroup(int nid)
{
	struct page_cgroup *base, *pc;
	unsigned long table_size;
	unsigned long start_pfn, nr_pages, index;
	struct page *page;
	unsigned int order;

	start_pfn = NODE_DATA(nid)->node_start_pfn;
	nr_pages = NODE_DATA(nid)->node_spanned_pages;

	if (!nr_pages)
		return 0;

	table_size = sizeof(struct page_cgroup) * nr_pages;
	order = get_order(table_size);
	page = alloc_pages_node(nid, GFP_NOWAIT | __GFP_ZERO, order);
	if (!page)
		page = alloc_pages_node(-1, GFP_NOWAIT | __GFP_ZERO, order);
	if (!page)
		return -ENOMEM;
	base = page_address(page);
	for (index = 0; index < nr_pages; index++) {
		pc = base + index;
		__init_page_cgroup(pc, start_pfn + index);
	}
	NODE_DATA(nid)->node_page_cgroup = base;
	total_usage += table_size;
	return 0;
}

void __init page_cgroup_init(void)
{

	int nid, fail;

	if (mem_cgroup_disabled())
		return;

	for_each_online_node(nid)  {
		fail = alloc_node_page_cgroup(nid);
		if (fail)
			goto fail;
	}
	printk(KERN_INFO "allocated %ld bytes of page_cgroup\n", total_usage);
	printk(KERN_INFO "please try cgroup_disable=memory option if you"
	" don't want\n");
	return;
fail:
	printk(KERN_CRIT "allocation of page_cgroup was failed.\n");
	printk(KERN_CRIT "please try cgroup_disable=memory boot option\n");
	panic("Out of memory");
}

#else /* CONFIG_FLAT_NODE_MEM_MAP */

struct page_cgroup *lookup_page_cgroup(struct page *page)
{
	unsigned long pfn = page_to_pfn(page);
	struct mem_section *section = __pfn_to_section(pfn);

	return section->page_cgroup + pfn;
}

/* __alloc_bootmem...() is protected by !slab_available() */
static int __init_refok init_section_page_cgroup(unsigned long pfn)
{
	struct mem_section *section = __pfn_to_section(pfn);
	struct page_cgroup *base, *pc;
	unsigned long table_size;
	int nid, index;

	if (!section->page_cgroup) {
		nid = page_to_nid(pfn_to_page(pfn));
		table_size = sizeof(struct page_cgroup) * PAGES_PER_SECTION;
		if (slab_is_available()) {
			base = kmalloc_node(table_size,
					GFP_KERNEL | __GFP_NOWARN, nid);
			if (!base)
				base = vmalloc_node(table_size, nid);
		} else {
			base = __alloc_bootmem_node_nopanic(NODE_DATA(nid),
				table_size,
				PAGE_SIZE, __pa(MAX_DMA_ADDRESS));
		}
	} else {
		/*
 		 * We don't have to allocate page_cgroup again, but
		 * address of memmap may be changed. So, we have to initialize
		 * again.
		 */
		base = section->page_cgroup + pfn;
		table_size = 0;
		/* check address of memmap is changed or not. */
		if (base->page == pfn_to_page(pfn))
			return 0;
	}

	if (!base) {
		printk(KERN_ERR "page cgroup allocation failure\n");
		return -ENOMEM;
	}

	for (index = 0; index < PAGES_PER_SECTION; index++) {
		pc = base + index;
		__init_page_cgroup(pc, pfn + index);
	}

	section->page_cgroup = base - pfn;
	total_usage += table_size;
	return 0;
}
#ifdef CONFIG_MEMORY_HOTPLUG
void __free_page_cgroup(unsigned long pfn)
{
	struct mem_section *ms;
	struct page_cgroup *base;

	ms = __pfn_to_section(pfn);
	if (!ms || !ms->page_cgroup)
		return;
	base = ms->page_cgroup + pfn;
	if (is_vmalloc_addr(base)) {
		vfree(base);
		ms->page_cgroup = NULL;
	} else {
		struct page *page = virt_to_page(base);
		if (!PageReserved(page)) { /* Is bootmem ? */
			kfree(base);
			ms->page_cgroup = NULL;
		}
	}
}

int __meminit online_page_cgroup(unsigned long start_pfn,
			unsigned long nr_pages,
			int nid)
{
	unsigned long start, end, pfn;
	int fail = 0;

	start = start_pfn & ~(PAGES_PER_SECTION - 1);
	end = ALIGN(start_pfn + nr_pages, PAGES_PER_SECTION);

	for (pfn = start; !fail && pfn < end; pfn += PAGES_PER_SECTION) {
		if (!pfn_present(pfn))
			continue;
		fail = init_section_page_cgroup(pfn);
	}
	if (!fail)
		return 0;

	/* rollback */
	for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION)
		__free_page_cgroup(pfn);

	return -ENOMEM;
}

int __meminit offline_page_cgroup(unsigned long start_pfn,
		unsigned long nr_pages, int nid)
{
	unsigned long start, end, pfn;

	start = start_pfn & ~(PAGES_PER_SECTION - 1);
	end = ALIGN(start_pfn + nr_pages, PAGES_PER_SECTION);

	for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION)
		__free_page_cgroup(pfn);
	return 0;

}

static int __meminit page_cgroup_callback(struct notifier_block *self,
			       unsigned long action, void *arg)
{
	struct memory_notify *mn = arg;
	int ret = 0;
	switch (action) {
	case MEM_GOING_ONLINE:
		ret = online_page_cgroup(mn->start_pfn,
				   mn->nr_pages, mn->status_change_nid);
		break;
	case MEM_OFFLINE:
		offline_page_cgroup(mn->start_pfn,
				mn->nr_pages, mn->status_change_nid);
		break;
	case MEM_CANCEL_ONLINE:
	case MEM_GOING_OFFLINE:
		break;
	case MEM_ONLINE:
	case MEM_CANCEL_OFFLINE:
		break;
	}

	if (ret)
		ret = notifier_from_errno(ret);
	else
		ret = NOTIFY_OK;

	return ret;
}

#endif

void __init page_cgroup_init(void)
{
	unsigned long pfn;
	int fail = 0;

	if (mem_cgroup_disabled())
		return;

	for (pfn = 0; !fail && pfn < max_pfn; pfn += PAGES_PER_SECTION) {
		if (!pfn_present(pfn))
			continue;
		fail = init_section_page_cgroup(pfn);
	}
	if (fail) {
		printk(KERN_CRIT "try cgroup_disable=memory boot option\n");
		panic("Out of memory");
	} else {
		hotplug_memory_notifier(page_cgroup_callback, 0);
	}
	printk(KERN_INFO "allocated %ld bytes of page_cgroup\n", total_usage);
	printk(KERN_INFO "please try cgroup_disable=memory option if you don't"
	" want\n");
}

void __meminit pgdat_page_cgroup_init(struct pglist_data *pgdat)
{
	return;
}

#endif


#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP

static DEFINE_MUTEX(swap_cgroup_mutex);
struct swap_cgroup_ctrl {
	struct page **map;
	unsigned long length;
};

struct swap_cgroup_ctrl swap_cgroup_ctrl[MAX_SWAPFILES];

struct swap_cgroup {
	unsigned short		id;
};
#define SC_PER_PAGE	(PAGE_SIZE/sizeof(struct swap_cgroup))
#define SC_POS_MASK	(SC_PER_PAGE - 1)

/*
 * SwapCgroup implements "lookup" and "exchange" operations.
 * In typical usage, this swap_cgroup is accessed via memcg's charge/uncharge
 * against SwapCache. At swap_free(), this is accessed directly from swap.
 *
 * This means,
 *  - we have no race in "exchange" when we're accessed via SwapCache because
 *    SwapCache(and its swp_entry) is under lock.
 *  - When called via swap_free(), there is no user of this entry and no race.
 * Then, we don't need lock around "exchange".
 *
 * TODO: we can push these buffers out to HIGHMEM.
 */

/*
 * allocate buffer for swap_cgroup.
 */
static int swap_cgroup_prepare(int type)
{
	struct page *page;
	struct swap_cgroup_ctrl *ctrl;
	unsigned long idx, max;

	if (!do_swap_account)
		return 0;
	ctrl = &swap_cgroup_ctrl[type];

	for (idx = 0; idx < ctrl->length; idx++) {
		page = alloc_page(GFP_KERNEL | __GFP_ZERO);
		if (!page)
			goto not_enough_page;
		ctrl->map[idx] = page;
	}
	return 0;
not_enough_page:
	max = idx;
	for (idx = 0; idx < max; idx++)
		__free_page(ctrl->map[idx]);

	return -ENOMEM;
}

/**
 * swap_cgroup_record - record mem_cgroup for this swp_entry.
 * @ent: swap entry to be recorded into
 * @mem: mem_cgroup to be recorded
 *
 * Returns old value at success, 0 at failure.
 * (Of course, old value can be 0.)
 */
unsigned short swap_cgroup_record(swp_entry_t ent, unsigned short id)
{
	int type = swp_type(ent);
	unsigned long offset = swp_offset(ent);
	unsigned long idx = offset / SC_PER_PAGE;
	unsigned long pos = offset & SC_POS_MASK;
	struct swap_cgroup_ctrl *ctrl;
	struct page *mappage;
	struct swap_cgroup *sc;
	unsigned short old;

	if (!do_swap_account)
		return 0;

	ctrl = &swap_cgroup_ctrl[type];

	mappage = ctrl->map[idx];
	sc = page_address(mappage);
	sc += pos;
	old = sc->id;
	sc->id = id;

	return old;
}

/**
 * lookup_swap_cgroup - lookup mem_cgroup tied to swap entry
 * @ent: swap entry to be looked up.
 *
 * Returns CSS ID of mem_cgroup at success. 0 at failure. (0 is invalid ID)
 */
unsigned short lookup_swap_cgroup(swp_entry_t ent)
{
	int type = swp_type(ent);
	unsigned long offset = swp_offset(ent);
	unsigned long idx = offset / SC_PER_PAGE;
	unsigned long pos = offset & SC_POS_MASK;
	struct swap_cgroup_ctrl *ctrl;
	struct page *mappage;
	struct swap_cgroup *sc;
	unsigned short ret;

	if (!do_swap_account)
		return 0;

	ctrl = &swap_cgroup_ctrl[type];
	mappage = ctrl->map[idx];
	sc = page_address(mappage);
	sc += pos;
	ret = sc->id;
	return ret;
}

int swap_cgroup_swapon(int type, unsigned long max_pages)
{
	void *array;
	unsigned long array_size;
	unsigned long length;
	struct swap_cgroup_ctrl *ctrl;

	if (!do_swap_account)
		return 0;

	length = ((max_pages/SC_PER_PAGE) + 1);
	array_size = length * sizeof(void *);

	array = vmalloc(array_size);
	if (!array)
		goto nomem;

	memset(array, 0, array_size);
	ctrl = &swap_cgroup_ctrl[type];
	mutex_lock(&swap_cgroup_mutex);
	ctrl->length = length;
	ctrl->map = array;
	if (swap_cgroup_prepare(type)) {
		/* memory shortage */
		ctrl->map = NULL;
		ctrl->length = 0;
		vfree(array);
		mutex_unlock(&swap_cgroup_mutex);
		goto nomem;
	}
	mutex_unlock(&swap_cgroup_mutex);

	return 0;
nomem:
	printk(KERN_INFO "couldn't allocate enough memory for swap_cgroup.\n");
	printk(KERN_INFO
		"swap_cgroup can be disabled by noswapaccount boot option\n");
	return -ENOMEM;
}

void swap_cgroup_swapoff(int type)
{
	int i;
	struct swap_cgroup_ctrl *ctrl;

	if (!do_swap_account)
		return;

	mutex_lock(&swap_cgroup_mutex);
	ctrl = &swap_cgroup_ctrl[type];
	if (ctrl->map) {
		for (i = 0; i < ctrl->length; i++) {
			struct page *page = ctrl->map[i];
			if (page)
				__free_page(page);
		}
		vfree(ctrl->map);
		ctrl->map = NULL;
		ctrl->length = 0;
	}
	mutex_unlock(&swap_cgroup_mutex);
}

#endif
Commit	Line	Data
52d4b9ac KH	1	#include <linux/mm.h>
	2	#include <linux/mmzone.h>
	3	#include <linux/bootmem.h>
	4	#include <linux/bit_spinlock.h>
	5	#include <linux/page_cgroup.h>
	6	#include <linux/hash.h>
94b6da5a	7	#include <linux/slab.h>
52d4b9ac	8	#include <linux/memory.h>
4c821042	9	#include <linux/vmalloc.h>
94b6da5a	10	#include <linux/cgroup.h>
27a7faa0	11	#include <linux/swapops.h>
52d4b9ac KH	12
	13	static void __meminit
	14	__init_page_cgroup(struct page_cgroup *pc, unsigned long pfn)
	15	{
	16	pc->flags = 0;
	17	pc->mem_cgroup = NULL;
	18	pc->page = pfn_to_page(pfn);
08e552c6	19	INIT_LIST_HEAD(&pc->lru);
52d4b9ac KH	20	}
	21	static unsigned long total_usage;
	22
	23	#if !defined(CONFIG_SPARSEMEM)
	24
	25
31168481	26	void __meminit pgdat_page_cgroup_init(struct pglist_data *pgdat)
52d4b9ac KH	27	{
	28	pgdat->node_page_cgroup = NULL;
	29	}
	30
	31	struct page_cgroup lookup_page_cgroup(struct page page)
	32	{
	33	unsigned long pfn = page_to_pfn(page);
	34	unsigned long offset;
	35	struct page_cgroup *base;
	36
	37	base = NODE_DATA(page_to_nid(page))->node_page_cgroup;
	38	if (unlikely(!base))
	39	return NULL;
	40
	41	offset = pfn - NODE_DATA(page_to_nid(page))->node_start_pfn;
	42	return base + offset;
	43	}
	44
	45	static int __init alloc_node_page_cgroup(int nid)
	46	{
	47	struct page_cgroup base, pc;
	48	unsigned long table_size;
	49	unsigned long start_pfn, nr_pages, index;
959982fe YL	50	struct page *page;
959982fe YL	51	unsigned int order;
52d4b9ac KH	52
	53	start_pfn = NODE_DATA(nid)->node_start_pfn;
	54	nr_pages = NODE_DATA(nid)->node_spanned_pages;
	55
653d22c0 KH	56	if (!nr_pages)
	57	return 0;
	58
52d4b9ac	59	table_size = sizeof(struct page_cgroup) * nr_pages;
959982fe YL	60	order = get_order(table_size);
	61	page = alloc_pages_node(nid, GFP_NOWAIT \| __GFP_ZERO, order);
	62	if (!page)
	63	page = alloc_pages_node(-1, GFP_NOWAIT \| __GFP_ZERO, order);
	64	if (!page)
52d4b9ac	65	return -ENOMEM;
959982fe	66	base = page_address(page);
52d4b9ac KH	67	for (index = 0; index < nr_pages; index++) {
	68	pc = base + index;
	69	__init_page_cgroup(pc, start_pfn + index);
	70	}
	71	NODE_DATA(nid)->node_page_cgroup = base;
	72	total_usage += table_size;
	73	return 0;
	74	}
	75
	76	void __init page_cgroup_init(void)
	77	{
	78
	79	int nid, fail;
	80
f8d66542	81	if (mem_cgroup_disabled())
94b6da5a KH	82	return;
94b6da5a KH	83
52d4b9ac KH	84	for_each_online_node(nid) {
	85	fail = alloc_node_page_cgroup(nid);
	86	if (fail)
	87	goto fail;
	88	}
	89	printk(KERN_INFO "allocated %ld bytes of page_cgroup\n", total_usage);
	90	printk(KERN_INFO "please try cgroup_disable=memory option if you"
	91	" don't want\n");
	92	return;
	93	fail:
	94	printk(KERN_CRIT "allocation of page_cgroup was failed.\n");
	95	printk(KERN_CRIT "please try cgroup_disable=memory boot option\n");
	96	panic("Out of memory");
	97	}
	98
	99	#else /* CONFIG_FLAT_NODE_MEM_MAP */
	100
	101	struct page_cgroup lookup_page_cgroup(struct page page)
	102	{
	103	unsigned long pfn = page_to_pfn(page);
	104	struct mem_section *section = __pfn_to_section(pfn);
	105
	106	return section->page_cgroup + pfn;
	107	}
	108
31168481	109	/* __alloc_bootmem...() is protected by !slab_available() */
feb16694	110	static int __init_refok init_section_page_cgroup(unsigned long pfn)
52d4b9ac	111	{
0753b0ef	112	struct mem_section *section = __pfn_to_section(pfn);
52d4b9ac KH	113	struct page_cgroup base, pc;
	114	unsigned long table_size;
	115	int nid, index;
	116
dc19f9db KH	117	if (!section->page_cgroup) {
	118	nid = page_to_nid(pfn_to_page(pfn));
	119	table_size = sizeof(struct page_cgroup) * PAGES_PER_SECTION;
	120	if (slab_is_available()) {
2e9c2372 KH	121	base = kmalloc_node(table_size,
2e9c2372 KH	122	GFP_KERNEL \| __GFP_NOWARN, nid);
dc19f9db KH	123	if (!base)
	124	base = vmalloc_node(table_size, nid);
	125	} else {
	126	base = __alloc_bootmem_node_nopanic(NODE_DATA(nid),
	127	table_size,
94b6da5a	128	PAGE_SIZE, __pa(MAX_DMA_ADDRESS));
dc19f9db KH	129	}
	130	} else {
	131	/*
	132	* We don't have to allocate page_cgroup again, but
	133	* address of memmap may be changed. So, we have to initialize
	134	* again.
	135	*/
	136	base = section->page_cgroup + pfn;
	137	table_size = 0;
	138	/* check address of memmap is changed or not. */
	139	if (base->page == pfn_to_page(pfn))
	140	return 0;
94b6da5a	141	}
52d4b9ac KH	142
	143	if (!base) {
	144	printk(KERN_ERR "page cgroup allocation failure\n");
	145	return -ENOMEM;
	146	}
	147
	148	for (index = 0; index < PAGES_PER_SECTION; index++) {
	149	pc = base + index;
	150	__init_page_cgroup(pc, pfn + index);
	151	}
	152
52d4b9ac KH	153	section->page_cgroup = base - pfn;
	154	total_usage += table_size;
	155	return 0;
	156	}
	157	#ifdef CONFIG_MEMORY_HOTPLUG
	158	void __free_page_cgroup(unsigned long pfn)
	159	{
	160	struct mem_section *ms;
	161	struct page_cgroup *base;
	162
	163	ms = __pfn_to_section(pfn);
	164	if (!ms \|\| !ms->page_cgroup)
	165	return;
	166	base = ms->page_cgroup + pfn;
94b6da5a	167	if (is_vmalloc_addr(base)) {
52d4b9ac	168	vfree(base);
94b6da5a KH	169	ms->page_cgroup = NULL;
	170	} else {
	171	struct page *page = virt_to_page(base);
	172	if (!PageReserved(page)) { /* Is bootmem ? */
	173	kfree(base);
	174	ms->page_cgroup = NULL;
	175	}
	176	}
52d4b9ac KH	177	}
52d4b9ac KH	178
31168481	179	int __meminit online_page_cgroup(unsigned long start_pfn,
52d4b9ac KH	180	unsigned long nr_pages,
	181	int nid)
	182	{
	183	unsigned long start, end, pfn;
	184	int fail = 0;
	185
33c5d3d6	186	start = start_pfn & ~(PAGES_PER_SECTION - 1);
52d4b9ac KH	187	end = ALIGN(start_pfn + nr_pages, PAGES_PER_SECTION);
	188
	189	for (pfn = start; !fail && pfn < end; pfn += PAGES_PER_SECTION) {
	190	if (!pfn_present(pfn))
	191	continue;
	192	fail = init_section_page_cgroup(pfn);
	193	}
	194	if (!fail)
	195	return 0;
	196
	197	/* rollback */
	198	for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION)
	199	__free_page_cgroup(pfn);
	200
	201	return -ENOMEM;
	202	}
	203
31168481	204	int __meminit offline_page_cgroup(unsigned long start_pfn,
52d4b9ac KH	205	unsigned long nr_pages, int nid)
	206	{
	207	unsigned long start, end, pfn;
	208
33c5d3d6	209	start = start_pfn & ~(PAGES_PER_SECTION - 1);
52d4b9ac KH	210	end = ALIGN(start_pfn + nr_pages, PAGES_PER_SECTION);
	211
	212	for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION)
	213	__free_page_cgroup(pfn);
	214	return 0;
	215
	216	}
	217
31168481	218	static int __meminit page_cgroup_callback(struct notifier_block *self,
52d4b9ac KH	219	unsigned long action, void *arg)
	220	{
	221	struct memory_notify *mn = arg;
	222	int ret = 0;
	223	switch (action) {
	224	case MEM_GOING_ONLINE:
	225	ret = online_page_cgroup(mn->start_pfn,
	226	mn->nr_pages, mn->status_change_nid);
	227	break;
52d4b9ac KH	228	case MEM_OFFLINE:
	229	offline_page_cgroup(mn->start_pfn,
	230	mn->nr_pages, mn->status_change_nid);
	231	break;
dc19f9db	232	case MEM_CANCEL_ONLINE:
52d4b9ac KH	233	case MEM_GOING_OFFLINE:
	234	break;
	235	case MEM_ONLINE:
	236	case MEM_CANCEL_OFFLINE:
	237	break;
	238	}
dc19f9db KH	239
	240	if (ret)
	241	ret = notifier_from_errno(ret);
	242	else
	243	ret = NOTIFY_OK;
	244
52d4b9ac KH	245	return ret;
	246	}
	247
	248	#endif
	249
	250	void __init page_cgroup_init(void)
	251	{
	252	unsigned long pfn;
	253	int fail = 0;
	254
f8d66542	255	if (mem_cgroup_disabled())
94b6da5a KH	256	return;
94b6da5a KH	257
52d4b9ac KH	258	for (pfn = 0; !fail && pfn < max_pfn; pfn += PAGES_PER_SECTION) {
	259	if (!pfn_present(pfn))
	260	continue;
	261	fail = init_section_page_cgroup(pfn);
	262	}
	263	if (fail) {
	264	printk(KERN_CRIT "try cgroup_disable=memory boot option\n");
	265	panic("Out of memory");
	266	} else {
	267	hotplug_memory_notifier(page_cgroup_callback, 0);
	268	}
	269	printk(KERN_INFO "allocated %ld bytes of page_cgroup\n", total_usage);
	270	printk(KERN_INFO "please try cgroup_disable=memory option if you don't"
	271	" want\n");
	272	}
	273
31168481	274	void __meminit pgdat_page_cgroup_init(struct pglist_data *pgdat)
52d4b9ac KH	275	{
	276	return;
	277	}
	278
	279	#endif
27a7faa0 KH	280
	281
	282	#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
	283
	284	static DEFINE_MUTEX(swap_cgroup_mutex);
	285	struct swap_cgroup_ctrl {
	286	struct page **map;
	287	unsigned long length;
	288	};
	289
	290	struct swap_cgroup_ctrl swap_cgroup_ctrl[MAX_SWAPFILES];
	291
27a7faa0	292	struct swap_cgroup {
a3b2d692	293	unsigned short id;
27a7faa0 KH	294	};
	295	#define SC_PER_PAGE (PAGE_SIZE/sizeof(struct swap_cgroup))
	296	#define SC_POS_MASK (SC_PER_PAGE - 1)
	297
	298	/*
	299	* SwapCgroup implements "lookup" and "exchange" operations.
	300	* In typical usage, this swap_cgroup is accessed via memcg's charge/uncharge
	301	* against SwapCache. At swap_free(), this is accessed directly from swap.
	302	*
	303	* This means,
	304	* - we have no race in "exchange" when we're accessed via SwapCache because
	305	* SwapCache(and its swp_entry) is under lock.
	306	* - When called via swap_free(), there is no user of this entry and no race.
	307	* Then, we don't need lock around "exchange".
	308	*
	309	* TODO: we can push these buffers out to HIGHMEM.
	310	*/
	311
	312	/*
	313	* allocate buffer for swap_cgroup.
	314	*/
	315	static int swap_cgroup_prepare(int type)
	316	{
	317	struct page *page;
	318	struct swap_cgroup_ctrl *ctrl;
	319	unsigned long idx, max;
	320
	321	if (!do_swap_account)
	322	return 0;
	323	ctrl = &swap_cgroup_ctrl[type];
	324
	325	for (idx = 0; idx < ctrl->length; idx++) {
	326	page = alloc_page(GFP_KERNEL \| __GFP_ZERO);
	327	if (!page)
	328	goto not_enough_page;
	329	ctrl->map[idx] = page;
	330	}
	331	return 0;
	332	not_enough_page:
	333	max = idx;
	334	for (idx = 0; idx < max; idx++)
	335	__free_page(ctrl->map[idx]);
	336
	337	return -ENOMEM;
	338	}
	339
	340	/**
	341	* swap_cgroup_record - record mem_cgroup for this swp_entry.
	342	* @ent: swap entry to be recorded into
	343	* @mem: mem_cgroup to be recorded
	344	*
a3b2d692 KH	345	* Returns old value at success, 0 at failure.
a3b2d692 KH	346	* (Of course, old value can be 0.)
27a7faa0	347	*/
a3b2d692	348	unsigned short swap_cgroup_record(swp_entry_t ent, unsigned short id)
27a7faa0 KH	349	{
	350	int type = swp_type(ent);
	351	unsigned long offset = swp_offset(ent);
	352	unsigned long idx = offset / SC_PER_PAGE;
	353	unsigned long pos = offset & SC_POS_MASK;
	354	struct swap_cgroup_ctrl *ctrl;
	355	struct page *mappage;
	356	struct swap_cgroup *sc;
a3b2d692	357	unsigned short old;
27a7faa0 KH	358
27a7faa0 KH	359	if (!do_swap_account)
a3b2d692	360	return 0;
27a7faa0 KH	361
	362	ctrl = &swap_cgroup_ctrl[type];
	363
	364	mappage = ctrl->map[idx];
	365	sc = page_address(mappage);
	366	sc += pos;
a3b2d692 KH	367	old = sc->id;
a3b2d692 KH	368	sc->id = id;
27a7faa0 KH	369
	370	return old;
	371	}
	372
	373	/**
	374	* lookup_swap_cgroup - lookup mem_cgroup tied to swap entry
	375	* @ent: swap entry to be looked up.
	376	*
a3b2d692	377	* Returns CSS ID of mem_cgroup at success. 0 at failure. (0 is invalid ID)
27a7faa0	378	*/
a3b2d692	379	unsigned short lookup_swap_cgroup(swp_entry_t ent)
27a7faa0 KH	380	{
	381	int type = swp_type(ent);
	382	unsigned long offset = swp_offset(ent);
	383	unsigned long idx = offset / SC_PER_PAGE;
	384	unsigned long pos = offset & SC_POS_MASK;
	385	struct swap_cgroup_ctrl *ctrl;
	386	struct page *mappage;
	387	struct swap_cgroup *sc;
a3b2d692	388	unsigned short ret;
27a7faa0 KH	389
27a7faa0 KH	390	if (!do_swap_account)
a3b2d692	391	return 0;
27a7faa0 KH	392
	393	ctrl = &swap_cgroup_ctrl[type];
	394	mappage = ctrl->map[idx];
	395	sc = page_address(mappage);
	396	sc += pos;
a3b2d692	397	ret = sc->id;
27a7faa0 KH	398	return ret;
	399	}
	400
	401	int swap_cgroup_swapon(int type, unsigned long max_pages)
	402	{
	403	void *array;
	404	unsigned long array_size;
	405	unsigned long length;
	406	struct swap_cgroup_ctrl *ctrl;
	407
	408	if (!do_swap_account)
	409	return 0;
	410
	411	length = ((max_pages/SC_PER_PAGE) + 1);
	412	array_size = length * sizeof(void *);
	413
	414	array = vmalloc(array_size);
	415	if (!array)
	416	goto nomem;
	417
	418	memset(array, 0, array_size);
	419	ctrl = &swap_cgroup_ctrl[type];
	420	mutex_lock(&swap_cgroup_mutex);
	421	ctrl->length = length;
	422	ctrl->map = array;
	423	if (swap_cgroup_prepare(type)) {
	424	/* memory shortage */
	425	ctrl->map = NULL;
	426	ctrl->length = 0;
	427	vfree(array);
	428	mutex_unlock(&swap_cgroup_mutex);
	429	goto nomem;
	430	}
	431	mutex_unlock(&swap_cgroup_mutex);
	432
27a7faa0 KH	433	return 0;
	434	nomem:
	435	printk(KERN_INFO "couldn't allocate enough memory for swap_cgroup.\n");
	436	printk(KERN_INFO
	437	"swap_cgroup can be disabled by noswapaccount boot option\n");
	438	return -ENOMEM;
	439	}
	440
	441	void swap_cgroup_swapoff(int type)
	442	{
	443	int i;
	444	struct swap_cgroup_ctrl *ctrl;
	445
	446	if (!do_swap_account)
	447	return;
	448
	449	mutex_lock(&swap_cgroup_mutex);
	450	ctrl = &swap_cgroup_ctrl[type];
	451	if (ctrl->map) {
	452	for (i = 0; i < ctrl->length; i++) {
	453	struct page *page = ctrl->map[i];
	454	if (page)
	455	__free_page(page);
	456	}
	457	vfree(ctrl->map);
	458	ctrl->map = NULL;
	459	ctrl->length = 0;
	460	}
	461	mutex_unlock(&swap_cgroup_mutex);
	462	}
	463
	464	#endif