[net-next-2.6.git] / mm / truncate.c

/*
 * mm/truncate.c - code for taking down pages from address_spaces
 *
 * Copyright (C) 2002, Linus Torvalds
 *
 * 10Sep2002	akpm@zip.com.au
 *		Initial version.
 */

#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/module.h>
#include <linux/pagemap.h>
#include <linux/pagevec.h>
#include <linux/buffer_head.h>	/* grr. try_to_release_page,
				   do_invalidatepage */


static inline void truncate_partial_page(struct page *page, unsigned partial)
{
	memclear_highpage_flush(page, partial, PAGE_CACHE_SIZE-partial);
	if (PagePrivate(page))
		do_invalidatepage(page, partial);
}

/*
 * If truncate cannot remove the fs-private metadata from the page, the page
 * becomes anonymous.  It will be left on the LRU and may even be mapped into
 * user pagetables if we're racing with filemap_nopage().
 *
 * We need to bale out if page->mapping is no longer equal to the original
 * mapping.  This happens a) when the VM reclaimed the page while we waited on
 * its lock, b) when a concurrent invalidate_inode_pages got there first and
 * c) when tmpfs swizzles a page between a tmpfs inode and swapper_space.
 */
static void
truncate_complete_page(struct address_space *mapping, struct page *page)
{
	if (page->mapping != mapping)
		return;

	if (PagePrivate(page))
		do_invalidatepage(page, 0);

	clear_page_dirty(page);
	ClearPageUptodate(page);
	ClearPageMappedToDisk(page);
	remove_from_page_cache(page);
	page_cache_release(page);	/* pagecache ref */
}

/*
 * This is for invalidate_inode_pages().  That function can be called at
 * any time, and is not supposed to throw away dirty pages.  But pages can
 * be marked dirty at any time too.  So we re-check the dirtiness inside
 * ->tree_lock.  That provides exclusion against the __set_page_dirty
 * functions.
 *
 * Returns non-zero if the page was successfully invalidated.
 */
static int
invalidate_complete_page(struct address_space *mapping, struct page *page)
{
	if (page->mapping != mapping)
		return 0;

	if (PagePrivate(page) && !try_to_release_page(page, 0))
		return 0;

	write_lock_irq(&mapping->tree_lock);
	if (PageDirty(page)) {
		write_unlock_irq(&mapping->tree_lock);
		return 0;
	}

	BUG_ON(PagePrivate(page));
	__remove_from_page_cache(page);
	write_unlock_irq(&mapping->tree_lock);
	ClearPageUptodate(page);
	page_cache_release(page);	/* pagecache ref */
	return 1;
}

/**
 * truncate_inode_pages - truncate range of pages specified by start and
 * end byte offsets
 * @mapping: mapping to truncate
 * @lstart: offset from which to truncate
 * @lend: offset to which to truncate
 *
 * Truncate the page cache, removing the pages that are between
 * specified offsets (and zeroing out partial page
 * (if lstart is not page aligned)).
 *
 * Truncate takes two passes - the first pass is nonblocking.  It will not
 * block on page locks and it will not block on writeback.  The second pass
 * will wait.  This is to prevent as much IO as possible in the affected region.
 * The first pass will remove most pages, so the search cost of the second pass
 * is low.
 *
 * When looking at page->index outside the page lock we need to be careful to
 * copy it into a local to avoid races (it could change at any time).
 *
 * We pass down the cache-hot hint to the page freeing code.  Even if the
 * mapping is large, it is probably the case that the final pages are the most
 * recently touched, and freeing happens in ascending file offset order.
 */
void truncate_inode_pages_range(struct address_space *mapping,
				loff_t lstart, loff_t lend)
{
	const pgoff_t start = (lstart + PAGE_CACHE_SIZE-1) >> PAGE_CACHE_SHIFT;
	pgoff_t end;
	const unsigned partial = lstart & (PAGE_CACHE_SIZE - 1);
	struct pagevec pvec;
	pgoff_t next;
	int i;

	if (mapping->nrpages == 0)
		return;

	BUG_ON((lend & (PAGE_CACHE_SIZE - 1)) != (PAGE_CACHE_SIZE - 1));
	end = (lend >> PAGE_CACHE_SHIFT);

	pagevec_init(&pvec, 0);
	next = start;
	while (next <= end &&
	       pagevec_lookup(&pvec, mapping, next, PAGEVEC_SIZE)) {
		for (i = 0; i < pagevec_count(&pvec); i++) {
			struct page *page = pvec.pages[i];
			pgoff_t page_index = page->index;

			if (page_index > end) {
				next = page_index;
				break;
			}

			if (page_index > next)
				next = page_index;
			next++;
			if (TestSetPageLocked(page))
				continue;
			if (PageWriteback(page)) {
				unlock_page(page);
				continue;
			}
			truncate_complete_page(mapping, page);
			unlock_page(page);
		}
		pagevec_release(&pvec);
		cond_resched();
	}

	if (partial) {
		struct page *page = find_lock_page(mapping, start - 1);
		if (page) {
			wait_on_page_writeback(page);
			truncate_partial_page(page, partial);
			unlock_page(page);
			page_cache_release(page);
		}
	}

	next = start;
	for ( ; ; ) {
		cond_resched();
		if (!pagevec_lookup(&pvec, mapping, next, PAGEVEC_SIZE)) {
			if (next == start)
				break;
			next = start;
			continue;
		}
		if (pvec.pages[0]->index > end) {
			pagevec_release(&pvec);
			break;
		}
		for (i = 0; i < pagevec_count(&pvec); i++) {
			struct page *page = pvec.pages[i];

			if (page->index > end)
				break;
			lock_page(page);
			wait_on_page_writeback(page);
			if (page->index > next)
				next = page->index;
			next++;
			truncate_complete_page(mapping, page);
			unlock_page(page);
		}
		pagevec_release(&pvec);
	}
}
EXPORT_SYMBOL(truncate_inode_pages_range);

/**
 * truncate_inode_pages - truncate *all* the pages from an offset
 * @mapping: mapping to truncate
 * @lstart: offset from which to truncate
 *
 * Called under (and serialised by) inode->i_sem.
 */
void truncate_inode_pages(struct address_space *mapping, loff_t lstart)
{
	truncate_inode_pages_range(mapping, lstart, (loff_t)-1);
}
EXPORT_SYMBOL(truncate_inode_pages);

/**
 * invalidate_mapping_pages - Invalidate all the unlocked pages of one inode
 * @mapping: the address_space which holds the pages to invalidate
 * @start: the offset 'from' which to invalidate
 * @end: the offset 'to' which to invalidate (inclusive)
 *
 * This function only removes the unlocked pages, if you want to
 * remove all the pages of one inode, you must call truncate_inode_pages.
 *
 * invalidate_mapping_pages() will not block on IO activity. It will not
 * invalidate pages which are dirty, locked, under writeback or mapped into
 * pagetables.
 */
unsigned long invalidate_mapping_pages(struct address_space *mapping,
				pgoff_t start, pgoff_t end)
{
	struct pagevec pvec;
	pgoff_t next = start;
	unsigned long ret = 0;
	int i;

	pagevec_init(&pvec, 0);
	while (next <= end &&
			pagevec_lookup(&pvec, mapping, next, PAGEVEC_SIZE)) {
		for (i = 0; i < pagevec_count(&pvec); i++) {
			struct page *page = pvec.pages[i];

			if (TestSetPageLocked(page)) {
				next++;
				continue;
			}
			if (page->index > next)
				next = page->index;
			next++;
			if (PageDirty(page) || PageWriteback(page))
				goto unlock;
			if (page_mapped(page))
				goto unlock;
			ret += invalidate_complete_page(mapping, page);
unlock:
			unlock_page(page);
			if (next > end)
				break;
		}
		pagevec_release(&pvec);
		cond_resched();
	}
	return ret;
}

unsigned long invalidate_inode_pages(struct address_space *mapping)
{
	return invalidate_mapping_pages(mapping, 0, ~0UL);
}

EXPORT_SYMBOL(invalidate_inode_pages);

/**
 * invalidate_inode_pages2_range - remove range of pages from an address_space
 * @mapping: the address_space
 * @start: the page offset 'from' which to invalidate
 * @end: the page offset 'to' which to invalidate (inclusive)
 *
 * Any pages which are found to be mapped into pagetables are unmapped prior to
 * invalidation.
 *
 * Returns -EIO if any pages could not be invalidated.
 */
int invalidate_inode_pages2_range(struct address_space *mapping,
				  pgoff_t start, pgoff_t end)
{
	struct pagevec pvec;
	pgoff_t next;
	int i;
	int ret = 0;
	int did_range_unmap = 0;
	int wrapped = 0;

	pagevec_init(&pvec, 0);
	next = start;
	while (next <= end && !ret && !wrapped &&
		pagevec_lookup(&pvec, mapping, next,
			min(end - next, (pgoff_t)PAGEVEC_SIZE - 1) + 1)) {
		for (i = 0; !ret && i < pagevec_count(&pvec); i++) {
			struct page *page = pvec.pages[i];
			pgoff_t page_index;
			int was_dirty;

			lock_page(page);
			if (page->mapping != mapping) {
				unlock_page(page);
				continue;
			}
			page_index = page->index;
			next = page_index + 1;
			if (next == 0)
				wrapped = 1;
			if (page_index > end) {
				unlock_page(page);
				break;
			}
			wait_on_page_writeback(page);
			while (page_mapped(page)) {
				if (!did_range_unmap) {
					/*
					 * Zap the rest of the file in one hit.
					 */
					unmap_mapping_range(mapping,
					   (loff_t)page_index<<PAGE_CACHE_SHIFT,
					   (loff_t)(end - page_index + 1)
							<< PAGE_CACHE_SHIFT,
					    0);
					did_range_unmap = 1;
				} else {
					/*
					 * Just zap this page
					 */
					unmap_mapping_range(mapping,
					  (loff_t)page_index<<PAGE_CACHE_SHIFT,
					  PAGE_CACHE_SIZE, 0);
				}
			}
			was_dirty = test_clear_page_dirty(page);
			if (!invalidate_complete_page(mapping, page)) {
				if (was_dirty)
					set_page_dirty(page);
				ret = -EIO;
			}
			unlock_page(page);
		}
		pagevec_release(&pvec);
		cond_resched();
	}
	return ret;
}
EXPORT_SYMBOL_GPL(invalidate_inode_pages2_range);

/**
 * invalidate_inode_pages2 - remove all pages from an address_space
 * @mapping: the address_space
 *
 * Any pages which are found to be mapped into pagetables are unmapped prior to
 * invalidation.
 *
 * Returns -EIO if any pages could not be invalidated.
 */
int invalidate_inode_pages2(struct address_space *mapping)
{
	return invalidate_inode_pages2_range(mapping, 0, -1);
}
EXPORT_SYMBOL_GPL(invalidate_inode_pages2);
Commit	Line	Data
1da177e4 LT	1	/*
	2	* mm/truncate.c - code for taking down pages from address_spaces
	3	*
	4	* Copyright (C) 2002, Linus Torvalds
	5	*
	6	* 10Sep2002 akpm@zip.com.au
	7	* Initial version.
	8	*/
	9
	10	#include <linux/kernel.h>
	11	#include <linux/mm.h>
	12	#include <linux/module.h>
	13	#include <linux/pagemap.h>
	14	#include <linux/pagevec.h>
	15	#include <linux/buffer_head.h> /* grr. try_to_release_page,
aaa4059b	16	do_invalidatepage */
1da177e4 LT	17
1da177e4 LT	18
1da177e4 LT	19	static inline void truncate_partial_page(struct page *page, unsigned partial)
	20	{
	21	memclear_highpage_flush(page, partial, PAGE_CACHE_SIZE-partial);
	22	if (PagePrivate(page))
	23	do_invalidatepage(page, partial);
	24	}
	25
	26	/*
	27	* If truncate cannot remove the fs-private metadata from the page, the page
	28	* becomes anonymous. It will be left on the LRU and may even be mapped into
	29	* user pagetables if we're racing with filemap_nopage().
	30	*
	31	* We need to bale out if page->mapping is no longer equal to the original
	32	* mapping. This happens a) when the VM reclaimed the page while we waited on
	33	* its lock, b) when a concurrent invalidate_inode_pages got there first and
	34	* c) when tmpfs swizzles a page between a tmpfs inode and swapper_space.
	35	*/
	36	static void
	37	truncate_complete_page(struct address_space mapping, struct page page)
	38	{
	39	if (page->mapping != mapping)
	40	return;
	41
	42	if (PagePrivate(page))
	43	do_invalidatepage(page, 0);
	44
	45	clear_page_dirty(page);
	46	ClearPageUptodate(page);
	47	ClearPageMappedToDisk(page);
	48	remove_from_page_cache(page);
	49	page_cache_release(page); /* pagecache ref */
	50	}
	51
	52	/*
	53	* This is for invalidate_inode_pages(). That function can be called at
	54	* any time, and is not supposed to throw away dirty pages. But pages can
	55	* be marked dirty at any time too. So we re-check the dirtiness inside
	56	* ->tree_lock. That provides exclusion against the __set_page_dirty
	57	* functions.
	58	*
	59	* Returns non-zero if the page was successfully invalidated.
	60	*/
	61	static int
	62	invalidate_complete_page(struct address_space mapping, struct page page)
	63	{
	64	if (page->mapping != mapping)
	65	return 0;
	66
	67	if (PagePrivate(page) && !try_to_release_page(page, 0))
	68	return 0;
	69
	70	write_lock_irq(&mapping->tree_lock);
	71	if (PageDirty(page)) {
	72	write_unlock_irq(&mapping->tree_lock);
	73	return 0;
	74	}
	75
	76	BUG_ON(PagePrivate(page));
	77	__remove_from_page_cache(page);
	78	write_unlock_irq(&mapping->tree_lock);
	79	ClearPageUptodate(page);
	80	page_cache_release(page); /* pagecache ref */
	81	return 1;
	82	}
83
84	/**
d7339071 HR	85	* truncate_inode_pages - truncate range of pages specified by start and
d7339071 HR	86	* end byte offsets
1da177e4 LT	87	* @mapping: mapping to truncate
1da177e4 LT	88	* @lstart: offset from which to truncate
d7339071	89	* @lend: offset to which to truncate
1da177e4	90	*
d7339071 HR	91	* Truncate the page cache, removing the pages that are between
	92	* specified offsets (and zeroing out partial page
	93	* (if lstart is not page aligned)).
1da177e4 LT	94	*
	95	* Truncate takes two passes - the first pass is nonblocking. It will not
	96	* block on page locks and it will not block on writeback. The second pass
	97	* will wait. This is to prevent as much IO as possible in the affected region.
	98	* The first pass will remove most pages, so the search cost of the second pass
	99	* is low.
	100	*
	101	* When looking at page->index outside the page lock we need to be careful to
	102	* copy it into a local to avoid races (it could change at any time).
	103	*
	104	* We pass down the cache-hot hint to the page freeing code. Even if the
	105	* mapping is large, it is probably the case that the final pages are the most
	106	* recently touched, and freeing happens in ascending file offset order.
1da177e4	107	*/
d7339071 HR	108	void truncate_inode_pages_range(struct address_space *mapping,
d7339071 HR	109	loff_t lstart, loff_t lend)
1da177e4 LT	110	{
1da177e4 LT	111	const pgoff_t start = (lstart + PAGE_CACHE_SIZE-1) >> PAGE_CACHE_SHIFT;
d7339071	112	pgoff_t end;
1da177e4 LT	113	const unsigned partial = lstart & (PAGE_CACHE_SIZE - 1);
	114	struct pagevec pvec;
	115	pgoff_t next;
	116	int i;
	117
	118	if (mapping->nrpages == 0)
	119	return;
	120
d7339071 HR	121	BUG_ON((lend & (PAGE_CACHE_SIZE - 1)) != (PAGE_CACHE_SIZE - 1));
	122	end = (lend >> PAGE_CACHE_SHIFT);
	123
1da177e4 LT	124	pagevec_init(&pvec, 0);
1da177e4 LT	125	next = start;
d7339071 HR	126	while (next <= end &&
d7339071 HR	127	pagevec_lookup(&pvec, mapping, next, PAGEVEC_SIZE)) {
1da177e4 LT	128	for (i = 0; i < pagevec_count(&pvec); i++) {
	129	struct page *page = pvec.pages[i];
	130	pgoff_t page_index = page->index;
	131
d7339071 HR	132	if (page_index > end) {
	133	next = page_index;
	134	break;
	135	}
	136
1da177e4 LT	137	if (page_index > next)
	138	next = page_index;
	139	next++;
	140	if (TestSetPageLocked(page))
	141	continue;
	142	if (PageWriteback(page)) {
	143	unlock_page(page);
	144	continue;
	145	}
	146	truncate_complete_page(mapping, page);
	147	unlock_page(page);
	148	}
	149	pagevec_release(&pvec);
	150	cond_resched();
	151	}
	152
	153	if (partial) {
	154	struct page *page = find_lock_page(mapping, start - 1);
	155	if (page) {
	156	wait_on_page_writeback(page);
	157	truncate_partial_page(page, partial);
	158	unlock_page(page);
	159	page_cache_release(page);
	160	}
	161	}
	162
	163	next = start;
	164	for ( ; ; ) {
	165	cond_resched();
	166	if (!pagevec_lookup(&pvec, mapping, next, PAGEVEC_SIZE)) {
	167	if (next == start)
	168	break;
	169	next = start;
	170	continue;
	171	}
d7339071 HR	172	if (pvec.pages[0]->index > end) {
	173	pagevec_release(&pvec);
	174	break;
	175	}
1da177e4 LT	176	for (i = 0; i < pagevec_count(&pvec); i++) {
	177	struct page *page = pvec.pages[i];
	178
d7339071 HR	179	if (page->index > end)
d7339071 HR	180	break;
1da177e4 LT	181	lock_page(page);
	182	wait_on_page_writeback(page);
	183	if (page->index > next)
	184	next = page->index;
	185	next++;
	186	truncate_complete_page(mapping, page);
	187	unlock_page(page);
	188	}
	189	pagevec_release(&pvec);
	190	}
	191	}
d7339071	192	EXPORT_SYMBOL(truncate_inode_pages_range);
1da177e4	193
d7339071 HR	194	/**
	195	* truncate_inode_pages - truncate all the pages from an offset
	196	* @mapping: mapping to truncate
	197	* @lstart: offset from which to truncate
	198	*
	199	* Called under (and serialised by) inode->i_sem.
	200	*/
	201	void truncate_inode_pages(struct address_space *mapping, loff_t lstart)
	202	{
	203	truncate_inode_pages_range(mapping, lstart, (loff_t)-1);
	204	}
1da177e4 LT	205	EXPORT_SYMBOL(truncate_inode_pages);
	206
	207	/**
	208	* invalidate_mapping_pages - Invalidate all the unlocked pages of one inode
	209	* @mapping: the address_space which holds the pages to invalidate
	210	* @start: the offset 'from' which to invalidate
	211	* @end: the offset 'to' which to invalidate (inclusive)
	212	*
	213	* This function only removes the unlocked pages, if you want to
	214	* remove all the pages of one inode, you must call truncate_inode_pages.
	215	*
	216	* invalidate_mapping_pages() will not block on IO activity. It will not
	217	* invalidate pages which are dirty, locked, under writeback or mapped into
	218	* pagetables.
	219	*/
	220	unsigned long invalidate_mapping_pages(struct address_space *mapping,
	221	pgoff_t start, pgoff_t end)
	222	{
	223	struct pagevec pvec;
	224	pgoff_t next = start;
	225	unsigned long ret = 0;
	226	int i;
	227
	228	pagevec_init(&pvec, 0);
	229	while (next <= end &&
	230	pagevec_lookup(&pvec, mapping, next, PAGEVEC_SIZE)) {
	231	for (i = 0; i < pagevec_count(&pvec); i++) {
	232	struct page *page = pvec.pages[i];
	233
	234	if (TestSetPageLocked(page)) {
	235	next++;
	236	continue;
	237	}
	238	if (page->index > next)
	239	next = page->index;
	240	next++;
	241	if (PageDirty(page) \|\| PageWriteback(page))
	242	goto unlock;
	243	if (page_mapped(page))
	244	goto unlock;
	245	ret += invalidate_complete_page(mapping, page);
	246	unlock:
	247	unlock_page(page);
	248	if (next > end)
	249	break;
	250	}
	251	pagevec_release(&pvec);
	252	cond_resched();
	253	}
	254	return ret;
	255	}
	256
	257	unsigned long invalidate_inode_pages(struct address_space *mapping)
	258	{
	259	return invalidate_mapping_pages(mapping, 0, ~0UL);
	260	}
	261
	262	EXPORT_SYMBOL(invalidate_inode_pages);
	263
	264	/**
	265	* invalidate_inode_pages2_range - remove range of pages from an address_space
67be2dd1	266	* @mapping: the address_space
1da177e4 LT	267	* @start: the page offset 'from' which to invalidate
	268	* @end: the page offset 'to' which to invalidate (inclusive)
	269	*
	270	* Any pages which are found to be mapped into pagetables are unmapped prior to
	271	* invalidation.
	272	*
	273	* Returns -EIO if any pages could not be invalidated.
	274	*/
	275	int invalidate_inode_pages2_range(struct address_space *mapping,
	276	pgoff_t start, pgoff_t end)
	277	{
	278	struct pagevec pvec;
	279	pgoff_t next;
	280	int i;
	281	int ret = 0;
	282	int did_range_unmap = 0;
	283	int wrapped = 0;
	284
	285	pagevec_init(&pvec, 0);
	286	next = start;
	287	while (next <= end && !ret && !wrapped &&
	288	pagevec_lookup(&pvec, mapping, next,
	289	min(end - next, (pgoff_t)PAGEVEC_SIZE - 1) + 1)) {
	290	for (i = 0; !ret && i < pagevec_count(&pvec); i++) {
	291	struct page *page = pvec.pages[i];
	292	pgoff_t page_index;
	293	int was_dirty;
	294
	295	lock_page(page);
	296	if (page->mapping != mapping) {
	297	unlock_page(page);
	298	continue;
	299	}
	300	page_index = page->index;
	301	next = page_index + 1;
	302	if (next == 0)
	303	wrapped = 1;
	304	if (page_index > end) {
	305	unlock_page(page);
	306	break;
	307	}
	308	wait_on_page_writeback(page);
	309	while (page_mapped(page)) {
	310	if (!did_range_unmap) {
	311	/*
	312	* Zap the rest of the file in one hit.
	313	*/
	314	unmap_mapping_range(mapping,
479ef592 OD	315	(loff_t)page_index<<PAGE_CACHE_SHIFT,
479ef592 OD	316	(loff_t)(end - page_index + 1)
1da177e4 LT	317	<< PAGE_CACHE_SHIFT,
	318	0);
	319	did_range_unmap = 1;
	320	} else {
	321	/*
	322	* Just zap this page
	323	*/
	324	unmap_mapping_range(mapping,
479ef592	325	(loff_t)page_index<<PAGE_CACHE_SHIFT,
1da177e4 LT	326	PAGE_CACHE_SIZE, 0);
	327	}
	328	}
	329	was_dirty = test_clear_page_dirty(page);
	330	if (!invalidate_complete_page(mapping, page)) {
	331	if (was_dirty)
	332	set_page_dirty(page);
	333	ret = -EIO;
	334	}
	335	unlock_page(page);
	336	}
	337	pagevec_release(&pvec);
	338	cond_resched();
	339	}
	340	return ret;
	341	}
	342	EXPORT_SYMBOL_GPL(invalidate_inode_pages2_range);
	343
	344	/**
	345	* invalidate_inode_pages2 - remove all pages from an address_space
67be2dd1	346	* @mapping: the address_space
1da177e4 LT	347	*
	348	* Any pages which are found to be mapped into pagetables are unmapped prior to
	349	* invalidation.
	350	*
	351	* Returns -EIO if any pages could not be invalidated.
	352	*/
	353	int invalidate_inode_pages2(struct address_space *mapping)
	354	{
	355	return invalidate_inode_pages2_range(mapping, 0, -1);
	356	}
	357	EXPORT_SYMBOL_GPL(invalidate_inode_pages2);