]> bbs.cooldavid.org Git - net-next-2.6.git/blame - fs/buffer.c
lockdep: fixup checking of dir inode annotation
[net-next-2.6.git] / fs / buffer.c
CommitLineData
1da177e4
LT
1/*
2 * linux/fs/buffer.c
3 *
4 * Copyright (C) 1991, 1992, 2002 Linus Torvalds
5 */
6
7/*
8 * Start bdflush() with kernel_thread not syscall - Paul Gortmaker, 12/95
9 *
10 * Removed a lot of unnecessary code and simplified things now that
11 * the buffer cache isn't our primary cache - Andrew Tridgell 12/96
12 *
13 * Speed up hash, lru, and free list operations. Use gfp() for allocating
14 * hash table, use SLAB cache for buffer heads. SMP threading. -DaveM
15 *
16 * Added 32k buffer block sizes - these are required older ARM systems. - RMK
17 *
18 * async buffer flushing, 1999 Andrea Arcangeli <andrea@suse.de>
19 */
20
1da177e4
LT
21#include <linux/kernel.h>
22#include <linux/syscalls.h>
23#include <linux/fs.h>
24#include <linux/mm.h>
25#include <linux/percpu.h>
26#include <linux/slab.h>
16f7e0fe 27#include <linux/capability.h>
1da177e4
LT
28#include <linux/blkdev.h>
29#include <linux/file.h>
30#include <linux/quotaops.h>
31#include <linux/highmem.h>
32#include <linux/module.h>
33#include <linux/writeback.h>
34#include <linux/hash.h>
35#include <linux/suspend.h>
36#include <linux/buffer_head.h>
55e829af 37#include <linux/task_io_accounting_ops.h>
1da177e4
LT
38#include <linux/bio.h>
39#include <linux/notifier.h>
40#include <linux/cpu.h>
41#include <linux/bitops.h>
42#include <linux/mpage.h>
fb1c8f93 43#include <linux/bit_spinlock.h>
1da177e4
LT
44
45static int fsync_buffers_list(spinlock_t *lock, struct list_head *list);
1da177e4
LT
46
47#define BH_ENTRY(list) list_entry((list), struct buffer_head, b_assoc_buffers)
48
49inline void
50init_buffer(struct buffer_head *bh, bh_end_io_t *handler, void *private)
51{
52 bh->b_end_io = handler;
53 bh->b_private = private;
54}
1fe72eaa 55EXPORT_SYMBOL(init_buffer);
1da177e4
LT
56
57static int sync_buffer(void *word)
58{
59 struct block_device *bd;
60 struct buffer_head *bh
61 = container_of(word, struct buffer_head, b_state);
62
63 smp_mb();
64 bd = bh->b_bdev;
65 if (bd)
66 blk_run_address_space(bd->bd_inode->i_mapping);
67 io_schedule();
68 return 0;
69}
70
fc9b52cd 71void __lock_buffer(struct buffer_head *bh)
1da177e4
LT
72{
73 wait_on_bit_lock(&bh->b_state, BH_Lock, sync_buffer,
74 TASK_UNINTERRUPTIBLE);
75}
76EXPORT_SYMBOL(__lock_buffer);
77
fc9b52cd 78void unlock_buffer(struct buffer_head *bh)
1da177e4 79{
51b07fc3 80 clear_bit_unlock(BH_Lock, &bh->b_state);
1da177e4
LT
81 smp_mb__after_clear_bit();
82 wake_up_bit(&bh->b_state, BH_Lock);
83}
1fe72eaa 84EXPORT_SYMBOL(unlock_buffer);
1da177e4
LT
85
86/*
87 * Block until a buffer comes unlocked. This doesn't stop it
88 * from becoming locked again - you have to lock it yourself
89 * if you want to preserve its state.
90 */
91void __wait_on_buffer(struct buffer_head * bh)
92{
93 wait_on_bit(&bh->b_state, BH_Lock, sync_buffer, TASK_UNINTERRUPTIBLE);
94}
1fe72eaa 95EXPORT_SYMBOL(__wait_on_buffer);
1da177e4
LT
96
97static void
98__clear_page_buffers(struct page *page)
99{
100 ClearPagePrivate(page);
4c21e2f2 101 set_page_private(page, 0);
1da177e4
LT
102 page_cache_release(page);
103}
104
08bafc03
KM
105
106static int quiet_error(struct buffer_head *bh)
107{
108 if (!test_bit(BH_Quiet, &bh->b_state) && printk_ratelimit())
109 return 0;
110 return 1;
111}
112
113
1da177e4
LT
114static void buffer_io_error(struct buffer_head *bh)
115{
116 char b[BDEVNAME_SIZE];
1da177e4
LT
117 printk(KERN_ERR "Buffer I/O error on device %s, logical block %Lu\n",
118 bdevname(bh->b_bdev, b),
119 (unsigned long long)bh->b_blocknr);
120}
121
122/*
68671f35
DM
123 * End-of-IO handler helper function which does not touch the bh after
124 * unlocking it.
125 * Note: unlock_buffer() sort-of does touch the bh after unlocking it, but
126 * a race there is benign: unlock_buffer() only use the bh's address for
127 * hashing after unlocking the buffer, so it doesn't actually touch the bh
128 * itself.
1da177e4 129 */
68671f35 130static void __end_buffer_read_notouch(struct buffer_head *bh, int uptodate)
1da177e4
LT
131{
132 if (uptodate) {
133 set_buffer_uptodate(bh);
134 } else {
135 /* This happens, due to failed READA attempts. */
136 clear_buffer_uptodate(bh);
137 }
138 unlock_buffer(bh);
68671f35
DM
139}
140
141/*
142 * Default synchronous end-of-IO handler.. Just mark it up-to-date and
143 * unlock the buffer. This is what ll_rw_block uses too.
144 */
145void end_buffer_read_sync(struct buffer_head *bh, int uptodate)
146{
147 __end_buffer_read_notouch(bh, uptodate);
1da177e4
LT
148 put_bh(bh);
149}
1fe72eaa 150EXPORT_SYMBOL(end_buffer_read_sync);
1da177e4
LT
151
152void end_buffer_write_sync(struct buffer_head *bh, int uptodate)
153{
154 char b[BDEVNAME_SIZE];
155
156 if (uptodate) {
157 set_buffer_uptodate(bh);
158 } else {
0edd55fa 159 if (!quiet_error(bh)) {
1da177e4
LT
160 buffer_io_error(bh);
161 printk(KERN_WARNING "lost page write due to "
162 "I/O error on %s\n",
163 bdevname(bh->b_bdev, b));
164 }
165 set_buffer_write_io_error(bh);
166 clear_buffer_uptodate(bh);
167 }
168 unlock_buffer(bh);
169 put_bh(bh);
170}
1fe72eaa 171EXPORT_SYMBOL(end_buffer_write_sync);
1da177e4 172
1da177e4
LT
173/*
174 * Various filesystems appear to want __find_get_block to be non-blocking.
175 * But it's the page lock which protects the buffers. To get around this,
176 * we get exclusion from try_to_free_buffers with the blockdev mapping's
177 * private_lock.
178 *
179 * Hack idea: for the blockdev mapping, i_bufferlist_lock contention
180 * may be quite high. This code could TryLock the page, and if that
181 * succeeds, there is no need to take private_lock. (But if
182 * private_lock is contended then so is mapping->tree_lock).
183 */
184static struct buffer_head *
385fd4c5 185__find_get_block_slow(struct block_device *bdev, sector_t block)
1da177e4
LT
186{
187 struct inode *bd_inode = bdev->bd_inode;
188 struct address_space *bd_mapping = bd_inode->i_mapping;
189 struct buffer_head *ret = NULL;
190 pgoff_t index;
191 struct buffer_head *bh;
192 struct buffer_head *head;
193 struct page *page;
194 int all_mapped = 1;
195
196 index = block >> (PAGE_CACHE_SHIFT - bd_inode->i_blkbits);
197 page = find_get_page(bd_mapping, index);
198 if (!page)
199 goto out;
200
201 spin_lock(&bd_mapping->private_lock);
202 if (!page_has_buffers(page))
203 goto out_unlock;
204 head = page_buffers(page);
205 bh = head;
206 do {
97f76d3d
NK
207 if (!buffer_mapped(bh))
208 all_mapped = 0;
209 else if (bh->b_blocknr == block) {
1da177e4
LT
210 ret = bh;
211 get_bh(bh);
212 goto out_unlock;
213 }
1da177e4
LT
214 bh = bh->b_this_page;
215 } while (bh != head);
216
217 /* we might be here because some of the buffers on this page are
218 * not mapped. This is due to various races between
219 * file io on the block device and getblk. It gets dealt with
220 * elsewhere, don't buffer_error if we had some unmapped buffers
221 */
222 if (all_mapped) {
223 printk("__find_get_block_slow() failed. "
224 "block=%llu, b_blocknr=%llu\n",
205f87f6
BP
225 (unsigned long long)block,
226 (unsigned long long)bh->b_blocknr);
227 printk("b_state=0x%08lx, b_size=%zu\n",
228 bh->b_state, bh->b_size);
1da177e4
LT
229 printk("device blocksize: %d\n", 1 << bd_inode->i_blkbits);
230 }
231out_unlock:
232 spin_unlock(&bd_mapping->private_lock);
233 page_cache_release(page);
234out:
235 return ret;
236}
237
238/* If invalidate_buffers() will trash dirty buffers, it means some kind
239 of fs corruption is going on. Trashing dirty data always imply losing
240 information that was supposed to be just stored on the physical layer
241 by the user.
242
243 Thus invalidate_buffers in general usage is not allwowed to trash
244 dirty buffers. For example ioctl(FLSBLKBUF) expects dirty data to
245 be preserved. These buffers are simply skipped.
246
247 We also skip buffers which are still in use. For example this can
248 happen if a userspace program is reading the block device.
249
250 NOTE: In the case where the user removed a removable-media-disk even if
251 there's still dirty data not synced on disk (due a bug in the device driver
252 or due an error of the user), by not destroying the dirty buffers we could
253 generate corruption also on the next media inserted, thus a parameter is
254 necessary to handle this case in the most safe way possible (trying
255 to not corrupt also the new disk inserted with the data belonging to
256 the old now corrupted disk). Also for the ramdisk the natural thing
257 to do in order to release the ramdisk memory is to destroy dirty buffers.
258
259 These are two special cases. Normal usage imply the device driver
260 to issue a sync on the device (without waiting I/O completion) and
261 then an invalidate_buffers call that doesn't trash dirty buffers.
262
263 For handling cache coherency with the blkdev pagecache the 'update' case
264 is been introduced. It is needed to re-read from disk any pinned
265 buffer. NOTE: re-reading from disk is destructive so we can do it only
266 when we assume nobody is changing the buffercache under our I/O and when
267 we think the disk contains more recent information than the buffercache.
268 The update == 1 pass marks the buffers we need to update, the update == 2
269 pass does the actual I/O. */
f98393a6 270void invalidate_bdev(struct block_device *bdev)
1da177e4 271{
0e1dfc66
AM
272 struct address_space *mapping = bdev->bd_inode->i_mapping;
273
274 if (mapping->nrpages == 0)
275 return;
276
1da177e4 277 invalidate_bh_lrus();
fa4b9074 278 lru_add_drain_all(); /* make sure all lru add caches are flushed */
fc0ecff6 279 invalidate_mapping_pages(mapping, 0, -1);
1da177e4 280}
1fe72eaa 281EXPORT_SYMBOL(invalidate_bdev);
1da177e4
LT
282
283/*
5b0830cb 284 * Kick the writeback threads then try to free up some ZONE_NORMAL memory.
1da177e4
LT
285 */
286static void free_more_memory(void)
287{
19770b32 288 struct zone *zone;
0e88460d 289 int nid;
1da177e4 290
03ba3782 291 wakeup_flusher_threads(1024);
1da177e4
LT
292 yield();
293
0e88460d 294 for_each_online_node(nid) {
19770b32
MG
295 (void)first_zones_zonelist(node_zonelist(nid, GFP_NOFS),
296 gfp_zone(GFP_NOFS), NULL,
297 &zone);
298 if (zone)
54a6eb5c 299 try_to_free_pages(node_zonelist(nid, GFP_NOFS), 0,
327c0e96 300 GFP_NOFS, NULL);
1da177e4
LT
301 }
302}
303
304/*
305 * I/O completion handler for block_read_full_page() - pages
306 * which come unlocked at the end of I/O.
307 */
308static void end_buffer_async_read(struct buffer_head *bh, int uptodate)
309{
1da177e4 310 unsigned long flags;
a3972203 311 struct buffer_head *first;
1da177e4
LT
312 struct buffer_head *tmp;
313 struct page *page;
314 int page_uptodate = 1;
315
316 BUG_ON(!buffer_async_read(bh));
317
318 page = bh->b_page;
319 if (uptodate) {
320 set_buffer_uptodate(bh);
321 } else {
322 clear_buffer_uptodate(bh);
08bafc03 323 if (!quiet_error(bh))
1da177e4
LT
324 buffer_io_error(bh);
325 SetPageError(page);
326 }
327
328 /*
329 * Be _very_ careful from here on. Bad things can happen if
330 * two buffer heads end IO at almost the same time and both
331 * decide that the page is now completely done.
332 */
a3972203
NP
333 first = page_buffers(page);
334 local_irq_save(flags);
335 bit_spin_lock(BH_Uptodate_Lock, &first->b_state);
1da177e4
LT
336 clear_buffer_async_read(bh);
337 unlock_buffer(bh);
338 tmp = bh;
339 do {
340 if (!buffer_uptodate(tmp))
341 page_uptodate = 0;
342 if (buffer_async_read(tmp)) {
343 BUG_ON(!buffer_locked(tmp));
344 goto still_busy;
345 }
346 tmp = tmp->b_this_page;
347 } while (tmp != bh);
a3972203
NP
348 bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
349 local_irq_restore(flags);
1da177e4
LT
350
351 /*
352 * If none of the buffers had errors and they are all
353 * uptodate then we can set the page uptodate.
354 */
355 if (page_uptodate && !PageError(page))
356 SetPageUptodate(page);
357 unlock_page(page);
358 return;
359
360still_busy:
a3972203
NP
361 bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
362 local_irq_restore(flags);
1da177e4
LT
363 return;
364}
365
366/*
367 * Completion handler for block_write_full_page() - pages which are unlocked
368 * during I/O, and which have PageWriteback cleared upon I/O completion.
369 */
35c80d5f 370void end_buffer_async_write(struct buffer_head *bh, int uptodate)
1da177e4
LT
371{
372 char b[BDEVNAME_SIZE];
1da177e4 373 unsigned long flags;
a3972203 374 struct buffer_head *first;
1da177e4
LT
375 struct buffer_head *tmp;
376 struct page *page;
377
378 BUG_ON(!buffer_async_write(bh));
379
380 page = bh->b_page;
381 if (uptodate) {
382 set_buffer_uptodate(bh);
383 } else {
08bafc03 384 if (!quiet_error(bh)) {
1da177e4
LT
385 buffer_io_error(bh);
386 printk(KERN_WARNING "lost page write due to "
387 "I/O error on %s\n",
388 bdevname(bh->b_bdev, b));
389 }
390 set_bit(AS_EIO, &page->mapping->flags);
58ff407b 391 set_buffer_write_io_error(bh);
1da177e4
LT
392 clear_buffer_uptodate(bh);
393 SetPageError(page);
394 }
395
a3972203
NP
396 first = page_buffers(page);
397 local_irq_save(flags);
398 bit_spin_lock(BH_Uptodate_Lock, &first->b_state);
399
1da177e4
LT
400 clear_buffer_async_write(bh);
401 unlock_buffer(bh);
402 tmp = bh->b_this_page;
403 while (tmp != bh) {
404 if (buffer_async_write(tmp)) {
405 BUG_ON(!buffer_locked(tmp));
406 goto still_busy;
407 }
408 tmp = tmp->b_this_page;
409 }
a3972203
NP
410 bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
411 local_irq_restore(flags);
1da177e4
LT
412 end_page_writeback(page);
413 return;
414
415still_busy:
a3972203
NP
416 bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
417 local_irq_restore(flags);
1da177e4
LT
418 return;
419}
1fe72eaa 420EXPORT_SYMBOL(end_buffer_async_write);
1da177e4
LT
421
422/*
423 * If a page's buffers are under async readin (end_buffer_async_read
424 * completion) then there is a possibility that another thread of
425 * control could lock one of the buffers after it has completed
426 * but while some of the other buffers have not completed. This
427 * locked buffer would confuse end_buffer_async_read() into not unlocking
428 * the page. So the absence of BH_Async_Read tells end_buffer_async_read()
429 * that this buffer is not under async I/O.
430 *
431 * The page comes unlocked when it has no locked buffer_async buffers
432 * left.
433 *
434 * PageLocked prevents anyone starting new async I/O reads any of
435 * the buffers.
436 *
437 * PageWriteback is used to prevent simultaneous writeout of the same
438 * page.
439 *
440 * PageLocked prevents anyone from starting writeback of a page which is
441 * under read I/O (PageWriteback is only ever set against a locked page).
442 */
443static void mark_buffer_async_read(struct buffer_head *bh)
444{
445 bh->b_end_io = end_buffer_async_read;
446 set_buffer_async_read(bh);
447}
448
1fe72eaa
HS
449static void mark_buffer_async_write_endio(struct buffer_head *bh,
450 bh_end_io_t *handler)
1da177e4 451{
35c80d5f 452 bh->b_end_io = handler;
1da177e4
LT
453 set_buffer_async_write(bh);
454}
35c80d5f
CM
455
456void mark_buffer_async_write(struct buffer_head *bh)
457{
458 mark_buffer_async_write_endio(bh, end_buffer_async_write);
459}
1da177e4
LT
460EXPORT_SYMBOL(mark_buffer_async_write);
461
462
463/*
464 * fs/buffer.c contains helper functions for buffer-backed address space's
465 * fsync functions. A common requirement for buffer-based filesystems is
466 * that certain data from the backing blockdev needs to be written out for
467 * a successful fsync(). For example, ext2 indirect blocks need to be
468 * written back and waited upon before fsync() returns.
469 *
470 * The functions mark_buffer_inode_dirty(), fsync_inode_buffers(),
471 * inode_has_buffers() and invalidate_inode_buffers() are provided for the
472 * management of a list of dependent buffers at ->i_mapping->private_list.
473 *
474 * Locking is a little subtle: try_to_free_buffers() will remove buffers
475 * from their controlling inode's queue when they are being freed. But
476 * try_to_free_buffers() will be operating against the *blockdev* mapping
477 * at the time, not against the S_ISREG file which depends on those buffers.
478 * So the locking for private_list is via the private_lock in the address_space
479 * which backs the buffers. Which is different from the address_space
480 * against which the buffers are listed. So for a particular address_space,
481 * mapping->private_lock does *not* protect mapping->private_list! In fact,
482 * mapping->private_list will always be protected by the backing blockdev's
483 * ->private_lock.
484 *
485 * Which introduces a requirement: all buffers on an address_space's
486 * ->private_list must be from the same address_space: the blockdev's.
487 *
488 * address_spaces which do not place buffers at ->private_list via these
489 * utility functions are free to use private_lock and private_list for
490 * whatever they want. The only requirement is that list_empty(private_list)
491 * be true at clear_inode() time.
492 *
493 * FIXME: clear_inode should not call invalidate_inode_buffers(). The
494 * filesystems should do that. invalidate_inode_buffers() should just go
495 * BUG_ON(!list_empty).
496 *
497 * FIXME: mark_buffer_dirty_inode() is a data-plane operation. It should
498 * take an address_space, not an inode. And it should be called
499 * mark_buffer_dirty_fsync() to clearly define why those buffers are being
500 * queued up.
501 *
502 * FIXME: mark_buffer_dirty_inode() doesn't need to add the buffer to the
503 * list if it is already on a list. Because if the buffer is on a list,
504 * it *must* already be on the right one. If not, the filesystem is being
505 * silly. This will save a ton of locking. But first we have to ensure
506 * that buffers are taken *off* the old inode's list when they are freed
507 * (presumably in truncate). That requires careful auditing of all
508 * filesystems (do it inside bforget()). It could also be done by bringing
509 * b_inode back.
510 */
511
512/*
513 * The buffer's backing address_space's private_lock must be held
514 */
dbacefc9 515static void __remove_assoc_queue(struct buffer_head *bh)
1da177e4
LT
516{
517 list_del_init(&bh->b_assoc_buffers);
58ff407b
JK
518 WARN_ON(!bh->b_assoc_map);
519 if (buffer_write_io_error(bh))
520 set_bit(AS_EIO, &bh->b_assoc_map->flags);
521 bh->b_assoc_map = NULL;
1da177e4
LT
522}
523
524int inode_has_buffers(struct inode *inode)
525{
526 return !list_empty(&inode->i_data.private_list);
527}
528
529/*
530 * osync is designed to support O_SYNC io. It waits synchronously for
531 * all already-submitted IO to complete, but does not queue any new
532 * writes to the disk.
533 *
534 * To do O_SYNC writes, just queue the buffer writes with ll_rw_block as
535 * you dirty the buffers, and then use osync_inode_buffers to wait for
536 * completion. Any other dirty buffers which are not yet queued for
537 * write will not be flushed to disk by the osync.
538 */
539static int osync_buffers_list(spinlock_t *lock, struct list_head *list)
540{
541 struct buffer_head *bh;
542 struct list_head *p;
543 int err = 0;
544
545 spin_lock(lock);
546repeat:
547 list_for_each_prev(p, list) {
548 bh = BH_ENTRY(p);
549 if (buffer_locked(bh)) {
550 get_bh(bh);
551 spin_unlock(lock);
552 wait_on_buffer(bh);
553 if (!buffer_uptodate(bh))
554 err = -EIO;
555 brelse(bh);
556 spin_lock(lock);
557 goto repeat;
558 }
559 }
560 spin_unlock(lock);
561 return err;
562}
563
01a05b33 564static void do_thaw_one(struct super_block *sb, void *unused)
c2d75438 565{
c2d75438 566 char b[BDEVNAME_SIZE];
01a05b33
AV
567 while (sb->s_bdev && !thaw_bdev(sb->s_bdev, sb))
568 printk(KERN_WARNING "Emergency Thaw on %s\n",
569 bdevname(sb->s_bdev, b));
570}
c2d75438 571
01a05b33
AV
572static void do_thaw_all(struct work_struct *work)
573{
574 iterate_supers(do_thaw_one, NULL);
053c525f 575 kfree(work);
c2d75438
ES
576 printk(KERN_WARNING "Emergency Thaw complete\n");
577}
578
579/**
580 * emergency_thaw_all -- forcibly thaw every frozen filesystem
581 *
582 * Used for emergency unfreeze of all filesystems via SysRq
583 */
584void emergency_thaw_all(void)
585{
053c525f
JA
586 struct work_struct *work;
587
588 work = kmalloc(sizeof(*work), GFP_ATOMIC);
589 if (work) {
590 INIT_WORK(work, do_thaw_all);
591 schedule_work(work);
592 }
c2d75438
ES
593}
594
1da177e4 595/**
78a4a50a 596 * sync_mapping_buffers - write out & wait upon a mapping's "associated" buffers
67be2dd1 597 * @mapping: the mapping which wants those buffers written
1da177e4
LT
598 *
599 * Starts I/O against the buffers at mapping->private_list, and waits upon
600 * that I/O.
601 *
67be2dd1
MW
602 * Basically, this is a convenience function for fsync().
603 * @mapping is a file or directory which needs those buffers to be written for
604 * a successful fsync().
1da177e4
LT
605 */
606int sync_mapping_buffers(struct address_space *mapping)
607{
608 struct address_space *buffer_mapping = mapping->assoc_mapping;
609
610 if (buffer_mapping == NULL || list_empty(&mapping->private_list))
611 return 0;
612
613 return fsync_buffers_list(&buffer_mapping->private_lock,
614 &mapping->private_list);
615}
616EXPORT_SYMBOL(sync_mapping_buffers);
617
618/*
619 * Called when we've recently written block `bblock', and it is known that
620 * `bblock' was for a buffer_boundary() buffer. This means that the block at
621 * `bblock + 1' is probably a dirty indirect block. Hunt it down and, if it's
622 * dirty, schedule it for IO. So that indirects merge nicely with their data.
623 */
624void write_boundary_block(struct block_device *bdev,
625 sector_t bblock, unsigned blocksize)
626{
627 struct buffer_head *bh = __find_get_block(bdev, bblock + 1, blocksize);
628 if (bh) {
629 if (buffer_dirty(bh))
630 ll_rw_block(WRITE, 1, &bh);
631 put_bh(bh);
632 }
633}
634
635void mark_buffer_dirty_inode(struct buffer_head *bh, struct inode *inode)
636{
637 struct address_space *mapping = inode->i_mapping;
638 struct address_space *buffer_mapping = bh->b_page->mapping;
639
640 mark_buffer_dirty(bh);
641 if (!mapping->assoc_mapping) {
642 mapping->assoc_mapping = buffer_mapping;
643 } else {
e827f923 644 BUG_ON(mapping->assoc_mapping != buffer_mapping);
1da177e4 645 }
535ee2fb 646 if (!bh->b_assoc_map) {
1da177e4
LT
647 spin_lock(&buffer_mapping->private_lock);
648 list_move_tail(&bh->b_assoc_buffers,
649 &mapping->private_list);
58ff407b 650 bh->b_assoc_map = mapping;
1da177e4
LT
651 spin_unlock(&buffer_mapping->private_lock);
652 }
653}
654EXPORT_SYMBOL(mark_buffer_dirty_inode);
655
787d2214
NP
656/*
657 * Mark the page dirty, and set it dirty in the radix tree, and mark the inode
658 * dirty.
659 *
660 * If warn is true, then emit a warning if the page is not uptodate and has
661 * not been truncated.
662 */
a8e7d49a 663static void __set_page_dirty(struct page *page,
787d2214
NP
664 struct address_space *mapping, int warn)
665{
19fd6231 666 spin_lock_irq(&mapping->tree_lock);
787d2214
NP
667 if (page->mapping) { /* Race with truncate? */
668 WARN_ON_ONCE(warn && !PageUptodate(page));
e3a7cca1 669 account_page_dirtied(page, mapping);
787d2214
NP
670 radix_tree_tag_set(&mapping->page_tree,
671 page_index(page), PAGECACHE_TAG_DIRTY);
672 }
19fd6231 673 spin_unlock_irq(&mapping->tree_lock);
787d2214 674 __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
787d2214
NP
675}
676
1da177e4
LT
677/*
678 * Add a page to the dirty page list.
679 *
680 * It is a sad fact of life that this function is called from several places
681 * deeply under spinlocking. It may not sleep.
682 *
683 * If the page has buffers, the uptodate buffers are set dirty, to preserve
684 * dirty-state coherency between the page and the buffers. It the page does
685 * not have buffers then when they are later attached they will all be set
686 * dirty.
687 *
688 * The buffers are dirtied before the page is dirtied. There's a small race
689 * window in which a writepage caller may see the page cleanness but not the
690 * buffer dirtiness. That's fine. If this code were to set the page dirty
691 * before the buffers, a concurrent writepage caller could clear the page dirty
692 * bit, see a bunch of clean buffers and we'd end up with dirty buffers/clean
693 * page on the dirty page list.
694 *
695 * We use private_lock to lock against try_to_free_buffers while using the
696 * page's buffer list. Also use this to protect against clean buffers being
697 * added to the page after it was set dirty.
698 *
699 * FIXME: may need to call ->reservepage here as well. That's rather up to the
700 * address_space though.
701 */
702int __set_page_dirty_buffers(struct page *page)
703{
a8e7d49a 704 int newly_dirty;
787d2214 705 struct address_space *mapping = page_mapping(page);
ebf7a227
NP
706
707 if (unlikely(!mapping))
708 return !TestSetPageDirty(page);
1da177e4
LT
709
710 spin_lock(&mapping->private_lock);
711 if (page_has_buffers(page)) {
712 struct buffer_head *head = page_buffers(page);
713 struct buffer_head *bh = head;
714
715 do {
716 set_buffer_dirty(bh);
717 bh = bh->b_this_page;
718 } while (bh != head);
719 }
a8e7d49a 720 newly_dirty = !TestSetPageDirty(page);
1da177e4
LT
721 spin_unlock(&mapping->private_lock);
722
a8e7d49a
LT
723 if (newly_dirty)
724 __set_page_dirty(page, mapping, 1);
725 return newly_dirty;
1da177e4
LT
726}
727EXPORT_SYMBOL(__set_page_dirty_buffers);
728
729/*
730 * Write out and wait upon a list of buffers.
731 *
732 * We have conflicting pressures: we want to make sure that all
733 * initially dirty buffers get waited on, but that any subsequently
734 * dirtied buffers don't. After all, we don't want fsync to last
735 * forever if somebody is actively writing to the file.
736 *
737 * Do this in two main stages: first we copy dirty buffers to a
738 * temporary inode list, queueing the writes as we go. Then we clean
739 * up, waiting for those writes to complete.
740 *
741 * During this second stage, any subsequent updates to the file may end
742 * up refiling the buffer on the original inode's dirty list again, so
743 * there is a chance we will end up with a buffer queued for write but
744 * not yet completed on that list. So, as a final cleanup we go through
745 * the osync code to catch these locked, dirty buffers without requeuing
746 * any newly dirty buffers for write.
747 */
748static int fsync_buffers_list(spinlock_t *lock, struct list_head *list)
749{
750 struct buffer_head *bh;
751 struct list_head tmp;
9cf6b720 752 struct address_space *mapping, *prev_mapping = NULL;
1da177e4
LT
753 int err = 0, err2;
754
755 INIT_LIST_HEAD(&tmp);
756
757 spin_lock(lock);
758 while (!list_empty(list)) {
759 bh = BH_ENTRY(list->next);
535ee2fb 760 mapping = bh->b_assoc_map;
58ff407b 761 __remove_assoc_queue(bh);
535ee2fb
JK
762 /* Avoid race with mark_buffer_dirty_inode() which does
763 * a lockless check and we rely on seeing the dirty bit */
764 smp_mb();
1da177e4
LT
765 if (buffer_dirty(bh) || buffer_locked(bh)) {
766 list_add(&bh->b_assoc_buffers, &tmp);
535ee2fb 767 bh->b_assoc_map = mapping;
1da177e4
LT
768 if (buffer_dirty(bh)) {
769 get_bh(bh);
770 spin_unlock(lock);
771 /*
772 * Ensure any pending I/O completes so that
9cb569d6
CH
773 * write_dirty_buffer() actually writes the
774 * current contents - it is a noop if I/O is
775 * still in flight on potentially older
776 * contents.
1da177e4 777 */
9cb569d6 778 write_dirty_buffer(bh, WRITE_SYNC_PLUG);
9cf6b720
JA
779
780 /*
781 * Kick off IO for the previous mapping. Note
782 * that we will not run the very last mapping,
783 * wait_on_buffer() will do that for us
784 * through sync_buffer().
785 */
786 if (prev_mapping && prev_mapping != mapping)
787 blk_run_address_space(prev_mapping);
788 prev_mapping = mapping;
789
1da177e4
LT
790 brelse(bh);
791 spin_lock(lock);
792 }
793 }
794 }
795
796 while (!list_empty(&tmp)) {
797 bh = BH_ENTRY(tmp.prev);
1da177e4 798 get_bh(bh);
535ee2fb
JK
799 mapping = bh->b_assoc_map;
800 __remove_assoc_queue(bh);
801 /* Avoid race with mark_buffer_dirty_inode() which does
802 * a lockless check and we rely on seeing the dirty bit */
803 smp_mb();
804 if (buffer_dirty(bh)) {
805 list_add(&bh->b_assoc_buffers,
e3892296 806 &mapping->private_list);
535ee2fb
JK
807 bh->b_assoc_map = mapping;
808 }
1da177e4
LT
809 spin_unlock(lock);
810 wait_on_buffer(bh);
811 if (!buffer_uptodate(bh))
812 err = -EIO;
813 brelse(bh);
814 spin_lock(lock);
815 }
816
817 spin_unlock(lock);
818 err2 = osync_buffers_list(lock, list);
819 if (err)
820 return err;
821 else
822 return err2;
823}
824
825/*
826 * Invalidate any and all dirty buffers on a given inode. We are
827 * probably unmounting the fs, but that doesn't mean we have already
828 * done a sync(). Just drop the buffers from the inode list.
829 *
830 * NOTE: we take the inode's blockdev's mapping's private_lock. Which
831 * assumes that all the buffers are against the blockdev. Not true
832 * for reiserfs.
833 */
834void invalidate_inode_buffers(struct inode *inode)
835{
836 if (inode_has_buffers(inode)) {
837 struct address_space *mapping = &inode->i_data;
838 struct list_head *list = &mapping->private_list;
839 struct address_space *buffer_mapping = mapping->assoc_mapping;
840
841 spin_lock(&buffer_mapping->private_lock);
842 while (!list_empty(list))
843 __remove_assoc_queue(BH_ENTRY(list->next));
844 spin_unlock(&buffer_mapping->private_lock);
845 }
846}
52b19ac9 847EXPORT_SYMBOL(invalidate_inode_buffers);
1da177e4
LT
848
849/*
850 * Remove any clean buffers from the inode's buffer list. This is called
851 * when we're trying to free the inode itself. Those buffers can pin it.
852 *
853 * Returns true if all buffers were removed.
854 */
855int remove_inode_buffers(struct inode *inode)
856{
857 int ret = 1;
858
859 if (inode_has_buffers(inode)) {
860 struct address_space *mapping = &inode->i_data;
861 struct list_head *list = &mapping->private_list;
862 struct address_space *buffer_mapping = mapping->assoc_mapping;
863
864 spin_lock(&buffer_mapping->private_lock);
865 while (!list_empty(list)) {
866 struct buffer_head *bh = BH_ENTRY(list->next);
867 if (buffer_dirty(bh)) {
868 ret = 0;
869 break;
870 }
871 __remove_assoc_queue(bh);
872 }
873 spin_unlock(&buffer_mapping->private_lock);
874 }
875 return ret;
876}
877
878/*
879 * Create the appropriate buffers when given a page for data area and
880 * the size of each buffer.. Use the bh->b_this_page linked list to
881 * follow the buffers created. Return NULL if unable to create more
882 * buffers.
883 *
884 * The retry flag is used to differentiate async IO (paging, swapping)
885 * which may not fail from ordinary buffer allocations.
886 */
887struct buffer_head *alloc_page_buffers(struct page *page, unsigned long size,
888 int retry)
889{
890 struct buffer_head *bh, *head;
891 long offset;
892
893try_again:
894 head = NULL;
895 offset = PAGE_SIZE;
896 while ((offset -= size) >= 0) {
897 bh = alloc_buffer_head(GFP_NOFS);
898 if (!bh)
899 goto no_grow;
900
901 bh->b_bdev = NULL;
902 bh->b_this_page = head;
903 bh->b_blocknr = -1;
904 head = bh;
905
906 bh->b_state = 0;
907 atomic_set(&bh->b_count, 0);
908 bh->b_size = size;
909
910 /* Link the buffer to its page */
911 set_bh_page(bh, page, offset);
912
01ffe339 913 init_buffer(bh, NULL, NULL);
1da177e4
LT
914 }
915 return head;
916/*
917 * In case anything failed, we just free everything we got.
918 */
919no_grow:
920 if (head) {
921 do {
922 bh = head;
923 head = head->b_this_page;
924 free_buffer_head(bh);
925 } while (head);
926 }
927
928 /*
929 * Return failure for non-async IO requests. Async IO requests
930 * are not allowed to fail, so we have to wait until buffer heads
931 * become available. But we don't want tasks sleeping with
932 * partially complete buffers, so all were released above.
933 */
934 if (!retry)
935 return NULL;
936
937 /* We're _really_ low on memory. Now we just
938 * wait for old buffer heads to become free due to
939 * finishing IO. Since this is an async request and
940 * the reserve list is empty, we're sure there are
941 * async buffer heads in use.
942 */
943 free_more_memory();
944 goto try_again;
945}
946EXPORT_SYMBOL_GPL(alloc_page_buffers);
947
948static inline void
949link_dev_buffers(struct page *page, struct buffer_head *head)
950{
951 struct buffer_head *bh, *tail;
952
953 bh = head;
954 do {
955 tail = bh;
956 bh = bh->b_this_page;
957 } while (bh);
958 tail->b_this_page = head;
959 attach_page_buffers(page, head);
960}
961
962/*
963 * Initialise the state of a blockdev page's buffers.
964 */
965static void
966init_page_buffers(struct page *page, struct block_device *bdev,
967 sector_t block, int size)
968{
969 struct buffer_head *head = page_buffers(page);
970 struct buffer_head *bh = head;
971 int uptodate = PageUptodate(page);
972
973 do {
974 if (!buffer_mapped(bh)) {
975 init_buffer(bh, NULL, NULL);
976 bh->b_bdev = bdev;
977 bh->b_blocknr = block;
978 if (uptodate)
979 set_buffer_uptodate(bh);
980 set_buffer_mapped(bh);
981 }
982 block++;
983 bh = bh->b_this_page;
984 } while (bh != head);
985}
986
987/*
988 * Create the page-cache page that contains the requested block.
989 *
990 * This is user purely for blockdev mappings.
991 */
992static struct page *
993grow_dev_page(struct block_device *bdev, sector_t block,
994 pgoff_t index, int size)
995{
996 struct inode *inode = bdev->bd_inode;
997 struct page *page;
998 struct buffer_head *bh;
999
ea125892 1000 page = find_or_create_page(inode->i_mapping, index,
769848c0 1001 (mapping_gfp_mask(inode->i_mapping) & ~__GFP_FS)|__GFP_MOVABLE);
1da177e4
LT
1002 if (!page)
1003 return NULL;
1004
e827f923 1005 BUG_ON(!PageLocked(page));
1da177e4
LT
1006
1007 if (page_has_buffers(page)) {
1008 bh = page_buffers(page);
1009 if (bh->b_size == size) {
1010 init_page_buffers(page, bdev, block, size);
1011 return page;
1012 }
1013 if (!try_to_free_buffers(page))
1014 goto failed;
1015 }
1016
1017 /*
1018 * Allocate some buffers for this page
1019 */
1020 bh = alloc_page_buffers(page, size, 0);
1021 if (!bh)
1022 goto failed;
1023
1024 /*
1025 * Link the page to the buffers and initialise them. Take the
1026 * lock to be atomic wrt __find_get_block(), which does not
1027 * run under the page lock.
1028 */
1029 spin_lock(&inode->i_mapping->private_lock);
1030 link_dev_buffers(page, bh);
1031 init_page_buffers(page, bdev, block, size);
1032 spin_unlock(&inode->i_mapping->private_lock);
1033 return page;
1034
1035failed:
1036 BUG();
1037 unlock_page(page);
1038 page_cache_release(page);
1039 return NULL;
1040}
1041
1042/*
1043 * Create buffers for the specified block device block's page. If
1044 * that page was dirty, the buffers are set dirty also.
1da177e4 1045 */
858119e1 1046static int
1da177e4
LT
1047grow_buffers(struct block_device *bdev, sector_t block, int size)
1048{
1049 struct page *page;
1050 pgoff_t index;
1051 int sizebits;
1052
1053 sizebits = -1;
1054 do {
1055 sizebits++;
1056 } while ((size << sizebits) < PAGE_SIZE);
1057
1058 index = block >> sizebits;
1da177e4 1059
e5657933
AM
1060 /*
1061 * Check for a block which wants to lie outside our maximum possible
1062 * pagecache index. (this comparison is done using sector_t types).
1063 */
1064 if (unlikely(index != block >> sizebits)) {
1065 char b[BDEVNAME_SIZE];
1066
1067 printk(KERN_ERR "%s: requested out-of-range block %llu for "
1068 "device %s\n",
8e24eea7 1069 __func__, (unsigned long long)block,
e5657933
AM
1070 bdevname(bdev, b));
1071 return -EIO;
1072 }
1073 block = index << sizebits;
1da177e4
LT
1074 /* Create a page with the proper size buffers.. */
1075 page = grow_dev_page(bdev, block, index, size);
1076 if (!page)
1077 return 0;
1078 unlock_page(page);
1079 page_cache_release(page);
1080 return 1;
1081}
1082
75c96f85 1083static struct buffer_head *
1da177e4
LT
1084__getblk_slow(struct block_device *bdev, sector_t block, int size)
1085{
1086 /* Size must be multiple of hard sectorsize */
e1defc4f 1087 if (unlikely(size & (bdev_logical_block_size(bdev)-1) ||
1da177e4
LT
1088 (size < 512 || size > PAGE_SIZE))) {
1089 printk(KERN_ERR "getblk(): invalid block size %d requested\n",
1090 size);
e1defc4f
MP
1091 printk(KERN_ERR "logical block size: %d\n",
1092 bdev_logical_block_size(bdev));
1da177e4
LT
1093
1094 dump_stack();
1095 return NULL;
1096 }
1097
1098 for (;;) {
1099 struct buffer_head * bh;
e5657933 1100 int ret;
1da177e4
LT
1101
1102 bh = __find_get_block(bdev, block, size);
1103 if (bh)
1104 return bh;
1105
e5657933
AM
1106 ret = grow_buffers(bdev, block, size);
1107 if (ret < 0)
1108 return NULL;
1109 if (ret == 0)
1da177e4
LT
1110 free_more_memory();
1111 }
1112}
1113
1114/*
1115 * The relationship between dirty buffers and dirty pages:
1116 *
1117 * Whenever a page has any dirty buffers, the page's dirty bit is set, and
1118 * the page is tagged dirty in its radix tree.
1119 *
1120 * At all times, the dirtiness of the buffers represents the dirtiness of
1121 * subsections of the page. If the page has buffers, the page dirty bit is
1122 * merely a hint about the true dirty state.
1123 *
1124 * When a page is set dirty in its entirety, all its buffers are marked dirty
1125 * (if the page has buffers).
1126 *
1127 * When a buffer is marked dirty, its page is dirtied, but the page's other
1128 * buffers are not.
1129 *
1130 * Also. When blockdev buffers are explicitly read with bread(), they
1131 * individually become uptodate. But their backing page remains not
1132 * uptodate - even if all of its buffers are uptodate. A subsequent
1133 * block_read_full_page() against that page will discover all the uptodate
1134 * buffers, will set the page uptodate and will perform no I/O.
1135 */
1136
1137/**
1138 * mark_buffer_dirty - mark a buffer_head as needing writeout
67be2dd1 1139 * @bh: the buffer_head to mark dirty
1da177e4
LT
1140 *
1141 * mark_buffer_dirty() will set the dirty bit against the buffer, then set its
1142 * backing page dirty, then tag the page as dirty in its address_space's radix
1143 * tree and then attach the address_space's inode to its superblock's dirty
1144 * inode list.
1145 *
1146 * mark_buffer_dirty() is atomic. It takes bh->b_page->mapping->private_lock,
1147 * mapping->tree_lock and the global inode_lock.
1148 */
fc9b52cd 1149void mark_buffer_dirty(struct buffer_head *bh)
1da177e4 1150{
787d2214 1151 WARN_ON_ONCE(!buffer_uptodate(bh));
1be62dc1
LT
1152
1153 /*
1154 * Very *carefully* optimize the it-is-already-dirty case.
1155 *
1156 * Don't let the final "is it dirty" escape to before we
1157 * perhaps modified the buffer.
1158 */
1159 if (buffer_dirty(bh)) {
1160 smp_mb();
1161 if (buffer_dirty(bh))
1162 return;
1163 }
1164
a8e7d49a
LT
1165 if (!test_set_buffer_dirty(bh)) {
1166 struct page *page = bh->b_page;
8e9d78ed
LT
1167 if (!TestSetPageDirty(page)) {
1168 struct address_space *mapping = page_mapping(page);
1169 if (mapping)
1170 __set_page_dirty(page, mapping, 0);
1171 }
a8e7d49a 1172 }
1da177e4 1173}
1fe72eaa 1174EXPORT_SYMBOL(mark_buffer_dirty);
1da177e4
LT
1175
1176/*
1177 * Decrement a buffer_head's reference count. If all buffers against a page
1178 * have zero reference count, are clean and unlocked, and if the page is clean
1179 * and unlocked then try_to_free_buffers() may strip the buffers from the page
1180 * in preparation for freeing it (sometimes, rarely, buffers are removed from
1181 * a page but it ends up not being freed, and buffers may later be reattached).
1182 */
1183void __brelse(struct buffer_head * buf)
1184{
1185 if (atomic_read(&buf->b_count)) {
1186 put_bh(buf);
1187 return;
1188 }
5c752ad9 1189 WARN(1, KERN_ERR "VFS: brelse: Trying to free free buffer\n");
1da177e4 1190}
1fe72eaa 1191EXPORT_SYMBOL(__brelse);
1da177e4
LT
1192
1193/*
1194 * bforget() is like brelse(), except it discards any
1195 * potentially dirty data.
1196 */
1197void __bforget(struct buffer_head *bh)
1198{
1199 clear_buffer_dirty(bh);
535ee2fb 1200 if (bh->b_assoc_map) {
1da177e4
LT
1201 struct address_space *buffer_mapping = bh->b_page->mapping;
1202
1203 spin_lock(&buffer_mapping->private_lock);
1204 list_del_init(&bh->b_assoc_buffers);
58ff407b 1205 bh->b_assoc_map = NULL;
1da177e4
LT
1206 spin_unlock(&buffer_mapping->private_lock);
1207 }
1208 __brelse(bh);
1209}
1fe72eaa 1210EXPORT_SYMBOL(__bforget);
1da177e4
LT
1211
1212static struct buffer_head *__bread_slow(struct buffer_head *bh)
1213{
1214 lock_buffer(bh);
1215 if (buffer_uptodate(bh)) {
1216 unlock_buffer(bh);
1217 return bh;
1218 } else {
1219 get_bh(bh);
1220 bh->b_end_io = end_buffer_read_sync;
1221 submit_bh(READ, bh);
1222 wait_on_buffer(bh);
1223 if (buffer_uptodate(bh))
1224 return bh;
1225 }
1226 brelse(bh);
1227 return NULL;
1228}
1229
1230/*
1231 * Per-cpu buffer LRU implementation. To reduce the cost of __find_get_block().
1232 * The bhs[] array is sorted - newest buffer is at bhs[0]. Buffers have their
1233 * refcount elevated by one when they're in an LRU. A buffer can only appear
1234 * once in a particular CPU's LRU. A single buffer can be present in multiple
1235 * CPU's LRUs at the same time.
1236 *
1237 * This is a transparent caching front-end to sb_bread(), sb_getblk() and
1238 * sb_find_get_block().
1239 *
1240 * The LRUs themselves only need locking against invalidate_bh_lrus. We use
1241 * a local interrupt disable for that.
1242 */
1243
1244#define BH_LRU_SIZE 8
1245
1246struct bh_lru {
1247 struct buffer_head *bhs[BH_LRU_SIZE];
1248};
1249
1250static DEFINE_PER_CPU(struct bh_lru, bh_lrus) = {{ NULL }};
1251
1252#ifdef CONFIG_SMP
1253#define bh_lru_lock() local_irq_disable()
1254#define bh_lru_unlock() local_irq_enable()
1255#else
1256#define bh_lru_lock() preempt_disable()
1257#define bh_lru_unlock() preempt_enable()
1258#endif
1259
1260static inline void check_irqs_on(void)
1261{
1262#ifdef irqs_disabled
1263 BUG_ON(irqs_disabled());
1264#endif
1265}
1266
1267/*
1268 * The LRU management algorithm is dopey-but-simple. Sorry.
1269 */
1270static void bh_lru_install(struct buffer_head *bh)
1271{
1272 struct buffer_head *evictee = NULL;
1273 struct bh_lru *lru;
1274
1275 check_irqs_on();
1276 bh_lru_lock();
1277 lru = &__get_cpu_var(bh_lrus);
1278 if (lru->bhs[0] != bh) {
1279 struct buffer_head *bhs[BH_LRU_SIZE];
1280 int in;
1281 int out = 0;
1282
1283 get_bh(bh);
1284 bhs[out++] = bh;
1285 for (in = 0; in < BH_LRU_SIZE; in++) {
1286 struct buffer_head *bh2 = lru->bhs[in];
1287
1288 if (bh2 == bh) {
1289 __brelse(bh2);
1290 } else {
1291 if (out >= BH_LRU_SIZE) {
1292 BUG_ON(evictee != NULL);
1293 evictee = bh2;
1294 } else {
1295 bhs[out++] = bh2;
1296 }
1297 }
1298 }
1299 while (out < BH_LRU_SIZE)
1300 bhs[out++] = NULL;
1301 memcpy(lru->bhs, bhs, sizeof(bhs));
1302 }
1303 bh_lru_unlock();
1304
1305 if (evictee)
1306 __brelse(evictee);
1307}
1308
1309/*
1310 * Look up the bh in this cpu's LRU. If it's there, move it to the head.
1311 */
858119e1 1312static struct buffer_head *
3991d3bd 1313lookup_bh_lru(struct block_device *bdev, sector_t block, unsigned size)
1da177e4
LT
1314{
1315 struct buffer_head *ret = NULL;
1316 struct bh_lru *lru;
3991d3bd 1317 unsigned int i;
1da177e4
LT
1318
1319 check_irqs_on();
1320 bh_lru_lock();
1321 lru = &__get_cpu_var(bh_lrus);
1322 for (i = 0; i < BH_LRU_SIZE; i++) {
1323 struct buffer_head *bh = lru->bhs[i];
1324
1325 if (bh && bh->b_bdev == bdev &&
1326 bh->b_blocknr == block && bh->b_size == size) {
1327 if (i) {
1328 while (i) {
1329 lru->bhs[i] = lru->bhs[i - 1];
1330 i--;
1331 }
1332 lru->bhs[0] = bh;
1333 }
1334 get_bh(bh);
1335 ret = bh;
1336 break;
1337 }
1338 }
1339 bh_lru_unlock();
1340 return ret;
1341}
1342
1343/*
1344 * Perform a pagecache lookup for the matching buffer. If it's there, refresh
1345 * it in the LRU and mark it as accessed. If it is not present then return
1346 * NULL
1347 */
1348struct buffer_head *
3991d3bd 1349__find_get_block(struct block_device *bdev, sector_t block, unsigned size)
1da177e4
LT
1350{
1351 struct buffer_head *bh = lookup_bh_lru(bdev, block, size);
1352
1353 if (bh == NULL) {
385fd4c5 1354 bh = __find_get_block_slow(bdev, block);
1da177e4
LT
1355 if (bh)
1356 bh_lru_install(bh);
1357 }
1358 if (bh)
1359 touch_buffer(bh);
1360 return bh;
1361}
1362EXPORT_SYMBOL(__find_get_block);
1363
1364/*
1365 * __getblk will locate (and, if necessary, create) the buffer_head
1366 * which corresponds to the passed block_device, block and size. The
1367 * returned buffer has its reference count incremented.
1368 *
1369 * __getblk() cannot fail - it just keeps trying. If you pass it an
1370 * illegal block number, __getblk() will happily return a buffer_head
1371 * which represents the non-existent block. Very weird.
1372 *
1373 * __getblk() will lock up the machine if grow_dev_page's try_to_free_buffers()
1374 * attempt is failing. FIXME, perhaps?
1375 */
1376struct buffer_head *
3991d3bd 1377__getblk(struct block_device *bdev, sector_t block, unsigned size)
1da177e4
LT
1378{
1379 struct buffer_head *bh = __find_get_block(bdev, block, size);
1380
1381 might_sleep();
1382 if (bh == NULL)
1383 bh = __getblk_slow(bdev, block, size);
1384 return bh;
1385}
1386EXPORT_SYMBOL(__getblk);
1387
1388/*
1389 * Do async read-ahead on a buffer..
1390 */
3991d3bd 1391void __breadahead(struct block_device *bdev, sector_t block, unsigned size)
1da177e4
LT
1392{
1393 struct buffer_head *bh = __getblk(bdev, block, size);
a3e713b5
AM
1394 if (likely(bh)) {
1395 ll_rw_block(READA, 1, &bh);
1396 brelse(bh);
1397 }
1da177e4
LT
1398}
1399EXPORT_SYMBOL(__breadahead);
1400
1401/**
1402 * __bread() - reads a specified block and returns the bh
67be2dd1 1403 * @bdev: the block_device to read from
1da177e4
LT
1404 * @block: number of block
1405 * @size: size (in bytes) to read
1406 *
1407 * Reads a specified block, and returns buffer head that contains it.
1408 * It returns NULL if the block was unreadable.
1409 */
1410struct buffer_head *
3991d3bd 1411__bread(struct block_device *bdev, sector_t block, unsigned size)
1da177e4
LT
1412{
1413 struct buffer_head *bh = __getblk(bdev, block, size);
1414
a3e713b5 1415 if (likely(bh) && !buffer_uptodate(bh))
1da177e4
LT
1416 bh = __bread_slow(bh);
1417 return bh;
1418}
1419EXPORT_SYMBOL(__bread);
1420
1421/*
1422 * invalidate_bh_lrus() is called rarely - but not only at unmount.
1423 * This doesn't race because it runs in each cpu either in irq
1424 * or with preempt disabled.
1425 */
1426static void invalidate_bh_lru(void *arg)
1427{
1428 struct bh_lru *b = &get_cpu_var(bh_lrus);
1429 int i;
1430
1431 for (i = 0; i < BH_LRU_SIZE; i++) {
1432 brelse(b->bhs[i]);
1433 b->bhs[i] = NULL;
1434 }
1435 put_cpu_var(bh_lrus);
1436}
1437
f9a14399 1438void invalidate_bh_lrus(void)
1da177e4 1439{
15c8b6c1 1440 on_each_cpu(invalidate_bh_lru, NULL, 1);
1da177e4 1441}
9db5579b 1442EXPORT_SYMBOL_GPL(invalidate_bh_lrus);
1da177e4
LT
1443
1444void set_bh_page(struct buffer_head *bh,
1445 struct page *page, unsigned long offset)
1446{
1447 bh->b_page = page;
e827f923 1448 BUG_ON(offset >= PAGE_SIZE);
1da177e4
LT
1449 if (PageHighMem(page))
1450 /*
1451 * This catches illegal uses and preserves the offset:
1452 */
1453 bh->b_data = (char *)(0 + offset);
1454 else
1455 bh->b_data = page_address(page) + offset;
1456}
1457EXPORT_SYMBOL(set_bh_page);
1458
1459/*
1460 * Called when truncating a buffer on a page completely.
1461 */
858119e1 1462static void discard_buffer(struct buffer_head * bh)
1da177e4
LT
1463{
1464 lock_buffer(bh);
1465 clear_buffer_dirty(bh);
1466 bh->b_bdev = NULL;
1467 clear_buffer_mapped(bh);
1468 clear_buffer_req(bh);
1469 clear_buffer_new(bh);
1470 clear_buffer_delay(bh);
33a266dd 1471 clear_buffer_unwritten(bh);
1da177e4
LT
1472 unlock_buffer(bh);
1473}
1474
1da177e4
LT
1475/**
1476 * block_invalidatepage - invalidate part of all of a buffer-backed page
1477 *
1478 * @page: the page which is affected
1479 * @offset: the index of the truncation point
1480 *
1481 * block_invalidatepage() is called when all or part of the page has become
1482 * invalidatedby a truncate operation.
1483 *
1484 * block_invalidatepage() does not have to release all buffers, but it must
1485 * ensure that no dirty buffer is left outside @offset and that no I/O
1486 * is underway against any of the blocks which are outside the truncation
1487 * point. Because the caller is about to free (and possibly reuse) those
1488 * blocks on-disk.
1489 */
2ff28e22 1490void block_invalidatepage(struct page *page, unsigned long offset)
1da177e4
LT
1491{
1492 struct buffer_head *head, *bh, *next;
1493 unsigned int curr_off = 0;
1da177e4
LT
1494
1495 BUG_ON(!PageLocked(page));
1496 if (!page_has_buffers(page))
1497 goto out;
1498
1499 head = page_buffers(page);
1500 bh = head;
1501 do {
1502 unsigned int next_off = curr_off + bh->b_size;
1503 next = bh->b_this_page;
1504
1505 /*
1506 * is this block fully invalidated?
1507 */
1508 if (offset <= curr_off)
1509 discard_buffer(bh);
1510 curr_off = next_off;
1511 bh = next;
1512 } while (bh != head);
1513
1514 /*
1515 * We release buffers only if the entire page is being invalidated.
1516 * The get_block cached value has been unconditionally invalidated,
1517 * so real IO is not possible anymore.
1518 */
1519 if (offset == 0)
2ff28e22 1520 try_to_release_page(page, 0);
1da177e4 1521out:
2ff28e22 1522 return;
1da177e4
LT
1523}
1524EXPORT_SYMBOL(block_invalidatepage);
1525
1526/*
1527 * We attach and possibly dirty the buffers atomically wrt
1528 * __set_page_dirty_buffers() via private_lock. try_to_free_buffers
1529 * is already excluded via the page lock.
1530 */
1531void create_empty_buffers(struct page *page,
1532 unsigned long blocksize, unsigned long b_state)
1533{
1534 struct buffer_head *bh, *head, *tail;
1535
1536 head = alloc_page_buffers(page, blocksize, 1);
1537 bh = head;
1538 do {
1539 bh->b_state |= b_state;
1540 tail = bh;
1541 bh = bh->b_this_page;
1542 } while (bh);
1543 tail->b_this_page = head;
1544
1545 spin_lock(&page->mapping->private_lock);
1546 if (PageUptodate(page) || PageDirty(page)) {
1547 bh = head;
1548 do {
1549 if (PageDirty(page))
1550 set_buffer_dirty(bh);
1551 if (PageUptodate(page))
1552 set_buffer_uptodate(bh);
1553 bh = bh->b_this_page;
1554 } while (bh != head);
1555 }
1556 attach_page_buffers(page, head);
1557 spin_unlock(&page->mapping->private_lock);
1558}
1559EXPORT_SYMBOL(create_empty_buffers);
1560
1561/*
1562 * We are taking a block for data and we don't want any output from any
1563 * buffer-cache aliases starting from return from that function and
1564 * until the moment when something will explicitly mark the buffer
1565 * dirty (hopefully that will not happen until we will free that block ;-)
1566 * We don't even need to mark it not-uptodate - nobody can expect
1567 * anything from a newly allocated buffer anyway. We used to used
1568 * unmap_buffer() for such invalidation, but that was wrong. We definitely
1569 * don't want to mark the alias unmapped, for example - it would confuse
1570 * anyone who might pick it with bread() afterwards...
1571 *
1572 * Also.. Note that bforget() doesn't lock the buffer. So there can
1573 * be writeout I/O going on against recently-freed buffers. We don't
1574 * wait on that I/O in bforget() - it's more efficient to wait on the I/O
1575 * only if we really need to. That happens here.
1576 */
1577void unmap_underlying_metadata(struct block_device *bdev, sector_t block)
1578{
1579 struct buffer_head *old_bh;
1580
1581 might_sleep();
1582
385fd4c5 1583 old_bh = __find_get_block_slow(bdev, block);
1da177e4
LT
1584 if (old_bh) {
1585 clear_buffer_dirty(old_bh);
1586 wait_on_buffer(old_bh);
1587 clear_buffer_req(old_bh);
1588 __brelse(old_bh);
1589 }
1590}
1591EXPORT_SYMBOL(unmap_underlying_metadata);
1592
1593/*
1594 * NOTE! All mapped/uptodate combinations are valid:
1595 *
1596 * Mapped Uptodate Meaning
1597 *
1598 * No No "unknown" - must do get_block()
1599 * No Yes "hole" - zero-filled
1600 * Yes No "allocated" - allocated on disk, not read in
1601 * Yes Yes "valid" - allocated and up-to-date in memory.
1602 *
1603 * "Dirty" is valid only with the last case (mapped+uptodate).
1604 */
1605
1606/*
1607 * While block_write_full_page is writing back the dirty buffers under
1608 * the page lock, whoever dirtied the buffers may decide to clean them
1609 * again at any time. We handle that by only looking at the buffer
1610 * state inside lock_buffer().
1611 *
1612 * If block_write_full_page() is called for regular writeback
1613 * (wbc->sync_mode == WB_SYNC_NONE) then it will redirty a page which has a
1614 * locked buffer. This only can happen if someone has written the buffer
1615 * directly, with submit_bh(). At the address_space level PageWriteback
1616 * prevents this contention from occurring.
6e34eedd
TT
1617 *
1618 * If block_write_full_page() is called with wbc->sync_mode ==
1619 * WB_SYNC_ALL, the writes are posted using WRITE_SYNC_PLUG; this
1620 * causes the writes to be flagged as synchronous writes, but the
1621 * block device queue will NOT be unplugged, since usually many pages
1622 * will be pushed to the out before the higher-level caller actually
1623 * waits for the writes to be completed. The various wait functions,
1624 * such as wait_on_writeback_range() will ultimately call sync_page()
1625 * which will ultimately call blk_run_backing_dev(), which will end up
1626 * unplugging the device queue.
1da177e4
LT
1627 */
1628static int __block_write_full_page(struct inode *inode, struct page *page,
35c80d5f
CM
1629 get_block_t *get_block, struct writeback_control *wbc,
1630 bh_end_io_t *handler)
1da177e4
LT
1631{
1632 int err;
1633 sector_t block;
1634 sector_t last_block;
f0fbd5fc 1635 struct buffer_head *bh, *head;
b0cf2321 1636 const unsigned blocksize = 1 << inode->i_blkbits;
1da177e4 1637 int nr_underway = 0;
6e34eedd
TT
1638 int write_op = (wbc->sync_mode == WB_SYNC_ALL ?
1639 WRITE_SYNC_PLUG : WRITE);
1da177e4
LT
1640
1641 BUG_ON(!PageLocked(page));
1642
1643 last_block = (i_size_read(inode) - 1) >> inode->i_blkbits;
1644
1645 if (!page_has_buffers(page)) {
b0cf2321 1646 create_empty_buffers(page, blocksize,
1da177e4
LT
1647 (1 << BH_Dirty)|(1 << BH_Uptodate));
1648 }
1649
1650 /*
1651 * Be very careful. We have no exclusion from __set_page_dirty_buffers
1652 * here, and the (potentially unmapped) buffers may become dirty at
1653 * any time. If a buffer becomes dirty here after we've inspected it
1654 * then we just miss that fact, and the page stays dirty.
1655 *
1656 * Buffers outside i_size may be dirtied by __set_page_dirty_buffers;
1657 * handle that here by just cleaning them.
1658 */
1659
54b21a79 1660 block = (sector_t)page->index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
1da177e4
LT
1661 head = page_buffers(page);
1662 bh = head;
1663
1664 /*
1665 * Get all the dirty buffers mapped to disk addresses and
1666 * handle any aliases from the underlying blockdev's mapping.
1667 */
1668 do {
1669 if (block > last_block) {
1670 /*
1671 * mapped buffers outside i_size will occur, because
1672 * this page can be outside i_size when there is a
1673 * truncate in progress.
1674 */
1675 /*
1676 * The buffer was zeroed by block_write_full_page()
1677 */
1678 clear_buffer_dirty(bh);
1679 set_buffer_uptodate(bh);
29a814d2
AT
1680 } else if ((!buffer_mapped(bh) || buffer_delay(bh)) &&
1681 buffer_dirty(bh)) {
b0cf2321 1682 WARN_ON(bh->b_size != blocksize);
1da177e4
LT
1683 err = get_block(inode, block, bh, 1);
1684 if (err)
1685 goto recover;
29a814d2 1686 clear_buffer_delay(bh);
1da177e4
LT
1687 if (buffer_new(bh)) {
1688 /* blockdev mappings never come here */
1689 clear_buffer_new(bh);
1690 unmap_underlying_metadata(bh->b_bdev,
1691 bh->b_blocknr);
1692 }
1693 }
1694 bh = bh->b_this_page;
1695 block++;
1696 } while (bh != head);
1697
1698 do {
1da177e4
LT
1699 if (!buffer_mapped(bh))
1700 continue;
1701 /*
1702 * If it's a fully non-blocking write attempt and we cannot
1703 * lock the buffer then redirty the page. Note that this can
5b0830cb
JA
1704 * potentially cause a busy-wait loop from writeback threads
1705 * and kswapd activity, but those code paths have their own
1706 * higher-level throttling.
1da177e4
LT
1707 */
1708 if (wbc->sync_mode != WB_SYNC_NONE || !wbc->nonblocking) {
1709 lock_buffer(bh);
ca5de404 1710 } else if (!trylock_buffer(bh)) {
1da177e4
LT
1711 redirty_page_for_writepage(wbc, page);
1712 continue;
1713 }
1714 if (test_clear_buffer_dirty(bh)) {
35c80d5f 1715 mark_buffer_async_write_endio(bh, handler);
1da177e4
LT
1716 } else {
1717 unlock_buffer(bh);
1718 }
1719 } while ((bh = bh->b_this_page) != head);
1720
1721 /*
1722 * The page and its buffers are protected by PageWriteback(), so we can
1723 * drop the bh refcounts early.
1724 */
1725 BUG_ON(PageWriteback(page));
1726 set_page_writeback(page);
1da177e4
LT
1727
1728 do {
1729 struct buffer_head *next = bh->b_this_page;
1730 if (buffer_async_write(bh)) {
a64c8610 1731 submit_bh(write_op, bh);
1da177e4
LT
1732 nr_underway++;
1733 }
1da177e4
LT
1734 bh = next;
1735 } while (bh != head);
05937baa 1736 unlock_page(page);
1da177e4
LT
1737
1738 err = 0;
1739done:
1740 if (nr_underway == 0) {
1741 /*
1742 * The page was marked dirty, but the buffers were
1743 * clean. Someone wrote them back by hand with
1744 * ll_rw_block/submit_bh. A rare case.
1745 */
1da177e4 1746 end_page_writeback(page);
3d67f2d7 1747
1da177e4
LT
1748 /*
1749 * The page and buffer_heads can be released at any time from
1750 * here on.
1751 */
1da177e4
LT
1752 }
1753 return err;
1754
1755recover:
1756 /*
1757 * ENOSPC, or some other error. We may already have added some
1758 * blocks to the file, so we need to write these out to avoid
1759 * exposing stale data.
1760 * The page is currently locked and not marked for writeback
1761 */
1762 bh = head;
1763 /* Recovery: lock and submit the mapped buffers */
1764 do {
29a814d2
AT
1765 if (buffer_mapped(bh) && buffer_dirty(bh) &&
1766 !buffer_delay(bh)) {
1da177e4 1767 lock_buffer(bh);
35c80d5f 1768 mark_buffer_async_write_endio(bh, handler);
1da177e4
LT
1769 } else {
1770 /*
1771 * The buffer may have been set dirty during
1772 * attachment to a dirty page.
1773 */
1774 clear_buffer_dirty(bh);
1775 }
1776 } while ((bh = bh->b_this_page) != head);
1777 SetPageError(page);
1778 BUG_ON(PageWriteback(page));
7e4c3690 1779 mapping_set_error(page->mapping, err);
1da177e4 1780 set_page_writeback(page);
1da177e4
LT
1781 do {
1782 struct buffer_head *next = bh->b_this_page;
1783 if (buffer_async_write(bh)) {
1784 clear_buffer_dirty(bh);
a64c8610 1785 submit_bh(write_op, bh);
1da177e4
LT
1786 nr_underway++;
1787 }
1da177e4
LT
1788 bh = next;
1789 } while (bh != head);
ffda9d30 1790 unlock_page(page);
1da177e4
LT
1791 goto done;
1792}
1793
afddba49
NP
1794/*
1795 * If a page has any new buffers, zero them out here, and mark them uptodate
1796 * and dirty so they'll be written out (in order to prevent uninitialised
1797 * block data from leaking). And clear the new bit.
1798 */
1799void page_zero_new_buffers(struct page *page, unsigned from, unsigned to)
1800{
1801 unsigned int block_start, block_end;
1802 struct buffer_head *head, *bh;
1803
1804 BUG_ON(!PageLocked(page));
1805 if (!page_has_buffers(page))
1806 return;
1807
1808 bh = head = page_buffers(page);
1809 block_start = 0;
1810 do {
1811 block_end = block_start + bh->b_size;
1812
1813 if (buffer_new(bh)) {
1814 if (block_end > from && block_start < to) {
1815 if (!PageUptodate(page)) {
1816 unsigned start, size;
1817
1818 start = max(from, block_start);
1819 size = min(to, block_end) - start;
1820
eebd2aa3 1821 zero_user(page, start, size);
afddba49
NP
1822 set_buffer_uptodate(bh);
1823 }
1824
1825 clear_buffer_new(bh);
1826 mark_buffer_dirty(bh);
1827 }
1828 }
1829
1830 block_start = block_end;
1831 bh = bh->b_this_page;
1832 } while (bh != head);
1833}
1834EXPORT_SYMBOL(page_zero_new_buffers);
1835
ebdec241 1836int __block_write_begin(struct page *page, loff_t pos, unsigned len,
6e1db88d 1837 get_block_t *get_block)
1da177e4 1838{
ebdec241
CH
1839 unsigned from = pos & (PAGE_CACHE_SIZE - 1);
1840 unsigned to = from + len;
6e1db88d 1841 struct inode *inode = page->mapping->host;
1da177e4
LT
1842 unsigned block_start, block_end;
1843 sector_t block;
1844 int err = 0;
1845 unsigned blocksize, bbits;
1846 struct buffer_head *bh, *head, *wait[2], **wait_bh=wait;
1847
1848 BUG_ON(!PageLocked(page));
1849 BUG_ON(from > PAGE_CACHE_SIZE);
1850 BUG_ON(to > PAGE_CACHE_SIZE);
1851 BUG_ON(from > to);
1852
1853 blocksize = 1 << inode->i_blkbits;
1854 if (!page_has_buffers(page))
1855 create_empty_buffers(page, blocksize, 0);
1856 head = page_buffers(page);
1857
1858 bbits = inode->i_blkbits;
1859 block = (sector_t)page->index << (PAGE_CACHE_SHIFT - bbits);
1860
1861 for(bh = head, block_start = 0; bh != head || !block_start;
1862 block++, block_start=block_end, bh = bh->b_this_page) {
1863 block_end = block_start + blocksize;
1864 if (block_end <= from || block_start >= to) {
1865 if (PageUptodate(page)) {
1866 if (!buffer_uptodate(bh))
1867 set_buffer_uptodate(bh);
1868 }
1869 continue;
1870 }
1871 if (buffer_new(bh))
1872 clear_buffer_new(bh);
1873 if (!buffer_mapped(bh)) {
b0cf2321 1874 WARN_ON(bh->b_size != blocksize);
1da177e4
LT
1875 err = get_block(inode, block, bh, 1);
1876 if (err)
f3ddbdc6 1877 break;
1da177e4 1878 if (buffer_new(bh)) {
1da177e4
LT
1879 unmap_underlying_metadata(bh->b_bdev,
1880 bh->b_blocknr);
1881 if (PageUptodate(page)) {
637aff46 1882 clear_buffer_new(bh);
1da177e4 1883 set_buffer_uptodate(bh);
637aff46 1884 mark_buffer_dirty(bh);
1da177e4
LT
1885 continue;
1886 }
eebd2aa3
CL
1887 if (block_end > to || block_start < from)
1888 zero_user_segments(page,
1889 to, block_end,
1890 block_start, from);
1da177e4
LT
1891 continue;
1892 }
1893 }
1894 if (PageUptodate(page)) {
1895 if (!buffer_uptodate(bh))
1896 set_buffer_uptodate(bh);
1897 continue;
1898 }
1899 if (!buffer_uptodate(bh) && !buffer_delay(bh) &&
33a266dd 1900 !buffer_unwritten(bh) &&
1da177e4
LT
1901 (block_start < from || block_end > to)) {
1902 ll_rw_block(READ, 1, &bh);
1903 *wait_bh++=bh;
1904 }
1905 }
1906 /*
1907 * If we issued read requests - let them complete.
1908 */
1909 while(wait_bh > wait) {
1910 wait_on_buffer(*--wait_bh);
1911 if (!buffer_uptodate(*wait_bh))
f3ddbdc6 1912 err = -EIO;
1da177e4 1913 }
6e1db88d 1914 if (unlikely(err)) {
afddba49 1915 page_zero_new_buffers(page, from, to);
6e1db88d
CH
1916 ClearPageUptodate(page);
1917 }
1da177e4
LT
1918 return err;
1919}
ebdec241 1920EXPORT_SYMBOL(__block_write_begin);
1da177e4
LT
1921
1922static int __block_commit_write(struct inode *inode, struct page *page,
1923 unsigned from, unsigned to)
1924{
1925 unsigned block_start, block_end;
1926 int partial = 0;
1927 unsigned blocksize;
1928 struct buffer_head *bh, *head;
1929
1930 blocksize = 1 << inode->i_blkbits;
1931
1932 for(bh = head = page_buffers(page), block_start = 0;
1933 bh != head || !block_start;
1934 block_start=block_end, bh = bh->b_this_page) {
1935 block_end = block_start + blocksize;
1936 if (block_end <= from || block_start >= to) {
1937 if (!buffer_uptodate(bh))
1938 partial = 1;
1939 } else {
1940 set_buffer_uptodate(bh);
1941 mark_buffer_dirty(bh);
1942 }
afddba49 1943 clear_buffer_new(bh);
1da177e4
LT
1944 }
1945
1946 /*
1947 * If this is a partial write which happened to make all buffers
1948 * uptodate then we can optimize away a bogus readpage() for
1949 * the next read(). Here we 'discover' whether the page went
1950 * uptodate as a result of this (potentially partial) write.
1951 */
1952 if (!partial)
1953 SetPageUptodate(page);
1954 return 0;
1955}
1956
afddba49 1957/*
155130a4
CH
1958 * block_write_begin takes care of the basic task of block allocation and
1959 * bringing partial write blocks uptodate first.
1960 *
7bb46a67 1961 * The filesystem needs to handle block truncation upon failure.
afddba49 1962 */
155130a4
CH
1963int block_write_begin(struct address_space *mapping, loff_t pos, unsigned len,
1964 unsigned flags, struct page **pagep, get_block_t *get_block)
afddba49 1965{
6e1db88d 1966 pgoff_t index = pos >> PAGE_CACHE_SHIFT;
afddba49 1967 struct page *page;
6e1db88d 1968 int status;
afddba49 1969
6e1db88d
CH
1970 page = grab_cache_page_write_begin(mapping, index, flags);
1971 if (!page)
1972 return -ENOMEM;
afddba49 1973
6e1db88d 1974 status = __block_write_begin(page, pos, len, get_block);
afddba49 1975 if (unlikely(status)) {
6e1db88d
CH
1976 unlock_page(page);
1977 page_cache_release(page);
1978 page = NULL;
afddba49
NP
1979 }
1980
6e1db88d 1981 *pagep = page;
afddba49
NP
1982 return status;
1983}
1984EXPORT_SYMBOL(block_write_begin);
1985
1986int block_write_end(struct file *file, struct address_space *mapping,
1987 loff_t pos, unsigned len, unsigned copied,
1988 struct page *page, void *fsdata)
1989{
1990 struct inode *inode = mapping->host;
1991 unsigned start;
1992
1993 start = pos & (PAGE_CACHE_SIZE - 1);
1994
1995 if (unlikely(copied < len)) {
1996 /*
1997 * The buffers that were written will now be uptodate, so we
1998 * don't have to worry about a readpage reading them and
1999 * overwriting a partial write. However if we have encountered
2000 * a short write and only partially written into a buffer, it
2001 * will not be marked uptodate, so a readpage might come in and
2002 * destroy our partial write.
2003 *
2004 * Do the simplest thing, and just treat any short write to a
2005 * non uptodate page as a zero-length write, and force the
2006 * caller to redo the whole thing.
2007 */
2008 if (!PageUptodate(page))
2009 copied = 0;
2010
2011 page_zero_new_buffers(page, start+copied, start+len);
2012 }
2013 flush_dcache_page(page);
2014
2015 /* This could be a short (even 0-length) commit */
2016 __block_commit_write(inode, page, start, start+copied);
2017
2018 return copied;
2019}
2020EXPORT_SYMBOL(block_write_end);
2021
2022int generic_write_end(struct file *file, struct address_space *mapping,
2023 loff_t pos, unsigned len, unsigned copied,
2024 struct page *page, void *fsdata)
2025{
2026 struct inode *inode = mapping->host;
c7d206b3 2027 int i_size_changed = 0;
afddba49
NP
2028
2029 copied = block_write_end(file, mapping, pos, len, copied, page, fsdata);
2030
2031 /*
2032 * No need to use i_size_read() here, the i_size
2033 * cannot change under us because we hold i_mutex.
2034 *
2035 * But it's important to update i_size while still holding page lock:
2036 * page writeout could otherwise come in and zero beyond i_size.
2037 */
2038 if (pos+copied > inode->i_size) {
2039 i_size_write(inode, pos+copied);
c7d206b3 2040 i_size_changed = 1;
afddba49
NP
2041 }
2042
2043 unlock_page(page);
2044 page_cache_release(page);
2045
c7d206b3
JK
2046 /*
2047 * Don't mark the inode dirty under page lock. First, it unnecessarily
2048 * makes the holding time of page lock longer. Second, it forces lock
2049 * ordering of page lock and transaction start for journaling
2050 * filesystems.
2051 */
2052 if (i_size_changed)
2053 mark_inode_dirty(inode);
2054
afddba49
NP
2055 return copied;
2056}
2057EXPORT_SYMBOL(generic_write_end);
2058
8ab22b9a
HH
2059/*
2060 * block_is_partially_uptodate checks whether buffers within a page are
2061 * uptodate or not.
2062 *
2063 * Returns true if all buffers which correspond to a file portion
2064 * we want to read are uptodate.
2065 */
2066int block_is_partially_uptodate(struct page *page, read_descriptor_t *desc,
2067 unsigned long from)
2068{
2069 struct inode *inode = page->mapping->host;
2070 unsigned block_start, block_end, blocksize;
2071 unsigned to;
2072 struct buffer_head *bh, *head;
2073 int ret = 1;
2074
2075 if (!page_has_buffers(page))
2076 return 0;
2077
2078 blocksize = 1 << inode->i_blkbits;
2079 to = min_t(unsigned, PAGE_CACHE_SIZE - from, desc->count);
2080 to = from + to;
2081 if (from < blocksize && to > PAGE_CACHE_SIZE - blocksize)
2082 return 0;
2083
2084 head = page_buffers(page);
2085 bh = head;
2086 block_start = 0;
2087 do {
2088 block_end = block_start + blocksize;
2089 if (block_end > from && block_start < to) {
2090 if (!buffer_uptodate(bh)) {
2091 ret = 0;
2092 break;
2093 }
2094 if (block_end >= to)
2095 break;
2096 }
2097 block_start = block_end;
2098 bh = bh->b_this_page;
2099 } while (bh != head);
2100
2101 return ret;
2102}
2103EXPORT_SYMBOL(block_is_partially_uptodate);
2104
1da177e4
LT
2105/*
2106 * Generic "read page" function for block devices that have the normal
2107 * get_block functionality. This is most of the block device filesystems.
2108 * Reads the page asynchronously --- the unlock_buffer() and
2109 * set/clear_buffer_uptodate() functions propagate buffer state into the
2110 * page struct once IO has completed.
2111 */
2112int block_read_full_page(struct page *page, get_block_t *get_block)
2113{
2114 struct inode *inode = page->mapping->host;
2115 sector_t iblock, lblock;
2116 struct buffer_head *bh, *head, *arr[MAX_BUF_PER_PAGE];
2117 unsigned int blocksize;
2118 int nr, i;
2119 int fully_mapped = 1;
2120
cd7619d6 2121 BUG_ON(!PageLocked(page));
1da177e4
LT
2122 blocksize = 1 << inode->i_blkbits;
2123 if (!page_has_buffers(page))
2124 create_empty_buffers(page, blocksize, 0);
2125 head = page_buffers(page);
2126
2127 iblock = (sector_t)page->index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
2128 lblock = (i_size_read(inode)+blocksize-1) >> inode->i_blkbits;
2129 bh = head;
2130 nr = 0;
2131 i = 0;
2132
2133 do {
2134 if (buffer_uptodate(bh))
2135 continue;
2136
2137 if (!buffer_mapped(bh)) {
c64610ba
AM
2138 int err = 0;
2139
1da177e4
LT
2140 fully_mapped = 0;
2141 if (iblock < lblock) {
b0cf2321 2142 WARN_ON(bh->b_size != blocksize);
c64610ba
AM
2143 err = get_block(inode, iblock, bh, 0);
2144 if (err)
1da177e4
LT
2145 SetPageError(page);
2146 }
2147 if (!buffer_mapped(bh)) {
eebd2aa3 2148 zero_user(page, i * blocksize, blocksize);
c64610ba
AM
2149 if (!err)
2150 set_buffer_uptodate(bh);
1da177e4
LT
2151 continue;
2152 }
2153 /*
2154 * get_block() might have updated the buffer
2155 * synchronously
2156 */
2157 if (buffer_uptodate(bh))
2158 continue;
2159 }
2160 arr[nr++] = bh;
2161 } while (i++, iblock++, (bh = bh->b_this_page) != head);
2162
2163 if (fully_mapped)
2164 SetPageMappedToDisk(page);
2165
2166 if (!nr) {
2167 /*
2168 * All buffers are uptodate - we can set the page uptodate
2169 * as well. But not if get_block() returned an error.
2170 */
2171 if (!PageError(page))
2172 SetPageUptodate(page);
2173 unlock_page(page);
2174 return 0;
2175 }
2176
2177 /* Stage two: lock the buffers */
2178 for (i = 0; i < nr; i++) {
2179 bh = arr[i];
2180 lock_buffer(bh);
2181 mark_buffer_async_read(bh);
2182 }
2183
2184 /*
2185 * Stage 3: start the IO. Check for uptodateness
2186 * inside the buffer lock in case another process reading
2187 * the underlying blockdev brought it uptodate (the sct fix).
2188 */
2189 for (i = 0; i < nr; i++) {
2190 bh = arr[i];
2191 if (buffer_uptodate(bh))
2192 end_buffer_async_read(bh, 1);
2193 else
2194 submit_bh(READ, bh);
2195 }
2196 return 0;
2197}
1fe72eaa 2198EXPORT_SYMBOL(block_read_full_page);
1da177e4
LT
2199
2200/* utility function for filesystems that need to do work on expanding
89e10787 2201 * truncates. Uses filesystem pagecache writes to allow the filesystem to
1da177e4
LT
2202 * deal with the hole.
2203 */
89e10787 2204int generic_cont_expand_simple(struct inode *inode, loff_t size)
1da177e4
LT
2205{
2206 struct address_space *mapping = inode->i_mapping;
2207 struct page *page;
89e10787 2208 void *fsdata;
1da177e4
LT
2209 int err;
2210
c08d3b0e 2211 err = inode_newsize_ok(inode, size);
2212 if (err)
1da177e4
LT
2213 goto out;
2214
89e10787
NP
2215 err = pagecache_write_begin(NULL, mapping, size, 0,
2216 AOP_FLAG_UNINTERRUPTIBLE|AOP_FLAG_CONT_EXPAND,
2217 &page, &fsdata);
2218 if (err)
05eb0b51 2219 goto out;
05eb0b51 2220
89e10787
NP
2221 err = pagecache_write_end(NULL, mapping, size, 0, 0, page, fsdata);
2222 BUG_ON(err > 0);
05eb0b51 2223
1da177e4
LT
2224out:
2225 return err;
2226}
1fe72eaa 2227EXPORT_SYMBOL(generic_cont_expand_simple);
1da177e4 2228
f1e3af72
AB
2229static int cont_expand_zero(struct file *file, struct address_space *mapping,
2230 loff_t pos, loff_t *bytes)
1da177e4 2231{
1da177e4 2232 struct inode *inode = mapping->host;
1da177e4 2233 unsigned blocksize = 1 << inode->i_blkbits;
89e10787
NP
2234 struct page *page;
2235 void *fsdata;
2236 pgoff_t index, curidx;
2237 loff_t curpos;
2238 unsigned zerofrom, offset, len;
2239 int err = 0;
1da177e4 2240
89e10787
NP
2241 index = pos >> PAGE_CACHE_SHIFT;
2242 offset = pos & ~PAGE_CACHE_MASK;
2243
2244 while (index > (curidx = (curpos = *bytes)>>PAGE_CACHE_SHIFT)) {
2245 zerofrom = curpos & ~PAGE_CACHE_MASK;
1da177e4
LT
2246 if (zerofrom & (blocksize-1)) {
2247 *bytes |= (blocksize-1);
2248 (*bytes)++;
2249 }
89e10787 2250 len = PAGE_CACHE_SIZE - zerofrom;
1da177e4 2251
89e10787
NP
2252 err = pagecache_write_begin(file, mapping, curpos, len,
2253 AOP_FLAG_UNINTERRUPTIBLE,
2254 &page, &fsdata);
2255 if (err)
2256 goto out;
eebd2aa3 2257 zero_user(page, zerofrom, len);
89e10787
NP
2258 err = pagecache_write_end(file, mapping, curpos, len, len,
2259 page, fsdata);
2260 if (err < 0)
2261 goto out;
2262 BUG_ON(err != len);
2263 err = 0;
061e9746
OH
2264
2265 balance_dirty_pages_ratelimited(mapping);
89e10787 2266 }
1da177e4 2267
89e10787
NP
2268 /* page covers the boundary, find the boundary offset */
2269 if (index == curidx) {
2270 zerofrom = curpos & ~PAGE_CACHE_MASK;
1da177e4 2271 /* if we will expand the thing last block will be filled */
89e10787
NP
2272 if (offset <= zerofrom) {
2273 goto out;
2274 }
2275 if (zerofrom & (blocksize-1)) {
1da177e4
LT
2276 *bytes |= (blocksize-1);
2277 (*bytes)++;
2278 }
89e10787 2279 len = offset - zerofrom;
1da177e4 2280
89e10787
NP
2281 err = pagecache_write_begin(file, mapping, curpos, len,
2282 AOP_FLAG_UNINTERRUPTIBLE,
2283 &page, &fsdata);
2284 if (err)
2285 goto out;
eebd2aa3 2286 zero_user(page, zerofrom, len);
89e10787
NP
2287 err = pagecache_write_end(file, mapping, curpos, len, len,
2288 page, fsdata);
2289 if (err < 0)
2290 goto out;
2291 BUG_ON(err != len);
2292 err = 0;
1da177e4 2293 }
89e10787
NP
2294out:
2295 return err;
2296}
2297
2298/*
2299 * For moronic filesystems that do not allow holes in file.
2300 * We may have to extend the file.
2301 */
282dc178 2302int cont_write_begin(struct file *file, struct address_space *mapping,
89e10787
NP
2303 loff_t pos, unsigned len, unsigned flags,
2304 struct page **pagep, void **fsdata,
2305 get_block_t *get_block, loff_t *bytes)
2306{
2307 struct inode *inode = mapping->host;
2308 unsigned blocksize = 1 << inode->i_blkbits;
2309 unsigned zerofrom;
2310 int err;
2311
2312 err = cont_expand_zero(file, mapping, pos, bytes);
2313 if (err)
155130a4 2314 return err;
89e10787
NP
2315
2316 zerofrom = *bytes & ~PAGE_CACHE_MASK;
2317 if (pos+len > *bytes && zerofrom & (blocksize-1)) {
2318 *bytes |= (blocksize-1);
2319 (*bytes)++;
1da177e4 2320 }
1da177e4 2321
155130a4 2322 return block_write_begin(mapping, pos, len, flags, pagep, get_block);
1da177e4 2323}
1fe72eaa 2324EXPORT_SYMBOL(cont_write_begin);
1da177e4 2325
1da177e4
LT
2326int block_commit_write(struct page *page, unsigned from, unsigned to)
2327{
2328 struct inode *inode = page->mapping->host;
2329 __block_commit_write(inode,page,from,to);
2330 return 0;
2331}
1fe72eaa 2332EXPORT_SYMBOL(block_commit_write);
1da177e4 2333
54171690
DC
2334/*
2335 * block_page_mkwrite() is not allowed to change the file size as it gets
2336 * called from a page fault handler when a page is first dirtied. Hence we must
2337 * be careful to check for EOF conditions here. We set the page up correctly
2338 * for a written page which means we get ENOSPC checking when writing into
2339 * holes and correct delalloc and unwritten extent mapping on filesystems that
2340 * support these features.
2341 *
2342 * We are not allowed to take the i_mutex here so we have to play games to
2343 * protect against truncate races as the page could now be beyond EOF. Because
7bb46a67 2344 * truncate writes the inode size before removing pages, once we have the
54171690
DC
2345 * page lock we can determine safely if the page is beyond EOF. If it is not
2346 * beyond EOF, then the page is guaranteed safe against truncation until we
2347 * unlock the page.
2348 */
2349int
c2ec175c 2350block_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf,
54171690
DC
2351 get_block_t get_block)
2352{
c2ec175c 2353 struct page *page = vmf->page;
54171690
DC
2354 struct inode *inode = vma->vm_file->f_path.dentry->d_inode;
2355 unsigned long end;
2356 loff_t size;
56a76f82 2357 int ret = VM_FAULT_NOPAGE; /* make the VM retry the fault */
54171690
DC
2358
2359 lock_page(page);
2360 size = i_size_read(inode);
2361 if ((page->mapping != inode->i_mapping) ||
18336338 2362 (page_offset(page) > size)) {
54171690 2363 /* page got truncated out from underneath us */
b827e496
NP
2364 unlock_page(page);
2365 goto out;
54171690
DC
2366 }
2367
2368 /* page is wholly or partially inside EOF */
2369 if (((page->index + 1) << PAGE_CACHE_SHIFT) > size)
2370 end = size & ~PAGE_CACHE_MASK;
2371 else
2372 end = PAGE_CACHE_SIZE;
2373
ebdec241 2374 ret = __block_write_begin(page, 0, end, get_block);
54171690
DC
2375 if (!ret)
2376 ret = block_commit_write(page, 0, end);
2377
56a76f82 2378 if (unlikely(ret)) {
b827e496 2379 unlock_page(page);
56a76f82
NP
2380 if (ret == -ENOMEM)
2381 ret = VM_FAULT_OOM;
2382 else /* -ENOSPC, -EIO, etc */
2383 ret = VM_FAULT_SIGBUS;
b827e496
NP
2384 } else
2385 ret = VM_FAULT_LOCKED;
c2ec175c 2386
b827e496 2387out:
54171690
DC
2388 return ret;
2389}
1fe72eaa 2390EXPORT_SYMBOL(block_page_mkwrite);
1da177e4
LT
2391
2392/*
03158cd7 2393 * nobh_write_begin()'s prereads are special: the buffer_heads are freed
1da177e4
LT
2394 * immediately, while under the page lock. So it needs a special end_io
2395 * handler which does not touch the bh after unlocking it.
1da177e4
LT
2396 */
2397static void end_buffer_read_nobh(struct buffer_head *bh, int uptodate)
2398{
68671f35 2399 __end_buffer_read_notouch(bh, uptodate);
1da177e4
LT
2400}
2401
03158cd7
NP
2402/*
2403 * Attach the singly-linked list of buffers created by nobh_write_begin, to
2404 * the page (converting it to circular linked list and taking care of page
2405 * dirty races).
2406 */
2407static void attach_nobh_buffers(struct page *page, struct buffer_head *head)
2408{
2409 struct buffer_head *bh;
2410
2411 BUG_ON(!PageLocked(page));
2412
2413 spin_lock(&page->mapping->private_lock);
2414 bh = head;
2415 do {
2416 if (PageDirty(page))
2417 set_buffer_dirty(bh);
2418 if (!bh->b_this_page)
2419 bh->b_this_page = head;
2420 bh = bh->b_this_page;
2421 } while (bh != head);
2422 attach_page_buffers(page, head);
2423 spin_unlock(&page->mapping->private_lock);
2424}
2425
1da177e4 2426/*
ea0f04e5
CH
2427 * On entry, the page is fully not uptodate.
2428 * On exit the page is fully uptodate in the areas outside (from,to)
7bb46a67 2429 * The filesystem needs to handle block truncation upon failure.
1da177e4 2430 */
ea0f04e5 2431int nobh_write_begin(struct address_space *mapping,
03158cd7
NP
2432 loff_t pos, unsigned len, unsigned flags,
2433 struct page **pagep, void **fsdata,
1da177e4
LT
2434 get_block_t *get_block)
2435{
03158cd7 2436 struct inode *inode = mapping->host;
1da177e4
LT
2437 const unsigned blkbits = inode->i_blkbits;
2438 const unsigned blocksize = 1 << blkbits;
a4b0672d 2439 struct buffer_head *head, *bh;
03158cd7
NP
2440 struct page *page;
2441 pgoff_t index;
2442 unsigned from, to;
1da177e4 2443 unsigned block_in_page;
a4b0672d 2444 unsigned block_start, block_end;
1da177e4 2445 sector_t block_in_file;
1da177e4 2446 int nr_reads = 0;
1da177e4
LT
2447 int ret = 0;
2448 int is_mapped_to_disk = 1;
1da177e4 2449
03158cd7
NP
2450 index = pos >> PAGE_CACHE_SHIFT;
2451 from = pos & (PAGE_CACHE_SIZE - 1);
2452 to = from + len;
2453
54566b2c 2454 page = grab_cache_page_write_begin(mapping, index, flags);
03158cd7
NP
2455 if (!page)
2456 return -ENOMEM;
2457 *pagep = page;
2458 *fsdata = NULL;
2459
2460 if (page_has_buffers(page)) {
2461 unlock_page(page);
2462 page_cache_release(page);
2463 *pagep = NULL;
155130a4
CH
2464 return block_write_begin(mapping, pos, len, flags, pagep,
2465 get_block);
03158cd7 2466 }
a4b0672d 2467
1da177e4
LT
2468 if (PageMappedToDisk(page))
2469 return 0;
2470
a4b0672d
NP
2471 /*
2472 * Allocate buffers so that we can keep track of state, and potentially
2473 * attach them to the page if an error occurs. In the common case of
2474 * no error, they will just be freed again without ever being attached
2475 * to the page (which is all OK, because we're under the page lock).
2476 *
2477 * Be careful: the buffer linked list is a NULL terminated one, rather
2478 * than the circular one we're used to.
2479 */
2480 head = alloc_page_buffers(page, blocksize, 0);
03158cd7
NP
2481 if (!head) {
2482 ret = -ENOMEM;
2483 goto out_release;
2484 }
a4b0672d 2485
1da177e4 2486 block_in_file = (sector_t)page->index << (PAGE_CACHE_SHIFT - blkbits);
1da177e4
LT
2487
2488 /*
2489 * We loop across all blocks in the page, whether or not they are
2490 * part of the affected region. This is so we can discover if the
2491 * page is fully mapped-to-disk.
2492 */
a4b0672d 2493 for (block_start = 0, block_in_page = 0, bh = head;
1da177e4 2494 block_start < PAGE_CACHE_SIZE;
a4b0672d 2495 block_in_page++, block_start += blocksize, bh = bh->b_this_page) {
1da177e4
LT
2496 int create;
2497
a4b0672d
NP
2498 block_end = block_start + blocksize;
2499 bh->b_state = 0;
1da177e4
LT
2500 create = 1;
2501 if (block_start >= to)
2502 create = 0;
2503 ret = get_block(inode, block_in_file + block_in_page,
a4b0672d 2504 bh, create);
1da177e4
LT
2505 if (ret)
2506 goto failed;
a4b0672d 2507 if (!buffer_mapped(bh))
1da177e4 2508 is_mapped_to_disk = 0;
a4b0672d
NP
2509 if (buffer_new(bh))
2510 unmap_underlying_metadata(bh->b_bdev, bh->b_blocknr);
2511 if (PageUptodate(page)) {
2512 set_buffer_uptodate(bh);
1da177e4 2513 continue;
a4b0672d
NP
2514 }
2515 if (buffer_new(bh) || !buffer_mapped(bh)) {
eebd2aa3
CL
2516 zero_user_segments(page, block_start, from,
2517 to, block_end);
1da177e4
LT
2518 continue;
2519 }
a4b0672d 2520 if (buffer_uptodate(bh))
1da177e4
LT
2521 continue; /* reiserfs does this */
2522 if (block_start < from || block_end > to) {
a4b0672d
NP
2523 lock_buffer(bh);
2524 bh->b_end_io = end_buffer_read_nobh;
2525 submit_bh(READ, bh);
2526 nr_reads++;
1da177e4
LT
2527 }
2528 }
2529
2530 if (nr_reads) {
1da177e4
LT
2531 /*
2532 * The page is locked, so these buffers are protected from
2533 * any VM or truncate activity. Hence we don't need to care
2534 * for the buffer_head refcounts.
2535 */
a4b0672d 2536 for (bh = head; bh; bh = bh->b_this_page) {
1da177e4
LT
2537 wait_on_buffer(bh);
2538 if (!buffer_uptodate(bh))
2539 ret = -EIO;
1da177e4
LT
2540 }
2541 if (ret)
2542 goto failed;
2543 }
2544
2545 if (is_mapped_to_disk)
2546 SetPageMappedToDisk(page);
1da177e4 2547
03158cd7 2548 *fsdata = head; /* to be released by nobh_write_end */
a4b0672d 2549
1da177e4
LT
2550 return 0;
2551
2552failed:
03158cd7 2553 BUG_ON(!ret);
1da177e4 2554 /*
a4b0672d
NP
2555 * Error recovery is a bit difficult. We need to zero out blocks that
2556 * were newly allocated, and dirty them to ensure they get written out.
2557 * Buffers need to be attached to the page at this point, otherwise
2558 * the handling of potential IO errors during writeout would be hard
2559 * (could try doing synchronous writeout, but what if that fails too?)
1da177e4 2560 */
03158cd7
NP
2561 attach_nobh_buffers(page, head);
2562 page_zero_new_buffers(page, from, to);
a4b0672d 2563
03158cd7
NP
2564out_release:
2565 unlock_page(page);
2566 page_cache_release(page);
2567 *pagep = NULL;
a4b0672d 2568
7bb46a67 2569 return ret;
2570}
03158cd7 2571EXPORT_SYMBOL(nobh_write_begin);
1da177e4 2572
03158cd7
NP
2573int nobh_write_end(struct file *file, struct address_space *mapping,
2574 loff_t pos, unsigned len, unsigned copied,
2575 struct page *page, void *fsdata)
1da177e4
LT
2576{
2577 struct inode *inode = page->mapping->host;
efdc3131 2578 struct buffer_head *head = fsdata;
03158cd7 2579 struct buffer_head *bh;
5b41e74a 2580 BUG_ON(fsdata != NULL && page_has_buffers(page));
1da177e4 2581
d4cf109f 2582 if (unlikely(copied < len) && head)
5b41e74a
DM
2583 attach_nobh_buffers(page, head);
2584 if (page_has_buffers(page))
2585 return generic_write_end(file, mapping, pos, len,
2586 copied, page, fsdata);
a4b0672d 2587
22c8ca78 2588 SetPageUptodate(page);
1da177e4 2589 set_page_dirty(page);
03158cd7
NP
2590 if (pos+copied > inode->i_size) {
2591 i_size_write(inode, pos+copied);
1da177e4
LT
2592 mark_inode_dirty(inode);
2593 }
03158cd7
NP
2594
2595 unlock_page(page);
2596 page_cache_release(page);
2597
03158cd7
NP
2598 while (head) {
2599 bh = head;
2600 head = head->b_this_page;
2601 free_buffer_head(bh);
2602 }
2603
2604 return copied;
1da177e4 2605}
03158cd7 2606EXPORT_SYMBOL(nobh_write_end);
1da177e4
LT
2607
2608/*
2609 * nobh_writepage() - based on block_full_write_page() except
2610 * that it tries to operate without attaching bufferheads to
2611 * the page.
2612 */
2613int nobh_writepage(struct page *page, get_block_t *get_block,
2614 struct writeback_control *wbc)
2615{
2616 struct inode * const inode = page->mapping->host;
2617 loff_t i_size = i_size_read(inode);
2618 const pgoff_t end_index = i_size >> PAGE_CACHE_SHIFT;
2619 unsigned offset;
1da177e4
LT
2620 int ret;
2621
2622 /* Is the page fully inside i_size? */
2623 if (page->index < end_index)
2624 goto out;
2625
2626 /* Is the page fully outside i_size? (truncate in progress) */
2627 offset = i_size & (PAGE_CACHE_SIZE-1);
2628 if (page->index >= end_index+1 || !offset) {
2629 /*
2630 * The page may have dirty, unmapped buffers. For example,
2631 * they may have been added in ext3_writepage(). Make them
2632 * freeable here, so the page does not leak.
2633 */
2634#if 0
2635 /* Not really sure about this - do we need this ? */
2636 if (page->mapping->a_ops->invalidatepage)
2637 page->mapping->a_ops->invalidatepage(page, offset);
2638#endif
2639 unlock_page(page);
2640 return 0; /* don't care */
2641 }
2642
2643 /*
2644 * The page straddles i_size. It must be zeroed out on each and every
2645 * writepage invocation because it may be mmapped. "A file is mapped
2646 * in multiples of the page size. For a file that is not a multiple of
2647 * the page size, the remaining memory is zeroed when mapped, and
2648 * writes to that region are not written out to the file."
2649 */
eebd2aa3 2650 zero_user_segment(page, offset, PAGE_CACHE_SIZE);
1da177e4
LT
2651out:
2652 ret = mpage_writepage(page, get_block, wbc);
2653 if (ret == -EAGAIN)
35c80d5f
CM
2654 ret = __block_write_full_page(inode, page, get_block, wbc,
2655 end_buffer_async_write);
1da177e4
LT
2656 return ret;
2657}
2658EXPORT_SYMBOL(nobh_writepage);
2659
03158cd7
NP
2660int nobh_truncate_page(struct address_space *mapping,
2661 loff_t from, get_block_t *get_block)
1da177e4 2662{
1da177e4
LT
2663 pgoff_t index = from >> PAGE_CACHE_SHIFT;
2664 unsigned offset = from & (PAGE_CACHE_SIZE-1);
03158cd7
NP
2665 unsigned blocksize;
2666 sector_t iblock;
2667 unsigned length, pos;
2668 struct inode *inode = mapping->host;
1da177e4 2669 struct page *page;
03158cd7
NP
2670 struct buffer_head map_bh;
2671 int err;
1da177e4 2672
03158cd7
NP
2673 blocksize = 1 << inode->i_blkbits;
2674 length = offset & (blocksize - 1);
2675
2676 /* Block boundary? Nothing to do */
2677 if (!length)
2678 return 0;
2679
2680 length = blocksize - length;
2681 iblock = (sector_t)index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
1da177e4 2682
1da177e4 2683 page = grab_cache_page(mapping, index);
03158cd7 2684 err = -ENOMEM;
1da177e4
LT
2685 if (!page)
2686 goto out;
2687
03158cd7
NP
2688 if (page_has_buffers(page)) {
2689has_buffers:
2690 unlock_page(page);
2691 page_cache_release(page);
2692 return block_truncate_page(mapping, from, get_block);
2693 }
2694
2695 /* Find the buffer that contains "offset" */
2696 pos = blocksize;
2697 while (offset >= pos) {
2698 iblock++;
2699 pos += blocksize;
2700 }
2701
460bcf57
TT
2702 map_bh.b_size = blocksize;
2703 map_bh.b_state = 0;
03158cd7
NP
2704 err = get_block(inode, iblock, &map_bh, 0);
2705 if (err)
2706 goto unlock;
2707 /* unmapped? It's a hole - nothing to do */
2708 if (!buffer_mapped(&map_bh))
2709 goto unlock;
2710
2711 /* Ok, it's mapped. Make sure it's up-to-date */
2712 if (!PageUptodate(page)) {
2713 err = mapping->a_ops->readpage(NULL, page);
2714 if (err) {
2715 page_cache_release(page);
2716 goto out;
2717 }
2718 lock_page(page);
2719 if (!PageUptodate(page)) {
2720 err = -EIO;
2721 goto unlock;
2722 }
2723 if (page_has_buffers(page))
2724 goto has_buffers;
1da177e4 2725 }
eebd2aa3 2726 zero_user(page, offset, length);
03158cd7
NP
2727 set_page_dirty(page);
2728 err = 0;
2729
2730unlock:
1da177e4
LT
2731 unlock_page(page);
2732 page_cache_release(page);
2733out:
03158cd7 2734 return err;
1da177e4
LT
2735}
2736EXPORT_SYMBOL(nobh_truncate_page);
2737
2738int block_truncate_page(struct address_space *mapping,
2739 loff_t from, get_block_t *get_block)
2740{
2741 pgoff_t index = from >> PAGE_CACHE_SHIFT;
2742 unsigned offset = from & (PAGE_CACHE_SIZE-1);
2743 unsigned blocksize;
54b21a79 2744 sector_t iblock;
1da177e4
LT
2745 unsigned length, pos;
2746 struct inode *inode = mapping->host;
2747 struct page *page;
2748 struct buffer_head *bh;
1da177e4
LT
2749 int err;
2750
2751 blocksize = 1 << inode->i_blkbits;
2752 length = offset & (blocksize - 1);
2753
2754 /* Block boundary? Nothing to do */
2755 if (!length)
2756 return 0;
2757
2758 length = blocksize - length;
54b21a79 2759 iblock = (sector_t)index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
1da177e4
LT
2760
2761 page = grab_cache_page(mapping, index);
2762 err = -ENOMEM;
2763 if (!page)
2764 goto out;
2765
2766 if (!page_has_buffers(page))
2767 create_empty_buffers(page, blocksize, 0);
2768
2769 /* Find the buffer that contains "offset" */
2770 bh = page_buffers(page);
2771 pos = blocksize;
2772 while (offset >= pos) {
2773 bh = bh->b_this_page;
2774 iblock++;
2775 pos += blocksize;
2776 }
2777
2778 err = 0;
2779 if (!buffer_mapped(bh)) {
b0cf2321 2780 WARN_ON(bh->b_size != blocksize);
1da177e4
LT
2781 err = get_block(inode, iblock, bh, 0);
2782 if (err)
2783 goto unlock;
2784 /* unmapped? It's a hole - nothing to do */
2785 if (!buffer_mapped(bh))
2786 goto unlock;
2787 }
2788
2789 /* Ok, it's mapped. Make sure it's up-to-date */
2790 if (PageUptodate(page))
2791 set_buffer_uptodate(bh);
2792
33a266dd 2793 if (!buffer_uptodate(bh) && !buffer_delay(bh) && !buffer_unwritten(bh)) {
1da177e4
LT
2794 err = -EIO;
2795 ll_rw_block(READ, 1, &bh);
2796 wait_on_buffer(bh);
2797 /* Uhhuh. Read error. Complain and punt. */
2798 if (!buffer_uptodate(bh))
2799 goto unlock;
2800 }
2801
eebd2aa3 2802 zero_user(page, offset, length);
1da177e4
LT
2803 mark_buffer_dirty(bh);
2804 err = 0;
2805
2806unlock:
2807 unlock_page(page);
2808 page_cache_release(page);
2809out:
2810 return err;
2811}
1fe72eaa 2812EXPORT_SYMBOL(block_truncate_page);
1da177e4
LT
2813
2814/*
2815 * The generic ->writepage function for buffer-backed address_spaces
35c80d5f 2816 * this form passes in the end_io handler used to finish the IO.
1da177e4 2817 */
35c80d5f
CM
2818int block_write_full_page_endio(struct page *page, get_block_t *get_block,
2819 struct writeback_control *wbc, bh_end_io_t *handler)
1da177e4
LT
2820{
2821 struct inode * const inode = page->mapping->host;
2822 loff_t i_size = i_size_read(inode);
2823 const pgoff_t end_index = i_size >> PAGE_CACHE_SHIFT;
2824 unsigned offset;
1da177e4
LT
2825
2826 /* Is the page fully inside i_size? */
2827 if (page->index < end_index)
35c80d5f
CM
2828 return __block_write_full_page(inode, page, get_block, wbc,
2829 handler);
1da177e4
LT
2830
2831 /* Is the page fully outside i_size? (truncate in progress) */
2832 offset = i_size & (PAGE_CACHE_SIZE-1);
2833 if (page->index >= end_index+1 || !offset) {
2834 /*
2835 * The page may have dirty, unmapped buffers. For example,
2836 * they may have been added in ext3_writepage(). Make them
2837 * freeable here, so the page does not leak.
2838 */
aaa4059b 2839 do_invalidatepage(page, 0);
1da177e4
LT
2840 unlock_page(page);
2841 return 0; /* don't care */
2842 }
2843
2844 /*
2845 * The page straddles i_size. It must be zeroed out on each and every
2a61aa40 2846 * writepage invocation because it may be mmapped. "A file is mapped
1da177e4
LT
2847 * in multiples of the page size. For a file that is not a multiple of
2848 * the page size, the remaining memory is zeroed when mapped, and
2849 * writes to that region are not written out to the file."
2850 */
eebd2aa3 2851 zero_user_segment(page, offset, PAGE_CACHE_SIZE);
35c80d5f 2852 return __block_write_full_page(inode, page, get_block, wbc, handler);
1da177e4 2853}
1fe72eaa 2854EXPORT_SYMBOL(block_write_full_page_endio);
1da177e4 2855
35c80d5f
CM
2856/*
2857 * The generic ->writepage function for buffer-backed address_spaces
2858 */
2859int block_write_full_page(struct page *page, get_block_t *get_block,
2860 struct writeback_control *wbc)
2861{
2862 return block_write_full_page_endio(page, get_block, wbc,
2863 end_buffer_async_write);
2864}
1fe72eaa 2865EXPORT_SYMBOL(block_write_full_page);
35c80d5f 2866
1da177e4
LT
2867sector_t generic_block_bmap(struct address_space *mapping, sector_t block,
2868 get_block_t *get_block)
2869{
2870 struct buffer_head tmp;
2871 struct inode *inode = mapping->host;
2872 tmp.b_state = 0;
2873 tmp.b_blocknr = 0;
b0cf2321 2874 tmp.b_size = 1 << inode->i_blkbits;
1da177e4
LT
2875 get_block(inode, block, &tmp, 0);
2876 return tmp.b_blocknr;
2877}
1fe72eaa 2878EXPORT_SYMBOL(generic_block_bmap);
1da177e4 2879
6712ecf8 2880static void end_bio_bh_io_sync(struct bio *bio, int err)
1da177e4
LT
2881{
2882 struct buffer_head *bh = bio->bi_private;
2883
1da177e4
LT
2884 if (err == -EOPNOTSUPP) {
2885 set_bit(BIO_EOPNOTSUPP, &bio->bi_flags);
1da177e4
LT
2886 }
2887
08bafc03
KM
2888 if (unlikely (test_bit(BIO_QUIET,&bio->bi_flags)))
2889 set_bit(BH_Quiet, &bh->b_state);
2890
1da177e4
LT
2891 bh->b_end_io(bh, test_bit(BIO_UPTODATE, &bio->bi_flags));
2892 bio_put(bio);
1da177e4
LT
2893}
2894
2895int submit_bh(int rw, struct buffer_head * bh)
2896{
2897 struct bio *bio;
2898 int ret = 0;
2899
2900 BUG_ON(!buffer_locked(bh));
2901 BUG_ON(!buffer_mapped(bh));
2902 BUG_ON(!bh->b_end_io);
8fb0e342
AK
2903 BUG_ON(buffer_delay(bh));
2904 BUG_ON(buffer_unwritten(bh));
1da177e4 2905
1da177e4 2906 /*
48fd4f93 2907 * Only clear out a write error when rewriting
1da177e4 2908 */
48fd4f93 2909 if (test_set_buffer_req(bh) && (rw & WRITE))
1da177e4
LT
2910 clear_buffer_write_io_error(bh);
2911
2912 /*
2913 * from here on down, it's all bio -- do the initial mapping,
2914 * submit_bio -> generic_make_request may further map this bio around
2915 */
2916 bio = bio_alloc(GFP_NOIO, 1);
2917
2918 bio->bi_sector = bh->b_blocknr * (bh->b_size >> 9);
2919 bio->bi_bdev = bh->b_bdev;
2920 bio->bi_io_vec[0].bv_page = bh->b_page;
2921 bio->bi_io_vec[0].bv_len = bh->b_size;
2922 bio->bi_io_vec[0].bv_offset = bh_offset(bh);
2923
2924 bio->bi_vcnt = 1;
2925 bio->bi_idx = 0;
2926 bio->bi_size = bh->b_size;
2927
2928 bio->bi_end_io = end_bio_bh_io_sync;
2929 bio->bi_private = bh;
2930
2931 bio_get(bio);
2932 submit_bio(rw, bio);
2933
2934 if (bio_flagged(bio, BIO_EOPNOTSUPP))
2935 ret = -EOPNOTSUPP;
2936
2937 bio_put(bio);
2938 return ret;
2939}
1fe72eaa 2940EXPORT_SYMBOL(submit_bh);
1da177e4
LT
2941
2942/**
2943 * ll_rw_block: low-level access to block devices (DEPRECATED)
9cb569d6 2944 * @rw: whether to %READ or %WRITE or maybe %READA (readahead)
1da177e4
LT
2945 * @nr: number of &struct buffer_heads in the array
2946 * @bhs: array of pointers to &struct buffer_head
2947 *
a7662236
JK
2948 * ll_rw_block() takes an array of pointers to &struct buffer_heads, and
2949 * requests an I/O operation on them, either a %READ or a %WRITE. The third
9cb569d6
CH
2950 * %READA option is described in the documentation for generic_make_request()
2951 * which ll_rw_block() calls.
1da177e4
LT
2952 *
2953 * This function drops any buffer that it cannot get a lock on (with the
9cb569d6
CH
2954 * BH_Lock state bit), any buffer that appears to be clean when doing a write
2955 * request, and any buffer that appears to be up-to-date when doing read
2956 * request. Further it marks as clean buffers that are processed for
2957 * writing (the buffer cache won't assume that they are actually clean
2958 * until the buffer gets unlocked).
1da177e4
LT
2959 *
2960 * ll_rw_block sets b_end_io to simple completion handler that marks
2961 * the buffer up-to-date (if approriate), unlocks the buffer and wakes
2962 * any waiters.
2963 *
2964 * All of the buffers must be for the same device, and must also be a
2965 * multiple of the current approved size for the device.
2966 */
2967void ll_rw_block(int rw, int nr, struct buffer_head *bhs[])
2968{
2969 int i;
2970
2971 for (i = 0; i < nr; i++) {
2972 struct buffer_head *bh = bhs[i];
2973
9cb569d6 2974 if (!trylock_buffer(bh))
1da177e4 2975 continue;
9cb569d6 2976 if (rw == WRITE) {
1da177e4 2977 if (test_clear_buffer_dirty(bh)) {
76c3073a 2978 bh->b_end_io = end_buffer_write_sync;
e60e5c50 2979 get_bh(bh);
9cb569d6 2980 submit_bh(WRITE, bh);
1da177e4
LT
2981 continue;
2982 }
2983 } else {
1da177e4 2984 if (!buffer_uptodate(bh)) {
76c3073a 2985 bh->b_end_io = end_buffer_read_sync;
e60e5c50 2986 get_bh(bh);
1da177e4
LT
2987 submit_bh(rw, bh);
2988 continue;
2989 }
2990 }
2991 unlock_buffer(bh);
1da177e4
LT
2992 }
2993}
1fe72eaa 2994EXPORT_SYMBOL(ll_rw_block);
1da177e4 2995
9cb569d6
CH
2996void write_dirty_buffer(struct buffer_head *bh, int rw)
2997{
2998 lock_buffer(bh);
2999 if (!test_clear_buffer_dirty(bh)) {
3000 unlock_buffer(bh);
3001 return;
3002 }
3003 bh->b_end_io = end_buffer_write_sync;
3004 get_bh(bh);
3005 submit_bh(rw, bh);
3006}
3007EXPORT_SYMBOL(write_dirty_buffer);
3008
1da177e4
LT
3009/*
3010 * For a data-integrity writeout, we need to wait upon any in-progress I/O
3011 * and then start new I/O and then wait upon it. The caller must have a ref on
3012 * the buffer_head.
3013 */
87e99511 3014int __sync_dirty_buffer(struct buffer_head *bh, int rw)
1da177e4
LT
3015{
3016 int ret = 0;
3017
3018 WARN_ON(atomic_read(&bh->b_count) < 1);
3019 lock_buffer(bh);
3020 if (test_clear_buffer_dirty(bh)) {
3021 get_bh(bh);
3022 bh->b_end_io = end_buffer_write_sync;
87e99511 3023 ret = submit_bh(rw, bh);
1da177e4 3024 wait_on_buffer(bh);
1da177e4
LT
3025 if (!ret && !buffer_uptodate(bh))
3026 ret = -EIO;
3027 } else {
3028 unlock_buffer(bh);
3029 }
3030 return ret;
3031}
87e99511
CH
3032EXPORT_SYMBOL(__sync_dirty_buffer);
3033
3034int sync_dirty_buffer(struct buffer_head *bh)
3035{
3036 return __sync_dirty_buffer(bh, WRITE_SYNC);
3037}
1fe72eaa 3038EXPORT_SYMBOL(sync_dirty_buffer);
1da177e4
LT
3039
3040/*
3041 * try_to_free_buffers() checks if all the buffers on this particular page
3042 * are unused, and releases them if so.
3043 *
3044 * Exclusion against try_to_free_buffers may be obtained by either
3045 * locking the page or by holding its mapping's private_lock.
3046 *
3047 * If the page is dirty but all the buffers are clean then we need to
3048 * be sure to mark the page clean as well. This is because the page
3049 * may be against a block device, and a later reattachment of buffers
3050 * to a dirty page will set *all* buffers dirty. Which would corrupt
3051 * filesystem data on the same device.
3052 *
3053 * The same applies to regular filesystem pages: if all the buffers are
3054 * clean then we set the page clean and proceed. To do that, we require
3055 * total exclusion from __set_page_dirty_buffers(). That is obtained with
3056 * private_lock.
3057 *
3058 * try_to_free_buffers() is non-blocking.
3059 */
3060static inline int buffer_busy(struct buffer_head *bh)
3061{
3062 return atomic_read(&bh->b_count) |
3063 (bh->b_state & ((1 << BH_Dirty) | (1 << BH_Lock)));
3064}
3065
3066static int
3067drop_buffers(struct page *page, struct buffer_head **buffers_to_free)
3068{
3069 struct buffer_head *head = page_buffers(page);
3070 struct buffer_head *bh;
3071
3072 bh = head;
3073 do {
de7d5a3b 3074 if (buffer_write_io_error(bh) && page->mapping)
1da177e4
LT
3075 set_bit(AS_EIO, &page->mapping->flags);
3076 if (buffer_busy(bh))
3077 goto failed;
3078 bh = bh->b_this_page;
3079 } while (bh != head);
3080
3081 do {
3082 struct buffer_head *next = bh->b_this_page;
3083
535ee2fb 3084 if (bh->b_assoc_map)
1da177e4
LT
3085 __remove_assoc_queue(bh);
3086 bh = next;
3087 } while (bh != head);
3088 *buffers_to_free = head;
3089 __clear_page_buffers(page);
3090 return 1;
3091failed:
3092 return 0;
3093}
3094
3095int try_to_free_buffers(struct page *page)
3096{
3097 struct address_space * const mapping = page->mapping;
3098 struct buffer_head *buffers_to_free = NULL;
3099 int ret = 0;
3100
3101 BUG_ON(!PageLocked(page));
ecdfc978 3102 if (PageWriteback(page))
1da177e4
LT
3103 return 0;
3104
3105 if (mapping == NULL) { /* can this still happen? */
3106 ret = drop_buffers(page, &buffers_to_free);
3107 goto out;
3108 }
3109
3110 spin_lock(&mapping->private_lock);
3111 ret = drop_buffers(page, &buffers_to_free);
ecdfc978
LT
3112
3113 /*
3114 * If the filesystem writes its buffers by hand (eg ext3)
3115 * then we can have clean buffers against a dirty page. We
3116 * clean the page here; otherwise the VM will never notice
3117 * that the filesystem did any IO at all.
3118 *
3119 * Also, during truncate, discard_buffer will have marked all
3120 * the page's buffers clean. We discover that here and clean
3121 * the page also.
87df7241
NP
3122 *
3123 * private_lock must be held over this entire operation in order
3124 * to synchronise against __set_page_dirty_buffers and prevent the
3125 * dirty bit from being lost.
ecdfc978
LT
3126 */
3127 if (ret)
3128 cancel_dirty_page(page, PAGE_CACHE_SIZE);
87df7241 3129 spin_unlock(&mapping->private_lock);
1da177e4
LT
3130out:
3131 if (buffers_to_free) {
3132 struct buffer_head *bh = buffers_to_free;
3133
3134 do {
3135 struct buffer_head *next = bh->b_this_page;
3136 free_buffer_head(bh);
3137 bh = next;
3138 } while (bh != buffers_to_free);
3139 }
3140 return ret;
3141}
3142EXPORT_SYMBOL(try_to_free_buffers);
3143
3978d717 3144void block_sync_page(struct page *page)
1da177e4
LT
3145{
3146 struct address_space *mapping;
3147
3148 smp_mb();
3149 mapping = page_mapping(page);
3150 if (mapping)
3151 blk_run_backing_dev(mapping->backing_dev_info, page);
1da177e4 3152}
1fe72eaa 3153EXPORT_SYMBOL(block_sync_page);
1da177e4
LT
3154
3155/*
3156 * There are no bdflush tunables left. But distributions are
3157 * still running obsolete flush daemons, so we terminate them here.
3158 *
3159 * Use of bdflush() is deprecated and will be removed in a future kernel.
5b0830cb 3160 * The `flush-X' kernel threads fully replace bdflush daemons and this call.
1da177e4 3161 */
bdc480e3 3162SYSCALL_DEFINE2(bdflush, int, func, long, data)
1da177e4
LT
3163{
3164 static int msg_count;
3165
3166 if (!capable(CAP_SYS_ADMIN))
3167 return -EPERM;
3168
3169 if (msg_count < 5) {
3170 msg_count++;
3171 printk(KERN_INFO
3172 "warning: process `%s' used the obsolete bdflush"
3173 " system call\n", current->comm);
3174 printk(KERN_INFO "Fix your initscripts?\n");
3175 }
3176
3177 if (func == 1)
3178 do_exit(0);
3179 return 0;
3180}
3181
3182/*
3183 * Buffer-head allocation
3184 */
e18b890b 3185static struct kmem_cache *bh_cachep;
1da177e4
LT
3186
3187/*
3188 * Once the number of bh's in the machine exceeds this level, we start
3189 * stripping them in writeback.
3190 */
3191static int max_buffer_heads;
3192
3193int buffer_heads_over_limit;
3194
3195struct bh_accounting {
3196 int nr; /* Number of live bh's */
3197 int ratelimit; /* Limit cacheline bouncing */
3198};
3199
3200static DEFINE_PER_CPU(struct bh_accounting, bh_accounting) = {0, 0};
3201
3202static void recalc_bh_state(void)
3203{
3204 int i;
3205 int tot = 0;
3206
3207 if (__get_cpu_var(bh_accounting).ratelimit++ < 4096)
3208 return;
3209 __get_cpu_var(bh_accounting).ratelimit = 0;
8a143426 3210 for_each_online_cpu(i)
1da177e4
LT
3211 tot += per_cpu(bh_accounting, i).nr;
3212 buffer_heads_over_limit = (tot > max_buffer_heads);
3213}
3214
dd0fc66f 3215struct buffer_head *alloc_buffer_head(gfp_t gfp_flags)
1da177e4 3216{
019b4d12 3217 struct buffer_head *ret = kmem_cache_zalloc(bh_cachep, gfp_flags);
1da177e4 3218 if (ret) {
a35afb83 3219 INIT_LIST_HEAD(&ret->b_assoc_buffers);
736c7b80 3220 get_cpu_var(bh_accounting).nr++;
1da177e4 3221 recalc_bh_state();
736c7b80 3222 put_cpu_var(bh_accounting);
1da177e4
LT
3223 }
3224 return ret;
3225}
3226EXPORT_SYMBOL(alloc_buffer_head);
3227
3228void free_buffer_head(struct buffer_head *bh)
3229{
3230 BUG_ON(!list_empty(&bh->b_assoc_buffers));
3231 kmem_cache_free(bh_cachep, bh);
736c7b80 3232 get_cpu_var(bh_accounting).nr--;
1da177e4 3233 recalc_bh_state();
736c7b80 3234 put_cpu_var(bh_accounting);
1da177e4
LT
3235}
3236EXPORT_SYMBOL(free_buffer_head);
3237
1da177e4
LT
3238static void buffer_exit_cpu(int cpu)
3239{
3240 int i;
3241 struct bh_lru *b = &per_cpu(bh_lrus, cpu);
3242
3243 for (i = 0; i < BH_LRU_SIZE; i++) {
3244 brelse(b->bhs[i]);
3245 b->bhs[i] = NULL;
3246 }
8a143426
ED
3247 get_cpu_var(bh_accounting).nr += per_cpu(bh_accounting, cpu).nr;
3248 per_cpu(bh_accounting, cpu).nr = 0;
3249 put_cpu_var(bh_accounting);
1da177e4
LT
3250}
3251
3252static int buffer_cpu_notify(struct notifier_block *self,
3253 unsigned long action, void *hcpu)
3254{
8bb78442 3255 if (action == CPU_DEAD || action == CPU_DEAD_FROZEN)
1da177e4
LT
3256 buffer_exit_cpu((unsigned long)hcpu);
3257 return NOTIFY_OK;
3258}
1da177e4 3259
389d1b08 3260/**
a6b91919 3261 * bh_uptodate_or_lock - Test whether the buffer is uptodate
389d1b08
AK
3262 * @bh: struct buffer_head
3263 *
3264 * Return true if the buffer is up-to-date and false,
3265 * with the buffer locked, if not.
3266 */
3267int bh_uptodate_or_lock(struct buffer_head *bh)
3268{
3269 if (!buffer_uptodate(bh)) {
3270 lock_buffer(bh);
3271 if (!buffer_uptodate(bh))
3272 return 0;
3273 unlock_buffer(bh);
3274 }
3275 return 1;
3276}
3277EXPORT_SYMBOL(bh_uptodate_or_lock);
3278
3279/**
a6b91919 3280 * bh_submit_read - Submit a locked buffer for reading
389d1b08
AK
3281 * @bh: struct buffer_head
3282 *
3283 * Returns zero on success and -EIO on error.
3284 */
3285int bh_submit_read(struct buffer_head *bh)
3286{
3287 BUG_ON(!buffer_locked(bh));
3288
3289 if (buffer_uptodate(bh)) {
3290 unlock_buffer(bh);
3291 return 0;
3292 }
3293
3294 get_bh(bh);
3295 bh->b_end_io = end_buffer_read_sync;
3296 submit_bh(READ, bh);
3297 wait_on_buffer(bh);
3298 if (buffer_uptodate(bh))
3299 return 0;
3300 return -EIO;
3301}
3302EXPORT_SYMBOL(bh_submit_read);
3303
1da177e4
LT
3304void __init buffer_init(void)
3305{
3306 int nrpages;
3307
b98938c3
CL
3308 bh_cachep = kmem_cache_create("buffer_head",
3309 sizeof(struct buffer_head), 0,
3310 (SLAB_RECLAIM_ACCOUNT|SLAB_PANIC|
3311 SLAB_MEM_SPREAD),
019b4d12 3312 NULL);
1da177e4
LT
3313
3314 /*
3315 * Limit the bh occupancy to 10% of ZONE_NORMAL
3316 */
3317 nrpages = (nr_free_buffer_pages() * 10) / 100;
3318 max_buffer_heads = nrpages * (PAGE_SIZE / sizeof(struct buffer_head));
3319 hotcpu_notifier(buffer_cpu_notify, 0);
3320}