fs/xfs/linux-2.6/xfs_buf.c

   1 /*
   2  * Copyright (c) 2000-2006 Silicon Graphics, Inc.
   3  * All Rights Reserved.
   4  *
   5  * This program is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU General Public License as
   7  * published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope that it would be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program; if not, write the Free Software Foundation,
  16  * Inc.,  51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  17  */
  18 #include "xfs.h"
  19 #include <linux/stddef.h>
  20 #include <linux/errno.h>
  21 #include <linux/slab.h>
  22 #include <linux/pagemap.h>
  23 #include <linux/init.h>
  24 #include <linux/vmalloc.h>
  25 #include <linux/bio.h>
  26 #include <linux/sysctl.h>
  27 #include <linux/proc_fs.h>
  28 #include <linux/workqueue.h>
  29 #include <linux/percpu.h>
  30 #include <linux/blkdev.h>
  31 #include <linux/hash.h>
  32 #include <linux/kthread.h>
  33 #include <linux/migrate.h>
  34 #include <linux/backing-dev.h>
  35 #include <linux/freezer.h>
  36
  37 #include "xfs_sb.h"
  38 #include "xfs_inum.h"
  39 #include "xfs_ag.h"
  40 #include "xfs_dmapi.h"
  41 #include "xfs_mount.h"
  42 #include "xfs_trace.h"
  43
  44 static kmem_zone_t *xfs_buf_zone;
  45 STATIC int xfsbufd(void *);
  46 STATIC int xfsbufd_wakeup(int, gfp_t);
  47 STATIC void xfs_buf_delwri_queue(xfs_buf_t *, int);
  48 static struct shrinker xfs_buf_shake = {
  49         .shrink = xfsbufd_wakeup,
  50         .seeks = DEFAULT_SEEKS,
  51 };
  52
  53 static struct workqueue_struct *xfslogd_workqueue;
  54 struct workqueue_struct *xfsdatad_workqueue;
  55 struct workqueue_struct *xfsconvertd_workqueue;
  56
  57 #ifdef XFS_BUF_LOCK_TRACKING
  58 # define XB_SET_OWNER(bp)       ((bp)->b_last_holder = current->pid)
  59 # define XB_CLEAR_OWNER(bp)     ((bp)->b_last_holder = -1)
  60 # define XB_GET_OWNER(bp)       ((bp)->b_last_holder)
  61 #else
  62 # define XB_SET_OWNER(bp)       do { } while (0)
  63 # define XB_CLEAR_OWNER(bp)     do { } while (0)
  64 # define XB_GET_OWNER(bp)       do { } while (0)
  65 #endif
  66
  67 #define xb_to_gfp(flags) \
  68         ((((flags) & XBF_READ_AHEAD) ? __GFP_NORETRY : \
  69           ((flags) & XBF_DONT_BLOCK) ? GFP_NOFS : GFP_KERNEL) | __GFP_NOWARN)
  70
  71 #define xb_to_km(flags) \
  72          (((flags) & XBF_DONT_BLOCK) ? KM_NOFS : KM_SLEEP)
  73
  74 #define xfs_buf_allocate(flags) \
  75         kmem_zone_alloc(xfs_buf_zone, xb_to_km(flags))
  76 #define xfs_buf_deallocate(bp) \
  77         kmem_zone_free(xfs_buf_zone, (bp));
  78
  79 /*
  80  *      Page Region interfaces.
  81  *
  82  *      For pages in filesystems where the blocksize is smaller than the
  83  *      pagesize, we use the page->private field (long) to hold a bitmap
  84  *      of uptodate regions within the page.
  85  *
  86  *      Each such region is "bytes per page / bits per long" bytes long.
  87  *
  88  *      NBPPR == number-of-bytes-per-page-region
  89  *      BTOPR == bytes-to-page-region (rounded up)
  90  *      BTOPRT == bytes-to-page-region-truncated (rounded down)
  91  */
  92 #if (BITS_PER_LONG == 32)
  93 #define PRSHIFT         (PAGE_CACHE_SHIFT - 5)  /* (32 == 1<<5) */
  94 #elif (BITS_PER_LONG == 64)
  95 #define PRSHIFT         (PAGE_CACHE_SHIFT - 6)  /* (64 == 1<<6) */
  96 #else
  97 #error BITS_PER_LONG must be 32 or 64
  98 #endif
  99 #define NBPPR           (PAGE_CACHE_SIZE/BITS_PER_LONG)
 100 #define BTOPR(b)        (((unsigned int)(b) + (NBPPR - 1)) >> PRSHIFT)
 101 #define BTOPRT(b)       (((unsigned int)(b) >> PRSHIFT))
 102
 103 STATIC unsigned long
 104 page_region_mask(
 105         size_t          offset,
 106         size_t          length)
 107 {
 108         unsigned long   mask;
 109         int             first, final;
 110
 111         first = BTOPR(offset);
 112         final = BTOPRT(offset + length - 1);
 113         first = min(first, final);
 114
 115         mask = ~0UL;
 116         mask <<= BITS_PER_LONG - (final - first);
 117         mask >>= BITS_PER_LONG - (final);
 118
 119         ASSERT(offset + length <= PAGE_CACHE_SIZE);
 120         ASSERT((final - first) < BITS_PER_LONG && (final - first) >= 0);
 121
 122         return mask;
 123 }
 124
 125 STATIC void
 126 set_page_region(
 127         struct page     *page,
 128         size_t          offset,
 129         size_t          length)
 130 {
 131         set_page_private(page,
 132                 page_private(page) | page_region_mask(offset, length));
 133         if (page_private(page) == ~0UL)
 134                 SetPageUptodate(page);
 135 }
 136
 137 STATIC int
 138 test_page_region(
 139         struct page     *page,
 140         size_t          offset,
 141         size_t          length)
 142 {
 143         unsigned long   mask = page_region_mask(offset, length);
 144
 145         return (mask && (page_private(page) & mask) == mask);
 146 }
 147
 148 /*
 149  *      Mapping of multi-page buffers into contiguous virtual space
 150  */
 151
 152 typedef struct a_list {
 153         void            *vm_addr;
 154         struct a_list   *next;
 155 } a_list_t;
 156
 157 static a_list_t         *as_free_head;
 158 static int              as_list_len;
 159 static DEFINE_SPINLOCK(as_lock);
 160
 161 /*
 162  *      Try to batch vunmaps because they are costly.
 163  */
 164 STATIC void
 165 free_address(
 166         void            *addr)
 167 {
 168         a_list_t        *aentry;
 169
 170 #ifdef CONFIG_XEN
 171         /*
 172          * Xen needs to be able to make sure it can get an exclusive
 173          * RO mapping of pages it wants to turn into a pagetable.  If
 174          * a newly allocated page is also still being vmap()ed by xfs,
 175          * it will cause pagetable construction to fail.  This is a
 176          * quick workaround to always eagerly unmap pages so that Xen
 177          * is happy.
 178          */
 179         vunmap(addr);
 180         return;
 181 #endif
 182
 183         aentry = kmalloc(sizeof(a_list_t), GFP_NOWAIT);
 184         if (likely(aentry)) {
 185                 spin_lock(&as_lock);
 186                 aentry->next = as_free_head;
 187                 aentry->vm_addr = addr;
 188                 as_free_head = aentry;
 189                 as_list_len++;
 190                 spin_unlock(&as_lock);
 191         } else {
 192                 vunmap(addr);
 193         }
 194 }
 195
 196 STATIC void
 197 purge_addresses(void)
 198 {
 199         a_list_t        *aentry, *old;
 200
 201         if (as_free_head == NULL)
 202                 return;
 203
 204         spin_lock(&as_lock);
 205         aentry = as_free_head;
 206         as_free_head = NULL;
 207         as_list_len = 0;
 208         spin_unlock(&as_lock);
 209
 210         while ((old = aentry) != NULL) {
 211                 vunmap(aentry->vm_addr);
 212                 aentry = aentry->next;
 213                 kfree(old);
 214         }
 215 }
 216
 217 /*
 218  *      Internal xfs_buf_t object manipulation
 219  */
 220
 221 STATIC void
 222 _xfs_buf_initialize(
 223         xfs_buf_t               *bp,
 224         xfs_buftarg_t           *target,
 225         xfs_off_t               range_base,
 226         size_t                  range_length,
 227         xfs_buf_flags_t         flags)
 228 {
 229         /*
 230          * We don't want certain flags to appear in b_flags.
 231          */
 232         flags &= ~(XBF_LOCK|XBF_MAPPED|XBF_DONT_BLOCK|XBF_READ_AHEAD);
 233
 234         memset(bp, 0, sizeof(xfs_buf_t));
 235         atomic_set(&bp->b_hold, 1);
 236         init_completion(&bp->b_iowait);
 237         INIT_LIST_HEAD(&bp->b_list);
 238         INIT_LIST_HEAD(&bp->b_hash_list);
 239         init_MUTEX_LOCKED(&bp->b_sema); /* held, no waiters */
 240         XB_SET_OWNER(bp);
 241         bp->b_target = target;
 242         bp->b_file_offset = range_base;
 243         /*
 244          * Set buffer_length and count_desired to the same value initially.
 245          * I/O routines should use count_desired, which will be the same in
 246          * most cases but may be reset (e.g. XFS recovery).
 247          */
 248         bp->b_buffer_length = bp->b_count_desired = range_length;
 249         bp->b_flags = flags;
 250         bp->b_bn = XFS_BUF_DADDR_NULL;
 251         atomic_set(&bp->b_pin_count, 0);
 252         init_waitqueue_head(&bp->b_waiters);
 253
 254         XFS_STATS_INC(xb_create);
 255
 256         trace_xfs_buf_init(bp, _RET_IP_);
 257 }
 258
 259 /*
 260  *      Allocate a page array capable of holding a specified number
 261  *      of pages, and point the page buf at it.
 262  */
 263 STATIC int
 264 _xfs_buf_get_pages(
 265         xfs_buf_t               *bp,
 266         int                     page_count,
 267         xfs_buf_flags_t         flags)
 268 {
 269         /* Make sure that we have a page list */
 270         if (bp->b_pages == NULL) {
 271                 bp->b_offset = xfs_buf_poff(bp->b_file_offset);
 272                 bp->b_page_count = page_count;
 273                 if (page_count <= XB_PAGES) {
 274                         bp->b_pages = bp->b_page_array;
 275                 } else {
 276                         bp->b_pages = kmem_alloc(sizeof(struct page *) *
 277                                         page_count, xb_to_km(flags));
 278                         if (bp->b_pages == NULL)
 279                                 return -ENOMEM;
 280                 }
 281                 memset(bp->b_pages, 0, sizeof(struct page *) * page_count);
 282         }
 283         return 0;
 284 }
 285
 286 /*
 287  *      Frees b_pages if it was allocated.
 288  */
 289 STATIC void
 290 _xfs_buf_free_pages(
 291         xfs_buf_t       *bp)
 292 {
 293         if (bp->b_pages != bp->b_page_array) {
 294                 kmem_free(bp->b_pages);
 295                 bp->b_pages = NULL;
 296         }
 297 }
 298
 299 /*
 300  *      Releases the specified buffer.
 301  *
 302  *      The modification state of any associated pages is left unchanged.
 303  *      The buffer most not be on any hash - use xfs_buf_rele instead for
 304  *      hashed and refcounted buffers
 305  */
 306 void
 307 xfs_buf_free(
 308         xfs_buf_t               *bp)
 309 {
 310         trace_xfs_buf_free(bp, _RET_IP_);
 311
 312         ASSERT(list_empty(&bp->b_hash_list));
 313
 314         if (bp->b_flags & (_XBF_PAGE_CACHE|_XBF_PAGES)) {
 315                 uint            i;
 316
 317                 if ((bp->b_flags & XBF_MAPPED) && (bp->b_page_count > 1))
 318                         free_address(bp->b_addr - bp->b_offset);
 319
 320                 for (i = 0; i < bp->b_page_count; i++) {
 321                         struct page     *page = bp->b_pages[i];
 322
 323                         if (bp->b_flags & _XBF_PAGE_CACHE)
 324                                 ASSERT(!PagePrivate(page));
 325                         page_cache_release(page);
 326                 }
 327         }
 328         _xfs_buf_free_pages(bp);
 329         xfs_buf_deallocate(bp);
 330 }
 331
 332 /*
 333  *      Finds all pages for buffer in question and builds it's page list.
 334  */
 335 STATIC int
 336 _xfs_buf_lookup_pages(
 337         xfs_buf_t               *bp,
 338         uint                    flags)
 339 {
 340         struct address_space    *mapping = bp->b_target->bt_mapping;
 341         size_t                  blocksize = bp->b_target->bt_bsize;
 342         size_t                  size = bp->b_count_desired;
 343         size_t                  nbytes, offset;
 344         gfp_t                   gfp_mask = xb_to_gfp(flags);
 345         unsigned short          page_count, i;
 346         pgoff_t                 first;
 347         xfs_off_t               end;
 348         int                     error;
 349
 350         end = bp->b_file_offset + bp->b_buffer_length;
 351         page_count = xfs_buf_btoc(end) - xfs_buf_btoct(bp->b_file_offset);
 352
 353         error = _xfs_buf_get_pages(bp, page_count, flags);
 354         if (unlikely(error))
 355                 return error;
 356         bp->b_flags |= _XBF_PAGE_CACHE;
 357
 358         offset = bp->b_offset;
 359         first = bp->b_file_offset >> PAGE_CACHE_SHIFT;
 360
 361         for (i = 0; i < bp->b_page_count; i++) {
 362                 struct page     *page;
 363                 uint            retries = 0;
 364
 365               retry:
 366                 page = find_or_create_page(mapping, first + i, gfp_mask);
 367                 if (unlikely(page == NULL)) {
 368                         if (flags & XBF_READ_AHEAD) {
 369                                 bp->b_page_count = i;
 370                                 for (i = 0; i < bp->b_page_count; i++)
 371                                         unlock_page(bp->b_pages[i]);
 372                                 return -ENOMEM;
 373                         }
 374
 375                         /*
 376                          * This could deadlock.
 377                          *
 378                          * But until all the XFS lowlevel code is revamped to
 379                          * handle buffer allocation failures we can't do much.
 380                          */
 381                         if (!(++retries % 100))
 382                                 printk(KERN_ERR
 383                                         "XFS: possible memory allocation "
 384                                         "deadlock in %s (mode:0x%x)\n",
 385                                         __func__, gfp_mask);
 386
 387                         XFS_STATS_INC(xb_page_retries);
 388                         xfsbufd_wakeup(0, gfp_mask);
 389                         congestion_wait(BLK_RW_ASYNC, HZ/50);
 390                         goto retry;
 391                 }
 392
 393                 XFS_STATS_INC(xb_page_found);
 394
 395                 nbytes = min_t(size_t, size, PAGE_CACHE_SIZE - offset);
 396                 size -= nbytes;
 397
 398                 ASSERT(!PagePrivate(page));
 399                 if (!PageUptodate(page)) {
 400                         page_count--;
 401                         if (blocksize >= PAGE_CACHE_SIZE) {
 402                                 if (flags & XBF_READ)
 403                                         bp->b_flags |= _XBF_PAGE_LOCKED;
 404                         } else if (!PagePrivate(page)) {
 405                                 if (test_page_region(page, offset, nbytes))
 406                                         page_count++;
 407                         }
 408                 }
 409
 410                 bp->b_pages[i] = page;
 411                 offset = 0;
 412         }
 413
 414         if (!(bp->b_flags & _XBF_PAGE_LOCKED)) {
 415                 for (i = 0; i < bp->b_page_count; i++)
 416                         unlock_page(bp->b_pages[i]);
 417         }
 418
 419         if (page_count == bp->b_page_count)
 420                 bp->b_flags |= XBF_DONE;
 421
 422         return error;
 423 }
 424
 425 /*
 426  *      Map buffer into kernel address-space if nessecary.
 427  */
 428 STATIC int
 429 _xfs_buf_map_pages(
 430         xfs_buf_t               *bp,
 431         uint                    flags)
 432 {
 433         /* A single page buffer is always mappable */
 434         if (bp->b_page_count == 1) {
 435                 bp->b_addr = page_address(bp->b_pages[0]) + bp->b_offset;
 436                 bp->b_flags |= XBF_MAPPED;
 437         } else if (flags & XBF_MAPPED) {
 438                 if (as_list_len > 64)
 439                         purge_addresses();
 440                 bp->b_addr = vmap(bp->b_pages, bp->b_page_count,
 441                                         VM_MAP, PAGE_KERNEL);
 442                 if (unlikely(bp->b_addr == NULL))
 443                         return -ENOMEM;
 444                 bp->b_addr += bp->b_offset;
 445                 bp->b_flags |= XBF_MAPPED;
 446         }
 447
 448         return 0;
 449 }
 450
 451 /*
 452  *      Finding and Reading Buffers
 453  */
 454
 455 /*
 456  *      Look up, and creates if absent, a lockable buffer for
 457  *      a given range of an inode.  The buffer is returned
 458  *      locked.  If other overlapping buffers exist, they are
 459  *      released before the new buffer is created and locked,
 460  *      which may imply that this call will block until those buffers
 461  *      are unlocked.  No I/O is implied by this call.
 462  */
 463 xfs_buf_t *
 464 _xfs_buf_find(
 465         xfs_buftarg_t           *btp,   /* block device target          */
 466         xfs_off_t               ioff,   /* starting offset of range     */
 467         size_t                  isize,  /* length of range              */
 468         xfs_buf_flags_t         flags,
 469         xfs_buf_t               *new_bp)
 470 {
 471         xfs_off_t               range_base;
 472         size_t                  range_length;
 473         xfs_bufhash_t           *hash;
 474         xfs_buf_t               *bp, *n;
 475
 476         range_base = (ioff << BBSHIFT);
 477         range_length = (isize << BBSHIFT);
 478
 479         /* Check for IOs smaller than the sector size / not sector aligned */
 480         ASSERT(!(range_length < (1 << btp->bt_sshift)));
 481         ASSERT(!(range_base & (xfs_off_t)btp->bt_smask));
 482
 483         hash = &btp->bt_hash[hash_long((unsigned long)ioff, btp->bt_hashshift)];
 484
 485         spin_lock(&hash->bh_lock);
 486
 487         list_for_each_entry_safe(bp, n, &hash->bh_list, b_hash_list) {
 488                 ASSERT(btp == bp->b_target);
 489                 if (bp->b_file_offset == range_base &&
 490                     bp->b_buffer_length == range_length) {
 491                         /*
 492                          * If we look at something, bring it to the
 493                          * front of the list for next time.
 494                          */
 495                         atomic_inc(&bp->b_hold);
 496                         list_move(&bp->b_hash_list, &hash->bh_list);
 497                         goto found;
 498                 }
 499         }
 500
 501         /* No match found */
 502         if (new_bp) {
 503                 _xfs_buf_initialize(new_bp, btp, range_base,
 504                                 range_length, flags);
 505                 new_bp->b_hash = hash;
 506                 list_add(&new_bp->b_hash_list, &hash->bh_list);
 507         } else {
 508                 XFS_STATS_INC(xb_miss_locked);
 509         }
 510
 511         spin_unlock(&hash->bh_lock);
 512         return new_bp;
 513
 514 found:
 515         spin_unlock(&hash->bh_lock);
 516
 517         /* Attempt to get the semaphore without sleeping,
 518          * if this does not work then we need to drop the
 519          * spinlock and do a hard attempt on the semaphore.
 520          */
 521         if (down_trylock(&bp->b_sema)) {
 522                 if (!(flags & XBF_TRYLOCK)) {
 523                         /* wait for buffer ownership */
 524                         xfs_buf_lock(bp);
 525                         XFS_STATS_INC(xb_get_locked_waited);
 526                 } else {
 527                         /* We asked for a trylock and failed, no need
 528                          * to look at file offset and length here, we
 529                          * know that this buffer at least overlaps our
 530                          * buffer and is locked, therefore our buffer
 531                          * either does not exist, or is this buffer.
 532                          */
 533                         xfs_buf_rele(bp);
 534                         XFS_STATS_INC(xb_busy_locked);
 535                         return NULL;
 536                 }
 537         } else {
 538                 /* trylock worked */
 539                 XB_SET_OWNER(bp);
 540         }
 541
 542         if (bp->b_flags & XBF_STALE) {
 543                 ASSERT((bp->b_flags & _XBF_DELWRI_Q) == 0);
 544                 bp->b_flags &= XBF_MAPPED;
 545         }
 546
 547         trace_xfs_buf_find(bp, flags, _RET_IP_);
 548         XFS_STATS_INC(xb_get_locked);
 549         return bp;
 550 }
 551
 552 /*
 553  *      Assembles a buffer covering the specified range.
 554  *      Storage in memory for all portions of the buffer will be allocated,
 555  *      although backing storage may not be.
 556  */
 557 xfs_buf_t *
 558 xfs_buf_get(
 559         xfs_buftarg_t           *target,/* target for buffer            */
 560         xfs_off_t               ioff,   /* starting offset of range     */
 561         size_t                  isize,  /* length of range              */
 562         xfs_buf_flags_t         flags)
 563 {
 564         xfs_buf_t               *bp, *new_bp;
 565         int                     error = 0, i;
 566
 567         new_bp = xfs_buf_allocate(flags);
 568         if (unlikely(!new_bp))
 569                 return NULL;
 570
 571         bp = _xfs_buf_find(target, ioff, isize, flags, new_bp);
 572         if (bp == new_bp) {
 573                 error = _xfs_buf_lookup_pages(bp, flags);
 574                 if (error)
 575                         goto no_buffer;
 576         } else {
 577                 xfs_buf_deallocate(new_bp);
 578                 if (unlikely(bp == NULL))
 579                         return NULL;
 580         }
 581
 582         for (i = 0; i < bp->b_page_count; i++)
 583                 mark_page_accessed(bp->b_pages[i]);
 584
 585         if (!(bp->b_flags & XBF_MAPPED)) {
 586                 error = _xfs_buf_map_pages(bp, flags);
 587                 if (unlikely(error)) {
 588                         printk(KERN_WARNING "%s: failed to map pages\n",
 589                                         __func__);
 590                         goto no_buffer;
 591                 }
 592         }
 593
 594         XFS_STATS_INC(xb_get);
 595
 596         /*
 597          * Always fill in the block number now, the mapped cases can do
 598          * their own overlay of this later.
 599          */
 600         bp->b_bn = ioff;
 601         bp->b_count_desired = bp->b_buffer_length;
 602
 603         trace_xfs_buf_get(bp, flags, _RET_IP_);
 604         return bp;
 605
 606  no_buffer:
 607         if (flags & (XBF_LOCK | XBF_TRYLOCK))
 608                 xfs_buf_unlock(bp);
 609         xfs_buf_rele(bp);
 610         return NULL;
 611 }
 612
 613 STATIC int
 614 _xfs_buf_read(
 615         xfs_buf_t               *bp,
 616         xfs_buf_flags_t         flags)
 617 {
 618         int                     status;
 619
 620         ASSERT(!(flags & (XBF_DELWRI|XBF_WRITE)));
 621         ASSERT(bp->b_bn != XFS_BUF_DADDR_NULL);
 622
 623         bp->b_flags &= ~(XBF_WRITE | XBF_ASYNC | XBF_DELWRI | \
 624                         XBF_READ_AHEAD | _XBF_RUN_QUEUES);
 625         bp->b_flags |= flags & (XBF_READ | XBF_ASYNC | \
 626                         XBF_READ_AHEAD | _XBF_RUN_QUEUES);
 627
 628         status = xfs_buf_iorequest(bp);
 629         if (!status && !(flags & XBF_ASYNC))
 630                 status = xfs_buf_iowait(bp);
 631         return status;
 632 }
 633
 634 xfs_buf_t *
 635 xfs_buf_read(
 636         xfs_buftarg_t           *target,
 637         xfs_off_t               ioff,
 638         size_t                  isize,
 639         xfs_buf_flags_t         flags)
 640 {
 641         xfs_buf_t               *bp;
 642
 643         flags |= XBF_READ;
 644
 645         bp = xfs_buf_get(target, ioff, isize, flags);
 646         if (bp) {
 647                 trace_xfs_buf_read(bp, flags, _RET_IP_);
 648
 649                 if (!XFS_BUF_ISDONE(bp)) {
 650                         XFS_STATS_INC(xb_get_read);
 651                         _xfs_buf_read(bp, flags);
 652                 } else if (flags & XBF_ASYNC) {
 653                         /*
 654                          * Read ahead call which is already satisfied,
 655                          * drop the buffer
 656                          */
 657                         goto no_buffer;
 658                 } else {
 659                         /* We do not want read in the flags */
 660                         bp->b_flags &= ~XBF_READ;
 661                 }
 662         }
 663
 664         return bp;
 665
 666  no_buffer:
 667         if (flags & (XBF_LOCK | XBF_TRYLOCK))
 668                 xfs_buf_unlock(bp);
 669         xfs_buf_rele(bp);
 670         return NULL;
 671 }
 672
 673 /*
 674  *      If we are not low on memory then do the readahead in a deadlock
 675  *      safe manner.
 676  */
 677 void
 678 xfs_buf_readahead(
 679         xfs_buftarg_t           *target,
 680         xfs_off_t               ioff,
 681         size_t                  isize,
 682         xfs_buf_flags_t         flags)
 683 {
 684         struct backing_dev_info *bdi;
 685
 686         bdi = target->bt_mapping->backing_dev_info;
 687         if (bdi_read_congested(bdi))
 688                 return;
 689
 690         flags |= (XBF_TRYLOCK|XBF_ASYNC|XBF_READ_AHEAD);
 691         xfs_buf_read(target, ioff, isize, flags);
 692 }
 693
 694 xfs_buf_t *
 695 xfs_buf_get_empty(
 696         size_t                  len,
 697         xfs_buftarg_t           *target)
 698 {
 699         xfs_buf_t               *bp;
 700
 701         bp = xfs_buf_allocate(0);
 702         if (bp)
 703                 _xfs_buf_initialize(bp, target, 0, len, 0);
 704         return bp;
 705 }
 706
 707 static inline struct page *
 708 mem_to_page(
 709         void                    *addr)
 710 {
 711         if ((!is_vmalloc_addr(addr))) {
 712                 return virt_to_page(addr);
 713         } else {
 714                 return vmalloc_to_page(addr);
 715         }
 716 }
 717
 718 int
 719 xfs_buf_associate_memory(
 720         xfs_buf_t               *bp,
 721         void                    *mem,
 722         size_t                  len)
 723 {
 724         int                     rval;
 725         int                     i = 0;
 726         unsigned long           pageaddr;
 727         unsigned long           offset;
 728         size_t                  buflen;
 729         int                     page_count;
 730
 731         pageaddr = (unsigned long)mem & PAGE_CACHE_MASK;
 732         offset = (unsigned long)mem - pageaddr;
 733         buflen = PAGE_CACHE_ALIGN(len + offset);
 734         page_count = buflen >> PAGE_CACHE_SHIFT;
 735
 736         /* Free any previous set of page pointers */
 737         if (bp->b_pages)
 738                 _xfs_buf_free_pages(bp);
 739
 740         bp->b_pages = NULL;
 741         bp->b_addr = mem;
 742
 743         rval = _xfs_buf_get_pages(bp, page_count, XBF_DONT_BLOCK);
 744         if (rval)
 745                 return rval;
 746
 747         bp->b_offset = offset;
 748
 749         for (i = 0; i < bp->b_page_count; i++) {
 750                 bp->b_pages[i] = mem_to_page((void *)pageaddr);
 751                 pageaddr += PAGE_CACHE_SIZE;
 752         }
 753
 754         bp->b_count_desired = len;
 755         bp->b_buffer_length = buflen;
 756         bp->b_flags |= XBF_MAPPED;
 757         bp->b_flags &= ~_XBF_PAGE_LOCKED;
 758
 759         return 0;
 760 }
 761
 762 xfs_buf_t *
 763 xfs_buf_get_noaddr(
 764         size_t                  len,
 765         xfs_buftarg_t           *target)
 766 {
 767         unsigned long           page_count = PAGE_ALIGN(len) >> PAGE_SHIFT;
 768         int                     error, i;
 769         xfs_buf_t               *bp;
 770
 771         bp = xfs_buf_allocate(0);
 772         if (unlikely(bp == NULL))
 773                 goto fail;
 774         _xfs_buf_initialize(bp, target, 0, len, 0);
 775
 776         error = _xfs_buf_get_pages(bp, page_count, 0);
 777         if (error)
 778                 goto fail_free_buf;
 779
 780         for (i = 0; i < page_count; i++) {
 781                 bp->b_pages[i] = alloc_page(GFP_KERNEL);
 782                 if (!bp->b_pages[i])
 783                         goto fail_free_mem;
 784         }
 785         bp->b_flags |= _XBF_PAGES;
 786
 787         error = _xfs_buf_map_pages(bp, XBF_MAPPED);
 788         if (unlikely(error)) {
 789                 printk(KERN_WARNING "%s: failed to map pages\n",
 790                                 __func__);
 791                 goto fail_free_mem;
 792         }
 793
 794         xfs_buf_unlock(bp);
 795
 796         trace_xfs_buf_get_noaddr(bp, _RET_IP_);
 797         return bp;
 798
 799  fail_free_mem:
 800         while (--i >= 0)
 801                 __free_page(bp->b_pages[i]);
 802         _xfs_buf_free_pages(bp);
 803  fail_free_buf:
 804         xfs_buf_deallocate(bp);
 805  fail:
 806         return NULL;
 807 }
 808
 809 /*
 810  *      Increment reference count on buffer, to hold the buffer concurrently
 811  *      with another thread which may release (free) the buffer asynchronously.
 812  *      Must hold the buffer already to call this function.
 813  */
 814 void
 815 xfs_buf_hold(
 816         xfs_buf_t               *bp)
 817 {
 818         trace_xfs_buf_hold(bp, _RET_IP_);
 819         atomic_inc(&bp->b_hold);
 820 }
 821
 822 /*
 823  *      Releases a hold on the specified buffer.  If the
 824  *      the hold count is 1, calls xfs_buf_free.
 825  */
 826 void
 827 xfs_buf_rele(
 828         xfs_buf_t               *bp)
 829 {
 830         xfs_bufhash_t           *hash = bp->b_hash;
 831
 832         trace_xfs_buf_rele(bp, _RET_IP_);
 833
 834         if (unlikely(!hash)) {
 835                 ASSERT(!bp->b_relse);
 836                 if (atomic_dec_and_test(&bp->b_hold))
 837                         xfs_buf_free(bp);
 838                 return;
 839         }
 840
 841         ASSERT(atomic_read(&bp->b_hold) > 0);
 842         if (atomic_dec_and_lock(&bp->b_hold, &hash->bh_lock)) {
 843                 if (bp->b_relse) {
 844                         atomic_inc(&bp->b_hold);
 845                         spin_unlock(&hash->bh_lock);
 846                         (*(bp->b_relse)) (bp);
 847                 } else if (bp->b_flags & XBF_FS_MANAGED) {
 848                         spin_unlock(&hash->bh_lock);
 849                 } else {
 850                         ASSERT(!(bp->b_flags & (XBF_DELWRI|_XBF_DELWRI_Q)));
 851                         list_del_init(&bp->b_hash_list);
 852                         spin_unlock(&hash->bh_lock);
 853                         xfs_buf_free(bp);
 854                 }
 855         }
 856 }
 857
 858
 859 /*
 860  *      Mutual exclusion on buffers.  Locking model:
 861  *
 862  *      Buffers associated with inodes for which buffer locking
 863  *      is not enabled are not protected by semaphores, and are
 864  *      assumed to be exclusively owned by the caller.  There is a
 865  *      spinlock in the buffer, used by the caller when concurrent
 866  *      access is possible.
 867  */
 868
 869 /*
 870  *      Locks a buffer object, if it is not already locked.
 871  *      Note that this in no way locks the underlying pages, so it is only
 872  *      useful for synchronizing concurrent use of buffer objects, not for
 873  *      synchronizing independent access to the underlying pages.
 874  */
 875 int
 876 xfs_buf_cond_lock(
 877         xfs_buf_t               *bp)
 878 {
 879         int                     locked;
 880
 881         locked = down_trylock(&bp->b_sema) == 0;
 882         if (locked)
 883                 XB_SET_OWNER(bp);
 884
 885         trace_xfs_buf_cond_lock(bp, _RET_IP_);
 886         return locked ? 0 : -EBUSY;
 887 }
 888
 889 int
 890 xfs_buf_lock_value(
 891         xfs_buf_t               *bp)
 892 {
 893         return bp->b_sema.count;
 894 }
 895
 896 /*
 897  *      Locks a buffer object.
 898  *      Note that this in no way locks the underlying pages, so it is only
 899  *      useful for synchronizing concurrent use of buffer objects, not for
 900  *      synchronizing independent access to the underlying pages.
 901  */
 902 void
 903 xfs_buf_lock(
 904         xfs_buf_t               *bp)
 905 {
 906         trace_xfs_buf_lock(bp, _RET_IP_);
 907
 908         if (atomic_read(&bp->b_io_remaining))
 909                 blk_run_address_space(bp->b_target->bt_mapping);
 910         down(&bp->b_sema);
 911         XB_SET_OWNER(bp);
 912
 913         trace_xfs_buf_lock_done(bp, _RET_IP_);
 914 }
 915
 916 /*
 917  *      Releases the lock on the buffer object.
 918  *      If the buffer is marked delwri but is not queued, do so before we
 919  *      unlock the buffer as we need to set flags correctly.  We also need to
 920  *      take a reference for the delwri queue because the unlocker is going to
 921  *      drop their's and they don't know we just queued it.
 922  */
 923 void
 924 xfs_buf_unlock(
 925         xfs_buf_t               *bp)
 926 {
 927         if ((bp->b_flags & (XBF_DELWRI|_XBF_DELWRI_Q)) == XBF_DELWRI) {
 928                 atomic_inc(&bp->b_hold);
 929                 bp->b_flags |= XBF_ASYNC;
 930                 xfs_buf_delwri_queue(bp, 0);
 931         }
 932
 933         XB_CLEAR_OWNER(bp);
 934         up(&bp->b_sema);
 935
 936         trace_xfs_buf_unlock(bp, _RET_IP_);
 937 }
 938
 939
 940 /*
 941  *      Pinning Buffer Storage in Memory
 942  *      Ensure that no attempt to force a buffer to disk will succeed.
 943  */
 944 void
 945 xfs_buf_pin(
 946         xfs_buf_t               *bp)
 947 {
 948         trace_xfs_buf_pin(bp, _RET_IP_);
 949         atomic_inc(&bp->b_pin_count);
 950 }
 951
 952 void
 953 xfs_buf_unpin(
 954         xfs_buf_t               *bp)
 955 {
 956         trace_xfs_buf_unpin(bp, _RET_IP_);
 957
 958         if (atomic_dec_and_test(&bp->b_pin_count))
 959                 wake_up_all(&bp->b_waiters);
 960 }
 961
 962 int
 963 xfs_buf_ispin(
 964         xfs_buf_t               *bp)
 965 {
 966         return atomic_read(&bp->b_pin_count);
 967 }
 968
 969 STATIC void
 970 xfs_buf_wait_unpin(
 971         xfs_buf_t               *bp)
 972 {
 973         DECLARE_WAITQUEUE       (wait, current);
 974
 975         if (atomic_read(&bp->b_pin_count) == 0)
 976                 return;
 977
 978         add_wait_queue(&bp->b_waiters, &wait);
 979         for (;;) {
 980                 set_current_state(TASK_UNINTERRUPTIBLE);
 981                 if (atomic_read(&bp->b_pin_count) == 0)
 982                         break;
 983                 if (atomic_read(&bp->b_io_remaining))
 984                         blk_run_address_space(bp->b_target->bt_mapping);
 985                 schedule();
 986         }
 987         remove_wait_queue(&bp->b_waiters, &wait);
 988         set_current_state(TASK_RUNNING);
 989 }
 990
 991 /*
 992  *      Buffer Utility Routines
 993  */
 994
 995 STATIC void
 996 xfs_buf_iodone_work(
 997         struct work_struct      *work)
 998 {
 999         xfs_buf_t               *bp =
1000                 container_of(work, xfs_buf_t, b_iodone_work);
1001
1002         /*
1003          * We can get an EOPNOTSUPP to ordered writes.  Here we clear the
1004          * ordered flag and reissue them.  Because we can't tell the higher
1005          * layers directly that they should not issue ordered I/O anymore, they
1006          * need to check if the _XFS_BARRIER_FAILED flag was set during I/O completion.
1007          */
1008         if ((bp->b_error == EOPNOTSUPP) &&
1009             (bp->b_flags & (XBF_ORDERED|XBF_ASYNC)) == (XBF_ORDERED|XBF_ASYNC)) {
1010                 trace_xfs_buf_ordered_retry(bp, _RET_IP_);
1011                 bp->b_flags &= ~XBF_ORDERED;
1012                 bp->b_flags |= _XFS_BARRIER_FAILED;
1013                 xfs_buf_iorequest(bp);
1014         } else if (bp->b_iodone)
1015                 (*(bp->b_iodone))(bp);
1016         else if (bp->b_flags & XBF_ASYNC)
1017                 xfs_buf_relse(bp);
1018 }
1019
1020 void
1021 xfs_buf_ioend(
1022         xfs_buf_t               *bp,
1023         int                     schedule)
1024 {
1025         trace_xfs_buf_iodone(bp, _RET_IP_);
1026
1027         bp->b_flags &= ~(XBF_READ | XBF_WRITE | XBF_READ_AHEAD);
1028         if (bp->b_error == 0)
1029                 bp->b_flags |= XBF_DONE;
1030
1031         if ((bp->b_iodone) || (bp->b_flags & XBF_ASYNC)) {
1032                 if (schedule) {
1033                         INIT_WORK(&bp->b_iodone_work, xfs_buf_iodone_work);
1034                         queue_work(xfslogd_workqueue, &bp->b_iodone_work);
1035                 } else {
1036                         xfs_buf_iodone_work(&bp->b_iodone_work);
1037                 }
1038         } else {
1039                 complete(&bp->b_iowait);
1040         }
1041 }
1042
1043 void
1044 xfs_buf_ioerror(
1045         xfs_buf_t               *bp,
1046         int                     error)
1047 {
1048         ASSERT(error >= 0 && error <= 0xffff);
1049         bp->b_error = (unsigned short)error;
1050         trace_xfs_buf_ioerror(bp, error, _RET_IP_);
1051 }
1052
1053 int
1054 xfs_bwrite(
1055         struct xfs_mount        *mp,
1056         struct xfs_buf          *bp)
1057 {
1058         int                     iowait = (bp->b_flags & XBF_ASYNC) == 0;
1059         int                     error = 0;
1060
1061         bp->b_strat = xfs_bdstrat_cb;
1062         bp->b_mount = mp;
1063         bp->b_flags |= XBF_WRITE;
1064         if (!iowait)
1065                 bp->b_flags |= _XBF_RUN_QUEUES;
1066
1067         xfs_buf_delwri_dequeue(bp);
1068         xfs_buf_iostrategy(bp);
1069
1070         if (iowait) {
1071                 error = xfs_buf_iowait(bp);
1072                 if (error)
1073                         xfs_force_shutdown(mp, SHUTDOWN_META_IO_ERROR);
1074                 xfs_buf_relse(bp);
1075         }
1076
1077         return error;
1078 }
1079
1080 int
1081 xfs_bawrite(
1082         void                    *mp,
1083         struct xfs_buf          *bp)
1084 {
1085         trace_xfs_buf_bawrite(bp, _RET_IP_);
1086
1087         ASSERT(bp->b_bn != XFS_BUF_DADDR_NULL);
1088
1089         xfs_buf_delwri_dequeue(bp);
1090
1091         bp->b_flags &= ~(XBF_READ | XBF_DELWRI | XBF_READ_AHEAD);
1092         bp->b_flags |= (XBF_WRITE | XBF_ASYNC | _XBF_RUN_QUEUES);
1093
1094         bp->b_mount = mp;
1095         bp->b_strat = xfs_bdstrat_cb;
1096         return xfs_bdstrat_cb(bp);
1097 }
1098
1099 void
1100 xfs_bdwrite(
1101         void                    *mp,
1102         struct xfs_buf          *bp)
1103 {
1104         trace_xfs_buf_bdwrite(bp, _RET_IP_);
1105
1106         bp->b_strat = xfs_bdstrat_cb;
1107         bp->b_mount = mp;
1108
1109         bp->b_flags &= ~XBF_READ;
1110         bp->b_flags |= (XBF_DELWRI | XBF_ASYNC);
1111
1112         xfs_buf_delwri_queue(bp, 1);
1113 }
1114
1115 /*
1116  * Called when we want to stop a buffer from getting written or read.
1117  * We attach the EIO error, muck with its flags, and call biodone
1118  * so that the proper iodone callbacks get called.
1119  */
1120 STATIC int
1121 xfs_bioerror(
1122         xfs_buf_t *bp)
1123 {
1124 #ifdef XFSERRORDEBUG
1125         ASSERT(XFS_BUF_ISREAD(bp) || bp->b_iodone);
1126 #endif
1127
1128         /*
1129          * No need to wait until the buffer is unpinned, we aren't flushing it.
1130          */
1131         XFS_BUF_ERROR(bp, EIO);
1132
1133         /*
1134          * We're calling biodone, so delete XBF_DONE flag.
1135          */
1136         XFS_BUF_UNREAD(bp);
1137         XFS_BUF_UNDELAYWRITE(bp);
1138         XFS_BUF_UNDONE(bp);
1139         XFS_BUF_STALE(bp);
1140
1141         XFS_BUF_CLR_BDSTRAT_FUNC(bp);
1142         xfs_biodone(bp);
1143
1144         return EIO;
1145 }
1146
1147 /*
1148  * Same as xfs_bioerror, except that we are releasing the buffer
1149  * here ourselves, and avoiding the biodone call.
1150  * This is meant for userdata errors; metadata bufs come with
1151  * iodone functions attached, so that we can track down errors.
1152  */
1153 STATIC int
1154 xfs_bioerror_relse(
1155         struct xfs_buf  *bp)
1156 {
1157         int64_t         fl = XFS_BUF_BFLAGS(bp);
1158         /*
1159          * No need to wait until the buffer is unpinned.
1160          * We aren't flushing it.
1161          *
1162          * chunkhold expects B_DONE to be set, whether
1163          * we actually finish the I/O or not. We don't want to
1164          * change that interface.
1165          */
1166         XFS_BUF_UNREAD(bp);
1167         XFS_BUF_UNDELAYWRITE(bp);
1168         XFS_BUF_DONE(bp);
1169         XFS_BUF_STALE(bp);
1170         XFS_BUF_CLR_IODONE_FUNC(bp);
1171         XFS_BUF_CLR_BDSTRAT_FUNC(bp);
1172         if (!(fl & XFS_B_ASYNC)) {
1173                 /*
1174                  * Mark b_error and B_ERROR _both_.
1175                  * Lot's of chunkcache code assumes that.
1176                  * There's no reason to mark error for
1177                  * ASYNC buffers.
1178                  */
1179                 XFS_BUF_ERROR(bp, EIO);
1180                 XFS_BUF_FINISH_IOWAIT(bp);
1181         } else {
1182                 xfs_buf_relse(bp);
1183         }
1184
1185         return EIO;
1186 }
1187
1188
1189 /*
1190  * All xfs metadata buffers except log state machine buffers
1191  * get this attached as their b_bdstrat callback function.
1192  * This is so that we can catch a buffer
1193  * after prematurely unpinning it to forcibly shutdown the filesystem.
1194  */
1195 int
1196 xfs_bdstrat_cb(
1197         struct xfs_buf  *bp)
1198 {
1199         if (XFS_FORCED_SHUTDOWN(bp->b_mount)) {
1200                 trace_xfs_bdstrat_shut(bp, _RET_IP_);
1201                 /*
1202                  * Metadata write that didn't get logged but
1203                  * written delayed anyway. These aren't associated
1204                  * with a transaction, and can be ignored.
1205                  */
1206                 if (!bp->b_iodone && !XFS_BUF_ISREAD(bp))
1207                         return xfs_bioerror_relse(bp);
1208                 else
1209                         return xfs_bioerror(bp);
1210         }
1211
1212         xfs_buf_iorequest(bp);
1213         return 0;
1214 }
1215
1216 /*
1217  * Wrapper around bdstrat so that we can stop data from going to disk in case
1218  * we are shutting down the filesystem.  Typically user data goes thru this
1219  * path; one of the exceptions is the superblock.
1220  */
1221 void
1222 xfsbdstrat(
1223         struct xfs_mount        *mp,
1224         struct xfs_buf          *bp)
1225 {
1226         if (XFS_FORCED_SHUTDOWN(mp)) {
1227                 trace_xfs_bdstrat_shut(bp, _RET_IP_);
1228                 xfs_bioerror_relse(bp);
1229                 return;
1230         }
1231
1232         xfs_buf_iorequest(bp);
1233 }
1234
1235 STATIC void
1236 _xfs_buf_ioend(
1237         xfs_buf_t               *bp,
1238         int                     schedule)
1239 {
1240         if (atomic_dec_and_test(&bp->b_io_remaining) == 1) {
1241                 bp->b_flags &= ~_XBF_PAGE_LOCKED;
1242                 xfs_buf_ioend(bp, schedule);
1243         }
1244 }
1245
1246 STATIC void
1247 xfs_buf_bio_end_io(
1248         struct bio              *bio,
1249         int                     error)
1250 {
1251         xfs_buf_t               *bp = (xfs_buf_t *)bio->bi_private;
1252         unsigned int            blocksize = bp->b_target->bt_bsize;
1253         struct bio_vec          *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
1254
1255         xfs_buf_ioerror(bp, -error);
1256
1257         do {
1258                 struct page     *page = bvec->bv_page;
1259
1260                 ASSERT(!PagePrivate(page));
1261                 if (unlikely(bp->b_error)) {
1262                         if (bp->b_flags & XBF_READ)
1263                                 ClearPageUptodate(page);
1264                 } else if (blocksize >= PAGE_CACHE_SIZE) {
1265                         SetPageUptodate(page);
1266                 } else if (!PagePrivate(page) &&
1267                                 (bp->b_flags & _XBF_PAGE_CACHE)) {
1268                         set_page_region(page, bvec->bv_offset, bvec->bv_len);
1269                 }
1270
1271                 if (--bvec >= bio->bi_io_vec)
1272                         prefetchw(&bvec->bv_page->flags);
1273
1274                 if (bp->b_flags & _XBF_PAGE_LOCKED)
1275                         unlock_page(page);
1276         } while (bvec >= bio->bi_io_vec);
1277
1278         _xfs_buf_ioend(bp, 1);
1279         bio_put(bio);
1280 }
1281
1282 STATIC void
1283 _xfs_buf_ioapply(
1284         xfs_buf_t               *bp)
1285 {
1286         int                     rw, map_i, total_nr_pages, nr_pages;
1287         struct bio              *bio;
1288         int                     offset = bp->b_offset;
1289         int                     size = bp->b_count_desired;
1290         sector_t                sector = bp->b_bn;
1291         unsigned int            blocksize = bp->b_target->bt_bsize;
1292
1293         total_nr_pages = bp->b_page_count;
1294         map_i = 0;
1295
1296         if (bp->b_flags & XBF_ORDERED) {
1297                 ASSERT(!(bp->b_flags & XBF_READ));
1298                 rw = WRITE_BARRIER;
1299         } else if (bp->b_flags & XBF_LOG_BUFFER) {
1300                 ASSERT(!(bp->b_flags & XBF_READ_AHEAD));
1301                 bp->b_flags &= ~_XBF_RUN_QUEUES;
1302                 rw = (bp->b_flags & XBF_WRITE) ? WRITE_SYNC : READ_SYNC;
1303         } else if (bp->b_flags & _XBF_RUN_QUEUES) {
1304                 ASSERT(!(bp->b_flags & XBF_READ_AHEAD));
1305                 bp->b_flags &= ~_XBF_RUN_QUEUES;
1306                 rw = (bp->b_flags & XBF_WRITE) ? WRITE_META : READ_META;
1307         } else {
1308                 rw = (bp->b_flags & XBF_WRITE) ? WRITE :
1309                      (bp->b_flags & XBF_READ_AHEAD) ? READA : READ;
1310         }
1311
1312         /* Special code path for reading a sub page size buffer in --
1313          * we populate up the whole page, and hence the other metadata
1314          * in the same page.  This optimization is only valid when the
1315          * filesystem block size is not smaller than the page size.
1316          */
1317         if ((bp->b_buffer_length < PAGE_CACHE_SIZE) &&
1318             ((bp->b_flags & (XBF_READ|_XBF_PAGE_LOCKED)) ==
1319               (XBF_READ|_XBF_PAGE_LOCKED)) &&
1320             (blocksize >= PAGE_CACHE_SIZE)) {
1321                 bio = bio_alloc(GFP_NOIO, 1);
1322
1323                 bio->bi_bdev = bp->b_target->bt_bdev;
1324                 bio->bi_sector = sector - (offset >> BBSHIFT);
1325                 bio->bi_end_io = xfs_buf_bio_end_io;
1326                 bio->bi_private = bp;
1327
1328                 bio_add_page(bio, bp->b_pages[0], PAGE_CACHE_SIZE, 0);
1329                 size = 0;
1330
1331                 atomic_inc(&bp->b_io_remaining);
1332
1333                 goto submit_io;
1334         }
1335
1336 next_chunk:
1337         atomic_inc(&bp->b_io_remaining);
1338         nr_pages = BIO_MAX_SECTORS >> (PAGE_SHIFT - BBSHIFT);
1339         if (nr_pages > total_nr_pages)
1340                 nr_pages = total_nr_pages;
1341
1342         bio = bio_alloc(GFP_NOIO, nr_pages);
1343         bio->bi_bdev = bp->b_target->bt_bdev;
1344         bio->bi_sector = sector;
1345         bio->bi_end_io = xfs_buf_bio_end_io;
1346         bio->bi_private = bp;
1347
1348         for (; size && nr_pages; nr_pages--, map_i++) {
1349                 int     rbytes, nbytes = PAGE_CACHE_SIZE - offset;
1350
1351                 if (nbytes > size)
1352                         nbytes = size;
1353
1354                 rbytes = bio_add_page(bio, bp->b_pages[map_i], nbytes, offset);
1355                 if (rbytes < nbytes)
1356                         break;
1357
1358                 offset = 0;
1359                 sector += nbytes >> BBSHIFT;
1360                 size -= nbytes;
1361                 total_nr_pages--;
1362         }
1363
1364 submit_io:
1365         if (likely(bio->bi_size)) {
1366                 submit_bio(rw, bio);
1367                 if (size)
1368                         goto next_chunk;
1369         } else {
1370                 bio_put(bio);
1371                 xfs_buf_ioerror(bp, EIO);
1372         }
1373 }
1374
1375 int
1376 xfs_buf_iorequest(
1377         xfs_buf_t               *bp)
1378 {
1379         trace_xfs_buf_iorequest(bp, _RET_IP_);
1380
1381         if (bp->b_flags & XBF_DELWRI) {
1382                 xfs_buf_delwri_queue(bp, 1);
1383                 return 0;
1384         }
1385
1386         if (bp->b_flags & XBF_WRITE) {
1387                 xfs_buf_wait_unpin(bp);
1388         }
1389
1390         xfs_buf_hold(bp);
1391
1392         /* Set the count to 1 initially, this will stop an I/O
1393          * completion callout which happens before we have started
1394          * all the I/O from calling xfs_buf_ioend too early.
1395          */
1396         atomic_set(&bp->b_io_remaining, 1);
1397         _xfs_buf_ioapply(bp);
1398         _xfs_buf_ioend(bp, 0);
1399
1400         xfs_buf_rele(bp);
1401         return 0;
1402 }
1403
1404 /*
1405  *      Waits for I/O to complete on the buffer supplied.
1406  *      It returns immediately if no I/O is pending.
1407  *      It returns the I/O error code, if any, or 0 if there was no error.
1408  */
1409 int
1410 xfs_buf_iowait(
1411         xfs_buf_t               *bp)
1412 {
1413         trace_xfs_buf_iowait(bp, _RET_IP_);
1414
1415         if (atomic_read(&bp->b_io_remaining))
1416                 blk_run_address_space(bp->b_target->bt_mapping);
1417         wait_for_completion(&bp->b_iowait);
1418
1419         trace_xfs_buf_iowait_done(bp, _RET_IP_);
1420         return bp->b_error;
1421 }
1422
1423 xfs_caddr_t
1424 xfs_buf_offset(
1425         xfs_buf_t               *bp,
1426         size_t                  offset)
1427 {
1428         struct page             *page;
1429
1430         if (bp->b_flags & XBF_MAPPED)
1431                 return XFS_BUF_PTR(bp) + offset;
1432
1433         offset += bp->b_offset;
1434         page = bp->b_pages[offset >> PAGE_CACHE_SHIFT];
1435         return (xfs_caddr_t)page_address(page) + (offset & (PAGE_CACHE_SIZE-1));
1436 }
1437
1438 /*
1439  *      Move data into or out of a buffer.
1440  */
1441 void
1442 xfs_buf_iomove(
1443         xfs_buf_t               *bp,    /* buffer to process            */
1444         size_t                  boff,   /* starting buffer offset       */
1445         size_t                  bsize,  /* length to copy               */
1446         void                    *data,  /* data address                 */
1447         xfs_buf_rw_t            mode)   /* read/write/zero flag         */
1448 {
1449         size_t                  bend, cpoff, csize;
1450         struct page             *page;
1451
1452         bend = boff + bsize;
1453         while (boff < bend) {
1454                 page = bp->b_pages[xfs_buf_btoct(boff + bp->b_offset)];
1455                 cpoff = xfs_buf_poff(boff + bp->b_offset);
1456                 csize = min_t(size_t,
1457                               PAGE_CACHE_SIZE-cpoff, bp->b_count_desired-boff);
1458
1459                 ASSERT(((csize + cpoff) <= PAGE_CACHE_SIZE));
1460
1461                 switch (mode) {
1462                 case XBRW_ZERO:
1463                         memset(page_address(page) + cpoff, 0, csize);
1464                         break;
1465                 case XBRW_READ:
1466                         memcpy(data, page_address(page) + cpoff, csize);
1467                         break;
1468                 case XBRW_WRITE:
1469                         memcpy(page_address(page) + cpoff, data, csize);
1470                 }
1471
1472                 boff += csize;
1473                 data += csize;
1474         }
1475 }
1476
1477 /*
1478  *      Handling of buffer targets (buftargs).
1479  */
1480
1481 /*
1482  *      Wait for any bufs with callbacks that have been submitted but
1483  *      have not yet returned... walk the hash list for the target.
1484  */
1485 void
1486 xfs_wait_buftarg(
1487         xfs_buftarg_t   *btp)
1488 {
1489         xfs_buf_t       *bp, *n;
1490         xfs_bufhash_t   *hash;
1491         uint            i;
1492
1493         for (i = 0; i < (1 << btp->bt_hashshift); i++) {
1494                 hash = &btp->bt_hash[i];
1495 again:
1496                 spin_lock(&hash->bh_lock);
1497                 list_for_each_entry_safe(bp, n, &hash->bh_list, b_hash_list) {
1498                         ASSERT(btp == bp->b_target);
1499                         if (!(bp->b_flags & XBF_FS_MANAGED)) {
1500                                 spin_unlock(&hash->bh_lock);
1501                                 /*
1502                                  * Catch superblock reference count leaks
1503                                  * immediately
1504                                  */
1505                                 BUG_ON(bp->b_bn == 0);
1506                                 delay(100);
1507                                 goto again;
1508                         }
1509                 }
1510                 spin_unlock(&hash->bh_lock);
1511         }
1512 }
1513
1514 /*
1515  *      Allocate buffer hash table for a given target.
1516  *      For devices containing metadata (i.e. not the log/realtime devices)
1517  *      we need to allocate a much larger hash table.
1518  */
1519 STATIC void
1520 xfs_alloc_bufhash(
1521         xfs_buftarg_t           *btp,
1522         int                     external)
1523 {
1524         unsigned int            i;
1525
1526         btp->bt_hashshift = external ? 3 : 8;   /* 8 or 256 buckets */
1527         btp->bt_hashmask = (1 << btp->bt_hashshift) - 1;
1528         btp->bt_hash = kmem_zalloc((1 << btp->bt_hashshift) *
1529                                         sizeof(xfs_bufhash_t), KM_SLEEP | KM_LARGE);
1530         for (i = 0; i < (1 << btp->bt_hashshift); i++) {
1531                 spin_lock_init(&btp->bt_hash[i].bh_lock);
1532                 INIT_LIST_HEAD(&btp->bt_hash[i].bh_list);
1533         }
1534 }
1535
1536 STATIC void
1537 xfs_free_bufhash(
1538         xfs_buftarg_t           *btp)
1539 {
1540         kmem_free(btp->bt_hash);
1541         btp->bt_hash = NULL;
1542 }
1543
1544 /*
1545  *      buftarg list for delwrite queue processing
1546  */
1547 static LIST_HEAD(xfs_buftarg_list);
1548 static DEFINE_SPINLOCK(xfs_buftarg_lock);
1549
1550 STATIC void
1551 xfs_register_buftarg(
1552         xfs_buftarg_t           *btp)
1553 {
1554         spin_lock(&xfs_buftarg_lock);
1555         list_add(&btp->bt_list, &xfs_buftarg_list);
1556         spin_unlock(&xfs_buftarg_lock);
1557 }
1558
1559 STATIC void
1560 xfs_unregister_buftarg(
1561         xfs_buftarg_t           *btp)
1562 {
1563         spin_lock(&xfs_buftarg_lock);
1564         list_del(&btp->bt_list);
1565         spin_unlock(&xfs_buftarg_lock);
1566 }
1567
1568 void
1569 xfs_free_buftarg(
1570         struct xfs_mount        *mp,
1571         struct xfs_buftarg      *btp)
1572 {
1573         xfs_flush_buftarg(btp, 1);
1574         if (mp->m_flags & XFS_MOUNT_BARRIER)
1575                 xfs_blkdev_issue_flush(btp);
1576         xfs_free_bufhash(btp);
1577         iput(btp->bt_mapping->host);
1578
1579         /* Unregister the buftarg first so that we don't get a
1580          * wakeup finding a non-existent task
1581          */
1582         xfs_unregister_buftarg(btp);
1583         kthread_stop(btp->bt_task);
1584
1585         kmem_free(btp);
1586 }
1587
1588 STATIC int
1589 xfs_setsize_buftarg_flags(
1590         xfs_buftarg_t           *btp,
1591         unsigned int            blocksize,
1592         unsigned int            sectorsize,
1593         int                     verbose)
1594 {
1595         btp->bt_bsize = blocksize;
1596         btp->bt_sshift = ffs(sectorsize) - 1;
1597         btp->bt_smask = sectorsize - 1;
1598
1599         if (set_blocksize(btp->bt_bdev, sectorsize)) {
1600                 printk(KERN_WARNING
1601                         "XFS: Cannot set_blocksize to %u on device %s\n",
1602                         sectorsize, XFS_BUFTARG_NAME(btp));
1603                 return EINVAL;
1604         }
1605
1606         if (verbose &&
1607             (PAGE_CACHE_SIZE / BITS_PER_LONG) > sectorsize) {
1608                 printk(KERN_WARNING
1609                         "XFS: %u byte sectors in use on device %s.  "
1610                         "This is suboptimal; %u or greater is ideal.\n",
1611                         sectorsize, XFS_BUFTARG_NAME(btp),
1612                         (unsigned int)PAGE_CACHE_SIZE / BITS_PER_LONG);
1613         }
1614
1615         return 0;
1616 }
1617
1618 /*
1619  *      When allocating the initial buffer target we have not yet
1620  *      read in the superblock, so don't know what sized sectors
1621  *      are being used is at this early stage.  Play safe.
1622  */
1623 STATIC int
1624 xfs_setsize_buftarg_early(
1625         xfs_buftarg_t           *btp,
1626         struct block_device     *bdev)
1627 {
1628         return xfs_setsize_buftarg_flags(btp,
1629                         PAGE_CACHE_SIZE, bdev_logical_block_size(bdev), 0);
1630 }
1631
1632 int
1633 xfs_setsize_buftarg(
1634         xfs_buftarg_t           *btp,
1635         unsigned int            blocksize,
1636         unsigned int            sectorsize)
1637 {
1638         return xfs_setsize_buftarg_flags(btp, blocksize, sectorsize, 1);
1639 }
1640
1641 STATIC int
1642 xfs_mapping_buftarg(
1643         xfs_buftarg_t           *btp,
1644         struct block_device     *bdev)
1645 {
1646         struct backing_dev_info *bdi;
1647         struct inode            *inode;
1648         struct address_space    *mapping;
1649         static const struct address_space_operations mapping_aops = {
1650                 .sync_page = block_sync_page,
1651                 .migratepage = fail_migrate_page,
1652         };
1653
1654         inode = new_inode(bdev->bd_inode->i_sb);
1655         if (!inode) {
1656                 printk(KERN_WARNING
1657                         "XFS: Cannot allocate mapping inode for device %s\n",
1658                         XFS_BUFTARG_NAME(btp));
1659                 return ENOMEM;
1660         }
1661         inode->i_mode = S_IFBLK;
1662         inode->i_bdev = bdev;
1663         inode->i_rdev = bdev->bd_dev;
1664         bdi = blk_get_backing_dev_info(bdev);
1665         if (!bdi)
1666                 bdi = &default_backing_dev_info;
1667         mapping = &inode->i_data;
1668         mapping->a_ops = &mapping_aops;
1669         mapping->backing_dev_info = bdi;
1670         mapping_set_gfp_mask(mapping, GFP_NOFS);
1671         btp->bt_mapping = mapping;
1672         return 0;
1673 }
1674
1675 STATIC int
1676 xfs_alloc_delwrite_queue(
1677         xfs_buftarg_t           *btp)
1678 {
1679         int     error = 0;
1680
1681         INIT_LIST_HEAD(&btp->bt_list);
1682         INIT_LIST_HEAD(&btp->bt_delwrite_queue);
1683         spin_lock_init(&btp->bt_delwrite_lock);
1684         btp->bt_flags = 0;
1685         btp->bt_task = kthread_run(xfsbufd, btp, "xfsbufd");
1686         if (IS_ERR(btp->bt_task)) {
1687                 error = PTR_ERR(btp->bt_task);
1688                 goto out_error;
1689         }
1690         xfs_register_buftarg(btp);
1691 out_error:
1692         return error;
1693 }
1694
1695 xfs_buftarg_t *
1696 xfs_alloc_buftarg(
1697         struct block_device     *bdev,
1698         int                     external)
1699 {
1700         xfs_buftarg_t           *btp;
1701
1702         btp = kmem_zalloc(sizeof(*btp), KM_SLEEP);
1703
1704         btp->bt_dev =  bdev->bd_dev;
1705         btp->bt_bdev = bdev;
1706         if (xfs_setsize_buftarg_early(btp, bdev))
1707                 goto error;
1708         if (xfs_mapping_buftarg(btp, bdev))
1709                 goto error;
1710         if (xfs_alloc_delwrite_queue(btp))
1711                 goto error;
1712         xfs_alloc_bufhash(btp, external);
1713         return btp;
1714
1715 error:
1716         kmem_free(btp);
1717         return NULL;
1718 }
1719
1720
1721 /*
1722  *      Delayed write buffer handling
1723  */
1724 STATIC void
1725 xfs_buf_delwri_queue(
1726         xfs_buf_t               *bp,
1727         int                     unlock)
1728 {
1729         struct list_head        *dwq = &bp->b_target->bt_delwrite_queue;
1730         spinlock_t              *dwlk = &bp->b_target->bt_delwrite_lock;
1731
1732         trace_xfs_buf_delwri_queue(bp, _RET_IP_);
1733
1734         ASSERT((bp->b_flags&(XBF_DELWRI|XBF_ASYNC)) == (XBF_DELWRI|XBF_ASYNC));
1735
1736         spin_lock(dwlk);
1737         /* If already in the queue, dequeue and place at tail */
1738         if (!list_empty(&bp->b_list)) {
1739                 ASSERT(bp->b_flags & _XBF_DELWRI_Q);
1740                 if (unlock)
1741                         atomic_dec(&bp->b_hold);
1742                 list_del(&bp->b_list);
1743         }
1744
1745         if (list_empty(dwq)) {
1746                 /* start xfsbufd as it is about to have something to do */
1747                 wake_up_process(bp->b_target->bt_task);
1748         }
1749
1750         bp->b_flags |= _XBF_DELWRI_Q;
1751         list_add_tail(&bp->b_list, dwq);
1752         bp->b_queuetime = jiffies;
1753         spin_unlock(dwlk);
1754
1755         if (unlock)
1756                 xfs_buf_unlock(bp);
1757 }
1758
1759 void
1760 xfs_buf_delwri_dequeue(
1761         xfs_buf_t               *bp)
1762 {
1763         spinlock_t              *dwlk = &bp->b_target->bt_delwrite_lock;
1764         int                     dequeued = 0;
1765
1766         spin_lock(dwlk);
1767         if ((bp->b_flags & XBF_DELWRI) && !list_empty(&bp->b_list)) {
1768                 ASSERT(bp->b_flags & _XBF_DELWRI_Q);
1769                 list_del_init(&bp->b_list);
1770                 dequeued = 1;
1771         }
1772         bp->b_flags &= ~(XBF_DELWRI|_XBF_DELWRI_Q);
1773         spin_unlock(dwlk);
1774
1775         if (dequeued)
1776                 xfs_buf_rele(bp);
1777
1778         trace_xfs_buf_delwri_dequeue(bp, _RET_IP_);
1779 }
1780
1781 STATIC void
1782 xfs_buf_runall_queues(
1783         struct workqueue_struct *queue)
1784 {
1785         flush_workqueue(queue);
1786 }
1787
1788 STATIC int
1789 xfsbufd_wakeup(
1790         int                     priority,
1791         gfp_t                   mask)
1792 {
1793         xfs_buftarg_t           *btp;
1794
1795         spin_lock(&xfs_buftarg_lock);
1796         list_for_each_entry(btp, &xfs_buftarg_list, bt_list) {
1797                 if (test_bit(XBT_FORCE_SLEEP, &btp->bt_flags))
1798                         continue;
1799                 if (list_empty(&btp->bt_delwrite_queue))
1800                         continue;
1801                 set_bit(XBT_FORCE_FLUSH, &btp->bt_flags);
1802                 wake_up_process(btp->bt_task);
1803         }
1804         spin_unlock(&xfs_buftarg_lock);
1805         return 0;
1806 }
1807
1808 /*
1809  * Move as many buffers as specified to the supplied list
1810  * idicating if we skipped any buffers to prevent deadlocks.
1811  */
1812 STATIC int
1813 xfs_buf_delwri_split(
1814         xfs_buftarg_t   *target,
1815         struct list_head *list,
1816         unsigned long   age)
1817 {
1818         xfs_buf_t       *bp, *n;
1819         struct list_head *dwq = &target->bt_delwrite_queue;
1820         spinlock_t      *dwlk = &target->bt_delwrite_lock;
1821         int             skipped = 0;
1822         int             force;
1823
1824         force = test_and_clear_bit(XBT_FORCE_FLUSH, &target->bt_flags);
1825         INIT_LIST_HEAD(list);
1826         spin_lock(dwlk);
1827         list_for_each_entry_safe(bp, n, dwq, b_list) {
1828                 trace_xfs_buf_delwri_split(bp, _RET_IP_);
1829                 ASSERT(bp->b_flags & XBF_DELWRI);
1830
1831                 if (!xfs_buf_ispin(bp) && !xfs_buf_cond_lock(bp)) {
1832                         if (!force &&
1833                             time_before(jiffies, bp->b_queuetime + age)) {
1834                                 xfs_buf_unlock(bp);
1835                                 break;
1836                         }
1837
1838                         bp->b_flags &= ~(XBF_DELWRI|_XBF_DELWRI_Q|
1839                                          _XBF_RUN_QUEUES);
1840                         bp->b_flags |= XBF_WRITE;
1841                         list_move_tail(&bp->b_list, list);
1842                 } else
1843                         skipped++;
1844         }
1845         spin_unlock(dwlk);
1846
1847         return skipped;
1848
1849 }
1850
1851 STATIC int
1852 xfsbufd(
1853         void            *data)
1854 {
1855         struct list_head tmp;
1856         xfs_buftarg_t   *target = (xfs_buftarg_t *)data;
1857         int             count;
1858         xfs_buf_t       *bp;
1859
1860         current->flags |= PF_MEMALLOC;
1861
1862         set_freezable();
1863
1864         do {
1865                 long    age = xfs_buf_age_centisecs * msecs_to_jiffies(10);
1866                 long    tout = xfs_buf_timer_centisecs * msecs_to_jiffies(10);
1867
1868                 if (unlikely(freezing(current))) {
1869                         set_bit(XBT_FORCE_SLEEP, &target->bt_flags);
1870                         refrigerator();
1871                 } else {
1872                         clear_bit(XBT_FORCE_SLEEP, &target->bt_flags);
1873                 }
1874
1875                 /* sleep for a long time if there is nothing to do. */
1876                 if (list_empty(&target->bt_delwrite_queue))
1877                         tout = MAX_SCHEDULE_TIMEOUT;
1878                 schedule_timeout_interruptible(tout);
1879
1880                 xfs_buf_delwri_split(target, &tmp, age);
1881                 count = 0;
1882                 while (!list_empty(&tmp)) {
1883                         bp = list_entry(tmp.next, xfs_buf_t, b_list);
1884                         ASSERT(target == bp->b_target);
1885
1886                         list_del_init(&bp->b_list);
1887                         xfs_buf_iostrategy(bp);
1888                         count++;
1889                 }
1890
1891                 if (as_list_len > 0)
1892                         purge_addresses();
1893                 if (count)
1894                         blk_run_address_space(target->bt_mapping);
1895
1896         } while (!kthread_should_stop());
1897
1898         return 0;
1899 }
1900
1901 /*
1902  *      Go through all incore buffers, and release buffers if they belong to
1903  *      the given device. This is used in filesystem error handling to
1904  *      preserve the consistency of its metadata.
1905  */
1906 int
1907 xfs_flush_buftarg(
1908         xfs_buftarg_t   *target,
1909         int             wait)
1910 {
1911         struct list_head tmp;
1912         xfs_buf_t       *bp, *n;
1913         int             pincount = 0;
1914
1915         xfs_buf_runall_queues(xfsconvertd_workqueue);
1916         xfs_buf_runall_queues(xfsdatad_workqueue);
1917         xfs_buf_runall_queues(xfslogd_workqueue);
1918
1919         set_bit(XBT_FORCE_FLUSH, &target->bt_flags);
1920         pincount = xfs_buf_delwri_split(target, &tmp, 0);
1921
1922         /*
1923          * Dropped the delayed write list lock, now walk the temporary list
1924          */
1925         list_for_each_entry_safe(bp, n, &tmp, b_list) {
1926                 ASSERT(target == bp->b_target);
1927                 if (wait)
1928                         bp->b_flags &= ~XBF_ASYNC;
1929                 else
1930                         list_del_init(&bp->b_list);
1931
1932                 xfs_buf_iostrategy(bp);
1933         }
1934
1935         if (wait)
1936                 blk_run_address_space(target->bt_mapping);
1937
1938         /*
1939          * Remaining list items must be flushed before returning
1940          */
1941         while (!list_empty(&tmp)) {
1942                 bp = list_entry(tmp.next, xfs_buf_t, b_list);
1943
1944                 list_del_init(&bp->b_list);
1945                 xfs_iowait(bp);
1946                 xfs_buf_relse(bp);
1947         }
1948
1949         return pincount;
1950 }
1951
1952 int __init
1953 xfs_buf_init(void)
1954 {
1955         xfs_buf_zone = kmem_zone_init_flags(sizeof(xfs_buf_t), "xfs_buf",
1956                                                 KM_ZONE_HWALIGN, NULL);
1957         if (!xfs_buf_zone)
1958                 goto out;
1959
1960         xfslogd_workqueue = create_workqueue("xfslogd");
1961         if (!xfslogd_workqueue)
1962                 goto out_free_buf_zone;
1963
1964         xfsdatad_workqueue = create_workqueue("xfsdatad");
1965         if (!xfsdatad_workqueue)
1966                 goto out_destroy_xfslogd_workqueue;
1967
1968         xfsconvertd_workqueue = create_workqueue("xfsconvertd");
1969         if (!xfsconvertd_workqueue)
1970                 goto out_destroy_xfsdatad_workqueue;
1971
1972         register_shrinker(&xfs_buf_shake);
1973         return 0;
1974
1975  out_destroy_xfsdatad_workqueue:
1976         destroy_workqueue(xfsdatad_workqueue);
1977  out_destroy_xfslogd_workqueue:
1978         destroy_workqueue(xfslogd_workqueue);
1979  out_free_buf_zone:
1980         kmem_zone_destroy(xfs_buf_zone);
1981  out:
1982         return -ENOMEM;
1983 }
1984
1985 void
1986 xfs_buf_terminate(void)
1987 {
1988         unregister_shrinker(&xfs_buf_shake);
1989         destroy_workqueue(xfsconvertd_workqueue);
1990         destroy_workqueue(xfsdatad_workqueue);
1991         destroy_workqueue(xfslogd_workqueue);
1992         kmem_zone_destroy(xfs_buf_zone);
1993 }
1994
1995 #ifdef CONFIG_KDB_MODULES
1996 struct list_head *
1997 xfs_get_buftarg_list(void)
1998 {
1999         return &xfs_buftarg_list;
2000 }
2001 #endif