fs/xfs/linux-2.6/xfs_buf.c

   1 /*
   2  * Copyright (c) 2000-2004 Silicon Graphics, Inc.  All Rights Reserved.
   3  *
   4  * This program is free software; you can redistribute it and/or modify it
   5  * under the terms of version 2 of the GNU General Public License as
   6  * published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it would be useful, but
   9  * WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
  11  *
  12  * Further, this software is distributed without any warranty that it is
  13  * free of the rightful claim of any third person regarding infringement
  14  * or the like.  Any license provided herein, whether implied or
  15  * otherwise, applies only to this software file.  Patent licenses, if
  16  * any, provided herein do not apply to combinations of this program with
  17  * other software, or any other product whatsoever.
  18  *
  19  * You should have received a copy of the GNU General Public License along
  20  * with this program; if not, write the Free Software Foundation, Inc., 59
  21  * Temple Place - Suite 330, Boston MA 02111-1307, USA.
  22  *
  23  * Contact information: Silicon Graphics, Inc., 1600 Amphitheatre Pkwy,
  24  * Mountain View, CA  94043, or:
  25  *
  26  * http://www.sgi.com
  27  *
  28  * For further information regarding this notice, see:
  29  *
  30  * http://oss.sgi.com/projects/GenInfo/SGIGPLNoticeExplan/
  31  */
  32
  33 /*
  34  *      The xfs_buf.c code provides an abstract buffer cache model on top
  35  *      of the Linux page cache.  Cached metadata blocks for a file system
  36  *      are hashed to the inode for the block device.  xfs_buf.c assembles
  37  *      buffers (xfs_buf_t) on demand to aggregate such cached pages for I/O.
  38  *
  39  *      Written by Steve Lord, Jim Mostek, Russell Cattelan
  40  *                  and Rajagopal Ananthanarayanan ("ananth") at SGI.
  41  *
  42  */
  43
  44 #include <linux/stddef.h>
  45 #include <linux/errno.h>
  46 #include <linux/slab.h>
  47 #include <linux/pagemap.h>
  48 #include <linux/init.h>
  49 #include <linux/vmalloc.h>
  50 #include <linux/bio.h>
  51 #include <linux/sysctl.h>
  52 #include <linux/proc_fs.h>
  53 #include <linux/workqueue.h>
  54 #include <linux/percpu.h>
  55 #include <linux/blkdev.h>
  56 #include <linux/hash.h>
  57
  58 #include "xfs_linux.h"
  59
  60 /*
  61  * File wide globals
  62  */
  63
  64 STATIC kmem_cache_t *pagebuf_zone;
  65 STATIC kmem_shaker_t pagebuf_shake;
  66 STATIC int xfsbufd_wakeup(int, unsigned int);
  67 STATIC void pagebuf_delwri_queue(xfs_buf_t *, int);
  68
  69 STATIC struct workqueue_struct *xfslogd_workqueue;
  70 STATIC struct workqueue_struct *xfsdatad_workqueue;
  71
  72 /*
  73  * Pagebuf debugging
  74  */
  75
  76 #ifdef PAGEBUF_TRACE
  77 void
  78 pagebuf_trace(
  79         xfs_buf_t       *pb,
  80         char            *id,
  81         void            *data,
  82         void            *ra)
  83 {
  84         ktrace_enter(pagebuf_trace_buf,
  85                 pb, id,
  86                 (void *)(unsigned long)pb->pb_flags,
  87                 (void *)(unsigned long)pb->pb_hold.counter,
  88                 (void *)(unsigned long)pb->pb_sema.count.counter,
  89                 (void *)current,
  90                 data, ra,
  91                 (void *)(unsigned long)((pb->pb_file_offset>>32) & 0xffffffff),
  92                 (void *)(unsigned long)(pb->pb_file_offset & 0xffffffff),
  93                 (void *)(unsigned long)pb->pb_buffer_length,
  94                 NULL, NULL, NULL, NULL, NULL);
  95 }
  96 ktrace_t *pagebuf_trace_buf;
  97 #define PAGEBUF_TRACE_SIZE      4096
  98 #define PB_TRACE(pb, id, data)  \
  99         pagebuf_trace(pb, id, (void *)data, (void *)__builtin_return_address(0))
 100 #else
 101 #define PB_TRACE(pb, id, data)  do { } while (0)
 102 #endif
 103
 104 #ifdef PAGEBUF_LOCK_TRACKING
 105 # define PB_SET_OWNER(pb)       ((pb)->pb_last_holder = current->pid)
 106 # define PB_CLEAR_OWNER(pb)     ((pb)->pb_last_holder = -1)
 107 # define PB_GET_OWNER(pb)       ((pb)->pb_last_holder)
 108 #else
 109 # define PB_SET_OWNER(pb)       do { } while (0)
 110 # define PB_CLEAR_OWNER(pb)     do { } while (0)
 111 # define PB_GET_OWNER(pb)       do { } while (0)
 112 #endif
 113
 114 /*
 115  * Pagebuf allocation / freeing.
 116  */
 117
 118 #define pb_to_gfp(flags) \
 119         ((((flags) & PBF_READ_AHEAD) ? __GFP_NORETRY : \
 120           ((flags) & PBF_DONT_BLOCK) ? GFP_NOFS : GFP_KERNEL) | __GFP_NOWARN)
 121
 122 #define pb_to_km(flags) \
 123          (((flags) & PBF_DONT_BLOCK) ? KM_NOFS : KM_SLEEP)
 124
 125
 126 #define pagebuf_allocate(flags) \
 127         kmem_zone_alloc(pagebuf_zone, pb_to_km(flags))
 128 #define pagebuf_deallocate(pb) \
 129         kmem_zone_free(pagebuf_zone, (pb));
 130
 131 /*
 132  * Page Region interfaces.
 133  *
 134  * For pages in filesystems where the blocksize is smaller than the
 135  * pagesize, we use the page->private field (long) to hold a bitmap
 136  * of uptodate regions within the page.
 137  *
 138  * Each such region is "bytes per page / bits per long" bytes long.
 139  *
 140  * NBPPR == number-of-bytes-per-page-region
 141  * BTOPR == bytes-to-page-region (rounded up)
 142  * BTOPRT == bytes-to-page-region-truncated (rounded down)
 143  */
 144 #if (BITS_PER_LONG == 32)
 145 #define PRSHIFT         (PAGE_CACHE_SHIFT - 5)  /* (32 == 1<<5) */
 146 #elif (BITS_PER_LONG == 64)
 147 #define PRSHIFT         (PAGE_CACHE_SHIFT - 6)  /* (64 == 1<<6) */
 148 #else
 149 #error BITS_PER_LONG must be 32 or 64
 150 #endif
 151 #define NBPPR           (PAGE_CACHE_SIZE/BITS_PER_LONG)
 152 #define BTOPR(b)        (((unsigned int)(b) + (NBPPR - 1)) >> PRSHIFT)
 153 #define BTOPRT(b)       (((unsigned int)(b) >> PRSHIFT))
 154
 155 STATIC unsigned long
 156 page_region_mask(
 157         size_t          offset,
 158         size_t          length)
 159 {
 160         unsigned long   mask;
 161         int             first, final;
 162
 163         first = BTOPR(offset);
 164         final = BTOPRT(offset + length - 1);
 165         first = min(first, final);
 166
 167         mask = ~0UL;
 168         mask <<= BITS_PER_LONG - (final - first);
 169         mask >>= BITS_PER_LONG - (final);
 170
 171         ASSERT(offset + length <= PAGE_CACHE_SIZE);
 172         ASSERT((final - first) < BITS_PER_LONG && (final - first) >= 0);
 173
 174         return mask;
 175 }
 176
 177 STATIC inline void
 178 set_page_region(
 179         struct page     *page,
 180         size_t          offset,
 181         size_t          length)
 182 {
 183         page->private |= page_region_mask(offset, length);
 184         if (page->private == ~0UL)
 185                 SetPageUptodate(page);
 186 }
 187
 188 STATIC inline int
 189 test_page_region(
 190         struct page     *page,
 191         size_t          offset,
 192         size_t          length)
 193 {
 194         unsigned long   mask = page_region_mask(offset, length);
 195
 196         return (mask && (page->private & mask) == mask);
 197 }
 198
 199 /*
 200  * Mapping of multi-page buffers into contiguous virtual space
 201  */
 202
 203 typedef struct a_list {
 204         void            *vm_addr;
 205         struct a_list   *next;
 206 } a_list_t;
 207
 208 STATIC a_list_t         *as_free_head;
 209 STATIC int              as_list_len;
 210 STATIC DEFINE_SPINLOCK(as_lock);
 211
 212 /*
 213  * Try to batch vunmaps because they are costly.
 214  */
 215 STATIC void
 216 free_address(
 217         void            *addr)
 218 {
 219         a_list_t        *aentry;
 220
 221         aentry = kmalloc(sizeof(a_list_t), GFP_ATOMIC & ~__GFP_HIGH);
 222         if (likely(aentry)) {
 223                 spin_lock(&as_lock);
 224                 aentry->next = as_free_head;
 225                 aentry->vm_addr = addr;
 226                 as_free_head = aentry;
 227                 as_list_len++;
 228                 spin_unlock(&as_lock);
 229         } else {
 230                 vunmap(addr);
 231         }
 232 }
 233
 234 STATIC void
 235 purge_addresses(void)
 236 {
 237         a_list_t        *aentry, *old;
 238
 239         if (as_free_head == NULL)
 240                 return;
 241
 242         spin_lock(&as_lock);
 243         aentry = as_free_head;
 244         as_free_head = NULL;
 245         as_list_len = 0;
 246         spin_unlock(&as_lock);
 247
 248         while ((old = aentry) != NULL) {
 249                 vunmap(aentry->vm_addr);
 250                 aentry = aentry->next;
 251                 kfree(old);
 252         }
 253 }
 254
 255 /*
 256  *      Internal pagebuf object manipulation
 257  */
 258
 259 STATIC void
 260 _pagebuf_initialize(
 261         xfs_buf_t               *pb,
 262         xfs_buftarg_t           *target,
 263         loff_t                  range_base,
 264         size_t                  range_length,
 265         page_buf_flags_t        flags)
 266 {
 267         /*
 268          * We don't want certain flags to appear in pb->pb_flags.
 269          */
 270         flags &= ~(PBF_LOCK|PBF_MAPPED|PBF_DONT_BLOCK|PBF_READ_AHEAD);
 271
 272         memset(pb, 0, sizeof(xfs_buf_t));
 273         atomic_set(&pb->pb_hold, 1);
 274         init_MUTEX_LOCKED(&pb->pb_iodonesema);
 275         INIT_LIST_HEAD(&pb->pb_list);
 276         INIT_LIST_HEAD(&pb->pb_hash_list);
 277         init_MUTEX_LOCKED(&pb->pb_sema); /* held, no waiters */
 278         PB_SET_OWNER(pb);
 279         pb->pb_target = target;
 280         pb->pb_file_offset = range_base;
 281         /*
 282          * Set buffer_length and count_desired to the same value initially.
 283          * I/O routines should use count_desired, which will be the same in
 284          * most cases but may be reset (e.g. XFS recovery).
 285          */
 286         pb->pb_buffer_length = pb->pb_count_desired = range_length;
 287         pb->pb_flags = flags | PBF_NONE;
 288         pb->pb_bn = XFS_BUF_DADDR_NULL;
 289         atomic_set(&pb->pb_pin_count, 0);
 290         init_waitqueue_head(&pb->pb_waiters);
 291
 292         XFS_STATS_INC(pb_create);
 293         PB_TRACE(pb, "initialize", target);
 294 }
 295
 296 /*
 297  * Allocate a page array capable of holding a specified number
 298  * of pages, and point the page buf at it.
 299  */
 300 STATIC int
 301 _pagebuf_get_pages(
 302         xfs_buf_t               *pb,
 303         int                     page_count,
 304         page_buf_flags_t        flags)
 305 {
 306         /* Make sure that we have a page list */
 307         if (pb->pb_pages == NULL) {
 308                 pb->pb_offset = page_buf_poff(pb->pb_file_offset);
 309                 pb->pb_page_count = page_count;
 310                 if (page_count <= PB_PAGES) {
 311                         pb->pb_pages = pb->pb_page_array;
 312                 } else {
 313                         pb->pb_pages = kmem_alloc(sizeof(struct page *) *
 314                                         page_count, pb_to_km(flags));
 315                         if (pb->pb_pages == NULL)
 316                                 return -ENOMEM;
 317                 }
 318                 memset(pb->pb_pages, 0, sizeof(struct page *) * page_count);
 319         }
 320         return 0;
 321 }
 322
 323 /*
 324  *      Frees pb_pages if it was malloced.
 325  */
 326 STATIC void
 327 _pagebuf_free_pages(
 328         xfs_buf_t       *bp)
 329 {
 330         if (bp->pb_pages != bp->pb_page_array) {
 331                 kmem_free(bp->pb_pages,
 332                           bp->pb_page_count * sizeof(struct page *));
 333         }
 334 }
 335
 336 /*
 337  *      Releases the specified buffer.
 338  *
 339  *      The modification state of any associated pages is left unchanged.
 340  *      The buffer most not be on any hash - use pagebuf_rele instead for
 341  *      hashed and refcounted buffers
 342  */
 343 void
 344 pagebuf_free(
 345         xfs_buf_t               *bp)
 346 {
 347         PB_TRACE(bp, "free", 0);
 348
 349         ASSERT(list_empty(&bp->pb_hash_list));
 350
 351         if (bp->pb_flags & _PBF_PAGE_CACHE) {
 352                 uint            i;
 353
 354                 if ((bp->pb_flags & PBF_MAPPED) && (bp->pb_page_count > 1))
 355                         free_address(bp->pb_addr - bp->pb_offset);
 356
 357                 for (i = 0; i < bp->pb_page_count; i++)
 358                         page_cache_release(bp->pb_pages[i]);
 359                 _pagebuf_free_pages(bp);
 360         } else if (bp->pb_flags & _PBF_KMEM_ALLOC) {
 361                  /*
 362                   * XXX(hch): bp->pb_count_desired might be incorrect (see
 363                   * pagebuf_associate_memory for details), but fortunately
 364                   * the Linux version of kmem_free ignores the len argument..
 365                   */
 366                 kmem_free(bp->pb_addr, bp->pb_count_desired);
 367                 _pagebuf_free_pages(bp);
 368         }
 369
 370         pagebuf_deallocate(bp);
 371 }
 372
 373 /*
 374  *      Finds all pages for buffer in question and builds it's page list.
 375  */
 376 STATIC int
 377 _pagebuf_lookup_pages(
 378         xfs_buf_t               *bp,
 379         uint                    flags)
 380 {
 381         struct address_space    *mapping = bp->pb_target->pbr_mapping;
 382         size_t                  blocksize = bp->pb_target->pbr_bsize;
 383         size_t                  size = bp->pb_count_desired;
 384         size_t                  nbytes, offset;
 385         int                     gfp_mask = pb_to_gfp(flags);
 386         unsigned short          page_count, i;
 387         pgoff_t                 first;
 388         loff_t                  end;
 389         int                     error;
 390
 391         end = bp->pb_file_offset + bp->pb_buffer_length;
 392         page_count = page_buf_btoc(end) - page_buf_btoct(bp->pb_file_offset);
 393
 394         error = _pagebuf_get_pages(bp, page_count, flags);
 395         if (unlikely(error))
 396                 return error;
 397         bp->pb_flags |= _PBF_PAGE_CACHE;
 398
 399         offset = bp->pb_offset;
 400         first = bp->pb_file_offset >> PAGE_CACHE_SHIFT;
 401
 402         for (i = 0; i < bp->pb_page_count; i++) {
 403                 struct page     *page;
 404                 uint            retries = 0;
 405
 406               retry:
 407                 page = find_or_create_page(mapping, first + i, gfp_mask);
 408                 if (unlikely(page == NULL)) {
 409                         if (flags & PBF_READ_AHEAD) {
 410                                 bp->pb_page_count = i;
 411                                 for (i = 0; i < bp->pb_page_count; i++)
 412                                         unlock_page(bp->pb_pages[i]);
 413                                 return -ENOMEM;
 414                         }
 415
 416                         /*
 417                          * This could deadlock.
 418                          *
 419                          * But until all the XFS lowlevel code is revamped to
 420                          * handle buffer allocation failures we can't do much.
 421                          */
 422                         if (!(++retries % 100))
 423                                 printk(KERN_ERR
 424                                         "XFS: possible memory allocation "
 425                                         "deadlock in %s (mode:0x%x)\n",
 426                                         __FUNCTION__, gfp_mask);
 427
 428                         XFS_STATS_INC(pb_page_retries);
 429                         xfsbufd_wakeup(0, gfp_mask);
 430                         blk_congestion_wait(WRITE, HZ/50);
 431                         goto retry;
 432                 }
 433
 434                 XFS_STATS_INC(pb_page_found);
 435
 436                 nbytes = min_t(size_t, size, PAGE_CACHE_SIZE - offset);
 437                 size -= nbytes;
 438
 439                 if (!PageUptodate(page)) {
 440                         page_count--;
 441                         if (blocksize >= PAGE_CACHE_SIZE) {
 442                                 if (flags & PBF_READ)
 443                                         bp->pb_locked = 1;
 444                         } else if (!PagePrivate(page)) {
 445                                 if (test_page_region(page, offset, nbytes))
 446                                         page_count++;
 447                         }
 448                 }
 449
 450                 bp->pb_pages[i] = page;
 451                 offset = 0;
 452         }
 453
 454         if (!bp->pb_locked) {
 455                 for (i = 0; i < bp->pb_page_count; i++)
 456                         unlock_page(bp->pb_pages[i]);
 457         }
 458
 459         if (page_count) {
 460                 /* if we have any uptodate pages, mark that in the buffer */
 461                 bp->pb_flags &= ~PBF_NONE;
 462
 463                 /* if some pages aren't uptodate, mark that in the buffer */
 464                 if (page_count != bp->pb_page_count)
 465                         bp->pb_flags |= PBF_PARTIAL;
 466         }
 467
 468         PB_TRACE(bp, "lookup_pages", (long)page_count);
 469         return error;
 470 }
 471
 472 /*
 473  *      Map buffer into kernel address-space if nessecary.
 474  */
 475 STATIC int
 476 _pagebuf_map_pages(
 477         xfs_buf_t               *bp,
 478         uint                    flags)
 479 {
 480         /* A single page buffer is always mappable */
 481         if (bp->pb_page_count == 1) {
 482                 bp->pb_addr = page_address(bp->pb_pages[0]) + bp->pb_offset;
 483                 bp->pb_flags |= PBF_MAPPED;
 484         } else if (flags & PBF_MAPPED) {
 485                 if (as_list_len > 64)
 486                         purge_addresses();
 487                 bp->pb_addr = vmap(bp->pb_pages, bp->pb_page_count,
 488                                 VM_MAP, PAGE_KERNEL);
 489                 if (unlikely(bp->pb_addr == NULL))
 490                         return -ENOMEM;
 491                 bp->pb_addr += bp->pb_offset;
 492                 bp->pb_flags |= PBF_MAPPED;
 493         }
 494
 495         return 0;
 496 }
 497
 498 /*
 499  *      Finding and Reading Buffers
 500  */
 501
 502 /*
 503  *      _pagebuf_find
 504  *
 505  *      Looks up, and creates if absent, a lockable buffer for
 506  *      a given range of an inode.  The buffer is returned
 507  *      locked.  If other overlapping buffers exist, they are
 508  *      released before the new buffer is created and locked,
 509  *      which may imply that this call will block until those buffers
 510  *      are unlocked.  No I/O is implied by this call.
 511  */
 512 xfs_buf_t *
 513 _pagebuf_find(
 514         xfs_buftarg_t           *btp,   /* block device target          */
 515         loff_t                  ioff,   /* starting offset of range     */
 516         size_t                  isize,  /* length of range              */
 517         page_buf_flags_t        flags,  /* PBF_TRYLOCK                  */
 518         xfs_buf_t               *new_pb)/* newly allocated buffer       */
 519 {
 520         loff_t                  range_base;
 521         size_t                  range_length;
 522         xfs_bufhash_t           *hash;
 523         xfs_buf_t               *pb, *n;
 524
 525         range_base = (ioff << BBSHIFT);
 526         range_length = (isize << BBSHIFT);
 527
 528         /* Check for IOs smaller than the sector size / not sector aligned */
 529         ASSERT(!(range_length < (1 << btp->pbr_sshift)));
 530         ASSERT(!(range_base & (loff_t)btp->pbr_smask));
 531
 532         hash = &btp->bt_hash[hash_long((unsigned long)ioff, btp->bt_hashshift)];
 533
 534         spin_lock(&hash->bh_lock);
 535
 536         list_for_each_entry_safe(pb, n, &hash->bh_list, pb_hash_list) {
 537                 ASSERT(btp == pb->pb_target);
 538                 if (pb->pb_file_offset == range_base &&
 539                     pb->pb_buffer_length == range_length) {
 540                         /*
 541                          * If we look at something bring it to the
 542                          * front of the list for next time.
 543                          */
 544                         atomic_inc(&pb->pb_hold);
 545                         list_move(&pb->pb_hash_list, &hash->bh_list);
 546                         goto found;
 547                 }
 548         }
 549
 550         /* No match found */
 551         if (new_pb) {
 552                 _pagebuf_initialize(new_pb, btp, range_base,
 553                                 range_length, flags);
 554                 new_pb->pb_hash = hash;
 555                 list_add(&new_pb->pb_hash_list, &hash->bh_list);
 556         } else {
 557                 XFS_STATS_INC(pb_miss_locked);
 558         }
 559
 560         spin_unlock(&hash->bh_lock);
 561         return new_pb;
 562
 563 found:
 564         spin_unlock(&hash->bh_lock);
 565
 566         /* Attempt to get the semaphore without sleeping,
 567          * if this does not work then we need to drop the
 568          * spinlock and do a hard attempt on the semaphore.
 569          */
 570         if (down_trylock(&pb->pb_sema)) {
 571                 if (!(flags & PBF_TRYLOCK)) {
 572                         /* wait for buffer ownership */
 573                         PB_TRACE(pb, "get_lock", 0);
 574                         pagebuf_lock(pb);
 575                         XFS_STATS_INC(pb_get_locked_waited);
 576                 } else {
 577                         /* We asked for a trylock and failed, no need
 578                          * to look at file offset and length here, we
 579                          * know that this pagebuf at least overlaps our
 580                          * pagebuf and is locked, therefore our buffer
 581                          * either does not exist, or is this buffer
 582                          */
 583
 584                         pagebuf_rele(pb);
 585                         XFS_STATS_INC(pb_busy_locked);
 586                         return (NULL);
 587                 }
 588         } else {
 589                 /* trylock worked */
 590                 PB_SET_OWNER(pb);
 591         }
 592
 593         if (pb->pb_flags & PBF_STALE)
 594                 pb->pb_flags &= PBF_MAPPED;
 595         PB_TRACE(pb, "got_lock", 0);
 596         XFS_STATS_INC(pb_get_locked);
 597         return (pb);
 598 }
 599
 600 /*
 601  *      xfs_buf_get_flags assembles a buffer covering the specified range.
 602  *
 603  *      Storage in memory for all portions of the buffer will be allocated,
 604  *      although backing storage may not be.
 605  */
 606 xfs_buf_t *
 607 xfs_buf_get_flags(                      /* allocate a buffer            */
 608         xfs_buftarg_t           *target,/* target for buffer            */
 609         loff_t                  ioff,   /* starting offset of range     */
 610         size_t                  isize,  /* length of range              */
 611         page_buf_flags_t        flags)  /* PBF_TRYLOCK                  */
 612 {
 613         xfs_buf_t               *pb, *new_pb;
 614         int                     error = 0, i;
 615
 616         new_pb = pagebuf_allocate(flags);
 617         if (unlikely(!new_pb))
 618                 return NULL;
 619
 620         pb = _pagebuf_find(target, ioff, isize, flags, new_pb);
 621         if (pb == new_pb) {
 622                 error = _pagebuf_lookup_pages(pb, flags);
 623                 if (error)
 624                         goto no_buffer;
 625         } else {
 626                 pagebuf_deallocate(new_pb);
 627                 if (unlikely(pb == NULL))
 628                         return NULL;
 629         }
 630
 631         for (i = 0; i < pb->pb_page_count; i++)
 632                 mark_page_accessed(pb->pb_pages[i]);
 633
 634         if (!(pb->pb_flags & PBF_MAPPED)) {
 635                 error = _pagebuf_map_pages(pb, flags);
 636                 if (unlikely(error)) {
 637                         printk(KERN_WARNING "%s: failed to map pages\n",
 638                                         __FUNCTION__);
 639                         goto no_buffer;
 640                 }
 641         }
 642
 643         XFS_STATS_INC(pb_get);
 644
 645         /*
 646          * Always fill in the block number now, the mapped cases can do
 647          * their own overlay of this later.
 648          */
 649         pb->pb_bn = ioff;
 650         pb->pb_count_desired = pb->pb_buffer_length;
 651
 652         PB_TRACE(pb, "get", (unsigned long)flags);
 653         return pb;
 654
 655  no_buffer:
 656         if (flags & (PBF_LOCK | PBF_TRYLOCK))
 657                 pagebuf_unlock(pb);
 658         pagebuf_rele(pb);
 659         return NULL;
 660 }
 661
 662 xfs_buf_t *
 663 xfs_buf_read_flags(
 664         xfs_buftarg_t           *target,
 665         loff_t                  ioff,
 666         size_t                  isize,
 667         page_buf_flags_t        flags)
 668 {
 669         xfs_buf_t               *pb;
 670
 671         flags |= PBF_READ;
 672
 673         pb = xfs_buf_get_flags(target, ioff, isize, flags);
 674         if (pb) {
 675                 if (PBF_NOT_DONE(pb)) {
 676                         PB_TRACE(pb, "read", (unsigned long)flags);
 677                         XFS_STATS_INC(pb_get_read);
 678                         pagebuf_iostart(pb, flags);
 679                 } else if (flags & PBF_ASYNC) {
 680                         PB_TRACE(pb, "read_async", (unsigned long)flags);
 681                         /*
 682                          * Read ahead call which is already satisfied,
 683                          * drop the buffer
 684                          */
 685                         goto no_buffer;
 686                 } else {
 687                         PB_TRACE(pb, "read_done", (unsigned long)flags);
 688                         /* We do not want read in the flags */
 689                         pb->pb_flags &= ~PBF_READ;
 690                 }
 691         }
 692
 693         return pb;
 694
 695  no_buffer:
 696         if (flags & (PBF_LOCK | PBF_TRYLOCK))
 697                 pagebuf_unlock(pb);
 698         pagebuf_rele(pb);
 699         return NULL;
 700 }
 701
 702 /*
 703  * Create a skeletal pagebuf (no pages associated with it).
 704  */
 705 xfs_buf_t *
 706 pagebuf_lookup(
 707         xfs_buftarg_t           *target,
 708         loff_t                  ioff,
 709         size_t                  isize,
 710         page_buf_flags_t        flags)
 711 {
 712         xfs_buf_t               *pb;
 713
 714         pb = pagebuf_allocate(flags);
 715         if (pb) {
 716                 _pagebuf_initialize(pb, target, ioff, isize, flags);
 717         }
 718         return pb;
 719 }
 720
 721 /*
 722  * If we are not low on memory then do the readahead in a deadlock
 723  * safe manner.
 724  */
 725 void
 726 pagebuf_readahead(
 727         xfs_buftarg_t           *target,
 728         loff_t                  ioff,
 729         size_t                  isize,
 730         page_buf_flags_t        flags)
 731 {
 732         struct backing_dev_info *bdi;
 733
 734         bdi = target->pbr_mapping->backing_dev_info;
 735         if (bdi_read_congested(bdi))
 736                 return;
 737
 738         flags |= (PBF_TRYLOCK|PBF_ASYNC|PBF_READ_AHEAD);
 739         xfs_buf_read_flags(target, ioff, isize, flags);
 740 }
 741
 742 xfs_buf_t *
 743 pagebuf_get_empty(
 744         size_t                  len,
 745         xfs_buftarg_t           *target)
 746 {
 747         xfs_buf_t               *pb;
 748
 749         pb = pagebuf_allocate(0);
 750         if (pb)
 751                 _pagebuf_initialize(pb, target, 0, len, 0);
 752         return pb;
 753 }
 754
 755 static inline struct page *
 756 mem_to_page(
 757         void                    *addr)
 758 {
 759         if (((unsigned long)addr < VMALLOC_START) ||
 760             ((unsigned long)addr >= VMALLOC_END)) {
 761                 return virt_to_page(addr);
 762         } else {
 763                 return vmalloc_to_page(addr);
 764         }
 765 }
 766
 767 int
 768 pagebuf_associate_memory(
 769         xfs_buf_t               *pb,
 770         void                    *mem,
 771         size_t                  len)
 772 {
 773         int                     rval;
 774         int                     i = 0;
 775         size_t                  ptr;
 776         size_t                  end, end_cur;
 777         off_t                   offset;
 778         int                     page_count;
 779
 780         page_count = PAGE_CACHE_ALIGN(len) >> PAGE_CACHE_SHIFT;
 781         offset = (off_t) mem - ((off_t)mem & PAGE_CACHE_MASK);
 782         if (offset && (len > PAGE_CACHE_SIZE))
 783                 page_count++;
 784
 785         /* Free any previous set of page pointers */
 786         if (pb->pb_pages)
 787                 _pagebuf_free_pages(pb);
 788
 789         pb->pb_pages = NULL;
 790         pb->pb_addr = mem;
 791
 792         rval = _pagebuf_get_pages(pb, page_count, 0);
 793         if (rval)
 794                 return rval;
 795
 796         pb->pb_offset = offset;
 797         ptr = (size_t) mem & PAGE_CACHE_MASK;
 798         end = PAGE_CACHE_ALIGN((size_t) mem + len);
 799         end_cur = end;
 800         /* set up first page */
 801         pb->pb_pages[0] = mem_to_page(mem);
 802
 803         ptr += PAGE_CACHE_SIZE;
 804         pb->pb_page_count = ++i;
 805         while (ptr < end) {
 806                 pb->pb_pages[i] = mem_to_page((void *)ptr);
 807                 pb->pb_page_count = ++i;
 808                 ptr += PAGE_CACHE_SIZE;
 809         }
 810         pb->pb_locked = 0;
 811
 812         pb->pb_count_desired = pb->pb_buffer_length = len;
 813         pb->pb_flags |= PBF_MAPPED;
 814
 815         return 0;
 816 }
 817
 818 xfs_buf_t *
 819 pagebuf_get_no_daddr(
 820         size_t                  len,
 821         xfs_buftarg_t           *target)
 822 {
 823         size_t                  malloc_len = len;
 824         xfs_buf_t               *bp;
 825         void                    *data;
 826         int                     error;
 827
 828         bp = pagebuf_allocate(0);
 829         if (unlikely(bp == NULL))
 830                 goto fail;
 831         _pagebuf_initialize(bp, target, 0, len, PBF_FORCEIO);
 832
 833  try_again:
 834         data = kmem_alloc(malloc_len, KM_SLEEP | KM_MAYFAIL);
 835         if (unlikely(data == NULL))
 836                 goto fail_free_buf;
 837
 838         /* check whether alignment matches.. */
 839         if ((__psunsigned_t)data !=
 840             ((__psunsigned_t)data & ~target->pbr_smask)) {
 841                 /* .. else double the size and try again */
 842                 kmem_free(data, malloc_len);
 843                 malloc_len <<= 1;
 844                 goto try_again;
 845         }
 846
 847         error = pagebuf_associate_memory(bp, data, len);
 848         if (error)
 849                 goto fail_free_mem;
 850         bp->pb_flags |= _PBF_KMEM_ALLOC;
 851
 852         pagebuf_unlock(bp);
 853
 854         PB_TRACE(bp, "no_daddr", data);
 855         return bp;
 856  fail_free_mem:
 857         kmem_free(data, malloc_len);
 858  fail_free_buf:
 859         pagebuf_free(bp);
 860  fail:
 861         return NULL;
 862 }
 863
 864 /*
 865  *      pagebuf_hold
 866  *
 867  *      Increment reference count on buffer, to hold the buffer concurrently
 868  *      with another thread which may release (free) the buffer asynchronously.
 869  *
 870  *      Must hold the buffer already to call this function.
 871  */
 872 void
 873 pagebuf_hold(
 874         xfs_buf_t               *pb)
 875 {
 876         atomic_inc(&pb->pb_hold);
 877         PB_TRACE(pb, "hold", 0);
 878 }
 879
 880 /*
 881  *      pagebuf_rele
 882  *
 883  *      pagebuf_rele releases a hold on the specified buffer.  If the
 884  *      the hold count is 1, pagebuf_rele calls pagebuf_free.
 885  */
 886 void
 887 pagebuf_rele(
 888         xfs_buf_t               *pb)
 889 {
 890         xfs_bufhash_t           *hash = pb->pb_hash;
 891
 892         PB_TRACE(pb, "rele", pb->pb_relse);
 893
 894         /*
 895          * pagebuf_lookup buffers are not hashed, not delayed write,
 896          * and don't have their own release routines.  Special case.
 897          */
 898         if (unlikely(!hash)) {
 899                 ASSERT(!pb->pb_relse);
 900                 if (atomic_dec_and_test(&pb->pb_hold))
 901                         xfs_buf_free(pb);
 902                 return;
 903         }
 904
 905         if (atomic_dec_and_lock(&pb->pb_hold, &hash->bh_lock)) {
 906                 int             do_free = 1;
 907
 908                 if (pb->pb_relse) {
 909                         atomic_inc(&pb->pb_hold);
 910                         spin_unlock(&hash->bh_lock);
 911                         (*(pb->pb_relse)) (pb);
 912                         spin_lock(&hash->bh_lock);
 913                         do_free = 0;
 914                 }
 915
 916                 if (pb->pb_flags & PBF_DELWRI) {
 917                         pb->pb_flags |= PBF_ASYNC;
 918                         atomic_inc(&pb->pb_hold);
 919                         pagebuf_delwri_queue(pb, 0);
 920                         do_free = 0;
 921                 } else if (pb->pb_flags & PBF_FS_MANAGED) {
 922                         do_free = 0;
 923                 }
 924
 925                 if (do_free) {
 926                         list_del_init(&pb->pb_hash_list);
 927                         spin_unlock(&hash->bh_lock);
 928                         pagebuf_free(pb);
 929                 } else {
 930                         spin_unlock(&hash->bh_lock);
 931                 }
 932         }
 933 }
 934
 935
 936 /*
 937  *      Mutual exclusion on buffers.  Locking model:
 938  *
 939  *      Buffers associated with inodes for which buffer locking
 940  *      is not enabled are not protected by semaphores, and are
 941  *      assumed to be exclusively owned by the caller.  There is a
 942  *      spinlock in the buffer, used by the caller when concurrent
 943  *      access is possible.
 944  */
 945
 946 /*
 947  *      pagebuf_cond_lock
 948  *
 949  *      pagebuf_cond_lock locks a buffer object, if it is not already locked.
 950  *      Note that this in no way
 951  *      locks the underlying pages, so it is only useful for synchronizing
 952  *      concurrent use of page buffer objects, not for synchronizing independent
 953  *      access to the underlying pages.
 954  */
 955 int
 956 pagebuf_cond_lock(                      /* lock buffer, if not locked   */
 957                                         /* returns -EBUSY if locked)    */
 958         xfs_buf_t               *pb)
 959 {
 960         int                     locked;
 961
 962         locked = down_trylock(&pb->pb_sema) == 0;
 963         if (locked) {
 964                 PB_SET_OWNER(pb);
 965         }
 966         PB_TRACE(pb, "cond_lock", (long)locked);
 967         return(locked ? 0 : -EBUSY);
 968 }
 969
 970 #if defined(DEBUG) || defined(XFS_BLI_TRACE)
 971 /*
 972  *      pagebuf_lock_value
 973  *
 974  *      Return lock value for a pagebuf
 975  */
 976 int
 977 pagebuf_lock_value(
 978         xfs_buf_t               *pb)
 979 {
 980         return(atomic_read(&pb->pb_sema.count));
 981 }
 982 #endif
 983
 984 /*
 985  *      pagebuf_lock
 986  *
 987  *      pagebuf_lock locks a buffer object.  Note that this in no way
 988  *      locks the underlying pages, so it is only useful for synchronizing
 989  *      concurrent use of page buffer objects, not for synchronizing independent
 990  *      access to the underlying pages.
 991  */
 992 int
 993 pagebuf_lock(
 994         xfs_buf_t               *pb)
 995 {
 996         PB_TRACE(pb, "lock", 0);
 997         if (atomic_read(&pb->pb_io_remaining))
 998                 blk_run_address_space(pb->pb_target->pbr_mapping);
 999         down(&pb->pb_sema);
1000         PB_SET_OWNER(pb);
1001         PB_TRACE(pb, "locked", 0);
1002         return 0;
1003 }
1004
1005 /*
1006  *      pagebuf_unlock
1007  *
1008  *      pagebuf_unlock releases the lock on the buffer object created by
1009  *      pagebuf_lock or pagebuf_cond_lock (not any
1010  *      pinning of underlying pages created by pagebuf_pin).
1011  */
1012 void
1013 pagebuf_unlock(                         /* unlock buffer                */
1014         xfs_buf_t               *pb)    /* buffer to unlock             */
1015 {
1016         PB_CLEAR_OWNER(pb);
1017         up(&pb->pb_sema);
1018         PB_TRACE(pb, "unlock", 0);
1019 }
1020
1021
1022 /*
1023  *      Pinning Buffer Storage in Memory
1024  */
1025
1026 /*
1027  *      pagebuf_pin
1028  *
1029  *      pagebuf_pin locks all of the memory represented by a buffer in
1030  *      memory.  Multiple calls to pagebuf_pin and pagebuf_unpin, for
1031  *      the same or different buffers affecting a given page, will
1032  *      properly count the number of outstanding "pin" requests.  The
1033  *      buffer may be released after the pagebuf_pin and a different
1034  *      buffer used when calling pagebuf_unpin, if desired.
1035  *      pagebuf_pin should be used by the file system when it wants be
1036  *      assured that no attempt will be made to force the affected
1037  *      memory to disk.  It does not assure that a given logical page
1038  *      will not be moved to a different physical page.
1039  */
1040 void
1041 pagebuf_pin(
1042         xfs_buf_t               *pb)
1043 {
1044         atomic_inc(&pb->pb_pin_count);
1045         PB_TRACE(pb, "pin", (long)pb->pb_pin_count.counter);
1046 }
1047
1048 /*
1049  *      pagebuf_unpin
1050  *
1051  *      pagebuf_unpin reverses the locking of memory performed by
1052  *      pagebuf_pin.  Note that both functions affected the logical
1053  *      pages associated with the buffer, not the buffer itself.
1054  */
1055 void
1056 pagebuf_unpin(
1057         xfs_buf_t               *pb)
1058 {
1059         if (atomic_dec_and_test(&pb->pb_pin_count)) {
1060                 wake_up_all(&pb->pb_waiters);
1061         }
1062         PB_TRACE(pb, "unpin", (long)pb->pb_pin_count.counter);
1063 }
1064
1065 int
1066 pagebuf_ispin(
1067         xfs_buf_t               *pb)
1068 {
1069         return atomic_read(&pb->pb_pin_count);
1070 }
1071
1072 /*
1073  *      pagebuf_wait_unpin
1074  *
1075  *      pagebuf_wait_unpin waits until all of the memory associated
1076  *      with the buffer is not longer locked in memory.  It returns
1077  *      immediately if none of the affected pages are locked.
1078  */
1079 static inline void
1080 _pagebuf_wait_unpin(
1081         xfs_buf_t               *pb)
1082 {
1083         DECLARE_WAITQUEUE       (wait, current);
1084
1085         if (atomic_read(&pb->pb_pin_count) == 0)
1086                 return;
1087
1088         add_wait_queue(&pb->pb_waiters, &wait);
1089         for (;;) {
1090                 set_current_state(TASK_UNINTERRUPTIBLE);
1091                 if (atomic_read(&pb->pb_pin_count) == 0)
1092                         break;
1093                 if (atomic_read(&pb->pb_io_remaining))
1094                         blk_run_address_space(pb->pb_target->pbr_mapping);
1095                 schedule();
1096         }
1097         remove_wait_queue(&pb->pb_waiters, &wait);
1098         set_current_state(TASK_RUNNING);
1099 }
1100
1101 /*
1102  *      Buffer Utility Routines
1103  */
1104
1105 /*
1106  *      pagebuf_iodone
1107  *
1108  *      pagebuf_iodone marks a buffer for which I/O is in progress
1109  *      done with respect to that I/O.  The pb_iodone routine, if
1110  *      present, will be called as a side-effect.
1111  */
1112 STATIC void
1113 pagebuf_iodone_work(
1114         void                    *v)
1115 {
1116         xfs_buf_t               *bp = (xfs_buf_t *)v;
1117
1118         if (bp->pb_iodone)
1119                 (*(bp->pb_iodone))(bp);
1120         else if (bp->pb_flags & PBF_ASYNC)
1121                 xfs_buf_relse(bp);
1122 }
1123
1124 void
1125 pagebuf_iodone(
1126         xfs_buf_t               *pb,
1127         int                     dataio,
1128         int                     schedule)
1129 {
1130         pb->pb_flags &= ~(PBF_READ | PBF_WRITE);
1131         if (pb->pb_error == 0) {
1132                 pb->pb_flags &= ~(PBF_PARTIAL | PBF_NONE);
1133         }
1134
1135         PB_TRACE(pb, "iodone", pb->pb_iodone);
1136
1137         if ((pb->pb_iodone) || (pb->pb_flags & PBF_ASYNC)) {
1138                 if (schedule) {
1139                         INIT_WORK(&pb->pb_iodone_work, pagebuf_iodone_work, pb);
1140                         queue_work(dataio ? xfsdatad_workqueue :
1141                                 xfslogd_workqueue, &pb->pb_iodone_work);
1142                 } else {
1143                         pagebuf_iodone_work(pb);
1144                 }
1145         } else {
1146                 up(&pb->pb_iodonesema);
1147         }
1148 }
1149
1150 /*
1151  *      pagebuf_ioerror
1152  *
1153  *      pagebuf_ioerror sets the error code for a buffer.
1154  */
1155 void
1156 pagebuf_ioerror(                        /* mark/clear buffer error flag */
1157         xfs_buf_t               *pb,    /* buffer to mark               */
1158         int                     error)  /* error to store (0 if none)   */
1159 {
1160         ASSERT(error >= 0 && error <= 0xffff);
1161         pb->pb_error = (unsigned short)error;
1162         PB_TRACE(pb, "ioerror", (unsigned long)error);
1163 }
1164
1165 /*
1166  *      pagebuf_iostart
1167  *
1168  *      pagebuf_iostart initiates I/O on a buffer, based on the flags supplied.
1169  *      If necessary, it will arrange for any disk space allocation required,
1170  *      and it will break up the request if the block mappings require it.
1171  *      The pb_iodone routine in the buffer supplied will only be called
1172  *      when all of the subsidiary I/O requests, if any, have been completed.
1173  *      pagebuf_iostart calls the pagebuf_ioinitiate routine or
1174  *      pagebuf_iorequest, if the former routine is not defined, to start
1175  *      the I/O on a given low-level request.
1176  */
1177 int
1178 pagebuf_iostart(                        /* start I/O on a buffer          */
1179         xfs_buf_t               *pb,    /* buffer to start                */
1180         page_buf_flags_t        flags)  /* PBF_LOCK, PBF_ASYNC, PBF_READ, */
1181                                         /* PBF_WRITE, PBF_DELWRI,         */
1182                                         /* PBF_DONT_BLOCK                 */
1183 {
1184         int                     status = 0;
1185
1186         PB_TRACE(pb, "iostart", (unsigned long)flags);
1187
1188         if (flags & PBF_DELWRI) {
1189                 pb->pb_flags &= ~(PBF_READ | PBF_WRITE | PBF_ASYNC);
1190                 pb->pb_flags |= flags & (PBF_DELWRI | PBF_ASYNC);
1191                 pagebuf_delwri_queue(pb, 1);
1192                 return status;
1193         }
1194
1195         pb->pb_flags &= ~(PBF_READ | PBF_WRITE | PBF_ASYNC | PBF_DELWRI | \
1196                         PBF_READ_AHEAD | _PBF_RUN_QUEUES);
1197         pb->pb_flags |= flags & (PBF_READ | PBF_WRITE | PBF_ASYNC | \
1198                         PBF_READ_AHEAD | _PBF_RUN_QUEUES);
1199
1200         BUG_ON(pb->pb_bn == XFS_BUF_DADDR_NULL);
1201
1202         /* For writes allow an alternate strategy routine to precede
1203          * the actual I/O request (which may not be issued at all in
1204          * a shutdown situation, for example).
1205          */
1206         status = (flags & PBF_WRITE) ?
1207                 pagebuf_iostrategy(pb) : pagebuf_iorequest(pb);
1208
1209         /* Wait for I/O if we are not an async request.
1210          * Note: async I/O request completion will release the buffer,
1211          * and that can already be done by this point.  So using the
1212          * buffer pointer from here on, after async I/O, is invalid.
1213          */
1214         if (!status && !(flags & PBF_ASYNC))
1215                 status = pagebuf_iowait(pb);
1216
1217         return status;
1218 }
1219
1220 /*
1221  * Helper routine for pagebuf_iorequest
1222  */
1223
1224 STATIC __inline__ int
1225 _pagebuf_iolocked(
1226         xfs_buf_t               *pb)
1227 {
1228         ASSERT(pb->pb_flags & (PBF_READ|PBF_WRITE));
1229         if (pb->pb_flags & PBF_READ)
1230                 return pb->pb_locked;
1231         return 0;
1232 }
1233
1234 STATIC __inline__ void
1235 _pagebuf_iodone(
1236         xfs_buf_t               *pb,
1237         int                     schedule)
1238 {
1239         if (atomic_dec_and_test(&pb->pb_io_remaining) == 1) {
1240                 pb->pb_locked = 0;
1241                 pagebuf_iodone(pb, (pb->pb_flags & PBF_FS_DATAIOD), schedule);
1242         }
1243 }
1244
1245 STATIC int
1246 bio_end_io_pagebuf(
1247         struct bio              *bio,
1248         unsigned int            bytes_done,
1249         int                     error)
1250 {
1251         xfs_buf_t               *pb = (xfs_buf_t *)bio->bi_private;
1252         unsigned int            i, blocksize = pb->pb_target->pbr_bsize;
1253         struct bio_vec          *bvec = bio->bi_io_vec;
1254
1255         if (bio->bi_size)
1256                 return 1;
1257
1258         if (!test_bit(BIO_UPTODATE, &bio->bi_flags))
1259                 pb->pb_error = EIO;
1260
1261         for (i = 0; i < bio->bi_vcnt; i++, bvec++) {
1262                 struct page     *page = bvec->bv_page;
1263
1264                 if (pb->pb_error) {
1265                         SetPageError(page);
1266                 } else if (blocksize == PAGE_CACHE_SIZE) {
1267                         SetPageUptodate(page);
1268                 } else if (!PagePrivate(page) &&
1269                                 (pb->pb_flags & _PBF_PAGE_CACHE)) {
1270                         set_page_region(page, bvec->bv_offset, bvec->bv_len);
1271                 }
1272
1273                 if (_pagebuf_iolocked(pb)) {
1274                         unlock_page(page);
1275                 }
1276         }
1277
1278         _pagebuf_iodone(pb, 1);
1279         bio_put(bio);
1280         return 0;
1281 }
1282
1283 STATIC void
1284 _pagebuf_ioapply(
1285         xfs_buf_t               *pb)
1286 {
1287         int                     i, rw, map_i, total_nr_pages, nr_pages;
1288         struct bio              *bio;
1289         int                     offset = pb->pb_offset;
1290         int                     size = pb->pb_count_desired;
1291         sector_t                sector = pb->pb_bn;
1292         unsigned int            blocksize = pb->pb_target->pbr_bsize;
1293         int                     locking = _pagebuf_iolocked(pb);
1294
1295         total_nr_pages = pb->pb_page_count;
1296         map_i = 0;
1297
1298         if (pb->pb_flags & _PBF_RUN_QUEUES) {
1299                 pb->pb_flags &= ~_PBF_RUN_QUEUES;
1300                 rw = (pb->pb_flags & PBF_READ) ? READ_SYNC : WRITE_SYNC;
1301         } else {
1302                 rw = (pb->pb_flags & PBF_READ) ? READ : WRITE;
1303         }
1304
1305         /* Special code path for reading a sub page size pagebuf in --
1306          * we populate up the whole page, and hence the other metadata
1307          * in the same page.  This optimization is only valid when the
1308          * filesystem block size and the page size are equal.
1309          */
1310         if ((pb->pb_buffer_length < PAGE_CACHE_SIZE) &&
1311             (pb->pb_flags & PBF_READ) && locking &&
1312             (blocksize == PAGE_CACHE_SIZE)) {
1313                 bio = bio_alloc(GFP_NOIO, 1);
1314
1315                 bio->bi_bdev = pb->pb_target->pbr_bdev;
1316                 bio->bi_sector = sector - (offset >> BBSHIFT);
1317                 bio->bi_end_io = bio_end_io_pagebuf;
1318                 bio->bi_private = pb;
1319
1320                 bio_add_page(bio, pb->pb_pages[0], PAGE_CACHE_SIZE, 0);
1321                 size = 0;
1322
1323                 atomic_inc(&pb->pb_io_remaining);
1324
1325                 goto submit_io;
1326         }
1327
1328         /* Lock down the pages which we need to for the request */
1329         if (locking && (pb->pb_flags & PBF_WRITE) && (pb->pb_locked == 0)) {
1330                 for (i = 0; size; i++) {
1331                         int             nbytes = PAGE_CACHE_SIZE - offset;
1332                         struct page     *page = pb->pb_pages[i];
1333
1334                         if (nbytes > size)
1335                                 nbytes = size;
1336
1337                         lock_page(page);
1338
1339                         size -= nbytes;
1340                         offset = 0;
1341                 }
1342                 offset = pb->pb_offset;
1343                 size = pb->pb_count_desired;
1344         }
1345
1346 next_chunk:
1347         atomic_inc(&pb->pb_io_remaining);
1348         nr_pages = BIO_MAX_SECTORS >> (PAGE_SHIFT - BBSHIFT);
1349         if (nr_pages > total_nr_pages)
1350                 nr_pages = total_nr_pages;
1351
1352         bio = bio_alloc(GFP_NOIO, nr_pages);
1353         bio->bi_bdev = pb->pb_target->pbr_bdev;
1354         bio->bi_sector = sector;
1355         bio->bi_end_io = bio_end_io_pagebuf;
1356         bio->bi_private = pb;
1357
1358         for (; size && nr_pages; nr_pages--, map_i++) {
1359                 int     nbytes = PAGE_CACHE_SIZE - offset;
1360
1361                 if (nbytes > size)
1362                         nbytes = size;
1363
1364                 if (bio_add_page(bio, pb->pb_pages[map_i],
1365                                         nbytes, offset) < nbytes)
1366                         break;
1367
1368                 offset = 0;
1369                 sector += nbytes >> BBSHIFT;
1370                 size -= nbytes;
1371                 total_nr_pages--;
1372         }
1373
1374 submit_io:
1375         if (likely(bio->bi_size)) {
1376                 submit_bio(rw, bio);
1377                 if (size)
1378                         goto next_chunk;
1379         } else {
1380                 bio_put(bio);
1381                 pagebuf_ioerror(pb, EIO);
1382         }
1383 }
1384
1385 /*
1386  *      pagebuf_iorequest -- the core I/O request routine.
1387  */
1388 int
1389 pagebuf_iorequest(                      /* start real I/O               */
1390         xfs_buf_t               *pb)    /* buffer to convey to device   */
1391 {
1392         PB_TRACE(pb, "iorequest", 0);
1393
1394         if (pb->pb_flags & PBF_DELWRI) {
1395                 pagebuf_delwri_queue(pb, 1);
1396                 return 0;
1397         }
1398
1399         if (pb->pb_flags & PBF_WRITE) {
1400                 _pagebuf_wait_unpin(pb);
1401         }
1402
1403         pagebuf_hold(pb);
1404
1405         /* Set the count to 1 initially, this will stop an I/O
1406          * completion callout which happens before we have started
1407          * all the I/O from calling pagebuf_iodone too early.
1408          */
1409         atomic_set(&pb->pb_io_remaining, 1);
1410         _pagebuf_ioapply(pb);
1411         _pagebuf_iodone(pb, 0);
1412
1413         pagebuf_rele(pb);
1414         return 0;
1415 }
1416
1417 /*
1418  *      pagebuf_iowait
1419  *
1420  *      pagebuf_iowait waits for I/O to complete on the buffer supplied.
1421  *      It returns immediately if no I/O is pending.  In any case, it returns
1422  *      the error code, if any, or 0 if there is no error.
1423  */
1424 int
1425 pagebuf_iowait(
1426         xfs_buf_t               *pb)
1427 {
1428         PB_TRACE(pb, "iowait", 0);
1429         if (atomic_read(&pb->pb_io_remaining))
1430                 blk_run_address_space(pb->pb_target->pbr_mapping);
1431         down(&pb->pb_iodonesema);
1432         PB_TRACE(pb, "iowaited", (long)pb->pb_error);
1433         return pb->pb_error;
1434 }
1435
1436 caddr_t
1437 pagebuf_offset(
1438         xfs_buf_t               *pb,
1439         size_t                  offset)
1440 {
1441         struct page             *page;
1442
1443         offset += pb->pb_offset;
1444
1445         page = pb->pb_pages[offset >> PAGE_CACHE_SHIFT];
1446         return (caddr_t) page_address(page) + (offset & (PAGE_CACHE_SIZE - 1));
1447 }
1448
1449 /*
1450  *      pagebuf_iomove
1451  *
1452  *      Move data into or out of a buffer.
1453  */
1454 void
1455 pagebuf_iomove(
1456         xfs_buf_t               *pb,    /* buffer to process            */
1457         size_t                  boff,   /* starting buffer offset       */
1458         size_t                  bsize,  /* length to copy               */
1459         caddr_t                 data,   /* data address                 */
1460         page_buf_rw_t           mode)   /* read/write flag              */
1461 {
1462         size_t                  bend, cpoff, csize;
1463         struct page             *page;
1464
1465         bend = boff + bsize;
1466         while (boff < bend) {
1467                 page = pb->pb_pages[page_buf_btoct(boff + pb->pb_offset)];
1468                 cpoff = page_buf_poff(boff + pb->pb_offset);
1469                 csize = min_t(size_t,
1470                               PAGE_CACHE_SIZE-cpoff, pb->pb_count_desired-boff);
1471
1472                 ASSERT(((csize + cpoff) <= PAGE_CACHE_SIZE));
1473
1474                 switch (mode) {
1475                 case PBRW_ZERO:
1476                         memset(page_address(page) + cpoff, 0, csize);
1477                         break;
1478                 case PBRW_READ:
1479                         memcpy(data, page_address(page) + cpoff, csize);
1480                         break;
1481                 case PBRW_WRITE:
1482                         memcpy(page_address(page) + cpoff, data, csize);
1483                 }
1484
1485                 boff += csize;
1486                 data += csize;
1487         }
1488 }
1489
1490 /*
1491  *      Handling of buftargs.
1492  */
1493
1494 /*
1495  * Wait for any bufs with callbacks that have been submitted but
1496  * have not yet returned... walk the hash list for the target.
1497  */
1498 void
1499 xfs_wait_buftarg(
1500         xfs_buftarg_t   *btp)
1501 {
1502         xfs_buf_t       *bp, *n;
1503         xfs_bufhash_t   *hash;
1504         uint            i;
1505
1506         for (i = 0; i < (1 << btp->bt_hashshift); i++) {
1507                 hash = &btp->bt_hash[i];
1508 again:
1509                 spin_lock(&hash->bh_lock);
1510                 list_for_each_entry_safe(bp, n, &hash->bh_list, pb_hash_list) {
1511                         ASSERT(btp == bp->pb_target);
1512                         if (!(bp->pb_flags & PBF_FS_MANAGED)) {
1513                                 spin_unlock(&hash->bh_lock);
1514                                 delay(100);
1515                                 goto again;
1516                         }
1517                 }
1518                 spin_unlock(&hash->bh_lock);
1519         }
1520 }
1521
1522 /*
1523  * Allocate buffer hash table for a given target.
1524  * For devices containing metadata (i.e. not the log/realtime devices)
1525  * we need to allocate a much larger hash table.
1526  */
1527 STATIC void
1528 xfs_alloc_bufhash(
1529         xfs_buftarg_t           *btp,
1530         int                     external)
1531 {
1532         unsigned int            i;
1533
1534         btp->bt_hashshift = external ? 3 : 8;   /* 8 or 256 buckets */
1535         btp->bt_hashmask = (1 << btp->bt_hashshift) - 1;
1536         btp->bt_hash = kmem_zalloc((1 << btp->bt_hashshift) *
1537                                         sizeof(xfs_bufhash_t), KM_SLEEP);
1538         for (i = 0; i < (1 << btp->bt_hashshift); i++) {
1539                 spin_lock_init(&btp->bt_hash[i].bh_lock);
1540                 INIT_LIST_HEAD(&btp->bt_hash[i].bh_list);
1541         }
1542 }
1543
1544 STATIC void
1545 xfs_free_bufhash(
1546         xfs_buftarg_t           *btp)
1547 {
1548         kmem_free(btp->bt_hash,
1549                         (1 << btp->bt_hashshift) * sizeof(xfs_bufhash_t));
1550         btp->bt_hash = NULL;
1551 }
1552
1553 void
1554 xfs_free_buftarg(
1555         xfs_buftarg_t           *btp,
1556         int                     external)
1557 {
1558         xfs_flush_buftarg(btp, 1);
1559         if (external)
1560                 xfs_blkdev_put(btp->pbr_bdev);
1561         xfs_free_bufhash(btp);
1562         iput(btp->pbr_mapping->host);
1563         kmem_free(btp, sizeof(*btp));
1564 }
1565
1566 STATIC int
1567 xfs_setsize_buftarg_flags(
1568         xfs_buftarg_t           *btp,
1569         unsigned int            blocksize,
1570         unsigned int            sectorsize,
1571         int                     verbose)
1572 {
1573         btp->pbr_bsize = blocksize;
1574         btp->pbr_sshift = ffs(sectorsize) - 1;
1575         btp->pbr_smask = sectorsize - 1;
1576
1577         if (set_blocksize(btp->pbr_bdev, sectorsize)) {
1578                 printk(KERN_WARNING
1579                         "XFS: Cannot set_blocksize to %u on device %s\n",
1580                         sectorsize, XFS_BUFTARG_NAME(btp));
1581                 return EINVAL;
1582         }
1583
1584         if (verbose &&
1585             (PAGE_CACHE_SIZE / BITS_PER_LONG) > sectorsize) {
1586                 printk(KERN_WARNING
1587                         "XFS: %u byte sectors in use on device %s.  "
1588                         "This is suboptimal; %u or greater is ideal.\n",
1589                         sectorsize, XFS_BUFTARG_NAME(btp),
1590                         (unsigned int)PAGE_CACHE_SIZE / BITS_PER_LONG);
1591         }
1592
1593         return 0;
1594 }
1595
1596 /*
1597 * When allocating the initial buffer target we have not yet
1598 * read in the superblock, so don't know what sized sectors
1599 * are being used is at this early stage.  Play safe.
1600 */
1601 STATIC int
1602 xfs_setsize_buftarg_early(
1603         xfs_buftarg_t           *btp,
1604         struct block_device     *bdev)
1605 {
1606         return xfs_setsize_buftarg_flags(btp,
1607                         PAGE_CACHE_SIZE, bdev_hardsect_size(bdev), 0);
1608 }
1609
1610 int
1611 xfs_setsize_buftarg(
1612         xfs_buftarg_t           *btp,
1613         unsigned int            blocksize,
1614         unsigned int            sectorsize)
1615 {
1616         return xfs_setsize_buftarg_flags(btp, blocksize, sectorsize, 1);
1617 }
1618
1619 STATIC int
1620 xfs_mapping_buftarg(
1621         xfs_buftarg_t           *btp,
1622         struct block_device     *bdev)
1623 {
1624         struct backing_dev_info *bdi;
1625         struct inode            *inode;
1626         struct address_space    *mapping;
1627         static struct address_space_operations mapping_aops = {
1628                 .sync_page = block_sync_page,
1629         };
1630
1631         inode = new_inode(bdev->bd_inode->i_sb);
1632         if (!inode) {
1633                 printk(KERN_WARNING
1634                         "XFS: Cannot allocate mapping inode for device %s\n",
1635                         XFS_BUFTARG_NAME(btp));
1636                 return ENOMEM;
1637         }
1638         inode->i_mode = S_IFBLK;
1639         inode->i_bdev = bdev;
1640         inode->i_rdev = bdev->bd_dev;
1641         bdi = blk_get_backing_dev_info(bdev);
1642         if (!bdi)
1643                 bdi = &default_backing_dev_info;
1644         mapping = &inode->i_data;
1645         mapping->a_ops = &mapping_aops;
1646         mapping->backing_dev_info = bdi;
1647         mapping_set_gfp_mask(mapping, GFP_NOFS);
1648         btp->pbr_mapping = mapping;
1649         return 0;
1650 }
1651
1652 xfs_buftarg_t *
1653 xfs_alloc_buftarg(
1654         struct block_device     *bdev,
1655         int                     external)
1656 {
1657         xfs_buftarg_t           *btp;
1658
1659         btp = kmem_zalloc(sizeof(*btp), KM_SLEEP);
1660
1661         btp->pbr_dev =  bdev->bd_dev;
1662         btp->pbr_bdev = bdev;
1663         if (xfs_setsize_buftarg_early(btp, bdev))
1664                 goto error;
1665         if (xfs_mapping_buftarg(btp, bdev))
1666                 goto error;
1667         xfs_alloc_bufhash(btp, external);
1668         return btp;
1669
1670 error:
1671         kmem_free(btp, sizeof(*btp));
1672         return NULL;
1673 }
1674
1675
1676 /*
1677  * Pagebuf delayed write buffer handling
1678  */
1679
1680 STATIC LIST_HEAD(pbd_delwrite_queue);
1681 STATIC DEFINE_SPINLOCK(pbd_delwrite_lock);
1682
1683 STATIC void
1684 pagebuf_delwri_queue(
1685         xfs_buf_t               *pb,
1686         int                     unlock)
1687 {
1688         PB_TRACE(pb, "delwri_q", (long)unlock);
1689         ASSERT(pb->pb_flags & PBF_DELWRI);
1690
1691         spin_lock(&pbd_delwrite_lock);
1692         /* If already in the queue, dequeue and place at tail */
1693         if (!list_empty(&pb->pb_list)) {
1694                 if (unlock) {
1695                         atomic_dec(&pb->pb_hold);
1696                 }
1697                 list_del(&pb->pb_list);
1698         }
1699
1700         list_add_tail(&pb->pb_list, &pbd_delwrite_queue);
1701         pb->pb_queuetime = jiffies;
1702         spin_unlock(&pbd_delwrite_lock);
1703
1704         if (unlock)
1705                 pagebuf_unlock(pb);
1706 }
1707
1708 void
1709 pagebuf_delwri_dequeue(
1710         xfs_buf_t               *pb)
1711 {
1712         int                     dequeued = 0;
1713
1714         spin_lock(&pbd_delwrite_lock);
1715         if ((pb->pb_flags & PBF_DELWRI) && !list_empty(&pb->pb_list)) {
1716                 list_del_init(&pb->pb_list);
1717                 dequeued = 1;
1718         }
1719         pb->pb_flags &= ~PBF_DELWRI;
1720         spin_unlock(&pbd_delwrite_lock);
1721
1722         if (dequeued)
1723                 pagebuf_rele(pb);
1724
1725         PB_TRACE(pb, "delwri_dq", (long)dequeued);
1726 }
1727
1728 STATIC void
1729 pagebuf_runall_queues(
1730         struct workqueue_struct *queue)
1731 {
1732         flush_workqueue(queue);
1733 }
1734
1735 /* Defines for pagebuf daemon */
1736 STATIC DECLARE_COMPLETION(xfsbufd_done);
1737 STATIC struct task_struct *xfsbufd_task;
1738 STATIC int xfsbufd_active;
1739 STATIC int xfsbufd_force_flush;
1740 STATIC int xfsbufd_force_sleep;
1741
1742 STATIC int
1743 xfsbufd_wakeup(
1744         int                     priority,
1745         unsigned int            mask)
1746 {
1747         if (xfsbufd_force_sleep)
1748                 return 0;
1749         xfsbufd_force_flush = 1;
1750         barrier();
1751         wake_up_process(xfsbufd_task);
1752         return 0;
1753 }
1754
1755 STATIC int
1756 xfsbufd(
1757         void                    *data)
1758 {
1759         struct list_head        tmp;
1760         unsigned long           age;
1761         xfs_buftarg_t           *target;
1762         xfs_buf_t               *pb, *n;
1763
1764         /*  Set up the thread  */
1765         daemonize("xfsbufd");
1766         current->flags |= PF_MEMALLOC;
1767
1768         xfsbufd_task = current;
1769         xfsbufd_active = 1;
1770         barrier();
1771
1772         INIT_LIST_HEAD(&tmp);
1773         do {
1774                 if (unlikely(freezing(current))) {
1775                         xfsbufd_force_sleep = 1;
1776                         refrigerator();
1777                 } else {
1778                         xfsbufd_force_sleep = 0;
1779                 }
1780
1781                 set_current_state(TASK_INTERRUPTIBLE);
1782                 schedule_timeout((xfs_buf_timer_centisecs * HZ) / 100);
1783
1784                 age = (xfs_buf_age_centisecs * HZ) / 100;
1785                 spin_lock(&pbd_delwrite_lock);
1786                 list_for_each_entry_safe(pb, n, &pbd_delwrite_queue, pb_list) {
1787                         PB_TRACE(pb, "walkq1", (long)pagebuf_ispin(pb));
1788                         ASSERT(pb->pb_flags & PBF_DELWRI);
1789
1790                         if (!pagebuf_ispin(pb) && !pagebuf_cond_lock(pb)) {
1791                                 if (!xfsbufd_force_flush &&
1792                                     time_before(jiffies,
1793                                                 pb->pb_queuetime + age)) {
1794                                         pagebuf_unlock(pb);
1795                                         break;
1796                                 }
1797
1798                                 pb->pb_flags &= ~PBF_DELWRI;
1799                                 pb->pb_flags |= PBF_WRITE;
1800                                 list_move(&pb->pb_list, &tmp);
1801                         }
1802                 }
1803                 spin_unlock(&pbd_delwrite_lock);
1804
1805                 while (!list_empty(&tmp)) {
1806                         pb = list_entry(tmp.next, xfs_buf_t, pb_list);
1807                         target = pb->pb_target;
1808
1809                         list_del_init(&pb->pb_list);
1810                         pagebuf_iostrategy(pb);
1811
1812                         blk_run_address_space(target->pbr_mapping);
1813                 }
1814
1815                 if (as_list_len > 0)
1816                         purge_addresses();
1817
1818                 xfsbufd_force_flush = 0;
1819         } while (xfsbufd_active);
1820
1821         complete_and_exit(&xfsbufd_done, 0);
1822 }
1823
1824 /*
1825  * Go through all incore buffers, and release buffers if they belong to
1826  * the given device. This is used in filesystem error handling to
1827  * preserve the consistency of its metadata.
1828  */
1829 int
1830 xfs_flush_buftarg(
1831         xfs_buftarg_t           *target,
1832         int                     wait)
1833 {
1834         struct list_head        tmp;
1835         xfs_buf_t               *pb, *n;
1836         int                     pincount = 0;
1837
1838         pagebuf_runall_queues(xfsdatad_workqueue);
1839         pagebuf_runall_queues(xfslogd_workqueue);
1840
1841         INIT_LIST_HEAD(&tmp);
1842         spin_lock(&pbd_delwrite_lock);
1843         list_for_each_entry_safe(pb, n, &pbd_delwrite_queue, pb_list) {
1844
1845                 if (pb->pb_target != target)
1846                         continue;
1847
1848                 ASSERT(pb->pb_flags & PBF_DELWRI);
1849                 PB_TRACE(pb, "walkq2", (long)pagebuf_ispin(pb));
1850                 if (pagebuf_ispin(pb)) {
1851                         pincount++;
1852                         continue;
1853                 }
1854
1855                 pb->pb_flags &= ~PBF_DELWRI;
1856                 pb->pb_flags |= PBF_WRITE;
1857                 list_move(&pb->pb_list, &tmp);
1858         }
1859         spin_unlock(&pbd_delwrite_lock);
1860
1861         /*
1862          * Dropped the delayed write list lock, now walk the temporary list
1863          */
1864         list_for_each_entry_safe(pb, n, &tmp, pb_list) {
1865                 if (wait)
1866                         pb->pb_flags &= ~PBF_ASYNC;
1867                 else
1868                         list_del_init(&pb->pb_list);
1869
1870                 pagebuf_lock(pb);
1871                 pagebuf_iostrategy(pb);
1872         }
1873
1874         /*
1875          * Remaining list items must be flushed before returning
1876          */
1877         while (!list_empty(&tmp)) {
1878                 pb = list_entry(tmp.next, xfs_buf_t, pb_list);
1879
1880                 list_del_init(&pb->pb_list);
1881                 xfs_iowait(pb);
1882                 xfs_buf_relse(pb);
1883         }
1884
1885         if (wait)
1886                 blk_run_address_space(target->pbr_mapping);
1887
1888         return pincount;
1889 }
1890
1891 STATIC int
1892 xfs_buf_daemons_start(void)
1893 {
1894         int             error = -ENOMEM;
1895
1896         xfslogd_workqueue = create_workqueue("xfslogd");
1897         if (!xfslogd_workqueue)
1898                 goto out;
1899
1900         xfsdatad_workqueue = create_workqueue("xfsdatad");
1901         if (!xfsdatad_workqueue)
1902                 goto out_destroy_xfslogd_workqueue;
1903
1904         error = kernel_thread(xfsbufd, NULL, CLONE_FS|CLONE_FILES);
1905         if (error < 0)
1906                 goto out_destroy_xfsdatad_workqueue;
1907         return 0;
1908
1909  out_destroy_xfsdatad_workqueue:
1910         destroy_workqueue(xfsdatad_workqueue);
1911  out_destroy_xfslogd_workqueue:
1912         destroy_workqueue(xfslogd_workqueue);
1913  out:
1914         return error;
1915 }
1916
1917 /*
1918  * Note: do not mark as __exit, it is called from pagebuf_terminate.
1919  */
1920 STATIC void
1921 xfs_buf_daemons_stop(void)
1922 {
1923         xfsbufd_active = 0;
1924         barrier();
1925         wait_for_completion(&xfsbufd_done);
1926
1927         destroy_workqueue(xfslogd_workqueue);
1928         destroy_workqueue(xfsdatad_workqueue);
1929 }
1930
1931 /*
1932  *      Initialization and Termination
1933  */
1934
1935 int __init
1936 pagebuf_init(void)
1937 {
1938         int             error = -ENOMEM;
1939
1940         pagebuf_zone = kmem_zone_init(sizeof(xfs_buf_t), "xfs_buf");
1941         if (!pagebuf_zone)
1942                 goto out;
1943
1944 #ifdef PAGEBUF_TRACE
1945         pagebuf_trace_buf = ktrace_alloc(PAGEBUF_TRACE_SIZE, KM_SLEEP);
1946 #endif
1947
1948         error = xfs_buf_daemons_start();
1949         if (error)
1950                 goto out_free_buf_zone;
1951
1952         pagebuf_shake = kmem_shake_register(xfsbufd_wakeup);
1953         if (!pagebuf_shake) {
1954                 error = -ENOMEM;
1955                 goto out_stop_daemons;
1956         }
1957
1958         return 0;
1959
1960  out_stop_daemons:
1961         xfs_buf_daemons_stop();
1962  out_free_buf_zone:
1963 #ifdef PAGEBUF_TRACE
1964         ktrace_free(pagebuf_trace_buf);
1965 #endif
1966         kmem_zone_destroy(pagebuf_zone);
1967  out:
1968         return error;
1969 }
1970
1971
1972 /*
1973  *      pagebuf_terminate.
1974  *
1975  *      Note: do not mark as __exit, this is also called from the __init code.
1976  */
1977 void
1978 pagebuf_terminate(void)
1979 {
1980         xfs_buf_daemons_stop();
1981
1982 #ifdef PAGEBUF_TRACE
1983         ktrace_free(pagebuf_trace_buf);
1984 #endif
1985
1986         kmem_zone_destroy(pagebuf_zone);
1987         kmem_shake_deregister(pagebuf_shake);
1988 }