fs/btrfs/scrub.c

   1 /*
   2  * Copyright (C) 2011, 2012 STRATO.  All rights reserved.
   3  *
   4  * This program is free software; you can redistribute it and/or
   5  * modify it under the terms of the GNU General Public
   6  * License v2 as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it will be useful,
   9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  11  * General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public
  14  * License along with this program; if not, write to the
  15  * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  16  * Boston, MA 021110-1307, USA.
  17  */
  18
  19 #include <linux/blkdev.h>
  20 #include <linux/ratelimit.h>
  21 #include "ctree.h"
  22 #include "volumes.h"
  23 #include "disk-io.h"
  24 #include "ordered-data.h"
  25 #include "transaction.h"
  26 #include "backref.h"
  27 #include "extent_io.h"
  28 #include "dev-replace.h"
  29 #include "check-integrity.h"
  30 #include "rcu-string.h"
  31 #include "raid56.h"
  32
  33 /*
  34  * This is only the first step towards a full-features scrub. It reads all
  35  * extent and super block and verifies the checksums. In case a bad checksum
  36  * is found or the extent cannot be read, good data will be written back if
  37  * any can be found.
  38  *
  39  * Future enhancements:
  40  *  - In case an unrepairable extent is encountered, track which files are
  41  *    affected and report them
  42  *  - track and record media errors, throw out bad devices
  43  *  - add a mode to also read unallocated space
  44  */
  45
  46 struct scrub_block;
  47 struct scrub_ctx;
  48
  49 /*
  50  * the following three values only influence the performance.
  51  * The last one configures the number of parallel and outstanding I/O
  52  * operations. The first two values configure an upper limit for the number
  53  * of (dynamically allocated) pages that are added to a bio.
  54  */
  55 #define SCRUB_PAGES_PER_RD_BIO  32      /* 128k per bio */
  56 #define SCRUB_PAGES_PER_WR_BIO  32      /* 128k per bio */
  57 #define SCRUB_BIOS_PER_SCTX     64      /* 8MB per device in flight */
  58
  59 /*
  60  * the following value times PAGE_SIZE needs to be large enough to match the
  61  * largest node/leaf/sector size that shall be supported.
  62  * Values larger than BTRFS_STRIPE_LEN are not supported.
  63  */
  64 #define SCRUB_MAX_PAGES_PER_BLOCK       16      /* 64k per node/leaf/sector */
  65
  66 struct scrub_page {
  67         struct scrub_block      *sblock;
  68         struct page             *page;
  69         struct btrfs_device     *dev;
  70         u64                     flags;  /* extent flags */
  71         u64                     generation;
  72         u64                     logical;
  73         u64                     physical;
  74         u64                     physical_for_dev_replace;
  75         atomic_t                ref_count;
  76         struct {
  77                 unsigned int    mirror_num:8;
  78                 unsigned int    have_csum:1;
  79                 unsigned int    io_error:1;
  80         };
  81         u8                      csum[BTRFS_CSUM_SIZE];
  82 };
  83
  84 struct scrub_bio {
  85         int                     index;
  86         struct scrub_ctx        *sctx;
  87         struct btrfs_device     *dev;
  88         struct bio              *bio;
  89         int                     err;
  90         u64                     logical;
  91         u64                     physical;
  92 #if SCRUB_PAGES_PER_WR_BIO >= SCRUB_PAGES_PER_RD_BIO
  93         struct scrub_page       *pagev[SCRUB_PAGES_PER_WR_BIO];
  94 #else
  95         struct scrub_page       *pagev[SCRUB_PAGES_PER_RD_BIO];
  96 #endif
  97         int                     page_count;
  98         int                     next_free;
  99         struct btrfs_work       work;
 100 };
 101
 102 struct scrub_block {
 103         struct scrub_page       *pagev[SCRUB_MAX_PAGES_PER_BLOCK];
 104         int                     page_count;
 105         atomic_t                outstanding_pages;
 106         atomic_t                ref_count; /* free mem on transition to zero */
 107         struct scrub_ctx        *sctx;
 108         struct {
 109                 unsigned int    header_error:1;
 110                 unsigned int    checksum_error:1;
 111                 unsigned int    no_io_error_seen:1;
 112                 unsigned int    generation_error:1; /* also sets header_error */
 113         };
 114 };
 115
 116 struct scrub_wr_ctx {
 117         struct scrub_bio *wr_curr_bio;
 118         struct btrfs_device *tgtdev;
 119         int pages_per_wr_bio; /* <= SCRUB_PAGES_PER_WR_BIO */
 120         atomic_t flush_all_writes;
 121         struct mutex wr_lock;
 122 };
 123
 124 struct scrub_ctx {
 125         struct scrub_bio        *bios[SCRUB_BIOS_PER_SCTX];
 126         struct btrfs_root       *dev_root;
 127         int                     first_free;
 128         int                     curr;
 129         atomic_t                bios_in_flight;
 130         atomic_t                workers_pending;
 131         spinlock_t              list_lock;
 132         wait_queue_head_t       list_wait;
 133         u16                     csum_size;
 134         struct list_head        csum_list;
 135         atomic_t                cancel_req;
 136         int                     readonly;
 137         int                     pages_per_rd_bio;
 138         u32                     sectorsize;
 139         u32                     nodesize;
 140         u32                     leafsize;
 141
 142         int                     is_dev_replace;
 143         struct scrub_wr_ctx     wr_ctx;
 144
 145         /*
 146          * statistics
 147          */
 148         struct btrfs_scrub_progress stat;
 149         spinlock_t              stat_lock;
 150 };
 151
 152 struct scrub_fixup_nodatasum {
 153         struct scrub_ctx        *sctx;
 154         struct btrfs_device     *dev;
 155         u64                     logical;
 156         struct btrfs_root       *root;
 157         struct btrfs_work       work;
 158         int                     mirror_num;
 159 };
 160
 161 struct scrub_nocow_inode {
 162         u64                     inum;
 163         u64                     offset;
 164         u64                     root;
 165         struct list_head        list;
 166 };
 167
 168 struct scrub_copy_nocow_ctx {
 169         struct scrub_ctx        *sctx;
 170         u64                     logical;
 171         u64                     len;
 172         int                     mirror_num;
 173         u64                     physical_for_dev_replace;
 174         struct list_head        inodes;
 175         struct btrfs_work       work;
 176 };
 177
 178 struct scrub_warning {
 179         struct btrfs_path       *path;
 180         u64                     extent_item_size;
 181         char                    *scratch_buf;
 182         char                    *msg_buf;
 183         const char              *errstr;
 184         sector_t                sector;
 185         u64                     logical;
 186         struct btrfs_device     *dev;
 187         int                     msg_bufsize;
 188         int                     scratch_bufsize;
 189 };
 190
 191
 192 static void scrub_pending_bio_inc(struct scrub_ctx *sctx);
 193 static void scrub_pending_bio_dec(struct scrub_ctx *sctx);
 194 static void scrub_pending_trans_workers_inc(struct scrub_ctx *sctx);
 195 static void scrub_pending_trans_workers_dec(struct scrub_ctx *sctx);
 196 static int scrub_handle_errored_block(struct scrub_block *sblock_to_check);
 197 static int scrub_setup_recheck_block(struct scrub_ctx *sctx,
 198                                      struct btrfs_fs_info *fs_info,
 199                                      struct scrub_block *original_sblock,
 200                                      u64 length, u64 logical,
 201                                      struct scrub_block *sblocks_for_recheck);
 202 static void scrub_recheck_block(struct btrfs_fs_info *fs_info,
 203                                 struct scrub_block *sblock, int is_metadata,
 204                                 int have_csum, u8 *csum, u64 generation,
 205                                 u16 csum_size);
 206 static void scrub_recheck_block_checksum(struct btrfs_fs_info *fs_info,
 207                                          struct scrub_block *sblock,
 208                                          int is_metadata, int have_csum,
 209                                          const u8 *csum, u64 generation,
 210                                          u16 csum_size);
 211 static int scrub_repair_block_from_good_copy(struct scrub_block *sblock_bad,
 212                                              struct scrub_block *sblock_good,
 213                                              int force_write);
 214 static int scrub_repair_page_from_good_copy(struct scrub_block *sblock_bad,
 215                                             struct scrub_block *sblock_good,
 216                                             int page_num, int force_write);
 217 static void scrub_write_block_to_dev_replace(struct scrub_block *sblock);
 218 static int scrub_write_page_to_dev_replace(struct scrub_block *sblock,
 219                                            int page_num);
 220 static int scrub_checksum_data(struct scrub_block *sblock);
 221 static int scrub_checksum_tree_block(struct scrub_block *sblock);
 222 static int scrub_checksum_super(struct scrub_block *sblock);
 223 static void scrub_block_get(struct scrub_block *sblock);
 224 static void scrub_block_put(struct scrub_block *sblock);
 225 static void scrub_page_get(struct scrub_page *spage);
 226 static void scrub_page_put(struct scrub_page *spage);
 227 static int scrub_add_page_to_rd_bio(struct scrub_ctx *sctx,
 228                                     struct scrub_page *spage);
 229 static int scrub_pages(struct scrub_ctx *sctx, u64 logical, u64 len,
 230                        u64 physical, struct btrfs_device *dev, u64 flags,
 231                        u64 gen, int mirror_num, u8 *csum, int force,
 232                        u64 physical_for_dev_replace);
 233 static void scrub_bio_end_io(struct bio *bio, int err);
 234 static void scrub_bio_end_io_worker(struct btrfs_work *work);
 235 static void scrub_block_complete(struct scrub_block *sblock);
 236 static void scrub_remap_extent(struct btrfs_fs_info *fs_info,
 237                                u64 extent_logical, u64 extent_len,
 238                                u64 *extent_physical,
 239                                struct btrfs_device **extent_dev,
 240                                int *extent_mirror_num);
 241 static int scrub_setup_wr_ctx(struct scrub_ctx *sctx,
 242                               struct scrub_wr_ctx *wr_ctx,
 243                               struct btrfs_fs_info *fs_info,
 244                               struct btrfs_device *dev,
 245                               int is_dev_replace);
 246 static void scrub_free_wr_ctx(struct scrub_wr_ctx *wr_ctx);
 247 static int scrub_add_page_to_wr_bio(struct scrub_ctx *sctx,
 248                                     struct scrub_page *spage);
 249 static void scrub_wr_submit(struct scrub_ctx *sctx);
 250 static void scrub_wr_bio_end_io(struct bio *bio, int err);
 251 static void scrub_wr_bio_end_io_worker(struct btrfs_work *work);
 252 static int write_page_nocow(struct scrub_ctx *sctx,
 253                             u64 physical_for_dev_replace, struct page *page);
 254 static int copy_nocow_pages_for_inode(u64 inum, u64 offset, u64 root,
 255                                       struct scrub_copy_nocow_ctx *ctx);
 256 static int copy_nocow_pages(struct scrub_ctx *sctx, u64 logical, u64 len,
 257                             int mirror_num, u64 physical_for_dev_replace);
 258 static void copy_nocow_pages_worker(struct btrfs_work *work);
 259 static void __scrub_blocked_if_needed(struct btrfs_fs_info *fs_info);
 260 static void scrub_blocked_if_needed(struct btrfs_fs_info *fs_info);
 261
 262
 263 static void scrub_pending_bio_inc(struct scrub_ctx *sctx)
 264 {
 265         atomic_inc(&sctx->bios_in_flight);
 266 }
 267
 268 static void scrub_pending_bio_dec(struct scrub_ctx *sctx)
 269 {
 270         atomic_dec(&sctx->bios_in_flight);
 271         wake_up(&sctx->list_wait);
 272 }
 273
 274 static void __scrub_blocked_if_needed(struct btrfs_fs_info *fs_info)
 275 {
 276         while (atomic_read(&fs_info->scrub_pause_req)) {
 277                 mutex_unlock(&fs_info->scrub_lock);
 278                 wait_event(fs_info->scrub_pause_wait,
 279                    atomic_read(&fs_info->scrub_pause_req) == 0);
 280                 mutex_lock(&fs_info->scrub_lock);
 281         }
 282 }
 283
 284 static void scrub_blocked_if_needed(struct btrfs_fs_info *fs_info)
 285 {
 286         atomic_inc(&fs_info->scrubs_paused);
 287         wake_up(&fs_info->scrub_pause_wait);
 288
 289         mutex_lock(&fs_info->scrub_lock);
 290         __scrub_blocked_if_needed(fs_info);
 291         atomic_dec(&fs_info->scrubs_paused);
 292         mutex_unlock(&fs_info->scrub_lock);
 293
 294         wake_up(&fs_info->scrub_pause_wait);
 295 }
 296
 297 /*
 298  * used for workers that require transaction commits (i.e., for the
 299  * NOCOW case)
 300  */
 301 static void scrub_pending_trans_workers_inc(struct scrub_ctx *sctx)
 302 {
 303         struct btrfs_fs_info *fs_info = sctx->dev_root->fs_info;
 304
 305         /*
 306          * increment scrubs_running to prevent cancel requests from
 307          * completing as long as a worker is running. we must also
 308          * increment scrubs_paused to prevent deadlocking on pause
 309          * requests used for transactions commits (as the worker uses a
 310          * transaction context). it is safe to regard the worker
 311          * as paused for all matters practical. effectively, we only
 312          * avoid cancellation requests from completing.
 313          */
 314         mutex_lock(&fs_info->scrub_lock);
 315         atomic_inc(&fs_info->scrubs_running);
 316         atomic_inc(&fs_info->scrubs_paused);
 317         mutex_unlock(&fs_info->scrub_lock);
 318         atomic_inc(&sctx->workers_pending);
 319 }
 320
 321 /* used for workers that require transaction commits */
 322 static void scrub_pending_trans_workers_dec(struct scrub_ctx *sctx)
 323 {
 324         struct btrfs_fs_info *fs_info = sctx->dev_root->fs_info;
 325
 326         /*
 327          * see scrub_pending_trans_workers_inc() why we're pretending
 328          * to be paused in the scrub counters
 329          */
 330         mutex_lock(&fs_info->scrub_lock);
 331         atomic_dec(&fs_info->scrubs_running);
 332         atomic_dec(&fs_info->scrubs_paused);
 333         mutex_unlock(&fs_info->scrub_lock);
 334         atomic_dec(&sctx->workers_pending);
 335         wake_up(&fs_info->scrub_pause_wait);
 336         wake_up(&sctx->list_wait);
 337 }
 338
 339 static void scrub_free_csums(struct scrub_ctx *sctx)
 340 {
 341         while (!list_empty(&sctx->csum_list)) {
 342                 struct btrfs_ordered_sum *sum;
 343                 sum = list_first_entry(&sctx->csum_list,
 344                                        struct btrfs_ordered_sum, list);
 345                 list_del(&sum->list);
 346                 kfree(sum);
 347         }
 348 }
 349
 350 static noinline_for_stack void scrub_free_ctx(struct scrub_ctx *sctx)
 351 {
 352         int i;
 353
 354         if (!sctx)
 355                 return;
 356
 357         scrub_free_wr_ctx(&sctx->wr_ctx);
 358
 359         /* this can happen when scrub is cancelled */
 360         if (sctx->curr != -1) {
 361                 struct scrub_bio *sbio = sctx->bios[sctx->curr];
 362
 363                 for (i = 0; i < sbio->page_count; i++) {
 364                         WARN_ON(!sbio->pagev[i]->page);
 365                         scrub_block_put(sbio->pagev[i]->sblock);
 366                 }
 367                 bio_put(sbio->bio);
 368         }
 369
 370         for (i = 0; i < SCRUB_BIOS_PER_SCTX; ++i) {
 371                 struct scrub_bio *sbio = sctx->bios[i];
 372
 373                 if (!sbio)
 374                         break;
 375                 kfree(sbio);
 376         }
 377
 378         scrub_free_csums(sctx);
 379         kfree(sctx);
 380 }
 381
 382 static noinline_for_stack
 383 struct scrub_ctx *scrub_setup_ctx(struct btrfs_device *dev, int is_dev_replace)
 384 {
 385         struct scrub_ctx *sctx;
 386         int             i;
 387         struct btrfs_fs_info *fs_info = dev->dev_root->fs_info;
 388         int pages_per_rd_bio;
 389         int ret;
 390
 391         /*
 392          * the setting of pages_per_rd_bio is correct for scrub but might
 393          * be wrong for the dev_replace code where we might read from
 394          * different devices in the initial huge bios. However, that
 395          * code is able to correctly handle the case when adding a page
 396          * to a bio fails.
 397          */
 398         if (dev->bdev)
 399                 pages_per_rd_bio = min_t(int, SCRUB_PAGES_PER_RD_BIO,
 400                                          bio_get_nr_vecs(dev->bdev));
 401         else
 402                 pages_per_rd_bio = SCRUB_PAGES_PER_RD_BIO;
 403         sctx = kzalloc(sizeof(*sctx), GFP_NOFS);
 404         if (!sctx)
 405                 goto nomem;
 406         sctx->is_dev_replace = is_dev_replace;
 407         sctx->pages_per_rd_bio = pages_per_rd_bio;
 408         sctx->curr = -1;
 409         sctx->dev_root = dev->dev_root;
 410         for (i = 0; i < SCRUB_BIOS_PER_SCTX; ++i) {
 411                 struct scrub_bio *sbio;
 412
 413                 sbio = kzalloc(sizeof(*sbio), GFP_NOFS);
 414                 if (!sbio)
 415                         goto nomem;
 416                 sctx->bios[i] = sbio;
 417
 418                 sbio->index = i;
 419                 sbio->sctx = sctx;
 420                 sbio->page_count = 0;
 421                 sbio->work.func = scrub_bio_end_io_worker;
 422
 423                 if (i != SCRUB_BIOS_PER_SCTX - 1)
 424                         sctx->bios[i]->next_free = i + 1;
 425                 else
 426                         sctx->bios[i]->next_free = -1;
 427         }
 428         sctx->first_free = 0;
 429         sctx->nodesize = dev->dev_root->nodesize;
 430         sctx->leafsize = dev->dev_root->leafsize;
 431         sctx->sectorsize = dev->dev_root->sectorsize;
 432         atomic_set(&sctx->bios_in_flight, 0);
 433         atomic_set(&sctx->workers_pending, 0);
 434         atomic_set(&sctx->cancel_req, 0);
 435         sctx->csum_size = btrfs_super_csum_size(fs_info->super_copy);
 436         INIT_LIST_HEAD(&sctx->csum_list);
 437
 438         spin_lock_init(&sctx->list_lock);
 439         spin_lock_init(&sctx->stat_lock);
 440         init_waitqueue_head(&sctx->list_wait);
 441
 442         ret = scrub_setup_wr_ctx(sctx, &sctx->wr_ctx, fs_info,
 443                                  fs_info->dev_replace.tgtdev, is_dev_replace);
 444         if (ret) {
 445                 scrub_free_ctx(sctx);
 446                 return ERR_PTR(ret);
 447         }
 448         return sctx;
 449
 450 nomem:
 451         scrub_free_ctx(sctx);
 452         return ERR_PTR(-ENOMEM);
 453 }
 454
 455 static int scrub_print_warning_inode(u64 inum, u64 offset, u64 root,
 456                                      void *warn_ctx)
 457 {
 458         u64 isize;
 459         u32 nlink;
 460         int ret;
 461         int i;
 462         struct extent_buffer *eb;
 463         struct btrfs_inode_item *inode_item;
 464         struct scrub_warning *swarn = warn_ctx;
 465         struct btrfs_fs_info *fs_info = swarn->dev->dev_root->fs_info;
 466         struct inode_fs_paths *ipath = NULL;
 467         struct btrfs_root *local_root;
 468         struct btrfs_key root_key;
 469
 470         root_key.objectid = root;
 471         root_key.type = BTRFS_ROOT_ITEM_KEY;
 472         root_key.offset = (u64)-1;
 473         local_root = btrfs_read_fs_root_no_name(fs_info, &root_key);
 474         if (IS_ERR(local_root)) {
 475                 ret = PTR_ERR(local_root);
 476                 goto err;
 477         }
 478
 479         ret = inode_item_info(inum, 0, local_root, swarn->path);
 480         if (ret) {
 481                 btrfs_release_path(swarn->path);
 482                 goto err;
 483         }
 484
 485         eb = swarn->path->nodes[0];
 486         inode_item = btrfs_item_ptr(eb, swarn->path->slots[0],
 487                                         struct btrfs_inode_item);
 488         isize = btrfs_inode_size(eb, inode_item);
 489         nlink = btrfs_inode_nlink(eb, inode_item);
 490         btrfs_release_path(swarn->path);
 491
 492         ipath = init_ipath(4096, local_root, swarn->path);
 493         if (IS_ERR(ipath)) {
 494                 ret = PTR_ERR(ipath);
 495                 ipath = NULL;
 496                 goto err;
 497         }
 498         ret = paths_from_inode(inum, ipath);
 499
 500         if (ret < 0)
 501                 goto err;
 502
 503         /*
 504          * we deliberately ignore the bit ipath might have been too small to
 505          * hold all of the paths here
 506          */
 507         for (i = 0; i < ipath->fspath->elem_cnt; ++i)
 508                 printk_in_rcu(KERN_WARNING "btrfs: %s at logical %llu on dev "
 509                         "%s, sector %llu, root %llu, inode %llu, offset %llu, "
 510                         "length %llu, links %u (path: %s)\n", swarn->errstr,
 511                         swarn->logical, rcu_str_deref(swarn->dev->name),
 512                         (unsigned long long)swarn->sector, root, inum, offset,
 513                         min(isize - offset, (u64)PAGE_SIZE), nlink,
 514                         (char *)(unsigned long)ipath->fspath->val[i]);
 515
 516         free_ipath(ipath);
 517         return 0;
 518
 519 err:
 520         printk_in_rcu(KERN_WARNING "btrfs: %s at logical %llu on dev "
 521                 "%s, sector %llu, root %llu, inode %llu, offset %llu: path "
 522                 "resolving failed with ret=%d\n", swarn->errstr,
 523                 swarn->logical, rcu_str_deref(swarn->dev->name),
 524                 (unsigned long long)swarn->sector, root, inum, offset, ret);
 525
 526         free_ipath(ipath);
 527         return 0;
 528 }
 529
 530 static void scrub_print_warning(const char *errstr, struct scrub_block *sblock)
 531 {
 532         struct btrfs_device *dev;
 533         struct btrfs_fs_info *fs_info;
 534         struct btrfs_path *path;
 535         struct btrfs_key found_key;
 536         struct extent_buffer *eb;
 537         struct btrfs_extent_item *ei;
 538         struct scrub_warning swarn;
 539         unsigned long ptr = 0;
 540         u64 extent_item_pos;
 541         u64 flags = 0;
 542         u64 ref_root;
 543         u32 item_size;
 544         u8 ref_level;
 545         const int bufsize = 4096;
 546         int ret;
 547
 548         WARN_ON(sblock->page_count < 1);
 549         dev = sblock->pagev[0]->dev;
 550         fs_info = sblock->sctx->dev_root->fs_info;
 551
 552         path = btrfs_alloc_path();
 553
 554         swarn.scratch_buf = kmalloc(bufsize, GFP_NOFS);
 555         swarn.msg_buf = kmalloc(bufsize, GFP_NOFS);
 556         swarn.sector = (sblock->pagev[0]->physical) >> 9;
 557         swarn.logical = sblock->pagev[0]->logical;
 558         swarn.errstr = errstr;
 559         swarn.dev = NULL;
 560         swarn.msg_bufsize = bufsize;
 561         swarn.scratch_bufsize = bufsize;
 562
 563         if (!path || !swarn.scratch_buf || !swarn.msg_buf)
 564                 goto out;
 565
 566         ret = extent_from_logical(fs_info, swarn.logical, path, &found_key,
 567                                   &flags);
 568         if (ret < 0)
 569                 goto out;
 570
 571         extent_item_pos = swarn.logical - found_key.objectid;
 572         swarn.extent_item_size = found_key.offset;
 573
 574         eb = path->nodes[0];
 575         ei = btrfs_item_ptr(eb, path->slots[0], struct btrfs_extent_item);
 576         item_size = btrfs_item_size_nr(eb, path->slots[0]);
 577
 578         if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {
 579                 do {
 580                         ret = tree_backref_for_extent(&ptr, eb, ei, item_size,
 581                                                         &ref_root, &ref_level);
 582                         printk_in_rcu(KERN_WARNING
 583                                 "btrfs: %s at logical %llu on dev %s, "
 584                                 "sector %llu: metadata %s (level %d) in tree "
 585                                 "%llu\n", errstr, swarn.logical,
 586                                 rcu_str_deref(dev->name),
 587                                 (unsigned long long)swarn.sector,
 588                                 ref_level ? "node" : "leaf",
 589                                 ret < 0 ? -1 : ref_level,
 590                                 ret < 0 ? -1 : ref_root);
 591                 } while (ret != 1);
 592                 btrfs_release_path(path);
 593         } else {
 594                 btrfs_release_path(path);
 595                 swarn.path = path;
 596                 swarn.dev = dev;
 597                 iterate_extent_inodes(fs_info, found_key.objectid,
 598                                         extent_item_pos, 1,
 599                                         scrub_print_warning_inode, &swarn);
 600         }
 601
 602 out:
 603         btrfs_free_path(path);
 604         kfree(swarn.scratch_buf);
 605         kfree(swarn.msg_buf);
 606 }
 607
 608 static int scrub_fixup_readpage(u64 inum, u64 offset, u64 root, void *fixup_ctx)
 609 {
 610         struct page *page = NULL;
 611         unsigned long index;
 612         struct scrub_fixup_nodatasum *fixup = fixup_ctx;
 613         int ret;
 614         int corrected = 0;
 615         struct btrfs_key key;
 616         struct inode *inode = NULL;
 617         struct btrfs_fs_info *fs_info;
 618         u64 end = offset + PAGE_SIZE - 1;
 619         struct btrfs_root *local_root;
 620         int srcu_index;
 621
 622         key.objectid = root;
 623         key.type = BTRFS_ROOT_ITEM_KEY;
 624         key.offset = (u64)-1;
 625
 626         fs_info = fixup->root->fs_info;
 627         srcu_index = srcu_read_lock(&fs_info->subvol_srcu);
 628
 629         local_root = btrfs_read_fs_root_no_name(fs_info, &key);
 630         if (IS_ERR(local_root)) {
 631                 srcu_read_unlock(&fs_info->subvol_srcu, srcu_index);
 632                 return PTR_ERR(local_root);
 633         }
 634
 635         key.type = BTRFS_INODE_ITEM_KEY;
 636         key.objectid = inum;
 637         key.offset = 0;
 638         inode = btrfs_iget(fs_info->sb, &key, local_root, NULL);
 639         srcu_read_unlock(&fs_info->subvol_srcu, srcu_index);
 640         if (IS_ERR(inode))
 641                 return PTR_ERR(inode);
 642
 643         index = offset >> PAGE_CACHE_SHIFT;
 644
 645         page = find_or_create_page(inode->i_mapping, index, GFP_NOFS);
 646         if (!page) {
 647                 ret = -ENOMEM;
 648                 goto out;
 649         }
 650
 651         if (PageUptodate(page)) {
 652                 if (PageDirty(page)) {
 653                         /*
 654                          * we need to write the data to the defect sector. the
 655                          * data that was in that sector is not in memory,
 656                          * because the page was modified. we must not write the
 657                          * modified page to that sector.
 658                          *
 659                          * TODO: what could be done here: wait for the delalloc
 660                          *       runner to write out that page (might involve
 661                          *       COW) and see whether the sector is still
 662                          *       referenced afterwards.
 663                          *
 664                          * For the meantime, we'll treat this error
 665                          * incorrectable, although there is a chance that a
 666                          * later scrub will find the bad sector again and that
 667                          * there's no dirty page in memory, then.
 668                          */
 669                         ret = -EIO;
 670                         goto out;
 671                 }
 672                 fs_info = BTRFS_I(inode)->root->fs_info;
 673                 ret = repair_io_failure(fs_info, offset, PAGE_SIZE,
 674                                         fixup->logical, page,
 675                                         fixup->mirror_num);
 676                 unlock_page(page);
 677                 corrected = !ret;
 678         } else {
 679                 /*
 680                  * we need to get good data first. the general readpage path
 681                  * will call repair_io_failure for us, we just have to make
 682                  * sure we read the bad mirror.
 683                  */
 684                 ret = set_extent_bits(&BTRFS_I(inode)->io_tree, offset, end,
 685                                         EXTENT_DAMAGED, GFP_NOFS);
 686                 if (ret) {
 687                         /* set_extent_bits should give proper error */
 688                         WARN_ON(ret > 0);
 689                         if (ret > 0)
 690                                 ret = -EFAULT;
 691                         goto out;
 692                 }
 693
 694                 ret = extent_read_full_page(&BTRFS_I(inode)->io_tree, page,
 695                                                 btrfs_get_extent,
 696                                                 fixup->mirror_num);
 697                 wait_on_page_locked(page);
 698
 699                 corrected = !test_range_bit(&BTRFS_I(inode)->io_tree, offset,
 700                                                 end, EXTENT_DAMAGED, 0, NULL);
 701                 if (!corrected)
 702                         clear_extent_bits(&BTRFS_I(inode)->io_tree, offset, end,
 703                                                 EXTENT_DAMAGED, GFP_NOFS);
 704         }
 705
 706 out:
 707         if (page)
 708                 put_page(page);
 709         if (inode)
 710                 iput(inode);
 711
 712         if (ret < 0)
 713                 return ret;
 714
 715         if (ret == 0 && corrected) {
 716                 /*
 717                  * we only need to call readpage for one of the inodes belonging
 718                  * to this extent. so make iterate_extent_inodes stop
 719                  */
 720                 return 1;
 721         }
 722
 723         return -EIO;
 724 }
 725
 726 static void scrub_fixup_nodatasum(struct btrfs_work *work)
 727 {
 728         int ret;
 729         struct scrub_fixup_nodatasum *fixup;
 730         struct scrub_ctx *sctx;
 731         struct btrfs_trans_handle *trans = NULL;
 732         struct btrfs_path *path;
 733         int uncorrectable = 0;
 734
 735         fixup = container_of(work, struct scrub_fixup_nodatasum, work);
 736         sctx = fixup->sctx;
 737
 738         path = btrfs_alloc_path();
 739         if (!path) {
 740                 spin_lock(&sctx->stat_lock);
 741                 ++sctx->stat.malloc_errors;
 742                 spin_unlock(&sctx->stat_lock);
 743                 uncorrectable = 1;
 744                 goto out;
 745         }
 746
 747         trans = btrfs_join_transaction(fixup->root);
 748         if (IS_ERR(trans)) {
 749                 uncorrectable = 1;
 750                 goto out;
 751         }
 752
 753         /*
 754          * the idea is to trigger a regular read through the standard path. we
 755          * read a page from the (failed) logical address by specifying the
 756          * corresponding copynum of the failed sector. thus, that readpage is
 757          * expected to fail.
 758          * that is the point where on-the-fly error correction will kick in
 759          * (once it's finished) and rewrite the failed sector if a good copy
 760          * can be found.
 761          */
 762         ret = iterate_inodes_from_logical(fixup->logical, fixup->root->fs_info,
 763                                                 path, scrub_fixup_readpage,
 764                                                 fixup);
 765         if (ret < 0) {
 766                 uncorrectable = 1;
 767                 goto out;
 768         }
 769         WARN_ON(ret != 1);
 770
 771         spin_lock(&sctx->stat_lock);
 772         ++sctx->stat.corrected_errors;
 773         spin_unlock(&sctx->stat_lock);
 774
 775 out:
 776         if (trans && !IS_ERR(trans))
 777                 btrfs_end_transaction(trans, fixup->root);
 778         if (uncorrectable) {
 779                 spin_lock(&sctx->stat_lock);
 780                 ++sctx->stat.uncorrectable_errors;
 781                 spin_unlock(&sctx->stat_lock);
 782                 btrfs_dev_replace_stats_inc(
 783                         &sctx->dev_root->fs_info->dev_replace.
 784                         num_uncorrectable_read_errors);
 785                 printk_ratelimited_in_rcu(KERN_ERR
 786                         "btrfs: unable to fixup (nodatasum) error at logical %llu on dev %s\n",
 787                         fixup->logical, rcu_str_deref(fixup->dev->name));
 788         }
 789
 790         btrfs_free_path(path);
 791         kfree(fixup);
 792
 793         scrub_pending_trans_workers_dec(sctx);
 794 }
 795
 796 /*
 797  * scrub_handle_errored_block gets called when either verification of the
 798  * pages failed or the bio failed to read, e.g. with EIO. In the latter
 799  * case, this function handles all pages in the bio, even though only one
 800  * may be bad.
 801  * The goal of this function is to repair the errored block by using the
 802  * contents of one of the mirrors.
 803  */
 804 static int scrub_handle_errored_block(struct scrub_block *sblock_to_check)
 805 {
 806         struct scrub_ctx *sctx = sblock_to_check->sctx;
 807         struct btrfs_device *dev;
 808         struct btrfs_fs_info *fs_info;
 809         u64 length;
 810         u64 logical;
 811         u64 generation;
 812         unsigned int failed_mirror_index;
 813         unsigned int is_metadata;
 814         unsigned int have_csum;
 815         u8 *csum;
 816         struct scrub_block *sblocks_for_recheck; /* holds one for each mirror */
 817         struct scrub_block *sblock_bad;
 818         int ret;
 819         int mirror_index;
 820         int page_num;
 821         int success;
 822         static DEFINE_RATELIMIT_STATE(_rs, DEFAULT_RATELIMIT_INTERVAL,
 823                                       DEFAULT_RATELIMIT_BURST);
 824
 825         BUG_ON(sblock_to_check->page_count < 1);
 826         fs_info = sctx->dev_root->fs_info;
 827         if (sblock_to_check->pagev[0]->flags & BTRFS_EXTENT_FLAG_SUPER) {
 828                 /*
 829                  * if we find an error in a super block, we just report it.
 830                  * They will get written with the next transaction commit
 831                  * anyway
 832                  */
 833                 spin_lock(&sctx->stat_lock);
 834                 ++sctx->stat.super_errors;
 835                 spin_unlock(&sctx->stat_lock);
 836                 return 0;
 837         }
 838         length = sblock_to_check->page_count * PAGE_SIZE;
 839         logical = sblock_to_check->pagev[0]->logical;
 840         generation = sblock_to_check->pagev[0]->generation;
 841         BUG_ON(sblock_to_check->pagev[0]->mirror_num < 1);
 842         failed_mirror_index = sblock_to_check->pagev[0]->mirror_num - 1;
 843         is_metadata = !(sblock_to_check->pagev[0]->flags &
 844                         BTRFS_EXTENT_FLAG_DATA);
 845         have_csum = sblock_to_check->pagev[0]->have_csum;
 846         csum = sblock_to_check->pagev[0]->csum;
 847         dev = sblock_to_check->pagev[0]->dev;
 848
 849         if (sctx->is_dev_replace && !is_metadata && !have_csum) {
 850                 sblocks_for_recheck = NULL;
 851                 goto nodatasum_case;
 852         }
 853
 854         /*
 855          * read all mirrors one after the other. This includes to
 856          * re-read the extent or metadata block that failed (that was
 857          * the cause that this fixup code is called) another time,
 858          * page by page this time in order to know which pages
 859          * caused I/O errors and which ones are good (for all mirrors).
 860          * It is the goal to handle the situation when more than one
 861          * mirror contains I/O errors, but the errors do not
 862          * overlap, i.e. the data can be repaired by selecting the
 863          * pages from those mirrors without I/O error on the
 864          * particular pages. One example (with blocks >= 2 * PAGE_SIZE)
 865          * would be that mirror #1 has an I/O error on the first page,
 866          * the second page is good, and mirror #2 has an I/O error on
 867          * the second page, but the first page is good.
 868          * Then the first page of the first mirror can be repaired by
 869          * taking the first page of the second mirror, and the
 870          * second page of the second mirror can be repaired by
 871          * copying the contents of the 2nd page of the 1st mirror.
 872          * One more note: if the pages of one mirror contain I/O
 873          * errors, the checksum cannot be verified. In order to get
 874          * the best data for repairing, the first attempt is to find
 875          * a mirror without I/O errors and with a validated checksum.
 876          * Only if this is not possible, the pages are picked from
 877          * mirrors with I/O errors without considering the checksum.
 878          * If the latter is the case, at the end, the checksum of the
 879          * repaired area is verified in order to correctly maintain
 880          * the statistics.
 881          */
 882
 883         sblocks_for_recheck = kzalloc(BTRFS_MAX_MIRRORS *
 884                                      sizeof(*sblocks_for_recheck),
 885                                      GFP_NOFS);
 886         if (!sblocks_for_recheck) {
 887                 spin_lock(&sctx->stat_lock);
 888                 sctx->stat.malloc_errors++;
 889                 sctx->stat.read_errors++;
 890                 sctx->stat.uncorrectable_errors++;
 891                 spin_unlock(&sctx->stat_lock);
 892                 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_READ_ERRS);
 893                 goto out;
 894         }
 895
 896         /* setup the context, map the logical blocks and alloc the pages */
 897         ret = scrub_setup_recheck_block(sctx, fs_info, sblock_to_check, length,
 898                                         logical, sblocks_for_recheck);
 899         if (ret) {
 900                 spin_lock(&sctx->stat_lock);
 901                 sctx->stat.read_errors++;
 902                 sctx->stat.uncorrectable_errors++;
 903                 spin_unlock(&sctx->stat_lock);
 904                 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_READ_ERRS);
 905                 goto out;
 906         }
 907         BUG_ON(failed_mirror_index >= BTRFS_MAX_MIRRORS);
 908         sblock_bad = sblocks_for_recheck + failed_mirror_index;
 909
 910         /* build and submit the bios for the failed mirror, check checksums */
 911         scrub_recheck_block(fs_info, sblock_bad, is_metadata, have_csum,
 912                             csum, generation, sctx->csum_size);
 913
 914         if (!sblock_bad->header_error && !sblock_bad->checksum_error &&
 915             sblock_bad->no_io_error_seen) {
 916                 /*
 917                  * the error disappeared after reading page by page, or
 918                  * the area was part of a huge bio and other parts of the
 919                  * bio caused I/O errors, or the block layer merged several
 920                  * read requests into one and the error is caused by a
 921                  * different bio (usually one of the two latter cases is
 922                  * the cause)
 923                  */
 924                 spin_lock(&sctx->stat_lock);
 925                 sctx->stat.unverified_errors++;
 926                 spin_unlock(&sctx->stat_lock);
 927
 928                 if (sctx->is_dev_replace)
 929                         scrub_write_block_to_dev_replace(sblock_bad);
 930                 goto out;
 931         }
 932
 933         if (!sblock_bad->no_io_error_seen) {
 934                 spin_lock(&sctx->stat_lock);
 935                 sctx->stat.read_errors++;
 936                 spin_unlock(&sctx->stat_lock);
 937                 if (__ratelimit(&_rs))
 938                         scrub_print_warning("i/o error", sblock_to_check);
 939                 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_READ_ERRS);
 940         } else if (sblock_bad->checksum_error) {
 941                 spin_lock(&sctx->stat_lock);
 942                 sctx->stat.csum_errors++;
 943                 spin_unlock(&sctx->stat_lock);
 944                 if (__ratelimit(&_rs))
 945                         scrub_print_warning("checksum error", sblock_to_check);
 946                 btrfs_dev_stat_inc_and_print(dev,
 947                                              BTRFS_DEV_STAT_CORRUPTION_ERRS);
 948         } else if (sblock_bad->header_error) {
 949                 spin_lock(&sctx->stat_lock);
 950                 sctx->stat.verify_errors++;
 951                 spin_unlock(&sctx->stat_lock);
 952                 if (__ratelimit(&_rs))
 953                         scrub_print_warning("checksum/header error",
 954                                             sblock_to_check);
 955                 if (sblock_bad->generation_error)
 956                         btrfs_dev_stat_inc_and_print(dev,
 957                                 BTRFS_DEV_STAT_GENERATION_ERRS);
 958                 else
 959                         btrfs_dev_stat_inc_and_print(dev,
 960                                 BTRFS_DEV_STAT_CORRUPTION_ERRS);
 961         }
 962
 963         if (sctx->readonly) {
 964                 ASSERT(!sctx->is_dev_replace);
 965                 goto out;
 966         }
 967
 968         if (!is_metadata && !have_csum) {
 969                 struct scrub_fixup_nodatasum *fixup_nodatasum;
 970
 971 nodatasum_case:
 972                 WARN_ON(sctx->is_dev_replace);
 973
 974                 /*
 975                  * !is_metadata and !have_csum, this means that the data
 976                  * might not be COW'ed, that it might be modified
 977                  * concurrently. The general strategy to work on the
 978                  * commit root does not help in the case when COW is not
 979                  * used.
 980                  */
 981                 fixup_nodatasum = kzalloc(sizeof(*fixup_nodatasum), GFP_NOFS);
 982                 if (!fixup_nodatasum)
 983                         goto did_not_correct_error;
 984                 fixup_nodatasum->sctx = sctx;
 985                 fixup_nodatasum->dev = dev;
 986                 fixup_nodatasum->logical = logical;
 987                 fixup_nodatasum->root = fs_info->extent_root;
 988                 fixup_nodatasum->mirror_num = failed_mirror_index + 1;
 989                 scrub_pending_trans_workers_inc(sctx);
 990                 fixup_nodatasum->work.func = scrub_fixup_nodatasum;
 991                 btrfs_queue_worker(&fs_info->scrub_workers,
 992                                    &fixup_nodatasum->work);
 993                 goto out;
 994         }
 995
 996         /*
 997          * now build and submit the bios for the other mirrors, check
 998          * checksums.
 999          * First try to pick the mirror which is completely without I/O
1000          * errors and also does not have a checksum error.
1001          * If one is found, and if a checksum is present, the full block
1002          * that is known to contain an error is rewritten. Afterwards
1003          * the block is known to be corrected.
1004          * If a mirror is found which is completely correct, and no
1005          * checksum is present, only those pages are rewritten that had
1006          * an I/O error in the block to be repaired, since it cannot be
1007          * determined, which copy of the other pages is better (and it
1008          * could happen otherwise that a correct page would be
1009          * overwritten by a bad one).
1010          */
1011         for (mirror_index = 0;
1012              mirror_index < BTRFS_MAX_MIRRORS &&
1013              sblocks_for_recheck[mirror_index].page_count > 0;
1014              mirror_index++) {
1015                 struct scrub_block *sblock_other;
1016
1017                 if (mirror_index == failed_mirror_index)
1018                         continue;
1019                 sblock_other = sblocks_for_recheck + mirror_index;
1020
1021                 /* build and submit the bios, check checksums */
1022                 scrub_recheck_block(fs_info, sblock_other, is_metadata,
1023                                     have_csum, csum, generation,
1024                                     sctx->csum_size);
1025
1026                 if (!sblock_other->header_error &&
1027                     !sblock_other->checksum_error &&
1028                     sblock_other->no_io_error_seen) {
1029                         if (sctx->is_dev_replace) {
1030                                 scrub_write_block_to_dev_replace(sblock_other);
1031                         } else {
1032                                 int force_write = is_metadata || have_csum;
1033
1034                                 ret = scrub_repair_block_from_good_copy(
1035                                                 sblock_bad, sblock_other,
1036                                                 force_write);
1037                         }
1038                         if (0 == ret)
1039                                 goto corrected_error;
1040                 }
1041         }
1042
1043         /*
1044          * for dev_replace, pick good pages and write to the target device.
1045          */
1046         if (sctx->is_dev_replace) {
1047                 success = 1;
1048                 for (page_num = 0; page_num < sblock_bad->page_count;
1049                      page_num++) {
1050                         int sub_success;
1051
1052                         sub_success = 0;
1053                         for (mirror_index = 0;
1054                              mirror_index < BTRFS_MAX_MIRRORS &&
1055                              sblocks_for_recheck[mirror_index].page_count > 0;
1056                              mirror_index++) {
1057                                 struct scrub_block *sblock_other =
1058                                         sblocks_for_recheck + mirror_index;
1059                                 struct scrub_page *page_other =
1060                                         sblock_other->pagev[page_num];
1061
1062                                 if (!page_other->io_error) {
1063                                         ret = scrub_write_page_to_dev_replace(
1064                                                         sblock_other, page_num);
1065                                         if (ret == 0) {
1066                                                 /* succeeded for this page */
1067                                                 sub_success = 1;
1068                                                 break;
1069                                         } else {
1070                                                 btrfs_dev_replace_stats_inc(
1071                                                         &sctx->dev_root->
1072                                                         fs_info->dev_replace.
1073                                                         num_write_errors);
1074                                         }
1075                                 }
1076                         }
1077
1078                         if (!sub_success) {
1079                                 /*
1080                                  * did not find a mirror to fetch the page
1081                                  * from. scrub_write_page_to_dev_replace()
1082                                  * handles this case (page->io_error), by
1083                                  * filling the block with zeros before
1084                                  * submitting the write request
1085                                  */
1086                                 success = 0;
1087                                 ret = scrub_write_page_to_dev_replace(
1088                                                 sblock_bad, page_num);
1089                                 if (ret)
1090                                         btrfs_dev_replace_stats_inc(
1091                                                 &sctx->dev_root->fs_info->
1092                                                 dev_replace.num_write_errors);
1093                         }
1094                 }
1095
1096                 goto out;
1097         }
1098
1099         /*
1100          * for regular scrub, repair those pages that are errored.
1101          * In case of I/O errors in the area that is supposed to be
1102          * repaired, continue by picking good copies of those pages.
1103          * Select the good pages from mirrors to rewrite bad pages from
1104          * the area to fix. Afterwards verify the checksum of the block
1105          * that is supposed to be repaired. This verification step is
1106          * only done for the purpose of statistic counting and for the
1107          * final scrub report, whether errors remain.
1108          * A perfect algorithm could make use of the checksum and try
1109          * all possible combinations of pages from the different mirrors
1110          * until the checksum verification succeeds. For example, when
1111          * the 2nd page of mirror #1 faces I/O errors, and the 2nd page
1112          * of mirror #2 is readable but the final checksum test fails,
1113          * then the 2nd page of mirror #3 could be tried, whether now
1114          * the final checksum succeedes. But this would be a rare
1115          * exception and is therefore not implemented. At least it is
1116          * avoided that the good copy is overwritten.
1117          * A more useful improvement would be to pick the sectors
1118          * without I/O error based on sector sizes (512 bytes on legacy
1119          * disks) instead of on PAGE_SIZE. Then maybe 512 byte of one
1120          * mirror could be repaired by taking 512 byte of a different
1121          * mirror, even if other 512 byte sectors in the same PAGE_SIZE
1122          * area are unreadable.
1123          */
1124
1125         /* can only fix I/O errors from here on */
1126         if (sblock_bad->no_io_error_seen)
1127                 goto did_not_correct_error;
1128
1129         success = 1;
1130         for (page_num = 0; page_num < sblock_bad->page_count; page_num++) {
1131                 struct scrub_page *page_bad = sblock_bad->pagev[page_num];
1132
1133                 if (!page_bad->io_error)
1134                         continue;
1135
1136                 for (mirror_index = 0;
1137                      mirror_index < BTRFS_MAX_MIRRORS &&
1138                      sblocks_for_recheck[mirror_index].page_count > 0;
1139                      mirror_index++) {
1140                         struct scrub_block *sblock_other = sblocks_for_recheck +
1141                                                            mirror_index;
1142                         struct scrub_page *page_other = sblock_other->pagev[
1143                                                         page_num];
1144
1145                         if (!page_other->io_error) {
1146                                 ret = scrub_repair_page_from_good_copy(
1147                                         sblock_bad, sblock_other, page_num, 0);
1148                                 if (0 == ret) {
1149                                         page_bad->io_error = 0;
1150                                         break; /* succeeded for this page */
1151                                 }
1152                         }
1153                 }
1154
1155                 if (page_bad->io_error) {
1156                         /* did not find a mirror to copy the page from */
1157                         success = 0;
1158                 }
1159         }
1160
1161         if (success) {
1162                 if (is_metadata || have_csum) {
1163                         /*
1164                          * need to verify the checksum now that all
1165                          * sectors on disk are repaired (the write
1166                          * request for data to be repaired is on its way).
1167                          * Just be lazy and use scrub_recheck_block()
1168                          * which re-reads the data before the checksum
1169                          * is verified, but most likely the data comes out
1170                          * of the page cache.
1171                          */
1172                         scrub_recheck_block(fs_info, sblock_bad,
1173                                             is_metadata, have_csum, csum,
1174                                             generation, sctx->csum_size);
1175                         if (!sblock_bad->header_error &&
1176                             !sblock_bad->checksum_error &&
1177                             sblock_bad->no_io_error_seen)
1178                                 goto corrected_error;
1179                         else
1180                                 goto did_not_correct_error;
1181                 } else {
1182 corrected_error:
1183                         spin_lock(&sctx->stat_lock);
1184                         sctx->stat.corrected_errors++;
1185                         spin_unlock(&sctx->stat_lock);
1186                         printk_ratelimited_in_rcu(KERN_ERR
1187                                 "btrfs: fixed up error at logical %llu on dev %s\n",
1188                                 logical, rcu_str_deref(dev->name));
1189                 }
1190         } else {
1191 did_not_correct_error:
1192                 spin_lock(&sctx->stat_lock);
1193                 sctx->stat.uncorrectable_errors++;
1194                 spin_unlock(&sctx->stat_lock);
1195                 printk_ratelimited_in_rcu(KERN_ERR
1196                         "btrfs: unable to fixup (regular) error at logical %llu on dev %s\n",
1197                         logical, rcu_str_deref(dev->name));
1198         }
1199
1200 out:
1201         if (sblocks_for_recheck) {
1202                 for (mirror_index = 0; mirror_index < BTRFS_MAX_MIRRORS;
1203                      mirror_index++) {
1204                         struct scrub_block *sblock = sblocks_for_recheck +
1205                                                      mirror_index;
1206                         int page_index;
1207
1208                         for (page_index = 0; page_index < sblock->page_count;
1209                              page_index++) {
1210                                 sblock->pagev[page_index]->sblock = NULL;
1211                                 scrub_page_put(sblock->pagev[page_index]);
1212                         }
1213                 }
1214                 kfree(sblocks_for_recheck);
1215         }
1216
1217         return 0;
1218 }
1219
1220 static int scrub_setup_recheck_block(struct scrub_ctx *sctx,
1221                                      struct btrfs_fs_info *fs_info,
1222                                      struct scrub_block *original_sblock,
1223                                      u64 length, u64 logical,
1224                                      struct scrub_block *sblocks_for_recheck)
1225 {
1226         int page_index;
1227         int mirror_index;
1228         int ret;
1229
1230         /*
1231          * note: the two members ref_count and outstanding_pages
1232          * are not used (and not set) in the blocks that are used for
1233          * the recheck procedure
1234          */
1235
1236         page_index = 0;
1237         while (length > 0) {
1238                 u64 sublen = min_t(u64, length, PAGE_SIZE);
1239                 u64 mapped_length = sublen;
1240                 struct btrfs_bio *bbio = NULL;
1241
1242                 /*
1243                  * with a length of PAGE_SIZE, each returned stripe
1244                  * represents one mirror
1245                  */
1246                 ret = btrfs_map_block(fs_info, REQ_GET_READ_MIRRORS, logical,
1247                                       &mapped_length, &bbio, 0);
1248                 if (ret || !bbio || mapped_length < sublen) {
1249                         kfree(bbio);
1250                         return -EIO;
1251                 }
1252
1253                 BUG_ON(page_index >= SCRUB_PAGES_PER_RD_BIO);
1254                 for (mirror_index = 0; mirror_index < (int)bbio->num_stripes;
1255                      mirror_index++) {
1256                         struct scrub_block *sblock;
1257                         struct scrub_page *page;
1258
1259                         if (mirror_index >= BTRFS_MAX_MIRRORS)
1260                                 continue;
1261
1262                         sblock = sblocks_for_recheck + mirror_index;
1263                         sblock->sctx = sctx;
1264                         page = kzalloc(sizeof(*page), GFP_NOFS);
1265                         if (!page) {
1266 leave_nomem:
1267                                 spin_lock(&sctx->stat_lock);
1268                                 sctx->stat.malloc_errors++;
1269                                 spin_unlock(&sctx->stat_lock);
1270                                 kfree(bbio);
1271                                 return -ENOMEM;
1272                         }
1273                         scrub_page_get(page);
1274                         sblock->pagev[page_index] = page;
1275                         page->logical = logical;
1276                         page->physical = bbio->stripes[mirror_index].physical;
1277                         BUG_ON(page_index >= original_sblock->page_count);
1278                         page->physical_for_dev_replace =
1279                                 original_sblock->pagev[page_index]->
1280                                 physical_for_dev_replace;
1281                         /* for missing devices, dev->bdev is NULL */
1282                         page->dev = bbio->stripes[mirror_index].dev;
1283                         page->mirror_num = mirror_index + 1;
1284                         sblock->page_count++;
1285                         page->page = alloc_page(GFP_NOFS);
1286                         if (!page->page)
1287                                 goto leave_nomem;
1288                 }
1289                 kfree(bbio);
1290                 length -= sublen;
1291                 logical += sublen;
1292                 page_index++;
1293         }
1294
1295         return 0;
1296 }
1297
1298 /*
1299  * this function will check the on disk data for checksum errors, header
1300  * errors and read I/O errors. If any I/O errors happen, the exact pages
1301  * which are errored are marked as being bad. The goal is to enable scrub
1302  * to take those pages that are not errored from all the mirrors so that
1303  * the pages that are errored in the just handled mirror can be repaired.
1304  */
1305 static void scrub_recheck_block(struct btrfs_fs_info *fs_info,
1306                                 struct scrub_block *sblock, int is_metadata,
1307                                 int have_csum, u8 *csum, u64 generation,
1308                                 u16 csum_size)
1309 {
1310         int page_num;
1311
1312         sblock->no_io_error_seen = 1;
1313         sblock->header_error = 0;
1314         sblock->checksum_error = 0;
1315
1316         for (page_num = 0; page_num < sblock->page_count; page_num++) {
1317                 struct bio *bio;
1318                 struct scrub_page *page = sblock->pagev[page_num];
1319
1320                 if (page->dev->bdev == NULL) {
1321                         page->io_error = 1;
1322                         sblock->no_io_error_seen = 0;
1323                         continue;
1324                 }
1325
1326                 WARN_ON(!page->page);
1327                 bio = btrfs_io_bio_alloc(GFP_NOFS, 1);
1328                 if (!bio) {
1329                         page->io_error = 1;
1330                         sblock->no_io_error_seen = 0;
1331                         continue;
1332                 }
1333                 bio->bi_bdev = page->dev->bdev;
1334                 bio->bi_sector = page->physical >> 9;
1335
1336                 bio_add_page(bio, page->page, PAGE_SIZE, 0);
1337                 if (btrfsic_submit_bio_wait(READ, bio))
1338                         sblock->no_io_error_seen = 0;
1339
1340                 bio_put(bio);
1341         }
1342
1343         if (sblock->no_io_error_seen)
1344                 scrub_recheck_block_checksum(fs_info, sblock, is_metadata,
1345                                              have_csum, csum, generation,
1346                                              csum_size);
1347
1348         return;
1349 }
1350
1351 static void scrub_recheck_block_checksum(struct btrfs_fs_info *fs_info,
1352                                          struct scrub_block *sblock,
1353                                          int is_metadata, int have_csum,
1354                                          const u8 *csum, u64 generation,
1355                                          u16 csum_size)
1356 {
1357         int page_num;
1358         u8 calculated_csum[BTRFS_CSUM_SIZE];
1359         u32 crc = ~(u32)0;
1360         void *mapped_buffer;
1361
1362         WARN_ON(!sblock->pagev[0]->page);
1363         if (is_metadata) {
1364                 struct btrfs_header *h;
1365
1366                 mapped_buffer = kmap_atomic(sblock->pagev[0]->page);
1367                 h = (struct btrfs_header *)mapped_buffer;
1368
1369                 if (sblock->pagev[0]->logical != btrfs_stack_header_bytenr(h) ||
1370                     memcmp(h->fsid, fs_info->fsid, BTRFS_UUID_SIZE) ||
1371                     memcmp(h->chunk_tree_uuid, fs_info->chunk_tree_uuid,
1372                            BTRFS_UUID_SIZE)) {
1373                         sblock->header_error = 1;
1374                 } else if (generation != btrfs_stack_header_generation(h)) {
1375                         sblock->header_error = 1;
1376                         sblock->generation_error = 1;
1377                 }
1378                 csum = h->csum;
1379         } else {
1380                 if (!have_csum)
1381                         return;
1382
1383                 mapped_buffer = kmap_atomic(sblock->pagev[0]->page);
1384         }
1385
1386         for (page_num = 0;;) {
1387                 if (page_num == 0 && is_metadata)
1388                         crc = btrfs_csum_data(
1389                                 ((u8 *)mapped_buffer) + BTRFS_CSUM_SIZE,
1390                                 crc, PAGE_SIZE - BTRFS_CSUM_SIZE);
1391                 else
1392                         crc = btrfs_csum_data(mapped_buffer, crc, PAGE_SIZE);
1393
1394                 kunmap_atomic(mapped_buffer);
1395                 page_num++;
1396                 if (page_num >= sblock->page_count)
1397                         break;
1398                 WARN_ON(!sblock->pagev[page_num]->page);
1399
1400                 mapped_buffer = kmap_atomic(sblock->pagev[page_num]->page);
1401         }
1402
1403         btrfs_csum_final(crc, calculated_csum);
1404         if (memcmp(calculated_csum, csum, csum_size))
1405                 sblock->checksum_error = 1;
1406 }
1407
1408 static int scrub_repair_block_from_good_copy(struct scrub_block *sblock_bad,
1409                                              struct scrub_block *sblock_good,
1410                                              int force_write)
1411 {
1412         int page_num;
1413         int ret = 0;
1414
1415         for (page_num = 0; page_num < sblock_bad->page_count; page_num++) {
1416                 int ret_sub;
1417
1418                 ret_sub = scrub_repair_page_from_good_copy(sblock_bad,
1419                                                            sblock_good,
1420                                                            page_num,
1421                                                            force_write);
1422                 if (ret_sub)
1423                         ret = ret_sub;
1424         }
1425
1426         return ret;
1427 }
1428
1429 static int scrub_repair_page_from_good_copy(struct scrub_block *sblock_bad,
1430                                             struct scrub_block *sblock_good,
1431                                             int page_num, int force_write)
1432 {
1433         struct scrub_page *page_bad = sblock_bad->pagev[page_num];
1434         struct scrub_page *page_good = sblock_good->pagev[page_num];
1435
1436         BUG_ON(page_bad->page == NULL);
1437         BUG_ON(page_good->page == NULL);
1438         if (force_write || sblock_bad->header_error ||
1439             sblock_bad->checksum_error || page_bad->io_error) {
1440                 struct bio *bio;
1441                 int ret;
1442
1443                 if (!page_bad->dev->bdev) {
1444                         printk_ratelimited(KERN_WARNING
1445                                 "btrfs: scrub_repair_page_from_good_copy(bdev == NULL) is unexpected!\n");
1446                         return -EIO;
1447                 }
1448
1449                 bio = btrfs_io_bio_alloc(GFP_NOFS, 1);
1450                 if (!bio)
1451                         return -EIO;
1452                 bio->bi_bdev = page_bad->dev->bdev;
1453                 bio->bi_sector = page_bad->physical >> 9;
1454
1455                 ret = bio_add_page(bio, page_good->page, PAGE_SIZE, 0);
1456                 if (PAGE_SIZE != ret) {
1457                         bio_put(bio);
1458                         return -EIO;
1459                 }
1460
1461                 if (btrfsic_submit_bio_wait(WRITE, bio)) {
1462                         btrfs_dev_stat_inc_and_print(page_bad->dev,
1463                                 BTRFS_DEV_STAT_WRITE_ERRS);
1464                         btrfs_dev_replace_stats_inc(
1465                                 &sblock_bad->sctx->dev_root->fs_info->
1466                                 dev_replace.num_write_errors);
1467                         bio_put(bio);
1468                         return -EIO;
1469                 }
1470                 bio_put(bio);
1471         }
1472
1473         return 0;
1474 }
1475
1476 static void scrub_write_block_to_dev_replace(struct scrub_block *sblock)
1477 {
1478         int page_num;
1479
1480         for (page_num = 0; page_num < sblock->page_count; page_num++) {
1481                 int ret;
1482
1483                 ret = scrub_write_page_to_dev_replace(sblock, page_num);
1484                 if (ret)
1485                         btrfs_dev_replace_stats_inc(
1486                                 &sblock->sctx->dev_root->fs_info->dev_replace.
1487                                 num_write_errors);
1488         }
1489 }
1490
1491 static int scrub_write_page_to_dev_replace(struct scrub_block *sblock,
1492                                            int page_num)
1493 {
1494         struct scrub_page *spage = sblock->pagev[page_num];
1495
1496         BUG_ON(spage->page == NULL);
1497         if (spage->io_error) {
1498                 void *mapped_buffer = kmap_atomic(spage->page);
1499
1500                 memset(mapped_buffer, 0, PAGE_CACHE_SIZE);
1501                 flush_dcache_page(spage->page);
1502                 kunmap_atomic(mapped_buffer);
1503         }
1504         return scrub_add_page_to_wr_bio(sblock->sctx, spage);
1505 }
1506
1507 static int scrub_add_page_to_wr_bio(struct scrub_ctx *sctx,
1508                                     struct scrub_page *spage)
1509 {
1510         struct scrub_wr_ctx *wr_ctx = &sctx->wr_ctx;
1511         struct scrub_bio *sbio;
1512         int ret;
1513
1514         mutex_lock(&wr_ctx->wr_lock);
1515 again:
1516         if (!wr_ctx->wr_curr_bio) {
1517                 wr_ctx->wr_curr_bio = kzalloc(sizeof(*wr_ctx->wr_curr_bio),
1518                                               GFP_NOFS);
1519                 if (!wr_ctx->wr_curr_bio) {
1520                         mutex_unlock(&wr_ctx->wr_lock);
1521                         return -ENOMEM;
1522                 }
1523                 wr_ctx->wr_curr_bio->sctx = sctx;
1524                 wr_ctx->wr_curr_bio->page_count = 0;
1525         }
1526         sbio = wr_ctx->wr_curr_bio;
1527         if (sbio->page_count == 0) {
1528                 struct bio *bio;
1529
1530                 sbio->physical = spage->physical_for_dev_replace;
1531                 sbio->logical = spage->logical;
1532                 sbio->dev = wr_ctx->tgtdev;
1533                 bio = sbio->bio;
1534                 if (!bio) {
1535                         bio = btrfs_io_bio_alloc(GFP_NOFS, wr_ctx->pages_per_wr_bio);
1536                         if (!bio) {
1537                                 mutex_unlock(&wr_ctx->wr_lock);
1538                                 return -ENOMEM;
1539                         }
1540                         sbio->bio = bio;
1541                 }
1542
1543                 bio->bi_private = sbio;
1544                 bio->bi_end_io = scrub_wr_bio_end_io;
1545                 bio->bi_bdev = sbio->dev->bdev;
1546                 bio->bi_sector = sbio->physical >> 9;
1547                 sbio->err = 0;
1548         } else if (sbio->physical + sbio->page_count * PAGE_SIZE !=
1549                    spage->physical_for_dev_replace ||
1550                    sbio->logical + sbio->page_count * PAGE_SIZE !=
1551                    spage->logical) {
1552                 scrub_wr_submit(sctx);
1553                 goto again;
1554         }
1555
1556         ret = bio_add_page(sbio->bio, spage->page, PAGE_SIZE, 0);
1557         if (ret != PAGE_SIZE) {
1558                 if (sbio->page_count < 1) {
1559                         bio_put(sbio->bio);
1560                         sbio->bio = NULL;
1561                         mutex_unlock(&wr_ctx->wr_lock);
1562                         return -EIO;
1563                 }
1564                 scrub_wr_submit(sctx);
1565                 goto again;
1566         }
1567
1568         sbio->pagev[sbio->page_count] = spage;
1569         scrub_page_get(spage);
1570         sbio->page_count++;
1571         if (sbio->page_count == wr_ctx->pages_per_wr_bio)
1572                 scrub_wr_submit(sctx);
1573         mutex_unlock(&wr_ctx->wr_lock);
1574
1575         return 0;
1576 }
1577
1578 static void scrub_wr_submit(struct scrub_ctx *sctx)
1579 {
1580         struct scrub_wr_ctx *wr_ctx = &sctx->wr_ctx;
1581         struct scrub_bio *sbio;
1582
1583         if (!wr_ctx->wr_curr_bio)
1584                 return;
1585
1586         sbio = wr_ctx->wr_curr_bio;
1587         wr_ctx->wr_curr_bio = NULL;
1588         WARN_ON(!sbio->bio->bi_bdev);
1589         scrub_pending_bio_inc(sctx);
1590         /* process all writes in a single worker thread. Then the block layer
1591          * orders the requests before sending them to the driver which
1592          * doubled the write performance on spinning disks when measured
1593          * with Linux 3.5 */
1594         btrfsic_submit_bio(WRITE, sbio->bio);
1595 }
1596
1597 static void scrub_wr_bio_end_io(struct bio *bio, int err)
1598 {
1599         struct scrub_bio *sbio = bio->bi_private;
1600         struct btrfs_fs_info *fs_info = sbio->dev->dev_root->fs_info;
1601
1602         sbio->err = err;
1603         sbio->bio = bio;
1604
1605         sbio->work.func = scrub_wr_bio_end_io_worker;
1606         btrfs_queue_worker(&fs_info->scrub_wr_completion_workers, &sbio->work);
1607 }
1608
1609 static void scrub_wr_bio_end_io_worker(struct btrfs_work *work)
1610 {
1611         struct scrub_bio *sbio = container_of(work, struct scrub_bio, work);
1612         struct scrub_ctx *sctx = sbio->sctx;
1613         int i;
1614
1615         WARN_ON(sbio->page_count > SCRUB_PAGES_PER_WR_BIO);
1616         if (sbio->err) {
1617                 struct btrfs_dev_replace *dev_replace =
1618                         &sbio->sctx->dev_root->fs_info->dev_replace;
1619
1620                 for (i = 0; i < sbio->page_count; i++) {
1621                         struct scrub_page *spage = sbio->pagev[i];
1622
1623                         spage->io_error = 1;
1624                         btrfs_dev_replace_stats_inc(&dev_replace->
1625                                                     num_write_errors);
1626                 }
1627         }
1628
1629         for (i = 0; i < sbio->page_count; i++)
1630                 scrub_page_put(sbio->pagev[i]);
1631
1632         bio_put(sbio->bio);
1633         kfree(sbio);
1634         scrub_pending_bio_dec(sctx);
1635 }
1636
1637 static int scrub_checksum(struct scrub_block *sblock)
1638 {
1639         u64 flags;
1640         int ret;
1641
1642         WARN_ON(sblock->page_count < 1);
1643         flags = sblock->pagev[0]->flags;
1644         ret = 0;
1645         if (flags & BTRFS_EXTENT_FLAG_DATA)
1646                 ret = scrub_checksum_data(sblock);
1647         else if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK)
1648                 ret = scrub_checksum_tree_block(sblock);
1649         else if (flags & BTRFS_EXTENT_FLAG_SUPER)
1650                 (void)scrub_checksum_super(sblock);
1651         else
1652                 WARN_ON(1);
1653         if (ret)
1654                 scrub_handle_errored_block(sblock);
1655
1656         return ret;
1657 }
1658
1659 static int scrub_checksum_data(struct scrub_block *sblock)
1660 {
1661         struct scrub_ctx *sctx = sblock->sctx;
1662         u8 csum[BTRFS_CSUM_SIZE];
1663         u8 *on_disk_csum;
1664         struct page *page;
1665         void *buffer;
1666         u32 crc = ~(u32)0;
1667         int fail = 0;
1668         u64 len;
1669         int index;
1670
1671         BUG_ON(sblock->page_count < 1);
1672         if (!sblock->pagev[0]->have_csum)
1673                 return 0;
1674
1675         on_disk_csum = sblock->pagev[0]->csum;
1676         page = sblock->pagev[0]->page;
1677         buffer = kmap_atomic(page);
1678
1679         len = sctx->sectorsize;
1680         index = 0;
1681         for (;;) {
1682                 u64 l = min_t(u64, len, PAGE_SIZE);
1683
1684                 crc = btrfs_csum_data(buffer, crc, l);
1685                 kunmap_atomic(buffer);
1686                 len -= l;
1687                 if (len == 0)
1688                         break;
1689                 index++;
1690                 BUG_ON(index >= sblock->page_count);
1691                 BUG_ON(!sblock->pagev[index]->page);
1692                 page = sblock->pagev[index]->page;
1693                 buffer = kmap_atomic(page);
1694         }
1695
1696         btrfs_csum_final(crc, csum);
1697         if (memcmp(csum, on_disk_csum, sctx->csum_size))
1698                 fail = 1;
1699
1700         return fail;
1701 }
1702
1703 static int scrub_checksum_tree_block(struct scrub_block *sblock)
1704 {
1705         struct scrub_ctx *sctx = sblock->sctx;
1706         struct btrfs_header *h;
1707         struct btrfs_root *root = sctx->dev_root;
1708         struct btrfs_fs_info *fs_info = root->fs_info;
1709         u8 calculated_csum[BTRFS_CSUM_SIZE];
1710         u8 on_disk_csum[BTRFS_CSUM_SIZE];
1711         struct page *page;
1712         void *mapped_buffer;
1713         u64 mapped_size;
1714         void *p;
1715         u32 crc = ~(u32)0;
1716         int fail = 0;
1717         int crc_fail = 0;
1718         u64 len;
1719         int index;
1720
1721         BUG_ON(sblock->page_count < 1);
1722         page = sblock->pagev[0]->page;
1723         mapped_buffer = kmap_atomic(page);
1724         h = (struct btrfs_header *)mapped_buffer;
1725         memcpy(on_disk_csum, h->csum, sctx->csum_size);
1726
1727         /*
1728          * we don't use the getter functions here, as we
1729          * a) don't have an extent buffer and
1730          * b) the page is already kmapped
1731          */
1732
1733         if (sblock->pagev[0]->logical != btrfs_stack_header_bytenr(h))
1734                 ++fail;
1735
1736         if (sblock->pagev[0]->generation != btrfs_stack_header_generation(h))
1737                 ++fail;
1738
1739         if (memcmp(h->fsid, fs_info->fsid, BTRFS_UUID_SIZE))
1740                 ++fail;
1741
1742         if (memcmp(h->chunk_tree_uuid, fs_info->chunk_tree_uuid,
1743                    BTRFS_UUID_SIZE))
1744                 ++fail;
1745
1746         WARN_ON(sctx->nodesize != sctx->leafsize);
1747         len = sctx->nodesize - BTRFS_CSUM_SIZE;
1748         mapped_size = PAGE_SIZE - BTRFS_CSUM_SIZE;
1749         p = ((u8 *)mapped_buffer) + BTRFS_CSUM_SIZE;
1750         index = 0;
1751         for (;;) {
1752                 u64 l = min_t(u64, len, mapped_size);
1753
1754                 crc = btrfs_csum_data(p, crc, l);
1755                 kunmap_atomic(mapped_buffer);
1756                 len -= l;
1757                 if (len == 0)
1758                         break;
1759                 index++;
1760                 BUG_ON(index >= sblock->page_count);
1761                 BUG_ON(!sblock->pagev[index]->page);
1762                 page = sblock->pagev[index]->page;
1763                 mapped_buffer = kmap_atomic(page);
1764                 mapped_size = PAGE_SIZE;
1765                 p = mapped_buffer;
1766         }
1767
1768         btrfs_csum_final(crc, calculated_csum);
1769         if (memcmp(calculated_csum, on_disk_csum, sctx->csum_size))
1770                 ++crc_fail;
1771
1772         return fail || crc_fail;
1773 }
1774
1775 static int scrub_checksum_super(struct scrub_block *sblock)
1776 {
1777         struct btrfs_super_block *s;
1778         struct scrub_ctx *sctx = sblock->sctx;
1779         struct btrfs_root *root = sctx->dev_root;
1780         struct btrfs_fs_info *fs_info = root->fs_info;
1781         u8 calculated_csum[BTRFS_CSUM_SIZE];
1782         u8 on_disk_csum[BTRFS_CSUM_SIZE];
1783         struct page *page;
1784         void *mapped_buffer;
1785         u64 mapped_size;
1786         void *p;
1787         u32 crc = ~(u32)0;
1788         int fail_gen = 0;
1789         int fail_cor = 0;
1790         u64 len;
1791         int index;
1792
1793         BUG_ON(sblock->page_count < 1);
1794         page = sblock->pagev[0]->page;
1795         mapped_buffer = kmap_atomic(page);
1796         s = (struct btrfs_super_block *)mapped_buffer;
1797         memcpy(on_disk_csum, s->csum, sctx->csum_size);
1798
1799         if (sblock->pagev[0]->logical != btrfs_super_bytenr(s))
1800                 ++fail_cor;
1801
1802         if (sblock->pagev[0]->generation != btrfs_super_generation(s))
1803                 ++fail_gen;
1804
1805         if (memcmp(s->fsid, fs_info->fsid, BTRFS_UUID_SIZE))
1806                 ++fail_cor;
1807
1808         len = BTRFS_SUPER_INFO_SIZE - BTRFS_CSUM_SIZE;
1809         mapped_size = PAGE_SIZE - BTRFS_CSUM_SIZE;
1810         p = ((u8 *)mapped_buffer) + BTRFS_CSUM_SIZE;
1811         index = 0;
1812         for (;;) {
1813                 u64 l = min_t(u64, len, mapped_size);
1814
1815                 crc = btrfs_csum_data(p, crc, l);
1816                 kunmap_atomic(mapped_buffer);
1817                 len -= l;
1818                 if (len == 0)
1819                         break;
1820                 index++;
1821                 BUG_ON(index >= sblock->page_count);
1822                 BUG_ON(!sblock->pagev[index]->page);
1823                 page = sblock->pagev[index]->page;
1824                 mapped_buffer = kmap_atomic(page);
1825                 mapped_size = PAGE_SIZE;
1826                 p = mapped_buffer;
1827         }
1828
1829         btrfs_csum_final(crc, calculated_csum);
1830         if (memcmp(calculated_csum, on_disk_csum, sctx->csum_size))
1831                 ++fail_cor;
1832
1833         if (fail_cor + fail_gen) {
1834                 /*
1835                  * if we find an error in a super block, we just report it.
1836                  * They will get written with the next transaction commit
1837                  * anyway
1838                  */
1839                 spin_lock(&sctx->stat_lock);
1840                 ++sctx->stat.super_errors;
1841                 spin_unlock(&sctx->stat_lock);
1842                 if (fail_cor)
1843                         btrfs_dev_stat_inc_and_print(sblock->pagev[0]->dev,
1844                                 BTRFS_DEV_STAT_CORRUPTION_ERRS);
1845                 else
1846                         btrfs_dev_stat_inc_and_print(sblock->pagev[0]->dev,
1847                                 BTRFS_DEV_STAT_GENERATION_ERRS);
1848         }
1849
1850         return fail_cor + fail_gen;
1851 }
1852
1853 static void scrub_block_get(struct scrub_block *sblock)
1854 {
1855         atomic_inc(&sblock->ref_count);
1856 }
1857
1858 static void scrub_block_put(struct scrub_block *sblock)
1859 {
1860         if (atomic_dec_and_test(&sblock->ref_count)) {
1861                 int i;
1862
1863                 for (i = 0; i < sblock->page_count; i++)
1864                         scrub_page_put(sblock->pagev[i]);
1865                 kfree(sblock);
1866         }
1867 }
1868
1869 static void scrub_page_get(struct scrub_page *spage)
1870 {
1871         atomic_inc(&spage->ref_count);
1872 }
1873
1874 static void scrub_page_put(struct scrub_page *spage)
1875 {
1876         if (atomic_dec_and_test(&spage->ref_count)) {
1877                 if (spage->page)
1878                         __free_page(spage->page);
1879                 kfree(spage);
1880         }
1881 }
1882
1883 static void scrub_submit(struct scrub_ctx *sctx)
1884 {
1885         struct scrub_bio *sbio;
1886
1887         if (sctx->curr == -1)
1888                 return;
1889
1890         sbio = sctx->bios[sctx->curr];
1891         sctx->curr = -1;
1892         scrub_pending_bio_inc(sctx);
1893
1894         if (!sbio->bio->bi_bdev) {
1895                 /*
1896                  * this case should not happen. If btrfs_map_block() is
1897                  * wrong, it could happen for dev-replace operations on
1898                  * missing devices when no mirrors are available, but in
1899                  * this case it should already fail the mount.
1900                  * This case is handled correctly (but _very_ slowly).
1901                  */
1902                 printk_ratelimited(KERN_WARNING
1903                         "btrfs: scrub_submit(bio bdev == NULL) is unexpected!\n");
1904                 bio_endio(sbio->bio, -EIO);
1905         } else {
1906                 btrfsic_submit_bio(READ, sbio->bio);
1907         }
1908 }
1909
1910 static int scrub_add_page_to_rd_bio(struct scrub_ctx *sctx,
1911                                     struct scrub_page *spage)
1912 {
1913         struct scrub_block *sblock = spage->sblock;
1914         struct scrub_bio *sbio;
1915         int ret;
1916
1917 again:
1918         /*
1919          * grab a fresh bio or wait for one to become available
1920          */
1921         while (sctx->curr == -1) {
1922                 spin_lock(&sctx->list_lock);
1923                 sctx->curr = sctx->first_free;
1924                 if (sctx->curr != -1) {
1925                         sctx->first_free = sctx->bios[sctx->curr]->next_free;
1926                         sctx->bios[sctx->curr]->next_free = -1;
1927                         sctx->bios[sctx->curr]->page_count = 0;
1928                         spin_unlock(&sctx->list_lock);
1929                 } else {
1930                         spin_unlock(&sctx->list_lock);
1931                         wait_event(sctx->list_wait, sctx->first_free != -1);
1932                 }
1933         }
1934         sbio = sctx->bios[sctx->curr];
1935         if (sbio->page_count == 0) {
1936                 struct bio *bio;
1937
1938                 sbio->physical = spage->physical;
1939                 sbio->logical = spage->logical;
1940                 sbio->dev = spage->dev;
1941                 bio = sbio->bio;
1942                 if (!bio) {
1943                         bio = btrfs_io_bio_alloc(GFP_NOFS, sctx->pages_per_rd_bio);
1944                         if (!bio)
1945                                 return -ENOMEM;
1946                         sbio->bio = bio;
1947                 }
1948
1949                 bio->bi_private = sbio;
1950                 bio->bi_end_io = scrub_bio_end_io;
1951                 bio->bi_bdev = sbio->dev->bdev;
1952                 bio->bi_sector = sbio->physical >> 9;
1953                 sbio->err = 0;
1954         } else if (sbio->physical + sbio->page_count * PAGE_SIZE !=
1955                    spage->physical ||
1956                    sbio->logical + sbio->page_count * PAGE_SIZE !=
1957                    spage->logical ||
1958                    sbio->dev != spage->dev) {
1959                 scrub_submit(sctx);
1960                 goto again;
1961         }
1962
1963         sbio->pagev[sbio->page_count] = spage;
1964         ret = bio_add_page(sbio->bio, spage->page, PAGE_SIZE, 0);
1965         if (ret != PAGE_SIZE) {
1966                 if (sbio->page_count < 1) {
1967                         bio_put(sbio->bio);
1968                         sbio->bio = NULL;
1969                         return -EIO;
1970                 }
1971                 scrub_submit(sctx);
1972                 goto again;
1973         }
1974
1975         scrub_block_get(sblock); /* one for the page added to the bio */
1976         atomic_inc(&sblock->outstanding_pages);
1977         sbio->page_count++;
1978         if (sbio->page_count == sctx->pages_per_rd_bio)
1979                 scrub_submit(sctx);
1980
1981         return 0;
1982 }
1983
1984 static int scrub_pages(struct scrub_ctx *sctx, u64 logical, u64 len,
1985                        u64 physical, struct btrfs_device *dev, u64 flags,
1986                        u64 gen, int mirror_num, u8 *csum, int force,
1987                        u64 physical_for_dev_replace)
1988 {
1989         struct scrub_block *sblock;
1990         int index;
1991
1992         sblock = kzalloc(sizeof(*sblock), GFP_NOFS);
1993         if (!sblock) {
1994                 spin_lock(&sctx->stat_lock);
1995                 sctx->stat.malloc_errors++;
1996                 spin_unlock(&sctx->stat_lock);
1997                 return -ENOMEM;
1998         }
1999
2000         /* one ref inside this function, plus one for each page added to
2001          * a bio later on */
2002         atomic_set(&sblock->ref_count, 1);
2003         sblock->sctx = sctx;
2004         sblock->no_io_error_seen = 1;
2005
2006         for (index = 0; len > 0; index++) {
2007                 struct scrub_page *spage;
2008                 u64 l = min_t(u64, len, PAGE_SIZE);
2009
2010                 spage = kzalloc(sizeof(*spage), GFP_NOFS);
2011                 if (!spage) {
2012 leave_nomem:
2013                         spin_lock(&sctx->stat_lock);
2014                         sctx->stat.malloc_errors++;
2015                         spin_unlock(&sctx->stat_lock);
2016                         scrub_block_put(sblock);
2017                         return -ENOMEM;
2018                 }
2019                 BUG_ON(index >= SCRUB_MAX_PAGES_PER_BLOCK);
2020                 scrub_page_get(spage);
2021                 sblock->pagev[index] = spage;
2022                 spage->sblock = sblock;
2023                 spage->dev = dev;
2024                 spage->flags = flags;
2025                 spage->generation = gen;
2026                 spage->logical = logical;
2027                 spage->physical = physical;
2028                 spage->physical_for_dev_replace = physical_for_dev_replace;
2029                 spage->mirror_num = mirror_num;
2030                 if (csum) {
2031                         spage->have_csum = 1;
2032                         memcpy(spage->csum, csum, sctx->csum_size);
2033                 } else {
2034                         spage->have_csum = 0;
2035                 }
2036                 sblock->page_count++;
2037                 spage->page = alloc_page(GFP_NOFS);
2038                 if (!spage->page)
2039                         goto leave_nomem;
2040                 len -= l;
2041                 logical += l;
2042                 physical += l;
2043                 physical_for_dev_replace += l;
2044         }
2045
2046         WARN_ON(sblock->page_count == 0);
2047         for (index = 0; index < sblock->page_count; index++) {
2048                 struct scrub_page *spage = sblock->pagev[index];
2049                 int ret;
2050
2051                 ret = scrub_add_page_to_rd_bio(sctx, spage);
2052                 if (ret) {
2053                         scrub_block_put(sblock);
2054                         return ret;
2055                 }
2056         }
2057
2058         if (force)
2059                 scrub_submit(sctx);
2060
2061         /* last one frees, either here or in bio completion for last page */
2062         scrub_block_put(sblock);
2063         return 0;
2064 }
2065
2066 static void scrub_bio_end_io(struct bio *bio, int err)
2067 {
2068         struct scrub_bio *sbio = bio->bi_private;
2069         struct btrfs_fs_info *fs_info = sbio->dev->dev_root->fs_info;
2070
2071         sbio->err = err;
2072         sbio->bio = bio;
2073
2074         btrfs_queue_worker(&fs_info->scrub_workers, &sbio->work);
2075 }
2076
2077 static void scrub_bio_end_io_worker(struct btrfs_work *work)
2078 {
2079         struct scrub_bio *sbio = container_of(work, struct scrub_bio, work);
2080         struct scrub_ctx *sctx = sbio->sctx;
2081         int i;
2082
2083         BUG_ON(sbio->page_count > SCRUB_PAGES_PER_RD_BIO);
2084         if (sbio->err) {
2085                 for (i = 0; i < sbio->page_count; i++) {
2086                         struct scrub_page *spage = sbio->pagev[i];
2087
2088                         spage->io_error = 1;
2089                         spage->sblock->no_io_error_seen = 0;
2090                 }
2091         }
2092
2093         /* now complete the scrub_block items that have all pages completed */
2094         for (i = 0; i < sbio->page_count; i++) {
2095                 struct scrub_page *spage = sbio->pagev[i];
2096                 struct scrub_block *sblock = spage->sblock;
2097
2098                 if (atomic_dec_and_test(&sblock->outstanding_pages))
2099                         scrub_block_complete(sblock);
2100                 scrub_block_put(sblock);
2101         }
2102
2103         bio_put(sbio->bio);
2104         sbio->bio = NULL;
2105         spin_lock(&sctx->list_lock);
2106         sbio->next_free = sctx->first_free;
2107         sctx->first_free = sbio->index;
2108         spin_unlock(&sctx->list_lock);
2109
2110         if (sctx->is_dev_replace &&
2111             atomic_read(&sctx->wr_ctx.flush_all_writes)) {
2112                 mutex_lock(&sctx->wr_ctx.wr_lock);
2113                 scrub_wr_submit(sctx);
2114                 mutex_unlock(&sctx->wr_ctx.wr_lock);
2115         }
2116
2117         scrub_pending_bio_dec(sctx);
2118 }
2119
2120 static void scrub_block_complete(struct scrub_block *sblock)
2121 {
2122         if (!sblock->no_io_error_seen) {
2123                 scrub_handle_errored_block(sblock);
2124         } else {
2125                 /*
2126                  * if has checksum error, write via repair mechanism in
2127                  * dev replace case, otherwise write here in dev replace
2128                  * case.
2129                  */
2130                 if (!scrub_checksum(sblock) && sblock->sctx->is_dev_replace)
2131                         scrub_write_block_to_dev_replace(sblock);
2132         }
2133 }
2134
2135 static int scrub_find_csum(struct scrub_ctx *sctx, u64 logical, u64 len,
2136                            u8 *csum)
2137 {
2138         struct btrfs_ordered_sum *sum = NULL;
2139         unsigned long index;
2140         unsigned long num_sectors;
2141
2142         while (!list_empty(&sctx->csum_list)) {
2143                 sum = list_first_entry(&sctx->csum_list,
2144                                        struct btrfs_ordered_sum, list);
2145                 if (sum->bytenr > logical)
2146                         return 0;
2147                 if (sum->bytenr + sum->len > logical)
2148                         break;
2149
2150                 ++sctx->stat.csum_discards;
2151                 list_del(&sum->list);
2152                 kfree(sum);
2153                 sum = NULL;
2154         }
2155         if (!sum)
2156                 return 0;
2157
2158         index = ((u32)(logical - sum->bytenr)) / sctx->sectorsize;
2159         num_sectors = sum->len / sctx->sectorsize;
2160         memcpy(csum, sum->sums + index, sctx->csum_size);
2161         if (index == num_sectors - 1) {
2162                 list_del(&sum->list);
2163                 kfree(sum);
2164         }
2165         return 1;
2166 }
2167
2168 /* scrub extent tries to collect up to 64 kB for each bio */
2169 static int scrub_extent(struct scrub_ctx *sctx, u64 logical, u64 len,
2170                         u64 physical, struct btrfs_device *dev, u64 flags,
2171                         u64 gen, int mirror_num, u64 physical_for_dev_replace)
2172 {
2173         int ret;
2174         u8 csum[BTRFS_CSUM_SIZE];
2175         u32 blocksize;
2176
2177         if (flags & BTRFS_EXTENT_FLAG_DATA) {
2178                 blocksize = sctx->sectorsize;
2179                 spin_lock(&sctx->stat_lock);
2180                 sctx->stat.data_extents_scrubbed++;
2181                 sctx->stat.data_bytes_scrubbed += len;
2182                 spin_unlock(&sctx->stat_lock);
2183         } else if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {
2184                 WARN_ON(sctx->nodesize != sctx->leafsize);
2185                 blocksize = sctx->nodesize;
2186                 spin_lock(&sctx->stat_lock);
2187                 sctx->stat.tree_extents_scrubbed++;
2188                 sctx->stat.tree_bytes_scrubbed += len;
2189                 spin_unlock(&sctx->stat_lock);
2190         } else {
2191                 blocksize = sctx->sectorsize;
2192                 WARN_ON(1);
2193         }
2194
2195         while (len) {
2196                 u64 l = min_t(u64, len, blocksize);
2197                 int have_csum = 0;
2198
2199                 if (flags & BTRFS_EXTENT_FLAG_DATA) {
2200                         /* push csums to sbio */
2201                         have_csum = scrub_find_csum(sctx, logical, l, csum);
2202                         if (have_csum == 0)
2203                                 ++sctx->stat.no_csum;
2204                         if (sctx->is_dev_replace && !have_csum) {
2205                                 ret = copy_nocow_pages(sctx, logical, l,
2206                                                        mirror_num,
2207                                                       physical_for_dev_replace);
2208                                 goto behind_scrub_pages;
2209                         }
2210                 }
2211                 ret = scrub_pages(sctx, logical, l, physical, dev, flags, gen,
2212                                   mirror_num, have_csum ? csum : NULL, 0,
2213                                   physical_for_dev_replace);
2214 behind_scrub_pages:
2215                 if (ret)
2216                         return ret;
2217                 len -= l;
2218                 logical += l;
2219                 physical += l;
2220                 physical_for_dev_replace += l;
2221         }
2222         return 0;
2223 }
2224
2225 static noinline_for_stack int scrub_stripe(struct scrub_ctx *sctx,
2226                                            struct map_lookup *map,
2227                                            struct btrfs_device *scrub_dev,
2228                                            int num, u64 base, u64 length,
2229                                            int is_dev_replace)
2230 {
2231         struct btrfs_path *path;
2232         struct btrfs_fs_info *fs_info = sctx->dev_root->fs_info;
2233         struct btrfs_root *root = fs_info->extent_root;
2234         struct btrfs_root *csum_root = fs_info->csum_root;
2235         struct btrfs_extent_item *extent;
2236         struct blk_plug plug;
2237         u64 flags;
2238         int ret;
2239         int slot;
2240         u64 nstripes;
2241         struct extent_buffer *l;
2242         struct btrfs_key key;
2243         u64 physical;
2244         u64 logical;
2245         u64 logic_end;
2246         u64 generation;
2247         int mirror_num;
2248         struct reada_control *reada1;
2249         struct reada_control *reada2;
2250         struct btrfs_key key_start;
2251         struct btrfs_key key_end;
2252         u64 increment = map->stripe_len;
2253         u64 offset;
2254         u64 extent_logical;
2255         u64 extent_physical;
2256         u64 extent_len;
2257         struct btrfs_device *extent_dev;
2258         int extent_mirror_num;
2259         int stop_loop;
2260
2261         if (map->type & (BTRFS_BLOCK_GROUP_RAID5 |
2262                          BTRFS_BLOCK_GROUP_RAID6)) {
2263                 if (num >= nr_data_stripes(map)) {
2264                         return 0;
2265                 }
2266         }
2267
2268         nstripes = length;
2269         offset = 0;
2270         do_div(nstripes, map->stripe_len);
2271         if (map->type & BTRFS_BLOCK_GROUP_RAID0) {
2272                 offset = map->stripe_len * num;
2273                 increment = map->stripe_len * map->num_stripes;
2274                 mirror_num = 1;
2275         } else if (map->type & BTRFS_BLOCK_GROUP_RAID10) {
2276                 int factor = map->num_stripes / map->sub_stripes;
2277                 offset = map->stripe_len * (num / map->sub_stripes);
2278                 increment = map->stripe_len * factor;
2279                 mirror_num = num % map->sub_stripes + 1;
2280         } else if (map->type & BTRFS_BLOCK_GROUP_RAID1) {
2281                 increment = map->stripe_len;
2282                 mirror_num = num % map->num_stripes + 1;
2283         } else if (map->type & BTRFS_BLOCK_GROUP_DUP) {
2284                 increment = map->stripe_len;
2285                 mirror_num = num % map->num_stripes + 1;
2286         } else {
2287                 increment = map->stripe_len;
2288                 mirror_num = 1;
2289         }
2290
2291         path = btrfs_alloc_path();
2292         if (!path)
2293                 return -ENOMEM;
2294
2295         /*
2296          * work on commit root. The related disk blocks are static as
2297          * long as COW is applied. This means, it is save to rewrite
2298          * them to repair disk errors without any race conditions
2299          */
2300         path->search_commit_root = 1;
2301         path->skip_locking = 1;
2302
2303         /*
2304          * trigger the readahead for extent tree csum tree and wait for
2305          * completion. During readahead, the scrub is officially paused
2306          * to not hold off transaction commits
2307          */
2308         logical = base + offset;
2309
2310         wait_event(sctx->list_wait,
2311                    atomic_read(&sctx->bios_in_flight) == 0);
2312         scrub_blocked_if_needed(fs_info);
2313
2314         /* FIXME it might be better to start readahead at commit root */
2315         key_start.objectid = logical;
2316         key_start.type = BTRFS_EXTENT_ITEM_KEY;
2317         key_start.offset = (u64)0;
2318         key_end.objectid = base + offset + nstripes * increment;
2319         key_end.type = BTRFS_METADATA_ITEM_KEY;
2320         key_end.offset = (u64)-1;
2321         reada1 = btrfs_reada_add(root, &key_start, &key_end);
2322
2323         key_start.objectid = BTRFS_EXTENT_CSUM_OBJECTID;
2324         key_start.type = BTRFS_EXTENT_CSUM_KEY;
2325         key_start.offset = logical;
2326         key_end.objectid = BTRFS_EXTENT_CSUM_OBJECTID;
2327         key_end.type = BTRFS_EXTENT_CSUM_KEY;
2328         key_end.offset = base + offset + nstripes * increment;
2329         reada2 = btrfs_reada_add(csum_root, &key_start, &key_end);
2330
2331         if (!IS_ERR(reada1))
2332                 btrfs_reada_wait(reada1);
2333         if (!IS_ERR(reada2))
2334                 btrfs_reada_wait(reada2);
2335
2336
2337         /*
2338          * collect all data csums for the stripe to avoid seeking during
2339          * the scrub. This might currently (crc32) end up to be about 1MB
2340          */
2341         blk_start_plug(&plug);
2342
2343         /*
2344          * now find all extents for each stripe and scrub them
2345          */
2346         logical = base + offset;
2347         physical = map->stripes[num].physical;
2348         logic_end = logical + increment * nstripes;
2349         ret = 0;
2350         while (logical < logic_end) {
2351                 /*
2352                  * canceled?
2353                  */
2354                 if (atomic_read(&fs_info->scrub_cancel_req) ||
2355                     atomic_read(&sctx->cancel_req)) {
2356                         ret = -ECANCELED;
2357                         goto out;
2358                 }
2359                 /*
2360                  * check to see if we have to pause
2361                  */
2362                 if (atomic_read(&fs_info->scrub_pause_req)) {
2363                         /* push queued extents */
2364                         atomic_set(&sctx->wr_ctx.flush_all_writes, 1);
2365                         scrub_submit(sctx);
2366                         mutex_lock(&sctx->wr_ctx.wr_lock);
2367                         scrub_wr_submit(sctx);
2368                         mutex_unlock(&sctx->wr_ctx.wr_lock);
2369                         wait_event(sctx->list_wait,
2370                                    atomic_read(&sctx->bios_in_flight) == 0);
2371                         atomic_set(&sctx->wr_ctx.flush_all_writes, 0);
2372                         scrub_blocked_if_needed(fs_info);
2373                 }
2374
2375                 key.objectid = logical;
2376                 key.type = BTRFS_EXTENT_ITEM_KEY;
2377                 key.offset = (u64)-1;
2378
2379                 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
2380                 if (ret < 0)
2381                         goto out;
2382
2383                 if (ret > 0) {
2384                         ret = btrfs_previous_item(root, path, 0,
2385                                                   BTRFS_EXTENT_ITEM_KEY);
2386                         if (ret < 0)
2387                                 goto out;
2388                         if (ret > 0) {
2389                                 /* there's no smaller item, so stick with the
2390                                  * larger one */
2391                                 btrfs_release_path(path);
2392                                 ret = btrfs_search_slot(NULL, root, &key,
2393                                                         path, 0, 0);
2394                                 if (ret < 0)
2395                                         goto out;
2396                         }
2397                 }
2398
2399                 stop_loop = 0;
2400                 while (1) {
2401                         u64 bytes;
2402
2403                         l = path->nodes[0];
2404                         slot = path->slots[0];
2405                         if (slot >= btrfs_header_nritems(l)) {
2406                                 ret = btrfs_next_leaf(root, path);
2407                                 if (ret == 0)
2408                                         continue;
2409                                 if (ret < 0)
2410                                         goto out;
2411
2412                                 stop_loop = 1;
2413                                 break;
2414                         }
2415                         btrfs_item_key_to_cpu(l, &key, slot);
2416
2417                         if (key.type == BTRFS_METADATA_ITEM_KEY)
2418                                 bytes = root->leafsize;
2419                         else
2420                                 bytes = key.offset;
2421
2422                         if (key.objectid + bytes <= logical)
2423                                 goto next;
2424
2425                         if (key.type != BTRFS_EXTENT_ITEM_KEY &&
2426                             key.type != BTRFS_METADATA_ITEM_KEY)
2427                                 goto next;
2428
2429                         if (key.objectid >= logical + map->stripe_len) {
2430                                 /* out of this device extent */
2431                                 if (key.objectid >= logic_end)
2432                                         stop_loop = 1;
2433                                 break;
2434                         }
2435
2436                         extent = btrfs_item_ptr(l, slot,
2437                                                 struct btrfs_extent_item);
2438                         flags = btrfs_extent_flags(l, extent);
2439                         generation = btrfs_extent_generation(l, extent);
2440
2441                         if (key.objectid < logical &&
2442                             (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK)) {
2443                                 printk(KERN_ERR
2444                                        "btrfs scrub: tree block %llu spanning "
2445                                        "stripes, ignored. logical=%llu\n",
2446                                        key.objectid, logical);
2447                                 goto next;
2448                         }
2449
2450 again:
2451                         extent_logical = key.objectid;
2452                         extent_len = bytes;
2453
2454                         /*
2455                          * trim extent to this stripe
2456                          */
2457                         if (extent_logical < logical) {
2458                                 extent_len -= logical - extent_logical;
2459                                 extent_logical = logical;
2460                         }
2461                         if (extent_logical + extent_len >
2462                             logical + map->stripe_len) {
2463                                 extent_len = logical + map->stripe_len -
2464                                              extent_logical;
2465                         }
2466
2467                         extent_physical = extent_logical - logical + physical;
2468                         extent_dev = scrub_dev;
2469                         extent_mirror_num = mirror_num;
2470                         if (is_dev_replace)
2471                                 scrub_remap_extent(fs_info, extent_logical,
2472                                                    extent_len, &extent_physical,
2473                                                    &extent_dev,
2474                                                    &extent_mirror_num);
2475
2476                         ret = btrfs_lookup_csums_range(csum_root, logical,
2477                                                 logical + map->stripe_len - 1,
2478                                                 &sctx->csum_list, 1);
2479                         if (ret)
2480                                 goto out;
2481
2482                         ret = scrub_extent(sctx, extent_logical, extent_len,
2483                                            extent_physical, extent_dev, flags,
2484                                            generation, extent_mirror_num,
2485                                            extent_logical - logical + physical);
2486                         if (ret)
2487                                 goto out;
2488
2489                         scrub_free_csums(sctx);
2490                         if (extent_logical + extent_len <
2491                             key.objectid + bytes) {
2492                                 logical += increment;
2493                                 physical += map->stripe_len;
2494
2495                                 if (logical < key.objectid + bytes) {
2496                                         cond_resched();
2497                                         goto again;
2498                                 }
2499
2500                                 if (logical >= logic_end) {
2501                                         stop_loop = 1;
2502                                         break;
2503                                 }
2504                         }
2505 next:
2506                         path->slots[0]++;
2507                 }
2508                 btrfs_release_path(path);
2509                 logical += increment;
2510                 physical += map->stripe_len;
2511                 spin_lock(&sctx->stat_lock);
2512                 if (stop_loop)
2513                         sctx->stat.last_physical = map->stripes[num].physical +
2514                                                    length;
2515                 else
2516                         sctx->stat.last_physical = physical;
2517                 spin_unlock(&sctx->stat_lock);
2518                 if (stop_loop)
2519                         break;
2520         }
2521 out:
2522         /* push queued extents */
2523         scrub_submit(sctx);
2524         mutex_lock(&sctx->wr_ctx.wr_lock);
2525         scrub_wr_submit(sctx);
2526         mutex_unlock(&sctx->wr_ctx.wr_lock);
2527
2528         blk_finish_plug(&plug);
2529         btrfs_free_path(path);
2530         return ret < 0 ? ret : 0;
2531 }
2532
2533 static noinline_for_stack int scrub_chunk(struct scrub_ctx *sctx,
2534                                           struct btrfs_device *scrub_dev,
2535                                           u64 chunk_tree, u64 chunk_objectid,
2536                                           u64 chunk_offset, u64 length,
2537                                           u64 dev_offset, int is_dev_replace)
2538 {
2539         struct btrfs_mapping_tree *map_tree =
2540                 &sctx->dev_root->fs_info->mapping_tree;
2541         struct map_lookup *map;
2542         struct extent_map *em;
2543         int i;
2544         int ret = 0;
2545
2546         read_lock(&map_tree->map_tree.lock);
2547         em = lookup_extent_mapping(&map_tree->map_tree, chunk_offset, 1);
2548         read_unlock(&map_tree->map_tree.lock);
2549
2550         if (!em)
2551                 return -EINVAL;
2552
2553         map = (struct map_lookup *)em->bdev;
2554         if (em->start != chunk_offset)
2555                 goto out;
2556
2557         if (em->len < length)
2558                 goto out;
2559
2560         for (i = 0; i < map->num_stripes; ++i) {
2561                 if (map->stripes[i].dev->bdev == scrub_dev->bdev &&
2562                     map->stripes[i].physical == dev_offset) {
2563                         ret = scrub_stripe(sctx, map, scrub_dev, i,
2564                                            chunk_offset, length,
2565                                            is_dev_replace);
2566                         if (ret)
2567                                 goto out;
2568                 }
2569         }
2570 out:
2571         free_extent_map(em);
2572
2573         return ret;
2574 }
2575
2576 static noinline_for_stack
2577 int scrub_enumerate_chunks(struct scrub_ctx *sctx,
2578                            struct btrfs_device *scrub_dev, u64 start, u64 end,
2579                            int is_dev_replace)
2580 {
2581         struct btrfs_dev_extent *dev_extent = NULL;
2582         struct btrfs_path *path;
2583         struct btrfs_root *root = sctx->dev_root;
2584         struct btrfs_fs_info *fs_info = root->fs_info;
2585         u64 length;
2586         u64 chunk_tree;
2587         u64 chunk_objectid;
2588         u64 chunk_offset;
2589         int ret;
2590         int slot;
2591         struct extent_buffer *l;
2592         struct btrfs_key key;
2593         struct btrfs_key found_key;
2594         struct btrfs_block_group_cache *cache;
2595         struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
2596
2597         path = btrfs_alloc_path();
2598         if (!path)
2599                 return -ENOMEM;
2600
2601         path->reada = 2;
2602         path->search_commit_root = 1;
2603         path->skip_locking = 1;
2604
2605         key.objectid = scrub_dev->devid;
2606         key.offset = 0ull;
2607         key.type = BTRFS_DEV_EXTENT_KEY;
2608
2609         while (1) {
2610                 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
2611                 if (ret < 0)
2612                         break;
2613                 if (ret > 0) {
2614                         if (path->slots[0] >=
2615                             btrfs_header_nritems(path->nodes[0])) {
2616                                 ret = btrfs_next_leaf(root, path);
2617                                 if (ret)
2618                                         break;
2619                         }
2620                 }
2621
2622                 l = path->nodes[0];
2623                 slot = path->slots[0];
2624
2625                 btrfs_item_key_to_cpu(l, &found_key, slot);
2626
2627                 if (found_key.objectid != scrub_dev->devid)
2628                         break;
2629
2630                 if (btrfs_key_type(&found_key) != BTRFS_DEV_EXTENT_KEY)
2631                         break;
2632
2633                 if (found_key.offset >= end)
2634                         break;
2635
2636                 if (found_key.offset < key.offset)
2637                         break;
2638
2639                 dev_extent = btrfs_item_ptr(l, slot, struct btrfs_dev_extent);
2640                 length = btrfs_dev_extent_length(l, dev_extent);
2641
2642                 if (found_key.offset + length <= start) {
2643                         key.offset = found_key.offset + length;
2644                         btrfs_release_path(path);
2645                         continue;
2646                 }
2647
2648                 chunk_tree = btrfs_dev_extent_chunk_tree(l, dev_extent);
2649                 chunk_objectid = btrfs_dev_extent_chunk_objectid(l, dev_extent);
2650                 chunk_offset = btrfs_dev_extent_chunk_offset(l, dev_extent);
2651
2652                 /*
2653                  * get a reference on the corresponding block group to prevent
2654                  * the chunk from going away while we scrub it
2655                  */
2656                 cache = btrfs_lookup_block_group(fs_info, chunk_offset);
2657                 if (!cache) {
2658                         ret = -ENOENT;
2659                         break;
2660                 }
2661                 dev_replace->cursor_right = found_key.offset + length;
2662                 dev_replace->cursor_left = found_key.offset;
2663                 dev_replace->item_needs_writeback = 1;
2664                 ret = scrub_chunk(sctx, scrub_dev, chunk_tree, chunk_objectid,
2665                                   chunk_offset, length, found_key.offset,
2666                                   is_dev_replace);
2667
2668                 /*
2669                  * flush, submit all pending read and write bios, afterwards
2670                  * wait for them.
2671                  * Note that in the dev replace case, a read request causes
2672                  * write requests that are submitted in the read completion
2673                  * worker. Therefore in the current situation, it is required
2674                  * that all write requests are flushed, so that all read and
2675                  * write requests are really completed when bios_in_flight
2676                  * changes to 0.
2677                  */
2678                 atomic_set(&sctx->wr_ctx.flush_all_writes, 1);
2679                 scrub_submit(sctx);
2680                 mutex_lock(&sctx->wr_ctx.wr_lock);
2681                 scrub_wr_submit(sctx);
2682                 mutex_unlock(&sctx->wr_ctx.wr_lock);
2683
2684                 wait_event(sctx->list_wait,
2685                            atomic_read(&sctx->bios_in_flight) == 0);
2686                 atomic_set(&sctx->wr_ctx.flush_all_writes, 0);
2687                 wait_event(sctx->list_wait,
2688                            atomic_read(&sctx->workers_pending) == 0);
2689                 scrub_blocked_if_needed(fs_info);
2690
2691                 btrfs_put_block_group(cache);
2692                 if (ret)
2693                         break;
2694                 if (is_dev_replace &&
2695                     atomic64_read(&dev_replace->num_write_errors) > 0) {
2696                         ret = -EIO;
2697                         break;
2698                 }
2699                 if (sctx->stat.malloc_errors > 0) {
2700                         ret = -ENOMEM;
2701                         break;
2702                 }
2703
2704                 dev_replace->cursor_left = dev_replace->cursor_right;
2705                 dev_replace->item_needs_writeback = 1;
2706
2707                 key.offset = found_key.offset + length;
2708                 btrfs_release_path(path);
2709         }
2710
2711         btrfs_free_path(path);
2712
2713         /*
2714          * ret can still be 1 from search_slot or next_leaf,
2715          * that's not an error
2716          */
2717         return ret < 0 ? ret : 0;
2718 }
2719
2720 static noinline_for_stack int scrub_supers(struct scrub_ctx *sctx,
2721                                            struct btrfs_device *scrub_dev)
2722 {
2723         int     i;
2724         u64     bytenr;
2725         u64     gen;
2726         int     ret;
2727         struct btrfs_root *root = sctx->dev_root;
2728
2729         if (test_bit(BTRFS_FS_STATE_ERROR, &root->fs_info->fs_state))
2730                 return -EIO;
2731
2732         gen = root->fs_info->last_trans_committed;
2733
2734         for (i = 0; i < BTRFS_SUPER_MIRROR_MAX; i++) {
2735                 bytenr = btrfs_sb_offset(i);
2736                 if (bytenr + BTRFS_SUPER_INFO_SIZE > scrub_dev->total_bytes)
2737                         break;
2738
2739                 ret = scrub_pages(sctx, bytenr, BTRFS_SUPER_INFO_SIZE, bytenr,
2740                                   scrub_dev, BTRFS_EXTENT_FLAG_SUPER, gen, i,
2741                                   NULL, 1, bytenr);
2742                 if (ret)
2743                         return ret;
2744         }
2745         wait_event(sctx->list_wait, atomic_read(&sctx->bios_in_flight) == 0);
2746
2747         return 0;
2748 }
2749
2750 /*
2751  * get a reference count on fs_info->scrub_workers. start worker if necessary
2752  */
2753 static noinline_for_stack int scrub_workers_get(struct btrfs_fs_info *fs_info,
2754                                                 int is_dev_replace)
2755 {
2756         int ret = 0;
2757
2758         if (fs_info->scrub_workers_refcnt == 0) {
2759                 if (is_dev_replace)
2760                         btrfs_init_workers(&fs_info->scrub_workers, "scrub", 1,
2761                                         &fs_info->generic_worker);
2762                 else
2763                         btrfs_init_workers(&fs_info->scrub_workers, "scrub",
2764                                         fs_info->thread_pool_size,
2765                                         &fs_info->generic_worker);
2766                 fs_info->scrub_workers.idle_thresh = 4;
2767                 ret = btrfs_start_workers(&fs_info->scrub_workers);
2768                 if (ret)
2769                         goto out;
2770                 btrfs_init_workers(&fs_info->scrub_wr_completion_workers,
2771                                    "scrubwrc",
2772                                    fs_info->thread_pool_size,
2773                                    &fs_info->generic_worker);
2774                 fs_info->scrub_wr_completion_workers.idle_thresh = 2;
2775                 ret = btrfs_start_workers(
2776                                 &fs_info->scrub_wr_completion_workers);
2777                 if (ret)
2778                         goto out;
2779                 btrfs_init_workers(&fs_info->scrub_nocow_workers, "scrubnc", 1,
2780                                    &fs_info->generic_worker);
2781                 ret = btrfs_start_workers(&fs_info->scrub_nocow_workers);
2782                 if (ret)
2783                         goto out;
2784         }
2785         ++fs_info->scrub_workers_refcnt;
2786 out:
2787         return ret;
2788 }
2789
2790 static noinline_for_stack void scrub_workers_put(struct btrfs_fs_info *fs_info)
2791 {
2792         if (--fs_info->scrub_workers_refcnt == 0) {
2793                 btrfs_stop_workers(&fs_info->scrub_workers);
2794                 btrfs_stop_workers(&fs_info->scrub_wr_completion_workers);
2795                 btrfs_stop_workers(&fs_info->scrub_nocow_workers);
2796         }
2797         WARN_ON(fs_info->scrub_workers_refcnt < 0);
2798 }
2799
2800 int btrfs_scrub_dev(struct btrfs_fs_info *fs_info, u64 devid, u64 start,
2801                     u64 end, struct btrfs_scrub_progress *progress,
2802                     int readonly, int is_dev_replace)
2803 {
2804         struct scrub_ctx *sctx;
2805         int ret;
2806         struct btrfs_device *dev;
2807
2808         if (btrfs_fs_closing(fs_info))
2809                 return -EINVAL;
2810
2811         /*
2812          * check some assumptions
2813          */
2814         if (fs_info->chunk_root->nodesize != fs_info->chunk_root->leafsize) {
2815                 printk(KERN_ERR
2816                        "btrfs_scrub: size assumption nodesize == leafsize (%d == %d) fails\n",
2817                        fs_info->chunk_root->nodesize,
2818                        fs_info->chunk_root->leafsize);
2819                 return -EINVAL;
2820         }
2821
2822         if (fs_info->chunk_root->nodesize > BTRFS_STRIPE_LEN) {
2823                 /*
2824                  * in this case scrub is unable to calculate the checksum
2825                  * the way scrub is implemented. Do not handle this
2826                  * situation at all because it won't ever happen.
2827                  */
2828                 printk(KERN_ERR
2829                        "btrfs_scrub: size assumption nodesize <= BTRFS_STRIPE_LEN (%d <= %d) fails\n",
2830                        fs_info->chunk_root->nodesize, BTRFS_STRIPE_LEN);
2831                 return -EINVAL;
2832         }
2833
2834         if (fs_info->chunk_root->sectorsize != PAGE_SIZE) {
2835                 /* not supported for data w/o checksums */
2836                 printk(KERN_ERR
2837                        "btrfs_scrub: size assumption sectorsize != PAGE_SIZE (%d != %lu) fails\n",
2838                        fs_info->chunk_root->sectorsize, PAGE_SIZE);
2839                 return -EINVAL;
2840         }
2841
2842         if (fs_info->chunk_root->nodesize >
2843             PAGE_SIZE * SCRUB_MAX_PAGES_PER_BLOCK ||
2844             fs_info->chunk_root->sectorsize >
2845             PAGE_SIZE * SCRUB_MAX_PAGES_PER_BLOCK) {
2846                 /*
2847                  * would exhaust the array bounds of pagev member in
2848                  * struct scrub_block
2849                  */
2850                 pr_err("btrfs_scrub: size assumption nodesize and sectorsize <= SCRUB_MAX_PAGES_PER_BLOCK (%d <= %d && %d <= %d) fails\n",
2851                        fs_info->chunk_root->nodesize,
2852                        SCRUB_MAX_PAGES_PER_BLOCK,
2853                        fs_info->chunk_root->sectorsize,
2854                        SCRUB_MAX_PAGES_PER_BLOCK);
2855                 return -EINVAL;
2856         }
2857
2858
2859         mutex_lock(&fs_info->fs_devices->device_list_mutex);
2860         dev = btrfs_find_device(fs_info, devid, NULL, NULL);
2861         if (!dev || (dev->missing && !is_dev_replace)) {
2862                 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
2863                 return -ENODEV;
2864         }
2865
2866         mutex_lock(&fs_info->scrub_lock);
2867         if (!dev->in_fs_metadata || dev->is_tgtdev_for_dev_replace) {
2868                 mutex_unlock(&fs_info->scrub_lock);
2869                 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
2870                 return -EIO;
2871         }
2872
2873         btrfs_dev_replace_lock(&fs_info->dev_replace);
2874         if (dev->scrub_device ||
2875             (!is_dev_replace &&
2876              btrfs_dev_replace_is_ongoing(&fs_info->dev_replace))) {
2877                 btrfs_dev_replace_unlock(&fs_info->dev_replace);
2878                 mutex_unlock(&fs_info->scrub_lock);
2879                 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
2880                 return -EINPROGRESS;
2881         }
2882         btrfs_dev_replace_unlock(&fs_info->dev_replace);
2883
2884         ret = scrub_workers_get(fs_info, is_dev_replace);
2885         if (ret) {
2886                 mutex_unlock(&fs_info->scrub_lock);
2887                 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
2888                 return ret;
2889         }
2890
2891         sctx = scrub_setup_ctx(dev, is_dev_replace);
2892         if (IS_ERR(sctx)) {
2893                 mutex_unlock(&fs_info->scrub_lock);
2894                 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
2895                 scrub_workers_put(fs_info);
2896                 return PTR_ERR(sctx);
2897         }
2898         sctx->readonly = readonly;
2899         dev->scrub_device = sctx;
2900         mutex_unlock(&fs_info->fs_devices->device_list_mutex);
2901
2902         /*
2903          * checking @scrub_pause_req here, we can avoid
2904          * race between committing transaction and scrubbing.
2905          */
2906         __scrub_blocked_if_needed(fs_info);
2907         atomic_inc(&fs_info->scrubs_running);
2908         mutex_unlock(&fs_info->scrub_lock);
2909
2910         if (!is_dev_replace) {
2911                 /*
2912                  * by holding device list mutex, we can
2913                  * kick off writing super in log tree sync.
2914                  */
2915                 mutex_lock(&fs_info->fs_devices->device_list_mutex);
2916                 ret = scrub_supers(sctx, dev);
2917                 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
2918         }
2919
2920         if (!ret)
2921                 ret = scrub_enumerate_chunks(sctx, dev, start, end,
2922                                              is_dev_replace);
2923
2924         wait_event(sctx->list_wait, atomic_read(&sctx->bios_in_flight) == 0);
2925         atomic_dec(&fs_info->scrubs_running);
2926         wake_up(&fs_info->scrub_pause_wait);
2927
2928         wait_event(sctx->list_wait, atomic_read(&sctx->workers_pending) == 0);
2929
2930         if (progress)
2931                 memcpy(progress, &sctx->stat, sizeof(*progress));
2932
2933         mutex_lock(&fs_info->scrub_lock);
2934         dev->scrub_device = NULL;
2935         scrub_workers_put(fs_info);
2936         mutex_unlock(&fs_info->scrub_lock);
2937
2938         scrub_free_ctx(sctx);
2939
2940         return ret;
2941 }
2942
2943 void btrfs_scrub_pause(struct btrfs_root *root)
2944 {
2945         struct btrfs_fs_info *fs_info = root->fs_info;
2946
2947         mutex_lock(&fs_info->scrub_lock);
2948         atomic_inc(&fs_info->scrub_pause_req);
2949         while (atomic_read(&fs_info->scrubs_paused) !=
2950                atomic_read(&fs_info->scrubs_running)) {
2951                 mutex_unlock(&fs_info->scrub_lock);
2952                 wait_event(fs_info->scrub_pause_wait,
2953                            atomic_read(&fs_info->scrubs_paused) ==
2954                            atomic_read(&fs_info->scrubs_running));
2955                 mutex_lock(&fs_info->scrub_lock);
2956         }
2957         mutex_unlock(&fs_info->scrub_lock);
2958 }
2959
2960 void btrfs_scrub_continue(struct btrfs_root *root)
2961 {
2962         struct btrfs_fs_info *fs_info = root->fs_info;
2963
2964         atomic_dec(&fs_info->scrub_pause_req);
2965         wake_up(&fs_info->scrub_pause_wait);
2966 }
2967
2968 int btrfs_scrub_cancel(struct btrfs_fs_info *fs_info)
2969 {
2970         mutex_lock(&fs_info->scrub_lock);
2971         if (!atomic_read(&fs_info->scrubs_running)) {
2972                 mutex_unlock(&fs_info->scrub_lock);
2973                 return -ENOTCONN;
2974         }
2975
2976         atomic_inc(&fs_info->scrub_cancel_req);
2977         while (atomic_read(&fs_info->scrubs_running)) {
2978                 mutex_unlock(&fs_info->scrub_lock);
2979                 wait_event(fs_info->scrub_pause_wait,
2980                            atomic_read(&fs_info->scrubs_running) == 0);
2981                 mutex_lock(&fs_info->scrub_lock);
2982         }
2983         atomic_dec(&fs_info->scrub_cancel_req);
2984         mutex_unlock(&fs_info->scrub_lock);
2985
2986         return 0;
2987 }
2988
2989 int btrfs_scrub_cancel_dev(struct btrfs_fs_info *fs_info,
2990                            struct btrfs_device *dev)
2991 {
2992         struct scrub_ctx *sctx;
2993
2994         mutex_lock(&fs_info->scrub_lock);
2995         sctx = dev->scrub_device;
2996         if (!sctx) {
2997                 mutex_unlock(&fs_info->scrub_lock);
2998                 return -ENOTCONN;
2999         }
3000         atomic_inc(&sctx->cancel_req);
3001         while (dev->scrub_device) {
3002                 mutex_unlock(&fs_info->scrub_lock);
3003                 wait_event(fs_info->scrub_pause_wait,
3004                            dev->scrub_device == NULL);
3005                 mutex_lock(&fs_info->scrub_lock);
3006         }
3007         mutex_unlock(&fs_info->scrub_lock);
3008
3009         return 0;
3010 }
3011
3012 int btrfs_scrub_progress(struct btrfs_root *root, u64 devid,
3013                          struct btrfs_scrub_progress *progress)
3014 {
3015         struct btrfs_device *dev;
3016         struct scrub_ctx *sctx = NULL;
3017
3018         mutex_lock(&root->fs_info->fs_devices->device_list_mutex);
3019         dev = btrfs_find_device(root->fs_info, devid, NULL, NULL);
3020         if (dev)
3021                 sctx = dev->scrub_device;
3022         if (sctx)
3023                 memcpy(progress, &sctx->stat, sizeof(*progress));
3024         mutex_unlock(&root->fs_info->fs_devices->device_list_mutex);
3025
3026         return dev ? (sctx ? 0 : -ENOTCONN) : -ENODEV;
3027 }
3028
3029 static void scrub_remap_extent(struct btrfs_fs_info *fs_info,
3030                                u64 extent_logical, u64 extent_len,
3031                                u64 *extent_physical,
3032                                struct btrfs_device **extent_dev,
3033                                int *extent_mirror_num)
3034 {
3035         u64 mapped_length;
3036         struct btrfs_bio *bbio = NULL;
3037         int ret;
3038
3039         mapped_length = extent_len;
3040         ret = btrfs_map_block(fs_info, READ, extent_logical,
3041                               &mapped_length, &bbio, 0);
3042         if (ret || !bbio || mapped_length < extent_len ||
3043             !bbio->stripes[0].dev->bdev) {
3044                 kfree(bbio);
3045                 return;
3046         }
3047
3048         *extent_physical = bbio->stripes[0].physical;
3049         *extent_mirror_num = bbio->mirror_num;
3050         *extent_dev = bbio->stripes[0].dev;
3051         kfree(bbio);
3052 }
3053
3054 static int scrub_setup_wr_ctx(struct scrub_ctx *sctx,
3055                               struct scrub_wr_ctx *wr_ctx,
3056                               struct btrfs_fs_info *fs_info,
3057                               struct btrfs_device *dev,
3058                               int is_dev_replace)
3059 {
3060         WARN_ON(wr_ctx->wr_curr_bio != NULL);
3061
3062         mutex_init(&wr_ctx->wr_lock);
3063         wr_ctx->wr_curr_bio = NULL;
3064         if (!is_dev_replace)
3065                 return 0;
3066
3067         WARN_ON(!dev->bdev);
3068         wr_ctx->pages_per_wr_bio = min_t(int, SCRUB_PAGES_PER_WR_BIO,
3069                                          bio_get_nr_vecs(dev->bdev));
3070         wr_ctx->tgtdev = dev;
3071         atomic_set(&wr_ctx->flush_all_writes, 0);
3072         return 0;
3073 }
3074
3075 static void scrub_free_wr_ctx(struct scrub_wr_ctx *wr_ctx)
3076 {
3077         mutex_lock(&wr_ctx->wr_lock);
3078         kfree(wr_ctx->wr_curr_bio);
3079         wr_ctx->wr_curr_bio = NULL;
3080         mutex_unlock(&wr_ctx->wr_lock);
3081 }
3082
3083 static int copy_nocow_pages(struct scrub_ctx *sctx, u64 logical, u64 len,
3084                             int mirror_num, u64 physical_for_dev_replace)
3085 {
3086         struct scrub_copy_nocow_ctx *nocow_ctx;
3087         struct btrfs_fs_info *fs_info = sctx->dev_root->fs_info;
3088
3089         nocow_ctx = kzalloc(sizeof(*nocow_ctx), GFP_NOFS);
3090         if (!nocow_ctx) {
3091                 spin_lock(&sctx->stat_lock);
3092                 sctx->stat.malloc_errors++;
3093                 spin_unlock(&sctx->stat_lock);
3094                 return -ENOMEM;
3095         }
3096
3097         scrub_pending_trans_workers_inc(sctx);
3098
3099         nocow_ctx->sctx = sctx;
3100         nocow_ctx->logical = logical;
3101         nocow_ctx->len = len;
3102         nocow_ctx->mirror_num = mirror_num;
3103         nocow_ctx->physical_for_dev_replace = physical_for_dev_replace;
3104         nocow_ctx->work.func = copy_nocow_pages_worker;
3105         INIT_LIST_HEAD(&nocow_ctx->inodes);
3106         btrfs_queue_worker(&fs_info->scrub_nocow_workers,
3107                            &nocow_ctx->work);
3108
3109         return 0;
3110 }
3111
3112 static int record_inode_for_nocow(u64 inum, u64 offset, u64 root, void *ctx)
3113 {
3114         struct scrub_copy_nocow_ctx *nocow_ctx = ctx;
3115         struct scrub_nocow_inode *nocow_inode;
3116
3117         nocow_inode = kzalloc(sizeof(*nocow_inode), GFP_NOFS);
3118         if (!nocow_inode)
3119                 return -ENOMEM;
3120         nocow_inode->inum = inum;
3121         nocow_inode->offset = offset;
3122         nocow_inode->root = root;
3123         list_add_tail(&nocow_inode->list, &nocow_ctx->inodes);
3124         return 0;
3125 }
3126
3127 #define COPY_COMPLETE 1
3128
3129 static void copy_nocow_pages_worker(struct btrfs_work *work)
3130 {
3131         struct scrub_copy_nocow_ctx *nocow_ctx =
3132                 container_of(work, struct scrub_copy_nocow_ctx, work);
3133         struct scrub_ctx *sctx = nocow_ctx->sctx;
3134         u64 logical = nocow_ctx->logical;
3135         u64 len = nocow_ctx->len;
3136         int mirror_num = nocow_ctx->mirror_num;
3137         u64 physical_for_dev_replace = nocow_ctx->physical_for_dev_replace;
3138         int ret;
3139         struct btrfs_trans_handle *trans = NULL;
3140         struct btrfs_fs_info *fs_info;
3141         struct btrfs_path *path;
3142         struct btrfs_root *root;
3143         int not_written = 0;
3144
3145         fs_info = sctx->dev_root->fs_info;
3146         root = fs_info->extent_root;
3147
3148         path = btrfs_alloc_path();
3149         if (!path) {
3150                 spin_lock(&sctx->stat_lock);
3151                 sctx->stat.malloc_errors++;
3152                 spin_unlock(&sctx->stat_lock);
3153                 not_written = 1;
3154                 goto out;
3155         }
3156
3157         trans = btrfs_join_transaction(root);
3158         if (IS_ERR(trans)) {
3159                 not_written = 1;
3160                 goto out;
3161         }
3162
3163         ret = iterate_inodes_from_logical(logical, fs_info, path,
3164                                           record_inode_for_nocow, nocow_ctx);
3165         if (ret != 0 && ret != -ENOENT) {
3166                 pr_warn("iterate_inodes_from_logical() failed: log %llu, phys %llu, len %llu, mir %u, ret %d\n",
3167                         logical, physical_for_dev_replace, len, mirror_num,
3168                         ret);
3169                 not_written = 1;
3170                 goto out;
3171         }
3172
3173         btrfs_end_transaction(trans, root);
3174         trans = NULL;
3175         while (!list_empty(&nocow_ctx->inodes)) {
3176                 struct scrub_nocow_inode *entry;
3177                 entry = list_first_entry(&nocow_ctx->inodes,
3178                                          struct scrub_nocow_inode,
3179                                          list);
3180                 list_del_init(&entry->list);
3181                 ret = copy_nocow_pages_for_inode(entry->inum, entry->offset,
3182                                                  entry->root, nocow_ctx);
3183                 kfree(entry);
3184                 if (ret == COPY_COMPLETE) {
3185                         ret = 0;
3186                         break;
3187                 } else if (ret) {
3188                         break;
3189                 }
3190         }
3191 out:
3192         while (!list_empty(&nocow_ctx->inodes)) {
3193                 struct scrub_nocow_inode *entry;
3194                 entry = list_first_entry(&nocow_ctx->inodes,
3195                                          struct scrub_nocow_inode,
3196                                          list);
3197                 list_del_init(&entry->list);
3198                 kfree(entry);
3199         }
3200         if (trans && !IS_ERR(trans))
3201                 btrfs_end_transaction(trans, root);
3202         if (not_written)
3203                 btrfs_dev_replace_stats_inc(&fs_info->dev_replace.
3204                                             num_uncorrectable_read_errors);
3205
3206         btrfs_free_path(path);
3207         kfree(nocow_ctx);
3208
3209         scrub_pending_trans_workers_dec(sctx);
3210 }
3211
3212 static int copy_nocow_pages_for_inode(u64 inum, u64 offset, u64 root,
3213                                       struct scrub_copy_nocow_ctx *nocow_ctx)
3214 {
3215         struct btrfs_fs_info *fs_info = nocow_ctx->sctx->dev_root->fs_info;
3216         struct btrfs_key key;
3217         struct inode *inode;
3218         struct page *page;
3219         struct btrfs_root *local_root;
3220         struct btrfs_ordered_extent *ordered;
3221         struct extent_map *em;
3222         struct extent_state *cached_state = NULL;
3223         struct extent_io_tree *io_tree;
3224         u64 physical_for_dev_replace;
3225         u64 len = nocow_ctx->len;
3226         u64 lockstart = offset, lockend = offset + len - 1;
3227         unsigned long index;
3228         int srcu_index;
3229         int ret = 0;
3230         int err = 0;
3231
3232         key.objectid = root;
3233         key.type = BTRFS_ROOT_ITEM_KEY;
3234         key.offset = (u64)-1;
3235
3236         srcu_index = srcu_read_lock(&fs_info->subvol_srcu);
3237
3238         local_root = btrfs_read_fs_root_no_name(fs_info, &key);
3239         if (IS_ERR(local_root)) {
3240                 srcu_read_unlock(&fs_info->subvol_srcu, srcu_index);
3241                 return PTR_ERR(local_root);
3242         }
3243
3244         key.type = BTRFS_INODE_ITEM_KEY;
3245         key.objectid = inum;
3246         key.offset = 0;
3247         inode = btrfs_iget(fs_info->sb, &key, local_root, NULL);
3248         srcu_read_unlock(&fs_info->subvol_srcu, srcu_index);
3249         if (IS_ERR(inode))
3250                 return PTR_ERR(inode);
3251
3252         /* Avoid truncate/dio/punch hole.. */
3253         mutex_lock(&inode->i_mutex);
3254         inode_dio_wait(inode);
3255
3256         physical_for_dev_replace = nocow_ctx->physical_for_dev_replace;
3257         io_tree = &BTRFS_I(inode)->io_tree;
3258
3259         lock_extent_bits(io_tree, lockstart, lockend, 0, &cached_state);
3260         ordered = btrfs_lookup_ordered_range(inode, lockstart, len);
3261         if (ordered) {
3262                 btrfs_put_ordered_extent(ordered);
3263                 goto out_unlock;
3264         }
3265
3266         em = btrfs_get_extent(inode, NULL, 0, lockstart, len, 0);
3267         if (IS_ERR(em)) {
3268                 ret = PTR_ERR(em);
3269                 goto out_unlock;
3270         }
3271
3272         /*
3273          * This extent does not actually cover the logical extent anymore,
3274          * move on to the next inode.
3275          */
3276         if (em->block_start > nocow_ctx->logical ||
3277             em->block_start + em->block_len < nocow_ctx->logical + len) {
3278                 free_extent_map(em);
3279                 goto out_unlock;
3280         }
3281         free_extent_map(em);
3282
3283         while (len >= PAGE_CACHE_SIZE) {
3284                 index = offset >> PAGE_CACHE_SHIFT;
3285 again:
3286                 page = find_or_create_page(inode->i_mapping, index, GFP_NOFS);
3287                 if (!page) {
3288                         pr_err("find_or_create_page() failed\n");
3289                         ret = -ENOMEM;
3290                         goto out;
3291                 }
3292
3293                 if (PageUptodate(page)) {
3294                         if (PageDirty(page))
3295                                 goto next_page;
3296                 } else {
3297                         ClearPageError(page);
3298                         err = extent_read_full_page_nolock(io_tree, page,
3299                                                            btrfs_get_extent,
3300                                                            nocow_ctx->mirror_num);
3301                         if (err) {
3302                                 ret = err;
3303                                 goto next_page;
3304                         }
3305
3306                         lock_page(page);
3307                         /*
3308                          * If the page has been remove from the page cache,
3309                          * the data on it is meaningless, because it may be
3310                          * old one, the new data may be written into the new
3311                          * page in the page cache.
3312                          */
3313                         if (page->mapping != inode->i_mapping) {
3314                                 unlock_page(page);
3315                                 page_cache_release(page);
3316                                 goto again;
3317                         }
3318                         if (!PageUptodate(page)) {
3319                                 ret = -EIO;
3320                                 goto next_page;
3321                         }
3322                 }
3323                 err = write_page_nocow(nocow_ctx->sctx,
3324                                        physical_for_dev_replace, page);
3325                 if (err)
3326                         ret = err;
3327 next_page:
3328                 unlock_page(page);
3329                 page_cache_release(page);
3330
3331                 if (ret)
3332                         break;
3333
3334                 offset += PAGE_CACHE_SIZE;
3335                 physical_for_dev_replace += PAGE_CACHE_SIZE;
3336                 len -= PAGE_CACHE_SIZE;
3337         }
3338         ret = COPY_COMPLETE;
3339 out_unlock:
3340         unlock_extent_cached(io_tree, lockstart, lockend, &cached_state,
3341                              GFP_NOFS);
3342 out:
3343         mutex_unlock(&inode->i_mutex);
3344         iput(inode);
3345         return ret;
3346 }
3347
3348 static int write_page_nocow(struct scrub_ctx *sctx,
3349                             u64 physical_for_dev_replace, struct page *page)
3350 {
3351         struct bio *bio;
3352         struct btrfs_device *dev;
3353         int ret;
3354
3355         dev = sctx->wr_ctx.tgtdev;
3356         if (!dev)
3357                 return -EIO;
3358         if (!dev->bdev) {
3359                 printk_ratelimited(KERN_WARNING
3360                         "btrfs: scrub write_page_nocow(bdev == NULL) is unexpected!\n");
3361                 return -EIO;
3362         }
3363         bio = btrfs_io_bio_alloc(GFP_NOFS, 1);
3364         if (!bio) {
3365                 spin_lock(&sctx->stat_lock);
3366                 sctx->stat.malloc_errors++;
3367                 spin_unlock(&sctx->stat_lock);
3368                 return -ENOMEM;
3369         }
3370         bio->bi_size = 0;
3371         bio->bi_sector = physical_for_dev_replace >> 9;
3372         bio->bi_bdev = dev->bdev;
3373         ret = bio_add_page(bio, page, PAGE_CACHE_SIZE, 0);
3374         if (ret != PAGE_CACHE_SIZE) {
3375 leave_with_eio:
3376                 bio_put(bio);
3377                 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_WRITE_ERRS);
3378                 return -EIO;
3379         }
3380
3381         if (btrfsic_submit_bio_wait(WRITE_SYNC, bio))
3382                 goto leave_with_eio;
3383
3384         bio_put(bio);
3385         return 0;
3386 }