drivers/md/raid5-cache.c

   1 /*
   2  * Copyright (C) 2015 Shaohua Li <shli@fb.com>
   3  *
   4  * This program is free software; you can redistribute it and/or modify it
   5  * under the terms and conditions of the GNU General Public License,
   6  * version 2, as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  11  * more details.
  12  *
  13  */
  14 #include <linux/kernel.h>
  15 #include <linux/wait.h>
  16 #include <linux/blkdev.h>
  17 #include <linux/slab.h>
  18 #include <linux/raid/md_p.h>
  19 #include <linux/crc32c.h>
  20 #include <linux/random.h>
  21 #include "md.h"
  22 #include "raid5.h"
  23
  24 /*
  25  * metadata/data stored in disk with 4k size unit (a block) regardless
  26  * underneath hardware sector size. only works with PAGE_SIZE == 4096
  27  */
  28 #define BLOCK_SECTORS (8)
  29
  30 /*
  31  * reclaim runs every 1/4 disk size or 10G reclaimable space. This can prevent
  32  * recovery scans a very long log
  33  */
  34 #define RECLAIM_MAX_FREE_SPACE (10 * 1024 * 1024 * 2) /* sector */
  35 #define RECLAIM_MAX_FREE_SPACE_SHIFT (2)
  36
  37 /*
  38  * We only need 2 bios per I/O unit to make progress, but ensure we
  39  * have a few more available to not get too tight.
  40  */
  41 #define R5L_POOL_SIZE   4
  42
  43 struct r5l_log {
  44         struct md_rdev *rdev;
  45
  46         u32 uuid_checksum;
  47
  48         sector_t device_size;           /* log device size, round to
  49                                          * BLOCK_SECTORS */
  50         sector_t max_free_space;        /* reclaim run if free space is at
  51                                          * this size */
  52
  53         sector_t last_checkpoint;       /* log tail. where recovery scan
  54                                          * starts from */
  55         u64 last_cp_seq;                /* log tail sequence */
  56
  57         sector_t log_start;             /* log head. where new data appends */
  58         u64 seq;                        /* log head sequence */
  59
  60         sector_t next_checkpoint;
  61         u64 next_cp_seq;
  62
  63         struct mutex io_mutex;
  64         struct r5l_io_unit *current_io; /* current io_unit accepting new data */
  65
  66         spinlock_t io_list_lock;
  67         struct list_head running_ios;   /* io_units which are still running,
  68                                          * and have not yet been completely
  69                                          * written to the log */
  70         struct list_head io_end_ios;    /* io_units which have been completely
  71                                          * written to the log but not yet written
  72                                          * to the RAID */
  73         struct list_head flushing_ios;  /* io_units which are waiting for log
  74                                          * cache flush */
  75         struct list_head finished_ios;  /* io_units which settle down in log disk */
  76         struct bio flush_bio;
  77
  78         struct list_head no_mem_stripes;   /* pending stripes, -ENOMEM */
  79
  80         struct kmem_cache *io_kc;
  81         mempool_t *io_pool;
  82         struct bio_set *bs;
  83         mempool_t *meta_pool;
  84
  85         struct md_thread *reclaim_thread;
  86         unsigned long reclaim_target;   /* number of space that need to be
  87                                          * reclaimed.  if it's 0, reclaim spaces
  88                                          * used by io_units which are in
  89                                          * IO_UNIT_STRIPE_END state (eg, reclaim
  90                                          * dones't wait for specific io_unit
  91                                          * switching to IO_UNIT_STRIPE_END
  92                                          * state) */
  93         wait_queue_head_t iounit_wait;
  94
  95         struct list_head no_space_stripes; /* pending stripes, log has no space */
  96         spinlock_t no_space_stripes_lock;
  97
  98         bool need_cache_flush;
  99         bool in_teardown;
 100 };
 101
 102 /*
 103  * an IO range starts from a meta data block and end at the next meta data
 104  * block. The io unit's the meta data block tracks data/parity followed it. io
 105  * unit is written to log disk with normal write, as we always flush log disk
 106  * first and then start move data to raid disks, there is no requirement to
 107  * write io unit with FLUSH/FUA
 108  */
 109 struct r5l_io_unit {
 110         struct r5l_log *log;
 111
 112         struct page *meta_page; /* store meta block */
 113         int meta_offset;        /* current offset in meta_page */
 114
 115         struct bio *current_bio;/* current_bio accepting new data */
 116
 117         atomic_t pending_stripe;/* how many stripes not flushed to raid */
 118         u64 seq;                /* seq number of the metablock */
 119         sector_t log_start;     /* where the io_unit starts */
 120         sector_t log_end;       /* where the io_unit ends */
 121         struct list_head log_sibling; /* log->running_ios */
 122         struct list_head stripe_list; /* stripes added to the io_unit */
 123
 124         int state;
 125         bool need_split_bio;
 126 };
 127
 128 /* r5l_io_unit state */
 129 enum r5l_io_unit_state {
 130         IO_UNIT_RUNNING = 0,    /* accepting new IO */
 131         IO_UNIT_IO_START = 1,   /* io_unit bio start writing to log,
 132                                  * don't accepting new bio */
 133         IO_UNIT_IO_END = 2,     /* io_unit bio finish writing to log */
 134         IO_UNIT_STRIPE_END = 3, /* stripes data finished writing to raid */
 135 };
 136
 137 static sector_t r5l_ring_add(struct r5l_log *log, sector_t start, sector_t inc)
 138 {
 139         start += inc;
 140         if (start >= log->device_size)
 141                 start = start - log->device_size;
 142         return start;
 143 }
 144
 145 static sector_t r5l_ring_distance(struct r5l_log *log, sector_t start,
 146                                   sector_t end)
 147 {
 148         if (end >= start)
 149                 return end - start;
 150         else
 151                 return end + log->device_size - start;
 152 }
 153
 154 static bool r5l_has_free_space(struct r5l_log *log, sector_t size)
 155 {
 156         sector_t used_size;
 157
 158         used_size = r5l_ring_distance(log, log->last_checkpoint,
 159                                         log->log_start);
 160
 161         return log->device_size > used_size + size;
 162 }
 163
 164 static void __r5l_set_io_unit_state(struct r5l_io_unit *io,
 165                                     enum r5l_io_unit_state state)
 166 {
 167         if (WARN_ON(io->state >= state))
 168                 return;
 169         io->state = state;
 170 }
 171
 172 static void r5l_io_run_stripes(struct r5l_io_unit *io)
 173 {
 174         struct stripe_head *sh, *next;
 175
 176         list_for_each_entry_safe(sh, next, &io->stripe_list, log_list) {
 177                 list_del_init(&sh->log_list);
 178                 set_bit(STRIPE_HANDLE, &sh->state);
 179                 raid5_release_stripe(sh);
 180         }
 181 }
 182
 183 static void r5l_log_run_stripes(struct r5l_log *log)
 184 {
 185         struct r5l_io_unit *io, *next;
 186
 187         assert_spin_locked(&log->io_list_lock);
 188
 189         list_for_each_entry_safe(io, next, &log->running_ios, log_sibling) {
 190                 /* don't change list order */
 191                 if (io->state < IO_UNIT_IO_END)
 192                         break;
 193
 194                 list_move_tail(&io->log_sibling, &log->finished_ios);
 195                 r5l_io_run_stripes(io);
 196         }
 197 }
 198
 199 static void r5l_move_to_end_ios(struct r5l_log *log)
 200 {
 201         struct r5l_io_unit *io, *next;
 202
 203         assert_spin_locked(&log->io_list_lock);
 204
 205         list_for_each_entry_safe(io, next, &log->running_ios, log_sibling) {
 206                 /* don't change list order */
 207                 if (io->state < IO_UNIT_IO_END)
 208                         break;
 209                 list_move_tail(&io->log_sibling, &log->io_end_ios);
 210         }
 211 }
 212
 213 static void r5l_log_endio(struct bio *bio)
 214 {
 215         struct r5l_io_unit *io = bio->bi_private;
 216         struct r5l_log *log = io->log;
 217         unsigned long flags;
 218
 219         if (bio->bi_error)
 220                 md_error(log->rdev->mddev, log->rdev);
 221
 222         bio_put(bio);
 223         mempool_free(io->meta_page, log->meta_pool);
 224
 225         spin_lock_irqsave(&log->io_list_lock, flags);
 226         __r5l_set_io_unit_state(io, IO_UNIT_IO_END);
 227         if (log->need_cache_flush)
 228                 r5l_move_to_end_ios(log);
 229         else
 230                 r5l_log_run_stripes(log);
 231         spin_unlock_irqrestore(&log->io_list_lock, flags);
 232
 233         if (log->need_cache_flush)
 234                 md_wakeup_thread(log->rdev->mddev->thread);
 235 }
 236
 237 static void r5l_submit_current_io(struct r5l_log *log)
 238 {
 239         struct r5l_io_unit *io = log->current_io;
 240         struct r5l_meta_block *block;
 241         unsigned long flags;
 242         u32 crc;
 243
 244         if (!io)
 245                 return;
 246
 247         block = page_address(io->meta_page);
 248         block->meta_size = cpu_to_le32(io->meta_offset);
 249         crc = crc32c_le(log->uuid_checksum, block, PAGE_SIZE);
 250         block->checksum = cpu_to_le32(crc);
 251
 252         log->current_io = NULL;
 253         spin_lock_irqsave(&log->io_list_lock, flags);
 254         __r5l_set_io_unit_state(io, IO_UNIT_IO_START);
 255         spin_unlock_irqrestore(&log->io_list_lock, flags);
 256
 257         submit_bio(WRITE, io->current_bio);
 258 }
 259
 260 static struct bio *r5l_bio_alloc(struct r5l_log *log)
 261 {
 262         struct bio *bio = bio_alloc_bioset(GFP_NOIO, BIO_MAX_PAGES, log->bs);
 263
 264         bio->bi_rw = WRITE;
 265         bio->bi_bdev = log->rdev->bdev;
 266         bio->bi_iter.bi_sector = log->rdev->data_offset + log->log_start;
 267
 268         return bio;
 269 }
 270
 271 static void r5_reserve_log_entry(struct r5l_log *log, struct r5l_io_unit *io)
 272 {
 273         log->log_start = r5l_ring_add(log, log->log_start, BLOCK_SECTORS);
 274
 275         /*
 276          * If we filled up the log device start from the beginning again,
 277          * which will require a new bio.
 278          *
 279          * Note: for this to work properly the log size needs to me a multiple
 280          * of BLOCK_SECTORS.
 281          */
 282         if (log->log_start == 0)
 283                 io->need_split_bio = true;
 284
 285         io->log_end = log->log_start;
 286 }
 287
 288 static struct r5l_io_unit *r5l_new_meta(struct r5l_log *log)
 289 {
 290         struct r5l_io_unit *io;
 291         struct r5l_meta_block *block;
 292
 293         io = mempool_alloc(log->io_pool, GFP_ATOMIC);
 294         if (!io)
 295                 return NULL;
 296         memset(io, 0, sizeof(*io));
 297
 298         io->log = log;
 299         INIT_LIST_HEAD(&io->log_sibling);
 300         INIT_LIST_HEAD(&io->stripe_list);
 301         io->state = IO_UNIT_RUNNING;
 302
 303         io->meta_page = mempool_alloc(log->meta_pool, GFP_NOIO);
 304         block = page_address(io->meta_page);
 305         clear_page(block);
 306         block->magic = cpu_to_le32(R5LOG_MAGIC);
 307         block->version = R5LOG_VERSION;
 308         block->seq = cpu_to_le64(log->seq);
 309         block->position = cpu_to_le64(log->log_start);
 310
 311         io->log_start = log->log_start;
 312         io->meta_offset = sizeof(struct r5l_meta_block);
 313         io->seq = log->seq++;
 314
 315         io->current_bio = r5l_bio_alloc(log);
 316         io->current_bio->bi_end_io = r5l_log_endio;
 317         io->current_bio->bi_private = io;
 318         bio_add_page(io->current_bio, io->meta_page, PAGE_SIZE, 0);
 319
 320         r5_reserve_log_entry(log, io);
 321
 322         spin_lock_irq(&log->io_list_lock);
 323         list_add_tail(&io->log_sibling, &log->running_ios);
 324         spin_unlock_irq(&log->io_list_lock);
 325
 326         return io;
 327 }
 328
 329 static int r5l_get_meta(struct r5l_log *log, unsigned int payload_size)
 330 {
 331         if (log->current_io &&
 332             log->current_io->meta_offset + payload_size > PAGE_SIZE)
 333                 r5l_submit_current_io(log);
 334
 335         if (!log->current_io) {
 336                 log->current_io = r5l_new_meta(log);
 337                 if (!log->current_io)
 338                         return -ENOMEM;
 339         }
 340
 341         return 0;
 342 }
 343
 344 static void r5l_append_payload_meta(struct r5l_log *log, u16 type,
 345                                     sector_t location,
 346                                     u32 checksum1, u32 checksum2,
 347                                     bool checksum2_valid)
 348 {
 349         struct r5l_io_unit *io = log->current_io;
 350         struct r5l_payload_data_parity *payload;
 351
 352         payload = page_address(io->meta_page) + io->meta_offset;
 353         payload->header.type = cpu_to_le16(type);
 354         payload->header.flags = cpu_to_le16(0);
 355         payload->size = cpu_to_le32((1 + !!checksum2_valid) <<
 356                                     (PAGE_SHIFT - 9));
 357         payload->location = cpu_to_le64(location);
 358         payload->checksum[0] = cpu_to_le32(checksum1);
 359         if (checksum2_valid)
 360                 payload->checksum[1] = cpu_to_le32(checksum2);
 361
 362         io->meta_offset += sizeof(struct r5l_payload_data_parity) +
 363                 sizeof(__le32) * (1 + !!checksum2_valid);
 364 }
 365
 366 static void r5l_append_payload_page(struct r5l_log *log, struct page *page)
 367 {
 368         struct r5l_io_unit *io = log->current_io;
 369
 370         if (io->need_split_bio) {
 371                 struct bio *prev = io->current_bio;
 372
 373                 io->current_bio = r5l_bio_alloc(log);
 374                 bio_chain(io->current_bio, prev);
 375
 376                 submit_bio(WRITE, prev);
 377         }
 378
 379         if (!bio_add_page(io->current_bio, page, PAGE_SIZE, 0))
 380                 BUG();
 381
 382         r5_reserve_log_entry(log, io);
 383 }
 384
 385 static int r5l_log_stripe(struct r5l_log *log, struct stripe_head *sh,
 386                            int data_pages, int parity_pages)
 387 {
 388         int i;
 389         int meta_size;
 390         int ret;
 391         struct r5l_io_unit *io;
 392
 393         meta_size =
 394                 ((sizeof(struct r5l_payload_data_parity) + sizeof(__le32))
 395                  * data_pages) +
 396                 sizeof(struct r5l_payload_data_parity) +
 397                 sizeof(__le32) * parity_pages;
 398
 399         ret = r5l_get_meta(log, meta_size);
 400         if (ret)
 401                 return ret;
 402
 403         io = log->current_io;
 404
 405         for (i = 0; i < sh->disks; i++) {
 406                 if (!test_bit(R5_Wantwrite, &sh->dev[i].flags))
 407                         continue;
 408                 if (i == sh->pd_idx || i == sh->qd_idx)
 409                         continue;
 410                 r5l_append_payload_meta(log, R5LOG_PAYLOAD_DATA,
 411                                         raid5_compute_blocknr(sh, i, 0),
 412                                         sh->dev[i].log_checksum, 0, false);
 413                 r5l_append_payload_page(log, sh->dev[i].page);
 414         }
 415
 416         if (sh->qd_idx >= 0) {
 417                 r5l_append_payload_meta(log, R5LOG_PAYLOAD_PARITY,
 418                                         sh->sector, sh->dev[sh->pd_idx].log_checksum,
 419                                         sh->dev[sh->qd_idx].log_checksum, true);
 420                 r5l_append_payload_page(log, sh->dev[sh->pd_idx].page);
 421                 r5l_append_payload_page(log, sh->dev[sh->qd_idx].page);
 422         } else {
 423                 r5l_append_payload_meta(log, R5LOG_PAYLOAD_PARITY,
 424                                         sh->sector, sh->dev[sh->pd_idx].log_checksum,
 425                                         0, false);
 426                 r5l_append_payload_page(log, sh->dev[sh->pd_idx].page);
 427         }
 428
 429         list_add_tail(&sh->log_list, &io->stripe_list);
 430         atomic_inc(&io->pending_stripe);
 431         sh->log_io = io;
 432
 433         return 0;
 434 }
 435
 436 static void r5l_wake_reclaim(struct r5l_log *log, sector_t space);
 437 /*
 438  * running in raid5d, where reclaim could wait for raid5d too (when it flushes
 439  * data from log to raid disks), so we shouldn't wait for reclaim here
 440  */
 441 int r5l_write_stripe(struct r5l_log *log, struct stripe_head *sh)
 442 {
 443         int write_disks = 0;
 444         int data_pages, parity_pages;
 445         int meta_size;
 446         int reserve;
 447         int i;
 448         int ret = 0;
 449
 450         if (!log)
 451                 return -EAGAIN;
 452         /* Don't support stripe batch */
 453         if (sh->log_io || !test_bit(R5_Wantwrite, &sh->dev[sh->pd_idx].flags) ||
 454             test_bit(STRIPE_SYNCING, &sh->state)) {
 455                 /* the stripe is written to log, we start writing it to raid */
 456                 clear_bit(STRIPE_LOG_TRAPPED, &sh->state);
 457                 return -EAGAIN;
 458         }
 459
 460         for (i = 0; i < sh->disks; i++) {
 461                 void *addr;
 462
 463                 if (!test_bit(R5_Wantwrite, &sh->dev[i].flags))
 464                         continue;
 465                 write_disks++;
 466                 /* checksum is already calculated in last run */
 467                 if (test_bit(STRIPE_LOG_TRAPPED, &sh->state))
 468                         continue;
 469                 addr = kmap_atomic(sh->dev[i].page);
 470                 sh->dev[i].log_checksum = crc32c_le(log->uuid_checksum,
 471                                                     addr, PAGE_SIZE);
 472                 kunmap_atomic(addr);
 473         }
 474         parity_pages = 1 + !!(sh->qd_idx >= 0);
 475         data_pages = write_disks - parity_pages;
 476
 477         meta_size =
 478                 ((sizeof(struct r5l_payload_data_parity) + sizeof(__le32))
 479                  * data_pages) +
 480                 sizeof(struct r5l_payload_data_parity) +
 481                 sizeof(__le32) * parity_pages;
 482         /* Doesn't work with very big raid array */
 483         if (meta_size + sizeof(struct r5l_meta_block) > PAGE_SIZE)
 484                 return -EINVAL;
 485
 486         set_bit(STRIPE_LOG_TRAPPED, &sh->state);
 487         /*
 488          * The stripe must enter state machine again to finish the write, so
 489          * don't delay.
 490          */
 491         clear_bit(STRIPE_DELAYED, &sh->state);
 492         atomic_inc(&sh->count);
 493
 494         mutex_lock(&log->io_mutex);
 495         /* meta + data */
 496         reserve = (1 + write_disks) << (PAGE_SHIFT - 9);
 497         if (!r5l_has_free_space(log, reserve)) {
 498                 spin_lock(&log->no_space_stripes_lock);
 499                 list_add_tail(&sh->log_list, &log->no_space_stripes);
 500                 spin_unlock(&log->no_space_stripes_lock);
 501
 502                 r5l_wake_reclaim(log, reserve);
 503         } else {
 504                 ret = r5l_log_stripe(log, sh, data_pages, parity_pages);
 505                 if (ret) {
 506                         spin_lock_irq(&log->io_list_lock);
 507                         list_add_tail(&sh->log_list, &log->no_mem_stripes);
 508                         spin_unlock_irq(&log->io_list_lock);
 509                 }
 510         }
 511
 512         mutex_unlock(&log->io_mutex);
 513         return 0;
 514 }
 515
 516 void r5l_write_stripe_run(struct r5l_log *log)
 517 {
 518         if (!log)
 519                 return;
 520         mutex_lock(&log->io_mutex);
 521         r5l_submit_current_io(log);
 522         mutex_unlock(&log->io_mutex);
 523 }
 524
 525 int r5l_handle_flush_request(struct r5l_log *log, struct bio *bio)
 526 {
 527         if (!log)
 528                 return -ENODEV;
 529         /*
 530          * we flush log disk cache first, then write stripe data to raid disks.
 531          * So if bio is finished, the log disk cache is flushed already. The
 532          * recovery guarantees we can recovery the bio from log disk, so we
 533          * don't need to flush again
 534          */
 535         if (bio->bi_iter.bi_size == 0) {
 536                 bio_endio(bio);
 537                 return 0;
 538         }
 539         bio->bi_rw &= ~REQ_FLUSH;
 540         return -EAGAIN;
 541 }
 542
 543 /* This will run after log space is reclaimed */
 544 static void r5l_run_no_space_stripes(struct r5l_log *log)
 545 {
 546         struct stripe_head *sh;
 547
 548         spin_lock(&log->no_space_stripes_lock);
 549         while (!list_empty(&log->no_space_stripes)) {
 550                 sh = list_first_entry(&log->no_space_stripes,
 551                                       struct stripe_head, log_list);
 552                 list_del_init(&sh->log_list);
 553                 set_bit(STRIPE_HANDLE, &sh->state);
 554                 raid5_release_stripe(sh);
 555         }
 556         spin_unlock(&log->no_space_stripes_lock);
 557 }
 558
 559 static sector_t r5l_reclaimable_space(struct r5l_log *log)
 560 {
 561         return r5l_ring_distance(log, log->last_checkpoint,
 562                                  log->next_checkpoint);
 563 }
 564
 565 static void r5l_run_no_mem_stripe(struct r5l_log *log)
 566 {
 567         struct stripe_head *sh;
 568
 569         assert_spin_locked(&log->io_list_lock);
 570
 571         if (!list_empty(&log->no_mem_stripes)) {
 572                 sh = list_first_entry(&log->no_mem_stripes,
 573                                       struct stripe_head, log_list);
 574                 list_del_init(&sh->log_list);
 575                 set_bit(STRIPE_HANDLE, &sh->state);
 576                 raid5_release_stripe(sh);
 577         }
 578 }
 579
 580 static bool r5l_complete_finished_ios(struct r5l_log *log)
 581 {
 582         struct r5l_io_unit *io, *next;
 583         bool found = false;
 584
 585         assert_spin_locked(&log->io_list_lock);
 586
 587         list_for_each_entry_safe(io, next, &log->finished_ios, log_sibling) {
 588                 /* don't change list order */
 589                 if (io->state < IO_UNIT_STRIPE_END)
 590                         break;
 591
 592                 log->next_checkpoint = io->log_start;
 593                 log->next_cp_seq = io->seq;
 594
 595                 list_del(&io->log_sibling);
 596                 mempool_free(io, log->io_pool);
 597                 r5l_run_no_mem_stripe(log);
 598
 599                 found = true;
 600         }
 601
 602         return found;
 603 }
 604
 605 static void __r5l_stripe_write_finished(struct r5l_io_unit *io)
 606 {
 607         struct r5l_log *log = io->log;
 608         unsigned long flags;
 609
 610         spin_lock_irqsave(&log->io_list_lock, flags);
 611         __r5l_set_io_unit_state(io, IO_UNIT_STRIPE_END);
 612
 613         if (!r5l_complete_finished_ios(log)) {
 614                 spin_unlock_irqrestore(&log->io_list_lock, flags);
 615                 return;
 616         }
 617
 618         if (r5l_reclaimable_space(log) > log->max_free_space)
 619                 r5l_wake_reclaim(log, 0);
 620
 621         spin_unlock_irqrestore(&log->io_list_lock, flags);
 622         wake_up(&log->iounit_wait);
 623 }
 624
 625 void r5l_stripe_write_finished(struct stripe_head *sh)
 626 {
 627         struct r5l_io_unit *io;
 628
 629         io = sh->log_io;
 630         sh->log_io = NULL;
 631
 632         if (io && atomic_dec_and_test(&io->pending_stripe))
 633                 __r5l_stripe_write_finished(io);
 634 }
 635
 636 static void r5l_log_flush_endio(struct bio *bio)
 637 {
 638         struct r5l_log *log = container_of(bio, struct r5l_log,
 639                 flush_bio);
 640         unsigned long flags;
 641         struct r5l_io_unit *io;
 642
 643         if (bio->bi_error)
 644                 md_error(log->rdev->mddev, log->rdev);
 645
 646         spin_lock_irqsave(&log->io_list_lock, flags);
 647         list_for_each_entry(io, &log->flushing_ios, log_sibling)
 648                 r5l_io_run_stripes(io);
 649         list_splice_tail_init(&log->flushing_ios, &log->finished_ios);
 650         spin_unlock_irqrestore(&log->io_list_lock, flags);
 651 }
 652
 653 /*
 654  * Starting dispatch IO to raid.
 655  * io_unit(meta) consists of a log. There is one situation we want to avoid. A
 656  * broken meta in the middle of a log causes recovery can't find meta at the
 657  * head of log. If operations require meta at the head persistent in log, we
 658  * must make sure meta before it persistent in log too. A case is:
 659  *
 660  * stripe data/parity is in log, we start write stripe to raid disks. stripe
 661  * data/parity must be persistent in log before we do the write to raid disks.
 662  *
 663  * The solution is we restrictly maintain io_unit list order. In this case, we
 664  * only write stripes of an io_unit to raid disks till the io_unit is the first
 665  * one whose data/parity is in log.
 666  */
 667 void r5l_flush_stripe_to_raid(struct r5l_log *log)
 668 {
 669         bool do_flush;
 670
 671         if (!log || !log->need_cache_flush)
 672                 return;
 673
 674         spin_lock_irq(&log->io_list_lock);
 675         /* flush bio is running */
 676         if (!list_empty(&log->flushing_ios)) {
 677                 spin_unlock_irq(&log->io_list_lock);
 678                 return;
 679         }
 680         list_splice_tail_init(&log->io_end_ios, &log->flushing_ios);
 681         do_flush = !list_empty(&log->flushing_ios);
 682         spin_unlock_irq(&log->io_list_lock);
 683
 684         if (!do_flush)
 685                 return;
 686         bio_reset(&log->flush_bio);
 687         log->flush_bio.bi_bdev = log->rdev->bdev;
 688         log->flush_bio.bi_end_io = r5l_log_flush_endio;
 689         submit_bio(WRITE_FLUSH, &log->flush_bio);
 690 }
 691
 692 static void r5l_write_super(struct r5l_log *log, sector_t cp);
 693 static void r5l_write_super_and_discard_space(struct r5l_log *log,
 694         sector_t end)
 695 {
 696         struct block_device *bdev = log->rdev->bdev;
 697         struct mddev *mddev;
 698
 699         r5l_write_super(log, end);
 700
 701         if (!blk_queue_discard(bdev_get_queue(bdev)))
 702                 return;
 703
 704         mddev = log->rdev->mddev;
 705         /*
 706          * This is to avoid a deadlock. r5l_quiesce holds reconfig_mutex and
 707          * wait for this thread to finish. This thread waits for
 708          * MD_CHANGE_PENDING clear, which is supposed to be done in
 709          * md_check_recovery(). md_check_recovery() tries to get
 710          * reconfig_mutex. Since r5l_quiesce already holds the mutex,
 711          * md_check_recovery() fails, so the PENDING never get cleared. The
 712          * in_teardown check workaround this issue.
 713          */
 714         if (!log->in_teardown) {
 715                 set_bit(MD_CHANGE_DEVS, &mddev->flags);
 716                 set_bit(MD_CHANGE_PENDING, &mddev->flags);
 717                 md_wakeup_thread(mddev->thread);
 718                 wait_event(mddev->sb_wait,
 719                         !test_bit(MD_CHANGE_PENDING, &mddev->flags) ||
 720                         log->in_teardown);
 721                 /*
 722                  * r5l_quiesce could run after in_teardown check and hold
 723                  * mutex first. Superblock might get updated twice.
 724                  */
 725                 if (log->in_teardown)
 726                         md_update_sb(mddev, 1);
 727         } else {
 728                 WARN_ON(!mddev_is_locked(mddev));
 729                 md_update_sb(mddev, 1);
 730         }
 731
 732         /* discard IO error really doesn't matter, ignore it */
 733         if (log->last_checkpoint < end) {
 734                 blkdev_issue_discard(bdev,
 735                                 log->last_checkpoint + log->rdev->data_offset,
 736                                 end - log->last_checkpoint, GFP_NOIO, 0);
 737         } else {
 738                 blkdev_issue_discard(bdev,
 739                                 log->last_checkpoint + log->rdev->data_offset,
 740                                 log->device_size - log->last_checkpoint,
 741                                 GFP_NOIO, 0);
 742                 blkdev_issue_discard(bdev, log->rdev->data_offset, end,
 743                                 GFP_NOIO, 0);
 744         }
 745 }
 746
 747
 748 static void r5l_do_reclaim(struct r5l_log *log)
 749 {
 750         sector_t reclaim_target = xchg(&log->reclaim_target, 0);
 751         sector_t reclaimable;
 752         sector_t next_checkpoint;
 753         u64 next_cp_seq;
 754
 755         spin_lock_irq(&log->io_list_lock);
 756         /*
 757          * move proper io_unit to reclaim list. We should not change the order.
 758          * reclaimable/unreclaimable io_unit can be mixed in the list, we
 759          * shouldn't reuse space of an unreclaimable io_unit
 760          */
 761         while (1) {
 762                 reclaimable = r5l_reclaimable_space(log);
 763                 if (reclaimable >= reclaim_target ||
 764                     (list_empty(&log->running_ios) &&
 765                      list_empty(&log->io_end_ios) &&
 766                      list_empty(&log->flushing_ios) &&
 767                      list_empty(&log->finished_ios)))
 768                         break;
 769
 770                 md_wakeup_thread(log->rdev->mddev->thread);
 771                 wait_event_lock_irq(log->iounit_wait,
 772                                     r5l_reclaimable_space(log) > reclaimable,
 773                                     log->io_list_lock);
 774         }
 775
 776         next_checkpoint = log->next_checkpoint;
 777         next_cp_seq = log->next_cp_seq;
 778         spin_unlock_irq(&log->io_list_lock);
 779
 780         BUG_ON(reclaimable < 0);
 781         if (reclaimable == 0)
 782                 return;
 783
 784         /*
 785          * write_super will flush cache of each raid disk. We must write super
 786          * here, because the log area might be reused soon and we don't want to
 787          * confuse recovery
 788          */
 789         r5l_write_super_and_discard_space(log, next_checkpoint);
 790
 791         mutex_lock(&log->io_mutex);
 792         log->last_checkpoint = next_checkpoint;
 793         log->last_cp_seq = next_cp_seq;
 794         mutex_unlock(&log->io_mutex);
 795
 796         r5l_run_no_space_stripes(log);
 797 }
 798
 799 static void r5l_reclaim_thread(struct md_thread *thread)
 800 {
 801         struct mddev *mddev = thread->mddev;
 802         struct r5conf *conf = mddev->private;
 803         struct r5l_log *log = conf->log;
 804
 805         if (!log)
 806                 return;
 807         r5l_do_reclaim(log);
 808 }
 809
 810 static void r5l_wake_reclaim(struct r5l_log *log, sector_t space)
 811 {
 812         unsigned long target;
 813         unsigned long new = (unsigned long)space; /* overflow in theory */
 814
 815         do {
 816                 target = log->reclaim_target;
 817                 if (new < target)
 818                         return;
 819         } while (cmpxchg(&log->reclaim_target, target, new) != target);
 820         md_wakeup_thread(log->reclaim_thread);
 821 }
 822
 823 void r5l_quiesce(struct r5l_log *log, int state)
 824 {
 825         struct mddev *mddev;
 826         if (!log || state == 2)
 827                 return;
 828         if (state == 0) {
 829                 log->in_teardown = 0;
 830                 /*
 831                  * This is a special case for hotadd. In suspend, the array has
 832                  * no journal. In resume, journal is initialized as well as the
 833                  * reclaim thread.
 834                  */
 835                 if (log->reclaim_thread)
 836                         return;
 837                 log->reclaim_thread = md_register_thread(r5l_reclaim_thread,
 838                                         log->rdev->mddev, "reclaim");
 839         } else if (state == 1) {
 840                 /*
 841                  * at this point all stripes are finished, so io_unit is at
 842                  * least in STRIPE_END state
 843                  */
 844                 log->in_teardown = 1;
 845                 /* make sure r5l_write_super_and_discard_space exits */
 846                 mddev = log->rdev->mddev;
 847                 wake_up(&mddev->sb_wait);
 848                 r5l_wake_reclaim(log, -1L);
 849                 md_unregister_thread(&log->reclaim_thread);
 850                 r5l_do_reclaim(log);
 851         }
 852 }
 853
 854 bool r5l_log_disk_error(struct r5conf *conf)
 855 {
 856         struct r5l_log *log;
 857         bool ret;
 858         /* don't allow write if journal disk is missing */
 859         rcu_read_lock();
 860         log = rcu_dereference(conf->log);
 861
 862         if (!log)
 863                 ret = test_bit(MD_HAS_JOURNAL, &conf->mddev->flags);
 864         else
 865                 ret = test_bit(Faulty, &log->rdev->flags);
 866         rcu_read_unlock();
 867         return ret;
 868 }
 869
 870 struct r5l_recovery_ctx {
 871         struct page *meta_page;         /* current meta */
 872         sector_t meta_total_blocks;     /* total size of current meta and data */
 873         sector_t pos;                   /* recovery position */
 874         u64 seq;                        /* recovery position seq */
 875 };
 876
 877 static int r5l_read_meta_block(struct r5l_log *log,
 878                                struct r5l_recovery_ctx *ctx)
 879 {
 880         struct page *page = ctx->meta_page;
 881         struct r5l_meta_block *mb;
 882         u32 crc, stored_crc;
 883
 884         if (!sync_page_io(log->rdev, ctx->pos, PAGE_SIZE, page, READ, false))
 885                 return -EIO;
 886
 887         mb = page_address(page);
 888         stored_crc = le32_to_cpu(mb->checksum);
 889         mb->checksum = 0;
 890
 891         if (le32_to_cpu(mb->magic) != R5LOG_MAGIC ||
 892             le64_to_cpu(mb->seq) != ctx->seq ||
 893             mb->version != R5LOG_VERSION ||
 894             le64_to_cpu(mb->position) != ctx->pos)
 895                 return -EINVAL;
 896
 897         crc = crc32c_le(log->uuid_checksum, mb, PAGE_SIZE);
 898         if (stored_crc != crc)
 899                 return -EINVAL;
 900
 901         if (le32_to_cpu(mb->meta_size) > PAGE_SIZE)
 902                 return -EINVAL;
 903
 904         ctx->meta_total_blocks = BLOCK_SECTORS;
 905
 906         return 0;
 907 }
 908
 909 static int r5l_recovery_flush_one_stripe(struct r5l_log *log,
 910                                          struct r5l_recovery_ctx *ctx,
 911                                          sector_t stripe_sect,
 912                                          int *offset, sector_t *log_offset)
 913 {
 914         struct r5conf *conf = log->rdev->mddev->private;
 915         struct stripe_head *sh;
 916         struct r5l_payload_data_parity *payload;
 917         int disk_index;
 918
 919         sh = raid5_get_active_stripe(conf, stripe_sect, 0, 0, 0);
 920         while (1) {
 921                 payload = page_address(ctx->meta_page) + *offset;
 922
 923                 if (le16_to_cpu(payload->header.type) == R5LOG_PAYLOAD_DATA) {
 924                         raid5_compute_sector(conf,
 925                                              le64_to_cpu(payload->location), 0,
 926                                              &disk_index, sh);
 927
 928                         sync_page_io(log->rdev, *log_offset, PAGE_SIZE,
 929                                      sh->dev[disk_index].page, READ, false);
 930                         sh->dev[disk_index].log_checksum =
 931                                 le32_to_cpu(payload->checksum[0]);
 932                         set_bit(R5_Wantwrite, &sh->dev[disk_index].flags);
 933                         ctx->meta_total_blocks += BLOCK_SECTORS;
 934                 } else {
 935                         disk_index = sh->pd_idx;
 936                         sync_page_io(log->rdev, *log_offset, PAGE_SIZE,
 937                                      sh->dev[disk_index].page, READ, false);
 938                         sh->dev[disk_index].log_checksum =
 939                                 le32_to_cpu(payload->checksum[0]);
 940                         set_bit(R5_Wantwrite, &sh->dev[disk_index].flags);
 941
 942                         if (sh->qd_idx >= 0) {
 943                                 disk_index = sh->qd_idx;
 944                                 sync_page_io(log->rdev,
 945                                              r5l_ring_add(log, *log_offset, BLOCK_SECTORS),
 946                                              PAGE_SIZE, sh->dev[disk_index].page,
 947                                              READ, false);
 948                                 sh->dev[disk_index].log_checksum =
 949                                         le32_to_cpu(payload->checksum[1]);
 950                                 set_bit(R5_Wantwrite,
 951                                         &sh->dev[disk_index].flags);
 952                         }
 953                         ctx->meta_total_blocks += BLOCK_SECTORS * conf->max_degraded;
 954                 }
 955
 956                 *log_offset = r5l_ring_add(log, *log_offset,
 957                                            le32_to_cpu(payload->size));
 958                 *offset += sizeof(struct r5l_payload_data_parity) +
 959                         sizeof(__le32) *
 960                         (le32_to_cpu(payload->size) >> (PAGE_SHIFT - 9));
 961                 if (le16_to_cpu(payload->header.type) == R5LOG_PAYLOAD_PARITY)
 962                         break;
 963         }
 964
 965         for (disk_index = 0; disk_index < sh->disks; disk_index++) {
 966                 void *addr;
 967                 u32 checksum;
 968
 969                 if (!test_bit(R5_Wantwrite, &sh->dev[disk_index].flags))
 970                         continue;
 971                 addr = kmap_atomic(sh->dev[disk_index].page);
 972                 checksum = crc32c_le(log->uuid_checksum, addr, PAGE_SIZE);
 973                 kunmap_atomic(addr);
 974                 if (checksum != sh->dev[disk_index].log_checksum)
 975                         goto error;
 976         }
 977
 978         for (disk_index = 0; disk_index < sh->disks; disk_index++) {
 979                 struct md_rdev *rdev, *rrdev;
 980
 981                 if (!test_and_clear_bit(R5_Wantwrite,
 982                                         &sh->dev[disk_index].flags))
 983                         continue;
 984
 985                 /* in case device is broken */
 986                 rdev = rcu_dereference(conf->disks[disk_index].rdev);
 987                 if (rdev)
 988                         sync_page_io(rdev, stripe_sect, PAGE_SIZE,
 989                                      sh->dev[disk_index].page, WRITE, false);
 990                 rrdev = rcu_dereference(conf->disks[disk_index].replacement);
 991                 if (rrdev)
 992                         sync_page_io(rrdev, stripe_sect, PAGE_SIZE,
 993                                      sh->dev[disk_index].page, WRITE, false);
 994         }
 995         raid5_release_stripe(sh);
 996         return 0;
 997
 998 error:
 999         for (disk_index = 0; disk_index < sh->disks; disk_index++)
1000                 sh->dev[disk_index].flags = 0;
1001         raid5_release_stripe(sh);
1002         return -EINVAL;
1003 }
1004
1005 static int r5l_recovery_flush_one_meta(struct r5l_log *log,
1006                                        struct r5l_recovery_ctx *ctx)
1007 {
1008         struct r5conf *conf = log->rdev->mddev->private;
1009         struct r5l_payload_data_parity *payload;
1010         struct r5l_meta_block *mb;
1011         int offset;
1012         sector_t log_offset;
1013         sector_t stripe_sector;
1014
1015         mb = page_address(ctx->meta_page);
1016         offset = sizeof(struct r5l_meta_block);
1017         log_offset = r5l_ring_add(log, ctx->pos, BLOCK_SECTORS);
1018
1019         while (offset < le32_to_cpu(mb->meta_size)) {
1020                 int dd;
1021
1022                 payload = (void *)mb + offset;
1023                 stripe_sector = raid5_compute_sector(conf,
1024                                                      le64_to_cpu(payload->location), 0, &dd, NULL);
1025                 if (r5l_recovery_flush_one_stripe(log, ctx, stripe_sector,
1026                                                   &offset, &log_offset))
1027                         return -EINVAL;
1028         }
1029         return 0;
1030 }
1031
1032 /* copy data/parity from log to raid disks */
1033 static void r5l_recovery_flush_log(struct r5l_log *log,
1034                                    struct r5l_recovery_ctx *ctx)
1035 {
1036         while (1) {
1037                 if (r5l_read_meta_block(log, ctx))
1038                         return;
1039                 if (r5l_recovery_flush_one_meta(log, ctx))
1040                         return;
1041                 ctx->seq++;
1042                 ctx->pos = r5l_ring_add(log, ctx->pos, ctx->meta_total_blocks);
1043         }
1044 }
1045
1046 static int r5l_log_write_empty_meta_block(struct r5l_log *log, sector_t pos,
1047                                           u64 seq)
1048 {
1049         struct page *page;
1050         struct r5l_meta_block *mb;
1051         u32 crc;
1052
1053         page = alloc_page(GFP_KERNEL | __GFP_ZERO);
1054         if (!page)
1055                 return -ENOMEM;
1056         mb = page_address(page);
1057         mb->magic = cpu_to_le32(R5LOG_MAGIC);
1058         mb->version = R5LOG_VERSION;
1059         mb->meta_size = cpu_to_le32(sizeof(struct r5l_meta_block));
1060         mb->seq = cpu_to_le64(seq);
1061         mb->position = cpu_to_le64(pos);
1062         crc = crc32c_le(log->uuid_checksum, mb, PAGE_SIZE);
1063         mb->checksum = cpu_to_le32(crc);
1064
1065         if (!sync_page_io(log->rdev, pos, PAGE_SIZE, page, WRITE_FUA, false)) {
1066                 __free_page(page);
1067                 return -EIO;
1068         }
1069         __free_page(page);
1070         return 0;
1071 }
1072
1073 static int r5l_recovery_log(struct r5l_log *log)
1074 {
1075         struct r5l_recovery_ctx ctx;
1076
1077         ctx.pos = log->last_checkpoint;
1078         ctx.seq = log->last_cp_seq;
1079         ctx.meta_page = alloc_page(GFP_KERNEL);
1080         if (!ctx.meta_page)
1081                 return -ENOMEM;
1082
1083         r5l_recovery_flush_log(log, &ctx);
1084         __free_page(ctx.meta_page);
1085
1086         /*
1087          * we did a recovery. Now ctx.pos points to an invalid meta block. New
1088          * log will start here. but we can't let superblock point to last valid
1089          * meta block. The log might looks like:
1090          * | meta 1| meta 2| meta 3|
1091          * meta 1 is valid, meta 2 is invalid. meta 3 could be valid. If
1092          * superblock points to meta 1, we write a new valid meta 2n.  if crash
1093          * happens again, new recovery will start from meta 1. Since meta 2n is
1094          * valid now, recovery will think meta 3 is valid, which is wrong.
1095          * The solution is we create a new meta in meta2 with its seq == meta
1096          * 1's seq + 10 and let superblock points to meta2. The same recovery will
1097          * not think meta 3 is a valid meta, because its seq doesn't match
1098          */
1099         if (ctx.seq > log->last_cp_seq + 1) {
1100                 int ret;
1101
1102                 ret = r5l_log_write_empty_meta_block(log, ctx.pos, ctx.seq + 10);
1103                 if (ret)
1104                         return ret;
1105                 log->seq = ctx.seq + 11;
1106                 log->log_start = r5l_ring_add(log, ctx.pos, BLOCK_SECTORS);
1107                 r5l_write_super(log, ctx.pos);
1108         } else {
1109                 log->log_start = ctx.pos;
1110                 log->seq = ctx.seq;
1111         }
1112         return 0;
1113 }
1114
1115 static void r5l_write_super(struct r5l_log *log, sector_t cp)
1116 {
1117         struct mddev *mddev = log->rdev->mddev;
1118
1119         log->rdev->journal_tail = cp;
1120         set_bit(MD_CHANGE_DEVS, &mddev->flags);
1121 }
1122
1123 static int r5l_load_log(struct r5l_log *log)
1124 {
1125         struct md_rdev *rdev = log->rdev;
1126         struct page *page;
1127         struct r5l_meta_block *mb;
1128         sector_t cp = log->rdev->journal_tail;
1129         u32 stored_crc, expected_crc;
1130         bool create_super = false;
1131         int ret;
1132
1133         /* Make sure it's valid */
1134         if (cp >= rdev->sectors || round_down(cp, BLOCK_SECTORS) != cp)
1135                 cp = 0;
1136         page = alloc_page(GFP_KERNEL);
1137         if (!page)
1138                 return -ENOMEM;
1139
1140         if (!sync_page_io(rdev, cp, PAGE_SIZE, page, READ, false)) {
1141                 ret = -EIO;
1142                 goto ioerr;
1143         }
1144         mb = page_address(page);
1145
1146         if (le32_to_cpu(mb->magic) != R5LOG_MAGIC ||
1147             mb->version != R5LOG_VERSION) {
1148                 create_super = true;
1149                 goto create;
1150         }
1151         stored_crc = le32_to_cpu(mb->checksum);
1152         mb->checksum = 0;
1153         expected_crc = crc32c_le(log->uuid_checksum, mb, PAGE_SIZE);
1154         if (stored_crc != expected_crc) {
1155                 create_super = true;
1156                 goto create;
1157         }
1158         if (le64_to_cpu(mb->position) != cp) {
1159                 create_super = true;
1160                 goto create;
1161         }
1162 create:
1163         if (create_super) {
1164                 log->last_cp_seq = prandom_u32();
1165                 cp = 0;
1166                 /*
1167                  * Make sure super points to correct address. Log might have
1168                  * data very soon. If super hasn't correct log tail address,
1169                  * recovery can't find the log
1170                  */
1171                 r5l_write_super(log, cp);
1172         } else
1173                 log->last_cp_seq = le64_to_cpu(mb->seq);
1174
1175         log->device_size = round_down(rdev->sectors, BLOCK_SECTORS);
1176         log->max_free_space = log->device_size >> RECLAIM_MAX_FREE_SPACE_SHIFT;
1177         if (log->max_free_space > RECLAIM_MAX_FREE_SPACE)
1178                 log->max_free_space = RECLAIM_MAX_FREE_SPACE;
1179         log->last_checkpoint = cp;
1180
1181         __free_page(page);
1182
1183         return r5l_recovery_log(log);
1184 ioerr:
1185         __free_page(page);
1186         return ret;
1187 }
1188
1189 int r5l_init_log(struct r5conf *conf, struct md_rdev *rdev)
1190 {
1191         struct request_queue *q = bdev_get_queue(rdev->bdev);
1192         struct r5l_log *log;
1193
1194         if (PAGE_SIZE != 4096)
1195                 return -EINVAL;
1196         log = kzalloc(sizeof(*log), GFP_KERNEL);
1197         if (!log)
1198                 return -ENOMEM;
1199         log->rdev = rdev;
1200
1201         log->need_cache_flush = test_bit(QUEUE_FLAG_WC, &q->queue_flags) != 0;
1202
1203         log->uuid_checksum = crc32c_le(~0, rdev->mddev->uuid,
1204                                        sizeof(rdev->mddev->uuid));
1205
1206         mutex_init(&log->io_mutex);
1207
1208         spin_lock_init(&log->io_list_lock);
1209         INIT_LIST_HEAD(&log->running_ios);
1210         INIT_LIST_HEAD(&log->io_end_ios);
1211         INIT_LIST_HEAD(&log->flushing_ios);
1212         INIT_LIST_HEAD(&log->finished_ios);
1213         bio_init(&log->flush_bio);
1214
1215         log->io_kc = KMEM_CACHE(r5l_io_unit, 0);
1216         if (!log->io_kc)
1217                 goto io_kc;
1218
1219         log->io_pool = mempool_create_slab_pool(R5L_POOL_SIZE, log->io_kc);
1220         if (!log->io_pool)
1221                 goto io_pool;
1222
1223         log->bs = bioset_create(R5L_POOL_SIZE, 0);
1224         if (!log->bs)
1225                 goto io_bs;
1226
1227         log->meta_pool = mempool_create_page_pool(R5L_POOL_SIZE, 0);
1228         if (!log->meta_pool)
1229                 goto out_mempool;
1230
1231         log->reclaim_thread = md_register_thread(r5l_reclaim_thread,
1232                                                  log->rdev->mddev, "reclaim");
1233         if (!log->reclaim_thread)
1234                 goto reclaim_thread;
1235         init_waitqueue_head(&log->iounit_wait);
1236
1237         INIT_LIST_HEAD(&log->no_mem_stripes);
1238
1239         INIT_LIST_HEAD(&log->no_space_stripes);
1240         spin_lock_init(&log->no_space_stripes_lock);
1241
1242         if (r5l_load_log(log))
1243                 goto error;
1244
1245         rcu_assign_pointer(conf->log, log);
1246         set_bit(MD_HAS_JOURNAL, &conf->mddev->flags);
1247         return 0;
1248
1249 error:
1250         md_unregister_thread(&log->reclaim_thread);
1251 reclaim_thread:
1252         mempool_destroy(log->meta_pool);
1253 out_mempool:
1254         bioset_free(log->bs);
1255 io_bs:
1256         mempool_destroy(log->io_pool);
1257 io_pool:
1258         kmem_cache_destroy(log->io_kc);
1259 io_kc:
1260         kfree(log);
1261         return -EINVAL;
1262 }
1263
1264 void r5l_exit_log(struct r5l_log *log)
1265 {
1266         md_unregister_thread(&log->reclaim_thread);
1267         mempool_destroy(log->meta_pool);
1268         bioset_free(log->bs);
1269         mempool_destroy(log->io_pool);
1270         kmem_cache_destroy(log->io_kc);
1271         kfree(log);
1272 }