fs/f2fs/segment.c

   1 /*
   2  * fs/f2fs/segment.c
   3  *
   4  * Copyright (c) 2012 Samsung Electronics Co., Ltd.
   5  *             http://www.samsung.com/
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License version 2 as
   9  * published by the Free Software Foundation.
  10  */
  11 #include <linux/fs.h>
  12 #include <linux/f2fs_fs.h>
  13 #include <linux/bio.h>
  14 #include <linux/blkdev.h>
  15 #include <linux/prefetch.h>
  16 #include <linux/kthread.h>
  17 #include <linux/swap.h>
  18 #include <linux/timer.h>
  19
  20 #include "f2fs.h"
  21 #include "segment.h"
  22 #include "node.h"
  23 #include "trace.h"
  24 #include <trace/events/f2fs.h>
  25
  26 #define __reverse_ffz(x) __reverse_ffs(~(x))
  27
  28 static struct kmem_cache *discard_entry_slab;
  29 static struct kmem_cache *sit_entry_set_slab;
  30 static struct kmem_cache *inmem_entry_slab;
  31
  32 static unsigned long __reverse_ulong(unsigned char *str)
  33 {
  34         unsigned long tmp = 0;
  35         int shift = 24, idx = 0;
  36
  37 #if BITS_PER_LONG == 64
  38         shift = 56;
  39 #endif
  40         while (shift >= 0) {
  41                 tmp |= (unsigned long)str[idx++] << shift;
  42                 shift -= BITS_PER_BYTE;
  43         }
  44         return tmp;
  45 }
  46
  47 /*
  48  * __reverse_ffs is copied from include/asm-generic/bitops/__ffs.h since
  49  * MSB and LSB are reversed in a byte by f2fs_set_bit.
  50  */
  51 static inline unsigned long __reverse_ffs(unsigned long word)
  52 {
  53         int num = 0;
  54
  55 #if BITS_PER_LONG == 64
  56         if ((word & 0xffffffff00000000UL) == 0)
  57                 num += 32;
  58         else
  59                 word >>= 32;
  60 #endif
  61         if ((word & 0xffff0000) == 0)
  62                 num += 16;
  63         else
  64                 word >>= 16;
  65
  66         if ((word & 0xff00) == 0)
  67                 num += 8;
  68         else
  69                 word >>= 8;
  70
  71         if ((word & 0xf0) == 0)
  72                 num += 4;
  73         else
  74                 word >>= 4;
  75
  76         if ((word & 0xc) == 0)
  77                 num += 2;
  78         else
  79                 word >>= 2;
  80
  81         if ((word & 0x2) == 0)
  82                 num += 1;
  83         return num;
  84 }
  85
  86 /*
  87  * __find_rev_next(_zero)_bit is copied from lib/find_next_bit.c because
  88  * f2fs_set_bit makes MSB and LSB reversed in a byte.
  89  * @size must be integral times of unsigned long.
  90  * Example:
  91  *                             MSB <--> LSB
  92  *   f2fs_set_bit(0, bitmap) => 1000 0000
  93  *   f2fs_set_bit(7, bitmap) => 0000 0001
  94  */
  95 static unsigned long __find_rev_next_bit(const unsigned long *addr,
  96                         unsigned long size, unsigned long offset)
  97 {
  98         const unsigned long *p = addr + BIT_WORD(offset);
  99         unsigned long result = size;
 100         unsigned long tmp;
 101
 102         if (offset >= size)
 103                 return size;
 104
 105         size -= (offset & ~(BITS_PER_LONG - 1));
 106         offset %= BITS_PER_LONG;
 107
 108         while (1) {
 109                 if (*p == 0)
 110                         goto pass;
 111
 112                 tmp = __reverse_ulong((unsigned char *)p);
 113
 114                 tmp &= ~0UL >> offset;
 115                 if (size < BITS_PER_LONG)
 116                         tmp &= (~0UL << (BITS_PER_LONG - size));
 117                 if (tmp)
 118                         goto found;
 119 pass:
 120                 if (size <= BITS_PER_LONG)
 121                         break;
 122                 size -= BITS_PER_LONG;
 123                 offset = 0;
 124                 p++;
 125         }
 126         return result;
 127 found:
 128         return result - size + __reverse_ffs(tmp);
 129 }
 130
 131 static unsigned long __find_rev_next_zero_bit(const unsigned long *addr,
 132                         unsigned long size, unsigned long offset)
 133 {
 134         const unsigned long *p = addr + BIT_WORD(offset);
 135         unsigned long result = size;
 136         unsigned long tmp;
 137
 138         if (offset >= size)
 139                 return size;
 140
 141         size -= (offset & ~(BITS_PER_LONG - 1));
 142         offset %= BITS_PER_LONG;
 143
 144         while (1) {
 145                 if (*p == ~0UL)
 146                         goto pass;
 147
 148                 tmp = __reverse_ulong((unsigned char *)p);
 149
 150                 if (offset)
 151                         tmp |= ~0UL << (BITS_PER_LONG - offset);
 152                 if (size < BITS_PER_LONG)
 153                         tmp |= ~0UL >> size;
 154                 if (tmp != ~0UL)
 155                         goto found;
 156 pass:
 157                 if (size <= BITS_PER_LONG)
 158                         break;
 159                 size -= BITS_PER_LONG;
 160                 offset = 0;
 161                 p++;
 162         }
 163         return result;
 164 found:
 165         return result - size + __reverse_ffz(tmp);
 166 }
 167
 168 void register_inmem_page(struct inode *inode, struct page *page)
 169 {
 170         struct f2fs_inode_info *fi = F2FS_I(inode);
 171         struct inmem_pages *new;
 172
 173         f2fs_trace_pid(page);
 174
 175         set_page_private(page, (unsigned long)ATOMIC_WRITTEN_PAGE);
 176         SetPagePrivate(page);
 177
 178         new = f2fs_kmem_cache_alloc(inmem_entry_slab, GFP_NOFS);
 179
 180         /* add atomic page indices to the list */
 181         new->page = page;
 182         INIT_LIST_HEAD(&new->list);
 183
 184         /* increase reference count with clean state */
 185         mutex_lock(&fi->inmem_lock);
 186         get_page(page);
 187         list_add_tail(&new->list, &fi->inmem_pages);
 188         inc_page_count(F2FS_I_SB(inode), F2FS_INMEM_PAGES);
 189         mutex_unlock(&fi->inmem_lock);
 190
 191         trace_f2fs_register_inmem_page(page, INMEM);
 192 }
 193
 194 static int __revoke_inmem_pages(struct inode *inode,
 195                                 struct list_head *head, bool drop, bool recover)
 196 {
 197         struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
 198         struct inmem_pages *cur, *tmp;
 199         int err = 0;
 200
 201         list_for_each_entry_safe(cur, tmp, head, list) {
 202                 struct page *page = cur->page;
 203
 204                 if (drop)
 205                         trace_f2fs_commit_inmem_page(page, INMEM_DROP);
 206
 207                 lock_page(page);
 208
 209                 if (recover) {
 210                         struct dnode_of_data dn;
 211                         struct node_info ni;
 212
 213                         trace_f2fs_commit_inmem_page(page, INMEM_REVOKE);
 214
 215                         set_new_dnode(&dn, inode, NULL, NULL, 0);
 216                         if (get_dnode_of_data(&dn, page->index, LOOKUP_NODE)) {
 217                                 err = -EAGAIN;
 218                                 goto next;
 219                         }
 220                         get_node_info(sbi, dn.nid, &ni);
 221                         f2fs_replace_block(sbi, &dn, dn.data_blkaddr,
 222                                         cur->old_addr, ni.version, true, true);
 223                         f2fs_put_dnode(&dn);
 224                 }
 225 next:
 226                 /* we don't need to invalidate this in the sccessful status */
 227                 if (drop || recover)
 228                         ClearPageUptodate(page);
 229                 set_page_private(page, 0);
 230                 ClearPagePrivate(page);
 231                 f2fs_put_page(page, 1);
 232
 233                 list_del(&cur->list);
 234                 kmem_cache_free(inmem_entry_slab, cur);
 235                 dec_page_count(F2FS_I_SB(inode), F2FS_INMEM_PAGES);
 236         }
 237         return err;
 238 }
 239
 240 void drop_inmem_pages(struct inode *inode)
 241 {
 242         struct f2fs_inode_info *fi = F2FS_I(inode);
 243
 244         clear_inode_flag(F2FS_I(inode), FI_ATOMIC_FILE);
 245
 246         mutex_lock(&fi->inmem_lock);
 247         __revoke_inmem_pages(inode, &fi->inmem_pages, true, false);
 248         mutex_unlock(&fi->inmem_lock);
 249 }
 250
 251 static int __commit_inmem_pages(struct inode *inode,
 252                                         struct list_head *revoke_list)
 253 {
 254         struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
 255         struct f2fs_inode_info *fi = F2FS_I(inode);
 256         struct inmem_pages *cur, *tmp;
 257         struct f2fs_io_info fio = {
 258                 .sbi = sbi,
 259                 .type = DATA,
 260                 .rw = WRITE_SYNC | REQ_PRIO,
 261                 .encrypted_page = NULL,
 262         };
 263         bool submit_bio = false;
 264         int err = 0;
 265
 266         list_for_each_entry_safe(cur, tmp, &fi->inmem_pages, list) {
 267                 struct page *page = cur->page;
 268
 269                 lock_page(page);
 270                 if (page->mapping == inode->i_mapping) {
 271                         trace_f2fs_commit_inmem_page(page, INMEM);
 272
 273                         set_page_dirty(page);
 274                         f2fs_wait_on_page_writeback(page, DATA, true);
 275                         if (clear_page_dirty_for_io(page))
 276                                 inode_dec_dirty_pages(inode);
 277
 278                         fio.page = page;
 279                         err = do_write_data_page(&fio);
 280                         if (err) {
 281                                 unlock_page(page);
 282                                 break;
 283                         }
 284
 285                         /* record old blkaddr for revoking */
 286                         cur->old_addr = fio.old_blkaddr;
 287
 288                         clear_cold_data(page);
 289                         submit_bio = true;
 290                 }
 291                 unlock_page(page);
 292                 list_move_tail(&cur->list, revoke_list);
 293         }
 294
 295         if (submit_bio)
 296                 f2fs_submit_merged_bio_cond(sbi, inode, NULL, 0, DATA, WRITE);
 297
 298         if (!err)
 299                 __revoke_inmem_pages(inode, revoke_list, false, false);
 300
 301         return err;
 302 }
 303
 304 int commit_inmem_pages(struct inode *inode)
 305 {
 306         struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
 307         struct f2fs_inode_info *fi = F2FS_I(inode);
 308         struct list_head revoke_list;
 309         int err;
 310
 311         INIT_LIST_HEAD(&revoke_list);
 312         f2fs_balance_fs(sbi, true);
 313         f2fs_lock_op(sbi);
 314
 315         mutex_lock(&fi->inmem_lock);
 316         err = __commit_inmem_pages(inode, &revoke_list);
 317         if (err) {
 318                 int ret;
 319                 /*
 320                  * try to revoke all committed pages, but still we could fail
 321                  * due to no memory or other reason, if that happened, EAGAIN
 322                  * will be returned, which means in such case, transaction is
 323                  * already not integrity, caller should use journal to do the
 324                  * recovery or rewrite & commit last transaction. For other
 325                  * error number, revoking was done by filesystem itself.
 326                  */
 327                 ret = __revoke_inmem_pages(inode, &revoke_list, false, true);
 328                 if (ret)
 329                         err = ret;
 330
 331                 /* drop all uncommitted pages */
 332                 __revoke_inmem_pages(inode, &fi->inmem_pages, true, false);
 333         }
 334         mutex_unlock(&fi->inmem_lock);
 335
 336         f2fs_unlock_op(sbi);
 337         return err;
 338 }
 339
 340 /*
 341  * This function balances dirty node and dentry pages.
 342  * In addition, it controls garbage collection.
 343  */
 344 void f2fs_balance_fs(struct f2fs_sb_info *sbi, bool need)
 345 {
 346         if (!need)
 347                 return;
 348         /*
 349          * We should do GC or end up with checkpoint, if there are so many dirty
 350          * dir/node pages without enough free segments.
 351          */
 352         if (has_not_enough_free_secs(sbi, 0)) {
 353                 mutex_lock(&sbi->gc_mutex);
 354                 f2fs_gc(sbi, false);
 355         }
 356 }
 357
 358 void f2fs_balance_fs_bg(struct f2fs_sb_info *sbi)
 359 {
 360         /* try to shrink extent cache when there is no enough memory */
 361         if (!available_free_memory(sbi, EXTENT_CACHE))
 362                 f2fs_shrink_extent_tree(sbi, EXTENT_CACHE_SHRINK_NUMBER);
 363
 364         /* check the # of cached NAT entries */
 365         if (!available_free_memory(sbi, NAT_ENTRIES))
 366                 try_to_free_nats(sbi, NAT_ENTRY_PER_BLOCK);
 367
 368         if (!available_free_memory(sbi, FREE_NIDS))
 369                 try_to_free_nids(sbi, NAT_ENTRY_PER_BLOCK * FREE_NID_PAGES);
 370
 371         /* checkpoint is the only way to shrink partial cached entries */
 372         if (!available_free_memory(sbi, NAT_ENTRIES) ||
 373                         !available_free_memory(sbi, INO_ENTRIES) ||
 374                         excess_prefree_segs(sbi) ||
 375                         excess_dirty_nats(sbi) ||
 376                         (is_idle(sbi) && f2fs_time_over(sbi, CP_TIME))) {
 377                 if (test_opt(sbi, DATA_FLUSH)) {
 378                         struct blk_plug plug;
 379
 380                         blk_start_plug(&plug);
 381                         sync_dirty_inodes(sbi, FILE_INODE);
 382                         blk_finish_plug(&plug);
 383                 }
 384                 f2fs_sync_fs(sbi->sb, true);
 385                 stat_inc_bg_cp_count(sbi->stat_info);
 386         }
 387 }
 388
 389 static int issue_flush_thread(void *data)
 390 {
 391         struct f2fs_sb_info *sbi = data;
 392         struct flush_cmd_control *fcc = SM_I(sbi)->cmd_control_info;
 393         wait_queue_head_t *q = &fcc->flush_wait_queue;
 394 repeat:
 395         if (kthread_should_stop())
 396                 return 0;
 397
 398         if (!llist_empty(&fcc->issue_list)) {
 399                 struct bio *bio;
 400                 struct flush_cmd *cmd, *next;
 401                 int ret;
 402
 403                 bio = f2fs_bio_alloc(0);
 404
 405                 fcc->dispatch_list = llist_del_all(&fcc->issue_list);
 406                 fcc->dispatch_list = llist_reverse_order(fcc->dispatch_list);
 407
 408                 bio->bi_bdev = sbi->sb->s_bdev;
 409                 ret = submit_bio_wait(WRITE_FLUSH, bio);
 410
 411                 llist_for_each_entry_safe(cmd, next,
 412                                           fcc->dispatch_list, llnode) {
 413                         cmd->ret = ret;
 414                         complete(&cmd->wait);
 415                 }
 416                 bio_put(bio);
 417                 fcc->dispatch_list = NULL;
 418         }
 419
 420         wait_event_interruptible(*q,
 421                 kthread_should_stop() || !llist_empty(&fcc->issue_list));
 422         goto repeat;
 423 }
 424
 425 int f2fs_issue_flush(struct f2fs_sb_info *sbi)
 426 {
 427         struct flush_cmd_control *fcc = SM_I(sbi)->cmd_control_info;
 428         struct flush_cmd cmd;
 429
 430         trace_f2fs_issue_flush(sbi->sb, test_opt(sbi, NOBARRIER),
 431                                         test_opt(sbi, FLUSH_MERGE));
 432
 433         if (test_opt(sbi, NOBARRIER))
 434                 return 0;
 435
 436         if (!test_opt(sbi, FLUSH_MERGE)) {
 437                 struct bio *bio = f2fs_bio_alloc(0);
 438                 int ret;
 439
 440                 bio->bi_bdev = sbi->sb->s_bdev;
 441                 ret = submit_bio_wait(WRITE_FLUSH, bio);
 442                 bio_put(bio);
 443                 return ret;
 444         }
 445
 446         init_completion(&cmd.wait);
 447
 448         llist_add(&cmd.llnode, &fcc->issue_list);
 449
 450         if (!fcc->dispatch_list)
 451                 wake_up(&fcc->flush_wait_queue);
 452
 453         wait_for_completion(&cmd.wait);
 454
 455         return cmd.ret;
 456 }
 457
 458 int create_flush_cmd_control(struct f2fs_sb_info *sbi)
 459 {
 460         dev_t dev = sbi->sb->s_bdev->bd_dev;
 461         struct flush_cmd_control *fcc;
 462         int err = 0;
 463
 464         fcc = kzalloc(sizeof(struct flush_cmd_control), GFP_KERNEL);
 465         if (!fcc)
 466                 return -ENOMEM;
 467         init_waitqueue_head(&fcc->flush_wait_queue);
 468         init_llist_head(&fcc->issue_list);
 469         SM_I(sbi)->cmd_control_info = fcc;
 470         fcc->f2fs_issue_flush = kthread_run(issue_flush_thread, sbi,
 471                                 "f2fs_flush-%u:%u", MAJOR(dev), MINOR(dev));
 472         if (IS_ERR(fcc->f2fs_issue_flush)) {
 473                 err = PTR_ERR(fcc->f2fs_issue_flush);
 474                 kfree(fcc);
 475                 SM_I(sbi)->cmd_control_info = NULL;
 476                 return err;
 477         }
 478
 479         return err;
 480 }
 481
 482 void destroy_flush_cmd_control(struct f2fs_sb_info *sbi)
 483 {
 484         struct flush_cmd_control *fcc = SM_I(sbi)->cmd_control_info;
 485
 486         if (fcc && fcc->f2fs_issue_flush)
 487                 kthread_stop(fcc->f2fs_issue_flush);
 488         kfree(fcc);
 489         SM_I(sbi)->cmd_control_info = NULL;
 490 }
 491
 492 static void __locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno,
 493                 enum dirty_type dirty_type)
 494 {
 495         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
 496
 497         /* need not be added */
 498         if (IS_CURSEG(sbi, segno))
 499                 return;
 500
 501         if (!test_and_set_bit(segno, dirty_i->dirty_segmap[dirty_type]))
 502                 dirty_i->nr_dirty[dirty_type]++;
 503
 504         if (dirty_type == DIRTY) {
 505                 struct seg_entry *sentry = get_seg_entry(sbi, segno);
 506                 enum dirty_type t = sentry->type;
 507
 508                 if (unlikely(t >= DIRTY)) {
 509                         f2fs_bug_on(sbi, 1);
 510                         return;
 511                 }
 512                 if (!test_and_set_bit(segno, dirty_i->dirty_segmap[t]))
 513                         dirty_i->nr_dirty[t]++;
 514         }
 515 }
 516
 517 static void __remove_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno,
 518                 enum dirty_type dirty_type)
 519 {
 520         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
 521
 522         if (test_and_clear_bit(segno, dirty_i->dirty_segmap[dirty_type]))
 523                 dirty_i->nr_dirty[dirty_type]--;
 524
 525         if (dirty_type == DIRTY) {
 526                 struct seg_entry *sentry = get_seg_entry(sbi, segno);
 527                 enum dirty_type t = sentry->type;
 528
 529                 if (test_and_clear_bit(segno, dirty_i->dirty_segmap[t]))
 530                         dirty_i->nr_dirty[t]--;
 531
 532                 if (get_valid_blocks(sbi, segno, sbi->segs_per_sec) == 0)
 533                         clear_bit(GET_SECNO(sbi, segno),
 534                                                 dirty_i->victim_secmap);
 535         }
 536 }
 537
 538 /*
 539  * Should not occur error such as -ENOMEM.
 540  * Adding dirty entry into seglist is not critical operation.
 541  * If a given segment is one of current working segments, it won't be added.
 542  */
 543 static void locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno)
 544 {
 545         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
 546         unsigned short valid_blocks;
 547
 548         if (segno == NULL_SEGNO || IS_CURSEG(sbi, segno))
 549                 return;
 550
 551         mutex_lock(&dirty_i->seglist_lock);
 552
 553         valid_blocks = get_valid_blocks(sbi, segno, 0);
 554
 555         if (valid_blocks == 0) {
 556                 __locate_dirty_segment(sbi, segno, PRE);
 557                 __remove_dirty_segment(sbi, segno, DIRTY);
 558         } else if (valid_blocks < sbi->blocks_per_seg) {
 559                 __locate_dirty_segment(sbi, segno, DIRTY);
 560         } else {
 561                 /* Recovery routine with SSR needs this */
 562                 __remove_dirty_segment(sbi, segno, DIRTY);
 563         }
 564
 565         mutex_unlock(&dirty_i->seglist_lock);
 566 }
 567
 568 static int f2fs_issue_discard(struct f2fs_sb_info *sbi,
 569                                 block_t blkstart, block_t blklen)
 570 {
 571         sector_t start = SECTOR_FROM_BLOCK(blkstart);
 572         sector_t len = SECTOR_FROM_BLOCK(blklen);
 573         struct seg_entry *se;
 574         unsigned int offset;
 575         block_t i;
 576
 577         for (i = blkstart; i < blkstart + blklen; i++) {
 578                 se = get_seg_entry(sbi, GET_SEGNO(sbi, i));
 579                 offset = GET_BLKOFF_FROM_SEG0(sbi, i);
 580
 581                 if (!f2fs_test_and_set_bit(offset, se->discard_map))
 582                         sbi->discard_blks--;
 583         }
 584         trace_f2fs_issue_discard(sbi->sb, blkstart, blklen);
 585         return blkdev_issue_discard(sbi->sb->s_bdev, start, len, GFP_NOFS, 0);
 586 }
 587
 588 bool discard_next_dnode(struct f2fs_sb_info *sbi, block_t blkaddr)
 589 {
 590         int err = -EOPNOTSUPP;
 591
 592         if (test_opt(sbi, DISCARD)) {
 593                 struct seg_entry *se = get_seg_entry(sbi,
 594                                 GET_SEGNO(sbi, blkaddr));
 595                 unsigned int offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr);
 596
 597                 if (f2fs_test_bit(offset, se->discard_map))
 598                         return false;
 599
 600                 err = f2fs_issue_discard(sbi, blkaddr, 1);
 601         }
 602
 603         if (err) {
 604                 update_meta_page(sbi, NULL, blkaddr);
 605                 return true;
 606         }
 607         return false;
 608 }
 609
 610 static void __add_discard_entry(struct f2fs_sb_info *sbi,
 611                 struct cp_control *cpc, struct seg_entry *se,
 612                 unsigned int start, unsigned int end)
 613 {
 614         struct list_head *head = &SM_I(sbi)->discard_list;
 615         struct discard_entry *new, *last;
 616
 617         if (!list_empty(head)) {
 618                 last = list_last_entry(head, struct discard_entry, list);
 619                 if (START_BLOCK(sbi, cpc->trim_start) + start ==
 620                                                 last->blkaddr + last->len) {
 621                         last->len += end - start;
 622                         goto done;
 623                 }
 624         }
 625
 626         new = f2fs_kmem_cache_alloc(discard_entry_slab, GFP_NOFS);
 627         INIT_LIST_HEAD(&new->list);
 628         new->blkaddr = START_BLOCK(sbi, cpc->trim_start) + start;
 629         new->len = end - start;
 630         list_add_tail(&new->list, head);
 631 done:
 632         SM_I(sbi)->nr_discards += end - start;
 633 }
 634
 635 static void add_discard_addrs(struct f2fs_sb_info *sbi, struct cp_control *cpc)
 636 {
 637         int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long);
 638         int max_blocks = sbi->blocks_per_seg;
 639         struct seg_entry *se = get_seg_entry(sbi, cpc->trim_start);
 640         unsigned long *cur_map = (unsigned long *)se->cur_valid_map;
 641         unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map;
 642         unsigned long *discard_map = (unsigned long *)se->discard_map;
 643         unsigned long *dmap = SIT_I(sbi)->tmp_map;
 644         unsigned int start = 0, end = -1;
 645         bool force = (cpc->reason == CP_DISCARD);
 646         int i;
 647
 648         if (se->valid_blocks == max_blocks)
 649                 return;
 650
 651         if (!force) {
 652                 if (!test_opt(sbi, DISCARD) || !se->valid_blocks ||
 653                     SM_I(sbi)->nr_discards >= SM_I(sbi)->max_discards)
 654                         return;
 655         }
 656
 657         /* SIT_VBLOCK_MAP_SIZE should be multiple of sizeof(unsigned long) */
 658         for (i = 0; i < entries; i++)
 659                 dmap[i] = force ? ~ckpt_map[i] & ~discard_map[i] :
 660                                 (cur_map[i] ^ ckpt_map[i]) & ckpt_map[i];
 661
 662         while (force || SM_I(sbi)->nr_discards <= SM_I(sbi)->max_discards) {
 663                 start = __find_rev_next_bit(dmap, max_blocks, end + 1);
 664                 if (start >= max_blocks)
 665                         break;
 666
 667                 end = __find_rev_next_zero_bit(dmap, max_blocks, start + 1);
 668                 __add_discard_entry(sbi, cpc, se, start, end);
 669         }
 670 }
 671
 672 void release_discard_addrs(struct f2fs_sb_info *sbi)
 673 {
 674         struct list_head *head = &(SM_I(sbi)->discard_list);
 675         struct discard_entry *entry, *this;
 676
 677         /* drop caches */
 678         list_for_each_entry_safe(entry, this, head, list) {
 679                 list_del(&entry->list);
 680                 kmem_cache_free(discard_entry_slab, entry);
 681         }
 682 }
 683
 684 /*
 685  * Should call clear_prefree_segments after checkpoint is done.
 686  */
 687 static void set_prefree_as_free_segments(struct f2fs_sb_info *sbi)
 688 {
 689         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
 690         unsigned int segno;
 691
 692         mutex_lock(&dirty_i->seglist_lock);
 693         for_each_set_bit(segno, dirty_i->dirty_segmap[PRE], MAIN_SEGS(sbi))
 694                 __set_test_and_free(sbi, segno);
 695         mutex_unlock(&dirty_i->seglist_lock);
 696 }
 697
 698 void clear_prefree_segments(struct f2fs_sb_info *sbi, struct cp_control *cpc)
 699 {
 700         struct list_head *head = &(SM_I(sbi)->discard_list);
 701         struct discard_entry *entry, *this;
 702         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
 703         unsigned long *prefree_map = dirty_i->dirty_segmap[PRE];
 704         unsigned int start = 0, end = -1;
 705
 706         mutex_lock(&dirty_i->seglist_lock);
 707
 708         while (1) {
 709                 int i;
 710                 start = find_next_bit(prefree_map, MAIN_SEGS(sbi), end + 1);
 711                 if (start >= MAIN_SEGS(sbi))
 712                         break;
 713                 end = find_next_zero_bit(prefree_map, MAIN_SEGS(sbi),
 714                                                                 start + 1);
 715
 716                 for (i = start; i < end; i++)
 717                         clear_bit(i, prefree_map);
 718
 719                 dirty_i->nr_dirty[PRE] -= end - start;
 720
 721                 if (!test_opt(sbi, DISCARD))
 722                         continue;
 723
 724                 f2fs_issue_discard(sbi, START_BLOCK(sbi, start),
 725                                 (end - start) << sbi->log_blocks_per_seg);
 726         }
 727         mutex_unlock(&dirty_i->seglist_lock);
 728
 729         /* send small discards */
 730         list_for_each_entry_safe(entry, this, head, list) {
 731                 if (cpc->reason == CP_DISCARD && entry->len < cpc->trim_minlen)
 732                         goto skip;
 733                 f2fs_issue_discard(sbi, entry->blkaddr, entry->len);
 734                 cpc->trimmed += entry->len;
 735 skip:
 736                 list_del(&entry->list);
 737                 SM_I(sbi)->nr_discards -= entry->len;
 738                 kmem_cache_free(discard_entry_slab, entry);
 739         }
 740 }
 741
 742 static bool __mark_sit_entry_dirty(struct f2fs_sb_info *sbi, unsigned int segno)
 743 {
 744         struct sit_info *sit_i = SIT_I(sbi);
 745
 746         if (!__test_and_set_bit(segno, sit_i->dirty_sentries_bitmap)) {
 747                 sit_i->dirty_sentries++;
 748                 return false;
 749         }
 750
 751         return true;
 752 }
 753
 754 static void __set_sit_entry_type(struct f2fs_sb_info *sbi, int type,
 755                                         unsigned int segno, int modified)
 756 {
 757         struct seg_entry *se = get_seg_entry(sbi, segno);
 758         se->type = type;
 759         if (modified)
 760                 __mark_sit_entry_dirty(sbi, segno);
 761 }
 762
 763 static void update_sit_entry(struct f2fs_sb_info *sbi, block_t blkaddr, int del)
 764 {
 765         struct seg_entry *se;
 766         unsigned int segno, offset;
 767         long int new_vblocks;
 768
 769         segno = GET_SEGNO(sbi, blkaddr);
 770
 771         se = get_seg_entry(sbi, segno);
 772         new_vblocks = se->valid_blocks + del;
 773         offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr);
 774
 775         f2fs_bug_on(sbi, (new_vblocks >> (sizeof(unsigned short) << 3) ||
 776                                 (new_vblocks > sbi->blocks_per_seg)));
 777
 778         se->valid_blocks = new_vblocks;
 779         se->mtime = get_mtime(sbi);
 780         SIT_I(sbi)->max_mtime = se->mtime;
 781
 782         /* Update valid block bitmap */
 783         if (del > 0) {
 784                 if (f2fs_test_and_set_bit(offset, se->cur_valid_map))
 785                         f2fs_bug_on(sbi, 1);
 786                 if (!f2fs_test_and_set_bit(offset, se->discard_map))
 787                         sbi->discard_blks--;
 788         } else {
 789                 if (!f2fs_test_and_clear_bit(offset, se->cur_valid_map))
 790                         f2fs_bug_on(sbi, 1);
 791                 if (f2fs_test_and_clear_bit(offset, se->discard_map))
 792                         sbi->discard_blks++;
 793         }
 794         if (!f2fs_test_bit(offset, se->ckpt_valid_map))
 795                 se->ckpt_valid_blocks += del;
 796
 797         __mark_sit_entry_dirty(sbi, segno);
 798
 799         /* update total number of valid blocks to be written in ckpt area */
 800         SIT_I(sbi)->written_valid_blocks += del;
 801
 802         if (sbi->segs_per_sec > 1)
 803                 get_sec_entry(sbi, segno)->valid_blocks += del;
 804 }
 805
 806 void refresh_sit_entry(struct f2fs_sb_info *sbi, block_t old, block_t new)
 807 {
 808         update_sit_entry(sbi, new, 1);
 809         if (GET_SEGNO(sbi, old) != NULL_SEGNO)
 810                 update_sit_entry(sbi, old, -1);
 811
 812         locate_dirty_segment(sbi, GET_SEGNO(sbi, old));
 813         locate_dirty_segment(sbi, GET_SEGNO(sbi, new));
 814 }
 815
 816 void invalidate_blocks(struct f2fs_sb_info *sbi, block_t addr)
 817 {
 818         unsigned int segno = GET_SEGNO(sbi, addr);
 819         struct sit_info *sit_i = SIT_I(sbi);
 820
 821         f2fs_bug_on(sbi, addr == NULL_ADDR);
 822         if (addr == NEW_ADDR)
 823                 return;
 824
 825         /* add it into sit main buffer */
 826         mutex_lock(&sit_i->sentry_lock);
 827
 828         update_sit_entry(sbi, addr, -1);
 829
 830         /* add it into dirty seglist */
 831         locate_dirty_segment(sbi, segno);
 832
 833         mutex_unlock(&sit_i->sentry_lock);
 834 }
 835
 836 bool is_checkpointed_data(struct f2fs_sb_info *sbi, block_t blkaddr)
 837 {
 838         struct sit_info *sit_i = SIT_I(sbi);
 839         unsigned int segno, offset;
 840         struct seg_entry *se;
 841         bool is_cp = false;
 842
 843         if (blkaddr == NEW_ADDR || blkaddr == NULL_ADDR)
 844                 return true;
 845
 846         mutex_lock(&sit_i->sentry_lock);
 847
 848         segno = GET_SEGNO(sbi, blkaddr);
 849         se = get_seg_entry(sbi, segno);
 850         offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr);
 851
 852         if (f2fs_test_bit(offset, se->ckpt_valid_map))
 853                 is_cp = true;
 854
 855         mutex_unlock(&sit_i->sentry_lock);
 856
 857         return is_cp;
 858 }
 859
 860 /*
 861  * This function should be resided under the curseg_mutex lock
 862  */
 863 static void __add_sum_entry(struct f2fs_sb_info *sbi, int type,
 864                                         struct f2fs_summary *sum)
 865 {
 866         struct curseg_info *curseg = CURSEG_I(sbi, type);
 867         void *addr = curseg->sum_blk;
 868         addr += curseg->next_blkoff * sizeof(struct f2fs_summary);
 869         memcpy(addr, sum, sizeof(struct f2fs_summary));
 870 }
 871
 872 /*
 873  * Calculate the number of current summary pages for writing
 874  */
 875 int npages_for_summary_flush(struct f2fs_sb_info *sbi, bool for_ra)
 876 {
 877         int valid_sum_count = 0;
 878         int i, sum_in_page;
 879
 880         for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
 881                 if (sbi->ckpt->alloc_type[i] == SSR)
 882                         valid_sum_count += sbi->blocks_per_seg;
 883                 else {
 884                         if (for_ra)
 885                                 valid_sum_count += le16_to_cpu(
 886                                         F2FS_CKPT(sbi)->cur_data_blkoff[i]);
 887                         else
 888                                 valid_sum_count += curseg_blkoff(sbi, i);
 889                 }
 890         }
 891
 892         sum_in_page = (PAGE_SIZE - 2 * SUM_JOURNAL_SIZE -
 893                         SUM_FOOTER_SIZE) / SUMMARY_SIZE;
 894         if (valid_sum_count <= sum_in_page)
 895                 return 1;
 896         else if ((valid_sum_count - sum_in_page) <=
 897                 (PAGE_SIZE - SUM_FOOTER_SIZE) / SUMMARY_SIZE)
 898                 return 2;
 899         return 3;
 900 }
 901
 902 /*
 903  * Caller should put this summary page
 904  */
 905 struct page *get_sum_page(struct f2fs_sb_info *sbi, unsigned int segno)
 906 {
 907         return get_meta_page(sbi, GET_SUM_BLOCK(sbi, segno));
 908 }
 909
 910 void update_meta_page(struct f2fs_sb_info *sbi, void *src, block_t blk_addr)
 911 {
 912         struct page *page = grab_meta_page(sbi, blk_addr);
 913         void *dst = page_address(page);
 914
 915         if (src)
 916                 memcpy(dst, src, PAGE_SIZE);
 917         else
 918                 memset(dst, 0, PAGE_SIZE);
 919         set_page_dirty(page);
 920         f2fs_put_page(page, 1);
 921 }
 922
 923 static void write_sum_page(struct f2fs_sb_info *sbi,
 924                         struct f2fs_summary_block *sum_blk, block_t blk_addr)
 925 {
 926         update_meta_page(sbi, (void *)sum_blk, blk_addr);
 927 }
 928
 929 static void write_current_sum_page(struct f2fs_sb_info *sbi,
 930                                                 int type, block_t blk_addr)
 931 {
 932         struct curseg_info *curseg = CURSEG_I(sbi, type);
 933         struct page *page = grab_meta_page(sbi, blk_addr);
 934         struct f2fs_summary_block *src = curseg->sum_blk;
 935         struct f2fs_summary_block *dst;
 936
 937         dst = (struct f2fs_summary_block *)page_address(page);
 938
 939         mutex_lock(&curseg->curseg_mutex);
 940
 941         down_read(&curseg->journal_rwsem);
 942         memcpy(&dst->journal, curseg->journal, SUM_JOURNAL_SIZE);
 943         up_read(&curseg->journal_rwsem);
 944
 945         memcpy(dst->entries, src->entries, SUM_ENTRY_SIZE);
 946         memcpy(&dst->footer, &src->footer, SUM_FOOTER_SIZE);
 947
 948         mutex_unlock(&curseg->curseg_mutex);
 949
 950         set_page_dirty(page);
 951         f2fs_put_page(page, 1);
 952 }
 953
 954 static int is_next_segment_free(struct f2fs_sb_info *sbi, int type)
 955 {
 956         struct curseg_info *curseg = CURSEG_I(sbi, type);
 957         unsigned int segno = curseg->segno + 1;
 958         struct free_segmap_info *free_i = FREE_I(sbi);
 959
 960         if (segno < MAIN_SEGS(sbi) && segno % sbi->segs_per_sec)
 961                 return !test_bit(segno, free_i->free_segmap);
 962         return 0;
 963 }
 964
 965 /*
 966  * Find a new segment from the free segments bitmap to right order
 967  * This function should be returned with success, otherwise BUG
 968  */
 969 static void get_new_segment(struct f2fs_sb_info *sbi,
 970                         unsigned int *newseg, bool new_sec, int dir)
 971 {
 972         struct free_segmap_info *free_i = FREE_I(sbi);
 973         unsigned int segno, secno, zoneno;
 974         unsigned int total_zones = MAIN_SECS(sbi) / sbi->secs_per_zone;
 975         unsigned int hint = *newseg / sbi->segs_per_sec;
 976         unsigned int old_zoneno = GET_ZONENO_FROM_SEGNO(sbi, *newseg);
 977         unsigned int left_start = hint;
 978         bool init = true;
 979         int go_left = 0;
 980         int i;
 981
 982         spin_lock(&free_i->segmap_lock);
 983
 984         if (!new_sec && ((*newseg + 1) % sbi->segs_per_sec)) {
 985                 segno = find_next_zero_bit(free_i->free_segmap,
 986                                 (hint + 1) * sbi->segs_per_sec, *newseg + 1);
 987                 if (segno < (hint + 1) * sbi->segs_per_sec)
 988                         goto got_it;
 989         }
 990 find_other_zone:
 991         secno = find_next_zero_bit(free_i->free_secmap, MAIN_SECS(sbi), hint);
 992         if (secno >= MAIN_SECS(sbi)) {
 993                 if (dir == ALLOC_RIGHT) {
 994                         secno = find_next_zero_bit(free_i->free_secmap,
 995                                                         MAIN_SECS(sbi), 0);
 996                         f2fs_bug_on(sbi, secno >= MAIN_SECS(sbi));
 997                 } else {
 998                         go_left = 1;
 999                         left_start = hint - 1;
1000                 }
1001         }
1002         if (go_left == 0)
1003                 goto skip_left;
1004
1005         while (test_bit(left_start, free_i->free_secmap)) {
1006                 if (left_start > 0) {
1007                         left_start--;
1008                         continue;
1009                 }
1010                 left_start = find_next_zero_bit(free_i->free_secmap,
1011                                                         MAIN_SECS(sbi), 0);
1012                 f2fs_bug_on(sbi, left_start >= MAIN_SECS(sbi));
1013                 break;
1014         }
1015         secno = left_start;
1016 skip_left:
1017         hint = secno;
1018         segno = secno * sbi->segs_per_sec;
1019         zoneno = secno / sbi->secs_per_zone;
1020
1021         /* give up on finding another zone */
1022         if (!init)
1023                 goto got_it;
1024         if (sbi->secs_per_zone == 1)
1025                 goto got_it;
1026         if (zoneno == old_zoneno)
1027                 goto got_it;
1028         if (dir == ALLOC_LEFT) {
1029                 if (!go_left && zoneno + 1 >= total_zones)
1030                         goto got_it;
1031                 if (go_left && zoneno == 0)
1032                         goto got_it;
1033         }
1034         for (i = 0; i < NR_CURSEG_TYPE; i++)
1035                 if (CURSEG_I(sbi, i)->zone == zoneno)
1036                         break;
1037
1038         if (i < NR_CURSEG_TYPE) {
1039                 /* zone is in user, try another */
1040                 if (go_left)
1041                         hint = zoneno * sbi->secs_per_zone - 1;
1042                 else if (zoneno + 1 >= total_zones)
1043                         hint = 0;
1044                 else
1045                         hint = (zoneno + 1) * sbi->secs_per_zone;
1046                 init = false;
1047                 goto find_other_zone;
1048         }
1049 got_it:
1050         /* set it as dirty segment in free segmap */
1051         f2fs_bug_on(sbi, test_bit(segno, free_i->free_segmap));
1052         __set_inuse(sbi, segno);
1053         *newseg = segno;
1054         spin_unlock(&free_i->segmap_lock);
1055 }
1056
1057 static void reset_curseg(struct f2fs_sb_info *sbi, int type, int modified)
1058 {
1059         struct curseg_info *curseg = CURSEG_I(sbi, type);
1060         struct summary_footer *sum_footer;
1061
1062         curseg->segno = curseg->next_segno;
1063         curseg->zone = GET_ZONENO_FROM_SEGNO(sbi, curseg->segno);
1064         curseg->next_blkoff = 0;
1065         curseg->next_segno = NULL_SEGNO;
1066
1067         sum_footer = &(curseg->sum_blk->footer);
1068         memset(sum_footer, 0, sizeof(struct summary_footer));
1069         if (IS_DATASEG(type))
1070                 SET_SUM_TYPE(sum_footer, SUM_TYPE_DATA);
1071         if (IS_NODESEG(type))
1072                 SET_SUM_TYPE(sum_footer, SUM_TYPE_NODE);
1073         __set_sit_entry_type(sbi, type, curseg->segno, modified);
1074 }
1075
1076 /*
1077  * Allocate a current working segment.
1078  * This function always allocates a free segment in LFS manner.
1079  */
1080 static void new_curseg(struct f2fs_sb_info *sbi, int type, bool new_sec)
1081 {
1082         struct curseg_info *curseg = CURSEG_I(sbi, type);
1083         unsigned int segno = curseg->segno;
1084         int dir = ALLOC_LEFT;
1085
1086         write_sum_page(sbi, curseg->sum_blk,
1087                                 GET_SUM_BLOCK(sbi, segno));
1088         if (type == CURSEG_WARM_DATA || type == CURSEG_COLD_DATA)
1089                 dir = ALLOC_RIGHT;
1090
1091         if (test_opt(sbi, NOHEAP))
1092                 dir = ALLOC_RIGHT;
1093
1094         get_new_segment(sbi, &segno, new_sec, dir);
1095         curseg->next_segno = segno;
1096         reset_curseg(sbi, type, 1);
1097         curseg->alloc_type = LFS;
1098 }
1099
1100 static void __next_free_blkoff(struct f2fs_sb_info *sbi,
1101                         struct curseg_info *seg, block_t start)
1102 {
1103         struct seg_entry *se = get_seg_entry(sbi, seg->segno);
1104         int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long);
1105         unsigned long *target_map = SIT_I(sbi)->tmp_map;
1106         unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map;
1107         unsigned long *cur_map = (unsigned long *)se->cur_valid_map;
1108         int i, pos;
1109
1110         for (i = 0; i < entries; i++)
1111                 target_map[i] = ckpt_map[i] | cur_map[i];
1112
1113         pos = __find_rev_next_zero_bit(target_map, sbi->blocks_per_seg, start);
1114
1115         seg->next_blkoff = pos;
1116 }
1117
1118 /*
1119  * If a segment is written by LFS manner, next block offset is just obtained
1120  * by increasing the current block offset. However, if a segment is written by
1121  * SSR manner, next block offset obtained by calling __next_free_blkoff
1122  */
1123 static void __refresh_next_blkoff(struct f2fs_sb_info *sbi,
1124                                 struct curseg_info *seg)
1125 {
1126         if (seg->alloc_type == SSR)
1127                 __next_free_blkoff(sbi, seg, seg->next_blkoff + 1);
1128         else
1129                 seg->next_blkoff++;
1130 }
1131
1132 /*
1133  * This function always allocates a used segment(from dirty seglist) by SSR
1134  * manner, so it should recover the existing segment information of valid blocks
1135  */
1136 static void change_curseg(struct f2fs_sb_info *sbi, int type, bool reuse)
1137 {
1138         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
1139         struct curseg_info *curseg = CURSEG_I(sbi, type);
1140         unsigned int new_segno = curseg->next_segno;
1141         struct f2fs_summary_block *sum_node;
1142         struct page *sum_page;
1143
1144         write_sum_page(sbi, curseg->sum_blk,
1145                                 GET_SUM_BLOCK(sbi, curseg->segno));
1146         __set_test_and_inuse(sbi, new_segno);
1147
1148         mutex_lock(&dirty_i->seglist_lock);
1149         __remove_dirty_segment(sbi, new_segno, PRE);
1150         __remove_dirty_segment(sbi, new_segno, DIRTY);
1151         mutex_unlock(&dirty_i->seglist_lock);
1152
1153         reset_curseg(sbi, type, 1);
1154         curseg->alloc_type = SSR;
1155         __next_free_blkoff(sbi, curseg, 0);
1156
1157         if (reuse) {
1158                 sum_page = get_sum_page(sbi, new_segno);
1159                 sum_node = (struct f2fs_summary_block *)page_address(sum_page);
1160                 memcpy(curseg->sum_blk, sum_node, SUM_ENTRY_SIZE);
1161                 f2fs_put_page(sum_page, 1);
1162         }
1163 }
1164
1165 static int get_ssr_segment(struct f2fs_sb_info *sbi, int type)
1166 {
1167         struct curseg_info *curseg = CURSEG_I(sbi, type);
1168         const struct victim_selection *v_ops = DIRTY_I(sbi)->v_ops;
1169
1170         if (IS_NODESEG(type) || !has_not_enough_free_secs(sbi, 0))
1171                 return v_ops->get_victim(sbi,
1172                                 &(curseg)->next_segno, BG_GC, type, SSR);
1173
1174         /* For data segments, let's do SSR more intensively */
1175         for (; type >= CURSEG_HOT_DATA; type--)
1176                 if (v_ops->get_victim(sbi, &(curseg)->next_segno,
1177                                                 BG_GC, type, SSR))
1178                         return 1;
1179         return 0;
1180 }
1181
1182 /*
1183  * flush out current segment and replace it with new segment
1184  * This function should be returned with success, otherwise BUG
1185  */
1186 static void allocate_segment_by_default(struct f2fs_sb_info *sbi,
1187                                                 int type, bool force)
1188 {
1189         struct curseg_info *curseg = CURSEG_I(sbi, type);
1190
1191         if (force)
1192                 new_curseg(sbi, type, true);
1193         else if (type == CURSEG_WARM_NODE)
1194                 new_curseg(sbi, type, false);
1195         else if (curseg->alloc_type == LFS && is_next_segment_free(sbi, type))
1196                 new_curseg(sbi, type, false);
1197         else if (need_SSR(sbi) && get_ssr_segment(sbi, type))
1198                 change_curseg(sbi, type, true);
1199         else
1200                 new_curseg(sbi, type, false);
1201
1202         stat_inc_seg_type(sbi, curseg);
1203 }
1204
1205 static void __allocate_new_segments(struct f2fs_sb_info *sbi, int type)
1206 {
1207         struct curseg_info *curseg = CURSEG_I(sbi, type);
1208         unsigned int old_segno;
1209
1210         old_segno = curseg->segno;
1211         SIT_I(sbi)->s_ops->allocate_segment(sbi, type, true);
1212         locate_dirty_segment(sbi, old_segno);
1213 }
1214
1215 void allocate_new_segments(struct f2fs_sb_info *sbi)
1216 {
1217         int i;
1218
1219         for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++)
1220                 __allocate_new_segments(sbi, i);
1221 }
1222
1223 static const struct segment_allocation default_salloc_ops = {
1224         .allocate_segment = allocate_segment_by_default,
1225 };
1226
1227 int f2fs_trim_fs(struct f2fs_sb_info *sbi, struct fstrim_range *range)
1228 {
1229         __u64 start = F2FS_BYTES_TO_BLK(range->start);
1230         __u64 end = start + F2FS_BYTES_TO_BLK(range->len) - 1;
1231         unsigned int start_segno, end_segno;
1232         struct cp_control cpc;
1233         int err = 0;
1234
1235         if (start >= MAX_BLKADDR(sbi) || range->len < sbi->blocksize)
1236                 return -EINVAL;
1237
1238         cpc.trimmed = 0;
1239         if (end <= MAIN_BLKADDR(sbi))
1240                 goto out;
1241
1242         /* start/end segment number in main_area */
1243         start_segno = (start <= MAIN_BLKADDR(sbi)) ? 0 : GET_SEGNO(sbi, start);
1244         end_segno = (end >= MAX_BLKADDR(sbi)) ? MAIN_SEGS(sbi) - 1 :
1245                                                 GET_SEGNO(sbi, end);
1246         cpc.reason = CP_DISCARD;
1247         cpc.trim_minlen = max_t(__u64, 1, F2FS_BYTES_TO_BLK(range->minlen));
1248
1249         /* do checkpoint to issue discard commands safely */
1250         for (; start_segno <= end_segno; start_segno = cpc.trim_end + 1) {
1251                 cpc.trim_start = start_segno;
1252
1253                 if (sbi->discard_blks == 0)
1254                         break;
1255                 else if (sbi->discard_blks < BATCHED_TRIM_BLOCKS(sbi))
1256                         cpc.trim_end = end_segno;
1257                 else
1258                         cpc.trim_end = min_t(unsigned int,
1259                                 rounddown(start_segno +
1260                                 BATCHED_TRIM_SEGMENTS(sbi),
1261                                 sbi->segs_per_sec) - 1, end_segno);
1262
1263                 mutex_lock(&sbi->gc_mutex);
1264                 err = write_checkpoint(sbi, &cpc);
1265                 mutex_unlock(&sbi->gc_mutex);
1266         }
1267 out:
1268         range->len = F2FS_BLK_TO_BYTES(cpc.trimmed);
1269         return err;
1270 }
1271
1272 static bool __has_curseg_space(struct f2fs_sb_info *sbi, int type)
1273 {
1274         struct curseg_info *curseg = CURSEG_I(sbi, type);
1275         if (curseg->next_blkoff < sbi->blocks_per_seg)
1276                 return true;
1277         return false;
1278 }
1279
1280 static int __get_segment_type_2(struct page *page, enum page_type p_type)
1281 {
1282         if (p_type == DATA)
1283                 return CURSEG_HOT_DATA;
1284         else
1285                 return CURSEG_HOT_NODE;
1286 }
1287
1288 static int __get_segment_type_4(struct page *page, enum page_type p_type)
1289 {
1290         if (p_type == DATA) {
1291                 struct inode *inode = page->mapping->host;
1292
1293                 if (S_ISDIR(inode->i_mode))
1294                         return CURSEG_HOT_DATA;
1295                 else
1296                         return CURSEG_COLD_DATA;
1297         } else {
1298                 if (IS_DNODE(page) && is_cold_node(page))
1299                         return CURSEG_WARM_NODE;
1300                 else
1301                         return CURSEG_COLD_NODE;
1302         }
1303 }
1304
1305 static int __get_segment_type_6(struct page *page, enum page_type p_type)
1306 {
1307         if (p_type == DATA) {
1308                 struct inode *inode = page->mapping->host;
1309
1310                 if (S_ISDIR(inode->i_mode))
1311                         return CURSEG_HOT_DATA;
1312                 else if (is_cold_data(page) || file_is_cold(inode))
1313                         return CURSEG_COLD_DATA;
1314                 else
1315                         return CURSEG_WARM_DATA;
1316         } else {
1317                 if (IS_DNODE(page))
1318                         return is_cold_node(page) ? CURSEG_WARM_NODE :
1319                                                 CURSEG_HOT_NODE;
1320                 else
1321                         return CURSEG_COLD_NODE;
1322         }
1323 }
1324
1325 static int __get_segment_type(struct page *page, enum page_type p_type)
1326 {
1327         switch (F2FS_P_SB(page)->active_logs) {
1328         case 2:
1329                 return __get_segment_type_2(page, p_type);
1330         case 4:
1331                 return __get_segment_type_4(page, p_type);
1332         }
1333         /* NR_CURSEG_TYPE(6) logs by default */
1334         f2fs_bug_on(F2FS_P_SB(page),
1335                 F2FS_P_SB(page)->active_logs != NR_CURSEG_TYPE);
1336         return __get_segment_type_6(page, p_type);
1337 }
1338
1339 void allocate_data_block(struct f2fs_sb_info *sbi, struct page *page,
1340                 block_t old_blkaddr, block_t *new_blkaddr,
1341                 struct f2fs_summary *sum, int type)
1342 {
1343         struct sit_info *sit_i = SIT_I(sbi);
1344         struct curseg_info *curseg;
1345         bool direct_io = (type == CURSEG_DIRECT_IO);
1346
1347         type = direct_io ? CURSEG_WARM_DATA : type;
1348
1349         curseg = CURSEG_I(sbi, type);
1350
1351         mutex_lock(&curseg->curseg_mutex);
1352         mutex_lock(&sit_i->sentry_lock);
1353
1354         /* direct_io'ed data is aligned to the segment for better performance */
1355         if (direct_io && curseg->next_blkoff &&
1356                                 !has_not_enough_free_secs(sbi, 0))
1357                 __allocate_new_segments(sbi, type);
1358
1359         *new_blkaddr = NEXT_FREE_BLKADDR(sbi, curseg);
1360
1361         /*
1362          * __add_sum_entry should be resided under the curseg_mutex
1363          * because, this function updates a summary entry in the
1364          * current summary block.
1365          */
1366         __add_sum_entry(sbi, type, sum);
1367
1368         __refresh_next_blkoff(sbi, curseg);
1369
1370         stat_inc_block_count(sbi, curseg);
1371
1372         if (!__has_curseg_space(sbi, type))
1373                 sit_i->s_ops->allocate_segment(sbi, type, false);
1374         /*
1375          * SIT information should be updated before segment allocation,
1376          * since SSR needs latest valid block information.
1377          */
1378         refresh_sit_entry(sbi, old_blkaddr, *new_blkaddr);
1379
1380         mutex_unlock(&sit_i->sentry_lock);
1381
1382         if (page && IS_NODESEG(type))
1383                 fill_node_footer_blkaddr(page, NEXT_FREE_BLKADDR(sbi, curseg));
1384
1385         mutex_unlock(&curseg->curseg_mutex);
1386 }
1387
1388 static void do_write_page(struct f2fs_summary *sum, struct f2fs_io_info *fio)
1389 {
1390         int type = __get_segment_type(fio->page, fio->type);
1391
1392         allocate_data_block(fio->sbi, fio->page, fio->old_blkaddr,
1393                                         &fio->new_blkaddr, sum, type);
1394
1395         /* writeout dirty page into bdev */
1396         f2fs_submit_page_mbio(fio);
1397 }
1398
1399 void write_meta_page(struct f2fs_sb_info *sbi, struct page *page)
1400 {
1401         struct f2fs_io_info fio = {
1402                 .sbi = sbi,
1403                 .type = META,
1404                 .rw = WRITE_SYNC | REQ_META | REQ_PRIO,
1405                 .old_blkaddr = page->index,
1406                 .new_blkaddr = page->index,
1407                 .page = page,
1408                 .encrypted_page = NULL,
1409         };
1410
1411         if (unlikely(page->index >= MAIN_BLKADDR(sbi)))
1412                 fio.rw &= ~REQ_META;
1413
1414         set_page_writeback(page);
1415         f2fs_submit_page_mbio(&fio);
1416 }
1417
1418 void write_node_page(unsigned int nid, struct f2fs_io_info *fio)
1419 {
1420         struct f2fs_summary sum;
1421
1422         set_summary(&sum, nid, 0, 0);
1423         do_write_page(&sum, fio);
1424 }
1425
1426 void write_data_page(struct dnode_of_data *dn, struct f2fs_io_info *fio)
1427 {
1428         struct f2fs_sb_info *sbi = fio->sbi;
1429         struct f2fs_summary sum;
1430         struct node_info ni;
1431
1432         f2fs_bug_on(sbi, dn->data_blkaddr == NULL_ADDR);
1433         get_node_info(sbi, dn->nid, &ni);
1434         set_summary(&sum, dn->nid, dn->ofs_in_node, ni.version);
1435         do_write_page(&sum, fio);
1436         f2fs_update_data_blkaddr(dn, fio->new_blkaddr);
1437 }
1438
1439 void rewrite_data_page(struct f2fs_io_info *fio)
1440 {
1441         fio->new_blkaddr = fio->old_blkaddr;
1442         stat_inc_inplace_blocks(fio->sbi);
1443         f2fs_submit_page_mbio(fio);
1444 }
1445
1446 void __f2fs_replace_block(struct f2fs_sb_info *sbi, struct f2fs_summary *sum,
1447                                 block_t old_blkaddr, block_t new_blkaddr,
1448                                 bool recover_curseg, bool recover_newaddr)
1449 {
1450         struct sit_info *sit_i = SIT_I(sbi);
1451         struct curseg_info *curseg;
1452         unsigned int segno, old_cursegno;
1453         struct seg_entry *se;
1454         int type;
1455         unsigned short old_blkoff;
1456
1457         segno = GET_SEGNO(sbi, new_blkaddr);
1458         se = get_seg_entry(sbi, segno);
1459         type = se->type;
1460
1461         if (!recover_curseg) {
1462                 /* for recovery flow */
1463                 if (se->valid_blocks == 0 && !IS_CURSEG(sbi, segno)) {
1464                         if (old_blkaddr == NULL_ADDR)
1465                                 type = CURSEG_COLD_DATA;
1466                         else
1467                                 type = CURSEG_WARM_DATA;
1468                 }
1469         } else {
1470                 if (!IS_CURSEG(sbi, segno))
1471                         type = CURSEG_WARM_DATA;
1472         }
1473
1474         curseg = CURSEG_I(sbi, type);
1475
1476         mutex_lock(&curseg->curseg_mutex);
1477         mutex_lock(&sit_i->sentry_lock);
1478
1479         old_cursegno = curseg->segno;
1480         old_blkoff = curseg->next_blkoff;
1481
1482         /* change the current segment */
1483         if (segno != curseg->segno) {
1484                 curseg->next_segno = segno;
1485                 change_curseg(sbi, type, true);
1486         }
1487
1488         curseg->next_blkoff = GET_BLKOFF_FROM_SEG0(sbi, new_blkaddr);
1489         __add_sum_entry(sbi, type, sum);
1490
1491         if (!recover_curseg || recover_newaddr)
1492                 update_sit_entry(sbi, new_blkaddr, 1);
1493         if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO)
1494                 update_sit_entry(sbi, old_blkaddr, -1);
1495
1496         locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr));
1497         locate_dirty_segment(sbi, GET_SEGNO(sbi, new_blkaddr));
1498
1499         locate_dirty_segment(sbi, old_cursegno);
1500
1501         if (recover_curseg) {
1502                 if (old_cursegno != curseg->segno) {
1503                         curseg->next_segno = old_cursegno;
1504                         change_curseg(sbi, type, true);
1505                 }
1506                 curseg->next_blkoff = old_blkoff;
1507         }
1508
1509         mutex_unlock(&sit_i->sentry_lock);
1510         mutex_unlock(&curseg->curseg_mutex);
1511 }
1512
1513 void f2fs_replace_block(struct f2fs_sb_info *sbi, struct dnode_of_data *dn,
1514                                 block_t old_addr, block_t new_addr,
1515                                 unsigned char version, bool recover_curseg,
1516                                 bool recover_newaddr)
1517 {
1518         struct f2fs_summary sum;
1519
1520         set_summary(&sum, dn->nid, dn->ofs_in_node, version);
1521
1522         __f2fs_replace_block(sbi, &sum, old_addr, new_addr,
1523                                         recover_curseg, recover_newaddr);
1524
1525         f2fs_update_data_blkaddr(dn, new_addr);
1526 }
1527
1528 void f2fs_wait_on_page_writeback(struct page *page,
1529                                 enum page_type type, bool ordered)
1530 {
1531         if (PageWriteback(page)) {
1532                 struct f2fs_sb_info *sbi = F2FS_P_SB(page);
1533
1534                 f2fs_submit_merged_bio_cond(sbi, NULL, page, 0, type, WRITE);
1535                 if (ordered)
1536                         wait_on_page_writeback(page);
1537                 else
1538                         wait_for_stable_page(page);
1539         }
1540 }
1541
1542 void f2fs_wait_on_encrypted_page_writeback(struct f2fs_sb_info *sbi,
1543                                                         block_t blkaddr)
1544 {
1545         struct page *cpage;
1546
1547         if (blkaddr == NEW_ADDR)
1548                 return;
1549
1550         f2fs_bug_on(sbi, blkaddr == NULL_ADDR);
1551
1552         cpage = find_lock_page(META_MAPPING(sbi), blkaddr);
1553         if (cpage) {
1554                 f2fs_wait_on_page_writeback(cpage, DATA, true);
1555                 f2fs_put_page(cpage, 1);
1556         }
1557 }
1558
1559 static int read_compacted_summaries(struct f2fs_sb_info *sbi)
1560 {
1561         struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
1562         struct curseg_info *seg_i;
1563         unsigned char *kaddr;
1564         struct page *page;
1565         block_t start;
1566         int i, j, offset;
1567
1568         start = start_sum_block(sbi);
1569
1570         page = get_meta_page(sbi, start++);
1571         kaddr = (unsigned char *)page_address(page);
1572
1573         /* Step 1: restore nat cache */
1574         seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA);
1575         memcpy(seg_i->journal, kaddr, SUM_JOURNAL_SIZE);
1576
1577         /* Step 2: restore sit cache */
1578         seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA);
1579         memcpy(seg_i->journal, kaddr + SUM_JOURNAL_SIZE, SUM_JOURNAL_SIZE);
1580         offset = 2 * SUM_JOURNAL_SIZE;
1581
1582         /* Step 3: restore summary entries */
1583         for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
1584                 unsigned short blk_off;
1585                 unsigned int segno;
1586
1587                 seg_i = CURSEG_I(sbi, i);
1588                 segno = le32_to_cpu(ckpt->cur_data_segno[i]);
1589                 blk_off = le16_to_cpu(ckpt->cur_data_blkoff[i]);
1590                 seg_i->next_segno = segno;
1591                 reset_curseg(sbi, i, 0);
1592                 seg_i->alloc_type = ckpt->alloc_type[i];
1593                 seg_i->next_blkoff = blk_off;
1594
1595                 if (seg_i->alloc_type == SSR)
1596                         blk_off = sbi->blocks_per_seg;
1597
1598                 for (j = 0; j < blk_off; j++) {
1599                         struct f2fs_summary *s;
1600                         s = (struct f2fs_summary *)(kaddr + offset);
1601                         seg_i->sum_blk->entries[j] = *s;
1602                         offset += SUMMARY_SIZE;
1603                         if (offset + SUMMARY_SIZE <= PAGE_SIZE -
1604                                                 SUM_FOOTER_SIZE)
1605                                 continue;
1606
1607                         f2fs_put_page(page, 1);
1608                         page = NULL;
1609
1610                         page = get_meta_page(sbi, start++);
1611                         kaddr = (unsigned char *)page_address(page);
1612                         offset = 0;
1613                 }
1614         }
1615         f2fs_put_page(page, 1);
1616         return 0;
1617 }
1618
1619 static int read_normal_summaries(struct f2fs_sb_info *sbi, int type)
1620 {
1621         struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
1622         struct f2fs_summary_block *sum;
1623         struct curseg_info *curseg;
1624         struct page *new;
1625         unsigned short blk_off;
1626         unsigned int segno = 0;
1627         block_t blk_addr = 0;
1628
1629         /* get segment number and block addr */
1630         if (IS_DATASEG(type)) {
1631                 segno = le32_to_cpu(ckpt->cur_data_segno[type]);
1632                 blk_off = le16_to_cpu(ckpt->cur_data_blkoff[type -
1633                                                         CURSEG_HOT_DATA]);
1634                 if (__exist_node_summaries(sbi))
1635                         blk_addr = sum_blk_addr(sbi, NR_CURSEG_TYPE, type);
1636                 else
1637                         blk_addr = sum_blk_addr(sbi, NR_CURSEG_DATA_TYPE, type);
1638         } else {
1639                 segno = le32_to_cpu(ckpt->cur_node_segno[type -
1640                                                         CURSEG_HOT_NODE]);
1641                 blk_off = le16_to_cpu(ckpt->cur_node_blkoff[type -
1642                                                         CURSEG_HOT_NODE]);
1643                 if (__exist_node_summaries(sbi))
1644                         blk_addr = sum_blk_addr(sbi, NR_CURSEG_NODE_TYPE,
1645                                                         type - CURSEG_HOT_NODE);
1646                 else
1647                         blk_addr = GET_SUM_BLOCK(sbi, segno);
1648         }
1649
1650         new = get_meta_page(sbi, blk_addr);
1651         sum = (struct f2fs_summary_block *)page_address(new);
1652
1653         if (IS_NODESEG(type)) {
1654                 if (__exist_node_summaries(sbi)) {
1655                         struct f2fs_summary *ns = &sum->entries[0];
1656                         int i;
1657                         for (i = 0; i < sbi->blocks_per_seg; i++, ns++) {
1658                                 ns->version = 0;
1659                                 ns->ofs_in_node = 0;
1660                         }
1661                 } else {
1662                         int err;
1663
1664                         err = restore_node_summary(sbi, segno, sum);
1665                         if (err) {
1666                                 f2fs_put_page(new, 1);
1667                                 return err;
1668                         }
1669                 }
1670         }
1671
1672         /* set uncompleted segment to curseg */
1673         curseg = CURSEG_I(sbi, type);
1674         mutex_lock(&curseg->curseg_mutex);
1675
1676         /* update journal info */
1677         down_write(&curseg->journal_rwsem);
1678         memcpy(curseg->journal, &sum->journal, SUM_JOURNAL_SIZE);
1679         up_write(&curseg->journal_rwsem);
1680
1681         memcpy(curseg->sum_blk->entries, sum->entries, SUM_ENTRY_SIZE);
1682         memcpy(&curseg->sum_blk->footer, &sum->footer, SUM_FOOTER_SIZE);
1683         curseg->next_segno = segno;
1684         reset_curseg(sbi, type, 0);
1685         curseg->alloc_type = ckpt->alloc_type[type];
1686         curseg->next_blkoff = blk_off;
1687         mutex_unlock(&curseg->curseg_mutex);
1688         f2fs_put_page(new, 1);
1689         return 0;
1690 }
1691
1692 static int restore_curseg_summaries(struct f2fs_sb_info *sbi)
1693 {
1694         int type = CURSEG_HOT_DATA;
1695         int err;
1696
1697         if (is_set_ckpt_flags(F2FS_CKPT(sbi), CP_COMPACT_SUM_FLAG)) {
1698                 int npages = npages_for_summary_flush(sbi, true);
1699
1700                 if (npages >= 2)
1701                         ra_meta_pages(sbi, start_sum_block(sbi), npages,
1702                                                         META_CP, true);
1703
1704                 /* restore for compacted data summary */
1705                 if (read_compacted_summaries(sbi))
1706                         return -EINVAL;
1707                 type = CURSEG_HOT_NODE;
1708         }
1709
1710         if (__exist_node_summaries(sbi))
1711                 ra_meta_pages(sbi, sum_blk_addr(sbi, NR_CURSEG_TYPE, type),
1712                                         NR_CURSEG_TYPE - type, META_CP, true);
1713
1714         for (; type <= CURSEG_COLD_NODE; type++) {
1715                 err = read_normal_summaries(sbi, type);
1716                 if (err)
1717                         return err;
1718         }
1719
1720         return 0;
1721 }
1722
1723 static void write_compacted_summaries(struct f2fs_sb_info *sbi, block_t blkaddr)
1724 {
1725         struct page *page;
1726         unsigned char *kaddr;
1727         struct f2fs_summary *summary;
1728         struct curseg_info *seg_i;
1729         int written_size = 0;
1730         int i, j;
1731
1732         page = grab_meta_page(sbi, blkaddr++);
1733         kaddr = (unsigned char *)page_address(page);
1734
1735         /* Step 1: write nat cache */
1736         seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA);
1737         memcpy(kaddr, seg_i->journal, SUM_JOURNAL_SIZE);
1738         written_size += SUM_JOURNAL_SIZE;
1739
1740         /* Step 2: write sit cache */
1741         seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA);
1742         memcpy(kaddr + written_size, seg_i->journal, SUM_JOURNAL_SIZE);
1743         written_size += SUM_JOURNAL_SIZE;
1744
1745         /* Step 3: write summary entries */
1746         for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
1747                 unsigned short blkoff;
1748                 seg_i = CURSEG_I(sbi, i);
1749                 if (sbi->ckpt->alloc_type[i] == SSR)
1750                         blkoff = sbi->blocks_per_seg;
1751                 else
1752                         blkoff = curseg_blkoff(sbi, i);
1753
1754                 for (j = 0; j < blkoff; j++) {
1755                         if (!page) {
1756                                 page = grab_meta_page(sbi, blkaddr++);
1757                                 kaddr = (unsigned char *)page_address(page);
1758                                 written_size = 0;
1759                         }
1760                         summary = (struct f2fs_summary *)(kaddr + written_size);
1761                         *summary = seg_i->sum_blk->entries[j];
1762                         written_size += SUMMARY_SIZE;
1763
1764                         if (written_size + SUMMARY_SIZE <= PAGE_SIZE -
1765                                                         SUM_FOOTER_SIZE)
1766                                 continue;
1767
1768                         set_page_dirty(page);
1769                         f2fs_put_page(page, 1);
1770                         page = NULL;
1771                 }
1772         }
1773         if (page) {
1774                 set_page_dirty(page);
1775                 f2fs_put_page(page, 1);
1776         }
1777 }
1778
1779 static void write_normal_summaries(struct f2fs_sb_info *sbi,
1780                                         block_t blkaddr, int type)
1781 {
1782         int i, end;
1783         if (IS_DATASEG(type))
1784                 end = type + NR_CURSEG_DATA_TYPE;
1785         else
1786                 end = type + NR_CURSEG_NODE_TYPE;
1787
1788         for (i = type; i < end; i++)
1789                 write_current_sum_page(sbi, i, blkaddr + (i - type));
1790 }
1791
1792 void write_data_summaries(struct f2fs_sb_info *sbi, block_t start_blk)
1793 {
1794         if (is_set_ckpt_flags(F2FS_CKPT(sbi), CP_COMPACT_SUM_FLAG))
1795                 write_compacted_summaries(sbi, start_blk);
1796         else
1797                 write_normal_summaries(sbi, start_blk, CURSEG_HOT_DATA);
1798 }
1799
1800 void write_node_summaries(struct f2fs_sb_info *sbi, block_t start_blk)
1801 {
1802         write_normal_summaries(sbi, start_blk, CURSEG_HOT_NODE);
1803 }
1804
1805 int lookup_journal_in_cursum(struct f2fs_journal *journal, int type,
1806                                         unsigned int val, int alloc)
1807 {
1808         int i;
1809
1810         if (type == NAT_JOURNAL) {
1811                 for (i = 0; i < nats_in_cursum(journal); i++) {
1812                         if (le32_to_cpu(nid_in_journal(journal, i)) == val)
1813                                 return i;
1814                 }
1815                 if (alloc && __has_cursum_space(journal, 1, NAT_JOURNAL))
1816                         return update_nats_in_cursum(journal, 1);
1817         } else if (type == SIT_JOURNAL) {
1818                 for (i = 0; i < sits_in_cursum(journal); i++)
1819                         if (le32_to_cpu(segno_in_journal(journal, i)) == val)
1820                                 return i;
1821                 if (alloc && __has_cursum_space(journal, 1, SIT_JOURNAL))
1822                         return update_sits_in_cursum(journal, 1);
1823         }
1824         return -1;
1825 }
1826
1827 static struct page *get_current_sit_page(struct f2fs_sb_info *sbi,
1828                                         unsigned int segno)
1829 {
1830         return get_meta_page(sbi, current_sit_addr(sbi, segno));
1831 }
1832
1833 static struct page *get_next_sit_page(struct f2fs_sb_info *sbi,
1834                                         unsigned int start)
1835 {
1836         struct sit_info *sit_i = SIT_I(sbi);
1837         struct page *src_page, *dst_page;
1838         pgoff_t src_off, dst_off;
1839         void *src_addr, *dst_addr;
1840
1841         src_off = current_sit_addr(sbi, start);
1842         dst_off = next_sit_addr(sbi, src_off);
1843
1844         /* get current sit block page without lock */
1845         src_page = get_meta_page(sbi, src_off);
1846         dst_page = grab_meta_page(sbi, dst_off);
1847         f2fs_bug_on(sbi, PageDirty(src_page));
1848
1849         src_addr = page_address(src_page);
1850         dst_addr = page_address(dst_page);
1851         memcpy(dst_addr, src_addr, PAGE_SIZE);
1852
1853         set_page_dirty(dst_page);
1854         f2fs_put_page(src_page, 1);
1855
1856         set_to_next_sit(sit_i, start);
1857
1858         return dst_page;
1859 }
1860
1861 static struct sit_entry_set *grab_sit_entry_set(void)
1862 {
1863         struct sit_entry_set *ses =
1864                         f2fs_kmem_cache_alloc(sit_entry_set_slab, GFP_NOFS);
1865
1866         ses->entry_cnt = 0;
1867         INIT_LIST_HEAD(&ses->set_list);
1868         return ses;
1869 }
1870
1871 static void release_sit_entry_set(struct sit_entry_set *ses)
1872 {
1873         list_del(&ses->set_list);
1874         kmem_cache_free(sit_entry_set_slab, ses);
1875 }
1876
1877 static void adjust_sit_entry_set(struct sit_entry_set *ses,
1878                                                 struct list_head *head)
1879 {
1880         struct sit_entry_set *next = ses;
1881
1882         if (list_is_last(&ses->set_list, head))
1883                 return;
1884
1885         list_for_each_entry_continue(next, head, set_list)
1886                 if (ses->entry_cnt <= next->entry_cnt)
1887                         break;
1888
1889         list_move_tail(&ses->set_list, &next->set_list);
1890 }
1891
1892 static void add_sit_entry(unsigned int segno, struct list_head *head)
1893 {
1894         struct sit_entry_set *ses;
1895         unsigned int start_segno = START_SEGNO(segno);
1896
1897         list_for_each_entry(ses, head, set_list) {
1898                 if (ses->start_segno == start_segno) {
1899                         ses->entry_cnt++;
1900                         adjust_sit_entry_set(ses, head);
1901                         return;
1902                 }
1903         }
1904
1905         ses = grab_sit_entry_set();
1906
1907         ses->start_segno = start_segno;
1908         ses->entry_cnt++;
1909         list_add(&ses->set_list, head);
1910 }
1911
1912 static void add_sits_in_set(struct f2fs_sb_info *sbi)
1913 {
1914         struct f2fs_sm_info *sm_info = SM_I(sbi);
1915         struct list_head *set_list = &sm_info->sit_entry_set;
1916         unsigned long *bitmap = SIT_I(sbi)->dirty_sentries_bitmap;
1917         unsigned int segno;
1918
1919         for_each_set_bit(segno, bitmap, MAIN_SEGS(sbi))
1920                 add_sit_entry(segno, set_list);
1921 }
1922
1923 static void remove_sits_in_journal(struct f2fs_sb_info *sbi)
1924 {
1925         struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
1926         struct f2fs_journal *journal = curseg->journal;
1927         int i;
1928
1929         down_write(&curseg->journal_rwsem);
1930         for (i = 0; i < sits_in_cursum(journal); i++) {
1931                 unsigned int segno;
1932                 bool dirtied;
1933
1934                 segno = le32_to_cpu(segno_in_journal(journal, i));
1935                 dirtied = __mark_sit_entry_dirty(sbi, segno);
1936
1937                 if (!dirtied)
1938                         add_sit_entry(segno, &SM_I(sbi)->sit_entry_set);
1939         }
1940         update_sits_in_cursum(journal, -i);
1941         up_write(&curseg->journal_rwsem);
1942 }
1943
1944 /*
1945  * CP calls this function, which flushes SIT entries including sit_journal,
1946  * and moves prefree segs to free segs.
1947  */
1948 void flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc)
1949 {
1950         struct sit_info *sit_i = SIT_I(sbi);
1951         unsigned long *bitmap = sit_i->dirty_sentries_bitmap;
1952         struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
1953         struct f2fs_journal *journal = curseg->journal;
1954         struct sit_entry_set *ses, *tmp;
1955         struct list_head *head = &SM_I(sbi)->sit_entry_set;
1956         bool to_journal = true;
1957         struct seg_entry *se;
1958
1959         mutex_lock(&sit_i->sentry_lock);
1960
1961         if (!sit_i->dirty_sentries)
1962                 goto out;
1963
1964         /*
1965          * add and account sit entries of dirty bitmap in sit entry
1966          * set temporarily
1967          */
1968         add_sits_in_set(sbi);
1969
1970         /*
1971          * if there are no enough space in journal to store dirty sit
1972          * entries, remove all entries from journal and add and account
1973          * them in sit entry set.
1974          */
1975         if (!__has_cursum_space(journal, sit_i->dirty_sentries, SIT_JOURNAL))
1976                 remove_sits_in_journal(sbi);
1977
1978         /*
1979          * there are two steps to flush sit entries:
1980          * #1, flush sit entries to journal in current cold data summary block.
1981          * #2, flush sit entries to sit page.
1982          */
1983         list_for_each_entry_safe(ses, tmp, head, set_list) {
1984                 struct page *page = NULL;
1985                 struct f2fs_sit_block *raw_sit = NULL;
1986                 unsigned int start_segno = ses->start_segno;
1987                 unsigned int end = min(start_segno + SIT_ENTRY_PER_BLOCK,
1988                                                 (unsigned long)MAIN_SEGS(sbi));
1989                 unsigned int segno = start_segno;
1990
1991                 if (to_journal &&
1992                         !__has_cursum_space(journal, ses->entry_cnt, SIT_JOURNAL))
1993                         to_journal = false;
1994
1995                 if (to_journal) {
1996                         down_write(&curseg->journal_rwsem);
1997                 } else {
1998                         page = get_next_sit_page(sbi, start_segno);
1999                         raw_sit = page_address(page);
2000                 }
2001
2002                 /* flush dirty sit entries in region of current sit set */
2003                 for_each_set_bit_from(segno, bitmap, end) {
2004                         int offset, sit_offset;
2005
2006                         se = get_seg_entry(sbi, segno);
2007
2008                         /* add discard candidates */
2009                         if (cpc->reason != CP_DISCARD) {
2010                                 cpc->trim_start = segno;
2011                                 add_discard_addrs(sbi, cpc);
2012                         }
2013
2014                         if (to_journal) {
2015                                 offset = lookup_journal_in_cursum(journal,
2016                                                         SIT_JOURNAL, segno, 1);
2017                                 f2fs_bug_on(sbi, offset < 0);
2018                                 segno_in_journal(journal, offset) =
2019                                                         cpu_to_le32(segno);
2020                                 seg_info_to_raw_sit(se,
2021                                         &sit_in_journal(journal, offset));
2022                         } else {
2023                                 sit_offset = SIT_ENTRY_OFFSET(sit_i, segno);
2024                                 seg_info_to_raw_sit(se,
2025                                                 &raw_sit->entries[sit_offset]);
2026                         }
2027
2028                         __clear_bit(segno, bitmap);
2029                         sit_i->dirty_sentries--;
2030                         ses->entry_cnt--;
2031                 }
2032
2033                 if (to_journal)
2034                         up_write(&curseg->journal_rwsem);
2035                 else
2036                         f2fs_put_page(page, 1);
2037
2038                 f2fs_bug_on(sbi, ses->entry_cnt);
2039                 release_sit_entry_set(ses);
2040         }
2041
2042         f2fs_bug_on(sbi, !list_empty(head));
2043         f2fs_bug_on(sbi, sit_i->dirty_sentries);
2044 out:
2045         if (cpc->reason == CP_DISCARD) {
2046                 for (; cpc->trim_start <= cpc->trim_end; cpc->trim_start++)
2047                         add_discard_addrs(sbi, cpc);
2048         }
2049         mutex_unlock(&sit_i->sentry_lock);
2050
2051         set_prefree_as_free_segments(sbi);
2052 }
2053
2054 static int build_sit_info(struct f2fs_sb_info *sbi)
2055 {
2056         struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi);
2057         struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
2058         struct sit_info *sit_i;
2059         unsigned int sit_segs, start;
2060         char *src_bitmap, *dst_bitmap;
2061         unsigned int bitmap_size;
2062
2063         /* allocate memory for SIT information */
2064         sit_i = kzalloc(sizeof(struct sit_info), GFP_KERNEL);
2065         if (!sit_i)
2066                 return -ENOMEM;
2067
2068         SM_I(sbi)->sit_info = sit_i;
2069
2070         sit_i->sentries = f2fs_kvzalloc(MAIN_SEGS(sbi) *
2071                                         sizeof(struct seg_entry), GFP_KERNEL);
2072         if (!sit_i->sentries)
2073                 return -ENOMEM;
2074
2075         bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));
2076         sit_i->dirty_sentries_bitmap = f2fs_kvzalloc(bitmap_size, GFP_KERNEL);
2077         if (!sit_i->dirty_sentries_bitmap)
2078                 return -ENOMEM;
2079
2080         for (start = 0; start < MAIN_SEGS(sbi); start++) {
2081                 sit_i->sentries[start].cur_valid_map
2082                         = kzalloc(SIT_VBLOCK_MAP_SIZE, GFP_KERNEL);
2083                 sit_i->sentries[start].ckpt_valid_map
2084                         = kzalloc(SIT_VBLOCK_MAP_SIZE, GFP_KERNEL);
2085                 sit_i->sentries[start].discard_map
2086                         = kzalloc(SIT_VBLOCK_MAP_SIZE, GFP_KERNEL);
2087                 if (!sit_i->sentries[start].cur_valid_map ||
2088                                 !sit_i->sentries[start].ckpt_valid_map ||
2089                                 !sit_i->sentries[start].discard_map)
2090                         return -ENOMEM;
2091         }
2092
2093         sit_i->tmp_map = kzalloc(SIT_VBLOCK_MAP_SIZE, GFP_KERNEL);
2094         if (!sit_i->tmp_map)
2095                 return -ENOMEM;
2096
2097         if (sbi->segs_per_sec > 1) {
2098                 sit_i->sec_entries = f2fs_kvzalloc(MAIN_SECS(sbi) *
2099                                         sizeof(struct sec_entry), GFP_KERNEL);
2100                 if (!sit_i->sec_entries)
2101                         return -ENOMEM;
2102         }
2103
2104         /* get information related with SIT */
2105         sit_segs = le32_to_cpu(raw_super->segment_count_sit) >> 1;
2106
2107         /* setup SIT bitmap from ckeckpoint pack */
2108         bitmap_size = __bitmap_size(sbi, SIT_BITMAP);
2109         src_bitmap = __bitmap_ptr(sbi, SIT_BITMAP);
2110
2111         dst_bitmap = kmemdup(src_bitmap, bitmap_size, GFP_KERNEL);
2112         if (!dst_bitmap)
2113                 return -ENOMEM;
2114
2115         /* init SIT information */
2116         sit_i->s_ops = &default_salloc_ops;
2117
2118         sit_i->sit_base_addr = le32_to_cpu(raw_super->sit_blkaddr);
2119         sit_i->sit_blocks = sit_segs << sbi->log_blocks_per_seg;
2120         sit_i->written_valid_blocks = le64_to_cpu(ckpt->valid_block_count);
2121         sit_i->sit_bitmap = dst_bitmap;
2122         sit_i->bitmap_size = bitmap_size;
2123         sit_i->dirty_sentries = 0;
2124         sit_i->sents_per_block = SIT_ENTRY_PER_BLOCK;
2125         sit_i->elapsed_time = le64_to_cpu(sbi->ckpt->elapsed_time);
2126         sit_i->mounted_time = CURRENT_TIME_SEC.tv_sec;
2127         mutex_init(&sit_i->sentry_lock);
2128         return 0;
2129 }
2130
2131 static int build_free_segmap(struct f2fs_sb_info *sbi)
2132 {
2133         struct free_segmap_info *free_i;
2134         unsigned int bitmap_size, sec_bitmap_size;
2135
2136         /* allocate memory for free segmap information */
2137         free_i = kzalloc(sizeof(struct free_segmap_info), GFP_KERNEL);
2138         if (!free_i)
2139                 return -ENOMEM;
2140
2141         SM_I(sbi)->free_info = free_i;
2142
2143         bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));
2144         free_i->free_segmap = f2fs_kvmalloc(bitmap_size, GFP_KERNEL);
2145         if (!free_i->free_segmap)
2146                 return -ENOMEM;
2147
2148         sec_bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi));
2149         free_i->free_secmap = f2fs_kvmalloc(sec_bitmap_size, GFP_KERNEL);
2150         if (!free_i->free_secmap)
2151                 return -ENOMEM;
2152
2153         /* set all segments as dirty temporarily */
2154         memset(free_i->free_segmap, 0xff, bitmap_size);
2155         memset(free_i->free_secmap, 0xff, sec_bitmap_size);
2156
2157         /* init free segmap information */
2158         free_i->start_segno = GET_SEGNO_FROM_SEG0(sbi, MAIN_BLKADDR(sbi));
2159         free_i->free_segments = 0;
2160         free_i->free_sections = 0;
2161         spin_lock_init(&free_i->segmap_lock);
2162         return 0;
2163 }
2164
2165 static int build_curseg(struct f2fs_sb_info *sbi)
2166 {
2167         struct curseg_info *array;
2168         int i;
2169
2170         array = kcalloc(NR_CURSEG_TYPE, sizeof(*array), GFP_KERNEL);
2171         if (!array)
2172                 return -ENOMEM;
2173
2174         SM_I(sbi)->curseg_array = array;
2175
2176         for (i = 0; i < NR_CURSEG_TYPE; i++) {
2177                 mutex_init(&array[i].curseg_mutex);
2178                 array[i].sum_blk = kzalloc(PAGE_SIZE, GFP_KERNEL);
2179                 if (!array[i].sum_blk)
2180                         return -ENOMEM;
2181                 init_rwsem(&array[i].journal_rwsem);
2182                 array[i].journal = kzalloc(sizeof(struct f2fs_journal),
2183                                                         GFP_KERNEL);
2184                 if (!array[i].journal)
2185                         return -ENOMEM;
2186                 array[i].segno = NULL_SEGNO;
2187                 array[i].next_blkoff = 0;
2188         }
2189         return restore_curseg_summaries(sbi);
2190 }
2191
2192 static void build_sit_entries(struct f2fs_sb_info *sbi)
2193 {
2194         struct sit_info *sit_i = SIT_I(sbi);
2195         struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
2196         struct f2fs_journal *journal = curseg->journal;
2197         int sit_blk_cnt = SIT_BLK_CNT(sbi);
2198         unsigned int i, start, end;
2199         unsigned int readed, start_blk = 0;
2200         int nrpages = MAX_BIO_BLOCKS(sbi) * 8;
2201
2202         do {
2203                 readed = ra_meta_pages(sbi, start_blk, nrpages, META_SIT, true);
2204
2205                 start = start_blk * sit_i->sents_per_block;
2206                 end = (start_blk + readed) * sit_i->sents_per_block;
2207
2208                 for (; start < end && start < MAIN_SEGS(sbi); start++) {
2209                         struct seg_entry *se = &sit_i->sentries[start];
2210                         struct f2fs_sit_block *sit_blk;
2211                         struct f2fs_sit_entry sit;
2212                         struct page *page;
2213
2214                         down_read(&curseg->journal_rwsem);
2215                         for (i = 0; i < sits_in_cursum(journal); i++) {
2216                                 if (le32_to_cpu(segno_in_journal(journal, i))
2217                                                                 == start) {
2218                                         sit = sit_in_journal(journal, i);
2219                                         up_read(&curseg->journal_rwsem);
2220                                         goto got_it;
2221                                 }
2222                         }
2223                         up_read(&curseg->journal_rwsem);
2224
2225                         page = get_current_sit_page(sbi, start);
2226                         sit_blk = (struct f2fs_sit_block *)page_address(page);
2227                         sit = sit_blk->entries[SIT_ENTRY_OFFSET(sit_i, start)];
2228                         f2fs_put_page(page, 1);
2229 got_it:
2230                         check_block_count(sbi, start, &sit);
2231                         seg_info_from_raw_sit(se, &sit);
2232
2233                         /* build discard map only one time */
2234                         memcpy(se->discard_map, se->cur_valid_map, SIT_VBLOCK_MAP_SIZE);
2235                         sbi->discard_blks += sbi->blocks_per_seg - se->valid_blocks;
2236
2237                         if (sbi->segs_per_sec > 1) {
2238                                 struct sec_entry *e = get_sec_entry(sbi, start);
2239                                 e->valid_blocks += se->valid_blocks;
2240                         }
2241                 }
2242                 start_blk += readed;
2243         } while (start_blk < sit_blk_cnt);
2244 }
2245
2246 static void init_free_segmap(struct f2fs_sb_info *sbi)
2247 {
2248         unsigned int start;
2249         int type;
2250
2251         for (start = 0; start < MAIN_SEGS(sbi); start++) {
2252                 struct seg_entry *sentry = get_seg_entry(sbi, start);
2253                 if (!sentry->valid_blocks)
2254                         __set_free(sbi, start);
2255         }
2256
2257         /* set use the current segments */
2258         for (type = CURSEG_HOT_DATA; type <= CURSEG_COLD_NODE; type++) {
2259                 struct curseg_info *curseg_t = CURSEG_I(sbi, type);
2260                 __set_test_and_inuse(sbi, curseg_t->segno);
2261         }
2262 }
2263
2264 static void init_dirty_segmap(struct f2fs_sb_info *sbi)
2265 {
2266         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
2267         struct free_segmap_info *free_i = FREE_I(sbi);
2268         unsigned int segno = 0, offset = 0;
2269         unsigned short valid_blocks;
2270
2271         while (1) {
2272                 /* find dirty segment based on free segmap */
2273                 segno = find_next_inuse(free_i, MAIN_SEGS(sbi), offset);
2274                 if (segno >= MAIN_SEGS(sbi))
2275                         break;
2276                 offset = segno + 1;
2277                 valid_blocks = get_valid_blocks(sbi, segno, 0);
2278                 if (valid_blocks == sbi->blocks_per_seg || !valid_blocks)
2279                         continue;
2280                 if (valid_blocks > sbi->blocks_per_seg) {
2281                         f2fs_bug_on(sbi, 1);
2282                         continue;
2283                 }
2284                 mutex_lock(&dirty_i->seglist_lock);
2285                 __locate_dirty_segment(sbi, segno, DIRTY);
2286                 mutex_unlock(&dirty_i->seglist_lock);
2287         }
2288 }
2289
2290 static int init_victim_secmap(struct f2fs_sb_info *sbi)
2291 {
2292         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
2293         unsigned int bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi));
2294
2295         dirty_i->victim_secmap = f2fs_kvzalloc(bitmap_size, GFP_KERNEL);
2296         if (!dirty_i->victim_secmap)
2297                 return -ENOMEM;
2298         return 0;
2299 }
2300
2301 static int build_dirty_segmap(struct f2fs_sb_info *sbi)
2302 {
2303         struct dirty_seglist_info *dirty_i;
2304         unsigned int bitmap_size, i;
2305
2306         /* allocate memory for dirty segments list information */
2307         dirty_i = kzalloc(sizeof(struct dirty_seglist_info), GFP_KERNEL);
2308         if (!dirty_i)
2309                 return -ENOMEM;
2310
2311         SM_I(sbi)->dirty_info = dirty_i;
2312         mutex_init(&dirty_i->seglist_lock);
2313
2314         bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));
2315
2316         for (i = 0; i < NR_DIRTY_TYPE; i++) {
2317                 dirty_i->dirty_segmap[i] = f2fs_kvzalloc(bitmap_size, GFP_KERNEL);
2318                 if (!dirty_i->dirty_segmap[i])
2319                         return -ENOMEM;
2320         }
2321
2322         init_dirty_segmap(sbi);
2323         return init_victim_secmap(sbi);
2324 }
2325
2326 /*
2327  * Update min, max modified time for cost-benefit GC algorithm
2328  */
2329 static void init_min_max_mtime(struct f2fs_sb_info *sbi)
2330 {
2331         struct sit_info *sit_i = SIT_I(sbi);
2332         unsigned int segno;
2333
2334         mutex_lock(&sit_i->sentry_lock);
2335
2336         sit_i->min_mtime = LLONG_MAX;
2337
2338         for (segno = 0; segno < MAIN_SEGS(sbi); segno += sbi->segs_per_sec) {
2339                 unsigned int i;
2340                 unsigned long long mtime = 0;
2341
2342                 for (i = 0; i < sbi->segs_per_sec; i++)
2343                         mtime += get_seg_entry(sbi, segno + i)->mtime;
2344
2345                 mtime = div_u64(mtime, sbi->segs_per_sec);
2346
2347                 if (sit_i->min_mtime > mtime)
2348                         sit_i->min_mtime = mtime;
2349         }
2350         sit_i->max_mtime = get_mtime(sbi);
2351         mutex_unlock(&sit_i->sentry_lock);
2352 }
2353
2354 int build_segment_manager(struct f2fs_sb_info *sbi)
2355 {
2356         struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi);
2357         struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
2358         struct f2fs_sm_info *sm_info;
2359         int err;
2360
2361         sm_info = kzalloc(sizeof(struct f2fs_sm_info), GFP_KERNEL);
2362         if (!sm_info)
2363                 return -ENOMEM;
2364
2365         /* init sm info */
2366         sbi->sm_info = sm_info;
2367         sm_info->seg0_blkaddr = le32_to_cpu(raw_super->segment0_blkaddr);
2368         sm_info->main_blkaddr = le32_to_cpu(raw_super->main_blkaddr);
2369         sm_info->segment_count = le32_to_cpu(raw_super->segment_count);
2370         sm_info->reserved_segments = le32_to_cpu(ckpt->rsvd_segment_count);
2371         sm_info->ovp_segments = le32_to_cpu(ckpt->overprov_segment_count);
2372         sm_info->main_segments = le32_to_cpu(raw_super->segment_count_main);
2373         sm_info->ssa_blkaddr = le32_to_cpu(raw_super->ssa_blkaddr);
2374         sm_info->rec_prefree_segments = sm_info->main_segments *
2375                                         DEF_RECLAIM_PREFREE_SEGMENTS / 100;
2376         sm_info->ipu_policy = 1 << F2FS_IPU_FSYNC;
2377         sm_info->min_ipu_util = DEF_MIN_IPU_UTIL;
2378         sm_info->min_fsync_blocks = DEF_MIN_FSYNC_BLOCKS;
2379
2380         INIT_LIST_HEAD(&sm_info->discard_list);
2381         sm_info->nr_discards = 0;
2382         sm_info->max_discards = 0;
2383
2384         sm_info->trim_sections = DEF_BATCHED_TRIM_SECTIONS;
2385
2386         INIT_LIST_HEAD(&sm_info->sit_entry_set);
2387
2388         if (test_opt(sbi, FLUSH_MERGE) && !f2fs_readonly(sbi->sb)) {
2389                 err = create_flush_cmd_control(sbi);
2390                 if (err)
2391                         return err;
2392         }
2393
2394         err = build_sit_info(sbi);
2395         if (err)
2396                 return err;
2397         err = build_free_segmap(sbi);
2398         if (err)
2399                 return err;
2400         err = build_curseg(sbi);
2401         if (err)
2402                 return err;
2403
2404         /* reinit free segmap based on SIT */
2405         build_sit_entries(sbi);
2406
2407         init_free_segmap(sbi);
2408         err = build_dirty_segmap(sbi);
2409         if (err)
2410                 return err;
2411
2412         init_min_max_mtime(sbi);
2413         return 0;
2414 }
2415
2416 static void discard_dirty_segmap(struct f2fs_sb_info *sbi,
2417                 enum dirty_type dirty_type)
2418 {
2419         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
2420
2421         mutex_lock(&dirty_i->seglist_lock);
2422         kvfree(dirty_i->dirty_segmap[dirty_type]);
2423         dirty_i->nr_dirty[dirty_type] = 0;
2424         mutex_unlock(&dirty_i->seglist_lock);
2425 }
2426
2427 static void destroy_victim_secmap(struct f2fs_sb_info *sbi)
2428 {
2429         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
2430         kvfree(dirty_i->victim_secmap);
2431 }
2432
2433 static void destroy_dirty_segmap(struct f2fs_sb_info *sbi)
2434 {
2435         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
2436         int i;
2437
2438         if (!dirty_i)
2439                 return;
2440
2441         /* discard pre-free/dirty segments list */
2442         for (i = 0; i < NR_DIRTY_TYPE; i++)
2443                 discard_dirty_segmap(sbi, i);
2444
2445         destroy_victim_secmap(sbi);
2446         SM_I(sbi)->dirty_info = NULL;
2447         kfree(dirty_i);
2448 }
2449
2450 static void destroy_curseg(struct f2fs_sb_info *sbi)
2451 {
2452         struct curseg_info *array = SM_I(sbi)->curseg_array;
2453         int i;
2454
2455         if (!array)
2456                 return;
2457         SM_I(sbi)->curseg_array = NULL;
2458         for (i = 0; i < NR_CURSEG_TYPE; i++) {
2459                 kfree(array[i].sum_blk);
2460                 kfree(array[i].journal);
2461         }
2462         kfree(array);
2463 }
2464
2465 static void destroy_free_segmap(struct f2fs_sb_info *sbi)
2466 {
2467         struct free_segmap_info *free_i = SM_I(sbi)->free_info;
2468         if (!free_i)
2469                 return;
2470         SM_I(sbi)->free_info = NULL;
2471         kvfree(free_i->free_segmap);
2472         kvfree(free_i->free_secmap);
2473         kfree(free_i);
2474 }
2475
2476 static void destroy_sit_info(struct f2fs_sb_info *sbi)
2477 {
2478         struct sit_info *sit_i = SIT_I(sbi);
2479         unsigned int start;
2480
2481         if (!sit_i)
2482                 return;
2483
2484         if (sit_i->sentries) {
2485                 for (start = 0; start < MAIN_SEGS(sbi); start++) {
2486                         kfree(sit_i->sentries[start].cur_valid_map);
2487                         kfree(sit_i->sentries[start].ckpt_valid_map);
2488                         kfree(sit_i->sentries[start].discard_map);
2489                 }
2490         }
2491         kfree(sit_i->tmp_map);
2492
2493         kvfree(sit_i->sentries);
2494         kvfree(sit_i->sec_entries);
2495         kvfree(sit_i->dirty_sentries_bitmap);
2496
2497         SM_I(sbi)->sit_info = NULL;
2498         kfree(sit_i->sit_bitmap);
2499         kfree(sit_i);
2500 }
2501
2502 void destroy_segment_manager(struct f2fs_sb_info *sbi)
2503 {
2504         struct f2fs_sm_info *sm_info = SM_I(sbi);
2505
2506         if (!sm_info)
2507                 return;
2508         destroy_flush_cmd_control(sbi);
2509         destroy_dirty_segmap(sbi);
2510         destroy_curseg(sbi);
2511         destroy_free_segmap(sbi);
2512         destroy_sit_info(sbi);
2513         sbi->sm_info = NULL;
2514         kfree(sm_info);
2515 }
2516
2517 int __init create_segment_manager_caches(void)
2518 {
2519         discard_entry_slab = f2fs_kmem_cache_create("discard_entry",
2520                         sizeof(struct discard_entry));
2521         if (!discard_entry_slab)
2522                 goto fail;
2523
2524         sit_entry_set_slab = f2fs_kmem_cache_create("sit_entry_set",
2525                         sizeof(struct sit_entry_set));
2526         if (!sit_entry_set_slab)
2527                 goto destory_discard_entry;
2528
2529         inmem_entry_slab = f2fs_kmem_cache_create("inmem_page_entry",
2530                         sizeof(struct inmem_pages));
2531         if (!inmem_entry_slab)
2532                 goto destroy_sit_entry_set;
2533         return 0;
2534
2535 destroy_sit_entry_set:
2536         kmem_cache_destroy(sit_entry_set_slab);
2537 destory_discard_entry:
2538         kmem_cache_destroy(discard_entry_slab);
2539 fail:
2540         return -ENOMEM;
2541 }
2542
2543 void destroy_segment_manager_caches(void)
2544 {
2545         kmem_cache_destroy(sit_entry_set_slab);
2546         kmem_cache_destroy(discard_entry_slab);
2547         kmem_cache_destroy(inmem_entry_slab);
2548 }