drivers/block/zram/zram_drv.c

   1 /*
   2  * Compressed RAM block device
   3  *
   4  * Copyright (C) 2008, 2009, 2010  Nitin Gupta
   5  *               2012, 2013 Minchan Kim
   6  *
   7  * This code is released using a dual license strategy: BSD/GPL
   8  * You can choose the licence that better fits your requirements.
   9  *
  10  * Released under the terms of 3-clause BSD License
  11  * Released under the terms of GNU General Public License Version 2.0
  12  *
  13  */
  14
  15 #define KMSG_COMPONENT "zram"
  16 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
  17
  18 #ifdef CONFIG_ZRAM_DEBUG
  19 #define DEBUG
  20 #endif
  21
  22 #include <linux/module.h>
  23 #include <linux/kernel.h>
  24 #include <linux/bio.h>
  25 #include <linux/bitops.h>
  26 #include <linux/blkdev.h>
  27 #include <linux/buffer_head.h>
  28 #include <linux/device.h>
  29 #include <linux/genhd.h>
  30 #include <linux/highmem.h>
  31 #include <linux/slab.h>
  32 #include <linux/string.h>
  33 #include <linux/vmalloc.h>
  34 #include <linux/err.h>
  35
  36 #include "zram_drv.h"
  37
  38 /* Globals */
  39 static int zram_major;
  40 static struct zram *zram_devices;
  41 static const char *default_compressor = "lzo";
  42
  43 /* Module params (documentation at end) */
  44 static unsigned int num_devices = 1;
  45
  46 #define ZRAM_ATTR_RO(name)                                              \
  47 static ssize_t zram_attr_##name##_show(struct device *d,                \
  48                                 struct device_attribute *attr, char *b) \
  49 {                                                                       \
  50         struct zram *zram = dev_to_zram(d);                             \
  51         return scnprintf(b, PAGE_SIZE, "%llu\n",                        \
  52                 (u64)atomic64_read(&zram->stats.name));                 \
  53 }                                                                       \
  54 static struct device_attribute dev_attr_##name =                        \
  55         __ATTR(name, S_IRUGO, zram_attr_##name##_show, NULL);
  56
  57 static inline int init_done(struct zram *zram)
  58 {
  59         return zram->meta != NULL;
  60 }
  61
  62 static inline struct zram *dev_to_zram(struct device *dev)
  63 {
  64         return (struct zram *)dev_to_disk(dev)->private_data;
  65 }
  66
  67 static ssize_t disksize_show(struct device *dev,
  68                 struct device_attribute *attr, char *buf)
  69 {
  70         struct zram *zram = dev_to_zram(dev);
  71
  72         return scnprintf(buf, PAGE_SIZE, "%llu\n", zram->disksize);
  73 }
  74
  75 static ssize_t initstate_show(struct device *dev,
  76                 struct device_attribute *attr, char *buf)
  77 {
  78         u32 val;
  79         struct zram *zram = dev_to_zram(dev);
  80
  81         down_read(&zram->init_lock);
  82         val = init_done(zram);
  83         up_read(&zram->init_lock);
  84
  85         return scnprintf(buf, PAGE_SIZE, "%u\n", val);
  86 }
  87
  88 static ssize_t orig_data_size_show(struct device *dev,
  89                 struct device_attribute *attr, char *buf)
  90 {
  91         struct zram *zram = dev_to_zram(dev);
  92
  93         return scnprintf(buf, PAGE_SIZE, "%llu\n",
  94                 (u64)(atomic64_read(&zram->stats.pages_stored)) << PAGE_SHIFT);
  95 }
  96
  97 static ssize_t mem_used_total_show(struct device *dev,
  98                 struct device_attribute *attr, char *buf)
  99 {
 100         u64 val = 0;
 101         struct zram *zram = dev_to_zram(dev);
 102         struct zram_meta *meta = zram->meta;
 103
 104         down_read(&zram->init_lock);
 105         if (init_done(zram))
 106                 val = zs_get_total_size_bytes(meta->mem_pool);
 107         up_read(&zram->init_lock);
 108
 109         return scnprintf(buf, PAGE_SIZE, "%llu\n", val);
 110 }
 111
 112 static ssize_t max_comp_streams_show(struct device *dev,
 113                 struct device_attribute *attr, char *buf)
 114 {
 115         int val;
 116         struct zram *zram = dev_to_zram(dev);
 117
 118         down_read(&zram->init_lock);
 119         val = zram->max_comp_streams;
 120         up_read(&zram->init_lock);
 121
 122         return scnprintf(buf, PAGE_SIZE, "%d\n", val);
 123 }
 124
 125 static ssize_t max_comp_streams_store(struct device *dev,
 126                 struct device_attribute *attr, const char *buf, size_t len)
 127 {
 128         int num;
 129         struct zram *zram = dev_to_zram(dev);
 130         int ret;
 131
 132         ret = kstrtoint(buf, 0, &num);
 133         if (ret < 0)
 134                 return ret;
 135         if (num < 1)
 136                 return -EINVAL;
 137
 138         down_write(&zram->init_lock);
 139         if (init_done(zram)) {
 140                 if (!zcomp_set_max_streams(zram->comp, num)) {
 141                         pr_info("Cannot change max compression streams\n");
 142                         ret = -EINVAL;
 143                         goto out;
 144                 }
 145         }
 146
 147         zram->max_comp_streams = num;
 148         ret = len;
 149 out:
 150         up_write(&zram->init_lock);
 151         return ret;
 152 }
 153
 154 static ssize_t comp_algorithm_show(struct device *dev,
 155                 struct device_attribute *attr, char *buf)
 156 {
 157         size_t sz;
 158         struct zram *zram = dev_to_zram(dev);
 159
 160         down_read(&zram->init_lock);
 161         sz = zcomp_available_show(zram->compressor, buf);
 162         up_read(&zram->init_lock);
 163
 164         return sz;
 165 }
 166
 167 static ssize_t comp_algorithm_store(struct device *dev,
 168                 struct device_attribute *attr, const char *buf, size_t len)
 169 {
 170         struct zram *zram = dev_to_zram(dev);
 171         down_write(&zram->init_lock);
 172         if (init_done(zram)) {
 173                 up_write(&zram->init_lock);
 174                 pr_info("Can't change algorithm for initialized device\n");
 175                 return -EBUSY;
 176         }
 177         strlcpy(zram->compressor, buf, sizeof(zram->compressor));
 178         up_write(&zram->init_lock);
 179         return len;
 180 }
 181
 182 /* flag operations needs meta->tb_lock */
 183 static int zram_test_flag(struct zram_meta *meta, u32 index,
 184                         enum zram_pageflags flag)
 185 {
 186         return meta->table[index].value & BIT(flag);
 187 }
 188
 189 static void zram_set_flag(struct zram_meta *meta, u32 index,
 190                         enum zram_pageflags flag)
 191 {
 192         meta->table[index].value |= BIT(flag);
 193 }
 194
 195 static void zram_clear_flag(struct zram_meta *meta, u32 index,
 196                         enum zram_pageflags flag)
 197 {
 198         meta->table[index].value &= ~BIT(flag);
 199 }
 200
 201 static size_t zram_get_obj_size(struct zram_meta *meta, u32 index)
 202 {
 203         return meta->table[index].value & (BIT(ZRAM_FLAG_SHIFT) - 1);
 204 }
 205
 206 static void zram_set_obj_size(struct zram_meta *meta,
 207                                         u32 index, size_t size)
 208 {
 209         unsigned long flags = meta->table[index].value >> ZRAM_FLAG_SHIFT;
 210
 211         meta->table[index].value = (flags << ZRAM_FLAG_SHIFT) | size;
 212 }
 213
 214 static inline int is_partial_io(struct bio_vec *bvec)
 215 {
 216         return bvec->bv_len != PAGE_SIZE;
 217 }
 218
 219 /*
 220  * Check if request is within bounds and aligned on zram logical blocks.
 221  */
 222 static inline int valid_io_request(struct zram *zram, struct bio *bio)
 223 {
 224         u64 start, end, bound;
 225
 226         /* unaligned request */
 227         if (unlikely(bio->bi_iter.bi_sector &
 228                      (ZRAM_SECTOR_PER_LOGICAL_BLOCK - 1)))
 229                 return 0;
 230         if (unlikely(bio->bi_iter.bi_size & (ZRAM_LOGICAL_BLOCK_SIZE - 1)))
 231                 return 0;
 232
 233         start = bio->bi_iter.bi_sector;
 234         end = start + (bio->bi_iter.bi_size >> SECTOR_SHIFT);
 235         bound = zram->disksize >> SECTOR_SHIFT;
 236         /* out of range range */
 237         if (unlikely(start >= bound || end > bound || start > end))
 238                 return 0;
 239
 240         /* I/O request is valid */
 241         return 1;
 242 }
 243
 244 static void zram_meta_free(struct zram_meta *meta)
 245 {
 246         zs_destroy_pool(meta->mem_pool);
 247         vfree(meta->table);
 248         kfree(meta);
 249 }
 250
 251 static struct zram_meta *zram_meta_alloc(u64 disksize)
 252 {
 253         size_t num_pages;
 254         struct zram_meta *meta = kmalloc(sizeof(*meta), GFP_KERNEL);
 255         if (!meta)
 256                 goto out;
 257
 258         num_pages = disksize >> PAGE_SHIFT;
 259         meta->table = vzalloc(num_pages * sizeof(*meta->table));
 260         if (!meta->table) {
 261                 pr_err("Error allocating zram address table\n");
 262                 goto free_meta;
 263         }
 264
 265         meta->mem_pool = zs_create_pool(GFP_NOIO | __GFP_HIGHMEM);
 266         if (!meta->mem_pool) {
 267                 pr_err("Error creating memory pool\n");
 268                 goto free_table;
 269         }
 270
 271         return meta;
 272
 273 free_table:
 274         vfree(meta->table);
 275 free_meta:
 276         kfree(meta);
 277         meta = NULL;
 278 out:
 279         return meta;
 280 }
 281
 282 static void update_position(u32 *index, int *offset, struct bio_vec *bvec)
 283 {
 284         if (*offset + bvec->bv_len >= PAGE_SIZE)
 285                 (*index)++;
 286         *offset = (*offset + bvec->bv_len) % PAGE_SIZE;
 287 }
 288
 289 static int page_zero_filled(void *ptr)
 290 {
 291         unsigned int pos;
 292         unsigned long *page;
 293
 294         page = (unsigned long *)ptr;
 295
 296         for (pos = 0; pos != PAGE_SIZE / sizeof(*page); pos++) {
 297                 if (page[pos])
 298                         return 0;
 299         }
 300
 301         return 1;
 302 }
 303
 304 static void handle_zero_page(struct bio_vec *bvec)
 305 {
 306         struct page *page = bvec->bv_page;
 307         void *user_mem;
 308
 309         user_mem = kmap_atomic(page);
 310         if (is_partial_io(bvec))
 311                 memset(user_mem + bvec->bv_offset, 0, bvec->bv_len);
 312         else
 313                 clear_page(user_mem);
 314         kunmap_atomic(user_mem);
 315
 316         flush_dcache_page(page);
 317 }
 318
 319
 320 /*
 321  * To protect concurrent access to the same index entry,
 322  * caller should hold this table index entry's bit_spinlock to
 323  * indicate this index entry is accessing.
 324  */
 325 static void zram_free_page(struct zram *zram, size_t index)
 326 {
 327         struct zram_meta *meta = zram->meta;
 328         unsigned long handle = meta->table[index].handle;
 329
 330         if (unlikely(!handle)) {
 331                 /*
 332                  * No memory is allocated for zero filled pages.
 333                  * Simply clear zero page flag.
 334                  */
 335                 if (zram_test_flag(meta, index, ZRAM_ZERO)) {
 336                         zram_clear_flag(meta, index, ZRAM_ZERO);
 337                         atomic64_dec(&zram->stats.zero_pages);
 338                 }
 339                 return;
 340         }
 341
 342         zs_free(meta->mem_pool, handle);
 343
 344         atomic64_sub(zram_get_obj_size(meta, index),
 345                         &zram->stats.compr_data_size);
 346         atomic64_dec(&zram->stats.pages_stored);
 347
 348         meta->table[index].handle = 0;
 349         zram_set_obj_size(meta, index, 0);
 350 }
 351
 352 static int zram_decompress_page(struct zram *zram, char *mem, u32 index)
 353 {
 354         int ret = 0;
 355         unsigned char *cmem;
 356         struct zram_meta *meta = zram->meta;
 357         unsigned long handle;
 358         size_t size;
 359
 360         bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
 361         handle = meta->table[index].handle;
 362         size = zram_get_obj_size(meta, index);
 363
 364         if (!handle || zram_test_flag(meta, index, ZRAM_ZERO)) {
 365                 bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
 366                 clear_page(mem);
 367                 return 0;
 368         }
 369
 370         cmem = zs_map_object(meta->mem_pool, handle, ZS_MM_RO);
 371         if (size == PAGE_SIZE)
 372                 copy_page(mem, cmem);
 373         else
 374                 ret = zcomp_decompress(zram->comp, cmem, size, mem);
 375         zs_unmap_object(meta->mem_pool, handle);
 376         bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
 377
 378         /* Should NEVER happen. Return bio error if it does. */
 379         if (unlikely(ret)) {
 380                 pr_err("Decompression failed! err=%d, page=%u\n", ret, index);
 381                 atomic64_inc(&zram->stats.failed_reads);
 382                 return ret;
 383         }
 384
 385         return 0;
 386 }
 387
 388 static int zram_bvec_read(struct zram *zram, struct bio_vec *bvec,
 389                           u32 index, int offset, struct bio *bio)
 390 {
 391         int ret;
 392         struct page *page;
 393         unsigned char *user_mem, *uncmem = NULL;
 394         struct zram_meta *meta = zram->meta;
 395         page = bvec->bv_page;
 396
 397         bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
 398         if (unlikely(!meta->table[index].handle) ||
 399                         zram_test_flag(meta, index, ZRAM_ZERO)) {
 400                 bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
 401                 handle_zero_page(bvec);
 402                 return 0;
 403         }
 404         bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
 405
 406         if (is_partial_io(bvec))
 407                 /* Use  a temporary buffer to decompress the page */
 408                 uncmem = kmalloc(PAGE_SIZE, GFP_NOIO);
 409
 410         user_mem = kmap_atomic(page);
 411         if (!is_partial_io(bvec))
 412                 uncmem = user_mem;
 413
 414         if (!uncmem) {
 415                 pr_info("Unable to allocate temp memory\n");
 416                 ret = -ENOMEM;
 417                 goto out_cleanup;
 418         }
 419
 420         ret = zram_decompress_page(zram, uncmem, index);
 421         /* Should NEVER happen. Return bio error if it does. */
 422         if (unlikely(ret))
 423                 goto out_cleanup;
 424
 425         if (is_partial_io(bvec))
 426                 memcpy(user_mem + bvec->bv_offset, uncmem + offset,
 427                                 bvec->bv_len);
 428
 429         flush_dcache_page(page);
 430         ret = 0;
 431 out_cleanup:
 432         kunmap_atomic(user_mem);
 433         if (is_partial_io(bvec))
 434                 kfree(uncmem);
 435         return ret;
 436 }
 437
 438 static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec, u32 index,
 439                            int offset)
 440 {
 441         int ret = 0;
 442         size_t clen;
 443         unsigned long handle;
 444         struct page *page;
 445         unsigned char *user_mem, *cmem, *src, *uncmem = NULL;
 446         struct zram_meta *meta = zram->meta;
 447         struct zcomp_strm *zstrm;
 448         bool locked = false;
 449
 450         page = bvec->bv_page;
 451         if (is_partial_io(bvec)) {
 452                 /*
 453                  * This is a partial IO. We need to read the full page
 454                  * before to write the changes.
 455                  */
 456                 uncmem = kmalloc(PAGE_SIZE, GFP_NOIO);
 457                 if (!uncmem) {
 458                         ret = -ENOMEM;
 459                         goto out;
 460                 }
 461                 ret = zram_decompress_page(zram, uncmem, index);
 462                 if (ret)
 463                         goto out;
 464         }
 465
 466         zstrm = zcomp_strm_find(zram->comp);
 467         locked = true;
 468         user_mem = kmap_atomic(page);
 469
 470         if (is_partial_io(bvec)) {
 471                 memcpy(uncmem + offset, user_mem + bvec->bv_offset,
 472                        bvec->bv_len);
 473                 kunmap_atomic(user_mem);
 474                 user_mem = NULL;
 475         } else {
 476                 uncmem = user_mem;
 477         }
 478
 479         if (page_zero_filled(uncmem)) {
 480                 kunmap_atomic(user_mem);
 481                 /* Free memory associated with this sector now. */
 482                 bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
 483                 zram_free_page(zram, index);
 484                 zram_set_flag(meta, index, ZRAM_ZERO);
 485                 bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
 486
 487                 atomic64_inc(&zram->stats.zero_pages);
 488                 ret = 0;
 489                 goto out;
 490         }
 491
 492         ret = zcomp_compress(zram->comp, zstrm, uncmem, &clen);
 493         if (!is_partial_io(bvec)) {
 494                 kunmap_atomic(user_mem);
 495                 user_mem = NULL;
 496                 uncmem = NULL;
 497         }
 498
 499         if (unlikely(ret)) {
 500                 pr_err("Compression failed! err=%d\n", ret);
 501                 goto out;
 502         }
 503         src = zstrm->buffer;
 504         if (unlikely(clen > max_zpage_size)) {
 505                 clen = PAGE_SIZE;
 506                 if (is_partial_io(bvec))
 507                         src = uncmem;
 508         }
 509
 510         handle = zs_malloc(meta->mem_pool, clen);
 511         if (!handle) {
 512                 pr_info("Error allocating memory for compressed page: %u, size=%zu\n",
 513                         index, clen);
 514                 ret = -ENOMEM;
 515                 goto out;
 516         }
 517         cmem = zs_map_object(meta->mem_pool, handle, ZS_MM_WO);
 518
 519         if ((clen == PAGE_SIZE) && !is_partial_io(bvec)) {
 520                 src = kmap_atomic(page);
 521                 copy_page(cmem, src);
 522                 kunmap_atomic(src);
 523         } else {
 524                 memcpy(cmem, src, clen);
 525         }
 526
 527         zcomp_strm_release(zram->comp, zstrm);
 528         locked = false;
 529         zs_unmap_object(meta->mem_pool, handle);
 530
 531         /*
 532          * Free memory associated with this sector
 533          * before overwriting unused sectors.
 534          */
 535         bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
 536         zram_free_page(zram, index);
 537
 538         meta->table[index].handle = handle;
 539         zram_set_obj_size(meta, index, clen);
 540         bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
 541
 542         /* Update stats */
 543         atomic64_add(clen, &zram->stats.compr_data_size);
 544         atomic64_inc(&zram->stats.pages_stored);
 545 out:
 546         if (locked)
 547                 zcomp_strm_release(zram->comp, zstrm);
 548         if (is_partial_io(bvec))
 549                 kfree(uncmem);
 550         if (ret)
 551                 atomic64_inc(&zram->stats.failed_writes);
 552         return ret;
 553 }
 554
 555 static int zram_bvec_rw(struct zram *zram, struct bio_vec *bvec, u32 index,
 556                         int offset, struct bio *bio)
 557 {
 558         int ret;
 559         int rw = bio_data_dir(bio);
 560
 561         if (rw == READ) {
 562                 atomic64_inc(&zram->stats.num_reads);
 563                 ret = zram_bvec_read(zram, bvec, index, offset, bio);
 564         } else {
 565                 atomic64_inc(&zram->stats.num_writes);
 566                 ret = zram_bvec_write(zram, bvec, index, offset);
 567         }
 568
 569         return ret;
 570 }
 571
 572 /*
 573  * zram_bio_discard - handler on discard request
 574  * @index: physical block index in PAGE_SIZE units
 575  * @offset: byte offset within physical block
 576  */
 577 static void zram_bio_discard(struct zram *zram, u32 index,
 578                              int offset, struct bio *bio)
 579 {
 580         size_t n = bio->bi_iter.bi_size;
 581         struct zram_meta *meta = zram->meta;
 582
 583         /*
 584          * zram manages data in physical block size units. Because logical block
 585          * size isn't identical with physical block size on some arch, we
 586          * could get a discard request pointing to a specific offset within a
 587          * certain physical block.  Although we can handle this request by
 588          * reading that physiclal block and decompressing and partially zeroing
 589          * and re-compressing and then re-storing it, this isn't reasonable
 590          * because our intent with a discard request is to save memory.  So
 591          * skipping this logical block is appropriate here.
 592          */
 593         if (offset) {
 594                 if (n <= (PAGE_SIZE - offset))
 595                         return;
 596
 597                 n -= (PAGE_SIZE - offset);
 598                 index++;
 599         }
 600
 601         while (n >= PAGE_SIZE) {
 602                 bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
 603                 zram_free_page(zram, index);
 604                 bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
 605                 index++;
 606                 n -= PAGE_SIZE;
 607         }
 608 }
 609
 610 static void zram_reset_device(struct zram *zram, bool reset_capacity)
 611 {
 612         size_t index;
 613         struct zram_meta *meta;
 614
 615         down_write(&zram->init_lock);
 616         if (!init_done(zram)) {
 617                 up_write(&zram->init_lock);
 618                 return;
 619         }
 620
 621         meta = zram->meta;
 622         /* Free all pages that are still in this zram device */
 623         for (index = 0; index < zram->disksize >> PAGE_SHIFT; index++) {
 624                 unsigned long handle = meta->table[index].handle;
 625                 if (!handle)
 626                         continue;
 627
 628                 zs_free(meta->mem_pool, handle);
 629         }
 630
 631         zcomp_destroy(zram->comp);
 632         zram->max_comp_streams = 1;
 633
 634         zram_meta_free(zram->meta);
 635         zram->meta = NULL;
 636         /* Reset stats */
 637         memset(&zram->stats, 0, sizeof(zram->stats));
 638
 639         zram->disksize = 0;
 640         if (reset_capacity)
 641                 set_capacity(zram->disk, 0);
 642
 643         up_write(&zram->init_lock);
 644
 645         /*
 646          * Revalidate disk out of the init_lock to avoid lockdep splat.
 647          * It's okay because disk's capacity is protected by init_lock
 648          * so that revalidate_disk always sees up-to-date capacity.
 649          */
 650         if (reset_capacity)
 651                 revalidate_disk(zram->disk);
 652 }
 653
 654 static ssize_t disksize_store(struct device *dev,
 655                 struct device_attribute *attr, const char *buf, size_t len)
 656 {
 657         u64 disksize;
 658         struct zcomp *comp;
 659         struct zram_meta *meta;
 660         struct zram *zram = dev_to_zram(dev);
 661         int err;
 662
 663         disksize = memparse(buf, NULL);
 664         if (!disksize)
 665                 return -EINVAL;
 666
 667         disksize = PAGE_ALIGN(disksize);
 668         meta = zram_meta_alloc(disksize);
 669         if (!meta)
 670                 return -ENOMEM;
 671
 672         comp = zcomp_create(zram->compressor, zram->max_comp_streams);
 673         if (IS_ERR(comp)) {
 674                 pr_info("Cannot initialise %s compressing backend\n",
 675                                 zram->compressor);
 676                 err = PTR_ERR(comp);
 677                 goto out_free_meta;
 678         }
 679
 680         down_write(&zram->init_lock);
 681         if (init_done(zram)) {
 682                 pr_info("Cannot change disksize for initialized device\n");
 683                 err = -EBUSY;
 684                 goto out_destroy_comp;
 685         }
 686
 687         zram->meta = meta;
 688         zram->comp = comp;
 689         zram->disksize = disksize;
 690         set_capacity(zram->disk, zram->disksize >> SECTOR_SHIFT);
 691         up_write(&zram->init_lock);
 692
 693         /*
 694          * Revalidate disk out of the init_lock to avoid lockdep splat.
 695          * It's okay because disk's capacity is protected by init_lock
 696          * so that revalidate_disk always sees up-to-date capacity.
 697          */
 698         revalidate_disk(zram->disk);
 699
 700         return len;
 701
 702 out_destroy_comp:
 703         up_write(&zram->init_lock);
 704         zcomp_destroy(comp);
 705 out_free_meta:
 706         zram_meta_free(meta);
 707         return err;
 708 }
 709
 710 static ssize_t reset_store(struct device *dev,
 711                 struct device_attribute *attr, const char *buf, size_t len)
 712 {
 713         int ret;
 714         unsigned short do_reset;
 715         struct zram *zram;
 716         struct block_device *bdev;
 717
 718         zram = dev_to_zram(dev);
 719         bdev = bdget_disk(zram->disk, 0);
 720
 721         if (!bdev)
 722                 return -ENOMEM;
 723
 724         /* Do not reset an active device! */
 725         if (bdev->bd_holders) {
 726                 ret = -EBUSY;
 727                 goto out;
 728         }
 729
 730         ret = kstrtou16(buf, 10, &do_reset);
 731         if (ret)
 732                 goto out;
 733
 734         if (!do_reset) {
 735                 ret = -EINVAL;
 736                 goto out;
 737         }
 738
 739         /* Make sure all pending I/O is finished */
 740         fsync_bdev(bdev);
 741         bdput(bdev);
 742
 743         zram_reset_device(zram, true);
 744         return len;
 745
 746 out:
 747         bdput(bdev);
 748         return ret;
 749 }
 750
 751 static void __zram_make_request(struct zram *zram, struct bio *bio)
 752 {
 753         int offset;
 754         u32 index;
 755         struct bio_vec bvec;
 756         struct bvec_iter iter;
 757
 758         index = bio->bi_iter.bi_sector >> SECTORS_PER_PAGE_SHIFT;
 759         offset = (bio->bi_iter.bi_sector &
 760                   (SECTORS_PER_PAGE - 1)) << SECTOR_SHIFT;
 761
 762         if (unlikely(bio->bi_rw & REQ_DISCARD)) {
 763                 zram_bio_discard(zram, index, offset, bio);
 764                 bio_endio(bio, 0);
 765                 return;
 766         }
 767
 768         bio_for_each_segment(bvec, bio, iter) {
 769                 int max_transfer_size = PAGE_SIZE - offset;
 770
 771                 if (bvec.bv_len > max_transfer_size) {
 772                         /*
 773                          * zram_bvec_rw() can only make operation on a single
 774                          * zram page. Split the bio vector.
 775                          */
 776                         struct bio_vec bv;
 777
 778                         bv.bv_page = bvec.bv_page;
 779                         bv.bv_len = max_transfer_size;
 780                         bv.bv_offset = bvec.bv_offset;
 781
 782                         if (zram_bvec_rw(zram, &bv, index, offset, bio) < 0)
 783                                 goto out;
 784
 785                         bv.bv_len = bvec.bv_len - max_transfer_size;
 786                         bv.bv_offset += max_transfer_size;
 787                         if (zram_bvec_rw(zram, &bv, index + 1, 0, bio) < 0)
 788                                 goto out;
 789                 } else
 790                         if (zram_bvec_rw(zram, &bvec, index, offset, bio) < 0)
 791                                 goto out;
 792
 793                 update_position(&index, &offset, &bvec);
 794         }
 795
 796         set_bit(BIO_UPTODATE, &bio->bi_flags);
 797         bio_endio(bio, 0);
 798         return;
 799
 800 out:
 801         bio_io_error(bio);
 802 }
 803
 804 /*
 805  * Handler function for all zram I/O requests.
 806  */
 807 static void zram_make_request(struct request_queue *queue, struct bio *bio)
 808 {
 809         struct zram *zram = queue->queuedata;
 810
 811         down_read(&zram->init_lock);
 812         if (unlikely(!init_done(zram)))
 813                 goto error;
 814
 815         if (!valid_io_request(zram, bio)) {
 816                 atomic64_inc(&zram->stats.invalid_io);
 817                 goto error;
 818         }
 819
 820         __zram_make_request(zram, bio);
 821         up_read(&zram->init_lock);
 822
 823         return;
 824
 825 error:
 826         up_read(&zram->init_lock);
 827         bio_io_error(bio);
 828 }
 829
 830 static void zram_slot_free_notify(struct block_device *bdev,
 831                                 unsigned long index)
 832 {
 833         struct zram *zram;
 834         struct zram_meta *meta;
 835
 836         zram = bdev->bd_disk->private_data;
 837         meta = zram->meta;
 838
 839         bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
 840         zram_free_page(zram, index);
 841         bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
 842         atomic64_inc(&zram->stats.notify_free);
 843 }
 844
 845 static const struct block_device_operations zram_devops = {
 846         .swap_slot_free_notify = zram_slot_free_notify,
 847         .owner = THIS_MODULE
 848 };
 849
 850 static DEVICE_ATTR(disksize, S_IRUGO | S_IWUSR,
 851                 disksize_show, disksize_store);
 852 static DEVICE_ATTR(initstate, S_IRUGO, initstate_show, NULL);
 853 static DEVICE_ATTR(reset, S_IWUSR, NULL, reset_store);
 854 static DEVICE_ATTR(orig_data_size, S_IRUGO, orig_data_size_show, NULL);
 855 static DEVICE_ATTR(mem_used_total, S_IRUGO, mem_used_total_show, NULL);
 856 static DEVICE_ATTR(max_comp_streams, S_IRUGO | S_IWUSR,
 857                 max_comp_streams_show, max_comp_streams_store);
 858 static DEVICE_ATTR(comp_algorithm, S_IRUGO | S_IWUSR,
 859                 comp_algorithm_show, comp_algorithm_store);
 860
 861 ZRAM_ATTR_RO(num_reads);
 862 ZRAM_ATTR_RO(num_writes);
 863 ZRAM_ATTR_RO(failed_reads);
 864 ZRAM_ATTR_RO(failed_writes);
 865 ZRAM_ATTR_RO(invalid_io);
 866 ZRAM_ATTR_RO(notify_free);
 867 ZRAM_ATTR_RO(zero_pages);
 868 ZRAM_ATTR_RO(compr_data_size);
 869
 870 static struct attribute *zram_disk_attrs[] = {
 871         &dev_attr_disksize.attr,
 872         &dev_attr_initstate.attr,
 873         &dev_attr_reset.attr,
 874         &dev_attr_num_reads.attr,
 875         &dev_attr_num_writes.attr,
 876         &dev_attr_failed_reads.attr,
 877         &dev_attr_failed_writes.attr,
 878         &dev_attr_invalid_io.attr,
 879         &dev_attr_notify_free.attr,
 880         &dev_attr_zero_pages.attr,
 881         &dev_attr_orig_data_size.attr,
 882         &dev_attr_compr_data_size.attr,
 883         &dev_attr_mem_used_total.attr,
 884         &dev_attr_max_comp_streams.attr,
 885         &dev_attr_comp_algorithm.attr,
 886         NULL,
 887 };
 888
 889 static struct attribute_group zram_disk_attr_group = {
 890         .attrs = zram_disk_attrs,
 891 };
 892
 893 static int create_device(struct zram *zram, int device_id)
 894 {
 895         int ret = -ENOMEM;
 896
 897         init_rwsem(&zram->init_lock);
 898
 899         zram->queue = blk_alloc_queue(GFP_KERNEL);
 900         if (!zram->queue) {
 901                 pr_err("Error allocating disk queue for device %d\n",
 902                         device_id);
 903                 goto out;
 904         }
 905
 906         blk_queue_make_request(zram->queue, zram_make_request);
 907         zram->queue->queuedata = zram;
 908
 909          /* gendisk structure */
 910         zram->disk = alloc_disk(1);
 911         if (!zram->disk) {
 912                 pr_warn("Error allocating disk structure for device %d\n",
 913                         device_id);
 914                 goto out_free_queue;
 915         }
 916
 917         zram->disk->major = zram_major;
 918         zram->disk->first_minor = device_id;
 919         zram->disk->fops = &zram_devops;
 920         zram->disk->queue = zram->queue;
 921         zram->disk->private_data = zram;
 922         snprintf(zram->disk->disk_name, 16, "zram%d", device_id);
 923
 924         /* Actual capacity set using syfs (/sys/block/zram<id>/disksize */
 925         set_capacity(zram->disk, 0);
 926         /* zram devices sort of resembles non-rotational disks */
 927         queue_flag_set_unlocked(QUEUE_FLAG_NONROT, zram->disk->queue);
 928         /*
 929          * To ensure that we always get PAGE_SIZE aligned
 930          * and n*PAGE_SIZED sized I/O requests.
 931          */
 932         blk_queue_physical_block_size(zram->disk->queue, PAGE_SIZE);
 933         blk_queue_logical_block_size(zram->disk->queue,
 934                                         ZRAM_LOGICAL_BLOCK_SIZE);
 935         blk_queue_io_min(zram->disk->queue, PAGE_SIZE);
 936         blk_queue_io_opt(zram->disk->queue, PAGE_SIZE);
 937         zram->disk->queue->limits.discard_granularity = PAGE_SIZE;
 938         zram->disk->queue->limits.max_discard_sectors = UINT_MAX;
 939         /*
 940          * zram_bio_discard() will clear all logical blocks if logical block
 941          * size is identical with physical block size(PAGE_SIZE). But if it is
 942          * different, we will skip discarding some parts of logical blocks in
 943          * the part of the request range which isn't aligned to physical block
 944          * size.  So we can't ensure that all discarded logical blocks are
 945          * zeroed.
 946          */
 947         if (ZRAM_LOGICAL_BLOCK_SIZE == PAGE_SIZE)
 948                 zram->disk->queue->limits.discard_zeroes_data = 1;
 949         else
 950                 zram->disk->queue->limits.discard_zeroes_data = 0;
 951         queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, zram->disk->queue);
 952
 953         add_disk(zram->disk);
 954
 955         ret = sysfs_create_group(&disk_to_dev(zram->disk)->kobj,
 956                                 &zram_disk_attr_group);
 957         if (ret < 0) {
 958                 pr_warn("Error creating sysfs group");
 959                 goto out_free_disk;
 960         }
 961         strlcpy(zram->compressor, default_compressor, sizeof(zram->compressor));
 962         zram->meta = NULL;
 963         zram->max_comp_streams = 1;
 964         return 0;
 965
 966 out_free_disk:
 967         del_gendisk(zram->disk);
 968         put_disk(zram->disk);
 969 out_free_queue:
 970         blk_cleanup_queue(zram->queue);
 971 out:
 972         return ret;
 973 }
 974
 975 static void destroy_device(struct zram *zram)
 976 {
 977         sysfs_remove_group(&disk_to_dev(zram->disk)->kobj,
 978                         &zram_disk_attr_group);
 979
 980         del_gendisk(zram->disk);
 981         put_disk(zram->disk);
 982
 983         blk_cleanup_queue(zram->queue);
 984 }
 985
 986 static int __init zram_init(void)
 987 {
 988         int ret, dev_id;
 989
 990         if (num_devices > max_num_devices) {
 991                 pr_warn("Invalid value for num_devices: %u\n",
 992                                 num_devices);
 993                 ret = -EINVAL;
 994                 goto out;
 995         }
 996
 997         zram_major = register_blkdev(0, "zram");
 998         if (zram_major <= 0) {
 999                 pr_warn("Unable to get major number\n");
1000                 ret = -EBUSY;
1001                 goto out;
1002         }
1003
1004         /* Allocate the device array and initialize each one */
1005         zram_devices = kzalloc(num_devices * sizeof(struct zram), GFP_KERNEL);
1006         if (!zram_devices) {
1007                 ret = -ENOMEM;
1008                 goto unregister;
1009         }
1010
1011         for (dev_id = 0; dev_id < num_devices; dev_id++) {
1012                 ret = create_device(&zram_devices[dev_id], dev_id);
1013                 if (ret)
1014                         goto free_devices;
1015         }
1016
1017         pr_info("Created %u device(s) ...\n", num_devices);
1018
1019         return 0;
1020
1021 free_devices:
1022         while (dev_id)
1023                 destroy_device(&zram_devices[--dev_id]);
1024         kfree(zram_devices);
1025 unregister:
1026         unregister_blkdev(zram_major, "zram");
1027 out:
1028         return ret;
1029 }
1030
1031 static void __exit zram_exit(void)
1032 {
1033         int i;
1034         struct zram *zram;
1035
1036         for (i = 0; i < num_devices; i++) {
1037                 zram = &zram_devices[i];
1038
1039                 destroy_device(zram);
1040                 /*
1041                  * Shouldn't access zram->disk after destroy_device
1042                  * because destroy_device already released zram->disk.
1043                  */
1044                 zram_reset_device(zram, false);
1045         }
1046
1047         unregister_blkdev(zram_major, "zram");
1048
1049         kfree(zram_devices);
1050         pr_debug("Cleanup done!\n");
1051 }
1052
1053 module_init(zram_init);
1054 module_exit(zram_exit);
1055
1056 module_param(num_devices, uint, 0);
1057 MODULE_PARM_DESC(num_devices, "Number of zram devices");
1058
1059 MODULE_LICENSE("Dual BSD/GPL");
1060 MODULE_AUTHOR("Nitin Gupta <ngupta@vflare.org>");
1061 MODULE_DESCRIPTION("Compressed RAM Block Device");