mm/huge_memory.c

   1 /*
   2  *  Copyright (C) 2009  Red Hat, Inc.
   3  *
   4  *  This work is licensed under the terms of the GNU GPL, version 2. See
   5  *  the COPYING file in the top-level directory.
   6  */
   7
   8 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
   9
  10 #include <linux/mm.h>
  11 #include <linux/sched.h>
  12 #include <linux/highmem.h>
  13 #include <linux/hugetlb.h>
  14 #include <linux/mmu_notifier.h>
  15 #include <linux/rmap.h>
  16 #include <linux/swap.h>
  17 #include <linux/shrinker.h>
  18 #include <linux/mm_inline.h>
  19 #include <linux/kthread.h>
  20 #include <linux/khugepaged.h>
  21 #include <linux/freezer.h>
  22 #include <linux/mman.h>
  23 #include <linux/pagemap.h>
  24 #include <linux/migrate.h>
  25 #include <linux/hashtable.h>
  26
  27 #include <asm/tlb.h>
  28 #include <asm/pgalloc.h>
  29 #include "internal.h"
  30
  31 /*
  32  * By default transparent hugepage support is disabled in order that avoid
  33  * to risk increase the memory footprint of applications without a guaranteed
  34  * benefit. When transparent hugepage support is enabled, is for all mappings,
  35  * and khugepaged scans all mappings.
  36  * Defrag is invoked by khugepaged hugepage allocations and by page faults
  37  * for all hugepage allocations.
  38  */
  39 unsigned long transparent_hugepage_flags __read_mostly =
  40 #ifdef CONFIG_TRANSPARENT_HUGEPAGE_ALWAYS
  41         (1<<TRANSPARENT_HUGEPAGE_FLAG)|
  42 #endif
  43 #ifdef CONFIG_TRANSPARENT_HUGEPAGE_MADVISE
  44         (1<<TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG)|
  45 #endif
  46         (1<<TRANSPARENT_HUGEPAGE_DEFRAG_FLAG)|
  47         (1<<TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG)|
  48         (1<<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG);
  49
  50 /* default scan 8*512 pte (or vmas) every 30 second */
  51 static unsigned int khugepaged_pages_to_scan __read_mostly = HPAGE_PMD_NR*8;
  52 static unsigned int khugepaged_pages_collapsed;
  53 static unsigned int khugepaged_full_scans;
  54 static unsigned int khugepaged_scan_sleep_millisecs __read_mostly = 10000;
  55 /* during fragmentation poll the hugepage allocator once every minute */
  56 static unsigned int khugepaged_alloc_sleep_millisecs __read_mostly = 60000;
  57 static struct task_struct *khugepaged_thread __read_mostly;
  58 static DEFINE_MUTEX(khugepaged_mutex);
  59 static DEFINE_SPINLOCK(khugepaged_mm_lock);
  60 static DECLARE_WAIT_QUEUE_HEAD(khugepaged_wait);
  61 /*
  62  * default collapse hugepages if there is at least one pte mapped like
  63  * it would have happened if the vma was large enough during page
  64  * fault.
  65  */
  66 static unsigned int khugepaged_max_ptes_none __read_mostly = HPAGE_PMD_NR-1;
  67
  68 static int khugepaged(void *none);
  69 static int khugepaged_slab_init(void);
  70 static void khugepaged_slab_exit(void);
  71
  72 #define MM_SLOTS_HASH_BITS 10
  73 static __read_mostly DEFINE_HASHTABLE(mm_slots_hash, MM_SLOTS_HASH_BITS);
  74
  75 static struct kmem_cache *mm_slot_cache __read_mostly;
  76
  77 /**
  78  * struct mm_slot - hash lookup from mm to mm_slot
  79  * @hash: hash collision list
  80  * @mm_node: khugepaged scan list headed in khugepaged_scan.mm_head
  81  * @mm: the mm that this information is valid for
  82  */
  83 struct mm_slot {
  84         struct hlist_node hash;
  85         struct list_head mm_node;
  86         struct mm_struct *mm;
  87 };
  88
  89 /**
  90  * struct khugepaged_scan - cursor for scanning
  91  * @mm_head: the head of the mm list to scan
  92  * @mm_slot: the current mm_slot we are scanning
  93  * @address: the next address inside that to be scanned
  94  *
  95  * There is only the one khugepaged_scan instance of this cursor structure.
  96  */
  97 struct khugepaged_scan {
  98         struct list_head mm_head;
  99         struct mm_slot *mm_slot;
 100         unsigned long address;
 101 };
 102 static struct khugepaged_scan khugepaged_scan = {
 103         .mm_head = LIST_HEAD_INIT(khugepaged_scan.mm_head),
 104 };
 105
 106
 107 static int set_recommended_min_free_kbytes(void)
 108 {
 109         struct zone *zone;
 110         int nr_zones = 0;
 111         unsigned long recommended_min;
 112
 113         if (!khugepaged_enabled())
 114                 return 0;
 115
 116         for_each_populated_zone(zone)
 117                 nr_zones++;
 118
 119         /* Make sure at least 2 hugepages are free for MIGRATE_RESERVE */
 120         recommended_min = pageblock_nr_pages * nr_zones * 2;
 121
 122         /*
 123          * Make sure that on average at least two pageblocks are almost free
 124          * of another type, one for a migratetype to fall back to and a
 125          * second to avoid subsequent fallbacks of other types There are 3
 126          * MIGRATE_TYPES we care about.
 127          */
 128         recommended_min += pageblock_nr_pages * nr_zones *
 129                            MIGRATE_PCPTYPES * MIGRATE_PCPTYPES;
 130
 131         /* don't ever allow to reserve more than 5% of the lowmem */
 132         recommended_min = min(recommended_min,
 133                               (unsigned long) nr_free_buffer_pages() / 20);
 134         recommended_min <<= (PAGE_SHIFT-10);
 135
 136         if (recommended_min > min_free_kbytes) {
 137                 if (user_min_free_kbytes >= 0)
 138                         pr_info("raising min_free_kbytes from %d to %lu "
 139                                 "to help transparent hugepage allocations\n",
 140                                 min_free_kbytes, recommended_min);
 141
 142                 min_free_kbytes = recommended_min;
 143         }
 144         setup_per_zone_wmarks();
 145         return 0;
 146 }
 147 late_initcall(set_recommended_min_free_kbytes);
 148
 149 static int start_khugepaged(void)
 150 {
 151         int err = 0;
 152         if (khugepaged_enabled()) {
 153                 if (!khugepaged_thread)
 154                         khugepaged_thread = kthread_run(khugepaged, NULL,
 155                                                         "khugepaged");
 156                 if (unlikely(IS_ERR(khugepaged_thread))) {
 157                         pr_err("khugepaged: kthread_run(khugepaged) failed\n");
 158                         err = PTR_ERR(khugepaged_thread);
 159                         khugepaged_thread = NULL;
 160                 }
 161
 162                 if (!list_empty(&khugepaged_scan.mm_head))
 163                         wake_up_interruptible(&khugepaged_wait);
 164
 165                 set_recommended_min_free_kbytes();
 166         } else if (khugepaged_thread) {
 167                 kthread_stop(khugepaged_thread);
 168                 khugepaged_thread = NULL;
 169         }
 170
 171         return err;
 172 }
 173
 174 static atomic_t huge_zero_refcount;
 175 struct page *huge_zero_page __read_mostly;
 176
 177 static inline bool is_huge_zero_pmd(pmd_t pmd)
 178 {
 179         return is_huge_zero_page(pmd_page(pmd));
 180 }
 181
 182 static struct page *get_huge_zero_page(void)
 183 {
 184         struct page *zero_page;
 185 retry:
 186         if (likely(atomic_inc_not_zero(&huge_zero_refcount)))
 187                 return READ_ONCE(huge_zero_page);
 188
 189         zero_page = alloc_pages((GFP_TRANSHUGE | __GFP_ZERO) & ~__GFP_MOVABLE,
 190                         HPAGE_PMD_ORDER);
 191         if (!zero_page) {
 192                 count_vm_event(THP_ZERO_PAGE_ALLOC_FAILED);
 193                 return NULL;
 194         }
 195         count_vm_event(THP_ZERO_PAGE_ALLOC);
 196         preempt_disable();
 197         if (cmpxchg(&huge_zero_page, NULL, zero_page)) {
 198                 preempt_enable();
 199                 __free_pages(zero_page, compound_order(zero_page));
 200                 goto retry;
 201         }
 202
 203         /* We take additional reference here. It will be put back by shrinker */
 204         atomic_set(&huge_zero_refcount, 2);
 205         preempt_enable();
 206         return READ_ONCE(huge_zero_page);
 207 }
 208
 209 static void put_huge_zero_page(void)
 210 {
 211         /*
 212          * Counter should never go to zero here. Only shrinker can put
 213          * last reference.
 214          */
 215         BUG_ON(atomic_dec_and_test(&huge_zero_refcount));
 216 }
 217
 218 static unsigned long shrink_huge_zero_page_count(struct shrinker *shrink,
 219                                         struct shrink_control *sc)
 220 {
 221         /* we can free zero page only if last reference remains */
 222         return atomic_read(&huge_zero_refcount) == 1 ? HPAGE_PMD_NR : 0;
 223 }
 224
 225 static unsigned long shrink_huge_zero_page_scan(struct shrinker *shrink,
 226                                        struct shrink_control *sc)
 227 {
 228         if (atomic_cmpxchg(&huge_zero_refcount, 1, 0) == 1) {
 229                 struct page *zero_page = xchg(&huge_zero_page, NULL);
 230                 BUG_ON(zero_page == NULL);
 231                 __free_pages(zero_page, compound_order(zero_page));
 232                 return HPAGE_PMD_NR;
 233         }
 234
 235         return 0;
 236 }
 237
 238 static struct shrinker huge_zero_page_shrinker = {
 239         .count_objects = shrink_huge_zero_page_count,
 240         .scan_objects = shrink_huge_zero_page_scan,
 241         .seeks = DEFAULT_SEEKS,
 242 };
 243
 244 #ifdef CONFIG_SYSFS
 245
 246 static ssize_t double_flag_show(struct kobject *kobj,
 247                                 struct kobj_attribute *attr, char *buf,
 248                                 enum transparent_hugepage_flag enabled,
 249                                 enum transparent_hugepage_flag req_madv)
 250 {
 251         if (test_bit(enabled, &transparent_hugepage_flags)) {
 252                 VM_BUG_ON(test_bit(req_madv, &transparent_hugepage_flags));
 253                 return sprintf(buf, "[always] madvise never\n");
 254         } else if (test_bit(req_madv, &transparent_hugepage_flags))
 255                 return sprintf(buf, "always [madvise] never\n");
 256         else
 257                 return sprintf(buf, "always madvise [never]\n");
 258 }
 259 static ssize_t double_flag_store(struct kobject *kobj,
 260                                  struct kobj_attribute *attr,
 261                                  const char *buf, size_t count,
 262                                  enum transparent_hugepage_flag enabled,
 263                                  enum transparent_hugepage_flag req_madv)
 264 {
 265         if (!memcmp("always", buf,
 266                     min(sizeof("always")-1, count))) {
 267                 set_bit(enabled, &transparent_hugepage_flags);
 268                 clear_bit(req_madv, &transparent_hugepage_flags);
 269         } else if (!memcmp("madvise", buf,
 270                            min(sizeof("madvise")-1, count))) {
 271                 clear_bit(enabled, &transparent_hugepage_flags);
 272                 set_bit(req_madv, &transparent_hugepage_flags);
 273         } else if (!memcmp("never", buf,
 274                            min(sizeof("never")-1, count))) {
 275                 clear_bit(enabled, &transparent_hugepage_flags);
 276                 clear_bit(req_madv, &transparent_hugepage_flags);
 277         } else
 278                 return -EINVAL;
 279
 280         return count;
 281 }
 282
 283 static ssize_t enabled_show(struct kobject *kobj,
 284                             struct kobj_attribute *attr, char *buf)
 285 {
 286         return double_flag_show(kobj, attr, buf,
 287                                 TRANSPARENT_HUGEPAGE_FLAG,
 288                                 TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG);
 289 }
 290 static ssize_t enabled_store(struct kobject *kobj,
 291                              struct kobj_attribute *attr,
 292                              const char *buf, size_t count)
 293 {
 294         ssize_t ret;
 295
 296         ret = double_flag_store(kobj, attr, buf, count,
 297                                 TRANSPARENT_HUGEPAGE_FLAG,
 298                                 TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG);
 299
 300         if (ret > 0) {
 301                 int err;
 302
 303                 mutex_lock(&khugepaged_mutex);
 304                 err = start_khugepaged();
 305                 mutex_unlock(&khugepaged_mutex);
 306
 307                 if (err)
 308                         ret = err;
 309         }
 310
 311         return ret;
 312 }
 313 static struct kobj_attribute enabled_attr =
 314         __ATTR(enabled, 0644, enabled_show, enabled_store);
 315
 316 static ssize_t single_flag_show(struct kobject *kobj,
 317                                 struct kobj_attribute *attr, char *buf,
 318                                 enum transparent_hugepage_flag flag)
 319 {
 320         return sprintf(buf, "%d\n",
 321                        !!test_bit(flag, &transparent_hugepage_flags));
 322 }
 323
 324 static ssize_t single_flag_store(struct kobject *kobj,
 325                                  struct kobj_attribute *attr,
 326                                  const char *buf, size_t count,
 327                                  enum transparent_hugepage_flag flag)
 328 {
 329         unsigned long value;
 330         int ret;
 331
 332         ret = kstrtoul(buf, 10, &value);
 333         if (ret < 0)
 334                 return ret;
 335         if (value > 1)
 336                 return -EINVAL;
 337
 338         if (value)
 339                 set_bit(flag, &transparent_hugepage_flags);
 340         else
 341                 clear_bit(flag, &transparent_hugepage_flags);
 342
 343         return count;
 344 }
 345
 346 /*
 347  * Currently defrag only disables __GFP_NOWAIT for allocation. A blind
 348  * __GFP_REPEAT is too aggressive, it's never worth swapping tons of
 349  * memory just to allocate one more hugepage.
 350  */
 351 static ssize_t defrag_show(struct kobject *kobj,
 352                            struct kobj_attribute *attr, char *buf)
 353 {
 354         return double_flag_show(kobj, attr, buf,
 355                                 TRANSPARENT_HUGEPAGE_DEFRAG_FLAG,
 356                                 TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG);
 357 }
 358 static ssize_t defrag_store(struct kobject *kobj,
 359                             struct kobj_attribute *attr,
 360                             const char *buf, size_t count)
 361 {
 362         return double_flag_store(kobj, attr, buf, count,
 363                                  TRANSPARENT_HUGEPAGE_DEFRAG_FLAG,
 364                                  TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG);
 365 }
 366 static struct kobj_attribute defrag_attr =
 367         __ATTR(defrag, 0644, defrag_show, defrag_store);
 368
 369 static ssize_t use_zero_page_show(struct kobject *kobj,
 370                 struct kobj_attribute *attr, char *buf)
 371 {
 372         return single_flag_show(kobj, attr, buf,
 373                                 TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG);
 374 }
 375 static ssize_t use_zero_page_store(struct kobject *kobj,
 376                 struct kobj_attribute *attr, const char *buf, size_t count)
 377 {
 378         return single_flag_store(kobj, attr, buf, count,
 379                                  TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG);
 380 }
 381 static struct kobj_attribute use_zero_page_attr =
 382         __ATTR(use_zero_page, 0644, use_zero_page_show, use_zero_page_store);
 383 #ifdef CONFIG_DEBUG_VM
 384 static ssize_t debug_cow_show(struct kobject *kobj,
 385                                 struct kobj_attribute *attr, char *buf)
 386 {
 387         return single_flag_show(kobj, attr, buf,
 388                                 TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG);
 389 }
 390 static ssize_t debug_cow_store(struct kobject *kobj,
 391                                struct kobj_attribute *attr,
 392                                const char *buf, size_t count)
 393 {
 394         return single_flag_store(kobj, attr, buf, count,
 395                                  TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG);
 396 }
 397 static struct kobj_attribute debug_cow_attr =
 398         __ATTR(debug_cow, 0644, debug_cow_show, debug_cow_store);
 399 #endif /* CONFIG_DEBUG_VM */
 400
 401 static struct attribute *hugepage_attr[] = {
 402         &enabled_attr.attr,
 403         &defrag_attr.attr,
 404         &use_zero_page_attr.attr,
 405 #ifdef CONFIG_DEBUG_VM
 406         &debug_cow_attr.attr,
 407 #endif
 408         NULL,
 409 };
 410
 411 static struct attribute_group hugepage_attr_group = {
 412         .attrs = hugepage_attr,
 413 };
 414
 415 static ssize_t scan_sleep_millisecs_show(struct kobject *kobj,
 416                                          struct kobj_attribute *attr,
 417                                          char *buf)
 418 {
 419         return sprintf(buf, "%u\n", khugepaged_scan_sleep_millisecs);
 420 }
 421
 422 static ssize_t scan_sleep_millisecs_store(struct kobject *kobj,
 423                                           struct kobj_attribute *attr,
 424                                           const char *buf, size_t count)
 425 {
 426         unsigned long msecs;
 427         int err;
 428
 429         err = kstrtoul(buf, 10, &msecs);
 430         if (err || msecs > UINT_MAX)
 431                 return -EINVAL;
 432
 433         khugepaged_scan_sleep_millisecs = msecs;
 434         wake_up_interruptible(&khugepaged_wait);
 435
 436         return count;
 437 }
 438 static struct kobj_attribute scan_sleep_millisecs_attr =
 439         __ATTR(scan_sleep_millisecs, 0644, scan_sleep_millisecs_show,
 440                scan_sleep_millisecs_store);
 441
 442 static ssize_t alloc_sleep_millisecs_show(struct kobject *kobj,
 443                                           struct kobj_attribute *attr,
 444                                           char *buf)
 445 {
 446         return sprintf(buf, "%u\n", khugepaged_alloc_sleep_millisecs);
 447 }
 448
 449 static ssize_t alloc_sleep_millisecs_store(struct kobject *kobj,
 450                                            struct kobj_attribute *attr,
 451                                            const char *buf, size_t count)
 452 {
 453         unsigned long msecs;
 454         int err;
 455
 456         err = kstrtoul(buf, 10, &msecs);
 457         if (err || msecs > UINT_MAX)
 458                 return -EINVAL;
 459
 460         khugepaged_alloc_sleep_millisecs = msecs;
 461         wake_up_interruptible(&khugepaged_wait);
 462
 463         return count;
 464 }
 465 static struct kobj_attribute alloc_sleep_millisecs_attr =
 466         __ATTR(alloc_sleep_millisecs, 0644, alloc_sleep_millisecs_show,
 467                alloc_sleep_millisecs_store);
 468
 469 static ssize_t pages_to_scan_show(struct kobject *kobj,
 470                                   struct kobj_attribute *attr,
 471                                   char *buf)
 472 {
 473         return sprintf(buf, "%u\n", khugepaged_pages_to_scan);
 474 }
 475 static ssize_t pages_to_scan_store(struct kobject *kobj,
 476                                    struct kobj_attribute *attr,
 477                                    const char *buf, size_t count)
 478 {
 479         int err;
 480         unsigned long pages;
 481
 482         err = kstrtoul(buf, 10, &pages);
 483         if (err || !pages || pages > UINT_MAX)
 484                 return -EINVAL;
 485
 486         khugepaged_pages_to_scan = pages;
 487
 488         return count;
 489 }
 490 static struct kobj_attribute pages_to_scan_attr =
 491         __ATTR(pages_to_scan, 0644, pages_to_scan_show,
 492                pages_to_scan_store);
 493
 494 static ssize_t pages_collapsed_show(struct kobject *kobj,
 495                                     struct kobj_attribute *attr,
 496                                     char *buf)
 497 {
 498         return sprintf(buf, "%u\n", khugepaged_pages_collapsed);
 499 }
 500 static struct kobj_attribute pages_collapsed_attr =
 501         __ATTR_RO(pages_collapsed);
 502
 503 static ssize_t full_scans_show(struct kobject *kobj,
 504                                struct kobj_attribute *attr,
 505                                char *buf)
 506 {
 507         return sprintf(buf, "%u\n", khugepaged_full_scans);
 508 }
 509 static struct kobj_attribute full_scans_attr =
 510         __ATTR_RO(full_scans);
 511
 512 static ssize_t khugepaged_defrag_show(struct kobject *kobj,
 513                                       struct kobj_attribute *attr, char *buf)
 514 {
 515         return single_flag_show(kobj, attr, buf,
 516                                 TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG);
 517 }
 518 static ssize_t khugepaged_defrag_store(struct kobject *kobj,
 519                                        struct kobj_attribute *attr,
 520                                        const char *buf, size_t count)
 521 {
 522         return single_flag_store(kobj, attr, buf, count,
 523                                  TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG);
 524 }
 525 static struct kobj_attribute khugepaged_defrag_attr =
 526         __ATTR(defrag, 0644, khugepaged_defrag_show,
 527                khugepaged_defrag_store);
 528
 529 /*
 530  * max_ptes_none controls if khugepaged should collapse hugepages over
 531  * any unmapped ptes in turn potentially increasing the memory
 532  * footprint of the vmas. When max_ptes_none is 0 khugepaged will not
 533  * reduce the available free memory in the system as it
 534  * runs. Increasing max_ptes_none will instead potentially reduce the
 535  * free memory in the system during the khugepaged scan.
 536  */
 537 static ssize_t khugepaged_max_ptes_none_show(struct kobject *kobj,
 538                                              struct kobj_attribute *attr,
 539                                              char *buf)
 540 {
 541         return sprintf(buf, "%u\n", khugepaged_max_ptes_none);
 542 }
 543 static ssize_t khugepaged_max_ptes_none_store(struct kobject *kobj,
 544                                               struct kobj_attribute *attr,
 545                                               const char *buf, size_t count)
 546 {
 547         int err;
 548         unsigned long max_ptes_none;
 549
 550         err = kstrtoul(buf, 10, &max_ptes_none);
 551         if (err || max_ptes_none > HPAGE_PMD_NR-1)
 552                 return -EINVAL;
 553
 554         khugepaged_max_ptes_none = max_ptes_none;
 555
 556         return count;
 557 }
 558 static struct kobj_attribute khugepaged_max_ptes_none_attr =
 559         __ATTR(max_ptes_none, 0644, khugepaged_max_ptes_none_show,
 560                khugepaged_max_ptes_none_store);
 561
 562 static struct attribute *khugepaged_attr[] = {
 563         &khugepaged_defrag_attr.attr,
 564         &khugepaged_max_ptes_none_attr.attr,
 565         &pages_to_scan_attr.attr,
 566         &pages_collapsed_attr.attr,
 567         &full_scans_attr.attr,
 568         &scan_sleep_millisecs_attr.attr,
 569         &alloc_sleep_millisecs_attr.attr,
 570         NULL,
 571 };
 572
 573 static struct attribute_group khugepaged_attr_group = {
 574         .attrs = khugepaged_attr,
 575         .name = "khugepaged",
 576 };
 577
 578 static int __init hugepage_init_sysfs(struct kobject **hugepage_kobj)
 579 {
 580         int err;
 581
 582         *hugepage_kobj = kobject_create_and_add("transparent_hugepage", mm_kobj);
 583         if (unlikely(!*hugepage_kobj)) {
 584                 pr_err("failed to create transparent hugepage kobject\n");
 585                 return -ENOMEM;
 586         }
 587
 588         err = sysfs_create_group(*hugepage_kobj, &hugepage_attr_group);
 589         if (err) {
 590                 pr_err("failed to register transparent hugepage group\n");
 591                 goto delete_obj;
 592         }
 593
 594         err = sysfs_create_group(*hugepage_kobj, &khugepaged_attr_group);
 595         if (err) {
 596                 pr_err("failed to register transparent hugepage group\n");
 597                 goto remove_hp_group;
 598         }
 599
 600         return 0;
 601
 602 remove_hp_group:
 603         sysfs_remove_group(*hugepage_kobj, &hugepage_attr_group);
 604 delete_obj:
 605         kobject_put(*hugepage_kobj);
 606         return err;
 607 }
 608
 609 static void __init hugepage_exit_sysfs(struct kobject *hugepage_kobj)
 610 {
 611         sysfs_remove_group(hugepage_kobj, &khugepaged_attr_group);
 612         sysfs_remove_group(hugepage_kobj, &hugepage_attr_group);
 613         kobject_put(hugepage_kobj);
 614 }
 615 #else
 616 static inline int hugepage_init_sysfs(struct kobject **hugepage_kobj)
 617 {
 618         return 0;
 619 }
 620
 621 static inline void hugepage_exit_sysfs(struct kobject *hugepage_kobj)
 622 {
 623 }
 624 #endif /* CONFIG_SYSFS */
 625
 626 static int __init hugepage_init(void)
 627 {
 628         int err;
 629         struct kobject *hugepage_kobj;
 630
 631         if (!has_transparent_hugepage()) {
 632                 transparent_hugepage_flags = 0;
 633                 return -EINVAL;
 634         }
 635
 636         err = hugepage_init_sysfs(&hugepage_kobj);
 637         if (err)
 638                 goto err_sysfs;
 639
 640         err = khugepaged_slab_init();
 641         if (err)
 642                 goto err_slab;
 643
 644         err = register_shrinker(&huge_zero_page_shrinker);
 645         if (err)
 646                 goto err_hzp_shrinker;
 647
 648         /*
 649          * By default disable transparent hugepages on smaller systems,
 650          * where the extra memory used could hurt more than TLB overhead
 651          * is likely to save.  The admin can still enable it through /sys.
 652          */
 653         if (totalram_pages < (512 << (20 - PAGE_SHIFT)))
 654                 transparent_hugepage_flags = 0;
 655
 656         err = start_khugepaged();
 657         if (err)
 658                 goto err_khugepaged;
 659
 660         return 0;
 661 err_khugepaged:
 662         unregister_shrinker(&huge_zero_page_shrinker);
 663 err_hzp_shrinker:
 664         khugepaged_slab_exit();
 665 err_slab:
 666         hugepage_exit_sysfs(hugepage_kobj);
 667 err_sysfs:
 668         return err;
 669 }
 670 subsys_initcall(hugepage_init);
 671
 672 static int __init setup_transparent_hugepage(char *str)
 673 {
 674         int ret = 0;
 675         if (!str)
 676                 goto out;
 677         if (!strcmp(str, "always")) {
 678                 set_bit(TRANSPARENT_HUGEPAGE_FLAG,
 679                         &transparent_hugepage_flags);
 680                 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
 681                           &transparent_hugepage_flags);
 682                 ret = 1;
 683         } else if (!strcmp(str, "madvise")) {
 684                 clear_bit(TRANSPARENT_HUGEPAGE_FLAG,
 685                           &transparent_hugepage_flags);
 686                 set_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
 687                         &transparent_hugepage_flags);
 688                 ret = 1;
 689         } else if (!strcmp(str, "never")) {
 690                 clear_bit(TRANSPARENT_HUGEPAGE_FLAG,
 691                           &transparent_hugepage_flags);
 692                 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
 693                           &transparent_hugepage_flags);
 694                 ret = 1;
 695         }
 696 out:
 697         if (!ret)
 698                 pr_warn("transparent_hugepage= cannot parse, ignored\n");
 699         return ret;
 700 }
 701 __setup("transparent_hugepage=", setup_transparent_hugepage);
 702
 703 pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma)
 704 {
 705         if (likely(vma->vm_flags & VM_WRITE))
 706                 pmd = pmd_mkwrite(pmd);
 707         return pmd;
 708 }
 709
 710 static inline pmd_t mk_huge_pmd(struct page *page, pgprot_t prot)
 711 {
 712         pmd_t entry;
 713         entry = mk_pmd(page, prot);
 714         entry = pmd_mkhuge(entry);
 715         return entry;
 716 }
 717
 718 static int __do_huge_pmd_anonymous_page(struct mm_struct *mm,
 719                                         struct vm_area_struct *vma,
 720                                         unsigned long haddr, pmd_t *pmd,
 721                                         struct page *page, gfp_t gfp)
 722 {
 723         struct mem_cgroup *memcg;
 724         pgtable_t pgtable;
 725         spinlock_t *ptl;
 726
 727         VM_BUG_ON_PAGE(!PageCompound(page), page);
 728
 729         if (mem_cgroup_try_charge(page, mm, gfp, &memcg))
 730                 return VM_FAULT_OOM;
 731
 732         pgtable = pte_alloc_one(mm, haddr);
 733         if (unlikely(!pgtable)) {
 734                 mem_cgroup_cancel_charge(page, memcg);
 735                 return VM_FAULT_OOM;
 736         }
 737
 738         clear_huge_page(page, haddr, HPAGE_PMD_NR);
 739         /*
 740          * The memory barrier inside __SetPageUptodate makes sure that
 741          * clear_huge_page writes become visible before the set_pmd_at()
 742          * write.
 743          */
 744         __SetPageUptodate(page);
 745
 746         ptl = pmd_lock(mm, pmd);
 747         if (unlikely(!pmd_none(*pmd))) {
 748                 spin_unlock(ptl);
 749                 mem_cgroup_cancel_charge(page, memcg);
 750                 put_page(page);
 751                 pte_free(mm, pgtable);
 752         } else {
 753                 pmd_t entry;
 754                 entry = mk_huge_pmd(page, vma->vm_page_prot);
 755                 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
 756                 page_add_new_anon_rmap(page, vma, haddr);
 757                 mem_cgroup_commit_charge(page, memcg, false);
 758                 lru_cache_add_active_or_unevictable(page, vma);
 759                 pgtable_trans_huge_deposit(mm, pmd, pgtable);
 760                 set_pmd_at(mm, haddr, pmd, entry);
 761                 add_mm_counter(mm, MM_ANONPAGES, HPAGE_PMD_NR);
 762                 atomic_long_inc(&mm->nr_ptes);
 763                 spin_unlock(ptl);
 764         }
 765
 766         return 0;
 767 }
 768
 769 static inline gfp_t alloc_hugepage_gfpmask(int defrag, gfp_t extra_gfp)
 770 {
 771         return (GFP_TRANSHUGE & ~(defrag ? 0 : __GFP_WAIT)) | extra_gfp;
 772 }
 773
 774 /* Caller must hold page table lock. */
 775 static bool set_huge_zero_page(pgtable_t pgtable, struct mm_struct *mm,
 776                 struct vm_area_struct *vma, unsigned long haddr, pmd_t *pmd,
 777                 struct page *zero_page)
 778 {
 779         pmd_t entry;
 780         if (!pmd_none(*pmd))
 781                 return false;
 782         entry = mk_pmd(zero_page, vma->vm_page_prot);
 783         entry = pmd_mkhuge(entry);
 784         pgtable_trans_huge_deposit(mm, pmd, pgtable);
 785         set_pmd_at(mm, haddr, pmd, entry);
 786         atomic_long_inc(&mm->nr_ptes);
 787         return true;
 788 }
 789
 790 int do_huge_pmd_anonymous_page(struct mm_struct *mm, struct vm_area_struct *vma,
 791                                unsigned long address, pmd_t *pmd,
 792                                unsigned int flags)
 793 {
 794         gfp_t gfp;
 795         struct page *page;
 796         unsigned long haddr = address & HPAGE_PMD_MASK;
 797
 798         if (haddr < vma->vm_start || haddr + HPAGE_PMD_SIZE > vma->vm_end)
 799                 return VM_FAULT_FALLBACK;
 800         if (unlikely(anon_vma_prepare(vma)))
 801                 return VM_FAULT_OOM;
 802         if (unlikely(khugepaged_enter(vma, vma->vm_flags)))
 803                 return VM_FAULT_OOM;
 804         if (!(flags & FAULT_FLAG_WRITE) && !mm_forbids_zeropage(mm) &&
 805                         transparent_hugepage_use_zero_page()) {
 806                 spinlock_t *ptl;
 807                 pgtable_t pgtable;
 808                 struct page *zero_page;
 809                 bool set;
 810                 pgtable = pte_alloc_one(mm, haddr);
 811                 if (unlikely(!pgtable))
 812                         return VM_FAULT_OOM;
 813                 zero_page = get_huge_zero_page();
 814                 if (unlikely(!zero_page)) {
 815                         pte_free(mm, pgtable);
 816                         count_vm_event(THP_FAULT_FALLBACK);
 817                         return VM_FAULT_FALLBACK;
 818                 }
 819                 ptl = pmd_lock(mm, pmd);
 820                 set = set_huge_zero_page(pgtable, mm, vma, haddr, pmd,
 821                                 zero_page);
 822                 spin_unlock(ptl);
 823                 if (!set) {
 824                         pte_free(mm, pgtable);
 825                         put_huge_zero_page();
 826                 }
 827                 return 0;
 828         }
 829         gfp = alloc_hugepage_gfpmask(transparent_hugepage_defrag(vma), 0);
 830         page = alloc_hugepage_vma(gfp, vma, haddr, HPAGE_PMD_ORDER);
 831         if (unlikely(!page)) {
 832                 count_vm_event(THP_FAULT_FALLBACK);
 833                 return VM_FAULT_FALLBACK;
 834         }
 835         if (unlikely(__do_huge_pmd_anonymous_page(mm, vma, haddr, pmd, page, gfp))) {
 836                 put_page(page);
 837                 count_vm_event(THP_FAULT_FALLBACK);
 838                 return VM_FAULT_FALLBACK;
 839         }
 840
 841         count_vm_event(THP_FAULT_ALLOC);
 842         return 0;
 843 }
 844
 845 int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm,
 846                   pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr,
 847                   struct vm_area_struct *vma)
 848 {
 849         spinlock_t *dst_ptl, *src_ptl;
 850         struct page *src_page;
 851         pmd_t pmd;
 852         pgtable_t pgtable;
 853         int ret;
 854
 855         ret = -ENOMEM;
 856         pgtable = pte_alloc_one(dst_mm, addr);
 857         if (unlikely(!pgtable))
 858                 goto out;
 859
 860         dst_ptl = pmd_lock(dst_mm, dst_pmd);
 861         src_ptl = pmd_lockptr(src_mm, src_pmd);
 862         spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
 863
 864         ret = -EAGAIN;
 865         pmd = *src_pmd;
 866         if (unlikely(!pmd_trans_huge(pmd))) {
 867                 pte_free(dst_mm, pgtable);
 868                 goto out_unlock;
 869         }
 870         /*
 871          * When page table lock is held, the huge zero pmd should not be
 872          * under splitting since we don't split the page itself, only pmd to
 873          * a page table.
 874          */
 875         if (is_huge_zero_pmd(pmd)) {
 876                 struct page *zero_page;
 877                 bool set;
 878                 /*
 879                  * get_huge_zero_page() will never allocate a new page here,
 880                  * since we already have a zero page to copy. It just takes a
 881                  * reference.
 882                  */
 883                 zero_page = get_huge_zero_page();
 884                 set = set_huge_zero_page(pgtable, dst_mm, vma, addr, dst_pmd,
 885                                 zero_page);
 886                 BUG_ON(!set); /* unexpected !pmd_none(dst_pmd) */
 887                 ret = 0;
 888                 goto out_unlock;
 889         }
 890
 891         if (unlikely(pmd_trans_splitting(pmd))) {
 892                 /* split huge page running from under us */
 893                 spin_unlock(src_ptl);
 894                 spin_unlock(dst_ptl);
 895                 pte_free(dst_mm, pgtable);
 896
 897                 wait_split_huge_page(vma->anon_vma, src_pmd); /* src_vma */
 898                 goto out;
 899         }
 900         src_page = pmd_page(pmd);
 901         VM_BUG_ON_PAGE(!PageHead(src_page), src_page);
 902         get_page(src_page);
 903         page_dup_rmap(src_page);
 904         add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR);
 905
 906         pmdp_set_wrprotect(src_mm, addr, src_pmd);
 907         pmd = pmd_mkold(pmd_wrprotect(pmd));
 908         pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable);
 909         set_pmd_at(dst_mm, addr, dst_pmd, pmd);
 910         atomic_long_inc(&dst_mm->nr_ptes);
 911
 912         ret = 0;
 913 out_unlock:
 914         spin_unlock(src_ptl);
 915         spin_unlock(dst_ptl);
 916 out:
 917         return ret;
 918 }
 919
 920 void huge_pmd_set_accessed(struct mm_struct *mm,
 921                            struct vm_area_struct *vma,
 922                            unsigned long address,
 923                            pmd_t *pmd, pmd_t orig_pmd,
 924                            int dirty)
 925 {
 926         spinlock_t *ptl;
 927         pmd_t entry;
 928         unsigned long haddr;
 929
 930         ptl = pmd_lock(mm, pmd);
 931         if (unlikely(!pmd_same(*pmd, orig_pmd)))
 932                 goto unlock;
 933
 934         entry = pmd_mkyoung(orig_pmd);
 935         haddr = address & HPAGE_PMD_MASK;
 936         if (pmdp_set_access_flags(vma, haddr, pmd, entry, dirty))
 937                 update_mmu_cache_pmd(vma, address, pmd);
 938
 939 unlock:
 940         spin_unlock(ptl);
 941 }
 942
 943 /*
 944  * Save CONFIG_DEBUG_PAGEALLOC from faulting falsely on tail pages
 945  * during copy_user_huge_page()'s copy_page_rep(): in the case when
 946  * the source page gets split and a tail freed before copy completes.
 947  * Called under pmd_lock of checked pmd, so safe from splitting itself.
 948  */
 949 static void get_user_huge_page(struct page *page)
 950 {
 951         if (IS_ENABLED(CONFIG_DEBUG_PAGEALLOC)) {
 952                 struct page *endpage = page + HPAGE_PMD_NR;
 953
 954                 atomic_add(HPAGE_PMD_NR, &page->_count);
 955                 while (++page < endpage)
 956                         get_huge_page_tail(page);
 957         } else {
 958                 get_page(page);
 959         }
 960 }
 961
 962 static void put_user_huge_page(struct page *page)
 963 {
 964         if (IS_ENABLED(CONFIG_DEBUG_PAGEALLOC)) {
 965                 struct page *endpage = page + HPAGE_PMD_NR;
 966
 967                 while (page < endpage)
 968                         put_page(page++);
 969         } else {
 970                 put_page(page);
 971         }
 972 }
 973
 974 static int do_huge_pmd_wp_page_fallback(struct mm_struct *mm,
 975                                         struct vm_area_struct *vma,
 976                                         unsigned long address,
 977                                         pmd_t *pmd, pmd_t orig_pmd,
 978                                         struct page *page,
 979                                         unsigned long haddr)
 980 {
 981         struct mem_cgroup *memcg;
 982         spinlock_t *ptl;
 983         pgtable_t pgtable;
 984         pmd_t _pmd;
 985         int ret = 0, i;
 986         struct page **pages;
 987         unsigned long mmun_start;       /* For mmu_notifiers */
 988         unsigned long mmun_end;         /* For mmu_notifiers */
 989
 990         pages = kmalloc(sizeof(struct page *) * HPAGE_PMD_NR,
 991                         GFP_KERNEL);
 992         if (unlikely(!pages)) {
 993                 ret |= VM_FAULT_OOM;
 994                 goto out;
 995         }
 996
 997         for (i = 0; i < HPAGE_PMD_NR; i++) {
 998                 pages[i] = alloc_page_vma_node(GFP_HIGHUSER_MOVABLE |
 999                                                __GFP_OTHER_NODE,
1000                                                vma, address, page_to_nid(page));
1001                 if (unlikely(!pages[i] ||
1002                              mem_cgroup_try_charge(pages[i], mm, GFP_KERNEL,
1003                                                    &memcg))) {
1004                         if (pages[i])
1005                                 put_page(pages[i]);
1006                         while (--i >= 0) {
1007                                 memcg = (void *)page_private(pages[i]);
1008                                 set_page_private(pages[i], 0);
1009                                 mem_cgroup_cancel_charge(pages[i], memcg);
1010                                 put_page(pages[i]);
1011                         }
1012                         kfree(pages);
1013                         ret |= VM_FAULT_OOM;
1014                         goto out;
1015                 }
1016                 set_page_private(pages[i], (unsigned long)memcg);
1017         }
1018
1019         for (i = 0; i < HPAGE_PMD_NR; i++) {
1020                 copy_user_highpage(pages[i], page + i,
1021                                    haddr + PAGE_SIZE * i, vma);
1022                 __SetPageUptodate(pages[i]);
1023                 cond_resched();
1024         }
1025
1026         mmun_start = haddr;
1027         mmun_end   = haddr + HPAGE_PMD_SIZE;
1028         mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
1029
1030         ptl = pmd_lock(mm, pmd);
1031         if (unlikely(!pmd_same(*pmd, orig_pmd)))
1032                 goto out_free_pages;
1033         VM_BUG_ON_PAGE(!PageHead(page), page);
1034
1035         pmdp_clear_flush_notify(vma, haddr, pmd);
1036         /* leave pmd empty until pte is filled */
1037
1038         pgtable = pgtable_trans_huge_withdraw(mm, pmd);
1039         pmd_populate(mm, &_pmd, pgtable);
1040
1041         for (i = 0; i < HPAGE_PMD_NR; i++, haddr += PAGE_SIZE) {
1042                 pte_t *pte, entry;
1043                 entry = mk_pte(pages[i], vma->vm_page_prot);
1044                 entry = maybe_mkwrite(pte_mkdirty(entry), vma);
1045                 memcg = (void *)page_private(pages[i]);
1046                 set_page_private(pages[i], 0);
1047                 page_add_new_anon_rmap(pages[i], vma, haddr);
1048                 mem_cgroup_commit_charge(pages[i], memcg, false);
1049                 lru_cache_add_active_or_unevictable(pages[i], vma);
1050                 pte = pte_offset_map(&_pmd, haddr);
1051                 VM_BUG_ON(!pte_none(*pte));
1052                 set_pte_at(mm, haddr, pte, entry);
1053                 pte_unmap(pte);
1054         }
1055         kfree(pages);
1056
1057         smp_wmb(); /* make pte visible before pmd */
1058         pmd_populate(mm, pmd, pgtable);
1059         page_remove_rmap(page);
1060         spin_unlock(ptl);
1061
1062         mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
1063
1064         ret |= VM_FAULT_WRITE;
1065         put_page(page);
1066
1067 out:
1068         return ret;
1069
1070 out_free_pages:
1071         spin_unlock(ptl);
1072         mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
1073         for (i = 0; i < HPAGE_PMD_NR; i++) {
1074                 memcg = (void *)page_private(pages[i]);
1075                 set_page_private(pages[i], 0);
1076                 mem_cgroup_cancel_charge(pages[i], memcg);
1077                 put_page(pages[i]);
1078         }
1079         kfree(pages);
1080         goto out;
1081 }
1082
1083 int do_huge_pmd_wp_page(struct mm_struct *mm, struct vm_area_struct *vma,
1084                         unsigned long address, pmd_t *pmd, pmd_t orig_pmd)
1085 {
1086         spinlock_t *ptl;
1087         int ret = 0;
1088         struct page *page = NULL, *new_page;
1089         struct mem_cgroup *memcg;
1090         unsigned long haddr;
1091         unsigned long mmun_start;       /* For mmu_notifiers */
1092         unsigned long mmun_end;         /* For mmu_notifiers */
1093         gfp_t huge_gfp;                 /* for allocation and charge */
1094
1095         ptl = pmd_lockptr(mm, pmd);
1096         VM_BUG_ON_VMA(!vma->anon_vma, vma);
1097         haddr = address & HPAGE_PMD_MASK;
1098         if (is_huge_zero_pmd(orig_pmd))
1099                 goto alloc;
1100         spin_lock(ptl);
1101         if (unlikely(!pmd_same(*pmd, orig_pmd)))
1102                 goto out_unlock;
1103
1104         page = pmd_page(orig_pmd);
1105         VM_BUG_ON_PAGE(!PageCompound(page) || !PageHead(page), page);
1106         if (page_mapcount(page) == 1) {
1107                 pmd_t entry;
1108                 entry = pmd_mkyoung(orig_pmd);
1109                 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
1110                 if (pmdp_set_access_flags(vma, haddr, pmd, entry,  1))
1111                         update_mmu_cache_pmd(vma, address, pmd);
1112                 ret |= VM_FAULT_WRITE;
1113                 goto out_unlock;
1114         }
1115         get_user_huge_page(page);
1116         spin_unlock(ptl);
1117 alloc:
1118         if (transparent_hugepage_enabled(vma) &&
1119             !transparent_hugepage_debug_cow()) {
1120                 huge_gfp = alloc_hugepage_gfpmask(transparent_hugepage_defrag(vma), 0);
1121                 new_page = alloc_hugepage_vma(huge_gfp, vma, haddr, HPAGE_PMD_ORDER);
1122         } else
1123                 new_page = NULL;
1124
1125         if (unlikely(!new_page)) {
1126                 if (!page) {
1127                         split_huge_page_pmd(vma, address, pmd);
1128                         ret |= VM_FAULT_FALLBACK;
1129                 } else {
1130                         ret = do_huge_pmd_wp_page_fallback(mm, vma, address,
1131                                         pmd, orig_pmd, page, haddr);
1132                         if (ret & VM_FAULT_OOM) {
1133                                 split_huge_page(page);
1134                                 ret |= VM_FAULT_FALLBACK;
1135                         }
1136                         put_user_huge_page(page);
1137                 }
1138                 count_vm_event(THP_FAULT_FALLBACK);
1139                 goto out;
1140         }
1141
1142         if (unlikely(mem_cgroup_try_charge(new_page, mm, huge_gfp, &memcg))) {
1143                 put_page(new_page);
1144                 if (page) {
1145                         split_huge_page(page);
1146                         put_user_huge_page(page);
1147                 } else
1148                         split_huge_page_pmd(vma, address, pmd);
1149                 ret |= VM_FAULT_FALLBACK;
1150                 count_vm_event(THP_FAULT_FALLBACK);
1151                 goto out;
1152         }
1153
1154         count_vm_event(THP_FAULT_ALLOC);
1155
1156         if (!page)
1157                 clear_huge_page(new_page, haddr, HPAGE_PMD_NR);
1158         else
1159                 copy_user_huge_page(new_page, page, haddr, vma, HPAGE_PMD_NR);
1160         __SetPageUptodate(new_page);
1161
1162         mmun_start = haddr;
1163         mmun_end   = haddr + HPAGE_PMD_SIZE;
1164         mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
1165
1166         spin_lock(ptl);
1167         if (page)
1168                 put_user_huge_page(page);
1169         if (unlikely(!pmd_same(*pmd, orig_pmd))) {
1170                 spin_unlock(ptl);
1171                 mem_cgroup_cancel_charge(new_page, memcg);
1172                 put_page(new_page);
1173                 goto out_mn;
1174         } else {
1175                 pmd_t entry;
1176                 entry = mk_huge_pmd(new_page, vma->vm_page_prot);
1177                 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
1178                 pmdp_clear_flush_notify(vma, haddr, pmd);
1179                 page_add_new_anon_rmap(new_page, vma, haddr);
1180                 mem_cgroup_commit_charge(new_page, memcg, false);
1181                 lru_cache_add_active_or_unevictable(new_page, vma);
1182                 set_pmd_at(mm, haddr, pmd, entry);
1183                 update_mmu_cache_pmd(vma, address, pmd);
1184                 if (!page) {
1185                         add_mm_counter(mm, MM_ANONPAGES, HPAGE_PMD_NR);
1186                         put_huge_zero_page();
1187                 } else {
1188                         VM_BUG_ON_PAGE(!PageHead(page), page);
1189                         page_remove_rmap(page);
1190                         put_page(page);
1191                 }
1192                 ret |= VM_FAULT_WRITE;
1193         }
1194         spin_unlock(ptl);
1195 out_mn:
1196         mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
1197 out:
1198         return ret;
1199 out_unlock:
1200         spin_unlock(ptl);
1201         return ret;
1202 }
1203
1204 struct page *follow_trans_huge_pmd(struct vm_area_struct *vma,
1205                                    unsigned long addr,
1206                                    pmd_t *pmd,
1207                                    unsigned int flags)
1208 {
1209         struct mm_struct *mm = vma->vm_mm;
1210         struct page *page = NULL;
1211
1212         assert_spin_locked(pmd_lockptr(mm, pmd));
1213
1214         if (flags & FOLL_WRITE && !pmd_write(*pmd))
1215                 goto out;
1216
1217         /* Avoid dumping huge zero page */
1218         if ((flags & FOLL_DUMP) && is_huge_zero_pmd(*pmd))
1219                 return ERR_PTR(-EFAULT);
1220
1221         /* Full NUMA hinting faults to serialise migration in fault paths */
1222         if ((flags & FOLL_NUMA) && pmd_protnone(*pmd))
1223                 goto out;
1224
1225         page = pmd_page(*pmd);
1226         VM_BUG_ON_PAGE(!PageHead(page), page);
1227         if (flags & FOLL_TOUCH) {
1228                 pmd_t _pmd;
1229                 /*
1230                  * We should set the dirty bit only for FOLL_WRITE but
1231                  * for now the dirty bit in the pmd is meaningless.
1232                  * And if the dirty bit will become meaningful and
1233                  * we'll only set it with FOLL_WRITE, an atomic
1234                  * set_bit will be required on the pmd to set the
1235                  * young bit, instead of the current set_pmd_at.
1236                  */
1237                 _pmd = pmd_mkyoung(pmd_mkdirty(*pmd));
1238                 if (pmdp_set_access_flags(vma, addr & HPAGE_PMD_MASK,
1239                                           pmd, _pmd,  1))
1240                         update_mmu_cache_pmd(vma, addr, pmd);
1241         }
1242         if ((flags & FOLL_POPULATE) && (vma->vm_flags & VM_LOCKED)) {
1243                 if (page->mapping && trylock_page(page)) {
1244                         lru_add_drain();
1245                         if (page->mapping)
1246                                 mlock_vma_page(page);
1247                         unlock_page(page);
1248                 }
1249         }
1250         page += (addr & ~HPAGE_PMD_MASK) >> PAGE_SHIFT;
1251         VM_BUG_ON_PAGE(!PageCompound(page), page);
1252         if (flags & FOLL_GET)
1253                 get_page_foll(page);
1254
1255 out:
1256         return page;
1257 }
1258
1259 /* NUMA hinting page fault entry point for trans huge pmds */
1260 int do_huge_pmd_numa_page(struct mm_struct *mm, struct vm_area_struct *vma,
1261                                 unsigned long addr, pmd_t pmd, pmd_t *pmdp)
1262 {
1263         spinlock_t *ptl;
1264         struct anon_vma *anon_vma = NULL;
1265         struct page *page;
1266         unsigned long haddr = addr & HPAGE_PMD_MASK;
1267         int page_nid = -1, this_nid = numa_node_id();
1268         int target_nid, last_cpupid = -1;
1269         bool page_locked;
1270         bool migrated = false;
1271         bool was_writable;
1272         int flags = 0;
1273
1274         /* A PROT_NONE fault should not end up here */
1275         BUG_ON(!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE)));
1276
1277         ptl = pmd_lock(mm, pmdp);
1278         if (unlikely(!pmd_same(pmd, *pmdp)))
1279                 goto out_unlock;
1280
1281         /*
1282          * If there are potential migrations, wait for completion and retry
1283          * without disrupting NUMA hinting information. Do not relock and
1284          * check_same as the page may no longer be mapped.
1285          */
1286         if (unlikely(pmd_trans_migrating(*pmdp))) {
1287                 page = pmd_page(*pmdp);
1288                 spin_unlock(ptl);
1289                 wait_on_page_locked(page);
1290                 goto out;
1291         }
1292
1293         page = pmd_page(pmd);
1294         BUG_ON(is_huge_zero_page(page));
1295         page_nid = page_to_nid(page);
1296         last_cpupid = page_cpupid_last(page);
1297         count_vm_numa_event(NUMA_HINT_FAULTS);
1298         if (page_nid == this_nid) {
1299                 count_vm_numa_event(NUMA_HINT_FAULTS_LOCAL);
1300                 flags |= TNF_FAULT_LOCAL;
1301         }
1302
1303         /* See similar comment in do_numa_page for explanation */
1304         if (!(vma->vm_flags & VM_WRITE))
1305                 flags |= TNF_NO_GROUP;
1306
1307         /*
1308          * Acquire the page lock to serialise THP migrations but avoid dropping
1309          * page_table_lock if at all possible
1310          */
1311         page_locked = trylock_page(page);
1312         target_nid = mpol_misplaced(page, vma, haddr);
1313         if (target_nid == -1) {
1314                 /* If the page was locked, there are no parallel migrations */
1315                 if (page_locked)
1316                         goto clear_pmdnuma;
1317         }
1318
1319         /* Migration could have started since the pmd_trans_migrating check */
1320         if (!page_locked) {
1321                 spin_unlock(ptl);
1322                 wait_on_page_locked(page);
1323                 page_nid = -1;
1324                 goto out;
1325         }
1326
1327         /*
1328          * Page is misplaced. Page lock serialises migrations. Acquire anon_vma
1329          * to serialises splits
1330          */
1331         get_page(page);
1332         spin_unlock(ptl);
1333         anon_vma = page_lock_anon_vma_read(page);
1334
1335         /* Confirm the PMD did not change while page_table_lock was released */
1336         spin_lock(ptl);
1337         if (unlikely(!pmd_same(pmd, *pmdp))) {
1338                 unlock_page(page);
1339                 put_page(page);
1340                 page_nid = -1;
1341                 goto out_unlock;
1342         }
1343
1344         /* Bail if we fail to protect against THP splits for any reason */
1345         if (unlikely(!anon_vma)) {
1346                 put_page(page);
1347                 page_nid = -1;
1348                 goto clear_pmdnuma;
1349         }
1350
1351         /*
1352          * Migrate the THP to the requested node, returns with page unlocked
1353          * and access rights restored.
1354          */
1355         spin_unlock(ptl);
1356         migrated = migrate_misplaced_transhuge_page(mm, vma,
1357                                 pmdp, pmd, addr, page, target_nid);
1358         if (migrated) {
1359                 flags |= TNF_MIGRATED;
1360                 page_nid = target_nid;
1361         } else
1362                 flags |= TNF_MIGRATE_FAIL;
1363
1364         goto out;
1365 clear_pmdnuma:
1366         BUG_ON(!PageLocked(page));
1367         was_writable = pmd_write(pmd);
1368         pmd = pmd_modify(pmd, vma->vm_page_prot);
1369         pmd = pmd_mkyoung(pmd);
1370         if (was_writable)
1371                 pmd = pmd_mkwrite(pmd);
1372         set_pmd_at(mm, haddr, pmdp, pmd);
1373         update_mmu_cache_pmd(vma, addr, pmdp);
1374         unlock_page(page);
1375 out_unlock:
1376         spin_unlock(ptl);
1377
1378 out:
1379         if (anon_vma)
1380                 page_unlock_anon_vma_read(anon_vma);
1381
1382         if (page_nid != -1)
1383                 task_numa_fault(last_cpupid, page_nid, HPAGE_PMD_NR, flags);
1384
1385         return 0;
1386 }
1387
1388 int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
1389                  pmd_t *pmd, unsigned long addr)
1390 {
1391         spinlock_t *ptl;
1392         int ret = 0;
1393
1394         if (__pmd_trans_huge_lock(pmd, vma, &ptl) == 1) {
1395                 struct page *page;
1396                 pgtable_t pgtable;
1397                 pmd_t orig_pmd;
1398                 /*
1399                  * For architectures like ppc64 we look at deposited pgtable
1400                  * when calling pmdp_get_and_clear. So do the
1401                  * pgtable_trans_huge_withdraw after finishing pmdp related
1402                  * operations.
1403                  */
1404                 orig_pmd = pmdp_get_and_clear_full(tlb->mm, addr, pmd,
1405                                                    tlb->fullmm);
1406                 tlb_remove_pmd_tlb_entry(tlb, pmd, addr);
1407                 pgtable = pgtable_trans_huge_withdraw(tlb->mm, pmd);
1408                 if (is_huge_zero_pmd(orig_pmd)) {
1409                         atomic_long_dec(&tlb->mm->nr_ptes);
1410                         spin_unlock(ptl);
1411                         put_huge_zero_page();
1412                 } else {
1413                         page = pmd_page(orig_pmd);
1414                         page_remove_rmap(page);
1415                         VM_BUG_ON_PAGE(page_mapcount(page) < 0, page);
1416                         add_mm_counter(tlb->mm, MM_ANONPAGES, -HPAGE_PMD_NR);
1417                         VM_BUG_ON_PAGE(!PageHead(page), page);
1418                         atomic_long_dec(&tlb->mm->nr_ptes);
1419                         spin_unlock(ptl);
1420                         tlb_remove_page(tlb, page);
1421                 }
1422                 pte_free(tlb->mm, pgtable);
1423                 ret = 1;
1424         }
1425         return ret;
1426 }
1427
1428 int move_huge_pmd(struct vm_area_struct *vma, struct vm_area_struct *new_vma,
1429                   unsigned long old_addr,
1430                   unsigned long new_addr, unsigned long old_end,
1431                   pmd_t *old_pmd, pmd_t *new_pmd)
1432 {
1433         spinlock_t *old_ptl, *new_ptl;
1434         int ret = 0;
1435         pmd_t pmd;
1436
1437         struct mm_struct *mm = vma->vm_mm;
1438
1439         if ((old_addr & ~HPAGE_PMD_MASK) ||
1440             (new_addr & ~HPAGE_PMD_MASK) ||
1441             old_end - old_addr < HPAGE_PMD_SIZE ||
1442             (new_vma->vm_flags & VM_NOHUGEPAGE))
1443                 goto out;
1444
1445         /*
1446          * The destination pmd shouldn't be established, free_pgtables()
1447          * should have release it.
1448          */
1449         if (WARN_ON(!pmd_none(*new_pmd))) {
1450                 VM_BUG_ON(pmd_trans_huge(*new_pmd));
1451                 goto out;
1452         }
1453
1454         /*
1455          * We don't have to worry about the ordering of src and dst
1456          * ptlocks because exclusive mmap_sem prevents deadlock.
1457          */
1458         ret = __pmd_trans_huge_lock(old_pmd, vma, &old_ptl);
1459         if (ret == 1) {
1460                 new_ptl = pmd_lockptr(mm, new_pmd);
1461                 if (new_ptl != old_ptl)
1462                         spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING);
1463                 pmd = pmdp_get_and_clear(mm, old_addr, old_pmd);
1464                 VM_BUG_ON(!pmd_none(*new_pmd));
1465
1466                 if (pmd_move_must_withdraw(new_ptl, old_ptl)) {
1467                         pgtable_t pgtable;
1468                         pgtable = pgtable_trans_huge_withdraw(mm, old_pmd);
1469                         pgtable_trans_huge_deposit(mm, new_pmd, pgtable);
1470                 }
1471                 set_pmd_at(mm, new_addr, new_pmd, pmd_mksoft_dirty(pmd));
1472                 if (new_ptl != old_ptl)
1473                         spin_unlock(new_ptl);
1474                 spin_unlock(old_ptl);
1475         }
1476 out:
1477         return ret;
1478 }
1479
1480 /*
1481  * Returns
1482  *  - 0 if PMD could not be locked
1483  *  - 1 if PMD was locked but protections unchange and TLB flush unnecessary
1484  *  - HPAGE_PMD_NR is protections changed and TLB flush necessary
1485  */
1486 int change_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
1487                 unsigned long addr, pgprot_t newprot, int prot_numa)
1488 {
1489         struct mm_struct *mm = vma->vm_mm;
1490         spinlock_t *ptl;
1491         int ret = 0;
1492
1493         if (__pmd_trans_huge_lock(pmd, vma, &ptl) == 1) {
1494                 pmd_t entry;
1495                 bool preserve_write = prot_numa && pmd_write(*pmd);
1496                 ret = 1;
1497
1498                 /*
1499                  * Avoid trapping faults against the zero page. The read-only
1500                  * data is likely to be read-cached on the local CPU and
1501                  * local/remote hits to the zero page are not interesting.
1502                  */
1503                 if (prot_numa && is_huge_zero_pmd(*pmd)) {
1504                         spin_unlock(ptl);
1505                         return ret;
1506                 }
1507
1508                 if (!prot_numa || !pmd_protnone(*pmd)) {
1509                         entry = pmdp_get_and_clear_notify(mm, addr, pmd);
1510                         entry = pmd_modify(entry, newprot);
1511                         if (preserve_write)
1512                                 entry = pmd_mkwrite(entry);
1513                         ret = HPAGE_PMD_NR;
1514                         set_pmd_at(mm, addr, pmd, entry);
1515                         BUG_ON(!preserve_write && pmd_write(entry));
1516                 }
1517                 spin_unlock(ptl);
1518         }
1519
1520         return ret;
1521 }
1522
1523 /*
1524  * Returns 1 if a given pmd maps a stable (not under splitting) thp.
1525  * Returns -1 if it maps a thp under splitting. Returns 0 otherwise.
1526  *
1527  * Note that if it returns 1, this routine returns without unlocking page
1528  * table locks. So callers must unlock them.
1529  */
1530 int __pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma,
1531                 spinlock_t **ptl)
1532 {
1533         *ptl = pmd_lock(vma->vm_mm, pmd);
1534         if (likely(pmd_trans_huge(*pmd))) {
1535                 if (unlikely(pmd_trans_splitting(*pmd))) {
1536                         spin_unlock(*ptl);
1537                         wait_split_huge_page(vma->anon_vma, pmd);
1538                         return -1;
1539                 } else {
1540                         /* Thp mapped by 'pmd' is stable, so we can
1541                          * handle it as it is. */
1542                         return 1;
1543                 }
1544         }
1545         spin_unlock(*ptl);
1546         return 0;
1547 }
1548
1549 /*
1550  * This function returns whether a given @page is mapped onto the @address
1551  * in the virtual space of @mm.
1552  *
1553  * When it's true, this function returns *pmd with holding the page table lock
1554  * and passing it back to the caller via @ptl.
1555  * If it's false, returns NULL without holding the page table lock.
1556  */
1557 pmd_t *page_check_address_pmd(struct page *page,
1558                               struct mm_struct *mm,
1559                               unsigned long address,
1560                               enum page_check_address_pmd_flag flag,
1561                               spinlock_t **ptl)
1562 {
1563         pgd_t *pgd;
1564         pud_t *pud;
1565         pmd_t *pmd;
1566
1567         if (address & ~HPAGE_PMD_MASK)
1568                 return NULL;
1569
1570         pgd = pgd_offset(mm, address);
1571         if (!pgd_present(*pgd))
1572                 return NULL;
1573         pud = pud_offset(pgd, address);
1574         if (!pud_present(*pud))
1575                 return NULL;
1576         pmd = pmd_offset(pud, address);
1577
1578         *ptl = pmd_lock(mm, pmd);
1579         if (!pmd_present(*pmd))
1580                 goto unlock;
1581         if (pmd_page(*pmd) != page)
1582                 goto unlock;
1583         /*
1584          * split_vma() may create temporary aliased mappings. There is
1585          * no risk as long as all huge pmd are found and have their
1586          * splitting bit set before __split_huge_page_refcount
1587          * runs. Finding the same huge pmd more than once during the
1588          * same rmap walk is not a problem.
1589          */
1590         if (flag == PAGE_CHECK_ADDRESS_PMD_NOTSPLITTING_FLAG &&
1591             pmd_trans_splitting(*pmd))
1592                 goto unlock;
1593         if (pmd_trans_huge(*pmd)) {
1594                 VM_BUG_ON(flag == PAGE_CHECK_ADDRESS_PMD_SPLITTING_FLAG &&
1595                           !pmd_trans_splitting(*pmd));
1596                 return pmd;
1597         }
1598 unlock:
1599         spin_unlock(*ptl);
1600         return NULL;
1601 }
1602
1603 static int __split_huge_page_splitting(struct page *page,
1604                                        struct vm_area_struct *vma,
1605                                        unsigned long address)
1606 {
1607         struct mm_struct *mm = vma->vm_mm;
1608         spinlock_t *ptl;
1609         pmd_t *pmd;
1610         int ret = 0;
1611         /* For mmu_notifiers */
1612         const unsigned long mmun_start = address;
1613         const unsigned long mmun_end   = address + HPAGE_PMD_SIZE;
1614
1615         mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
1616         pmd = page_check_address_pmd(page, mm, address,
1617                         PAGE_CHECK_ADDRESS_PMD_NOTSPLITTING_FLAG, &ptl);
1618         if (pmd) {
1619                 /*
1620                  * We can't temporarily set the pmd to null in order
1621                  * to split it, the pmd must remain marked huge at all
1622                  * times or the VM won't take the pmd_trans_huge paths
1623                  * and it won't wait on the anon_vma->root->rwsem to
1624                  * serialize against split_huge_page*.
1625                  */
1626                 pmdp_splitting_flush(vma, address, pmd);
1627
1628                 ret = 1;
1629                 spin_unlock(ptl);
1630         }
1631         mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
1632
1633         return ret;
1634 }
1635
1636 static void __split_huge_page_refcount(struct page *page,
1637                                        struct list_head *list)
1638 {
1639         int i;
1640         struct zone *zone = page_zone(page);
1641         struct lruvec *lruvec;
1642         int tail_count = 0;
1643
1644         /* prevent PageLRU to go away from under us, and freeze lru stats */
1645         spin_lock_irq(&zone->lru_lock);
1646         lruvec = mem_cgroup_page_lruvec(page, zone);
1647
1648         compound_lock(page);
1649         /* complete memcg works before add pages to LRU */
1650         mem_cgroup_split_huge_fixup(page);
1651
1652         for (i = HPAGE_PMD_NR - 1; i >= 1; i--) {
1653                 struct page *page_tail = page + i;
1654
1655                 /* tail_page->_mapcount cannot change */
1656                 BUG_ON(page_mapcount(page_tail) < 0);
1657                 tail_count += page_mapcount(page_tail);
1658                 /* check for overflow */
1659                 BUG_ON(tail_count < 0);
1660                 BUG_ON(atomic_read(&page_tail->_count) != 0);
1661                 /*
1662                  * tail_page->_count is zero and not changing from
1663                  * under us. But get_page_unless_zero() may be running
1664                  * from under us on the tail_page. If we used
1665                  * atomic_set() below instead of atomic_add(), we
1666                  * would then run atomic_set() concurrently with
1667                  * get_page_unless_zero(), and atomic_set() is
1668                  * implemented in C not using locked ops. spin_unlock
1669                  * on x86 sometime uses locked ops because of PPro
1670                  * errata 66, 92, so unless somebody can guarantee
1671                  * atomic_set() here would be safe on all archs (and
1672                  * not only on x86), it's safer to use atomic_add().
1673                  */
1674                 atomic_add(page_mapcount(page) + page_mapcount(page_tail) + 1,
1675                            &page_tail->_count);
1676
1677                 /* after clearing PageTail the gup refcount can be released */
1678                 smp_mb__after_atomic();
1679
1680                 /*
1681                  * retain hwpoison flag of the poisoned tail page:
1682                  *   fix for the unsuitable process killed on Guest Machine(KVM)
1683                  *   by the memory-failure.
1684                  */
1685                 page_tail->flags &= ~PAGE_FLAGS_CHECK_AT_PREP | __PG_HWPOISON;
1686                 page_tail->flags |= (page->flags &
1687                                      ((1L << PG_referenced) |
1688                                       (1L << PG_swapbacked) |
1689                                       (1L << PG_mlocked) |
1690                                       (1L << PG_uptodate) |
1691                                       (1L << PG_active) |
1692                                       (1L << PG_unevictable)));
1693                 page_tail->flags |= (1L << PG_dirty);
1694
1695                 /* clear PageTail before overwriting first_page */
1696                 smp_wmb();
1697
1698                 /*
1699                  * __split_huge_page_splitting() already set the
1700                  * splitting bit in all pmd that could map this
1701                  * hugepage, that will ensure no CPU can alter the
1702                  * mapcount on the head page. The mapcount is only
1703                  * accounted in the head page and it has to be
1704                  * transferred to all tail pages in the below code. So
1705                  * for this code to be safe, the split the mapcount
1706                  * can't change. But that doesn't mean userland can't
1707                  * keep changing and reading the page contents while
1708                  * we transfer the mapcount, so the pmd splitting
1709                  * status is achieved setting a reserved bit in the
1710                  * pmd, not by clearing the present bit.
1711                 */
1712                 page_tail->_mapcount = page->_mapcount;
1713
1714                 BUG_ON(page_tail->mapping);
1715                 page_tail->mapping = page->mapping;
1716
1717                 page_tail->index = page->index + i;
1718                 page_cpupid_xchg_last(page_tail, page_cpupid_last(page));
1719
1720                 BUG_ON(!PageAnon(page_tail));
1721                 BUG_ON(!PageUptodate(page_tail));
1722                 BUG_ON(!PageDirty(page_tail));
1723                 BUG_ON(!PageSwapBacked(page_tail));
1724
1725                 lru_add_page_tail(page, page_tail, lruvec, list);
1726         }
1727         atomic_sub(tail_count, &page->_count);
1728         BUG_ON(atomic_read(&page->_count) <= 0);
1729
1730         __mod_zone_page_state(zone, NR_ANON_TRANSPARENT_HUGEPAGES, -1);
1731
1732         ClearPageCompound(page);
1733         compound_unlock(page);
1734         spin_unlock_irq(&zone->lru_lock);
1735
1736         for (i = 1; i < HPAGE_PMD_NR; i++) {
1737                 struct page *page_tail = page + i;
1738                 BUG_ON(page_count(page_tail) <= 0);
1739                 /*
1740                  * Tail pages may be freed if there wasn't any mapping
1741                  * like if add_to_swap() is running on a lru page that
1742                  * had its mapping zapped. And freeing these pages
1743                  * requires taking the lru_lock so we do the put_page
1744                  * of the tail pages after the split is complete.
1745                  */
1746                 put_page(page_tail);
1747         }
1748
1749         /*
1750          * Only the head page (now become a regular page) is required
1751          * to be pinned by the caller.
1752          */
1753         BUG_ON(page_count(page) <= 0);
1754 }
1755
1756 static int __split_huge_page_map(struct page *page,
1757                                  struct vm_area_struct *vma,
1758                                  unsigned long address)
1759 {
1760         struct mm_struct *mm = vma->vm_mm;
1761         spinlock_t *ptl;
1762         pmd_t *pmd, _pmd;
1763         int ret = 0, i;
1764         pgtable_t pgtable;
1765         unsigned long haddr;
1766
1767         pmd = page_check_address_pmd(page, mm, address,
1768                         PAGE_CHECK_ADDRESS_PMD_SPLITTING_FLAG, &ptl);
1769         if (pmd) {
1770                 pgtable = pgtable_trans_huge_withdraw(mm, pmd);
1771                 pmd_populate(mm, &_pmd, pgtable);
1772                 if (pmd_write(*pmd))
1773                         BUG_ON(page_mapcount(page) != 1);
1774
1775                 haddr = address;
1776                 for (i = 0; i < HPAGE_PMD_NR; i++, haddr += PAGE_SIZE) {
1777                         pte_t *pte, entry;
1778                         BUG_ON(PageCompound(page+i));
1779                         /*
1780                          * Note that NUMA hinting access restrictions are not
1781                          * transferred to avoid any possibility of altering
1782                          * permissions across VMAs.
1783                          */
1784                         entry = mk_pte(page + i, vma->vm_page_prot);
1785                         entry = maybe_mkwrite(pte_mkdirty(entry), vma);
1786                         if (!pmd_write(*pmd))
1787                                 entry = pte_wrprotect(entry);
1788                         if (!pmd_young(*pmd))
1789                                 entry = pte_mkold(entry);
1790                         pte = pte_offset_map(&_pmd, haddr);
1791                         BUG_ON(!pte_none(*pte));
1792                         set_pte_at(mm, haddr, pte, entry);
1793                         pte_unmap(pte);
1794                 }
1795
1796                 smp_wmb(); /* make pte visible before pmd */
1797                 /*
1798                  * Up to this point the pmd is present and huge and
1799                  * userland has the whole access to the hugepage
1800                  * during the split (which happens in place). If we
1801                  * overwrite the pmd with the not-huge version
1802                  * pointing to the pte here (which of course we could
1803                  * if all CPUs were bug free), userland could trigger
1804                  * a small page size TLB miss on the small sized TLB
1805                  * while the hugepage TLB entry is still established
1806                  * in the huge TLB. Some CPU doesn't like that. See
1807                  * http://support.amd.com/us/Processor_TechDocs/41322.pdf,
1808                  * Erratum 383 on page 93. Intel should be safe but is
1809                  * also warns that it's only safe if the permission
1810                  * and cache attributes of the two entries loaded in
1811                  * the two TLB is identical (which should be the case
1812                  * here). But it is generally safer to never allow
1813                  * small and huge TLB entries for the same virtual
1814                  * address to be loaded simultaneously. So instead of
1815                  * doing "pmd_populate(); flush_tlb_range();" we first
1816                  * mark the current pmd notpresent (atomically because
1817                  * here the pmd_trans_huge and pmd_trans_splitting
1818                  * must remain set at all times on the pmd until the
1819                  * split is complete for this pmd), then we flush the
1820                  * SMP TLB and finally we write the non-huge version
1821                  * of the pmd entry with pmd_populate.
1822                  */
1823                 pmdp_invalidate(vma, address, pmd);
1824                 pmd_populate(mm, pmd, pgtable);
1825                 ret = 1;
1826                 spin_unlock(ptl);
1827         }
1828
1829         return ret;
1830 }
1831
1832 /* must be called with anon_vma->root->rwsem held */
1833 static void __split_huge_page(struct page *page,
1834                               struct anon_vma *anon_vma,
1835                               struct list_head *list)
1836 {
1837         int mapcount, mapcount2;
1838         pgoff_t pgoff = page->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT);
1839         struct anon_vma_chain *avc;
1840
1841         BUG_ON(!PageHead(page));
1842         BUG_ON(PageTail(page));
1843
1844         mapcount = 0;
1845         anon_vma_interval_tree_foreach(avc, &anon_vma->rb_root, pgoff, pgoff) {
1846                 struct vm_area_struct *vma = avc->vma;
1847                 unsigned long addr = vma_address(page, vma);
1848                 BUG_ON(is_vma_temporary_stack(vma));
1849                 mapcount += __split_huge_page_splitting(page, vma, addr);
1850         }
1851         /*
1852          * It is critical that new vmas are added to the tail of the
1853          * anon_vma list. This guarantes that if copy_huge_pmd() runs
1854          * and establishes a child pmd before
1855          * __split_huge_page_splitting() freezes the parent pmd (so if
1856          * we fail to prevent copy_huge_pmd() from running until the
1857          * whole __split_huge_page() is complete), we will still see
1858          * the newly established pmd of the child later during the
1859          * walk, to be able to set it as pmd_trans_splitting too.
1860          */
1861         if (mapcount != page_mapcount(page)) {
1862                 pr_err("mapcount %d page_mapcount %d\n",
1863                         mapcount, page_mapcount(page));
1864                 BUG();
1865         }
1866
1867         __split_huge_page_refcount(page, list);
1868
1869         mapcount2 = 0;
1870         anon_vma_interval_tree_foreach(avc, &anon_vma->rb_root, pgoff, pgoff) {
1871                 struct vm_area_struct *vma = avc->vma;
1872                 unsigned long addr = vma_address(page, vma);
1873                 BUG_ON(is_vma_temporary_stack(vma));
1874                 mapcount2 += __split_huge_page_map(page, vma, addr);
1875         }
1876         if (mapcount != mapcount2) {
1877                 pr_err("mapcount %d mapcount2 %d page_mapcount %d\n",
1878                         mapcount, mapcount2, page_mapcount(page));
1879                 BUG();
1880         }
1881 }
1882
1883 /*
1884  * Split a hugepage into normal pages. This doesn't change the position of head
1885  * page. If @list is null, tail pages will be added to LRU list, otherwise, to
1886  * @list. Both head page and tail pages will inherit mapping, flags, and so on
1887  * from the hugepage.
1888  * Return 0 if the hugepage is split successfully otherwise return 1.
1889  */
1890 int split_huge_page_to_list(struct page *page, struct list_head *list)
1891 {
1892         struct anon_vma *anon_vma;
1893         int ret = 1;
1894
1895         BUG_ON(is_huge_zero_page(page));
1896         BUG_ON(!PageAnon(page));
1897
1898         /*
1899          * The caller does not necessarily hold an mmap_sem that would prevent
1900          * the anon_vma disappearing so we first we take a reference to it
1901          * and then lock the anon_vma for write. This is similar to
1902          * page_lock_anon_vma_read except the write lock is taken to serialise
1903          * against parallel split or collapse operations.
1904          */
1905         anon_vma = page_get_anon_vma(page);
1906         if (!anon_vma)
1907                 goto out;
1908         anon_vma_lock_write(anon_vma);
1909
1910         ret = 0;
1911         if (!PageCompound(page))
1912                 goto out_unlock;
1913
1914         BUG_ON(!PageSwapBacked(page));
1915         __split_huge_page(page, anon_vma, list);
1916         count_vm_event(THP_SPLIT);
1917
1918         BUG_ON(PageCompound(page));
1919 out_unlock:
1920         anon_vma_unlock_write(anon_vma);
1921         put_anon_vma(anon_vma);
1922 out:
1923         return ret;
1924 }
1925
1926 #define VM_NO_THP (VM_SPECIAL | VM_HUGETLB | VM_SHARED | VM_MAYSHARE)
1927
1928 int hugepage_madvise(struct vm_area_struct *vma,
1929                      unsigned long *vm_flags, int advice)
1930 {
1931         switch (advice) {
1932         case MADV_HUGEPAGE:
1933 #ifdef CONFIG_S390
1934                 /*
1935                  * qemu blindly sets MADV_HUGEPAGE on all allocations, but s390
1936                  * can't handle this properly after s390_enable_sie, so we simply
1937                  * ignore the madvise to prevent qemu from causing a SIGSEGV.
1938                  */
1939                 if (mm_has_pgste(vma->vm_mm))
1940                         return 0;
1941 #endif
1942                 /*
1943                  * Be somewhat over-protective like KSM for now!
1944                  */
1945                 if (*vm_flags & (VM_HUGEPAGE | VM_NO_THP))
1946                         return -EINVAL;
1947                 *vm_flags &= ~VM_NOHUGEPAGE;
1948                 *vm_flags |= VM_HUGEPAGE;
1949                 /*
1950                  * If the vma become good for khugepaged to scan,
1951                  * register it here without waiting a page fault that
1952                  * may not happen any time soon.
1953                  */
1954                 if (unlikely(khugepaged_enter_vma_merge(vma, *vm_flags)))
1955                         return -ENOMEM;
1956                 break;
1957         case MADV_NOHUGEPAGE:
1958                 /*
1959                  * Be somewhat over-protective like KSM for now!
1960                  */
1961                 if (*vm_flags & (VM_NOHUGEPAGE | VM_NO_THP))
1962                         return -EINVAL;
1963                 *vm_flags &= ~VM_HUGEPAGE;
1964                 *vm_flags |= VM_NOHUGEPAGE;
1965                 /*
1966                  * Setting VM_NOHUGEPAGE will prevent khugepaged from scanning
1967                  * this vma even if we leave the mm registered in khugepaged if
1968                  * it got registered before VM_NOHUGEPAGE was set.
1969                  */
1970                 break;
1971         }
1972
1973         return 0;
1974 }
1975
1976 static int __init khugepaged_slab_init(void)
1977 {
1978         mm_slot_cache = kmem_cache_create("khugepaged_mm_slot",
1979                                           sizeof(struct mm_slot),
1980                                           __alignof__(struct mm_slot), 0, NULL);
1981         if (!mm_slot_cache)
1982                 return -ENOMEM;
1983
1984         return 0;
1985 }
1986
1987 static void __init khugepaged_slab_exit(void)
1988 {
1989         kmem_cache_destroy(mm_slot_cache);
1990 }
1991
1992 static inline struct mm_slot *alloc_mm_slot(void)
1993 {
1994         if (!mm_slot_cache)     /* initialization failed */
1995                 return NULL;
1996         return kmem_cache_zalloc(mm_slot_cache, GFP_KERNEL);
1997 }
1998
1999 static inline void free_mm_slot(struct mm_slot *mm_slot)
2000 {
2001         kmem_cache_free(mm_slot_cache, mm_slot);
2002 }
2003
2004 static struct mm_slot *get_mm_slot(struct mm_struct *mm)
2005 {
2006         struct mm_slot *mm_slot;
2007
2008         hash_for_each_possible(mm_slots_hash, mm_slot, hash, (unsigned long)mm)
2009                 if (mm == mm_slot->mm)
2010                         return mm_slot;
2011
2012         return NULL;
2013 }
2014
2015 static void insert_to_mm_slots_hash(struct mm_struct *mm,
2016                                     struct mm_slot *mm_slot)
2017 {
2018         mm_slot->mm = mm;
2019         hash_add(mm_slots_hash, &mm_slot->hash, (long)mm);
2020 }
2021
2022 static inline int khugepaged_test_exit(struct mm_struct *mm)
2023 {
2024         return atomic_read(&mm->mm_users) == 0;
2025 }
2026
2027 int __khugepaged_enter(struct mm_struct *mm)
2028 {
2029         struct mm_slot *mm_slot;
2030         int wakeup;
2031
2032         mm_slot = alloc_mm_slot();
2033         if (!mm_slot)
2034                 return -ENOMEM;
2035
2036         /* __khugepaged_exit() must not run from under us */
2037         VM_BUG_ON_MM(khugepaged_test_exit(mm), mm);
2038         if (unlikely(test_and_set_bit(MMF_VM_HUGEPAGE, &mm->flags))) {
2039                 free_mm_slot(mm_slot);
2040                 return 0;
2041         }
2042
2043         spin_lock(&khugepaged_mm_lock);
2044         insert_to_mm_slots_hash(mm, mm_slot);
2045         /*
2046          * Insert just behind the scanning cursor, to let the area settle
2047          * down a little.
2048          */
2049         wakeup = list_empty(&khugepaged_scan.mm_head);
2050         list_add_tail(&mm_slot->mm_node, &khugepaged_scan.mm_head);
2051         spin_unlock(&khugepaged_mm_lock);
2052
2053         atomic_inc(&mm->mm_count);
2054         if (wakeup)
2055                 wake_up_interruptible(&khugepaged_wait);
2056
2057         return 0;
2058 }
2059
2060 int khugepaged_enter_vma_merge(struct vm_area_struct *vma,
2061                                unsigned long vm_flags)
2062 {
2063         unsigned long hstart, hend;
2064         if (!vma->anon_vma)
2065                 /*
2066                  * Not yet faulted in so we will register later in the
2067                  * page fault if needed.
2068                  */
2069                 return 0;
2070         if (vma->vm_ops)
2071                 /* khugepaged not yet working on file or special mappings */
2072                 return 0;
2073         VM_BUG_ON_VMA(vm_flags & VM_NO_THP, vma);
2074         hstart = (vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK;
2075         hend = vma->vm_end & HPAGE_PMD_MASK;
2076         if (hstart < hend)
2077                 return khugepaged_enter(vma, vm_flags);
2078         return 0;
2079 }
2080
2081 void __khugepaged_exit(struct mm_struct *mm)
2082 {
2083         struct mm_slot *mm_slot;
2084         int free = 0;
2085
2086         spin_lock(&khugepaged_mm_lock);
2087         mm_slot = get_mm_slot(mm);
2088         if (mm_slot && khugepaged_scan.mm_slot != mm_slot) {
2089                 hash_del(&mm_slot->hash);
2090                 list_del(&mm_slot->mm_node);
2091                 free = 1;
2092         }
2093         spin_unlock(&khugepaged_mm_lock);
2094
2095         if (free) {
2096                 clear_bit(MMF_VM_HUGEPAGE, &mm->flags);
2097                 free_mm_slot(mm_slot);
2098                 mmdrop(mm);
2099         } else if (mm_slot) {
2100                 /*
2101                  * This is required to serialize against
2102                  * khugepaged_test_exit() (which is guaranteed to run
2103                  * under mmap sem read mode). Stop here (after we
2104                  * return all pagetables will be destroyed) until
2105                  * khugepaged has finished working on the pagetables
2106                  * under the mmap_sem.
2107                  */
2108                 down_write(&mm->mmap_sem);
2109                 up_write(&mm->mmap_sem);
2110         }
2111 }
2112
2113 static void release_pte_page(struct page *page)
2114 {
2115         /* 0 stands for page_is_file_cache(page) == false */
2116         dec_zone_page_state(page, NR_ISOLATED_ANON + 0);
2117         unlock_page(page);
2118         putback_lru_page(page);
2119 }
2120
2121 static void release_pte_pages(pte_t *pte, pte_t *_pte)
2122 {
2123         while (--_pte >= pte) {
2124                 pte_t pteval = *_pte;
2125                 if (!pte_none(pteval) && !is_zero_pfn(pte_pfn(pteval)))
2126                         release_pte_page(pte_page(pteval));
2127         }
2128 }
2129
2130 static int __collapse_huge_page_isolate(struct vm_area_struct *vma,
2131                                         unsigned long address,
2132                                         pte_t *pte)
2133 {
2134         struct page *page;
2135         pte_t *_pte;
2136         int none_or_zero = 0;
2137         bool referenced = false, writable = false;
2138         for (_pte = pte; _pte < pte+HPAGE_PMD_NR;
2139              _pte++, address += PAGE_SIZE) {
2140                 pte_t pteval = *_pte;
2141                 if (pte_none(pteval) || is_zero_pfn(pte_pfn(pteval))) {
2142                         if (++none_or_zero <= khugepaged_max_ptes_none)
2143                                 continue;
2144                         else
2145                                 goto out;
2146                 }
2147                 if (!pte_present(pteval))
2148                         goto out;
2149                 page = vm_normal_page(vma, address, pteval);
2150                 if (unlikely(!page))
2151                         goto out;
2152
2153                 VM_BUG_ON_PAGE(PageCompound(page), page);
2154                 VM_BUG_ON_PAGE(!PageAnon(page), page);
2155                 VM_BUG_ON_PAGE(!PageSwapBacked(page), page);
2156
2157                 /*
2158                  * We can do it before isolate_lru_page because the
2159                  * page can't be freed from under us. NOTE: PG_lock
2160                  * is needed to serialize against split_huge_page
2161                  * when invoked from the VM.
2162                  */
2163                 if (!trylock_page(page))
2164                         goto out;
2165
2166                 /*
2167                  * cannot use mapcount: can't collapse if there's a gup pin.
2168                  * The page must only be referenced by the scanned process
2169                  * and page swap cache.
2170                  */
2171                 if (page_count(page) != 1 + !!PageSwapCache(page)) {
2172                         unlock_page(page);
2173                         goto out;
2174                 }
2175                 if (pte_write(pteval)) {
2176                         writable = true;
2177                 } else {
2178                         if (PageSwapCache(page) && !reuse_swap_page(page)) {
2179                                 unlock_page(page);
2180                                 goto out;
2181                         }
2182                         /*
2183                          * Page is not in the swap cache. It can be collapsed
2184                          * into a THP.
2185                          */
2186                 }
2187
2188                 /*
2189                  * Isolate the page to avoid collapsing an hugepage
2190                  * currently in use by the VM.
2191                  */
2192                 if (isolate_lru_page(page)) {
2193                         unlock_page(page);
2194                         goto out;
2195                 }
2196                 /* 0 stands for page_is_file_cache(page) == false */
2197                 inc_zone_page_state(page, NR_ISOLATED_ANON + 0);
2198                 VM_BUG_ON_PAGE(!PageLocked(page), page);
2199                 VM_BUG_ON_PAGE(PageLRU(page), page);
2200
2201                 /* If there is no mapped pte young don't collapse the page */
2202                 if (pte_young(pteval) || PageReferenced(page) ||
2203                     mmu_notifier_test_young(vma->vm_mm, address))
2204                         referenced = true;
2205         }
2206         if (likely(referenced && writable))
2207                 return 1;
2208 out:
2209         release_pte_pages(pte, _pte);
2210         return 0;
2211 }
2212
2213 static void __collapse_huge_page_copy(pte_t *pte, struct page *page,
2214                                       struct vm_area_struct *vma,
2215                                       unsigned long address,
2216                                       spinlock_t *ptl)
2217 {
2218         pte_t *_pte;
2219         for (_pte = pte; _pte < pte+HPAGE_PMD_NR; _pte++) {
2220                 pte_t pteval = *_pte;
2221                 struct page *src_page;
2222
2223                 if (pte_none(pteval) || is_zero_pfn(pte_pfn(pteval))) {
2224                         clear_user_highpage(page, address);
2225                         add_mm_counter(vma->vm_mm, MM_ANONPAGES, 1);
2226                         if (is_zero_pfn(pte_pfn(pteval))) {
2227                                 /*
2228                                  * ptl mostly unnecessary.
2229                                  */
2230                                 spin_lock(ptl);
2231                                 /*
2232                                  * paravirt calls inside pte_clear here are
2233                                  * superfluous.
2234                                  */
2235                                 pte_clear(vma->vm_mm, address, _pte);
2236                                 spin_unlock(ptl);
2237                         }
2238                 } else {
2239                         src_page = pte_page(pteval);
2240                         copy_user_highpage(page, src_page, address, vma);
2241                         VM_BUG_ON_PAGE(page_mapcount(src_page) != 1, src_page);
2242                         release_pte_page(src_page);
2243                         /*
2244                          * ptl mostly unnecessary, but preempt has to
2245                          * be disabled to update the per-cpu stats
2246                          * inside page_remove_rmap().
2247                          */
2248                         spin_lock(ptl);
2249                         /*
2250                          * paravirt calls inside pte_clear here are
2251                          * superfluous.
2252                          */
2253                         pte_clear(vma->vm_mm, address, _pte);
2254                         page_remove_rmap(src_page);
2255                         spin_unlock(ptl);
2256                         free_page_and_swap_cache(src_page);
2257                 }
2258
2259                 address += PAGE_SIZE;
2260                 page++;
2261         }
2262 }
2263
2264 static void khugepaged_alloc_sleep(void)
2265 {
2266         wait_event_freezable_timeout(khugepaged_wait, false,
2267                         msecs_to_jiffies(khugepaged_alloc_sleep_millisecs));
2268 }
2269
2270 static int khugepaged_node_load[MAX_NUMNODES];
2271
2272 static bool khugepaged_scan_abort(int nid)
2273 {
2274         int i;
2275
2276         /*
2277          * If zone_reclaim_mode is disabled, then no extra effort is made to
2278          * allocate memory locally.
2279          */
2280         if (!zone_reclaim_mode)
2281                 return false;
2282
2283         /* If there is a count for this node already, it must be acceptable */
2284         if (khugepaged_node_load[nid])
2285                 return false;
2286
2287         for (i = 0; i < MAX_NUMNODES; i++) {
2288                 if (!khugepaged_node_load[i])
2289                         continue;
2290                 if (node_distance(nid, i) > RECLAIM_DISTANCE)
2291                         return true;
2292         }
2293         return false;
2294 }
2295
2296 #ifdef CONFIG_NUMA
2297 static int khugepaged_find_target_node(void)
2298 {
2299         static int last_khugepaged_target_node = NUMA_NO_NODE;
2300         int nid, target_node = 0, max_value = 0;
2301
2302         /* find first node with max normal pages hit */
2303         for (nid = 0; nid < MAX_NUMNODES; nid++)
2304                 if (khugepaged_node_load[nid] > max_value) {
2305                         max_value = khugepaged_node_load[nid];
2306                         target_node = nid;
2307                 }
2308
2309         /* do some balance if several nodes have the same hit record */
2310         if (target_node <= last_khugepaged_target_node)
2311                 for (nid = last_khugepaged_target_node + 1; nid < MAX_NUMNODES;
2312                                 nid++)
2313                         if (max_value == khugepaged_node_load[nid]) {
2314                                 target_node = nid;
2315                                 break;
2316                         }
2317
2318         last_khugepaged_target_node = target_node;
2319         return target_node;
2320 }
2321
2322 static bool khugepaged_prealloc_page(struct page **hpage, bool *wait)
2323 {
2324         if (IS_ERR(*hpage)) {
2325                 if (!*wait)
2326                         return false;
2327
2328                 *wait = false;
2329                 *hpage = NULL;
2330                 khugepaged_alloc_sleep();
2331         } else if (*hpage) {
2332                 put_page(*hpage);
2333                 *hpage = NULL;
2334         }
2335
2336         return true;
2337 }
2338
2339 static struct page *
2340 khugepaged_alloc_page(struct page **hpage, gfp_t gfp, struct mm_struct *mm,
2341                        struct vm_area_struct *vma, unsigned long address,
2342                        int node)
2343 {
2344         VM_BUG_ON_PAGE(*hpage, *hpage);
2345
2346         /*
2347          * Before allocating the hugepage, release the mmap_sem read lock.
2348          * The allocation can take potentially a long time if it involves
2349          * sync compaction, and we do not need to hold the mmap_sem during
2350          * that. We will recheck the vma after taking it again in write mode.
2351          */
2352         up_read(&mm->mmap_sem);
2353
2354         *hpage = alloc_pages_exact_node(node, gfp, HPAGE_PMD_ORDER);
2355         if (unlikely(!*hpage)) {
2356                 count_vm_event(THP_COLLAPSE_ALLOC_FAILED);
2357                 *hpage = ERR_PTR(-ENOMEM);
2358                 return NULL;
2359         }
2360
2361         count_vm_event(THP_COLLAPSE_ALLOC);
2362         return *hpage;
2363 }
2364 #else
2365 static int khugepaged_find_target_node(void)
2366 {
2367         return 0;
2368 }
2369
2370 static inline struct page *alloc_hugepage(int defrag)
2371 {
2372         return alloc_pages(alloc_hugepage_gfpmask(defrag, 0),
2373                            HPAGE_PMD_ORDER);
2374 }
2375
2376 static struct page *khugepaged_alloc_hugepage(bool *wait)
2377 {
2378         struct page *hpage;
2379
2380         do {
2381                 hpage = alloc_hugepage(khugepaged_defrag());
2382                 if (!hpage) {
2383                         count_vm_event(THP_COLLAPSE_ALLOC_FAILED);
2384                         if (!*wait)
2385                                 return NULL;
2386
2387                         *wait = false;
2388                         khugepaged_alloc_sleep();
2389                 } else
2390                         count_vm_event(THP_COLLAPSE_ALLOC);
2391         } while (unlikely(!hpage) && likely(khugepaged_enabled()));
2392
2393         return hpage;
2394 }
2395
2396 static bool khugepaged_prealloc_page(struct page **hpage, bool *wait)
2397 {
2398         if (!*hpage)
2399                 *hpage = khugepaged_alloc_hugepage(wait);
2400
2401         if (unlikely(!*hpage))
2402                 return false;
2403
2404         return true;
2405 }
2406
2407 static struct page *
2408 khugepaged_alloc_page(struct page **hpage, gfp_t gfp, struct mm_struct *mm,
2409                        struct vm_area_struct *vma, unsigned long address,
2410                        int node)
2411 {
2412         up_read(&mm->mmap_sem);
2413         VM_BUG_ON(!*hpage);
2414
2415         return  *hpage;
2416 }
2417 #endif
2418
2419 static bool hugepage_vma_check(struct vm_area_struct *vma)
2420 {
2421         if ((!(vma->vm_flags & VM_HUGEPAGE) && !khugepaged_always()) ||
2422             (vma->vm_flags & VM_NOHUGEPAGE))
2423                 return false;
2424
2425         if (!vma->anon_vma || vma->vm_ops)
2426                 return false;
2427         if (is_vma_temporary_stack(vma))
2428                 return false;
2429         VM_BUG_ON_VMA(vma->vm_flags & VM_NO_THP, vma);
2430         return true;
2431 }
2432
2433 static void collapse_huge_page(struct mm_struct *mm,
2434                                    unsigned long address,
2435                                    struct page **hpage,
2436                                    struct vm_area_struct *vma,
2437                                    int node)
2438 {
2439         pmd_t *pmd, _pmd;
2440         pte_t *pte;
2441         pgtable_t pgtable;
2442         struct page *new_page;
2443         spinlock_t *pmd_ptl, *pte_ptl;
2444         int isolated;
2445         unsigned long hstart, hend;
2446         struct mem_cgroup *memcg;
2447         unsigned long mmun_start;       /* For mmu_notifiers */
2448         unsigned long mmun_end;         /* For mmu_notifiers */
2449         gfp_t gfp;
2450
2451         VM_BUG_ON(address & ~HPAGE_PMD_MASK);
2452
2453         /* Only allocate from the target node */
2454         gfp = alloc_hugepage_gfpmask(khugepaged_defrag(), __GFP_OTHER_NODE) |
2455                 __GFP_THISNODE;
2456
2457         /* release the mmap_sem read lock. */
2458         new_page = khugepaged_alloc_page(hpage, gfp, mm, vma, address, node);
2459         if (!new_page)
2460                 return;
2461
2462         if (unlikely(mem_cgroup_try_charge(new_page, mm,
2463                                            gfp, &memcg)))
2464                 return;
2465
2466         /*
2467          * Prevent all access to pagetables with the exception of
2468          * gup_fast later hanlded by the ptep_clear_flush and the VM
2469          * handled by the anon_vma lock + PG_lock.
2470          */
2471         down_write(&mm->mmap_sem);
2472         if (unlikely(khugepaged_test_exit(mm)))
2473                 goto out;
2474
2475         vma = find_vma(mm, address);
2476         if (!vma)
2477                 goto out;
2478         hstart = (vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK;
2479         hend = vma->vm_end & HPAGE_PMD_MASK;
2480         if (address < hstart || address + HPAGE_PMD_SIZE > hend)
2481                 goto out;
2482         if (!hugepage_vma_check(vma))
2483                 goto out;
2484         pmd = mm_find_pmd(mm, address);
2485         if (!pmd)
2486                 goto out;
2487
2488         anon_vma_lock_write(vma->anon_vma);
2489
2490         pte = pte_offset_map(pmd, address);
2491         pte_ptl = pte_lockptr(mm, pmd);
2492
2493         mmun_start = address;
2494         mmun_end   = address + HPAGE_PMD_SIZE;
2495         mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
2496         pmd_ptl = pmd_lock(mm, pmd); /* probably unnecessary */
2497         /*
2498          * After this gup_fast can't run anymore. This also removes
2499          * any huge TLB entry from the CPU so we won't allow
2500          * huge and small TLB entries for the same virtual address
2501          * to avoid the risk of CPU bugs in that area.
2502          */
2503         _pmd = pmdp_clear_flush(vma, address, pmd);
2504         spin_unlock(pmd_ptl);
2505         mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
2506
2507         spin_lock(pte_ptl);
2508         isolated = __collapse_huge_page_isolate(vma, address, pte);
2509         spin_unlock(pte_ptl);
2510
2511         if (unlikely(!isolated)) {
2512                 pte_unmap(pte);
2513                 spin_lock(pmd_ptl);
2514                 BUG_ON(!pmd_none(*pmd));
2515                 /*
2516                  * We can only use set_pmd_at when establishing
2517                  * hugepmds and never for establishing regular pmds that
2518                  * points to regular pagetables. Use pmd_populate for that
2519                  */
2520                 pmd_populate(mm, pmd, pmd_pgtable(_pmd));
2521                 spin_unlock(pmd_ptl);
2522                 anon_vma_unlock_write(vma->anon_vma);
2523                 goto out;
2524         }
2525
2526         /*
2527          * All pages are isolated and locked so anon_vma rmap
2528          * can't run anymore.
2529          */
2530         anon_vma_unlock_write(vma->anon_vma);
2531
2532         __collapse_huge_page_copy(pte, new_page, vma, address, pte_ptl);
2533         pte_unmap(pte);
2534         __SetPageUptodate(new_page);
2535         pgtable = pmd_pgtable(_pmd);
2536
2537         _pmd = mk_huge_pmd(new_page, vma->vm_page_prot);
2538         _pmd = maybe_pmd_mkwrite(pmd_mkdirty(_pmd), vma);
2539
2540         /*
2541          * spin_lock() below is not the equivalent of smp_wmb(), so
2542          * this is needed to avoid the copy_huge_page writes to become
2543          * visible after the set_pmd_at() write.
2544          */
2545         smp_wmb();
2546
2547         spin_lock(pmd_ptl);
2548         BUG_ON(!pmd_none(*pmd));
2549         page_add_new_anon_rmap(new_page, vma, address);
2550         mem_cgroup_commit_charge(new_page, memcg, false);
2551         lru_cache_add_active_or_unevictable(new_page, vma);
2552         pgtable_trans_huge_deposit(mm, pmd, pgtable);
2553         set_pmd_at(mm, address, pmd, _pmd);
2554         update_mmu_cache_pmd(vma, address, pmd);
2555         spin_unlock(pmd_ptl);
2556
2557         *hpage = NULL;
2558
2559         khugepaged_pages_collapsed++;
2560 out_up_write:
2561         up_write(&mm->mmap_sem);
2562         return;
2563
2564 out:
2565         mem_cgroup_cancel_charge(new_page, memcg);
2566         goto out_up_write;
2567 }
2568
2569 static int khugepaged_scan_pmd(struct mm_struct *mm,
2570                                struct vm_area_struct *vma,
2571                                unsigned long address,
2572                                struct page **hpage)
2573 {
2574         pmd_t *pmd;
2575         pte_t *pte, *_pte;
2576         int ret = 0, none_or_zero = 0;
2577         struct page *page;
2578         unsigned long _address;
2579         spinlock_t *ptl;
2580         int node = NUMA_NO_NODE;
2581         bool writable = false, referenced = false;
2582
2583         VM_BUG_ON(address & ~HPAGE_PMD_MASK);
2584
2585         pmd = mm_find_pmd(mm, address);
2586         if (!pmd)
2587                 goto out;
2588
2589         memset(khugepaged_node_load, 0, sizeof(khugepaged_node_load));
2590         pte = pte_offset_map_lock(mm, pmd, address, &ptl);
2591         for (_address = address, _pte = pte; _pte < pte+HPAGE_PMD_NR;
2592              _pte++, _address += PAGE_SIZE) {
2593                 pte_t pteval = *_pte;
2594                 if (pte_none(pteval) || is_zero_pfn(pte_pfn(pteval))) {
2595                         if (++none_or_zero <= khugepaged_max_ptes_none)
2596                                 continue;
2597                         else
2598                                 goto out_unmap;
2599                 }
2600                 if (!pte_present(pteval))
2601                         goto out_unmap;
2602                 if (pte_write(pteval))
2603                         writable = true;
2604
2605                 page = vm_normal_page(vma, _address, pteval);
2606                 if (unlikely(!page))
2607                         goto out_unmap;
2608                 /*
2609                  * Record which node the original page is from and save this
2610                  * information to khugepaged_node_load[].
2611                  * Khupaged will allocate hugepage from the node has the max
2612                  * hit record.
2613                  */
2614                 node = page_to_nid(page);
2615                 if (khugepaged_scan_abort(node))
2616                         goto out_unmap;
2617                 khugepaged_node_load[node]++;
2618                 VM_BUG_ON_PAGE(PageCompound(page), page);
2619                 if (!PageLRU(page) || PageLocked(page) || !PageAnon(page))
2620                         goto out_unmap;
2621                 /*
2622                  * cannot use mapcount: can't collapse if there's a gup pin.
2623                  * The page must only be referenced by the scanned process
2624                  * and page swap cache.
2625                  */
2626                 if (page_count(page) != 1 + !!PageSwapCache(page))
2627                         goto out_unmap;
2628                 if (pte_young(pteval) || PageReferenced(page) ||
2629                     mmu_notifier_test_young(vma->vm_mm, address))
2630                         referenced = true;
2631         }
2632         if (referenced && writable)
2633                 ret = 1;
2634 out_unmap:
2635         pte_unmap_unlock(pte, ptl);
2636         if (ret) {
2637                 node = khugepaged_find_target_node();
2638                 /* collapse_huge_page will return with the mmap_sem released */
2639                 collapse_huge_page(mm, address, hpage, vma, node);
2640         }
2641 out:
2642         return ret;
2643 }
2644
2645 static void collect_mm_slot(struct mm_slot *mm_slot)
2646 {
2647         struct mm_struct *mm = mm_slot->mm;
2648
2649         VM_BUG_ON(NR_CPUS != 1 && !spin_is_locked(&khugepaged_mm_lock));
2650
2651         if (khugepaged_test_exit(mm)) {
2652                 /* free mm_slot */
2653                 hash_del(&mm_slot->hash);
2654                 list_del(&mm_slot->mm_node);
2655
2656                 /*
2657                  * Not strictly needed because the mm exited already.
2658                  *
2659                  * clear_bit(MMF_VM_HUGEPAGE, &mm->flags);
2660                  */
2661
2662                 /* khugepaged_mm_lock actually not necessary for the below */
2663                 free_mm_slot(mm_slot);
2664                 mmdrop(mm);
2665         }
2666 }
2667
2668 static unsigned int khugepaged_scan_mm_slot(unsigned int pages,
2669                                             struct page **hpage)
2670         __releases(&khugepaged_mm_lock)
2671         __acquires(&khugepaged_mm_lock)
2672 {
2673         struct mm_slot *mm_slot;
2674         struct mm_struct *mm;
2675         struct vm_area_struct *vma;
2676         int progress = 0;
2677
2678         VM_BUG_ON(!pages);
2679         VM_BUG_ON(NR_CPUS != 1 && !spin_is_locked(&khugepaged_mm_lock));
2680
2681         if (khugepaged_scan.mm_slot)
2682                 mm_slot = khugepaged_scan.mm_slot;
2683         else {
2684                 mm_slot = list_entry(khugepaged_scan.mm_head.next,
2685                                      struct mm_slot, mm_node);
2686                 khugepaged_scan.address = 0;
2687                 khugepaged_scan.mm_slot = mm_slot;
2688         }
2689         spin_unlock(&khugepaged_mm_lock);
2690
2691         mm = mm_slot->mm;
2692         down_read(&mm->mmap_sem);
2693         if (unlikely(khugepaged_test_exit(mm)))
2694                 vma = NULL;
2695         else
2696                 vma = find_vma(mm, khugepaged_scan.address);
2697
2698         progress++;
2699         for (; vma; vma = vma->vm_next) {
2700                 unsigned long hstart, hend;
2701
2702                 cond_resched();
2703                 if (unlikely(khugepaged_test_exit(mm))) {
2704                         progress++;
2705                         break;
2706                 }
2707                 if (!hugepage_vma_check(vma)) {
2708 skip:
2709                         progress++;
2710                         continue;
2711                 }
2712                 hstart = (vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK;
2713                 hend = vma->vm_end & HPAGE_PMD_MASK;
2714                 if (hstart >= hend)
2715                         goto skip;
2716                 if (khugepaged_scan.address > hend)
2717                         goto skip;
2718                 if (khugepaged_scan.address < hstart)
2719                         khugepaged_scan.address = hstart;
2720                 VM_BUG_ON(khugepaged_scan.address & ~HPAGE_PMD_MASK);
2721
2722                 while (khugepaged_scan.address < hend) {
2723                         int ret;
2724                         cond_resched();
2725                         if (unlikely(khugepaged_test_exit(mm)))
2726                                 goto breakouterloop;
2727
2728                         VM_BUG_ON(khugepaged_scan.address < hstart ||
2729                                   khugepaged_scan.address + HPAGE_PMD_SIZE >
2730                                   hend);
2731                         ret = khugepaged_scan_pmd(mm, vma,
2732                                                   khugepaged_scan.address,
2733                                                   hpage);
2734                         /* move to next address */
2735                         khugepaged_scan.address += HPAGE_PMD_SIZE;
2736                         progress += HPAGE_PMD_NR;
2737                         if (ret)
2738                                 /* we released mmap_sem so break loop */
2739                                 goto breakouterloop_mmap_sem;
2740                         if (progress >= pages)
2741                                 goto breakouterloop;
2742                 }
2743         }
2744 breakouterloop:
2745         up_read(&mm->mmap_sem); /* exit_mmap will destroy ptes after this */
2746 breakouterloop_mmap_sem:
2747
2748         spin_lock(&khugepaged_mm_lock);
2749         VM_BUG_ON(khugepaged_scan.mm_slot != mm_slot);
2750         /*
2751          * Release the current mm_slot if this mm is about to die, or
2752          * if we scanned all vmas of this mm.
2753          */
2754         if (khugepaged_test_exit(mm) || !vma) {
2755                 /*
2756                  * Make sure that if mm_users is reaching zero while
2757                  * khugepaged runs here, khugepaged_exit will find
2758                  * mm_slot not pointing to the exiting mm.
2759                  */
2760                 if (mm_slot->mm_node.next != &khugepaged_scan.mm_head) {
2761                         khugepaged_scan.mm_slot = list_entry(
2762                                 mm_slot->mm_node.next,
2763                                 struct mm_slot, mm_node);
2764                         khugepaged_scan.address = 0;
2765                 } else {
2766                         khugepaged_scan.mm_slot = NULL;
2767                         khugepaged_full_scans++;
2768                 }
2769
2770                 collect_mm_slot(mm_slot);
2771         }
2772
2773         return progress;
2774 }
2775
2776 static int khugepaged_has_work(void)
2777 {
2778         return !list_empty(&khugepaged_scan.mm_head) &&
2779                 khugepaged_enabled();
2780 }
2781
2782 static int khugepaged_wait_event(void)
2783 {
2784         return !list_empty(&khugepaged_scan.mm_head) ||
2785                 kthread_should_stop();
2786 }
2787
2788 static void khugepaged_do_scan(void)
2789 {
2790         struct page *hpage = NULL;
2791         unsigned int progress = 0, pass_through_head = 0;
2792         unsigned int pages = khugepaged_pages_to_scan;
2793         bool wait = true;
2794
2795         barrier(); /* write khugepaged_pages_to_scan to local stack */
2796
2797         while (progress < pages) {
2798                 if (!khugepaged_prealloc_page(&hpage, &wait))
2799                         break;
2800
2801                 cond_resched();
2802
2803                 if (unlikely(kthread_should_stop() || freezing(current)))
2804                         break;
2805
2806                 spin_lock(&khugepaged_mm_lock);
2807                 if (!khugepaged_scan.mm_slot)
2808                         pass_through_head++;
2809                 if (khugepaged_has_work() &&
2810                     pass_through_head < 2)
2811                         progress += khugepaged_scan_mm_slot(pages - progress,
2812                                                             &hpage);
2813                 else
2814                         progress = pages;
2815                 spin_unlock(&khugepaged_mm_lock);
2816         }
2817
2818         if (!IS_ERR_OR_NULL(hpage))
2819                 put_page(hpage);
2820 }
2821
2822 static void khugepaged_wait_work(void)
2823 {
2824         try_to_freeze();
2825
2826         if (khugepaged_has_work()) {
2827                 if (!khugepaged_scan_sleep_millisecs)
2828                         return;
2829
2830                 wait_event_freezable_timeout(khugepaged_wait,
2831                                              kthread_should_stop(),
2832                         msecs_to_jiffies(khugepaged_scan_sleep_millisecs));
2833                 return;
2834         }
2835
2836         if (khugepaged_enabled())
2837                 wait_event_freezable(khugepaged_wait, khugepaged_wait_event());
2838 }
2839
2840 static int khugepaged(void *none)
2841 {
2842         struct mm_slot *mm_slot;
2843
2844         set_freezable();
2845         set_user_nice(current, MAX_NICE);
2846
2847         while (!kthread_should_stop()) {
2848                 khugepaged_do_scan();
2849                 khugepaged_wait_work();
2850         }
2851
2852         spin_lock(&khugepaged_mm_lock);
2853         mm_slot = khugepaged_scan.mm_slot;
2854         khugepaged_scan.mm_slot = NULL;
2855         if (mm_slot)
2856                 collect_mm_slot(mm_slot);
2857         spin_unlock(&khugepaged_mm_lock);
2858         return 0;
2859 }
2860
2861 static void __split_huge_zero_page_pmd(struct vm_area_struct *vma,
2862                 unsigned long haddr, pmd_t *pmd)
2863 {
2864         struct mm_struct *mm = vma->vm_mm;
2865         pgtable_t pgtable;
2866         pmd_t _pmd;
2867         int i;
2868
2869         pmdp_clear_flush_notify(vma, haddr, pmd);
2870         /* leave pmd empty until pte is filled */
2871
2872         pgtable = pgtable_trans_huge_withdraw(mm, pmd);
2873         pmd_populate(mm, &_pmd, pgtable);
2874
2875         for (i = 0; i < HPAGE_PMD_NR; i++, haddr += PAGE_SIZE) {
2876                 pte_t *pte, entry;
2877                 entry = pfn_pte(my_zero_pfn(haddr), vma->vm_page_prot);
2878                 entry = pte_mkspecial(entry);
2879                 pte = pte_offset_map(&_pmd, haddr);
2880                 VM_BUG_ON(!pte_none(*pte));
2881                 set_pte_at(mm, haddr, pte, entry);
2882                 pte_unmap(pte);
2883         }
2884         smp_wmb(); /* make pte visible before pmd */
2885         pmd_populate(mm, pmd, pgtable);
2886         put_huge_zero_page();
2887 }
2888
2889 void __split_huge_page_pmd(struct vm_area_struct *vma, unsigned long address,
2890                 pmd_t *pmd)
2891 {
2892         spinlock_t *ptl;
2893         struct page *page;
2894         struct mm_struct *mm = vma->vm_mm;
2895         unsigned long haddr = address & HPAGE_PMD_MASK;
2896         unsigned long mmun_start;       /* For mmu_notifiers */
2897         unsigned long mmun_end;         /* For mmu_notifiers */
2898
2899         BUG_ON(vma->vm_start > haddr || vma->vm_end < haddr + HPAGE_PMD_SIZE);
2900
2901         mmun_start = haddr;
2902         mmun_end   = haddr + HPAGE_PMD_SIZE;
2903 again:
2904         mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
2905         ptl = pmd_lock(mm, pmd);
2906         if (unlikely(!pmd_trans_huge(*pmd))) {
2907                 spin_unlock(ptl);
2908                 mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
2909                 return;
2910         }
2911         if (is_huge_zero_pmd(*pmd)) {
2912                 __split_huge_zero_page_pmd(vma, haddr, pmd);
2913                 spin_unlock(ptl);
2914                 mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
2915                 return;
2916         }
2917         page = pmd_page(*pmd);
2918         VM_BUG_ON_PAGE(!page_count(page), page);
2919         get_page(page);
2920         spin_unlock(ptl);
2921         mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
2922
2923         split_huge_page(page);
2924
2925         put_page(page);
2926
2927         /*
2928          * We don't always have down_write of mmap_sem here: a racing
2929          * do_huge_pmd_wp_page() might have copied-on-write to another
2930          * huge page before our split_huge_page() got the anon_vma lock.
2931          */
2932         if (unlikely(pmd_trans_huge(*pmd)))
2933                 goto again;
2934 }
2935
2936 void split_huge_page_pmd_mm(struct mm_struct *mm, unsigned long address,
2937                 pmd_t *pmd)
2938 {
2939         struct vm_area_struct *vma;
2940
2941         vma = find_vma(mm, address);
2942         BUG_ON(vma == NULL);
2943         split_huge_page_pmd(vma, address, pmd);
2944 }
2945
2946 static void split_huge_page_address(struct mm_struct *mm,
2947                                     unsigned long address)
2948 {
2949         pgd_t *pgd;
2950         pud_t *pud;
2951         pmd_t *pmd;
2952
2953         VM_BUG_ON(!(address & ~HPAGE_PMD_MASK));
2954
2955         pgd = pgd_offset(mm, address);
2956         if (!pgd_present(*pgd))
2957                 return;
2958
2959         pud = pud_offset(pgd, address);
2960         if (!pud_present(*pud))
2961                 return;
2962
2963         pmd = pmd_offset(pud, address);
2964         if (!pmd_present(*pmd))
2965                 return;
2966         /*
2967          * Caller holds the mmap_sem write mode, so a huge pmd cannot
2968          * materialize from under us.
2969          */
2970         split_huge_page_pmd_mm(mm, address, pmd);
2971 }
2972
2973 void __vma_adjust_trans_huge(struct vm_area_struct *vma,
2974                              unsigned long start,
2975                              unsigned long end,
2976                              long adjust_next)
2977 {
2978         /*
2979          * If the new start address isn't hpage aligned and it could
2980          * previously contain an hugepage: check if we need to split
2981          * an huge pmd.
2982          */
2983         if (start & ~HPAGE_PMD_MASK &&
2984             (start & HPAGE_PMD_MASK) >= vma->vm_start &&
2985             (start & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE <= vma->vm_end)
2986                 split_huge_page_address(vma->vm_mm, start);
2987
2988         /*
2989          * If the new end address isn't hpage aligned and it could
2990          * previously contain an hugepage: check if we need to split
2991          * an huge pmd.
2992          */
2993         if (end & ~HPAGE_PMD_MASK &&
2994             (end & HPAGE_PMD_MASK) >= vma->vm_start &&
2995             (end & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE <= vma->vm_end)
2996                 split_huge_page_address(vma->vm_mm, end);
2997
2998         /*
2999          * If we're also updating the vma->vm_next->vm_start, if the new
3000          * vm_next->vm_start isn't page aligned and it could previously
3001          * contain an hugepage: check if we need to split an huge pmd.
3002          */
3003         if (adjust_next > 0) {
3004                 struct vm_area_struct *next = vma->vm_next;
3005                 unsigned long nstart = next->vm_start;
3006                 nstart += adjust_next << PAGE_SHIFT;
3007                 if (nstart & ~HPAGE_PMD_MASK &&
3008                     (nstart & HPAGE_PMD_MASK) >= next->vm_start &&
3009                     (nstart & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE <= next->vm_end)
3010                         split_huge_page_address(next->vm_mm, nstart);
3011         }
3012 }