mm/hugetlb.c

   1 /*
   2  * Generic hugetlb support.
   3  * (C) William Irwin, April 2004
   4  */
   5 #include <linux/gfp.h>
   6 #include <linux/list.h>
   7 #include <linux/init.h>
   8 #include <linux/module.h>
   9 #include <linux/mm.h>
  10 #include <linux/sysctl.h>
  11 #include <linux/highmem.h>
  12 #include <linux/nodemask.h>
  13 #include <linux/pagemap.h>
  14 #include <linux/mempolicy.h>
  15 #include <linux/cpuset.h>
  16 #include <linux/mutex.h>
  17
  18 #include <asm/page.h>
  19 #include <asm/pgtable.h>
  20
  21 #include <linux/hugetlb.h>
  22 #include "internal.h"
  23
  24 const unsigned long hugetlb_zero = 0, hugetlb_infinity = ~0UL;
  25 static unsigned long nr_huge_pages, free_huge_pages, resv_huge_pages;
  26 unsigned long max_huge_pages;
  27 static struct list_head hugepage_freelists[MAX_NUMNODES];
  28 static unsigned int nr_huge_pages_node[MAX_NUMNODES];
  29 static unsigned int free_huge_pages_node[MAX_NUMNODES];
  30 /*
  31  * Protects updates to hugepage_freelists, nr_huge_pages, and free_huge_pages
  32  */
  33 static DEFINE_SPINLOCK(hugetlb_lock);
  34
  35 static void clear_huge_page(struct page *page, unsigned long addr)
  36 {
  37         int i;
  38
  39         might_sleep();
  40         for (i = 0; i < (HPAGE_SIZE/PAGE_SIZE); i++) {
  41                 cond_resched();
  42                 clear_user_highpage(page + i, addr);
  43         }
  44 }
  45
  46 static void copy_huge_page(struct page *dst, struct page *src,
  47                            unsigned long addr, struct vm_area_struct *vma)
  48 {
  49         int i;
  50
  51         might_sleep();
  52         for (i = 0; i < HPAGE_SIZE/PAGE_SIZE; i++) {
  53                 cond_resched();
  54                 copy_user_highpage(dst + i, src + i, addr + i*PAGE_SIZE, vma);
  55         }
  56 }
  57
  58 static void enqueue_huge_page(struct page *page)
  59 {
  60         int nid = page_to_nid(page);
  61         list_add(&page->lru, &hugepage_freelists[nid]);
  62         free_huge_pages++;
  63         free_huge_pages_node[nid]++;
  64 }
  65
  66 static struct page *dequeue_huge_page(struct vm_area_struct *vma,
  67                                 unsigned long address)
  68 {
  69         int nid;
  70         struct page *page = NULL;
  71         struct zonelist *zonelist = huge_zonelist(vma, address);
  72         struct zone **z;
  73
  74         for (z = zonelist->zones; *z; z++) {
  75                 nid = zone_to_nid(*z);
  76                 if (cpuset_zone_allowed_softwall(*z, GFP_HIGHUSER) &&
  77                     !list_empty(&hugepage_freelists[nid]))
  78                         break;
  79         }
  80
  81         if (*z) {
  82                 page = list_entry(hugepage_freelists[nid].next,
  83                                   struct page, lru);
  84                 list_del(&page->lru);
  85                 free_huge_pages--;
  86                 free_huge_pages_node[nid]--;
  87         }
  88         return page;
  89 }
  90
  91 static void free_huge_page(struct page *page)
  92 {
  93         BUG_ON(page_count(page));
  94
  95         INIT_LIST_HEAD(&page->lru);
  96
  97         spin_lock(&hugetlb_lock);
  98         enqueue_huge_page(page);
  99         spin_unlock(&hugetlb_lock);
 100 }
 101
 102 static int alloc_fresh_huge_page(void)
 103 {
 104         static int prev_nid;
 105         struct page *page;
 106         static DEFINE_SPINLOCK(nid_lock);
 107         int nid;
 108
 109         spin_lock(&nid_lock);
 110         nid = next_node(prev_nid, node_online_map);
 111         if (nid == MAX_NUMNODES)
 112                 nid = first_node(node_online_map);
 113         prev_nid = nid;
 114         spin_unlock(&nid_lock);
 115
 116         page = alloc_pages_node(nid, GFP_HIGHUSER|__GFP_COMP|__GFP_NOWARN,
 117                                         HUGETLB_PAGE_ORDER);
 118         if (page) {
 119                 set_compound_page_dtor(page, free_huge_page);
 120                 spin_lock(&hugetlb_lock);
 121                 nr_huge_pages++;
 122                 nr_huge_pages_node[page_to_nid(page)]++;
 123                 spin_unlock(&hugetlb_lock);
 124                 put_page(page); /* free it into the hugepage allocator */
 125                 return 1;
 126         }
 127         return 0;
 128 }
 129
 130 static struct page *alloc_huge_page(struct vm_area_struct *vma,
 131                                     unsigned long addr)
 132 {
 133         struct page *page;
 134
 135         spin_lock(&hugetlb_lock);
 136         if (vma->vm_flags & VM_MAYSHARE)
 137                 resv_huge_pages--;
 138         else if (free_huge_pages <= resv_huge_pages)
 139                 goto fail;
 140
 141         page = dequeue_huge_page(vma, addr);
 142         if (!page)
 143                 goto fail;
 144
 145         spin_unlock(&hugetlb_lock);
 146         set_page_refcounted(page);
 147         return page;
 148
 149 fail:
 150         if (vma->vm_flags & VM_MAYSHARE)
 151                 resv_huge_pages++;
 152         spin_unlock(&hugetlb_lock);
 153         return NULL;
 154 }
 155
 156 static int __init hugetlb_init(void)
 157 {
 158         unsigned long i;
 159
 160         if (HPAGE_SHIFT == 0)
 161                 return 0;
 162
 163         for (i = 0; i < MAX_NUMNODES; ++i)
 164                 INIT_LIST_HEAD(&hugepage_freelists[i]);
 165
 166         for (i = 0; i < max_huge_pages; ++i) {
 167                 if (!alloc_fresh_huge_page())
 168                         break;
 169         }
 170         max_huge_pages = free_huge_pages = nr_huge_pages = i;
 171         printk("Total HugeTLB memory allocated, %ld\n", free_huge_pages);
 172         return 0;
 173 }
 174 module_init(hugetlb_init);
 175
 176 static int __init hugetlb_setup(char *s)
 177 {
 178         if (sscanf(s, "%lu", &max_huge_pages) <= 0)
 179                 max_huge_pages = 0;
 180         return 1;
 181 }
 182 __setup("hugepages=", hugetlb_setup);
 183
 184 static unsigned int cpuset_mems_nr(unsigned int *array)
 185 {
 186         int node;
 187         unsigned int nr = 0;
 188
 189         for_each_node_mask(node, cpuset_current_mems_allowed)
 190                 nr += array[node];
 191
 192         return nr;
 193 }
 194
 195 #ifdef CONFIG_SYSCTL
 196 static void update_and_free_page(struct page *page)
 197 {
 198         int i;
 199         nr_huge_pages--;
 200         nr_huge_pages_node[page_to_nid(page)]--;
 201         for (i = 0; i < (HPAGE_SIZE / PAGE_SIZE); i++) {
 202                 page[i].flags &= ~(1 << PG_locked | 1 << PG_error | 1 << PG_referenced |
 203                                 1 << PG_dirty | 1 << PG_active | 1 << PG_reserved |
 204                                 1 << PG_private | 1<< PG_writeback);
 205         }
 206         page[1].lru.next = NULL;
 207         set_page_refcounted(page);
 208         __free_pages(page, HUGETLB_PAGE_ORDER);
 209 }
 210
 211 #ifdef CONFIG_HIGHMEM
 212 static void try_to_free_low(unsigned long count)
 213 {
 214         int i;
 215
 216         for (i = 0; i < MAX_NUMNODES; ++i) {
 217                 struct page *page, *next;
 218                 list_for_each_entry_safe(page, next, &hugepage_freelists[i], lru) {
 219                         if (PageHighMem(page))
 220                                 continue;
 221                         list_del(&page->lru);
 222                         update_and_free_page(page);
 223                         free_huge_pages--;
 224                         free_huge_pages_node[page_to_nid(page)]--;
 225                         if (count >= nr_huge_pages)
 226                                 return;
 227                 }
 228         }
 229 }
 230 #else
 231 static inline void try_to_free_low(unsigned long count)
 232 {
 233 }
 234 #endif
 235
 236 static unsigned long set_max_huge_pages(unsigned long count)
 237 {
 238         while (count > nr_huge_pages) {
 239                 if (!alloc_fresh_huge_page())
 240                         return nr_huge_pages;
 241         }
 242         if (count >= nr_huge_pages)
 243                 return nr_huge_pages;
 244
 245         spin_lock(&hugetlb_lock);
 246         count = max(count, resv_huge_pages);
 247         try_to_free_low(count);
 248         while (count < nr_huge_pages) {
 249                 struct page *page = dequeue_huge_page(NULL, 0);
 250                 if (!page)
 251                         break;
 252                 update_and_free_page(page);
 253         }
 254         spin_unlock(&hugetlb_lock);
 255         return nr_huge_pages;
 256 }
 257
 258 int hugetlb_sysctl_handler(struct ctl_table *table, int write,
 259                            struct file *file, void __user *buffer,
 260                            size_t *length, loff_t *ppos)
 261 {
 262         proc_doulongvec_minmax(table, write, file, buffer, length, ppos);
 263         max_huge_pages = set_max_huge_pages(max_huge_pages);
 264         return 0;
 265 }
 266 #endif /* CONFIG_SYSCTL */
 267
 268 int hugetlb_report_meminfo(char *buf)
 269 {
 270         return sprintf(buf,
 271                         "HugePages_Total: %5lu\n"
 272                         "HugePages_Free:  %5lu\n"
 273                         "HugePages_Rsvd:  %5lu\n"
 274                         "Hugepagesize:    %5lu kB\n",
 275                         nr_huge_pages,
 276                         free_huge_pages,
 277                         resv_huge_pages,
 278                         HPAGE_SIZE/1024);
 279 }
 280
 281 int hugetlb_report_node_meminfo(int nid, char *buf)
 282 {
 283         return sprintf(buf,
 284                 "Node %d HugePages_Total: %5u\n"
 285                 "Node %d HugePages_Free:  %5u\n",
 286                 nid, nr_huge_pages_node[nid],
 287                 nid, free_huge_pages_node[nid]);
 288 }
 289
 290 /* Return the number pages of memory we physically have, in PAGE_SIZE units. */
 291 unsigned long hugetlb_total_pages(void)
 292 {
 293         return nr_huge_pages * (HPAGE_SIZE / PAGE_SIZE);
 294 }
 295
 296 /*
 297  * We cannot handle pagefaults against hugetlb pages at all.  They cause
 298  * handle_mm_fault() to try to instantiate regular-sized pages in the
 299  * hugegpage VMA.  do_page_fault() is supposed to trap this, so BUG is we get
 300  * this far.
 301  */
 302 static struct page *hugetlb_nopage(struct vm_area_struct *vma,
 303                                 unsigned long address, int *unused)
 304 {
 305         BUG();
 306         return NULL;
 307 }
 308
 309 struct vm_operations_struct hugetlb_vm_ops = {
 310         .nopage = hugetlb_nopage,
 311 };
 312
 313 static pte_t make_huge_pte(struct vm_area_struct *vma, struct page *page,
 314                                 int writable)
 315 {
 316         pte_t entry;
 317
 318         if (writable) {
 319                 entry =
 320                     pte_mkwrite(pte_mkdirty(mk_pte(page, vma->vm_page_prot)));
 321         } else {
 322                 entry = pte_wrprotect(mk_pte(page, vma->vm_page_prot));
 323         }
 324         entry = pte_mkyoung(entry);
 325         entry = pte_mkhuge(entry);
 326
 327         return entry;
 328 }
 329
 330 static void set_huge_ptep_writable(struct vm_area_struct *vma,
 331                                    unsigned long address, pte_t *ptep)
 332 {
 333         pte_t entry;
 334
 335         entry = pte_mkwrite(pte_mkdirty(*ptep));
 336         if (ptep_set_access_flags(vma, address, ptep, entry, 1)) {
 337                 update_mmu_cache(vma, address, entry);
 338                 lazy_mmu_prot_update(entry);
 339         }
 340 }
 341
 342
 343 int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
 344                             struct vm_area_struct *vma)
 345 {
 346         pte_t *src_pte, *dst_pte, entry;
 347         struct page *ptepage;
 348         unsigned long addr;
 349         int cow;
 350
 351         cow = (vma->vm_flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE;
 352
 353         for (addr = vma->vm_start; addr < vma->vm_end; addr += HPAGE_SIZE) {
 354                 src_pte = huge_pte_offset(src, addr);
 355                 if (!src_pte)
 356                         continue;
 357                 dst_pte = huge_pte_alloc(dst, addr);
 358                 if (!dst_pte)
 359                         goto nomem;
 360                 spin_lock(&dst->page_table_lock);
 361                 spin_lock(&src->page_table_lock);
 362                 if (!pte_none(*src_pte)) {
 363                         if (cow)
 364                                 ptep_set_wrprotect(src, addr, src_pte);
 365                         entry = *src_pte;
 366                         ptepage = pte_page(entry);
 367                         get_page(ptepage);
 368                         set_huge_pte_at(dst, addr, dst_pte, entry);
 369                 }
 370                 spin_unlock(&src->page_table_lock);
 371                 spin_unlock(&dst->page_table_lock);
 372         }
 373         return 0;
 374
 375 nomem:
 376         return -ENOMEM;
 377 }
 378
 379 void __unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,
 380                             unsigned long end)
 381 {
 382         struct mm_struct *mm = vma->vm_mm;
 383         unsigned long address;
 384         pte_t *ptep;
 385         pte_t pte;
 386         struct page *page;
 387         struct page *tmp;
 388         /*
 389          * A page gathering list, protected by per file i_mmap_lock. The
 390          * lock is used to avoid list corruption from multiple unmapping
 391          * of the same page since we are using page->lru.
 392          */
 393         LIST_HEAD(page_list);
 394
 395         WARN_ON(!is_vm_hugetlb_page(vma));
 396         BUG_ON(start & ~HPAGE_MASK);
 397         BUG_ON(end & ~HPAGE_MASK);
 398
 399         spin_lock(&mm->page_table_lock);
 400         for (address = start; address < end; address += HPAGE_SIZE) {
 401                 ptep = huge_pte_offset(mm, address);
 402                 if (!ptep)
 403                         continue;
 404
 405                 if (huge_pmd_unshare(mm, &address, ptep))
 406                         continue;
 407
 408                 pte = huge_ptep_get_and_clear(mm, address, ptep);
 409                 if (pte_none(pte))
 410                         continue;
 411
 412                 page = pte_page(pte);
 413                 if (pte_dirty(pte))
 414                         set_page_dirty(page);
 415                 list_add(&page->lru, &page_list);
 416         }
 417         spin_unlock(&mm->page_table_lock);
 418         flush_tlb_range(vma, start, end);
 419         list_for_each_entry_safe(page, tmp, &page_list, lru) {
 420                 list_del(&page->lru);
 421                 put_page(page);
 422         }
 423 }
 424
 425 void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,
 426                           unsigned long end)
 427 {
 428         /*
 429          * It is undesirable to test vma->vm_file as it should be non-null
 430          * for valid hugetlb area. However, vm_file will be NULL in the error
 431          * cleanup path of do_mmap_pgoff. When hugetlbfs ->mmap method fails,
 432          * do_mmap_pgoff() nullifies vma->vm_file before calling this function
 433          * to clean up. Since no pte has actually been setup, it is safe to
 434          * do nothing in this case.
 435          */
 436         if (vma->vm_file) {
 437                 spin_lock(&vma->vm_file->f_mapping->i_mmap_lock);
 438                 __unmap_hugepage_range(vma, start, end);
 439                 spin_unlock(&vma->vm_file->f_mapping->i_mmap_lock);
 440         }
 441 }
 442
 443 static int hugetlb_cow(struct mm_struct *mm, struct vm_area_struct *vma,
 444                         unsigned long address, pte_t *ptep, pte_t pte)
 445 {
 446         struct page *old_page, *new_page;
 447         int avoidcopy;
 448
 449         old_page = pte_page(pte);
 450
 451         /* If no-one else is actually using this page, avoid the copy
 452          * and just make the page writable */
 453         avoidcopy = (page_count(old_page) == 1);
 454         if (avoidcopy) {
 455                 set_huge_ptep_writable(vma, address, ptep);
 456                 return VM_FAULT_MINOR;
 457         }
 458
 459         page_cache_get(old_page);
 460         new_page = alloc_huge_page(vma, address);
 461
 462         if (!new_page) {
 463                 page_cache_release(old_page);
 464                 return VM_FAULT_OOM;
 465         }
 466
 467         spin_unlock(&mm->page_table_lock);
 468         copy_huge_page(new_page, old_page, address, vma);
 469         spin_lock(&mm->page_table_lock);
 470
 471         ptep = huge_pte_offset(mm, address & HPAGE_MASK);
 472         if (likely(pte_same(*ptep, pte))) {
 473                 /* Break COW */
 474                 set_huge_pte_at(mm, address, ptep,
 475                                 make_huge_pte(vma, new_page, 1));
 476                 /* Make the old page be freed below */
 477                 new_page = old_page;
 478         }
 479         page_cache_release(new_page);
 480         page_cache_release(old_page);
 481         return VM_FAULT_MINOR;
 482 }
 483
 484 int hugetlb_no_page(struct mm_struct *mm, struct vm_area_struct *vma,
 485                         unsigned long address, pte_t *ptep, int write_access)
 486 {
 487         int ret = VM_FAULT_SIGBUS;
 488         unsigned long idx;
 489         unsigned long size;
 490         struct page *page;
 491         struct address_space *mapping;
 492         pte_t new_pte;
 493
 494         mapping = vma->vm_file->f_mapping;
 495         idx = ((address - vma->vm_start) >> HPAGE_SHIFT)
 496                 + (vma->vm_pgoff >> (HPAGE_SHIFT - PAGE_SHIFT));
 497
 498         /*
 499          * Use page lock to guard against racing truncation
 500          * before we get page_table_lock.
 501          */
 502 retry:
 503         page = find_lock_page(mapping, idx);
 504         if (!page) {
 505                 size = i_size_read(mapping->host) >> HPAGE_SHIFT;
 506                 if (idx >= size)
 507                         goto out;
 508                 if (hugetlb_get_quota(mapping))
 509                         goto out;
 510                 page = alloc_huge_page(vma, address);
 511                 if (!page) {
 512                         hugetlb_put_quota(mapping);
 513                         ret = VM_FAULT_OOM;
 514                         goto out;
 515                 }
 516                 clear_huge_page(page, address);
 517
 518                 if (vma->vm_flags & VM_SHARED) {
 519                         int err;
 520
 521                         err = add_to_page_cache(page, mapping, idx, GFP_KERNEL);
 522                         if (err) {
 523                                 put_page(page);
 524                                 hugetlb_put_quota(mapping);
 525                                 if (err == -EEXIST)
 526                                         goto retry;
 527                                 goto out;
 528                         }
 529                 } else
 530                         lock_page(page);
 531         }
 532
 533         spin_lock(&mm->page_table_lock);
 534         size = i_size_read(mapping->host) >> HPAGE_SHIFT;
 535         if (idx >= size)
 536                 goto backout;
 537
 538         ret = VM_FAULT_MINOR;
 539         if (!pte_none(*ptep))
 540                 goto backout;
 541
 542         new_pte = make_huge_pte(vma, page, ((vma->vm_flags & VM_WRITE)
 543                                 && (vma->vm_flags & VM_SHARED)));
 544         set_huge_pte_at(mm, address, ptep, new_pte);
 545
 546         if (write_access && !(vma->vm_flags & VM_SHARED)) {
 547                 /* Optimization, do the COW without a second fault */
 548                 ret = hugetlb_cow(mm, vma, address, ptep, new_pte);
 549         }
 550
 551         spin_unlock(&mm->page_table_lock);
 552         unlock_page(page);
 553 out:
 554         return ret;
 555
 556 backout:
 557         spin_unlock(&mm->page_table_lock);
 558         hugetlb_put_quota(mapping);
 559         unlock_page(page);
 560         put_page(page);
 561         goto out;
 562 }
 563
 564 int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
 565                         unsigned long address, int write_access)
 566 {
 567         pte_t *ptep;
 568         pte_t entry;
 569         int ret;
 570         static DEFINE_MUTEX(hugetlb_instantiation_mutex);
 571
 572         ptep = huge_pte_alloc(mm, address);
 573         if (!ptep)
 574                 return VM_FAULT_OOM;
 575
 576         /*
 577          * Serialize hugepage allocation and instantiation, so that we don't
 578          * get spurious allocation failures if two CPUs race to instantiate
 579          * the same page in the page cache.
 580          */
 581         mutex_lock(&hugetlb_instantiation_mutex);
 582         entry = *ptep;
 583         if (pte_none(entry)) {
 584                 ret = hugetlb_no_page(mm, vma, address, ptep, write_access);
 585                 mutex_unlock(&hugetlb_instantiation_mutex);
 586                 return ret;
 587         }
 588
 589         ret = VM_FAULT_MINOR;
 590
 591         spin_lock(&mm->page_table_lock);
 592         /* Check for a racing update before calling hugetlb_cow */
 593         if (likely(pte_same(entry, *ptep)))
 594                 if (write_access && !pte_write(entry))
 595                         ret = hugetlb_cow(mm, vma, address, ptep, entry);
 596         spin_unlock(&mm->page_table_lock);
 597         mutex_unlock(&hugetlb_instantiation_mutex);
 598
 599         return ret;
 600 }
 601
 602 int follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
 603                         struct page **pages, struct vm_area_struct **vmas,
 604                         unsigned long *position, int *length, int i)
 605 {
 606         unsigned long pfn_offset;
 607         unsigned long vaddr = *position;
 608         int remainder = *length;
 609
 610         spin_lock(&mm->page_table_lock);
 611         while (vaddr < vma->vm_end && remainder) {
 612                 pte_t *pte;
 613                 struct page *page;
 614
 615                 /*
 616                  * Some archs (sparc64, sh*) have multiple pte_ts to
 617                  * each hugepage.  We have to make * sure we get the
 618                  * first, for the page indexing below to work.
 619                  */
 620                 pte = huge_pte_offset(mm, vaddr & HPAGE_MASK);
 621
 622                 if (!pte || pte_none(*pte)) {
 623                         int ret;
 624
 625                         spin_unlock(&mm->page_table_lock);
 626                         ret = hugetlb_fault(mm, vma, vaddr, 0);
 627                         spin_lock(&mm->page_table_lock);
 628                         if (ret == VM_FAULT_MINOR)
 629                                 continue;
 630
 631                         remainder = 0;
 632                         if (!i)
 633                                 i = -EFAULT;
 634                         break;
 635                 }
 636
 637                 pfn_offset = (vaddr & ~HPAGE_MASK) >> PAGE_SHIFT;
 638                 page = pte_page(*pte);
 639 same_page:
 640                 if (pages) {
 641                         get_page(page);
 642                         pages[i] = page + pfn_offset;
 643                 }
 644
 645                 if (vmas)
 646                         vmas[i] = vma;
 647
 648                 vaddr += PAGE_SIZE;
 649                 ++pfn_offset;
 650                 --remainder;
 651                 ++i;
 652                 if (vaddr < vma->vm_end && remainder &&
 653                                 pfn_offset < HPAGE_SIZE/PAGE_SIZE) {
 654                         /*
 655                          * We use pfn_offset to avoid touching the pageframes
 656                          * of this compound page.
 657                          */
 658                         goto same_page;
 659                 }
 660         }
 661         spin_unlock(&mm->page_table_lock);
 662         *length = remainder;
 663         *position = vaddr;
 664
 665         return i;
 666 }
 667
 668 void hugetlb_change_protection(struct vm_area_struct *vma,
 669                 unsigned long address, unsigned long end, pgprot_t newprot)
 670 {
 671         struct mm_struct *mm = vma->vm_mm;
 672         unsigned long start = address;
 673         pte_t *ptep;
 674         pte_t pte;
 675
 676         BUG_ON(address >= end);
 677         flush_cache_range(vma, address, end);
 678
 679         spin_lock(&vma->vm_file->f_mapping->i_mmap_lock);
 680         spin_lock(&mm->page_table_lock);
 681         for (; address < end; address += HPAGE_SIZE) {
 682                 ptep = huge_pte_offset(mm, address);
 683                 if (!ptep)
 684                         continue;
 685                 if (huge_pmd_unshare(mm, &address, ptep))
 686                         continue;
 687                 if (!pte_none(*ptep)) {
 688                         pte = huge_ptep_get_and_clear(mm, address, ptep);
 689                         pte = pte_mkhuge(pte_modify(pte, newprot));
 690                         set_huge_pte_at(mm, address, ptep, pte);
 691                         lazy_mmu_prot_update(pte);
 692                 }
 693         }
 694         spin_unlock(&mm->page_table_lock);
 695         spin_unlock(&vma->vm_file->f_mapping->i_mmap_lock);
 696
 697         flush_tlb_range(vma, start, end);
 698 }
 699
 700 struct file_region {
 701         struct list_head link;
 702         long from;
 703         long to;
 704 };
 705
 706 static long region_add(struct list_head *head, long f, long t)
 707 {
 708         struct file_region *rg, *nrg, *trg;
 709
 710         /* Locate the region we are either in or before. */
 711         list_for_each_entry(rg, head, link)
 712                 if (f <= rg->to)
 713                         break;
 714
 715         /* Round our left edge to the current segment if it encloses us. */
 716         if (f > rg->from)
 717                 f = rg->from;
 718
 719         /* Check for and consume any regions we now overlap with. */
 720         nrg = rg;
 721         list_for_each_entry_safe(rg, trg, rg->link.prev, link) {
 722                 if (&rg->link == head)
 723                         break;
 724                 if (rg->from > t)
 725                         break;
 726
 727                 /* If this area reaches higher then extend our area to
 728                  * include it completely.  If this is not the first area
 729                  * which we intend to reuse, free it. */
 730                 if (rg->to > t)
 731                         t = rg->to;
 732                 if (rg != nrg) {
 733                         list_del(&rg->link);
 734                         kfree(rg);
 735                 }
 736         }
 737         nrg->from = f;
 738         nrg->to = t;
 739         return 0;
 740 }
 741
 742 static long region_chg(struct list_head *head, long f, long t)
 743 {
 744         struct file_region *rg, *nrg;
 745         long chg = 0;
 746
 747         /* Locate the region we are before or in. */
 748         list_for_each_entry(rg, head, link)
 749                 if (f <= rg->to)
 750                         break;
 751
 752         /* If we are below the current region then a new region is required.
 753          * Subtle, allocate a new region at the position but make it zero
 754          * size such that we can guarentee to record the reservation. */
 755         if (&rg->link == head || t < rg->from) {
 756                 nrg = kmalloc(sizeof(*nrg), GFP_KERNEL);
 757                 if (nrg == 0)
 758                         return -ENOMEM;
 759                 nrg->from = f;
 760                 nrg->to   = f;
 761                 INIT_LIST_HEAD(&nrg->link);
 762                 list_add(&nrg->link, rg->link.prev);
 763
 764                 return t - f;
 765         }
 766
 767         /* Round our left edge to the current segment if it encloses us. */
 768         if (f > rg->from)
 769                 f = rg->from;
 770         chg = t - f;
 771
 772         /* Check for and consume any regions we now overlap with. */
 773         list_for_each_entry(rg, rg->link.prev, link) {
 774                 if (&rg->link == head)
 775                         break;
 776                 if (rg->from > t)
 777                         return chg;
 778
 779                 /* We overlap with this area, if it extends futher than
 780                  * us then we must extend ourselves.  Account for its
 781                  * existing reservation. */
 782                 if (rg->to > t) {
 783                         chg += rg->to - t;
 784                         t = rg->to;
 785                 }
 786                 chg -= rg->to - rg->from;
 787         }
 788         return chg;
 789 }
 790
 791 static long region_truncate(struct list_head *head, long end)
 792 {
 793         struct file_region *rg, *trg;
 794         long chg = 0;
 795
 796         /* Locate the region we are either in or before. */
 797         list_for_each_entry(rg, head, link)
 798                 if (end <= rg->to)
 799                         break;
 800         if (&rg->link == head)
 801                 return 0;
 802
 803         /* If we are in the middle of a region then adjust it. */
 804         if (end > rg->from) {
 805                 chg = rg->to - end;
 806                 rg->to = end;
 807                 rg = list_entry(rg->link.next, typeof(*rg), link);
 808         }
 809
 810         /* Drop any remaining regions. */
 811         list_for_each_entry_safe(rg, trg, rg->link.prev, link) {
 812                 if (&rg->link == head)
 813                         break;
 814                 chg += rg->to - rg->from;
 815                 list_del(&rg->link);
 816                 kfree(rg);
 817         }
 818         return chg;
 819 }
 820
 821 static int hugetlb_acct_memory(long delta)
 822 {
 823         int ret = -ENOMEM;
 824
 825         spin_lock(&hugetlb_lock);
 826         if ((delta + resv_huge_pages) <= free_huge_pages) {
 827                 resv_huge_pages += delta;
 828                 ret = 0;
 829         }
 830         spin_unlock(&hugetlb_lock);
 831         return ret;
 832 }
 833
 834 int hugetlb_reserve_pages(struct inode *inode, long from, long to)
 835 {
 836         long ret, chg;
 837
 838         chg = region_chg(&inode->i_mapping->private_list, from, to);
 839         if (chg < 0)
 840                 return chg;
 841         /*
 842          * When cpuset is configured, it breaks the strict hugetlb page
 843          * reservation as the accounting is done on a global variable. Such
 844          * reservation is completely rubbish in the presence of cpuset because
 845          * the reservation is not checked against page availability for the
 846          * current cpuset. Application can still potentially OOM'ed by kernel
 847          * with lack of free htlb page in cpuset that the task is in.
 848          * Attempt to enforce strict accounting with cpuset is almost
 849          * impossible (or too ugly) because cpuset is too fluid that
 850          * task or memory node can be dynamically moved between cpusets.
 851          *
 852          * The change of semantics for shared hugetlb mapping with cpuset is
 853          * undesirable. However, in order to preserve some of the semantics,
 854          * we fall back to check against current free page availability as
 855          * a best attempt and hopefully to minimize the impact of changing
 856          * semantics that cpuset has.
 857          */
 858         if (chg > cpuset_mems_nr(free_huge_pages_node))
 859                 return -ENOMEM;
 860
 861         ret = hugetlb_acct_memory(chg);
 862         if (ret < 0)
 863                 return ret;
 864         region_add(&inode->i_mapping->private_list, from, to);
 865         return 0;
 866 }
 867
 868 void hugetlb_unreserve_pages(struct inode *inode, long offset, long freed)
 869 {
 870         long chg = region_truncate(&inode->i_mapping->private_list, offset);
 871         hugetlb_acct_memory(freed - chg);
 872 }