Merge tag 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/borntraeger...

[cascardo/linux.git] / mm / gup.c

diff --git a/mm/gup.c b/mm/gup.c
index f2305de..a900759 100644 (file)
--- a/mm/gup.c
+++ b/mm/gup.c
@@ -3,7 +3,6 @@
 #include <linux/err.h>
 #include <linux/spinlock.h>
 
 #include <linux/err.h>
 #include <linux/spinlock.h>
 

-#include <linux/hugetlb.h>

 #include <linux/mm.h>
 #include <linux/pagemap.h>
 #include <linux/rmap.h>
 #include <linux/mm.h>
 #include <linux/pagemap.h>
 #include <linux/rmap.h>


@@ -12,6 +11,7 @@
 
 #include <linux/sched.h>
 #include <linux/rwsem.h>
 
 #include <linux/sched.h>
 #include <linux/rwsem.h>

+#include <linux/hugetlb.h>

 #include <asm/pgtable.h>
 
 #include "internal.h"
 #include <asm/pgtable.h>
 
 #include "internal.h"


@@ -875,6 +875,49 @@ static int gup_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr,
        return 1;
 }
 
        return 1;
 }
 

+static int gup_huge_pgd(pgd_t orig, pgd_t *pgdp, unsigned long addr,
+                       unsigned long end, int write,
+                       struct page **pages, int *nr)
+{
+       int refs;
+       struct page *head, *page, *tail;
+
+       if (write && !pgd_write(orig))
+               return 0;
+
+       refs = 0;
+       head = pgd_page(orig);
+       page = head + ((addr & ~PGDIR_MASK) >> PAGE_SHIFT);
+       tail = page;
+       do {
+               VM_BUG_ON_PAGE(compound_head(page) != head, page);
+               pages[*nr] = page;
+               (*nr)++;
+               page++;
+               refs++;
+       } while (addr += PAGE_SIZE, addr != end);
+
+       if (!page_cache_add_speculative(head, refs)) {
+               *nr -= refs;
+               return 0;
+       }
+
+       if (unlikely(pgd_val(orig) != pgd_val(*pgdp))) {
+               *nr -= refs;
+               while (refs--)
+                       put_page(head);
+               return 0;
+       }
+
+       while (refs--) {
+               if (PageTail(tail))
+                       get_huge_page_tail(tail);
+               tail++;
+       }
+
+       return 1;
+}
+

 static int gup_pmd_range(pud_t pud, unsigned long addr, unsigned long end,
                int write, struct page **pages, int *nr)
 {
 static int gup_pmd_range(pud_t pud, unsigned long addr, unsigned long end,
                int write, struct page **pages, int *nr)
 {


@@ -902,6 +945,14 @@ static int gup_pmd_range(pud_t pud, unsigned long addr, unsigned long end,
                                pages, nr))
                                return 0;
 
                                pages, nr))
                                return 0;
 

+               } else if (unlikely(is_hugepd(__hugepd(pmd_val(pmd))))) {
+                       /*
+                        * architecture have different format for hugetlbfs
+                        * pmd format and THP pmd format
+                        */
+                       if (!gup_huge_pd(__hugepd(pmd_val(pmd)), addr,
+                                        PMD_SHIFT, next, write, pages, nr))
+                               return 0;

                } else if (!gup_pte_range(pmd, addr, next, write, pages, nr))
                                return 0;
        } while (pmdp++, addr = next, addr != end);
                } else if (!gup_pte_range(pmd, addr, next, write, pages, nr))
                                return 0;
        } while (pmdp++, addr = next, addr != end);


@@ -909,22 +960,26 @@ static int gup_pmd_range(pud_t pud, unsigned long addr, unsigned long end,
        return 1;
 }
 
        return 1;
 }
 

-static int gup_pud_range(pgd_t *pgdp, unsigned long addr, unsigned long end,
-               int write, struct page **pages, int *nr)
+static int gup_pud_range(pgd_t pgd, unsigned long addr, unsigned long end,
+                        int write, struct page **pages, int *nr)

 {
        unsigned long next;
        pud_t *pudp;
 
 {
        unsigned long next;
        pud_t *pudp;
 

-       pudp = pud_offset(pgdp, addr);
+       pudp = pud_offset(&pgd, addr);

        do {
                pud_t pud = READ_ONCE(*pudp);
 
                next = pud_addr_end(addr, end);
                if (pud_none(pud))
                        return 0;
        do {
                pud_t pud = READ_ONCE(*pudp);
 
                next = pud_addr_end(addr, end);
                if (pud_none(pud))
                        return 0;

-               if (pud_huge(pud)) {
+               if (unlikely(pud_huge(pud))) {

                        if (!gup_huge_pud(pud, pudp, addr, next, write,
                        if (!gup_huge_pud(pud, pudp, addr, next, write,

-                                       pages, nr))
+                                         pages, nr))
+                               return 0;
+               } else if (unlikely(is_hugepd(__hugepd(pud_val(pud))))) {
+                       if (!gup_huge_pd(__hugepd(pud_val(pud)), addr,
+                                        PUD_SHIFT, next, write, pages, nr))

                                return 0;
                } else if (!gup_pmd_range(pud, addr, next, write, pages, nr))
                        return 0;
                                return 0;
                } else if (!gup_pmd_range(pud, addr, next, write, pages, nr))
                        return 0;


@@ -970,10 +1025,20 @@ int __get_user_pages_fast(unsigned long start, int nr_pages, int write,
        local_irq_save(flags);
        pgdp = pgd_offset(mm, addr);
        do {
        local_irq_save(flags);
        pgdp = pgd_offset(mm, addr);
        do {

+               pgd_t pgd = ACCESS_ONCE(*pgdp);
+

                next = pgd_addr_end(addr, end);
                next = pgd_addr_end(addr, end);

-               if (pgd_none(*pgdp))
+               if (pgd_none(pgd))

                        break;
                        break;

-               else if (!gup_pud_range(pgdp, addr, next, write, pages, &nr))
+               if (unlikely(pgd_huge(pgd))) {
+                       if (!gup_huge_pgd(pgd, pgdp, addr, next, write,
+                                         pages, &nr))
+                               break;
+               } else if (unlikely(is_hugepd(__hugepd(pgd_val(pgd))))) {
+                       if (!gup_huge_pd(__hugepd(pgd_val(pgd)), addr,
+                                        PGDIR_SHIFT, next, write, pages, &nr))
+                               break;
+               } else if (!gup_pud_range(pgd, addr, next, write, pages, &nr))

                        break;
        } while (pgdp++, addr = next, addr != end);
        local_irq_restore(flags);
                        break;
        } while (pgdp++, addr = next, addr != end);
        local_irq_restore(flags);