s390/mm: add support for 2GB hugepages

[cascardo/linux.git] / arch / s390 / mm / gup.c

diff --git a/arch/s390/mm/gup.c b/arch/s390/mm/gup.c
index a8a6765..adb0c34 100644 (file)
--- a/arch/s390/mm/gup.c
+++ b/arch/s390/mm/gup.c
@@ -128,6 +128,44 @@ static inline int gup_pmd_range(pud_t *pudp, pud_t pud, unsigned long addr,
        return 1;
 }
 
+static int gup_huge_pud(pud_t *pudp, pud_t pud, unsigned long addr,
+               unsigned long end, int write, struct page **pages, int *nr)
+{
+       struct page *head, *page;
+       unsigned long mask;
+       int refs;
+
+       mask = (write ? _REGION_ENTRY_PROTECT : 0) | _REGION_ENTRY_INVALID;
+       if ((pud_val(pud) & mask) != 0)
+               return 0;
+       VM_BUG_ON(!pfn_valid(pud_pfn(pud)));
+
+       refs = 0;
+       head = pud_page(pud);
+       page = head + ((addr & ~PUD_MASK) >> PAGE_SHIFT);
+       do {
+               VM_BUG_ON_PAGE(compound_head(page) != head, page);
+               pages[*nr] = page;
+               (*nr)++;
+               page++;
+               refs++;
+       } while (addr += PAGE_SIZE, addr != end);
+
+       if (!page_cache_add_speculative(head, refs)) {
+               *nr -= refs;
+               return 0;
+       }
+
+       if (unlikely(pud_val(pud) != pud_val(*pudp))) {
+               *nr -= refs;
+               while (refs--)
+                       put_page(head);
+               return 0;
+       }
+
+       return 1;
+}
+
 static inline int gup_pud_range(pgd_t *pgdp, pgd_t pgd, unsigned long addr,
                unsigned long end, int write, struct page **pages, int *nr)
 {
@@ -144,7 +182,12 @@ static inline int gup_pud_range(pgd_t *pgdp, pgd_t pgd, unsigned long addr,
                next = pud_addr_end(addr, end);
                if (pud_none(pud))
                        return 0;
-               if (!gup_pmd_range(pudp, pud, addr, next, write, pages, nr))
+               if (unlikely(pud_large(pud))) {
+                       if (!gup_huge_pud(pudp, pud, addr, next, write, pages,
+                                         nr))
+                               return 0;
+               } else if (!gup_pmd_range(pudp, pud, addr, next, write, pages,
+                                         nr))
                        return 0;
        } while (pudp++, addr = next, addr != end);