mm: replace get_user_pages_unlocked() write/force parameters with gup_flags

[cascardo/linux.git] / mm / gup.c

diff --git a/mm/gup.c b/mm/gup.c
index 96b2b2f..373d1ec 100644 (file)
--- a/mm/gup.c
+++ b/mm/gup.c
@@ -60,6 +60,16 @@ static int follow_pfn_pte(struct vm_area_struct *vma, unsigned long address,
        return -EEXIST;
 }
 
+/*
+ * FOLL_FORCE can write to even unwritable pte's, but only
+ * after we've gone through a COW cycle and they are dirty.
+ */
+static inline bool can_follow_write_pte(pte_t pte, unsigned int flags)
+{
+       return pte_write(pte) ||
+               ((flags & FOLL_FORCE) && (flags & FOLL_COW) && pte_dirty(pte));
+}
+
 static struct page *follow_page_pte(struct vm_area_struct *vma,
                unsigned long address, pmd_t *pmd, unsigned int flags)
 {
@@ -95,7 +105,7 @@ retry:
        }
        if ((flags & FOLL_NUMA) && pte_protnone(pte))
                goto no_page;
-       if ((flags & FOLL_WRITE) && !pte_write(pte)) {
+       if ((flags & FOLL_WRITE) && !can_follow_write_pte(pte, flags)) {
                pte_unmap_unlock(ptep, ptl);
                return NULL;
        }
@@ -412,7 +422,7 @@ static int faultin_page(struct task_struct *tsk, struct vm_area_struct *vma,
         * reCOWed by userspace write).
         */
        if ((ret & VM_FAULT_WRITE) && !(vma->vm_flags & VM_WRITE))
-               *flags &= ~FOLL_WRITE;
+               *flags |= FOLL_COW;
        return 0;
 }
 
@@ -729,7 +739,6 @@ static __always_inline long __get_user_pages_locked(struct task_struct *tsk,
                                                struct mm_struct *mm,
                                                unsigned long start,
                                                unsigned long nr_pages,
-                                               int write, int force,
                                                struct page **pages,
                                                struct vm_area_struct **vmas,
                                                int *locked, bool notify_drop,
@@ -747,10 +756,6 @@ static __always_inline long __get_user_pages_locked(struct task_struct *tsk,
 
        if (pages)
                flags |= FOLL_GET;
-       if (write)
-               flags |= FOLL_WRITE;
-       if (force)
-               flags |= FOLL_FORCE;
 
        pages_done = 0;
        lock_dropped = false;
@@ -846,9 +851,15 @@ long get_user_pages_locked(unsigned long start, unsigned long nr_pages,
                           int write, int force, struct page **pages,
                           int *locked)
 {
+       unsigned int flags = FOLL_TOUCH;
+
+       if (write)
+               flags |= FOLL_WRITE;
+       if (force)
+               flags |= FOLL_FORCE;
+
        return __get_user_pages_locked(current, current->mm, start, nr_pages,
-                                      write, force, pages, NULL, locked, true,
-                                      FOLL_TOUCH);
+                                      pages, NULL, locked, true, flags);
 }
 EXPORT_SYMBOL(get_user_pages_locked);
 
@@ -864,14 +875,14 @@ EXPORT_SYMBOL(get_user_pages_locked);
  */
 __always_inline long __get_user_pages_unlocked(struct task_struct *tsk, struct mm_struct *mm,
                                               unsigned long start, unsigned long nr_pages,
-                                              int write, int force, struct page **pages,
-                                              unsigned int gup_flags)
+                                              struct page **pages, unsigned int gup_flags)
 {
        long ret;
        int locked = 1;
+
        down_read(&mm->mmap_sem);
-       ret = __get_user_pages_locked(tsk, mm, start, nr_pages, write, force,
-                                     pages, NULL, &locked, false, gup_flags);
+       ret = __get_user_pages_locked(tsk, mm, start, nr_pages, pages, NULL,
+                                     &locked, false, gup_flags);
        if (locked)
                up_read(&mm->mmap_sem);
        return ret;
@@ -896,10 +907,10 @@ EXPORT_SYMBOL(__get_user_pages_unlocked);
  * "force" parameter).
  */
 long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages,
-                            int write, int force, struct page **pages)
+                            struct page **pages, unsigned int gup_flags)
 {
        return __get_user_pages_unlocked(current, current->mm, start, nr_pages,
-                                        write, force, pages, FOLL_TOUCH);
+                                        pages, gup_flags | FOLL_TOUCH);
 }
 EXPORT_SYMBOL(get_user_pages_unlocked);
 
@@ -963,9 +974,15 @@ long get_user_pages_remote(struct task_struct *tsk, struct mm_struct *mm,
                int write, int force, struct page **pages,
                struct vm_area_struct **vmas)
 {
-       return __get_user_pages_locked(tsk, mm, start, nr_pages, write, force,
-                                      pages, vmas, NULL, false,
-                                      FOLL_TOUCH | FOLL_REMOTE);
+       unsigned int flags = FOLL_TOUCH | FOLL_REMOTE;
+
+       if (write)
+               flags |= FOLL_WRITE;
+       if (force)
+               flags |= FOLL_FORCE;
+
+       return __get_user_pages_locked(tsk, mm, start, nr_pages, pages, vmas,
+                                      NULL, false, flags);
 }
 EXPORT_SYMBOL(get_user_pages_remote);
 
@@ -979,9 +996,15 @@ long get_user_pages(unsigned long start, unsigned long nr_pages,
                int write, int force, struct page **pages,
                struct vm_area_struct **vmas)
 {
+       unsigned int flags = FOLL_TOUCH;
+
+       if (write)
+               flags |= FOLL_WRITE;
+       if (force)
+               flags |= FOLL_FORCE;
+
        return __get_user_pages_locked(current, current->mm, start, nr_pages,
-                                      write, force, pages, vmas, NULL, false,
-                                      FOLL_TOUCH);
+                                      pages, vmas, NULL, false, flags);
 }
 EXPORT_SYMBOL(get_user_pages);
 
@@ -1505,7 +1528,8 @@ int get_user_pages_fast(unsigned long start, int nr_pages, int write,
                start += nr << PAGE_SHIFT;
                pages += nr;
 
-               ret = get_user_pages_unlocked(start, nr_pages - nr, write, 0, pages);
+               ret = get_user_pages_unlocked(start, nr_pages - nr, pages,
+                               write ? FOLL_WRITE : 0);
 
                /* Have to be a bit careful with return values */
                if (nr > 0) {