Merge tag 'libnvdimm-for-4.3' of git://git.kernel.org/pub/scm/linux/kernel/git/nvdimm...

[cascardo/linux.git] / fs / dax.c

diff --git a/fs/dax.c b/fs/dax.c
index a7f77e1..57bb70b 100644 (file)
--- a/fs/dax.c
+++ b/fs/dax.c
@@ -23,6 +23,7 @@
 #include <linux/memcontrol.h>
 #include <linux/mm.h>
 #include <linux/mutex.h>
+#include <linux/pmem.h>
 #include <linux/sched.h>
 #include <linux/uio.h>
 #include <linux/vmstat.h>
@@ -34,7 +35,7 @@ int dax_clear_blocks(struct inode *inode, sector_t block, long size)
 
        might_sleep();
        do {
-               void *addr;
+               void __pmem *addr;
                unsigned long pfn;
                long count;
 
@@ -46,10 +47,7 @@ int dax_clear_blocks(struct inode *inode, sector_t block, long size)
                        unsigned pgsz = PAGE_SIZE - offset_in_page(addr);
                        if (pgsz > count)
                                pgsz = count;
-                       if (pgsz < PAGE_SIZE)
-                               memset(addr, 0, pgsz);
-                       else
-                               clear_page(addr);
+                       clear_pmem(addr, pgsz);
                        addr += pgsz;
                        size -= pgsz;
                        count -= pgsz;
@@ -59,26 +57,29 @@ int dax_clear_blocks(struct inode *inode, sector_t block, long size)
                }
        } while (size);
 
+       wmb_pmem();
        return 0;
 }
 EXPORT_SYMBOL_GPL(dax_clear_blocks);
 
-static long dax_get_addr(struct buffer_head *bh, void **addr, unsigned blkbits)
+static long dax_get_addr(struct buffer_head *bh, void __pmem **addr,
+               unsigned blkbits)
 {
        unsigned long pfn;
        sector_t sector = bh->b_blocknr << (blkbits - 9);
        return bdev_direct_access(bh->b_bdev, sector, addr, &pfn, bh->b_size);
 }
 
-static void dax_new_buf(void *addr, unsigned size, unsigned first, loff_t pos,
-                       loff_t end)
+/* the clear_pmem() calls are ordered by a wmb_pmem() in the caller */
+static void dax_new_buf(void __pmem *addr, unsigned size, unsigned first,
+               loff_t pos, loff_t end)
 {
        loff_t final = end - pos + first; /* The final byte of the buffer */
 
        if (first > 0)
-               memset(addr, 0, first);
+               clear_pmem(addr, first);
        if (final < size)
-               memset(addr + final, 0, size - final);
+               clear_pmem(addr + final, size - final);
 }
 
 static bool buffer_written(struct buffer_head *bh)
@@ -106,14 +107,15 @@ static ssize_t dax_io(struct inode *inode, struct iov_iter *iter,
        loff_t pos = start;
        loff_t max = start;
        loff_t bh_max = start;
-       void *addr;
+       void __pmem *addr;
        bool hole = false;
+       bool need_wmb = false;
 
        if (iov_iter_rw(iter) != WRITE)
                end = min(end, i_size_read(inode));
 
        while (pos < end) {
-               unsigned len;
+               size_t len;
                if (pos == max) {
                        unsigned blkbits = inode->i_blkbits;
                        sector_t block = pos >> blkbits;
@@ -145,19 +147,23 @@ static ssize_t dax_io(struct inode *inode, struct iov_iter *iter,
                                retval = dax_get_addr(bh, &addr, blkbits);
                                if (retval < 0)
                                        break;
-                               if (buffer_unwritten(bh) || buffer_new(bh))
+                               if (buffer_unwritten(bh) || buffer_new(bh)) {
                                        dax_new_buf(addr, retval, first, pos,
                                                                        end);
+                                       need_wmb = true;
+                               }
                                addr += first;
                                size = retval - first;
                        }
                        max = min(pos + size, end);
                }
 
-               if (iov_iter_rw(iter) == WRITE)
-                       len = copy_from_iter_nocache(addr, max - pos, iter);
-               else if (!hole)
-                       len = copy_to_iter(addr, max - pos, iter);
+               if (iov_iter_rw(iter) == WRITE) {
+                       len = copy_from_iter_pmem(addr, max - pos, iter);
+                       need_wmb = true;
+               } else if (!hole)
+                       len = copy_to_iter((void __force *)addr, max - pos,
+                                       iter);
                else
                        len = iov_iter_zero(max - pos, iter);
 
@@ -168,6 +174,9 @@ static ssize_t dax_io(struct inode *inode, struct iov_iter *iter,
                addr += len;
        }
 
+       if (need_wmb)
+               wmb_pmem();
+
        return (pos == start) ? retval : pos - start;
 }
 
@@ -260,11 +269,13 @@ static int dax_load_hole(struct address_space *mapping, struct page *page,
 static int copy_user_bh(struct page *to, struct buffer_head *bh,
                        unsigned blkbits, unsigned long vaddr)
 {
-       void *vfrom, *vto;
+       void __pmem *vfrom;
+       void *vto;
+
        if (dax_get_addr(bh, &vfrom, blkbits) < 0)
                return -EIO;
        vto = kmap_atomic(to);
-       copy_user_page(vto, vfrom, vaddr, to);
+       copy_user_page(vto, (void __force *)vfrom, vaddr, to);
        kunmap_atomic(vto);
        return 0;
 }
@@ -275,7 +286,7 @@ static int dax_insert_mapping(struct inode *inode, struct buffer_head *bh,
        struct address_space *mapping = inode->i_mapping;
        sector_t sector = bh->b_blocknr << (inode->i_blkbits - 9);
        unsigned long vaddr = (unsigned long)vmf->virtual_address;
-       void *addr;
+       void __pmem *addr;
        unsigned long pfn;
        pgoff_t size;
        int error;
@@ -303,8 +314,10 @@ static int dax_insert_mapping(struct inode *inode, struct buffer_head *bh,
                goto out;
        }
 
-       if (buffer_unwritten(bh) || buffer_new(bh))
-               clear_page(addr);
+       if (buffer_unwritten(bh) || buffer_new(bh)) {
+               clear_pmem(addr, PAGE_SIZE);
+               wmb_pmem();
+       }
 
        error = vm_insert_mixed(vma, vaddr, pfn);
 
@@ -548,11 +561,12 @@ int dax_zero_page_range(struct inode *inode, loff_t from, unsigned length,
        if (err < 0)
                return err;
        if (buffer_written(&bh)) {
-               void *addr;
+               void __pmem *addr;
                err = dax_get_addr(&bh, &addr, inode->i_blkbits);
                if (err < 0)
                        return err;
-               memset(addr + offset, 0, length);
+               clear_pmem(addr + offset, length);
+               wmb_pmem();
        }
 
        return 0;