#include <linux/ioctl.h>
#include <linux/security.h>
+static struct kmem_cache *userfaultfd_ctx_cachep __read_mostly;
+
enum userfaultfd_state {
UFFD_STATE_WAIT_API,
UFFD_STATE_RUNNING,
};
+/*
+ * Start with fault_pending_wqh and fault_wqh so they're more likely
+ * to be in the same cacheline.
+ */
struct userfaultfd_ctx {
- /* pseudo fd refcounting */
- atomic_t refcount;
- /* waitqueue head for the userfaultfd page faults */
+ /* waitqueue head for the pending (i.e. not read) userfaults */
+ wait_queue_head_t fault_pending_wqh;
+ /* waitqueue head for the userfaults */
wait_queue_head_t fault_wqh;
/* waitqueue head for the pseudo fd to wakeup poll/read */
wait_queue_head_t fd_wqh;
+ /* a refile sequence protected by fault_pending_wqh lock */
+ struct seqcount refile_seq;
+ /* pseudo fd refcounting */
+ atomic_t refcount;
/* userfaultfd syscall flags */
unsigned int flags;
/* state machine */
};
struct userfaultfd_wait_queue {
- unsigned long address;
+ struct uffd_msg msg;
wait_queue_t wq;
- bool pending;
struct userfaultfd_ctx *ctx;
};
uwq = container_of(wq, struct userfaultfd_wait_queue, wq);
ret = 0;
- /* don't wake the pending ones to avoid reads to block */
- if (uwq->pending && !ACCESS_ONCE(uwq->ctx->released))
- goto out;
/* len == 0 means wake all */
start = range->start;
len = range->len;
- if (len && (start > uwq->address || start + len <= uwq->address))
+ if (len && (start > uwq->msg.arg.pagefault.address ||
+ start + len <= uwq->msg.arg.pagefault.address))
goto out;
ret = wake_up_state(wq->private, mode);
if (ret)
VM_BUG_ON(spin_is_locked(&ctx->fd_wqh.lock));
VM_BUG_ON(waitqueue_active(&ctx->fd_wqh));
mmput(ctx->mm);
- kfree(ctx);
+ kmem_cache_free(userfaultfd_ctx_cachep, ctx);
}
}
-static inline unsigned long userfault_address(unsigned long address,
- unsigned int flags,
- unsigned long reason)
+static inline void msg_init(struct uffd_msg *msg)
+{
+ BUILD_BUG_ON(sizeof(struct uffd_msg) != 32);
+ /*
+ * Must use memset to zero out the paddings or kernel data is
+ * leaked to userland.
+ */
+ memset(msg, 0, sizeof(struct uffd_msg));
+}
+
+static inline struct uffd_msg userfault_msg(unsigned long address,
+ unsigned int flags,
+ unsigned long reason)
{
- BUILD_BUG_ON(PAGE_SHIFT < UFFD_BITS);
- address &= PAGE_MASK;
+ struct uffd_msg msg;
+ msg_init(&msg);
+ msg.event = UFFD_EVENT_PAGEFAULT;
+ msg.arg.pagefault.address = address;
if (flags & FAULT_FLAG_WRITE)
/*
- * Encode "write" fault information in the LSB of the
- * address read by userland, without depending on
- * FAULT_FLAG_WRITE kernel internal value.
+ * If UFFD_FEATURE_PAGEFAULT_FLAG_WRITE was set in the
+ * uffdio_api.features and UFFD_PAGEFAULT_FLAG_WRITE
+ * was not set in a UFFD_EVENT_PAGEFAULT, it means it
+ * was a read fault, otherwise if set it means it's
+ * a write fault.
*/
- address |= UFFD_BIT_WRITE;
+ msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_WRITE;
if (reason & VM_UFFD_WP)
/*
- * Encode "reason" fault information as bit number 1
- * in the address read by userland. If bit number 1 is
- * clear it means the reason is a VM_FAULT_MISSING
- * fault.
+ * If UFFD_FEATURE_PAGEFAULT_FLAG_WP was set in the
+ * uffdio_api.features and UFFD_PAGEFAULT_FLAG_WP was
+ * not set in a UFFD_EVENT_PAGEFAULT, it means it was
+ * a missing fault, otherwise if set it means it's a
+ * write protect fault.
*/
- address |= UFFD_BIT_WP;
- return address;
+ msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_WP;
+ return msg;
+}
+
+/*
+ * Verify the pagetables are still not ok after having reigstered into
+ * the fault_pending_wqh to avoid userland having to UFFDIO_WAKE any
+ * userfault that has already been resolved, if userfaultfd_read and
+ * UFFDIO_COPY|ZEROPAGE are being run simultaneously on two different
+ * threads.
+ */
+static inline bool userfaultfd_must_wait(struct userfaultfd_ctx *ctx,
+ unsigned long address,
+ unsigned long flags,
+ unsigned long reason)
+{
+ struct mm_struct *mm = ctx->mm;
+ pgd_t *pgd;
+ pud_t *pud;
+ pmd_t *pmd, _pmd;
+ pte_t *pte;
+ bool ret = true;
+
+ VM_BUG_ON(!rwsem_is_locked(&mm->mmap_sem));
+
+ pgd = pgd_offset(mm, address);
+ if (!pgd_present(*pgd))
+ goto out;
+ pud = pud_offset(pgd, address);
+ if (!pud_present(*pud))
+ goto out;
+ pmd = pmd_offset(pud, address);
+ /*
+ * READ_ONCE must function as a barrier with narrower scope
+ * and it must be equivalent to:
+ * _pmd = *pmd; barrier();
+ *
+ * This is to deal with the instability (as in
+ * pmd_trans_unstable) of the pmd.
+ */
+ _pmd = READ_ONCE(*pmd);
+ if (!pmd_present(_pmd))
+ goto out;
+
+ ret = false;
+ if (pmd_trans_huge(_pmd))
+ goto out;
+
+ /*
+ * the pmd is stable (as in !pmd_trans_unstable) so we can re-read it
+ * and use the standard pte_offset_map() instead of parsing _pmd.
+ */
+ pte = pte_offset_map(pmd, address);
+ /*
+ * Lockless access: we're in a wait_event so it's ok if it
+ * changes under us.
+ */
+ if (pte_none(*pte))
+ ret = true;
+ pte_unmap(pte);
+
+out:
+ return ret;
}
/*
struct mm_struct *mm = vma->vm_mm;
struct userfaultfd_ctx *ctx;
struct userfaultfd_wait_queue uwq;
+ int ret;
+ bool must_wait, return_to_userland;
BUG_ON(!rwsem_is_locked(&mm->mmap_sem));
+ ret = VM_FAULT_SIGBUS;
ctx = vma->vm_userfaultfd_ctx.ctx;
if (!ctx)
- return VM_FAULT_SIGBUS;
+ goto out;
BUG_ON(ctx->mm != mm);
* caller of handle_userfault to release the mmap_sem.
*/
if (unlikely(ACCESS_ONCE(ctx->released)))
- return VM_FAULT_SIGBUS;
+ goto out;
/*
* Check that we can return VM_FAULT_RETRY.
dump_stack();
}
#endif
- return VM_FAULT_SIGBUS;
+ goto out;
}
/*
* Handle nowait, not much to do other than tell it to retry
* and wait.
*/
+ ret = VM_FAULT_RETRY;
if (flags & FAULT_FLAG_RETRY_NOWAIT)
- return VM_FAULT_RETRY;
+ goto out;
/* take the reference before dropping the mmap_sem */
userfaultfd_ctx_get(ctx);
- /* be gentle and immediately relinquish the mmap_sem */
- up_read(&mm->mmap_sem);
-
init_waitqueue_func_entry(&uwq.wq, userfaultfd_wake_function);
uwq.wq.private = current;
- uwq.address = userfault_address(address, flags, reason);
- uwq.pending = true;
+ uwq.msg = userfault_msg(address, flags, reason);
uwq.ctx = ctx;
- spin_lock(&ctx->fault_wqh.lock);
+ return_to_userland = (flags & (FAULT_FLAG_USER|FAULT_FLAG_KILLABLE)) ==
+ (FAULT_FLAG_USER|FAULT_FLAG_KILLABLE);
+
+ spin_lock(&ctx->fault_pending_wqh.lock);
/*
* After the __add_wait_queue the uwq is visible to userland
* through poll/read().
*/
- __add_wait_queue(&ctx->fault_wqh, &uwq.wq);
- for (;;) {
- set_current_state(TASK_KILLABLE);
- if (!uwq.pending || ACCESS_ONCE(ctx->released) ||
- fatal_signal_pending(current))
- break;
- spin_unlock(&ctx->fault_wqh.lock);
+ __add_wait_queue(&ctx->fault_pending_wqh, &uwq.wq);
+ /*
+ * The smp_mb() after __set_current_state prevents the reads
+ * following the spin_unlock to happen before the list_add in
+ * __add_wait_queue.
+ */
+ set_current_state(return_to_userland ? TASK_INTERRUPTIBLE :
+ TASK_KILLABLE);
+ spin_unlock(&ctx->fault_pending_wqh.lock);
+
+ must_wait = userfaultfd_must_wait(ctx, address, flags, reason);
+ up_read(&mm->mmap_sem);
+ if (likely(must_wait && !ACCESS_ONCE(ctx->released) &&
+ (return_to_userland ? !signal_pending(current) :
+ !fatal_signal_pending(current)))) {
wake_up_poll(&ctx->fd_wqh, POLLIN);
schedule();
-
- spin_lock(&ctx->fault_wqh.lock);
+ ret |= VM_FAULT_MAJOR;
}
- __remove_wait_queue(&ctx->fault_wqh, &uwq.wq);
+
__set_current_state(TASK_RUNNING);
- spin_unlock(&ctx->fault_wqh.lock);
+
+ if (return_to_userland) {
+ if (signal_pending(current) &&
+ !fatal_signal_pending(current)) {
+ /*
+ * If we got a SIGSTOP or SIGCONT and this is
+ * a normal userland page fault, just let
+ * userland return so the signal will be
+ * handled and gdb debugging works. The page
+ * fault code immediately after we return from
+ * this function is going to release the
+ * mmap_sem and it's not depending on it
+ * (unlike gup would if we were not to return
+ * VM_FAULT_RETRY).
+ *
+ * If a fatal signal is pending we still take
+ * the streamlined VM_FAULT_RETRY failure path
+ * and there's no need to retake the mmap_sem
+ * in such case.
+ */
+ down_read(&mm->mmap_sem);
+ ret = 0;
+ }
+ }
+
+ /*
+ * Here we race with the list_del; list_add in
+ * userfaultfd_ctx_read(), however because we don't ever run
+ * list_del_init() to refile across the two lists, the prev
+ * and next pointers will never point to self. list_add also
+ * would never let any of the two pointers to point to
+ * self. So list_empty_careful won't risk to see both pointers
+ * pointing to self at any time during the list refile. The
+ * only case where list_del_init() is called is the full
+ * removal in the wake function and there we don't re-list_add
+ * and it's fine not to block on the spinlock. The uwq on this
+ * kernel stack can be released after the list_del_init.
+ */
+ if (!list_empty_careful(&uwq.wq.task_list)) {
+ spin_lock(&ctx->fault_pending_wqh.lock);
+ /*
+ * No need of list_del_init(), the uwq on the stack
+ * will be freed shortly anyway.
+ */
+ list_del(&uwq.wq.task_list);
+ spin_unlock(&ctx->fault_pending_wqh.lock);
+ }
/*
* ctx may go away after this if the userfault pseudo fd is
*/
userfaultfd_ctx_put(ctx);
- return VM_FAULT_RETRY;
+out:
+ return ret;
}
static int userfaultfd_release(struct inode *inode, struct file *file)
up_write(&mm->mmap_sem);
/*
- * After no new page faults can wait on this fault_wqh, flush
+ * After no new page faults can wait on this fault_*wqh, flush
* the last page faults that may have been already waiting on
- * the fault_wqh.
+ * the fault_*wqh.
*/
- spin_lock(&ctx->fault_wqh.lock);
+ spin_lock(&ctx->fault_pending_wqh.lock);
+ __wake_up_locked_key(&ctx->fault_pending_wqh, TASK_NORMAL, 0, &range);
__wake_up_locked_key(&ctx->fault_wqh, TASK_NORMAL, 0, &range);
- spin_unlock(&ctx->fault_wqh.lock);
+ spin_unlock(&ctx->fault_pending_wqh.lock);
wake_up_poll(&ctx->fd_wqh, POLLHUP);
userfaultfd_ctx_put(ctx);
return 0;
}
-/* fault_wqh.lock must be hold by the caller */
-static inline unsigned int find_userfault(struct userfaultfd_ctx *ctx,
- struct userfaultfd_wait_queue **uwq)
+/* fault_pending_wqh.lock must be hold by the caller */
+static inline struct userfaultfd_wait_queue *find_userfault(
+ struct userfaultfd_ctx *ctx)
{
wait_queue_t *wq;
- struct userfaultfd_wait_queue *_uwq;
- unsigned int ret = 0;
+ struct userfaultfd_wait_queue *uwq;
- VM_BUG_ON(!spin_is_locked(&ctx->fault_wqh.lock));
+ VM_BUG_ON(!spin_is_locked(&ctx->fault_pending_wqh.lock));
- list_for_each_entry(wq, &ctx->fault_wqh.task_list, task_list) {
- _uwq = container_of(wq, struct userfaultfd_wait_queue, wq);
- if (_uwq->pending) {
- ret = POLLIN;
- if (!uwq)
- /*
- * If there's at least a pending and
- * we don't care which one it is,
- * break immediately and leverage the
- * efficiency of the LIFO walk.
- */
- break;
- /*
- * If we need to find which one was pending we
- * keep walking until we find the first not
- * pending one, so we read() them in FIFO order.
- */
- *uwq = _uwq;
- } else
- /*
- * break the loop at the first not pending
- * one, there cannot be pending userfaults
- * after the first not pending one, because
- * all new pending ones are inserted at the
- * head and we walk it in LIFO.
- */
- break;
- }
-
- return ret;
+ uwq = NULL;
+ if (!waitqueue_active(&ctx->fault_pending_wqh))
+ goto out;
+ /* walk in reverse to provide FIFO behavior to read userfaults */
+ wq = list_last_entry(&ctx->fault_pending_wqh.task_list,
+ typeof(*wq), task_list);
+ uwq = container_of(wq, struct userfaultfd_wait_queue, wq);
+out:
+ return uwq;
}
static unsigned int userfaultfd_poll(struct file *file, poll_table *wait)
case UFFD_STATE_WAIT_API:
return POLLERR;
case UFFD_STATE_RUNNING:
- spin_lock(&ctx->fault_wqh.lock);
- ret = find_userfault(ctx, NULL);
- spin_unlock(&ctx->fault_wqh.lock);
+ /*
+ * poll() never guarantees that read won't block.
+ * userfaults can be waken before they're read().
+ */
+ if (unlikely(!(file->f_flags & O_NONBLOCK)))
+ return POLLERR;
+ /*
+ * lockless access to see if there are pending faults
+ * __pollwait last action is the add_wait_queue but
+ * the spin_unlock would allow the waitqueue_active to
+ * pass above the actual list_add inside
+ * add_wait_queue critical section. So use a full
+ * memory barrier to serialize the list_add write of
+ * add_wait_queue() with the waitqueue_active read
+ * below.
+ */
+ ret = 0;
+ smp_mb();
+ if (waitqueue_active(&ctx->fault_pending_wqh))
+ ret = POLLIN;
return ret;
default:
BUG();
}
static ssize_t userfaultfd_ctx_read(struct userfaultfd_ctx *ctx, int no_wait,
- __u64 *addr)
+ struct uffd_msg *msg)
{
ssize_t ret;
DECLARE_WAITQUEUE(wait, current);
- struct userfaultfd_wait_queue *uwq = NULL;
+ struct userfaultfd_wait_queue *uwq;
- /* always take the fd_wqh lock before the fault_wqh lock */
+ /* always take the fd_wqh lock before the fault_pending_wqh lock */
spin_lock(&ctx->fd_wqh.lock);
__add_wait_queue(&ctx->fd_wqh, &wait);
for (;;) {
set_current_state(TASK_INTERRUPTIBLE);
- spin_lock(&ctx->fault_wqh.lock);
- if (find_userfault(ctx, &uwq)) {
+ spin_lock(&ctx->fault_pending_wqh.lock);
+ uwq = find_userfault(ctx);
+ if (uwq) {
+ /*
+ * Use a seqcount to repeat the lockless check
+ * in wake_userfault() to avoid missing
+ * wakeups because during the refile both
+ * waitqueue could become empty if this is the
+ * only userfault.
+ */
+ write_seqcount_begin(&ctx->refile_seq);
+
/*
- * The fault_wqh.lock prevents the uwq to
- * disappear from under us.
+ * The fault_pending_wqh.lock prevents the uwq
+ * to disappear from under us.
+ *
+ * Refile this userfault from
+ * fault_pending_wqh to fault_wqh, it's not
+ * pending anymore after we read it.
+ *
+ * Use list_del() by hand (as
+ * userfaultfd_wake_function also uses
+ * list_del_init() by hand) to be sure nobody
+ * changes __remove_wait_queue() to use
+ * list_del_init() in turn breaking the
+ * !list_empty_careful() check in
+ * handle_userfault(). The uwq->wq.task_list
+ * must never be empty at any time during the
+ * refile, or the waitqueue could disappear
+ * from under us. The "wait_queue_head_t"
+ * parameter of __remove_wait_queue() is unused
+ * anyway.
*/
- uwq->pending = false;
- /* careful to always initialize addr if ret == 0 */
- *addr = uwq->address;
- spin_unlock(&ctx->fault_wqh.lock);
+ list_del(&uwq->wq.task_list);
+ __add_wait_queue(&ctx->fault_wqh, &uwq->wq);
+
+ write_seqcount_end(&ctx->refile_seq);
+
+ /* careful to always initialize msg if ret == 0 */
+ *msg = uwq->msg;
+ spin_unlock(&ctx->fault_pending_wqh.lock);
ret = 0;
break;
}
- spin_unlock(&ctx->fault_wqh.lock);
+ spin_unlock(&ctx->fault_pending_wqh.lock);
if (signal_pending(current)) {
ret = -ERESTARTSYS;
break;
{
struct userfaultfd_ctx *ctx = file->private_data;
ssize_t _ret, ret = 0;
- /* careful to always initialize addr if ret == 0 */
- __u64 uninitialized_var(addr);
+ struct uffd_msg msg;
int no_wait = file->f_flags & O_NONBLOCK;
if (ctx->state == UFFD_STATE_WAIT_API)
return -EINVAL;
- BUG_ON(ctx->state != UFFD_STATE_RUNNING);
for (;;) {
- if (count < sizeof(addr))
+ if (count < sizeof(msg))
return ret ? ret : -EINVAL;
- _ret = userfaultfd_ctx_read(ctx, no_wait, &addr);
+ _ret = userfaultfd_ctx_read(ctx, no_wait, &msg);
if (_ret < 0)
return ret ? ret : _ret;
- if (put_user(addr, (__u64 __user *) buf))
+ if (copy_to_user((__u64 __user *) buf, &msg, sizeof(msg)))
return ret ? ret : -EFAULT;
- ret += sizeof(addr);
- buf += sizeof(addr);
- count -= sizeof(addr);
+ ret += sizeof(msg);
+ buf += sizeof(msg);
+ count -= sizeof(msg);
/*
* Allow to read more than one fault at time but only
* block if waiting for the very first one.
start = range->start;
end = range->start + range->len;
- spin_lock(&ctx->fault_wqh.lock);
+ spin_lock(&ctx->fault_pending_wqh.lock);
/* wake all in the range and autoremove */
- __wake_up_locked_key(&ctx->fault_wqh, TASK_NORMAL, 0, range);
- spin_unlock(&ctx->fault_wqh.lock);
+ if (waitqueue_active(&ctx->fault_pending_wqh))
+ __wake_up_locked_key(&ctx->fault_pending_wqh, TASK_NORMAL, 0,
+ range);
+ if (waitqueue_active(&ctx->fault_wqh))
+ __wake_up_locked_key(&ctx->fault_wqh, TASK_NORMAL, 0, range);
+ spin_unlock(&ctx->fault_pending_wqh.lock);
}
static __always_inline void wake_userfault(struct userfaultfd_ctx *ctx,
struct userfaultfd_wake_range *range)
{
+ unsigned seq;
+ bool need_wakeup;
+
/*
* To be sure waitqueue_active() is not reordered by the CPU
* before the pagetable update, use an explicit SMP memory
* userfaults yet. So we take the spinlock only when we're
* sure we've userfaults to wake.
*/
- if (waitqueue_active(&ctx->fault_wqh))
+ do {
+ seq = read_seqcount_begin(&ctx->refile_seq);
+ need_wakeup = waitqueue_active(&ctx->fault_pending_wqh) ||
+ waitqueue_active(&ctx->fault_wqh);
+ cond_resched();
+ } while (read_seqcount_retry(&ctx->refile_seq, seq));
+ if (need_wakeup)
__wake_userfault(ctx, range);
}
}
/*
- * This is mostly needed to re-wakeup those userfaults that were still
- * pending when userland wake them up the first time. We don't wake
- * the pending one to avoid blocking reads to block, or non blocking
- * read to return -EAGAIN, if used with POLLIN, to avoid userland
- * doubts on why POLLIN wasn't reliable.
+ * userfaultfd_wake may be used in combination with the
+ * UFFDIO_*_MODE_DONTWAKE to wakeup userfaults in batches.
*/
static int userfaultfd_wake(struct userfaultfd_ctx *ctx,
unsigned long arg)
return ret;
}
+static int userfaultfd_copy(struct userfaultfd_ctx *ctx,
+ unsigned long arg)
+{
+ __s64 ret;
+ struct uffdio_copy uffdio_copy;
+ struct uffdio_copy __user *user_uffdio_copy;
+ struct userfaultfd_wake_range range;
+
+ user_uffdio_copy = (struct uffdio_copy __user *) arg;
+
+ ret = -EFAULT;
+ if (copy_from_user(&uffdio_copy, user_uffdio_copy,
+ /* don't copy "copy" last field */
+ sizeof(uffdio_copy)-sizeof(__s64)))
+ goto out;
+
+ ret = validate_range(ctx->mm, uffdio_copy.dst, uffdio_copy.len);
+ if (ret)
+ goto out;
+ /*
+ * double check for wraparound just in case. copy_from_user()
+ * will later check uffdio_copy.src + uffdio_copy.len to fit
+ * in the userland range.
+ */
+ ret = -EINVAL;
+ if (uffdio_copy.src + uffdio_copy.len <= uffdio_copy.src)
+ goto out;
+ if (uffdio_copy.mode & ~UFFDIO_COPY_MODE_DONTWAKE)
+ goto out;
+
+ ret = mcopy_atomic(ctx->mm, uffdio_copy.dst, uffdio_copy.src,
+ uffdio_copy.len);
+ if (unlikely(put_user(ret, &user_uffdio_copy->copy)))
+ return -EFAULT;
+ if (ret < 0)
+ goto out;
+ BUG_ON(!ret);
+ /* len == 0 would wake all */
+ range.len = ret;
+ if (!(uffdio_copy.mode & UFFDIO_COPY_MODE_DONTWAKE)) {
+ range.start = uffdio_copy.dst;
+ wake_userfault(ctx, &range);
+ }
+ ret = range.len == uffdio_copy.len ? 0 : -EAGAIN;
+out:
+ return ret;
+}
+
+static int userfaultfd_zeropage(struct userfaultfd_ctx *ctx,
+ unsigned long arg)
+{
+ __s64 ret;
+ struct uffdio_zeropage uffdio_zeropage;
+ struct uffdio_zeropage __user *user_uffdio_zeropage;
+ struct userfaultfd_wake_range range;
+
+ user_uffdio_zeropage = (struct uffdio_zeropage __user *) arg;
+
+ ret = -EFAULT;
+ if (copy_from_user(&uffdio_zeropage, user_uffdio_zeropage,
+ /* don't copy "zeropage" last field */
+ sizeof(uffdio_zeropage)-sizeof(__s64)))
+ goto out;
+
+ ret = validate_range(ctx->mm, uffdio_zeropage.range.start,
+ uffdio_zeropage.range.len);
+ if (ret)
+ goto out;
+ ret = -EINVAL;
+ if (uffdio_zeropage.mode & ~UFFDIO_ZEROPAGE_MODE_DONTWAKE)
+ goto out;
+
+ ret = mfill_zeropage(ctx->mm, uffdio_zeropage.range.start,
+ uffdio_zeropage.range.len);
+ if (unlikely(put_user(ret, &user_uffdio_zeropage->zeropage)))
+ return -EFAULT;
+ if (ret < 0)
+ goto out;
+ /* len == 0 would wake all */
+ BUG_ON(!ret);
+ range.len = ret;
+ if (!(uffdio_zeropage.mode & UFFDIO_ZEROPAGE_MODE_DONTWAKE)) {
+ range.start = uffdio_zeropage.range.start;
+ wake_userfault(ctx, &range);
+ }
+ ret = range.len == uffdio_zeropage.range.len ? 0 : -EAGAIN;
+out:
+ return ret;
+}
+
/*
* userland asks for a certain API version and we return which bits
* and ioctl commands are implemented in this kernel for such API
if (ctx->state != UFFD_STATE_WAIT_API)
goto out;
ret = -EFAULT;
- if (copy_from_user(&uffdio_api, buf, sizeof(__u64)))
+ if (copy_from_user(&uffdio_api, buf, sizeof(uffdio_api)))
goto out;
- if (uffdio_api.api != UFFD_API) {
- /* careful not to leak info, we only read the first 8 bytes */
+ if (uffdio_api.api != UFFD_API || uffdio_api.features) {
memset(&uffdio_api, 0, sizeof(uffdio_api));
if (copy_to_user(buf, &uffdio_api, sizeof(uffdio_api)))
goto out;
ret = -EINVAL;
goto out;
}
- /* careful not to leak info, we only read the first 8 bytes */
uffdio_api.features = UFFD_API_FEATURES;
uffdio_api.ioctls = UFFD_API_IOCTLS;
ret = -EFAULT;
int ret = -EINVAL;
struct userfaultfd_ctx *ctx = file->private_data;
+ if (cmd != UFFDIO_API && ctx->state == UFFD_STATE_WAIT_API)
+ return -EINVAL;
+
switch(cmd) {
case UFFDIO_API:
ret = userfaultfd_api(ctx, arg);
case UFFDIO_WAKE:
ret = userfaultfd_wake(ctx, arg);
break;
+ case UFFDIO_COPY:
+ ret = userfaultfd_copy(ctx, arg);
+ break;
+ case UFFDIO_ZEROPAGE:
+ ret = userfaultfd_zeropage(ctx, arg);
+ break;
}
return ret;
}
struct userfaultfd_wait_queue *uwq;
unsigned long pending = 0, total = 0;
- spin_lock(&ctx->fault_wqh.lock);
+ spin_lock(&ctx->fault_pending_wqh.lock);
+ list_for_each_entry(wq, &ctx->fault_pending_wqh.task_list, task_list) {
+ uwq = container_of(wq, struct userfaultfd_wait_queue, wq);
+ pending++;
+ total++;
+ }
list_for_each_entry(wq, &ctx->fault_wqh.task_list, task_list) {
uwq = container_of(wq, struct userfaultfd_wait_queue, wq);
- if (uwq->pending)
- pending++;
total++;
}
- spin_unlock(&ctx->fault_wqh.lock);
+ spin_unlock(&ctx->fault_pending_wqh.lock);
/*
* If more protocols will be added, there will be all shown
.llseek = noop_llseek,
};
+static void init_once_userfaultfd_ctx(void *mem)
+{
+ struct userfaultfd_ctx *ctx = (struct userfaultfd_ctx *) mem;
+
+ init_waitqueue_head(&ctx->fault_pending_wqh);
+ init_waitqueue_head(&ctx->fault_wqh);
+ init_waitqueue_head(&ctx->fd_wqh);
+ seqcount_init(&ctx->refile_seq);
+}
+
/**
* userfaultfd_file_create - Creates an userfaultfd file pointer.
* @flags: Flags for the userfaultfd file.
goto out;
file = ERR_PTR(-ENOMEM);
- ctx = kmalloc(sizeof(*ctx), GFP_KERNEL);
+ ctx = kmem_cache_alloc(userfaultfd_ctx_cachep, GFP_KERNEL);
if (!ctx)
goto out;
atomic_set(&ctx->refcount, 1);
- init_waitqueue_head(&ctx->fault_wqh);
- init_waitqueue_head(&ctx->fd_wqh);
ctx->flags = flags;
ctx->state = UFFD_STATE_WAIT_API;
ctx->released = false;
file = anon_inode_getfile("[userfaultfd]", &userfaultfd_fops, ctx,
O_RDWR | (flags & UFFD_SHARED_FCNTL_FLAGS));
if (IS_ERR(file))
- kfree(ctx);
+ kmem_cache_free(userfaultfd_ctx_cachep, ctx);
out:
return file;
}
return error;
}
+
+static int __init userfaultfd_init(void)
+{
+ userfaultfd_ctx_cachep = kmem_cache_create("userfaultfd_ctx_cache",
+ sizeof(struct userfaultfd_ctx),
+ 0,
+ SLAB_HWCACHE_ALIGN|SLAB_PANIC,
+ init_once_userfaultfd_ctx);
+ return 0;
+}
+__initcall(userfaultfd_init);
projects / cascardo / linux.git / blobdiff