crypto: rsa-pkcs1pad - Avoid copying output when possible

[cascardo/linux.git] / mm / vmalloc.c

diff --git a/mm/vmalloc.c b/mm/vmalloc.c
index ae7d20b..cf7ad1a 100644 (file)
--- a/mm/vmalloc.c
+++ b/mm/vmalloc.c
@@ -21,6 +21,7 @@
 #include <linux/debugobjects.h>
 #include <linux/kallsyms.h>
 #include <linux/list.h>
+#include <linux/notifier.h>
 #include <linux/rbtree.h>
 #include <linux/radix-tree.h>
 #include <linux/rcupdate.h>
@@ -274,13 +275,12 @@ EXPORT_SYMBOL(vmalloc_to_pfn);
 
 /*** Global kva allocator ***/
 
-#define VM_LAZY_FREE   0x01
-#define VM_LAZY_FREEING        0x02
 #define VM_VM_AREA     0x04
 
 static DEFINE_SPINLOCK(vmap_area_lock);
 /* Export for kexec only */
 LIST_HEAD(vmap_area_list);
+static LLIST_HEAD(vmap_purge_list);
 static struct rb_root vmap_area_root = RB_ROOT;
 
 /* The vmap cache globals are protected by vmap_area_lock */
@@ -344,6 +344,8 @@ static void __insert_vmap_area(struct vmap_area *va)
 
 static void purge_vmap_area_lazy(void);
 
+static BLOCKING_NOTIFIER_HEAD(vmap_notify_list);
+
 /*
  * Allocate a region of KVA of the specified size and alignment, within the
  * vstart and vend.
@@ -363,6 +365,8 @@ static struct vmap_area *alloc_vmap_area(unsigned long size,
        BUG_ON(offset_in_page(size));
        BUG_ON(!is_power_of_2(align));
 
+       might_sleep_if(gfpflags_allow_blocking(gfp_mask));
+
        va = kmalloc_node(sizeof(struct vmap_area),
                        gfp_mask & GFP_RECLAIM_MASK, node);
        if (unlikely(!va))
@@ -468,6 +472,16 @@ overflow:
                purged = 1;
                goto retry;
        }
+
+       if (gfpflags_allow_blocking(gfp_mask)) {
+               unsigned long freed = 0;
+               blocking_notifier_call_chain(&vmap_notify_list, 0, &freed);
+               if (freed > 0) {
+                       purged = 0;
+                       goto retry;
+               }
+       }
+
        if (printk_ratelimit())
                pr_warn("vmap allocation for size %lu failed: use vmalloc=<size> to increase size\n",
                        size);
@@ -475,6 +489,18 @@ overflow:
        return ERR_PTR(-EBUSY);
 }
 
+int register_vmap_purge_notifier(struct notifier_block *nb)
+{
+       return blocking_notifier_chain_register(&vmap_notify_list, nb);
+}
+EXPORT_SYMBOL_GPL(register_vmap_purge_notifier);
+
+int unregister_vmap_purge_notifier(struct notifier_block *nb)
+{
+       return blocking_notifier_chain_unregister(&vmap_notify_list, nb);
+}
+EXPORT_SYMBOL_GPL(unregister_vmap_purge_notifier);
+
 static void __free_vmap_area(struct vmap_area *va)
 {
        BUG_ON(RB_EMPTY_NODE(&va->rb_node));
@@ -601,7 +627,7 @@ static void __purge_vmap_area_lazy(unsigned long *start, unsigned long *end,
                                        int sync, int force_flush)
 {
        static DEFINE_SPINLOCK(purge_lock);
-       LIST_HEAD(valist);
+       struct llist_node *valist;
        struct vmap_area *va;
        struct vmap_area *n_va;
        int nr = 0;
@@ -620,20 +646,14 @@ static void __purge_vmap_area_lazy(unsigned long *start, unsigned long *end,
        if (sync)
                purge_fragmented_blocks_allcpus();
 
-       rcu_read_lock();
-       list_for_each_entry_rcu(va, &vmap_area_list, list) {
-               if (va->flags & VM_LAZY_FREE) {
-                       if (va->va_start < *start)
-                               *start = va->va_start;
-                       if (va->va_end > *end)
-                               *end = va->va_end;
-                       nr += (va->va_end - va->va_start) >> PAGE_SHIFT;
-                       list_add_tail(&va->purge_list, &valist);
-                       va->flags |= VM_LAZY_FREEING;
-                       va->flags &= ~VM_LAZY_FREE;
-               }
+       valist = llist_del_all(&vmap_purge_list);
+       llist_for_each_entry(va, valist, purge_list) {
+               if (va->va_start < *start)
+                       *start = va->va_start;
+               if (va->va_end > *end)
+                       *end = va->va_end;
+               nr += (va->va_end - va->va_start) >> PAGE_SHIFT;
        }
-       rcu_read_unlock();
 
        if (nr)
                atomic_sub(nr, &vmap_lazy_nr);
@@ -643,7 +663,7 @@ static void __purge_vmap_area_lazy(unsigned long *start, unsigned long *end,
 
        if (nr) {
                spin_lock(&vmap_area_lock);
-               list_for_each_entry_safe(va, n_va, &valist, purge_list)
+               llist_for_each_entry_safe(va, n_va, valist, purge_list)
                        __free_vmap_area(va);
                spin_unlock(&vmap_area_lock);
        }
@@ -678,9 +698,15 @@ static void purge_vmap_area_lazy(void)
  */
 static void free_vmap_area_noflush(struct vmap_area *va)
 {
-       va->flags |= VM_LAZY_FREE;
-       atomic_add((va->va_end - va->va_start) >> PAGE_SHIFT, &vmap_lazy_nr);
-       if (unlikely(atomic_read(&vmap_lazy_nr) > lazy_max_pages()))
+       int nr_lazy;
+
+       nr_lazy = atomic_add_return((va->va_end - va->va_start) >> PAGE_SHIFT,
+                                   &vmap_lazy_nr);
+
+       /* After this point, we may free va at any time */
+       llist_add(&va->purge_list, &vmap_purge_list);
+
+       if (unlikely(nr_lazy > lazy_max_pages()))
                try_purge_vmap_area_lazy();
 }