2 * Simple NUMA memory policy for the Linux kernel.
4 * Copyright 2003,2004 Andi Kleen, SuSE Labs.
5 * (C) Copyright 2005 Christoph Lameter, Silicon Graphics, Inc.
6 * Subject to the GNU Public License, version 2.
8 * NUMA policy allows the user to give hints in which node(s) memory should
11 * Support four policies per VMA and per process:
13 * The VMA policy has priority over the process policy for a page fault.
15 * interleave Allocate memory interleaved over a set of nodes,
16 * with normal fallback if it fails.
17 * For VMA based allocations this interleaves based on the
18 * offset into the backing object or offset into the mapping
19 * for anonymous memory. For process policy an process counter
22 * bind Only allocate memory on a specific set of nodes,
24 * FIXME: memory is allocated starting with the first node
25 * to the last. It would be better if bind would truly restrict
26 * the allocation to memory nodes instead
28 * preferred Try a specific node first before normal fallback.
29 * As a special case NUMA_NO_NODE here means do the allocation
30 * on the local CPU. This is normally identical to default,
31 * but useful to set in a VMA when you have a non default
34 * default Allocate on the local node first, or when on a VMA
35 * use the process policy. This is what Linux always did
36 * in a NUMA aware kernel and still does by, ahem, default.
38 * The process policy is applied for most non interrupt memory allocations
39 * in that process' context. Interrupts ignore the policies and always
40 * try to allocate on the local CPU. The VMA policy is only applied for memory
41 * allocations for a VMA in the VM.
43 * Currently there are a few corner cases in swapping where the policy
44 * is not applied, but the majority should be handled. When process policy
45 * is used it is not remembered over swap outs/swap ins.
47 * Only the highest zone in the zone hierarchy gets policied. Allocations
48 * requesting a lower zone just use default policy. This implies that
49 * on systems with highmem kernel lowmem allocation don't get policied.
50 * Same with GFP_DMA allocations.
52 * For shmfs/tmpfs/hugetlbfs shared memory the policy is shared between
53 * all users and remembered even when nobody has memory mapped.
57 fix mmap readahead to honour policy and enable policy for any page cache
59 statistics for bigpages
60 global policy for page cache? currently it uses process policy. Requires
62 handle mremap for shared memory (currently ignored for the policy)
64 make bind policy root only? It can trigger oom much faster and the
65 kernel is not always grateful with that.
68 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
70 #include <linux/mempolicy.h>
72 #include <linux/highmem.h>
73 #include <linux/hugetlb.h>
74 #include <linux/kernel.h>
75 #include <linux/sched.h>
76 #include <linux/nodemask.h>
77 #include <linux/cpuset.h>
78 #include <linux/slab.h>
79 #include <linux/string.h>
80 #include <linux/export.h>
81 #include <linux/nsproxy.h>
82 #include <linux/interrupt.h>
83 #include <linux/init.h>
84 #include <linux/compat.h>
85 #include <linux/swap.h>
86 #include <linux/seq_file.h>
87 #include <linux/proc_fs.h>
88 #include <linux/migrate.h>
89 #include <linux/ksm.h>
90 #include <linux/rmap.h>
91 #include <linux/security.h>
92 #include <linux/syscalls.h>
93 #include <linux/ctype.h>
94 #include <linux/mm_inline.h>
95 #include <linux/mmu_notifier.h>
96 #include <linux/printk.h>
98 #include <asm/tlbflush.h>
99 #include <asm/uaccess.h>
101 #include "internal.h"
104 #define MPOL_MF_DISCONTIG_OK (MPOL_MF_INTERNAL << 0) /* Skip checks for continuous vmas */
105 #define MPOL_MF_INVERT (MPOL_MF_INTERNAL << 1) /* Invert check for nodemask */
107 static struct kmem_cache *policy_cache;
108 static struct kmem_cache *sn_cache;
110 /* Highest zone. An specific allocation for a zone below that is not
112 enum zone_type policy_zone = 0;
115 * run-time system-wide default policy => local allocation
117 static struct mempolicy default_policy = {
118 .refcnt = ATOMIC_INIT(1), /* never free it */
119 .mode = MPOL_PREFERRED,
120 .flags = MPOL_F_LOCAL,
123 static struct mempolicy preferred_node_policy[MAX_NUMNODES];
125 struct mempolicy *get_task_policy(struct task_struct *p)
127 struct mempolicy *pol = p->mempolicy;
133 node = numa_node_id();
134 if (node != NUMA_NO_NODE) {
135 pol = &preferred_node_policy[node];
136 /* preferred_node_policy is not initialised early in boot */
141 return &default_policy;
144 static const struct mempolicy_operations {
145 int (*create)(struct mempolicy *pol, const nodemask_t *nodes);
147 * If read-side task has no lock to protect task->mempolicy, write-side
148 * task will rebind the task->mempolicy by two step. The first step is
149 * setting all the newly nodes, and the second step is cleaning all the
150 * disallowed nodes. In this way, we can avoid finding no node to alloc
152 * If we have a lock to protect task->mempolicy in read-side, we do
156 * MPOL_REBIND_ONCE - do rebind work at once
157 * MPOL_REBIND_STEP1 - set all the newly nodes
158 * MPOL_REBIND_STEP2 - clean all the disallowed nodes
160 void (*rebind)(struct mempolicy *pol, const nodemask_t *nodes,
161 enum mpol_rebind_step step);
162 } mpol_ops[MPOL_MAX];
164 static inline int mpol_store_user_nodemask(const struct mempolicy *pol)
166 return pol->flags & MPOL_MODE_FLAGS;
169 static void mpol_relative_nodemask(nodemask_t *ret, const nodemask_t *orig,
170 const nodemask_t *rel)
173 nodes_fold(tmp, *orig, nodes_weight(*rel));
174 nodes_onto(*ret, tmp, *rel);
177 static int mpol_new_interleave(struct mempolicy *pol, const nodemask_t *nodes)
179 if (nodes_empty(*nodes))
181 pol->v.nodes = *nodes;
185 static int mpol_new_preferred(struct mempolicy *pol, const nodemask_t *nodes)
188 pol->flags |= MPOL_F_LOCAL; /* local allocation */
189 else if (nodes_empty(*nodes))
190 return -EINVAL; /* no allowed nodes */
192 pol->v.preferred_node = first_node(*nodes);
196 static int mpol_new_bind(struct mempolicy *pol, const nodemask_t *nodes)
198 if (nodes_empty(*nodes))
200 pol->v.nodes = *nodes;
205 * mpol_set_nodemask is called after mpol_new() to set up the nodemask, if
206 * any, for the new policy. mpol_new() has already validated the nodes
207 * parameter with respect to the policy mode and flags. But, we need to
208 * handle an empty nodemask with MPOL_PREFERRED here.
210 * Must be called holding task's alloc_lock to protect task's mems_allowed
211 * and mempolicy. May also be called holding the mmap_semaphore for write.
213 static int mpol_set_nodemask(struct mempolicy *pol,
214 const nodemask_t *nodes, struct nodemask_scratch *nsc)
218 /* if mode is MPOL_DEFAULT, pol is NULL. This is right. */
222 nodes_and(nsc->mask1,
223 cpuset_current_mems_allowed, node_states[N_MEMORY]);
226 if (pol->mode == MPOL_PREFERRED && nodes_empty(*nodes))
227 nodes = NULL; /* explicit local allocation */
229 if (pol->flags & MPOL_F_RELATIVE_NODES)
230 mpol_relative_nodemask(&nsc->mask2, nodes, &nsc->mask1);
232 nodes_and(nsc->mask2, *nodes, nsc->mask1);
234 if (mpol_store_user_nodemask(pol))
235 pol->w.user_nodemask = *nodes;
237 pol->w.cpuset_mems_allowed =
238 cpuset_current_mems_allowed;
242 ret = mpol_ops[pol->mode].create(pol, &nsc->mask2);
244 ret = mpol_ops[pol->mode].create(pol, NULL);
249 * This function just creates a new policy, does some check and simple
250 * initialization. You must invoke mpol_set_nodemask() to set nodes.
252 static struct mempolicy *mpol_new(unsigned short mode, unsigned short flags,
255 struct mempolicy *policy;
257 pr_debug("setting mode %d flags %d nodes[0] %lx\n",
258 mode, flags, nodes ? nodes_addr(*nodes)[0] : NUMA_NO_NODE);
260 if (mode == MPOL_DEFAULT) {
261 if (nodes && !nodes_empty(*nodes))
262 return ERR_PTR(-EINVAL);
268 * MPOL_PREFERRED cannot be used with MPOL_F_STATIC_NODES or
269 * MPOL_F_RELATIVE_NODES if the nodemask is empty (local allocation).
270 * All other modes require a valid pointer to a non-empty nodemask.
272 if (mode == MPOL_PREFERRED) {
273 if (nodes_empty(*nodes)) {
274 if (((flags & MPOL_F_STATIC_NODES) ||
275 (flags & MPOL_F_RELATIVE_NODES)))
276 return ERR_PTR(-EINVAL);
278 } else if (mode == MPOL_LOCAL) {
279 if (!nodes_empty(*nodes))
280 return ERR_PTR(-EINVAL);
281 mode = MPOL_PREFERRED;
282 } else if (nodes_empty(*nodes))
283 return ERR_PTR(-EINVAL);
284 policy = kmem_cache_alloc(policy_cache, GFP_KERNEL);
286 return ERR_PTR(-ENOMEM);
287 atomic_set(&policy->refcnt, 1);
289 policy->flags = flags;
294 /* Slow path of a mpol destructor. */
295 void __mpol_put(struct mempolicy *p)
297 if (!atomic_dec_and_test(&p->refcnt))
299 kmem_cache_free(policy_cache, p);
302 static void mpol_rebind_default(struct mempolicy *pol, const nodemask_t *nodes,
303 enum mpol_rebind_step step)
309 * MPOL_REBIND_ONCE - do rebind work at once
310 * MPOL_REBIND_STEP1 - set all the newly nodes
311 * MPOL_REBIND_STEP2 - clean all the disallowed nodes
313 static void mpol_rebind_nodemask(struct mempolicy *pol, const nodemask_t *nodes,
314 enum mpol_rebind_step step)
318 if (pol->flags & MPOL_F_STATIC_NODES)
319 nodes_and(tmp, pol->w.user_nodemask, *nodes);
320 else if (pol->flags & MPOL_F_RELATIVE_NODES)
321 mpol_relative_nodemask(&tmp, &pol->w.user_nodemask, nodes);
324 * if step == 1, we use ->w.cpuset_mems_allowed to cache the
327 if (step == MPOL_REBIND_ONCE || step == MPOL_REBIND_STEP1) {
328 nodes_remap(tmp, pol->v.nodes,
329 pol->w.cpuset_mems_allowed, *nodes);
330 pol->w.cpuset_mems_allowed = step ? tmp : *nodes;
331 } else if (step == MPOL_REBIND_STEP2) {
332 tmp = pol->w.cpuset_mems_allowed;
333 pol->w.cpuset_mems_allowed = *nodes;
338 if (nodes_empty(tmp))
341 if (step == MPOL_REBIND_STEP1)
342 nodes_or(pol->v.nodes, pol->v.nodes, tmp);
343 else if (step == MPOL_REBIND_ONCE || step == MPOL_REBIND_STEP2)
348 if (!node_isset(current->il_next, tmp)) {
349 current->il_next = next_node_in(current->il_next, tmp);
350 if (current->il_next >= MAX_NUMNODES)
351 current->il_next = numa_node_id();
355 static void mpol_rebind_preferred(struct mempolicy *pol,
356 const nodemask_t *nodes,
357 enum mpol_rebind_step step)
361 if (pol->flags & MPOL_F_STATIC_NODES) {
362 int node = first_node(pol->w.user_nodemask);
364 if (node_isset(node, *nodes)) {
365 pol->v.preferred_node = node;
366 pol->flags &= ~MPOL_F_LOCAL;
368 pol->flags |= MPOL_F_LOCAL;
369 } else if (pol->flags & MPOL_F_RELATIVE_NODES) {
370 mpol_relative_nodemask(&tmp, &pol->w.user_nodemask, nodes);
371 pol->v.preferred_node = first_node(tmp);
372 } else if (!(pol->flags & MPOL_F_LOCAL)) {
373 pol->v.preferred_node = node_remap(pol->v.preferred_node,
374 pol->w.cpuset_mems_allowed,
376 pol->w.cpuset_mems_allowed = *nodes;
381 * mpol_rebind_policy - Migrate a policy to a different set of nodes
383 * If read-side task has no lock to protect task->mempolicy, write-side
384 * task will rebind the task->mempolicy by two step. The first step is
385 * setting all the newly nodes, and the second step is cleaning all the
386 * disallowed nodes. In this way, we can avoid finding no node to alloc
388 * If we have a lock to protect task->mempolicy in read-side, we do
392 * MPOL_REBIND_ONCE - do rebind work at once
393 * MPOL_REBIND_STEP1 - set all the newly nodes
394 * MPOL_REBIND_STEP2 - clean all the disallowed nodes
396 static void mpol_rebind_policy(struct mempolicy *pol, const nodemask_t *newmask,
397 enum mpol_rebind_step step)
401 if (!mpol_store_user_nodemask(pol) && step == MPOL_REBIND_ONCE &&
402 nodes_equal(pol->w.cpuset_mems_allowed, *newmask))
405 if (step == MPOL_REBIND_STEP1 && (pol->flags & MPOL_F_REBINDING))
408 if (step == MPOL_REBIND_STEP2 && !(pol->flags & MPOL_F_REBINDING))
411 if (step == MPOL_REBIND_STEP1)
412 pol->flags |= MPOL_F_REBINDING;
413 else if (step == MPOL_REBIND_STEP2)
414 pol->flags &= ~MPOL_F_REBINDING;
415 else if (step >= MPOL_REBIND_NSTEP)
418 mpol_ops[pol->mode].rebind(pol, newmask, step);
422 * Wrapper for mpol_rebind_policy() that just requires task
423 * pointer, and updates task mempolicy.
425 * Called with task's alloc_lock held.
428 void mpol_rebind_task(struct task_struct *tsk, const nodemask_t *new,
429 enum mpol_rebind_step step)
431 mpol_rebind_policy(tsk->mempolicy, new, step);
435 * Rebind each vma in mm to new nodemask.
437 * Call holding a reference to mm. Takes mm->mmap_sem during call.
440 void mpol_rebind_mm(struct mm_struct *mm, nodemask_t *new)
442 struct vm_area_struct *vma;
444 down_write(&mm->mmap_sem);
445 for (vma = mm->mmap; vma; vma = vma->vm_next)
446 mpol_rebind_policy(vma->vm_policy, new, MPOL_REBIND_ONCE);
447 up_write(&mm->mmap_sem);
450 static const struct mempolicy_operations mpol_ops[MPOL_MAX] = {
452 .rebind = mpol_rebind_default,
454 [MPOL_INTERLEAVE] = {
455 .create = mpol_new_interleave,
456 .rebind = mpol_rebind_nodemask,
459 .create = mpol_new_preferred,
460 .rebind = mpol_rebind_preferred,
463 .create = mpol_new_bind,
464 .rebind = mpol_rebind_nodemask,
468 static void migrate_page_add(struct page *page, struct list_head *pagelist,
469 unsigned long flags);
472 struct list_head *pagelist;
475 struct vm_area_struct *prev;
479 * Scan through pages checking if pages follow certain conditions,
480 * and move them to the pagelist if they do.
482 static int queue_pages_pte_range(pmd_t *pmd, unsigned long addr,
483 unsigned long end, struct mm_walk *walk)
485 struct vm_area_struct *vma = walk->vma;
487 struct queue_pages *qp = walk->private;
488 unsigned long flags = qp->flags;
493 if (pmd_trans_huge(*pmd)) {
494 ptl = pmd_lock(walk->mm, pmd);
495 if (pmd_trans_huge(*pmd)) {
496 page = pmd_page(*pmd);
497 if (is_huge_zero_page(page)) {
499 split_huge_pmd(vma, pmd, addr);
504 ret = split_huge_page(page);
516 pte = pte_offset_map_lock(walk->mm, pmd, addr, &ptl);
517 for (; addr != end; pte++, addr += PAGE_SIZE) {
518 if (!pte_present(*pte))
520 page = vm_normal_page(vma, addr, *pte);
524 * vm_normal_page() filters out zero pages, but there might
525 * still be PageReserved pages to skip, perhaps in a VDSO.
527 if (PageReserved(page))
529 nid = page_to_nid(page);
530 if (node_isset(nid, *qp->nmask) == !!(flags & MPOL_MF_INVERT))
532 if (PageTransCompound(page) && PageAnon(page)) {
534 pte_unmap_unlock(pte, ptl);
536 ret = split_huge_page(page);
539 /* Failed to split -- skip. */
541 pte = pte_offset_map_lock(walk->mm, pmd,
548 migrate_page_add(page, qp->pagelist, flags);
550 pte_unmap_unlock(pte - 1, ptl);
555 static int queue_pages_hugetlb(pte_t *pte, unsigned long hmask,
556 unsigned long addr, unsigned long end,
557 struct mm_walk *walk)
559 #ifdef CONFIG_HUGETLB_PAGE
560 struct queue_pages *qp = walk->private;
561 unsigned long flags = qp->flags;
567 ptl = huge_pte_lock(hstate_vma(walk->vma), walk->mm, pte);
568 entry = huge_ptep_get(pte);
569 if (!pte_present(entry))
571 page = pte_page(entry);
572 nid = page_to_nid(page);
573 if (node_isset(nid, *qp->nmask) == !!(flags & MPOL_MF_INVERT))
575 /* With MPOL_MF_MOVE, we migrate only unshared hugepage. */
576 if (flags & (MPOL_MF_MOVE_ALL) ||
577 (flags & MPOL_MF_MOVE && page_mapcount(page) == 1))
578 isolate_huge_page(page, qp->pagelist);
587 #ifdef CONFIG_NUMA_BALANCING
589 * This is used to mark a range of virtual addresses to be inaccessible.
590 * These are later cleared by a NUMA hinting fault. Depending on these
591 * faults, pages may be migrated for better NUMA placement.
593 * This is assuming that NUMA faults are handled using PROT_NONE. If
594 * an architecture makes a different choice, it will need further
595 * changes to the core.
597 unsigned long change_prot_numa(struct vm_area_struct *vma,
598 unsigned long addr, unsigned long end)
602 nr_updated = change_protection(vma, addr, end, PAGE_NONE, 0, 1);
604 count_vm_numa_events(NUMA_PTE_UPDATES, nr_updated);
609 static unsigned long change_prot_numa(struct vm_area_struct *vma,
610 unsigned long addr, unsigned long end)
614 #endif /* CONFIG_NUMA_BALANCING */
616 static int queue_pages_test_walk(unsigned long start, unsigned long end,
617 struct mm_walk *walk)
619 struct vm_area_struct *vma = walk->vma;
620 struct queue_pages *qp = walk->private;
621 unsigned long endvma = vma->vm_end;
622 unsigned long flags = qp->flags;
624 if (!vma_migratable(vma))
629 if (vma->vm_start > start)
630 start = vma->vm_start;
632 if (!(flags & MPOL_MF_DISCONTIG_OK)) {
633 if (!vma->vm_next && vma->vm_end < end)
635 if (qp->prev && qp->prev->vm_end < vma->vm_start)
641 if (flags & MPOL_MF_LAZY) {
642 /* Similar to task_numa_work, skip inaccessible VMAs */
643 if (!is_vm_hugetlb_page(vma) &&
644 (vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE)) &&
645 !(vma->vm_flags & VM_MIXEDMAP))
646 change_prot_numa(vma, start, endvma);
650 /* queue pages from current vma */
651 if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL))
657 * Walk through page tables and collect pages to be migrated.
659 * If pages found in a given range are on a set of nodes (determined by
660 * @nodes and @flags,) it's isolated and queued to the pagelist which is
661 * passed via @private.)
664 queue_pages_range(struct mm_struct *mm, unsigned long start, unsigned long end,
665 nodemask_t *nodes, unsigned long flags,
666 struct list_head *pagelist)
668 struct queue_pages qp = {
669 .pagelist = pagelist,
674 struct mm_walk queue_pages_walk = {
675 .hugetlb_entry = queue_pages_hugetlb,
676 .pmd_entry = queue_pages_pte_range,
677 .test_walk = queue_pages_test_walk,
682 return walk_page_range(start, end, &queue_pages_walk);
686 * Apply policy to a single VMA
687 * This must be called with the mmap_sem held for writing.
689 static int vma_replace_policy(struct vm_area_struct *vma,
690 struct mempolicy *pol)
693 struct mempolicy *old;
694 struct mempolicy *new;
696 pr_debug("vma %lx-%lx/%lx vm_ops %p vm_file %p set_policy %p\n",
697 vma->vm_start, vma->vm_end, vma->vm_pgoff,
698 vma->vm_ops, vma->vm_file,
699 vma->vm_ops ? vma->vm_ops->set_policy : NULL);
705 if (vma->vm_ops && vma->vm_ops->set_policy) {
706 err = vma->vm_ops->set_policy(vma, new);
711 old = vma->vm_policy;
712 vma->vm_policy = new; /* protected by mmap_sem */
721 /* Step 2: apply policy to a range and do splits. */
722 static int mbind_range(struct mm_struct *mm, unsigned long start,
723 unsigned long end, struct mempolicy *new_pol)
725 struct vm_area_struct *next;
726 struct vm_area_struct *prev;
727 struct vm_area_struct *vma;
730 unsigned long vmstart;
733 vma = find_vma(mm, start);
734 if (!vma || vma->vm_start > start)
738 if (start > vma->vm_start)
741 for (; vma && vma->vm_start < end; prev = vma, vma = next) {
743 vmstart = max(start, vma->vm_start);
744 vmend = min(end, vma->vm_end);
746 if (mpol_equal(vma_policy(vma), new_pol))
749 pgoff = vma->vm_pgoff +
750 ((vmstart - vma->vm_start) >> PAGE_SHIFT);
751 prev = vma_merge(mm, prev, vmstart, vmend, vma->vm_flags,
752 vma->anon_vma, vma->vm_file, pgoff,
753 new_pol, vma->vm_userfaultfd_ctx);
757 if (mpol_equal(vma_policy(vma), new_pol))
759 /* vma_merge() joined vma && vma->next, case 8 */
762 if (vma->vm_start != vmstart) {
763 err = split_vma(vma->vm_mm, vma, vmstart, 1);
767 if (vma->vm_end != vmend) {
768 err = split_vma(vma->vm_mm, vma, vmend, 0);
773 err = vma_replace_policy(vma, new_pol);
782 /* Set the process memory policy */
783 static long do_set_mempolicy(unsigned short mode, unsigned short flags,
786 struct mempolicy *new, *old;
787 NODEMASK_SCRATCH(scratch);
793 new = mpol_new(mode, flags, nodes);
800 ret = mpol_set_nodemask(new, nodes, scratch);
802 task_unlock(current);
806 old = current->mempolicy;
807 current->mempolicy = new;
808 if (new && new->mode == MPOL_INTERLEAVE &&
809 nodes_weight(new->v.nodes))
810 current->il_next = first_node(new->v.nodes);
811 task_unlock(current);
815 NODEMASK_SCRATCH_FREE(scratch);
820 * Return nodemask for policy for get_mempolicy() query
822 * Called with task's alloc_lock held
824 static void get_policy_nodemask(struct mempolicy *p, nodemask_t *nodes)
827 if (p == &default_policy)
833 case MPOL_INTERLEAVE:
837 if (!(p->flags & MPOL_F_LOCAL))
838 node_set(p->v.preferred_node, *nodes);
839 /* else return empty node mask for local allocation */
846 static int lookup_node(unsigned long addr)
851 err = get_user_pages(addr & PAGE_MASK, 1, 0, 0, &p, NULL);
853 err = page_to_nid(p);
859 /* Retrieve NUMA policy */
860 static long do_get_mempolicy(int *policy, nodemask_t *nmask,
861 unsigned long addr, unsigned long flags)
864 struct mm_struct *mm = current->mm;
865 struct vm_area_struct *vma = NULL;
866 struct mempolicy *pol = current->mempolicy;
869 ~(unsigned long)(MPOL_F_NODE|MPOL_F_ADDR|MPOL_F_MEMS_ALLOWED))
872 if (flags & MPOL_F_MEMS_ALLOWED) {
873 if (flags & (MPOL_F_NODE|MPOL_F_ADDR))
875 *policy = 0; /* just so it's initialized */
877 *nmask = cpuset_current_mems_allowed;
878 task_unlock(current);
882 if (flags & MPOL_F_ADDR) {
884 * Do NOT fall back to task policy if the
885 * vma/shared policy at addr is NULL. We
886 * want to return MPOL_DEFAULT in this case.
888 down_read(&mm->mmap_sem);
889 vma = find_vma_intersection(mm, addr, addr+1);
891 up_read(&mm->mmap_sem);
894 if (vma->vm_ops && vma->vm_ops->get_policy)
895 pol = vma->vm_ops->get_policy(vma, addr);
897 pol = vma->vm_policy;
902 pol = &default_policy; /* indicates default behavior */
904 if (flags & MPOL_F_NODE) {
905 if (flags & MPOL_F_ADDR) {
906 err = lookup_node(addr);
910 } else if (pol == current->mempolicy &&
911 pol->mode == MPOL_INTERLEAVE) {
912 *policy = current->il_next;
918 *policy = pol == &default_policy ? MPOL_DEFAULT :
921 * Internal mempolicy flags must be masked off before exposing
922 * the policy to userspace.
924 *policy |= (pol->flags & MPOL_MODE_FLAGS);
928 up_read(¤t->mm->mmap_sem);
934 if (mpol_store_user_nodemask(pol)) {
935 *nmask = pol->w.user_nodemask;
938 get_policy_nodemask(pol, nmask);
939 task_unlock(current);
946 up_read(¤t->mm->mmap_sem);
950 #ifdef CONFIG_MIGRATION
954 static void migrate_page_add(struct page *page, struct list_head *pagelist,
958 * Avoid migrating a page that is shared with others.
960 if ((flags & MPOL_MF_MOVE_ALL) || page_mapcount(page) == 1) {
961 if (!isolate_lru_page(page)) {
962 list_add_tail(&page->lru, pagelist);
963 inc_zone_page_state(page, NR_ISOLATED_ANON +
964 page_is_file_cache(page));
969 static struct page *new_node_page(struct page *page, unsigned long node, int **x)
972 return alloc_huge_page_node(page_hstate(compound_head(page)),
975 return __alloc_pages_node(node, GFP_HIGHUSER_MOVABLE |
980 * Migrate pages from one node to a target node.
981 * Returns error or the number of pages not migrated.
983 static int migrate_to_node(struct mm_struct *mm, int source, int dest,
991 node_set(source, nmask);
994 * This does not "check" the range but isolates all pages that
995 * need migration. Between passing in the full user address
996 * space range and MPOL_MF_DISCONTIG_OK, this call can not fail.
998 VM_BUG_ON(!(flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)));
999 queue_pages_range(mm, mm->mmap->vm_start, mm->task_size, &nmask,
1000 flags | MPOL_MF_DISCONTIG_OK, &pagelist);
1002 if (!list_empty(&pagelist)) {
1003 err = migrate_pages(&pagelist, new_node_page, NULL, dest,
1004 MIGRATE_SYNC, MR_SYSCALL);
1006 putback_movable_pages(&pagelist);
1013 * Move pages between the two nodesets so as to preserve the physical
1014 * layout as much as possible.
1016 * Returns the number of page that could not be moved.
1018 int do_migrate_pages(struct mm_struct *mm, const nodemask_t *from,
1019 const nodemask_t *to, int flags)
1025 err = migrate_prep();
1029 down_read(&mm->mmap_sem);
1032 * Find a 'source' bit set in 'tmp' whose corresponding 'dest'
1033 * bit in 'to' is not also set in 'tmp'. Clear the found 'source'
1034 * bit in 'tmp', and return that <source, dest> pair for migration.
1035 * The pair of nodemasks 'to' and 'from' define the map.
1037 * If no pair of bits is found that way, fallback to picking some
1038 * pair of 'source' and 'dest' bits that are not the same. If the
1039 * 'source' and 'dest' bits are the same, this represents a node
1040 * that will be migrating to itself, so no pages need move.
1042 * If no bits are left in 'tmp', or if all remaining bits left
1043 * in 'tmp' correspond to the same bit in 'to', return false
1044 * (nothing left to migrate).
1046 * This lets us pick a pair of nodes to migrate between, such that
1047 * if possible the dest node is not already occupied by some other
1048 * source node, minimizing the risk of overloading the memory on a
1049 * node that would happen if we migrated incoming memory to a node
1050 * before migrating outgoing memory source that same node.
1052 * A single scan of tmp is sufficient. As we go, we remember the
1053 * most recent <s, d> pair that moved (s != d). If we find a pair
1054 * that not only moved, but what's better, moved to an empty slot
1055 * (d is not set in tmp), then we break out then, with that pair.
1056 * Otherwise when we finish scanning from_tmp, we at least have the
1057 * most recent <s, d> pair that moved. If we get all the way through
1058 * the scan of tmp without finding any node that moved, much less
1059 * moved to an empty node, then there is nothing left worth migrating.
1063 while (!nodes_empty(tmp)) {
1065 int source = NUMA_NO_NODE;
1068 for_each_node_mask(s, tmp) {
1071 * do_migrate_pages() tries to maintain the relative
1072 * node relationship of the pages established between
1073 * threads and memory areas.
1075 * However if the number of source nodes is not equal to
1076 * the number of destination nodes we can not preserve
1077 * this node relative relationship. In that case, skip
1078 * copying memory from a node that is in the destination
1081 * Example: [2,3,4] -> [3,4,5] moves everything.
1082 * [0-7] - > [3,4,5] moves only 0,1,2,6,7.
1085 if ((nodes_weight(*from) != nodes_weight(*to)) &&
1086 (node_isset(s, *to)))
1089 d = node_remap(s, *from, *to);
1093 source = s; /* Node moved. Memorize */
1096 /* dest not in remaining from nodes? */
1097 if (!node_isset(dest, tmp))
1100 if (source == NUMA_NO_NODE)
1103 node_clear(source, tmp);
1104 err = migrate_to_node(mm, source, dest, flags);
1110 up_read(&mm->mmap_sem);
1118 * Allocate a new page for page migration based on vma policy.
1119 * Start by assuming the page is mapped by the same vma as contains @start.
1120 * Search forward from there, if not. N.B., this assumes that the
1121 * list of pages handed to migrate_pages()--which is how we get here--
1122 * is in virtual address order.
1124 static struct page *new_page(struct page *page, unsigned long start, int **x)
1126 struct vm_area_struct *vma;
1127 unsigned long uninitialized_var(address);
1129 vma = find_vma(current->mm, start);
1131 address = page_address_in_vma(page, vma);
1132 if (address != -EFAULT)
1137 if (PageHuge(page)) {
1139 return alloc_huge_page_noerr(vma, address, 1);
1142 * if !vma, alloc_page_vma() will use task or system default policy
1144 return alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, address);
1148 static void migrate_page_add(struct page *page, struct list_head *pagelist,
1149 unsigned long flags)
1153 int do_migrate_pages(struct mm_struct *mm, const nodemask_t *from,
1154 const nodemask_t *to, int flags)
1159 static struct page *new_page(struct page *page, unsigned long start, int **x)
1165 static long do_mbind(unsigned long start, unsigned long len,
1166 unsigned short mode, unsigned short mode_flags,
1167 nodemask_t *nmask, unsigned long flags)
1169 struct mm_struct *mm = current->mm;
1170 struct mempolicy *new;
1173 LIST_HEAD(pagelist);
1175 if (flags & ~(unsigned long)MPOL_MF_VALID)
1177 if ((flags & MPOL_MF_MOVE_ALL) && !capable(CAP_SYS_NICE))
1180 if (start & ~PAGE_MASK)
1183 if (mode == MPOL_DEFAULT)
1184 flags &= ~MPOL_MF_STRICT;
1186 len = (len + PAGE_SIZE - 1) & PAGE_MASK;
1194 new = mpol_new(mode, mode_flags, nmask);
1196 return PTR_ERR(new);
1198 if (flags & MPOL_MF_LAZY)
1199 new->flags |= MPOL_F_MOF;
1202 * If we are using the default policy then operation
1203 * on discontinuous address spaces is okay after all
1206 flags |= MPOL_MF_DISCONTIG_OK;
1208 pr_debug("mbind %lx-%lx mode:%d flags:%d nodes:%lx\n",
1209 start, start + len, mode, mode_flags,
1210 nmask ? nodes_addr(*nmask)[0] : NUMA_NO_NODE);
1212 if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) {
1214 err = migrate_prep();
1219 NODEMASK_SCRATCH(scratch);
1221 down_write(&mm->mmap_sem);
1223 err = mpol_set_nodemask(new, nmask, scratch);
1224 task_unlock(current);
1226 up_write(&mm->mmap_sem);
1229 NODEMASK_SCRATCH_FREE(scratch);
1234 err = queue_pages_range(mm, start, end, nmask,
1235 flags | MPOL_MF_INVERT, &pagelist);
1237 err = mbind_range(mm, start, end, new);
1242 if (!list_empty(&pagelist)) {
1243 WARN_ON_ONCE(flags & MPOL_MF_LAZY);
1244 nr_failed = migrate_pages(&pagelist, new_page, NULL,
1245 start, MIGRATE_SYNC, MR_MEMPOLICY_MBIND);
1247 putback_movable_pages(&pagelist);
1250 if (nr_failed && (flags & MPOL_MF_STRICT))
1253 putback_movable_pages(&pagelist);
1255 up_write(&mm->mmap_sem);
1262 * User space interface with variable sized bitmaps for nodelists.
1265 /* Copy a node mask from user space. */
1266 static int get_nodes(nodemask_t *nodes, const unsigned long __user *nmask,
1267 unsigned long maxnode)
1270 unsigned long nlongs;
1271 unsigned long endmask;
1274 nodes_clear(*nodes);
1275 if (maxnode == 0 || !nmask)
1277 if (maxnode > PAGE_SIZE*BITS_PER_BYTE)
1280 nlongs = BITS_TO_LONGS(maxnode);
1281 if ((maxnode % BITS_PER_LONG) == 0)
1284 endmask = (1UL << (maxnode % BITS_PER_LONG)) - 1;
1286 /* When the user specified more nodes than supported just check
1287 if the non supported part is all zero. */
1288 if (nlongs > BITS_TO_LONGS(MAX_NUMNODES)) {
1289 if (nlongs > PAGE_SIZE/sizeof(long))
1291 for (k = BITS_TO_LONGS(MAX_NUMNODES); k < nlongs; k++) {
1293 if (get_user(t, nmask + k))
1295 if (k == nlongs - 1) {
1301 nlongs = BITS_TO_LONGS(MAX_NUMNODES);
1305 if (copy_from_user(nodes_addr(*nodes), nmask, nlongs*sizeof(unsigned long)))
1307 nodes_addr(*nodes)[nlongs-1] &= endmask;
1311 /* Copy a kernel node mask to user space */
1312 static int copy_nodes_to_user(unsigned long __user *mask, unsigned long maxnode,
1315 unsigned long copy = ALIGN(maxnode-1, 64) / 8;
1316 const int nbytes = BITS_TO_LONGS(MAX_NUMNODES) * sizeof(long);
1318 if (copy > nbytes) {
1319 if (copy > PAGE_SIZE)
1321 if (clear_user((char __user *)mask + nbytes, copy - nbytes))
1325 return copy_to_user(mask, nodes_addr(*nodes), copy) ? -EFAULT : 0;
1328 SYSCALL_DEFINE6(mbind, unsigned long, start, unsigned long, len,
1329 unsigned long, mode, const unsigned long __user *, nmask,
1330 unsigned long, maxnode, unsigned, flags)
1334 unsigned short mode_flags;
1336 mode_flags = mode & MPOL_MODE_FLAGS;
1337 mode &= ~MPOL_MODE_FLAGS;
1338 if (mode >= MPOL_MAX)
1340 if ((mode_flags & MPOL_F_STATIC_NODES) &&
1341 (mode_flags & MPOL_F_RELATIVE_NODES))
1343 err = get_nodes(&nodes, nmask, maxnode);
1346 return do_mbind(start, len, mode, mode_flags, &nodes, flags);
1349 /* Set the process memory policy */
1350 SYSCALL_DEFINE3(set_mempolicy, int, mode, const unsigned long __user *, nmask,
1351 unsigned long, maxnode)
1355 unsigned short flags;
1357 flags = mode & MPOL_MODE_FLAGS;
1358 mode &= ~MPOL_MODE_FLAGS;
1359 if ((unsigned int)mode >= MPOL_MAX)
1361 if ((flags & MPOL_F_STATIC_NODES) && (flags & MPOL_F_RELATIVE_NODES))
1363 err = get_nodes(&nodes, nmask, maxnode);
1366 return do_set_mempolicy(mode, flags, &nodes);
1369 SYSCALL_DEFINE4(migrate_pages, pid_t, pid, unsigned long, maxnode,
1370 const unsigned long __user *, old_nodes,
1371 const unsigned long __user *, new_nodes)
1373 const struct cred *cred = current_cred(), *tcred;
1374 struct mm_struct *mm = NULL;
1375 struct task_struct *task;
1376 nodemask_t task_nodes;
1380 NODEMASK_SCRATCH(scratch);
1385 old = &scratch->mask1;
1386 new = &scratch->mask2;
1388 err = get_nodes(old, old_nodes, maxnode);
1392 err = get_nodes(new, new_nodes, maxnode);
1396 /* Find the mm_struct */
1398 task = pid ? find_task_by_vpid(pid) : current;
1404 get_task_struct(task);
1409 * Check if this process has the right to modify the specified
1410 * process. The right exists if the process has administrative
1411 * capabilities, superuser privileges or the same
1412 * userid as the target process.
1414 tcred = __task_cred(task);
1415 if (!uid_eq(cred->euid, tcred->suid) && !uid_eq(cred->euid, tcred->uid) &&
1416 !uid_eq(cred->uid, tcred->suid) && !uid_eq(cred->uid, tcred->uid) &&
1417 !capable(CAP_SYS_NICE)) {
1424 task_nodes = cpuset_mems_allowed(task);
1425 /* Is the user allowed to access the target nodes? */
1426 if (!nodes_subset(*new, task_nodes) && !capable(CAP_SYS_NICE)) {
1431 if (!nodes_subset(*new, node_states[N_MEMORY])) {
1436 err = security_task_movememory(task);
1440 mm = get_task_mm(task);
1441 put_task_struct(task);
1448 err = do_migrate_pages(mm, old, new,
1449 capable(CAP_SYS_NICE) ? MPOL_MF_MOVE_ALL : MPOL_MF_MOVE);
1453 NODEMASK_SCRATCH_FREE(scratch);
1458 put_task_struct(task);
1464 /* Retrieve NUMA policy */
1465 SYSCALL_DEFINE5(get_mempolicy, int __user *, policy,
1466 unsigned long __user *, nmask, unsigned long, maxnode,
1467 unsigned long, addr, unsigned long, flags)
1470 int uninitialized_var(pval);
1473 if (nmask != NULL && maxnode < MAX_NUMNODES)
1476 err = do_get_mempolicy(&pval, &nodes, addr, flags);
1481 if (policy && put_user(pval, policy))
1485 err = copy_nodes_to_user(nmask, maxnode, &nodes);
1490 #ifdef CONFIG_COMPAT
1492 COMPAT_SYSCALL_DEFINE5(get_mempolicy, int __user *, policy,
1493 compat_ulong_t __user *, nmask,
1494 compat_ulong_t, maxnode,
1495 compat_ulong_t, addr, compat_ulong_t, flags)
1498 unsigned long __user *nm = NULL;
1499 unsigned long nr_bits, alloc_size;
1500 DECLARE_BITMAP(bm, MAX_NUMNODES);
1502 nr_bits = min_t(unsigned long, maxnode-1, MAX_NUMNODES);
1503 alloc_size = ALIGN(nr_bits, BITS_PER_LONG) / 8;
1506 nm = compat_alloc_user_space(alloc_size);
1508 err = sys_get_mempolicy(policy, nm, nr_bits+1, addr, flags);
1510 if (!err && nmask) {
1511 unsigned long copy_size;
1512 copy_size = min_t(unsigned long, sizeof(bm), alloc_size);
1513 err = copy_from_user(bm, nm, copy_size);
1514 /* ensure entire bitmap is zeroed */
1515 err |= clear_user(nmask, ALIGN(maxnode-1, 8) / 8);
1516 err |= compat_put_bitmap(nmask, bm, nr_bits);
1522 COMPAT_SYSCALL_DEFINE3(set_mempolicy, int, mode, compat_ulong_t __user *, nmask,
1523 compat_ulong_t, maxnode)
1526 unsigned long __user *nm = NULL;
1527 unsigned long nr_bits, alloc_size;
1528 DECLARE_BITMAP(bm, MAX_NUMNODES);
1530 nr_bits = min_t(unsigned long, maxnode-1, MAX_NUMNODES);
1531 alloc_size = ALIGN(nr_bits, BITS_PER_LONG) / 8;
1534 err = compat_get_bitmap(bm, nmask, nr_bits);
1535 nm = compat_alloc_user_space(alloc_size);
1536 err |= copy_to_user(nm, bm, alloc_size);
1542 return sys_set_mempolicy(mode, nm, nr_bits+1);
1545 COMPAT_SYSCALL_DEFINE6(mbind, compat_ulong_t, start, compat_ulong_t, len,
1546 compat_ulong_t, mode, compat_ulong_t __user *, nmask,
1547 compat_ulong_t, maxnode, compat_ulong_t, flags)
1550 unsigned long __user *nm = NULL;
1551 unsigned long nr_bits, alloc_size;
1554 nr_bits = min_t(unsigned long, maxnode-1, MAX_NUMNODES);
1555 alloc_size = ALIGN(nr_bits, BITS_PER_LONG) / 8;
1558 err = compat_get_bitmap(nodes_addr(bm), nmask, nr_bits);
1559 nm = compat_alloc_user_space(alloc_size);
1560 err |= copy_to_user(nm, nodes_addr(bm), alloc_size);
1566 return sys_mbind(start, len, mode, nm, nr_bits+1, flags);
1571 struct mempolicy *__get_vma_policy(struct vm_area_struct *vma,
1574 struct mempolicy *pol = NULL;
1577 if (vma->vm_ops && vma->vm_ops->get_policy) {
1578 pol = vma->vm_ops->get_policy(vma, addr);
1579 } else if (vma->vm_policy) {
1580 pol = vma->vm_policy;
1583 * shmem_alloc_page() passes MPOL_F_SHARED policy with
1584 * a pseudo vma whose vma->vm_ops=NULL. Take a reference
1585 * count on these policies which will be dropped by
1586 * mpol_cond_put() later
1588 if (mpol_needs_cond_ref(pol))
1597 * get_vma_policy(@vma, @addr)
1598 * @vma: virtual memory area whose policy is sought
1599 * @addr: address in @vma for shared policy lookup
1601 * Returns effective policy for a VMA at specified address.
1602 * Falls back to current->mempolicy or system default policy, as necessary.
1603 * Shared policies [those marked as MPOL_F_SHARED] require an extra reference
1604 * count--added by the get_policy() vm_op, as appropriate--to protect against
1605 * freeing by another task. It is the caller's responsibility to free the
1606 * extra reference for shared policies.
1608 static struct mempolicy *get_vma_policy(struct vm_area_struct *vma,
1611 struct mempolicy *pol = __get_vma_policy(vma, addr);
1614 pol = get_task_policy(current);
1619 bool vma_policy_mof(struct vm_area_struct *vma)
1621 struct mempolicy *pol;
1623 if (vma->vm_ops && vma->vm_ops->get_policy) {
1626 pol = vma->vm_ops->get_policy(vma, vma->vm_start);
1627 if (pol && (pol->flags & MPOL_F_MOF))
1634 pol = vma->vm_policy;
1636 pol = get_task_policy(current);
1638 return pol->flags & MPOL_F_MOF;
1641 static int apply_policy_zone(struct mempolicy *policy, enum zone_type zone)
1643 enum zone_type dynamic_policy_zone = policy_zone;
1645 BUG_ON(dynamic_policy_zone == ZONE_MOVABLE);
1648 * if policy->v.nodes has movable memory only,
1649 * we apply policy when gfp_zone(gfp) = ZONE_MOVABLE only.
1651 * policy->v.nodes is intersect with node_states[N_MEMORY].
1652 * so if the following test faile, it implies
1653 * policy->v.nodes has movable memory only.
1655 if (!nodes_intersects(policy->v.nodes, node_states[N_HIGH_MEMORY]))
1656 dynamic_policy_zone = ZONE_MOVABLE;
1658 return zone >= dynamic_policy_zone;
1662 * Return a nodemask representing a mempolicy for filtering nodes for
1665 static nodemask_t *policy_nodemask(gfp_t gfp, struct mempolicy *policy)
1667 /* Lower zones don't get a nodemask applied for MPOL_BIND */
1668 if (unlikely(policy->mode == MPOL_BIND) &&
1669 apply_policy_zone(policy, gfp_zone(gfp)) &&
1670 cpuset_nodemask_valid_mems_allowed(&policy->v.nodes))
1671 return &policy->v.nodes;
1676 /* Return a zonelist indicated by gfp for node representing a mempolicy */
1677 static struct zonelist *policy_zonelist(gfp_t gfp, struct mempolicy *policy,
1680 switch (policy->mode) {
1681 case MPOL_PREFERRED:
1682 if (!(policy->flags & MPOL_F_LOCAL))
1683 nd = policy->v.preferred_node;
1687 * Normally, MPOL_BIND allocations are node-local within the
1688 * allowed nodemask. However, if __GFP_THISNODE is set and the
1689 * current node isn't part of the mask, we use the zonelist for
1690 * the first node in the mask instead.
1692 if (unlikely(gfp & __GFP_THISNODE) &&
1693 unlikely(!node_isset(nd, policy->v.nodes)))
1694 nd = first_node(policy->v.nodes);
1699 return node_zonelist(nd, gfp);
1702 /* Do dynamic interleaving for a process */
1703 static unsigned interleave_nodes(struct mempolicy *policy)
1706 struct task_struct *me = current;
1709 next = next_node_in(nid, policy->v.nodes);
1710 if (next < MAX_NUMNODES)
1716 * Depending on the memory policy provide a node from which to allocate the
1719 unsigned int mempolicy_slab_node(void)
1721 struct mempolicy *policy;
1722 int node = numa_mem_id();
1727 policy = current->mempolicy;
1728 if (!policy || policy->flags & MPOL_F_LOCAL)
1731 switch (policy->mode) {
1732 case MPOL_PREFERRED:
1734 * handled MPOL_F_LOCAL above
1736 return policy->v.preferred_node;
1738 case MPOL_INTERLEAVE:
1739 return interleave_nodes(policy);
1743 * Follow bind policy behavior and start allocation at the
1746 struct zonelist *zonelist;
1748 enum zone_type highest_zoneidx = gfp_zone(GFP_KERNEL);
1749 zonelist = &NODE_DATA(node)->node_zonelists[0];
1750 (void)first_zones_zonelist(zonelist, highest_zoneidx,
1753 return zone ? zone->node : node;
1761 /* Do static interleaving for a VMA with known offset. */
1762 static unsigned offset_il_node(struct mempolicy *pol,
1763 struct vm_area_struct *vma, unsigned long off)
1765 unsigned nnodes = nodes_weight(pol->v.nodes);
1768 int nid = NUMA_NO_NODE;
1771 return numa_node_id();
1772 target = (unsigned int)off % nnodes;
1775 nid = next_node(nid, pol->v.nodes);
1777 } while (c <= target);
1781 /* Determine a node number for interleave */
1782 static inline unsigned interleave_nid(struct mempolicy *pol,
1783 struct vm_area_struct *vma, unsigned long addr, int shift)
1789 * for small pages, there is no difference between
1790 * shift and PAGE_SHIFT, so the bit-shift is safe.
1791 * for huge pages, since vm_pgoff is in units of small
1792 * pages, we need to shift off the always 0 bits to get
1795 BUG_ON(shift < PAGE_SHIFT);
1796 off = vma->vm_pgoff >> (shift - PAGE_SHIFT);
1797 off += (addr - vma->vm_start) >> shift;
1798 return offset_il_node(pol, vma, off);
1800 return interleave_nodes(pol);
1803 #ifdef CONFIG_HUGETLBFS
1805 * huge_zonelist(@vma, @addr, @gfp_flags, @mpol)
1806 * @vma: virtual memory area whose policy is sought
1807 * @addr: address in @vma for shared policy lookup and interleave policy
1808 * @gfp_flags: for requested zone
1809 * @mpol: pointer to mempolicy pointer for reference counted mempolicy
1810 * @nodemask: pointer to nodemask pointer for MPOL_BIND nodemask
1812 * Returns a zonelist suitable for a huge page allocation and a pointer
1813 * to the struct mempolicy for conditional unref after allocation.
1814 * If the effective policy is 'BIND, returns a pointer to the mempolicy's
1815 * @nodemask for filtering the zonelist.
1817 * Must be protected by read_mems_allowed_begin()
1819 struct zonelist *huge_zonelist(struct vm_area_struct *vma, unsigned long addr,
1820 gfp_t gfp_flags, struct mempolicy **mpol,
1821 nodemask_t **nodemask)
1823 struct zonelist *zl;
1825 *mpol = get_vma_policy(vma, addr);
1826 *nodemask = NULL; /* assume !MPOL_BIND */
1828 if (unlikely((*mpol)->mode == MPOL_INTERLEAVE)) {
1829 zl = node_zonelist(interleave_nid(*mpol, vma, addr,
1830 huge_page_shift(hstate_vma(vma))), gfp_flags);
1832 zl = policy_zonelist(gfp_flags, *mpol, numa_node_id());
1833 if ((*mpol)->mode == MPOL_BIND)
1834 *nodemask = &(*mpol)->v.nodes;
1840 * init_nodemask_of_mempolicy
1842 * If the current task's mempolicy is "default" [NULL], return 'false'
1843 * to indicate default policy. Otherwise, extract the policy nodemask
1844 * for 'bind' or 'interleave' policy into the argument nodemask, or
1845 * initialize the argument nodemask to contain the single node for
1846 * 'preferred' or 'local' policy and return 'true' to indicate presence
1847 * of non-default mempolicy.
1849 * We don't bother with reference counting the mempolicy [mpol_get/put]
1850 * because the current task is examining it's own mempolicy and a task's
1851 * mempolicy is only ever changed by the task itself.
1853 * N.B., it is the caller's responsibility to free a returned nodemask.
1855 bool init_nodemask_of_mempolicy(nodemask_t *mask)
1857 struct mempolicy *mempolicy;
1860 if (!(mask && current->mempolicy))
1864 mempolicy = current->mempolicy;
1865 switch (mempolicy->mode) {
1866 case MPOL_PREFERRED:
1867 if (mempolicy->flags & MPOL_F_LOCAL)
1868 nid = numa_node_id();
1870 nid = mempolicy->v.preferred_node;
1871 init_nodemask_of_node(mask, nid);
1876 case MPOL_INTERLEAVE:
1877 *mask = mempolicy->v.nodes;
1883 task_unlock(current);
1890 * mempolicy_nodemask_intersects
1892 * If tsk's mempolicy is "default" [NULL], return 'true' to indicate default
1893 * policy. Otherwise, check for intersection between mask and the policy
1894 * nodemask for 'bind' or 'interleave' policy. For 'perferred' or 'local'
1895 * policy, always return true since it may allocate elsewhere on fallback.
1897 * Takes task_lock(tsk) to prevent freeing of its mempolicy.
1899 bool mempolicy_nodemask_intersects(struct task_struct *tsk,
1900 const nodemask_t *mask)
1902 struct mempolicy *mempolicy;
1908 mempolicy = tsk->mempolicy;
1912 switch (mempolicy->mode) {
1913 case MPOL_PREFERRED:
1915 * MPOL_PREFERRED and MPOL_F_LOCAL are only preferred nodes to
1916 * allocate from, they may fallback to other nodes when oom.
1917 * Thus, it's possible for tsk to have allocated memory from
1922 case MPOL_INTERLEAVE:
1923 ret = nodes_intersects(mempolicy->v.nodes, *mask);
1933 /* Allocate a page in interleaved policy.
1934 Own path because it needs to do special accounting. */
1935 static struct page *alloc_page_interleave(gfp_t gfp, unsigned order,
1938 struct zonelist *zl;
1941 zl = node_zonelist(nid, gfp);
1942 page = __alloc_pages(gfp, order, zl);
1943 if (page && page_zone(page) == zonelist_zone(&zl->_zonerefs[0]))
1944 inc_zone_page_state(page, NUMA_INTERLEAVE_HIT);
1949 * alloc_pages_vma - Allocate a page for a VMA.
1952 * %GFP_USER user allocation.
1953 * %GFP_KERNEL kernel allocations,
1954 * %GFP_HIGHMEM highmem/user allocations,
1955 * %GFP_FS allocation should not call back into a file system.
1956 * %GFP_ATOMIC don't sleep.
1958 * @order:Order of the GFP allocation.
1959 * @vma: Pointer to VMA or NULL if not available.
1960 * @addr: Virtual Address of the allocation. Must be inside the VMA.
1961 * @node: Which node to prefer for allocation (modulo policy).
1962 * @hugepage: for hugepages try only the preferred node if possible
1964 * This function allocates a page from the kernel page pool and applies
1965 * a NUMA policy associated with the VMA or the current process.
1966 * When VMA is not NULL caller must hold down_read on the mmap_sem of the
1967 * mm_struct of the VMA to prevent it from going away. Should be used for
1968 * all allocations for pages that will be mapped into user space. Returns
1969 * NULL when no page can be allocated.
1972 alloc_pages_vma(gfp_t gfp, int order, struct vm_area_struct *vma,
1973 unsigned long addr, int node, bool hugepage)
1975 struct mempolicy *pol;
1977 unsigned int cpuset_mems_cookie;
1978 struct zonelist *zl;
1982 pol = get_vma_policy(vma, addr);
1983 cpuset_mems_cookie = read_mems_allowed_begin();
1985 if (pol->mode == MPOL_INTERLEAVE) {
1988 nid = interleave_nid(pol, vma, addr, PAGE_SHIFT + order);
1990 page = alloc_page_interleave(gfp, order, nid);
1994 if (unlikely(IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) && hugepage)) {
1995 int hpage_node = node;
1998 * For hugepage allocation and non-interleave policy which
1999 * allows the current node (or other explicitly preferred
2000 * node) we only try to allocate from the current/preferred
2001 * node and don't fall back to other nodes, as the cost of
2002 * remote accesses would likely offset THP benefits.
2004 * If the policy is interleave, or does not allow the current
2005 * node in its nodemask, we allocate the standard way.
2007 if (pol->mode == MPOL_PREFERRED &&
2008 !(pol->flags & MPOL_F_LOCAL))
2009 hpage_node = pol->v.preferred_node;
2011 nmask = policy_nodemask(gfp, pol);
2012 if (!nmask || node_isset(hpage_node, *nmask)) {
2014 page = __alloc_pages_node(hpage_node,
2015 gfp | __GFP_THISNODE, order);
2020 nmask = policy_nodemask(gfp, pol);
2021 zl = policy_zonelist(gfp, pol, node);
2023 page = __alloc_pages_nodemask(gfp, order, zl, nmask);
2025 if (unlikely(!page && read_mems_allowed_retry(cpuset_mems_cookie)))
2031 * alloc_pages_current - Allocate pages.
2034 * %GFP_USER user allocation,
2035 * %GFP_KERNEL kernel allocation,
2036 * %GFP_HIGHMEM highmem allocation,
2037 * %GFP_FS don't call back into a file system.
2038 * %GFP_ATOMIC don't sleep.
2039 * @order: Power of two of allocation size in pages. 0 is a single page.
2041 * Allocate a page from the kernel page pool. When not in
2042 * interrupt context and apply the current process NUMA policy.
2043 * Returns NULL when no page can be allocated.
2045 * Don't call cpuset_update_task_memory_state() unless
2046 * 1) it's ok to take cpuset_sem (can WAIT), and
2047 * 2) allocating for current task (not interrupt).
2049 struct page *alloc_pages_current(gfp_t gfp, unsigned order)
2051 struct mempolicy *pol = &default_policy;
2053 unsigned int cpuset_mems_cookie;
2055 if (!in_interrupt() && !(gfp & __GFP_THISNODE))
2056 pol = get_task_policy(current);
2059 cpuset_mems_cookie = read_mems_allowed_begin();
2062 * No reference counting needed for current->mempolicy
2063 * nor system default_policy
2065 if (pol->mode == MPOL_INTERLEAVE)
2066 page = alloc_page_interleave(gfp, order, interleave_nodes(pol));
2068 page = __alloc_pages_nodemask(gfp, order,
2069 policy_zonelist(gfp, pol, numa_node_id()),
2070 policy_nodemask(gfp, pol));
2072 if (unlikely(!page && read_mems_allowed_retry(cpuset_mems_cookie)))
2077 EXPORT_SYMBOL(alloc_pages_current);
2079 int vma_dup_policy(struct vm_area_struct *src, struct vm_area_struct *dst)
2081 struct mempolicy *pol = mpol_dup(vma_policy(src));
2084 return PTR_ERR(pol);
2085 dst->vm_policy = pol;
2090 * If mpol_dup() sees current->cpuset == cpuset_being_rebound, then it
2091 * rebinds the mempolicy its copying by calling mpol_rebind_policy()
2092 * with the mems_allowed returned by cpuset_mems_allowed(). This
2093 * keeps mempolicies cpuset relative after its cpuset moves. See
2094 * further kernel/cpuset.c update_nodemask().
2096 * current's mempolicy may be rebinded by the other task(the task that changes
2097 * cpuset's mems), so we needn't do rebind work for current task.
2100 /* Slow path of a mempolicy duplicate */
2101 struct mempolicy *__mpol_dup(struct mempolicy *old)
2103 struct mempolicy *new = kmem_cache_alloc(policy_cache, GFP_KERNEL);
2106 return ERR_PTR(-ENOMEM);
2108 /* task's mempolicy is protected by alloc_lock */
2109 if (old == current->mempolicy) {
2112 task_unlock(current);
2116 if (current_cpuset_is_being_rebound()) {
2117 nodemask_t mems = cpuset_mems_allowed(current);
2118 if (new->flags & MPOL_F_REBINDING)
2119 mpol_rebind_policy(new, &mems, MPOL_REBIND_STEP2);
2121 mpol_rebind_policy(new, &mems, MPOL_REBIND_ONCE);
2123 atomic_set(&new->refcnt, 1);
2127 /* Slow path of a mempolicy comparison */
2128 bool __mpol_equal(struct mempolicy *a, struct mempolicy *b)
2132 if (a->mode != b->mode)
2134 if (a->flags != b->flags)
2136 if (mpol_store_user_nodemask(a))
2137 if (!nodes_equal(a->w.user_nodemask, b->w.user_nodemask))
2143 case MPOL_INTERLEAVE:
2144 return !!nodes_equal(a->v.nodes, b->v.nodes);
2145 case MPOL_PREFERRED:
2146 return a->v.preferred_node == b->v.preferred_node;
2154 * Shared memory backing store policy support.
2156 * Remember policies even when nobody has shared memory mapped.
2157 * The policies are kept in Red-Black tree linked from the inode.
2158 * They are protected by the sp->lock rwlock, which should be held
2159 * for any accesses to the tree.
2163 * lookup first element intersecting start-end. Caller holds sp->lock for
2164 * reading or for writing
2166 static struct sp_node *
2167 sp_lookup(struct shared_policy *sp, unsigned long start, unsigned long end)
2169 struct rb_node *n = sp->root.rb_node;
2172 struct sp_node *p = rb_entry(n, struct sp_node, nd);
2174 if (start >= p->end)
2176 else if (end <= p->start)
2184 struct sp_node *w = NULL;
2185 struct rb_node *prev = rb_prev(n);
2188 w = rb_entry(prev, struct sp_node, nd);
2189 if (w->end <= start)
2193 return rb_entry(n, struct sp_node, nd);
2197 * Insert a new shared policy into the list. Caller holds sp->lock for
2200 static void sp_insert(struct shared_policy *sp, struct sp_node *new)
2202 struct rb_node **p = &sp->root.rb_node;
2203 struct rb_node *parent = NULL;
2208 nd = rb_entry(parent, struct sp_node, nd);
2209 if (new->start < nd->start)
2211 else if (new->end > nd->end)
2212 p = &(*p)->rb_right;
2216 rb_link_node(&new->nd, parent, p);
2217 rb_insert_color(&new->nd, &sp->root);
2218 pr_debug("inserting %lx-%lx: %d\n", new->start, new->end,
2219 new->policy ? new->policy->mode : 0);
2222 /* Find shared policy intersecting idx */
2224 mpol_shared_policy_lookup(struct shared_policy *sp, unsigned long idx)
2226 struct mempolicy *pol = NULL;
2229 if (!sp->root.rb_node)
2231 read_lock(&sp->lock);
2232 sn = sp_lookup(sp, idx, idx+1);
2234 mpol_get(sn->policy);
2237 read_unlock(&sp->lock);
2241 static void sp_free(struct sp_node *n)
2243 mpol_put(n->policy);
2244 kmem_cache_free(sn_cache, n);
2248 * mpol_misplaced - check whether current page node is valid in policy
2250 * @page: page to be checked
2251 * @vma: vm area where page mapped
2252 * @addr: virtual address where page mapped
2254 * Lookup current policy node id for vma,addr and "compare to" page's
2258 * -1 - not misplaced, page is in the right node
2259 * node - node id where the page should be
2261 * Policy determination "mimics" alloc_page_vma().
2262 * Called from fault path where we know the vma and faulting address.
2264 int mpol_misplaced(struct page *page, struct vm_area_struct *vma, unsigned long addr)
2266 struct mempolicy *pol;
2268 int curnid = page_to_nid(page);
2269 unsigned long pgoff;
2270 int thiscpu = raw_smp_processor_id();
2271 int thisnid = cpu_to_node(thiscpu);
2277 pol = get_vma_policy(vma, addr);
2278 if (!(pol->flags & MPOL_F_MOF))
2281 switch (pol->mode) {
2282 case MPOL_INTERLEAVE:
2283 BUG_ON(addr >= vma->vm_end);
2284 BUG_ON(addr < vma->vm_start);
2286 pgoff = vma->vm_pgoff;
2287 pgoff += (addr - vma->vm_start) >> PAGE_SHIFT;
2288 polnid = offset_il_node(pol, vma, pgoff);
2291 case MPOL_PREFERRED:
2292 if (pol->flags & MPOL_F_LOCAL)
2293 polnid = numa_node_id();
2295 polnid = pol->v.preferred_node;
2300 * allows binding to multiple nodes.
2301 * use current page if in policy nodemask,
2302 * else select nearest allowed node, if any.
2303 * If no allowed nodes, use current [!misplaced].
2305 if (node_isset(curnid, pol->v.nodes))
2307 (void)first_zones_zonelist(
2308 node_zonelist(numa_node_id(), GFP_HIGHUSER),
2309 gfp_zone(GFP_HIGHUSER),
2310 &pol->v.nodes, &zone);
2311 polnid = zone->node;
2318 /* Migrate the page towards the node whose CPU is referencing it */
2319 if (pol->flags & MPOL_F_MORON) {
2322 if (!should_numa_migrate_memory(current, page, curnid, thiscpu))
2326 if (curnid != polnid)
2334 static void sp_delete(struct shared_policy *sp, struct sp_node *n)
2336 pr_debug("deleting %lx-l%lx\n", n->start, n->end);
2337 rb_erase(&n->nd, &sp->root);
2341 static void sp_node_init(struct sp_node *node, unsigned long start,
2342 unsigned long end, struct mempolicy *pol)
2344 node->start = start;
2349 static struct sp_node *sp_alloc(unsigned long start, unsigned long end,
2350 struct mempolicy *pol)
2353 struct mempolicy *newpol;
2355 n = kmem_cache_alloc(sn_cache, GFP_KERNEL);
2359 newpol = mpol_dup(pol);
2360 if (IS_ERR(newpol)) {
2361 kmem_cache_free(sn_cache, n);
2364 newpol->flags |= MPOL_F_SHARED;
2365 sp_node_init(n, start, end, newpol);
2370 /* Replace a policy range. */
2371 static int shared_policy_replace(struct shared_policy *sp, unsigned long start,
2372 unsigned long end, struct sp_node *new)
2375 struct sp_node *n_new = NULL;
2376 struct mempolicy *mpol_new = NULL;
2380 write_lock(&sp->lock);
2381 n = sp_lookup(sp, start, end);
2382 /* Take care of old policies in the same range. */
2383 while (n && n->start < end) {
2384 struct rb_node *next = rb_next(&n->nd);
2385 if (n->start >= start) {
2391 /* Old policy spanning whole new range. */
2396 *mpol_new = *n->policy;
2397 atomic_set(&mpol_new->refcnt, 1);
2398 sp_node_init(n_new, end, n->end, mpol_new);
2400 sp_insert(sp, n_new);
2409 n = rb_entry(next, struct sp_node, nd);
2413 write_unlock(&sp->lock);
2420 kmem_cache_free(sn_cache, n_new);
2425 write_unlock(&sp->lock);
2427 n_new = kmem_cache_alloc(sn_cache, GFP_KERNEL);
2430 mpol_new = kmem_cache_alloc(policy_cache, GFP_KERNEL);
2437 * mpol_shared_policy_init - initialize shared policy for inode
2438 * @sp: pointer to inode shared policy
2439 * @mpol: struct mempolicy to install
2441 * Install non-NULL @mpol in inode's shared policy rb-tree.
2442 * On entry, the current task has a reference on a non-NULL @mpol.
2443 * This must be released on exit.
2444 * This is called at get_inode() calls and we can use GFP_KERNEL.
2446 void mpol_shared_policy_init(struct shared_policy *sp, struct mempolicy *mpol)
2450 sp->root = RB_ROOT; /* empty tree == default mempolicy */
2451 rwlock_init(&sp->lock);
2454 struct vm_area_struct pvma;
2455 struct mempolicy *new;
2456 NODEMASK_SCRATCH(scratch);
2460 /* contextualize the tmpfs mount point mempolicy */
2461 new = mpol_new(mpol->mode, mpol->flags, &mpol->w.user_nodemask);
2463 goto free_scratch; /* no valid nodemask intersection */
2466 ret = mpol_set_nodemask(new, &mpol->w.user_nodemask, scratch);
2467 task_unlock(current);
2471 /* Create pseudo-vma that contains just the policy */
2472 memset(&pvma, 0, sizeof(struct vm_area_struct));
2473 pvma.vm_end = TASK_SIZE; /* policy covers entire file */
2474 mpol_set_shared_policy(sp, &pvma, new); /* adds ref */
2477 mpol_put(new); /* drop initial ref */
2479 NODEMASK_SCRATCH_FREE(scratch);
2481 mpol_put(mpol); /* drop our incoming ref on sb mpol */
2485 int mpol_set_shared_policy(struct shared_policy *info,
2486 struct vm_area_struct *vma, struct mempolicy *npol)
2489 struct sp_node *new = NULL;
2490 unsigned long sz = vma_pages(vma);
2492 pr_debug("set_shared_policy %lx sz %lu %d %d %lx\n",
2494 sz, npol ? npol->mode : -1,
2495 npol ? npol->flags : -1,
2496 npol ? nodes_addr(npol->v.nodes)[0] : NUMA_NO_NODE);
2499 new = sp_alloc(vma->vm_pgoff, vma->vm_pgoff + sz, npol);
2503 err = shared_policy_replace(info, vma->vm_pgoff, vma->vm_pgoff+sz, new);
2509 /* Free a backing policy store on inode delete. */
2510 void mpol_free_shared_policy(struct shared_policy *p)
2513 struct rb_node *next;
2515 if (!p->root.rb_node)
2517 write_lock(&p->lock);
2518 next = rb_first(&p->root);
2520 n = rb_entry(next, struct sp_node, nd);
2521 next = rb_next(&n->nd);
2524 write_unlock(&p->lock);
2527 #ifdef CONFIG_NUMA_BALANCING
2528 static int __initdata numabalancing_override;
2530 static void __init check_numabalancing_enable(void)
2532 bool numabalancing_default = false;
2534 if (IS_ENABLED(CONFIG_NUMA_BALANCING_DEFAULT_ENABLED))
2535 numabalancing_default = true;
2537 /* Parsed by setup_numabalancing. override == 1 enables, -1 disables */
2538 if (numabalancing_override)
2539 set_numabalancing_state(numabalancing_override == 1);
2541 if (num_online_nodes() > 1 && !numabalancing_override) {
2542 pr_info("%s automatic NUMA balancing. Configure with numa_balancing= or the kernel.numa_balancing sysctl\n",
2543 numabalancing_default ? "Enabling" : "Disabling");
2544 set_numabalancing_state(numabalancing_default);
2548 static int __init setup_numabalancing(char *str)
2554 if (!strcmp(str, "enable")) {
2555 numabalancing_override = 1;
2557 } else if (!strcmp(str, "disable")) {
2558 numabalancing_override = -1;
2563 pr_warn("Unable to parse numa_balancing=\n");
2567 __setup("numa_balancing=", setup_numabalancing);
2569 static inline void __init check_numabalancing_enable(void)
2572 #endif /* CONFIG_NUMA_BALANCING */
2574 /* assumes fs == KERNEL_DS */
2575 void __init numa_policy_init(void)
2577 nodemask_t interleave_nodes;
2578 unsigned long largest = 0;
2579 int nid, prefer = 0;
2581 policy_cache = kmem_cache_create("numa_policy",
2582 sizeof(struct mempolicy),
2583 0, SLAB_PANIC, NULL);
2585 sn_cache = kmem_cache_create("shared_policy_node",
2586 sizeof(struct sp_node),
2587 0, SLAB_PANIC, NULL);
2589 for_each_node(nid) {
2590 preferred_node_policy[nid] = (struct mempolicy) {
2591 .refcnt = ATOMIC_INIT(1),
2592 .mode = MPOL_PREFERRED,
2593 .flags = MPOL_F_MOF | MPOL_F_MORON,
2594 .v = { .preferred_node = nid, },
2599 * Set interleaving policy for system init. Interleaving is only
2600 * enabled across suitably sized nodes (default is >= 16MB), or
2601 * fall back to the largest node if they're all smaller.
2603 nodes_clear(interleave_nodes);
2604 for_each_node_state(nid, N_MEMORY) {
2605 unsigned long total_pages = node_present_pages(nid);
2607 /* Preserve the largest node */
2608 if (largest < total_pages) {
2609 largest = total_pages;
2613 /* Interleave this node? */
2614 if ((total_pages << PAGE_SHIFT) >= (16 << 20))
2615 node_set(nid, interleave_nodes);
2618 /* All too small, use the largest */
2619 if (unlikely(nodes_empty(interleave_nodes)))
2620 node_set(prefer, interleave_nodes);
2622 if (do_set_mempolicy(MPOL_INTERLEAVE, 0, &interleave_nodes))
2623 pr_err("%s: interleaving failed\n", __func__);
2625 check_numabalancing_enable();
2628 /* Reset policy of current process to default */
2629 void numa_default_policy(void)
2631 do_set_mempolicy(MPOL_DEFAULT, 0, NULL);
2635 * Parse and format mempolicy from/to strings
2639 * "local" is implemented internally by MPOL_PREFERRED with MPOL_F_LOCAL flag.
2641 static const char * const policy_modes[] =
2643 [MPOL_DEFAULT] = "default",
2644 [MPOL_PREFERRED] = "prefer",
2645 [MPOL_BIND] = "bind",
2646 [MPOL_INTERLEAVE] = "interleave",
2647 [MPOL_LOCAL] = "local",
2653 * mpol_parse_str - parse string to mempolicy, for tmpfs mpol mount option.
2654 * @str: string containing mempolicy to parse
2655 * @mpol: pointer to struct mempolicy pointer, returned on success.
2658 * <mode>[=<flags>][:<nodelist>]
2660 * On success, returns 0, else 1
2662 int mpol_parse_str(char *str, struct mempolicy **mpol)
2664 struct mempolicy *new = NULL;
2665 unsigned short mode;
2666 unsigned short mode_flags;
2668 char *nodelist = strchr(str, ':');
2669 char *flags = strchr(str, '=');
2673 /* NUL-terminate mode or flags string */
2675 if (nodelist_parse(nodelist, nodes))
2677 if (!nodes_subset(nodes, node_states[N_MEMORY]))
2683 *flags++ = '\0'; /* terminate mode string */
2685 for (mode = 0; mode < MPOL_MAX; mode++) {
2686 if (!strcmp(str, policy_modes[mode])) {
2690 if (mode >= MPOL_MAX)
2694 case MPOL_PREFERRED:
2696 * Insist on a nodelist of one node only
2699 char *rest = nodelist;
2700 while (isdigit(*rest))
2706 case MPOL_INTERLEAVE:
2708 * Default to online nodes with memory if no nodelist
2711 nodes = node_states[N_MEMORY];
2715 * Don't allow a nodelist; mpol_new() checks flags
2719 mode = MPOL_PREFERRED;
2723 * Insist on a empty nodelist
2730 * Insist on a nodelist
2739 * Currently, we only support two mutually exclusive
2742 if (!strcmp(flags, "static"))
2743 mode_flags |= MPOL_F_STATIC_NODES;
2744 else if (!strcmp(flags, "relative"))
2745 mode_flags |= MPOL_F_RELATIVE_NODES;
2750 new = mpol_new(mode, mode_flags, &nodes);
2755 * Save nodes for mpol_to_str() to show the tmpfs mount options
2756 * for /proc/mounts, /proc/pid/mounts and /proc/pid/mountinfo.
2758 if (mode != MPOL_PREFERRED)
2759 new->v.nodes = nodes;
2761 new->v.preferred_node = first_node(nodes);
2763 new->flags |= MPOL_F_LOCAL;
2766 * Save nodes for contextualization: this will be used to "clone"
2767 * the mempolicy in a specific context [cpuset] at a later time.
2769 new->w.user_nodemask = nodes;
2774 /* Restore string for error message */
2783 #endif /* CONFIG_TMPFS */
2786 * mpol_to_str - format a mempolicy structure for printing
2787 * @buffer: to contain formatted mempolicy string
2788 * @maxlen: length of @buffer
2789 * @pol: pointer to mempolicy to be formatted
2791 * Convert @pol into a string. If @buffer is too short, truncate the string.
2792 * Recommend a @maxlen of at least 32 for the longest mode, "interleave", the
2793 * longest flag, "relative", and to display at least a few node ids.
2795 void mpol_to_str(char *buffer, int maxlen, struct mempolicy *pol)
2798 nodemask_t nodes = NODE_MASK_NONE;
2799 unsigned short mode = MPOL_DEFAULT;
2800 unsigned short flags = 0;
2802 if (pol && pol != &default_policy && !(pol->flags & MPOL_F_MORON)) {
2810 case MPOL_PREFERRED:
2811 if (flags & MPOL_F_LOCAL)
2814 node_set(pol->v.preferred_node, nodes);
2817 case MPOL_INTERLEAVE:
2818 nodes = pol->v.nodes;
2822 snprintf(p, maxlen, "unknown");
2826 p += snprintf(p, maxlen, "%s", policy_modes[mode]);
2828 if (flags & MPOL_MODE_FLAGS) {
2829 p += snprintf(p, buffer + maxlen - p, "=");
2832 * Currently, the only defined flags are mutually exclusive
2834 if (flags & MPOL_F_STATIC_NODES)
2835 p += snprintf(p, buffer + maxlen - p, "static");
2836 else if (flags & MPOL_F_RELATIVE_NODES)
2837 p += snprintf(p, buffer + maxlen - p, "relative");
2840 if (!nodes_empty(nodes))
2841 p += scnprintf(p, buffer + maxlen - p, ":%*pbl",
2842 nodemask_pr_args(&nodes));