mm: change anon_vma linking to fix multi-process server scalability issue

[cascardo/linux.git] / include / linux / mm_types.h

diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index 36f9627..048b462 100644 (file)
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -24,12 +24,6 @@ struct address_space;
 
 #define USE_SPLIT_PTLOCKS      (NR_CPUS >= CONFIG_SPLIT_PTLOCK_CPUS)
 
-#if USE_SPLIT_PTLOCKS
-typedef atomic_long_t mm_counter_t;
-#else  /* !USE_SPLIT_PTLOCKS */
-typedef unsigned long mm_counter_t;
-#endif /* !USE_SPLIT_PTLOCKS */
-
 /*
  * Each physical page in the system has a struct page associated with
  * it to keep track of whatever it is we are using the page for at the
@@ -169,7 +163,8 @@ struct vm_area_struct {
         * can only be in the i_mmap tree.  An anonymous MAP_PRIVATE, stack
         * or brk vma (with NULL file) can only be in an anon_vma list.
         */
-       struct list_head anon_vma_node; /* Serialized by anon_vma->lock */
+       struct list_head anon_vma_chain; /* Serialized by mmap_sem &
+                                         * page_table_lock */
        struct anon_vma *anon_vma;      /* Serialized by page_table_lock */
 
        /* Function pointers to deal with this struct. */
@@ -201,6 +196,29 @@ struct core_state {
        struct completion startup;
 };
 
+enum {
+       MM_FILEPAGES,
+       MM_ANONPAGES,
+       MM_SWAPENTS,
+       NR_MM_COUNTERS
+};
+
+#if USE_SPLIT_PTLOCKS
+#define SPLIT_RSS_COUNTING
+struct mm_rss_stat {
+       atomic_long_t count[NR_MM_COUNTERS];
+};
+/* per-thread cached information, */
+struct task_rss_stat {
+       int events;     /* for synchronization threshold */
+       int count[NR_MM_COUNTERS];
+};
+#else  /* !USE_SPLIT_PTLOCKS */
+struct mm_rss_stat {
+       unsigned long count[NR_MM_COUNTERS];
+};
+#endif /* !USE_SPLIT_PTLOCKS */
+
 struct mm_struct {
        struct vm_area_struct * mmap;           /* list of VMAs */
        struct rb_root mm_rb;
@@ -227,11 +245,6 @@ struct mm_struct {
                                                 * by mmlist_lock
                                                 */
 
-       /* Special counters, in some configurations protected by the
-        * page_table_lock, in other configurations by being atomic.
-        */
-       mm_counter_t _file_rss;
-       mm_counter_t _anon_rss;
 
        unsigned long hiwater_rss;      /* High-watermark of RSS usage */
        unsigned long hiwater_vm;       /* High-water virtual memory usage */
@@ -244,6 +257,12 @@ struct mm_struct {
 
        unsigned long saved_auxv[AT_VECTOR_SIZE]; /* for /proc/PID/auxv */
 
+       /*
+        * Special counters, in some configurations protected by the
+        * page_table_lock, in other configurations by being atomic.
+        */
+       struct mm_rss_stat rss_stat;
+
        struct linux_binfmt *binfmt;
 
        cpumask_t cpu_vm_mask;