Merge branch 'upstream' of git://git.linux-mips.org/pub/scm/ralf/upstream-linus

[cascardo/linux.git] / include / linux / list_lru.h

diff --git a/include/linux/list_lru.h b/include/linux/list_lru.h
index f343453..2a6b994 100644 (file)
--- a/include/linux/list_lru.h
+++ b/include/linux/list_lru.h
@@ -9,6 +9,9 @@
 
 #include <linux/list.h>
 #include <linux/nodemask.h>
+#include <linux/shrinker.h>
+
+struct mem_cgroup;
 
 /* list_lru_walk_cb has to always return one of those */
 enum lru_status {
@@ -21,24 +24,45 @@ enum lru_status {
                                   internally, but has to return locked. */
 };
 
-struct list_lru_node {
-       spinlock_t              lock;
+struct list_lru_one {
        struct list_head        list;
-       /* kept as signed so we can catch imbalance bugs */
+       /* may become negative during memcg reparenting */
        long                    nr_items;
+};
+
+struct list_lru_memcg {
+       /* array of per cgroup lists, indexed by memcg_cache_id */
+       struct list_lru_one     *lru[0];
+};
+
+struct list_lru_node {
+       /* protects all lists on the node, including per cgroup */
+       spinlock_t              lock;
+       /* global list, used for the root cgroup in cgroup aware lrus */
+       struct list_lru_one     lru;
+#ifdef CONFIG_MEMCG_KMEM
+       /* for cgroup aware lrus points to per cgroup lists, otherwise NULL */
+       struct list_lru_memcg   *memcg_lrus;
+#endif
 } ____cacheline_aligned_in_smp;
 
 struct list_lru {
        struct list_lru_node    *node;
-       nodemask_t              active_nodes;
+#ifdef CONFIG_MEMCG_KMEM
+       struct list_head        list;
+#endif
 };
 
 void list_lru_destroy(struct list_lru *lru);
-int list_lru_init_key(struct list_lru *lru, struct lock_class_key *key);
-static inline int list_lru_init(struct list_lru *lru)
-{
-       return list_lru_init_key(lru, NULL);
-}
+int __list_lru_init(struct list_lru *lru, bool memcg_aware,
+                   struct lock_class_key *key);
+
+#define list_lru_init(lru)             __list_lru_init((lru), false, NULL)
+#define list_lru_init_key(lru, key)    __list_lru_init((lru), false, (key))
+#define list_lru_init_memcg(lru)       __list_lru_init((lru), true, NULL)
+
+int memcg_update_all_list_lrus(int num_memcgs);
+void memcg_drain_all_list_lrus(int src_idx, int dst_idx);
 
 /**
  * list_lru_add: add an element to the lru list's tail
@@ -72,32 +96,48 @@ bool list_lru_add(struct list_lru *lru, struct list_head *item);
 bool list_lru_del(struct list_lru *lru, struct list_head *item);
 
 /**
- * list_lru_count_node: return the number of objects currently held by @lru
+ * list_lru_count_one: return the number of objects currently held by @lru
  * @lru: the lru pointer.
  * @nid: the node id to count from.
+ * @memcg: the cgroup to count from.
  *
  * Always return a non-negative number, 0 for empty lists. There is no
  * guarantee that the list is not updated while the count is being computed.
  * Callers that want such a guarantee need to provide an outer lock.
  */
+unsigned long list_lru_count_one(struct list_lru *lru,
+                                int nid, struct mem_cgroup *memcg);
 unsigned long list_lru_count_node(struct list_lru *lru, int nid);
+
+static inline unsigned long list_lru_shrink_count(struct list_lru *lru,
+                                                 struct shrink_control *sc)
+{
+       return list_lru_count_one(lru, sc->nid, sc->memcg);
+}
+
 static inline unsigned long list_lru_count(struct list_lru *lru)
 {
        long count = 0;
        int nid;
 
-       for_each_node_mask(nid, lru->active_nodes)
+       for_each_node_state(nid, N_NORMAL_MEMORY)
                count += list_lru_count_node(lru, nid);
 
        return count;
 }
 
-typedef enum lru_status
-(*list_lru_walk_cb)(struct list_head *item, spinlock_t *lock, void *cb_arg);
+void list_lru_isolate(struct list_lru_one *list, struct list_head *item);
+void list_lru_isolate_move(struct list_lru_one *list, struct list_head *item,
+                          struct list_head *head);
+
+typedef enum lru_status (*list_lru_walk_cb)(struct list_head *item,
+               struct list_lru_one *list, spinlock_t *lock, void *cb_arg);
+
 /**
- * list_lru_walk_node: walk a list_lru, isolating and disposing freeable items.
+ * list_lru_walk_one: walk a list_lru, isolating and disposing freeable items.
  * @lru: the lru pointer.
  * @nid: the node id to scan from.
+ * @memcg: the cgroup to scan from.
  * @isolate: callback function that is resposible for deciding what to do with
  *  the item currently being scanned
  * @cb_arg: opaque type that will be passed to @isolate
@@ -115,10 +155,22 @@ typedef enum lru_status
  *
  * Return value: the number of objects effectively removed from the LRU.
  */
+unsigned long list_lru_walk_one(struct list_lru *lru,
+                               int nid, struct mem_cgroup *memcg,
+                               list_lru_walk_cb isolate, void *cb_arg,
+                               unsigned long *nr_to_walk);
 unsigned long list_lru_walk_node(struct list_lru *lru, int nid,
                                 list_lru_walk_cb isolate, void *cb_arg,
                                 unsigned long *nr_to_walk);
 
+static inline unsigned long
+list_lru_shrink_walk(struct list_lru *lru, struct shrink_control *sc,
+                    list_lru_walk_cb isolate, void *cb_arg)
+{
+       return list_lru_walk_one(lru, sc->nid, sc->memcg, isolate, cb_arg,
+                                &sc->nr_to_scan);
+}
+
 static inline unsigned long
 list_lru_walk(struct list_lru *lru, list_lru_walk_cb isolate,
              void *cb_arg, unsigned long nr_to_walk)
@@ -126,7 +178,7 @@ list_lru_walk(struct list_lru *lru, list_lru_walk_cb isolate,
        long isolated = 0;
        int nid;
 
-       for_each_node_mask(nid, lru->active_nodes) {
+       for_each_node_state(nid, N_NORMAL_MEMORY) {
                isolated += list_lru_walk_node(lru, nid, isolate,
                                               cb_arg, &nr_to_walk);
                if (nr_to_walk <= 0)