mm: make lru_add_drain_all() selective

[cascardo/linux.git] / include / linux / swap.h

diff --git a/include/linux/swap.h b/include/linux/swap.h
index d95cde5..46ba0c6 100644 (file)
--- a/include/linux/swap.h
+++ b/include/linux/swap.h
@@ -181,6 +181,33 @@ enum {
 #define COUNT_CONTINUED        0x80    /* See swap_map continuation for full count */
 #define SWAP_MAP_SHMEM 0xbf    /* Owned by shmem/tmpfs, in first swap_map */
 
+/*
+ * We use this to track usage of a cluster. A cluster is a block of swap disk
+ * space with SWAPFILE_CLUSTER pages long and naturally aligns in disk. All
+ * free clusters are organized into a list. We fetch an entry from the list to
+ * get a free cluster.
+ *
+ * The data field stores next cluster if the cluster is free or cluster usage
+ * counter otherwise. The flags field determines if a cluster is free. This is
+ * protected by swap_info_struct.lock.
+ */
+struct swap_cluster_info {
+       unsigned int data:24;
+       unsigned int flags:8;
+};
+#define CLUSTER_FLAG_FREE 1 /* This cluster is free */
+#define CLUSTER_FLAG_NEXT_NULL 2 /* This cluster has no next cluster */
+
+/*
+ * We assign a cluster to each CPU, so each CPU can allocate swap entry from
+ * its own cluster and swapout sequentially. The purpose is to optimize swapout
+ * throughput.
+ */
+struct percpu_cluster {
+       struct swap_cluster_info index; /* Current cluster index */
+       unsigned int next; /* Likely next allocation offset */
+};
+
 /*
  * The in-memory structure used to track swap areas.
  */
@@ -191,14 +218,16 @@ struct swap_info_struct {
        signed char     next;           /* next type on the swap list */
        unsigned int    max;            /* extent of the swap_map */
        unsigned char *swap_map;        /* vmalloc'ed array of usage counts */
+       struct swap_cluster_info *cluster_info; /* cluster info. Only for SSD */
+       struct swap_cluster_info free_cluster_head; /* free cluster list head */
+       struct swap_cluster_info free_cluster_tail; /* free cluster list tail */
        unsigned int lowest_bit;        /* index of first free in swap_map */
        unsigned int highest_bit;       /* index of last free in swap_map */
        unsigned int pages;             /* total of usable pages of swap */
        unsigned int inuse_pages;       /* number of those currently in use */
        unsigned int cluster_next;      /* likely index for next allocation */
        unsigned int cluster_nr;        /* countdown to next cluster search */
-       unsigned int lowest_alloc;      /* while preparing discard cluster */
-       unsigned int highest_alloc;     /* while preparing discard cluster */
+       struct percpu_cluster __percpu *percpu_cluster; /* per cpu's swap location */
        struct swap_extent *curr_swap_extent;
        struct swap_extent first_swap_extent;
        struct block_device *bdev;      /* swap device or bdev of swap file */
@@ -212,14 +241,18 @@ struct swap_info_struct {
                                         * protect map scan related fields like
                                         * swap_map, lowest_bit, highest_bit,
                                         * inuse_pages, cluster_next,
-                                        * cluster_nr, lowest_alloc and
-                                        * highest_alloc. other fields are only
-                                        * changed at swapon/swapoff, so are
-                                        * protected by swap_lock. changing
-                                        * flags need hold this lock and
-                                        * swap_lock. If both locks need hold,
-                                        * hold swap_lock first.
+                                        * cluster_nr, lowest_alloc,
+                                        * highest_alloc, free/discard cluster
+                                        * list. other fields are only changed
+                                        * at swapon/swapoff, so are protected
+                                        * by swap_lock. changing flags need
+                                        * hold this lock and swap_lock. If
+                                        * both locks need hold, hold swap_lock
+                                        * first.
                                         */
+       struct work_struct discard_work; /* discard worker */
+       struct swap_cluster_info discard_cluster_head; /* list head of discard clusters */
+       struct swap_cluster_info discard_cluster_tail; /* list tail of discard clusters */
 };
 
 struct swap_list_t {
@@ -247,7 +280,7 @@ extern void activate_page(struct page *);
 extern void mark_page_accessed(struct page *);
 extern void lru_add_drain(void);
 extern void lru_add_drain_cpu(int cpu);
-extern int lru_add_drain_all(void);
+extern void lru_add_drain_all(void);
 extern void rotate_reclaimable_page(struct page *page);
 extern void deactivate_page(struct page *page);
 extern void swap_setup(void);
@@ -414,6 +447,7 @@ mem_cgroup_uncharge_swapcache(struct page *page, swp_entry_t ent, bool swapout)
 
 #else /* CONFIG_SWAP */
 
+#define swap_address_space(entry)              (NULL)
 #define get_nr_swap_pages()                    0L
 #define total_swap_pages                       0L
 #define total_swapcache_pages()                        0UL