cfq-iosched: Make IO merge related stats per cpu

Make BLKIO_STAT_MERGED per cpu hence gettring rid of need of taking
blkg->stats_lock.

Signed-off-by: Vivek Goyal <vgoyal@redhat.com>
Signed-off-by: Jens Axboe <jaxboe@fusionio.com>

diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c
index e41cc6f..07371cf 100644 (file)
--- a/block/blk-cgroup.c
+++ b/block/blk-cgroup.c
@@ -441,15 +441,27 @@ void blkiocg_update_completion_stats(struct blkio_group *blkg,
 }
 EXPORT_SYMBOL_GPL(blkiocg_update_completion_stats);
 
+/*  Merged stats are per cpu.  */
 void blkiocg_update_io_merged_stats(struct blkio_group *blkg, bool direction,
                                        bool sync)
 {
+       struct blkio_group_stats_cpu *stats_cpu;
        unsigned long flags;
 
-       spin_lock_irqsave(&blkg->stats_lock, flags);
-       blkio_add_stat(blkg->stats.stat_arr[BLKIO_STAT_MERGED], 1, direction,
-                       sync);
-       spin_unlock_irqrestore(&blkg->stats_lock, flags);
+       /*
+        * Disabling interrupts to provide mutual exclusion between two
+        * writes on same cpu. It probably is not needed for 64bit. Not
+        * optimizing that case yet.
+        */
+       local_irq_save(flags);
+
+       stats_cpu = this_cpu_ptr(blkg->stats_cpu);
+
+       u64_stats_update_begin(&stats_cpu->syncp);
+       blkio_add_stat(stats_cpu->stat_arr_cpu[BLKIO_STAT_CPU_MERGED], 1,
+                               direction, sync);
+       u64_stats_update_end(&stats_cpu->syncp);
+       local_irq_restore(flags);
 }
 EXPORT_SYMBOL_GPL(blkiocg_update_io_merged_stats);
 
@@ -1244,7 +1256,7 @@ static int blkiocg_file_read_map(struct cgroup *cgrp, struct cftype *cft,
                                                BLKIO_STAT_WAIT_TIME, 1, 0);
                case BLKIO_PROP_io_merged:
                        return blkio_read_blkg_stats(blkcg, cft, cb,
-                                               BLKIO_STAT_MERGED, 1, 0);
+                                               BLKIO_STAT_CPU_MERGED, 1, 1);
                case BLKIO_PROP_io_queued:
                        return blkio_read_blkg_stats(blkcg, cft, cb,
                                                BLKIO_STAT_QUEUED, 1, 0);

diff --git a/block/blk-cgroup.h b/block/blk-cgroup.h
index 2622267..a71d290 100644 (file)
--- a/block/blk-cgroup.h
+++ b/block/blk-cgroup.h
@@ -39,8 +39,6 @@ enum stat_type {
        BLKIO_STAT_SERVICE_TIME = 0,
        /* Total time spent waiting in scheduler queue in ns */
        BLKIO_STAT_WAIT_TIME,
-       /* Number of IOs merged */
-       BLKIO_STAT_MERGED,
        /* Number of IOs queued up */
        BLKIO_STAT_QUEUED,
        /* All the single valued stats go below this */
@@ -63,6 +61,8 @@ enum stat_type_cpu {
        BLKIO_STAT_CPU_SERVICE_BYTES,
        /* Total IOs serviced, post merge */
        BLKIO_STAT_CPU_SERVICED,
+       /* Number of IOs merged */
+       BLKIO_STAT_CPU_MERGED,
        BLKIO_STAT_CPU_NR
 };