block: shrink struct bio down to 2 cache lines again

Commit bcf2843b3f8f added ->bi_error to cleanup the error passing
for struct bio, but that ended up adding 4 bytes and a 4 byte hole
to the size of struct bio. For a clean config, that bumped it from
128 bytes, to 136 bytes, on x86-64.

The ->bi_flags member is currently an unsigned long, but it fits
easily within an int. Change it to an unsigned int, adjust the
the pool offset code, and move ->bi_error into the new hole. Then
we end up with a 128 byte bio again.

Change the bio flag set/clear to use cmpxchg to ensure we don't
lose any flags when manipulating them.

Signed-off-by: Jens Axboe <axboe@fb.com>

diff --git a/include/linux/bio.h b/include/linux/bio.h
index 986e6e1..b7892a1 100644 (file)
--- a/include/linux/bio.h
+++ b/include/linux/bio.h
@@ -306,17 +306,17 @@ static inline void bio_cnt_set(struct bio *bio, unsigned int count)
 
 static inline bool bio_flagged(struct bio *bio, unsigned int bit)
 {
-       return (bio->bi_flags & (1UL << bit)) != 0;
+       return (bio->bi_flags & (1U << bit)) != 0;
 }
 
 static inline void bio_set_flag(struct bio *bio, unsigned int bit)
 {
-       bio->bi_flags |= (1UL << bit);
+       bio->bi_flags |= (1U << bit);
 }
 
 static inline void bio_clear_flag(struct bio *bio, unsigned int bit)
 {
-       bio->bi_flags &= ~(1UL << bit);
+       bio->bi_flags &= ~(1U << bit);
 }
 
 enum bip_flags {

diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h
index a765a50..4b7b4eb 100644 (file)
--- a/include/linux/blk_types.h
+++ b/include/linux/blk_types.h
@@ -46,14 +46,14 @@ struct bvec_iter {
 struct bio {
        struct bio              *bi_next;       /* request queue link */
        struct block_device     *bi_bdev;
-       unsigned long           bi_flags;       /* status, command, etc */
+       unsigned int            bi_flags;       /* status, command, etc */
+       int                     bi_error;
        unsigned long           bi_rw;          /* bottom bits READ/WRITE,
                                                 * top bits priority
                                                 */
 
        struct bvec_iter        bi_iter;
 
-       int                     bi_error;
        /* Number of segments in this BIO after
         * physical address coalescing is performed.
         */
@@ -134,7 +134,7 @@ struct bio {
  */
 #define BIO_POOL_BITS          (4)
 #define BIO_POOL_NONE          ((1UL << BIO_POOL_BITS) - 1)
-#define BIO_POOL_OFFSET                (BITS_PER_LONG - BIO_POOL_BITS)
+#define BIO_POOL_OFFSET                (32 - BIO_POOL_BITS)
 #define BIO_POOL_MASK          (1UL << BIO_POOL_OFFSET)
 #define BIO_POOL_IDX(bio)      ((bio)->bi_flags >> BIO_POOL_OFFSET)