mm/page_alloc.c: remove unlikely() from the current_order test

[cascardo/linux.git] / mm / page_alloc.c

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 327516b..b5855e5 100644 (file)
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -1046,7 +1046,7 @@ __rmqueue_fallback(struct zone *zone, int order, int start_migratetype)
                         * MIGRATE_CMA areas.
                         */
                        if (!is_migrate_cma(migratetype) &&
-                           (unlikely(current_order >= pageblock_order / 2) ||
+                           (current_order >= pageblock_order / 2 ||
                             start_migratetype == MIGRATE_RECLAIMABLE ||
                             page_group_by_mobility_disabled)) {
                                int pages;
@@ -3153,12 +3153,10 @@ static void zoneref_set_zone(struct zone *zone, struct zoneref *zoneref)
  * Add all populated zones of a node to the zonelist.
  */
 static int build_zonelists_node(pg_data_t *pgdat, struct zonelist *zonelist,
-                               int nr_zones, enum zone_type zone_type)
+                               int nr_zones)
 {
        struct zone *zone;
-
-       BUG_ON(zone_type >= MAX_NR_ZONES);
-       zone_type++;
+       enum zone_type zone_type = MAX_NR_ZONES;
 
        do {
                zone_type--;
@@ -3168,8 +3166,8 @@ static int build_zonelists_node(pg_data_t *pgdat, struct zonelist *zonelist,
                                &zonelist->_zonerefs[nr_zones++]);
                        check_highest_zone(zone_type);
                }
-
        } while (zone_type);
+
        return nr_zones;
 }
 
@@ -3363,8 +3361,7 @@ static void build_zonelists_in_node_order(pg_data_t *pgdat, int node)
        zonelist = &pgdat->node_zonelists[0];
        for (j = 0; zonelist->_zonerefs[j].zone != NULL; j++)
                ;
-       j = build_zonelists_node(NODE_DATA(node), zonelist, j,
-                                                       MAX_NR_ZONES - 1);
+       j = build_zonelists_node(NODE_DATA(node), zonelist, j);
        zonelist->_zonerefs[j].zone = NULL;
        zonelist->_zonerefs[j].zone_idx = 0;
 }
@@ -3378,7 +3375,7 @@ static void build_thisnode_zonelists(pg_data_t *pgdat)
        struct zonelist *zonelist;
 
        zonelist = &pgdat->node_zonelists[1];
-       j = build_zonelists_node(pgdat, zonelist, 0, MAX_NR_ZONES - 1);
+       j = build_zonelists_node(pgdat, zonelist, 0);
        zonelist->_zonerefs[j].zone = NULL;
        zonelist->_zonerefs[j].zone_idx = 0;
 }
@@ -3586,7 +3583,7 @@ static void build_zonelists(pg_data_t *pgdat)
        local_node = pgdat->node_id;
 
        zonelist = &pgdat->node_zonelists[0];
-       j = build_zonelists_node(pgdat, zonelist, 0, MAX_NR_ZONES - 1);
+       j = build_zonelists_node(pgdat, zonelist, 0);
 
        /*
         * Now we build the zonelist so that it contains the zones
@@ -3599,14 +3596,12 @@ static void build_zonelists(pg_data_t *pgdat)
        for (node = local_node + 1; node < MAX_NUMNODES; node++) {
                if (!node_online(node))
                        continue;
-               j = build_zonelists_node(NODE_DATA(node), zonelist, j,
-                                                       MAX_NR_ZONES - 1);
+               j = build_zonelists_node(NODE_DATA(node), zonelist, j);
        }
        for (node = 0; node < local_node; node++) {
                if (!node_online(node))
                        continue;
-               j = build_zonelists_node(NODE_DATA(node), zonelist, j,
-                                                       MAX_NR_ZONES - 1);
+               j = build_zonelists_node(NODE_DATA(node), zonelist, j);
        }
 
        zonelist->_zonerefs[j].zone = NULL;
@@ -4421,13 +4416,13 @@ static void __meminit adjust_zone_range_for_zone_movable(int nid,
  */
 static unsigned long __meminit zone_spanned_pages_in_node(int nid,
                                        unsigned long zone_type,
+                                       unsigned long node_start_pfn,
+                                       unsigned long node_end_pfn,
                                        unsigned long *ignored)
 {
-       unsigned long node_start_pfn, node_end_pfn;
        unsigned long zone_start_pfn, zone_end_pfn;
 
-       /* Get the start and end of the node and zone */
-       get_pfn_range_for_nid(nid, &node_start_pfn, &node_end_pfn);
+       /* Get the start and end of the zone */
        zone_start_pfn = arch_zone_lowest_possible_pfn[zone_type];
        zone_end_pfn = arch_zone_highest_possible_pfn[zone_type];
        adjust_zone_range_for_zone_movable(nid, zone_type,
@@ -4482,14 +4477,14 @@ unsigned long __init absent_pages_in_range(unsigned long start_pfn,
 /* Return the number of page frames in holes in a zone on a node */
 static unsigned long __meminit zone_absent_pages_in_node(int nid,
                                        unsigned long zone_type,
+                                       unsigned long node_start_pfn,
+                                       unsigned long node_end_pfn,
                                        unsigned long *ignored)
 {
        unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type];
        unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type];
-       unsigned long node_start_pfn, node_end_pfn;
        unsigned long zone_start_pfn, zone_end_pfn;
 
-       get_pfn_range_for_nid(nid, &node_start_pfn, &node_end_pfn);
        zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high);
        zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high);
 
@@ -4502,6 +4497,8 @@ static unsigned long __meminit zone_absent_pages_in_node(int nid,
 #else /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
 static inline unsigned long __meminit zone_spanned_pages_in_node(int nid,
                                        unsigned long zone_type,
+                                       unsigned long node_start_pfn,
+                                       unsigned long node_end_pfn,
                                        unsigned long *zones_size)
 {
        return zones_size[zone_type];
@@ -4509,6 +4506,8 @@ static inline unsigned long __meminit zone_spanned_pages_in_node(int nid,
 
 static inline unsigned long __meminit zone_absent_pages_in_node(int nid,
                                                unsigned long zone_type,
+                                               unsigned long node_start_pfn,
+                                               unsigned long node_end_pfn,
                                                unsigned long *zholes_size)
 {
        if (!zholes_size)
@@ -4520,21 +4519,27 @@ static inline unsigned long __meminit zone_absent_pages_in_node(int nid,
 #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
 
 static void __meminit calculate_node_totalpages(struct pglist_data *pgdat,
-               unsigned long *zones_size, unsigned long *zholes_size)
+                                               unsigned long node_start_pfn,
+                                               unsigned long node_end_pfn,
+                                               unsigned long *zones_size,
+                                               unsigned long *zholes_size)
 {
        unsigned long realtotalpages, totalpages = 0;
        enum zone_type i;
 
        for (i = 0; i < MAX_NR_ZONES; i++)
                totalpages += zone_spanned_pages_in_node(pgdat->node_id, i,
-                                                               zones_size);
+                                                        node_start_pfn,
+                                                        node_end_pfn,
+                                                        zones_size);
        pgdat->node_spanned_pages = totalpages;
 
        realtotalpages = totalpages;
        for (i = 0; i < MAX_NR_ZONES; i++)
                realtotalpages -=
                        zone_absent_pages_in_node(pgdat->node_id, i,
-                                                               zholes_size);
+                                                 node_start_pfn, node_end_pfn,
+                                                 zholes_size);
        pgdat->node_present_pages = realtotalpages;
        printk(KERN_DEBUG "On node %d totalpages: %lu\n", pgdat->node_id,
                                                        realtotalpages);
@@ -4643,6 +4648,7 @@ static unsigned long __paginginit calc_memmap_size(unsigned long spanned_pages,
  * NOTE: pgdat should get zeroed by caller.
  */
 static void __paginginit free_area_init_core(struct pglist_data *pgdat,
+               unsigned long node_start_pfn, unsigned long node_end_pfn,
                unsigned long *zones_size, unsigned long *zholes_size)
 {
        enum zone_type j;
@@ -4664,8 +4670,11 @@ static void __paginginit free_area_init_core(struct pglist_data *pgdat,
                struct zone *zone = pgdat->node_zones + j;
                unsigned long size, realsize, freesize, memmap_pages;
 
-               size = zone_spanned_pages_in_node(nid, j, zones_size);
+               size = zone_spanned_pages_in_node(nid, j, node_start_pfn,
+                                                 node_end_pfn, zones_size);
                realsize = freesize = size - zone_absent_pages_in_node(nid, j,
+                                                               node_start_pfn,
+                                                               node_end_pfn,
                                                                zholes_size);
 
                /*
@@ -4779,6 +4788,8 @@ void __paginginit free_area_init_node(int nid, unsigned long *zones_size,
                unsigned long node_start_pfn, unsigned long *zholes_size)
 {
        pg_data_t *pgdat = NODE_DATA(nid);
+       unsigned long start_pfn = 0;
+       unsigned long end_pfn = 0;
 
        /* pg_data_t should be reset to zero when it's allocated */
        WARN_ON(pgdat->nr_zones || pgdat->classzone_idx);
@@ -4786,7 +4797,11 @@ void __paginginit free_area_init_node(int nid, unsigned long *zones_size,
        pgdat->node_id = nid;
        pgdat->node_start_pfn = node_start_pfn;
        init_zone_allows_reclaim(nid);
-       calculate_node_totalpages(pgdat, zones_size, zholes_size);
+#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
+       get_pfn_range_for_nid(nid, &start_pfn, &end_pfn);
+#endif
+       calculate_node_totalpages(pgdat, start_pfn, end_pfn,
+                                 zones_size, zholes_size);
 
        alloc_node_mem_map(pgdat);
 #ifdef CONFIG_FLAT_NODE_MEM_MAP
@@ -4795,7 +4810,8 @@ void __paginginit free_area_init_node(int nid, unsigned long *zones_size,
                (unsigned long)pgdat->node_mem_map);
 #endif
 
-       free_area_init_core(pgdat, zones_size, zholes_size);
+       free_area_init_core(pgdat, start_pfn, end_pfn,
+                           zones_size, zholes_size);
 }
 
 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP