mm, page_alloc: Introduce ZONELIST_FALLBACK_SAME_TYPE fallback listept-idle-v2-4.14.99

On system with heterogeneous memory, reasonable fall back lists woul be:
a. No fall back, to stick to current running node.
b. Fall back to other nodes of the same type or different type
   e.g. DRAM node 0 -> DRAM node 1 -> PMEM node 2 -> PMEM node 3
c. Fall back to other nodes of the same type only.
   e.g. DRAM node 0 -> DRAM node 1

a. is already in place, previous patch implement b. providing way to
satisfy memory request as best effort by default. And this patch of
writing build c. to fallback to the same node type when user specify
GFP_SAME_NODE_TYPE only.

Dump all aviable fallback list:
[    0.059267] ========== Dump   fallback zone list for node :0
[    0.059268]  node: 0 zone:Normal
[    0.059268]  node: 0 zone:DMA32
[    0.059269]  node: 0 zone:DMA
[    0.059270]  node: 1 zone:Normal
[    0.059270]  node: 2 zone:Normal
[    0.059271]  node: 3 zone:Normal
[    0.059271] ========== Dump   fallback zone list for node with same type :0
[    0.059272]  node: 0 zone:Normal
[    0.059272]  node: 0 zone:DMA32
[    0.059273]  node: 0 zone:DMA
[    0.059273]  node: 1 zone:Normal
[    0.059274] ========== Dump nofallback zone list for node :0
[    0.059274]  node: 0 zone:Normal
[    0.059275]  node: 0 zone:DMA32
[    0.059275]  node: 0 zone:DMA
[    0.059277] ========== Dump   fallback zone list for node :1
[    0.059278]  node: 1 zone:Normal
[    0.059278]  node: 0 zone:Normal
[    0.059279]  node: 0 zone:DMA32
[    0.059279]  node: 0 zone:DMA
[    0.059280]  node: 3 zone:Normal
[    0.059280]  node: 2 zone:Normal
[    0.059281] ========== Dump   fallback zone list for node with same type :1
[    0.059282]  node: 1 zone:Normal
[    0.059282]  node: 0 zone:Normal
[    0.059283]  node: 0 zone:DMA32
[    0.059283]  node: 0 zone:DMA
[    0.059284] ========== Dump nofallback zone list for node :1
[    0.059284]  node: 1 zone:Normal
[    0.059287] ========== Dump   fallback zone list for node :2
[    0.059287]  node: 2 zone:Normal
[    0.059288]  node: 3 zone:Normal
[    0.059288]  node: 0 zone:Normal
[    0.059289]  node: 0 zone:DMA32
[    0.059289]  node: 0 zone:DMA
[    0.059290]  node: 1 zone:Normal
[    0.059290] ========== Dump   fallback zone list for node with same type :2
[    0.059291]  node: 2 zone:Normal
[    0.059291]  node: 3 zone:Normal
[    0.059292] ========== Dump nofallback zone list for node :2
[    0.059292]  node: 2 zone:Normal
[    0.059294] ========== Dump   fallback zone list for node :3
[    0.059295]  node: 3 zone:Normal
[    0.059295]  node: 2 zone:Normal
[    0.059296]  node: 1 zone:Normal
[    0.059296]  node: 0 zone:Normal
[    0.059297]  node: 0 zone:DMA32
[    0.059297]  node: 0 zone:DMA
[    0.059298] ========== Dump   fallback zone list for node with same type :3
[    0.059298]  node: 3 zone:Normal
[    0.059299]  node: 2 zone:Normal
[    0.059299] ========== Dump nofallback zone list for node :3
[    0.059300]  node: 3 zone:Normal
[    0.059306] Built 4 zonelists, mobility grouping on.  Total pages: 129978506
[    0.059307] Policy zone: Normal

Signed-off-by: Fan Du <fan.du@intel.com>

3 files changed, 29 insertions, 0 deletions

diff --git a/include/linux/gfp.h b/include/linux/gfp.h
index b041f94678de5..2ec75231b9983 100644
--- a/include/linux/gfp.h
+++ b/include/linux/gfp.h
@@ -45,6 +45,8 @@ struct vm_area_struct;
 #else
 #define ___GFP_NOLOCKDEP	0
 #endif
+#define ___GFP_SAME_NODE_TYPE	0x1000000u
+
 /* If the above are modified, __GFP_BITS_SHIFT may need updating */
 
 /*
@@ -208,6 +210,7 @@ struct vm_area_struct;
 
 /* Disable lockdep for GFP context tracking */
 #define __GFP_NOLOCKDEP ((__force gfp_t)___GFP_NOLOCKDEP)
+#define __GFP_SAME_NODE_TYPE ((__force gfp_t)___GFP_SAME_NODE_TYPE)
 
 /* Room for N __GFP_FOO bits */
 #define __GFP_BITS_SHIFT (25 + IS_ENABLED(CONFIG_LOCKDEP))
@@ -289,6 +292,8 @@ struct vm_area_struct;
 			 __GFP_NOMEMALLOC | __GFP_NOWARN) & ~__GFP_RECLAIM)
 #define GFP_TRANSHUGE	(GFP_TRANSHUGE_LIGHT | __GFP_DIRECT_RECLAIM)
 
+#define GFP_SAME_NODE_TYPE (__GFP_SAME_NODE_TYPE)
+
 /* Convert GFP flags to their corresponding migrate type */
 #define GFP_MOVABLE_MASK (__GFP_RECLAIMABLE|__GFP_MOVABLE)
 #define GFP_MOVABLE_SHIFT 3
@@ -426,6 +431,8 @@ static inline int gfp_zonelist(gfp_t flags)
 #ifdef CONFIG_NUMA
 	if (unlikely(flags & __GFP_THISNODE))
 		return ZONELIST_NOFALLBACK;
+	if (unlikely(flags & __GFP_SAME_NODE_TYPE))
+		return ZONELIST_FALLBACK_SAME_TYPE;
 #endif
 	return ZONELIST_FALLBACK;
 }
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index b1a00246b6586..baf932972da1e 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -572,6 +572,7 @@ static inline bool zone_intersects(struct zone *zone,
 
 enum {
 	ZONELIST_FALLBACK,	/* zonelist with fallback */
+	ZONELIST_FALLBACK_SAME_TYPE,	/* zonelist with fallback to the same type node */
 #ifdef CONFIG_NUMA
 	/*
 	 * The NUMA zonelists are doubled because we need zonelists that
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 9be7d3757e991..cb95a2519d100 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -5024,6 +5024,21 @@ static void build_zonelists_in_node_order(pg_data_t *pgdat, int *node_order,
 	}
 	zonerefs->zone = NULL;
 	zonerefs->zone_idx = 0;
+
+	zonerefs = pgdat->node_zonelists[ZONELIST_FALLBACK_SAME_TYPE]._zonerefs;
+
+	for (i = 0; i < nr_nodes; i++) {
+		int nr_zones;
+
+		pg_data_t *node = NODE_DATA(node_order[i]);
+
+		if (!is_node_same_type(node->node_id, pgdat->node_id))
+			continue;
+		nr_zones = build_zonerefs_node(node, zonerefs);
+		zonerefs += nr_zones;
+	}
+	zonerefs->zone = NULL;
+	zonerefs->zone_idx = 0;
 }
 
 /*
@@ -5093,6 +5108,12 @@ static void build_zonelists(pg_data_t *pgdat)
 			printk("	node: %d zone:%s\n", zonelist_node_idx(zonerefs), zonelist_zone(zonerefs)->name);
 		}
 
+		printk("========== Dump   fallback zone list for node with same type :%d\n", pgdat->node_id);
+		zonerefs = pgdat->node_zonelists[ZONELIST_FALLBACK_SAME_TYPE]._zonerefs;
+		for (; zonerefs->zone; zonerefs++) {
+			printk("	node: %d zone:%s\n", zonelist_node_idx(zonerefs), zonelist_zone(zonerefs)->name);
+		}
+
 		printk("========== Dump nofallback zone list for node :%d\n", pgdat->node_id);
 		zonerefs = pgdat->node_zonelists[ZONELIST_NOFALLBACK]._zonerefs;
 		for (; zonerefs->zone; zonerefs++) {