10 Dec
2021
10 Dec
'21
11:53 a.m.
From: lijiawei <lijiawei49(a)huawei.com>
mm: add cma reuse feature
Signed-off-by: lijiawei <lijiawei49(a)huawei.com>
---
include/linux/gfp.h | 21 +++-----
include/linux/mmzone.h | 15 ++++--
mm/Kconfig | 10 ++++
mm/compaction.c | 2 +-
mm/page_alloc.c | 118 ++++++++++++++++++++++-------------------
5 files changed, 92 insertions(+), 74 deletions(-)
diff --git a/include/linux/gfp.h b/include/linux/gfp.h
index e80b7d2f5..4506d5c67 100644
--- a/include/linux/gfp.h
+++ b/include/linux/gfp.h
@@ -39,21 +39,12 @@ struct vm_area_struct;
#define ___GFP_HARDWALL 0x100000u
#define ___GFP_THISNODE 0x200000u
#define ___GFP_ACCOUNT 0x400000u
-#ifdef CONFIG_CMA
#define ___GFP_CMA 0x800000u
-#else
-#define ___GFP_CMA 0
-#endif
#ifdef CONFIG_LOCKDEP
-#ifdef CONFIG_CMA
#define ___GFP_NOLOCKDEP 0x1000000u
#else
-#define ___GFP_NOLOCKDEP 0x800000u
-#endif
-#else
#define ___GFP_NOLOCKDEP 0
#endif
-
/* If the above are modified, __GFP_BITS_SHIFT may need updating */
/*
@@ -235,11 +226,7 @@ struct vm_area_struct;
#define __GFP_NOLOCKDEP ((__force gfp_t)___GFP_NOLOCKDEP)
/* Room for N __GFP_FOO bits */
-#ifdef CONFIG_CMA
#define __GFP_BITS_SHIFT (24 + IS_ENABLED(CONFIG_LOCKDEP))
-#else
-#define __GFP_BITS_SHIFT (23 + IS_ENABLED(CONFIG_LOCKDEP))
-#endif
#define __GFP_BITS_MASK ((__force gfp_t)((1 << __GFP_BITS_SHIFT) - 1))
/**
@@ -339,7 +326,13 @@ static inline int gfp_migratetype(const gfp_t gfp_flags)
return MIGRATE_UNMOVABLE;
/* Group based on mobility */
- return (gfp_flags & GFP_MOVABLE_MASK) >> GFP_MOVABLE_SHIFT;
+ unsigned int ret_mt = (gfp_flags & GFP_MOVABLE_MASK) >> GFP_MOVABLE_SHIFT;
+
+ if (IS_ENABLED(CONFIG_CMA_REUSE) && ret_mt == MIGRATE_MOVABLE &&
+ (gfp_flags & __GFP_CMA))
+ return MIGRATE_CMA;
+
+ return ret_mt;
}
#undef GFP_MOVABLE_MASK
#undef GFP_MOVABLE_SHIFT
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index b6bcef45b..3ac2799dc 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -42,9 +42,12 @@ enum migratetype {
MIGRATE_UNMOVABLE,
MIGRATE_MOVABLE,
MIGRATE_RECLAIMABLE,
+#ifdef CONFIG_CMA_REUSE
+ MIGRATE_CMA,
+#endif
MIGRATE_PCPTYPES, /* the number of types on the pcp lists */
MIGRATE_HIGHATOMIC = MIGRATE_PCPTYPES,
-#ifdef CONFIG_CMA
+#if defined(CONFIG_CMA) && !defined(CONFIG_CMA_REUSE)
/*
* MIGRATE_CMA migration type is designed to mimic the way
* ZONE_MOVABLE works. Only movable pages can be allocated
@@ -77,6 +80,12 @@ extern const char * const migratetype_names[MIGRATE_TYPES];
# define is_migrate_cma_page(_page) false
#endif
+#ifdef CONFIG_CMA_REUSE
+# define get_cma_migratetype() MIGRATE_CMA
+#else
+# define get_cma_migratetype() MIGRATE_MOVABLE
+#endif
+
static inline bool is_migrate_movable(int mt)
{
return is_migrate_cma(mt) || mt == MIGRATE_MOVABLE;
@@ -451,10 +460,6 @@ struct zone {
struct pglist_data *zone_pgdat;
struct per_cpu_pageset __percpu *pageset;
-#ifdef CONFIG_CMA
- bool cma_alloc;
-#endif
-
#ifndef CONFIG_SPARSEMEM
/*
* Flags for a pageblock_nr_pages block. See pageblock-flags.h.
diff --git a/mm/Kconfig b/mm/Kconfig
index 9d606d258..acfc5e88a 100644
--- a/mm/Kconfig
+++ b/mm/Kconfig
@@ -528,6 +528,16 @@ config CMA_AREAS
If unsure, leave the default value "7" in UMA and "19" in NUMA.
+config CMA_REUSE
+ bool "CMA reuse feature"
+ depends on CMA
+ help
+ If enabled, it will add MIGRATE_CMA to pcp lists and movable
+ allocations with __GFP_CMA flag will use cma areas prior to
+ movable areas.
+
+ It improves the utilization ratio of cma areas.
+
config MEM_SOFT_DIRTY
bool "Track memory changes"
depends on CHECKPOINT_RESTORE && HAVE_ARCH_SOFT_DIRTY && PROC_FS
diff --git a/mm/compaction.c b/mm/compaction.c
index dba424447..22e6a6e21 100644
--- a/mm/compaction.c
+++ b/mm/compaction.c
@@ -2021,7 +2021,7 @@ static enum compact_result __compact_finished(struct compact_control
*cc)
#ifdef CONFIG_CMA
/* MIGRATE_MOVABLE can fallback on MIGRATE_CMA */
- if (migratetype == MIGRATE_MOVABLE &&
+ if (migratetype == get_cma_migratetype() &&
!free_area_empty(area, MIGRATE_CMA))
return COMPACT_SUCCESS;
#endif
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 7250a0b59..83c0146cb 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -324,8 +324,11 @@ const char * const migratetype_names[MIGRATE_TYPES] = {
"Unmovable",
"Movable",
"Reclaimable",
+#ifdef CONFIG_CMA_REUSE
+ "CMA",
+#endif
"HighAtomic",
-#ifdef CONFIG_CMA
+#if defined(CONFIG_CMA) && !defined(CONFIG_CMA_REUSE)
"CMA",
#endif
#ifdef CONFIG_MEMORY_ISOLATION
@@ -2834,6 +2837,27 @@ __rmqueue_fallback(struct zone *zone, int order, int
start_migratetype,
}
+static __always_inline struct page *
+__rmqueue_with_cma_reuse(struct zone *zone, unsigned int order,
+ int migratetype, unsigned int alloc_flags)
+{
+ struct page *page = NULL;
+retry:
+ page = __rmqueue_smallest(zone, order, migratetype);
+
+ if (unlikely(!page) && is_migrate_cma(migratetype)) {
+ migratetype = MIGRATE_MOVABLE;
+ alloc_flags &= ~ALLOC_CMA;
+ page = __rmqueue_smallest(zone, order, migratetype);
+ }
+
+ if (unlikely(!page) &&
+ __rmqueue_fallback(zone, order, migratetype, alloc_flags))
+ goto retry;
+
+ return page;
+}
+
/*
* Do the hard work of removing an element from the buddy allocator.
* Call me with the zone->lock already held.
@@ -2844,34 +2868,41 @@ __rmqueue(struct zone *zone, unsigned int order, int migratetype,
{
struct page *page;
+#ifdef CONFIG_CMA_REUSE
+ page = __rmqueue_with_cma_reuse(zone, order, migratetype, alloc_flags);
+ goto out;
+#endif
+
+ if (IS_ENABLED(CONFIG_CMA)) {
+ /*
+ * Balance movable allocations between regular and CMA areas by
+ * allocating from CMA when over half of the zone's free memory
+ * is in the CMA area.
+ */
+ if (alloc_flags & ALLOC_CMA &&
+ zone_page_state(zone, NR_FREE_CMA_PAGES) >
+ zone_page_state(zone, NR_FREE_PAGES) / 2) {
+ page = __rmqueue_cma_fallback(zone, order);
+ if (page)
+ goto out;
+ }
+ }
retry:
page = __rmqueue_smallest(zone, order, migratetype);
- if (unlikely(!page) && __rmqueue_fallback(zone, order, migratetype,
- alloc_flags))
- goto retry;
+ if (unlikely(!page)) {
+ if (alloc_flags & ALLOC_CMA)
+ page = __rmqueue_cma_fallback(zone, order);
+
+ if (!page && __rmqueue_fallback(zone, order, migratetype,
+ alloc_flags))
+ goto retry;
+ }
out:
if (page)
trace_mm_page_alloc_zone_locked(page, order, migratetype);
return page;
}
-static struct page *__rmqueue_cma(struct zone *zone, unsigned int order,
- int migratetype,
- unsigned int alloc_flags)
-{
- struct page *page = 0;
-
-#ifdef CONFIG_CMA
- if (migratetype == MIGRATE_MOVABLE && !zone->cma_alloc)
- page = __rmqueue_cma_fallback(zone, order);
- else
-#endif
- page = __rmqueue_smallest(zone, order, migratetype);
-
- trace_mm_page_alloc_zone_locked(page, order, MIGRATE_CMA);
- return page;
-}
-
/*
* Obtain a specified number of elements from the buddy allocator, all under
* a single hold of the lock, for efficiency. Add them to the supplied list.
@@ -2879,20 +2910,14 @@ static struct page *__rmqueue_cma(struct zone *zone, unsigned int
order,
*/
static int rmqueue_bulk(struct zone *zone, unsigned int order,
unsigned long count, struct list_head *list,
- int migratetype, unsigned int alloc_flags, int cma)
+ int migratetype, unsigned int alloc_flags)
{
int i, alloced = 0;
spin_lock(&zone->lock);
for (i = 0; i < count; ++i) {
- struct page *page = NULL;
-
- if (cma)
- page = __rmqueue_cma(zone, order, migratetype,
- alloc_flags);
- else
- page = __rmqueue(zone, order, migratetype, alloc_flags);
-
+ struct page *page = __rmqueue(zone, order, migratetype,
+ alloc_flags);
if (unlikely(page == NULL))
break;
@@ -3378,8 +3403,7 @@ static inline void zone_statistics(struct zone *preferred_zone,
struct zone *z)
static struct page *__rmqueue_pcplist(struct zone *zone, int migratetype,
unsigned int alloc_flags,
struct per_cpu_pages *pcp,
- struct list_head *list,
- gfp_t gfp_flags)
+ struct list_head *list)
{
struct page *page;
@@ -3387,8 +3411,7 @@ static struct page *__rmqueue_pcplist(struct zone *zone, int
migratetype,
if (list_empty(list)) {
pcp->count += rmqueue_bulk(zone, 0,
pcp->batch, list,
- migratetype, alloc_flags,
- gfp_flags && __GFP_CMA);
+ migratetype, alloc_flags);
if (unlikely(list_empty(list)))
return NULL;
}
@@ -3414,8 +3437,7 @@ static struct page *rmqueue_pcplist(struct zone *preferred_zone,
local_irq_save(flags);
pcp = &this_cpu_ptr(zone->pageset)->pcp;
list = &pcp->lists[migratetype];
- page = __rmqueue_pcplist(zone, migratetype, alloc_flags, pcp, list,
- gfp_flags);
+ page = __rmqueue_pcplist(zone, migratetype, alloc_flags, pcp, list);
if (page) {
__count_zid_vm_events(PGALLOC, page_zonenum(page), 1);
zone_statistics(preferred_zone, zone);
@@ -3441,8 +3463,9 @@ struct page *rmqueue(struct zone *preferred_zone,
* MIGRATE_MOVABLE pcplist could have the pages on CMA area and
* we need to skip it when CMA area isn't allowed.
*/
- if (!IS_ENABLED(CONFIG_CMA) || gfp_flags & __GFP_CMA ||
- migratetype != MIGRATE_MOVABLE) {
+ if (!IS_ENABLED(CONFIG_CMA) || alloc_flags & ALLOC_CMA ||
+ migratetype != MIGRATE_MOVABLE ||
+ IS_ENABLED(CONFIG_CMA_REUSE)) {
page = rmqueue_pcplist(preferred_zone, zone, gfp_flags,
migratetype, alloc_flags);
goto out;
@@ -3469,14 +3492,8 @@ struct page *rmqueue(struct zone *preferred_zone,
if (page)
trace_mm_page_alloc_zone_locked(page, order, migratetype);
}
- if (!page) {
- if (gfp_flags & __GFP_CMA)
- page = __rmqueue_cma(zone, order, migratetype,
- alloc_flags);
- else
- page = __rmqueue(zone, order, migratetype,
- alloc_flags);
- }
+ if (!page)
+ page = __rmqueue(zone, order, migratetype, alloc_flags);
} while (page && check_new_pages(page, order));
spin_unlock(&zone->lock);
if (!page)
@@ -3789,8 +3806,7 @@ static inline unsigned int current_alloc_flags(gfp_t gfp_mask,
unsigned int pflags = current->flags;
if (!(pflags & PF_MEMALLOC_NOCMA) &&
- gfp_migratetype(gfp_mask) == MIGRATE_MOVABLE &&
- gfp_mask & __GFP_CMA)
+ gfp_migratetype(gfp_mask) == get_cma_migratetype())
alloc_flags |= ALLOC_CMA;
#endif
@@ -8547,9 +8563,6 @@ int alloc_contig_range(unsigned long start, unsigned long end,
if (ret)
return ret;
-#ifdef CONFIG_CMA
- cc.zone->cma_alloc = 1;
-#endif
/*
* In case of -EBUSY, we'd like to know which page causes problem.
* So, just fall through. test_pages_isolated() has a tracepoint
@@ -8631,9 +8644,6 @@ int alloc_contig_range(unsigned long start, unsigned long end,
done:
undo_isolate_page_range(pfn_max_align_down(start),
pfn_max_align_up(end), migratetype);
-#ifdef CONFIG_CMA
- cc.zone->cma_alloc = 0;
-#endif
return ret;
}
EXPORT_SYMBOL(alloc_contig_range);
--
2.25.1