diff --git a/include/linux/gfp.h b/include/linux/gfp.h index 53f8ecadf996..fbed5dd273db 100644 --- a/include/linux/gfp.h +++ b/include/linux/gfp.h @@ -642,6 +642,8 @@ static inline bool pm_suspended_storage(void) #endif /* CONFIG_PM_SLEEP */ #ifdef CONFIG_CONTIG_ALLOC +extern unsigned long pfn_max_align_up(unsigned long pfn); + #define ACR_ERR_ISOLATE (1 << 0) #define ACR_ERR_MIGRATE (1 << 1) #define ACR_ERR_TEST (1 << 2) @@ -651,6 +653,7 @@ struct acr_info { unsigned long nr_migrated; unsigned long nr_reclaimed; unsigned int err; + unsigned long failed_pfn; }; /* The below functions must be run on a range from a single zone. */ diff --git a/include/linux/page-isolation.h b/include/linux/page-isolation.h index 572458016331..ccd3ed46434f 100644 --- a/include/linux/page-isolation.h +++ b/include/linux/page-isolation.h @@ -44,7 +44,8 @@ int move_freepages_block(struct zone *zone, struct page *page, */ int start_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn, - unsigned migratetype, int flags); + unsigned migratetype, int flags, + unsigned long *failed_pfn); /* * Changes MIGRATE_ISOLATE to MIGRATE_MOVABLE. @@ -58,7 +59,7 @@ undo_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn, * Test all pages in [start_pfn, end_pfn) are isolated or not. */ int test_pages_isolated(unsigned long start_pfn, unsigned long end_pfn, - int isol_flags); + int isol_flags, unsigned long *failed_pfn); struct page *alloc_migrate_target(struct page *page, unsigned long private); diff --git a/mm/cma.c b/mm/cma.c index 00c1b110a39e..0df6554bf0fc 100644 --- a/mm/cma.c +++ b/mm/cma.c @@ -532,8 +532,16 @@ struct page *cma_alloc(struct cma *cma, size_t count, unsigned int align, trace_cma_alloc_busy_retry(cma->name, pfn, pfn_to_page(pfn), count, align); - /* try again with a bit different memory target */ - start = bitmap_no + mask + 1; + + if (info.failed_pfn && gfp_mask & __GFP_NORETRY) { + /* try again from following failed page */ + start = (pfn_max_align_up(info.failed_pfn + 1) - + cma->base_pfn) >> cma->order_per_bit; + + } else { + /* try again with a bit different memory target */ + start = bitmap_no + mask + 1; + } } trace_cma_alloc_finish(cma->name, pfn, page, count, align); diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c index d0747f0b6a41..41ac69bc0299 100644 --- a/mm/memory_hotplug.c +++ b/mm/memory_hotplug.c @@ -1541,7 +1541,7 @@ int __ref offline_pages(unsigned long start_pfn, unsigned long nr_pages) /* set above range as isolated */ ret = start_isolate_page_range(start_pfn, end_pfn, MIGRATE_MOVABLE, - MEMORY_OFFLINE | REPORT_FAILURE); + MEMORY_OFFLINE | REPORT_FAILURE, NULL); if (ret) { reason = "failure to isolate range"; goto failed_removal; @@ -1608,7 +1608,7 @@ int __ref offline_pages(unsigned long start_pfn, unsigned long nr_pages) * because has_unmovable_pages explicitly checks for * PageBuddy on freed pages on other zones. */ - ret = test_pages_isolated(start_pfn, end_pfn, MEMORY_OFFLINE); + ret = test_pages_isolated(start_pfn, end_pfn, MEMORY_OFFLINE, NULL); if (ret) drain_all_pages(zone); } while (ret); diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 7c01170eeca5..cdebaa666ece 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -8547,7 +8547,7 @@ static unsigned long pfn_max_align_down(unsigned long pfn) pageblock_nr_pages) - 1); } -static unsigned long pfn_max_align_up(unsigned long pfn) +unsigned long pfn_max_align_up(unsigned long pfn) { return ALIGN(pfn, max_t(unsigned long, MAX_ORDER_NR_PAGES, pageblock_nr_pages)); @@ -8646,6 +8646,11 @@ static int __alloc_contig_migrate_range(struct compact_control *cc, page_pinner_mark_migration_failed_pages(&cc->migratepages); } + if (!list_empty(&cc->migratepages)) { + page = list_first_entry(&cc->migratepages, struct page , lru); + info->failed_pfn = page_to_pfn(page); + } + putback_movable_pages(&cc->migratepages); info->err |= ACR_ERR_MIGRATE; return ret; @@ -8719,7 +8724,8 @@ int alloc_contig_range(unsigned long start, unsigned long end, */ ret = start_isolate_page_range(pfn_max_align_down(start), - pfn_max_align_up(end), migratetype, 0); + pfn_max_align_up(end), migratetype, 0, + &info->failed_pfn); if (ret) { info->err |= ACR_ERR_ISOLATE; return ret; @@ -8783,7 +8789,7 @@ int alloc_contig_range(unsigned long start, unsigned long end, } /* Make sure the range is really isolated. */ - if (test_pages_isolated(outer_start, end, 0)) { + if (test_pages_isolated(outer_start, end, 0, &info->failed_pfn)) { pr_info_ratelimited("%s: [%lx, %lx) PFNs busy\n", __func__, outer_start, end); ret = -EBUSY; diff --git a/mm/page_isolation.c b/mm/page_isolation.c index 2399e3bf6393..b391091bfd5c 100644 --- a/mm/page_isolation.c +++ b/mm/page_isolation.c @@ -181,7 +181,8 @@ __first_valid_page(unsigned long pfn, unsigned long nr_pages) * Return: 0 on success and -EBUSY if any part of range cannot be isolated. */ int start_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn, - unsigned migratetype, int flags) + unsigned migratetype, int flags, + unsigned long *failed_pfn) { unsigned long pfn; unsigned long undo_pfn; @@ -197,6 +198,8 @@ int start_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn, if (page) { if (set_migratetype_isolate(page, migratetype, flags)) { undo_pfn = pfn; + if (failed_pfn) + *failed_pfn = page_to_pfn(page); goto undo; } } @@ -282,7 +285,7 @@ __test_page_isolated_in_pageblock(unsigned long pfn, unsigned long end_pfn, /* Caller should ensure that requested range is in a single zone */ int test_pages_isolated(unsigned long start_pfn, unsigned long end_pfn, - int isol_flags) + int isol_flags, unsigned long *failed_pfn) { unsigned long pfn, flags; struct page *page; @@ -310,6 +313,8 @@ int test_pages_isolated(unsigned long start_pfn, unsigned long end_pfn, trace_test_pages_isolated(start_pfn, end_pfn, pfn); if (pfn < end_pfn) { page_pinner_failure_detect(pfn_to_page(pfn)); + if (failed_pfn) + *failed_pfn = pfn; return -EBUSY; }