diff --git a/drivers/android/vendor_hooks.c b/drivers/android/vendor_hooks.c index b42e7f6ee83e..0c754ba4dd5d 100644 --- a/drivers/android/vendor_hooks.c +++ b/drivers/android/vendor_hooks.c @@ -327,3 +327,4 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_exit_signal_whether_wake); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_exit_check); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_freeze_whether_wake); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_use_amu_fie); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_scan_abort_check_wmarks); diff --git a/include/trace/hooks/vmscan.h b/include/trace/hooks/vmscan.h index e6594ca0d299..625d30d75757 100644 --- a/include/trace/hooks/vmscan.h +++ b/include/trace/hooks/vmscan.h @@ -46,6 +46,9 @@ DECLARE_HOOK(android_vh_tune_scan_type, DECLARE_HOOK(android_vh_tune_swappiness, TP_PROTO(int *swappiness), TP_ARGS(swappiness)); +DECLARE_HOOK(android_vh_scan_abort_check_wmarks, + TP_PROTO(bool *check_wmarks), + TP_ARGS(check_wmarks)); #endif /* _TRACE_HOOK_VMSCAN_H */ /* This part must be outside protection */ #include diff --git a/mm/vmscan.c b/mm/vmscan.c index 6be9bee3eca1..647e4c0aa0bd 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -5277,11 +5277,51 @@ static unsigned long get_nr_to_reclaim(struct scan_control *sc) return max(sc->nr_to_reclaim, compact_gap(sc->order)); } +static bool should_abort_scan(struct lruvec *lruvec, struct scan_control *sc) +{ + unsigned long nr_to_reclaim = get_nr_to_reclaim(sc); + bool check_wmarks = false; + int i; + + if (sc->nr_reclaimed >= nr_to_reclaim) + return true; + + trace_android_vh_scan_abort_check_wmarks(&check_wmarks); + + if (!check_wmarks) + return false; + + if (!current_is_kswapd()) + return false; + + for (i = 0; i <= sc->reclaim_idx; i++) { + unsigned long wmark; + struct zone *zone = lruvec_pgdat(lruvec)->node_zones + i; + + if (!managed_zone(zone)) + continue; + + if (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) + wmark = wmark_pages(zone, WMARK_PROMO); + else + wmark = high_wmark_pages(zone); + + /* + * Abort scan once the target number of order zero pages are met. + * Reclaim MIN_LRU_BATCH << 2 to facilitate immediate kswapd sleep. + */ + wmark += MIN_LRU_BATCH << 2; + if (!zone_watermark_ok_safe(zone, 0, wmark, sc->reclaim_idx)) + return false; + } + + return true; +} + static bool try_to_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) { long nr_to_scan; unsigned long scanned = 0; - unsigned long nr_to_reclaim = get_nr_to_reclaim(sc); int swappiness = get_swappiness(lruvec, sc); /* clean file folios are more likely to exist */ @@ -5303,7 +5343,7 @@ static bool try_to_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) if (scanned >= nr_to_scan) break; - if (sc->nr_reclaimed >= nr_to_reclaim) + if (should_abort_scan(lruvec, sc)) break; cond_resched(); @@ -5362,10 +5402,9 @@ static void shrink_many(struct pglist_data *pgdat, struct scan_control *sc) int bin; int first_bin; struct lruvec *lruvec; - struct lru_gen_folio *lrugen; + struct lru_gen_folio *lrugen = NULL; struct mem_cgroup *memcg; const struct hlist_nulls_node *pos; - unsigned long nr_to_reclaim = get_nr_to_reclaim(sc); bin = first_bin = get_random_u32_below(MEMCG_NR_BINS); restart: @@ -5396,7 +5435,7 @@ static void shrink_many(struct pglist_data *pgdat, struct scan_control *sc) rcu_read_lock(); - if (sc->nr_reclaimed >= nr_to_reclaim) + if (should_abort_scan(lruvec, sc)) break; } @@ -5407,7 +5446,7 @@ static void shrink_many(struct pglist_data *pgdat, struct scan_control *sc) mem_cgroup_put(memcg); - if (sc->nr_reclaimed >= nr_to_reclaim) + if (lruvec && should_abort_scan(lruvec, sc)) return; /* restart if raced with lru_gen_rotate_memcg() */