From 6316af1012e7e52b40aa18e4277525c3c4974f63 Mon Sep 17 00:00:00 2001 From: Minchan Kim Date: Tue, 7 May 2024 20:52:33 +0000 Subject: [PATCH] ANDROID: add vendor hooks and expoert reclaim_pages to reclaim memory Add vendor hooks to reclaim MADV_PAGEOUT memory for asynchrnous device. It also exports reclaim_pages to reclaim memory. Bug: 326662423 Change-Id: Ic2516c64a9dbd53173a3bfb19b6cd21636916c27 Signed-off-by: Minchan Kim --- drivers/android/vendor_hooks.c | 3 +++ include/linux/swap.h | 1 + include/trace/hooks/mm.h | 9 +++++++++ mm/madvise.c | 12 ++++++++++-- mm/vmscan.c | 29 +++++++++++++++++++++-------- 5 files changed, 44 insertions(+), 10 deletions(-) diff --git a/drivers/android/vendor_hooks.c b/drivers/android/vendor_hooks.c index ff9a0a78977a..e8af7b774588 100644 --- a/drivers/android/vendor_hooks.c +++ b/drivers/android/vendor_hooks.c @@ -331,6 +331,9 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_alloc_pages_failure_bypass); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_madvise_pageout_swap_entry); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_madvise_swapin_walk_pmd_entry); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_process_madvise_end); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_madvise_pageout_begin); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_madvise_pageout_end); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_reclaim_folio_list); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_smaps_pte_entry); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_show_smap); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_ctl_dirty_rate); diff --git a/include/linux/swap.h b/include/linux/swap.h index 6d17e7bb330c..671853df78ac 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -439,6 +439,7 @@ extern int vm_swappiness; long remove_mapping(struct address_space *mapping, struct folio *folio); extern unsigned long reclaim_pages(struct list_head *page_list); +extern unsigned long __reclaim_pages(struct list_head *page_list, void *private); #ifdef CONFIG_NUMA extern int node_reclaim_mode; extern int sysctl_min_unmapped_ratio; diff --git a/include/trace/hooks/mm.h b/include/trace/hooks/mm.h index 73a544af12b7..81ce1268c4fd 100644 --- a/include/trace/hooks/mm.h +++ b/include/trace/hooks/mm.h @@ -99,6 +99,15 @@ DECLARE_HOOK(android_vh_madvise_swapin_walk_pmd_entry, DECLARE_HOOK(android_vh_process_madvise_end, TP_PROTO(int behavior, ssize_t *ret), TP_ARGS(behavior, ret)); +DECLARE_RESTRICTED_HOOK(android_rvh_madvise_pageout_begin, + TP_PROTO(void **private), + TP_ARGS(private), 1); +DECLARE_RESTRICTED_HOOK(android_rvh_madvise_pageout_end, + TP_PROTO(void *private, struct list_head *folio_list), + TP_ARGS(private, folio_list), 1); +DECLARE_RESTRICTED_HOOK(android_rvh_reclaim_folio_list, + TP_PROTO(struct list_head *folio_list, void *private), + TP_ARGS(folio_list, private), 1); DECLARE_HOOK(android_vh_smaps_pte_entry, TP_PROTO(swp_entry_t entry, unsigned long *writeback, unsigned long *same, unsigned long *huge), diff --git a/mm/madvise.c b/mm/madvise.c index fa716030ba1d..d084cea48ffe 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -42,6 +42,7 @@ struct madvise_walk_private { struct mmu_gather *tlb; bool pageout; + void *private; }; /* @@ -426,7 +427,7 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd, huge_unlock: spin_unlock(ptl); if (pageout) - reclaim_pages(&page_list); + __reclaim_pages(&page_list, private->private); return 0; } @@ -534,7 +535,7 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd, arch_leave_lazy_mmu_mode(); pte_unmap_unlock(orig_pte, ptl); if (pageout) - reclaim_pages(&page_list); + __reclaim_pages(&page_list, private->private); cond_resched(); return 0; @@ -591,10 +592,17 @@ static void madvise_pageout_page_range(struct mmu_gather *tlb, .pageout = true, .tlb = tlb, }; + LIST_HEAD(folio_list); + + trace_android_rvh_madvise_pageout_begin(&walk_private.private); tlb_start_vma(tlb, vma); walk_page_range(vma->vm_mm, addr, end, &cold_walk_ops, &walk_private); tlb_end_vma(tlb, vma); + + trace_android_rvh_madvise_pageout_end(walk_private.private, &folio_list); + if (!list_empty(&folio_list)) + reclaim_pages(&folio_list); } static long madvise_pageout(struct vm_area_struct *vma, diff --git a/mm/vmscan.c b/mm/vmscan.c index 29ae2be21936..6061aa4fb1b6 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -73,6 +73,9 @@ #undef CREATE_TRACE_POINTS #include +#undef CREATE_TRACE_POINTS +#include + EXPORT_TRACEPOINT_SYMBOL_GPL(mm_vmscan_direct_reclaim_begin); EXPORT_TRACEPOINT_SYMBOL_GPL(mm_vmscan_direct_reclaim_end); EXPORT_TRACEPOINT_SYMBOL_GPL(mm_vmscan_kswapd_wake); @@ -2717,7 +2720,8 @@ static void shrink_active_list(unsigned long nr_to_scan, } static unsigned int reclaim_folio_list(struct list_head *folio_list, - struct pglist_data *pgdat) + struct pglist_data *pgdat, + void *private) { struct reclaim_stat dummy_stat; unsigned int nr_reclaimed; @@ -2731,16 +2735,20 @@ static unsigned int reclaim_folio_list(struct list_head *folio_list, }; nr_reclaimed = shrink_folio_list(folio_list, pgdat, &sc, &dummy_stat, false); - while (!list_empty(folio_list)) { - folio = lru_to_folio(folio_list); - list_del(&folio->lru); - folio_putback_lru(folio); + if (private) { + trace_android_rvh_reclaim_folio_list(folio_list, private); + } else { + while (!list_empty(folio_list)) { + folio = lru_to_folio(folio_list); + list_del(&folio->lru); + folio_putback_lru(folio); + } } return nr_reclaimed; } -unsigned long reclaim_pages(struct list_head *folio_list) +unsigned long __reclaim_pages(struct list_head *folio_list, void *private) { int nid; unsigned int nr_reclaimed = 0; @@ -2762,17 +2770,22 @@ unsigned long reclaim_pages(struct list_head *folio_list) continue; } - nr_reclaimed += reclaim_folio_list(&node_folio_list, NODE_DATA(nid)); + nr_reclaimed += reclaim_folio_list(&node_folio_list, NODE_DATA(nid), private); nid = folio_nid(lru_to_folio(folio_list)); } while (!list_empty(folio_list)); - nr_reclaimed += reclaim_folio_list(&node_folio_list, NODE_DATA(nid)); + nr_reclaimed += reclaim_folio_list(&node_folio_list, NODE_DATA(nid), private); memalloc_noreclaim_restore(noreclaim_flag); return nr_reclaimed; } +unsigned long reclaim_pages(struct list_head *folio_list) +{ + return __reclaim_pages(folio_list, NULL); +} + static unsigned long shrink_list(enum lru_list lru, unsigned long nr_to_scan, struct lruvec *lruvec, struct scan_control *sc) {