From 3f775b9367045cabb51247fbbcb561b24841dcb7 Mon Sep 17 00:00:00 2001 From: Peifeng Li Date: Thu, 23 Jun 2022 14:15:35 +0800 Subject: [PATCH] ANDROID: vendor_hooks: account page-mapcount Support five hooks as follows to account the amount of multi-mapped pages in kernel: - android_vh_show_mapcount_pages - android_vh_do_traversal_lruvec - android_vh_update_page_mapcount - android_vh_add_page_to_lrulist - android_vh_del_page_from_lrulist Bug: 236578020 Signed-off-by: Peifeng Li Change-Id: Ia2c7015aab442be7dbb496b8b630b9dff59ab935 --- drivers/android/vendor_hooks.c | 5 ++ include/linux/mm_inline.h | 7 +++ include/linux/rmap.h | 11 ++++- include/trace/hooks/mm.h | 16 +++++++ mm/huge_memory.c | 27 ++++++++--- mm/page_alloc.c | 1 + mm/rmap.c | 83 ++++++++++++++++++++++++++++------ mm/vmscan.c | 2 + 8 files changed, 129 insertions(+), 23 deletions(-) diff --git a/drivers/android/vendor_hooks.c b/drivers/android/vendor_hooks.c index 8c0bbcde78ae..ae0a674fe6f7 100644 --- a/drivers/android/vendor_hooks.c +++ b/drivers/android/vendor_hooks.c @@ -274,6 +274,8 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_include_reserved_zone); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_alloc_pages_slowpath_begin); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_alloc_pages_slowpath_end); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_show_mem); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_show_mapcount_pages); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_do_traversal_lruvec); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_typec_tcpci_override_toggling); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_typec_tcpci_chk_contaminant); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_typec_tcpci_get_vbus); @@ -393,6 +395,9 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_tcp_recvmsg_stat); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_pci_d3_sleep); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_kmalloc_slab); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mmap_region); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_update_page_mapcount); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_add_page_to_lrulist); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_del_page_from_lrulist); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_try_to_unmap_one); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mem_cgroup_id_remove); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mem_cgroup_css_offline); diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h index 8fc71e9d7bb0..835e4f83e7a4 100644 --- a/include/linux/mm_inline.h +++ b/include/linux/mm_inline.h @@ -4,6 +4,10 @@ #include #include +#ifndef __GENKSYMS__ +#define PROTECT_TRACE_INCLUDE_PATH +#include +#endif /** * page_is_file_lru - should the page be on a file LRU or anon LRU? @@ -48,6 +52,7 @@ static __always_inline void update_lru_size(struct lruvec *lruvec, static __always_inline void add_page_to_lru_list(struct page *page, struct lruvec *lruvec, enum lru_list lru) { + trace_android_vh_add_page_to_lrulist(page, false, lru); update_lru_size(lruvec, lru, page_zonenum(page), thp_nr_pages(page)); list_add(&page->lru, &lruvec->lists[lru]); } @@ -55,6 +60,7 @@ static __always_inline void add_page_to_lru_list(struct page *page, static __always_inline void add_page_to_lru_list_tail(struct page *page, struct lruvec *lruvec, enum lru_list lru) { + trace_android_vh_add_page_to_lrulist(page, false, lru); update_lru_size(lruvec, lru, page_zonenum(page), thp_nr_pages(page)); list_add_tail(&page->lru, &lruvec->lists[lru]); } @@ -62,6 +68,7 @@ static __always_inline void add_page_to_lru_list_tail(struct page *page, static __always_inline void del_page_from_lru_list(struct page *page, struct lruvec *lruvec, enum lru_list lru) { + trace_android_vh_del_page_from_lrulist(page, false, lru); list_del(&page->lru); update_lru_size(lruvec, lru, page_zonenum(page), -thp_nr_pages(page)); } diff --git a/include/linux/rmap.h b/include/linux/rmap.h index 7482da1ea67b..7dee138fbf0f 100644 --- a/include/linux/rmap.h +++ b/include/linux/rmap.h @@ -11,6 +11,10 @@ #include #include #include +#ifndef __GENKSYMS__ +#define PROTECT_TRACE_INCLUDE_PATH +#include +#endif /* * The anon_vma heads a list of private "related" vmas, to scan if @@ -194,7 +198,12 @@ void hugepage_add_new_anon_rmap(struct page *, struct vm_area_struct *, static inline void page_dup_rmap(struct page *page, bool compound) { - atomic_inc(compound ? compound_mapcount_ptr(page) : &page->_mapcount); + bool success = false; + + if (!compound) + trace_android_vh_update_page_mapcount(page, true, compound, NULL, &success); + if (!success) + atomic_inc(compound ? compound_mapcount_ptr(page) : &page->_mapcount); } /* diff --git a/include/trace/hooks/mm.h b/include/trace/hooks/mm.h index 6d25e458edb3..68d792378b50 100644 --- a/include/trace/hooks/mm.h +++ b/include/trace/hooks/mm.h @@ -153,6 +153,22 @@ DECLARE_HOOK(android_vh_drain_all_pages_bypass, int migratetype, unsigned long did_some_progress, bool *bypass), TP_ARGS(gfp_mask, order, alloc_flags, migratetype, did_some_progress, bypass)); +DECLARE_HOOK(android_vh_update_page_mapcount, + TP_PROTO(struct page *page, bool inc_size, bool compound, + bool *first_mapping, bool *success), + TP_ARGS(page, inc_size, compound, first_mapping, success)); +DECLARE_HOOK(android_vh_add_page_to_lrulist, + TP_PROTO(struct page *page, bool compound, enum lru_list lru), + TP_ARGS(page, compound, lru)); +DECLARE_HOOK(android_vh_del_page_from_lrulist, + TP_PROTO(struct page *page, bool compound, enum lru_list lru), + TP_ARGS(page, compound, lru)); +DECLARE_HOOK(android_vh_show_mapcount_pages, + TP_PROTO(void *unused), + TP_ARGS(unused)); +DECLARE_HOOK(android_vh_do_traversal_lruvec, + TP_PROTO(struct lruvec *lruvec), + TP_ARGS(lruvec)); DECLARE_HOOK(android_vh_cma_drain_all_pages_bypass, TP_PROTO(unsigned int migratetype, bool *bypass), TP_ARGS(migratetype, bypass)); diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 1eea5787fd63..c24716571a93 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -33,7 +33,7 @@ #include #include #include - +#include #include #include #include "internal.h" @@ -2033,6 +2033,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, bool young, write, soft_dirty, pmd_migration = false, uffd_wp = false; unsigned long addr; int i; + bool success = false; VM_BUG_ON(haddr & ~HPAGE_PMD_MASK); VM_BUG_ON_VMA(vma->vm_start > haddr, vma); @@ -2164,8 +2165,12 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, pte = pte_offset_map(&_pmd, addr); BUG_ON(!pte_none(*pte)); set_pte_at(mm, addr, pte, entry); - if (!pmd_migration) - atomic_inc(&page[i]._mapcount); + if (!pmd_migration) { + trace_android_vh_update_page_mapcount(&page[i], true, + false, NULL, &success); + if (!success) + atomic_inc(&page[i]._mapcount); + } pte_unmap(pte); } @@ -2176,8 +2181,12 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, */ if (compound_mapcount(page) > 1 && !TestSetPageDoubleMap(page)) { - for (i = 0; i < HPAGE_PMD_NR; i++) - atomic_inc(&page[i]._mapcount); + for (i = 0; i < HPAGE_PMD_NR; i++) { + trace_android_vh_update_page_mapcount(&page[i], true, + false, NULL, &success); + if (!success) + atomic_inc(&page[i]._mapcount); + } } lock_page_memcg(page); @@ -2186,8 +2195,12 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, __dec_lruvec_page_state(page, NR_ANON_THPS); if (TestClearPageDoubleMap(page)) { /* No need in mapcount reference anymore */ - for (i = 0; i < HPAGE_PMD_NR; i++) - atomic_dec(&page[i]._mapcount); + for (i = 0; i < HPAGE_PMD_NR; i++) { + trace_android_vh_update_page_mapcount(&page[i], + false, false, NULL, &success); + if (!success) + atomic_dec(&page[i]._mapcount); + } } } unlock_page_memcg(page); diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 1a5d222a7d6a..d2b7eca3a2bf 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -5646,6 +5646,7 @@ void show_free_areas(unsigned int filter, nodemask_t *nodemask) free_pcp, global_zone_page_state(NR_FREE_CMA_PAGES)); + trace_android_vh_show_mapcount_pages(NULL); for_each_online_pgdat(pgdat) { if (show_mem_node_skip(filter, pgdat->node_id, nodemask)) continue; diff --git a/mm/rmap.c b/mm/rmap.c index e526fbc7826e..ea1fd71b03df 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -1113,6 +1113,7 @@ void do_page_add_anon_rmap(struct page *page, { bool compound = flags & RMAP_COMPOUND; bool first; + bool success = false; if (unlikely(PageKsm(page))) lock_page_memcg(page); @@ -1126,7 +1127,10 @@ void do_page_add_anon_rmap(struct page *page, mapcount = compound_mapcount_ptr(page); first = atomic_inc_and_test(mapcount); } else { - first = atomic_inc_and_test(&page->_mapcount); + trace_android_vh_update_page_mapcount(page, true, compound, + &first, &success); + if (!success) + first = atomic_inc_and_test(&page->_mapcount); } if (first) { @@ -1200,13 +1204,22 @@ void __page_add_new_anon_rmap(struct page *page, void page_add_file_rmap(struct page *page, bool compound) { int i, nr = 1; + bool first_mapping; + bool success = false; VM_BUG_ON_PAGE(compound && !PageTransHuge(page), page); lock_page_memcg(page); if (compound && PageTransHuge(page)) { for (i = 0, nr = 0; i < thp_nr_pages(page); i++) { - if (atomic_inc_and_test(&page[i]._mapcount)) - nr++; + trace_android_vh_update_page_mapcount(&page[i], true, + compound, &first_mapping, &success); + if ((success)) { + if (first_mapping) + nr++; + } else { + if (atomic_inc_and_test(&page[i]._mapcount)) + nr++; + } } if (!atomic_inc_and_test(compound_mapcount_ptr(page))) goto out; @@ -1222,8 +1235,15 @@ void page_add_file_rmap(struct page *page, bool compound) if (PageMlocked(page)) clear_page_mlock(compound_head(page)); } - if (!atomic_inc_and_test(&page->_mapcount)) - goto out; + trace_android_vh_update_page_mapcount(page, true, + compound, &first_mapping, &success); + if (success) { + if (!first_mapping) + goto out; + } else { + if (!atomic_inc_and_test(&page->_mapcount)) + goto out; + } } __mod_lruvec_page_state(page, NR_FILE_MAPPED, nr); out: @@ -1233,6 +1253,8 @@ void page_add_file_rmap(struct page *page, bool compound) static void page_remove_file_rmap(struct page *page, bool compound) { int i, nr = 1; + bool first_mapping; + bool success = false; VM_BUG_ON_PAGE(compound && !PageHead(page), page); @@ -1246,8 +1268,15 @@ static void page_remove_file_rmap(struct page *page, bool compound) /* page still mapped by someone else? */ if (compound && PageTransHuge(page)) { for (i = 0, nr = 0; i < thp_nr_pages(page); i++) { - if (atomic_add_negative(-1, &page[i]._mapcount)) - nr++; + trace_android_vh_update_page_mapcount(&page[i], false, + compound, &first_mapping, &success); + if (success) { + if (first_mapping) + nr++; + } else { + if (atomic_add_negative(-1, &page[i]._mapcount)) + nr++; + } } if (!atomic_add_negative(-1, compound_mapcount_ptr(page))) return; @@ -1256,8 +1285,15 @@ static void page_remove_file_rmap(struct page *page, bool compound) else __dec_node_page_state(page, NR_FILE_PMDMAPPED); } else { - if (!atomic_add_negative(-1, &page->_mapcount)) - return; + trace_android_vh_update_page_mapcount(page, false, + compound, &first_mapping, &success); + if (success) { + if (!first_mapping) + return; + } else { + if (!atomic_add_negative(-1, &page->_mapcount)) + return; + } } /* @@ -1274,6 +1310,8 @@ static void page_remove_file_rmap(struct page *page, bool compound) static void page_remove_anon_compound_rmap(struct page *page) { int i, nr; + bool first_mapping; + bool success = false; if (!atomic_add_negative(-1, compound_mapcount_ptr(page))) return; @@ -1293,8 +1331,15 @@ static void page_remove_anon_compound_rmap(struct page *page) * them are still mapped. */ for (i = 0, nr = 0; i < thp_nr_pages(page); i++) { - if (atomic_add_negative(-1, &page[i]._mapcount)) - nr++; + trace_android_vh_update_page_mapcount(&page[i], false, + false, &first_mapping, &success); + if (success) { + if (first_mapping) + nr++; + } else { + if (atomic_add_negative(-1, &page[i]._mapcount)) + nr++; + } } /* @@ -1324,6 +1369,8 @@ static void page_remove_anon_compound_rmap(struct page *page) */ void page_remove_rmap(struct page *page, bool compound) { + bool first_mapping; + bool success = false; lock_page_memcg(page); if (!PageAnon(page)) { @@ -1336,10 +1383,16 @@ void page_remove_rmap(struct page *page, bool compound) goto out; } - /* page still mapped by someone else? */ - if (!atomic_add_negative(-1, &page->_mapcount)) - goto out; - + trace_android_vh_update_page_mapcount(page, false, + compound, &first_mapping, &success); + if (success) { + if (!first_mapping) + goto out; + } else { + /* page still mapped by someone else? */ + if (!atomic_add_negative(-1, &page->_mapcount)) + goto out; + } /* * We use the irq-unsafe __{inc|mod}_zone_page_stat because * these counters are not modified in interrupt context, and diff --git a/mm/vmscan.c b/mm/vmscan.c index e9789e8e7a6a..fef55a9ed60f 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -1727,6 +1727,7 @@ static unsigned long isolate_lru_pages(unsigned long nr_to_scan, case 0: nr_taken += nr_pages; nr_zone_taken[page_zonenum(page)] += nr_pages; + trace_android_vh_del_page_from_lrulist(page, false, lru); list_move(&page->lru, dst); break; @@ -1901,6 +1902,7 @@ static unsigned noinline_for_stack move_pages_to_lru(struct lruvec *lruvec, nr_pages = thp_nr_pages(page); update_lru_size(lruvec, lru, page_zonenum(page), nr_pages); list_move(&page->lru, &lruvec->lists[lru]); + trace_android_vh_add_page_to_lrulist(page, false, lru); if (put_page_testzero(page)) { __ClearPageLRU(page);