android/android_kernel_xiaomi_sm8450
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

percpu: make pcpu_nr_empty_pop_pages per chunk type

Browse Source 
commit 0760fa3d8f7fceeea508b98899f1c826e10ffe78 upstream.

nr_empty_pop_pages is used to guarantee that there are some free
populated pages to satisfy atomic allocations. Accounted and
non-accounted allocations are using separate sets of chunks,
so both need to have a surplus of empty pages.

This commit makes pcpu_nr_empty_pop_pages and the corresponding logic
per chunk type.

[Dennis]
This issue came up as I was reviewing [1] and realized I missed this.
Simultaneously, it was reported btrfs was seeing failed atomic
allocations in fsstress tests [2] and [3].

[1] https://lore.kernel.org/linux-mm/20210324190626.564297-1-guro@fb.com/
[2] https://lore.kernel.org/linux-mm/20210401185158.3275.409509F4@e16-tech.com/
[3] https://lore.kernel.org/linux-mm/CAL3q7H5RNBjCi708GH7jnczAOe0BLnacT9C+OBgA-Dx9jhB6SQ@mail.gmail.com/

Fixes: 3c7be18ac9 ("mm: memcg/percpu: account percpu memory to memory cgroups")
Cc: stable@vger.kernel.org # 5.9+
Signed-off-by: Roman Gushchin <guro@fb.com>
Tested-by: Filipe Manana <fdmanana@suse.com>
Signed-off-by: Dennis Zhou <dennis@kernel.org>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
This commit is contained in:
Roman Gushchin 2021-04-07 20:57:33 -07:00 committed by Greg Kroah-Hartman
parent c441949184
commit efa869b68b
3 changed files with 15 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
mm/percpu-internal.h
View File

@ -87,7 +87,7 @@ extern spinlock_t pcpu_lock;
extern struct list_head *pcpu_chunk_lists; extern struct list_head *pcpu_chunk_lists;
extern int pcpu_nr_slots; extern int pcpu_nr_slots;
extern int pcpu_nr_empty_pop_pages; extern int pcpu_nr_empty_pop_pages[];
extern struct pcpu_chunk *pcpu_first_chunk; extern struct pcpu_chunk *pcpu_first_chunk;
extern struct pcpu_chunk *pcpu_reserved_chunk; extern struct pcpu_chunk *pcpu_reserved_chunk;

9
mm/percpu-stats.c
View File

@ -145,6 +145,7 @@ static int percpu_stats_show(struct seq_file *m, void *v)
int slot, max_nr_alloc; int slot, max_nr_alloc;
int *buffer; int *buffer;
enum pcpu_chunk_type type; enum pcpu_chunk_type type;
int nr_empty_pop_pages;
alloc_buffer: alloc_buffer:
spin_lock_irq(&pcpu_lock); spin_lock_irq(&pcpu_lock);
@ -165,7 +166,11 @@ static int percpu_stats_show(struct seq_file *m, void *v)
goto alloc_buffer; goto alloc_buffer;
} }
#define PL(X) \ nr_empty_pop_pages = 0;
for (type = 0; type < PCPU_NR_CHUNK_TYPES; type++)
nr_empty_pop_pages += pcpu_nr_empty_pop_pages[type];
#define PL(X) \
seq_printf(m, " %-20s: %12lld\n", #X, (long long int)pcpu_stats_ai.X) seq_printf(m, " %-20s: %12lld\n", #X, (long long int)pcpu_stats_ai.X)
seq_printf(m, seq_printf(m,
@ -196,7 +201,7 @@ static int percpu_stats_show(struct seq_file *m, void *v)
PU(nr_max_chunks); PU(nr_max_chunks);
PU(min_alloc_size); PU(min_alloc_size);
PU(max_alloc_size); PU(max_alloc_size);
P("empty_pop_pages", pcpu_nr_empty_pop_pages); P("empty_pop_pages", nr_empty_pop_pages);
seq_putc(m, '\n'); seq_putc(m, '\n');
#undef PU #undef PU

14
mm/percpu.c
View File

@ -172,10 +172,10 @@ struct list_head *pcpu_chunk_lists __ro_after_init; /* chunk list slots */
static LIST_HEAD(pcpu_map_extend_chunks); static LIST_HEAD(pcpu_map_extend_chunks);
/* /*
* The number of empty populated pages, protected by pcpu_lock. The * The number of empty populated pages by chunk type, protected by pcpu_lock.
* reserved chunk doesn't contribute to the count. * The reserved chunk doesn't contribute to the count.
*/ */
int pcpu_nr_empty_pop_pages; int pcpu_nr_empty_pop_pages[PCPU_NR_CHUNK_TYPES];
/* /*
* The number of populated pages in use by the allocator, protected by * The number of populated pages in use by the allocator, protected by
@ -555,7 +555,7 @@ static inline void pcpu_update_empty_pages(struct pcpu_chunk *chunk, int nr)
{ {
chunk->nr_empty_pop_pages += nr; chunk->nr_empty_pop_pages += nr;
if (chunk != pcpu_reserved_chunk) if (chunk != pcpu_reserved_chunk)
pcpu_nr_empty_pop_pages += nr; pcpu_nr_empty_pop_pages[pcpu_chunk_type(chunk)] += nr;
} }
/* /*
@ -1831,7 +1831,7 @@ static void __percpu *pcpu_alloc(size_t size, size_t align, bool reserved,
mutex_unlock(&pcpu_alloc_mutex); mutex_unlock(&pcpu_alloc_mutex);
} }
if (pcpu_nr_empty_pop_pages < PCPU_EMPTY_POP_PAGES_LOW) if (pcpu_nr_empty_pop_pages[type] < PCPU_EMPTY_POP_PAGES_LOW)
pcpu_schedule_balance_work(); pcpu_schedule_balance_work();
/* clear the areas and return address relative to base address */ /* clear the areas and return address relative to base address */
@ -1999,7 +1999,7 @@ static void __pcpu_balance_workfn(enum pcpu_chunk_type type)
pcpu_atomic_alloc_failed = false; pcpu_atomic_alloc_failed = false;
} else { } else {
nr_to_pop = clamp(PCPU_EMPTY_POP_PAGES_HIGH - nr_to_pop = clamp(PCPU_EMPTY_POP_PAGES_HIGH -
pcpu_nr_empty_pop_pages, pcpu_nr_empty_pop_pages[type],
0, PCPU_EMPTY_POP_PAGES_HIGH); 0, PCPU_EMPTY_POP_PAGES_HIGH);
} }
@ -2579,7 +2579,7 @@ void __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai,
/* link the first chunk in */ /* link the first chunk in */
pcpu_first_chunk = chunk; pcpu_first_chunk = chunk;
pcpu_nr_empty_pop_pages = pcpu_first_chunk->nr_empty_pop_pages; pcpu_nr_empty_pop_pages[PCPU_CHUNK_ROOT] = pcpu_first_chunk->nr_empty_pop_pages;
pcpu_chunk_relocate(pcpu_first_chunk, -1); pcpu_chunk_relocate(pcpu_first_chunk, -1);
/* include all regions of the first chunk */ /* include all regions of the first chunk */