From 98d5fcb8d099a1a868e032c89891c395a2f365c5 Mon Sep 17 00:00:00 2001 From: Adhemerval Zanella Date: Mon, 16 Aug 2021 15:08:27 -0300 Subject: malloc: Add Huge Page support for mmap With the morecore hook removed, there is not easy way to provide huge pages support on with glibc allocator without resorting to transparent huge pages. And some users and programs do prefer to use the huge pages directly instead of THP for multiple reasons: no splitting, re-merging by the VM, no TLB shootdowns for running processes, fast allocation from the reserve pool, no competition with the rest of the processes unlike THP, no swapping all, etc. This patch extends the 'glibc.malloc.hugetlb' tunable: the value '2' means to use huge pages directly with the system default size, while a positive value means and specific page size that is matched against the supported ones by the system. Currently only memory allocated on sysmalloc() is handled, the arenas still uses the default system page size. To test is a new rule is added tests-malloc-hugetlb2, which run the addes tests with the required GLIBC_TUNABLE setting. On systems without a reserved huge pages pool, is just stress the mmap(MAP_HUGETLB) allocation failure. To improve test coverage it is required to create a pool with some allocated pages. Checked on x86_64-linux-gnu. Reviewed-by: DJ Delorie --- malloc/Makefile | 8 +++++--- malloc/arena.c | 4 ++-- malloc/malloc.c | 31 +++++++++++++++++++++++++++---- 3 files changed, 34 insertions(+), 9 deletions(-) (limited to 'malloc') diff --git a/malloc/Makefile b/malloc/Makefile index 0137595e17..e9a6666d22 100644 --- a/malloc/Makefile +++ b/malloc/Makefile @@ -78,9 +78,9 @@ tests-exclude-malloc-check = tst-malloc-check tst-malloc-usable \ tests-malloc-check = $(filter-out $(tests-exclude-malloc-check) \ $(tests-static),$(tests)) -# Run all testes with GLIBC_TUNABLES=glibc.malloc.hugetlb=1 that check the -# Transparent Huge Pages support. We need exclude some tests that define -# the ENV vars. +# Run all tests with GLIBC_TUNABLES=glibc.malloc.hugetlb={1,2} which check +# the Transparent Huge Pages support (1) or automatic huge page support (2). +# We need exclude some tests that define the ENV vars. tests-exclude-hugetlb1 = \ tst-compathooks-off \ tst-compathooks-on \ @@ -93,6 +93,8 @@ tests-exclude-hugetlb1 = \ tst-mallocstate tests-malloc-hugetlb1 = \ $(filter-out $(tests-exclude-hugetlb1), $(tests)) +tests-malloc-hugetlb2 = \ + $(filter-out $(tests-exclude-hugetlb1), $(tests)) # -lmcheck needs __malloc_initialize_hook, which was deprecated in 2.24. ifeq ($(have-GLIBC_2.23)$(build-shared),yesyes) diff --git a/malloc/arena.c b/malloc/arena.c index cd00c7bef4..9a6e1af2bd 100644 --- a/malloc/arena.c +++ b/malloc/arena.c @@ -230,7 +230,7 @@ TUNABLE_CALLBACK_FNDECL (set_tcache_count, size_t) TUNABLE_CALLBACK_FNDECL (set_tcache_unsorted_limit, size_t) #endif TUNABLE_CALLBACK_FNDECL (set_mxfast, size_t) -TUNABLE_CALLBACK_FNDECL (set_hugetlb, int32_t) +TUNABLE_CALLBACK_FNDECL (set_hugetlb, size_t) #else /* Initialization routine. */ #include @@ -331,7 +331,7 @@ ptmalloc_init (void) TUNABLE_CALLBACK (set_tcache_unsorted_limit)); # endif TUNABLE_GET (mxfast, size_t, TUNABLE_CALLBACK (set_mxfast)); - TUNABLE_GET (hugetlb, int32_t, TUNABLE_CALLBACK (set_hugetlb)); + TUNABLE_GET (hugetlb, size_t, TUNABLE_CALLBACK (set_hugetlb)); #else if (__glibc_likely (_environ != NULL)) { diff --git a/malloc/malloc.c b/malloc/malloc.c index 6b6ec53db1..75efdc2ee7 100644 --- a/malloc/malloc.c +++ b/malloc/malloc.c @@ -1883,6 +1883,10 @@ struct malloc_par #if HAVE_TUNABLES /* Transparent Large Page support. */ INTERNAL_SIZE_T thp_pagesize; + /* A value different than 0 means to align mmap allocation to hp_pagesize + add hp_flags on flags. */ + INTERNAL_SIZE_T hp_pagesize; + int hp_flags; #endif /* Memory map support */ @@ -2440,7 +2444,10 @@ sysmalloc_mmap (INTERNAL_SIZE_T nb, size_t pagesize, int extra_flags, mstate av) if (mm == MAP_FAILED) return mm; - madvise_thp (mm, size); +#ifdef MAP_HUGETLB + if (!(extra_flags & MAP_HUGETLB)) + madvise_thp (mm, size); +#endif /* The offset to the start of the mmapped region is stored in the prev_size @@ -2528,7 +2535,18 @@ sysmalloc (INTERNAL_SIZE_T nb, mstate av) || ((unsigned long) (nb) >= (unsigned long) (mp_.mmap_threshold) && (mp_.n_mmaps < mp_.n_mmaps_max))) { - char *mm = sysmalloc_mmap (nb, pagesize, 0, av); + char *mm; +#if HAVE_TUNABLES + if (mp_.hp_pagesize > 0 && nb >= mp_.hp_pagesize) + { + /* There is no need to isse the THP madvise call if Huge Pages are + used directly. */ + mm = sysmalloc_mmap (nb, mp_.hp_pagesize, mp_.hp_flags, av); + if (mm != MAP_FAILED) + return mm; + } +#endif + mm = sysmalloc_mmap (nb, pagesize, 0, av); if (mm != MAP_FAILED) return mm; tried_mmap = true; @@ -2609,7 +2627,9 @@ sysmalloc (INTERNAL_SIZE_T nb, mstate av) } else if (!tried_mmap) { - /* We can at least try to use to mmap memory. */ + /* We can at least try to use to mmap memory. If new_heap fails + it is unlikely that trying to allocate huge pages will + succeed. */ char *mm = sysmalloc_mmap (nb, pagesize, 0, av); if (mm != MAP_FAILED) return mm; @@ -5383,7 +5403,7 @@ do_set_mxfast (size_t value) #if HAVE_TUNABLES static __always_inline int -do_set_hugetlb (int32_t value) +do_set_hugetlb (size_t value) { if (value == 1) { @@ -5395,6 +5415,9 @@ do_set_hugetlb (int32_t value) if (thp_mode == malloc_thp_mode_madvise) mp_.thp_pagesize = __malloc_default_thp_pagesize (); } + else if (value >= 2) + __malloc_hugepage_config (value == 2 ? 0 : value, &mp_.hp_pagesize, + &mp_.hp_flags); return 0; } #endif -- cgit v1.2.3