Expose mi_theap_heap function

Allows accessing the mi_heap_t* that owns the mi_theap_t*.

The internal function has been renamed to _mi_theap_heap,
and all existing usages have been updated to use that.
This means the internal usages are still eligible for inlining.

Author: Techcable

contrib/rust/sys/src/lib.rs

@@ -438,6 +438,7 @@ unsafe extern "C" {
 
     // thread-local heaps
     pub fn mi_heap_theap(heap: *mut mi_heap_t) -> *mut mi_theap_t;
+    pub fn mi_theap_heap(heap: *const mi_theap_t) -> *mut mi_heap_t;
     pub fn mi_theap_set_default(theap: *mut mi_theap_t) -> *mut mi_theap_t;
     pub safe fn mi_theap_get_default() -> *mut mi_theap_t;
     pub fn mi_theap_collect(theap: *mut mi_theap_t, force: bool);

doc/mimalloc-doc.h

@@ -1410,6 +1410,11 @@ typedef struct mi_theap_s mi_theap_t;
 /// the theap at each allocation call.
 mi_theap_t* mi_heap_theap(mi_heap_t* heap);
 
+/// @brief Return the owning heap for the given thread-local heap.
+/// @param theap the thread-local heap
+/// @return The heap that owns the `theap`
+mi_theap_t* mi_theap_heap(const mi_heap_t* heap);
+
 /// Release outstanding resources in a specific theap.
 void mi_theap_collect(mi_theap_t* theap, bool force);
 

include/mimalloc.h

@@ -392,6 +392,7 @@ struct mi_theap_s;
 typedef struct mi_theap_s mi_theap_t;
 
 mi_decl_export mi_theap_t* mi_heap_theap(mi_heap_t* heap);
+mi_decl_export mi_heap_t*  mi_theap_heap(const mi_theap_t* theap);
 mi_decl_export mi_theap_t* mi_theap_set_default(mi_theap_t* theap);
 mi_decl_export mi_theap_t* mi_theap_get_default(void);
 mi_decl_export void        mi_theap_collect(mi_theap_t* theap, bool force) mi_attr_noexcept;

include/mimalloc/heap/theap.h

@@ -36,7 +36,7 @@ static inline mi_heap_t* mi_theap_heap_peek(const mi_theap_t* theap) {
   return mi_atomic_load_ptr_relaxed(mi_heap_t,&theap->heap);
 }
 
-static inline mi_heap_t* mi_theap_heap(const mi_theap_t* theap) {
+static inline mi_heap_t* _mi_theap_heap(const mi_theap_t* theap) {
   mi_heap_t* const heap = mi_theap_heap_peek(theap);
   mi_assert_internal(heap!=NULL);
   return heap;
@@ -48,7 +48,7 @@ static inline bool mi_theap_is_initialized(const mi_theap_t* theap) {
 
 static inline mi_subproc_t* mi_theap_subproc(const mi_theap_t* theap) {
   mi_subproc_t* const subproc = mi_atomic_load_ptr_relaxed(mi_subproc_t,&theap->subproc);
-  mi_assert_internal(!mi_theap_is_initialized(theap) || mi_theap_heap(theap)->subproc == subproc);
+  mi_assert_internal(!mi_theap_is_initialized(theap) || _mi_theap_heap(theap)->subproc == subproc);
   return subproc;
 }
 

src/alloc/alloc-generic.c

@@ -343,7 +343,7 @@ static mi_theap_t* mi_malloc_generic_admin(mi_theap_t* theap)
 
     // check if the profiler is enabled (unless this theap is permanently excluded, see `mi_theap_profile_never`)
     if (!theap->profile_never) {
-      mi_heap_t* const heap = mi_theap_heap(theap);
+      mi_heap_t* const heap = _mi_theap_heap(theap);
       mi_profiler_t* prof = mi_atomic_load_ptr_acquire(mi_profiler_t, &heap->profiler);
       const bool prof_enabled = (prof!=NULL && mi_profiler_is_enabled(prof));
       if (prof_enabled && theap->profile_sample_rate==0) {

src/alloc/free.c

@@ -615,7 +615,7 @@ static mi_decl_noinline bool mi_abandoned_page_try_reclaim(mi_page_t* page, long
   else if (reclaim_on_free == 1 &&               // if cross-thread is allowed
             !theap->tld->is_in_threadpool &&     // and we are not part of a threadpool
             !mi_page_is_used_at_frac(page,4) &&  // and at least 1/4th of the page is free
-            _mi_arena_memid_is_suitable(page->memid, mi_theap_heap(theap)->exclusive_arena)) {   // and it fits our memory
+            _mi_arena_memid_is_suitable(page->memid, _mi_theap_heap(theap)->exclusive_arena)) {   // and it fits our memory
     // across threads
     max_reclaim = _mi_option_get_fast(mi_option_page_cross_thread_max_reclaim);
   }

src/alloc/page.c

@@ -929,7 +929,7 @@ bool _mi_page_extend_free(mi_theap_t* theap, mi_page_t* page) {
 mi_decl_nodiscard bool _mi_page_init(mi_theap_t* theap, mi_page_t* page) {
   mi_assert(page != NULL);
   mi_assert(theap!=NULL);
-  // page->heap = (mi_is_heap_main(mi_theap_heap(theap)) ? NULL : mi_theap_heap(theap)); // faster for `mi_page_associated_theap`
+  // page->heap = (mi_is_heap_main(_mi_theap_heap(theap)) ? NULL : _mi_theap_heap(theap)); // faster for `mi_page_associated_theap`
   // mi_page_set_theap(page, theap);
 
   size_t page_size;
@@ -951,7 +951,7 @@ mi_decl_nodiscard bool _mi_page_init(mi_theap_t* theap, mi_page_t* page) {
   #endif
 
   mi_assert_internal(page->heap != NULL);
-  mi_assert_internal(page->heap == mi_theap_heap(theap));
+  mi_assert_internal(page->heap == _mi_theap_heap(theap));
   mi_assert_internal(page->theap!=NULL);
   mi_assert_internal(page->theap == mi_page_theap(page));
   mi_assert_internal(page->capacity == 0);

src/arena/arena-page.c

@@ -176,7 +176,7 @@ static mi_page_guards_t mi_theap_page_guards(const mi_theap_t* theap, size_t siz
   const size_t gsize = _mi_os_secure_guard_page_size();
   const bool guard_end = (theap->guard_end || (singleton && MI_SECURE >= 1));  // in secure mode, huge blocks are always followed by a guard page
   mi_page_guards_t guards = { 0, 0 };
-  const mi_arena_t* const req_arena = mi_theap_heap(theap)->exclusive_arena;
+  const mi_arena_t* const req_arena = _mi_theap_heap(theap)->exclusive_arena;
   if (req_arena != NULL && req_arena->memid.is_pinned) return guards;
   if (theap->guard_front && !os_align && gsize <= size/4) { guards.front = gsize; }
   if (guard_end && guards.front + gsize <= size/4) { guards.end = gsize; }
@@ -235,7 +235,7 @@ static bool mi_arena_try_find_abandoned_visit(mi_arena_t* arena, const void* arg
   const mi_arena_abandoned_visit_info_t* info = (const mi_arena_abandoned_visit_info_t*)arg;
   mi_assert_internal(result != NULL);
   mi_theap_t* const theap = info->theap;
-  mi_heap_t* const heap = mi_theap_heap(theap);
+  mi_heap_t* const heap = _mi_theap_heap(theap);
   const size_t tseq = theap->tld->thread_seq;
   mi_arena_pages_t* const arena_pages = mi_heap_arena_pages(heap, arena);
   if (arena_pages != NULL) {
@@ -270,7 +270,7 @@ static bool mi_arena_try_find_abandoned_visit(mi_arena_t* arena, const void* arg
 
 static mi_page_t* mi_arenas_page_try_find_abandoned(mi_theap_t* theap, size_t slice_count, size_t block_size, mi_page_guards_t guards)
 {
-  mi_heap_t* const heap = mi_theap_heap(theap);
+  mi_heap_t* const heap = _mi_theap_heap(theap);
   const size_t tseq = theap->tld->thread_seq;
   mi_arena_t* const req_arena = heap->exclusive_arena;
 
@@ -442,7 +442,7 @@ static bool mi_arena_page_meta_ensure_committed(mi_arena_t* arena, uint8_t* slic
 // Try to allocate slices from free space in the arena's, and return the separate page meta data (if available) in `*ppage_meta`.
 static uint8_t* mi_arenas_page_alloc_from_slices(mi_theap_t* theap, size_t slice_count, bool commit, bool allow_large,
                                                  mi_memid_t* memid, mi_arena_pages_t** parena_pages, mi_page_t** ppage_meta) {
-  mi_heap_t* const heap = mi_theap_heap(theap);
+  mi_heap_t* const heap = _mi_theap_heap(theap);
   mi_tld_t*  const tld  = theap->tld;
   const int numa_node = (heap->numa_node >= 0 ? heap->numa_node : tld->numa_node);
 
@@ -571,7 +571,7 @@ static uint8_t* mi_arenas_page_alloc_fresh_area(mi_theap_t* theap, size_t slice_
   *parena_pages = NULL;
   *ppage_meta = NULL;
   *memid = _mi_memid_none();
-  mi_heap_t* const heap = mi_theap_heap(theap);
+  mi_heap_t* const heap = _mi_theap_heap(theap);
 
   // try to allocate from free space in arena's
   uint8_t* slice_start = NULL;
@@ -697,7 +697,7 @@ static void mi_arenas_page_init_meta(mi_theap_t* theap, mi_page_t* page, uint8_t
   page->free_is_zero = memid.initially_zero;
   page->slice_pcommitted = (uint16_t)(commit_size / _mi_os_page_size());
 
-  page->heap = mi_theap_heap(theap);
+  page->heap = _mi_theap_heap(theap);
   if (block_size <= MI_LARGE_MAX_OBJ_SIZE && mi_memtag_use_for(memid.is_memtag, block_size)) {  // huge blocks are not tagged (but have an end guard page in secure mode); over-aligned singletons up to this size are tagged
     mi_page_flags_set(page, true, MI_PAGE_IS_MEMTAG);
   }
@@ -1392,7 +1392,7 @@ static bool mi_heap_delete_page(const mi_heap_t* heap, const mi_heap_area_t* are
   MI_UNUSED(block); MI_UNUSED(block_size); MI_UNUSED(heap);
   const mi_heap_delete_visit_info_t* info = (const mi_heap_delete_visit_info_t*)arg;
   mi_heap_t*  heap_target           = info->heap_target;
-  mi_theap_t* const theap           = NULL; // info->theap;       mi_assert_internal(mi_theap_heap(theap) == heap);
+  mi_theap_t* const theap           = NULL; // info->theap;       mi_assert_internal(_mi_theap_heap(theap) == heap);
   mi_page_t*  const page            = mi_heap_area_page(area);
 
   mi_page_claim_ownership(page);       // claim ownership

src/heap/heap.c

@@ -45,7 +45,7 @@ void mi_heap_stats_merge_to_main(mi_heap_t* heap) {
 }
 
 bool _mi_heap_theap_set(mi_heap_t* heap, mi_theap_t* theap) {
-  mi_assert_internal(mi_addr(theap) == 1 || mi_theap_heap(theap)==heap);
+  mi_assert_internal(mi_addr(theap) == 1 || _mi_theap_heap(theap)==heap);
   mi_assert_internal(!_mi_is_empty_theap(theap));
   mi_assert_internal(heap->theap != 0);
   return _mi_thread_local_set(heap->theap,theap);

src/heap/process.c

@@ -531,7 +531,7 @@ static void mi_thread_theaps_done(mi_tld_t* tld)
       mi_theap_t* next = theap->tnext;
       // never destroy theaps; if a dll is linked statically with mimalloc,
       // there may still be delete/free calls after the mi_fls_done is called. Issue #207
-      mi_heap_t* const heap = mi_theap_heap(theap);
+      mi_heap_t* const heap = _mi_theap_heap(theap);
       mi_lock(&heap->theaps_lock) { // keep page queues stable during a heap walk
         _mi_theap_collect_abandon(theap);
       }