// tiny_heap_box.h - TinyHeap front (mimalloc 風) の共通 Box // 役割: // - クラスごとの TinyHeap コンテキスト(current/partial/full + freelist)を TLS で保持。 // - ホットパス alloc/free は TinyHeap 内で完結させ、ページ枯渇/全 free の境界でのみ // Superslab/Tier/Warm などの下層 Box に触れる。 // - C7HotBox はこの TinyHeapBox 上のクラス7ラッパとして実装する。 #pragma once #include #include #include #include #include #include #include "../hakmem_build_flags.h" #include "../hakmem_tiny_superslab.h" // SuperSlab, TinySlabMeta, superslab_refill #include "../hakmem_super_registry.h" // hak_super_lookup #include "../superslab/superslab_inline.h" // slab_index_for #include "../tiny_tls.h" // TinyTLSSlab #include "../tiny_box_geometry.h" // tiny_stride_for_class #include "../tiny_region_id.h" // tiny_region_id_write_header #include "tiny_layout_box.h" // tiny_user_offset #include "tiny_next_ptr_box.h" // tiny_next_read/write // Forward decls for SuperSlab active counters (definitions in hakmem_tiny_ss_active_box.inc) void ss_active_add(SuperSlab* ss, uint32_t n); #ifndef TINY_HEAP_MAX_PAGES_PER_CLASS #define TINY_HEAP_MAX_PAGES_PER_CLASS 4 #endif typedef struct tiny_heap_page_t { void* free_list; // ページ内 free list 先頭 (BASE) uint16_t used; // 現在の使用数(active count) uint16_t capacity; // ページ内の総ブロック数 uint16_t slab_idx; // 対応する slab index uint16_t _pad; uint8_t* base; // slab 先頭(データ領域先頭) TinySlabMeta* meta; // Superslab メタ(owner/Tier 判定用) SuperSlab* ss; // 所有する Superslab struct tiny_heap_page_t* next; int32_t c7_active_delta; // C7 meta-light 用: total_active_blocks の差分 int32_t c7_used_delta; // C7 meta-light 用: meta->used の差分 } tiny_heap_page_t; typedef struct tiny_heap_class_t { tiny_heap_page_t* current_page; tiny_heap_page_t* partial_pages; tiny_heap_page_t* full_pages; tiny_heap_page_t nodes[TINY_HEAP_MAX_PAGES_PER_CLASS]; uint8_t node_in_use[TINY_HEAP_MAX_PAGES_PER_CLASS]; uint16_t stride; // cached block stride for this class uint16_t _pad; } tiny_heap_class_t; typedef struct tiny_heap_ctx_t { tiny_heap_class_t cls[TINY_NUM_CLASSES]; uint8_t initialized; } tiny_heap_ctx_t; // TLS state (定義は core/hakmem_tiny.c) extern __thread tiny_heap_ctx_t g_tiny_heap_ctx; extern __thread int g_tiny_heap_ctx_init; extern __thread TinyTLSSlab g_tls_slabs[TINY_NUM_CLASSES]; static inline int tiny_c7_heap_stats_enabled(void) { static int g = -1; if (__builtin_expect(g == -1, 0)) { const char* e = getenv("HAKMEM_TINY_C7_HEAP_STATS"); g = (e && *e && *e != '0') ? 1 : 0; } return g; } static inline int tiny_c7_meta_light_enabled(void) { static int g = -1; if (__builtin_expect(g == -1, 0)) { const char* e = getenv("HAKMEM_TINY_C7_META_LIGHT"); g = (e && *e && *e != '0') ? 1 : 0; } return g; } static inline int tiny_c7_delta_debug_enabled(void) { static int g = -1; if (__builtin_expect(g == -1, 0)) { const char* e = getenv("HAKMEM_TINY_C7_DELTA_DEBUG"); g = (e && *e && *e != '0') ? 1 : 0; } return g; } typedef struct { _Atomic uint64_t alloc_fast_current; _Atomic uint64_t alloc_slow_prepare; _Atomic uint64_t free_fast_local; _Atomic uint64_t free_slow_fallback; _Atomic uint64_t alloc_prepare_fail; _Atomic uint64_t alloc_fail; } TinyC7HeapStats; extern TinyC7HeapStats g_c7_heap_stats; static inline int tiny_heap_cold_drain_and_free(int class_idx, void* base) { (void)class_idx; (void)base; return 0; } static inline tiny_heap_ctx_t* tiny_heap_ctx_for_thread(void) { tiny_heap_ctx_t* ctx = &g_tiny_heap_ctx; if (!g_tiny_heap_ctx_init) { memset(ctx, 0, sizeof(*ctx)); g_tiny_heap_ctx_init = 1; ctx->initialized = 1; for (int c = 0; c < TINY_NUM_CLASSES; c++) { tiny_heap_class_t* hcls = &ctx->cls[c]; hcls->stride = (uint16_t)tiny_stride_for_class(c); } } return ctx; } static inline tiny_heap_class_t* tiny_heap_class(tiny_heap_ctx_t* ctx, int class_idx) { if (!ctx || class_idx < 0 || class_idx >= TINY_NUM_CLASSES) return NULL; return &ctx->cls[class_idx]; } static inline size_t tiny_heap_block_stride(int class_idx) { return tiny_stride_for_class(class_idx); } static inline void tiny_heap_page_clear(tiny_heap_page_t* page) { if (!page) return; page->free_list = NULL; page->used = 0; page->capacity = 0; page->slab_idx = 0; page->base = NULL; page->meta = NULL; page->ss = NULL; page->next = NULL; page->c7_active_delta = 0; page->c7_used_delta = 0; } static inline void tiny_c7_meta_flush_page(tiny_heap_page_t* page) { if (!page || !tiny_c7_meta_light_enabled()) return; if (!page->meta || !page->ss) return; int32_t active_delta = page->c7_active_delta; int32_t used_delta = page->c7_used_delta; if (active_delta == 0 && used_delta == 0) { return; } if (used_delta != 0) { atomic_fetch_add_explicit(&page->meta->used, used_delta, memory_order_relaxed); page->c7_used_delta = 0; } if (active_delta != 0) { if (active_delta > 0) { ss_active_add(page->ss, (uint32_t)active_delta); } else { int32_t n = -active_delta; for (int32_t i = 0; i < n; i++) { ss_active_dec_one(page->ss); } } page->c7_active_delta = 0; } } static inline int tiny_c7_delta_should_flush(tiny_heap_page_t* page) { if (!page) return 0; if (!tiny_c7_meta_light_enabled()) return 0; int32_t ud = page->c7_used_delta; int32_t ad = page->c7_active_delta; int32_t abs_ud = (ud >= 0) ? ud : -ud; int32_t abs_ad = (ad >= 0) ? ad : -ad; int32_t abs_max = (abs_ud > abs_ad) ? abs_ud : abs_ad; if (abs_max == 0) { return 0; } uint16_t cap = page->capacity; int32_t base_th = (cap > 0) ? ((int32_t)cap * 16) : 256; if (base_th < 256) { base_th = 256; } return abs_max >= base_th; } static __attribute__((noinline, unused)) void tiny_c7_heap_debug_dump_deltas(void) { if (!tiny_c7_meta_light_enabled() || !tiny_c7_delta_debug_enabled()) { return; } tiny_heap_ctx_t* ctx = tiny_heap_ctx_for_thread(); int class_idx = 7; tiny_heap_class_t* hcls = tiny_heap_class(ctx, class_idx); if (!hcls) return; uint64_t nonzero_pages = 0; int64_t used_delta_sum = 0; int64_t active_delta_sum = 0; for (int i = 0; i < TINY_HEAP_MAX_PAGES_PER_CLASS; i++) { tiny_heap_page_t* p = &hcls->nodes[i]; if (p->c7_used_delta != 0 || p->c7_active_delta != 0) { nonzero_pages++; used_delta_sum += (int64_t)p->c7_used_delta; active_delta_sum += (int64_t)p->c7_active_delta; fprintf(stderr, "[C7_DELTA_PAGE] idx=%d used_delta=%d active_delta=%d used=%u cap=%u ss=%p slab=%u\n", i, p->c7_used_delta, p->c7_active_delta, (unsigned)p->used, (unsigned)p->capacity, (void*)p->ss, (unsigned)p->slab_idx); } } fprintf(stderr, "[C7_DELTA_SUMMARY] nonzero_pages=%llu used_delta_sum=%lld active_delta_sum=%lld\n", (unsigned long long)nonzero_pages, (long long)used_delta_sum, (long long)active_delta_sum); } static inline tiny_heap_page_t* tiny_heap_class_acquire_node(tiny_heap_class_t* hcls) { if (!hcls) return NULL; for (int i = 0; i < TINY_HEAP_MAX_PAGES_PER_CLASS; i++) { if (hcls->node_in_use[i] == 0) { hcls->node_in_use[i] = 1; tiny_heap_page_t* p = &hcls->nodes[i]; tiny_heap_page_clear(p); return p; } } return NULL; } static inline void tiny_heap_class_release_node(tiny_heap_class_t* hcls, tiny_heap_page_t* page) { if (!hcls || !page) return; if (tiny_c7_meta_light_enabled()) { tiny_c7_meta_flush_page(page); } intptr_t idx = page - hcls->nodes; if (idx >= 0 && idx < TINY_HEAP_MAX_PAGES_PER_CLASS) { hcls->node_in_use[idx] = 0; tiny_heap_page_clear(page); } } static inline tiny_heap_page_t* tiny_heap_class_find_page(tiny_heap_class_t* hcls, SuperSlab* ss, int slab_idx) { if (!hcls || !ss || slab_idx < 0) return NULL; tiny_heap_page_t* lists[] = {hcls->current_page, hcls->partial_pages, hcls->full_pages}; for (size_t li = 0; li < sizeof(lists) / sizeof(lists[0]); li++) { tiny_heap_page_t* p = lists[li]; while (p) { if (p->ss == ss && (int)p->slab_idx == slab_idx) { return p; } p = p->next; } } return NULL; } static inline tiny_heap_page_t* tiny_heap_attach_page(tiny_heap_ctx_t* ctx, int class_idx, SuperSlab* ss, int slab_idx) { tiny_heap_class_t* hcls = tiny_heap_class(ctx, class_idx); if (!hcls || !ss || slab_idx < 0) return NULL; TinySlabMeta* meta = &ss->slabs[slab_idx]; if (meta->class_idx != class_idx) { return NULL; } tiny_heap_page_t* page = tiny_heap_class_find_page(hcls, ss, slab_idx); if (!page) { page = tiny_heap_class_acquire_node(hcls); if (!page) return NULL; page->ss = ss; page->slab_idx = (uint16_t)slab_idx; page->meta = meta; page->base = tiny_slab_base_for(ss, slab_idx); page->capacity = meta->capacity; page->used = atomic_load_explicit(&meta->used, memory_order_relaxed); page->free_list = atomic_load_explicit(&meta->freelist, memory_order_acquire); page->next = hcls->partial_pages; hcls->partial_pages = page; } else { page->capacity = meta->capacity; page->used = atomic_load_explicit(&meta->used, memory_order_relaxed); page->free_list = atomic_load_explicit(&meta->freelist, memory_order_acquire); if (!page->base) { page->base = tiny_slab_base_for(ss, slab_idx); } } if (class_idx == 7 && tiny_c7_meta_light_enabled()) { if (page->capacity > 0 && page->used > page->capacity) { page->used = page->capacity; } page->c7_used_delta = 0; page->c7_active_delta = 0; } return page; } static inline tiny_heap_page_t* tiny_heap_take_current(tiny_heap_class_t* hcls) { if (!hcls) return NULL; if (hcls->current_page) { return hcls->current_page; } if (hcls->partial_pages) { tiny_heap_page_t* page = hcls->partial_pages; hcls->partial_pages = page->next; page->next = NULL; hcls->current_page = page; return page; } return NULL; } static inline void tiny_heap_class_unlink(tiny_heap_class_t* hcls, tiny_heap_page_t* page) { if (!hcls || !page) return; if (hcls->current_page == page) { hcls->current_page = NULL; } tiny_heap_page_t** lists[] = {&hcls->partial_pages, &hcls->full_pages}; for (size_t i = 0; i < sizeof(lists) / sizeof(lists[0]); i++) { tiny_heap_page_t** head = lists[i]; tiny_heap_page_t* prev = NULL; tiny_heap_page_t* cur = head ? *head : NULL; while (cur) { if (cur == page) { if (prev) { prev->next = cur->next; } else { *head = cur->next; } break; } prev = cur; cur = cur->next; } } } static inline tiny_heap_page_t* tiny_heap_page_of(tiny_heap_ctx_t* ctx, int class_idx, void* base_ptr) { if (!ctx || !base_ptr || class_idx < 0 || class_idx >= TINY_NUM_CLASSES) return NULL; tiny_heap_class_t* hcls = tiny_heap_class(ctx, class_idx); if (!hcls) return NULL; // TLS fast path: TLS slab の範囲内なら lookup を避ける TinyTLSSlab* tls = &g_tls_slabs[class_idx]; if (tls->ss && tls->meta && tls->slab_base) { size_t stride = hcls->stride; if (stride == 0) { stride = tiny_heap_block_stride(class_idx); hcls->stride = (uint16_t)stride; } uint16_t cap = tls->meta->capacity; if (cap > 0) { uint8_t* slab_base = tls->slab_base; size_t slab_size = stride * (size_t)cap; uint8_t* base_u8 = (uint8_t*)base_ptr; if (base_u8 >= slab_base && base_u8 < (slab_base + slab_size)) { return tiny_heap_attach_page(ctx, class_idx, tls->ss, tls->slab_idx); } } } // fallback: Superslab lookup SuperSlab* ss = hak_super_lookup(base_ptr); if (!ss || ss->magic != SUPERSLAB_MAGIC) { return NULL; } int slab_idx = slab_index_for(ss, base_ptr); if (slab_idx < 0 || slab_idx >= ss_slabs_capacity(ss)) { return NULL; } return tiny_heap_attach_page(ctx, class_idx, ss, slab_idx); } static inline void tiny_heap_page_push_to_full(tiny_heap_class_t* hcls, tiny_heap_page_t* page) { if (!hcls || !page) return; page->next = hcls->full_pages; hcls->full_pages = page; } static inline void tiny_heap_page_push_to_partial(tiny_heap_class_t* hcls, tiny_heap_page_t* page) { if (!hcls || !page) return; page->next = hcls->partial_pages; hcls->partial_pages = page; } static inline void tiny_heap_page_becomes_empty(tiny_heap_ctx_t* ctx, int class_idx, tiny_heap_page_t* page) { tiny_heap_class_t* hcls = tiny_heap_class(ctx, class_idx); if (!hcls || !page) return; if (class_idx == 7 && tiny_c7_meta_light_enabled()) { tiny_c7_meta_flush_page(page); } if (page->meta && page->ss) { ss_partial_publish(class_idx, page->ss); } tiny_heap_class_unlink(hcls, page); tiny_heap_class_release_node(hcls, page); } static inline void tiny_heap_page_mark_full(tiny_heap_class_t* hcls, tiny_heap_page_t* page) { if (!hcls || !page) return; tiny_heap_class_unlink(hcls, page); tiny_heap_page_push_to_full(hcls, page); } static inline void* tiny_heap_page_pop(tiny_heap_class_t* hcls, int class_idx, tiny_heap_page_t* page) { if (!hcls || !page || !page->meta || !page->ss || !page->base) return NULL; const int meta_light = (class_idx == 7 && tiny_c7_meta_light_enabled()); void* block = NULL; if (page->free_list) { block = page->free_list; void* next = tiny_next_read(class_idx, block); page->free_list = next; atomic_store_explicit(&page->meta->freelist, next, memory_order_release); } else if (page->used < page->capacity) { size_t stride = hcls->stride; if (stride == 0) { stride = tiny_heap_block_stride(class_idx); hcls->stride = (uint16_t)stride; } block = (void*)(page->base + ((size_t)page->used * stride)); if (page->meta->carved < page->capacity) { page->meta->carved++; } } else { return NULL; } page->used++; if (__builtin_expect(meta_light, 0)) { page->c7_used_delta++; page->c7_active_delta++; if (tiny_c7_delta_should_flush(page)) { tiny_c7_meta_flush_page(page); } return tiny_region_id_write_header(block, class_idx); } atomic_fetch_add_explicit(&page->meta->used, 1, memory_order_relaxed); ss_active_add(page->ss, 1u); return tiny_region_id_write_header(block, class_idx); } static inline void tiny_heap_page_push_free(int class_idx, tiny_heap_page_t* page, void* base_ptr) { if (!page || !base_ptr || !page->meta) return; tiny_next_write(class_idx, base_ptr, page->free_list); page->free_list = base_ptr; atomic_store_explicit(&page->meta->freelist, base_ptr, memory_order_release); } static inline void tiny_heap_page_free_local(tiny_heap_ctx_t* ctx, int class_idx, tiny_heap_page_t* page, void* base_ptr) { tiny_heap_class_t* hcls = tiny_heap_class(ctx, class_idx); if (!hcls || !page || !base_ptr) return; const int stats_on = (class_idx == 7 && tiny_c7_heap_stats_enabled()); if (__builtin_expect(stats_on, 0)) { atomic_fetch_add_explicit(&g_c7_heap_stats.free_fast_local, 1, memory_order_relaxed); } const int meta_light = (class_idx == 7 && tiny_c7_meta_light_enabled()); tiny_heap_page_push_free(class_idx, page, base_ptr); if (page->used > 0) { page->used--; if (!__builtin_expect(meta_light, 0)) { atomic_fetch_sub_explicit(&page->meta->used, 1, memory_order_relaxed); ss_active_dec_one(page->ss); } else { page->c7_used_delta--; page->c7_active_delta--; if (tiny_c7_delta_should_flush(page)) { tiny_c7_meta_flush_page(page); } } } if (page->used == 0) { tiny_heap_page_becomes_empty(ctx, class_idx, page); } else if (page->used < page->capacity && page->free_list) { tiny_heap_page_t* old_cur = hcls->current_page; tiny_heap_class_unlink(hcls, page); page->next = NULL; if (class_idx == 7) { hcls->current_page = page; if (old_cur && old_cur != page) { tiny_heap_page_push_to_partial(hcls, old_cur); } } else { if (!hcls->current_page) { hcls->current_page = page; } else { tiny_heap_page_push_to_partial(hcls, page); } } } } static inline tiny_heap_page_t* tiny_heap_prepare_page(tiny_heap_ctx_t* ctx, int class_idx) { tiny_heap_class_t* hcls = tiny_heap_class(ctx, class_idx); const int stats_on = (class_idx == 7 && tiny_c7_heap_stats_enabled()); if (!hcls) { if (__builtin_expect(stats_on, 0)) { atomic_fetch_add_explicit(&g_c7_heap_stats.alloc_prepare_fail, 1, memory_order_relaxed); } return NULL; } tiny_heap_page_t* page = tiny_heap_take_current(hcls); if (page) return page; TinyTLSSlab* tls = &g_tls_slabs[class_idx]; if (!tls->ss) { if (superslab_refill(class_idx) == NULL) { if (__builtin_expect(stats_on, 0)) { atomic_fetch_add_explicit(&g_c7_heap_stats.alloc_prepare_fail, 1, memory_order_relaxed); } return NULL; } } tls = &g_tls_slabs[class_idx]; // superslab_refill で更新されるため再取得 if (!tls->ss || !tls->meta) { if (__builtin_expect(stats_on, 0)) { atomic_fetch_add_explicit(&g_c7_heap_stats.alloc_prepare_fail, 1, memory_order_relaxed); } return NULL; } page = tiny_heap_attach_page(ctx, class_idx, tls->ss, tls->slab_idx); if (page) { tiny_heap_class_unlink(hcls, page); page->next = NULL; hcls->current_page = page; } return page; } // class_idx 固定での alloc ホットパス static inline void* tiny_heap_alloc_slow_from_class(tiny_heap_ctx_t* ctx, int class_idx) { tiny_heap_class_t* hcls = tiny_heap_class(ctx, class_idx); if (!hcls) return NULL; const int stats_on = (class_idx == 7 && tiny_c7_heap_stats_enabled()); tiny_heap_page_t* page = tiny_heap_prepare_page(ctx, class_idx); if (!page) { if (__builtin_expect(stats_on, 0)) { atomic_fetch_add_explicit(&g_c7_heap_stats.alloc_fail, 1, memory_order_relaxed); } return NULL; } void* user = tiny_heap_page_pop(hcls, class_idx, page); if (user && page->used >= page->capacity && page->free_list == NULL) { tiny_heap_page_mark_full(hcls, page); } return user; } // class_idx 固定での alloc ホットパス __attribute__((always_inline)) static inline void* tiny_heap_alloc_class_fast(tiny_heap_ctx_t* ctx, int class_idx, size_t size) { (void)size; tiny_heap_class_t* hcls = tiny_heap_class(ctx, class_idx); tiny_heap_page_t* page = hcls ? hcls->current_page : NULL; const int stats_on = (class_idx == 7 && tiny_c7_heap_stats_enabled()); if (page) { if (page->free_list || page->used < page->capacity) { void* user = tiny_heap_page_pop(hcls, class_idx, page); if (user) { if (page->used >= page->capacity && page->free_list == NULL) { tiny_heap_page_mark_full(hcls, page); } if (__builtin_expect(stats_on, 0)) { atomic_fetch_add_explicit(&g_c7_heap_stats.alloc_fast_current, 1, memory_order_relaxed); } return user; } } } if (__builtin_expect(stats_on, 0)) { atomic_fetch_add_explicit(&g_c7_heap_stats.alloc_slow_prepare, 1, memory_order_relaxed); } return tiny_heap_alloc_slow_from_class(ctx, class_idx); } // Superslab/Slab メタが既に分かっている場合の free(Gate から渡されるホットパス用) static inline void tiny_heap_free_class_fast_with_meta(tiny_heap_ctx_t* ctx, int class_idx, SuperSlab* ss, int slab_idx, void* base) { tiny_heap_class_t* hcls = tiny_heap_class(ctx, class_idx); if (!base || !ss || slab_idx < 0 || !hcls) return; const int stats_on = (class_idx == 7 && tiny_c7_heap_stats_enabled()); tiny_heap_page_t* page = tiny_heap_attach_page(ctx, class_idx, ss, slab_idx); if (!page) { if (__builtin_expect(stats_on, 0)) { atomic_fetch_add_explicit(&g_c7_heap_stats.free_slow_fallback, 1, memory_order_relaxed); } tiny_heap_cold_drain_and_free(class_idx, base); return; } tiny_heap_page_free_local(ctx, class_idx, page, base); } // class_idx 固定での free ホットパス(ptr は USER ポインタ) static inline void tiny_heap_free_class_fast(tiny_heap_ctx_t* ctx, int class_idx, void* ptr) { if (!ptr) return; #if HAKMEM_TINY_HEADERLESS void* base = ptr; #else void* base = (void*)((uint8_t*)ptr - 1); #endif SuperSlab* ss = hak_super_lookup(base); if (!ss || ss->magic != SUPERSLAB_MAGIC) { if (__builtin_expect((class_idx == 7 && tiny_c7_heap_stats_enabled()), 0)) { atomic_fetch_add_explicit(&g_c7_heap_stats.free_slow_fallback, 1, memory_order_relaxed); } tiny_heap_cold_drain_and_free(class_idx, base); return; } int slab_idx = slab_index_for(ss, base); if (slab_idx < 0 || slab_idx >= ss_slabs_capacity(ss)) { if (__builtin_expect((class_idx == 7 && tiny_c7_heap_stats_enabled()), 0)) { atomic_fetch_add_explicit(&g_c7_heap_stats.free_slow_fallback, 1, memory_order_relaxed); } tiny_heap_cold_drain_and_free(class_idx, base); return; } tiny_heap_free_class_fast_with_meta(ctx, class_idx, ss, slab_idx, base); }