656 lines
23 KiB
C
656 lines
23 KiB
C
// tiny_heap_box.h - TinyHeap front (mimalloc 風) の共通 Box
|
||
// 役割:
|
||
// - クラスごとの TinyHeap コンテキスト（current/partial/full + freelist）を TLS で保持。
|
||
// - ホットパス alloc/free は TinyHeap 内で完結させ、ページ枯渇/全 free の境界でのみ
|
||
// Superslab/Tier/Warm などの下層 Box に触れる。
|
||
// - C7HotBox はこの TinyHeapBox 上のクラス7ラッパとして実装する。
|
||
#pragma once
|
||
|
||
#include <stdint.h>
|
||
#include <stddef.h>
|
||
#include <stdatomic.h>
|
||
#include <stdlib.h>
|
||
#include <string.h>
|
||
#include <stdio.h>
|
||
|
||
#include "../hakmem_build_flags.h"
|
||
#include "../hakmem_tiny_superslab.h" // SuperSlab, TinySlabMeta, superslab_refill
|
||
#include "../hakmem_super_registry.h" // hak_super_lookup
|
||
#include "../superslab/superslab_inline.h" // slab_index_for
|
||
#include "../tiny_tls.h" // TinyTLSSlab
|
||
#include "../tiny_box_geometry.h" // tiny_stride_for_class
|
||
#include "../tiny_region_id.h" // tiny_region_id_write_header
|
||
#include "tiny_layout_box.h" // tiny_user_offset
|
||
#include "tiny_next_ptr_box.h" // tiny_next_read/write
|
||
|
||
// Forward decls for SuperSlab active counters (definitions in hakmem_tiny_ss_active_box.inc)
|
||
void ss_active_add(SuperSlab* ss, uint32_t n);
|
||
|
||
#ifndef TINY_HEAP_MAX_PAGES_PER_CLASS
|
||
#define TINY_HEAP_MAX_PAGES_PER_CLASS 4
|
||
#endif
|
||
|
||
typedef struct tiny_heap_page_t {
|
||
void* free_list; // ページ内 free list 先頭 (BASE)
|
||
uint16_t used; // 現在の使用数（active count）
|
||
uint16_t capacity; // ページ内の総ブロック数
|
||
uint16_t slab_idx; // 対応する slab index
|
||
uint16_t _pad;
|
||
uint8_t* base; // slab 先頭（データ領域先頭）
|
||
TinySlabMeta* meta; // Superslab メタ（owner/Tier 判定用）
|
||
SuperSlab* ss; // 所有する Superslab
|
||
struct tiny_heap_page_t* next;
|
||
int32_t c7_active_delta; // C7 meta-light 用: total_active_blocks の差分
|
||
int32_t c7_used_delta; // C7 meta-light 用: meta->used の差分
|
||
} tiny_heap_page_t;
|
||
|
||
typedef struct tiny_heap_class_t {
|
||
tiny_heap_page_t* current_page;
|
||
tiny_heap_page_t* partial_pages;
|
||
tiny_heap_page_t* full_pages;
|
||
tiny_heap_page_t nodes[TINY_HEAP_MAX_PAGES_PER_CLASS];
|
||
uint8_t node_in_use[TINY_HEAP_MAX_PAGES_PER_CLASS];
|
||
uint16_t stride; // cached block stride for this class
|
||
uint16_t _pad;
|
||
} tiny_heap_class_t;
|
||
|
||
typedef struct tiny_heap_ctx_t {
|
||
tiny_heap_class_t cls[TINY_NUM_CLASSES];
|
||
uint8_t initialized;
|
||
} tiny_heap_ctx_t;
|
||
|
||
// TLS state (定義は core/hakmem_tiny.c)
|
||
extern __thread tiny_heap_ctx_t g_tiny_heap_ctx;
|
||
extern __thread int g_tiny_heap_ctx_init;
|
||
extern __thread TinyTLSSlab g_tls_slabs[TINY_NUM_CLASSES];
|
||
|
||
static inline int tiny_c7_heap_stats_enabled(void) {
|
||
static int g = -1;
|
||
if (__builtin_expect(g == -1, 0)) {
|
||
const char* e = getenv("HAKMEM_TINY_C7_HEAP_STATS");
|
||
g = (e && *e && *e != '0') ? 1 : 0;
|
||
}
|
||
return g;
|
||
}
|
||
|
||
static inline int tiny_c7_meta_light_enabled(void) {
|
||
static int g = -1;
|
||
if (__builtin_expect(g == -1, 0)) {
|
||
const char* e = getenv("HAKMEM_TINY_C7_META_LIGHT");
|
||
g = (e && *e && *e != '0') ? 1 : 0;
|
||
}
|
||
return g;
|
||
}
|
||
|
||
static inline int tiny_c7_delta_debug_enabled(void) {
|
||
static int g = -1;
|
||
if (__builtin_expect(g == -1, 0)) {
|
||
const char* e = getenv("HAKMEM_TINY_C7_DELTA_DEBUG");
|
||
g = (e && *e && *e != '0') ? 1 : 0;
|
||
}
|
||
return g;
|
||
}
|
||
|
||
typedef struct {
|
||
_Atomic uint64_t alloc_fast_current;
|
||
_Atomic uint64_t alloc_slow_prepare;
|
||
_Atomic uint64_t free_fast_local;
|
||
_Atomic uint64_t free_slow_fallback;
|
||
_Atomic uint64_t alloc_prepare_fail;
|
||
_Atomic uint64_t alloc_fail;
|
||
} TinyC7HeapStats;
|
||
|
||
extern TinyC7HeapStats g_c7_heap_stats;
|
||
|
||
static inline int tiny_heap_cold_drain_and_free(int class_idx, void* base) {
|
||
(void)class_idx;
|
||
(void)base;
|
||
return 0;
|
||
}
|
||
|
||
static inline tiny_heap_ctx_t* tiny_heap_ctx_for_thread(void) {
|
||
tiny_heap_ctx_t* ctx = &g_tiny_heap_ctx;
|
||
if (!g_tiny_heap_ctx_init) {
|
||
memset(ctx, 0, sizeof(*ctx));
|
||
g_tiny_heap_ctx_init = 1;
|
||
ctx->initialized = 1;
|
||
for (int c = 0; c < TINY_NUM_CLASSES; c++) {
|
||
tiny_heap_class_t* hcls = &ctx->cls[c];
|
||
hcls->stride = (uint16_t)tiny_stride_for_class(c);
|
||
}
|
||
}
|
||
return ctx;
|
||
}
|
||
|
||
static inline tiny_heap_class_t* tiny_heap_class(tiny_heap_ctx_t* ctx, int class_idx) {
|
||
if (!ctx || class_idx < 0 || class_idx >= TINY_NUM_CLASSES) return NULL;
|
||
return &ctx->cls[class_idx];
|
||
}
|
||
|
||
static inline size_t tiny_heap_block_stride(int class_idx) {
|
||
return tiny_stride_for_class(class_idx);
|
||
}
|
||
|
||
static inline void tiny_heap_page_clear(tiny_heap_page_t* page) {
|
||
if (!page) return;
|
||
page->free_list = NULL;
|
||
page->used = 0;
|
||
page->capacity = 0;
|
||
page->slab_idx = 0;
|
||
page->base = NULL;
|
||
page->meta = NULL;
|
||
page->ss = NULL;
|
||
page->next = NULL;
|
||
page->c7_active_delta = 0;
|
||
page->c7_used_delta = 0;
|
||
}
|
||
|
||
static inline void tiny_c7_meta_flush_page(tiny_heap_page_t* page) {
|
||
if (!page || !tiny_c7_meta_light_enabled()) return;
|
||
if (!page->meta || !page->ss) return;
|
||
|
||
int32_t active_delta = page->c7_active_delta;
|
||
int32_t used_delta = page->c7_used_delta;
|
||
if (active_delta == 0 && used_delta == 0) {
|
||
return;
|
||
}
|
||
|
||
if (used_delta != 0) {
|
||
atomic_fetch_add_explicit(&page->meta->used, used_delta, memory_order_relaxed);
|
||
page->c7_used_delta = 0;
|
||
}
|
||
if (active_delta != 0) {
|
||
if (active_delta > 0) {
|
||
ss_active_add(page->ss, (uint32_t)active_delta);
|
||
} else {
|
||
int32_t n = -active_delta;
|
||
for (int32_t i = 0; i < n; i++) {
|
||
ss_active_dec_one(page->ss);
|
||
}
|
||
}
|
||
page->c7_active_delta = 0;
|
||
}
|
||
}
|
||
|
||
static inline int tiny_c7_delta_should_flush(tiny_heap_page_t* page) {
|
||
if (!page) return 0;
|
||
if (!tiny_c7_meta_light_enabled()) return 0;
|
||
|
||
int32_t ud = page->c7_used_delta;
|
||
int32_t ad = page->c7_active_delta;
|
||
int32_t abs_ud = (ud >= 0) ? ud : -ud;
|
||
int32_t abs_ad = (ad >= 0) ? ad : -ad;
|
||
int32_t abs_max = (abs_ud > abs_ad) ? abs_ud : abs_ad;
|
||
if (abs_max == 0) {
|
||
return 0;
|
||
}
|
||
|
||
uint16_t cap = page->capacity;
|
||
int32_t base_th = (cap > 0) ? ((int32_t)cap * 16) : 256;
|
||
if (base_th < 256) {
|
||
base_th = 256;
|
||
}
|
||
return abs_max >= base_th;
|
||
}
|
||
|
||
static __attribute__((noinline, unused)) void tiny_c7_heap_debug_dump_deltas(void) {
|
||
if (!tiny_c7_meta_light_enabled() || !tiny_c7_delta_debug_enabled()) {
|
||
return;
|
||
}
|
||
|
||
tiny_heap_ctx_t* ctx = tiny_heap_ctx_for_thread();
|
||
int class_idx = 7;
|
||
tiny_heap_class_t* hcls = tiny_heap_class(ctx, class_idx);
|
||
if (!hcls) return;
|
||
|
||
uint64_t nonzero_pages = 0;
|
||
int64_t used_delta_sum = 0;
|
||
int64_t active_delta_sum = 0;
|
||
|
||
for (int i = 0; i < TINY_HEAP_MAX_PAGES_PER_CLASS; i++) {
|
||
tiny_heap_page_t* p = &hcls->nodes[i];
|
||
if (p->c7_used_delta != 0 || p->c7_active_delta != 0) {
|
||
nonzero_pages++;
|
||
used_delta_sum += (int64_t)p->c7_used_delta;
|
||
active_delta_sum += (int64_t)p->c7_active_delta;
|
||
fprintf(stderr,
|
||
"[C7_DELTA_PAGE] idx=%d used_delta=%d active_delta=%d used=%u cap=%u ss=%p slab=%u\n",
|
||
i,
|
||
p->c7_used_delta,
|
||
p->c7_active_delta,
|
||
(unsigned)p->used,
|
||
(unsigned)p->capacity,
|
||
(void*)p->ss,
|
||
(unsigned)p->slab_idx);
|
||
}
|
||
}
|
||
|
||
fprintf(stderr,
|
||
"[C7_DELTA_SUMMARY] nonzero_pages=%llu used_delta_sum=%lld active_delta_sum=%lld\n",
|
||
(unsigned long long)nonzero_pages,
|
||
(long long)used_delta_sum,
|
||
(long long)active_delta_sum);
|
||
}
|
||
|
||
static inline tiny_heap_page_t* tiny_heap_class_acquire_node(tiny_heap_class_t* hcls) {
|
||
if (!hcls) return NULL;
|
||
for (int i = 0; i < TINY_HEAP_MAX_PAGES_PER_CLASS; i++) {
|
||
if (hcls->node_in_use[i] == 0) {
|
||
hcls->node_in_use[i] = 1;
|
||
tiny_heap_page_t* p = &hcls->nodes[i];
|
||
tiny_heap_page_clear(p);
|
||
return p;
|
||
}
|
||
}
|
||
return NULL;
|
||
}
|
||
|
||
static inline void tiny_heap_class_release_node(tiny_heap_class_t* hcls, tiny_heap_page_t* page) {
|
||
if (!hcls || !page) return;
|
||
if (tiny_c7_meta_light_enabled()) {
|
||
tiny_c7_meta_flush_page(page);
|
||
}
|
||
intptr_t idx = page - hcls->nodes;
|
||
if (idx >= 0 && idx < TINY_HEAP_MAX_PAGES_PER_CLASS) {
|
||
hcls->node_in_use[idx] = 0;
|
||
tiny_heap_page_clear(page);
|
||
}
|
||
}
|
||
|
||
static inline tiny_heap_page_t* tiny_heap_class_find_page(tiny_heap_class_t* hcls, SuperSlab* ss, int slab_idx) {
|
||
if (!hcls || !ss || slab_idx < 0) return NULL;
|
||
tiny_heap_page_t* lists[] = {hcls->current_page, hcls->partial_pages, hcls->full_pages};
|
||
for (size_t li = 0; li < sizeof(lists) / sizeof(lists[0]); li++) {
|
||
tiny_heap_page_t* p = lists[li];
|
||
while (p) {
|
||
if (p->ss == ss && (int)p->slab_idx == slab_idx) {
|
||
return p;
|
||
}
|
||
p = p->next;
|
||
}
|
||
}
|
||
return NULL;
|
||
}
|
||
|
||
static inline tiny_heap_page_t* tiny_heap_attach_page(tiny_heap_ctx_t* ctx,
|
||
int class_idx,
|
||
SuperSlab* ss,
|
||
int slab_idx) {
|
||
tiny_heap_class_t* hcls = tiny_heap_class(ctx, class_idx);
|
||
if (!hcls || !ss || slab_idx < 0) return NULL;
|
||
TinySlabMeta* meta = &ss->slabs[slab_idx];
|
||
if (meta->class_idx != class_idx) {
|
||
return NULL;
|
||
}
|
||
|
||
tiny_heap_page_t* page = tiny_heap_class_find_page(hcls, ss, slab_idx);
|
||
if (!page) {
|
||
page = tiny_heap_class_acquire_node(hcls);
|
||
if (!page) return NULL;
|
||
page->ss = ss;
|
||
page->slab_idx = (uint16_t)slab_idx;
|
||
page->meta = meta;
|
||
page->base = tiny_slab_base_for(ss, slab_idx);
|
||
page->capacity = meta->capacity;
|
||
page->used = atomic_load_explicit(&meta->used, memory_order_relaxed);
|
||
page->free_list = atomic_load_explicit(&meta->freelist, memory_order_acquire);
|
||
page->next = hcls->partial_pages;
|
||
hcls->partial_pages = page;
|
||
} else {
|
||
page->capacity = meta->capacity;
|
||
page->used = atomic_load_explicit(&meta->used, memory_order_relaxed);
|
||
page->free_list = atomic_load_explicit(&meta->freelist, memory_order_acquire);
|
||
if (!page->base) {
|
||
page->base = tiny_slab_base_for(ss, slab_idx);
|
||
}
|
||
}
|
||
if (class_idx == 7 && tiny_c7_meta_light_enabled()) {
|
||
if (page->capacity > 0 && page->used > page->capacity) {
|
||
page->used = page->capacity;
|
||
}
|
||
page->c7_used_delta = 0;
|
||
page->c7_active_delta = 0;
|
||
}
|
||
return page;
|
||
}
|
||
|
||
static inline tiny_heap_page_t* tiny_heap_take_current(tiny_heap_class_t* hcls) {
|
||
if (!hcls) return NULL;
|
||
if (hcls->current_page) {
|
||
return hcls->current_page;
|
||
}
|
||
if (hcls->partial_pages) {
|
||
tiny_heap_page_t* page = hcls->partial_pages;
|
||
hcls->partial_pages = page->next;
|
||
page->next = NULL;
|
||
hcls->current_page = page;
|
||
return page;
|
||
}
|
||
return NULL;
|
||
}
|
||
|
||
static inline void tiny_heap_class_unlink(tiny_heap_class_t* hcls, tiny_heap_page_t* page) {
|
||
if (!hcls || !page) return;
|
||
if (hcls->current_page == page) {
|
||
hcls->current_page = NULL;
|
||
}
|
||
tiny_heap_page_t** lists[] = {&hcls->partial_pages, &hcls->full_pages};
|
||
for (size_t i = 0; i < sizeof(lists) / sizeof(lists[0]); i++) {
|
||
tiny_heap_page_t** head = lists[i];
|
||
tiny_heap_page_t* prev = NULL;
|
||
tiny_heap_page_t* cur = head ? *head : NULL;
|
||
while (cur) {
|
||
if (cur == page) {
|
||
if (prev) {
|
||
prev->next = cur->next;
|
||
} else {
|
||
*head = cur->next;
|
||
}
|
||
break;
|
||
}
|
||
prev = cur;
|
||
cur = cur->next;
|
||
}
|
||
}
|
||
}
|
||
|
||
static inline tiny_heap_page_t* tiny_heap_page_of(tiny_heap_ctx_t* ctx, int class_idx, void* base_ptr) {
|
||
if (!ctx || !base_ptr || class_idx < 0 || class_idx >= TINY_NUM_CLASSES) return NULL;
|
||
tiny_heap_class_t* hcls = tiny_heap_class(ctx, class_idx);
|
||
if (!hcls) return NULL;
|
||
|
||
// TLS fast path: TLS slab の範囲内なら lookup を避ける
|
||
TinyTLSSlab* tls = &g_tls_slabs[class_idx];
|
||
if (tls->ss && tls->meta && tls->slab_base) {
|
||
size_t stride = hcls->stride;
|
||
if (stride == 0) {
|
||
stride = tiny_heap_block_stride(class_idx);
|
||
hcls->stride = (uint16_t)stride;
|
||
}
|
||
uint16_t cap = tls->meta->capacity;
|
||
if (cap > 0) {
|
||
uint8_t* slab_base = tls->slab_base;
|
||
size_t slab_size = stride * (size_t)cap;
|
||
uint8_t* base_u8 = (uint8_t*)base_ptr;
|
||
if (base_u8 >= slab_base && base_u8 < (slab_base + slab_size)) {
|
||
return tiny_heap_attach_page(ctx, class_idx, tls->ss, tls->slab_idx);
|
||
}
|
||
}
|
||
}
|
||
|
||
// fallback: Superslab lookup
|
||
SuperSlab* ss = hak_super_lookup(base_ptr);
|
||
if (!ss || ss->magic != SUPERSLAB_MAGIC) {
|
||
return NULL;
|
||
}
|
||
int slab_idx = slab_index_for(ss, base_ptr);
|
||
if (slab_idx < 0 || slab_idx >= ss_slabs_capacity(ss)) {
|
||
return NULL;
|
||
}
|
||
return tiny_heap_attach_page(ctx, class_idx, ss, slab_idx);
|
||
}
|
||
|
||
static inline void tiny_heap_page_push_to_full(tiny_heap_class_t* hcls, tiny_heap_page_t* page) {
|
||
if (!hcls || !page) return;
|
||
page->next = hcls->full_pages;
|
||
hcls->full_pages = page;
|
||
}
|
||
|
||
static inline void tiny_heap_page_push_to_partial(tiny_heap_class_t* hcls, tiny_heap_page_t* page) {
|
||
if (!hcls || !page) return;
|
||
page->next = hcls->partial_pages;
|
||
hcls->partial_pages = page;
|
||
}
|
||
|
||
static inline void tiny_heap_page_becomes_empty(tiny_heap_ctx_t* ctx, int class_idx, tiny_heap_page_t* page) {
|
||
tiny_heap_class_t* hcls = tiny_heap_class(ctx, class_idx);
|
||
if (!hcls || !page) return;
|
||
|
||
if (class_idx == 7 && tiny_c7_meta_light_enabled()) {
|
||
tiny_c7_meta_flush_page(page);
|
||
}
|
||
if (page->meta && page->ss) {
|
||
ss_partial_publish(class_idx, page->ss);
|
||
}
|
||
tiny_heap_class_unlink(hcls, page);
|
||
tiny_heap_class_release_node(hcls, page);
|
||
}
|
||
|
||
static inline void tiny_heap_page_mark_full(tiny_heap_class_t* hcls, tiny_heap_page_t* page) {
|
||
if (!hcls || !page) return;
|
||
tiny_heap_class_unlink(hcls, page);
|
||
tiny_heap_page_push_to_full(hcls, page);
|
||
}
|
||
|
||
static inline void* tiny_heap_page_pop(tiny_heap_class_t* hcls, int class_idx, tiny_heap_page_t* page) {
|
||
if (!hcls || !page || !page->meta || !page->ss || !page->base) return NULL;
|
||
const int meta_light = (class_idx == 7 && tiny_c7_meta_light_enabled());
|
||
void* block = NULL;
|
||
if (page->free_list) {
|
||
block = page->free_list;
|
||
void* next = tiny_next_read(class_idx, block);
|
||
page->free_list = next;
|
||
atomic_store_explicit(&page->meta->freelist, next, memory_order_release);
|
||
} else if (page->used < page->capacity) {
|
||
size_t stride = hcls->stride;
|
||
if (stride == 0) {
|
||
stride = tiny_heap_block_stride(class_idx);
|
||
hcls->stride = (uint16_t)stride;
|
||
}
|
||
block = (void*)(page->base + ((size_t)page->used * stride));
|
||
if (page->meta->carved < page->capacity) {
|
||
page->meta->carved++;
|
||
}
|
||
} else {
|
||
return NULL;
|
||
}
|
||
|
||
page->used++;
|
||
if (__builtin_expect(meta_light, 0)) {
|
||
page->c7_used_delta++;
|
||
page->c7_active_delta++;
|
||
if (tiny_c7_delta_should_flush(page)) {
|
||
tiny_c7_meta_flush_page(page);
|
||
}
|
||
return tiny_region_id_write_header(block, class_idx);
|
||
}
|
||
atomic_fetch_add_explicit(&page->meta->used, 1, memory_order_relaxed);
|
||
ss_active_add(page->ss, 1u);
|
||
|
||
return tiny_region_id_write_header(block, class_idx);
|
||
}
|
||
|
||
static inline void tiny_heap_page_push_free(int class_idx, tiny_heap_page_t* page, void* base_ptr) {
|
||
if (!page || !base_ptr || !page->meta) return;
|
||
tiny_next_write(class_idx, base_ptr, page->free_list);
|
||
page->free_list = base_ptr;
|
||
atomic_store_explicit(&page->meta->freelist, base_ptr, memory_order_release);
|
||
}
|
||
|
||
static inline void tiny_heap_page_free_local(tiny_heap_ctx_t* ctx,
|
||
int class_idx,
|
||
tiny_heap_page_t* page,
|
||
void* base_ptr) {
|
||
tiny_heap_class_t* hcls = tiny_heap_class(ctx, class_idx);
|
||
if (!hcls || !page || !base_ptr) return;
|
||
const int stats_on = (class_idx == 7 && tiny_c7_heap_stats_enabled());
|
||
if (__builtin_expect(stats_on, 0)) {
|
||
atomic_fetch_add_explicit(&g_c7_heap_stats.free_fast_local, 1, memory_order_relaxed);
|
||
}
|
||
const int meta_light = (class_idx == 7 && tiny_c7_meta_light_enabled());
|
||
tiny_heap_page_push_free(class_idx, page, base_ptr);
|
||
if (page->used > 0) {
|
||
page->used--;
|
||
if (!__builtin_expect(meta_light, 0)) {
|
||
atomic_fetch_sub_explicit(&page->meta->used, 1, memory_order_relaxed);
|
||
ss_active_dec_one(page->ss);
|
||
} else {
|
||
page->c7_used_delta--;
|
||
page->c7_active_delta--;
|
||
if (tiny_c7_delta_should_flush(page)) {
|
||
tiny_c7_meta_flush_page(page);
|
||
}
|
||
}
|
||
}
|
||
|
||
if (page->used == 0) {
|
||
tiny_heap_page_becomes_empty(ctx, class_idx, page);
|
||
} else if (page->used < page->capacity && page->free_list) {
|
||
tiny_heap_page_t* old_cur = hcls->current_page;
|
||
tiny_heap_class_unlink(hcls, page);
|
||
page->next = NULL;
|
||
if (class_idx == 7) {
|
||
hcls->current_page = page;
|
||
if (old_cur && old_cur != page) {
|
||
tiny_heap_page_push_to_partial(hcls, old_cur);
|
||
}
|
||
} else {
|
||
if (!hcls->current_page) {
|
||
hcls->current_page = page;
|
||
} else {
|
||
tiny_heap_page_push_to_partial(hcls, page);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
static inline tiny_heap_page_t* tiny_heap_prepare_page(tiny_heap_ctx_t* ctx, int class_idx) {
|
||
tiny_heap_class_t* hcls = tiny_heap_class(ctx, class_idx);
|
||
const int stats_on = (class_idx == 7 && tiny_c7_heap_stats_enabled());
|
||
if (!hcls) {
|
||
if (__builtin_expect(stats_on, 0)) {
|
||
atomic_fetch_add_explicit(&g_c7_heap_stats.alloc_prepare_fail, 1, memory_order_relaxed);
|
||
}
|
||
return NULL;
|
||
}
|
||
|
||
tiny_heap_page_t* page = tiny_heap_take_current(hcls);
|
||
if (page) return page;
|
||
|
||
TinyTLSSlab* tls = &g_tls_slabs[class_idx];
|
||
if (!tls->ss) {
|
||
if (superslab_refill(class_idx) == NULL) {
|
||
if (__builtin_expect(stats_on, 0)) {
|
||
atomic_fetch_add_explicit(&g_c7_heap_stats.alloc_prepare_fail, 1, memory_order_relaxed);
|
||
}
|
||
return NULL;
|
||
}
|
||
}
|
||
tls = &g_tls_slabs[class_idx]; // superslab_refill で更新されるため再取得
|
||
if (!tls->ss || !tls->meta) {
|
||
if (__builtin_expect(stats_on, 0)) {
|
||
atomic_fetch_add_explicit(&g_c7_heap_stats.alloc_prepare_fail, 1, memory_order_relaxed);
|
||
}
|
||
return NULL;
|
||
}
|
||
page = tiny_heap_attach_page(ctx, class_idx, tls->ss, tls->slab_idx);
|
||
if (page) {
|
||
tiny_heap_class_unlink(hcls, page);
|
||
page->next = NULL;
|
||
hcls->current_page = page;
|
||
}
|
||
return page;
|
||
}
|
||
|
||
// class_idx 固定での alloc ホットパス
|
||
static inline void* tiny_heap_alloc_slow_from_class(tiny_heap_ctx_t* ctx, int class_idx) {
|
||
tiny_heap_class_t* hcls = tiny_heap_class(ctx, class_idx);
|
||
if (!hcls) return NULL;
|
||
|
||
const int stats_on = (class_idx == 7 && tiny_c7_heap_stats_enabled());
|
||
tiny_heap_page_t* page = tiny_heap_prepare_page(ctx, class_idx);
|
||
if (!page) {
|
||
if (__builtin_expect(stats_on, 0)) {
|
||
atomic_fetch_add_explicit(&g_c7_heap_stats.alloc_fail, 1, memory_order_relaxed);
|
||
}
|
||
return NULL;
|
||
}
|
||
|
||
void* user = tiny_heap_page_pop(hcls, class_idx, page);
|
||
if (user && page->used >= page->capacity && page->free_list == NULL) {
|
||
tiny_heap_page_mark_full(hcls, page);
|
||
}
|
||
return user;
|
||
}
|
||
|
||
// class_idx 固定での alloc ホットパス
|
||
__attribute__((always_inline)) static inline void* tiny_heap_alloc_class_fast(tiny_heap_ctx_t* ctx,
|
||
int class_idx,
|
||
size_t size) {
|
||
(void)size;
|
||
tiny_heap_class_t* hcls = tiny_heap_class(ctx, class_idx);
|
||
tiny_heap_page_t* page = hcls ? hcls->current_page : NULL;
|
||
const int stats_on = (class_idx == 7 && tiny_c7_heap_stats_enabled());
|
||
|
||
if (page) {
|
||
if (page->free_list || page->used < page->capacity) {
|
||
void* user = tiny_heap_page_pop(hcls, class_idx, page);
|
||
if (user) {
|
||
if (page->used >= page->capacity && page->free_list == NULL) {
|
||
tiny_heap_page_mark_full(hcls, page);
|
||
}
|
||
if (__builtin_expect(stats_on, 0)) {
|
||
atomic_fetch_add_explicit(&g_c7_heap_stats.alloc_fast_current, 1, memory_order_relaxed);
|
||
}
|
||
return user;
|
||
}
|
||
}
|
||
}
|
||
|
||
if (__builtin_expect(stats_on, 0)) {
|
||
atomic_fetch_add_explicit(&g_c7_heap_stats.alloc_slow_prepare, 1, memory_order_relaxed);
|
||
}
|
||
return tiny_heap_alloc_slow_from_class(ctx, class_idx);
|
||
}
|
||
|
||
// Superslab/Slab メタが既に分かっている場合の free（Gate から渡されるホットパス用）
|
||
static inline void tiny_heap_free_class_fast_with_meta(tiny_heap_ctx_t* ctx,
|
||
int class_idx,
|
||
SuperSlab* ss,
|
||
int slab_idx,
|
||
void* base) {
|
||
tiny_heap_class_t* hcls = tiny_heap_class(ctx, class_idx);
|
||
if (!base || !ss || slab_idx < 0 || !hcls) return;
|
||
const int stats_on = (class_idx == 7 && tiny_c7_heap_stats_enabled());
|
||
|
||
tiny_heap_page_t* page = tiny_heap_attach_page(ctx, class_idx, ss, slab_idx);
|
||
if (!page) {
|
||
if (__builtin_expect(stats_on, 0)) {
|
||
atomic_fetch_add_explicit(&g_c7_heap_stats.free_slow_fallback, 1, memory_order_relaxed);
|
||
}
|
||
tiny_heap_cold_drain_and_free(class_idx, base);
|
||
return;
|
||
}
|
||
|
||
tiny_heap_page_free_local(ctx, class_idx, page, base);
|
||
}
|
||
|
||
// class_idx 固定での free ホットパス（ptr は USER ポインタ）
|
||
static inline void tiny_heap_free_class_fast(tiny_heap_ctx_t* ctx, int class_idx, void* ptr) {
|
||
if (!ptr) return;
|
||
|
||
#if HAKMEM_TINY_HEADERLESS
|
||
void* base = ptr;
|
||
#else
|
||
void* base = (void*)((uint8_t*)ptr - 1);
|
||
#endif
|
||
SuperSlab* ss = hak_super_lookup(base);
|
||
if (!ss || ss->magic != SUPERSLAB_MAGIC) {
|
||
if (__builtin_expect((class_idx == 7 && tiny_c7_heap_stats_enabled()), 0)) {
|
||
atomic_fetch_add_explicit(&g_c7_heap_stats.free_slow_fallback, 1, memory_order_relaxed);
|
||
}
|
||
tiny_heap_cold_drain_and_free(class_idx, base);
|
||
return;
|
||
}
|
||
int slab_idx = slab_index_for(ss, base);
|
||
if (slab_idx < 0 || slab_idx >= ss_slabs_capacity(ss)) {
|
||
if (__builtin_expect((class_idx == 7 && tiny_c7_heap_stats_enabled()), 0)) {
|
||
atomic_fetch_add_explicit(&g_c7_heap_stats.free_slow_fallback, 1, memory_order_relaxed);
|
||
}
|
||
tiny_heap_cold_drain_and_free(class_idx, base);
|
||
return;
|
||
}
|
||
|
||
tiny_heap_free_class_fast_with_meta(ctx, class_idx, ss, slab_idx, base);
|
||
}
|