tomoaki/hakmem
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

C7 v2: add lease helpers and v2 page reset

Browse Source
This commit is contained in:
Moe Charm (CI)
2025-12-08 14:40:03 +09:00
parent 9502501842
commit 34a8fd69b6
5 changed files with 1171 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files

674
core/box/tiny_heap_box.h
View File

@ -22,6 +22,7 @@
#include "../tiny_region_id.h" // tiny_region_id_write_header
#include "tiny_layout_box.h" // tiny_user_offset
#include "tiny_next_ptr_box.h" // tiny_next_read/write
#include "tiny_heap_env_box.h" // profile gates
// Forward decls for SuperSlab active counters (definitions in hakmem_tiny_ss_active_box.inc)
void ss_active_add(SuperSlab* ss, uint32_t n);
@ -40,8 +41,11 @@ typedef struct tiny_heap_page_t {
TinySlabMeta* meta; // Superslab メタ（owner/Tier 判定用）
SuperSlab* ss; // 所有する Superslab
struct tiny_heap_page_t* next;
int32_t c7_active_delta; // C7 meta-light 用: total_active_blocks の差分
int32_t c7_used_delta; // C7 meta-light 用: meta->used の差分
int32_t active_delta; // meta-light 用: total_active_blocks の差分（主に C6/C7）
int32_t used_delta; // meta-light 用: meta->used の差分（主に C6/C7）
int32_t stats_active_pending; // Cold Stats Box 用 pending（C7 SAFE）
int32_t stats_used_pending; // Cold Stats Box 用 pending（C7 SAFE）
int32_t last_delta_site; // debug: 直近で delta を触ったサイト（C6/C7 meta-light 用）
} tiny_heap_page_t;
typedef struct tiny_heap_class_t {
@ -59,27 +63,104 @@ typedef struct tiny_heap_ctx_t {
uint8_t initialized;
} tiny_heap_ctx_t;
typedef struct {
TinySlabMeta* meta;
SuperSlab* ss;
uint8_t* base;
void* freelist;
uint16_t capacity;
uint16_t slab_idx;
tiny_heap_page_t* page; // v1 TinyHeap の page 構造体（lease 元）
} TinyHeapPageLease;
// Stats Box（Cold 側への集計フック）
#include "tiny_stats_box.h"
// TLS state (定義は core/hakmem_tiny.c)
extern __thread tiny_heap_ctx_t g_tiny_heap_ctx;
extern __thread int g_tiny_heap_ctx_init;
extern __thread TinyTLSSlab g_tls_slabs[TINY_NUM_CLASSES];
static inline int tiny_c7_heap_stats_enabled(void) {
static inline int tiny_heap_stats_enabled(void) {
static int g = -1;
if (__builtin_expect(g == -1, 0)) {
const char* eh = getenv("HAKMEM_TINY_HEAP_STATS");
const char* e = getenv("HAKMEM_TINY_C7_HEAP_STATS");
g = (e && *e && *e != '0') ? 1 : 0;
g = ((eh && *eh && *eh != '0') || (e && *e && *e != '0')) ? 1 : 0;
}
return g;
}
static inline int tiny_c7_meta_light_enabled(void) {
static int g = -1;
if (__builtin_expect(g == -1, 0)) {
const char* e = getenv("HAKMEM_TINY_C7_META_LIGHT");
g = (e && *e && *e != '0') ? 1 : 0;
static inline int tiny_c7_heap_stats_enabled(void) {
return tiny_heap_stats_enabled();
}
// meta mode: 0=OFF, 1=SAFE meta-light (delta + flush), 2=ULTRA (bench only, meta/active をほぼ触らない)
// meta mode: 0=OFF, 1=SAFE meta-light (delta + flush), 2=ULTRA (bench only, meta/active をほぼ触らない)
static inline int tiny_c7_meta_mode(void);
static inline int tiny_heap_meta_mode_for_class(int class_idx) {
if (class_idx == 7) {
return tiny_c7_meta_mode();
}
return g;
if (class_idx == 6) {
static int g_c6_mode = -1;
if (__builtin_expect(g_c6_mode == -1, 0)) {
const char* e = getenv("HAKMEM_TINY_C6_META_MODE");
if (e && *e && *e != '0') {
g_c6_mode = 1;
} else {
g_c6_mode = 0;
}
}
// NOTE: mode 1 for class 6 is experimental (bench-only, may crash). See CURRENT_TASK for guidance.
return g_c6_mode;
}
return 0;
}
static inline int tiny_c7_meta_mode(void) {
static int g_mode = -1;
if (__builtin_expect(g_mode == -1, 0)) {
const char* e = getenv("HAKMEM_TINY_C7_META_MODE");
if (!e || !*e) {
const char* l = getenv("HAKMEM_TINY_C7_META_LIGHT");
if (l && *l && *l != '0') {
g_mode = 1;
} else {
g_mode = tiny_heap_profile_default_c7_meta_mode(tiny_heap_profile_mode());
}
} else {
int v = atoi(e);
if (v < 0) v = 0;
if (v > 2) v = 2;
g_mode = v;
}
}
return g_mode;
}
static inline int tiny_heap_behavior_mode_for_class(int class_idx) {
return tiny_heap_meta_mode_for_class(class_idx);
}
static inline int tiny_heap_meta_light_enabled_for_class(int class_idx) {
return tiny_heap_behavior_mode_for_class(class_idx) >= 1;
}
static inline int tiny_heap_meta_ultra_enabled_for_class(int class_idx) {
return (class_idx == 7) && tiny_heap_meta_mode_for_class(class_idx) == 2;
}
static inline int tiny_heap_meta_mode_effective(int class_idx) {
return tiny_heap_meta_mode_for_class(class_idx);
}
static inline int tiny_c7_meta_light_enabled(void) {
return tiny_heap_meta_light_enabled_for_class(7);
}
static inline int tiny_c7_meta_ultra_enabled(void) {
return tiny_heap_meta_ultra_enabled_for_class(7);
}
static inline int tiny_c7_delta_debug_enabled(void) {
@ -91,6 +172,33 @@ static inline int tiny_c7_delta_debug_enabled(void) {
return g;
}
static inline int tiny_c6_delta_debug_enabled(void) {
static int g = -1;
if (__builtin_expect(g == -1, 0)) {
const char* e = getenv("HAKMEM_TINY_C6_DELTA_DEBUG");
g = (e && *e && *e != '0') ? 1 : 0;
}
return g;
}
static inline int tiny_c6_delta_trace_enabled(void) {
static int g = -1;
if (__builtin_expect(g == -1, 0)) {
const char* e = getenv("HAKMEM_TINY_C6_DELTA_TRACE");
g = (e && *e && *e != '0') ? 1 : 0;
}
return g;
}
static inline int tiny_c6_debug_pop_enabled(void) {
static int g = -1;
if (__builtin_expect(g == -1, 0)) {
const char* e = getenv("HAKMEM_TINY_C6_DEBUG_POP");
g = (e && *e && *e != '0') ? 1 : 0;
}
return g;
}
typedef struct {
_Atomic uint64_t alloc_fast_current;
_Atomic uint64_t alloc_slow_prepare;
@ -98,9 +206,38 @@ typedef struct {
_Atomic uint64_t free_slow_fallback;
_Atomic uint64_t alloc_prepare_fail;
_Atomic uint64_t alloc_fail;
} TinyC7HeapStats;
} TinyHeapClassStats;
extern TinyC7HeapStats g_c7_heap_stats;
extern TinyHeapClassStats g_tiny_heap_stats[TINY_NUM_CLASSES];
typedef struct {
_Atomic uint64_t prepare_calls;
_Atomic uint64_t prepare_with_current_null;
_Atomic uint64_t prepare_from_partial;
_Atomic uint64_t current_set_from_free;
_Atomic uint64_t current_dropped_to_partial;
} TinyC7PageStats;
extern TinyC7PageStats g_c7_page_stats;
typedef enum {
C6_DELTA_NONE = 0,
C6_DELTA_ALLOC = 1,
C6_DELTA_FREE = 2,
C6_DELTA_EMPTY = 3,
C6_DELTA_ATTACH = 4,
C6_DELTA_THRESHOLD = 5,
} tiny_c6_delta_site_t;
static inline int tiny_c7_page_stats_enabled(void) {
return tiny_heap_stats_enabled();
}
static inline TinyHeapClassStats* tiny_heap_stats_for_class(int class_idx) {
if (!tiny_heap_stats_enabled()) return NULL;
if (class_idx < 0 || class_idx >= TINY_NUM_CLASSES) return NULL;
return &g_tiny_heap_stats[class_idx];
}
static inline int tiny_heap_cold_drain_and_free(int class_idx, void* base) {
(void)class_idx;
@ -127,6 +264,28 @@ static inline tiny_heap_class_t* tiny_heap_class(tiny_heap_ctx_t* ctx, int class
return &ctx->cls[class_idx];
}
static inline int tiny_heap_page_is_valid(tiny_heap_class_t* hcls, tiny_heap_page_t* page) {
if (!hcls || !page) return 0;
return (page >= hcls->nodes) && (page < (hcls->nodes + TINY_HEAP_MAX_PAGES_PER_CLASS));
}
static inline int tiny_heap_ptr_in_page_range(tiny_heap_page_t* page, void* ptr) {
if (!page || !ptr || page->capacity == 0 || !page->base) return 0;
uintptr_t p = (uintptr_t)ptr;
uintptr_t low = (uintptr_t)page->base;
size_t stride = 0;
// stride は caller が hcls->stride を更新済みの前提。0 の場合は range 判定を諦める。
// (debug 用なので false positive でも安全側に倒す)
if (page->meta) {
stride = tiny_stride_for_class((int)page->meta->class_idx);
}
if (stride == 0) {
return 0;
}
uintptr_t high = low + (uintptr_t)(stride * (size_t)page->capacity);
return (p >= low) && (p < high);
}
static inline size_t tiny_heap_block_stride(int class_idx) {
return tiny_stride_for_class(class_idx);
}
@ -141,23 +300,45 @@ static inline void tiny_heap_page_clear(tiny_heap_page_t* page) {
page->meta = NULL;
page->ss = NULL;
page->next = NULL;
page->c7_active_delta = 0;
page->c7_used_delta = 0;
page->active_delta = 0;
page->used_delta = 0;
page->stats_active_pending = 0;
page->stats_used_pending = 0;
page->last_delta_site = 0;
}
static inline void tiny_c7_meta_flush_page(tiny_heap_page_t* page) {
if (!page || !tiny_c7_meta_light_enabled()) return;
static inline void tiny_c6_mark_delta_site(tiny_heap_page_t* page, tiny_c6_delta_site_t site) {
if (!page) return;
if (!tiny_c6_delta_trace_enabled()) return;
page->last_delta_site = (int32_t)site;
}
static inline void tiny_heap_meta_flush_page(int class_idx, tiny_heap_page_t* page) {
if (tiny_heap_meta_ultra_enabled_for_class(class_idx)) {
return;
}
if (!page || !tiny_heap_meta_light_enabled_for_class(class_idx)) return;
if (!page->meta || !page->ss) return;
int32_t active_delta = page->c7_active_delta;
int32_t used_delta = page->c7_used_delta;
int32_t active_delta = page->active_delta;
int32_t used_delta = page->used_delta;
if (active_delta == 0 && used_delta == 0) {
if (class_idx == 7 && tiny_stats_box_enabled() && tiny_stats_batch_enabled()) {
tiny_stats_maybe_flush_for_page(class_idx, page, page->used == 0);
}
return;
}
page->active_delta = 0;
page->used_delta = 0;
if (class_idx == 7 && tiny_stats_box_enabled()) {
tiny_stats_flush_for_page(class_idx, page, used_delta, active_delta);
return;
}
if (used_delta != 0) {
atomic_fetch_add_explicit(&page->meta->used, used_delta, memory_order_relaxed);
page->c7_used_delta = 0;
}
if (active_delta != 0) {
if (active_delta > 0) {
@ -168,16 +349,16 @@ static inline void tiny_c7_meta_flush_page(tiny_heap_page_t* page) {
ss_active_dec_one(page->ss);
}
}
page->c7_active_delta = 0;
}
}
static inline int tiny_c7_delta_should_flush(tiny_heap_page_t* page) {
static inline int tiny_heap_delta_should_flush(int class_idx, tiny_heap_page_t* page) {
if (!page) return 0;
if (!tiny_c7_meta_light_enabled()) return 0;
if (!tiny_heap_meta_light_enabled_for_class(class_idx)) return 0;
if (tiny_heap_meta_ultra_enabled_for_class(class_idx)) return 0;
int32_t ud = page->c7_used_delta;
int32_t ad = page->c7_active_delta;
int32_t ud = page->used_delta;
int32_t ad = page->active_delta;
int32_t abs_ud = (ud >= 0) ? ud : -ud;
int32_t abs_ad = (ad >= 0) ? ad : -ad;
int32_t abs_max = (abs_ud > abs_ad) ? abs_ud : abs_ad;
@ -193,13 +374,13 @@ static inline int tiny_c7_delta_should_flush(tiny_heap_page_t* page) {
return abs_max >= base_th;
}
static __attribute__((noinline, unused)) void tiny_c7_heap_debug_dump_deltas(void) {
if (!tiny_c7_meta_light_enabled() || !tiny_c7_delta_debug_enabled()) {
return;
}
static __attribute__((noinline, unused)) void tiny_heap_debug_dump_deltas_for_class(int class_idx, const char* tag) {
if (class_idx < 0 || class_idx >= TINY_NUM_CLASSES) return;
if (!tiny_heap_meta_light_enabled_for_class(class_idx)) return;
if (class_idx == 7 && !tiny_c7_delta_debug_enabled()) return;
if (class_idx == 6 && !tiny_c6_delta_debug_enabled()) return;
tiny_heap_ctx_t* ctx = tiny_heap_ctx_for_thread();
int class_idx = 7;
tiny_heap_class_t* hcls = tiny_heap_class(ctx, class_idx);
if (!hcls) return;
@ -209,15 +390,16 @@ static __attribute__((noinline, unused)) void tiny_c7_heap_debug_dump_deltas(voi
for (int i = 0; i < TINY_HEAP_MAX_PAGES_PER_CLASS; i++) {
tiny_heap_page_t* p = &hcls->nodes[i];
if (p->c7_used_delta != 0 || p->c7_active_delta != 0) {
if (p->used_delta != 0 || p->active_delta != 0) {
nonzero_pages++;
used_delta_sum += (int64_t)p->c7_used_delta;
active_delta_sum += (int64_t)p->c7_active_delta;
used_delta_sum += (int64_t)p->used_delta;
active_delta_sum += (int64_t)p->active_delta;
fprintf(stderr,
"[C7_DELTA_PAGE] idx=%d used_delta=%d active_delta=%d used=%u cap=%u ss=%p slab=%u\n",
"[%s_DELTA_PAGE] idx=%d used_delta=%d active_delta=%d used=%u cap=%u ss=%p slab=%u\n",
tag ? tag : "C?_DELTA",
i,
p->c7_used_delta,
p->c7_active_delta,
p->used_delta,
p->active_delta,
(unsigned)p->used,
(unsigned)p->capacity,
(void*)p->ss,
@ -226,12 +408,27 @@ static __attribute__((noinline, unused)) void tiny_c7_heap_debug_dump_deltas(voi
}
fprintf(stderr,
"[C7_DELTA_SUMMARY] nonzero_pages=%llu used_delta_sum=%lld active_delta_sum=%lld\n",
"[%s_DELTA_SUMMARY] nonzero_pages=%llu used_delta_sum=%lld active_delta_sum=%lld\n",
tag ? tag : "C?_DELTA",
(unsigned long long)nonzero_pages,
(long long)used_delta_sum,
(long long)active_delta_sum);
}
static __attribute__((noinline, unused)) void tiny_c7_heap_debug_dump_deltas(void) {
if (!tiny_c7_meta_light_enabled()) {
return;
}
tiny_heap_debug_dump_deltas_for_class(7, "C7");
}
static __attribute__((noinline, unused)) void tiny_c6_heap_debug_dump_deltas(void) {
if (!tiny_heap_meta_light_enabled_for_class(6)) {
return;
}
tiny_heap_debug_dump_deltas_for_class(6, "C6");
}
static inline tiny_heap_page_t* tiny_heap_class_acquire_node(tiny_heap_class_t* hcls) {
if (!hcls) return NULL;
for (int i = 0; i < TINY_HEAP_MAX_PAGES_PER_CLASS; i++) {
@ -247,8 +444,15 @@ static inline tiny_heap_page_t* tiny_heap_class_acquire_node(tiny_heap_class_t*
static inline void tiny_heap_class_release_node(tiny_heap_class_t* hcls, tiny_heap_page_t* page) {
if (!hcls || !page) return;
if (tiny_c7_meta_light_enabled()) {
tiny_c7_meta_flush_page(page);
int class_idx = (page->meta) ? page->meta->class_idx : -1;
if (tiny_heap_meta_ultra_enabled_for_class(class_idx)) {
page->active_delta = 0;
page->used_delta = 0;
} else if (tiny_heap_meta_light_enabled_for_class(class_idx)) {
if (class_idx == 6) {
tiny_c6_mark_delta_site(page, C6_DELTA_EMPTY);
}
tiny_heap_meta_flush_page(class_idx, page);
}
intptr_t idx = page - hcls->nodes;
if (idx >= 0 && idx < TINY_HEAP_MAX_PAGES_PER_CLASS) {
@ -304,12 +508,33 @@ static inline tiny_heap_page_t* tiny_heap_attach_page(tiny_heap_ctx_t* ctx,
page->base = tiny_slab_base_for(ss, slab_idx);
}
}
if (class_idx == 7 && tiny_c7_meta_light_enabled()) {
if (page->capacity > 0 && page->used > page->capacity) {
page->used = page->capacity;
if (class_idx == 6 && tiny_heap_meta_mode_effective(class_idx) == 1) {
// Fail-Fast: meta->freelist が範囲外なら破棄しておく（古い slab の残骸を拾わない）
if (page->free_list && !tiny_heap_ptr_in_page_range(page, page->free_list)) {
if (tiny_c6_debug_pop_enabled()) {
fprintf(stderr,
"[C6_ATTACH_OOB] freelist=%p base=%p cap=%u ss=%p slab=%d\n",
page->free_list,
(void*)page->base,
(unsigned)page->capacity,
(void*)page->ss,
slab_idx);
}
page->free_list = NULL;
atomic_store_explicit(&meta->freelist, NULL, memory_order_release);
}
}
if (tiny_heap_meta_light_enabled_for_class(class_idx)) {
if (!tiny_heap_meta_ultra_enabled_for_class(class_idx)) {
if (page->capacity > 0 && page->used > page->capacity) {
page->used = page->capacity;
}
}
page->used_delta = 0;
page->active_delta = 0;
if (class_idx == 6) {
tiny_c6_mark_delta_site(page, C6_DELTA_ATTACH);
}
page->c7_used_delta = 0;
page->c7_active_delta = 0;
}
return page;
}
@ -406,8 +631,33 @@ static inline void tiny_heap_page_becomes_empty(tiny_heap_ctx_t* ctx, int class_
tiny_heap_class_t* hcls = tiny_heap_class(ctx, class_idx);
if (!hcls || !page) return;
if (class_idx == 7 && tiny_c7_meta_light_enabled()) {
tiny_c7_meta_flush_page(page);
if (tiny_heap_meta_ultra_enabled_for_class(class_idx)) {
// ULTRA: C7 は 1 ページ前提で保持し続ける。publish/unlink/release を避ける。
hcls->current_page = page;
return;
}
if (tiny_heap_meta_light_enabled_for_class(class_idx)) {
// SAFE: delta を反映
if (class_idx == 6) {
tiny_c6_mark_delta_site(page, C6_DELTA_EMPTY);
}
tiny_heap_meta_flush_page(class_idx, page);
if (class_idx == 7) {
// C7 SAFE: current を保持し、頻繁な detach を避ける
tiny_heap_class_unlink(hcls, page);
hcls->current_page = page;
page->next = NULL;
return;
}
if (class_idx == 6) {
// C6 SAFE: freelist の残骸を次回 attach へ渡さない
atomic_store_explicit(&page->meta->freelist, NULL, memory_order_release);
if (tiny_c6_debug_pop_enabled()) {
page->free_list = (void*)0xDEAD6EED;
} else {
page->free_list = NULL;
}
}
}
if (page->meta && page->ss) {
ss_partial_publish(class_idx, page->ss);
@ -418,19 +668,91 @@ static inline void tiny_heap_page_becomes_empty(tiny_heap_ctx_t* ctx, int class_
static inline void tiny_heap_page_mark_full(tiny_heap_class_t* hcls, tiny_heap_page_t* page) {
if (!hcls || !page) return;
int class_idx = -1;
if (page->meta) {
class_idx = page->meta->class_idx;
}
if (tiny_heap_meta_ultra_enabled_for_class(class_idx)) {
// ULTRA: full 判定で current を追い出さない
return;
}
if (tiny_heap_meta_light_enabled_for_class(class_idx) && hcls->current_page == page) {
// SAFE: current page は可能な限り保持し、頻繁な unlink を避ける（C6/C7）
return;
}
tiny_heap_class_unlink(hcls, page);
tiny_heap_page_push_to_full(hcls, page);
}
static inline void* tiny_heap_page_pop(tiny_heap_class_t* hcls, int class_idx, tiny_heap_page_t* page) {
if (!hcls || !page || !page->meta || !page->ss || !page->base) return NULL;
const int meta_light = (class_idx == 7 && tiny_c7_meta_light_enabled());
const int mode = tiny_heap_meta_mode_effective(class_idx);
const int c6_pop_dbg = (class_idx == 6) && tiny_c6_debug_pop_enabled();
if (c6_pop_dbg) {
static _Atomic uint32_t g_pop_dbg = 0;
uint32_t pop_n = atomic_fetch_add_explicit(&g_pop_dbg, 1, memory_order_relaxed);
if (pop_n < 8) {
fprintf(stderr, "[POP_ENTRY] cls=%d page=%p\n", class_idx, (void*)page);
}
}
if (!tiny_heap_page_is_valid(hcls, page)) return NULL;
if (!page->meta || !page->ss || !page->base) return NULL;
if (class_idx == 6 && mode == 1) {
int fail = 0;
const char* reason = NULL;
SuperSlab* ss_chk = hak_super_lookup(page->base);
if (!page->meta || page->meta->class_idx != class_idx) {
fail = 1; reason = "meta_cls";
} else if (page->capacity == 0) {
fail = 1; reason = "cap0";
} else if (!page->free_list && page->used >= page->capacity) {
fail = 1; reason = "exhausted";
} else if (!ss_chk) {
fail = 1; reason = "ss_lookup_null";
} else if (page->ss && ss_chk != page->ss) {
fail = 1; reason = "ss_mismatch";
} else if (page->free_list && !tiny_heap_ptr_in_page_range(page, page->free_list)) {
fail = 1; reason = "freelist_oob";
}
if (fail) {
fprintf(stderr,
"[C6_POP_FAIL] reason=%s page=%p base=%p freelist=%p used=%u cap=%u ss=%p ss_chk=%p meta=%p last_site=%d\n",
reason ? reason : "unknown",
(void*)page,
(void*)page->base,
page->free_list,
(unsigned)page->used,
(unsigned)page->capacity,
(void*)page->ss,
(void*)ss_chk,
(void*)page->meta,
page->last_delta_site);
abort();
}
}
void* block = NULL;
if (class_idx == 6 && __builtin_expect(mode == 1, 0) && c6_pop_dbg) {
static _Atomic uint32_t g_c6_pop_dbg = 0;
uint32_t n = atomic_fetch_add_explicit(&g_c6_pop_dbg, 1, memory_order_relaxed);
if (n < 512) {
fprintf(stderr,
"[C6_POP_DEBUG] page=%p used=%u cap=%u free_list=%p meta=%p ss=%p base=%p last_site=%d\n",
(void*)page,
(unsigned)page->used,
(unsigned)page->capacity,
page->free_list,
(void*)page->meta,
(void*)page->ss,
(void*)page->base,
page->last_delta_site);
}
}
if (page->free_list) {
block = page->free_list;
void* next = tiny_next_read(class_idx, block);
page->free_list = next;
atomic_store_explicit(&page->meta->freelist, next, memory_order_release);
if (__builtin_expect(mode != 2, 1)) {
atomic_store_explicit(&page->meta->freelist, next, memory_order_release);
}
} else if (page->used < page->capacity) {
size_t stride = hcls->stride;
if (stride == 0) {
@ -438,7 +760,7 @@ static inline void* tiny_heap_page_pop(tiny_heap_class_t* hcls, int class_idx, t
hcls->stride = (uint16_t)stride;
}
block = (void*)(page->base + ((size_t)page->used * stride));
if (page->meta->carved < page->capacity) {
if (__builtin_expect(mode != 2, 1) && page->meta->carved < page->capacity) {
page->meta->carved++;
}
} else {
@ -446,11 +768,20 @@ static inline void* tiny_heap_page_pop(tiny_heap_class_t* hcls, int class_idx, t
}
page->used++;
if (__builtin_expect(meta_light, 0)) {
page->c7_used_delta++;
page->c7_active_delta++;
if (tiny_c7_delta_should_flush(page)) {
tiny_c7_meta_flush_page(page);
if (__builtin_expect(mode == 2, 0)) {
return tiny_region_id_write_header(block, class_idx);
}
if (__builtin_expect(mode == 1, 0)) {
if (class_idx == 6) {
tiny_c6_mark_delta_site(page, C6_DELTA_ALLOC);
}
page->used_delta++;
page->active_delta++;
if (tiny_heap_delta_should_flush(class_idx, page)) {
if (class_idx == 6) {
tiny_c6_mark_delta_site(page, C6_DELTA_THRESHOLD);
}
tiny_heap_meta_flush_page(class_idx, page);
}
return tiny_region_id_write_header(block, class_idx);
}
@ -462,9 +793,12 @@ static inline void* tiny_heap_page_pop(tiny_heap_class_t* hcls, int class_idx, t
static inline void tiny_heap_page_push_free(int class_idx, tiny_heap_page_t* page, void* base_ptr) {
if (!page || !base_ptr || !page->meta) return;
const int ultra = tiny_heap_meta_ultra_enabled_for_class(class_idx);
tiny_next_write(class_idx, base_ptr, page->free_list);
page->free_list = base_ptr;
atomic_store_explicit(&page->meta->freelist, base_ptr, memory_order_release);
if (!__builtin_expect(ultra, 0)) {
atomic_store_explicit(&page->meta->freelist, base_ptr, memory_order_release);
}
}
static inline void tiny_heap_page_free_local(tiny_heap_ctx_t* ctx,
@ -473,25 +807,131 @@ static inline void tiny_heap_page_free_local(tiny_heap_ctx_t* ctx,
void* base_ptr) {
tiny_heap_class_t* hcls = tiny_heap_class(ctx, class_idx);
if (!hcls || !page || !base_ptr) return;
const int stats_on = (class_idx == 7 && tiny_c7_heap_stats_enabled());
if (__builtin_expect(stats_on, 0)) {
atomic_fetch_add_explicit(&g_c7_heap_stats.free_fast_local, 1, memory_order_relaxed);
if (!tiny_heap_page_is_valid(hcls, page)) return;
TinyHeapClassStats* stats = tiny_heap_stats_for_class(class_idx);
if (__builtin_expect(stats != NULL, 0)) {
atomic_fetch_add_explicit(&stats->free_fast_local, 1, memory_order_relaxed);
}
const int behavior_mode = tiny_heap_behavior_mode_for_class(class_idx);
const int mode = tiny_heap_meta_mode_effective(class_idx);
const int page_stats_on = (class_idx == 7 && tiny_c7_page_stats_enabled());
if (class_idx == 6 && __builtin_expect(mode == 1, 0) && tiny_c6_debug_pop_enabled()) {
SuperSlab* ss_chk = hak_super_lookup(base_ptr);
if (!ss_chk || ss_chk != page->ss || !page->meta || page->meta->class_idx != class_idx) {
fprintf(stderr,
"[C6_FREE_FAIL] ss_chk=%p page_ss=%p meta=%p meta_cls=%d base=%p page=%p used=%u cap=%u last_site=%d\n",
(void*)ss_chk,
(void*)page->ss,
(void*)page->meta,
page->meta ? page->meta->class_idx : -1,
base_ptr,
(void*)page,
(unsigned)page->used,
(unsigned)page->capacity,
page->last_delta_site);
abort();
}
}
const int meta_light = (class_idx == 7 && tiny_c7_meta_light_enabled());
tiny_heap_page_push_free(class_idx, page, base_ptr);
if (page->used > 0) {
page->used--;
if (!__builtin_expect(meta_light, 0)) {
if (__builtin_expect(mode == 2, 0)) {
// ULTRA: meta/active は触らない
} else if (!__builtin_expect(mode == 1, 0)) {
atomic_fetch_sub_explicit(&page->meta->used, 1, memory_order_relaxed);
ss_active_dec_one(page->ss);
} else {
page->c7_used_delta--;
page->c7_active_delta--;
if (tiny_c7_delta_should_flush(page)) {
tiny_c7_meta_flush_page(page);
if (class_idx == 6) {
tiny_c6_mark_delta_site(page, C6_DELTA_FREE);
}
page->used_delta--;
page->active_delta--;
if (tiny_heap_delta_should_flush(class_idx, page)) {
if (class_idx == 6) {
tiny_c6_mark_delta_site(page, C6_DELTA_THRESHOLD);
}
tiny_heap_meta_flush_page(class_idx, page);
}
}
}
if (class_idx == 6 && __builtin_expect(mode == 1, 0) && tiny_c6_debug_pop_enabled()) {
static _Atomic uint32_t g_c6_free_dbg = 0;
uint32_t n = atomic_fetch_add_explicit(&g_c6_free_dbg, 1, memory_order_relaxed);
if (n < 512) {
fprintf(stderr,
"[C6_FREE_DEBUG] page=%p used=%u cap=%u free_list=%p meta=%p ss=%p base=%p used_delta=%d active_delta=%d\n",
(void*)page,
(unsigned)page->used,
(unsigned)page->capacity,
page->free_list,
(void*)page->meta,
(void*)page->ss,
(void*)page->base,
page->used_delta,
page->active_delta);
}
}
if (class_idx == 7 && __builtin_expect(behavior_mode == 2, 0)) {
tiny_heap_page_t* old_cur = hcls->current_page;
if (__builtin_expect(page_stats_on, 0)) {
if (!old_cur) {
atomic_fetch_add_explicit(&g_c7_page_stats.current_set_from_free, 1, memory_order_relaxed);
} else if (old_cur != page) {
atomic_fetch_add_explicit(&g_c7_page_stats.current_dropped_to_partial, 1, memory_order_relaxed);
}
}
// ULTRA: 1ページで回す前提。current を強制保持し、partial/full には追い出さない。
hcls->current_page = page;
if (old_cur && old_cur != page) {
tiny_heap_page_push_to_partial(hcls, old_cur);
}
// partial list に page が重複して残っている場合を避ける
tiny_heap_page_t** pp = &hcls->partial_pages;
while (*pp) {
if (*pp == page) {
*pp = (*pp)->next;
break;
}
pp = &(*pp)->next;
}
return;
}
if (__builtin_expect(behavior_mode == 1, 0)) {
if (page->used == 0) {
tiny_heap_page_becomes_empty(ctx, class_idx, page);
return;
}
if (page->used < page->capacity && page->free_list) {
tiny_heap_page_t* old_cur = hcls->current_page;
if (class_idx == 7 && __builtin_expect(page_stats_on, 0)) {
if (!old_cur) {
atomic_fetch_add_explicit(&g_c7_page_stats.current_set_from_free, 1, memory_order_relaxed);
} else if (old_cur != page) {
atomic_fetch_add_explicit(&g_c7_page_stats.current_dropped_to_partial, 1, memory_order_relaxed);
}
}
tiny_heap_class_unlink(hcls, page);
page->next = NULL;
if (class_idx == 7) {
hcls->current_page = page;
if (old_cur && old_cur != page) {
tiny_heap_page_push_to_partial(hcls, old_cur);
}
} else if (class_idx == 6) {
hcls->current_page = page;
if (old_cur && old_cur != page) {
tiny_heap_page_push_to_partial(hcls, old_cur);
}
} else if (!hcls->current_page) {
hcls->current_page = page;
} else {
tiny_heap_page_push_to_partial(hcls, page);
}
}
return;
}
if (page->used == 0) {
tiny_heap_page_becomes_empty(ctx, class_idx, page);
@ -500,6 +940,13 @@ static inline void tiny_heap_page_free_local(tiny_heap_ctx_t* ctx,
tiny_heap_class_unlink(hcls, page);
page->next = NULL;
if (class_idx == 7) {
if (__builtin_expect(page_stats_on, 0)) {
if (!old_cur) {
atomic_fetch_add_explicit(&g_c7_page_stats.current_set_from_free, 1, memory_order_relaxed);
} else if (old_cur != page) {
atomic_fetch_add_explicit(&g_c7_page_stats.current_dropped_to_partial, 1, memory_order_relaxed);
}
}
hcls->current_page = page;
if (old_cur && old_cur != page) {
tiny_heap_page_push_to_partial(hcls, old_cur);
@ -516,30 +963,53 @@ static inline void tiny_heap_page_free_local(tiny_heap_ctx_t* ctx,
static inline tiny_heap_page_t* tiny_heap_prepare_page(tiny_heap_ctx_t* ctx, int class_idx) {
tiny_heap_class_t* hcls = tiny_heap_class(ctx, class_idx);
const int stats_on = (class_idx == 7 && tiny_c7_heap_stats_enabled());
TinyHeapClassStats* stats = tiny_heap_stats_for_class(class_idx);
const int page_stats_on = (class_idx == 7 && tiny_c7_page_stats_enabled());
if (!hcls) {
if (__builtin_expect(stats_on, 0)) {
atomic_fetch_add_explicit(&g_c7_heap_stats.alloc_prepare_fail, 1, memory_order_relaxed);
if (__builtin_expect(stats != NULL, 0)) {
atomic_fetch_add_explicit(&stats->alloc_prepare_fail, 1, memory_order_relaxed);
}
return NULL;
}
if (__builtin_expect(page_stats_on, 0)) {
atomic_fetch_add_explicit(&g_c7_page_stats.prepare_calls, 1, memory_order_relaxed);
if (!hcls->current_page) {
atomic_fetch_add_explicit(&g_c7_page_stats.prepare_with_current_null, 1, memory_order_relaxed);
if (hcls->partial_pages) {
atomic_fetch_add_explicit(&g_c7_page_stats.prepare_from_partial, 1, memory_order_relaxed);
}
}
}
if (tiny_heap_meta_ultra_enabled_for_class(class_idx)) {
if (hcls->current_page) {
return hcls->current_page;
}
}
if (tiny_heap_meta_light_enabled_for_class(class_idx)) {
if (hcls->current_page &&
(hcls->current_page->free_list || hcls->current_page->used < hcls->current_page->capacity)) {
return hcls->current_page;
}
}
tiny_heap_page_t* page = tiny_heap_take_current(hcls);
if (page) return page;
TinyTLSSlab* tls = &g_tls_slabs[class_idx];
if (!tls->ss) {
if (superslab_refill(class_idx) == NULL) {
if (__builtin_expect(stats_on, 0)) {
atomic_fetch_add_explicit(&g_c7_heap_stats.alloc_prepare_fail, 1, memory_order_relaxed);
if (__builtin_expect(stats != NULL, 0)) {
atomic_fetch_add_explicit(&stats->alloc_prepare_fail, 1, memory_order_relaxed);
}
return NULL;
}
}
tls = &g_tls_slabs[class_idx]; // superslab_refill で更新されるため再取得
if (!tls->ss || !tls->meta) {
if (__builtin_expect(stats_on, 0)) {
atomic_fetch_add_explicit(&g_c7_heap_stats.alloc_prepare_fail, 1, memory_order_relaxed);
if (__builtin_expect(stats != NULL, 0)) {
atomic_fetch_add_explicit(&stats->alloc_prepare_fail, 1, memory_order_relaxed);
}
return NULL;
}
@ -552,16 +1022,46 @@ static inline tiny_heap_page_t* tiny_heap_prepare_page(tiny_heap_ctx_t* ctx, int
return page;
}
// =============================================================================
// C7 HotHeap v2 page lease helpers (Phase 32)
// =============================================================================
static inline TinyHeapPageLease tiny_heap_page_lease_nil(void) {
TinyHeapPageLease lease;
memset(&lease, 0, sizeof(lease));
return lease;
}
static inline TinyHeapPageLease tiny_heap_c7_lease_page_for_v2(void) {
TinyHeapPageLease lease = tiny_heap_page_lease_nil();
tiny_heap_ctx_t* ctx = tiny_heap_ctx_for_thread();
if (!ctx) return lease;
tiny_heap_page_t* page = tiny_heap_prepare_page(ctx, 7);
if (!page) return lease;
lease.page = page;
lease.meta = page->meta;
lease.ss = page->ss;
lease.base = page->base;
lease.capacity = page->capacity;
lease.slab_idx = page->slab_idx;
lease.freelist = page->free_list;
return lease;
}
static inline void tiny_heap_c7_return_page_from_v2(TinyHeapPageLease* lease) {
(void)lease;
// Phase32: C7 v2 は 1 枚使い切り前提。返却処理はまだ持たない。
}
// class_idx 固定での alloc ホットパス
static inline void* tiny_heap_alloc_slow_from_class(tiny_heap_ctx_t* ctx, int class_idx) {
tiny_heap_class_t* hcls = tiny_heap_class(ctx, class_idx);
if (!hcls) return NULL;
const int stats_on = (class_idx == 7 && tiny_c7_heap_stats_enabled());
TinyHeapClassStats* stats = tiny_heap_stats_for_class(class_idx);
tiny_heap_page_t* page = tiny_heap_prepare_page(ctx, class_idx);
if (!page) {
if (__builtin_expect(stats_on, 0)) {
atomic_fetch_add_explicit(&g_c7_heap_stats.alloc_fail, 1, memory_order_relaxed);
if (__builtin_expect(stats != NULL, 0)) {
atomic_fetch_add_explicit(&stats->alloc_fail, 1, memory_order_relaxed);
}
return NULL;
}
@ -580,7 +1080,7 @@ __attribute__((always_inline)) static inline void* tiny_heap_alloc_class_fast(ti
(void)size;
tiny_heap_class_t* hcls = tiny_heap_class(ctx, class_idx);
tiny_heap_page_t* page = hcls ? hcls->current_page : NULL;
const int stats_on = (class_idx == 7 && tiny_c7_heap_stats_enabled());
TinyHeapClassStats* stats = tiny_heap_stats_for_class(class_idx);
if (page) {
if (page->free_list || page->used < page->capacity) {
@ -589,16 +1089,16 @@ __attribute__((always_inline)) static inline void* tiny_heap_alloc_class_fast(ti
if (page->used >= page->capacity && page->free_list == NULL) {
tiny_heap_page_mark_full(hcls, page);
}
if (__builtin_expect(stats_on, 0)) {
atomic_fetch_add_explicit(&g_c7_heap_stats.alloc_fast_current, 1, memory_order_relaxed);
if (__builtin_expect(stats != NULL, 0)) {
atomic_fetch_add_explicit(&stats->alloc_fast_current, 1, memory_order_relaxed);
}
return user;
}
}
}
if (__builtin_expect(stats_on, 0)) {
atomic_fetch_add_explicit(&g_c7_heap_stats.alloc_slow_prepare, 1, memory_order_relaxed);
if (__builtin_expect(stats != NULL, 0)) {
atomic_fetch_add_explicit(&stats->alloc_slow_prepare, 1, memory_order_relaxed);
}
return tiny_heap_alloc_slow_from_class(ctx, class_idx);
}
@ -611,12 +1111,12 @@ static inline void tiny_heap_free_class_fast_with_meta(tiny_heap_ctx_t* ctx,
void* base) {
tiny_heap_class_t* hcls = tiny_heap_class(ctx, class_idx);
if (!base || !ss || slab_idx < 0 || !hcls) return;
const int stats_on = (class_idx == 7 && tiny_c7_heap_stats_enabled());
TinyHeapClassStats* stats = tiny_heap_stats_for_class(class_idx);
tiny_heap_page_t* page = tiny_heap_attach_page(ctx, class_idx, ss, slab_idx);
if (!page) {
if (__builtin_expect(stats_on, 0)) {
atomic_fetch_add_explicit(&g_c7_heap_stats.free_slow_fallback, 1, memory_order_relaxed);
if (__builtin_expect(stats != NULL, 0)) {
atomic_fetch_add_explicit(&stats->free_slow_fallback, 1, memory_order_relaxed);
}
tiny_heap_cold_drain_and_free(class_idx, base);
return;
@ -636,16 +1136,18 @@ static inline void tiny_heap_free_class_fast(tiny_heap_ctx_t* ctx, int class_idx
#endif
SuperSlab* ss = hak_super_lookup(base);
if (!ss || ss->magic != SUPERSLAB_MAGIC) {
if (__builtin_expect((class_idx == 7 && tiny_c7_heap_stats_enabled()), 0)) {
atomic_fetch_add_explicit(&g_c7_heap_stats.free_slow_fallback, 1, memory_order_relaxed);
TinyHeapClassStats* stats = tiny_heap_stats_for_class(class_idx);
if (__builtin_expect(stats != NULL, 0)) {
atomic_fetch_add_explicit(&stats->free_slow_fallback, 1, memory_order_relaxed);
}
tiny_heap_cold_drain_and_free(class_idx, base);
return;
}
int slab_idx = slab_index_for(ss, base);
if (slab_idx < 0 || slab_idx >= ss_slabs_capacity(ss)) {
if (__builtin_expect((class_idx == 7 && tiny_c7_heap_stats_enabled()), 0)) {
atomic_fetch_add_explicit(&g_c7_heap_stats.free_slow_fallback, 1, memory_order_relaxed);
TinyHeapClassStats* stats = tiny_heap_stats_for_class(class_idx);
if (__builtin_expect(stats != NULL, 0)) {
atomic_fetch_add_explicit(&stats->free_slow_fallback, 1, memory_order_relaxed);
}
tiny_heap_cold_drain_and_free(class_idx, base);
return;