tomoaki/hakmem
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

C7 v2: add lease helpers and v2 page reset

Browse Source
This commit is contained in:
Moe Charm (CI)
2025-12-08 14:40:03 +09:00
parent 9502501842
commit 34a8fd69b6
5 changed files with 1171 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files

242
core/hakmem_tiny.c
View File

@ -23,11 +23,14 @@
#include "tiny_debug_ring.h"
#include "tiny_route.h"
#include "front/tiny_heap_v2.h"
#include "box/tiny_front_stats_box.h"
#include "tiny_tls_guard.h"
#include "tiny_ready.h"
#include "box/c7_meta_used_counter_box.h"
#include "box/tiny_c7_hotbox.h"
#include "box/tiny_heap_box.h"
#include "box/tiny_hotheap_v2_box.h"
#include "box/tiny_route_env_box.h"
#include "box/super_reg_box.h"
#include "hakmem_tiny_tls_list.h"
#include "hakmem_tiny_remote_target.h" // Phase 2C-1: Remote target queue
@ -49,47 +52,240 @@ extern uint64_t g_bytes_allocated; // from hakmem_tiny_superslab.c
__thread hak_base_ptr_t s_tls_sll_last_push[TINY_NUM_CLASSES] = {0};
__thread tiny_heap_ctx_t g_tiny_heap_ctx;
__thread int g_tiny_heap_ctx_init = 0;
TinyC7HeapStats g_c7_heap_stats = {0};
__thread tiny_hotheap_ctx_v2* g_tiny_hotheap_ctx_v2 = NULL;
TinyHeapClassStats g_tiny_heap_stats[TINY_NUM_CLASSES] = {0};
TinyC7PageStats g_c7_page_stats = {0};
tiny_route_kind_t g_tiny_route_class[TINY_NUM_CLASSES] = {0};
int g_tiny_route_snapshot_done = 0;
_Atomic uint64_t g_tiny_front_alloc_class[TINY_NUM_CLASSES] = {0};
_Atomic uint64_t g_tiny_front_free_class[TINY_NUM_CLASSES] = {0};
static int tiny_c7_heap_stats_dump_enabled(void) {
static int tiny_heap_stats_dump_enabled(void) {
static int g = -1;
if (__builtin_expect(g == -1, 0)) {
const char* eh = getenv("HAKMEM_TINY_HEAP_STATS_DUMP");
const char* e = getenv("HAKMEM_TINY_C7_HEAP_STATS_DUMP");
g = (e && *e && *e != '0') ? 1 : 0;
g = ((eh && *eh && *eh != '0') || (e && *e && *e != '0')) ? 1 : 0;
}
return g;
}
__attribute__((destructor))
static void tiny_c7_heap_stats_dump(void) {
if (!tiny_c7_heap_stats_enabled() || !tiny_c7_heap_stats_dump_enabled()) {
static void tiny_heap_stats_dump(void) {
if (!tiny_heap_stats_enabled() || !tiny_heap_stats_dump_enabled()) {
return;
}
TinyC7HeapStats snap = {
.alloc_fast_current = atomic_load_explicit(&g_c7_heap_stats.alloc_fast_current, memory_order_relaxed),
.alloc_slow_prepare = atomic_load_explicit(&g_c7_heap_stats.alloc_slow_prepare, memory_order_relaxed),
.free_fast_local = atomic_load_explicit(&g_c7_heap_stats.free_fast_local, memory_order_relaxed),
.free_slow_fallback = atomic_load_explicit(&g_c7_heap_stats.free_slow_fallback, memory_order_relaxed),
.alloc_prepare_fail = atomic_load_explicit(&g_c7_heap_stats.alloc_prepare_fail, memory_order_relaxed),
.alloc_fail = atomic_load_explicit(&g_c7_heap_stats.alloc_fail, memory_order_relaxed),
for (int cls = 0; cls < TINY_NUM_CLASSES; cls++) {
TinyHeapClassStats snap = {
.alloc_fast_current = atomic_load_explicit(&g_tiny_heap_stats[cls].alloc_fast_current, memory_order_relaxed),
.alloc_slow_prepare = atomic_load_explicit(&g_tiny_heap_stats[cls].alloc_slow_prepare, memory_order_relaxed),
.free_fast_local = atomic_load_explicit(&g_tiny_heap_stats[cls].free_fast_local, memory_order_relaxed),
.free_slow_fallback = atomic_load_explicit(&g_tiny_heap_stats[cls].free_slow_fallback, memory_order_relaxed),
.alloc_prepare_fail = atomic_load_explicit(&g_tiny_heap_stats[cls].alloc_prepare_fail, memory_order_relaxed),
.alloc_fail = atomic_load_explicit(&g_tiny_heap_stats[cls].alloc_fail, memory_order_relaxed),
};
if (snap.alloc_fast_current == 0 && snap.alloc_slow_prepare == 0 &&
snap.free_fast_local == 0 && snap.free_slow_fallback == 0 &&
snap.alloc_prepare_fail == 0 && snap.alloc_fail == 0) {
continue;
}
fprintf(stderr,
"[HEAP_STATS cls=%d] alloc_fast_current=%llu alloc_slow_prepare=%llu free_fast_local=%llu free_slow_fallback=%llu alloc_prepare_fail=%llu alloc_fail=%llu\n",
cls,
(unsigned long long)snap.alloc_fast_current,
(unsigned long long)snap.alloc_slow_prepare,
(unsigned long long)snap.free_fast_local,
(unsigned long long)snap.free_slow_fallback,
(unsigned long long)snap.alloc_prepare_fail,
(unsigned long long)snap.alloc_fail);
}
TinyC7PageStats ps = {
.prepare_calls = atomic_load_explicit(&g_c7_page_stats.prepare_calls, memory_order_relaxed),
.prepare_with_current_null = atomic_load_explicit(&g_c7_page_stats.prepare_with_current_null, memory_order_relaxed),
.prepare_from_partial = atomic_load_explicit(&g_c7_page_stats.prepare_from_partial, memory_order_relaxed),
.current_set_from_free = atomic_load_explicit(&g_c7_page_stats.current_set_from_free, memory_order_relaxed),
.current_dropped_to_partial = atomic_load_explicit(&g_c7_page_stats.current_dropped_to_partial, memory_order_relaxed),
};
fprintf(stderr,
"[C7_HEAP_STATS] alloc_fast_current=%llu alloc_slow_prepare=%llu free_fast_local=%llu free_slow_fallback=%llu alloc_prepare_fail=%llu alloc_fail=%llu\n",
(unsigned long long)snap.alloc_fast_current,
(unsigned long long)snap.alloc_slow_prepare,
(unsigned long long)snap.free_fast_local,
(unsigned long long)snap.free_slow_fallback,
(unsigned long long)snap.alloc_prepare_fail,
(unsigned long long)snap.alloc_fail);
fflush(stderr);
if (ps.prepare_calls || ps.prepare_with_current_null || ps.prepare_from_partial ||
ps.current_set_from_free || ps.current_dropped_to_partial) {
fprintf(stderr,
"[C7_PAGE_STATS] prepare_calls=%llu prepare_with_current_null=%llu prepare_from_partial=%llu current_set_from_free=%llu current_dropped_to_partial=%llu\n",
(unsigned long long)ps.prepare_calls,
(unsigned long long)ps.prepare_with_current_null,
(unsigned long long)ps.prepare_from_partial,
(unsigned long long)ps.current_set_from_free,
(unsigned long long)ps.current_dropped_to_partial);
fflush(stderr);
}
}
__attribute__((destructor))
static void tiny_front_class_stats_dump(void) {
if (!tiny_front_class_stats_dump_enabled()) {
return;
}
for (int cls = 0; cls < TINY_NUM_CLASSES; cls++) {
uint64_t a = atomic_load_explicit(&g_tiny_front_alloc_class[cls], memory_order_relaxed);
uint64_t f = atomic_load_explicit(&g_tiny_front_free_class[cls], memory_order_relaxed);
if (a == 0 && f == 0) {
continue;
}
fprintf(stderr, "[FRONT_CLASS cls=%d] alloc=%llu free=%llu\n",
cls, (unsigned long long)a, (unsigned long long)f);
}
}
__attribute__((destructor))
static void tiny_c7_delta_debug_destructor(void) {
if (!tiny_c7_meta_light_enabled() || !tiny_c7_delta_debug_enabled()) {
if (tiny_c7_meta_light_enabled() && tiny_c7_delta_debug_enabled()) {
tiny_c7_heap_debug_dump_deltas();
}
if (tiny_heap_meta_light_enabled_for_class(6) && tiny_c6_delta_debug_enabled()) {
tiny_c6_heap_debug_dump_deltas();
}
}
// =============================================================================
// TinyHotHeap v2 (Phase30/31 wiring). Currently C7-only thin wrapper.
// =============================================================================
static _Atomic uint64_t g_tiny_hotheap_v2_c7_alloc = 0;
static _Atomic uint64_t g_tiny_hotheap_v2_c7_free = 0;
static _Atomic uint64_t g_tiny_hotheap_v2_c7_alloc_fallback = 0;
static _Atomic uint64_t g_tiny_hotheap_v2_c7_free_fallback = 0;
tiny_hotheap_ctx_v2* tiny_hotheap_v2_tls_get(void) {
tiny_hotheap_ctx_v2* ctx = g_tiny_hotheap_ctx_v2;
if (__builtin_expect(ctx == NULL, 0)) {
ctx = (tiny_hotheap_ctx_v2*)calloc(1, sizeof(tiny_hotheap_ctx_v2));
if (__builtin_expect(ctx == NULL, 0)) {
fprintf(stderr, "[TinyHotHeapV2] TLS alloc failed (OOM)\n");
abort();
}
g_tiny_hotheap_ctx_v2 = ctx;
// C7 用 stride を最初にだけ設定（他クラスは未使用のまま）
ctx->cls[7].stride = (uint16_t)tiny_stride_for_class(7);
}
return ctx;
}
void* tiny_hotheap_v2_alloc(uint8_t class_idx) {
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_c7_alloc, 1, memory_order_relaxed);
if (__builtin_expect(class_idx != 7, 0)) {
return NULL; // いまは C7 専用
}
tiny_hotheap_ctx_v2* v2ctx = tiny_hotheap_v2_tls_get();
tiny_hotheap_class_v2* vhcls = &v2ctx->cls[7];
tiny_hotheap_page_v2* v2page = vhcls ? vhcls->current_page : NULL;
if (!v2page && vhcls) {
v2page = &vhcls->storage_page;
tiny_hotheap_v2_page_reset(v2page);
vhcls->current_page = v2page;
}
tiny_heap_ctx_t* v1ctx = tiny_heap_ctx_for_thread();
tiny_heap_class_t* v1hcls = tiny_heap_class(v1ctx, 7);
TinyHeapClassStats* stats = tiny_heap_stats_for_class(7);
// Hot path: current_page の lease をそのまま使う
if (v2page && v2page->lease_page && v1hcls) {
tiny_heap_page_t* ipage = v2page->lease_page;
if (ipage->free_list || ipage->used < ipage->capacity) {
void* user = tiny_heap_page_pop(v1hcls, 7, ipage);
if (user) {
if (ipage->used >= ipage->capacity && ipage->free_list == NULL) {
tiny_heap_page_mark_full(v1hcls, ipage);
}
if (__builtin_expect(stats != NULL, 0)) {
atomic_fetch_add_explicit(&stats->alloc_fast_current, 1, memory_order_relaxed);
}
v2page->freelist = ipage->free_list;
v2page->used = ipage->used;
return user;
}
}
}
if (__builtin_expect(stats != NULL, 0)) {
atomic_fetch_add_explicit(&stats->alloc_slow_prepare, 1, memory_order_relaxed);
}
// Lease a page from v1 (C7 SAFE) and wrap it
TinyHeapPageLease lease = tiny_heap_c7_lease_page_for_v2();
if (!lease.page || !vhcls || !v1hcls) {
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_c7_alloc_fallback, 1, memory_order_relaxed);
size_t size = vhcls ? (vhcls->stride ? vhcls->stride : tiny_stride_for_class(7)) : tiny_stride_for_class(7);
return tiny_c7_alloc_fast(size); // safety fallback to v1
}
if (!v2page) {
v2page = &vhcls->storage_page;
tiny_hotheap_v2_page_reset(v2page);
vhcls->current_page = v2page;
}
v2page->lease_page = lease.page;
v2page->meta = lease.meta;
v2page->ss = lease.ss;
v2page->base = lease.base;
v2page->capacity = lease.capacity;
v2page->slab_idx = lease.slab_idx;
v2page->freelist = lease.freelist;
v2page->used = lease.page->used;
if (lease.page->free_list || lease.page->used < lease.page->capacity) {
void* user = tiny_heap_page_pop(v1hcls, 7, lease.page);
if (user) {
if (lease.page->used >= lease.page->capacity && lease.page->free_list == NULL) {
tiny_heap_page_mark_full(v1hcls, lease.page);
}
if (__builtin_expect(stats != NULL, 0)) {
atomic_fetch_add_explicit(&stats->alloc_fast_current, 1, memory_order_relaxed);
}
v2page->freelist = lease.page->free_list;
v2page->used = lease.page->used;
return user;
}
}
// Lease 取得後でも pop できなければ v1 に委譲
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_c7_alloc_fallback, 1, memory_order_relaxed);
size_t size = vhcls ? (vhcls->stride ? vhcls->stride : tiny_stride_for_class(7)) : tiny_stride_for_class(7);
return tiny_c7_alloc_fast(size);
}
void tiny_hotheap_v2_free(uint8_t class_idx, void* p, void* meta) {
if (__builtin_expect(class_idx != 7, 0)) {
return;
}
tiny_c7_heap_debug_dump_deltas();
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_c7_free, 1, memory_order_relaxed);
tiny_hotheap_ctx_v2* v2ctx = tiny_hotheap_v2_tls_get();
tiny_hotheap_class_v2* vhcls = v2ctx ? &v2ctx->cls[7] : NULL;
tiny_hotheap_page_v2* v2page = vhcls ? vhcls->current_page : NULL;
TinySlabMeta* meta_ptr = (TinySlabMeta*)meta;
tiny_heap_ctx_t* v1ctx = tiny_heap_ctx_for_thread();
tiny_heap_class_t* v1hcls = tiny_heap_class(v1ctx, 7);
if (v2page && v2page->lease_page && meta_ptr && v1hcls &&
v2page->meta == meta_ptr && tiny_heap_ptr_in_page_range(v2page->lease_page, p)) {
tiny_heap_page_free_local(v1ctx, 7, v2page->lease_page, p);
v2page->freelist = v2page->lease_page->free_list;
v2page->used = v2page->lease_page->used;
vhcls->current_page = v2page; // keep pinned
return;
}
// Fallback: mimic v1 free path
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_c7_free_fallback, 1, memory_order_relaxed);
SuperSlab* ss = hak_super_lookup(p);
if (ss && ss->magic == SUPERSLAB_MAGIC) {
int slab_idx = slab_index_for(ss, p);
if (slab_idx >= 0 && slab_idx < ss_slabs_capacity(ss)) {
tiny_c7_free_fast_with_meta(ss, slab_idx, p);
return;
}
}
tiny_c7_free_fast(p);
}
#if !HAKMEM_BUILD_RELEASE