Phase 36-37: TinyHotHeap v2 HotBox redesign and C7 current_page policy fixes
- Redefine TinyHotHeap v2 as per-thread Hot Box with clear boundaries - Add comprehensive OS statistics tracking for SS allocations - Implement route-based free handling for TinyHeap v2 - Add C6/C7 debugging and statistics improvements - Update documentation with implementation guidelines and analysis - Add new box headers for stats, routing, and front-end management
This commit is contained in:
Moe Charm (CI)
@ -147,11 +147,331 @@ static void tiny_c7_delta_debug_destructor(void) {
|
||||
|
||||
// =============================================================================
|
||||
// TinyHotHeap v2 (Phase30/31 wiring). Currently C7-only thin wrapper.
|
||||
// NOTE: Phase34/35 時点では v2 は C7-only でも v1 より遅く、mixed では大きな回帰がある。
|
||||
// 実験用フラグを明示 ON にしたときだけ使う前提で、デフォルトは v1 を推奨。
|
||||
// =============================================================================
|
||||
static _Atomic uint64_t g_tiny_hotheap_v2_c7_alloc = 0;
|
||||
static _Atomic uint64_t g_tiny_hotheap_v2_c7_free = 0;
|
||||
static _Atomic uint64_t g_tiny_hotheap_v2_c7_alloc_fallback = 0;
|
||||
static _Atomic uint64_t g_tiny_hotheap_v2_c7_free_fallback = 0;
|
||||
static inline int tiny_hotheap_v2_stats_enabled(void) {
|
||||
static int g = -1;
|
||||
if (__builtin_expect(g == -1, 0)) {
|
||||
const char* e = getenv("HAKMEM_TINY_HOTHEAP_V2_STATS");
|
||||
g = (e && *e && *e != '0') ? 1 : 0;
|
||||
}
|
||||
return g;
|
||||
}
|
||||
|
||||
static _Atomic uint64_t g_tiny_hotheap_v2_c7_alloc_calls = 0;
|
||||
static _Atomic uint64_t g_tiny_hotheap_v2_c7_route_hits = 0;
|
||||
static _Atomic uint64_t g_tiny_hotheap_v2_c7_alloc_fast = 0;
|
||||
static _Atomic uint64_t g_tiny_hotheap_v2_c7_alloc_lease = 0;
|
||||
static _Atomic uint64_t g_tiny_hotheap_v2_c7_alloc_fallback_v1 = 0;
|
||||
static _Atomic uint64_t g_tiny_hotheap_v2_c7_alloc_refill = 0;
|
||||
static _Atomic uint64_t g_tiny_hotheap_v2_c7_alloc_route_fb = 0;
|
||||
static _Atomic uint64_t g_tiny_hotheap_v2_c7_free_calls = 0;
|
||||
static _Atomic uint64_t g_tiny_hotheap_v2_c7_free_fast = 0;
|
||||
static _Atomic uint64_t g_tiny_hotheap_v2_c7_free_fallback_v1 = 0;
|
||||
|
||||
typedef struct {
|
||||
_Atomic uint64_t prepare_calls;
|
||||
_Atomic uint64_t prepare_with_current_null;
|
||||
_Atomic uint64_t prepare_from_partial;
|
||||
_Atomic uint64_t free_made_current;
|
||||
_Atomic uint64_t page_retired;
|
||||
} TinyHotHeapV2PageStats;
|
||||
|
||||
static TinyHotHeapV2PageStats g_tiny_hotheap_v2_page_stats = {0};
|
||||
|
||||
void tiny_hotheap_v2_record_route_fallback(void) {
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_c7_alloc_route_fb, 1, memory_order_relaxed);
|
||||
}
|
||||
|
||||
void tiny_hotheap_v2_record_free_fallback(void) {
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_c7_free_fallback_v1, 1, memory_order_relaxed);
|
||||
}
|
||||
|
||||
void tiny_hotheap_v2_debug_snapshot(tiny_hotheap_v2_stats_snapshot_t* out) {
|
||||
if (!out) return;
|
||||
memset(out, 0, sizeof(*out));
|
||||
out->route_hits = atomic_load_explicit(&g_tiny_hotheap_v2_c7_route_hits, memory_order_relaxed);
|
||||
out->alloc_calls = atomic_load_explicit(&g_tiny_hotheap_v2_c7_alloc_calls, memory_order_relaxed);
|
||||
out->alloc_fast = atomic_load_explicit(&g_tiny_hotheap_v2_c7_alloc_fast, memory_order_relaxed);
|
||||
out->alloc_lease = atomic_load_explicit(&g_tiny_hotheap_v2_c7_alloc_lease, memory_order_relaxed);
|
||||
out->alloc_refill = atomic_load_explicit(&g_tiny_hotheap_v2_c7_alloc_refill, memory_order_relaxed);
|
||||
out->alloc_fallback_v1 = atomic_load_explicit(&g_tiny_hotheap_v2_c7_alloc_fallback_v1, memory_order_relaxed);
|
||||
out->alloc_route_fb = atomic_load_explicit(&g_tiny_hotheap_v2_c7_alloc_route_fb, memory_order_relaxed);
|
||||
out->free_calls = atomic_load_explicit(&g_tiny_hotheap_v2_c7_free_calls, memory_order_relaxed);
|
||||
out->free_fast = atomic_load_explicit(&g_tiny_hotheap_v2_c7_free_fast, memory_order_relaxed);
|
||||
out->free_fallback_v1 = atomic_load_explicit(&g_tiny_hotheap_v2_c7_free_fallback_v1, memory_order_relaxed);
|
||||
out->prepare_calls = atomic_load_explicit(&g_tiny_hotheap_v2_page_stats.prepare_calls, memory_order_relaxed);
|
||||
out->prepare_with_current_null = atomic_load_explicit(&g_tiny_hotheap_v2_page_stats.prepare_with_current_null, memory_order_relaxed);
|
||||
out->prepare_from_partial = atomic_load_explicit(&g_tiny_hotheap_v2_page_stats.prepare_from_partial, memory_order_relaxed);
|
||||
out->free_made_current = atomic_load_explicit(&g_tiny_hotheap_v2_page_stats.free_made_current, memory_order_relaxed);
|
||||
out->page_retired = atomic_load_explicit(&g_tiny_hotheap_v2_page_stats.page_retired, memory_order_relaxed);
|
||||
}
|
||||
|
||||
static tiny_hotheap_page_v2* tiny_hotheap_v2_acquire_page_node(tiny_hotheap_class_v2* hc) {
|
||||
if (!hc) return NULL;
|
||||
if (hc->storage_page.meta == NULL && hc->storage_page.freelist == NULL &&
|
||||
hc->storage_page.capacity == 0) {
|
||||
tiny_hotheap_v2_page_reset(&hc->storage_page);
|
||||
return &hc->storage_page;
|
||||
}
|
||||
tiny_hotheap_page_v2* node = (tiny_hotheap_page_v2*)calloc(1, sizeof(tiny_hotheap_page_v2));
|
||||
if (!node) {
|
||||
return NULL;
|
||||
}
|
||||
tiny_hotheap_v2_page_reset(node);
|
||||
return node;
|
||||
}
|
||||
|
||||
static tiny_hotheap_page_v2* tiny_hotheap_v2_find_page(tiny_hotheap_class_v2* hc,
|
||||
uint8_t class_idx,
|
||||
void* p,
|
||||
TinySlabMeta* meta) {
|
||||
if (!hc || !p) return NULL;
|
||||
const size_t stride = hc->stride ? hc->stride : tiny_stride_for_class(class_idx);
|
||||
const size_t max_span = stride * (size_t)(hc->current_page ? hc->current_page->capacity : 0);
|
||||
tiny_hotheap_page_v2* candidates[3] = {hc->current_page, hc->partial_pages, hc->full_pages};
|
||||
for (int i = 0; i < 3; i++) {
|
||||
for (tiny_hotheap_page_v2* page = candidates[i]; page; page = page->next) {
|
||||
if (meta && page->meta && page->meta != meta) continue;
|
||||
if (!page->base || page->capacity == 0) continue;
|
||||
uint8_t* base = (uint8_t*)page->base;
|
||||
size_t span = stride * (size_t)page->capacity;
|
||||
if ((uint8_t*)p >= base && (uint8_t*)p < base + span) {
|
||||
(void)max_span; // silence unused warning in case stride==0
|
||||
return page;
|
||||
}
|
||||
}
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static inline void tiny_hotheap_v2_build_freelist(tiny_hotheap_page_v2* page,
|
||||
uint8_t class_idx,
|
||||
uint16_t stride) {
|
||||
if (!page || stride == 0) {
|
||||
return;
|
||||
}
|
||||
if (page->used >= page->capacity) {
|
||||
page->freelist = NULL;
|
||||
return;
|
||||
}
|
||||
void* head = NULL;
|
||||
size_t start = page->capacity;
|
||||
while (start > page->used) {
|
||||
start--;
|
||||
uint8_t* block = (uint8_t*)page->base + (start * (size_t)stride);
|
||||
tiny_next_write(class_idx, block, head);
|
||||
head = block;
|
||||
}
|
||||
page->freelist = head;
|
||||
if (page->lease_page) {
|
||||
page->lease_page->free_list = head;
|
||||
page->lease_page->used = page->used;
|
||||
if (page->lease_page->meta) {
|
||||
atomic_store_explicit(&page->lease_page->meta->freelist, head, memory_order_release);
|
||||
if (page->lease_page->meta->carved < page->capacity) {
|
||||
page->lease_page->meta->carved = page->capacity;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static void tiny_hotheap_v2_unlink_page(tiny_hotheap_class_v2* hc, tiny_hotheap_page_v2* target) {
|
||||
if (!hc || !target) return;
|
||||
if (hc->current_page == target) {
|
||||
hc->current_page = NULL;
|
||||
}
|
||||
tiny_hotheap_page_v2** lists[2] = {&hc->partial_pages, &hc->full_pages};
|
||||
for (int i = 0; i < 2; i++) {
|
||||
tiny_hotheap_page_v2** head = lists[i];
|
||||
tiny_hotheap_page_v2* prev = NULL;
|
||||
tiny_hotheap_page_v2* cur = *head;
|
||||
while (cur) {
|
||||
if (cur == target) {
|
||||
if (prev) {
|
||||
prev->next = cur->next;
|
||||
} else {
|
||||
*head = cur->next;
|
||||
}
|
||||
cur->next = NULL;
|
||||
break;
|
||||
}
|
||||
prev = cur;
|
||||
cur = cur->next;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static tiny_hotheap_page_v2* tiny_hotheap_v2_refill_slow(tiny_hotheap_ctx_v2* ctx, uint8_t class_idx) {
|
||||
if (!ctx || class_idx != 7) {
|
||||
return NULL;
|
||||
}
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_c7_alloc_refill, 1, memory_order_relaxed);
|
||||
TinyHeapClassStats* stats = tiny_heap_stats_for_class(7);
|
||||
if (__builtin_expect(stats != NULL, 0)) {
|
||||
atomic_fetch_add_explicit(&stats->alloc_slow_prepare, 1, memory_order_relaxed);
|
||||
}
|
||||
tiny_hotheap_class_v2* hc = &ctx->cls[class_idx];
|
||||
TinyHeapPageLease lease = tiny_heap_c7_lease_page_for_v2();
|
||||
if (!lease.page) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
if (hc->stride == 0) {
|
||||
hc->stride = (uint16_t)tiny_stride_for_class(class_idx);
|
||||
}
|
||||
|
||||
tiny_hotheap_page_v2* page = tiny_hotheap_v2_acquire_page_node(hc);
|
||||
if (!page) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
page->lease_page = lease.page;
|
||||
page->meta = lease.meta;
|
||||
page->ss = lease.ss;
|
||||
page->base = lease.base;
|
||||
page->capacity = lease.capacity;
|
||||
page->slab_idx = lease.slab_idx;
|
||||
page->freelist = lease.freelist;
|
||||
page->used = lease.page->used;
|
||||
if (page->lease_page) {
|
||||
page->lease_page->capacity = page->capacity;
|
||||
page->lease_page->free_list = page->freelist;
|
||||
page->lease_page->base = (uint8_t*)page->base;
|
||||
}
|
||||
const uint16_t stride = hc->stride ? hc->stride : (uint16_t)tiny_stride_for_class(class_idx);
|
||||
if (page->freelist == NULL && page->base && page->capacity > page->used) {
|
||||
tiny_hotheap_v2_build_freelist(page, class_idx, stride);
|
||||
} else if (page->lease_page && page->lease_page->meta) {
|
||||
atomic_store_explicit(&page->lease_page->meta->freelist, page->freelist, memory_order_release);
|
||||
}
|
||||
|
||||
tiny_hotheap_page_v2* old_cur = hc->current_page;
|
||||
hc->current_page = page;
|
||||
page->next = NULL;
|
||||
if (old_cur && old_cur != page) {
|
||||
old_cur->next = hc->partial_pages;
|
||||
hc->partial_pages = old_cur;
|
||||
}
|
||||
if (!hc->current_page || !hc->current_page->freelist || hc->current_page->capacity == 0 ||
|
||||
hc->current_page->used > hc->current_page->capacity) {
|
||||
fprintf(stderr, "[HOTHEAP_V2_REFILL_ASSERT] current_page missing freelist (page=%p freelist=%p cap=%u used=%u)\n",
|
||||
(void*)hc->current_page,
|
||||
hc->current_page ? hc->current_page->freelist : NULL,
|
||||
hc->current_page ? (unsigned)hc->current_page->capacity : 0u,
|
||||
hc->current_page ? (unsigned)hc->current_page->used : 0u);
|
||||
abort();
|
||||
}
|
||||
return hc->current_page;
|
||||
}
|
||||
|
||||
static void tiny_hotheap_v2_page_retire_slow(tiny_hotheap_ctx_v2* ctx,
|
||||
uint8_t class_idx,
|
||||
tiny_hotheap_page_v2* page) {
|
||||
if (!ctx || !page) return;
|
||||
tiny_hotheap_class_v2* hc = &ctx->cls[class_idx];
|
||||
tiny_hotheap_v2_unlink_page(hc, page);
|
||||
TinyHeapPageLease lease = tiny_heap_page_lease_nil();
|
||||
lease.page = page->lease_page;
|
||||
lease.meta = page->meta;
|
||||
lease.ss = page->ss;
|
||||
lease.base = page->base;
|
||||
lease.capacity = page->capacity;
|
||||
lease.slab_idx = page->slab_idx;
|
||||
lease.freelist = page->freelist;
|
||||
tiny_heap_c7_return_page_from_v2(&lease);
|
||||
if (page != &hc->storage_page) {
|
||||
free(page);
|
||||
} else {
|
||||
tiny_hotheap_v2_page_reset(page);
|
||||
}
|
||||
if (!hc->current_page && hc->partial_pages) {
|
||||
hc->current_page = hc->partial_pages;
|
||||
hc->partial_pages = hc->partial_pages->next;
|
||||
if (hc->current_page) {
|
||||
hc->current_page->next = NULL;
|
||||
}
|
||||
}
|
||||
if (tiny_hotheap_v2_stats_enabled()) {
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_page_stats.page_retired, 1, memory_order_relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
static inline void* tiny_hotheap_v2_try_pop(tiny_hotheap_page_v2* candidate,
|
||||
tiny_heap_class_t* v1hcls,
|
||||
TinyHeapClassStats* stats,
|
||||
int stats_on) {
|
||||
if (!candidate || !candidate->lease_page || !v1hcls) {
|
||||
return NULL;
|
||||
}
|
||||
tiny_heap_page_t* ipage = candidate->lease_page;
|
||||
v1hcls->current_page = ipage; // keep v1 hot page pinned to avoid mark_full churn
|
||||
if (!(ipage->free_list || ipage->used < ipage->capacity)) {
|
||||
return NULL;
|
||||
}
|
||||
void* user = tiny_heap_page_pop(v1hcls, 7, ipage);
|
||||
if (!user) {
|
||||
return NULL;
|
||||
}
|
||||
if (ipage->used >= ipage->capacity && ipage->free_list == NULL) {
|
||||
tiny_heap_page_mark_full(v1hcls, ipage);
|
||||
}
|
||||
if (__builtin_expect(stats != NULL, 0)) {
|
||||
atomic_fetch_add_explicit(&stats->alloc_fast_current, 1, memory_order_relaxed);
|
||||
}
|
||||
if (stats_on) {
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_c7_alloc_fast, 1, memory_order_relaxed);
|
||||
}
|
||||
candidate->freelist = ipage->free_list;
|
||||
candidate->used = ipage->used;
|
||||
return tiny_region_id_write_header(user, 7);
|
||||
}
|
||||
|
||||
__attribute__((destructor))
|
||||
static void tiny_hotheap_v2_stats_dump(void) {
|
||||
if (!tiny_hotheap_v2_stats_enabled()) {
|
||||
return;
|
||||
}
|
||||
uint64_t alloc_calls = atomic_load_explicit(&g_tiny_hotheap_v2_c7_alloc_calls, memory_order_relaxed);
|
||||
uint64_t route_hits = atomic_load_explicit(&g_tiny_hotheap_v2_c7_route_hits, memory_order_relaxed);
|
||||
uint64_t alloc_fast = atomic_load_explicit(&g_tiny_hotheap_v2_c7_alloc_fast, memory_order_relaxed);
|
||||
uint64_t alloc_lease = atomic_load_explicit(&g_tiny_hotheap_v2_c7_alloc_lease, memory_order_relaxed);
|
||||
uint64_t alloc_fb = atomic_load_explicit(&g_tiny_hotheap_v2_c7_alloc_fallback_v1, memory_order_relaxed);
|
||||
uint64_t free_calls = atomic_load_explicit(&g_tiny_hotheap_v2_c7_free_calls, memory_order_relaxed);
|
||||
uint64_t free_fast = atomic_load_explicit(&g_tiny_hotheap_v2_c7_free_fast, memory_order_relaxed);
|
||||
uint64_t free_fb = atomic_load_explicit(&g_tiny_hotheap_v2_c7_free_fallback_v1, memory_order_relaxed);
|
||||
|
||||
TinyHotHeapV2PageStats ps = {
|
||||
.prepare_calls = atomic_load_explicit(&g_tiny_hotheap_v2_page_stats.prepare_calls, memory_order_relaxed),
|
||||
.prepare_with_current_null = atomic_load_explicit(&g_tiny_hotheap_v2_page_stats.prepare_with_current_null, memory_order_relaxed),
|
||||
.prepare_from_partial = atomic_load_explicit(&g_tiny_hotheap_v2_page_stats.prepare_from_partial, memory_order_relaxed),
|
||||
.free_made_current = atomic_load_explicit(&g_tiny_hotheap_v2_page_stats.free_made_current, memory_order_relaxed),
|
||||
.page_retired = atomic_load_explicit(&g_tiny_hotheap_v2_page_stats.page_retired, memory_order_relaxed),
|
||||
};
|
||||
|
||||
if (alloc_calls || alloc_fast || alloc_lease || alloc_fb || free_calls || free_fast || free_fb ||
|
||||
ps.prepare_calls || ps.prepare_with_current_null || ps.prepare_from_partial ||
|
||||
ps.free_made_current || ps.page_retired) {
|
||||
fprintf(stderr,
|
||||
"[HOTHEAP_V2_C7_STATS] route_hits=%llu alloc_calls=%llu alloc_fast=%llu alloc_lease=%llu alloc_refill=%llu alloc_fb_v1=%llu alloc_route_fb=%llu free_calls=%llu free_fast=%llu free_fb_v1=%llu prep_calls=%llu prep_null=%llu prep_from_partial=%llu free_made_current=%llu page_retired=%llu\n",
|
||||
(unsigned long long)route_hits,
|
||||
(unsigned long long)alloc_calls,
|
||||
(unsigned long long)alloc_fast,
|
||||
(unsigned long long)alloc_lease,
|
||||
(unsigned long long)atomic_load_explicit(&g_tiny_hotheap_v2_c7_alloc_refill, memory_order_relaxed),
|
||||
(unsigned long long)alloc_fb,
|
||||
(unsigned long long)atomic_load_explicit(&g_tiny_hotheap_v2_c7_alloc_route_fb, memory_order_relaxed),
|
||||
(unsigned long long)free_calls,
|
||||
(unsigned long long)free_fast,
|
||||
(unsigned long long)free_fb,
|
||||
(unsigned long long)ps.prepare_calls,
|
||||
(unsigned long long)ps.prepare_with_current_null,
|
||||
(unsigned long long)ps.prepare_from_partial,
|
||||
(unsigned long long)ps.free_made_current,
|
||||
(unsigned long long)ps.page_retired);
|
||||
}
|
||||
}
|
||||
tiny_hotheap_ctx_v2* tiny_hotheap_v2_tls_get(void) {
|
||||
tiny_hotheap_ctx_v2* ctx = g_tiny_hotheap_ctx_v2;
|
||||
if (__builtin_expect(ctx == NULL, 0)) {
|
||||
@ -161,94 +481,81 @@ tiny_hotheap_ctx_v2* tiny_hotheap_v2_tls_get(void) {
|
||||
abort();
|
||||
}
|
||||
g_tiny_hotheap_ctx_v2 = ctx;
|
||||
// C7 用 stride を最初にだけ設定（他クラスは未使用のまま）
|
||||
ctx->cls[7].stride = (uint16_t)tiny_stride_for_class(7);
|
||||
for (int i = 0; i < TINY_HOTHEAP_MAX_CLASSES; i++) {
|
||||
tiny_hotheap_v2_page_reset(&ctx->cls[i].storage_page);
|
||||
ctx->cls[i].stride = (uint16_t)tiny_stride_for_class(i);
|
||||
}
|
||||
}
|
||||
return ctx;
|
||||
}
|
||||
|
||||
void* tiny_hotheap_v2_alloc(uint8_t class_idx) {
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_c7_alloc, 1, memory_order_relaxed);
|
||||
int stats_on = tiny_hotheap_v2_stats_enabled();
|
||||
if (stats_on) {
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_c7_route_hits, 1, memory_order_relaxed);
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_c7_alloc_calls, 1, memory_order_relaxed);
|
||||
}
|
||||
if (__builtin_expect(class_idx != 7, 0)) {
|
||||
return NULL; // いまは C7 専用
|
||||
}
|
||||
|
||||
tiny_hotheap_ctx_v2* v2ctx = tiny_hotheap_v2_tls_get();
|
||||
tiny_hotheap_class_v2* vhcls = &v2ctx->cls[7];
|
||||
tiny_hotheap_class_v2* vhcls = v2ctx ? &v2ctx->cls[7] : NULL;
|
||||
tiny_hotheap_page_v2* v2page = vhcls ? vhcls->current_page : NULL;
|
||||
if (!v2page && vhcls) {
|
||||
v2page = &vhcls->storage_page;
|
||||
tiny_hotheap_v2_page_reset(v2page);
|
||||
vhcls->current_page = v2page;
|
||||
}
|
||||
|
||||
tiny_heap_ctx_t* v1ctx = tiny_heap_ctx_for_thread();
|
||||
tiny_heap_class_t* v1hcls = tiny_heap_class(v1ctx, 7);
|
||||
TinyHeapClassStats* stats = tiny_heap_stats_for_class(7);
|
||||
|
||||
// Hot path: current_page の lease をそのまま使う
|
||||
if (v2page && v2page->lease_page && v1hcls) {
|
||||
tiny_heap_page_t* ipage = v2page->lease_page;
|
||||
if (ipage->free_list || ipage->used < ipage->capacity) {
|
||||
void* user = tiny_heap_page_pop(v1hcls, 7, ipage);
|
||||
if (user) {
|
||||
if (ipage->used >= ipage->capacity && ipage->free_list == NULL) {
|
||||
tiny_heap_page_mark_full(v1hcls, ipage);
|
||||
}
|
||||
if (__builtin_expect(stats != NULL, 0)) {
|
||||
atomic_fetch_add_explicit(&stats->alloc_fast_current, 1, memory_order_relaxed);
|
||||
}
|
||||
v2page->freelist = ipage->free_list;
|
||||
v2page->used = ipage->used;
|
||||
return user;
|
||||
// Hot path: current_page → partial → refill
|
||||
void* user = tiny_hotheap_v2_try_pop(v2page, v1hcls, stats, stats_on);
|
||||
if (user) {
|
||||
return user;
|
||||
}
|
||||
while (vhcls && vhcls->partial_pages) {
|
||||
if (stats_on) {
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_page_stats.prepare_calls, 1, memory_order_relaxed);
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_page_stats.prepare_from_partial, 1, memory_order_relaxed);
|
||||
if (vhcls->current_page == NULL) {
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_page_stats.prepare_with_current_null, 1, memory_order_relaxed);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (__builtin_expect(stats != NULL, 0)) {
|
||||
atomic_fetch_add_explicit(&stats->alloc_slow_prepare, 1, memory_order_relaxed);
|
||||
}
|
||||
|
||||
// Lease a page from v1 (C7 SAFE) and wrap it
|
||||
TinyHeapPageLease lease = tiny_heap_c7_lease_page_for_v2();
|
||||
if (!lease.page || !vhcls || !v1hcls) {
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_c7_alloc_fallback, 1, memory_order_relaxed);
|
||||
size_t size = vhcls ? (vhcls->stride ? vhcls->stride : tiny_stride_for_class(7)) : tiny_stride_for_class(7);
|
||||
return tiny_c7_alloc_fast(size); // safety fallback to v1
|
||||
}
|
||||
|
||||
if (!v2page) {
|
||||
v2page = &vhcls->storage_page;
|
||||
tiny_hotheap_v2_page_reset(v2page);
|
||||
v2page = vhcls->partial_pages;
|
||||
vhcls->partial_pages = vhcls->partial_pages->next;
|
||||
v2page->next = NULL;
|
||||
vhcls->current_page = v2page;
|
||||
}
|
||||
|
||||
v2page->lease_page = lease.page;
|
||||
v2page->meta = lease.meta;
|
||||
v2page->ss = lease.ss;
|
||||
v2page->base = lease.base;
|
||||
v2page->capacity = lease.capacity;
|
||||
v2page->slab_idx = lease.slab_idx;
|
||||
v2page->freelist = lease.freelist;
|
||||
v2page->used = lease.page->used;
|
||||
|
||||
if (lease.page->free_list || lease.page->used < lease.page->capacity) {
|
||||
void* user = tiny_heap_page_pop(v1hcls, 7, lease.page);
|
||||
user = tiny_hotheap_v2_try_pop(v2page, v1hcls, stats, stats_on);
|
||||
if (user) {
|
||||
if (lease.page->used >= lease.page->capacity && lease.page->free_list == NULL) {
|
||||
tiny_heap_page_mark_full(v1hcls, lease.page);
|
||||
}
|
||||
if (__builtin_expect(stats != NULL, 0)) {
|
||||
atomic_fetch_add_explicit(&stats->alloc_fast_current, 1, memory_order_relaxed);
|
||||
}
|
||||
v2page->freelist = lease.page->free_list;
|
||||
v2page->used = lease.page->used;
|
||||
return user;
|
||||
}
|
||||
}
|
||||
|
||||
// Lease 取得後でも pop できなければ v1 に委譲
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_c7_alloc_fallback, 1, memory_order_relaxed);
|
||||
// Lease a page from v1 (C7 SAFE) and wrap it
|
||||
tiny_hotheap_page_v2* leased = tiny_hotheap_v2_refill_slow(v2ctx, 7);
|
||||
if (!leased || !v1hcls) {
|
||||
if (stats_on) {
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_c7_alloc_fallback_v1, 1, memory_order_relaxed);
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_c7_alloc_route_fb, 1, memory_order_relaxed);
|
||||
}
|
||||
size_t size = vhcls ? (vhcls->stride ? vhcls->stride : tiny_stride_for_class(7)) : tiny_stride_for_class(7);
|
||||
return tiny_c7_alloc_fast(size); // safety fallback to v1
|
||||
}
|
||||
vhcls->current_page = leased;
|
||||
v2page = leased;
|
||||
if (v1hcls && v2page && v2page->lease_page) {
|
||||
v1hcls->current_page = v2page->lease_page;
|
||||
}
|
||||
if (stats_on) {
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_c7_alloc_lease, 1, memory_order_relaxed);
|
||||
}
|
||||
|
||||
user = tiny_hotheap_v2_try_pop(v2page, v1hcls, stats, stats_on);
|
||||
if (user) {
|
||||
return user;
|
||||
}
|
||||
|
||||
if (stats_on) {
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_c7_alloc_fallback_v1, 1, memory_order_relaxed);
|
||||
}
|
||||
size_t size = vhcls ? (vhcls->stride ? vhcls->stride : tiny_stride_for_class(7)) : tiny_stride_for_class(7);
|
||||
return tiny_c7_alloc_fast(size);
|
||||
}
|
||||
@ -257,26 +564,61 @@ void tiny_hotheap_v2_free(uint8_t class_idx, void* p, void* meta) {
|
||||
if (__builtin_expect(class_idx != 7, 0)) {
|
||||
return;
|
||||
}
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_c7_free, 1, memory_order_relaxed);
|
||||
int stats_on = tiny_hotheap_v2_stats_enabled();
|
||||
if (stats_on) {
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_c7_free_calls, 1, memory_order_relaxed);
|
||||
}
|
||||
tiny_hotheap_ctx_v2* v2ctx = tiny_hotheap_v2_tls_get();
|
||||
tiny_hotheap_class_v2* vhcls = v2ctx ? &v2ctx->cls[7] : NULL;
|
||||
tiny_hotheap_page_v2* v2page = vhcls ? vhcls->current_page : NULL;
|
||||
TinySlabMeta* meta_ptr = (TinySlabMeta*)meta;
|
||||
|
||||
tiny_heap_ctx_t* v1ctx = tiny_heap_ctx_for_thread();
|
||||
tiny_heap_class_t* v1hcls = tiny_heap_class(v1ctx, 7);
|
||||
|
||||
if (v2page && v2page->lease_page && meta_ptr && v1hcls &&
|
||||
v2page->meta == meta_ptr && tiny_heap_ptr_in_page_range(v2page->lease_page, p)) {
|
||||
tiny_heap_page_free_local(v1ctx, 7, v2page->lease_page, p);
|
||||
v2page->freelist = v2page->lease_page->free_list;
|
||||
v2page->used = v2page->lease_page->used;
|
||||
vhcls->current_page = v2page; // keep pinned
|
||||
tiny_hotheap_page_v2* page = tiny_hotheap_v2_find_page(vhcls, 7, p, meta_ptr);
|
||||
if (page && page->lease_page && v1hcls && tiny_heap_ptr_in_page_range(page->lease_page, p)) {
|
||||
tiny_heap_page_free_local(v1ctx, 7, page->lease_page, p);
|
||||
page->freelist = page->lease_page->free_list;
|
||||
page->used = page->lease_page->used;
|
||||
if (v1hcls) {
|
||||
v1hcls->current_page = page->lease_page;
|
||||
}
|
||||
if (vhcls && vhcls->current_page != page) {
|
||||
tiny_hotheap_v2_unlink_page(vhcls, page);
|
||||
page->next = vhcls->current_page;
|
||||
vhcls->current_page = page;
|
||||
if (stats_on) {
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_page_stats.free_made_current, 1, memory_order_relaxed);
|
||||
}
|
||||
} else if (stats_on) {
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_page_stats.free_made_current, 1, memory_order_relaxed);
|
||||
}
|
||||
// C7-only: keep the page hot even when empty to avoid churn
|
||||
if (vhcls) {
|
||||
if (!vhcls->current_page) {
|
||||
vhcls->current_page = page;
|
||||
} else if (vhcls->current_page != page) {
|
||||
tiny_hotheap_v2_unlink_page(vhcls, page);
|
||||
page->next = vhcls->current_page;
|
||||
vhcls->current_page = page;
|
||||
}
|
||||
}
|
||||
if (page->used == 0 && vhcls && vhcls->partial_pages != page && vhcls->current_page == page) {
|
||||
// park empty page in partial to allow re-use without immediate Superslab return
|
||||
page->next = vhcls->partial_pages;
|
||||
vhcls->partial_pages = page;
|
||||
vhcls->current_page = page; // still treat as current
|
||||
}
|
||||
if (stats_on) {
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_c7_free_fast, 1, memory_order_relaxed);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// Fallback: mimic v1 free path
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_c7_free_fallback, 1, memory_order_relaxed);
|
||||
if (stats_on) {
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_c7_free_fallback_v1, 1, memory_order_relaxed);
|
||||
}
|
||||
SuperSlab* ss = hak_super_lookup(p);
|
||||
if (ss && ss->magic == SUPERSLAB_MAGIC) {
|
||||
int slab_idx = slab_index_for(ss, p);
|
||||
|
||||
Reference in New Issue
Block a user