Guard madvise ENOMEM and stabilize pool/tiny front v3
This commit is contained in:
Moe Charm (CI)
@ -11,6 +11,7 @@
|
||||
#include "box/tiny_next_ptr_box.h" // Box API: next pointer read/write
|
||||
#include "box/ptr_conversion_box.h" // Box API: pointer conversion
|
||||
#include "hakmem_env_cache.h" // Priority-2: ENV cache
|
||||
#include "box/tiny_cold_iface_v1.h" // Cold boundary wrapper for TinyHotHeap v2
|
||||
// Phase 1 modules (must come AFTER hakmem_tiny.h for TinyPool definition)
|
||||
#include "hakmem_tiny_batch_refill.h" // Phase 1: Batch refill/spill for mini-magazine
|
||||
#include "hakmem_tiny_stats.h" // Phase 1: Batched statistics (replaces XOR RNG)
|
||||
@ -24,6 +25,8 @@
|
||||
#include "tiny_route.h"
|
||||
#include "front/tiny_heap_v2.h"
|
||||
#include "box/tiny_front_stats_box.h"
|
||||
#include "box/tiny_front_v3_env_box.h"
|
||||
#include "box/ss_os_acquire_box.h"
|
||||
#include "tiny_tls_guard.h"
|
||||
#include "tiny_ready.h"
|
||||
#include "box/c7_meta_used_counter_box.h"
|
||||
@ -32,6 +35,8 @@
|
||||
#include "box/tiny_hotheap_v2_box.h"
|
||||
#include "box/tiny_route_env_box.h"
|
||||
#include "box/super_reg_box.h"
|
||||
#include "tiny_region_id.h"
|
||||
#include "tiny_debug_api.h"
|
||||
#include "hakmem_tiny_tls_list.h"
|
||||
#include "hakmem_tiny_remote_target.h" // Phase 2C-1: Remote target queue
|
||||
#include "hakmem_tiny_bg_spill.h" // Phase 2C-2: Background spill queue
|
||||
@ -59,6 +64,13 @@ tiny_route_kind_t g_tiny_route_class[TINY_NUM_CLASSES] = {0};
|
||||
int g_tiny_route_snapshot_done = 0;
|
||||
_Atomic uint64_t g_tiny_front_alloc_class[TINY_NUM_CLASSES] = {0};
|
||||
_Atomic uint64_t g_tiny_front_free_class[TINY_NUM_CLASSES] = {0};
|
||||
TinyFrontV3Snapshot g_tiny_front_v3_snapshot = {0};
|
||||
int g_tiny_front_v3_snapshot_ready = 0;
|
||||
static TinyFrontV3SizeClassEntry g_tiny_front_v3_lut[TINY_MAX_SIZE + 1] = {0};
|
||||
static int g_tiny_front_v3_lut_ready = 0;
|
||||
|
||||
// Forward decls (to keep deps light in this TU)
|
||||
int unified_cache_enabled(void);
|
||||
|
||||
static int tiny_heap_stats_dump_enabled(void) {
|
||||
static int g = -1;
|
||||
@ -70,6 +82,59 @@ static int tiny_heap_stats_dump_enabled(void) {
|
||||
return g;
|
||||
}
|
||||
|
||||
void tiny_front_v3_snapshot_init(void) {
|
||||
if (g_tiny_front_v3_snapshot_ready) {
|
||||
return;
|
||||
}
|
||||
TinyFrontV3Snapshot snap = {
|
||||
.unified_cache_on = unified_cache_enabled(),
|
||||
.tiny_guard_on = tiny_guard_is_enabled(),
|
||||
.header_mode = (uint8_t)tiny_header_mode(),
|
||||
.header_v3_enabled = tiny_header_v3_enabled(),
|
||||
.header_v3_skip_c7 = tiny_header_v3_skip_c7(),
|
||||
};
|
||||
g_tiny_front_v3_snapshot = snap;
|
||||
g_tiny_front_v3_snapshot_ready = 1;
|
||||
}
|
||||
|
||||
void tiny_front_v3_size_class_lut_init(void) {
|
||||
if (g_tiny_front_v3_lut_ready) {
|
||||
return;
|
||||
}
|
||||
tiny_route_snapshot_init();
|
||||
size_t max_size = tiny_get_max_size();
|
||||
if (max_size > TINY_MAX_SIZE) {
|
||||
max_size = TINY_MAX_SIZE;
|
||||
}
|
||||
for (size_t sz = 0; sz <= TINY_MAX_SIZE; sz++) {
|
||||
TinyFrontV3SizeClassEntry e = {
|
||||
.class_idx = TINY_FRONT_V3_INVALID_CLASS,
|
||||
.route_kind = (uint8_t)TINY_ROUTE_LEGACY,
|
||||
};
|
||||
if (sz == 0 || sz > max_size) {
|
||||
g_tiny_front_v3_lut[sz] = e;
|
||||
continue;
|
||||
}
|
||||
int cls = hak_tiny_size_to_class((int)sz);
|
||||
if (cls >= 0 && cls < TINY_NUM_CLASSES) {
|
||||
e.class_idx = (uint8_t)cls;
|
||||
e.route_kind = (uint8_t)tiny_route_for_class((uint8_t)cls);
|
||||
}
|
||||
g_tiny_front_v3_lut[sz] = e;
|
||||
}
|
||||
g_tiny_front_v3_lut_ready = 1;
|
||||
}
|
||||
|
||||
const TinyFrontV3SizeClassEntry* tiny_front_v3_lut_lookup(size_t size) {
|
||||
if (__builtin_expect(!g_tiny_front_v3_lut_ready, 0)) {
|
||||
tiny_front_v3_size_class_lut_init();
|
||||
}
|
||||
if (size == 0 || size > TINY_MAX_SIZE) {
|
||||
return NULL;
|
||||
}
|
||||
return &g_tiny_front_v3_lut[size];
|
||||
}
|
||||
|
||||
__attribute__((destructor))
|
||||
static void tiny_heap_stats_dump(void) {
|
||||
if (!tiny_heap_stats_enabled() || !tiny_heap_stats_dump_enabled()) {
|
||||
@ -159,16 +224,24 @@ static inline int tiny_hotheap_v2_stats_enabled(void) {
|
||||
return g;
|
||||
}
|
||||
|
||||
static _Atomic uint64_t g_tiny_hotheap_v2_c7_alloc_calls = 0;
|
||||
static _Atomic uint64_t g_tiny_hotheap_v2_c7_route_hits = 0;
|
||||
static _Atomic uint64_t g_tiny_hotheap_v2_c7_alloc_fast = 0;
|
||||
static _Atomic uint64_t g_tiny_hotheap_v2_c7_alloc_lease = 0;
|
||||
static _Atomic uint64_t g_tiny_hotheap_v2_c7_alloc_fallback_v1 = 0;
|
||||
static _Atomic uint64_t g_tiny_hotheap_v2_c7_alloc_refill = 0;
|
||||
static _Atomic uint64_t g_tiny_hotheap_v2_c7_alloc_route_fb = 0;
|
||||
static _Atomic uint64_t g_tiny_hotheap_v2_c7_free_calls = 0;
|
||||
static _Atomic uint64_t g_tiny_hotheap_v2_c7_free_fast = 0;
|
||||
static _Atomic uint64_t g_tiny_hotheap_v2_c7_free_fallback_v1 = 0;
|
||||
static _Atomic uint64_t g_tiny_hotheap_v2_route_hits[TINY_HOTHEAP_MAX_CLASSES] = {0};
|
||||
static _Atomic uint64_t g_tiny_hotheap_v2_alloc_calls[TINY_HOTHEAP_MAX_CLASSES] = {0};
|
||||
static _Atomic uint64_t g_tiny_hotheap_v2_alloc_fast[TINY_HOTHEAP_MAX_CLASSES] = {0};
|
||||
static _Atomic uint64_t g_tiny_hotheap_v2_alloc_lease[TINY_HOTHEAP_MAX_CLASSES] = {0};
|
||||
static _Atomic uint64_t g_tiny_hotheap_v2_alloc_fallback_v1[TINY_HOTHEAP_MAX_CLASSES] = {0};
|
||||
static _Atomic uint64_t g_tiny_hotheap_v2_alloc_refill[TINY_HOTHEAP_MAX_CLASSES] = {0};
|
||||
static _Atomic uint64_t g_tiny_hotheap_v2_refill_with_current[TINY_HOTHEAP_MAX_CLASSES] = {0};
|
||||
static _Atomic uint64_t g_tiny_hotheap_v2_refill_with_partial[TINY_HOTHEAP_MAX_CLASSES] = {0};
|
||||
static _Atomic uint64_t g_tiny_hotheap_v2_alloc_route_fb[TINY_HOTHEAP_MAX_CLASSES] = {0};
|
||||
static _Atomic uint64_t g_tiny_hotheap_v2_free_calls[TINY_HOTHEAP_MAX_CLASSES] = {0};
|
||||
static _Atomic uint64_t g_tiny_hotheap_v2_free_fast[TINY_HOTHEAP_MAX_CLASSES] = {0};
|
||||
static _Atomic uint64_t g_tiny_hotheap_v2_free_fallback_v1[TINY_HOTHEAP_MAX_CLASSES] = {0};
|
||||
static _Atomic uint64_t g_tiny_hotheap_v2_cold_refill_fail[TINY_HOTHEAP_MAX_CLASSES] = {0};
|
||||
static _Atomic uint64_t g_tiny_hotheap_v2_cold_retire_calls[TINY_HOTHEAP_MAX_CLASSES] = {0};
|
||||
static _Atomic uint64_t g_tiny_hotheap_v2_retire_calls_v2[TINY_HOTHEAP_MAX_CLASSES] = {0};
|
||||
static _Atomic uint64_t g_tiny_hotheap_v2_partial_pushes[TINY_HOTHEAP_MAX_CLASSES] = {0};
|
||||
static _Atomic uint64_t g_tiny_hotheap_v2_partial_pops[TINY_HOTHEAP_MAX_CLASSES] = {0};
|
||||
static _Atomic uint64_t g_tiny_hotheap_v2_partial_peak[TINY_HOTHEAP_MAX_CLASSES] = {0};
|
||||
|
||||
typedef struct {
|
||||
_Atomic uint64_t prepare_calls;
|
||||
@ -178,34 +251,54 @@ typedef struct {
|
||||
_Atomic uint64_t page_retired;
|
||||
} TinyHotHeapV2PageStats;
|
||||
|
||||
static TinyHotHeapV2PageStats g_tiny_hotheap_v2_page_stats = {0};
|
||||
static TinyHotHeapV2PageStats g_tiny_hotheap_v2_page_stats[TINY_HOTHEAP_MAX_CLASSES] = {0};
|
||||
static void tiny_hotheap_v2_page_retire_slow(tiny_hotheap_ctx_v2* ctx,
|
||||
uint8_t class_idx,
|
||||
tiny_hotheap_page_v2* page);
|
||||
|
||||
void tiny_hotheap_v2_record_route_fallback(void) {
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_c7_alloc_route_fb, 1, memory_order_relaxed);
|
||||
static inline uint8_t tiny_hotheap_v2_idx(uint8_t class_idx) {
|
||||
return (class_idx < TINY_HOTHEAP_MAX_CLASSES) ? class_idx : 0;
|
||||
}
|
||||
|
||||
void tiny_hotheap_v2_record_free_fallback(void) {
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_c7_free_fallback_v1, 1, memory_order_relaxed);
|
||||
void tiny_hotheap_v2_record_route_fallback(uint8_t class_idx) {
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_alloc_route_fb[tiny_hotheap_v2_idx(class_idx)],
|
||||
1,
|
||||
memory_order_relaxed);
|
||||
}
|
||||
|
||||
void tiny_hotheap_v2_record_free_fallback(uint8_t class_idx) {
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_free_fallback_v1[tiny_hotheap_v2_idx(class_idx)],
|
||||
1,
|
||||
memory_order_relaxed);
|
||||
}
|
||||
|
||||
void tiny_hotheap_v2_debug_snapshot(tiny_hotheap_v2_stats_snapshot_t* out) {
|
||||
if (!out) return;
|
||||
memset(out, 0, sizeof(*out));
|
||||
out->route_hits = atomic_load_explicit(&g_tiny_hotheap_v2_c7_route_hits, memory_order_relaxed);
|
||||
out->alloc_calls = atomic_load_explicit(&g_tiny_hotheap_v2_c7_alloc_calls, memory_order_relaxed);
|
||||
out->alloc_fast = atomic_load_explicit(&g_tiny_hotheap_v2_c7_alloc_fast, memory_order_relaxed);
|
||||
out->alloc_lease = atomic_load_explicit(&g_tiny_hotheap_v2_c7_alloc_lease, memory_order_relaxed);
|
||||
out->alloc_refill = atomic_load_explicit(&g_tiny_hotheap_v2_c7_alloc_refill, memory_order_relaxed);
|
||||
out->alloc_fallback_v1 = atomic_load_explicit(&g_tiny_hotheap_v2_c7_alloc_fallback_v1, memory_order_relaxed);
|
||||
out->alloc_route_fb = atomic_load_explicit(&g_tiny_hotheap_v2_c7_alloc_route_fb, memory_order_relaxed);
|
||||
out->free_calls = atomic_load_explicit(&g_tiny_hotheap_v2_c7_free_calls, memory_order_relaxed);
|
||||
out->free_fast = atomic_load_explicit(&g_tiny_hotheap_v2_c7_free_fast, memory_order_relaxed);
|
||||
out->free_fallback_v1 = atomic_load_explicit(&g_tiny_hotheap_v2_c7_free_fallback_v1, memory_order_relaxed);
|
||||
out->prepare_calls = atomic_load_explicit(&g_tiny_hotheap_v2_page_stats.prepare_calls, memory_order_relaxed);
|
||||
out->prepare_with_current_null = atomic_load_explicit(&g_tiny_hotheap_v2_page_stats.prepare_with_current_null, memory_order_relaxed);
|
||||
out->prepare_from_partial = atomic_load_explicit(&g_tiny_hotheap_v2_page_stats.prepare_from_partial, memory_order_relaxed);
|
||||
out->free_made_current = atomic_load_explicit(&g_tiny_hotheap_v2_page_stats.free_made_current, memory_order_relaxed);
|
||||
out->page_retired = atomic_load_explicit(&g_tiny_hotheap_v2_page_stats.page_retired, memory_order_relaxed);
|
||||
uint8_t ci = 7;
|
||||
out->route_hits = atomic_load_explicit(&g_tiny_hotheap_v2_route_hits[ci], memory_order_relaxed);
|
||||
out->alloc_calls = atomic_load_explicit(&g_tiny_hotheap_v2_alloc_calls[ci], memory_order_relaxed);
|
||||
out->alloc_fast = atomic_load_explicit(&g_tiny_hotheap_v2_alloc_fast[ci], memory_order_relaxed);
|
||||
out->alloc_lease = atomic_load_explicit(&g_tiny_hotheap_v2_alloc_lease[ci], memory_order_relaxed);
|
||||
out->alloc_refill = atomic_load_explicit(&g_tiny_hotheap_v2_alloc_refill[ci], memory_order_relaxed);
|
||||
out->refill_with_current = atomic_load_explicit(&g_tiny_hotheap_v2_refill_with_current[ci], memory_order_relaxed);
|
||||
out->refill_with_partial = atomic_load_explicit(&g_tiny_hotheap_v2_refill_with_partial[ci], memory_order_relaxed);
|
||||
out->alloc_fallback_v1 = atomic_load_explicit(&g_tiny_hotheap_v2_alloc_fallback_v1[ci], memory_order_relaxed);
|
||||
out->alloc_route_fb = atomic_load_explicit(&g_tiny_hotheap_v2_alloc_route_fb[ci], memory_order_relaxed);
|
||||
out->free_calls = atomic_load_explicit(&g_tiny_hotheap_v2_free_calls[ci], memory_order_relaxed);
|
||||
out->free_fast = atomic_load_explicit(&g_tiny_hotheap_v2_free_fast[ci], memory_order_relaxed);
|
||||
out->free_fallback_v1 = atomic_load_explicit(&g_tiny_hotheap_v2_free_fallback_v1[ci], memory_order_relaxed);
|
||||
out->cold_refill_fail = atomic_load_explicit(&g_tiny_hotheap_v2_cold_refill_fail[ci], memory_order_relaxed);
|
||||
out->cold_retire_calls = atomic_load_explicit(&g_tiny_hotheap_v2_cold_retire_calls[ci], memory_order_relaxed);
|
||||
out->retire_calls_v2 = atomic_load_explicit(&g_tiny_hotheap_v2_retire_calls_v2[ci], memory_order_relaxed);
|
||||
out->prepare_calls = atomic_load_explicit(&g_tiny_hotheap_v2_page_stats[ci].prepare_calls, memory_order_relaxed);
|
||||
out->prepare_with_current_null = atomic_load_explicit(&g_tiny_hotheap_v2_page_stats[ci].prepare_with_current_null, memory_order_relaxed);
|
||||
out->prepare_from_partial = atomic_load_explicit(&g_tiny_hotheap_v2_page_stats[ci].prepare_from_partial, memory_order_relaxed);
|
||||
out->free_made_current = atomic_load_explicit(&g_tiny_hotheap_v2_page_stats[ci].free_made_current, memory_order_relaxed);
|
||||
out->page_retired = atomic_load_explicit(&g_tiny_hotheap_v2_page_stats[ci].page_retired, memory_order_relaxed);
|
||||
out->partial_pushes = atomic_load_explicit(&g_tiny_hotheap_v2_partial_pushes[ci], memory_order_relaxed);
|
||||
out->partial_pops = atomic_load_explicit(&g_tiny_hotheap_v2_partial_pops[ci], memory_order_relaxed);
|
||||
out->partial_peak = atomic_load_explicit(&g_tiny_hotheap_v2_partial_peak[ci], memory_order_relaxed);
|
||||
}
|
||||
|
||||
static tiny_hotheap_page_v2* tiny_hotheap_v2_acquire_page_node(tiny_hotheap_class_v2* hc) {
|
||||
@ -246,6 +339,57 @@ static tiny_hotheap_page_v2* tiny_hotheap_v2_find_page(tiny_hotheap_class_v2* hc
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static inline void tiny_hotheap_v2_partial_push(tiny_hotheap_class_v2* hc,
|
||||
tiny_hotheap_page_v2* page,
|
||||
uint8_t class_idx,
|
||||
int stats_on) {
|
||||
if (!hc || !page) return;
|
||||
page->next = hc->partial_pages;
|
||||
hc->partial_pages = page;
|
||||
if (hc->partial_count < UINT16_MAX) {
|
||||
hc->partial_count++;
|
||||
}
|
||||
if (stats_on) {
|
||||
uint8_t idx = tiny_hotheap_v2_idx(class_idx);
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_partial_pushes[idx], 1, memory_order_relaxed);
|
||||
uint64_t cur = hc->partial_count;
|
||||
uint64_t old = atomic_load_explicit(&g_tiny_hotheap_v2_partial_peak[idx], memory_order_relaxed);
|
||||
while (cur > old &&
|
||||
!atomic_compare_exchange_weak_explicit(&g_tiny_hotheap_v2_partial_peak[idx],
|
||||
&old,
|
||||
cur,
|
||||
memory_order_relaxed,
|
||||
memory_order_relaxed)) {
|
||||
old = atomic_load_explicit(&g_tiny_hotheap_v2_partial_peak[idx], memory_order_relaxed);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static inline void tiny_hotheap_v2_maybe_trim_partial(tiny_hotheap_ctx_v2* ctx,
|
||||
tiny_hotheap_class_v2* hc,
|
||||
uint8_t class_idx,
|
||||
int stats_on) {
|
||||
if (!ctx || !hc) return;
|
||||
uint16_t limit = hc->max_partial_pages;
|
||||
if (limit == 0) {
|
||||
return;
|
||||
}
|
||||
while (hc->partial_count > limit && hc->partial_pages) {
|
||||
tiny_hotheap_page_v2* victim = hc->partial_pages;
|
||||
hc->partial_pages = victim->next;
|
||||
if (hc->partial_count > 0) {
|
||||
hc->partial_count--;
|
||||
}
|
||||
victim->next = NULL;
|
||||
if (stats_on) {
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_partial_pops[tiny_hotheap_v2_idx(class_idx)],
|
||||
1,
|
||||
memory_order_relaxed);
|
||||
}
|
||||
tiny_hotheap_v2_page_retire_slow(ctx, class_idx, victim);
|
||||
}
|
||||
}
|
||||
|
||||
static inline void tiny_hotheap_v2_build_freelist(tiny_hotheap_page_v2* page,
|
||||
uint8_t class_idx,
|
||||
uint16_t stride) {
|
||||
@ -265,16 +409,6 @@ static inline void tiny_hotheap_v2_build_freelist(tiny_hotheap_page_v2* page,
|
||||
head = block;
|
||||
}
|
||||
page->freelist = head;
|
||||
if (page->lease_page) {
|
||||
page->lease_page->free_list = head;
|
||||
page->lease_page->used = page->used;
|
||||
if (page->lease_page->meta) {
|
||||
atomic_store_explicit(&page->lease_page->meta->freelist, head, memory_order_release);
|
||||
if (page->lease_page->meta->carved < page->capacity) {
|
||||
page->lease_page->meta->carved = page->capacity;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static void tiny_hotheap_v2_unlink_page(tiny_hotheap_class_v2* hc, tiny_hotheap_page_v2* target) {
|
||||
@ -295,6 +429,9 @@ static void tiny_hotheap_v2_unlink_page(tiny_hotheap_class_v2* hc, tiny_hotheap_
|
||||
*head = cur->next;
|
||||
}
|
||||
cur->next = NULL;
|
||||
if (i == 0 && hc->partial_count > 0) {
|
||||
hc->partial_count--;
|
||||
}
|
||||
break;
|
||||
}
|
||||
prev = cur;
|
||||
@ -304,17 +441,35 @@ static void tiny_hotheap_v2_unlink_page(tiny_hotheap_class_v2* hc, tiny_hotheap_
|
||||
}
|
||||
|
||||
static tiny_hotheap_page_v2* tiny_hotheap_v2_refill_slow(tiny_hotheap_ctx_v2* ctx, uint8_t class_idx) {
|
||||
if (!ctx || class_idx != 7) {
|
||||
if (!ctx || class_idx >= TINY_HOTHEAP_MAX_CLASSES) {
|
||||
return NULL;
|
||||
}
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_c7_alloc_refill, 1, memory_order_relaxed);
|
||||
TinyHeapClassStats* stats = tiny_heap_stats_for_class(7);
|
||||
int stats_on = tiny_hotheap_v2_stats_enabled();
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_alloc_refill[class_idx], 1, memory_order_relaxed);
|
||||
TinyHeapClassStats* stats = tiny_heap_stats_for_class(class_idx);
|
||||
if (__builtin_expect(stats != NULL, 0)) {
|
||||
atomic_fetch_add_explicit(&stats->alloc_slow_prepare, 1, memory_order_relaxed);
|
||||
}
|
||||
tiny_hotheap_class_v2* hc = &ctx->cls[class_idx];
|
||||
TinyHeapPageLease lease = tiny_heap_c7_lease_page_for_v2();
|
||||
if (!lease.page) {
|
||||
if (hc) {
|
||||
if (hc->current_page) {
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_refill_with_current[class_idx],
|
||||
1,
|
||||
memory_order_relaxed);
|
||||
}
|
||||
if (hc->partial_pages) {
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_refill_with_partial[class_idx],
|
||||
1,
|
||||
memory_order_relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
// Cold iface (v1 TinyHeap) からページを 1 枚借りる
|
||||
TinyColdIface cold = tiny_cold_iface_v1();
|
||||
tiny_heap_ctx_t* cold_ctx = tiny_heap_ctx_for_thread();
|
||||
tiny_heap_page_t* ipage = cold.refill_page ? cold.refill_page(cold_ctx, class_idx) : NULL;
|
||||
if (!ipage || !ipage->base || ipage->capacity == 0 || ipage->meta == NULL) {
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_cold_refill_fail[class_idx], 1, memory_order_relaxed);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
@ -327,33 +482,25 @@ static tiny_hotheap_page_v2* tiny_hotheap_v2_refill_slow(tiny_hotheap_ctx_v2* ct
|
||||
return NULL;
|
||||
}
|
||||
|
||||
page->lease_page = lease.page;
|
||||
page->meta = lease.meta;
|
||||
page->ss = lease.ss;
|
||||
page->base = lease.base;
|
||||
page->capacity = lease.capacity;
|
||||
page->slab_idx = lease.slab_idx;
|
||||
page->freelist = lease.freelist;
|
||||
page->used = lease.page->used;
|
||||
if (page->lease_page) {
|
||||
page->lease_page->capacity = page->capacity;
|
||||
page->lease_page->free_list = page->freelist;
|
||||
page->lease_page->base = (uint8_t*)page->base;
|
||||
}
|
||||
page->lease_page = ipage;
|
||||
page->meta = ipage->meta;
|
||||
page->ss = ipage->ss;
|
||||
page->base = ipage->base;
|
||||
page->capacity = ipage->capacity;
|
||||
page->slab_idx = ipage->slab_idx;
|
||||
page->freelist = NULL;
|
||||
page->used = 0;
|
||||
|
||||
const uint16_t stride = hc->stride ? hc->stride : (uint16_t)tiny_stride_for_class(class_idx);
|
||||
if (page->freelist == NULL && page->base && page->capacity > page->used) {
|
||||
tiny_hotheap_v2_build_freelist(page, class_idx, stride);
|
||||
} else if (page->lease_page && page->lease_page->meta) {
|
||||
atomic_store_explicit(&page->lease_page->meta->freelist, page->freelist, memory_order_release);
|
||||
}
|
||||
tiny_hotheap_v2_build_freelist(page, class_idx, stride);
|
||||
|
||||
tiny_hotheap_page_v2* old_cur = hc->current_page;
|
||||
hc->current_page = page;
|
||||
page->next = NULL;
|
||||
if (old_cur && old_cur != page) {
|
||||
old_cur->next = hc->partial_pages;
|
||||
hc->partial_pages = old_cur;
|
||||
tiny_hotheap_v2_partial_push(hc, old_cur, class_idx, stats_on);
|
||||
}
|
||||
tiny_hotheap_v2_maybe_trim_partial(ctx, hc, class_idx, stats_on);
|
||||
if (!hc->current_page || !hc->current_page->freelist || hc->current_page->capacity == 0 ||
|
||||
hc->current_page->used > hc->current_page->capacity) {
|
||||
fprintf(stderr, "[HOTHEAP_V2_REFILL_ASSERT] current_page missing freelist (page=%p freelist=%p cap=%u used=%u)\n",
|
||||
@ -361,7 +508,7 @@ static tiny_hotheap_page_v2* tiny_hotheap_v2_refill_slow(tiny_hotheap_ctx_v2* ct
|
||||
hc->current_page ? hc->current_page->freelist : NULL,
|
||||
hc->current_page ? (unsigned)hc->current_page->capacity : 0u,
|
||||
hc->current_page ? (unsigned)hc->current_page->used : 0u);
|
||||
abort();
|
||||
return NULL;
|
||||
}
|
||||
return hc->current_page;
|
||||
}
|
||||
@ -370,17 +517,26 @@ static void tiny_hotheap_v2_page_retire_slow(tiny_hotheap_ctx_v2* ctx,
|
||||
uint8_t class_idx,
|
||||
tiny_hotheap_page_v2* page) {
|
||||
if (!ctx || !page) return;
|
||||
uint8_t idx = tiny_hotheap_v2_idx(class_idx);
|
||||
tiny_hotheap_class_v2* hc = &ctx->cls[class_idx];
|
||||
tiny_hotheap_v2_unlink_page(hc, page);
|
||||
TinyHeapPageLease lease = tiny_heap_page_lease_nil();
|
||||
lease.page = page->lease_page;
|
||||
lease.meta = page->meta;
|
||||
lease.ss = page->ss;
|
||||
lease.base = page->base;
|
||||
lease.capacity = page->capacity;
|
||||
lease.slab_idx = page->slab_idx;
|
||||
lease.freelist = page->freelist;
|
||||
tiny_heap_c7_return_page_from_v2(&lease);
|
||||
if (page->lease_page) {
|
||||
page->lease_page->used = page->used;
|
||||
page->lease_page->free_list = page->freelist;
|
||||
if (page->lease_page->meta) {
|
||||
atomic_store_explicit(&page->lease_page->meta->freelist, page->freelist, memory_order_release);
|
||||
atomic_store_explicit(&page->lease_page->meta->used, page->used, memory_order_relaxed);
|
||||
}
|
||||
}
|
||||
TinyColdIface cold = tiny_cold_iface_v1();
|
||||
tiny_heap_ctx_t* cold_ctx = tiny_heap_ctx_for_thread();
|
||||
if (cold.retire_page) {
|
||||
cold.retire_page(cold_ctx, class_idx, page->lease_page);
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_cold_retire_calls[idx], 1, memory_order_relaxed);
|
||||
}
|
||||
if (tiny_hotheap_v2_stats_enabled()) {
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_retire_calls_v2[idx], 1, memory_order_relaxed);
|
||||
}
|
||||
if (page != &hc->storage_page) {
|
||||
free(page);
|
||||
} else {
|
||||
@ -394,38 +550,42 @@ static void tiny_hotheap_v2_page_retire_slow(tiny_hotheap_ctx_v2* ctx,
|
||||
}
|
||||
}
|
||||
if (tiny_hotheap_v2_stats_enabled()) {
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_page_stats.page_retired, 1, memory_order_relaxed);
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_page_stats[idx].page_retired, 1, memory_order_relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
static inline void* tiny_hotheap_v2_try_pop(tiny_hotheap_page_v2* candidate,
|
||||
tiny_heap_class_t* v1hcls,
|
||||
static inline void* tiny_hotheap_v2_try_pop(tiny_hotheap_class_v2* hc,
|
||||
tiny_hotheap_page_v2* page,
|
||||
uint8_t class_idx,
|
||||
TinyHeapClassStats* stats,
|
||||
int stats_on) {
|
||||
if (!candidate || !candidate->lease_page || !v1hcls) {
|
||||
if (!hc || !page || !page->base || page->capacity == 0) {
|
||||
return NULL;
|
||||
}
|
||||
tiny_heap_page_t* ipage = candidate->lease_page;
|
||||
v1hcls->current_page = ipage; // keep v1 hot page pinned to avoid mark_full churn
|
||||
if (!(ipage->free_list || ipage->used < ipage->capacity)) {
|
||||
if (hc->stride == 0) {
|
||||
hc->stride = (uint16_t)tiny_stride_for_class(class_idx);
|
||||
}
|
||||
const uint16_t stride = hc->stride;
|
||||
void* block = NULL;
|
||||
if (page->freelist) {
|
||||
block = page->freelist;
|
||||
void* next = tiny_next_read(class_idx, block);
|
||||
page->freelist = next;
|
||||
} else if (page->used < page->capacity) {
|
||||
block = (void*)((uint8_t*)page->base + ((size_t)page->used * stride));
|
||||
} else {
|
||||
return NULL;
|
||||
}
|
||||
void* user = tiny_heap_page_pop(v1hcls, 7, ipage);
|
||||
if (!user) {
|
||||
return NULL;
|
||||
}
|
||||
if (ipage->used >= ipage->capacity && ipage->free_list == NULL) {
|
||||
tiny_heap_page_mark_full(v1hcls, ipage);
|
||||
}
|
||||
page->used++;
|
||||
if (__builtin_expect(stats != NULL, 0)) {
|
||||
atomic_fetch_add_explicit(&stats->alloc_fast_current, 1, memory_order_relaxed);
|
||||
}
|
||||
if (stats_on) {
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_c7_alloc_fast, 1, memory_order_relaxed);
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_alloc_fast[tiny_hotheap_v2_idx(class_idx)],
|
||||
1,
|
||||
memory_order_relaxed);
|
||||
}
|
||||
candidate->freelist = ipage->free_list;
|
||||
candidate->used = ipage->used;
|
||||
return tiny_region_id_write_header(user, 7);
|
||||
return tiny_region_id_write_header(block, class_idx);
|
||||
}
|
||||
|
||||
__attribute__((destructor))
|
||||
@ -433,35 +593,55 @@ static void tiny_hotheap_v2_stats_dump(void) {
|
||||
if (!tiny_hotheap_v2_stats_enabled()) {
|
||||
return;
|
||||
}
|
||||
uint64_t alloc_calls = atomic_load_explicit(&g_tiny_hotheap_v2_c7_alloc_calls, memory_order_relaxed);
|
||||
uint64_t route_hits = atomic_load_explicit(&g_tiny_hotheap_v2_c7_route_hits, memory_order_relaxed);
|
||||
uint64_t alloc_fast = atomic_load_explicit(&g_tiny_hotheap_v2_c7_alloc_fast, memory_order_relaxed);
|
||||
uint64_t alloc_lease = atomic_load_explicit(&g_tiny_hotheap_v2_c7_alloc_lease, memory_order_relaxed);
|
||||
uint64_t alloc_fb = atomic_load_explicit(&g_tiny_hotheap_v2_c7_alloc_fallback_v1, memory_order_relaxed);
|
||||
uint64_t free_calls = atomic_load_explicit(&g_tiny_hotheap_v2_c7_free_calls, memory_order_relaxed);
|
||||
uint64_t free_fast = atomic_load_explicit(&g_tiny_hotheap_v2_c7_free_fast, memory_order_relaxed);
|
||||
uint64_t free_fb = atomic_load_explicit(&g_tiny_hotheap_v2_c7_free_fallback_v1, memory_order_relaxed);
|
||||
for (uint8_t ci = 0; ci < TINY_HOTHEAP_MAX_CLASSES; ci++) {
|
||||
uint64_t alloc_calls = atomic_load_explicit(&g_tiny_hotheap_v2_alloc_calls[ci], memory_order_relaxed);
|
||||
uint64_t route_hits = atomic_load_explicit(&g_tiny_hotheap_v2_route_hits[ci], memory_order_relaxed);
|
||||
uint64_t alloc_fast = atomic_load_explicit(&g_tiny_hotheap_v2_alloc_fast[ci], memory_order_relaxed);
|
||||
uint64_t alloc_lease = atomic_load_explicit(&g_tiny_hotheap_v2_alloc_lease[ci], memory_order_relaxed);
|
||||
uint64_t alloc_fb = atomic_load_explicit(&g_tiny_hotheap_v2_alloc_fallback_v1[ci], memory_order_relaxed);
|
||||
uint64_t free_calls = atomic_load_explicit(&g_tiny_hotheap_v2_free_calls[ci], memory_order_relaxed);
|
||||
uint64_t free_fast = atomic_load_explicit(&g_tiny_hotheap_v2_free_fast[ci], memory_order_relaxed);
|
||||
uint64_t free_fb = atomic_load_explicit(&g_tiny_hotheap_v2_free_fallback_v1[ci], memory_order_relaxed);
|
||||
uint64_t cold_refill_fail = atomic_load_explicit(&g_tiny_hotheap_v2_cold_refill_fail[ci], memory_order_relaxed);
|
||||
uint64_t cold_retire_calls = atomic_load_explicit(&g_tiny_hotheap_v2_cold_retire_calls[ci], memory_order_relaxed);
|
||||
uint64_t retire_calls_v2 = atomic_load_explicit(&g_tiny_hotheap_v2_retire_calls_v2[ci], memory_order_relaxed);
|
||||
uint64_t partial_pushes = atomic_load_explicit(&g_tiny_hotheap_v2_partial_pushes[ci], memory_order_relaxed);
|
||||
uint64_t partial_pops = atomic_load_explicit(&g_tiny_hotheap_v2_partial_pops[ci], memory_order_relaxed);
|
||||
uint64_t partial_peak = atomic_load_explicit(&g_tiny_hotheap_v2_partial_peak[ci], memory_order_relaxed);
|
||||
uint64_t refill_with_cur = atomic_load_explicit(&g_tiny_hotheap_v2_refill_with_current[ci], memory_order_relaxed);
|
||||
uint64_t refill_with_partial = atomic_load_explicit(&g_tiny_hotheap_v2_refill_with_partial[ci], memory_order_relaxed);
|
||||
|
||||
TinyHotHeapV2PageStats ps = {
|
||||
.prepare_calls = atomic_load_explicit(&g_tiny_hotheap_v2_page_stats.prepare_calls, memory_order_relaxed),
|
||||
.prepare_with_current_null = atomic_load_explicit(&g_tiny_hotheap_v2_page_stats.prepare_with_current_null, memory_order_relaxed),
|
||||
.prepare_from_partial = atomic_load_explicit(&g_tiny_hotheap_v2_page_stats.prepare_from_partial, memory_order_relaxed),
|
||||
.free_made_current = atomic_load_explicit(&g_tiny_hotheap_v2_page_stats.free_made_current, memory_order_relaxed),
|
||||
.page_retired = atomic_load_explicit(&g_tiny_hotheap_v2_page_stats.page_retired, memory_order_relaxed),
|
||||
};
|
||||
TinyHotHeapV2PageStats ps = {
|
||||
.prepare_calls = atomic_load_explicit(&g_tiny_hotheap_v2_page_stats[ci].prepare_calls, memory_order_relaxed),
|
||||
.prepare_with_current_null = atomic_load_explicit(&g_tiny_hotheap_v2_page_stats[ci].prepare_with_current_null, memory_order_relaxed),
|
||||
.prepare_from_partial = atomic_load_explicit(&g_tiny_hotheap_v2_page_stats[ci].prepare_from_partial, memory_order_relaxed),
|
||||
.free_made_current = atomic_load_explicit(&g_tiny_hotheap_v2_page_stats[ci].free_made_current, memory_order_relaxed),
|
||||
.page_retired = atomic_load_explicit(&g_tiny_hotheap_v2_page_stats[ci].page_retired, memory_order_relaxed),
|
||||
};
|
||||
|
||||
if (alloc_calls || alloc_fast || alloc_lease || alloc_fb || free_calls || free_fast || free_fb ||
|
||||
ps.prepare_calls || ps.prepare_with_current_null || ps.prepare_from_partial ||
|
||||
ps.free_made_current || ps.page_retired) {
|
||||
if (!(alloc_calls || alloc_fast || alloc_lease || alloc_fb || free_calls || free_fast || free_fb ||
|
||||
ps.prepare_calls || ps.prepare_with_current_null || ps.prepare_from_partial ||
|
||||
ps.free_made_current || ps.page_retired || retire_calls_v2 || partial_pushes || partial_pops || partial_peak)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
tiny_route_kind_t route_kind = tiny_route_for_class(ci);
|
||||
fprintf(stderr,
|
||||
"[HOTHEAP_V2_C7_STATS] route_hits=%llu alloc_calls=%llu alloc_fast=%llu alloc_lease=%llu alloc_refill=%llu alloc_fb_v1=%llu alloc_route_fb=%llu free_calls=%llu free_fast=%llu free_fb_v1=%llu prep_calls=%llu prep_null=%llu prep_from_partial=%llu free_made_current=%llu page_retired=%llu\n",
|
||||
"[HOTHEAP_V2_STATS cls=%u route=%d] route_hits=%llu alloc_calls=%llu alloc_fast=%llu alloc_lease=%llu alloc_refill=%llu refill_cur=%llu refill_partial=%llu alloc_fb_v1=%llu alloc_route_fb=%llu cold_refill_fail=%llu cold_retire_calls=%llu retire_v2=%llu free_calls=%llu free_fast=%llu free_fb_v1=%llu prep_calls=%llu prep_null=%llu prep_from_partial=%llu free_made_current=%llu page_retired=%llu partial_push=%llu partial_pop=%llu partial_peak=%llu\n",
|
||||
(unsigned)ci,
|
||||
(int)route_kind,
|
||||
(unsigned long long)route_hits,
|
||||
(unsigned long long)alloc_calls,
|
||||
(unsigned long long)alloc_fast,
|
||||
(unsigned long long)alloc_lease,
|
||||
(unsigned long long)atomic_load_explicit(&g_tiny_hotheap_v2_c7_alloc_refill, memory_order_relaxed),
|
||||
(unsigned long long)atomic_load_explicit(&g_tiny_hotheap_v2_alloc_refill[ci], memory_order_relaxed),
|
||||
(unsigned long long)refill_with_cur,
|
||||
(unsigned long long)refill_with_partial,
|
||||
(unsigned long long)alloc_fb,
|
||||
(unsigned long long)atomic_load_explicit(&g_tiny_hotheap_v2_c7_alloc_route_fb, memory_order_relaxed),
|
||||
(unsigned long long)atomic_load_explicit(&g_tiny_hotheap_v2_alloc_route_fb[ci], memory_order_relaxed),
|
||||
(unsigned long long)cold_refill_fail,
|
||||
(unsigned long long)cold_retire_calls,
|
||||
(unsigned long long)retire_calls_v2,
|
||||
(unsigned long long)free_calls,
|
||||
(unsigned long long)free_fast,
|
||||
(unsigned long long)free_fb,
|
||||
@ -469,7 +649,10 @@ static void tiny_hotheap_v2_stats_dump(void) {
|
||||
(unsigned long long)ps.prepare_with_current_null,
|
||||
(unsigned long long)ps.prepare_from_partial,
|
||||
(unsigned long long)ps.free_made_current,
|
||||
(unsigned long long)ps.page_retired);
|
||||
(unsigned long long)ps.page_retired,
|
||||
(unsigned long long)partial_pushes,
|
||||
(unsigned long long)partial_pops,
|
||||
(unsigned long long)partial_peak);
|
||||
}
|
||||
}
|
||||
tiny_hotheap_ctx_v2* tiny_hotheap_v2_tls_get(void) {
|
||||
@ -484,6 +667,8 @@ tiny_hotheap_ctx_v2* tiny_hotheap_v2_tls_get(void) {
|
||||
for (int i = 0; i < TINY_HOTHEAP_MAX_CLASSES; i++) {
|
||||
tiny_hotheap_v2_page_reset(&ctx->cls[i].storage_page);
|
||||
ctx->cls[i].stride = (uint16_t)tiny_stride_for_class(i);
|
||||
ctx->cls[i].max_partial_pages = (i == 7 || i == 6) ? 2 : 0; // C6/C7 は 1〜2 枚握る
|
||||
ctx->cls[i].partial_count = 0;
|
||||
}
|
||||
}
|
||||
return ctx;
|
||||
@ -491,143 +676,174 @@ tiny_hotheap_ctx_v2* tiny_hotheap_v2_tls_get(void) {
|
||||
|
||||
void* tiny_hotheap_v2_alloc(uint8_t class_idx) {
|
||||
int stats_on = tiny_hotheap_v2_stats_enabled();
|
||||
uint8_t idx = tiny_hotheap_v2_idx(class_idx);
|
||||
if (stats_on) {
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_c7_route_hits, 1, memory_order_relaxed);
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_c7_alloc_calls, 1, memory_order_relaxed);
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_route_hits[idx], 1, memory_order_relaxed);
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_alloc_calls[idx], 1, memory_order_relaxed);
|
||||
}
|
||||
if (__builtin_expect(class_idx != 7, 0)) {
|
||||
return NULL; // いまは C7 専用
|
||||
if (__builtin_expect(!(class_idx == 6 || class_idx == 7), 0)) {
|
||||
return NULL; // いまは C6/C7 のみ
|
||||
}
|
||||
|
||||
tiny_hotheap_ctx_v2* v2ctx = tiny_hotheap_v2_tls_get();
|
||||
tiny_hotheap_class_v2* vhcls = v2ctx ? &v2ctx->cls[7] : NULL;
|
||||
tiny_hotheap_class_v2* vhcls = v2ctx ? &v2ctx->cls[class_idx] : NULL;
|
||||
tiny_hotheap_page_v2* v2page = vhcls ? vhcls->current_page : NULL;
|
||||
tiny_heap_ctx_t* v1ctx = tiny_heap_ctx_for_thread();
|
||||
tiny_heap_class_t* v1hcls = tiny_heap_class(v1ctx, 7);
|
||||
TinyHeapClassStats* stats = tiny_heap_stats_for_class(7);
|
||||
TinyHeapClassStats* stats = tiny_heap_stats_for_class(class_idx);
|
||||
|
||||
// current_page が壊れていそうなら一度捨てて slow に降りる
|
||||
if (v2page && (!v2page->base || v2page->capacity == 0)) {
|
||||
vhcls->current_page = NULL;
|
||||
v2page = NULL;
|
||||
}
|
||||
|
||||
// Hot path: current_page → partial → refill
|
||||
void* user = tiny_hotheap_v2_try_pop(v2page, v1hcls, stats, stats_on);
|
||||
void* user = tiny_hotheap_v2_try_pop(vhcls, v2page, class_idx, stats, stats_on);
|
||||
if (user) {
|
||||
return user;
|
||||
}
|
||||
|
||||
// move exhausted current_page to full list if needed
|
||||
if (vhcls && v2page && v2page->used >= v2page->capacity && vhcls->current_page == v2page) {
|
||||
vhcls->current_page = NULL;
|
||||
v2page->next = vhcls->full_pages;
|
||||
vhcls->full_pages = v2page;
|
||||
}
|
||||
|
||||
while (vhcls && vhcls->partial_pages) {
|
||||
if (stats_on) {
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_page_stats.prepare_calls, 1, memory_order_relaxed);
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_page_stats.prepare_from_partial, 1, memory_order_relaxed);
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_page_stats[idx].prepare_calls, 1, memory_order_relaxed);
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_page_stats[idx].prepare_from_partial, 1, memory_order_relaxed);
|
||||
if (vhcls->current_page == NULL) {
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_page_stats.prepare_with_current_null, 1, memory_order_relaxed);
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_page_stats[idx].prepare_with_current_null, 1, memory_order_relaxed);
|
||||
}
|
||||
}
|
||||
v2page = vhcls->partial_pages;
|
||||
vhcls->partial_pages = vhcls->partial_pages->next;
|
||||
if (vhcls->partial_count > 0) {
|
||||
vhcls->partial_count--;
|
||||
}
|
||||
if (stats_on) {
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_partial_pops[idx], 1, memory_order_relaxed);
|
||||
}
|
||||
v2page->next = NULL;
|
||||
vhcls->current_page = v2page;
|
||||
user = tiny_hotheap_v2_try_pop(v2page, v1hcls, stats, stats_on);
|
||||
user = tiny_hotheap_v2_try_pop(vhcls, v2page, class_idx, stats, stats_on);
|
||||
if (user) {
|
||||
return user;
|
||||
}
|
||||
if (v2page->used >= v2page->capacity) {
|
||||
v2page->next = vhcls->full_pages;
|
||||
vhcls->full_pages = v2page;
|
||||
vhcls->current_page = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
// Lease a page from v1 (C7 SAFE) and wrap it
|
||||
tiny_hotheap_page_v2* leased = tiny_hotheap_v2_refill_slow(v2ctx, 7);
|
||||
if (!leased || !v1hcls) {
|
||||
tiny_hotheap_page_v2* leased = tiny_hotheap_v2_refill_slow(v2ctx, class_idx);
|
||||
if (!leased) {
|
||||
if (stats_on) {
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_c7_alloc_fallback_v1, 1, memory_order_relaxed);
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_c7_alloc_route_fb, 1, memory_order_relaxed);
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_alloc_fallback_v1[idx], 1, memory_order_relaxed);
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_alloc_route_fb[idx], 1, memory_order_relaxed);
|
||||
}
|
||||
size_t size = vhcls ? (vhcls->stride ? vhcls->stride : tiny_stride_for_class(7)) : tiny_stride_for_class(7);
|
||||
return tiny_c7_alloc_fast(size); // safety fallback to v1
|
||||
size_t size = vhcls ? (vhcls->stride ? vhcls->stride : tiny_stride_for_class(class_idx)) : tiny_stride_for_class(class_idx);
|
||||
if (class_idx == 7) {
|
||||
return tiny_c7_alloc_fast(size); // safety fallback to v1
|
||||
}
|
||||
tiny_heap_ctx_t* cold_ctx = tiny_heap_ctx_for_thread();
|
||||
return tiny_heap_alloc_class_fast(cold_ctx, class_idx, size);
|
||||
}
|
||||
vhcls->current_page = leased;
|
||||
v2page = leased;
|
||||
if (v1hcls && v2page && v2page->lease_page) {
|
||||
v1hcls->current_page = v2page->lease_page;
|
||||
}
|
||||
if (stats_on) {
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_c7_alloc_lease, 1, memory_order_relaxed);
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_alloc_lease[idx], 1, memory_order_relaxed);
|
||||
}
|
||||
|
||||
user = tiny_hotheap_v2_try_pop(v2page, v1hcls, stats, stats_on);
|
||||
user = tiny_hotheap_v2_try_pop(vhcls, v2page, class_idx, stats, stats_on);
|
||||
if (user) {
|
||||
return user;
|
||||
}
|
||||
|
||||
if (stats_on) {
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_c7_alloc_fallback_v1, 1, memory_order_relaxed);
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_alloc_fallback_v1[idx], 1, memory_order_relaxed);
|
||||
}
|
||||
size_t size = vhcls ? (vhcls->stride ? vhcls->stride : tiny_stride_for_class(7)) : tiny_stride_for_class(7);
|
||||
return tiny_c7_alloc_fast(size);
|
||||
size_t size = vhcls ? (vhcls->stride ? vhcls->stride : tiny_stride_for_class(class_idx)) : tiny_stride_for_class(class_idx);
|
||||
if (class_idx == 7) {
|
||||
return tiny_c7_alloc_fast(size);
|
||||
}
|
||||
tiny_heap_ctx_t* cold_ctx = tiny_heap_ctx_for_thread();
|
||||
return tiny_heap_alloc_class_fast(cold_ctx, class_idx, size);
|
||||
}
|
||||
|
||||
void tiny_hotheap_v2_free(uint8_t class_idx, void* p, void* meta) {
|
||||
if (__builtin_expect(class_idx != 7, 0)) {
|
||||
if (__builtin_expect(!(class_idx == 6 || class_idx == 7), 0)) {
|
||||
return;
|
||||
}
|
||||
uint8_t idx = tiny_hotheap_v2_idx(class_idx);
|
||||
int stats_on = tiny_hotheap_v2_stats_enabled();
|
||||
if (stats_on) {
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_c7_free_calls, 1, memory_order_relaxed);
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_free_calls[idx], 1, memory_order_relaxed);
|
||||
}
|
||||
tiny_hotheap_ctx_v2* v2ctx = tiny_hotheap_v2_tls_get();
|
||||
tiny_hotheap_class_v2* vhcls = v2ctx ? &v2ctx->cls[7] : NULL;
|
||||
tiny_hotheap_class_v2* vhcls = v2ctx ? &v2ctx->cls[class_idx] : NULL;
|
||||
TinySlabMeta* meta_ptr = (TinySlabMeta*)meta;
|
||||
|
||||
tiny_heap_ctx_t* v1ctx = tiny_heap_ctx_for_thread();
|
||||
tiny_heap_class_t* v1hcls = tiny_heap_class(v1ctx, 7);
|
||||
|
||||
tiny_hotheap_page_v2* page = tiny_hotheap_v2_find_page(vhcls, 7, p, meta_ptr);
|
||||
if (page && page->lease_page && v1hcls && tiny_heap_ptr_in_page_range(page->lease_page, p)) {
|
||||
tiny_heap_page_free_local(v1ctx, 7, page->lease_page, p);
|
||||
page->freelist = page->lease_page->free_list;
|
||||
page->used = page->lease_page->used;
|
||||
if (v1hcls) {
|
||||
v1hcls->current_page = page->lease_page;
|
||||
tiny_hotheap_page_v2* page = tiny_hotheap_v2_find_page(vhcls, class_idx, p, meta_ptr);
|
||||
if (page && page->base && page->capacity > 0) {
|
||||
tiny_next_write(class_idx, p, page->freelist);
|
||||
page->freelist = p;
|
||||
if (page->used > 0) {
|
||||
page->used--;
|
||||
}
|
||||
if (vhcls && vhcls->current_page != page) {
|
||||
tiny_hotheap_v2_unlink_page(vhcls, page);
|
||||
page->next = vhcls->current_page;
|
||||
vhcls->current_page = page;
|
||||
if (stats_on) {
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_page_stats.free_made_current, 1, memory_order_relaxed);
|
||||
}
|
||||
} else if (stats_on) {
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_page_stats.free_made_current, 1, memory_order_relaxed);
|
||||
}
|
||||
// C7-only: keep the page hot even when empty to avoid churn
|
||||
if (vhcls) {
|
||||
if (!vhcls->current_page) {
|
||||
vhcls->current_page = page;
|
||||
} else if (vhcls->current_page != page) {
|
||||
tiny_hotheap_v2_unlink_page(vhcls, page);
|
||||
page->next = vhcls->current_page;
|
||||
vhcls->current_page = page;
|
||||
}
|
||||
}
|
||||
if (page->used == 0 && vhcls && vhcls->partial_pages != page && vhcls->current_page == page) {
|
||||
// park empty page in partial to allow re-use without immediate Superslab return
|
||||
page->next = vhcls->partial_pages;
|
||||
vhcls->partial_pages = page;
|
||||
vhcls->current_page = page; // still treat as current
|
||||
}
|
||||
if (stats_on) {
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_c7_free_fast, 1, memory_order_relaxed);
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_page_stats[idx].free_made_current, 1, memory_order_relaxed);
|
||||
}
|
||||
if (page->used == 0) {
|
||||
// 空ページは一度 partial に温存し、上限を超えたら retire
|
||||
tiny_hotheap_v2_unlink_page(vhcls, page);
|
||||
page->next = NULL;
|
||||
if (vhcls && vhcls->current_page == NULL) {
|
||||
vhcls->current_page = page;
|
||||
} else if (vhcls) {
|
||||
tiny_hotheap_v2_partial_push(vhcls, page, class_idx, stats_on);
|
||||
tiny_hotheap_v2_maybe_trim_partial(v2ctx, vhcls, class_idx, stats_on);
|
||||
}
|
||||
} else if (stats_on) {
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_free_fast[idx], 1, memory_order_relaxed);
|
||||
}
|
||||
if (stats_on && page->used == 0) {
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_free_fast[idx], 1, memory_order_relaxed);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// Fallback: mimic v1 free path
|
||||
if (stats_on) {
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_c7_free_fallback_v1, 1, memory_order_relaxed);
|
||||
atomic_fetch_add_explicit(&g_tiny_hotheap_v2_free_fallback_v1[idx], 1, memory_order_relaxed);
|
||||
}
|
||||
SuperSlab* ss = hak_super_lookup(p);
|
||||
if (ss && ss->magic == SUPERSLAB_MAGIC) {
|
||||
int slab_idx = slab_index_for(ss, p);
|
||||
if (slab_idx >= 0 && slab_idx < ss_slabs_capacity(ss)) {
|
||||
tiny_c7_free_fast_with_meta(ss, slab_idx, p);
|
||||
if (class_idx == 7) {
|
||||
tiny_c7_free_fast_with_meta(ss, slab_idx, p);
|
||||
} else {
|
||||
tiny_heap_ctx_t* cold_ctx = tiny_heap_ctx_for_thread();
|
||||
tiny_heap_free_class_fast_with_meta(cold_ctx, class_idx, ss, slab_idx, p);
|
||||
}
|
||||
return;
|
||||
}
|
||||
}
|
||||
tiny_c7_free_fast(p);
|
||||
if (class_idx == 7) {
|
||||
tiny_c7_free_fast(p);
|
||||
} else {
|
||||
tiny_heap_ctx_t* cold_ctx = tiny_heap_ctx_for_thread();
|
||||
tiny_heap_free_class_fast(cold_ctx, class_idx, p);
|
||||
}
|
||||
}
|
||||
|
||||
#if !HAKMEM_BUILD_RELEASE
|
||||
|
||||
Reference in New Issue
Block a user