Phase TLS-UNIFY-3: C6 intrusive freelist implementation (完成)
Implement C6 ULTRA intrusive LIFO freelist with ENV gating: - Single-linked LIFO using next pointer at USER+1 offset - tiny_next_store/tiny_next_load for pointer access (single source of truth) - Segment learning via ss_fast_lookup (per-class seg_base/seg_end) - ENV gate: HAKMEM_TINY_C6_ULTRA_INTRUSIVE_FL (default OFF) - Counters: c6_ifl_push/pop/fallback in FREE_PATH_STATS Files: - core/box/tiny_ultra_tls_box.h: Added c6_head field for intrusive LIFO - core/box/tiny_ultra_tls_box.c: Pop/push with intrusive branching (case 6) - core/box/tiny_c6_ultra_intrusive_env_box.h: ENV gate (new) - core/box/tiny_c6_intrusive_freelist_box.h: L1 pure LIFO (new) - core/tiny_debug_ring.h: C6_IFL events - core/box/free_path_stats_box.h/c: c6_ifl_* counters A/B Test Results (1M iterations, ws=200, 257-512B): - ENV_OFF (array): 56.6 Mop/s avg - ENV_ON (intrusive): 57.6 Mop/s avg (+1.8%, within noise) - Counters verified: c6_ifl_push=265890, c6_ifl_pop=265815, fallback=0 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Haiku 4.5 <noreply@anthropic.com>
This commit is contained in:
@ -52,6 +52,7 @@
|
||||
#include "../box/tiny_c6_ultra_free_box.h" // Phase 4-2: C6 ULTRA-free (free-only, C6-only)
|
||||
#include "../box/tiny_c5_ultra_free_box.h" // Phase 5-1/5-2: C5 ULTRA-free + alloc integration
|
||||
#include "../box/tiny_c4_ultra_free_box.h" // Phase 6: C4 ULTRA-free + alloc integration (cap=64)
|
||||
#include "../box/tiny_ultra_tls_box.h" // Phase TLS-UNIFY-1: Unified ULTRA TLS API
|
||||
#include "../box/tiny_ultra_classes_box.h" // Phase REFACTOR-1: Named constants for C4-C7
|
||||
#include "../box/tiny_legacy_fallback_box.h" // Phase REFACTOR-2: Legacy fallback logic unification
|
||||
#include "../box/tiny_ptr_convert_box.h" // Phase REFACTOR-3: Inline pointer macro centralization
|
||||
@ -130,175 +131,84 @@ static inline void* malloc_tiny_fast(size_t size) {
|
||||
// Phase ALLOC-GATE-OPT-1: カウンタ散布 (1. 関数入口)
|
||||
ALLOC_GATE_STAT_INC(total_calls);
|
||||
|
||||
const int front_v3_on = tiny_front_v3_enabled();
|
||||
const TinyFrontV3Snapshot* front_snap =
|
||||
__builtin_expect(front_v3_on, 0) ? tiny_front_v3_snapshot_get() : NULL;
|
||||
const bool route_fast_on = front_v3_on && tiny_front_v3_lut_enabled() &&
|
||||
tiny_front_v3_route_fast_enabled();
|
||||
|
||||
int class_idx = -1;
|
||||
tiny_route_kind_t route = TINY_ROUTE_LEGACY;
|
||||
bool route_trusted = false;
|
||||
|
||||
if (front_v3_on && tiny_front_v3_lut_enabled()) {
|
||||
const TinyFrontV3SizeClassEntry* e = tiny_front_v3_lut_lookup(size);
|
||||
if (e && e->class_idx != TINY_FRONT_V3_INVALID_CLASS) {
|
||||
class_idx = (int)e->class_idx;
|
||||
route = (tiny_route_kind_t)e->route_kind;
|
||||
route_trusted = route_fast_on;
|
||||
}
|
||||
}
|
||||
|
||||
// Phase v11a-5: Simplified hot path with C7 ULTRA early-exit
|
||||
// 1. size → class_idx (single call)
|
||||
ALLOC_GATE_STAT_INC(size_to_class_calls);
|
||||
int class_idx = hak_tiny_size_to_class(size);
|
||||
if (__builtin_expect(class_idx < 0 || class_idx >= TINY_NUM_CLASSES, 0)) {
|
||||
// Phase ALLOC-GATE-OPT-1: カウンタ散布 (2. size→class 変換)
|
||||
ALLOC_GATE_STAT_INC(size_to_class_calls);
|
||||
class_idx = hak_tiny_size_to_class(size);
|
||||
if (__builtin_expect(class_idx < 0 || class_idx >= TINY_NUM_CLASSES, 0)) {
|
||||
return NULL;
|
||||
}
|
||||
// Phase ALLOC-GATE-OPT-1: カウンタ散布 (3. route_for_class 呼び出し)
|
||||
ALLOC_GATE_STAT_INC(route_for_class_calls);
|
||||
route = tiny_route_for_class((uint8_t)class_idx);
|
||||
route_trusted = false;
|
||||
} else if (!route_trusted &&
|
||||
route != TINY_ROUTE_LEGACY && route != TINY_ROUTE_HEAP &&
|
||||
route != TINY_ROUTE_HOTHEAP_V2 &&
|
||||
route != TINY_ROUTE_SMALL_HEAP_V6 && route != TINY_ROUTE_SMALL_HEAP_V7) {
|
||||
// Phase ALLOC-GATE-OPT-1: カウンタ散布 (3. route_for_class 呼び出し)
|
||||
// Note: v3/v4/v5 removed in Phase v10
|
||||
ALLOC_GATE_STAT_INC(route_for_class_calls);
|
||||
route = tiny_route_for_class((uint8_t)class_idx);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
tiny_front_alloc_stat_inc(class_idx);
|
||||
|
||||
// Phase ALLOC-GATE-OPT-1: カウンタ散布 (4. クラス別分布)
|
||||
ALLOC_GATE_STAT_INC_CLASS(class_idx);
|
||||
|
||||
// C7 ULTRA allocation path (ENV: HAKMEM_TINY_C7_ULTRA_ENABLED, default ON)
|
||||
if (tiny_class_is_c7(class_idx) && tiny_c7_ultra_enabled_env()) {
|
||||
// Phase ALLOC-GATE-OPT-1: カウンタ散布 (5. C7 ULTRA ENV check)
|
||||
ALLOC_GATE_STAT_INC(env_checks);
|
||||
// Phase v11a-5b: C7 ULTRA early-exit (skip policy snapshot for common case)
|
||||
// This is the most common hot path - avoids TLS policy overhead
|
||||
if (class_idx == 7 && tiny_c7_ultra_enabled_env()) {
|
||||
void* ultra_p = tiny_c7_ultra_alloc(size);
|
||||
if (TINY_HOT_LIKELY(ultra_p != NULL)) {
|
||||
return ultra_p;
|
||||
}
|
||||
// fallback to route on miss
|
||||
// C7 ULTRA miss → fall through to policy-based routing
|
||||
}
|
||||
|
||||
// Phase 4-4: C6 ULTRA free+alloc integration (寄生型 TLS キャッシュ pop)
|
||||
if (tiny_class_is_c6(class_idx) && tiny_c6_ultra_free_enabled()) {
|
||||
// Phase ALLOC-GATE-OPT-1: カウンタ散布 (5. C6 ULTRA ENV check)
|
||||
ALLOC_GATE_STAT_INC(env_checks);
|
||||
TinyC6UltraFreeTLS* ctx = tiny_c6_ultra_free_tls();
|
||||
if (TINY_HOT_LIKELY(ctx->count > 0)) {
|
||||
void* base = ctx->freelist[--ctx->count];
|
||||
// Phase 4-4: カウンタ散布 (TLS pop)
|
||||
FREE_PATH_STAT_INC(c6_ultra_alloc_hit);
|
||||
// BASE pointer のまま、USER pointer に変換して返す
|
||||
// (header は既に base[0] にある前提)
|
||||
return tiny_base_to_user_inline(base);
|
||||
}
|
||||
}
|
||||
|
||||
// Phase 5-2: C5 ULTRA free+alloc integration (same pattern as C6)
|
||||
if (tiny_class_is_c5(class_idx) && tiny_c5_ultra_free_enabled()) {
|
||||
// Phase ALLOC-GATE-OPT-1: カウンタ散布 (5. C5 ULTRA ENV check)
|
||||
ALLOC_GATE_STAT_INC(env_checks);
|
||||
TinyC5UltraFreeTLS* ctx = tiny_c5_ultra_free_tls();
|
||||
if (TINY_HOT_LIKELY(ctx->count > 0)) {
|
||||
void* base = ctx->freelist[--ctx->count];
|
||||
FREE_PATH_STAT_INC(c5_ultra_alloc_hit);
|
||||
return tiny_base_to_user_inline(base);
|
||||
}
|
||||
}
|
||||
|
||||
// Phase 6: C4 ULTRA free+alloc integration (same pattern as C5/C6, cap=64)
|
||||
if (tiny_class_is_c4(class_idx) && tiny_c4_ultra_free_enabled()) {
|
||||
// Phase ALLOC-GATE-OPT-1: カウンタ散布 (5. C4 ULTRA ENV check)
|
||||
ALLOC_GATE_STAT_INC(env_checks);
|
||||
TinyC4UltraFreeTLS* ctx = tiny_c4_ultra_free_tls();
|
||||
if (TINY_HOT_LIKELY(ctx->count > 0)) {
|
||||
void* base = ctx->freelist[--ctx->count];
|
||||
FREE_PATH_STAT_INC(c4_ultra_alloc_hit);
|
||||
return tiny_base_to_user_inline(base);
|
||||
}
|
||||
}
|
||||
|
||||
// Phase v7-4: Check Policy Box for v7/MID_V35 routing (before switch)
|
||||
// 2. Policy snapshot (TLS cached, single read)
|
||||
const SmallPolicyV7* policy = small_policy_v7_snapshot();
|
||||
SmallRouteKind route_kind = policy->route_kind[class_idx];
|
||||
|
||||
// Phase v11a-3: MID v3.5 routing
|
||||
if (policy->route_kind[class_idx] == SMALL_ROUTE_MID_V35) {
|
||||
void* v35p = small_mid_v35_alloc(class_idx, size);
|
||||
if (TINY_HOT_LIKELY(v35p != NULL)) {
|
||||
return v35p;
|
||||
// 3. Single switch on route_kind (all ENV checks moved to Policy init)
|
||||
switch (route_kind) {
|
||||
case SMALL_ROUTE_ULTRA: {
|
||||
// Phase TLS-UNIFY-1: Unified ULTRA TLS pop for C4-C6 (C7 handled above)
|
||||
void* base = tiny_ultra_tls_pop((uint8_t)class_idx);
|
||||
if (TINY_HOT_LIKELY(base != NULL)) {
|
||||
if (class_idx == 6) FREE_PATH_STAT_INC(c6_ultra_alloc_hit);
|
||||
else if (class_idx == 5) FREE_PATH_STAT_INC(c5_ultra_alloc_hit);
|
||||
else if (class_idx == 4) FREE_PATH_STAT_INC(c4_ultra_alloc_hit);
|
||||
return tiny_base_to_user_inline(base);
|
||||
}
|
||||
// ULTRA miss → fallback to LEGACY
|
||||
break;
|
||||
}
|
||||
// v35 returned NULL -> fallback to legacy
|
||||
}
|
||||
|
||||
if (policy->route_kind[class_idx] == SMALL_ROUTE_V7) {
|
||||
void* v7p = small_heap_alloc_fast_v7_stub(size, (uint8_t)class_idx);
|
||||
if (TINY_HOT_LIKELY(v7p != NULL)) {
|
||||
return v7p;
|
||||
case SMALL_ROUTE_MID_V35: {
|
||||
// Phase v11a-3: MID v3.5 allocation
|
||||
void* v35p = small_mid_v35_alloc(class_idx, size);
|
||||
if (TINY_HOT_LIKELY(v35p != NULL)) {
|
||||
return v35p;
|
||||
}
|
||||
// MID v3.5 miss → fallback to LEGACY
|
||||
break;
|
||||
}
|
||||
// v7 stub returned NULL -> fallback to legacy
|
||||
}
|
||||
|
||||
switch (route) {
|
||||
case TINY_ROUTE_SMALL_HEAP_V7: {
|
||||
// Phase v7-4: v7 routing now handled by Policy Box above (kept for legacy compatibility)
|
||||
case SMALL_ROUTE_V7: {
|
||||
// Phase v7: SmallObject v7 allocation (research box)
|
||||
void* v7p = small_heap_alloc_fast_v7_stub(size, (uint8_t)class_idx);
|
||||
if (TINY_HOT_LIKELY(v7p != NULL)) {
|
||||
return v7p;
|
||||
}
|
||||
// v7 stub returned NULL -> fallback to legacy
|
||||
// V7 miss → fallback to LEGACY
|
||||
break;
|
||||
}
|
||||
case TINY_ROUTE_SMALL_HEAP_V6: {
|
||||
// Phase V6-HDR-2: Headerless alloc (ENV gated)
|
||||
if (small_v6_headerless_route_enabled((uint8_t)class_idx)) {
|
||||
SmallHeapCtxV6* ctx_v6 = small_heap_ctx_v6();
|
||||
void* v6p = small_v6_headerless_alloc(ctx_v6, (uint8_t)class_idx);
|
||||
if (TINY_HOT_LIKELY(v6p != NULL)) {
|
||||
return v6p; // No header write needed - done in refill
|
||||
}
|
||||
// v6 returned NULL -> fallback to legacy
|
||||
}
|
||||
__attribute__((fallthrough));
|
||||
}
|
||||
// Phase v10: v3/v4/v5 removed - routes now handled as LEGACY
|
||||
case TINY_ROUTE_HOTHEAP_V2: {
|
||||
void* v2p = tiny_hotheap_v2_alloc((uint8_t)class_idx);
|
||||
if (TINY_HOT_LIKELY(v2p != NULL)) {
|
||||
return v2p;
|
||||
}
|
||||
tiny_hotheap_v2_record_route_fallback((uint8_t)class_idx);
|
||||
// fallthrough to TinyHeap v1
|
||||
__attribute__((fallthrough));
|
||||
}
|
||||
case TINY_ROUTE_HEAP: {
|
||||
void* heap_ptr = NULL;
|
||||
if (class_idx == 7) {
|
||||
heap_ptr = tiny_c7_alloc_fast(size);
|
||||
} else {
|
||||
heap_ptr = tiny_heap_alloc_class_fast(tiny_heap_ctx_for_thread(), class_idx, size);
|
||||
}
|
||||
if (heap_ptr) {
|
||||
return heap_ptr;
|
||||
|
||||
case SMALL_ROUTE_MID_V3: {
|
||||
// Phase MID-V3: MID v3 allocation (257-768B, C5-C6)
|
||||
// Note: MID v3 uses same segment infrastructure as MID v3.5
|
||||
// For now, delegate to MID v3.5 which handles both
|
||||
void* v3p = small_mid_v35_alloc(class_idx, size);
|
||||
if (TINY_HOT_LIKELY(v3p != NULL)) {
|
||||
return v3p;
|
||||
}
|
||||
break;
|
||||
}
|
||||
case TINY_ROUTE_LEGACY:
|
||||
|
||||
case SMALL_ROUTE_LEGACY:
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
// Legacy Tiny front
|
||||
void* ptr = NULL;
|
||||
if (!front_snap || front_snap->unified_cache_on) {
|
||||
ptr = tiny_hot_alloc_fast(class_idx);
|
||||
}
|
||||
// LEGACY fallback: Unified Cache hot/cold path
|
||||
void* ptr = tiny_hot_alloc_fast(class_idx);
|
||||
if (TINY_HOT_LIKELY(ptr != NULL)) {
|
||||
return ptr;
|
||||
}
|
||||
@ -353,85 +263,58 @@ static inline int free_tiny_fast(void* ptr) {
|
||||
// Phase FREE-LEGACY-BREAKDOWN-1: カウンタ散布 (1. 関数入口)
|
||||
FREE_PATH_STAT_INC(total_calls);
|
||||
|
||||
// C7 ULTRA free path (ENV: HAKMEM_TINY_C7_ULTRA_ENABLED, default ON)
|
||||
if (tiny_class_is_c7(class_idx) && tiny_c7_ultra_enabled_env()) {
|
||||
// Phase v11b-1: C7 ULTRA early-exit (skip policy snapshot for most common case)
|
||||
if (class_idx == 7 && tiny_c7_ultra_enabled_env()) {
|
||||
tiny_c7_ultra_free(ptr);
|
||||
return 1;
|
||||
}
|
||||
|
||||
// Phase 4-2: C6 ULTRA-free (C6-only, free-only, ENV gated)
|
||||
if (tiny_class_is_c6(class_idx) && tiny_c6_ultra_free_enabled()) {
|
||||
tiny_c6_ultra_free_fast(base, class_idx);
|
||||
return 1;
|
||||
}
|
||||
|
||||
// Phase 5-1: C5 ULTRA-free (C5-only, free-only, ENV gated, same pattern as C6)
|
||||
if (tiny_class_is_c5(class_idx) && tiny_c5_ultra_free_enabled()) {
|
||||
tiny_c5_ultra_free_fast(base, class_idx);
|
||||
return 1;
|
||||
}
|
||||
|
||||
// Phase 6: C4 ULTRA-free (C4-only, free-only, ENV gated, same pattern as C5/C6)
|
||||
if (tiny_class_is_c4(class_idx) && tiny_c4_ultra_free_enabled()) {
|
||||
tiny_c4_ultra_free_fast(base, class_idx);
|
||||
return 1;
|
||||
}
|
||||
|
||||
// C7 v3 fast classify: bypass classify_ptr/ss_map_lookup for clear hits
|
||||
if (class_idx == 7 &&
|
||||
tiny_front_v3_enabled() &&
|
||||
tiny_ptr_fast_classify_enabled() &&
|
||||
small_heap_v3_c7_enabled() &&
|
||||
smallobject_hotbox_v3_can_own_c7(base)) {
|
||||
so_free(7, base);
|
||||
// Phase FREE-LEGACY-BREAKDOWN-1: カウンタ散布 (3. C7 v3 fast classify)
|
||||
FREE_PATH_STAT_INC(smallheap_v3_fast);
|
||||
return 1;
|
||||
}
|
||||
|
||||
// Phase v11a-3: Try MID v3.5 free for C5/C6/C7
|
||||
// For v11a-3: simple ownership check (assumes current route)
|
||||
// For v11b: will add proper segment-based ownership check
|
||||
// Phase v11b-1: Policy-based single switch (replaces serial ULTRA checks)
|
||||
const SmallPolicyV7* policy_free = small_policy_v7_snapshot();
|
||||
if ((class_idx >= 5 && class_idx <= 7) &&
|
||||
policy_free->route_kind[class_idx] == SMALL_ROUTE_MID_V35) {
|
||||
small_mid_v35_free(ptr, class_idx);
|
||||
FREE_PATH_STAT_INC(smallheap_v7_fast); // Reuse counter for now
|
||||
return 1;
|
||||
}
|
||||
SmallRouteKind route_kind_free = policy_free->route_kind[class_idx];
|
||||
|
||||
// Phase v7-5b/v7-7: Always try V7 free for supported classes (C5/C6)
|
||||
// V7 returns false if ptr is not in V7 segment.
|
||||
// This is necessary because Learner may switch routes dynamically,
|
||||
// but pointers allocated before the switch still need V7 free.
|
||||
if (SMALL_V7_CLASS_SUPPORTED(class_idx)) {
|
||||
if (small_heap_free_fast_v7_stub(ptr, (uint8_t)class_idx)) {
|
||||
switch (route_kind_free) {
|
||||
case SMALL_ROUTE_ULTRA: {
|
||||
// Phase TLS-UNIFY-1: Unified ULTRA TLS push for C4-C6 (C7 handled above)
|
||||
if (class_idx >= 4 && class_idx <= 6) {
|
||||
tiny_ultra_tls_push((uint8_t)class_idx, base);
|
||||
return 1;
|
||||
}
|
||||
// ULTRA for other classes → fallback to LEGACY
|
||||
break;
|
||||
}
|
||||
|
||||
case SMALL_ROUTE_MID_V35: {
|
||||
// Phase v11a-3: MID v3.5 free
|
||||
small_mid_v35_free(ptr, class_idx);
|
||||
FREE_PATH_STAT_INC(smallheap_v7_fast);
|
||||
return 1;
|
||||
}
|
||||
// v7 returned false (ptr not in v7 segment) -> fallback to legacy below
|
||||
|
||||
case SMALL_ROUTE_V7: {
|
||||
// Phase v7: SmallObject v7 free (research box)
|
||||
if (small_heap_free_fast_v7_stub(ptr, (uint8_t)class_idx)) {
|
||||
FREE_PATH_STAT_INC(smallheap_v7_fast);
|
||||
return 1;
|
||||
}
|
||||
// V7 miss → fallback to LEGACY
|
||||
break;
|
||||
}
|
||||
|
||||
case SMALL_ROUTE_MID_V3: {
|
||||
// Phase MID-V3: delegate to MID v3.5
|
||||
small_mid_v35_free(ptr, class_idx);
|
||||
FREE_PATH_STAT_INC(smallheap_v7_fast);
|
||||
return 1;
|
||||
}
|
||||
|
||||
case SMALL_ROUTE_LEGACY:
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
// LEGACY fallback path
|
||||
tiny_route_kind_t route = tiny_route_for_class((uint8_t)class_idx);
|
||||
|
||||
// Phase v7-2: v7 early-exit for C6 (legacy path, now handled by Policy Box above)
|
||||
// Kept for compatibility with old routing system
|
||||
if (class_idx == 6 && route == TINY_ROUTE_SMALL_HEAP_V7) {
|
||||
if (small_heap_free_fast_v7_stub(ptr, (uint8_t)class_idx)) {
|
||||
FREE_PATH_STAT_INC(smallheap_v7_fast);
|
||||
return 1;
|
||||
}
|
||||
// v7 returned false (ptr not in v7 segment) -> fallback to legacy below
|
||||
}
|
||||
|
||||
if ((class_idx == 7 || class_idx == 6) &&
|
||||
route == TINY_ROUTE_SMALL_HEAP_V4 &&
|
||||
tiny_ptr_fast_classify_v4_enabled() &&
|
||||
smallobject_hotbox_v4_can_own(class_idx, base)) {
|
||||
small_heap_free_fast_v4(small_heap_ctx_v4_get(), class_idx, base);
|
||||
return 1;
|
||||
}
|
||||
|
||||
const int use_tiny_heap = tiny_route_is_heap_kind(route);
|
||||
const TinyFrontV3Snapshot* front_snap =
|
||||
__builtin_expect(tiny_front_v3_enabled(), 0) ? tiny_front_v3_snapshot_get() : NULL;
|
||||
|
||||
Reference in New Issue
Block a user