// tiny_ultra_tls_box.c - Phase TLS-UNIFY-2a + TLS-UNIFY-3: Unified ULTRA TLS implementation // // Phase 1: Thin wrapper delegating to per-class TLS (completed) // Phase 2a: Unified struct with array magazines for C4-C6 (completed) // C7 remains in separate TinyC7Ultra box. // Phase 3: C6 intrusive LIFO (current) - ENV gated // #include "tiny_ultra_tls_box.h" #include "tiny_c7_ultra_box.h" #include "free_path_stats_box.h" #include "tiny_c6_ultra_intrusive_env_box.h" // Phase 3: ENV gate #include "tiny_c6_intrusive_freelist_box.h" // Phase 3: L1 box #include "../superslab/superslab_inline.h" // For ss_fast_lookup #include "../tiny_debug_ring.h" // For ring visualization #ifndef likely #define likely(x) __builtin_expect(!!(x), 1) #define unlikely(x) __builtin_expect(!!(x), 0) #endif // ============================================================================ // Phase TLS-UNIFY-2a: Unified TLS context for C4-C6 // ============================================================================ static __thread TinyUltraTlsCtx g_ultra_tls_ctx = {0}; TinyUltraTlsCtx* tiny_ultra_tls_ctx(void) { return &g_ultra_tls_ctx; } // ============================================================================ // Phase TLS-UNIFY-2a: Pop from unified TLS (C4-C6) or C7 separate box // ============================================================================ void* tiny_ultra_tls_pop(uint8_t class_idx) { TinyUltraTlsCtx* ctx = &g_ultra_tls_ctx; switch (class_idx) { case 4: if (likely(ctx->c4_count > 0)) { return ctx->c4_freelist[--ctx->c4_count]; } return NULL; case 5: if (likely(ctx->c5_count > 0)) { return ctx->c5_freelist[--ctx->c5_count]; } return NULL; case 6: if (tiny_c6_ultra_intrusive_enabled()) { // Phase 3: intrusive LIFO void* base = c6_ifl_pop(&ctx->c6_head); if (base) { ctx->c6_count--; FREE_PATH_STAT_INC(c6_ifl_pop); tiny_debug_ring_record(TINY_RING_EVENT_C6_IFL_POP, 6, (uintptr_t)base, ctx->c6_count); } else { tiny_debug_ring_record(TINY_RING_EVENT_C6_IFL_EMPTY, 6, 0, ctx->c6_count); } return base; } else { // Fallback: array magazine if (likely(ctx->c6_count > 0)) { return ctx->c6_freelist[--ctx->c6_count]; } return NULL; } case 7: { // C7 uses separate TinyC7Ultra box (not unified) tiny_c7_ultra_tls_t* c7ctx = tiny_c7_ultra_tls_get(); if (likely(c7ctx->count > 0)) { return c7ctx->freelist[--c7ctx->count]; } return NULL; } default: return NULL; } } // ============================================================================ // Phase TLS-UNIFY-2a: Push to unified TLS (C4-C6) or C7 separate box // ============================================================================ // Forward declaration for slow path extern void so_free(int class_idx, void* ptr); // Slow path: flush half of TLS cache and push to segment static void tiny_ultra_tls_push_slow(uint8_t class_idx, void* base) { // Convert BASE to USER pointer for so_free void* user_ptr = (uint8_t*)base + 1; so_free(class_idx, user_ptr); } void tiny_ultra_tls_push(uint8_t class_idx, void* base) { TinyUltraTlsCtx* ctx = &g_ultra_tls_ctx; uintptr_t addr = (uintptr_t)base; switch (class_idx) { case 4: { // Learn segment on first C4 free if (unlikely(ctx->c4_seg_base == 0)) { SuperSlab* ss = ss_fast_lookup(base); if (ss != NULL) { ctx->c4_seg_base = (uintptr_t)ss; ctx->c4_seg_end = ctx->c4_seg_base + (1u << ss->lg_size); } } // Check segment range and capacity if (likely(ctx->c4_seg_base != 0 && addr >= ctx->c4_seg_base && addr < ctx->c4_seg_end && ctx->c4_count < TINY_ULTRA_C4_CAP)) { ctx->c4_freelist[ctx->c4_count++] = base; FREE_PATH_STAT_INC(c4_ultra_free_fast); return; } tiny_ultra_tls_push_slow(class_idx, base); break; } case 5: { // Learn segment on first C5 free if (unlikely(ctx->c5_seg_base == 0)) { SuperSlab* ss = ss_fast_lookup(base); if (ss != NULL) { ctx->c5_seg_base = (uintptr_t)ss; ctx->c5_seg_end = ctx->c5_seg_base + (1u << ss->lg_size); } } if (likely(ctx->c5_seg_base != 0 && addr >= ctx->c5_seg_base && addr < ctx->c5_seg_end && ctx->c5_count < TINY_ULTRA_C5_CAP)) { ctx->c5_freelist[ctx->c5_count++] = base; FREE_PATH_STAT_INC(c5_ultra_free_fast); return; } tiny_ultra_tls_push_slow(class_idx, base); break; } case 6: { // Learn segment on first C6 free (common for both modes) if (unlikely(ctx->c6_seg_base == 0)) { SuperSlab* ss = ss_fast_lookup(base); if (ss != NULL) { ctx->c6_seg_base = (uintptr_t)ss; ctx->c6_seg_end = ctx->c6_seg_base + (1u << ss->lg_size); } } // Check segment range and capacity (common) if (likely(ctx->c6_seg_base != 0 && addr >= ctx->c6_seg_base && addr < ctx->c6_seg_end && ctx->c6_count < TINY_ULTRA_C6_CAP)) { if (tiny_c6_ultra_intrusive_enabled()) { // Phase 3: intrusive LIFO c6_ifl_push(&ctx->c6_head, base); ctx->c6_count++; FREE_PATH_STAT_INC(c6_ifl_push); FREE_PATH_STAT_INC(c6_ultra_free_fast); tiny_debug_ring_record(TINY_RING_EVENT_C6_IFL_PUSH, 6, (uintptr_t)base, ctx->c6_count); } else { // Fallback: array magazine ctx->c6_freelist[ctx->c6_count++] = base; FREE_PATH_STAT_INC(c6_ultra_free_fast); } return; } // Slow path (range out or cap exceeded) if (tiny_c6_ultra_intrusive_enabled()) { FREE_PATH_STAT_INC(c6_ifl_fallback); } tiny_ultra_tls_push_slow(class_idx, base); break; } case 7: { // C7 uses separate TinyC7Ultra box (not unified) tiny_c7_ultra_tls_t* c7ctx = tiny_c7_ultra_tls_get(); if (unlikely(c7ctx->seg_base == 0)) { SuperSlab* ss = ss_fast_lookup(base); if (ss != NULL) { c7ctx->seg_base = (uintptr_t)ss; c7ctx->seg_end = c7ctx->seg_base + (1u << ss->lg_size); } } if (likely(c7ctx->seg_base != 0 && addr >= c7ctx->seg_base && addr < c7ctx->seg_end && c7ctx->count < TINY_C7_ULTRA_CAP)) { c7ctx->freelist[c7ctx->count++] = base; FREE_PATH_STAT_INC(c7_ultra_fast); return; } // Slow path for C7 void* user_ptr = (uint8_t*)base + 1; so_free(7, user_ptr); break; } default: break; } }