// smallobject_core_v6.c - SmallObject Core v6 実装 // // Phase V6-HDR-0: C6-only headerless core with OBSERVE mode #include #include #include #include "box/smallobject_core_v6_box.h" #include "box/smallobject_cold_iface_v6.h" #include "box/smallsegment_v6_box.h" #include "box/tiny_route_env_box.h" #include "box/region_id_v6_box.h" #include "box/smallobject_v6_env_box.h" #ifndef likely #define likely(x) __builtin_expect(!!(x), 1) #define unlikely(x) __builtin_expect(!!(x), 0) #endif // ============================================================================ // OBSERVE Mode (V6-HDR-0) // ============================================================================ // ENV: HAKMEM_SMALL_V6_OBSERVE=1 enables logging at free entry #define V6_OBSERVE_UNINIT (-1) #define V6_OBSERVE_OFF 0 #define V6_OBSERVE_ON 1 static int g_v6_observe = V6_OBSERVE_UNINIT; static inline int small_v6_observe_enabled(void) { if (unlikely(g_v6_observe == V6_OBSERVE_UNINIT)) { const char* env = getenv("HAKMEM_SMALL_V6_OBSERVE"); g_v6_observe = (env && env[0] == '1') ? V6_OBSERVE_ON : V6_OBSERVE_OFF; } return g_v6_observe == V6_OBSERVE_ON; } /// Log free entry (called when OBSERVE=1) static void small_v6_observe_free(void* ptr, uint32_t class_idx, int tls_owned) { fprintf(stderr, "[V6_OBSERVE] free ptr=%p class=%u tls_owned=%d\n", ptr, class_idx, tls_owned); } // ============================================================================ // REGION_OBSERVE Validation (V6-HDR-1) // ============================================================================ // Note: small_v6_region_observe_enabled() is now in smallobject_v6_env_box.h /// Validate class_idx via RegionIdBox lookup (called when REGION_OBSERVE=1) /// @param ptr: USER pointer /// @param class_idx_hint: class_idx from front caller static void small_v6_region_observe_validate(void* ptr, uint32_t class_idx_hint) { RegionLookupV6 lk = region_id_lookup_v6(ptr); // Log the lookup to REGION_ID_BOX observe (if enabled) region_id_observe_lookup(ptr, &lk); if (lk.kind == REGION_KIND_SMALL_V6 && lk.page_meta != NULL) { SmallPageMetaV6* page = (SmallPageMetaV6*)lk.page_meta; if (page->class_idx != class_idx_hint) { fprintf(stderr, "[V6_REGION_OBSERVE] MISMATCH ptr=%p " "hint=%u actual=%u page_meta=%p\n", ptr, class_idx_hint, page->class_idx, (void*)page); } } else if (lk.kind != REGION_KIND_UNKNOWN) { // ptr is in a different kind of region (not v6) fprintf(stderr, "[V6_REGION_OBSERVE] KIND_MISMATCH ptr=%p " "kind=%s (expected SMALL_V6)\n", ptr, region_kind_to_string(lk.kind)); } // REGION_KIND_UNKNOWN: ptr not in any v6 segment (OK for now) } // TLS context static __thread struct SmallHeapCtxV6 g_small_heap_ctx_v6; static __thread int g_small_heap_ctx_v6_init = 0; // TLS policy snapshot static __thread struct SmallPolicySnapshotV6 g_snap_v6; static __thread int g_snap_v6_init = 0; /// Get TLS heap context for v6 (lazy initialization) /// @return: TLS context pointer (never NULL) SmallHeapCtxV6* small_heap_ctx_v6(void) { if (!g_small_heap_ctx_v6_init) { memset(&g_small_heap_ctx_v6, 0, sizeof(g_small_heap_ctx_v6)); // Initialize TLS segment ownership range SmallSegmentV6* seg = small_segment_v6_acquire_for_thread(); if (seg && small_segment_v6_valid(seg)) { g_small_heap_ctx_v6.tls_seg_base = seg->base; g_small_heap_ctx_v6.tls_seg_end = seg->base + SMALL_SEGMENT_V6_SIZE; // Phase V6-HDR-3: Register segment with RegionIdBox (TLS scope) region_id_register_v6_segment(seg); } g_small_heap_ctx_v6_init = 1; } return &g_small_heap_ctx_v6; } /// Get TLS policy snapshot for v6 (lazy initialization) /// @return: Policy snapshot pointer (never NULL) const SmallPolicySnapshotV6* tiny_policy_snapshot_v6(void) { if (!g_snap_v6_init) { memset(&g_snap_v6, 0, sizeof(g_snap_v6)); // Initialize route_kind from tiny_route API (this ensures init is done) for (int i = 0; i < 8; i++) { g_snap_v6.route_kind[i] = (uint8_t)tiny_route_for_class((uint8_t)i); } g_snap_v6_init = 1; } return &g_snap_v6; } // Forward declarations for pool v1 fallback extern void* hak_pool_try_alloc(size_t size, uintptr_t site_id); extern void hak_pool_free(void* ptr, size_t size, uintptr_t site_id); // ============================================================================ // Allocation Implementation // ============================================================================ /// Allocate block from C6 v6 TLS freelist or refill /// @param size: requested size (unused, class_idx determines size) /// @param class_idx: size class index (must be C6 for v6 route) /// @param ctx: TLS context /// @param snap: policy snapshot /// @return: USER pointer (BASE+1) or NULL on fallback void* small_alloc_fast_v6(size_t size, uint32_t class_idx, SmallHeapCtxV6* ctx, const SmallPolicySnapshotV6* snap) { (void)size; // Bounds check if (unlikely(class_idx >= 8)) { return hak_pool_try_alloc(size, 0); } uint8_t route = snap->route_kind[class_idx]; // v6-5: Support C6 and C5 classes if (route != TINY_ROUTE_SMALL_HEAP_V6) { return hak_pool_try_alloc(size, 0); } // C6 fast path if (class_idx == SMALL_V6_C6_CLASS_IDX) { // Fast path: TLS freelist hit if (likely(ctx->tls_count_c6 > 0)) { void* blk = ctx->tls_freelist_c6[--ctx->tls_count_c6]; // v6-3: Header already written during refill, just return USER pointer return SMALL_V6_USER_FROM_BASE(blk); } } // C5 fast path (Phase v6-5) else if (class_idx == SMALL_V6_C5_CLASS_IDX) { // Fast path: TLS freelist hit if (likely(ctx->tls_count_c5 > 0)) { void* blk = ctx->tls_freelist_c5[--ctx->tls_count_c5]; return SMALL_V6_USER_FROM_BASE(blk); } } // C4 fast path (Phase v6-6) else if (class_idx == SMALL_V6_C4_CLASS_IDX) { // Fast path: TLS freelist hit if (likely(ctx->tls_count_c4 > 0)) { void* blk = ctx->tls_freelist_c4[--ctx->tls_count_c4]; return SMALL_V6_USER_FROM_BASE(blk); } } else { // Unsupported class for v6 return hak_pool_try_alloc(size, 0); } // Slow path: refill TLS with multiple blocks (batching) SmallPageMetaV6* page = small_cold_v6_refill_page(class_idx); if (!page || !page->free_list) { return hak_pool_try_alloc(size, 0); // Safety fallback } // v6-5: Batch refill - support C6 and C5 uint8_t header_byte = SMALL_V6_HEADER_FROM_CLASS(class_idx); if (class_idx == SMALL_V6_C6_CLASS_IDX) { // C6 refill path int max_fill = SMALL_V6_TLS_CAP - ctx->tls_count_c6; int filled = 0; // Fill TLS (leave room for 1 to return) while (page->free_list && filled < max_fill - 1) { void* blk = page->free_list; page->free_list = *(void**)blk; ((uint8_t*)blk)[0] = header_byte; ctx->tls_freelist_c6[ctx->tls_count_c6++] = blk; filled++; } page->used += filled; // Pop one more to return to caller if (page->free_list) { void* blk = page->free_list; page->free_list = *(void**)blk; page->used++; ((uint8_t*)blk)[0] = header_byte; return SMALL_V6_USER_FROM_BASE(blk); } // If we filled TLS but no more blocks, pop from TLS if (ctx->tls_count_c6 > 0) { void* blk = ctx->tls_freelist_c6[--ctx->tls_count_c6]; return SMALL_V6_USER_FROM_BASE(blk); } } else if (class_idx == SMALL_V6_C5_CLASS_IDX) { // C5 refill path (Phase v6-5) int max_fill = SMALL_V6_TLS_CAP - ctx->tls_count_c5; int filled = 0; // Fill TLS (leave room for 1 to return) while (page->free_list && filled < max_fill - 1) { void* blk = page->free_list; page->free_list = *(void**)blk; ((uint8_t*)blk)[0] = header_byte; ctx->tls_freelist_c5[ctx->tls_count_c5++] = blk; filled++; } page->used += filled; // Pop one more to return to caller if (page->free_list) { void* blk = page->free_list; page->free_list = *(void**)blk; page->used++; ((uint8_t*)blk)[0] = header_byte; return SMALL_V6_USER_FROM_BASE(blk); } // If we filled TLS but no more blocks, pop from TLS if (ctx->tls_count_c5 > 0) { void* blk = ctx->tls_freelist_c5[--ctx->tls_count_c5]; return SMALL_V6_USER_FROM_BASE(blk); } } else if (class_idx == SMALL_V6_C4_CLASS_IDX) { // C4 refill path (Phase v6-6) int max_fill = SMALL_V6_TLS_CAP - ctx->tls_count_c4; int filled = 0; // Fill TLS (leave room for 1 to return) while (page->free_list && filled < max_fill - 1) { void* blk = page->free_list; page->free_list = *(void**)blk; ((uint8_t*)blk)[0] = header_byte; ctx->tls_freelist_c4[ctx->tls_count_c4++] = blk; filled++; } page->used += filled; // Pop one more to return to caller if (page->free_list) { void* blk = page->free_list; page->free_list = *(void**)blk; page->used++; ((uint8_t*)blk)[0] = header_byte; return SMALL_V6_USER_FROM_BASE(blk); } // If we filled TLS but no more blocks, pop from TLS if (ctx->tls_count_c4 > 0) { void* blk = ctx->tls_freelist_c4[--ctx->tls_count_c4]; return SMALL_V6_USER_FROM_BASE(blk); } } // Should not reach here return hak_pool_try_alloc(size, 0); } // ============================================================================ // Free Implementation // ============================================================================ /// Free block to C6 v6 TLS freelist or page freelist /// @param ptr: USER pointer to free /// @param class_idx: size class index /// @param ctx: TLS context /// @param snap: policy snapshot void small_free_fast_v6(void* ptr, uint32_t class_idx, SmallHeapCtxV6* ctx, const SmallPolicySnapshotV6* snap) { // Bounds check if (unlikely(class_idx >= 8)) { hak_pool_free(ptr, 0, 0); return; } uint8_t route = snap->route_kind[class_idx]; // v6-5: Check if this is CORE_V6 route if (route != TINY_ROUTE_SMALL_HEAP_V6) { hak_pool_free(ptr, 0, 0); return; } // Convert USER pointer to BASE pointer void* base = SMALL_V6_BASE_FROM_USER(ptr); // V6-HDR-0: OBSERVE mode logging (check TLS ownership first for log) int tls_owned = small_tls_owns_ptr_v6(ctx, ptr); if (unlikely(small_v6_observe_enabled())) { small_v6_observe_free(ptr, class_idx, tls_owned); } // V6-HDR-1: REGION_OBSERVE mode - validate class_idx via RegionIdBox if (unlikely(small_v6_region_observe_enabled())) { small_v6_region_observe_validate(ptr, class_idx); } // Fast path: TLS segment ownership + TLS push if (likely(tls_owned)) { // C6 TLS push if (class_idx == SMALL_V6_C6_CLASS_IDX && ctx->tls_count_c6 < SMALL_V6_TLS_CAP) { ctx->tls_freelist_c6[ctx->tls_count_c6++] = base; return; } // C5 TLS push (Phase v6-5) if (class_idx == SMALL_V6_C5_CLASS_IDX && ctx->tls_count_c5 < SMALL_V6_TLS_CAP) { ctx->tls_freelist_c5[ctx->tls_count_c5++] = base; return; } // C4 TLS push (Phase v6-6) if (class_idx == SMALL_V6_C4_CLASS_IDX && ctx->tls_count_c4 < SMALL_V6_TLS_CAP) { ctx->tls_freelist_c4[ctx->tls_count_c4++] = base; return; } } // Slow path: page_meta lookup and push to page freelist SmallPageMetaV6* page = small_page_meta_v6_of(ptr); if (!page) { hak_pool_free(ptr, 0, 0); return; } // Push to page freelist (using BASE pointer) *(void**)base = page->free_list; page->free_list = base; if (page->used > 0) page->used--; // Retire empty page if (page->used == 0) { small_cold_v6_retire_page(page); } } // ============================================================================ // Cold Path Implementation (Phase v6-6) // ============================================================================ /// Cold path: alloc with refill - called when TLS is empty /// @param class_idx: C4, C5 or C6 /// @param ctx: TLS context /// @return: USER pointer or NULL void* small_alloc_cold_v6(uint32_t class_idx, SmallHeapCtxV6* ctx) { // Refill TLS from page SmallPageMetaV6* page = small_cold_v6_refill_page(class_idx); if (!page || !page->free_list) { return hak_pool_try_alloc(class_idx == SMALL_V6_C6_CLASS_IDX ? 512 : (class_idx == SMALL_V6_C5_CLASS_IDX ? 256 : 128), 0); } uint8_t header_byte = SMALL_V6_HEADER_FROM_CLASS(class_idx); if (class_idx == SMALL_V6_C6_CLASS_IDX) { int max_fill = SMALL_V6_TLS_CAP - ctx->tls_count_c6; int filled = 0; while (page->free_list && filled < max_fill - 1) { void* blk = page->free_list; page->free_list = *(void**)blk; ((uint8_t*)blk)[0] = header_byte; ctx->tls_freelist_c6[ctx->tls_count_c6++] = blk; filled++; } page->used += filled; if (page->free_list) { void* blk = page->free_list; page->free_list = *(void**)blk; page->used++; ((uint8_t*)blk)[0] = header_byte; return SMALL_V6_USER_FROM_BASE(blk); } if (ctx->tls_count_c6 > 0) { void* blk = ctx->tls_freelist_c6[--ctx->tls_count_c6]; return SMALL_V6_USER_FROM_BASE(blk); } } else if (class_idx == SMALL_V6_C5_CLASS_IDX) { int max_fill = SMALL_V6_TLS_CAP - ctx->tls_count_c5; int filled = 0; while (page->free_list && filled < max_fill - 1) { void* blk = page->free_list; page->free_list = *(void**)blk; ((uint8_t*)blk)[0] = header_byte; ctx->tls_freelist_c5[ctx->tls_count_c5++] = blk; filled++; } page->used += filled; if (page->free_list) { void* blk = page->free_list; page->free_list = *(void**)blk; page->used++; ((uint8_t*)blk)[0] = header_byte; return SMALL_V6_USER_FROM_BASE(blk); } if (ctx->tls_count_c5 > 0) { void* blk = ctx->tls_freelist_c5[--ctx->tls_count_c5]; return SMALL_V6_USER_FROM_BASE(blk); } } else if (class_idx == SMALL_V6_C4_CLASS_IDX) { int max_fill = SMALL_V6_TLS_CAP - ctx->tls_count_c4; int filled = 0; while (page->free_list && filled < max_fill - 1) { void* blk = page->free_list; page->free_list = *(void**)blk; ((uint8_t*)blk)[0] = header_byte; ctx->tls_freelist_c4[ctx->tls_count_c4++] = blk; filled++; } page->used += filled; if (page->free_list) { void* blk = page->free_list; page->free_list = *(void**)blk; page->used++; ((uint8_t*)blk)[0] = header_byte; return SMALL_V6_USER_FROM_BASE(blk); } if (ctx->tls_count_c4 > 0) { void* blk = ctx->tls_freelist_c4[--ctx->tls_count_c4]; return SMALL_V6_USER_FROM_BASE(blk); } } return hak_pool_try_alloc(class_idx == SMALL_V6_C6_CLASS_IDX ? 512 : (class_idx == SMALL_V6_C5_CLASS_IDX ? 256 : 128), 0); } /// Cold path: free to page freelist - called when TLS full or cross-thread /// @param ptr: USER pointer /// @param class_idx: C5 or C6 void small_free_cold_v6(void* ptr, uint32_t class_idx) { (void)class_idx; // Not needed for page lookup void* base = SMALL_V6_BASE_FROM_USER(ptr); SmallPageMetaV6* page = small_page_meta_v6_of(ptr); if (!page) { hak_pool_free(ptr, 0, 0); return; } *(void**)base = page->free_list; page->free_list = base; if (page->used > 0) page->used--; if (page->used == 0) { small_cold_v6_retire_page(page); } } // ============================================================================ // Phase V6-HDR-2: Headerless Free/Alloc Implementation // ============================================================================ /// Headerless free: uses RegionIdBox for ptr classification /// @param ctx: TLS context /// @param ptr: USER pointer to free /// @param class_idx_hint: class_idx from front (header byte) /// @return: true if handled by v6, false if fallback needed bool small_v6_headerless_free(SmallHeapCtxV6* ctx, void* ptr, uint8_t class_idx_hint) { // Step 1: RegionIdBox lookup (no header read) RegionLookupV6 lk = region_id_lookup_v6(ptr); if (lk.kind != REGION_KIND_SMALL_V6) { // Not a v6 managed region -> front should fallback return false; } SmallPageMetaV6* page = (SmallPageMetaV6*)lk.page_meta; if (!page) { return false; } uint8_t class_idx = page->class_idx; // Step 2: OBSERVE mode - validate class_idx hint if (unlikely(small_v6_region_observe_enabled())) { if (class_idx != class_idx_hint) { fprintf(stderr, "[V6_HDR_FREE] MISMATCH ptr=%p hint=%u actual=%u\n", ptr, class_idx_hint, class_idx); } } // Step 3: Convert USER -> BASE (no header touch) void* base = SMALL_V6_BASE_FROM_USER(ptr); // Step 4: TLS ownership check + TLS push if (small_tls_owns_ptr_v6(ctx, ptr)) { // C6 TLS push if (class_idx == SMALL_V6_C6_CLASS_IDX && ctx->tls_count_c6 < SMALL_V6_TLS_CAP) { ctx->tls_freelist_c6[ctx->tls_count_c6++] = base; return true; } // C5 TLS push if (class_idx == SMALL_V6_C5_CLASS_IDX && ctx->tls_count_c5 < SMALL_V6_TLS_CAP) { ctx->tls_freelist_c5[ctx->tls_count_c5++] = base; return true; } // C4 TLS push if (class_idx == SMALL_V6_C4_CLASS_IDX && ctx->tls_count_c4 < SMALL_V6_TLS_CAP) { ctx->tls_freelist_c4[ctx->tls_count_c4++] = base; return true; } } // Step 5: Cold path - push to page freelist *(void**)base = page->free_list; page->free_list = base; if (page->used > 0) page->used--; // Retire empty page if (page->used == 0) { small_cold_v6_retire_page(page); } return true; } /// Headerless alloc: TLS pop without header write /// Header is already written during carve/refill /// @param ctx: TLS context /// @param class_idx: class index (4=C4, 5=C5, 6=C6) /// @return: USER pointer or NULL (fallback needed) void* small_v6_headerless_alloc(SmallHeapCtxV6* ctx, uint8_t class_idx) { // TLS fast path (no header write - already done in refill) if (class_idx == SMALL_V6_C6_CLASS_IDX) { if (likely(ctx->tls_count_c6 > 0)) { void* blk = ctx->tls_freelist_c6[--ctx->tls_count_c6]; return SMALL_V6_USER_FROM_BASE(blk); } } else if (class_idx == SMALL_V6_C5_CLASS_IDX) { if (likely(ctx->tls_count_c5 > 0)) { void* blk = ctx->tls_freelist_c5[--ctx->tls_count_c5]; return SMALL_V6_USER_FROM_BASE(blk); } } else if (class_idx == SMALL_V6_C4_CLASS_IDX) { if (likely(ctx->tls_count_c4 > 0)) { void* blk = ctx->tls_freelist_c4[--ctx->tls_count_c4]; return SMALL_V6_USER_FROM_BASE(blk); } } // TLS empty -> need refill from cold path // NOTE: Refill writes header, so alloc doesn't need to SmallPageMetaV6* page = small_cold_v6_refill_page(class_idx); if (!page || !page->free_list) { return NULL; // Front should fallback to legacy } uint8_t header_byte = SMALL_V6_HEADER_FROM_CLASS(class_idx); int max_fill = SMALL_V6_TLS_CAP; int filled = 0; // Refill TLS from page if (class_idx == SMALL_V6_C6_CLASS_IDX) { max_fill -= ctx->tls_count_c6; while (page->free_list && filled < max_fill - 1) { void* blk = page->free_list; page->free_list = *(void**)blk; ((uint8_t*)blk)[0] = header_byte; // Header write on refill only ctx->tls_freelist_c6[ctx->tls_count_c6++] = blk; filled++; } page->used += filled; if (page->free_list) { void* blk = page->free_list; page->free_list = *(void**)blk; page->used++; ((uint8_t*)blk)[0] = header_byte; return SMALL_V6_USER_FROM_BASE(blk); } if (ctx->tls_count_c6 > 0) { void* blk = ctx->tls_freelist_c6[--ctx->tls_count_c6]; return SMALL_V6_USER_FROM_BASE(blk); } } else if (class_idx == SMALL_V6_C5_CLASS_IDX) { max_fill -= ctx->tls_count_c5; while (page->free_list && filled < max_fill - 1) { void* blk = page->free_list; page->free_list = *(void**)blk; ((uint8_t*)blk)[0] = header_byte; ctx->tls_freelist_c5[ctx->tls_count_c5++] = blk; filled++; } page->used += filled; if (page->free_list) { void* blk = page->free_list; page->free_list = *(void**)blk; page->used++; ((uint8_t*)blk)[0] = header_byte; return SMALL_V6_USER_FROM_BASE(blk); } if (ctx->tls_count_c5 > 0) { void* blk = ctx->tls_freelist_c5[--ctx->tls_count_c5]; return SMALL_V6_USER_FROM_BASE(blk); } } else if (class_idx == SMALL_V6_C4_CLASS_IDX) { max_fill -= ctx->tls_count_c4; while (page->free_list && filled < max_fill - 1) { void* blk = page->free_list; page->free_list = *(void**)blk; ((uint8_t*)blk)[0] = header_byte; ctx->tls_freelist_c4[ctx->tls_count_c4++] = blk; filled++; } page->used += filled; if (page->free_list) { void* blk = page->free_list; page->free_list = *(void**)blk; page->used++; ((uint8_t*)blk)[0] = header_byte; return SMALL_V6_USER_FROM_BASE(blk); } if (ctx->tls_count_c4 > 0) { void* blk = ctx->tls_freelist_c4[--ctx->tls_count_c4]; return SMALL_V6_USER_FROM_BASE(blk); } } return NULL; }