// smallobject_hotbox_v5.c - SmallObject HotBox v5 Full Implementation (Phase v5-2) // // Phase v5-2: C6-only full implementation with segment-based allocation #include #include #include #include "box/smallsegment_v5_box.h" #include "box/smallobject_hotbox_v5_box.h" #include "box/smallobject_cold_iface_v5.h" #include "box/smallobject_v5_env_box.h" #include "tiny_region_id.h" // For HEADER_MAGIC and HEADER_CLASS_MASK #ifndef likely #define likely(x) __builtin_expect(!!(x), 1) #define unlikely(x) __builtin_expect(!!(x), 0) #endif // TLS context static __thread SmallHeapCtxV5 g_small_heap_ctx_v5; static __thread int g_small_heap_ctx_v5_init = 0; SmallHeapCtxV5* small_heap_ctx_v5(void) { // Phase v5-4/v5-5/v5-6: Lazy initialization of cached ENV flags if (unlikely(!g_small_heap_ctx_v5_init)) { g_small_heap_ctx_v5.header_mode = (uint8_t)small_heap_v5_header_mode(); g_small_heap_ctx_v5.tls_cache_enabled = small_heap_v5_tls_cache_enabled(); g_small_heap_ctx_v5.c6_cached_block = NULL; // Initialize cache to empty g_small_heap_ctx_v5.batch_enabled = small_heap_v5_batch_enabled(); g_small_heap_ctx_v5.c6_batch.count = 0; // Initialize batch to empty for (int i = 0; i < SMALL_V5_BATCH_CAP; i++) { g_small_heap_ctx_v5.c6_batch.slots[i] = NULL; } g_small_heap_ctx_v5_init = 1; } return &g_small_heap_ctx_v5; } // Forward declarations for pool v1 fallback extern void* hak_pool_try_alloc(size_t size, uintptr_t site_id); extern void hak_pool_free(void* ptr, size_t size, uintptr_t site_id); // ============================================================================ // Helper: Slow path (refill from partial or cold) // ============================================================================ static SmallPageMetaV5* alloc_slow_v5(SmallHeapCtxV5* ctx, uint32_t class_idx) { SmallClassHeapV5* h = &ctx->cls[class_idx]; SmallPageMetaV5* cur = h->current; // If current exists but is exhausted, move to full list only // (exhausted pages are fully allocated, not partially free) if (cur && !cur->free_list) { SMALL_PAGE_V5_PUSH_FULL(h, cur); h->current = NULL; } // Try to pop from partial list (pages with some free blocks) SmallPageMetaV5* from_partial = SMALL_PAGE_V5_POP_PARTIAL(h); if (from_partial) { h->current = from_partial; return from_partial; } // Refill from cold interface (allocates new page) SmallPageMetaV5* page = small_cold_v5_refill_page(ctx, class_idx); if (!page) return NULL; h->current = page; return page; } // ============================================================================ // Phase v5-2: Fast alloc (C6-only full implementation) // ============================================================================ void* small_alloc_fast_v5(size_t size, uint32_t class_idx, SmallHeapCtxV5* ctx) { (void)size; // Not used in fast path // C6-only check if (unlikely(class_idx != SMALL_HEAP_V5_C6_CLASS_IDX)) { // Fallback to pool v1 for non-C6 classes return hak_pool_try_alloc(size, 0); } // Phase v5-5: TLS cache hit path (C6 only) if (unlikely(ctx->tls_cache_enabled)) { void* cached = ctx->c6_cached_block; if (likely(cached != NULL)) { ctx->c6_cached_block = NULL; // Consume cache slot // NOTE: cached is BASE pointer (same as freelist format), convert to USER pointer // This is consistent with the free path which stores (ptr - 1) as BASE // Header mode handling (same logic as freelist path) uint8_t* header_ptr = (uint8_t*)cached; uint8_t desired_header = (uint8_t)(HEADER_MAGIC | (class_idx & HEADER_CLASS_MASK)); if (ctx->header_mode == SMALL_HEAP_V5_HEADER_MODE_LIGHT) { // light mode: only write if invalid uint8_t existing = *header_ptr; if (existing != desired_header) { *header_ptr = desired_header; } } else { // full mode: always write header *header_ptr = desired_header; } return header_ptr + 1; } } // Phase v5-6: Batch alloc path (C6 only, after cache) if (ctx->batch_enabled && class_idx == SMALL_HEAP_V5_C6_CLASS_IDX && ctx->c6_batch.count > 0) { uint8_t idx = --ctx->c6_batch.count; void* b = ctx->c6_batch.slots[idx]; ctx->c6_batch.slots[idx] = NULL; // b is BASE pointer, return based on header mode if (ctx->header_mode == SMALL_HEAP_V5_HEADER_MODE_LIGHT) { return (uint8_t*)b + 1; } else { // full mode: write header uint8_t* header_ptr = (uint8_t*)b; uint8_t desired_header = (uint8_t)(HEADER_MAGIC | (class_idx & HEADER_CLASS_MASK)); *header_ptr = desired_header; return header_ptr + 1; } } // Cache miss - proceed to existing page_meta path SmallClassHeapV5* h = &ctx->cls[SMALL_HEAP_V5_C6_CLASS_IDX]; SmallPageMetaV5* page = h->current; // Fast path: Try current page freelist if (likely(page && page->free_list)) { void* blk = page->free_list; void* next = NULL; memcpy(&next, blk, sizeof(void*)); page->free_list = next; page->used++; // Phase v5-4: Header mode handling uint8_t* header_ptr = (uint8_t*)blk; uint8_t desired_header = (uint8_t)(HEADER_MAGIC | (class_idx & HEADER_CLASS_MASK)); if (ctx->header_mode == SMALL_HEAP_V5_HEADER_MODE_LIGHT) { // light mode: only write header if it's invalid/incorrect // This saves redundant writes when blocks are recycled uint8_t existing = *header_ptr; if (existing != desired_header) { *header_ptr = desired_header; } } else { // full mode: always write header (safety first) *header_ptr = desired_header; } return header_ptr + 1; } // Slow path: Current exhausted or NULL page = alloc_slow_v5(ctx, class_idx); if (unlikely(!page || !page->free_list)) { // Cold refill failed, fallback to pool v1 return hak_pool_try_alloc(size, 0); } // Allocate from newly acquired page void* blk = page->free_list; void* next = NULL; memcpy(&next, blk, sizeof(void*)); page->free_list = next; page->used++; // Phase v5-4: Header mode handling (same logic as fast path) uint8_t* header_ptr = (uint8_t*)blk; uint8_t desired_header = (uint8_t)(HEADER_MAGIC | (class_idx & HEADER_CLASS_MASK)); if (ctx->header_mode == SMALL_HEAP_V5_HEADER_MODE_LIGHT) { // light mode: only write if invalid uint8_t existing = *header_ptr; if (existing != desired_header) { *header_ptr = desired_header; } } else { // full mode: always write header *header_ptr = desired_header; } return header_ptr + 1; } // ============================================================================ // Helper: Determine page location in heap lists (Phase v5-3) // ============================================================================ static inline page_loc_t get_page_location(SmallClassHeapV5* h, SmallPageMetaV5* page, SmallPageMetaV5** prev_out) { if (prev_out) *prev_out = NULL; if (!h || !page) return LOC_NONE; // Check current (O(1)) if (h->current == page) { return LOC_CURRENT; } // Check partial list (typically 0-1 pages in v5-3) SmallPageMetaV5* prev = NULL; for (SmallPageMetaV5* p = h->partial_head; p; prev = p, p = p->next) { if (p == page) { if (prev_out) *prev_out = prev; return LOC_PARTIAL; } } // Check full list prev = NULL; for (SmallPageMetaV5* p = h->full_head; p; prev = p, p = p->next) { if (p == page) { if (prev_out) *prev_out = prev; return LOC_FULL; } } return LOC_NONE; } // ============================================================================ // Phase v5-3: Fast free (C6-only O(1) implementation) // ============================================================================ void small_free_fast_v5(void* ptr, uint32_t class_idx, SmallHeapCtxV5* ctx) { if (unlikely(!ptr)) { return; } // C6-only check if (unlikely(class_idx != SMALL_HEAP_V5_C6_CLASS_IDX)) { hak_pool_free(ptr, 0, 0); return; } // Phase v5-3: O(1) segment lookup (no list search) SmallPageMetaV5* page = small_segment_v5_page_meta_of(ptr); if (unlikely(!page)) { // Not in v5 segment, fallback to pool v1 hak_pool_free(ptr, 0, 0); return; } SmallClassHeapV5* h = &ctx->cls[SMALL_HEAP_V5_C6_CLASS_IDX]; // Phase v5-5: TLS cache refill path (before pushing to freelist) if (unlikely(ctx->tls_cache_enabled)) { if (ctx->c6_cached_block == NULL) { // Cache is empty, refill it with this block // NOTE: ptr is USER pointer, convert to BASE pointer for cache storage // (consistent with freelist storage format) void* base = (uint8_t*)ptr - 1; ctx->c6_cached_block = base; // IMPORTANT: Do NOT decrement page->used here! // The cached block is still logically "allocated" until it's: // - consumed during alloc (at which point it becomes allocated again) // - evicted to freelist (at which point page->used is decremented) // This prevents premature page retirement while holding a cached reference return; } // Cache full - evict cached block to freelist first, then cache this one else { void* evicted = ctx->c6_cached_block; // Evicted block is BASE pointer, convert to USER pointer for freelist push void* evicted_user = (uint8_t*)evicted + 1; // Look up the page for the evicted block (might be different from current page) SmallPageMetaV5* evicted_page = small_segment_v5_page_meta_of(evicted_user); if (evicted_page) { // Push evicted block to its page's freelist void* evicted_head = evicted_page->free_list; memcpy(evicted_user, &evicted_head, sizeof(void*)); evicted_page->free_list = evicted_user; if (evicted_page->used > 0) { evicted_page->used--; } // Note: We don't handle empty page transition here for evicted page // to keep this path fast. Empty pages will be handled on next alloc/free. } // Now cache the new block void* base = (uint8_t*)ptr - 1; ctx->c6_cached_block = base; return; } } // Phase v5-6: Batch free path (C6 only, after cache, before freelist) SmallV5Batch* batch = &ctx->c6_batch; if (ctx->batch_enabled && class_idx == SMALL_HEAP_V5_C6_CLASS_IDX && batch->count < SMALL_V5_BATCH_CAP) { // ptr is USER pointer, convert to BASE pointer for batch storage void* base = (uint8_t*)ptr - 1; batch->slots[batch->count++] = base; return; } // Cache disabled or batch full - push to freelist (standard path) void* head = page->free_list; memcpy(ptr, &head, sizeof(void*)); page->free_list = ptr; if (page->used > 0) { page->used--; } // Handle empty page (used == 0) if (page->used == 0) { // Fast path: if this is current, just keep it if (h->current == page) { return; } // Determine location and unlink (rare path) SmallPageMetaV5* prev = NULL; page_loc_t loc = get_page_location(h, page, &prev); if (loc != LOC_NONE && loc != LOC_CURRENT) { SMALL_PAGE_V5_UNLINK(h, loc, prev, page); } // Promote to current if empty if (!h->current) { h->current = page; page->next = NULL; return; } // Try partial (limit 1) if (h->partial_count < SMALL_HEAP_V5_C6_PARTIAL_LIMIT) { SMALL_PAGE_V5_PUSH_PARTIAL(h, page); return; } // Retire to cold small_cold_v5_retire_page(ctx, page); return; } // Page not empty - handle full→partial transition if (h->current != page) { SmallPageMetaV5* prev = NULL; page_loc_t loc = get_page_location(h, page, &prev); if (loc == LOC_FULL && page->free_list) { // Move from full to partial SMALL_PAGE_V5_UNLINK(h, loc, prev, page); if (h->partial_count < SMALL_HEAP_V5_C6_PARTIAL_LIMIT) { SMALL_PAGE_V5_PUSH_PARTIAL(h, page); } else { SMALL_PAGE_V5_PUSH_FULL(h, page); } } else if (!h->current) { // No current, promote this if (loc != LOC_NONE) { SMALL_PAGE_V5_UNLINK(h, loc, prev, page); } h->current = page; page->next = NULL; } } } // ============================================================================ // Helper: C6 block size query // ============================================================================ uint32_t small_heap_v5_c6_block_size(void) { return SMALL_HEAP_V5_C6_BLOCK_SIZE; }