// smallobject_cold_iface_v7.c - SmallObject ColdIface v7 implementation (Phase v7-5b) // // Purpose: // - Page refill: acquire page from segment, carve freelist // - Page retire: release empty page back to segment, publish stats // // v7-5a optimizations: // - Header written at alloc time (not carve - freelist overlaps block[0]) // - Stats collected at retire time (not on hot path) // // v7-5b additions: // - C5 support (256B blocks) alongside C6 (512B blocks) // - Same segment shared between C5/C6, page_meta.class_idx distinguishes #include #include #include #include #include #include "box/smallobject_cold_iface_v7_box.h" #include "box/smallsegment_v7_box.h" #include "box/region_id_v6_box.h" #include "box/smallobject_policy_v7_box.h" // v7-7: Learner integration #include "tiny_region_id.h" // v7-5a: For HEADER_MAGIC, HEADER_CLASS_MASK #ifndef likely #define likely(x) __builtin_expect(!!(x), 1) #define unlikely(x) __builtin_expect(!!(x), 0) #endif // ============================================================================ // V7 Stats (for debugging/OBSERVE) // ============================================================================ static uint64_t g_v7_alloc_count = 0; static uint64_t g_v7_free_count = 0; static uint64_t g_v7_refill_count = 0; static uint64_t g_v7_retire_count = 0; // Destructor to print stats at exit __attribute__((destructor)) static void small_v7_stats_report(void) { if (g_v7_alloc_count > 0 || g_v7_free_count > 0) { fprintf(stderr, "[SMALL_V7] alloc=%lu free=%lu refill=%lu retire=%lu\n", (unsigned long)g_v7_alloc_count, (unsigned long)g_v7_free_count, (unsigned long)g_v7_refill_count, (unsigned long)g_v7_retire_count); } } void small_v7_stat_alloc(void) { __sync_fetch_and_add(&g_v7_alloc_count, 1); } void small_v7_stat_free(void) { __sync_fetch_and_add(&g_v7_free_count, 1); } void small_v7_stat_refill(void) { __sync_fetch_and_add(&g_v7_refill_count, 1); } void small_v7_stat_retire(void) { __sync_fetch_and_add(&g_v7_retire_count, 1); } // ============================================================================ // TLS Heap Context // ============================================================================ static __thread SmallHeapCtx_v7 g_small_heap_ctx_v7; static __thread int g_small_heap_ctx_v7_init = 0; SmallHeapCtx_v7* small_heap_ctx_v7(void) { if (unlikely(!g_small_heap_ctx_v7_init)) { // Initialize context memset(&g_small_heap_ctx_v7, 0, sizeof(g_small_heap_ctx_v7)); for (int i = 0; i < HAK_SMALL_NUM_CLASSES_V7; i++) { g_small_heap_ctx_v7.cls[i].class_idx = (uint16_t)i; g_small_heap_ctx_v7.cls[i].current = NULL; g_small_heap_ctx_v7.cls[i].partial_head = NULL; g_small_heap_ctx_v7.cls[i].full_head = NULL; g_small_heap_ctx_v7.cls[i].local_freelist = NULL; g_small_heap_ctx_v7.cls[i].local_freelist_count = 0; g_small_heap_ctx_v7.cls[i].local_freelist_cap = 0; g_small_heap_ctx_v7.cls[i].flags = 0; } g_small_heap_ctx_v7.segment = NULL; // Phase v7-3: Initialize TLS fast hints and cache g_small_heap_ctx_v7.tls_seg_base = 0; g_small_heap_ctx_v7.tls_seg_end = 0; g_small_heap_ctx_v7.last_page_base = 0; g_small_heap_ctx_v7.last_page_end = 0; g_small_heap_ctx_v7.last_page_meta = NULL; g_small_heap_ctx_v7_init = 1; } return &g_small_heap_ctx_v7; } // ============================================================================ // Segment Acquisition (lazy) // ============================================================================ static SmallSegment_v7* cold_v7_ensure_segment(SmallHeapCtx_v7* ctx) { if (likely(ctx->segment != NULL)) { return ctx->segment; } // Acquire segment for this thread uint32_t tid = (uint32_t)getpid(); SmallSegment_v7* seg = small_segment_alloc_v7(tid); if (unlikely(!seg)) { return NULL; } ctx->segment = seg; // Phase v7-3: Set TLS segment fast hints ctx->tls_seg_base = seg->base; ctx->tls_seg_end = seg->base + SMALL_SEGMENT_V7_SIZE; // Register with RegionIdBox for ptr->region lookup // This enables region_id_lookup_v6() to recognize v7 pointers region_id_register_v6( (void*)seg->base, SMALL_SEGMENT_V7_SIZE, REGION_KIND_SMALL_V7, seg ); return seg; } // ============================================================================ // Page Refill // ============================================================================ SmallPageMeta_v7* small_cold_v7_refill_page(SmallHeapCtx_v7* ctx, uint32_t class_idx) { if (unlikely(!ctx)) { return NULL; } // v7-5b: C5 (256B) and C6 (512B) supported size_t block_size = small_v7_block_size(class_idx); if (unlikely(block_size == 0)) { return NULL; // Unsupported class } // Ensure we have a segment SmallSegment_v7* seg = cold_v7_ensure_segment(ctx); if (unlikely(!seg)) { return NULL; } // Take a page from segment's free stack SmallPageMeta_v7* page = small_segment_take_page_v7(seg, class_idx); if (unlikely(!page)) { return NULL; // No free pages } // Calculate page base address uintptr_t page_offset = (uintptr_t)page->page_idx * SMALL_PAGE_V7_SIZE; uintptr_t page_base = seg->base + page_offset; uint8_t* base = (uint8_t*)page_base; // Calculate capacity uint32_t capacity = SMALL_PAGE_V7_SIZE / block_size; page->capacity = capacity; page->used = 0; // Build intrusive freelist (last to first for cache locality on pop) // freelist points to BASE pointers (block start) // // Note: Cannot write header at carve time because freelist next pointer // is stored at block[0..7], which overlaps with header byte at block[0]. // Header must be written at alloc time after popping from freelist. void* freelist = NULL; for (int i = (int)capacity - 1; i >= 0; i--) { uint8_t* block = base + ((size_t)i * block_size); // Link: block->next = freelist void* next = freelist; memcpy(block, &next, sizeof(void*)); freelist = block; } page->free_list = freelist; // Set as current page for this class SmallClassHeap_v7* heap = &ctx->cls[class_idx]; heap->current = page; // v7-7: Feed refill to Learner for workload detection small_learner_v7_record_refill(class_idx, capacity); return page; } // ============================================================================ // Page Retire // ============================================================================ void small_cold_v7_retire_page(SmallHeapCtx_v7* ctx, SmallPageMeta_v7* page) { if (unlikely(!ctx || !page)) { return; } // Only retire pages that are fully empty if (page->used != 0) { return; } SmallSegment_v7* seg = page->segment; if (unlikely(!seg)) { return; } // Publish stats before retiring SmallPageStatsV7 stats = { .class_idx = (uint8_t)page->class_idx, .reserved0 = 0, .page_idx = page->page_idx, .capacity = page->capacity, .alloc_count = page->alloc_count, .free_count = page->free_count, .remote_free_count = page->remote_free_count, .peak_live = page->peak_live, .remote_burst_max = page->remote_burst_max, .lifetime_ms = 0 // TODO: Calculate from epoch }; small_cold_v7_publish_stats(&stats); // Clear current if this was the current page SmallClassHeap_v7* heap = &ctx->cls[page->class_idx]; if (heap->current == page) { heap->current = NULL; } // Release page back to segment small_segment_release_page_v7(seg, page); } // ============================================================================ // Stats Publishing (v7-7: Learner integration) // ============================================================================ void small_cold_v7_publish_stats(const SmallPageStatsV7* stats) { if (!stats) return; // v7-7: Feed stats to Learner for dynamic route switching // Note: v7-5a removed alloc_count from hot path, use capacity as traffic proxy small_learner_v7_record_retire(stats->class_idx, stats->capacity); }