hakmem/core/smallobject_core_v6.c

// smallobject_core_v6.c - SmallObject Core v6 実装（Phase v6-3）

#include <stdlib.h>
#include <string.h>
#include "box/smallobject_core_v6_box.h"
#include "box/smallobject_cold_iface_v6.h"
#include "box/smallsegment_v6_box.h"
#include "box/tiny_route_env_box.h"

#ifndef likely
#define likely(x)   __builtin_expect(!!(x), 1)
#define unlikely(x) __builtin_expect(!!(x), 0)
#endif

// TLS context
static __thread struct SmallHeapCtxV6 g_small_heap_ctx_v6;
static __thread int g_small_heap_ctx_v6_init = 0;

// TLS policy snapshot
static __thread struct SmallPolicySnapshotV6 g_snap_v6;
static __thread int g_snap_v6_init = 0;

/// Get TLS heap context for v6 (lazy initialization)
/// @return: TLS context pointer (never NULL)
SmallHeapCtxV6* small_heap_ctx_v6(void) {
    if (!g_small_heap_ctx_v6_init) {
        memset(&g_small_heap_ctx_v6, 0, sizeof(g_small_heap_ctx_v6));

        // Initialize TLS segment ownership range
        SmallSegmentV6* seg = small_segment_v6_acquire_for_thread();
        if (seg && small_segment_v6_valid(seg)) {
            g_small_heap_ctx_v6.tls_seg_base = seg->base;
            g_small_heap_ctx_v6.tls_seg_end = seg->base + SMALL_SEGMENT_V6_SIZE;
        }

        g_small_heap_ctx_v6_init = 1;
    }
    return &g_small_heap_ctx_v6;
}

/// Get TLS policy snapshot for v6 (lazy initialization)
/// @return: Policy snapshot pointer (never NULL)
const SmallPolicySnapshotV6* tiny_policy_snapshot_v6(void) {
    if (!g_snap_v6_init) {
        memset(&g_snap_v6, 0, sizeof(g_snap_v6));

        // Initialize route_kind from tiny_route API (this ensures init is done)
        for (int i = 0; i < 8; i++) {
            g_snap_v6.route_kind[i] = (uint8_t)tiny_route_for_class((uint8_t)i);
        }

        g_snap_v6_init = 1;
    }
    return &g_snap_v6;
}

// Forward declarations for pool v1 fallback
extern void* hak_pool_try_alloc(size_t size, uintptr_t site_id);
extern void hak_pool_free(void* ptr, size_t size, uintptr_t site_id);

// ============================================================================
// Allocation Implementation
// ============================================================================

/// Allocate block from C6 v6 TLS freelist or refill
/// @param size: requested size (unused, class_idx determines size)
/// @param class_idx: size class index (must be C6 for v6 route)
/// @param ctx: TLS context
/// @param snap: policy snapshot
/// @return: USER pointer (BASE+1) or NULL on fallback
void* small_alloc_fast_v6(size_t size,
                          uint32_t class_idx,
                          SmallHeapCtxV6* ctx,
                          const SmallPolicySnapshotV6* snap) {
    (void)size;

    // Bounds check
    if (unlikely(class_idx >= 8)) {
        return hak_pool_try_alloc(size, 0);
    }

    uint8_t route = snap->route_kind[class_idx];

    // v6-5: Support C6 and C5 classes
    if (route != TINY_ROUTE_SMALL_HEAP_V6) {
        return hak_pool_try_alloc(size, 0);
    }

    // C6 fast path
    if (class_idx == SMALL_V6_C6_CLASS_IDX) {
        // Fast path: TLS freelist hit
        if (likely(ctx->tls_count_c6 > 0)) {
            void* blk = ctx->tls_freelist_c6[--ctx->tls_count_c6];
            // v6-3: Header already written during refill, just return USER pointer
            return SMALL_V6_USER_FROM_BASE(blk);
        }
    }
    // C5 fast path (Phase v6-5)
    else if (class_idx == SMALL_V6_C5_CLASS_IDX) {
        // Fast path: TLS freelist hit
        if (likely(ctx->tls_count_c5 > 0)) {
            void* blk = ctx->tls_freelist_c5[--ctx->tls_count_c5];
            return SMALL_V6_USER_FROM_BASE(blk);
        }
    }
    else {
        // Unsupported class for v6
        return hak_pool_try_alloc(size, 0);
    }

    // Slow path: refill TLS with multiple blocks (batching)
    SmallPageMetaV6* page = small_cold_v6_refill_page(class_idx);
    if (!page || !page->free_list) {
        return hak_pool_try_alloc(size, 0);  // Safety fallback
    }

    // v6-5: Batch refill - support C6 and C5
    uint8_t header_byte = SMALL_V6_HEADER_FROM_CLASS(class_idx);

    if (class_idx == SMALL_V6_C6_CLASS_IDX) {
        // C6 refill path
        int max_fill = SMALL_V6_TLS_CAP - ctx->tls_count_c6;
        int filled = 0;

        // Fill TLS (leave room for 1 to return)
        while (page->free_list && filled < max_fill - 1) {
            void* blk = page->free_list;
            page->free_list = *(void**)blk;
            ((uint8_t*)blk)[0] = header_byte;
            ctx->tls_freelist_c6[ctx->tls_count_c6++] = blk;
            filled++;
        }
        page->used += filled;

        // Pop one more to return to caller
        if (page->free_list) {
            void* blk = page->free_list;
            page->free_list = *(void**)blk;
            page->used++;
            ((uint8_t*)blk)[0] = header_byte;
            return SMALL_V6_USER_FROM_BASE(blk);
        }

        // If we filled TLS but no more blocks, pop from TLS
        if (ctx->tls_count_c6 > 0) {
            void* blk = ctx->tls_freelist_c6[--ctx->tls_count_c6];
            return SMALL_V6_USER_FROM_BASE(blk);
        }
    }
    else if (class_idx == SMALL_V6_C5_CLASS_IDX) {
        // C5 refill path (Phase v6-5)
        int max_fill = SMALL_V6_TLS_CAP - ctx->tls_count_c5;
        int filled = 0;

        // Fill TLS (leave room for 1 to return)
        while (page->free_list && filled < max_fill - 1) {
            void* blk = page->free_list;
            page->free_list = *(void**)blk;
            ((uint8_t*)blk)[0] = header_byte;
            ctx->tls_freelist_c5[ctx->tls_count_c5++] = blk;
            filled++;
        }
        page->used += filled;

        // Pop one more to return to caller
        if (page->free_list) {
            void* blk = page->free_list;
            page->free_list = *(void**)blk;
            page->used++;
            ((uint8_t*)blk)[0] = header_byte;
            return SMALL_V6_USER_FROM_BASE(blk);
        }

        // If we filled TLS but no more blocks, pop from TLS
        if (ctx->tls_count_c5 > 0) {
            void* blk = ctx->tls_freelist_c5[--ctx->tls_count_c5];
            return SMALL_V6_USER_FROM_BASE(blk);
        }
    }

    // Should not reach here
    return hak_pool_try_alloc(size, 0);
}

// ============================================================================
// Free Implementation
// ============================================================================

/// Free block to C6 v6 TLS freelist or page freelist
/// @param ptr: USER pointer to free
/// @param class_idx: size class index
/// @param ctx: TLS context
/// @param snap: policy snapshot
void small_free_fast_v6(void* ptr,
                        uint32_t class_idx,
                        SmallHeapCtxV6* ctx,
                        const SmallPolicySnapshotV6* snap) {
    // Bounds check
    if (unlikely(class_idx >= 8)) {
        hak_pool_free(ptr, 0, 0);
        return;
    }

    uint8_t route = snap->route_kind[class_idx];

    // v6-5: Check if this is CORE_V6 route
    if (route != TINY_ROUTE_SMALL_HEAP_V6) {
        hak_pool_free(ptr, 0, 0);
        return;
    }

    // Convert USER pointer to BASE pointer
    void* base = SMALL_V6_BASE_FROM_USER(ptr);

    // Fast path: TLS segment ownership + TLS push
    if (likely(small_tls_owns_ptr_v6(ctx, ptr))) {
        // C6 TLS push
        if (class_idx == SMALL_V6_C6_CLASS_IDX && ctx->tls_count_c6 < SMALL_V6_TLS_CAP) {
            ctx->tls_freelist_c6[ctx->tls_count_c6++] = base;
            return;
        }
        // C5 TLS push (Phase v6-5)
        if (class_idx == SMALL_V6_C5_CLASS_IDX && ctx->tls_count_c5 < SMALL_V6_TLS_CAP) {
            ctx->tls_freelist_c5[ctx->tls_count_c5++] = base;
            return;
        }
    }

    // Slow path: page_meta lookup and push to page freelist
    SmallPageMetaV6* page = small_page_meta_v6_of(ptr);
    if (!page) {
        hak_pool_free(ptr, 0, 0);
        return;
    }

    // Push to page freelist (using BASE pointer)
    *(void**)base = page->free_list;
    page->free_list = base;
    if (page->used > 0) page->used--;

    // Retire empty page
    if (page->used == 0) {
        small_cold_v6_retire_page(page);
    }
}

// ============================================================================
// Cold Path Implementation (Phase v6-6)
// ============================================================================

/// Cold path: alloc with refill - called when TLS is empty
/// @param class_idx: C5 or C6
/// @param ctx: TLS context
/// @return: USER pointer or NULL
void* small_alloc_cold_v6(uint32_t class_idx, SmallHeapCtxV6* ctx) {
    // Refill TLS from page
    SmallPageMetaV6* page = small_cold_v6_refill_page(class_idx);
    if (!page || !page->free_list) {
        return hak_pool_try_alloc(class_idx == SMALL_V6_C6_CLASS_IDX ? 512 : 256, 0);
    }

    uint8_t header_byte = SMALL_V6_HEADER_FROM_CLASS(class_idx);

    if (class_idx == SMALL_V6_C6_CLASS_IDX) {
        int max_fill = SMALL_V6_TLS_CAP - ctx->tls_count_c6;
        int filled = 0;

        while (page->free_list && filled < max_fill - 1) {
            void* blk = page->free_list;
            page->free_list = *(void**)blk;
            ((uint8_t*)blk)[0] = header_byte;
            ctx->tls_freelist_c6[ctx->tls_count_c6++] = blk;
            filled++;
        }
        page->used += filled;

        if (page->free_list) {
            void* blk = page->free_list;
            page->free_list = *(void**)blk;
            page->used++;
            ((uint8_t*)blk)[0] = header_byte;
            return SMALL_V6_USER_FROM_BASE(blk);
        }

        if (ctx->tls_count_c6 > 0) {
            void* blk = ctx->tls_freelist_c6[--ctx->tls_count_c6];
            return SMALL_V6_USER_FROM_BASE(blk);
        }
    }
    else if (class_idx == SMALL_V6_C5_CLASS_IDX) {
        int max_fill = SMALL_V6_TLS_CAP - ctx->tls_count_c5;
        int filled = 0;

        while (page->free_list && filled < max_fill - 1) {
            void* blk = page->free_list;
            page->free_list = *(void**)blk;
            ((uint8_t*)blk)[0] = header_byte;
            ctx->tls_freelist_c5[ctx->tls_count_c5++] = blk;
            filled++;
        }
        page->used += filled;

        if (page->free_list) {
            void* blk = page->free_list;
            page->free_list = *(void**)blk;
            page->used++;
            ((uint8_t*)blk)[0] = header_byte;
            return SMALL_V6_USER_FROM_BASE(blk);
        }

        if (ctx->tls_count_c5 > 0) {
            void* blk = ctx->tls_freelist_c5[--ctx->tls_count_c5];
            return SMALL_V6_USER_FROM_BASE(blk);
        }
    }

    return hak_pool_try_alloc(class_idx == SMALL_V6_C6_CLASS_IDX ? 512 : 256, 0);
}

/// Cold path: free to page freelist - called when TLS full or cross-thread
/// @param ptr: USER pointer
/// @param class_idx: C5 or C6
void small_free_cold_v6(void* ptr, uint32_t class_idx) {
    (void)class_idx;  // Not needed for page lookup

    void* base = SMALL_V6_BASE_FROM_USER(ptr);

    SmallPageMetaV6* page = small_page_meta_v6_of(ptr);
    if (!page) {
        hak_pool_free(ptr, 0, 0);
        return;
    }

    *(void**)base = page->free_list;
    page->free_list = base;
    if (page->used > 0) page->used--;

    if (page->used == 0) {
        small_cold_v6_retire_page(page);
    }
}