hakmem/core/smallobject_hotbox_v4.c

// smallobject_hotbox_v4.c - SmallObject HotHeap v4 (C5/C6/C7 opt-in)
//
// Phase v4-3.1: C7 は v4 独自の freelist/current/partial で完結。C6/C5 は強ゲート付きで同形パスを使う。

#include <stdlib.h>
#include <string.h>

#include "box/smallobject_hotbox_v4_box.h"
#include "box/smallobject_hotbox_v4_env_box.h"
#include "box/smallobject_cold_iface_v4.h"
#include "box/smallobject_hotbox_v3_env_box.h"
#include "box/tiny_heap_box.h"
#include "box/tiny_cold_iface_v1.h"
#include "box/tiny_geometry_box.h"
#include "tiny_region_id.h"

// TLS context
static __thread small_heap_ctx_v4 g_ctx_v4;

small_heap_ctx_v4* small_heap_ctx_v4_get(void) {
    return &g_ctx_v4;
}

// -----------------------------------------------------------------------------
// helpers
// -----------------------------------------------------------------------------

static inline int v4_class_supported(int class_idx) {
    return class_idx == 7 || class_idx == 6 || class_idx == 5;
}

static inline void v4_page_push_partial(small_class_heap_v4* h, small_page_v4* page) {
    if (!h || !page) return;
    page->next = h->partial_head;
    h->partial_head = page;
    h->partial_count++;
}

static inline small_page_v4* v4_page_pop_partial(small_class_heap_v4* h) {
    if (!h) return NULL;
    small_page_v4* p = h->partial_head;
    if (p) {
        h->partial_head = p->next;
        p->next = NULL;
        if (h->partial_count > 0) {
            h->partial_count--;
        }
    }
    return p;
}

static inline void v4_page_push_full(small_class_heap_v4* h, small_page_v4* page) {
    if (!h || !page) return;
    page->next = h->full_head;
    h->full_head = page;
}

static inline uint32_t v4_partial_limit(int class_idx) {
    // C7 は refill/retire を抑えるため少し広めに保持
    return (class_idx == 7) ? 2u : 1u;
}

static inline int v4_ptr_in_page(const small_page_v4* page, const uint8_t* ptr) {
    if (!page || !ptr) return 0;
    uint8_t* base = page->base;
    size_t span = (size_t)page->block_size * (size_t)page->capacity;
    if (ptr < base || ptr >= base + span) return 0;
    size_t off = (size_t)(ptr - base);
    return (off % page->block_size) == 0;
}

static inline void* v4_build_freelist(uint8_t* base, uint16_t capacity, size_t stride) {
    void* head = NULL;
    for (int i = capacity - 1; i >= 0; i--) {
        uint8_t* blk = base + ((size_t)i * stride);
        void* next = head;
        head = blk;
        memcpy(blk, &next, sizeof(void*));
    }
    return head;
}

typedef enum {
    V4_LOC_NONE = 0,
    V4_LOC_CURRENT,
    V4_LOC_PARTIAL,
    V4_LOC_FULL,
} v4_loc_t;

static small_page_v4* v4_find_page(small_class_heap_v4* h, const uint8_t* ptr, v4_loc_t* loc, small_page_v4** prev_out) {
    if (loc) *loc = V4_LOC_NONE;
    if (prev_out) *prev_out = NULL;
    if (!h || !ptr) return NULL;

    if (h->current && v4_ptr_in_page(h->current, ptr)) {
        if (loc) *loc = V4_LOC_CURRENT;
        return h->current;
    }
    small_page_v4* prev = NULL;
    for (small_page_v4* p = h->partial_head; p; prev = p, p = p->next) {
        if (v4_ptr_in_page(p, ptr)) {
            if (loc) *loc = V4_LOC_PARTIAL;
            if (prev_out) *prev_out = prev;
            return p;
        }
    }
    for (small_page_v4* p = h->full_head; p; prev = p, p = p->next) {
        if (v4_ptr_in_page(p, ptr)) {
            if (loc) *loc = V4_LOC_FULL;
            if (prev_out) *prev_out = prev;
            return p;
        }
    }
    return NULL;
}

int smallobject_hotbox_v4_can_own(int class_idx, void* ptr) {
    if (__builtin_expect(!v4_class_supported(class_idx), 0)) return 0;
    if (!small_heap_v4_class_enabled((uint8_t)class_idx)) return 0;
    if (!ptr) return 0;
    small_heap_ctx_v4* ctx = small_heap_ctx_v4_get();
    if (!ctx) return 0;
    small_class_heap_v4* h = &ctx->cls[class_idx];
    return v4_find_page(h, (const uint8_t*)ptr, NULL, NULL) != NULL;
}

// -----------------------------------------------------------------------------
// Cold iface (C5/C6/C7, Tiny v1 経由)
// -----------------------------------------------------------------------------

static small_page_v4* cold_refill_page_v4(small_heap_ctx_v4* hot_ctx, uint32_t class_idx) {
    if (__builtin_expect(!v4_class_supported((int)class_idx), 0)) return NULL;
    (void)hot_ctx;
    tiny_heap_ctx_t* tctx = tiny_heap_ctx_for_thread();
    if (!tctx) return NULL;

    tiny_heap_page_t* lease = tiny_heap_prepare_page(tctx, (int)class_idx);
    if (!lease) return NULL;

    small_page_v4* page = (small_page_v4*)malloc(sizeof(small_page_v4));
    if (!page) {
        return NULL;
    }
    memset(page, 0, sizeof(*page));
    page->class_idx = (uint8_t)class_idx;
    page->capacity = lease->capacity;
    page->used = 0;
    page->block_size = (uint32_t)tiny_stride_for_class((int)class_idx);
    page->base = lease->base;
    page->slab_ref = lease;
    page->freelist = v4_build_freelist(lease->base, lease->capacity, page->block_size);
    if (!page->freelist) {
        free(page);
        return NULL;
    }
    page->next = NULL;
    page->flags = 0;
    return page;
}

static void cold_retire_page_v4(small_heap_ctx_v4* hot_ctx, uint32_t class_idx, small_page_v4* page) {
    (void)hot_ctx;
    if (!page) return;
    tiny_heap_ctx_t* tctx = tiny_heap_ctx_for_thread();
    tiny_heap_page_t* lease = (tiny_heap_page_t*)page->slab_ref;
    if (tctx && lease) {
        tiny_heap_page_becomes_empty(tctx, (int)class_idx, lease);
    }
    free(page);
}

static const SmallColdIfaceV4 g_cold_iface_v4 = {
    .refill_page = cold_refill_page_v4,
    .retire_page = cold_retire_page_v4,
    .remote_push = NULL,
    .remote_drain = NULL,
};

const SmallColdIfaceV4* small_cold_iface_v4_get(void) {
    return &g_cold_iface_v4;
}

// -----------------------------------------------------------------------------
// alloc/free
// -----------------------------------------------------------------------------

static small_page_v4* small_alloc_slow_v4(small_heap_ctx_v4* ctx, int class_idx) {
    small_class_heap_v4* h = &ctx->cls[class_idx];
    const uint32_t partial_limit = v4_partial_limit(class_idx);
    small_page_v4* cur = h->current;
    if (cur && cur->freelist) {
        return cur;  // usable current
    }
    if (cur && !cur->freelist) {
        // current をいったん partial/full に退避（partial を優先）
        if (h->partial_count < partial_limit) {
            v4_page_push_partial(h, cur);
        } else {
            v4_page_push_full(h, cur);
        }
        h->current = NULL;
    }

    // partial から 1 ページだけ復帰
    small_page_v4* from_partial = v4_page_pop_partial(h);
    if (from_partial) {
        h->current = from_partial;
        return from_partial;
    }

    const SmallColdIfaceV4* cold = small_cold_iface_v4_get();
    if (!cold || !cold->refill_page) return NULL;
    small_page_v4* page = cold->refill_page(ctx, (uint32_t)class_idx);
    if (!page) return NULL;
    h->current = page;
    return page;
}

void* small_heap_alloc_fast_v4(small_heap_ctx_v4* ctx, int class_idx) {
    if (__builtin_expect(!v4_class_supported(class_idx), 0)) {
        return NULL;  // C5/C6/C7 以外は未対応
    }
    if (!small_heap_v4_class_enabled((uint8_t)class_idx)) return NULL;
    small_class_heap_v4* h = &ctx->cls[class_idx];
    small_page_v4* page = h->current;

    if (!page || !page->freelist) {
        page = small_alloc_slow_v4(ctx, class_idx);
    }
    if (!page || !page->freelist) {
        return NULL;
    }

    void* blk = page->freelist;
    void* next = NULL;
    memcpy(&next, blk, sizeof(void*));
    page->freelist = next;
    page->used++;

    return tiny_region_id_write_header(blk, class_idx);
}

static void v4_unlink_from_list(small_class_heap_v4* h, v4_loc_t loc, small_page_v4* prev, small_page_v4* page) {
    if (!h || !page) return;
    switch (loc) {
        case V4_LOC_CURRENT:
            h->current = NULL;
            break;
        case V4_LOC_PARTIAL:
            if (prev) prev->next = page->next;
            else h->partial_head = page->next;
            if (h->partial_count > 0) {
                h->partial_count--;
            }
            break;
        case V4_LOC_FULL:
            if (prev) prev->next = page->next;
            else h->full_head = page->next;
            break;
        default:
            break;
    }
    page->next = NULL;
}

void small_heap_free_fast_v4(small_heap_ctx_v4* ctx, int class_idx, void* ptr) {
    if (__builtin_expect(!v4_class_supported(class_idx), 0)) {
        return;
    }
    if (!small_heap_v4_class_enabled((uint8_t)class_idx)) return;
    if (!ptr) return;

    small_class_heap_v4* h = &ctx->cls[class_idx];
    small_page_v4* prev = NULL;
    v4_loc_t loc = V4_LOC_NONE;
    small_page_v4* page = v4_find_page(h, (const uint8_t*)ptr, &loc, &prev);
    if (!page) return;

    const uint32_t partial_limit = v4_partial_limit(class_idx);

    // freelist push
    void* head = page->freelist;
    memcpy(ptr, &head, sizeof(void*));
    page->freelist = ptr;
    if (page->used > 0) {
        page->used--;
    }

    if (page->used == 0) {
        const SmallColdIfaceV4* cold = small_cold_iface_v4_get();
        if (loc != V4_LOC_CURRENT) {
            v4_unlink_from_list(h, loc, prev, page);
        }
        if (!h->current) {
            h->current = page;
            page->next = NULL;
            return;
        }
        if (h->current == page) {
            page->next = NULL;
            return;
        }
        if (h->partial_count < partial_limit) {
            v4_page_push_partial(h, page);
            return;
        }
        if (cold && cold->retire_page) {
            cold->retire_page(ctx, (uint32_t)class_idx, page);
        } else {
            free(page);
        }
        return;
    }

    if (!h->current) {
        // このページを current に据える
        if (loc != V4_LOC_CURRENT) {
            v4_unlink_from_list(h, loc, prev, page);
        }
        h->current = page;
        page->next = NULL;
    } else if (loc == V4_LOC_FULL && page->freelist) {
        // full → partial に戻す（current があっても partial 上限までは復帰）
        v4_unlink_from_list(h, loc, prev, page);
        if (h->partial_count < partial_limit) {
            v4_page_push_partial(h, page);
        } else {
            v4_page_push_full(h, page);  // 上限超なら戻す
        }
    }
}