hakmem/core/tiny_c7_ultra.c

// tiny_c7_ultra.c - UF-2: C7 ULTRA TLS freelist (coldは既存 v3 を使用)

#include <stddef.h>
#include <stdint.h>
#include <stdbool.h>
#include <string.h>
#include "box/tiny_c7_ultra_box.h"
#include "box/smallobject_hotbox_v3_box.h"
#include "box/tiny_geometry_box.h"
#include "tiny_region_id.h"
#include "box/tiny_c7_ultra_segment_box.h"
#include "box/tiny_front_v3_env_box.h"

static __thread tiny_c7_ultra_tls_t g_tiny_c7_ultra_tls;

static inline void tiny_c7_ultra_clear(tiny_c7_ultra_tls_t* tls) {
    tls->page_base = NULL;
    tls->block_size = 0;
    tls->capacity = 0;
    tls->used = 0;
    tls->freelist = NULL;
    tls->page_idx = 0;
    tls->page_meta = NULL;
    tls->headers_initialized = false;
}

tiny_c7_ultra_tls_t* tiny_c7_ultra_tls_get(void) {
    return &g_tiny_c7_ultra_tls;
}

// freelist next をヘッダを壊さずに保持する（ヘッダ byte の直後に保存）
static inline void ultra_store_next(void* base, void* next) {
    memcpy((uint8_t*)base + 1, &next, sizeof(next));
}

static inline void* ultra_load_next(void* base) {
    void* next = NULL;
    memcpy(&next, (uint8_t*)base + 1, sizeof(next));
    return next;
}

// セグメントから C7 ページを 1 枚借りて自前で carve する
static bool tiny_c7_ultra_lease_page(tiny_c7_ultra_tls_t* tls) {
    tiny_c7_ultra_segment_t* seg = tls->seg;
    if (!seg) {
        seg = tiny_c7_ultra_segment_acquire();
        if (!seg) return false;
        tls->seg = seg;
    }

    size_t block_sz = tls->block_size ? tls->block_size
                                      : (size_t)tiny_stride_for_class(7);
    if (block_sz == 0) return false;
    uint32_t capacity = (uint32_t)(seg->page_size / block_sz);
    if (capacity == 0) return false;

    const bool header_light = tiny_front_v3_c7_ultra_header_light_enabled();

    // 空きページを 1 枚だけ拾う（UF-3 では最初の空きを線形探索）
    uint32_t chosen = seg->num_pages;
    for (uint32_t i = 0; i < seg->num_pages; i++) {
        tiny_c7_ultra_page_meta_t* pm = &seg->pages[i];
        if (pm->capacity == 0 || pm->used == 0) {
            chosen = i;
            break;
        }
    }
    if (chosen == seg->num_pages) {
        return false;
    }

    tiny_c7_ultra_page_meta_t* page = &seg->pages[chosen];
    uint8_t* base = (uint8_t*)seg->base + ((size_t)chosen * seg->page_size);

    // freelist を自前で carve
    void* head = NULL;
    for (int i = (int)capacity - 1; i >= 0; i--) {
        uint8_t* blk = base + ((size_t)i * block_sz);
        if (header_light) {
            // header_light 時は carve で 1 度だけヘッダを書き込む
            tiny_region_id_write_header(blk, 7);
        }
        ultra_store_next(blk, head);
        head = blk;
    }
    if (!head) {
        return false;
    }

    page->freelist = head;
    page->capacity = capacity;
    page->used = 0;

    tls->page_base = base;
    tls->block_size = block_sz;
    tls->capacity = capacity;
    tls->used = 0;
    tls->freelist = head;
    tls->page_idx = chosen;
    tls->page_meta = page;
    tls->headers_initialized = header_light;
    return true;
}

void* tiny_c7_ultra_alloc(size_t size) {
    (void)size;  // C7 専用のため未使用
    tiny_c7_ultra_tls_t* tls = tiny_c7_ultra_tls_get();
    const bool header_light = tiny_front_v3_c7_ultra_header_light_enabled();

    // 1) freelist hit
    void* p = tls->freelist;
    if (__builtin_expect(p != NULL, 1)) {
        void* next = ultra_load_next(p);
        tls->freelist = next;
        if (tls->page_meta) {
            tls->page_meta->freelist = next;
            if (tls->page_meta->used < tls->page_meta->capacity) {
                tls->page_meta->used++;
            }
        }
        if (tls->used < tls->capacity) {
            tls->used++;
        }
        if (header_light && tls->headers_initialized) {
            return (uint8_t*)p + 1;
        }
        return tiny_region_id_write_header(p, 7);
    }

    // 2) lease page from existing v3 cold path
    if (!tiny_c7_ultra_lease_page(tls)) {
        // safety fallback to v3
        return so_alloc(7);
    }

    p = tls->freelist;
    if (__builtin_expect(p == NULL, 0)) {
        return so_alloc(7);
    }
    void* next = ultra_load_next(p);
    tls->freelist = next;
    if (tls->page_meta) {
        tls->page_meta->freelist = next;
        if (tls->page_meta->used < tls->page_meta->capacity) {
            tls->page_meta->used++;
        }
    }
    if (tls->used < tls->capacity) {
        tls->used++;
    }
    if (header_light && tls->headers_initialized) {
        return (uint8_t*)p + 1;
    }
    return tiny_region_id_write_header(p, 7);
}

void tiny_c7_ultra_free(void* ptr) {
    tiny_c7_ultra_tls_t* tls = tiny_c7_ultra_tls_get();
    if (!ptr) {
        so_free(7, ptr);
        return;
    }

    tiny_c7_ultra_segment_t* seg = tiny_c7_ultra_segment_from_ptr(ptr);
    if (!seg) {
        so_free(7, ptr);
        return;
    }

    uint32_t page_idx = 0;
    tiny_c7_ultra_page_meta_t* page = tiny_c7_ultra_page_of(ptr, &seg, &page_idx);
    if (!page) {
        so_free(7, ptr);
        return;
    }

    const size_t block_sz = tls->block_size ? tls->block_size
                                            : (size_t)tiny_stride_for_class(7);
    const uint32_t cap = page->capacity ? page->capacity
                                        : (uint32_t)(seg->page_size / block_sz);
    uintptr_t base = (uintptr_t)seg->base + ((size_t)page_idx * seg->page_size);
    uintptr_t paddr = (uintptr_t)ptr;
    size_t span = block_sz * (size_t)cap;
    if (paddr < base || paddr >= base + span || ((paddr - base) % block_sz) != 0) {
        so_free(7, ptr);
        return;
    }

    ultra_store_next(ptr, page->freelist);
    page->freelist = ptr;
    if (page->used > 0) {
        page->used--;
    }

    tls->page_meta = page;
    tls->page_idx = page_idx;
    tls->page_base = (void*)base;
    tls->capacity = cap;
    tls->block_size = block_sz;
    tls->freelist = page->freelist;
    tls->used = page->used;
}