305 lines
9.0 KiB
C
305 lines
9.0 KiB
C
|
// smallobject_hotbox_v4.c - SmallObject HotHeap v4 (C7-only real path)
|
||
|
|
//
|
||
|
|
// Phase v4-3: C7 クラスについては v4 独自の freelist/current/partial で完結させる。
|
||
|
|
|
||
|
|
#include <stdlib.h>
|
||
|
|
#include <string.h>
|
||
|
|
|
||
|
|
#include "box/smallobject_hotbox_v4_box.h"
|
||
|
|
#include "box/smallobject_hotbox_v4_env_box.h"
|
||
|
|
#include "box/smallobject_cold_iface_v4.h"
|
||
|
|
#include "box/smallobject_hotbox_v3_env_box.h"
|
||
|
|
#include "box/tiny_heap_box.h"
|
||
|
|
#include "box/tiny_cold_iface_v1.h"
|
||
|
|
#include "box/tiny_geometry_box.h"
|
||
|
|
#include "tiny_region_id.h"
|
||
|
|
|
||
|
|
// TLS context
|
||
|
|
static __thread small_heap_ctx_v4 g_ctx_v4;
|
||
|
|
|
||
|
|
#define V4_MAX_PARTIAL_PAGES 2
|
||
|
|
|
||
|
|
small_heap_ctx_v4* small_heap_ctx_v4_get(void) {
|
||
|
|
return &g_ctx_v4;
|
||
|
|
}
|
||
|
|
|
||
|
|
// -----------------------------------------------------------------------------
|
||
|
|
// helpers
|
||
|
|
// -----------------------------------------------------------------------------
|
||
|
|
|
||
|
|
static inline void v4_page_push_partial(small_class_heap_v4* h, small_page_v4* page) {
|
||
|
|
if (!h || !page) return;
|
||
|
|
page->next = h->partial_head;
|
||
|
|
h->partial_head = page;
|
||
|
|
h->partial_count++;
|
||
|
|
}
|
||
|
|
|
||
|
|
static inline small_page_v4* v4_page_pop_partial(small_class_heap_v4* h) {
|
||
|
|
if (!h) return NULL;
|
||
|
|
small_page_v4* p = h->partial_head;
|
||
|
|
if (p) {
|
||
|
|
h->partial_head = p->next;
|
||
|
|
p->next = NULL;
|
||
|
|
if (h->partial_count > 0) {
|
||
|
|
h->partial_count--;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return p;
|
||
|
|
}
|
||
|
|
|
||
|
|
static inline void v4_page_push_full(small_class_heap_v4* h, small_page_v4* page) {
|
||
|
|
if (!h || !page) return;
|
||
|
|
page->next = h->full_head;
|
||
|
|
h->full_head = page;
|
||
|
|
}
|
||
|
|
|
||
|
|
static inline int v4_ptr_in_page(const small_page_v4* page, const uint8_t* ptr) {
|
||
|
|
if (!page || !ptr) return 0;
|
||
|
|
uint8_t* base = page->base;
|
||
|
|
size_t span = (size_t)page->block_size * (size_t)page->capacity;
|
||
|
|
if (ptr < base || ptr >= base + span) return 0;
|
||
|
|
size_t off = (size_t)(ptr - base);
|
||
|
|
return (off % page->block_size) == 0;
|
||
|
|
}
|
||
|
|
|
||
|
|
static inline void* v4_build_freelist(uint8_t* base, uint16_t capacity, size_t stride) {
|
||
|
|
void* head = NULL;
|
||
|
|
for (int i = capacity - 1; i >= 0; i--) {
|
||
|
|
uint8_t* blk = base + ((size_t)i * stride);
|
||
|
|
void* next = head;
|
||
|
|
head = blk;
|
||
|
|
memcpy(blk, &next, sizeof(void*));
|
||
|
|
}
|
||
|
|
return head;
|
||
|
|
}
|
||
|
|
|
||
|
|
typedef enum {
|
||
|
|
V4_LOC_NONE = 0,
|
||
|
|
V4_LOC_CURRENT,
|
||
|
|
V4_LOC_PARTIAL,
|
||
|
|
V4_LOC_FULL,
|
||
|
|
} v4_loc_t;
|
||
|
|
|
||
|
|
static small_page_v4* v4_find_page(small_class_heap_v4* h, const uint8_t* ptr, v4_loc_t* loc, small_page_v4** prev_out) {
|
||
|
|
if (loc) *loc = V4_LOC_NONE;
|
||
|
|
if (prev_out) *prev_out = NULL;
|
||
|
|
if (!h || !ptr) return NULL;
|
||
|
|
|
||
|
|
if (h->current && v4_ptr_in_page(h->current, ptr)) {
|
||
|
|
if (loc) *loc = V4_LOC_CURRENT;
|
||
|
|
return h->current;
|
||
|
|
}
|
||
|
|
small_page_v4* prev = NULL;
|
||
|
|
for (small_page_v4* p = h->partial_head; p; prev = p, p = p->next) {
|
||
|
|
if (v4_ptr_in_page(p, ptr)) {
|
||
|
|
if (loc) *loc = V4_LOC_PARTIAL;
|
||
|
|
if (prev_out) *prev_out = prev;
|
||
|
|
return p;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
prev = NULL;
|
||
|
|
for (small_page_v4* p = h->full_head; p; prev = p, p = p->next) {
|
||
|
|
if (v4_ptr_in_page(p, ptr)) {
|
||
|
|
if (loc) *loc = V4_LOC_FULL;
|
||
|
|
if (prev_out) *prev_out = prev;
|
||
|
|
return p;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return NULL;
|
||
|
|
}
|
||
|
|
|
||
|
|
// -----------------------------------------------------------------------------
|
||
|
|
// Cold iface (C7-only, Tiny v1 経由)
|
||
|
|
// -----------------------------------------------------------------------------
|
||
|
|
|
||
|
|
static small_page_v4* cold_refill_page_v4(small_heap_ctx_v4* hot_ctx, uint32_t class_idx) {
|
||
|
|
if (__builtin_expect(class_idx != 7, 0)) return NULL;
|
||
|
|
(void)hot_ctx;
|
||
|
|
tiny_heap_ctx_t* tctx = tiny_heap_ctx_for_thread();
|
||
|
|
if (!tctx) return NULL;
|
||
|
|
|
||
|
|
tiny_heap_page_t* lease = tiny_heap_prepare_page(tctx, (int)class_idx);
|
||
|
|
if (!lease) return NULL;
|
||
|
|
|
||
|
|
small_page_v4* page = (small_page_v4*)malloc(sizeof(small_page_v4));
|
||
|
|
if (!page) {
|
||
|
|
return NULL;
|
||
|
|
}
|
||
|
|
memset(page, 0, sizeof(*page));
|
||
|
|
page->class_idx = (uint8_t)class_idx;
|
||
|
|
page->capacity = lease->capacity;
|
||
|
|
page->used = 0;
|
||
|
|
page->block_size = (uint32_t)tiny_stride_for_class((int)class_idx);
|
||
|
|
page->base = lease->base;
|
||
|
|
page->slab_ref = lease;
|
||
|
|
page->freelist = v4_build_freelist(lease->base, lease->capacity, page->block_size);
|
||
|
|
if (!page->freelist) {
|
||
|
|
free(page);
|
||
|
|
return NULL;
|
||
|
|
}
|
||
|
|
page->next = NULL;
|
||
|
|
page->flags = 0;
|
||
|
|
return page;
|
||
|
|
}
|
||
|
|
|
||
|
|
static void cold_retire_page_v4(small_heap_ctx_v4* hot_ctx, uint32_t class_idx, small_page_v4* page) {
|
||
|
|
(void)hot_ctx;
|
||
|
|
if (!page) return;
|
||
|
|
tiny_heap_ctx_t* tctx = tiny_heap_ctx_for_thread();
|
||
|
|
tiny_heap_page_t* lease = (tiny_heap_page_t*)page->slab_ref;
|
||
|
|
if (tctx && lease) {
|
||
|
|
tiny_heap_page_becomes_empty(tctx, (int)class_idx, lease);
|
||
|
|
}
|
||
|
|
free(page);
|
||
|
|
}
|
||
|
|
|
||
|
|
static const SmallColdIfaceV4 g_cold_iface_v4 = {
|
||
|
|
.refill_page = cold_refill_page_v4,
|
||
|
|
.retire_page = cold_retire_page_v4,
|
||
|
|
.remote_push = NULL,
|
||
|
|
.remote_drain = NULL,
|
||
|
|
};
|
||
|
|
|
||
|
|
const SmallColdIfaceV4* small_cold_iface_v4_get(void) {
|
||
|
|
return &g_cold_iface_v4;
|
||
|
|
}
|
||
|
|
|
||
|
|
// -----------------------------------------------------------------------------
|
||
|
|
// alloc/free
|
||
|
|
// -----------------------------------------------------------------------------
|
||
|
|
|
||
|
|
static small_page_v4* small_alloc_slow_v4(small_heap_ctx_v4* ctx, int class_idx) {
|
||
|
|
small_class_heap_v4* h = &ctx->cls[class_idx];
|
||
|
|
small_page_v4* cur = h->current;
|
||
|
|
if (cur && cur->freelist) {
|
||
|
|
return cur; // usable current
|
||
|
|
}
|
||
|
|
if (cur && !cur->freelist) {
|
||
|
|
// current を full list に残しておき、free で戻す
|
||
|
|
v4_page_push_full(h, cur);
|
||
|
|
h->current = NULL;
|
||
|
|
}
|
||
|
|
|
||
|
|
// partial から 1 ページだけ復帰
|
||
|
|
small_page_v4* from_partial = v4_page_pop_partial(h);
|
||
|
|
if (from_partial) {
|
||
|
|
h->current = from_partial;
|
||
|
|
return from_partial;
|
||
|
|
}
|
||
|
|
|
||
|
|
const SmallColdIfaceV4* cold = small_cold_iface_v4_get();
|
||
|
|
if (!cold || !cold->refill_page) return NULL;
|
||
|
|
small_page_v4* page = cold->refill_page(ctx, (uint32_t)class_idx);
|
||
|
|
if (!page) return NULL;
|
||
|
|
h->current = page;
|
||
|
|
return page;
|
||
|
|
}
|
||
|
|
|
||
|
|
void* small_heap_alloc_fast_v4(small_heap_ctx_v4* ctx, int class_idx) {
|
||
|
|
if (__builtin_expect(class_idx != 7, 0)) {
|
||
|
|
return NULL; // C7 専用
|
||
|
|
}
|
||
|
|
if (!small_heap_v4_c7_enabled()) return NULL;
|
||
|
|
small_class_heap_v4* h = &ctx->cls[class_idx];
|
||
|
|
small_page_v4* page = h->current;
|
||
|
|
|
||
|
|
if (!page || !page->freelist) {
|
||
|
|
page = small_alloc_slow_v4(ctx, class_idx);
|
||
|
|
}
|
||
|
|
if (!page || !page->freelist) {
|
||
|
|
return NULL;
|
||
|
|
}
|
||
|
|
|
||
|
|
void* blk = page->freelist;
|
||
|
|
void* next = NULL;
|
||
|
|
memcpy(&next, blk, sizeof(void*));
|
||
|
|
page->freelist = next;
|
||
|
|
page->used++;
|
||
|
|
|
||
|
|
return tiny_region_id_write_header(blk, class_idx);
|
||
|
|
}
|
||
|
|
|
||
|
|
static void v4_unlink_from_list(small_class_heap_v4* h, v4_loc_t loc, small_page_v4* prev, small_page_v4* page) {
|
||
|
|
if (!h || !page) return;
|
||
|
|
switch (loc) {
|
||
|
|
case V4_LOC_CURRENT:
|
||
|
|
h->current = NULL;
|
||
|
|
break;
|
||
|
|
case V4_LOC_PARTIAL:
|
||
|
|
if (prev) prev->next = page->next;
|
||
|
|
else h->partial_head = page->next;
|
||
|
|
if (h->partial_count > 0) {
|
||
|
|
h->partial_count--;
|
||
|
|
}
|
||
|
|
break;
|
||
|
|
case V4_LOC_FULL:
|
||
|
|
if (prev) prev->next = page->next;
|
||
|
|
else h->full_head = page->next;
|
||
|
|
break;
|
||
|
|
default:
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
page->next = NULL;
|
||
|
|
}
|
||
|
|
|
||
|
|
void small_heap_free_fast_v4(small_heap_ctx_v4* ctx, int class_idx, void* ptr) {
|
||
|
|
if (__builtin_expect(class_idx != 7, 0)) {
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
if (!small_heap_v4_c7_enabled()) return;
|
||
|
|
if (!ptr) return;
|
||
|
|
|
||
|
|
small_class_heap_v4* h = &ctx->cls[class_idx];
|
||
|
|
small_page_v4* prev = NULL;
|
||
|
|
v4_loc_t loc = V4_LOC_NONE;
|
||
|
|
small_page_v4* page = v4_find_page(h, (const uint8_t*)ptr, &loc, &prev);
|
||
|
|
if (!page) return;
|
||
|
|
|
||
|
|
// freelist push
|
||
|
|
void* head = page->freelist;
|
||
|
|
memcpy(ptr, &head, sizeof(void*));
|
||
|
|
page->freelist = ptr;
|
||
|
|
if (page->used > 0) {
|
||
|
|
page->used--;
|
||
|
|
}
|
||
|
|
|
||
|
|
if (page->used == 0) {
|
||
|
|
const SmallColdIfaceV4* cold = small_cold_iface_v4_get();
|
||
|
|
if (loc != V4_LOC_CURRENT) {
|
||
|
|
v4_unlink_from_list(h, loc, prev, page);
|
||
|
|
}
|
||
|
|
if (!h->current) {
|
||
|
|
h->current = page;
|
||
|
|
page->next = NULL;
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
if (h->current == page) {
|
||
|
|
page->next = NULL;
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
if (h->partial_count < V4_MAX_PARTIAL_PAGES) {
|
||
|
|
v4_page_push_partial(h, page);
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
if (cold && cold->retire_page) {
|
||
|
|
cold->retire_page(ctx, (uint32_t)class_idx, page);
|
||
|
|
} else {
|
||
|
|
free(page);
|
||
|
|
}
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
|
||
|
|
if (!h->current) {
|
||
|
|
// このページを current に据える
|
||
|
|
if (loc != V4_LOC_CURRENT) {
|
||
|
|
v4_unlink_from_list(h, loc, prev, page);
|
||
|
|
}
|
||
|
|
h->current = page;
|
||
|
|
page->next = NULL;
|
||
|
|
} else if (loc == V4_LOC_FULL && page->freelist) {
|
||
|
|
// full → partial に戻す
|
||
|
|
v4_unlink_from_list(h, loc, prev, page);
|
||
|
|
v4_page_push_partial(h, page);
|
||
|
|
}
|
||
|
|
}
|