Phase v4-mid-6: Implement C6 v4 TLS Fastlist (Gated)

- Implemented TLS fastlist logic for C6 in smallobject_hotbox_v4.c (alloc/free).
- Added SmallC6FastState struct and g_small_c6_fast TLS variable.
- Gated the fastlist logic with HAKMEM_SMALL_HEAP_V4_FASTLIST (default OFF) due to observed instability in mixed workloads.
- Fixed a memory leak in small_heap_free_fast_v4 fallback path by calling hak_pool_free.
- Updated CURRENT_TASK.md with phase report.
This commit is contained in:
Moe Charm (CI)
2025-12-11 01:44:08 +09:00
parent dd974b49c5
commit e486dd2c55
4 changed files with 113 additions and 1 deletions

View File

@ -5,6 +5,11 @@
#include <stdlib.h>
#include <string.h>
#ifndef likely
#define likely(x) __builtin_expect(!!(x), 1)
#define unlikely(x) __builtin_expect(!!(x), 0)
#endif
#include "box/smallobject_hotbox_v4_box.h"
#include "box/smallobject_hotbox_v4_env_box.h"
#include "box/smallobject_hotbox_v4_stats_box.h"
@ -34,6 +39,13 @@ small_heap_v4_class_stats_t g_small_heap_v4_stats[8];
// TLS context
static __thread small_heap_ctx_v4 g_ctx_v4;
// Phase v4-mid-6: C6 TLS Fastlist
static __thread SmallC6FastState g_small_c6_fast;
static inline SmallC6FastState* small_c6_fast_state(void) {
return &g_small_c6_fast;
}
// Internal segment structure (internal use only, not exposed via public box API)
typedef struct small_segment_v4_internal {
int class_idx;
@ -366,6 +378,22 @@ void* small_heap_alloc_fast_v4(small_heap_ctx_v4* ctx, int class_idx) {
// Phase v4-mid-5: Add stats instrumentation
small_heap_v4_stat_alloc_call(class_idx);
// Phase v4-mid-6: C6 Fastlist Path
if (class_idx == 6 && small_heap_v4_fastlist_enabled()) {
SmallC6FastState* s = &g_small_c6_fast;
if (likely(s->freelist)) {
void* b = s->freelist;
s->freelist = *(void**)b;
s->used++;
small_heap_v4_stat_alloc_success(class_idx);
return tiny_region_id_write_header(b, class_idx);
}
// Fastlist empty: sync used back to meta before slow path
if (s->meta) {
s->meta->used = (uint16_t)s->used;
}
}
// Phase v4-mid-2: C6-only full SmallHeapCtx v4 implementation
if (__builtin_expect(!v4_class_supported(class_idx), 0)) {
small_heap_v4_stat_alloc_fallback_pool(class_idx);
@ -398,6 +426,32 @@ void* small_heap_alloc_fast_v4(small_heap_ctx_v4* ctx, int class_idx) {
return NULL;
}
// Phase v4-mid-6: Promote to C6 Fastlist
if (class_idx == 6 && small_heap_v4_fastlist_enabled()) {
if (!page) {
// Should not happen
} else if (!page->freelist) {
return NULL;
} else {
SmallC6FastState* s = &g_small_c6_fast;
s->meta = page;
s->page_base = page->base;
s->capacity = page->capacity;
s->used = page->used;
s->freelist = page->freelist;
page->freelist = NULL; // Steal freelist ownership
// Retry fast path
if (likely(s->freelist)) {
void* b = s->freelist;
s->freelist = *(void**)b;
s->used++;
small_heap_v4_stat_alloc_success(class_idx);
return tiny_region_id_write_header(b, class_idx);
}
}
}
// Allocate from newly acquired/promoted page
void* blk = page->freelist;
void* next = NULL;
@ -432,10 +486,29 @@ static void v4_unlink_from_list(small_class_heap_v4* h, v4_loc_t loc, small_page
page->next = NULL;
}
extern void hak_pool_free(void* ptr, size_t size, uintptr_t site_id);
void small_heap_free_fast_v4(small_heap_ctx_v4* ctx, int class_idx, void* ptr) {
// Phase v4-mid-5: Add stats instrumentation
small_heap_v4_stat_free_call(class_idx);
// Phase v4-mid-6: C6 Fastlist Path
if (class_idx == 6 && small_heap_v4_fastlist_enabled()) {
SmallC6FastState* s = &g_small_c6_fast;
if (s->page_base && (uintptr_t)ptr >= (uintptr_t)s->page_base) {
// Use actual block size from meta
uint32_t bsize = (s->meta) ? s->meta->block_size : 512;
size_t span = (size_t)s->capacity * bsize;
if ((uintptr_t)ptr < (uintptr_t)s->page_base + span) {
*(void**)ptr = s->freelist;
s->freelist = ptr;
s->used--;
small_heap_v4_stat_free_page_found(class_idx);
return;
}
}
}
// Phase v4-mid-2: C6-only full SmallHeapCtx v4 implementation
if (__builtin_expect(!v4_class_supported(class_idx), 0)) {
return;
@ -459,7 +532,8 @@ void small_heap_free_fast_v4(small_heap_ctx_v4* ctx, int class_idx, void* ptr) {
small_heap_v4_stat_free_page_not_found(class_idx);
// Try to find via segment mask+shift (requires segment to be initialized)
// For now, this is a fallback for future segment-based allocation
// Return without freeing (pool v1 will handle)
// Fallback to pool v1 (avoid recursion via free())
hak_pool_free(base_ptr, 0, 0);
return;
}