Phase v4-mid-6: Implement C6 v4 TLS Fastlist (Gated)
- Implemented TLS fastlist logic for C6 in smallobject_hotbox_v4.c (alloc/free). - Added SmallC6FastState struct and g_small_c6_fast TLS variable. - Gated the fastlist logic with HAKMEM_SMALL_HEAP_V4_FASTLIST (default OFF) due to observed instability in mixed workloads. - Fixed a memory leak in small_heap_free_fast_v4 fallback path by calling hak_pool_free. - Updated CURRENT_TASK.md with phase report.
This commit is contained in:
@ -5,6 +5,11 @@
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#ifndef likely
|
||||
#define likely(x) __builtin_expect(!!(x), 1)
|
||||
#define unlikely(x) __builtin_expect(!!(x), 0)
|
||||
#endif
|
||||
|
||||
#include "box/smallobject_hotbox_v4_box.h"
|
||||
#include "box/smallobject_hotbox_v4_env_box.h"
|
||||
#include "box/smallobject_hotbox_v4_stats_box.h"
|
||||
@ -34,6 +39,13 @@ small_heap_v4_class_stats_t g_small_heap_v4_stats[8];
|
||||
// TLS context
|
||||
static __thread small_heap_ctx_v4 g_ctx_v4;
|
||||
|
||||
// Phase v4-mid-6: C6 TLS Fastlist
|
||||
static __thread SmallC6FastState g_small_c6_fast;
|
||||
|
||||
static inline SmallC6FastState* small_c6_fast_state(void) {
|
||||
return &g_small_c6_fast;
|
||||
}
|
||||
|
||||
// Internal segment structure (internal use only, not exposed via public box API)
|
||||
typedef struct small_segment_v4_internal {
|
||||
int class_idx;
|
||||
@ -366,6 +378,22 @@ void* small_heap_alloc_fast_v4(small_heap_ctx_v4* ctx, int class_idx) {
|
||||
// Phase v4-mid-5: Add stats instrumentation
|
||||
small_heap_v4_stat_alloc_call(class_idx);
|
||||
|
||||
// Phase v4-mid-6: C6 Fastlist Path
|
||||
if (class_idx == 6 && small_heap_v4_fastlist_enabled()) {
|
||||
SmallC6FastState* s = &g_small_c6_fast;
|
||||
if (likely(s->freelist)) {
|
||||
void* b = s->freelist;
|
||||
s->freelist = *(void**)b;
|
||||
s->used++;
|
||||
small_heap_v4_stat_alloc_success(class_idx);
|
||||
return tiny_region_id_write_header(b, class_idx);
|
||||
}
|
||||
// Fastlist empty: sync used back to meta before slow path
|
||||
if (s->meta) {
|
||||
s->meta->used = (uint16_t)s->used;
|
||||
}
|
||||
}
|
||||
|
||||
// Phase v4-mid-2: C6-only full SmallHeapCtx v4 implementation
|
||||
if (__builtin_expect(!v4_class_supported(class_idx), 0)) {
|
||||
small_heap_v4_stat_alloc_fallback_pool(class_idx);
|
||||
@ -398,6 +426,32 @@ void* small_heap_alloc_fast_v4(small_heap_ctx_v4* ctx, int class_idx) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
// Phase v4-mid-6: Promote to C6 Fastlist
|
||||
if (class_idx == 6 && small_heap_v4_fastlist_enabled()) {
|
||||
if (!page) {
|
||||
// Should not happen
|
||||
} else if (!page->freelist) {
|
||||
return NULL;
|
||||
} else {
|
||||
SmallC6FastState* s = &g_small_c6_fast;
|
||||
s->meta = page;
|
||||
s->page_base = page->base;
|
||||
s->capacity = page->capacity;
|
||||
s->used = page->used;
|
||||
s->freelist = page->freelist;
|
||||
page->freelist = NULL; // Steal freelist ownership
|
||||
|
||||
// Retry fast path
|
||||
if (likely(s->freelist)) {
|
||||
void* b = s->freelist;
|
||||
s->freelist = *(void**)b;
|
||||
s->used++;
|
||||
small_heap_v4_stat_alloc_success(class_idx);
|
||||
return tiny_region_id_write_header(b, class_idx);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Allocate from newly acquired/promoted page
|
||||
void* blk = page->freelist;
|
||||
void* next = NULL;
|
||||
@ -432,10 +486,29 @@ static void v4_unlink_from_list(small_class_heap_v4* h, v4_loc_t loc, small_page
|
||||
page->next = NULL;
|
||||
}
|
||||
|
||||
extern void hak_pool_free(void* ptr, size_t size, uintptr_t site_id);
|
||||
|
||||
void small_heap_free_fast_v4(small_heap_ctx_v4* ctx, int class_idx, void* ptr) {
|
||||
// Phase v4-mid-5: Add stats instrumentation
|
||||
small_heap_v4_stat_free_call(class_idx);
|
||||
|
||||
// Phase v4-mid-6: C6 Fastlist Path
|
||||
if (class_idx == 6 && small_heap_v4_fastlist_enabled()) {
|
||||
SmallC6FastState* s = &g_small_c6_fast;
|
||||
if (s->page_base && (uintptr_t)ptr >= (uintptr_t)s->page_base) {
|
||||
// Use actual block size from meta
|
||||
uint32_t bsize = (s->meta) ? s->meta->block_size : 512;
|
||||
size_t span = (size_t)s->capacity * bsize;
|
||||
if ((uintptr_t)ptr < (uintptr_t)s->page_base + span) {
|
||||
*(void**)ptr = s->freelist;
|
||||
s->freelist = ptr;
|
||||
s->used--;
|
||||
small_heap_v4_stat_free_page_found(class_idx);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Phase v4-mid-2: C6-only full SmallHeapCtx v4 implementation
|
||||
if (__builtin_expect(!v4_class_supported(class_idx), 0)) {
|
||||
return;
|
||||
@ -459,7 +532,8 @@ void small_heap_free_fast_v4(small_heap_ctx_v4* ctx, int class_idx, void* ptr) {
|
||||
small_heap_v4_stat_free_page_not_found(class_idx);
|
||||
// Try to find via segment mask+shift (requires segment to be initialized)
|
||||
// For now, this is a fallback for future segment-based allocation
|
||||
// Return without freeing (pool v1 will handle)
|
||||
// Fallback to pool v1 (avoid recursion via free())
|
||||
hak_pool_free(base_ptr, 0, 0);
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user