tomoaki/hakmem
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Boxify superslab registry, add bench profile, and document C7 hotpath experiments

Browse Source
This commit is contained in:
Moe Charm (CI)
2025-12-07 03:12:27 +09:00
parent 18faa6a1c4
commit fda6cd2e67
71 changed files with 2052 additions and 286 deletions
Download Patch File Download Diff File Expand all files Collapse all files

96
core/hakmem_shared_pool_acquire.c
View File

@ -12,6 +12,10 @@
#include "front/tiny_warm_pool.h" // Warm Pool: Prefill during registry scans
#include "box/ss_slab_reset_box.h" // Box: Reset slab metadata on reuse (C7 guard)
#include "box/tiny_class_stats_box.h" // OBSERVE: per-class shared lock stats
#include "box/ss_stats_box.h" // OBSERVE: Superslab/slab event counters
#include "box/ss_budget_box.h" // Budget guard for Superslab growth (larson_guard)
#include "box/super_reg_box.h" // Logical limit for registry scan
#include "box/shared_pool_box.h" // Logical cap for shared pool slots (bench profile)
#include <stdint.h>
#include <stdlib.h>
@ -22,8 +26,8 @@
_Atomic uintptr_t g_c7_stage3_magic_ss = 0;
static inline void sp_lock_with_stats(int class_idx) {
pthread_mutex_lock(&g_shared_pool.alloc_lock);
tiny_class_stats_on_shared_lock(class_idx);
sp_lock_with_stats(class_idx);
}
static inline void c7_log_meta_state(const char* tag, SuperSlab* ss, int slab_idx) {
@ -159,6 +163,37 @@ static inline int c7_reset_and_log_if_needed(SuperSlab* ss,
return 0;
}
static inline void sp_reset_superslab_all_slabs(SuperSlab* ss,
int class_idx,
int from_lru) {
if (!ss) {
return;
}
int cap = ss_slabs_capacity(ss);
ss->slab_bitmap = 0;
ss->nonempty_mask = 0;
ss->freelist_mask = 0;
ss->empty_mask = 0;
ss->empty_count = 0;
ss->active_slabs = 0;
ss->hot_count = 0;
ss->cold_count = 0;
for (int s = 0; s < cap; s++) {
ss_slab_reset_meta_for_tiny(ss, s, class_idx);
}
ss_stats_on_ss_scan(class_idx, 0, 1);
static _Atomic uint32_t rel_stage3_reset_logs = 0;
uint32_t n = atomic_fetch_add_explicit(&rel_stage3_reset_logs, 1, memory_order_relaxed);
if (n < 4) {
fprintf(stderr,
"[REL_STAGE3_RESET] class=%d ss=%p from_lru=%d cap=%d\n",
class_idx,
(void*)ss,
from_lru,
cap);
}
}
// ============================================================================
// Performance Measurement: Shared Pool Lock Contention (ENV-gated)
// ============================================================================
@ -208,10 +243,13 @@ sp_acquire_from_empty_scan(int class_idx, SuperSlab** ss_out, int* slab_idx_out,
return -1;
}
extern SuperSlab* g_super_reg_by_class[TINY_NUM_CLASSES][SUPER_REG_PER_CLASS];
extern int g_super_reg_class_size[TINY_NUM_CLASSES];
int reg_size = (class_idx < TINY_NUM_CLASSES) ? g_super_reg_class_size[class_idx] : 0;
int reg_cap = super_reg_effective_per_class();
if (reg_cap > 0 && reg_size > reg_cap) {
reg_size = reg_cap;
}
// Priority-2: Use cached ENV
int scan_limit = HAK_ENV_SS_EMPTY_SCAN_LIMIT();
if (scan_limit > reg_size) scan_limit = reg_size;
@ -229,7 +267,7 @@ sp_acquire_from_empty_scan(int class_idx, SuperSlab** ss_out, int* slab_idx_out,
int primary_slab_idx = -1;
for (int i = 0; i < scan_limit; i++) {
SuperSlab* ss = g_super_reg_by_class[class_idx][i];
SuperSlab* ss = super_reg_by_class_at(class_idx, i);
if (!(ss && ss->magic == SUPERSLAB_MAGIC)) continue;
// P-Tier: Skip DRAINING tier SuperSlabs
if (!ss_tier_is_hot(ss)) continue;
@ -769,6 +807,26 @@ stage2_scan:
1, memory_order_relaxed);
}
// bench プロファイルでは Shared Pool の論理上限を軽くかけておく
uint32_t total_limit = shared_pool_effective_total_slots();
if (total_limit > 0 && g_shared_pool.total_count >= total_limit) {
if (g_lock_stats_enabled == 1) {
atomic_fetch_add(&g_lock_release_count, 1);
}
pthread_mutex_unlock(&g_shared_pool.alloc_lock);
return -1;
}
uint32_t class_limit = shared_pool_effective_class_slots(class_idx);
if (class_limit > 0 &&
class_idx < TINY_NUM_CLASSES_SS &&
(uint32_t)g_shared_pool.class_active_slots[class_idx] >= class_limit) {
if (g_lock_stats_enabled == 1) {
atomic_fetch_add(&g_lock_release_count, 1);
}
pthread_mutex_unlock(&g_shared_pool.alloc_lock);
return -1;
}
// ========== Stage 3: Get new SuperSlab ==========
// Try LRU cache first, then mmap
SuperSlab* new_ss = NULL;
@ -786,6 +844,13 @@ stage2_scan:
// Stage 3b: If LRU miss, allocate new SuperSlab
if (!new_ss) {
if (!ss_budget_on_alloc(class_idx)) {
if (g_lock_stats_enabled == 1) {
atomic_fetch_add(&g_lock_release_count, 1);
}
pthread_mutex_unlock(&g_shared_pool.alloc_lock);
return -1;
}
// Release the alloc_lock to avoid deadlock with registry during superslab_allocate
if (g_lock_stats_enabled == 1) {
atomic_fetch_add(&g_lock_release_count, 1);
@ -834,27 +899,10 @@ stage2_scan:
g_shared_pool.total_count++;
}
// C7: LRU 再利用・新規確保いずれでも、空スラブに完全リセットしてから返す
if (class_idx == 7 && new_ss) {
int cap = ss_slabs_capacity(new_ss);
new_ss->slab_bitmap = 0;
new_ss->nonempty_mask = 0;
new_ss->freelist_mask = 0;
new_ss->empty_mask = 0;
new_ss->empty_count = 0;
new_ss->active_slabs = 0;
new_ss->hot_count = 0;
new_ss->cold_count = 0;
for (int s = 0; s < cap; s++) {
ss_slab_reset_meta_for_tiny(new_ss, s, class_idx);
}
static _Atomic uint32_t rel_stage3_reset_logs = 0;
uint32_t n = atomic_fetch_add_explicit(&rel_stage3_reset_logs, 1, memory_order_relaxed);
if (n < 4) {
fprintf(stderr,
"[REL_C7_STAGE3_RESET] ss=%p from_lru=%d cap=%d\n",
(void*)new_ss, from_lru, cap);
}
// Stage3 から返す前に、LRU 再利用分は必ず空スラブ化する。
// C7 以外でも from_lru の場合は全スラブをリセットしておく。
if (new_ss && (from_lru || class_idx == 7)) {
sp_reset_superslab_all_slabs(new_ss, class_idx, from_lru);
}
#if !HAKMEM_BUILD_RELEASE