tomoaki/hakmem
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add OBSERVE stats and auto tiny policy profile

Browse Source
This commit is contained in:
Moe Charm (CI)
2025-12-06 01:44:05 +09:00
parent 03538055ae
commit 18faa6a1c4
8 changed files with 182 additions and 144 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
bench_random_mixed.c
View File

@ -13,6 +13,7 @@
#include <stdint.h> #include <stdint.h>
#include <time.h> #include <time.h>
#include <string.h> #include <string.h>
#include <strings.h>
#include <stdatomic.h> #include <stdatomic.h>
#define C7_META_COUNTER_DEFINE #define C7_META_COUNTER_DEFINE
#include "core/box/c7_meta_used_counter_box.h" #include "core/box/c7_meta_used_counter_box.h"
@ -23,6 +24,8 @@
#include "hakmem.h" #include "hakmem.h"
#include "hakmem_build_flags.h" #include "hakmem_build_flags.h"
#include "core/box/c7_meta_used_counter_box.h" #include "core/box/c7_meta_used_counter_box.h"
#include "core/box/tiny_class_stats_box.h"
#include "core/box/tiny_class_policy_box.h"
// Box BenchMeta: Benchmark metadata management (bypass hakmem wrapper) // Box BenchMeta: Benchmark metadata management (bypass hakmem wrapper)
// Phase 15: Separate BenchMeta (slots array) from CoreAlloc (user workload) // Phase 15: Separate BenchMeta (slots array) from CoreAlloc (user workload)
@ -256,10 +259,26 @@ int main(int argc, char** argv){
tls_sll_print_measurements(); tls_sll_print_measurements();
shared_pool_print_measurements(); shared_pool_print_measurements();
// OBSERVE: per-class class stats (thread/global) for policy tuning
const char* stats_dump_env = getenv("HAKMEM_TINY_STATS_DUMP");
const char* policy_profile_env = getenv("HAKMEM_TINY_POLICY_PROFILE");
int policy_is_auto = (policy_profile_env &&
strcasecmp(policy_profile_env, "auto") == 0);
int dump_stats = (stats_dump_env && *stats_dump_env && *stats_dump_env != '0') || policy_is_auto;
if (dump_stats) {
tiny_class_stats_dump_thread(stderr, "[CLASS_STATS_THREAD]");
tiny_class_stats_dump_global(stderr, "[CLASS_STATS_GLOBAL]");
}
// Warm Pool Stats (ENV-gated: HAKMEM_WARM_POOL_STATS=1) // Warm Pool Stats (ENV-gated: HAKMEM_WARM_POOL_STATS=1)
extern void tiny_warm_pool_print_stats_public(void); extern void tiny_warm_pool_print_stats_public(void);
tiny_warm_pool_print_stats_public(); tiny_warm_pool_print_stats_public();
if (policy_is_auto) {
tiny_class_policy_refresh_auto();
tiny_class_policy_dump("[POLICY_AUTO]");
}
#if HAKMEM_BUILD_RELEASE #if HAKMEM_BUILD_RELEASE
// Minimal Release-side telemetry to verify Warm path usage (C7-only) // Minimal Release-side telemetry to verify Warm path usage (C7-only)
extern _Atomic uint64_t g_rel_c7_warm_pop; extern _Atomic uint64_t g_rel_c7_warm_pop;

47
core/box/tiny_class_policy_box.c
View File

@ -5,10 +5,12 @@
#include <stdlib.h> #include <stdlib.h>
#include <string.h> #include <string.h>
#include <strings.h> #include <strings.h>
#include "tiny_policy_learner_box.h"
TinyClassPolicy g_tiny_class_policy[TINY_NUM_CLASSES]; TinyClassPolicy g_tiny_class_policy[TINY_NUM_CLASSES];
static _Atomic int g_tiny_class_policy_init_done = 0; static _Atomic int g_tiny_class_policy_init_done = 0;
static _Atomic int g_tiny_class_policy_logged = 0; static _Atomic int g_tiny_class_policy_logged = 0;
static _Atomic int g_tiny_class_policy_profile_auto = 0;
static inline TinyClassPolicy tiny_class_policy_default_entry(void) { static inline TinyClassPolicy tiny_class_policy_default_entry(void) {
TinyClassPolicy p = {0}; TinyClassPolicy p = {0};
@ -51,23 +53,50 @@ static void tiny_class_policy_set_tinyplus_all(void) {
tiny_class_policy_set_legacy(); tiny_class_policy_set_legacy();
} }
static void tiny_class_policy_set_auto(void) {
// auto プロファイルは legacy をベースにして、後段の learner に委譲
tiny_class_policy_set_legacy();
atomic_store_explicit(&g_tiny_class_policy_profile_auto, 1, memory_order_release);
}
static const char* tiny_class_policy_set_profile(const char* profile) { static const char* tiny_class_policy_set_profile(const char* profile) {
if (profile == NULL || *profile == '\0' || strcasecmp(profile, "legacy") == 0) { if (profile == NULL || *profile == '\0' || strcasecmp(profile, "legacy") == 0) {
tiny_class_policy_set_legacy(); tiny_class_policy_set_legacy();
atomic_store_explicit(&g_tiny_class_policy_profile_auto, 0, memory_order_release);
return "legacy"; return "legacy";
} else if (strcasecmp(profile, "c5_7_only") == 0) { } else if (strcasecmp(profile, "c5_7_only") == 0) {
tiny_class_policy_set_c5_7_only(); tiny_class_policy_set_c5_7_only();
atomic_store_explicit(&g_tiny_class_policy_profile_auto, 0, memory_order_release);
return "c5_7_only"; return "c5_7_only";
} else if (strcasecmp(profile, "tinyplus_all") == 0) { } else if (strcasecmp(profile, "tinyplus_all") == 0) {
tiny_class_policy_set_tinyplus_all(); tiny_class_policy_set_tinyplus_all();
atomic_store_explicit(&g_tiny_class_policy_profile_auto, 0, memory_order_release);
return "tinyplus_all"; return "tinyplus_all";
} else if (strcasecmp(profile, "auto") == 0) {
tiny_class_policy_set_auto();
return "auto";
} else { } else {
// 不明な値は安全側で legacy にフォールバック。 // 不明な値は安全側で legacy にフォールバック。
tiny_class_policy_set_legacy(); tiny_class_policy_set_legacy();
atomic_store_explicit(&g_tiny_class_policy_profile_auto, 0, memory_order_release);
return "legacy"; return "legacy";
} }
} }
void tiny_class_policy_dump(const char* tag) {
const char* header = tag ? tag : "[POLICY_DUMP]";
fprintf(stderr, "%s\n", header);
for (int cls = 0; cls < TINY_NUM_CLASSES; cls++) {
TinyClassPolicy* p = &g_tiny_class_policy[cls];
fprintf(stderr,
" C%d: page=%u warm=%u cap=%u\n",
cls,
p->page_box_enabled,
p->warm_enabled,
p->warm_cap);
}
}
void tiny_class_policy_init_once(void) { void tiny_class_policy_init_once(void) {
if (atomic_load_explicit(&g_tiny_class_policy_init_done, memory_order_acquire)) { if (atomic_load_explicit(&g_tiny_class_policy_init_done, memory_order_acquire)) {
return; return;
@ -79,16 +108,16 @@ void tiny_class_policy_init_once(void) {
// 1-shot ダンプでポリシーの内容を可視化（デバッグ用） // 1-shot ダンプでポリシーの内容を可視化（デバッグ用）
if (atomic_exchange_explicit(&g_tiny_class_policy_logged, 1, memory_order_acq_rel) == 0) { if (atomic_exchange_explicit(&g_tiny_class_policy_logged, 1, memory_order_acq_rel) == 0) {
fprintf(stderr, "[POLICY_INIT] profile=%s\n", active_profile); fprintf(stderr, "[POLICY_INIT] profile=%s\n", active_profile);
for (int cls = 0; cls < TINY_NUM_CLASSES; cls++) { tiny_class_policy_dump(NULL);
TinyClassPolicy* p = &g_tiny_class_policy[cls];
fprintf(stderr,
" C%d: page=%u warm=%u cap=%u\n",
cls,
p->page_box_enabled,
p->warm_enabled,
p->warm_cap);
}
} }
atomic_store_explicit(&g_tiny_class_policy_init_done, 1, memory_order_release); atomic_store_explicit(&g_tiny_class_policy_init_done, 1, memory_order_release);
} }
void tiny_class_policy_refresh_auto(void) {
tiny_class_policy_init_once();
if (!atomic_load_explicit(&g_tiny_class_policy_profile_auto, memory_order_acquire)) {
return;
}
tiny_policy_learner_tick();
}

6
core/box/tiny_class_policy_box.h
View File

@ -29,6 +29,12 @@ extern TinyClassPolicy g_tiny_class_policy[TINY_NUM_CLASSES];
// Initialize policy table once (idempotent). // Initialize policy table once (idempotent).
void tiny_class_policy_init_once(void); void tiny_class_policy_init_once(void);
// Refresh auto profile based on learner output (no-op for non-auto profiles)
void tiny_class_policy_refresh_auto(void);
// Debug helper: dump current policy (tag optional)
void tiny_class_policy_dump(const char* tag);
// Lightweight accessor for hot paths. // Lightweight accessor for hot paths.
static inline const TinyClassPolicy* tiny_policy_get(int class_idx) { static inline const TinyClassPolicy* tiny_policy_get(int class_idx) {
if (class_idx < 0 || class_idx >= TINY_NUM_CLASSES) { if (class_idx < 0 || class_idx >= TINY_NUM_CLASSES) {

49
core/box/tiny_class_stats_box.c
View File

@ -1,10 +1,59 @@
// tiny_class_stats_box.c - Thread-local stats storage for Tiny classes // tiny_class_stats_box.c - Thread-local stats storage for Tiny classes
#include "tiny_class_stats_box.h" #include "tiny_class_stats_box.h"
#include <stdio.h>
#include <string.h> #include <string.h>
__thread TinyClassStatsThread g_tiny_class_stats = {0}; __thread TinyClassStatsThread g_tiny_class_stats = {0};
_Atomic uint64_t g_tiny_class_stats_uc_miss_global[TINY_NUM_CLASSES] = {0};
_Atomic uint64_t g_tiny_class_stats_warm_hit_global[TINY_NUM_CLASSES] = {0};
_Atomic uint64_t g_tiny_class_stats_shared_lock_global[TINY_NUM_CLASSES] = {0};
void tiny_class_stats_reset_thread(void) { void tiny_class_stats_reset_thread(void) {
memset(&g_tiny_class_stats, 0, sizeof(g_tiny_class_stats)); memset(&g_tiny_class_stats, 0, sizeof(g_tiny_class_stats));
} }
void tiny_class_stats_snapshot_thread(TinyClassStatsThread* out) {
if (!out) return;
memcpy(out, &g_tiny_class_stats, sizeof(*out));
}
void tiny_class_stats_snapshot_global(TinyClassStatsThread* out) {
if (!out) return;
for (int i = 0; i < TINY_NUM_CLASSES; i++) {
out->uc_miss[i] = atomic_load_explicit(&g_tiny_class_stats_uc_miss_global[i],
memory_order_relaxed);
out->warm_hit[i] = atomic_load_explicit(&g_tiny_class_stats_warm_hit_global[i],
memory_order_relaxed);
out->shared_lock[i] = atomic_load_explicit(&g_tiny_class_stats_shared_lock_global[i],
memory_order_relaxed);
}
}
static void tiny_class_stats_dump_common(FILE* out,
const char* tag,
const TinyClassStatsThread* stats) {
if (!(out && stats)) return;
fprintf(out, "%s class uc_miss warm_hit shared_lock\n", tag ? tag : "[STATS]");
for (int c = 0; c < TINY_NUM_CLASSES; c++) {
if (stats->uc_miss[c] || stats->warm_hit[c] || stats->shared_lock[c]) {
fprintf(out, " C%d: %llu %llu %llu\n",
c,
(unsigned long long)stats->uc_miss[c],
(unsigned long long)stats->warm_hit[c],
(unsigned long long)stats->shared_lock[c]);
}
}
}
void tiny_class_stats_dump_thread(FILE* out, const char* tag) {
TinyClassStatsThread snap = {0};
tiny_class_stats_snapshot_thread(&snap);
tiny_class_stats_dump_common(out, tag ? tag : "[THREAD_STATS]", &snap);
}
void tiny_class_stats_dump_global(FILE* out, const char* tag) {
TinyClassStatsThread snap = {0};
tiny_class_stats_snapshot_global(&snap);
tiny_class_stats_dump_common(out, tag ? tag : "[GLOBAL_STATS]", &snap);
}

21
core/box/tiny_class_stats_box.h
View File

@ -8,6 +8,8 @@
#define TINY_CLASS_STATS_BOX_H #define TINY_CLASS_STATS_BOX_H
#include <stdint.h> #include <stdint.h>
#include <stdatomic.h>
#include <stdio.h>
#include "../hakmem_tiny_config.h" #include "../hakmem_tiny_config.h"
typedef struct TinyClassStatsThread { typedef struct TinyClassStatsThread {
@ -18,25 +20,44 @@ typedef struct TinyClassStatsThread {
extern __thread TinyClassStatsThread g_tiny_class_stats; extern __thread TinyClassStatsThread g_tiny_class_stats;
// Global (cross-thread) aggregates for OBSERVE/LEARN
extern _Atomic uint64_t g_tiny_class_stats_uc_miss_global[TINY_NUM_CLASSES];
extern _Atomic uint64_t g_tiny_class_stats_warm_hit_global[TINY_NUM_CLASSES];
extern _Atomic uint64_t g_tiny_class_stats_shared_lock_global[TINY_NUM_CLASSES];
static inline void tiny_class_stats_on_uc_miss(int ci) { static inline void tiny_class_stats_on_uc_miss(int ci) {
if (ci >= 0 && ci < TINY_NUM_CLASSES) { if (ci >= 0 && ci < TINY_NUM_CLASSES) {
g_tiny_class_stats.uc_miss[ci]++; g_tiny_class_stats.uc_miss[ci]++;
atomic_fetch_add_explicit(&g_tiny_class_stats_uc_miss_global[ci],
1, memory_order_relaxed);
} }
} }
static inline void tiny_class_stats_on_warm_hit(int ci) { static inline void tiny_class_stats_on_warm_hit(int ci) {
if (ci >= 0 && ci < TINY_NUM_CLASSES) { if (ci >= 0 && ci < TINY_NUM_CLASSES) {
g_tiny_class_stats.warm_hit[ci]++; g_tiny_class_stats.warm_hit[ci]++;
atomic_fetch_add_explicit(&g_tiny_class_stats_warm_hit_global[ci],
1, memory_order_relaxed);
} }
} }
static inline void tiny_class_stats_on_shared_lock(int ci) { static inline void tiny_class_stats_on_shared_lock(int ci) {
if (ci >= 0 && ci < TINY_NUM_CLASSES) { if (ci >= 0 && ci < TINY_NUM_CLASSES) {
g_tiny_class_stats.shared_lock[ci]++; g_tiny_class_stats.shared_lock[ci]++;
atomic_fetch_add_explicit(&g_tiny_class_stats_shared_lock_global[ci],
1, memory_order_relaxed);
} }
} }
// Optional: reset per-thread counters (cold path only). // Optional: reset per-thread counters (cold path only).
void tiny_class_stats_reset_thread(void); void tiny_class_stats_reset_thread(void);
// Snapshot helpers (cold path): copy current counters into caller-provided struct.
void tiny_class_stats_snapshot_thread(TinyClassStatsThread* out);
void tiny_class_stats_snapshot_global(TinyClassStatsThread* out);
// Simple stderr dump helpers (cold path)
void tiny_class_stats_dump_thread(FILE* out, const char* tag);
void tiny_class_stats_dump_global(FILE* out, const char* tag);
#endif // TINY_CLASS_STATS_BOX_H #endif // TINY_CLASS_STATS_BOX_H

37
core/box/tiny_policy_learner_box.c
View File

@ -1,7 +1,42 @@
// tiny_policy_learner_box.c - Placeholder learner hook // tiny_policy_learner_box.c - Placeholder learner hook
#include "tiny_policy_learner_box.h" #include "tiny_policy_learner_box.h"
#include "tiny_class_policy_box.h"
#include "tiny_class_stats_box.h"
#include <stdint.h>
// Simple OBSERVE/LEARN rule:
// - Choose top-2 classes by shared_pool_lock and enable Page Box for them.
// - Always keep existing warm_enabled / warm_cap (policy table is already seeded).
void tiny_policy_learner_tick(void) { void tiny_policy_learner_tick(void) {
// FROZEN/OBSERVE: intentionally empty. TinyClassStatsThread snap = {0};
tiny_class_stats_snapshot_global(&snap);
int top1 = -1, top2 = -1;
uint64_t v1 = 0, v2 = 0;
for (int i = 0; i < TINY_NUM_CLASSES; i++) {
uint64_t v = snap.shared_lock[i];
if (v > v1) {
top2 = top1;
v2 = v1;
top1 = i;
v1 = v;
} else if (v > v2) {
top2 = i;
v2 = v;
}
}
// Nothing observed yet → leave policy untouched
if (v1 == 0) {
return;
}
for (int c = 0; c < TINY_NUM_CLASSES; c++) {
TinyClassPolicy* p = &g_tiny_class_policy[c];
if (c == top1 || c == top2) {
p->page_box_enabled = 1;
p->warm_enabled = 1;
}
}
} }

127
core/front/tiny_unified_cache.d
View File

@ -1,127 +0,0 @@
core/front/tiny_unified_cache.o: core/front/tiny_unified_cache.c \
core/front/tiny_unified_cache.h core/front/../hakmem_build_flags.h \
core/front/../hakmem_tiny_config.h core/front/../box/ptr_type_box.h \
core/front/../box/tiny_front_config_box.h \
core/front/../box/../hakmem_build_flags.h core/front/tiny_warm_pool.h \
core/front/../superslab/superslab_types.h \
core/hakmem_tiny_superslab_constants.h core/front/../tiny_tls.h \
core/front/../hakmem_tiny_superslab.h \
core/front/../superslab/superslab_types.h \
core/front/../superslab/superslab_inline.h \
core/front/../superslab/superslab_types.h \
core/front/../superslab/../tiny_box_geometry.h \
core/front/../superslab/../hakmem_tiny_superslab_constants.h \
core/front/../superslab/../hakmem_tiny_config.h \
core/front/../tiny_debug_ring.h core/front/../hakmem_build_flags.h \
core/front/../tiny_remote.h \
core/front/../hakmem_tiny_superslab_constants.h \
core/front/../tiny_box_geometry.h core/front/../box/tiny_next_ptr_box.h \
core/hakmem_tiny_config.h core/tiny_nextptr.h core/hakmem_build_flags.h \
core/tiny_region_id.h core/tiny_box_geometry.h core/ptr_track.h \
core/hakmem_super_registry.h core/hakmem_tiny_superslab.h \
core/box/ss_addr_map_box.h core/box/../hakmem_build_flags.h \
core/superslab/superslab_inline.h core/hakmem_tiny.h core/hakmem_trace.h \
core/hakmem_tiny_mini_mag.h core/box/hak_lane_classify.inc.h \
core/box/ptr_type_box.h core/tiny_debug_api.h core/box/tiny_layout_box.h \
core/box/../hakmem_tiny_config.h core/box/tiny_header_box.h \
core/box/tiny_layout_box.h core/box/../tiny_region_id.h \
core/front/../hakmem_tiny_superslab.h \
core/front/../superslab/superslab_inline.h \
core/front/../box/pagefault_telemetry_box.h \
core/front/../box/ss_tier_box.h \
core/front/../box/../superslab/superslab_types.h \
core/front/../box/ss_slab_meta_box.h \
core/front/../box/slab_freelist_atomic.h \
core/front/../box/warm_pool_stats_box.h \
core/front/../box/../hakmem_tiny_config.h \
core/front/../box/../front/tiny_warm_pool.h \
core/front/../box/slab_carve_box.h \
core/front/../box/../hakmem_tiny_superslab.h \
core/front/../box/../superslab/superslab_inline.h \
core/front/../box/../tiny_box_geometry.h \
core/front/../box/../box/pagefault_telemetry_box.h \
core/front/../box/c7_meta_used_counter_box.h \
core/front/../box/warm_pool_rel_counters_box.h \
core/front/../box/warm_pool_prefill_box.h \
core/front/../box/../tiny_tls.h \
core/front/../box/../box/warm_pool_stats_box.h \
core/front/../hakmem_env_cache.h core/front/../box/tiny_page_box.h \
core/front/../box/ss_tls_bind_box.h \
core/front/../box/../box/tiny_page_box.h \
core/front/../box/tiny_tls_carve_one_block_box.h \
core/front/../box/../tiny_debug_api.h \
core/front/../box/warm_tls_bind_logger_box.h \
core/front/../box/warm_pool_dbg_box.h
core/front/tiny_unified_cache.h:
core/front/../hakmem_build_flags.h:
core/front/../hakmem_tiny_config.h:
core/front/../box/ptr_type_box.h:
core/front/../box/tiny_front_config_box.h:
core/front/../box/../hakmem_build_flags.h:
core/front/tiny_warm_pool.h:
core/front/../superslab/superslab_types.h:
core/hakmem_tiny_superslab_constants.h:
core/front/../tiny_tls.h:
core/front/../hakmem_tiny_superslab.h:
core/front/../superslab/superslab_types.h:
core/front/../superslab/superslab_inline.h:
core/front/../superslab/superslab_types.h:
core/front/../superslab/../tiny_box_geometry.h:
core/front/../superslab/../hakmem_tiny_superslab_constants.h:
core/front/../superslab/../hakmem_tiny_config.h:
core/front/../tiny_debug_ring.h:
core/front/../hakmem_build_flags.h:
core/front/../tiny_remote.h:
core/front/../hakmem_tiny_superslab_constants.h:
core/front/../tiny_box_geometry.h:
core/front/../box/tiny_next_ptr_box.h:
core/hakmem_tiny_config.h:
core/tiny_nextptr.h:
core/hakmem_build_flags.h:
core/tiny_region_id.h:
core/tiny_box_geometry.h:
core/ptr_track.h:
core/hakmem_super_registry.h:
core/hakmem_tiny_superslab.h:
core/box/ss_addr_map_box.h:
core/box/../hakmem_build_flags.h:
core/superslab/superslab_inline.h:
core/hakmem_tiny.h:
core/hakmem_trace.h:
core/hakmem_tiny_mini_mag.h:
core/box/hak_lane_classify.inc.h:
core/box/ptr_type_box.h:
core/tiny_debug_api.h:
core/box/tiny_layout_box.h:
core/box/../hakmem_tiny_config.h:
core/box/tiny_header_box.h:
core/box/tiny_layout_box.h:
core/box/../tiny_region_id.h:
core/front/../hakmem_tiny_superslab.h:
core/front/../superslab/superslab_inline.h:
core/front/../box/pagefault_telemetry_box.h:
core/front/../box/ss_tier_box.h:
core/front/../box/../superslab/superslab_types.h:
core/front/../box/ss_slab_meta_box.h:
core/front/../box/slab_freelist_atomic.h:
core/front/../box/warm_pool_stats_box.h:
core/front/../box/../hakmem_tiny_config.h:
core/front/../box/../front/tiny_warm_pool.h:
core/front/../box/slab_carve_box.h:
core/front/../box/../hakmem_tiny_superslab.h:
core/front/../box/../superslab/superslab_inline.h:
core/front/../box/../tiny_box_geometry.h:
core/front/../box/../box/pagefault_telemetry_box.h:
core/front/../box/c7_meta_used_counter_box.h:
core/front/../box/warm_pool_rel_counters_box.h:
core/front/../box/warm_pool_prefill_box.h:
core/front/../box/../tiny_tls.h:
core/front/../box/../box/warm_pool_stats_box.h:
core/front/../hakmem_env_cache.h:
core/front/../box/tiny_page_box.h:
core/front/../box/ss_tls_bind_box.h:
core/front/../box/../box/tiny_page_box.h:
core/front/../box/tiny_tls_carve_one_block_box.h:
core/front/../box/../tiny_debug_api.h:
core/front/../box/warm_tls_bind_logger_box.h:
core/front/../box/warm_pool_dbg_box.h:

20
core/hakmem_shared_pool_acquire.c
View File

@ -11,6 +11,7 @@
#include "hakmem_env_cache.h" // Priority-2: ENV cache #include "hakmem_env_cache.h" // Priority-2: ENV cache
#include "front/tiny_warm_pool.h" // Warm Pool: Prefill during registry scans #include "front/tiny_warm_pool.h" // Warm Pool: Prefill during registry scans
#include "box/ss_slab_reset_box.h" // Box: Reset slab metadata on reuse (C7 guard) #include "box/ss_slab_reset_box.h" // Box: Reset slab metadata on reuse (C7 guard)
#include "box/tiny_class_stats_box.h" // OBSERVE: per-class shared lock stats
#include <stdint.h> #include <stdint.h>
#include <stdlib.h> #include <stdlib.h>
@ -20,6 +21,11 @@
// Stage3(LRU) 由来の Superslab をトレースするための簡易マジック // Stage3(LRU) 由来の Superslab をトレースするための簡易マジック
_Atomic uintptr_t g_c7_stage3_magic_ss = 0; _Atomic uintptr_t g_c7_stage3_magic_ss = 0;
static inline void sp_lock_with_stats(int class_idx) {
tiny_class_stats_on_shared_lock(class_idx);
sp_lock_with_stats(class_idx);
}
static inline void c7_log_meta_state(const char* tag, SuperSlab* ss, int slab_idx) { static inline void c7_log_meta_state(const char* tag, SuperSlab* ss, int slab_idx) {
if (!ss) return; if (!ss) return;
#if HAKMEM_BUILD_RELEASE #if HAKMEM_BUILD_RELEASE
@ -351,7 +357,7 @@ stage1_retry_after_tension_drain:
atomic_fetch_add(&g_lock_acquire_slab_count, 1); atomic_fetch_add(&g_lock_acquire_slab_count, 1);
} }
pthread_mutex_lock(&g_shared_pool.alloc_lock); sp_lock_with_stats(class_idx);
// P0.3: Guard against TLS SLL orphaned pointers before reusing slab // P0.3: Guard against TLS SLL orphaned pointers before reusing slab
// RACE FIX: Load SuperSlab pointer atomically BEFORE guard (consistency) // RACE FIX: Load SuperSlab pointer atomically BEFORE guard (consistency)
@ -498,7 +504,7 @@ stage2_fallback:
atomic_fetch_add(&g_lock_acquire_slab_count, 1); atomic_fetch_add(&g_lock_acquire_slab_count, 1);
} }
pthread_mutex_lock(&g_shared_pool.alloc_lock); sp_lock_with_stats(class_idx);
// Performance measurement: count Stage 2 lock acquisitions // Performance measurement: count Stage 2 lock acquisitions
if (__builtin_expect(sp_measure_enabled(), 0)) { if (__builtin_expect(sp_measure_enabled(), 0)) {
@ -626,11 +632,11 @@ stage2_scan:
// P0 instrumentation: count lock acquisitions // P0 instrumentation: count lock acquisitions
lock_stats_init(); lock_stats_init();
if (g_lock_stats_enabled == 1) { if (g_lock_stats_enabled == 1) {
atomic_fetch_add(&g_lock_acquire_count, 1); atomic_fetch_add(&g_lock_acquire_count, 1);
atomic_fetch_add(&g_lock_acquire_slab_count, 1); atomic_fetch_add(&g_lock_acquire_slab_count, 1);
} }
pthread_mutex_lock(&g_shared_pool.alloc_lock); sp_lock_with_stats(class_idx);
// Performance measurement: count Stage 2 scan lock acquisitions // Performance measurement: count Stage 2 scan lock acquisitions
if (__builtin_expect(sp_measure_enabled(), 0)) { if (__builtin_expect(sp_measure_enabled(), 0)) {
@ -793,7 +799,7 @@ stage2_scan:
atomic_fetch_add(&g_lock_acquire_count, 1); atomic_fetch_add(&g_lock_acquire_count, 1);
atomic_fetch_add(&g_lock_acquire_slab_count, 1); // This is part of acquisition path atomic_fetch_add(&g_lock_acquire_slab_count, 1); // This is part of acquisition path
} }
pthread_mutex_lock(&g_shared_pool.alloc_lock); sp_lock_with_stats(class_idx);
if (!allocated_ss) { if (!allocated_ss) {
// Allocation failed; return now. // Allocation failed; return now.