diff --git a/bench_random_mixed.c b/bench_random_mixed.c index 319122be..812ee33f 100644 --- a/bench_random_mixed.c +++ b/bench_random_mixed.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #define C7_META_COUNTER_DEFINE #include "core/box/c7_meta_used_counter_box.h" @@ -23,6 +24,8 @@ #include "hakmem.h" #include "hakmem_build_flags.h" #include "core/box/c7_meta_used_counter_box.h" +#include "core/box/tiny_class_stats_box.h" +#include "core/box/tiny_class_policy_box.h" // Box BenchMeta: Benchmark metadata management (bypass hakmem wrapper) // Phase 15: Separate BenchMeta (slots array) from CoreAlloc (user workload) @@ -256,10 +259,26 @@ int main(int argc, char** argv){ tls_sll_print_measurements(); shared_pool_print_measurements(); + // OBSERVE: per-class class stats (thread/global) for policy tuning + const char* stats_dump_env = getenv("HAKMEM_TINY_STATS_DUMP"); + const char* policy_profile_env = getenv("HAKMEM_TINY_POLICY_PROFILE"); + int policy_is_auto = (policy_profile_env && + strcasecmp(policy_profile_env, "auto") == 0); + int dump_stats = (stats_dump_env && *stats_dump_env && *stats_dump_env != '0') || policy_is_auto; + if (dump_stats) { + tiny_class_stats_dump_thread(stderr, "[CLASS_STATS_THREAD]"); + tiny_class_stats_dump_global(stderr, "[CLASS_STATS_GLOBAL]"); + } + // Warm Pool Stats (ENV-gated: HAKMEM_WARM_POOL_STATS=1) extern void tiny_warm_pool_print_stats_public(void); tiny_warm_pool_print_stats_public(); + if (policy_is_auto) { + tiny_class_policy_refresh_auto(); + tiny_class_policy_dump("[POLICY_AUTO]"); + } + #if HAKMEM_BUILD_RELEASE // Minimal Release-side telemetry to verify Warm path usage (C7-only) extern _Atomic uint64_t g_rel_c7_warm_pop; diff --git a/core/box/tiny_class_policy_box.c b/core/box/tiny_class_policy_box.c index 01939997..17246df0 100644 --- a/core/box/tiny_class_policy_box.c +++ b/core/box/tiny_class_policy_box.c @@ -5,10 +5,12 @@ #include #include #include +#include "tiny_policy_learner_box.h" TinyClassPolicy g_tiny_class_policy[TINY_NUM_CLASSES]; static _Atomic int g_tiny_class_policy_init_done = 0; static _Atomic int g_tiny_class_policy_logged = 0; +static _Atomic int g_tiny_class_policy_profile_auto = 0; static inline TinyClassPolicy tiny_class_policy_default_entry(void) { TinyClassPolicy p = {0}; @@ -51,23 +53,50 @@ static void tiny_class_policy_set_tinyplus_all(void) { tiny_class_policy_set_legacy(); } +static void tiny_class_policy_set_auto(void) { + // auto プロファイルは legacy をベースにして、後段の learner に委譲 + tiny_class_policy_set_legacy(); + atomic_store_explicit(&g_tiny_class_policy_profile_auto, 1, memory_order_release); +} + static const char* tiny_class_policy_set_profile(const char* profile) { if (profile == NULL || *profile == '\0' || strcasecmp(profile, "legacy") == 0) { tiny_class_policy_set_legacy(); + atomic_store_explicit(&g_tiny_class_policy_profile_auto, 0, memory_order_release); return "legacy"; } else if (strcasecmp(profile, "c5_7_only") == 0) { tiny_class_policy_set_c5_7_only(); + atomic_store_explicit(&g_tiny_class_policy_profile_auto, 0, memory_order_release); return "c5_7_only"; } else if (strcasecmp(profile, "tinyplus_all") == 0) { tiny_class_policy_set_tinyplus_all(); + atomic_store_explicit(&g_tiny_class_policy_profile_auto, 0, memory_order_release); return "tinyplus_all"; + } else if (strcasecmp(profile, "auto") == 0) { + tiny_class_policy_set_auto(); + return "auto"; } else { // 不明な値は安全側で legacy にフォールバック。 tiny_class_policy_set_legacy(); + atomic_store_explicit(&g_tiny_class_policy_profile_auto, 0, memory_order_release); return "legacy"; } } +void tiny_class_policy_dump(const char* tag) { + const char* header = tag ? tag : "[POLICY_DUMP]"; + fprintf(stderr, "%s\n", header); + for (int cls = 0; cls < TINY_NUM_CLASSES; cls++) { + TinyClassPolicy* p = &g_tiny_class_policy[cls]; + fprintf(stderr, + " C%d: page=%u warm=%u cap=%u\n", + cls, + p->page_box_enabled, + p->warm_enabled, + p->warm_cap); + } +} + void tiny_class_policy_init_once(void) { if (atomic_load_explicit(&g_tiny_class_policy_init_done, memory_order_acquire)) { return; @@ -79,16 +108,16 @@ void tiny_class_policy_init_once(void) { // 1-shot ダンプでポリシーの内容を可視化(デバッグ用) if (atomic_exchange_explicit(&g_tiny_class_policy_logged, 1, memory_order_acq_rel) == 0) { fprintf(stderr, "[POLICY_INIT] profile=%s\n", active_profile); - for (int cls = 0; cls < TINY_NUM_CLASSES; cls++) { - TinyClassPolicy* p = &g_tiny_class_policy[cls]; - fprintf(stderr, - " C%d: page=%u warm=%u cap=%u\n", - cls, - p->page_box_enabled, - p->warm_enabled, - p->warm_cap); - } + tiny_class_policy_dump(NULL); } atomic_store_explicit(&g_tiny_class_policy_init_done, 1, memory_order_release); } + +void tiny_class_policy_refresh_auto(void) { + tiny_class_policy_init_once(); + if (!atomic_load_explicit(&g_tiny_class_policy_profile_auto, memory_order_acquire)) { + return; + } + tiny_policy_learner_tick(); +} diff --git a/core/box/tiny_class_policy_box.h b/core/box/tiny_class_policy_box.h index 292cd81a..16618bbd 100644 --- a/core/box/tiny_class_policy_box.h +++ b/core/box/tiny_class_policy_box.h @@ -29,6 +29,12 @@ extern TinyClassPolicy g_tiny_class_policy[TINY_NUM_CLASSES]; // Initialize policy table once (idempotent). void tiny_class_policy_init_once(void); +// Refresh auto profile based on learner output (no-op for non-auto profiles) +void tiny_class_policy_refresh_auto(void); + +// Debug helper: dump current policy (tag optional) +void tiny_class_policy_dump(const char* tag); + // Lightweight accessor for hot paths. static inline const TinyClassPolicy* tiny_policy_get(int class_idx) { if (class_idx < 0 || class_idx >= TINY_NUM_CLASSES) { diff --git a/core/box/tiny_class_stats_box.c b/core/box/tiny_class_stats_box.c index 3ebc5de8..7da00796 100644 --- a/core/box/tiny_class_stats_box.c +++ b/core/box/tiny_class_stats_box.c @@ -1,10 +1,59 @@ // tiny_class_stats_box.c - Thread-local stats storage for Tiny classes #include "tiny_class_stats_box.h" +#include #include __thread TinyClassStatsThread g_tiny_class_stats = {0}; +_Atomic uint64_t g_tiny_class_stats_uc_miss_global[TINY_NUM_CLASSES] = {0}; +_Atomic uint64_t g_tiny_class_stats_warm_hit_global[TINY_NUM_CLASSES] = {0}; +_Atomic uint64_t g_tiny_class_stats_shared_lock_global[TINY_NUM_CLASSES] = {0}; void tiny_class_stats_reset_thread(void) { memset(&g_tiny_class_stats, 0, sizeof(g_tiny_class_stats)); } + +void tiny_class_stats_snapshot_thread(TinyClassStatsThread* out) { + if (!out) return; + memcpy(out, &g_tiny_class_stats, sizeof(*out)); +} + +void tiny_class_stats_snapshot_global(TinyClassStatsThread* out) { + if (!out) return; + for (int i = 0; i < TINY_NUM_CLASSES; i++) { + out->uc_miss[i] = atomic_load_explicit(&g_tiny_class_stats_uc_miss_global[i], + memory_order_relaxed); + out->warm_hit[i] = atomic_load_explicit(&g_tiny_class_stats_warm_hit_global[i], + memory_order_relaxed); + out->shared_lock[i] = atomic_load_explicit(&g_tiny_class_stats_shared_lock_global[i], + memory_order_relaxed); + } +} + +static void tiny_class_stats_dump_common(FILE* out, + const char* tag, + const TinyClassStatsThread* stats) { + if (!(out && stats)) return; + fprintf(out, "%s class uc_miss warm_hit shared_lock\n", tag ? tag : "[STATS]"); + for (int c = 0; c < TINY_NUM_CLASSES; c++) { + if (stats->uc_miss[c] || stats->warm_hit[c] || stats->shared_lock[c]) { + fprintf(out, " C%d: %llu %llu %llu\n", + c, + (unsigned long long)stats->uc_miss[c], + (unsigned long long)stats->warm_hit[c], + (unsigned long long)stats->shared_lock[c]); + } + } +} + +void tiny_class_stats_dump_thread(FILE* out, const char* tag) { + TinyClassStatsThread snap = {0}; + tiny_class_stats_snapshot_thread(&snap); + tiny_class_stats_dump_common(out, tag ? tag : "[THREAD_STATS]", &snap); +} + +void tiny_class_stats_dump_global(FILE* out, const char* tag) { + TinyClassStatsThread snap = {0}; + tiny_class_stats_snapshot_global(&snap); + tiny_class_stats_dump_common(out, tag ? tag : "[GLOBAL_STATS]", &snap); +} diff --git a/core/box/tiny_class_stats_box.h b/core/box/tiny_class_stats_box.h index 0f74c3ef..91e37205 100644 --- a/core/box/tiny_class_stats_box.h +++ b/core/box/tiny_class_stats_box.h @@ -8,6 +8,8 @@ #define TINY_CLASS_STATS_BOX_H #include +#include +#include #include "../hakmem_tiny_config.h" typedef struct TinyClassStatsThread { @@ -18,25 +20,44 @@ typedef struct TinyClassStatsThread { extern __thread TinyClassStatsThread g_tiny_class_stats; +// Global (cross-thread) aggregates for OBSERVE/LEARN +extern _Atomic uint64_t g_tiny_class_stats_uc_miss_global[TINY_NUM_CLASSES]; +extern _Atomic uint64_t g_tiny_class_stats_warm_hit_global[TINY_NUM_CLASSES]; +extern _Atomic uint64_t g_tiny_class_stats_shared_lock_global[TINY_NUM_CLASSES]; + static inline void tiny_class_stats_on_uc_miss(int ci) { if (ci >= 0 && ci < TINY_NUM_CLASSES) { g_tiny_class_stats.uc_miss[ci]++; + atomic_fetch_add_explicit(&g_tiny_class_stats_uc_miss_global[ci], + 1, memory_order_relaxed); } } static inline void tiny_class_stats_on_warm_hit(int ci) { if (ci >= 0 && ci < TINY_NUM_CLASSES) { g_tiny_class_stats.warm_hit[ci]++; + atomic_fetch_add_explicit(&g_tiny_class_stats_warm_hit_global[ci], + 1, memory_order_relaxed); } } static inline void tiny_class_stats_on_shared_lock(int ci) { if (ci >= 0 && ci < TINY_NUM_CLASSES) { g_tiny_class_stats.shared_lock[ci]++; + atomic_fetch_add_explicit(&g_tiny_class_stats_shared_lock_global[ci], + 1, memory_order_relaxed); } } // Optional: reset per-thread counters (cold path only). void tiny_class_stats_reset_thread(void); +// Snapshot helpers (cold path): copy current counters into caller-provided struct. +void tiny_class_stats_snapshot_thread(TinyClassStatsThread* out); +void tiny_class_stats_snapshot_global(TinyClassStatsThread* out); + +// Simple stderr dump helpers (cold path) +void tiny_class_stats_dump_thread(FILE* out, const char* tag); +void tiny_class_stats_dump_global(FILE* out, const char* tag); + #endif // TINY_CLASS_STATS_BOX_H diff --git a/core/box/tiny_policy_learner_box.c b/core/box/tiny_policy_learner_box.c index 7c877081..0cc60b55 100644 --- a/core/box/tiny_policy_learner_box.c +++ b/core/box/tiny_policy_learner_box.c @@ -1,7 +1,42 @@ // tiny_policy_learner_box.c - Placeholder learner hook #include "tiny_policy_learner_box.h" +#include "tiny_class_policy_box.h" +#include "tiny_class_stats_box.h" +#include +// Simple OBSERVE/LEARN rule: +// - Choose top-2 classes by shared_pool_lock and enable Page Box for them. +// - Always keep existing warm_enabled / warm_cap (policy table is already seeded). void tiny_policy_learner_tick(void) { - // FROZEN/OBSERVE: intentionally empty. + TinyClassStatsThread snap = {0}; + tiny_class_stats_snapshot_global(&snap); + + int top1 = -1, top2 = -1; + uint64_t v1 = 0, v2 = 0; + for (int i = 0; i < TINY_NUM_CLASSES; i++) { + uint64_t v = snap.shared_lock[i]; + if (v > v1) { + top2 = top1; + v2 = v1; + top1 = i; + v1 = v; + } else if (v > v2) { + top2 = i; + v2 = v; + } + } + + // Nothing observed yet → leave policy untouched + if (v1 == 0) { + return; + } + + for (int c = 0; c < TINY_NUM_CLASSES; c++) { + TinyClassPolicy* p = &g_tiny_class_policy[c]; + if (c == top1 || c == top2) { + p->page_box_enabled = 1; + p->warm_enabled = 1; + } + } } diff --git a/core/front/tiny_unified_cache.d b/core/front/tiny_unified_cache.d deleted file mode 100644 index 6fa3ca67..00000000 --- a/core/front/tiny_unified_cache.d +++ /dev/null @@ -1,127 +0,0 @@ -core/front/tiny_unified_cache.o: core/front/tiny_unified_cache.c \ - core/front/tiny_unified_cache.h core/front/../hakmem_build_flags.h \ - core/front/../hakmem_tiny_config.h core/front/../box/ptr_type_box.h \ - core/front/../box/tiny_front_config_box.h \ - core/front/../box/../hakmem_build_flags.h core/front/tiny_warm_pool.h \ - core/front/../superslab/superslab_types.h \ - core/hakmem_tiny_superslab_constants.h core/front/../tiny_tls.h \ - core/front/../hakmem_tiny_superslab.h \ - core/front/../superslab/superslab_types.h \ - core/front/../superslab/superslab_inline.h \ - core/front/../superslab/superslab_types.h \ - core/front/../superslab/../tiny_box_geometry.h \ - core/front/../superslab/../hakmem_tiny_superslab_constants.h \ - core/front/../superslab/../hakmem_tiny_config.h \ - core/front/../tiny_debug_ring.h core/front/../hakmem_build_flags.h \ - core/front/../tiny_remote.h \ - core/front/../hakmem_tiny_superslab_constants.h \ - core/front/../tiny_box_geometry.h core/front/../box/tiny_next_ptr_box.h \ - core/hakmem_tiny_config.h core/tiny_nextptr.h core/hakmem_build_flags.h \ - core/tiny_region_id.h core/tiny_box_geometry.h core/ptr_track.h \ - core/hakmem_super_registry.h core/hakmem_tiny_superslab.h \ - core/box/ss_addr_map_box.h core/box/../hakmem_build_flags.h \ - core/superslab/superslab_inline.h core/hakmem_tiny.h core/hakmem_trace.h \ - core/hakmem_tiny_mini_mag.h core/box/hak_lane_classify.inc.h \ - core/box/ptr_type_box.h core/tiny_debug_api.h core/box/tiny_layout_box.h \ - core/box/../hakmem_tiny_config.h core/box/tiny_header_box.h \ - core/box/tiny_layout_box.h core/box/../tiny_region_id.h \ - core/front/../hakmem_tiny_superslab.h \ - core/front/../superslab/superslab_inline.h \ - core/front/../box/pagefault_telemetry_box.h \ - core/front/../box/ss_tier_box.h \ - core/front/../box/../superslab/superslab_types.h \ - core/front/../box/ss_slab_meta_box.h \ - core/front/../box/slab_freelist_atomic.h \ - core/front/../box/warm_pool_stats_box.h \ - core/front/../box/../hakmem_tiny_config.h \ - core/front/../box/../front/tiny_warm_pool.h \ - core/front/../box/slab_carve_box.h \ - core/front/../box/../hakmem_tiny_superslab.h \ - core/front/../box/../superslab/superslab_inline.h \ - core/front/../box/../tiny_box_geometry.h \ - core/front/../box/../box/pagefault_telemetry_box.h \ - core/front/../box/c7_meta_used_counter_box.h \ - core/front/../box/warm_pool_rel_counters_box.h \ - core/front/../box/warm_pool_prefill_box.h \ - core/front/../box/../tiny_tls.h \ - core/front/../box/../box/warm_pool_stats_box.h \ - core/front/../hakmem_env_cache.h core/front/../box/tiny_page_box.h \ - core/front/../box/ss_tls_bind_box.h \ - core/front/../box/../box/tiny_page_box.h \ - core/front/../box/tiny_tls_carve_one_block_box.h \ - core/front/../box/../tiny_debug_api.h \ - core/front/../box/warm_tls_bind_logger_box.h \ - core/front/../box/warm_pool_dbg_box.h -core/front/tiny_unified_cache.h: -core/front/../hakmem_build_flags.h: -core/front/../hakmem_tiny_config.h: -core/front/../box/ptr_type_box.h: -core/front/../box/tiny_front_config_box.h: -core/front/../box/../hakmem_build_flags.h: -core/front/tiny_warm_pool.h: -core/front/../superslab/superslab_types.h: -core/hakmem_tiny_superslab_constants.h: -core/front/../tiny_tls.h: -core/front/../hakmem_tiny_superslab.h: -core/front/../superslab/superslab_types.h: -core/front/../superslab/superslab_inline.h: -core/front/../superslab/superslab_types.h: -core/front/../superslab/../tiny_box_geometry.h: -core/front/../superslab/../hakmem_tiny_superslab_constants.h: -core/front/../superslab/../hakmem_tiny_config.h: -core/front/../tiny_debug_ring.h: -core/front/../hakmem_build_flags.h: -core/front/../tiny_remote.h: -core/front/../hakmem_tiny_superslab_constants.h: -core/front/../tiny_box_geometry.h: -core/front/../box/tiny_next_ptr_box.h: -core/hakmem_tiny_config.h: -core/tiny_nextptr.h: -core/hakmem_build_flags.h: -core/tiny_region_id.h: -core/tiny_box_geometry.h: -core/ptr_track.h: -core/hakmem_super_registry.h: -core/hakmem_tiny_superslab.h: -core/box/ss_addr_map_box.h: -core/box/../hakmem_build_flags.h: -core/superslab/superslab_inline.h: -core/hakmem_tiny.h: -core/hakmem_trace.h: -core/hakmem_tiny_mini_mag.h: -core/box/hak_lane_classify.inc.h: -core/box/ptr_type_box.h: -core/tiny_debug_api.h: -core/box/tiny_layout_box.h: -core/box/../hakmem_tiny_config.h: -core/box/tiny_header_box.h: -core/box/tiny_layout_box.h: -core/box/../tiny_region_id.h: -core/front/../hakmem_tiny_superslab.h: -core/front/../superslab/superslab_inline.h: -core/front/../box/pagefault_telemetry_box.h: -core/front/../box/ss_tier_box.h: -core/front/../box/../superslab/superslab_types.h: -core/front/../box/ss_slab_meta_box.h: -core/front/../box/slab_freelist_atomic.h: -core/front/../box/warm_pool_stats_box.h: -core/front/../box/../hakmem_tiny_config.h: -core/front/../box/../front/tiny_warm_pool.h: -core/front/../box/slab_carve_box.h: -core/front/../box/../hakmem_tiny_superslab.h: -core/front/../box/../superslab/superslab_inline.h: -core/front/../box/../tiny_box_geometry.h: -core/front/../box/../box/pagefault_telemetry_box.h: -core/front/../box/c7_meta_used_counter_box.h: -core/front/../box/warm_pool_rel_counters_box.h: -core/front/../box/warm_pool_prefill_box.h: -core/front/../box/../tiny_tls.h: -core/front/../box/../box/warm_pool_stats_box.h: -core/front/../hakmem_env_cache.h: -core/front/../box/tiny_page_box.h: -core/front/../box/ss_tls_bind_box.h: -core/front/../box/../box/tiny_page_box.h: -core/front/../box/tiny_tls_carve_one_block_box.h: -core/front/../box/../tiny_debug_api.h: -core/front/../box/warm_tls_bind_logger_box.h: -core/front/../box/warm_pool_dbg_box.h: diff --git a/core/hakmem_shared_pool_acquire.c b/core/hakmem_shared_pool_acquire.c index 8ea1892f..0a0c6faa 100644 --- a/core/hakmem_shared_pool_acquire.c +++ b/core/hakmem_shared_pool_acquire.c @@ -11,6 +11,7 @@ #include "hakmem_env_cache.h" // Priority-2: ENV cache #include "front/tiny_warm_pool.h" // Warm Pool: Prefill during registry scans #include "box/ss_slab_reset_box.h" // Box: Reset slab metadata on reuse (C7 guard) +#include "box/tiny_class_stats_box.h" // OBSERVE: per-class shared lock stats #include #include @@ -20,6 +21,11 @@ // Stage3(LRU) 由来の Superslab をトレースするための簡易マジック _Atomic uintptr_t g_c7_stage3_magic_ss = 0; +static inline void sp_lock_with_stats(int class_idx) { + tiny_class_stats_on_shared_lock(class_idx); + sp_lock_with_stats(class_idx); +} + static inline void c7_log_meta_state(const char* tag, SuperSlab* ss, int slab_idx) { if (!ss) return; #if HAKMEM_BUILD_RELEASE @@ -351,7 +357,7 @@ stage1_retry_after_tension_drain: atomic_fetch_add(&g_lock_acquire_slab_count, 1); } - pthread_mutex_lock(&g_shared_pool.alloc_lock); + sp_lock_with_stats(class_idx); // P0.3: Guard against TLS SLL orphaned pointers before reusing slab // RACE FIX: Load SuperSlab pointer atomically BEFORE guard (consistency) @@ -498,7 +504,7 @@ stage2_fallback: atomic_fetch_add(&g_lock_acquire_slab_count, 1); } - pthread_mutex_lock(&g_shared_pool.alloc_lock); + sp_lock_with_stats(class_idx); // Performance measurement: count Stage 2 lock acquisitions if (__builtin_expect(sp_measure_enabled(), 0)) { @@ -626,11 +632,11 @@ stage2_scan: // P0 instrumentation: count lock acquisitions lock_stats_init(); if (g_lock_stats_enabled == 1) { - atomic_fetch_add(&g_lock_acquire_count, 1); - atomic_fetch_add(&g_lock_acquire_slab_count, 1); - } + atomic_fetch_add(&g_lock_acquire_count, 1); + atomic_fetch_add(&g_lock_acquire_slab_count, 1); + } - pthread_mutex_lock(&g_shared_pool.alloc_lock); + sp_lock_with_stats(class_idx); // Performance measurement: count Stage 2 scan lock acquisitions if (__builtin_expect(sp_measure_enabled(), 0)) { @@ -793,7 +799,7 @@ stage2_scan: atomic_fetch_add(&g_lock_acquire_count, 1); atomic_fetch_add(&g_lock_acquire_slab_count, 1); // This is part of acquisition path } - pthread_mutex_lock(&g_shared_pool.alloc_lock); + sp_lock_with_stats(class_idx); if (!allocated_ss) { // Allocation failed; return now.