diff --git a/core/hakmem_tiny.c b/core/hakmem_tiny.c index 378d8264..82d34df0 100644 --- a/core/hakmem_tiny.c +++ b/core/hakmem_tiny.c @@ -80,12 +80,10 @@ static void tiny_remote_drain_locked(struct TinySlab* slab); __attribute__((always_inline)) static inline void* hak_tiny_alloc_wrapper(int class_idx); // Helpers for SuperSlab active block accounting (atomic, saturating dec) -void ss_active_add(SuperSlab* ss, uint32_t n) { - atomic_fetch_add_explicit(&ss->total_active_blocks, n, memory_order_relaxed); -} -static inline __attribute__((always_inline)) void ss_active_inc(SuperSlab* ss) { - atomic_fetch_add_explicit(&ss->total_active_blocks, 1u, memory_order_relaxed); -} + +// SuperSlab Active Counter Helpers - EXTRACTED to hakmem_tiny_ss_active_box.inc +#include "hakmem_tiny_ss_active_box.inc" + // EXTRACTED: ss_active_dec_one() moved to hakmem_tiny_superslab.h (Phase 2C-2) // Front refill count global config (declare before init.inc uses them) @@ -240,38 +238,10 @@ _Atomic uint32_t g_frontend_fill_target[TINY_NUM_CLASSES]; static inline int ultra_batch_for_class(int class_idx); enum { HAK_TIER_SLL=1, HAK_TIER_MAG=2, HAK_TIER_SLAB=3, HAK_TIER_SUPER=4, HAK_TIER_FRONT=5 }; -static inline uint16_t hak_thread_id16(void) { - // best-effort compress cached thread id to 16 bits - uint32_t tid = tiny_self_u32(); - return (uint16_t)(tid ^ (tid >> 16)); -} -static inline void eventq_push_ex(int class_idx, uint32_t size, uint8_t tier, uint8_t flags, - uint32_t site_id, uint16_t lat_bucket) { - (void)flags; +// Event Queue & Telemetry Helpers - EXTRACTED to hakmem_tiny_eventq_box.inc +#include "hakmem_tiny_eventq_box.inc" - (void)lat_bucket; - (void)site_id; - - if (!g_int_engine) return; - // Lightweight sampling: if mask set, log 1 out of 2^N - unsigned m = g_int_sample_mask; - if (m != 0) { - unsigned x = g_tls_ev_seq++; - if ((x & m) != 0) return; - } - uint32_t t = atomic_fetch_add_explicit(&g_ev_tail, 1u, memory_order_relaxed); - AllocEvent ev; - ev.ts_ns = g_int_event_ts ? hak_now_ns() : 0; - ev.size = size; - ev.site_id = 0; // keep minimal - ev.latency_bucket = 0; - ev.tier_hit = tier; - ev.flags = 0; - ev.class_idx = (uint16_t)class_idx; - ev.thread_id = 0; - g_ev_ring[t & EVENTQ_MASK] = ev; // best-effort overwrite on overflow -} // Background refill workers and intelligence engine #include "hakmem_tiny_background.inc" @@ -344,18 +314,10 @@ static inline void* hak_tiny_alloc_superslab_try_fast(int class_idx) { // SLL capacity policy: for hot tiny classes (0..3), allow larger SLL up to multiplier * mag_cap // for >=4 keep current conservative half (to limit footprint). -static inline uint32_t sll_cap_for_class(int class_idx, uint32_t mag_cap) { - // Phase12: g_sll_cap_override は非推奨。ここでは無視して通常capを返す。 - uint32_t cap = mag_cap; - if (class_idx <= 3) { - uint32_t mult = (g_sll_multiplier > 0 ? (uint32_t)g_sll_multiplier : 1u); - uint64_t want = (uint64_t)cap * (uint64_t)mult; - if (want > (uint64_t)TINY_TLS_MAG_CAP) cap = TINY_TLS_MAG_CAP; else cap = (uint32_t)want; - } else if (class_idx >= 4) { - cap = (mag_cap > 1u ? (mag_cap / 2u) : 1u); - } - return cap; -} + +// SLL Capacity Policy - EXTRACTED to hakmem_tiny_sll_cap_box.inc +#include "hakmem_tiny_sll_cap_box.inc" + // ============================================================================ // EXTRACTED TO hakmem_tiny_refill.inc.h (Phase 2D-1) @@ -366,25 +328,9 @@ static inline uint32_t sll_cap_for_class(int class_idx, uint32_t mag_cap) { static inline int ultra_sll_cap_for_class(int class_idx); static inline int ultra_validate_sll_head(int class_idx, void* head); -// Ultra-mode (SLL-only) helpers -// Ultra batch overrides via env: HAKMEM_TINY_ULTRA_BATCH_C{0..7} -static int g_ultra_batch_override[TINY_NUM_CLASSES] = {0}; -static int g_ultra_sll_cap_override[TINY_NUM_CLASSES] = {0}; +// Ultra-Mode Batch Configuration - EXTRACTED to hakmem_tiny_ultra_batch_box.inc +#include "hakmem_tiny_ultra_batch_box.inc" -static inline int ultra_batch_for_class(int class_idx) { - int ov = g_ultra_batch_override[class_idx]; - if (ov > 0) return ov; - switch (class_idx) { - case 0: return 64; // 8B - case 1: return 96; // 16B（A/B最良） - case 2: return 96; // 32B（A/B最良） - case 3: return 224; // 64B（A/B最良） - case 4: return 96; // 128B (promote front refill a bit) - case 5: return 64; // 256B (promote front refill) - case 6: return 64; // 512B (promote front refill) - default: return 32; // 1024B and others - } -} // ============================================================================ // EXTRACTED TO hakmem_tiny_refill.inc.h (Phase 2D-1) diff --git a/core/hakmem_tiny_eventq_box.inc b/core/hakmem_tiny_eventq_box.inc new file mode 100644 index 00000000..5929c3f1 --- /dev/null +++ b/core/hakmem_tiny_eventq_box.inc @@ -0,0 +1,32 @@ +static inline uint16_t hak_thread_id16(void) { + // best-effort compress cached thread id to 16 bits + uint32_t tid = tiny_self_u32(); + return (uint16_t)(tid ^ (tid >> 16)); +} + +static inline void eventq_push_ex(int class_idx, uint32_t size, uint8_t tier, uint8_t flags, + uint32_t site_id, uint16_t lat_bucket) { + (void)flags; + + (void)lat_bucket; + (void)site_id; + + if (!g_int_engine) return; + // Lightweight sampling: if mask set, log 1 out of 2^N + unsigned m = g_int_sample_mask; + if (m != 0) { + unsigned x = g_tls_ev_seq++; + if ((x & m) != 0) return; + } + uint32_t t = atomic_fetch_add_explicit(&g_ev_tail, 1u, memory_order_relaxed); + AllocEvent ev; + ev.ts_ns = g_int_event_ts ? hak_now_ns() : 0; + ev.size = size; + ev.site_id = 0; // keep minimal + ev.latency_bucket = 0; + ev.tier_hit = tier; + ev.flags = 0; + ev.class_idx = (uint16_t)class_idx; + ev.thread_id = 0; + g_ev_ring[t & EVENTQ_MASK] = ev; // best-effort overwrite on overflow +} diff --git a/core/hakmem_tiny_sll_cap_box.inc b/core/hakmem_tiny_sll_cap_box.inc new file mode 100644 index 00000000..9a2a6ba4 --- /dev/null +++ b/core/hakmem_tiny_sll_cap_box.inc @@ -0,0 +1,12 @@ +static inline uint32_t sll_cap_for_class(int class_idx, uint32_t mag_cap) { + // Phase12: g_sll_cap_override は非推奨。ここでは無視して通常capを返す。 + uint32_t cap = mag_cap; + if (class_idx <= 3) { + uint32_t mult = (g_sll_multiplier > 0 ? (uint32_t)g_sll_multiplier : 1u); + uint64_t want = (uint64_t)cap * (uint64_t)mult; + if (want > (uint64_t)TINY_TLS_MAG_CAP) cap = TINY_TLS_MAG_CAP; else cap = (uint32_t)want; + } else if (class_idx >= 4) { + cap = (mag_cap > 1u ? (mag_cap / 2u) : 1u); + } + return cap; +} diff --git a/core/hakmem_tiny_ss_active_box.inc b/core/hakmem_tiny_ss_active_box.inc new file mode 100644 index 00000000..6e08c886 --- /dev/null +++ b/core/hakmem_tiny_ss_active_box.inc @@ -0,0 +1,6 @@ +void ss_active_add(SuperSlab* ss, uint32_t n) { + atomic_fetch_add_explicit(&ss->total_active_blocks, n, memory_order_relaxed); +} +static inline __attribute__((always_inline)) void ss_active_inc(SuperSlab* ss) { + atomic_fetch_add_explicit(&ss->total_active_blocks, 1u, memory_order_relaxed); +} diff --git a/core/hakmem_tiny_ultra_batch_box.inc b/core/hakmem_tiny_ultra_batch_box.inc new file mode 100644 index 00000000..ad2c691d --- /dev/null +++ b/core/hakmem_tiny_ultra_batch_box.inc @@ -0,0 +1,20 @@ + +// Ultra-mode (SLL-only) helpers +// Ultra batch overrides via env: HAKMEM_TINY_ULTRA_BATCH_C{0..7} +static int g_ultra_batch_override[TINY_NUM_CLASSES] = {0}; +static int g_ultra_sll_cap_override[TINY_NUM_CLASSES] = {0}; + +static inline int ultra_batch_for_class(int class_idx) { + int ov = g_ultra_batch_override[class_idx]; + if (ov > 0) return ov; + switch (class_idx) { + case 0: return 64; // 8B + case 1: return 96; // 16B（A/B最良） + case 2: return 96; // 32B（A/B最良） + case 3: return 224; // 64B（A/B最良） + case 4: return 96; // 128B (promote front refill a bit) + case 5: return 64; // 256B (promote front refill) + case 6: return 64; // 512B (promote front refill) + default: return 32; // 1024B and others + } +}