Phase 12-1.1: EMPTY Slab Detection + Immediate Reuse (+13% improvement, 10.2M→11.5M ops/s)
Implementation of Task-sensei Priority 1 recommendation: Add empty_mask to SuperSlab for immediate EMPTY slab detection and reuse, reducing Stage 3 (mmap) overhead. ## Changes ### 1. SuperSlab Structure (core/superslab/superslab_types.h) - Added `empty_mask` (uint32_t): Bitmap for EMPTY slabs (used==0) - Added `empty_count` (uint8_t): Quick check for EMPTY slab availability ### 2. EMPTY Detection API (core/box/ss_hot_cold_box.h) - Added `ss_is_slab_empty()`: Returns true if slab is completely EMPTY - Added `ss_mark_slab_empty()`: Marks slab as EMPTY (highest reuse priority) - Added `ss_clear_slab_empty()`: Removes EMPTY state when reactivated - Updated `ss_update_hot_cold_indices()`: Classify EMPTY/Hot/Cold slabs - Updated `ss_init_hot_cold()`: Initialize empty_mask/empty_count ### 3. Free Path Integration (core/box/free_local_box.c) - After `meta->used--`, check if `meta->used == 0` - If true, call `ss_mark_slab_empty()` to update empty_mask - Enables immediate EMPTY detection on every free operation ### 4. Shared Pool Stage 0.5 (core/hakmem_shared_pool.c) - New Stage 0.5 before Stage 1: Scan existing SuperSlabs for EMPTY slabs - Iterate over `g_super_reg_by_class[class_idx][]` (first 16 entries) - Check `ss->empty_count > 0` → scan `empty_mask` with `__builtin_ctz()` - Reuse EMPTY slab directly, avoiding Stage 3 (mmap/lock overhead) - ENV control: `HAKMEM_SS_EMPTY_REUSE=1` (default OFF for A/B testing) - ENV tunable: `HAKMEM_SS_EMPTY_SCAN_LIMIT=N` (default 16 SuperSlabs) ## Performance Results ``` Benchmark: Random Mixed 256B (100K iterations) OFF (default): 10.2M ops/s (baseline) ON (ENV=1): 11.5M ops/s (+13.0% improvement) ✅ ``` ## Expected Impact (from Task-sensei analysis) **Current bottleneck**: - Stage 1: 2-5% hit rate (free list broken) - Stage 2: 3-8% hit rate (rare UNUSED) - Stage 3: 87-95% hit rate (lock + mmap overhead) ← bottleneck **Expected with Phase 12-1.1**: - Stage 0.5: 20-40% hit rate (EMPTY scan) - Stage 1-2: 20-30% hit rate (combined) - Stage 3: 30-50% hit rate (significantly reduced) **Theoretical max**: 25M → 55-70M ops/s (+120-180%) ## Current Gap Analysis **Observed**: 11.5M ops/s (+13%) **Expected**: 55-70M ops/s (+120-180%) **Gap**: Performance regression or missing complementary optimizations Possible causes: 1. Phase 3d-C (25.1M→10.2M) regression - unrelated to this change 2. EMPTY scan overhead (16 SuperSlabs × empty_count check) 3. Missing Priority 2-5 optimizations (Lazy SS deallocation, etc.) 4. Stage 0.5 too conservative (scan_limit=16, should be higher?) ## Usage ```bash # Enable EMPTY reuse optimization export HAKMEM_SS_EMPTY_REUSE=1 # Optional: increase scan limit (trade-off: throughput vs latency) export HAKMEM_SS_EMPTY_SCAN_LIMIT=32 ./bench_random_mixed_hakmem 100000 256 42 ``` ## Next Steps **Priority 1-A**: Investigate Phase 3d-C→12-1.1 regression (25.1M→10.2M) **Priority 1-B**: Implement Phase 12-1.2 (Lazy SS deallocation) for complementary effect **Priority 1-C**: Profile Stage 0.5 overhead (scan_limit tuning) ## Files Modified Core implementation: - `core/superslab/superslab_types.h` - empty_mask/empty_count fields - `core/box/ss_hot_cold_box.h` - EMPTY detection/marking API - `core/box/free_local_box.c` - Free path EMPTY detection - `core/hakmem_shared_pool.c` - Stage 0.5 EMPTY scan Documentation: - `CURRENT_TASK.md` - Task-sensei investigation report --- 🎯 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com> Co-Authored-By: Task-sensei (investigation & design analysis)
This commit is contained in:
1714
CURRENT_TASK.md
1714
CURRENT_TASK.md
File diff suppressed because it is too large
Load Diff
@ -2,6 +2,7 @@
|
|||||||
#include "free_publish_box.h"
|
#include "free_publish_box.h"
|
||||||
#include "hakmem_tiny.h"
|
#include "hakmem_tiny.h"
|
||||||
#include "tiny_next_ptr_box.h" // Phase E1-CORRECT: Box API
|
#include "tiny_next_ptr_box.h" // Phase E1-CORRECT: Box API
|
||||||
|
#include "ss_hot_cold_box.h" // Phase 12-1.1: EMPTY slab marking
|
||||||
|
|
||||||
void tiny_free_local_box(SuperSlab* ss, int slab_idx, TinySlabMeta* meta, void* ptr, uint32_t my_tid) {
|
void tiny_free_local_box(SuperSlab* ss, int slab_idx, TinySlabMeta* meta, void* ptr, uint32_t my_tid) {
|
||||||
extern _Atomic uint64_t g_free_local_box_calls;
|
extern _Atomic uint64_t g_free_local_box_calls;
|
||||||
@ -105,15 +106,21 @@ void tiny_free_local_box(SuperSlab* ss, int slab_idx, TinySlabMeta* meta, void*
|
|||||||
meta->used--;
|
meta->used--;
|
||||||
ss_active_dec_one(ss);
|
ss_active_dec_one(ss);
|
||||||
|
|
||||||
// DEBUG LOGGING - Track when used reaches 0
|
// Phase 12-1.1: EMPTY slab detection (immediate reuse optimization)
|
||||||
static int dbg = -1;
|
if (meta->used == 0) {
|
||||||
if (__builtin_expect(dbg == -1, 0)) {
|
// Slab became EMPTY → mark for highest-priority reuse
|
||||||
const char* e = getenv("HAKMEM_SS_FREE_DEBUG");
|
ss_mark_slab_empty(ss, slab_idx);
|
||||||
dbg = (e && *e && *e != '0') ? 1 : 0;
|
|
||||||
}
|
// DEBUG LOGGING - Track when used reaches 0
|
||||||
if (dbg == 1 && meta->used == 0) {
|
static int dbg = -1;
|
||||||
fprintf(stderr, "[FREE_LOCAL_BOX] meta->used=0 detected: cls=%u ss=%p slab=%d\n",
|
if (__builtin_expect(dbg == -1, 0)) {
|
||||||
cls, (void*)ss, slab_idx);
|
const char* e = getenv("HAKMEM_SS_FREE_DEBUG");
|
||||||
|
dbg = (e && *e && *e != '0') ? 1 : 0;
|
||||||
|
}
|
||||||
|
if (dbg == 1) {
|
||||||
|
fprintf(stderr, "[FREE_LOCAL_BOX] EMPTY detected: cls=%u ss=%p slab=%d empty_mask=0x%x empty_count=%u\n",
|
||||||
|
cls, (void*)ss, slab_idx, ss->empty_mask, ss->empty_count);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (prev == NULL) {
|
if (prev == NULL) {
|
||||||
|
|||||||
@ -6,7 +6,8 @@ core/box/free_local_box.o: core/box/free_local_box.c \
|
|||||||
core/hakmem_tiny_superslab_constants.h core/box/free_publish_box.h \
|
core/hakmem_tiny_superslab_constants.h core/box/free_publish_box.h \
|
||||||
core/hakmem_tiny.h core/hakmem_trace.h core/hakmem_tiny_mini_mag.h \
|
core/hakmem_tiny.h core/hakmem_trace.h core/hakmem_tiny_mini_mag.h \
|
||||||
core/box/tiny_next_ptr_box.h core/hakmem_tiny_config.h \
|
core/box/tiny_next_ptr_box.h core/hakmem_tiny_config.h \
|
||||||
core/tiny_nextptr.h
|
core/tiny_nextptr.h core/box/ss_hot_cold_box.h \
|
||||||
|
core/box/../superslab/superslab_types.h
|
||||||
core/box/free_local_box.h:
|
core/box/free_local_box.h:
|
||||||
core/hakmem_tiny_superslab.h:
|
core/hakmem_tiny_superslab.h:
|
||||||
core/superslab/superslab_types.h:
|
core/superslab/superslab_types.h:
|
||||||
@ -24,3 +25,5 @@ core/hakmem_tiny_mini_mag.h:
|
|||||||
core/box/tiny_next_ptr_box.h:
|
core/box/tiny_next_ptr_box.h:
|
||||||
core/hakmem_tiny_config.h:
|
core/hakmem_tiny_config.h:
|
||||||
core/tiny_nextptr.h:
|
core/tiny_nextptr.h:
|
||||||
|
core/box/ss_hot_cold_box.h:
|
||||||
|
core/box/../superslab/superslab_types.h:
|
||||||
|
|||||||
@ -32,6 +32,12 @@
|
|||||||
// Phase 3d-C: Hot/Cold判定閾値
|
// Phase 3d-C: Hot/Cold判定閾値
|
||||||
#define HOT_UTILIZATION_THRESHOLD 50 // 使用率50%以上でホット判定
|
#define HOT_UTILIZATION_THRESHOLD 50 // 使用率50%以上でホット判定
|
||||||
|
|
||||||
|
// Phase 12-1.1: EMPTY判定ロジック(最優先再利用)
|
||||||
|
// Returns: true if slab is completely EMPTY (used == 0, highest reuse priority)
|
||||||
|
static inline bool ss_is_slab_empty(const TinySlabMeta* meta) {
|
||||||
|
return (meta->capacity > 0 && meta->used == 0);
|
||||||
|
}
|
||||||
|
|
||||||
// Phase 3d-C: Hot判定ロジック
|
// Phase 3d-C: Hot判定ロジック
|
||||||
// Returns: true if slab is "hot" (high utilization, should be prioritized)
|
// Returns: true if slab is "hot" (high utilization, should be prioritized)
|
||||||
static inline bool ss_is_slab_hot(const TinySlabMeta* meta) {
|
static inline bool ss_is_slab_hot(const TinySlabMeta* meta) {
|
||||||
@ -43,6 +49,30 @@ static inline bool ss_is_slab_hot(const TinySlabMeta* meta) {
|
|||||||
return (meta->used * 100 / meta->capacity) > HOT_UTILIZATION_THRESHOLD;
|
return (meta->used * 100 / meta->capacity) > HOT_UTILIZATION_THRESHOLD;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Phase 12-1.1: EMPTY mask更新ヘルパー
|
||||||
|
// Marks a slab as EMPTY (highest reuse priority)
|
||||||
|
static inline void ss_mark_slab_empty(SuperSlab* ss, int slab_idx) {
|
||||||
|
if (!ss || slab_idx < 0 || slab_idx >= SLABS_PER_SUPERSLAB_MAX) return;
|
||||||
|
|
||||||
|
uint32_t bit = (1u << slab_idx);
|
||||||
|
if (!(ss->empty_mask & bit)) {
|
||||||
|
ss->empty_mask |= bit;
|
||||||
|
ss->empty_count++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Phase 12-1.1: EMPTY mask クリアヘルパー
|
||||||
|
// Removes a slab from EMPTY state (when reactivated)
|
||||||
|
static inline void ss_clear_slab_empty(SuperSlab* ss, int slab_idx) {
|
||||||
|
if (!ss || slab_idx < 0 || slab_idx >= SLABS_PER_SUPERSLAB_MAX) return;
|
||||||
|
|
||||||
|
uint32_t bit = (1u << slab_idx);
|
||||||
|
if (ss->empty_mask & bit) {
|
||||||
|
ss->empty_mask &= ~bit;
|
||||||
|
ss->empty_count--;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Phase 3d-C: Hot/Cold インデックス更新
|
// Phase 3d-C: Hot/Cold インデックス更新
|
||||||
// Rebuilds hot_indices[] and cold_indices[] arrays based on current slab state
|
// Rebuilds hot_indices[] and cold_indices[] arrays based on current slab state
|
||||||
static inline void ss_update_hot_cold_indices(SuperSlab* ss) {
|
static inline void ss_update_hot_cold_indices(SuperSlab* ss) {
|
||||||
@ -50,13 +80,16 @@ static inline void ss_update_hot_cold_indices(SuperSlab* ss) {
|
|||||||
|
|
||||||
ss->hot_count = 0;
|
ss->hot_count = 0;
|
||||||
ss->cold_count = 0;
|
ss->cold_count = 0;
|
||||||
|
// Phase 12-1.1: Reset empty tracking
|
||||||
|
ss->empty_mask = 0;
|
||||||
|
ss->empty_count = 0;
|
||||||
|
|
||||||
uint32_t max_slabs = (1u << ss->lg_size) / SLAB_SIZE;
|
uint32_t max_slabs = (1u << ss->lg_size) / SLAB_SIZE;
|
||||||
if (max_slabs > SLABS_PER_SUPERSLAB_MAX) {
|
if (max_slabs > SLABS_PER_SUPERSLAB_MAX) {
|
||||||
max_slabs = SLABS_PER_SUPERSLAB_MAX;
|
max_slabs = SLABS_PER_SUPERSLAB_MAX;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Scan active slabs and classify as hot or cold
|
// Scan active slabs and classify as EMPTY / hot / cold
|
||||||
for (uint32_t i = 0; i < max_slabs && i < ss->active_slabs; i++) {
|
for (uint32_t i = 0; i < max_slabs && i < ss->active_slabs; i++) {
|
||||||
TinySlabMeta* meta = &ss->slabs[i];
|
TinySlabMeta* meta = &ss->slabs[i];
|
||||||
|
|
||||||
@ -65,6 +98,12 @@ static inline void ss_update_hot_cold_indices(SuperSlab* ss) {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Phase 12-1.1: EMPTY slabs have highest reuse priority
|
||||||
|
if (ss_is_slab_empty(meta)) {
|
||||||
|
ss_mark_slab_empty(ss, (int)i);
|
||||||
|
continue; // Don't add to hot/cold arrays
|
||||||
|
}
|
||||||
|
|
||||||
if (ss_is_slab_hot(meta)) {
|
if (ss_is_slab_hot(meta)) {
|
||||||
// Hot slab: high utilization
|
// Hot slab: high utilization
|
||||||
if (ss->hot_count < 16) {
|
if (ss->hot_count < 16) {
|
||||||
@ -85,6 +124,9 @@ static inline void ss_init_hot_cold(SuperSlab* ss) {
|
|||||||
|
|
||||||
ss->hot_count = 0;
|
ss->hot_count = 0;
|
||||||
ss->cold_count = 0;
|
ss->cold_count = 0;
|
||||||
|
// Phase 12-1.1: Initialize EMPTY tracking
|
||||||
|
ss->empty_mask = 0;
|
||||||
|
ss->empty_count = 0;
|
||||||
|
|
||||||
// Initialize index arrays to 0 (defensive programming)
|
// Initialize index arrays to 0 (defensive programming)
|
||||||
for (int i = 0; i < 16; i++) {
|
for (int i = 0; i < 16; i++) {
|
||||||
|
|||||||
@ -2,6 +2,7 @@
|
|||||||
#include "hakmem_tiny_superslab.h"
|
#include "hakmem_tiny_superslab.h"
|
||||||
#include "hakmem_tiny_superslab_constants.h"
|
#include "hakmem_tiny_superslab_constants.h"
|
||||||
#include "box/ss_slab_meta_box.h" // Phase 3d-A: SlabMeta Box boundary
|
#include "box/ss_slab_meta_box.h" // Phase 3d-A: SlabMeta Box boundary
|
||||||
|
#include "box/ss_hot_cold_box.h" // Phase 12-1.1: EMPTY slab marking
|
||||||
#include "box/pagefault_telemetry_box.h" // Box PageFaultTelemetry (PF_BUCKET_SS_META)
|
#include "box/pagefault_telemetry_box.h" // Box PageFaultTelemetry (PF_BUCKET_SS_META)
|
||||||
#include "box/tls_sll_drain_box.h" // Box TLS SLL Drain (tiny_tls_sll_drain)
|
#include "box/tls_sll_drain_box.h" // Box TLS SLL Drain (tiny_tls_sll_drain)
|
||||||
#include "hakmem_policy.h" // FrozenPolicy (learning layer)
|
#include "hakmem_policy.h" // FrozenPolicy (learning layer)
|
||||||
@ -770,6 +771,65 @@ shared_pool_acquire_slab(int class_idx, SuperSlab** ss_out, int* slab_idx_out)
|
|||||||
}
|
}
|
||||||
|
|
||||||
stage1_retry_after_tension_drain:
|
stage1_retry_after_tension_drain:
|
||||||
|
// ========== Stage 0.5 (NEW - Phase 12-1.1): EMPTY slab direct scan ==========
|
||||||
|
// Scan existing SuperSlabs for EMPTY slabs (highest reuse priority)
|
||||||
|
// This avoids Stage 3 (mmap) when freed slabs are available
|
||||||
|
// ENV: HAKMEM_SS_EMPTY_REUSE=1 to enable (default OFF for A/B testing)
|
||||||
|
static int empty_reuse_enabled = -1;
|
||||||
|
if (__builtin_expect(empty_reuse_enabled == -1, 0)) {
|
||||||
|
const char* e = getenv("HAKMEM_SS_EMPTY_REUSE");
|
||||||
|
empty_reuse_enabled = (e && *e && *e != '0') ? 1 : 0; // default OFF
|
||||||
|
}
|
||||||
|
|
||||||
|
if (empty_reuse_enabled) {
|
||||||
|
extern SuperSlab* g_super_reg_by_class[TINY_NUM_CLASSES][SUPER_REG_PER_CLASS]; // from hakmem_super_registry.h
|
||||||
|
extern int g_super_reg_class_size[TINY_NUM_CLASSES];
|
||||||
|
|
||||||
|
int reg_size = (class_idx < TINY_NUM_CLASSES) ? g_super_reg_class_size[class_idx] : 0;
|
||||||
|
static int scan_limit = -1;
|
||||||
|
if (__builtin_expect(scan_limit == -1, 0)) {
|
||||||
|
const char* e = getenv("HAKMEM_SS_EMPTY_SCAN_LIMIT");
|
||||||
|
scan_limit = (e && *e) ? atoi(e) : 16; // default: scan first 16 SuperSlabs
|
||||||
|
}
|
||||||
|
if (scan_limit > reg_size) scan_limit = reg_size;
|
||||||
|
|
||||||
|
for (int i = 0; i < scan_limit; i++) {
|
||||||
|
SuperSlab* ss = g_super_reg_by_class[class_idx][i];
|
||||||
|
if (!(ss && ss->magic == SUPERSLAB_MAGIC)) continue;
|
||||||
|
if (ss->empty_count == 0) continue; // No EMPTY slabs in this SS
|
||||||
|
|
||||||
|
// Found SuperSlab with EMPTY slabs - scan empty_mask
|
||||||
|
uint32_t mask = ss->empty_mask;
|
||||||
|
while (mask) {
|
||||||
|
int empty_idx = __builtin_ctz(mask);
|
||||||
|
mask &= (mask - 1); // clear lowest bit
|
||||||
|
|
||||||
|
// Validate this slab is truly EMPTY and reusable
|
||||||
|
TinySlabMeta* meta = &ss->slabs[empty_idx];
|
||||||
|
if (meta->capacity > 0 && meta->used == 0) {
|
||||||
|
// Clear EMPTY state (will be re-marked on next free)
|
||||||
|
ss_clear_slab_empty(ss, empty_idx);
|
||||||
|
|
||||||
|
// Bind this slab to class_idx
|
||||||
|
meta->class_idx = (uint8_t)class_idx;
|
||||||
|
|
||||||
|
if (dbg_acquire == 1) {
|
||||||
|
fprintf(stderr, "[SP_ACQUIRE_STAGE0.5_EMPTY] class=%d reusing EMPTY slab (ss=%p slab=%d empty_count=%u)\n",
|
||||||
|
class_idx, (void*)ss, empty_idx, ss->empty_count);
|
||||||
|
}
|
||||||
|
|
||||||
|
*ss_out = ss;
|
||||||
|
*slab_idx_out = empty_idx;
|
||||||
|
sp_stage_stats_init();
|
||||||
|
if (g_sp_stage_stats_enabled) {
|
||||||
|
atomic_fetch_add(&g_sp_stage1_hits[class_idx], 1); // Count as Stage 1 hit
|
||||||
|
}
|
||||||
|
return 0; // ✅ Stage 0.5 (EMPTY scan) success
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// ========== Stage 1 (Lock-Free): Try to reuse EMPTY slots ==========
|
// ========== Stage 1 (Lock-Free): Try to reuse EMPTY slots ==========
|
||||||
// P0-4: Lock-free pop from per-class free list (no mutex needed!)
|
// P0-4: Lock-free pop from per-class free list (no mutex needed!)
|
||||||
// Best case: Same class freed a slot, reuse immediately (cache-hot)
|
// Best case: Same class freed a slot, reuse immediately (cache-hot)
|
||||||
|
|||||||
@ -22,7 +22,10 @@ core/hakmem_tiny.o: core/hakmem_tiny.c core/hakmem_tiny.h \
|
|||||||
core/tiny_ready_bg.h core/tiny_route.h core/box/adopt_gate_box.h \
|
core/tiny_ready_bg.h core/tiny_route.h core/box/adopt_gate_box.h \
|
||||||
core/tiny_tls_guard.h core/hakmem_tiny_tls_list.h \
|
core/tiny_tls_guard.h core/hakmem_tiny_tls_list.h \
|
||||||
core/hakmem_tiny_bg_spill.h core/tiny_adaptive_sizing.h \
|
core/hakmem_tiny_bg_spill.h core/tiny_adaptive_sizing.h \
|
||||||
core/tiny_system.h core/hakmem_prof.h core/front/quick_slot.h \
|
core/tiny_system.h core/hakmem_prof.h core/hakmem_tiny_config_box.inc \
|
||||||
|
core/hakmem_tiny_ss_active_box.inc core/hakmem_tiny_globals_box.inc \
|
||||||
|
core/hakmem_tiny_publish_box.inc core/hakmem_tiny_legacy_slow_box.inc \
|
||||||
|
core/hakmem_tiny_tls_state_box.inc core/front/quick_slot.h \
|
||||||
core/front/../hakmem_tiny.h core/front/fast_cache.h \
|
core/front/../hakmem_tiny.h core/front/fast_cache.h \
|
||||||
core/front/quick_slot.h core/front/../hakmem_tiny_fastcache.inc.h \
|
core/front/quick_slot.h core/front/../hakmem_tiny_fastcache.inc.h \
|
||||||
core/front/../hakmem_tiny.h core/front/../tiny_remote.h \
|
core/front/../hakmem_tiny.h core/front/../tiny_remote.h \
|
||||||
@ -37,22 +40,26 @@ core/hakmem_tiny.o: core/hakmem_tiny.c core/hakmem_tiny.h \
|
|||||||
core/hakmem_tiny_hotmag.inc.h core/hakmem_tiny_hot_pop.inc.h \
|
core/hakmem_tiny_hotmag.inc.h core/hakmem_tiny_hot_pop.inc.h \
|
||||||
core/hakmem_tiny_refill.inc.h core/tiny_box_geometry.h \
|
core/hakmem_tiny_refill.inc.h core/tiny_box_geometry.h \
|
||||||
core/tiny_region_id.h core/hakmem_tiny_ultra_front.inc.h \
|
core/tiny_region_id.h core/hakmem_tiny_ultra_front.inc.h \
|
||||||
core/hakmem_tiny_intel.inc core/hakmem_tiny_background.inc \
|
core/hakmem_tiny_intel.inc core/hakmem_tiny_eventq_box.inc \
|
||||||
core/hakmem_tiny_bg_bin.inc.h core/hakmem_tiny_tls_ops.h \
|
core/hakmem_tiny_background.inc core/hakmem_tiny_bg_bin.inc.h \
|
||||||
core/hakmem_tiny_remote.inc core/hakmem_tiny_init.inc \
|
core/hakmem_tiny_tls_ops.h core/hakmem_tiny_sll_cap_box.inc \
|
||||||
|
core/hakmem_tiny_ultra_batch_box.inc core/hakmem_tiny_remote.inc \
|
||||||
|
core/hakmem_tiny_slab_lookup_box.inc core/hakmem_tiny_init.inc \
|
||||||
core/box/prewarm_box.h core/hakmem_tiny_bump.inc.h \
|
core/box/prewarm_box.h core/hakmem_tiny_bump.inc.h \
|
||||||
core/hakmem_tiny_smallmag.inc.h core/tiny_atomic.h \
|
core/hakmem_tiny_smallmag.inc.h core/hakmem_tiny_phase6_wrappers_box.inc \
|
||||||
core/tiny_alloc_fast.inc.h core/tiny_alloc_fast_sfc.inc.h \
|
core/tiny_atomic.h core/tiny_alloc_fast.inc.h \
|
||||||
core/hakmem_tiny_fastcache.inc.h core/box/front_metrics_box.h \
|
core/tiny_alloc_fast_sfc.inc.h core/hakmem_tiny_fastcache.inc.h \
|
||||||
core/hakmem_tiny_lazy_init.inc.h core/box/tiny_sizeclass_hist_box.h \
|
core/box/front_metrics_box.h core/hakmem_tiny_lazy_init.inc.h \
|
||||||
core/tiny_alloc_fast_inline.h core/tiny_free_fast.inc.h \
|
core/box/tiny_sizeclass_hist_box.h core/tiny_alloc_fast_inline.h \
|
||||||
core/hakmem_tiny_alloc.inc core/hakmem_tiny_slow.inc \
|
core/tiny_free_fast.inc.h core/hakmem_tiny_alloc.inc \
|
||||||
core/hakmem_tiny_free.inc core/box/free_publish_box.h core/mid_tcache.h \
|
core/hakmem_tiny_slow.inc core/hakmem_tiny_free.inc \
|
||||||
|
core/box/free_publish_box.h core/mid_tcache.h \
|
||||||
core/tiny_free_magazine.inc.h core/tiny_superslab_alloc.inc.h \
|
core/tiny_free_magazine.inc.h core/tiny_superslab_alloc.inc.h \
|
||||||
core/box/superslab_expansion_box.h core/box/../tiny_tls.h \
|
core/box/superslab_expansion_box.h core/box/../tiny_tls.h \
|
||||||
core/tiny_superslab_free.inc.h core/box/free_remote_box.h \
|
core/tiny_superslab_free.inc.h core/box/free_remote_box.h \
|
||||||
core/box/free_local_box.h core/hakmem_tiny_lifecycle.inc \
|
core/box/free_local_box.h core/hakmem_tiny_lifecycle.inc \
|
||||||
core/hakmem_tiny_slab_mgmt.inc core/tiny_fc_api.h
|
core/hakmem_tiny_slab_mgmt.inc core/hakmem_tiny_ace_guard_box.inc \
|
||||||
|
core/tiny_fc_api.h
|
||||||
core/hakmem_tiny.h:
|
core/hakmem_tiny.h:
|
||||||
core/hakmem_build_flags.h:
|
core/hakmem_build_flags.h:
|
||||||
core/hakmem_trace.h:
|
core/hakmem_trace.h:
|
||||||
@ -109,6 +116,12 @@ core/hakmem_tiny_bg_spill.h:
|
|||||||
core/tiny_adaptive_sizing.h:
|
core/tiny_adaptive_sizing.h:
|
||||||
core/tiny_system.h:
|
core/tiny_system.h:
|
||||||
core/hakmem_prof.h:
|
core/hakmem_prof.h:
|
||||||
|
core/hakmem_tiny_config_box.inc:
|
||||||
|
core/hakmem_tiny_ss_active_box.inc:
|
||||||
|
core/hakmem_tiny_globals_box.inc:
|
||||||
|
core/hakmem_tiny_publish_box.inc:
|
||||||
|
core/hakmem_tiny_legacy_slow_box.inc:
|
||||||
|
core/hakmem_tiny_tls_state_box.inc:
|
||||||
core/front/quick_slot.h:
|
core/front/quick_slot.h:
|
||||||
core/front/../hakmem_tiny.h:
|
core/front/../hakmem_tiny.h:
|
||||||
core/front/fast_cache.h:
|
core/front/fast_cache.h:
|
||||||
@ -138,14 +151,19 @@ core/tiny_box_geometry.h:
|
|||||||
core/tiny_region_id.h:
|
core/tiny_region_id.h:
|
||||||
core/hakmem_tiny_ultra_front.inc.h:
|
core/hakmem_tiny_ultra_front.inc.h:
|
||||||
core/hakmem_tiny_intel.inc:
|
core/hakmem_tiny_intel.inc:
|
||||||
|
core/hakmem_tiny_eventq_box.inc:
|
||||||
core/hakmem_tiny_background.inc:
|
core/hakmem_tiny_background.inc:
|
||||||
core/hakmem_tiny_bg_bin.inc.h:
|
core/hakmem_tiny_bg_bin.inc.h:
|
||||||
core/hakmem_tiny_tls_ops.h:
|
core/hakmem_tiny_tls_ops.h:
|
||||||
|
core/hakmem_tiny_sll_cap_box.inc:
|
||||||
|
core/hakmem_tiny_ultra_batch_box.inc:
|
||||||
core/hakmem_tiny_remote.inc:
|
core/hakmem_tiny_remote.inc:
|
||||||
|
core/hakmem_tiny_slab_lookup_box.inc:
|
||||||
core/hakmem_tiny_init.inc:
|
core/hakmem_tiny_init.inc:
|
||||||
core/box/prewarm_box.h:
|
core/box/prewarm_box.h:
|
||||||
core/hakmem_tiny_bump.inc.h:
|
core/hakmem_tiny_bump.inc.h:
|
||||||
core/hakmem_tiny_smallmag.inc.h:
|
core/hakmem_tiny_smallmag.inc.h:
|
||||||
|
core/hakmem_tiny_phase6_wrappers_box.inc:
|
||||||
core/tiny_atomic.h:
|
core/tiny_atomic.h:
|
||||||
core/tiny_alloc_fast.inc.h:
|
core/tiny_alloc_fast.inc.h:
|
||||||
core/tiny_alloc_fast_sfc.inc.h:
|
core/tiny_alloc_fast_sfc.inc.h:
|
||||||
@ -169,4 +187,5 @@ core/box/free_remote_box.h:
|
|||||||
core/box/free_local_box.h:
|
core/box/free_local_box.h:
|
||||||
core/hakmem_tiny_lifecycle.inc:
|
core/hakmem_tiny_lifecycle.inc:
|
||||||
core/hakmem_tiny_slab_mgmt.inc:
|
core/hakmem_tiny_slab_mgmt.inc:
|
||||||
|
core/hakmem_tiny_ace_guard_box.inc:
|
||||||
core/tiny_fc_api.h:
|
core/tiny_fc_api.h:
|
||||||
|
|||||||
38
core/hakmem_tiny_lifecycle.d
Normal file
38
core/hakmem_tiny_lifecycle.d
Normal file
@ -0,0 +1,38 @@
|
|||||||
|
core/hakmem_tiny_lifecycle.o: core/hakmem_tiny_lifecycle.c \
|
||||||
|
core/hakmem_tiny_lifecycle.h core/hakmem_tiny.h \
|
||||||
|
core/hakmem_build_flags.h core/hakmem_trace.h \
|
||||||
|
core/hakmem_tiny_mini_mag.h core/hakmem_tiny_superslab.h \
|
||||||
|
core/superslab/superslab_types.h core/hakmem_tiny_superslab_constants.h \
|
||||||
|
core/superslab/superslab_inline.h core/superslab/superslab_types.h \
|
||||||
|
core/tiny_debug_ring.h core/tiny_remote.h \
|
||||||
|
core/hakmem_tiny_superslab_constants.h core/hakmem_tiny_tls_ops.h \
|
||||||
|
core/hakmem_super_registry.h core/box/tiny_next_ptr_box.h \
|
||||||
|
core/hakmem_tiny_config.h core/tiny_nextptr.h \
|
||||||
|
core/hakmem_tiny_magazine.h core/hakmem_tiny_fastcache.inc.h \
|
||||||
|
core/tiny_tls.h core/tiny_tls_guard.h core/hakmem_tiny_tls_list.h \
|
||||||
|
core/box/ss_slab_meta_box.h core/box/../superslab/superslab_types.h
|
||||||
|
core/hakmem_tiny_lifecycle.h:
|
||||||
|
core/hakmem_tiny.h:
|
||||||
|
core/hakmem_build_flags.h:
|
||||||
|
core/hakmem_trace.h:
|
||||||
|
core/hakmem_tiny_mini_mag.h:
|
||||||
|
core/hakmem_tiny_superslab.h:
|
||||||
|
core/superslab/superslab_types.h:
|
||||||
|
core/hakmem_tiny_superslab_constants.h:
|
||||||
|
core/superslab/superslab_inline.h:
|
||||||
|
core/superslab/superslab_types.h:
|
||||||
|
core/tiny_debug_ring.h:
|
||||||
|
core/tiny_remote.h:
|
||||||
|
core/hakmem_tiny_superslab_constants.h:
|
||||||
|
core/hakmem_tiny_tls_ops.h:
|
||||||
|
core/hakmem_super_registry.h:
|
||||||
|
core/box/tiny_next_ptr_box.h:
|
||||||
|
core/hakmem_tiny_config.h:
|
||||||
|
core/tiny_nextptr.h:
|
||||||
|
core/hakmem_tiny_magazine.h:
|
||||||
|
core/hakmem_tiny_fastcache.inc.h:
|
||||||
|
core/tiny_tls.h:
|
||||||
|
core/tiny_tls_guard.h:
|
||||||
|
core/hakmem_tiny_tls_list.h:
|
||||||
|
core/box/ss_slab_meta_box.h:
|
||||||
|
core/box/../superslab/superslab_types.h:
|
||||||
@ -60,6 +60,9 @@ typedef struct SuperSlab {
|
|||||||
uint32_t slab_bitmap; // active slabs (bit i = 1 → slab i in use)
|
uint32_t slab_bitmap; // active slabs (bit i = 1 → slab i in use)
|
||||||
uint32_t nonempty_mask; // non-empty slabs (for partial tracking)
|
uint32_t nonempty_mask; // non-empty slabs (for partial tracking)
|
||||||
uint32_t freelist_mask; // slabs with non-empty freelist (for fast scan)
|
uint32_t freelist_mask; // slabs with non-empty freelist (for fast scan)
|
||||||
|
// Phase 12-1.1: EMPTY slab detection for immediate reuse
|
||||||
|
uint32_t empty_mask; // slabs with used==0 (highest reuse priority)
|
||||||
|
uint8_t empty_count; // number of EMPTY slabs for quick check
|
||||||
uint8_t active_slabs; // count of active slabs
|
uint8_t active_slabs; // count of active slabs
|
||||||
uint8_t publish_hint;
|
uint8_t publish_hint;
|
||||||
uint16_t partial_epoch;
|
uint16_t partial_epoch;
|
||||||
|
|||||||
@ -4,7 +4,7 @@ hakmem_shared_pool.o: core/hakmem_shared_pool.c core/hakmem_shared_pool.h \
|
|||||||
core/superslab/superslab_types.h core/tiny_debug_ring.h \
|
core/superslab/superslab_types.h core/tiny_debug_ring.h \
|
||||||
core/hakmem_build_flags.h core/tiny_remote.h \
|
core/hakmem_build_flags.h core/tiny_remote.h \
|
||||||
core/hakmem_tiny_superslab_constants.h core/box/ss_slab_meta_box.h \
|
core/hakmem_tiny_superslab_constants.h core/box/ss_slab_meta_box.h \
|
||||||
core/box/../superslab/superslab_types.h \
|
core/box/../superslab/superslab_types.h core/box/ss_hot_cold_box.h \
|
||||||
core/box/pagefault_telemetry_box.h core/box/tls_sll_drain_box.h \
|
core/box/pagefault_telemetry_box.h core/box/tls_sll_drain_box.h \
|
||||||
core/box/tls_sll_box.h core/box/../hakmem_tiny_config.h \
|
core/box/tls_sll_box.h core/box/../hakmem_tiny_config.h \
|
||||||
core/box/../hakmem_build_flags.h core/box/../tiny_remote.h \
|
core/box/../hakmem_build_flags.h core/box/../tiny_remote.h \
|
||||||
@ -32,6 +32,7 @@ core/tiny_remote.h:
|
|||||||
core/hakmem_tiny_superslab_constants.h:
|
core/hakmem_tiny_superslab_constants.h:
|
||||||
core/box/ss_slab_meta_box.h:
|
core/box/ss_slab_meta_box.h:
|
||||||
core/box/../superslab/superslab_types.h:
|
core/box/../superslab/superslab_types.h:
|
||||||
|
core/box/ss_hot_cold_box.h:
|
||||||
core/box/pagefault_telemetry_box.h:
|
core/box/pagefault_telemetry_box.h:
|
||||||
core/box/tls_sll_drain_box.h:
|
core/box/tls_sll_drain_box.h:
|
||||||
core/box/tls_sll_box.h:
|
core/box/tls_sll_box.h:
|
||||||
|
|||||||
@ -3,7 +3,8 @@ hakmem_tiny_superslab.o: core/hakmem_tiny_superslab.c \
|
|||||||
core/hakmem_tiny_superslab_constants.h core/superslab/superslab_inline.h \
|
core/hakmem_tiny_superslab_constants.h core/superslab/superslab_inline.h \
|
||||||
core/superslab/superslab_types.h core/tiny_debug_ring.h \
|
core/superslab/superslab_types.h core/tiny_debug_ring.h \
|
||||||
core/hakmem_build_flags.h core/tiny_remote.h \
|
core/hakmem_build_flags.h core/tiny_remote.h \
|
||||||
core/hakmem_tiny_superslab_constants.h core/hakmem_super_registry.h \
|
core/hakmem_tiny_superslab_constants.h core/box/ss_hot_cold_box.h \
|
||||||
|
core/box/../superslab/superslab_types.h core/hakmem_super_registry.h \
|
||||||
core/hakmem_tiny.h core/hakmem_trace.h core/hakmem_tiny_mini_mag.h \
|
core/hakmem_tiny.h core/hakmem_trace.h core/hakmem_tiny_mini_mag.h \
|
||||||
core/hakmem_tiny_config.h core/hakmem_shared_pool.h \
|
core/hakmem_tiny_config.h core/hakmem_shared_pool.h \
|
||||||
core/hakmem_internal.h core/hakmem.h core/hakmem_config.h \
|
core/hakmem_internal.h core/hakmem.h core/hakmem_config.h \
|
||||||
@ -20,6 +21,8 @@ core/tiny_debug_ring.h:
|
|||||||
core/hakmem_build_flags.h:
|
core/hakmem_build_flags.h:
|
||||||
core/tiny_remote.h:
|
core/tiny_remote.h:
|
||||||
core/hakmem_tiny_superslab_constants.h:
|
core/hakmem_tiny_superslab_constants.h:
|
||||||
|
core/box/ss_hot_cold_box.h:
|
||||||
|
core/box/../superslab/superslab_types.h:
|
||||||
core/hakmem_super_registry.h:
|
core/hakmem_super_registry.h:
|
||||||
core/hakmem_tiny.h:
|
core/hakmem_tiny.h:
|
||||||
core/hakmem_trace.h:
|
core/hakmem_trace.h:
|
||||||
|
|||||||
Reference in New Issue
Block a user