adb5913af5
Problem: - Legacy-allocated SuperSlabs had slot states stuck at SLOT_UNUSED - sp_slot_mark_empty() failed, preventing EMPTY transition - Slots never returned to freelist → registry exhaustion - "SuperSlab registry full" errors flooded the system Root Cause: - Dual management: Legacy path vs Shared Pool path - Legacy SuperSlabs not synced with Shared Pool metadata - Inconsistent slot state tracking Solution (Workaround): - Added sp_meta_sync_slots_from_ss(): Syncs SP metadata from SuperSlab - Modified shared_pool_release_slab(): Detects SLOT_ACTIVE mismatch - On mismatch: Syncs from SuperSlab bitmap/class_map, then proceeds - Allows EMPTY transition → freelist insertion → registry unregister Implementation: 1. sp_meta_sync_slots_from_ss() (core/hakmem_shared_pool.c:418-452) - Rebuilds slot states from SuperSlab->slab_bitmap - Updates total_slots, active_slots, class_idx - Handles SLOT_ACTIVE, SLOT_EMPTY, SLOT_UNUSED states 2. shared_pool_release_slab() (core/hakmem_shared_pool.c:1336-1349) - Checks slot_state != SLOT_ACTIVE but slab_bitmap set - Calls sp_meta_sync_slots_from_ss() to rebuild state - Allows normal EMPTY flow to proceed Results (verified by testing): - "SuperSlab registry full" errors: ELIMINATED (0 occurrences) - Throughput: 118-125 M ops/sec (stable) - 3 consecutive stress tests: All passed - Medium load test (15K iterations): Success Nature of Fix: - WORKAROUND (not root cause fix) - Detects and repairs inconsistency at release time - Root fix would require: Legacy path elimination + unified architecture - This fix ensures stability while preserving existing code paths Next Steps: - Benchmark performance improvement vs Phase 9-1 baseline - Plan root cause fix (Phase 10): Unify SuperSlab management - Consider gradual Legacy path deprecation Credit: ChatGPT for root cause analysis and implementation 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
1425 lines
54 KiB
C
1425 lines
54 KiB
C
#include "hakmem_shared_pool.h"
|
||
#include "hakmem_tiny_superslab.h"
|
||
#include "hakmem_tiny_superslab_constants.h"
|
||
#include "hakmem_debug_master.h" // Phase 4b: Master debug control
|
||
#include "hakmem_stats_master.h" // Phase 4d: Master stats control
|
||
#include "box/ss_slab_meta_box.h" // Phase 3d-A: SlabMeta Box boundary
|
||
#include "box/ss_hot_cold_box.h" // Phase 12-1.1: EMPTY slab marking
|
||
#include "box/pagefault_telemetry_box.h" // Box PageFaultTelemetry (PF_BUCKET_SS_META)
|
||
#include "box/tls_sll_drain_box.h" // Box TLS SLL Drain (tiny_tls_sll_drain)
|
||
#include "box/tls_slab_reuse_guard_box.h" // Box TLS Slab Reuse Guard (P0.3)
|
||
#include "hakmem_policy.h" // FrozenPolicy (learning layer)
|
||
|
||
#include <stdlib.h>
|
||
#include <string.h>
|
||
#include <stdatomic.h>
|
||
#include <stdio.h>
|
||
#include <sys/mman.h> // For mmap/munmap (used in shared_pool_ensure_capacity_unlocked)
|
||
|
||
// ============================================================================
|
||
// P0 Lock Contention Instrumentation (Debug build only; counters defined always)
|
||
// ============================================================================
|
||
static _Atomic uint64_t g_lock_acquire_count = 0; // Total lock acquisitions
|
||
static _Atomic uint64_t g_lock_release_count = 0; // Total lock releases
|
||
static _Atomic uint64_t g_lock_acquire_slab_count = 0; // Locks from acquire_slab path
|
||
static _Atomic uint64_t g_lock_release_slab_count = 0; // Locks from release_slab path
|
||
static int g_lock_stats_enabled = -1; // -1=uninitialized, 0=off, 1=on
|
||
|
||
#if !HAKMEM_BUILD_RELEASE
|
||
// Initialize lock stats from environment variable
|
||
// Phase 4b: Now uses hak_debug_check() for master debug control support
|
||
static inline void lock_stats_init(void) {
|
||
if (__builtin_expect(g_lock_stats_enabled == -1, 0)) {
|
||
g_lock_stats_enabled = hak_debug_check("HAKMEM_SHARED_POOL_LOCK_STATS");
|
||
}
|
||
}
|
||
|
||
// Report lock statistics at shutdown
|
||
static void __attribute__((destructor)) lock_stats_report(void) {
|
||
if (g_lock_stats_enabled != 1) {
|
||
return;
|
||
}
|
||
|
||
uint64_t acquires = atomic_load(&g_lock_acquire_count);
|
||
uint64_t releases = atomic_load(&g_lock_release_count);
|
||
uint64_t acquire_path = atomic_load(&g_lock_acquire_slab_count);
|
||
uint64_t release_path = atomic_load(&g_lock_release_slab_count);
|
||
|
||
fprintf(stderr, "\n=== SHARED POOL LOCK STATISTICS ===\n");
|
||
fprintf(stderr, "Total lock ops: %lu (acquire) + %lu (release) = %lu\n",
|
||
acquires, releases, acquires + releases);
|
||
fprintf(stderr, "Balance: %ld (should be 0)\n",
|
||
(int64_t)acquires - (int64_t)releases);
|
||
fprintf(stderr, "\n--- Breakdown by Code Path ---\n");
|
||
fprintf(stderr, "acquire_slab(): %lu (%.1f%%)\n",
|
||
acquire_path, 100.0 * acquire_path / (acquires ? acquires : 1));
|
||
fprintf(stderr, "release_slab(): %lu (%.1f%%)\n",
|
||
release_path, 100.0 * release_path / (acquires ? acquires : 1));
|
||
fprintf(stderr, "===================================\n");
|
||
fflush(stderr);
|
||
}
|
||
#else
|
||
// Release build: No-op stubs
|
||
static inline void lock_stats_init(void) {
|
||
if (__builtin_expect(g_lock_stats_enabled == -1, 0)) {
|
||
g_lock_stats_enabled = 0;
|
||
}
|
||
}
|
||
#endif
|
||
|
||
// ============================================================================
|
||
// SP Acquire Stage Statistics (Stage1/2/3 breakdown)
|
||
// ============================================================================
|
||
static _Atomic uint64_t g_sp_stage1_hits[TINY_NUM_CLASSES_SS];
|
||
static _Atomic uint64_t g_sp_stage2_hits[TINY_NUM_CLASSES_SS];
|
||
static _Atomic uint64_t g_sp_stage3_hits[TINY_NUM_CLASSES_SS];
|
||
// Data collection gate (0=off, 1=on). 学習層からも有効化される。
|
||
static int g_sp_stage_stats_enabled = 0;
|
||
|
||
#if !HAKMEM_BUILD_RELEASE
|
||
// Logging gate for destructor（ENV: HAKMEM_SHARED_POOL_STAGE_STATS）
|
||
static int g_sp_stage_stats_log_enabled = -1; // -1=uninitialized, 0=off, 1=on
|
||
|
||
static inline void sp_stage_stats_init(void) {
|
||
// Phase 4d: Now uses hak_stats_check() for unified stats control
|
||
if (__builtin_expect(g_sp_stage_stats_log_enabled == -1, 0)) {
|
||
g_sp_stage_stats_log_enabled = hak_stats_check("HAKMEM_SHARED_POOL_STAGE_STATS", "pool");
|
||
if (g_sp_stage_stats_log_enabled == 1) {
|
||
// ログが有効なら計測も必ず有効化する。
|
||
g_sp_stage_stats_enabled = 1;
|
||
}
|
||
}
|
||
}
|
||
|
||
static void __attribute__((destructor)) sp_stage_stats_report(void) {
|
||
if (g_sp_stage_stats_log_enabled != 1) {
|
||
return;
|
||
}
|
||
|
||
fprintf(stderr, "\n=== SHARED POOL STAGE STATISTICS ===\n");
|
||
fprintf(stderr, "Per-class acquire_slab() stage hits (Stage1=EMPTY, Stage2=UNUSED, Stage3=new SS)\n");
|
||
|
||
for (int cls = 0; cls < TINY_NUM_CLASSES_SS; cls++) {
|
||
uint64_t s1 = atomic_load(&g_sp_stage1_hits[cls]);
|
||
uint64_t s2 = atomic_load(&g_sp_stage2_hits[cls]);
|
||
uint64_t s3 = atomic_load(&g_sp_stage3_hits[cls]);
|
||
uint64_t total = s1 + s2 + s3;
|
||
if (total == 0) continue; // Skip unused classes
|
||
|
||
double p1 = 100.0 * (double)s1 / (double)total;
|
||
double p2 = 100.0 * (double)s2 / (double)total;
|
||
double p3 = 100.0 * (double)s3 / (double)total;
|
||
|
||
fprintf(stderr,
|
||
"Class %d: total=%llu S1=%llu (%.1f%%) S2=%llu (%.1f%%) S3=%llu (%.1f%%)\n",
|
||
cls,
|
||
(unsigned long long)total,
|
||
(unsigned long long)s1, p1,
|
||
(unsigned long long)s2, p2,
|
||
(unsigned long long)s3, p3);
|
||
}
|
||
fprintf(stderr, "====================================\n");
|
||
fflush(stderr);
|
||
}
|
||
#else
|
||
// Release build: No-op stubs
|
||
static inline void sp_stage_stats_init(void) {}
|
||
#endif
|
||
|
||
// Snapshot Tiny-related backend metrics for learner / observability.
|
||
void
|
||
shared_pool_tiny_metrics_snapshot(uint64_t stage1[TINY_NUM_CLASSES_SS],
|
||
uint64_t stage2[TINY_NUM_CLASSES_SS],
|
||
uint64_t stage3[TINY_NUM_CLASSES_SS],
|
||
uint32_t active_slots[TINY_NUM_CLASSES_SS])
|
||
{
|
||
// Ensure env-based logging設定の初期化だけ先に済ませる。
|
||
sp_stage_stats_init();
|
||
// 学習層から呼ばれた場合は、計測自体は常に有効化する（ログは env で制御）。
|
||
g_sp_stage_stats_enabled = 1;
|
||
|
||
for (int cls = 0; cls < TINY_NUM_CLASSES_SS; cls++) {
|
||
if (stage1) {
|
||
stage1[cls] = atomic_load_explicit(&g_sp_stage1_hits[cls],
|
||
memory_order_relaxed);
|
||
}
|
||
if (stage2) {
|
||
stage2[cls] = atomic_load_explicit(&g_sp_stage2_hits[cls],
|
||
memory_order_relaxed);
|
||
}
|
||
if (stage3) {
|
||
stage3[cls] = atomic_load_explicit(&g_sp_stage3_hits[cls],
|
||
memory_order_relaxed);
|
||
}
|
||
if (active_slots) {
|
||
active_slots[cls] = g_shared_pool.class_active_slots[cls];
|
||
}
|
||
}
|
||
}
|
||
|
||
// Helper: return per-class active slot limit from FrozenPolicy.tiny_cap[]
|
||
// Semantics:
|
||
// - tiny_cap[class] == 0 → no limit (unbounded)
|
||
// - otherwise: soft cap on ACTIVE slots managed by shared pool for this class.
|
||
static inline uint32_t sp_class_active_limit(int class_idx) {
|
||
const FrozenPolicy* pol = hkm_policy_get();
|
||
if (!pol) {
|
||
return 0; // no limit
|
||
}
|
||
if (class_idx < 0 || class_idx >= 8) {
|
||
return 0;
|
||
}
|
||
return (uint32_t)pol->tiny_cap[class_idx];
|
||
}
|
||
|
||
// ============================================================================
|
||
// Superslab L0 Cache (per-thread, per-class hot slot)
|
||
// ============================================================================
|
||
//
|
||
// Goal:
|
||
// - Avoid calling shared_pool_acquire_slab()'s full 3-stage logic on every
|
||
// allocation when the same (ss, slab_idx) still has room.
|
||
// - Keep Box boundaries: slot ownership/state is still managed by SP-SLOT,
|
||
// L0 では「既に ACTIVE な slot を再利用するだけ」（UNUSED/EMPTY には触れない）。
|
||
//
|
||
// Design:
|
||
// - Per-thread TLS for each tiny class (0..TINY_NUM_CLASSES_SS-1):
|
||
// - SharedSSMeta* meta
|
||
// - uint8_t slot_idx
|
||
// - Stage 0 in shared_pool_acquire_slab():
|
||
// - If L0 entry exists and meta->ss is non-NULL and
|
||
// ss->slabs[slot_idx] is still bound to this class,
|
||
// return (ss, slot_idx) directly without touching locks or lists.
|
||
// - If SuperSlab has been freed (meta->ss == NULL) or slot reused,
|
||
// L0 エントリを破棄して通常の Stage 1-3 にフォールバック。
|
||
//
|
||
// Env:
|
||
// - HAKMEM_SS_L0=0 → L0 無効
|
||
// - HAKMEM_SS_L0=1 → L0 有効（デフォルト）
|
||
|
||
static __thread SharedSSMeta* g_sp_l0_meta[TINY_NUM_CLASSES_SS];
|
||
static __thread uint8_t g_sp_l0_slot[TINY_NUM_CLASSES_SS];
|
||
|
||
// NOTE: L0 は実験段階のため、現行ビルドでは常に無効化したままにする。
|
||
// 将来の安定版で再度有効化する場合は、実装と検証をやり直すこと。
|
||
static inline int sp_l0_enabled(void) {
|
||
(void)g_sp_l0_meta;
|
||
(void)g_sp_l0_slot;
|
||
return 0; // Disabled for now
|
||
}
|
||
|
||
// ============================================================================
|
||
// P0-4: Lock-Free Free Slot List - Node Pool
|
||
// ============================================================================
|
||
|
||
// Pre-allocated node pools (one per class, to avoid malloc/free)
|
||
FreeSlotNode g_free_node_pool[TINY_NUM_CLASSES_SS][MAX_FREE_NODES_PER_CLASS];
|
||
_Atomic uint32_t g_node_alloc_index[TINY_NUM_CLASSES_SS] = {0};
|
||
|
||
// Recycle list for FreeSlotNode (per class, lock-free LIFO).
|
||
// node_alloc() はまずこのリストから再利用を試み、枯渇時のみ新規ノードを切り出す。
|
||
static _Atomic(FreeSlotNode*) g_node_free_head[TINY_NUM_CLASSES_SS] = {
|
||
[0 ... TINY_NUM_CLASSES_SS-1] = ATOMIC_VAR_INIT(NULL)
|
||
};
|
||
|
||
// Allocate a node from pool (lock-free fast path, may fall back to legacy path)
|
||
static inline FreeSlotNode* node_alloc(int class_idx) {
|
||
if (class_idx < 0 || class_idx >= TINY_NUM_CLASSES_SS) {
|
||
return NULL;
|
||
}
|
||
|
||
// First, try to pop from recycle list (nodes returned by pop_lockfree).
|
||
FreeSlotNode* free_head = atomic_load_explicit(
|
||
&g_node_free_head[class_idx],
|
||
memory_order_acquire);
|
||
while (free_head != NULL) {
|
||
FreeSlotNode* next = free_head->next;
|
||
if (atomic_compare_exchange_weak_explicit(
|
||
&g_node_free_head[class_idx],
|
||
&free_head,
|
||
next,
|
||
memory_order_acq_rel,
|
||
memory_order_acquire)) {
|
||
return free_head; // Recycled node
|
||
}
|
||
// CAS failed: free_head is updated; retry with new head.
|
||
}
|
||
|
||
uint32_t idx = atomic_fetch_add(&g_node_alloc_index[class_idx], 1);
|
||
if (idx >= MAX_FREE_NODES_PER_CLASS) {
|
||
// Pool exhausted - should be rare. Caller must fall back to legacy
|
||
// mutex-protected free list to preserve correctness.
|
||
#if !HAKMEM_BUILD_RELEASE
|
||
static _Atomic int warn_once = 0;
|
||
if (atomic_exchange(&warn_once, 1) == 0) {
|
||
fprintf(stderr, "[P0-4 WARN] Node pool exhausted for class %d\n", class_idx);
|
||
}
|
||
#endif
|
||
return NULL;
|
||
}
|
||
|
||
return &g_free_node_pool[class_idx][idx];
|
||
}
|
||
|
||
// ============================================================================
|
||
// Phase 12-2: SharedSuperSlabPool skeleton implementation
|
||
// Goal:
|
||
// - Centralize SuperSlab allocation/registration
|
||
// - Provide acquire_slab/release_slab APIs for later refill/free integration
|
||
// - Keep logic simple & conservative; correctness and observability first.
|
||
//
|
||
// Notes:
|
||
// - Concurrency: protected by g_shared_pool.alloc_lock for now.
|
||
// - class_hints is best-effort: read lock-free, written under lock.
|
||
// - LRU hooks left as no-op placeholders.
|
||
|
||
SharedSuperSlabPool g_shared_pool = {
|
||
.slabs = NULL,
|
||
.capacity = 0,
|
||
.total_count = 0,
|
||
.active_count = 0,
|
||
.alloc_lock = PTHREAD_MUTEX_INITIALIZER,
|
||
.class_hints = { NULL },
|
||
.lru_head = NULL,
|
||
.lru_tail = NULL,
|
||
.lru_count = 0,
|
||
// P0-4: Lock-free free slot lists (zero-initialized atomic pointers)
|
||
.free_slots_lockfree = {{.head = ATOMIC_VAR_INIT(NULL)}},
|
||
// Legacy: mutex-protected free lists
|
||
.free_slots = {{.entries = {{0}}, .count = 0}},
|
||
// Phase 12: SP-SLOT fields (ss_metadata is fixed-size array, auto-zeroed)
|
||
.ss_meta_count = 0
|
||
};
|
||
|
||
static void
|
||
shared_pool_ensure_capacity_unlocked(uint32_t min_capacity)
|
||
{
|
||
if (g_shared_pool.capacity >= min_capacity) {
|
||
return;
|
||
}
|
||
|
||
uint32_t new_cap = g_shared_pool.capacity ? g_shared_pool.capacity : 16;
|
||
while (new_cap < min_capacity) {
|
||
new_cap *= 2;
|
||
}
|
||
|
||
// CRITICAL FIX: Use system mmap() directly to avoid recursion!
|
||
// Problem: realloc() goes through HAKMEM allocator → hak_alloc_at(128)
|
||
// → needs Shared Pool init → calls realloc() → INFINITE RECURSION!
|
||
// Solution: Allocate Shared Pool metadata using system mmap, not HAKMEM allocator
|
||
size_t new_size = new_cap * sizeof(SuperSlab*);
|
||
SuperSlab** new_slabs = (SuperSlab**)mmap(NULL, new_size,
|
||
PROT_READ | PROT_WRITE,
|
||
MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
|
||
if (new_slabs == MAP_FAILED) {
|
||
// Allocation failure: keep old state; caller must handle NULL later.
|
||
return;
|
||
}
|
||
|
||
// Copy old data if exists
|
||
if (g_shared_pool.slabs != NULL) {
|
||
memcpy(new_slabs, g_shared_pool.slabs,
|
||
g_shared_pool.capacity * sizeof(SuperSlab*));
|
||
// Free old mapping (also use system munmap, not free!)
|
||
size_t old_size = g_shared_pool.capacity * sizeof(SuperSlab*);
|
||
munmap(g_shared_pool.slabs, old_size);
|
||
}
|
||
|
||
// Zero new entries to keep scanning logic simple.
|
||
memset(new_slabs + g_shared_pool.capacity, 0,
|
||
(new_cap - g_shared_pool.capacity) * sizeof(SuperSlab*));
|
||
|
||
g_shared_pool.slabs = new_slabs;
|
||
g_shared_pool.capacity = new_cap;
|
||
}
|
||
|
||
void
|
||
shared_pool_init(void)
|
||
{
|
||
// Idempotent init; safe to call from multiple early paths.
|
||
// pthread_mutex_t with static initializer is already valid.
|
||
pthread_mutex_lock(&g_shared_pool.alloc_lock);
|
||
if (g_shared_pool.capacity == 0 && g_shared_pool.slabs == NULL) {
|
||
shared_pool_ensure_capacity_unlocked(16);
|
||
}
|
||
pthread_mutex_unlock(&g_shared_pool.alloc_lock);
|
||
}
|
||
|
||
// ============================================================================
|
||
// Phase 12: SP-SLOT Box - Modular Helper Functions
|
||
// ============================================================================
|
||
|
||
// ---------- Layer 1: Slot Operations (Low-level) ----------
|
||
|
||
// Find first unused slot in SharedSSMeta
|
||
// P0-5: Uses atomic load for state check
|
||
// Returns: slot_idx on success, -1 if no unused slots
|
||
static int sp_slot_find_unused(SharedSSMeta* meta) {
|
||
if (!meta) return -1;
|
||
|
||
for (int i = 0; i < meta->total_slots; i++) {
|
||
SlotState state = atomic_load_explicit(&meta->slots[i].state, memory_order_acquire);
|
||
if (state == SLOT_UNUSED) {
|
||
return i;
|
||
}
|
||
}
|
||
return -1;
|
||
}
|
||
|
||
// Mark slot as ACTIVE (UNUSED→ACTIVE or EMPTY→ACTIVE)
|
||
// P0-5: Uses atomic store for state transition (caller must hold mutex!)
|
||
// Returns: 0 on success, -1 on error
|
||
static int sp_slot_mark_active(SharedSSMeta* meta, int slot_idx, int class_idx) {
|
||
if (!meta || slot_idx < 0 || slot_idx >= meta->total_slots) return -1;
|
||
if (class_idx < 0 || class_idx >= TINY_NUM_CLASSES_SS) return -1;
|
||
|
||
SharedSlot* slot = &meta->slots[slot_idx];
|
||
|
||
// Load state atomically
|
||
SlotState state = atomic_load_explicit(&slot->state, memory_order_acquire);
|
||
|
||
// Transition: UNUSED→ACTIVE or EMPTY→ACTIVE
|
||
if (state == SLOT_UNUSED || state == SLOT_EMPTY) {
|
||
atomic_store_explicit(&slot->state, SLOT_ACTIVE, memory_order_release);
|
||
slot->class_idx = (uint8_t)class_idx;
|
||
slot->slab_idx = (uint8_t)slot_idx;
|
||
meta->active_slots++;
|
||
return 0;
|
||
}
|
||
|
||
return -1; // Already ACTIVE or invalid state
|
||
}
|
||
|
||
// Mark slot as EMPTY (ACTIVE→EMPTY)
|
||
// P0-5: Uses atomic store for state transition (caller must hold mutex!)
|
||
// Returns: 0 on success, -1 on error
|
||
static int sp_slot_mark_empty(SharedSSMeta* meta, int slot_idx) {
|
||
if (!meta || slot_idx < 0 || slot_idx >= meta->total_slots) return -1;
|
||
|
||
SharedSlot* slot = &meta->slots[slot_idx];
|
||
|
||
// Load state atomically
|
||
SlotState state = atomic_load_explicit(&slot->state, memory_order_acquire);
|
||
|
||
if (state == SLOT_ACTIVE) {
|
||
atomic_store_explicit(&slot->state, SLOT_EMPTY, memory_order_release);
|
||
if (meta->active_slots > 0) {
|
||
meta->active_slots--;
|
||
}
|
||
return 0;
|
||
}
|
||
|
||
return -1; // Not ACTIVE
|
||
}
|
||
|
||
// Sync SP-SLOT view from an existing SuperSlab.
|
||
// This is needed when a legacy-allocated SuperSlab reaches the shared-pool
|
||
// release path for the first time (slot states are still SLOT_UNUSED).
|
||
static void sp_meta_sync_slots_from_ss(SharedSSMeta* meta, SuperSlab* ss) {
|
||
if (!meta || !ss) return;
|
||
|
||
int cap = ss_slabs_capacity(ss);
|
||
if (cap > MAX_SLOTS_PER_SS) {
|
||
cap = MAX_SLOTS_PER_SS;
|
||
}
|
||
|
||
meta->total_slots = (uint8_t)cap;
|
||
meta->active_slots = 0;
|
||
|
||
for (int i = 0; i < cap; i++) {
|
||
SlotState state = SLOT_UNUSED;
|
||
uint32_t bit = (1u << i);
|
||
if (ss->slab_bitmap & bit) {
|
||
state = SLOT_ACTIVE;
|
||
meta->active_slots++;
|
||
} else {
|
||
TinySlabMeta* smeta = &ss->slabs[i];
|
||
uint16_t used = atomic_load_explicit(&smeta->used, memory_order_relaxed);
|
||
if (smeta->capacity > 0 && used == 0) {
|
||
state = SLOT_EMPTY;
|
||
}
|
||
}
|
||
|
||
uint8_t cls = ss->class_map[i];
|
||
if (cls == 255) {
|
||
cls = ss->slabs[i].class_idx;
|
||
}
|
||
|
||
meta->slots[i].class_idx = cls;
|
||
meta->slots[i].slab_idx = (uint8_t)i;
|
||
atomic_store_explicit(&meta->slots[i].state, state, memory_order_release);
|
||
}
|
||
}
|
||
|
||
// ---------- Layer 2: Metadata Management (Mid-level) ----------
|
||
|
||
// Ensure ss_metadata array has capacity for at least min_count entries
|
||
// Caller must hold alloc_lock
|
||
// Returns: 0 on success, -1 if capacity exceeded
|
||
// RACE FIX: No realloc! Fixed-size array prevents race with lock-free Stage 2
|
||
static int sp_meta_ensure_capacity(uint32_t min_count) {
|
||
if (min_count > MAX_SS_METADATA_ENTRIES) {
|
||
#if !HAKMEM_BUILD_RELEASE
|
||
static int warn_once = 0;
|
||
if (warn_once == 0) {
|
||
fprintf(stderr, "[SP_META_CAPACITY_ERROR] Exceeded MAX_SS_METADATA_ENTRIES=%d\n",
|
||
MAX_SS_METADATA_ENTRIES);
|
||
warn_once = 1;
|
||
}
|
||
#endif
|
||
return -1;
|
||
}
|
||
return 0;
|
||
}
|
||
|
||
// Find SharedSSMeta for given SuperSlab, or create if not exists
|
||
// Caller must hold alloc_lock
|
||
// Returns: SharedSSMeta* on success, NULL on error
|
||
static SharedSSMeta* sp_meta_find_or_create(SuperSlab* ss) {
|
||
if (!ss) return NULL;
|
||
|
||
// RACE FIX: Load count atomically for consistency (even under mutex)
|
||
uint32_t count = atomic_load_explicit(&g_shared_pool.ss_meta_count, memory_order_relaxed);
|
||
|
||
// Search existing metadata
|
||
for (uint32_t i = 0; i < count; i++) {
|
||
// RACE FIX: Load pointer atomically for consistency
|
||
SuperSlab* meta_ss = atomic_load_explicit(&g_shared_pool.ss_metadata[i].ss, memory_order_relaxed);
|
||
if (meta_ss == ss) {
|
||
return &g_shared_pool.ss_metadata[i];
|
||
}
|
||
}
|
||
|
||
// Create new metadata entry
|
||
if (sp_meta_ensure_capacity(count + 1) != 0) {
|
||
return NULL;
|
||
}
|
||
|
||
// RACE FIX: Read current count atomically (even under mutex for consistency)
|
||
uint32_t current_count = atomic_load_explicit(&g_shared_pool.ss_meta_count, memory_order_relaxed);
|
||
SharedSSMeta* meta = &g_shared_pool.ss_metadata[current_count];
|
||
|
||
// RACE FIX: Store SuperSlab pointer atomically (visible to lock-free Stage 2)
|
||
atomic_store_explicit(&meta->ss, ss, memory_order_relaxed);
|
||
meta->total_slots = (uint8_t)ss_slabs_capacity(ss);
|
||
meta->active_slots = 0;
|
||
|
||
// Initialize all slots as UNUSED
|
||
// P0-5: Use atomic store for state initialization
|
||
for (int i = 0; i < meta->total_slots; i++) {
|
||
atomic_store_explicit(&meta->slots[i].state, SLOT_UNUSED, memory_order_relaxed);
|
||
meta->slots[i].class_idx = 0;
|
||
meta->slots[i].slab_idx = (uint8_t)i;
|
||
}
|
||
|
||
// RACE FIX: Atomic increment with release semantics
|
||
// This ensures all writes to metadata[current_count] (lines 268-278) are visible
|
||
// before the count increment is visible to lock-free Stage 2 readers
|
||
atomic_fetch_add_explicit(&g_shared_pool.ss_meta_count, 1, memory_order_release);
|
||
return meta;
|
||
}
|
||
|
||
// ---------- Layer 3: Free List Management ----------
|
||
|
||
// Push empty slot to per-class free list
|
||
// Caller must hold alloc_lock
|
||
// Returns: 0 on success, -1 if list is full
|
||
static int sp_freelist_push(int class_idx, SharedSSMeta* meta, int slot_idx) {
|
||
if (class_idx < 0 || class_idx >= TINY_NUM_CLASSES_SS) return -1;
|
||
if (!meta || slot_idx < 0 || slot_idx >= meta->total_slots) return -1;
|
||
|
||
FreeSlotList* list = &g_shared_pool.free_slots[class_idx];
|
||
|
||
if (list->count >= MAX_FREE_SLOTS_PER_CLASS) {
|
||
return -1; // List full
|
||
}
|
||
|
||
list->entries[list->count].meta = meta;
|
||
list->entries[list->count].slot_idx = (uint8_t)slot_idx;
|
||
list->count++;
|
||
return 0;
|
||
}
|
||
|
||
// Pop empty slot from per-class free list
|
||
// Caller must hold alloc_lock
|
||
// Returns: 1 if popped (out params filled), 0 if list empty
|
||
static int sp_freelist_pop(int class_idx, SharedSSMeta** out_meta, int* out_slot_idx) {
|
||
if (class_idx < 0 || class_idx >= TINY_NUM_CLASSES_SS) return 0;
|
||
if (!out_meta || !out_slot_idx) return 0;
|
||
|
||
FreeSlotList* list = &g_shared_pool.free_slots[class_idx];
|
||
|
||
if (list->count == 0) {
|
||
return 0; // List empty
|
||
}
|
||
|
||
// Pop from end (LIFO for cache locality)
|
||
list->count--;
|
||
*out_meta = list->entries[list->count].meta;
|
||
*out_slot_idx = list->entries[list->count].slot_idx;
|
||
return 1;
|
||
}
|
||
|
||
// ============================================================================
|
||
// P0-5: Lock-Free Slot Claiming (Stage 2 Optimization)
|
||
// ============================================================================
|
||
|
||
// Try to claim an UNUSED slot via lock-free CAS
|
||
// Returns: slot_idx on success, -1 if no UNUSED slots available
|
||
// LOCK-FREE: Can be called from any thread without mutex
|
||
static int sp_slot_claim_lockfree(SharedSSMeta* meta, int class_idx) {
|
||
if (!meta) return -1;
|
||
if (class_idx < 0 || class_idx >= TINY_NUM_CLASSES_SS) return -1;
|
||
|
||
// Scan all slots for UNUSED state
|
||
for (int i = 0; i < meta->total_slots; i++) {
|
||
SlotState expected = SLOT_UNUSED;
|
||
|
||
// Try to claim this slot atomically (UNUSED → ACTIVE)
|
||
if (atomic_compare_exchange_strong_explicit(
|
||
&meta->slots[i].state,
|
||
&expected,
|
||
SLOT_ACTIVE,
|
||
memory_order_acq_rel, // Success: acquire+release semantics
|
||
memory_order_relaxed // Failure: just retry next slot
|
||
)) {
|
||
// Successfully claimed! Update non-atomic fields
|
||
// (Safe because we now own this slot)
|
||
meta->slots[i].class_idx = (uint8_t)class_idx;
|
||
meta->slots[i].slab_idx = (uint8_t)i;
|
||
|
||
// Increment active_slots counter atomically
|
||
// (Multiple threads may claim slots concurrently)
|
||
atomic_fetch_add_explicit(
|
||
(_Atomic uint8_t*)&meta->active_slots, 1,
|
||
memory_order_relaxed
|
||
);
|
||
|
||
return i; // Return claimed slot index
|
||
}
|
||
|
||
// CAS failed (slot was not UNUSED) - continue to next slot
|
||
}
|
||
|
||
return -1; // No UNUSED slots available
|
||
}
|
||
|
||
// ============================================================================
|
||
// P0-4: Lock-Free Free Slot List Operations
|
||
// ============================================================================
|
||
|
||
// Push empty slot to lock-free per-class free list (LIFO)
|
||
// LOCK-FREE: Can be called from any thread without mutex
|
||
// Returns: 0 on success, -1 on failure (node pool exhausted)
|
||
static int sp_freelist_push_lockfree(int class_idx, SharedSSMeta* meta, int slot_idx) {
|
||
if (class_idx < 0 || class_idx >= TINY_NUM_CLASSES_SS) return -1;
|
||
if (!meta || slot_idx < 0 || slot_idx >= meta->total_slots) return -1;
|
||
|
||
// Allocate node from pool
|
||
FreeSlotNode* node = node_alloc(class_idx);
|
||
if (!node) {
|
||
// Fallback: push into legacy per-class free list
|
||
// ASSUME: Caller already holds alloc_lock (e.g., shared_pool_release_slab:772)
|
||
// Do NOT lock again to avoid deadlock on non-recursive mutex!
|
||
(void)sp_freelist_push(class_idx, meta, slot_idx);
|
||
return 0;
|
||
}
|
||
|
||
// Fill node data
|
||
node->meta = meta;
|
||
node->slot_idx = (uint8_t)slot_idx;
|
||
|
||
// Lock-free LIFO push using CAS loop
|
||
LockFreeFreeList* list = &g_shared_pool.free_slots_lockfree[class_idx];
|
||
FreeSlotNode* old_head = atomic_load_explicit(&list->head, memory_order_relaxed);
|
||
|
||
do {
|
||
node->next = old_head;
|
||
} while (!atomic_compare_exchange_weak_explicit(
|
||
&list->head, &old_head, node,
|
||
memory_order_release, // Success: publish node to other threads
|
||
memory_order_relaxed // Failure: retry with updated old_head
|
||
));
|
||
|
||
return 0; // Success
|
||
}
|
||
|
||
// Pop empty slot from lock-free per-class free list (LIFO)
|
||
// LOCK-FREE: Can be called from any thread without mutex
|
||
// Returns: 1 if popped (out params filled), 0 if list empty
|
||
static int sp_freelist_pop_lockfree(int class_idx, SharedSSMeta** out_meta, int* out_slot_idx) {
|
||
if (class_idx < 0 || class_idx >= TINY_NUM_CLASSES_SS) return 0;
|
||
if (!out_meta || !out_slot_idx) return 0;
|
||
|
||
LockFreeFreeList* list = &g_shared_pool.free_slots_lockfree[class_idx];
|
||
FreeSlotNode* old_head = atomic_load_explicit(&list->head, memory_order_acquire);
|
||
|
||
// Lock-free LIFO pop using CAS loop
|
||
do {
|
||
if (old_head == NULL) {
|
||
return 0; // List empty
|
||
}
|
||
} while (!atomic_compare_exchange_weak_explicit(
|
||
&list->head, &old_head, old_head->next,
|
||
memory_order_acquire, // Success: acquire node data
|
||
memory_order_acquire // Failure: retry with updated old_head
|
||
));
|
||
|
||
// Extract data from popped node
|
||
*out_meta = old_head->meta;
|
||
*out_slot_idx = old_head->slot_idx;
|
||
|
||
// Recycle node back into per-class free list so that long-running workloads
|
||
// do not permanently consume new nodes on every EMPTY event.
|
||
FreeSlotNode* free_head = atomic_load_explicit(
|
||
&g_node_free_head[class_idx],
|
||
memory_order_acquire);
|
||
do {
|
||
old_head->next = free_head;
|
||
} while (!atomic_compare_exchange_weak_explicit(
|
||
&g_node_free_head[class_idx],
|
||
&free_head,
|
||
old_head,
|
||
memory_order_release,
|
||
memory_order_acquire));
|
||
|
||
return 1; // Success
|
||
}
|
||
|
||
/*
|
||
* Internal: allocate and register a new SuperSlab for the shared pool.
|
||
*
|
||
* Phase 12 NOTE:
|
||
* - We MUST use the real superslab_allocate() path so that:
|
||
* - backing memory is a full SuperSlab region (1–2MB),
|
||
* - header/layout are initialized correctly,
|
||
* - registry integration stays consistent.
|
||
* - shared_pool is responsible only for:
|
||
* - tracking pointers,
|
||
* - marking per-slab class_idx as UNASSIGNED initially.
|
||
* It does NOT bypass registry/LRU.
|
||
*
|
||
* Caller must hold alloc_lock.
|
||
*/
|
||
static SuperSlab*
|
||
shared_pool_allocate_superslab_unlocked(void)
|
||
{
|
||
// Use size_class 0 as a neutral hint; Phase 12 per-slab class_idx is authoritative.
|
||
extern SuperSlab* superslab_allocate(uint8_t size_class);
|
||
SuperSlab* ss = superslab_allocate(0);
|
||
|
||
if (!ss) {
|
||
return NULL;
|
||
}
|
||
|
||
// PageFaultTelemetry: mark all backing pages for this Superslab (approximate)
|
||
size_t ss_bytes = (size_t)1 << ss->lg_size;
|
||
for (size_t off = 0; off < ss_bytes; off += 4096) {
|
||
pagefault_telemetry_touch(PF_BUCKET_SS_META, (char*)ss + off);
|
||
}
|
||
|
||
// superslab_allocate() already:
|
||
// - zeroes slab metadata / remote queues,
|
||
// - sets magic/lg_size/etc,
|
||
// - registers in global registry.
|
||
// For shared-pool semantics we normalize all slab class_idx to UNASSIGNED.
|
||
int max_slabs = ss_slabs_capacity(ss);
|
||
for (int i = 0; i < max_slabs; i++) {
|
||
ss_slab_meta_class_idx_set(ss, i, 255); // UNASSIGNED
|
||
// P1.1: Initialize class_map to UNASSIGNED as well
|
||
ss->class_map[i] = 255;
|
||
}
|
||
|
||
if (g_shared_pool.total_count >= g_shared_pool.capacity) {
|
||
shared_pool_ensure_capacity_unlocked(g_shared_pool.total_count + 1);
|
||
if (g_shared_pool.total_count >= g_shared_pool.capacity) {
|
||
// Pool table expansion failed; leave ss alive (registry-owned),
|
||
// but do not treat it as part of shared_pool.
|
||
return NULL;
|
||
}
|
||
}
|
||
|
||
g_shared_pool.slabs[g_shared_pool.total_count] = ss;
|
||
g_shared_pool.total_count++;
|
||
// Not counted as active until at least one slab is assigned.
|
||
return ss;
|
||
}
|
||
|
||
SuperSlab*
|
||
shared_pool_acquire_superslab(void)
|
||
{
|
||
// Phase 12 debug safety:
|
||
// If shared backend is disabled at Box API level, this function SHOULD NOT be called.
|
||
// But since bench currently SEGVs here even with legacy forced, treat this as a hard guard:
|
||
// we early-return error instead of touching potentially-bad state.
|
||
//
|
||
// This isolates shared_pool from the current crash so we can validate legacy path first.
|
||
// FIXED: Remove the return -1; that was preventing operation
|
||
|
||
shared_pool_init();
|
||
|
||
pthread_mutex_lock(&g_shared_pool.alloc_lock);
|
||
|
||
// For now, always allocate a fresh SuperSlab and register it.
|
||
// More advanced reuse/GC comes later.
|
||
SuperSlab* ss = shared_pool_allocate_superslab_unlocked();
|
||
|
||
pthread_mutex_unlock(&g_shared_pool.alloc_lock);
|
||
return ss;
|
||
}
|
||
|
||
// ---------- Layer 4: Public API (High-level) ----------
|
||
|
||
// Ensure slab geometry matches current class stride (handles upgrades like C7 1024->2048).
|
||
static inline void sp_fix_geometry_if_needed(SuperSlab* ss, int slab_idx, int class_idx)
|
||
{
|
||
if (!ss || slab_idx < 0 || class_idx < 0 || class_idx >= TINY_NUM_CLASSES_SS) {
|
||
return;
|
||
}
|
||
TinySlabMeta* meta = &ss->slabs[slab_idx];
|
||
size_t stride = g_tiny_class_sizes[class_idx];
|
||
size_t usable = (slab_idx == 0) ? SUPERSLAB_SLAB0_USABLE_SIZE : SUPERSLAB_SLAB_USABLE_SIZE;
|
||
uint16_t expect_cap = (uint16_t)(usable / stride);
|
||
|
||
// Reinitialize if capacity is off or class_idx mismatches.
|
||
if (meta->class_idx != (uint8_t)class_idx || meta->capacity != expect_cap) {
|
||
#if !HAKMEM_BUILD_RELEASE
|
||
extern __thread int g_hakmem_lock_depth;
|
||
g_hakmem_lock_depth++;
|
||
fprintf(stderr, "[SP_FIX_GEOMETRY] ss=%p slab=%d cls=%d: old_cls=%u old_cap=%u -> new_cls=%d new_cap=%u (stride=%zu)\n",
|
||
(void*)ss, slab_idx, class_idx,
|
||
meta->class_idx, meta->capacity,
|
||
class_idx, expect_cap, stride);
|
||
g_hakmem_lock_depth--;
|
||
#endif
|
||
|
||
superslab_init_slab(ss, slab_idx, stride, 0 /*owner_tid*/);
|
||
meta->class_idx = (uint8_t)class_idx;
|
||
// P1.1: Update class_map after geometry fix
|
||
ss->class_map[slab_idx] = (uint8_t)class_idx;
|
||
}
|
||
}
|
||
|
||
int
|
||
shared_pool_acquire_slab(int class_idx, SuperSlab** ss_out, int* slab_idx_out)
|
||
{
|
||
// Phase 12: SP-SLOT Box - 3-Stage Acquire Logic
|
||
//
|
||
// Stage 1: Reuse EMPTY slots from per-class free list (EMPTY→ACTIVE)
|
||
// Stage 2: Find UNUSED slots in existing SuperSlabs
|
||
// Stage 3: Get new SuperSlab (LRU pop or mmap)
|
||
//
|
||
// Invariants:
|
||
// - On success: *ss_out != NULL, 0 <= *slab_idx_out < total_slots
|
||
// - The chosen slab has meta->class_idx == class_idx
|
||
|
||
if (!ss_out || !slab_idx_out) {
|
||
return -1;
|
||
}
|
||
if (class_idx < 0 || class_idx >= TINY_NUM_CLASSES_SS) {
|
||
return -1;
|
||
}
|
||
|
||
shared_pool_init();
|
||
|
||
// Debug logging / stage stats
|
||
#if !HAKMEM_BUILD_RELEASE
|
||
static int dbg_acquire = -1;
|
||
if (__builtin_expect(dbg_acquire == -1, 0)) {
|
||
const char* e = getenv("HAKMEM_SS_ACQUIRE_DEBUG");
|
||
dbg_acquire = (e && *e && *e != '0') ? 1 : 0;
|
||
}
|
||
#else
|
||
static const int dbg_acquire = 0;
|
||
#endif
|
||
sp_stage_stats_init();
|
||
|
||
// ========== Stage 0: Per-thread hot slot (L0) reuse ==========
|
||
//
|
||
// 既に ACTIVE な slot で、かつ class_idx が一致し、まだ capacity に余裕がある場合のみ
|
||
// そのまま (ss, slab_idx) を返す。slot state の遷移や lock は一切触らない。
|
||
if (sp_l0_enabled()) {
|
||
SharedSSMeta* meta = g_sp_l0_meta[class_idx];
|
||
int l0_idx = (int)g_sp_l0_slot[class_idx];
|
||
if (meta && l0_idx >= 0) {
|
||
SuperSlab* ss = atomic_load_explicit(&meta->ss, memory_order_acquire);
|
||
if (ss && l0_idx < ss_slabs_capacity(ss)) {
|
||
TinySlabMeta* slab_meta = &ss->slabs[l0_idx];
|
||
if (slab_meta->class_idx == (uint8_t)class_idx &&
|
||
slab_meta->capacity > 0 &&
|
||
slab_meta->used < slab_meta->capacity) {
|
||
sp_fix_geometry_if_needed(ss, l0_idx, class_idx);
|
||
#if !HAKMEM_BUILD_RELEASE
|
||
if (dbg_acquire == 1) {
|
||
fprintf(stderr,
|
||
"[SP_ACQUIRE_STAGE0_L0] class=%d reuse hot slot (ss=%p slab=%d used=%u cap=%u)\n",
|
||
class_idx,
|
||
(void*)ss,
|
||
l0_idx,
|
||
(unsigned)slab_meta->used,
|
||
(unsigned)slab_meta->capacity);
|
||
}
|
||
#endif
|
||
*ss_out = ss;
|
||
*slab_idx_out = l0_idx;
|
||
return 0;
|
||
}
|
||
}
|
||
// 熱スロットが無効になっているのでクリアして通常経路へ
|
||
g_sp_l0_meta[class_idx] = NULL;
|
||
}
|
||
}
|
||
|
||
stage1_retry_after_tension_drain:
|
||
// ========== Stage 0.5 (NEW - Phase 12-1.1): EMPTY slab direct scan ==========
|
||
// Scan existing SuperSlabs for EMPTY slabs (highest reuse priority)
|
||
// This avoids Stage 3 (mmap) when freed slabs are available
|
||
// ENV: HAKMEM_SS_EMPTY_REUSE=0 to disable (default ON, +557% performance)
|
||
static int empty_reuse_enabled = -1;
|
||
if (__builtin_expect(empty_reuse_enabled == -1, 0)) {
|
||
const char* e = getenv("HAKMEM_SS_EMPTY_REUSE");
|
||
empty_reuse_enabled = (e && *e && *e == '0') ? 0 : 1; // default ON
|
||
}
|
||
|
||
if (empty_reuse_enabled) {
|
||
extern SuperSlab* g_super_reg_by_class[TINY_NUM_CLASSES][SUPER_REG_PER_CLASS]; // from hakmem_super_registry.h
|
||
extern int g_super_reg_class_size[TINY_NUM_CLASSES];
|
||
|
||
int reg_size = (class_idx < TINY_NUM_CLASSES) ? g_super_reg_class_size[class_idx] : 0;
|
||
static int scan_limit = -1;
|
||
if (__builtin_expect(scan_limit == -1, 0)) {
|
||
const char* e = getenv("HAKMEM_SS_EMPTY_SCAN_LIMIT");
|
||
scan_limit = (e && *e) ? atoi(e) : 16; // default: scan first 16 SuperSlabs
|
||
}
|
||
if (scan_limit > reg_size) scan_limit = reg_size;
|
||
|
||
for (int i = 0; i < scan_limit; i++) {
|
||
SuperSlab* ss = g_super_reg_by_class[class_idx][i];
|
||
if (!(ss && ss->magic == SUPERSLAB_MAGIC)) continue;
|
||
if (ss->empty_count == 0) continue; // No EMPTY slabs in this SS
|
||
|
||
// Found SuperSlab with EMPTY slabs - scan empty_mask
|
||
uint32_t mask = ss->empty_mask;
|
||
while (mask) {
|
||
int empty_idx = __builtin_ctz(mask);
|
||
mask &= (mask - 1); // clear lowest bit
|
||
|
||
// Validate this slab is truly EMPTY and reusable
|
||
TinySlabMeta* meta = &ss->slabs[empty_idx];
|
||
if (meta->capacity > 0 && meta->used == 0) {
|
||
// P0.3: Guard against TLS SLL orphaned pointers before reusing slab
|
||
tiny_tls_slab_reuse_guard(ss);
|
||
|
||
// Clear EMPTY state (will be re-marked on next free)
|
||
ss_clear_slab_empty(ss, empty_idx);
|
||
|
||
// Bind this slab to class_idx
|
||
meta->class_idx = (uint8_t)class_idx;
|
||
// P1.1: Update class_map for EMPTY slab reuse
|
||
ss->class_map[empty_idx] = (uint8_t)class_idx;
|
||
|
||
#if !HAKMEM_BUILD_RELEASE
|
||
if (dbg_acquire == 1) {
|
||
fprintf(stderr, "[SP_ACQUIRE_STAGE0.5_EMPTY] class=%d reusing EMPTY slab (ss=%p slab=%d empty_count=%u)\n",
|
||
class_idx, (void*)ss, empty_idx, ss->empty_count);
|
||
}
|
||
#endif
|
||
|
||
*ss_out = ss;
|
||
*slab_idx_out = empty_idx;
|
||
sp_stage_stats_init();
|
||
if (g_sp_stage_stats_enabled) {
|
||
atomic_fetch_add(&g_sp_stage1_hits[class_idx], 1); // Count as Stage 1 hit
|
||
}
|
||
return 0; // ✅ Stage 0.5 (EMPTY scan) success
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// ========== Stage 1 (Lock-Free): Try to reuse EMPTY slots ==========
|
||
// P0-4: Lock-free pop from per-class free list (no mutex needed!)
|
||
// Best case: Same class freed a slot, reuse immediately (cache-hot)
|
||
SharedSSMeta* reuse_meta = NULL;
|
||
int reuse_slot_idx = -1;
|
||
|
||
if (sp_freelist_pop_lockfree(class_idx, &reuse_meta, &reuse_slot_idx)) {
|
||
// Found EMPTY slot from lock-free list!
|
||
// Now acquire mutex ONLY for slot activation and metadata update
|
||
|
||
// P0 instrumentation: count lock acquisitions
|
||
lock_stats_init();
|
||
if (g_lock_stats_enabled == 1) {
|
||
atomic_fetch_add(&g_lock_acquire_count, 1);
|
||
atomic_fetch_add(&g_lock_acquire_slab_count, 1);
|
||
}
|
||
|
||
pthread_mutex_lock(&g_shared_pool.alloc_lock);
|
||
|
||
// P0.3: Guard against TLS SLL orphaned pointers before reusing slab
|
||
// RACE FIX: Load SuperSlab pointer atomically BEFORE guard (consistency)
|
||
SuperSlab* ss_guard = atomic_load_explicit(&reuse_meta->ss, memory_order_relaxed);
|
||
if (ss_guard) {
|
||
tiny_tls_slab_reuse_guard(ss_guard);
|
||
}
|
||
|
||
// Activate slot under mutex (slot state transition requires protection)
|
||
if (sp_slot_mark_active(reuse_meta, reuse_slot_idx, class_idx) == 0) {
|
||
// RACE FIX: Load SuperSlab pointer atomically (consistency)
|
||
SuperSlab* ss = atomic_load_explicit(&reuse_meta->ss, memory_order_relaxed);
|
||
|
||
// RACE FIX: Check if SuperSlab was freed (NULL pointer)
|
||
// This can happen if Thread A freed the SuperSlab after pushing slot to freelist,
|
||
// but Thread B popped the stale slot before the freelist was cleared.
|
||
if (!ss) {
|
||
// SuperSlab freed - skip and fall through to Stage 2/3
|
||
if (g_lock_stats_enabled == 1) {
|
||
atomic_fetch_add(&g_lock_release_count, 1);
|
||
}
|
||
pthread_mutex_unlock(&g_shared_pool.alloc_lock);
|
||
goto stage2_fallback;
|
||
}
|
||
|
||
#if !HAKMEM_BUILD_RELEASE
|
||
if (dbg_acquire == 1) {
|
||
fprintf(stderr, "[SP_ACQUIRE_STAGE1_LOCKFREE] class=%d reusing EMPTY slot (ss=%p slab=%d)\n",
|
||
class_idx, (void*)ss, reuse_slot_idx);
|
||
}
|
||
#endif
|
||
|
||
// Update SuperSlab metadata
|
||
ss->slab_bitmap |= (1u << reuse_slot_idx);
|
||
ss_slab_meta_class_idx_set(ss, reuse_slot_idx, (uint8_t)class_idx);
|
||
|
||
if (ss->active_slabs == 0) {
|
||
// Was empty, now active again
|
||
ss->active_slabs = 1;
|
||
g_shared_pool.active_count++;
|
||
}
|
||
// Track per-class active slots (approximate, under alloc_lock)
|
||
if (class_idx < TINY_NUM_CLASSES_SS) {
|
||
g_shared_pool.class_active_slots[class_idx]++;
|
||
}
|
||
|
||
// Update hint
|
||
g_shared_pool.class_hints[class_idx] = ss;
|
||
|
||
// Update per-thread hot slot (L0)
|
||
if (sp_l0_enabled()) {
|
||
g_sp_l0_meta[class_idx] = reuse_meta;
|
||
g_sp_l0_slot[class_idx] = (uint8_t)reuse_slot_idx;
|
||
}
|
||
|
||
*ss_out = ss;
|
||
*slab_idx_out = reuse_slot_idx;
|
||
|
||
if (g_lock_stats_enabled == 1) {
|
||
atomic_fetch_add(&g_lock_release_count, 1);
|
||
}
|
||
pthread_mutex_unlock(&g_shared_pool.alloc_lock);
|
||
if (g_sp_stage_stats_enabled) {
|
||
atomic_fetch_add(&g_sp_stage1_hits[class_idx], 1);
|
||
}
|
||
return 0; // ✅ Stage 1 (lock-free) success
|
||
}
|
||
|
||
// Slot activation failed (race condition?) - release lock and fall through
|
||
if (g_lock_stats_enabled == 1) {
|
||
atomic_fetch_add(&g_lock_release_count, 1);
|
||
}
|
||
pthread_mutex_unlock(&g_shared_pool.alloc_lock);
|
||
}
|
||
|
||
stage2_fallback:
|
||
// ========== Stage 2 (Lock-Free): Try to claim UNUSED slots ==========
|
||
// P0-5: Lock-free atomic CAS claiming (no mutex needed for slot state transition!)
|
||
// RACE FIX: Read ss_meta_count atomically (now properly declared as _Atomic)
|
||
// No cast needed! memory_order_acquire synchronizes with release in sp_meta_find_or_create
|
||
uint32_t meta_count = atomic_load_explicit(
|
||
&g_shared_pool.ss_meta_count,
|
||
memory_order_acquire
|
||
);
|
||
|
||
for (uint32_t i = 0; i < meta_count; i++) {
|
||
SharedSSMeta* meta = &g_shared_pool.ss_metadata[i];
|
||
|
||
// Try lock-free claiming (UNUSED → ACTIVE via CAS)
|
||
int claimed_idx = sp_slot_claim_lockfree(meta, class_idx);
|
||
if (claimed_idx >= 0) {
|
||
// RACE FIX: Load SuperSlab pointer atomically (critical for lock-free Stage 2)
|
||
// Use memory_order_acquire to synchronize with release in sp_meta_find_or_create
|
||
SuperSlab* ss = atomic_load_explicit(&meta->ss, memory_order_acquire);
|
||
if (!ss) {
|
||
// SuperSlab was freed between claiming and loading - skip this entry
|
||
continue;
|
||
}
|
||
|
||
#if !HAKMEM_BUILD_RELEASE
|
||
if (dbg_acquire == 1) {
|
||
fprintf(stderr, "[SP_ACQUIRE_STAGE2_LOCKFREE] class=%d claimed UNUSED slot (ss=%p slab=%d)\n",
|
||
class_idx, (void*)ss, claimed_idx);
|
||
}
|
||
#endif
|
||
|
||
// P0 instrumentation: count lock acquisitions
|
||
lock_stats_init();
|
||
if (g_lock_stats_enabled == 1) {
|
||
atomic_fetch_add(&g_lock_acquire_count, 1);
|
||
atomic_fetch_add(&g_lock_acquire_slab_count, 1);
|
||
}
|
||
|
||
pthread_mutex_lock(&g_shared_pool.alloc_lock);
|
||
|
||
// Update SuperSlab metadata under mutex
|
||
ss->slab_bitmap |= (1u << claimed_idx);
|
||
ss_slab_meta_class_idx_set(ss, claimed_idx, (uint8_t)class_idx);
|
||
|
||
if (ss->active_slabs == 0) {
|
||
ss->active_slabs = 1;
|
||
g_shared_pool.active_count++;
|
||
}
|
||
if (class_idx < TINY_NUM_CLASSES_SS) {
|
||
g_shared_pool.class_active_slots[class_idx]++;
|
||
}
|
||
|
||
// Update hint
|
||
g_shared_pool.class_hints[class_idx] = ss;
|
||
|
||
// Update per-thread hot slot (L0)
|
||
if (sp_l0_enabled()) {
|
||
g_sp_l0_meta[class_idx] = meta;
|
||
g_sp_l0_slot[class_idx] = (uint8_t)claimed_idx;
|
||
}
|
||
|
||
*ss_out = ss;
|
||
*slab_idx_out = claimed_idx;
|
||
sp_fix_geometry_if_needed(ss, claimed_idx, class_idx);
|
||
|
||
if (g_lock_stats_enabled == 1) {
|
||
atomic_fetch_add(&g_lock_release_count, 1);
|
||
}
|
||
pthread_mutex_unlock(&g_shared_pool.alloc_lock);
|
||
if (g_sp_stage_stats_enabled) {
|
||
atomic_fetch_add(&g_sp_stage2_hits[class_idx], 1);
|
||
}
|
||
return 0; // ✅ Stage 2 (lock-free) success
|
||
}
|
||
|
||
// Claim failed (no UNUSED slots in this meta) - continue to next SuperSlab
|
||
}
|
||
|
||
// ========== Tension-Based Drain: Try to create EMPTY slots before Stage 3 ==========
|
||
// If TLS SLL has accumulated blocks, drain them to enable EMPTY slot detection
|
||
// This can avoid allocating new SuperSlabs by reusing EMPTY slots in Stage 1
|
||
// ENV: HAKMEM_TINY_TENSION_DRAIN_ENABLE=0 to disable (default=1)
|
||
// ENV: HAKMEM_TINY_TENSION_DRAIN_THRESHOLD=N to set threshold (default=1024)
|
||
{
|
||
static int tension_drain_enabled = -1;
|
||
static uint32_t tension_threshold = 1024;
|
||
|
||
if (tension_drain_enabled < 0) {
|
||
const char* env = getenv("HAKMEM_TINY_TENSION_DRAIN_ENABLE");
|
||
tension_drain_enabled = (env == NULL || atoi(env) != 0) ? 1 : 0;
|
||
|
||
const char* thresh_env = getenv("HAKMEM_TINY_TENSION_DRAIN_THRESHOLD");
|
||
if (thresh_env) {
|
||
tension_threshold = (uint32_t)atoi(thresh_env);
|
||
if (tension_threshold < 64) tension_threshold = 64;
|
||
if (tension_threshold > 65536) tension_threshold = 65536;
|
||
}
|
||
}
|
||
|
||
if (tension_drain_enabled) {
|
||
extern __thread TinyTLSSLL g_tls_sll[TINY_NUM_CLASSES];
|
||
extern uint32_t tiny_tls_sll_drain(int class_idx, uint32_t batch_size);
|
||
|
||
uint32_t sll_count = (class_idx < TINY_NUM_CLASSES) ? g_tls_sll[class_idx].count : 0;
|
||
|
||
if (sll_count >= tension_threshold) {
|
||
// Drain all blocks to maximize EMPTY slot creation
|
||
uint32_t drained = tiny_tls_sll_drain(class_idx, 0); // 0 = drain all
|
||
|
||
if (drained > 0) {
|
||
// Retry Stage 1 (EMPTY reuse) after drain
|
||
// Some slabs might have become EMPTY (meta->used == 0)
|
||
goto stage1_retry_after_tension_drain;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// ========== Stage 3: Mutex-protected fallback (new SuperSlab allocation) ==========
|
||
// All existing SuperSlabs have no UNUSED slots → need new SuperSlab
|
||
// P0 instrumentation: count lock acquisitions
|
||
lock_stats_init();
|
||
if (g_lock_stats_enabled == 1) {
|
||
atomic_fetch_add(&g_lock_acquire_count, 1);
|
||
atomic_fetch_add(&g_lock_acquire_slab_count, 1);
|
||
}
|
||
|
||
pthread_mutex_lock(&g_shared_pool.alloc_lock);
|
||
|
||
// ========== Stage 3: Get new SuperSlab ==========
|
||
// Try LRU cache first, then mmap
|
||
SuperSlab* new_ss = NULL;
|
||
|
||
// Stage 3a: Try LRU cache
|
||
extern SuperSlab* hak_ss_lru_pop(uint8_t size_class);
|
||
new_ss = hak_ss_lru_pop((uint8_t)class_idx);
|
||
|
||
int from_lru = (new_ss != NULL);
|
||
|
||
// Stage 3b: If LRU miss, allocate new SuperSlab
|
||
if (!new_ss) {
|
||
new_ss = shared_pool_allocate_superslab_unlocked();
|
||
}
|
||
|
||
#if !HAKMEM_BUILD_RELEASE
|
||
if (dbg_acquire == 1 && new_ss) {
|
||
fprintf(stderr, "[SP_ACQUIRE_STAGE3] class=%d new SuperSlab (ss=%p from_lru=%d)\n",
|
||
class_idx, (void*)new_ss, from_lru);
|
||
}
|
||
#endif
|
||
|
||
if (!new_ss) {
|
||
if (g_lock_stats_enabled == 1) {
|
||
atomic_fetch_add(&g_lock_release_count, 1);
|
||
}
|
||
pthread_mutex_unlock(&g_shared_pool.alloc_lock);
|
||
return -1; // ❌ Out of memory
|
||
}
|
||
|
||
// Before creating a new SuperSlab, consult learning-layer soft cap.
|
||
// If current active slots for this class already exceed the policy cap,
|
||
// fail early so caller can fall back to legacy backend.
|
||
uint32_t limit = sp_class_active_limit(class_idx);
|
||
if (limit > 0) {
|
||
uint32_t cur = g_shared_pool.class_active_slots[class_idx];
|
||
if (cur >= limit) {
|
||
if (g_lock_stats_enabled == 1) {
|
||
atomic_fetch_add(&g_lock_release_count, 1);
|
||
}
|
||
pthread_mutex_unlock(&g_shared_pool.alloc_lock);
|
||
return -1; // Soft cap reached for this class
|
||
}
|
||
}
|
||
|
||
// Create metadata for this new SuperSlab
|
||
SharedSSMeta* new_meta = sp_meta_find_or_create(new_ss);
|
||
if (!new_meta) {
|
||
if (g_lock_stats_enabled == 1) {
|
||
atomic_fetch_add(&g_lock_release_count, 1);
|
||
}
|
||
pthread_mutex_unlock(&g_shared_pool.alloc_lock);
|
||
return -1; // ❌ Metadata allocation failed
|
||
}
|
||
|
||
// Assign first slot to this class
|
||
int first_slot = 0;
|
||
if (sp_slot_mark_active(new_meta, first_slot, class_idx) != 0) {
|
||
if (g_lock_stats_enabled == 1) {
|
||
atomic_fetch_add(&g_lock_release_count, 1);
|
||
}
|
||
pthread_mutex_unlock(&g_shared_pool.alloc_lock);
|
||
return -1; // ❌ Should not happen
|
||
}
|
||
|
||
// Update SuperSlab metadata
|
||
new_ss->slab_bitmap |= (1u << first_slot);
|
||
ss_slab_meta_class_idx_set(new_ss, first_slot, (uint8_t)class_idx);
|
||
new_ss->active_slabs = 1;
|
||
g_shared_pool.active_count++;
|
||
if (class_idx < TINY_NUM_CLASSES_SS) {
|
||
g_shared_pool.class_active_slots[class_idx]++;
|
||
}
|
||
|
||
// Update hint
|
||
g_shared_pool.class_hints[class_idx] = new_ss;
|
||
|
||
// Update per-thread hot slot (L0)
|
||
if (sp_l0_enabled()) {
|
||
g_sp_l0_meta[class_idx] = new_meta;
|
||
g_sp_l0_slot[class_idx] = (uint8_t)first_slot;
|
||
}
|
||
|
||
*ss_out = new_ss;
|
||
*slab_idx_out = first_slot;
|
||
sp_fix_geometry_if_needed(new_ss, first_slot, class_idx);
|
||
|
||
if (g_lock_stats_enabled == 1) {
|
||
atomic_fetch_add(&g_lock_release_count, 1);
|
||
}
|
||
pthread_mutex_unlock(&g_shared_pool.alloc_lock);
|
||
if (g_sp_stage_stats_enabled) {
|
||
atomic_fetch_add(&g_sp_stage3_hits[class_idx], 1);
|
||
}
|
||
return 0; // ✅ Stage 3 success
|
||
}
|
||
|
||
void
|
||
shared_pool_release_slab(SuperSlab* ss, int slab_idx)
|
||
{
|
||
// Phase 12: SP-SLOT Box - Slot-based Release
|
||
//
|
||
// Flow:
|
||
// 1. Validate inputs and check meta->used == 0
|
||
// 2. Find SharedSSMeta for this SuperSlab
|
||
// 3. Mark slot ACTIVE → EMPTY
|
||
// 4. Push to per-class free list (enables same-class reuse)
|
||
// 5. If all slots EMPTY → superslab_free() → LRU cache
|
||
|
||
if (!ss) {
|
||
return;
|
||
}
|
||
if (slab_idx < 0 || slab_idx >= SLABS_PER_SUPERSLAB_MAX) {
|
||
return;
|
||
}
|
||
|
||
// Debug logging
|
||
#if !HAKMEM_BUILD_RELEASE
|
||
static int dbg = -1;
|
||
if (__builtin_expect(dbg == -1, 0)) {
|
||
const char* e = getenv("HAKMEM_SS_FREE_DEBUG");
|
||
dbg = (e && *e && *e != '0') ? 1 : 0;
|
||
}
|
||
#else
|
||
static const int dbg = 0;
|
||
#endif
|
||
|
||
// P0 instrumentation: count lock acquisitions
|
||
lock_stats_init();
|
||
if (g_lock_stats_enabled == 1) {
|
||
atomic_fetch_add(&g_lock_acquire_count, 1);
|
||
atomic_fetch_add(&g_lock_release_slab_count, 1);
|
||
}
|
||
|
||
pthread_mutex_lock(&g_shared_pool.alloc_lock);
|
||
|
||
TinySlabMeta* slab_meta = &ss->slabs[slab_idx];
|
||
if (slab_meta->used != 0) {
|
||
// Not actually empty; nothing to do
|
||
if (g_lock_stats_enabled == 1) {
|
||
atomic_fetch_add(&g_lock_release_count, 1);
|
||
}
|
||
pthread_mutex_unlock(&g_shared_pool.alloc_lock);
|
||
return;
|
||
}
|
||
|
||
uint8_t class_idx = slab_meta->class_idx;
|
||
|
||
#if !HAKMEM_BUILD_RELEASE
|
||
if (dbg == 1) {
|
||
fprintf(stderr, "[SP_SLOT_RELEASE] ss=%p slab_idx=%d class=%d used=0 (marking EMPTY)\n",
|
||
(void*)ss, slab_idx, class_idx);
|
||
}
|
||
#endif
|
||
|
||
// Find SharedSSMeta for this SuperSlab
|
||
SharedSSMeta* sp_meta = NULL;
|
||
uint32_t count = atomic_load_explicit(&g_shared_pool.ss_meta_count, memory_order_relaxed);
|
||
for (uint32_t i = 0; i < count; i++) {
|
||
// RACE FIX: Load pointer atomically
|
||
SuperSlab* meta_ss = atomic_load_explicit(&g_shared_pool.ss_metadata[i].ss, memory_order_relaxed);
|
||
if (meta_ss == ss) {
|
||
sp_meta = &g_shared_pool.ss_metadata[i];
|
||
break;
|
||
}
|
||
}
|
||
|
||
if (!sp_meta) {
|
||
// SuperSlab not in SP-SLOT system yet - create metadata
|
||
sp_meta = sp_meta_find_or_create(ss);
|
||
if (!sp_meta) {
|
||
pthread_mutex_unlock(&g_shared_pool.alloc_lock);
|
||
return; // Failed to create metadata
|
||
}
|
||
}
|
||
|
||
// Mark slot as EMPTY (ACTIVE → EMPTY)
|
||
uint32_t slab_bit = (1u << slab_idx);
|
||
SlotState slot_state = atomic_load_explicit(
|
||
&sp_meta->slots[slab_idx].state,
|
||
memory_order_acquire);
|
||
if (slot_state != SLOT_ACTIVE && (ss->slab_bitmap & slab_bit)) {
|
||
// Legacy path import: rebuild slot states from SuperSlab bitmap/class_map
|
||
sp_meta_sync_slots_from_ss(sp_meta, ss);
|
||
slot_state = atomic_load_explicit(
|
||
&sp_meta->slots[slab_idx].state,
|
||
memory_order_acquire);
|
||
}
|
||
|
||
if (slot_state != SLOT_ACTIVE || sp_slot_mark_empty(sp_meta, slab_idx) != 0) {
|
||
if (g_lock_stats_enabled == 1) {
|
||
atomic_fetch_add(&g_lock_release_count, 1);
|
||
}
|
||
pthread_mutex_unlock(&g_shared_pool.alloc_lock);
|
||
return; // Slot wasn't ACTIVE
|
||
}
|
||
|
||
// Update SuperSlab metadata
|
||
uint32_t bit = (1u << slab_idx);
|
||
if (ss->slab_bitmap & bit) {
|
||
ss->slab_bitmap &= ~bit;
|
||
slab_meta->class_idx = 255; // UNASSIGNED
|
||
// P1.1: Mark class_map as UNASSIGNED when releasing slab
|
||
ss->class_map[slab_idx] = 255;
|
||
|
||
if (ss->active_slabs > 0) {
|
||
ss->active_slabs--;
|
||
if (ss->active_slabs == 0 && g_shared_pool.active_count > 0) {
|
||
g_shared_pool.active_count--;
|
||
}
|
||
}
|
||
if (class_idx < TINY_NUM_CLASSES_SS &&
|
||
g_shared_pool.class_active_slots[class_idx] > 0) {
|
||
g_shared_pool.class_active_slots[class_idx]--;
|
||
}
|
||
}
|
||
|
||
// P0-4: Push to lock-free per-class free list (enables reuse by same class)
|
||
// Note: push BEFORE releasing mutex (slot state already updated under lock)
|
||
if (class_idx < TINY_NUM_CLASSES_SS) {
|
||
sp_freelist_push_lockfree(class_idx, sp_meta, slab_idx);
|
||
|
||
#if !HAKMEM_BUILD_RELEASE
|
||
if (dbg == 1) {
|
||
fprintf(stderr, "[SP_SLOT_FREELIST_LOCKFREE] class=%d pushed slot (ss=%p slab=%d) active_slots=%u/%u\n",
|
||
class_idx, (void*)ss, slab_idx,
|
||
sp_meta->active_slots, sp_meta->total_slots);
|
||
}
|
||
#endif
|
||
}
|
||
|
||
// Check if SuperSlab is now completely empty (all slots EMPTY or UNUSED)
|
||
if (sp_meta->active_slots == 0) {
|
||
#if !HAKMEM_BUILD_RELEASE
|
||
if (dbg == 1) {
|
||
fprintf(stderr, "[SP_SLOT_COMPLETELY_EMPTY] ss=%p active_slots=0 (calling superslab_free)\n",
|
||
(void*)ss);
|
||
}
|
||
#endif
|
||
|
||
if (g_lock_stats_enabled == 1) {
|
||
atomic_fetch_add(&g_lock_release_count, 1);
|
||
}
|
||
|
||
// RACE FIX: Set meta->ss to NULL BEFORE unlocking mutex
|
||
// This prevents Stage 2 from accessing freed SuperSlab
|
||
atomic_store_explicit(&sp_meta->ss, NULL, memory_order_release);
|
||
|
||
pthread_mutex_unlock(&g_shared_pool.alloc_lock);
|
||
|
||
// Free SuperSlab:
|
||
// 1. Try LRU cache (hak_ss_lru_push) - lazy deallocation
|
||
// 2. Or munmap if LRU is full - eager deallocation
|
||
extern void superslab_free(SuperSlab* ss);
|
||
superslab_free(ss);
|
||
return;
|
||
}
|
||
|
||
if (g_lock_stats_enabled == 1) {
|
||
atomic_fetch_add(&g_lock_release_count, 1);
|
||
}
|
||
pthread_mutex_unlock(&g_shared_pool.alloc_lock);
|
||
}
|