2025-11-20 07:32:30 +09:00
|
|
|
// ss_os_acquire_box.h - SuperSlab OS Memory Acquisition Box
|
|
|
|
|
// Purpose: Low-level OS memory allocation (mmap/munmap) for SuperSlabs
|
|
|
|
|
// Box Theory: Encapsulates platform-specific aligned memory allocation
|
|
|
|
|
//
|
|
|
|
|
// Responsibilities:
|
|
|
|
|
// - Aligned mmap allocation (2MB boundary)
|
|
|
|
|
// - OOM diagnostics and error reporting
|
|
|
|
|
// - Global mmap counters
|
|
|
|
|
//
|
|
|
|
|
// Dependencies: None (pure OS interface)
|
|
|
|
|
//
|
|
|
|
|
// License: MIT
|
|
|
|
|
// Date: 2025-11-19
|
|
|
|
|
|
|
|
|
|
#ifndef HAKMEM_SS_OS_ACQUIRE_BOX_H
|
|
|
|
|
#define HAKMEM_SS_OS_ACQUIRE_BOX_H
|
|
|
|
|
|
|
|
|
|
#include <stdint.h>
|
|
|
|
|
#include <stddef.h>
|
|
|
|
|
#include <stdatomic.h>
|
2025-12-09 21:50:15 +09:00
|
|
|
#include <stdbool.h>
|
2025-12-08 21:30:21 +09:00
|
|
|
#include <stdlib.h>
|
2025-12-09 21:50:15 +09:00
|
|
|
#include <sys/mman.h>
|
2025-12-10 09:08:18 +09:00
|
|
|
|
|
|
|
|
#include "madvise_guard_box.h"
|
2025-11-20 07:32:30 +09:00
|
|
|
|
|
|
|
|
// ============================================================================
|
|
|
|
|
// Global Counters (for debugging/diagnostics)
|
|
|
|
|
// ============================================================================
|
|
|
|
|
|
|
|
|
|
extern _Atomic uint64_t g_ss_mmap_count;
|
|
|
|
|
extern _Atomic uint64_t g_final_fallback_mmap_count;
|
2025-12-08 21:30:21 +09:00
|
|
|
extern _Atomic uint64_t g_ss_os_alloc_calls;
|
|
|
|
|
extern _Atomic uint64_t g_ss_os_free_calls;
|
|
|
|
|
extern _Atomic uint64_t g_ss_os_madvise_calls;
|
2025-12-09 21:50:15 +09:00
|
|
|
extern _Atomic uint64_t g_ss_os_madvise_fail_enomem;
|
|
|
|
|
extern _Atomic uint64_t g_ss_os_madvise_fail_other;
|
2025-12-08 21:30:21 +09:00
|
|
|
extern _Atomic uint64_t g_ss_os_huge_alloc_calls;
|
|
|
|
|
extern _Atomic uint64_t g_ss_os_huge_fail_calls;
|
2025-12-09 21:50:15 +09:00
|
|
|
extern _Atomic bool g_ss_madvise_disabled;
|
Phase 54-60: Memory-Lean mode, Balanced mode stabilization, M1 (50%) achievement
## Summary
Completed Phase 54-60 optimization work:
**Phase 54-56: Memory-Lean mode (LEAN+OFF prewarm suppression)**
- Implemented ss_mem_lean_env_box.h with ENV gates
- Balanced mode (LEAN+OFF) promoted as production default
- Result: +1.2% throughput, better stability, zero syscall overhead
- Added to bench_profile.h: MIXED_TINYV3_C7_BALANCED preset
**Phase 57: 60-min soak finalization**
- Balanced mode: 60-min soak, RSS drift 0%, CV 5.38%
- Speed-first mode: 60-min soak, RSS drift 0%, CV 1.58%
- Syscall budget: 1.25e-7/op (800× under target)
- Status: PRODUCTION-READY
**Phase 59: 50% recovery baseline rebase**
- hakmem FAST (Balanced): 59.184M ops/s, CV 1.31%
- mimalloc: 120.466M ops/s, CV 3.50%
- Ratio: 49.13% (M1 ACHIEVED within statistical noise)
- Superior stability: 2.68× better CV than mimalloc
**Phase 60: Alloc pass-down SSOT (NO-GO)**
- Implemented alloc_passdown_ssot_env_box.h
- Modified malloc_tiny_fast.h for SSOT pattern
- Result: -0.46% (NO-GO)
- Key lesson: SSOT not applicable where early-exit already optimized
## Key Metrics
- Performance: 49.13% of mimalloc (M1 effectively achieved)
- Stability: CV 1.31% (superior to mimalloc 3.50%)
- Syscall budget: 1.25e-7/op (excellent)
- RSS: 33MB stable, 0% drift over 60 minutes
## Files Added/Modified
New boxes:
- core/box/ss_mem_lean_env_box.h
- core/box/ss_release_policy_box.{h,c}
- core/box/alloc_passdown_ssot_env_box.h
Scripts:
- scripts/soak_mixed_single_process.sh
- scripts/analyze_epoch_tail_csv.py
- scripts/soak_mixed_rss.sh
- scripts/calculate_percentiles.py
- scripts/analyze_soak.py
Documentation: Phase 40-60 analysis documents
## Design Decisions
1. Profile separation (core/bench_profile.h):
- MIXED_TINYV3_C7_SAFE: Speed-first (no LEAN)
- MIXED_TINYV3_C7_BALANCED: Balanced mode (LEAN+OFF)
2. Box Theory compliance:
- All ENV gates reversible (HAKMEM_SS_MEM_LEAN, HAKMEM_ALLOC_PASSDOWN_SSOT)
- Single conversion points maintained
- No physical deletions (compile-out only)
3. Lessons learned:
- SSOT effective only where redundancy exists (Phase 60 showed limits)
- Branch prediction extremely effective (~0 cycles for well-predicted branches)
- Early-exit pattern valuable even when seemingly redundant
🤖 Generated with [Claude Code](https://claude.com/claude-code)
Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
2025-12-17 06:24:01 +09:00
|
|
|
extern _Atomic uint64_t g_ss_lean_decommit_calls;
|
|
|
|
|
extern _Atomic uint64_t g_ss_lean_retire_calls;
|
2025-12-08 21:30:21 +09:00
|
|
|
|
|
|
|
|
static inline int ss_os_stats_enabled(void) {
|
|
|
|
|
static int g_ss_os_stats_enabled = -1;
|
|
|
|
|
if (__builtin_expect(g_ss_os_stats_enabled == -1, 0)) {
|
|
|
|
|
const char* e = getenv("HAKMEM_SS_OS_STATS");
|
|
|
|
|
g_ss_os_stats_enabled = (e && *e && *e != '0') ? 1 : 0;
|
|
|
|
|
}
|
|
|
|
|
return g_ss_os_stats_enabled;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline void ss_os_stats_record_alloc(void) {
|
|
|
|
|
if (!ss_os_stats_enabled()) {
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
atomic_fetch_add_explicit(&g_ss_os_alloc_calls, 1, memory_order_relaxed);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline void ss_os_stats_record_free(void) {
|
|
|
|
|
if (!ss_os_stats_enabled()) {
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
atomic_fetch_add_explicit(&g_ss_os_free_calls, 1, memory_order_relaxed);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline void ss_os_stats_record_madvise(void) {
|
|
|
|
|
if (!ss_os_stats_enabled()) {
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
atomic_fetch_add_explicit(&g_ss_os_madvise_calls, 1, memory_order_relaxed);
|
|
|
|
|
}
|
|
|
|
|
|
Phase 54-60: Memory-Lean mode, Balanced mode stabilization, M1 (50%) achievement
## Summary
Completed Phase 54-60 optimization work:
**Phase 54-56: Memory-Lean mode (LEAN+OFF prewarm suppression)**
- Implemented ss_mem_lean_env_box.h with ENV gates
- Balanced mode (LEAN+OFF) promoted as production default
- Result: +1.2% throughput, better stability, zero syscall overhead
- Added to bench_profile.h: MIXED_TINYV3_C7_BALANCED preset
**Phase 57: 60-min soak finalization**
- Balanced mode: 60-min soak, RSS drift 0%, CV 5.38%
- Speed-first mode: 60-min soak, RSS drift 0%, CV 1.58%
- Syscall budget: 1.25e-7/op (800× under target)
- Status: PRODUCTION-READY
**Phase 59: 50% recovery baseline rebase**
- hakmem FAST (Balanced): 59.184M ops/s, CV 1.31%
- mimalloc: 120.466M ops/s, CV 3.50%
- Ratio: 49.13% (M1 ACHIEVED within statistical noise)
- Superior stability: 2.68× better CV than mimalloc
**Phase 60: Alloc pass-down SSOT (NO-GO)**
- Implemented alloc_passdown_ssot_env_box.h
- Modified malloc_tiny_fast.h for SSOT pattern
- Result: -0.46% (NO-GO)
- Key lesson: SSOT not applicable where early-exit already optimized
## Key Metrics
- Performance: 49.13% of mimalloc (M1 effectively achieved)
- Stability: CV 1.31% (superior to mimalloc 3.50%)
- Syscall budget: 1.25e-7/op (excellent)
- RSS: 33MB stable, 0% drift over 60 minutes
## Files Added/Modified
New boxes:
- core/box/ss_mem_lean_env_box.h
- core/box/ss_release_policy_box.{h,c}
- core/box/alloc_passdown_ssot_env_box.h
Scripts:
- scripts/soak_mixed_single_process.sh
- scripts/analyze_epoch_tail_csv.py
- scripts/soak_mixed_rss.sh
- scripts/calculate_percentiles.py
- scripts/analyze_soak.py
Documentation: Phase 40-60 analysis documents
## Design Decisions
1. Profile separation (core/bench_profile.h):
- MIXED_TINYV3_C7_SAFE: Speed-first (no LEAN)
- MIXED_TINYV3_C7_BALANCED: Balanced mode (LEAN+OFF)
2. Box Theory compliance:
- All ENV gates reversible (HAKMEM_SS_MEM_LEAN, HAKMEM_ALLOC_PASSDOWN_SSOT)
- Single conversion points maintained
- No physical deletions (compile-out only)
3. Lessons learned:
- SSOT effective only where redundancy exists (Phase 60 showed limits)
- Branch prediction extremely effective (~0 cycles for well-predicted branches)
- Early-exit pattern valuable even when seemingly redundant
🤖 Generated with [Claude Code](https://claude.com/claude-code)
Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
2025-12-17 06:24:01 +09:00
|
|
|
static inline void ss_os_stats_record_lean_decommit(void) {
|
|
|
|
|
if (!ss_os_stats_enabled()) {
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
atomic_fetch_add_explicit(&g_ss_lean_decommit_calls, 1, memory_order_relaxed);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline void ss_os_stats_record_lean_retire(void) {
|
|
|
|
|
if (!ss_os_stats_enabled()) {
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
atomic_fetch_add_explicit(&g_ss_lean_retire_calls, 1, memory_order_relaxed);
|
|
|
|
|
}
|
|
|
|
|
|
2025-12-09 21:50:15 +09:00
|
|
|
// ============================================================================
|
2025-12-08 21:30:21 +09:00
|
|
|
// HugePage Experiment (research-only)
|
|
|
|
|
// ============================================================================
|
|
|
|
|
|
|
|
|
|
static inline int ss_os_huge_enabled(void) {
|
|
|
|
|
static int g_ss_os_huge_enabled = -1;
|
|
|
|
|
if (__builtin_expect(g_ss_os_huge_enabled == -1, 0)) {
|
|
|
|
|
const char* e = getenv("HAKMEM_SS_HUGEPAGE_EXPERIMENT");
|
|
|
|
|
g_ss_os_huge_enabled = (e && *e && *e != '0') ? 1 : 0;
|
|
|
|
|
}
|
|
|
|
|
return g_ss_os_huge_enabled;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Parse HAKMEM_SS_HUGEPAGE_SIZE (only "2M" supported explicitly; otherwise
|
|
|
|
|
// falls back to default 2MB). This is intentionally soft/experimental.
|
|
|
|
|
static inline size_t ss_os_huge_size_bytes(void) {
|
|
|
|
|
static size_t g_huge_size = 0;
|
|
|
|
|
if (__builtin_expect(g_huge_size == 0, 0)) {
|
|
|
|
|
const char* e = getenv("HAKMEM_SS_HUGEPAGE_SIZE");
|
|
|
|
|
if (e && *e) {
|
|
|
|
|
char* end = NULL;
|
|
|
|
|
unsigned long long v = strtoull(e, &end, 0);
|
|
|
|
|
if (end && (*end == 'M' || *end == 'm')) {
|
|
|
|
|
v *= 1024ULL * 1024ULL;
|
|
|
|
|
}
|
|
|
|
|
if (v > 0) {
|
|
|
|
|
g_huge_size = (size_t)v;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if (g_huge_size == 0) {
|
|
|
|
|
g_huge_size = (size_t)(2ULL << 20); // default 2MB
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return g_huge_size;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline void ss_os_stats_record_huge_alloc(void) {
|
|
|
|
|
if (!ss_os_stats_enabled()) {
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
atomic_fetch_add_explicit(&g_ss_os_huge_alloc_calls, 1, memory_order_relaxed);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline void ss_os_stats_record_huge_fail(void) {
|
|
|
|
|
if (!ss_os_stats_enabled()) {
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
atomic_fetch_add_explicit(&g_ss_os_huge_fail_calls, 1, memory_order_relaxed);
|
|
|
|
|
}
|
2025-11-20 07:32:30 +09:00
|
|
|
|
|
|
|
|
// ============================================================================
|
|
|
|
|
// OS Acquisition API
|
|
|
|
|
// ============================================================================
|
|
|
|
|
|
|
|
|
|
// Acquire aligned SuperSlab memory from OS via mmap
|
|
|
|
|
//
|
|
|
|
|
// Parameters:
|
|
|
|
|
// size_class: Size class index (0-7, for statistics)
|
|
|
|
|
// ss_size: SuperSlab size in bytes (e.g., 2^21 = 2MB)
|
|
|
|
|
// ss_mask: Alignment mask (ss_size - 1)
|
|
|
|
|
// populate: If true, use MAP_POPULATE to prefault pages
|
|
|
|
|
//
|
|
|
|
|
// Returns: Aligned pointer or NULL on OOM
|
|
|
|
|
//
|
|
|
|
|
// Guarantees:
|
|
|
|
|
// - Returns NULL on OOM (never crashes)
|
|
|
|
|
// - Returned pointer is aligned to ss_size boundary
|
|
|
|
|
// - Logs OOM once per process (not spammy)
|
|
|
|
|
// - Updates g_ss_mmap_count counter
|
|
|
|
|
//
|
|
|
|
|
// Thread-safe: Yes (no shared state mutations except atomic counters)
|
|
|
|
|
void* ss_os_acquire(uint8_t size_class, size_t ss_size, uintptr_t ss_mask, int populate);
|
|
|
|
|
|
|
|
|
|
#endif // HAKMEM_SS_OS_ACQUIRE_BOX_H
|