diff --git a/core/box/ss_allocation_box.c b/core/box/ss_allocation_box.c index 609152bf..1e71a875 100644 --- a/core/box/ss_allocation_box.c +++ b/core/box/ss_allocation_box.c @@ -3,6 +3,7 @@ #include "ss_allocation_box.h" #include "ss_os_acquire_box.h" +#include "ss_prefault_box.h" #include "ss_cache_box.h" #include "ss_stats_box.h" #include "ss_ace_box.h" @@ -167,11 +168,28 @@ SuperSlab* superslab_allocate(uint8_t size_class) { } if (!ptr) { - int populate = atomic_exchange_explicit(&g_ss_populate_once, 0, memory_order_acq_rel); + // Prefault policy: decide MAP_POPULATE and optional manual touch + SSPrefaultPolicy pf_policy = ss_prefault_policy(); + int populate = 0; + + if (pf_policy == SS_PREFAULT_POPULATE || + pf_policy == SS_PREFAULT_TOUCH || + pf_policy == SS_PREFAULT_ASYNC) { + // 常時 MAP_POPULATE(+必要なら手動 touch) + populate = 1; + } else { + // OFF の場合のみ、従来の「ワンショット populate」挙動を温存 + populate = atomic_exchange_explicit(&g_ss_populate_once, 0, memory_order_acq_rel); + } + ptr = ss_os_acquire(size_class, ss_size, ss_mask, populate); if (!ptr) { return NULL; } + // 手動 prefault が要求されている場合は、mmap 直後に SuperSlab 全域を page-in + if (pf_policy == SS_PREFAULT_TOUCH || pf_policy == SS_PREFAULT_ASYNC) { + ss_prefault_region(ptr, ss_size); + } // Debug logging for REFILL with new allocation if (dbg == 1) { fprintf(stderr, "[REFILL] class=%d new_alloc=1 ss=%p\n", diff --git a/core/box/ss_prefault_box.h b/core/box/ss_prefault_box.h new file mode 100644 index 00000000..58bfcce5 --- /dev/null +++ b/core/box/ss_prefault_box.h @@ -0,0 +1,80 @@ +// ss_prefault_box.h - Box: SuperSlab Page Prefaulting Policy +// Purpose: Control SuperSlab prefault strategy (MAP_POPULATE / manual touch) +// Box Theory: +// - This box owns「いつ / どの程度 pf を前倒しするか」というポリシーだけを担当。 +// - 実際の mmap は ss_os_acquire_box(下層)、Superslab 初期化は ss_allocation_box(上層)が担当。 +// +// Responsibilities: +// - HAKMEM_SS_PREFAULT の値から prefault ポリシーを決定 +// - 必要に応じて SuperSlab 全域を 1 回だけ touch して page-in する +// +// ENV: +// HAKMEM_SS_PREFAULT=0 Off (既存動作に近い。g_ss_populate_once のワンショットのみ) +// HAKMEM_SS_PREFAULT=1 POPULATE (MAP_POPULATE のみ使用) +// HAKMEM_SS_PREFAULT=2 TOUCH (MAP_POPULATE + 手動4KB touch) +// HAKMEM_SS_PREFAULT=3 ASYNC (予約値。現状は TOUCH と同等に扱う) +// +// Default: +// - 未指定時は 1 (POPULATE) とし、SuperSlab 取得時の page fault を大きく削減する。 + +#ifndef HAKMEM_SS_PREFAULT_BOX_H +#define HAKMEM_SS_PREFAULT_BOX_H + +#include +#include +#include + +typedef enum { + SS_PREFAULT_OFF = 0, // No prefault (legacy: g_ss_populate_once のみ) + SS_PREFAULT_POPULATE = 1, // MAP_POPULATE only + SS_PREFAULT_TOUCH = 2, // MAP_POPULATE + manual page touch + SS_PREFAULT_ASYNC = 3 // Reserved for future background prefault +} SSPrefaultPolicy; + +// Return current prefault policy (parsed once from HAKMEM_SS_PREFAULT). +static inline SSPrefaultPolicy ss_prefault_policy(void) +{ + static int cached = -1; + if (cached != -1) { + return (SSPrefaultPolicy)cached; + } + + // 一時的な安全デフォルト: OFF + // (4MB MAP_POPULATE 問題の影響を避けつつ、必要なときだけ明示的に ON にする) + int policy = SS_PREFAULT_OFF; + const char* env = getenv("HAKMEM_SS_PREFAULT"); + if (env && *env) { + int v = atoi(env); + if (v < 0) v = 0; + if (v > 3) v = 3; + policy = v; + } + cached = policy; + return (SSPrefaultPolicy)policy; +} + +// Prefault a contiguous SuperSlab region by touching one byte per page. +// Caller is expected to call this「mmap直後」に 1 回だけ実行する。 +static inline void ss_prefault_region(void* addr, size_t size) +{ + SSPrefaultPolicy policy = ss_prefault_policy(); + if (policy < SS_PREFAULT_TOUCH) { + // POPULATE または OFF の場合は手動 touch は行わない。 + return; + } + if (!addr || size == 0) { + return; + } + + long ps = sysconf(_SC_PAGESIZE); + size_t page = (ps > 0) ? (size_t)ps : (size_t)4096; + + volatile char* p = (volatile char*)addr; + for (size_t off = 0; off < size; off += page) { + (void)p[off]; + } + // Ensure last byte is also touched (covers exact multiples). + (void)p[size - 1]; +} + +#endif // HAKMEM_SS_PREFAULT_BOX_H diff --git a/core/superslab_cache.c b/core/superslab_cache.c index 92dc808f..9df33863 100644 --- a/core/superslab_cache.c +++ b/core/superslab_cache.c @@ -76,11 +76,6 @@ void* ss_os_acquire(uint8_t size_class, size_t ss_size, uintptr_t ss_mask, int p size_t alloc_size = ss_size * 2; int flags = MAP_PRIVATE | MAP_ANONYMOUS; -#ifdef MAP_POPULATE - if (populate) { - flags |= MAP_POPULATE; - } -#endif void* raw = mmap(NULL, alloc_size, PROT_READ | PROT_WRITE, flags, @@ -110,13 +105,15 @@ void* ss_os_acquire(uint8_t size_class, size_t ss_size, uintptr_t ss_mask, int p } size_t suffix_size = alloc_size - prefix_size - ss_size; if (suffix_size > 0) { - if (populate) { -#ifdef MADV_DONTNEED - madvise((char*)ptr + ss_size, suffix_size, MADV_DONTNEED); + // 余剰領域は常に munmap して、実際に使用する SuperSlab サイズだけを残す。 + munmap((char*)ptr + ss_size, suffix_size); + } + + // populate が要求されている場合は、実際に使う SuperSlab 領域だけを事前 fault-in する。 + if (populate) { +#ifdef MADV_WILLNEED + madvise(ptr, ss_size, MADV_WILLNEED); #endif - } else { - munmap((char*)ptr + ss_size, suffix_size); - } } ss_stats_os_alloc(size_class, ss_size); diff --git a/docs/specs/ENV_VARS.md b/docs/specs/ENV_VARS.md index 7ff84560..a2bfe4d6 100644 --- a/docs/specs/ENV_VARS.md +++ b/docs/specs/ENV_VARS.md @@ -176,6 +176,14 @@ SuperSlab cache / precharge(Phase 6.24+) - HAKMEM_TINY_SS_POPULATE_ONCE=1 - 次回 `mmap` で取得する SuperSlab を 1 回だけ `MAP_POPULATE` で fault-in(A/B 用のワンショットプリタッチ)。 +SuperSlab prefault(PF 削減のための前倒し) +- HAKMEM_SS_PREFAULT=0/1/2/3 + - 0: OFF(安全デフォルト。`g_ss_populate_once` によるワンショット `MAP_POPULATE` のみ) + - 1: POPULATE — 新規 SuperSlab の `mmap` で常に `MAP_POPULATE` を付与し、page fault を kernel 側で事前解消(perf 計測用)。 + - 2: TOUCH — `MAP_POPULATE` に加えて `ss_prefault_region()` で SuperSlab 全域を 1 回 4KB stride で touch(PF をほぼゼロにしたい実験用)。 + - 3: ASYNC — 予約値(現状は TOUCH と同等の扱いだが、将来的に BG thread prefault 用に拡張予定)。 + - Box: `core/box/ss_prefault_box.h`(ポリシー決定)+ `core/box/ss_allocation_box.c`(mmap 直後の実行)。 + Harvest / Guard(mmap前の収穫ゲート) - HAKMEM_TINY_SS_CAP=N - Tiny 各クラスにおける SuperSlab 上限(0=無制限)。 diff --git a/docs/specs/ENV_VARS_COMPLETE.md b/docs/specs/ENV_VARS_COMPLETE.md index efc0222b..1c693104 100644 --- a/docs/specs/ENV_VARS_COMPLETE.md +++ b/docs/specs/ENV_VARS_COMPLETE.md @@ -151,6 +151,44 @@ From `/mnt/workdisk/public_share/hakmem/core/hakmem_tiny_stats.h`: - **Purpose**: Interval between partial slab checks - **Impact**: Lower = more aggressive trimming +#### HAKMEM_TINY_SS_CACHE +- **Default**: 0 (disabled) +- **Purpose**: Per-class SuperSlab cache capacity +- **Impact**: Limits how many freed SuperSlabs are kept in LRU cache before munmap + +#### HAKMEM_TINY_SS_CACHE_C{0..7} +- **Default**: unset (inherits `HAKMEM_TINY_SS_CACHE`) +- **Purpose**: Per-class overrides for cache capacity +- **Impact**: Fine-grained control of cache size per Tiny class + +#### HAKMEM_TINY_SS_PRECHARGE +- **Default**: 0 +- **Purpose**: Precharge (pre-allocate) SuperSlabs into cache at startup/runtime +- **Impact**: Reduces first-use page faults by having warm SuperSlabs ready + +#### HAKMEM_TINY_SS_PRECHARGE_C{0..7} +- **Default**: unset (inherits `HAKMEM_TINY_SS_PRECHARGE`) +- **Purpose**: Per-class precharge targets +- **Impact**: e.g., `HAKMEM_TINY_SS_PRECHARGE_C0=4` precharges 4 SuperSlabs for class 0 + +#### HAKMEM_TINY_SS_POPULATE_ONCE +- **Default**: 0 +- **Purpose**: Use `MAP_POPULATE` for the next SuperSlab allocation only +- **Impact**: One-shot prefault for A/B testing; superseded by `HAKMEM_SS_PREFAULT` for常時運用 + +#### HAKMEM_SS_PREFAULT +- **Default**: `0` (OFF, safety-first default) +- **Type**: integer (0–3) +- **Purpose**: Control SuperSlab prefault strategy to reduce kernel page fault overhead (enabled explicitly when tuning). +- **Values**: + - `0` = OFF — legacy behavior, only `HAKMEM_TINY_SS_POPULATE_ONCE` may trigger a one-shot `MAP_POPULATE`(現状の安全デフォルト)。 + - `1` = POPULATE — always pass `populate=1` to `ss_os_acquire()` (use `MAP_POPULATE` for every new SuperSlab). **要 perf 確認。** + - `2` = TOUCH — POPULATE + `ss_prefault_region()` touches each page once (4KB stride) after `mmap`(実験用)。 + - `3` = ASYNC — reserved for future background-prefault implementation (currently treated as TOUCH). +- **Implementation**: + - Policy Box: `core/box/ss_prefault_box.h` + - Integration: `core/box/ss_allocation_box.c` calls `ss_prefault_policy()` to set `populate` and `ss_prefault_region()` immediately after `ss_os_acquire()`. + --- ### 4. Remote Free & Background Processing