diff --git a/build.sh b/build.sh index 29dd0f00..772bc18f 100755 --- a/build.sh +++ b/build.sh @@ -1,29 +1,114 @@ #!/usr/bin/env bash -# build.sh - Unified build wrapper to eliminate flag drift +# build.sh - Unified build wrapper (Phase 7 + Pool TLS) with discoverable help +# +# Quick use: +# ./build.sh bench_pool_tls_hakmem # Recommended target +# ./build.sh help # Show usage/hints/ENV +# ./build.sh verify bench_pool_tls_hakmem # Check freshness +# +# Notes: +# - Flags are pinned to avoid drift (see below). You can pass extra make flags via +# EXTRA_MAKEFLAGS, e.g. EXTRA_MAKEFLAGS="HAKMEM_DEBUG_VERBOSE=1" ./build.sh +# - Arena ENV (Pool TLS): HAKMEM_POOL_TLS_ARENA_MB_INIT/MAX/GROWTH_LEVELS +# - See also: docs/BUILDING_QUICKSTART.md + set -euo pipefail TARGET="${1:-bench_mid_large_mt_hakmem}" +usage() { + cat <<'USAGE' +========================================= + HAKMEM Build Script (help) +========================================= +Usage: + ./build.sh + ./build.sh help # Show this help + ./build.sh list # Show common targets + ./build.sh verify # Verify binary freshness + +Common targets (curated): + - bench_random_mixed_hakmem + - bench_pool_tls_hakmem + - bench_mid_large_mt_hakmem + - larson_hakmem + +Pinned build flags (by default): + POOL_TLS_PHASE1=1 HEADER_CLASSIDX=1 AGGRESSIVE_INLINE=1 PREWARM_TLS=1 POOL_TLS_PREWARM=1 + +Extra flags (optional): + Use environment var EXTRA_MAKEFLAGS, e.g.: + EXTRA_MAKEFLAGS="HAKMEM_DEBUG_VERBOSE=1" ./build.sh bench_pool_tls_hakmem + EXTRA_MAKEFLAGS="HAKMEM_TINY_SAFE_FREE=1" ./build.sh bench_random_mixed_hakmem + +Pool TLS Arena ENV (A/B friendly): + export HAKMEM_POOL_TLS_ARENA_MB_INIT=2 # default 1 + export HAKMEM_POOL_TLS_ARENA_MB_MAX=16 # default 8 + export HAKMEM_POOL_TLS_ARENA_GROWTH_LEVELS=4 # default 3 + +Verify & perf tips: + make print-flags + ./verify_build.sh + perf stat -e cycles,instructions,branches,branch-misses,cache-misses -r 3 -- ./ ... + strace -e trace=mmap,madvise,munmap -c ./ ... +USAGE +} + +list_targets() { + cat <<'LIST' +Common build targets: + bench_random_mixed_hakmem # Tiny 1T mixed + bench_pool_tls_hakmem # Pool TLS (8–52KB) + bench_mid_large_mt_hakmem # Mid-Large MT (8–32KB) + larson_hakmem # Larson mixed + bench_random_mixed_system # glibc baseline + bench_pool_tls_system # glibc baseline (PoolTLS workload) + bench_mid_large_mt_system # glibc baseline (Mid-Large workload) +LIST +} + +if [[ "${TARGET}" == "help" || "${TARGET}" == "-h" || "${TARGET}" == "--help" ]]; then + usage + exit 0 +fi + +if [[ "${TARGET}" == "list" ]]; then + list_targets + exit 0 +fi + +if [[ "${TARGET}" == "verify" ]]; then + BIN="${2:-}" + if [[ -z "${BIN}" ]]; then + echo "Usage: ./build.sh verify " >&2 + exit 2 + fi + ./verify_build.sh "${BIN}" + exit 0 +fi + echo "=========================================" echo " HAKMEM Build Script" echo " Target: ${TARGET}" +echo " Flags: POOL_TLS_PHASE1=1 POOL_TLS_PREWARM=1 HEADER_CLASSIDX=1 AGGRESSIVE_INLINE=1 PREWARM_TLS=1 ${EXTRA_MAKEFLAGS:-}" echo "=========================================" # Always clean to avoid stale objects when toggling flags make clean >/dev/null 2>&1 || true -# Phase 7 + Pool TLS Phase 1.5b defaults +# Phase 7 + Pool TLS defaults (pinned) + user extras make \ POOL_TLS_PHASE1=1 \ POOL_TLS_PREWARM=1 \ HEADER_CLASSIDX=1 \ AGGRESSIVE_INLINE=1 \ PREWARM_TLS=1 \ + ${EXTRA_MAKEFLAGS:-} \ "${TARGET}" echo "" echo "=========================================" echo " ✅ Build successful" echo " Run: ./${TARGET}" +echo " Tip: ./build.sh help # flags, ENV, targets" echo "=========================================" - diff --git a/core/pool_refill.c b/core/pool_refill.c index a5bed62f..39e6f31d 100644 --- a/core/pool_refill.c +++ b/core/pool_refill.c @@ -1,5 +1,7 @@ #include "pool_refill.h" #include "pool_tls.h" +#include "pool_tls_arena.h" +#include "pool_tls_remote.h" #include #include #include @@ -12,6 +14,26 @@ void* pool_refill_and_alloc(int class_idx) { int count = pool_get_refill_count(class_idx); if (count <= 0) return NULL; + // Refill boundary integration: try draining remote frees first. + // If we can satisfy from remote queue, avoid backend carve (OS pressure). + { + void* rchain = NULL; + int rgot = pool_remote_pop_chain(class_idx, count, &rchain); + if (rgot > 0 && rchain) { + // Pop one to return, install the rest into TLS + void* ret = rchain; + rchain = *(void**)rchain; + rgot--; + if (rgot > 0 && rchain) { + pool_install_chain(class_idx, rchain, rgot); + } + #if POOL_USE_HEADERS + *((uint8_t*)ret - POOL_HEADER_SIZE) = POOL_MAGIC | class_idx; + #endif + return ret; + } + } + // Batch allocate from existing Pool backend void* chain = backend_batch_carve(class_idx, count); if (!chain) return NULL; // OOM @@ -34,65 +56,36 @@ void* pool_refill_and_alloc(int class_idx) { return ret; } -// Backend batch carve - Phase 1: Direct mmap allocation +// Backend batch carve - Phase 1.5a: TLS Arena with chunk carving void* backend_batch_carve(int class_idx, int count) { if (class_idx < 0 || class_idx >= POOL_SIZE_CLASSES || count <= 0) { return NULL; } - // Get the class size - size_t block_size = POOL_CLASS_SIZES[class_idx]; + // Allocate blocks array on stack + void* blocks[64]; // Max refill count is 64 + if (count > 64) count = 64; - // For Phase 1: Allocate a single large chunk via mmap - // and carve it into blocks - #if POOL_USE_HEADERS - size_t total_block_size = block_size + POOL_HEADER_SIZE; - #else - size_t total_block_size = block_size; - #endif - - // Allocate enough for all requested blocks - size_t total_size = total_block_size * count; - - // Round up to page size - size_t page_size = 4096; - total_size = (total_size + page_size - 1) & ~(page_size - 1); - - // Allocate memory via mmap - void* chunk = mmap(NULL, total_size, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - if (chunk == MAP_FAILED) { - return NULL; + // Carve from TLS Arena (Phase 1.5a) + int carved = arena_batch_carve(class_idx, blocks, count); + if (carved == 0) { + return NULL; // OOM } - // Carve into blocks and chain them + // Chain the carved blocks void* head = NULL; void* tail = NULL; - char* ptr = (char*)chunk; - for (int i = 0; i < count; i++) { - #if POOL_USE_HEADERS - // Skip header space - user data starts after header - void* user_ptr = ptr + POOL_HEADER_SIZE; - #else - void* user_ptr = ptr; - #endif + for (int i = 0; i < carved; i++) { + void* block = blocks[i]; - // Chain the blocks + // Chain the block if (!head) { - head = user_ptr; - tail = user_ptr; + head = block; + tail = block; } else { - *(void**)tail = user_ptr; - tail = user_ptr; - } - - // Move to next block - ptr += total_block_size; - - // Stop if we'd go past the allocated chunk - if ((ptr + total_block_size) > ((char*)chunk + total_size)) { - break; + *(void**)tail = block; + tail = block; } } @@ -102,4 +95,4 @@ void* backend_batch_carve(int class_idx, int count) { } return head; -} \ No newline at end of file +} diff --git a/core/slab_handle.h b/core/slab_handle.h index d810e342..c69d10ef 100644 --- a/core/slab_handle.h +++ b/core/slab_handle.h @@ -266,6 +266,17 @@ static inline void* slab_freelist_pop(SlabHandle* h) { } void* ptr = h->meta->freelist; + // Option B: Defense-in-depth against sentinel leakage + if (__builtin_expect((uintptr_t)ptr == TINY_REMOTE_SENTINEL, 0)) { + if (__builtin_expect(g_debug_remote_guard, 0)) { + fprintf(stderr, "[FREELIST_POP] sentinel detected in freelist (cls=%u slab=%u) -> break chain\n", + h->ss ? h->ss->size_class : 0u, + (unsigned)h->slab_idx); + } + h->meta->freelist = NULL; // break the chain to avoid propagating corruption + if (__builtin_expect(g_tiny_safe_free_strict, 0)) { raise(SIGUSR2); } + return NULL; + } if (ptr) { void* next = *(void**)ptr; h->meta->freelist = next; diff --git a/core/superslab/superslab_inline.h b/core/superslab/superslab_inline.h index 397689d3..f3acec74 100644 --- a/core/superslab/superslab_inline.h +++ b/core/superslab/superslab_inline.h @@ -344,6 +344,18 @@ static inline void _ss_remote_drain_to_freelist_unsafe(SuperSlab* ss, int slab_i _Atomic(uintptr_t)* head = &ss->remote_heads[slab_idx]; uintptr_t p = atomic_exchange_explicit(head, (uintptr_t)NULL, memory_order_acq_rel); if (p == 0) return; + // Option A: Fail-fast guard against sentinel leaking into freelist + if (__builtin_expect(p == TINY_REMOTE_SENTINEL, 0)) { + if (__builtin_expect(g_debug_remote_guard, 0)) { + fprintf(stderr, "[REMOTE_DRAIN] head is sentinel! cls=%u slab=%d head=%p\n", + ss ? ss->size_class : 0u, + slab_idx, + (void*)p); + } + if (__builtin_expect(g_tiny_safe_free_strict, 0)) { raise(SIGUSR2); } + // Drop this drain attempt to prevent corrupting freelist + return; + } uint32_t drained = 0; uintptr_t base = (uintptr_t)ss; @@ -369,6 +381,15 @@ static inline void _ss_remote_drain_to_freelist_unsafe(SuperSlab* ss, int slab_i } } void* node = (void*)p; + // Additional defensive check (should be redundant with head guard) + if (__builtin_expect((uintptr_t)node == TINY_REMOTE_SENTINEL, 0)) { + if (__builtin_expect(g_debug_remote_guard, 0)) { + fprintf(stderr, "[REMOTE_DRAIN] node sentinel detected, abort chain (cls=%u slab=%d)\n", + ss ? ss->size_class : 0u, slab_idx); + } + if (__builtin_expect(g_tiny_safe_free_strict, 0)) { raise(SIGUSR2); } + break; + } uintptr_t next = tiny_remote_side_get(ss, slab_idx, node); tiny_remote_watch_note("drain_pull", ss, slab_idx, node, 0xA238u, drain_tid, 0); if (__builtin_expect(g_remote_side_enable, 0)) { @@ -378,20 +399,24 @@ static inline void _ss_remote_drain_to_freelist_unsafe(SuperSlab* ss, int slab_i uintptr_t observed = atomic_load_explicit((_Atomic uintptr_t*)node, memory_order_relaxed); tiny_remote_report_corruption("drain", node, observed); TinySlabMeta* meta = &ss->slabs[slab_idx]; - fprintf(stderr, - "[REMOTE_SENTINEL-DRAIN] cls=%u slab=%d node=%p drained=%u observed=0x%016" PRIxPTR " owner=%u used=%u freelist=%p\n", - ss->size_class, - slab_idx, - node, - drained, - observed, - meta->owner_tid, - (unsigned)meta->used, - meta->freelist); + if (__builtin_expect(g_debug_remote_guard, 0)) { + fprintf(stderr, + "[REMOTE_SENTINEL-DRAIN] cls=%u slab=%d node=%p drained=%u observed=0x%016" PRIxPTR " owner=%u used=%u freelist=%p\n", + ss->size_class, + slab_idx, + node, + drained, + observed, + meta->owner_tid, + (unsigned)meta->used, + meta->freelist); + } if (g_tiny_safe_free_strict) { raise(SIGUSR2); return; } } tiny_remote_side_clear(ss, slab_idx, node); } + // Always sanitize node header before linking into freelist (defense-in-depth) + // Overwrite any stale sentinel/value in node[0] with the local chain link. tiny_remote_watch_note("drain_link", ss, slab_idx, node, 0xA239u, drain_tid, 0); tiny_remote_track_on_remote_drain(ss, slab_idx, node, "remote_drain", drain_tid); if (__builtin_expect(g_debug_remote_guard && drained < 3, 0)) { diff --git a/core/tiny_adaptive_sizing.c b/core/tiny_adaptive_sizing.c index e8803930..b96dd467 100644 --- a/core/tiny_adaptive_sizing.c +++ b/core/tiny_adaptive_sizing.c @@ -12,8 +12,8 @@ __thread TLSCacheStats g_tls_cache_stats[TINY_NUM_CLASSES]; // Global enable flag (default: enabled, can disable via env) int g_adaptive_sizing_enabled = 1; -// Logging enable flag (default: enabled for debugging) -static int g_adaptive_logging_enabled = 1; +// Logging enable flag (default: disabled; enable via HAKMEM_ADAPTIVE_LOG=1) +static int g_adaptive_logging_enabled = 0; // Forward declaration for draining blocks extern void tiny_superslab_return_block(void* ptr, int class_idx); diff --git a/core/tiny_alloc_fast_inline.h b/core/tiny_alloc_fast_inline.h index 94fc1613..5479a1d7 100644 --- a/core/tiny_alloc_fast_inline.h +++ b/core/tiny_alloc_fast_inline.h @@ -8,6 +8,7 @@ #include #include "hakmem_build_flags.h" +#include "tiny_remote.h" // for TINY_REMOTE_SENTINEL (defense-in-depth) // External TLS variables (defined in hakmem_tiny.c) extern __thread void* g_tls_sll_head[TINY_NUM_CLASSES]; @@ -42,12 +43,19 @@ extern __thread uint32_t g_tls_sll_count[TINY_NUM_CLASSES]; #define TINY_ALLOC_FAST_POP_INLINE(class_idx, ptr_out) do { \ void* _head = g_tls_sll_head[(class_idx)]; \ if (__builtin_expect(_head != NULL, 1)) { \ - void* _next = *(void**)_head; \ - g_tls_sll_head[(class_idx)] = _next; \ - if (g_tls_sll_count[(class_idx)] > 0) { \ - g_tls_sll_count[(class_idx)]--; \ + if (__builtin_expect((uintptr_t)_head == TINY_REMOTE_SENTINEL, 0)) { \ + /* Break the chain defensively if sentinel leaked into TLS SLL */ \ + g_tls_sll_head[(class_idx)] = NULL; \ + if (g_tls_sll_count[(class_idx)] > 0) g_tls_sll_count[(class_idx)]--; \ + (ptr_out) = NULL; \ + } else { \ + void* _next = *(void**)_head; \ + g_tls_sll_head[(class_idx)] = _next; \ + if (g_tls_sll_count[(class_idx)] > 0) { \ + g_tls_sll_count[(class_idx)]--; \ + } \ + (ptr_out) = _head; \ } \ - (ptr_out) = _head; \ } else { \ (ptr_out) = NULL; \ } \ diff --git a/core/tiny_superslab_free.inc.h b/core/tiny_superslab_free.inc.h index 2f88dc18..fd0cd71f 100644 --- a/core/tiny_superslab_free.inc.h +++ b/core/tiny_superslab_free.inc.h @@ -198,19 +198,21 @@ static inline void hak_tiny_free_superslab(void* ptr, SuperSlab* ss) { tiny_debug_ring_record(TINY_RING_EVENT_REMOTE_INVALID, (uint16_t)ss->size_class, (void*)cur, aux); uintptr_t observed = atomic_load_explicit((_Atomic uintptr_t*)(void*)cur, memory_order_relaxed); tiny_remote_report_corruption("scan", (void*)cur, observed); - fprintf(stderr, - "[REMOTE_SENTINEL] cls=%u slab=%d cur=%p head=%p ptr=%p scanned=%d observed=0x%016" PRIxPTR " owner=%u used=%u freelist=%p remote_head=%p\n", - ss->size_class, - slab_idx, - (void*)cur, - (void*)head, - ptr, - scanned, - observed, - meta->owner_tid, - (unsigned)meta->used, - meta->freelist, - (void*)atomic_load_explicit(&ss->remote_heads[slab_idx], memory_order_relaxed)); + if (__builtin_expect(g_debug_remote_guard, 0)) { + fprintf(stderr, + "[REMOTE_SENTINEL] cls=%u slab=%d cur=%p head=%p ptr=%p scanned=%d observed=0x%016" PRIxPTR " owner=%u used=%u freelist=%p remote_head=%p\n", + ss->size_class, + slab_idx, + (void*)cur, + (void*)head, + ptr, + scanned, + observed, + meta->owner_tid, + (unsigned)meta->used, + meta->freelist, + (void*)atomic_load_explicit(&ss->remote_heads[slab_idx], memory_order_relaxed)); + } if (g_tiny_safe_free_strict) { raise(SIGUSR2); return; } break; } diff --git a/docs/BUILDING_QUICKSTART.md b/docs/BUILDING_QUICKSTART.md new file mode 100644 index 00000000..c81225e5 --- /dev/null +++ b/docs/BUILDING_QUICKSTART.md @@ -0,0 +1,31 @@ +# BUILDING Quickstart + +One‑liner (recommended) +- `./build.sh ` + - Pins: `POOL_TLS_PHASE1=1 HEADER_CLASSIDX=1 AGGRESSIVE_INLINE=1 PREWARM_TLS=1 POOL_TLS_PREWARM=1` +- Help/targets: `./build.sh help`, `./build.sh list` +- Verify freshness: `./build.sh verify ` + +Common targets +- `bench_random_mixed_hakmem` (Tiny 1T mixed) +- `bench_pool_tls_hakmem` (Pool TLS 8–52KB) +- `bench_mid_large_mt_hakmem` (Mid‑Large MT 8–32KB) +- `larson_hakmem` (Larson) +- System baselines: `bench_*_system` + +Pool TLS Arena ENV (A/B) +- `export HAKMEM_POOL_TLS_ARENA_MB_INIT=2` # default 1 +- `export HAKMEM_POOL_TLS_ARENA_MB_MAX=16` # default 8 +- `export HAKMEM_POOL_TLS_ARENA_GROWTH_LEVELS=4` # default 3 + +Runtime safety/verbosity (optional) +- `EXTRA_MAKEFLAGS="HAKMEM_TINY_SAFE_FREE=1" ./build.sh ` +- `EXTRA_MAKEFLAGS="HAKMEM_DEBUG_VERBOSE=1" ./build.sh ` + +Perf & strace +- `perf stat -e cycles,instructions,branches,branch-misses,cache-misses -r 3 -- ./ ...` +- `strace -e trace=mmap,madvise,munmap -c ./ ...` + +Troubleshooting +- `make print-flags` to inspect flags +- `./verify_build.sh ` to check binary freshness