Summary: - Phase 24 (alloc stats): +0.93% GO - Phase 25 (free stats): +1.07% GO - Phase 26 (diagnostics): -0.33% NEUTRAL (code cleanliness) - Total: 11 atomics compiled-out, +2.00% improvement Phase 24: OBSERVE tax prune (tiny_class_stats_box.h) - Added HAKMEM_TINY_CLASS_STATS_COMPILED (default: 0) - Wrapped 5 stats functions: uc_miss, warm_hit, shared_lock, tls_carve_* - Result: +0.93% (baseline 56.675M vs compiled-in 56.151M ops/s) Phase 25: Tiny free stats prune (tiny_superslab_free.inc.h) - Added HAKMEM_TINY_FREE_STATS_COMPILED (default: 0) - Wrapped g_free_ss_enter atomic in free hot path - Result: +1.07% (baseline 57.017M vs compiled-in 56.415M ops/s) Phase 26: Hot path diagnostic atomics prune - Added 5 compile gates for low-frequency error counters: - HAKMEM_TINY_C7_FREE_COUNT_COMPILED - HAKMEM_TINY_HDR_MISMATCH_LOG_COMPILED - HAKMEM_TINY_HDR_META_MISMATCH_COMPILED - HAKMEM_TINY_METRIC_BAD_CLASS_COMPILED - HAKMEM_TINY_HDR_META_FAST_COMPILED - Result: -0.33% NEUTRAL (within noise, kept for cleanliness) Alignment with mimalloc principles: - "No atomics on hot path" - telemetry moved to compile-time opt-in - Fixed per-op tax elimination - Production builds: maximum performance (atomics compiled-out) - Research builds: full diagnostics (COMPILED=1) Generated with Claude Code https://claude.com/claude-code Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
233 lines
8.2 KiB
C
233 lines
8.2 KiB
C
// tiny_header_box.h - Header Box: Single Source of Truth for Header Operations
|
|
//
|
|
// Design Principles:
|
|
// 1. All header logic flows from tiny_nextptr.h specification
|
|
// 2. Encapsulates "which classes preserve headers" knowledge
|
|
// 3. Eliminates hardcoded class_idx checks (class_idx != 7, etc.)
|
|
// 4. Provides type-safe header read/write/validate operations
|
|
//
|
|
// Background:
|
|
// - C0 (8B): next_off=0 → header overwritten by next pointer
|
|
// - C1-C6 (16B-1024B): next_off=1 → header preserved in freelist
|
|
// - C7 (2048B): next_off=0 → header overwritten by next pointer
|
|
//
|
|
// Migration:
|
|
// ❌ FORBIDDEN: class_idx != 7, class_idx == 0 || class_idx == 7
|
|
// ❌ FORBIDDEN: *(uint8_t*)base = HEADER_MAGIC | ...
|
|
// ✅ USE: tiny_class_preserves_header(class_idx)
|
|
// ✅ USE: tiny_header_write_if_preserved(base, class_idx)
|
|
// ✅ USE: tiny_header_validate(base, class_idx, ...)
|
|
|
|
#ifndef TINY_HEADER_BOX_H
|
|
#define TINY_HEADER_BOX_H
|
|
|
|
#include <stdint.h>
|
|
#include <stdbool.h>
|
|
#include "../hakmem_build_flags.h"
|
|
#include "tiny_layout_box.h"
|
|
#include "../tiny_region_id.h"
|
|
|
|
// ============================================================================
|
|
// Core Predicate: Does this class preserve headers in freelist?
|
|
// ============================================================================
|
|
//
|
|
// This is the SINGLE SOURCE OF TRUTH for header preservation logic.
|
|
// All code must use this instead of hardcoded class_idx checks.
|
|
//
|
|
// Implementation:
|
|
// - Delegates to tiny_nextptr_offset() from tiny_layout_box.h
|
|
// - offset=0 → header overwritten by next pointer → false
|
|
// - offset!=0 → header preserved → true
|
|
//
|
|
// Returns:
|
|
// true - C1-C6: Header preserved at offset 0, next at offset 1
|
|
// false - C0: Header overwritten by next pointer at offset 0
|
|
// Phase 13 v1: C7 returns false (default) or true (HAKMEM_TINY_C7_PRESERVE_HEADER=1)
|
|
|
|
static inline bool tiny_class_preserves_header(int class_idx) {
|
|
#if HAKMEM_TINY_HEADER_CLASSIDX
|
|
// Delegate to tiny_layout_box.h specification (Single Source of Truth)
|
|
// next_off=0 → header overwritten (C0, C7 default)
|
|
// next_off=1 → header preserved (C1-C6, C7 with HAKMEM_TINY_C7_PRESERVE_HEADER=1)
|
|
return tiny_nextptr_offset(class_idx) != 0;
|
|
#else
|
|
// Headers disabled globally
|
|
(void)class_idx;
|
|
return false;
|
|
#endif
|
|
}
|
|
|
|
// ============================================================================
|
|
// Header Write (Conditional - for freelist/TLS SLL operations)
|
|
// ============================================================================
|
|
//
|
|
// Writes header ONLY if this class preserves headers.
|
|
// For C0/C7, writing header is pointless (next pointer will overwrite it).
|
|
//
|
|
// Use this when:
|
|
// - Pushing blocks to TLS SLL
|
|
// - Moving blocks from freelist to TLS SLL
|
|
// - Splicing chains into TLS SLL
|
|
//
|
|
// DO NOT use this for:
|
|
// - Allocation path (use tiny_header_write_for_alloc instead)
|
|
|
|
static inline void tiny_header_write_if_preserved(void* base, int class_idx) {
|
|
#if HAKMEM_TINY_HEADER_CLASSIDX
|
|
if (tiny_class_preserves_header(class_idx)) {
|
|
*(uint8_t*)base = HEADER_MAGIC | (class_idx & HEADER_CLASS_MASK);
|
|
}
|
|
#else
|
|
(void)base;
|
|
(void)class_idx;
|
|
#endif
|
|
}
|
|
|
|
// ============================================================================
|
|
// Header Validate (Conditional - for TLS SLL pop operations)
|
|
// ============================================================================
|
|
//
|
|
// Validates header ONLY if this class preserves headers.
|
|
// For C0, validation is impossible (next pointer is stored at offset 0).
|
|
// Phase 13 v1: C7 validation depends on HAKMEM_TINY_C7_PRESERVE_HEADER.
|
|
//
|
|
// Arguments:
|
|
// base - BASE pointer (not user pointer)
|
|
// class_idx - Expected class index
|
|
// out_got - [optional] Store actual header byte read
|
|
// out_expect- [optional] Store expected header byte
|
|
//
|
|
// Returns:
|
|
// true - Header valid OR class doesn't preserve headers (C0/C7)
|
|
// false - Header mismatch (corruption detected)
|
|
//
|
|
// Use this when:
|
|
// - Popping blocks from TLS SLL
|
|
// - Validating freelist integrity
|
|
|
|
static inline bool tiny_header_validate(const void* base, int class_idx,
|
|
uint8_t* out_got, uint8_t* out_expect) {
|
|
#if HAKMEM_TINY_HEADER_CLASSIDX
|
|
// C0/C7: Validation impossible (next pointer stored at offset 0)
|
|
if (!tiny_class_preserves_header(class_idx)) {
|
|
return true; // Always valid (no header to check)
|
|
}
|
|
|
|
// C1-C6: Validate header
|
|
uint8_t got = *(const uint8_t*)base;
|
|
uint8_t expect = HEADER_MAGIC | (class_idx & HEADER_CLASS_MASK);
|
|
|
|
if (out_got) *out_got = got;
|
|
if (out_expect) *out_expect = expect;
|
|
|
|
return got == expect;
|
|
#else
|
|
(void)base;
|
|
(void)class_idx;
|
|
(void)out_got;
|
|
(void)out_expect;
|
|
return true;
|
|
#endif
|
|
}
|
|
|
|
// ============================================================================
|
|
// Header Write (Unconditional - for allocation path)
|
|
// ============================================================================
|
|
//
|
|
// ALWAYS writes header, regardless of class.
|
|
// For C0/C7, header will be overwritten when block enters freelist,
|
|
// but must be valid when returned to user.
|
|
//
|
|
// Use this ONLY in allocation path:
|
|
// - HAK_RET_ALLOC_BLOCK macro
|
|
// - HAK_RET_ALLOC_BLOCK_TRACED macro
|
|
// - Before returning block to user
|
|
//
|
|
// DO NOT use this for:
|
|
// - Freelist operations (use tiny_header_write_if_preserved)
|
|
// - TLS SLL operations (use tiny_header_write_if_preserved)
|
|
|
|
static inline void tiny_header_write_for_alloc(void* base, int class_idx) {
|
|
#if HAKMEM_TINY_HEADER_CLASSIDX
|
|
*(uint8_t*)base = HEADER_MAGIC | (class_idx & HEADER_CLASS_MASK);
|
|
#else
|
|
(void)base;
|
|
(void)class_idx;
|
|
#endif
|
|
}
|
|
|
|
// ============================================================================
|
|
// Header Read (for diagnostics/debugging)
|
|
// ============================================================================
|
|
//
|
|
// Reads header byte without validation.
|
|
// Returns -1 if headers disabled or class doesn't preserve headers.
|
|
//
|
|
// Use this for:
|
|
// - Diagnostics
|
|
// - Debug logging
|
|
// - Corruption analysis
|
|
//
|
|
// DO NOT use this for:
|
|
// - Validation (use tiny_header_validate)
|
|
|
|
static inline int tiny_header_read(const void* base, int class_idx) {
|
|
#if HAKMEM_TINY_HEADER_CLASSIDX
|
|
if (!tiny_class_preserves_header(class_idx)) {
|
|
return -1; // No header to read
|
|
}
|
|
return (int)(*(const uint8_t*)base);
|
|
#else
|
|
(void)base;
|
|
(void)class_idx;
|
|
return -1;
|
|
#endif
|
|
}
|
|
|
|
// ============================================================================
|
|
// Header Finalize for Allocation (Phase 5 E5-2: Write-Once Optimization)
|
|
// ============================================================================
|
|
//
|
|
// Replaces direct calls to tiny_region_id_write_header() in allocation paths.
|
|
// Enables header write-once optimization:
|
|
// - C1-C6: Skip header write if already prefilled at refill boundary
|
|
// - C0, C7: Always write header (next pointer overwrites it anyway)
|
|
//
|
|
// Use this in allocation hot paths:
|
|
// - tiny_hot_alloc_fast()
|
|
// - unified_cache_pop()
|
|
// - All other allocation returns
|
|
//
|
|
// DO NOT use this for:
|
|
// - Freelist operations (use tiny_header_write_if_preserved)
|
|
// - Refill boundary (use direct write in unified_cache_refill)
|
|
|
|
// Forward declaration from tiny_region_id.h
|
|
void* tiny_region_id_write_header(void* base, int class_idx);
|
|
|
|
// Forward declaration from tiny_header_write_once_env_box.h
|
|
// NOTE: This is static inline in tiny_header_write_once_env_box.h, not extern
|
|
// Must include the header instead of forward declaring
|
|
#include "tiny_header_write_once_env_box.h"
|
|
|
|
static inline void* tiny_header_finalize_alloc(void* base, int class_idx) {
|
|
#if HAKMEM_TINY_HEADER_CLASSIDX
|
|
#if HAKMEM_TINY_HEADER_WRITE_ONCE_COMPILED
|
|
// Phase 23: Write-once optimization (compile-out when disabled, default OFF)
|
|
// Evaluate class check first (short-circuit), then ENV check
|
|
if (tiny_class_preserves_header(class_idx) && tiny_header_write_once_enabled()) {
|
|
// Header already written at refill boundary → skip write, return USER pointer
|
|
return (void*)((uint8_t*)base + 1);
|
|
}
|
|
#endif
|
|
|
|
// Traditional path: C0, C7, or WRITE_ONCE compiled-out/disabled
|
|
return tiny_region_id_write_header(base, class_idx);
|
|
#else
|
|
(void)class_idx;
|
|
return base;
|
|
#endif
|
|
}
|
|
|
|
#endif // TINY_HEADER_BOX_H
|