2025-11-05 12:31:14 +09:00
|
|
|
// hakmem_tiny_superslab.h - SuperSlab allocator for Tiny Pool (Phase 6.22)
|
|
|
|
|
// Purpose: mimalloc-inspired 2MB aligned slab allocation for fast pointer→slab lookup
|
|
|
|
|
// License: MIT
|
|
|
|
|
// Date: 2025-10-24
|
2025-11-07 23:05:33 +09:00
|
|
|
// Phase 6-2.8: Refactored into modular headers (types, inline)
|
2025-11-05 12:31:14 +09:00
|
|
|
|
|
|
|
|
#ifndef HAKMEM_TINY_SUPERSLAB_H
|
|
|
|
|
#define HAKMEM_TINY_SUPERSLAB_H
|
|
|
|
|
|
|
|
|
|
#include <stdint.h>
|
|
|
|
|
#include <stddef.h>
|
|
|
|
|
#include <stdbool.h>
|
|
|
|
|
#include <stdatomic.h>
|
|
|
|
|
#include <stdlib.h>
|
|
|
|
|
#include <time.h> // Phase 8.3: For clock_gettime() in hak_now_ns()
|
|
|
|
|
#include <signal.h>
|
|
|
|
|
#include <stdio.h> // For fprintf() debugging
|
|
|
|
|
#include <pthread.h>
|
2025-11-07 23:05:33 +09:00
|
|
|
|
|
|
|
|
// Phase 6-2.8: Modular headers (types, inline functions)
|
|
|
|
|
#include "superslab/superslab_types.h"
|
|
|
|
|
#include "superslab/superslab_inline.h"
|
|
|
|
|
|
|
|
|
|
// Legacy includes (for backward compatibility)
|
2025-11-05 12:31:14 +09:00
|
|
|
#include "tiny_debug_ring.h"
|
|
|
|
|
#include "tiny_remote.h"
|
2025-11-07 21:45:20 +09:00
|
|
|
#include "hakmem_tiny_superslab_constants.h" // Phase 6-2.5: Centralized layout constants
|
2025-11-05 12:31:14 +09:00
|
|
|
|
|
|
|
|
// Debug instrumentation flags (defined in hakmem_tiny.c)
|
|
|
|
|
extern int g_debug_remote_guard;
|
|
|
|
|
extern int g_tiny_safe_free_strict;
|
2025-11-07 01:27:04 +09:00
|
|
|
extern _Atomic uint64_t g_ss_active_dec_calls;
|
|
|
|
|
|
2025-11-07 23:05:33 +09:00
|
|
|
uint32_t tiny_remote_drain_threshold(void);
|
2025-11-05 12:31:14 +09:00
|
|
|
|
feat: Phase 7 + Phase 2 - Massive performance & stability improvements
Performance Achievements:
- Tiny allocations: +180-280% (21M → 59-70M ops/s random mixed)
- Single-thread: +24% (2.71M → 3.36M ops/s Larson)
- 4T stability: 0% → 95% (19/20 success rate)
- Overall: 91.3% of System malloc average (target was 40-55%) ✓
Phase 7 (Tasks 1-3): Core Optimizations
- Task 1: Header validation removal (Region-ID direct lookup)
- Task 2: Aggressive inline (TLS cache access optimization)
- Task 3: Pre-warm TLS cache (eliminate cold-start penalty)
Result: +180-280% improvement, 85-146% of System malloc
Critical Bug Fixes:
- Fix 64B allocation crash (size-to-class +1 for header)
- Fix 4T wrapper recursion bugs (BUG #7, #8, #10, #11)
- Remove malloc fallback (30% → 50% stability)
Phase 2a: SuperSlab Dynamic Expansion (CRITICAL)
- Implement mimalloc-style chunk linking
- Unlimited slab expansion (no more OOM at 32 slabs)
- Fix chunk initialization bug (bitmap=0x00000001 after expansion)
Files: core/hakmem_tiny_superslab.c/h, core/superslab/superslab_types.h
Result: 50% → 95% stability (19/20 4T success)
Phase 2b: TLS Cache Adaptive Sizing
- Dynamic capacity: 16-2048 slots based on usage
- High-water mark tracking + exponential growth/shrink
- Expected: +3-10% performance, -30-50% memory
Files: core/tiny_adaptive_sizing.c/h (new)
Phase 2c: BigCache Dynamic Hash Table
- Migrate from fixed 256×8 array to dynamic hash table
- Auto-resize: 256 → 512 → 1024 → 65,536 buckets
- Improved hash function (FNV-1a) + collision chaining
Files: core/hakmem_bigcache.c/h
Expected: +10-20% cache hit rate
Design Flaws Analysis:
- Identified 6 components with fixed-capacity bottlenecks
- SuperSlab (CRITICAL), TLS Cache (HIGH), BigCache/L2.5 (MEDIUM)
- Report: DESIGN_FLAWS_ANALYSIS.md (11 chapters)
Documentation:
- 13 comprehensive reports (PHASE*.md, DESIGN_FLAWS*.md)
- Implementation guides, test results, production readiness
- Bug fix reports, root cause analysis
Build System:
- Makefile: phase7 targets, PREWARM_TLS flag
- Auto dependency generation (-MMD -MP) for .inc files
Known Issues:
- 4T stability: 19/20 (95%) - investigating 1 failure for 100%
- L2.5 Pool dynamic sharding: design only (needs 2-3 days integration)
🤖 Generated with Claude Code (https://claude.com/claude-code)
Co-Authored-By: Claude <noreply@anthropic.com>
2025-11-08 17:08:00 +09:00
|
|
|
// ============================================================================
|
|
|
|
|
// Phase 2a: Dynamic Expansion - Global per-class SuperSlabHeads
|
|
|
|
|
// ============================================================================
|
|
|
|
|
|
|
|
|
|
extern SuperSlabHead* g_superslab_heads[TINY_NUM_CLASSES_SS];
|
|
|
|
|
|
2025-11-05 12:31:14 +09:00
|
|
|
// ============================================================================
|
|
|
|
|
// SuperSlab Management Functions
|
|
|
|
|
// ============================================================================
|
|
|
|
|
|
|
|
|
|
// Allocate a new SuperSlab (2MB aligned)
|
|
|
|
|
SuperSlab* superslab_allocate(uint8_t size_class);
|
|
|
|
|
|
|
|
|
|
// Free a SuperSlab
|
|
|
|
|
void superslab_free(SuperSlab* ss);
|
|
|
|
|
|
feat: Phase 7 + Phase 2 - Massive performance & stability improvements
Performance Achievements:
- Tiny allocations: +180-280% (21M → 59-70M ops/s random mixed)
- Single-thread: +24% (2.71M → 3.36M ops/s Larson)
- 4T stability: 0% → 95% (19/20 success rate)
- Overall: 91.3% of System malloc average (target was 40-55%) ✓
Phase 7 (Tasks 1-3): Core Optimizations
- Task 1: Header validation removal (Region-ID direct lookup)
- Task 2: Aggressive inline (TLS cache access optimization)
- Task 3: Pre-warm TLS cache (eliminate cold-start penalty)
Result: +180-280% improvement, 85-146% of System malloc
Critical Bug Fixes:
- Fix 64B allocation crash (size-to-class +1 for header)
- Fix 4T wrapper recursion bugs (BUG #7, #8, #10, #11)
- Remove malloc fallback (30% → 50% stability)
Phase 2a: SuperSlab Dynamic Expansion (CRITICAL)
- Implement mimalloc-style chunk linking
- Unlimited slab expansion (no more OOM at 32 slabs)
- Fix chunk initialization bug (bitmap=0x00000001 after expansion)
Files: core/hakmem_tiny_superslab.c/h, core/superslab/superslab_types.h
Result: 50% → 95% stability (19/20 4T success)
Phase 2b: TLS Cache Adaptive Sizing
- Dynamic capacity: 16-2048 slots based on usage
- High-water mark tracking + exponential growth/shrink
- Expected: +3-10% performance, -30-50% memory
Files: core/tiny_adaptive_sizing.c/h (new)
Phase 2c: BigCache Dynamic Hash Table
- Migrate from fixed 256×8 array to dynamic hash table
- Auto-resize: 256 → 512 → 1024 → 65,536 buckets
- Improved hash function (FNV-1a) + collision chaining
Files: core/hakmem_bigcache.c/h
Expected: +10-20% cache hit rate
Design Flaws Analysis:
- Identified 6 components with fixed-capacity bottlenecks
- SuperSlab (CRITICAL), TLS Cache (HIGH), BigCache/L2.5 (MEDIUM)
- Report: DESIGN_FLAWS_ANALYSIS.md (11 chapters)
Documentation:
- 13 comprehensive reports (PHASE*.md, DESIGN_FLAWS*.md)
- Implementation guides, test results, production readiness
- Bug fix reports, root cause analysis
Build System:
- Makefile: phase7 targets, PREWARM_TLS flag
- Auto dependency generation (-MMD -MP) for .inc files
Known Issues:
- 4T stability: 19/20 (95%) - investigating 1 failure for 100%
- L2.5 Pool dynamic sharding: design only (needs 2-3 days integration)
🤖 Generated with Claude Code (https://claude.com/claude-code)
Co-Authored-By: Claude <noreply@anthropic.com>
2025-11-08 17:08:00 +09:00
|
|
|
// Phase 2a: Dynamic Expansion Functions
|
|
|
|
|
// Initialize SuperSlabHead for a class (called once per class)
|
|
|
|
|
SuperSlabHead* init_superslab_head(int class_idx);
|
|
|
|
|
|
|
|
|
|
// Expand SuperSlabHead by allocating and linking a new chunk
|
|
|
|
|
// Returns 0 on success, -1 on OOM
|
|
|
|
|
int expand_superslab_head(SuperSlabHead* head);
|
|
|
|
|
|
|
|
|
|
// Find which chunk a pointer belongs to
|
|
|
|
|
// Returns the chunk containing ptr, or NULL if not found
|
|
|
|
|
SuperSlab* find_chunk_for_ptr(void* ptr, int class_idx);
|
|
|
|
|
|
2025-11-05 12:31:14 +09:00
|
|
|
// Initialize a slab within SuperSlab
|
|
|
|
|
void superslab_init_slab(SuperSlab* ss, int slab_idx, size_t block_size, uint32_t owner_tid);
|
|
|
|
|
|
|
|
|
|
// Mark a slab as active
|
|
|
|
|
void superslab_activate_slab(SuperSlab* ss, int slab_idx);
|
|
|
|
|
|
|
|
|
|
// Mark a slab as inactive
|
|
|
|
|
void superslab_deactivate_slab(SuperSlab* ss, int slab_idx);
|
|
|
|
|
|
|
|
|
|
// Find first free slab index (-1 if none)
|
|
|
|
|
int superslab_find_free_slab(SuperSlab* ss);
|
|
|
|
|
|
|
|
|
|
// Statistics
|
|
|
|
|
void superslab_print_stats(SuperSlab* ss);
|
|
|
|
|
|
|
|
|
|
// Phase 8.3: ACE statistics
|
|
|
|
|
void superslab_ace_print_stats(void);
|
|
|
|
|
|
|
|
|
|
// ============================================================================
|
|
|
|
|
// Phase 8.3: ACE (Adaptive Cache Engine) - SuperSlab adaptive sizing
|
|
|
|
|
// ============================================================================
|
|
|
|
|
|
|
|
|
|
// ACE tick function (called periodically, ~150ms interval)
|
|
|
|
|
// Observes metrics and decides promotion (1MB→2MB) or demotion (2MB→1MB)
|
|
|
|
|
void hak_tiny_superslab_ace_tick(int class_idx, uint64_t now_ns);
|
|
|
|
|
|
|
|
|
|
// Phase 8.4: ACE Observer (called from Learner thread - zero hot-path overhead)
|
|
|
|
|
void hak_tiny_superslab_ace_observe_all(void);
|
|
|
|
|
|
2025-11-07 23:05:33 +09:00
|
|
|
// ============================================================================
|
2025-11-05 12:31:14 +09:00
|
|
|
// Partial SuperSlab adopt/publish (per-class single-slot)
|
2025-11-07 23:05:33 +09:00
|
|
|
// ============================================================================
|
|
|
|
|
|
2025-11-05 12:31:14 +09:00
|
|
|
// Publish a SuperSlab with available freelist for other threads to adopt.
|
|
|
|
|
void ss_partial_publish(int class_idx, SuperSlab* ss);
|
2025-11-07 23:05:33 +09:00
|
|
|
|
2025-11-05 12:31:14 +09:00
|
|
|
// Adopt published SuperSlab for the class (returns NULL if none).
|
|
|
|
|
SuperSlab* ss_partial_adopt(int class_idx);
|
|
|
|
|
|
2025-11-07 23:05:33 +09:00
|
|
|
// ============================================================================
|
2025-11-05 12:31:14 +09:00
|
|
|
// SuperSlab adopt gate (publish/adopt wiring helper)
|
2025-11-07 23:05:33 +09:00
|
|
|
// ============================================================================
|
|
|
|
|
|
2025-11-05 12:31:14 +09:00
|
|
|
// Environment-aware switch that keeps free/alloc sides in sync. Default:
|
|
|
|
|
// - Disabled until cross-thread free is observed.
|
|
|
|
|
// - `HAKMEM_TINY_SS_ADOPT=1` forces ON, `=0` forces OFF.
|
|
|
|
|
int tiny_adopt_gate_should_publish(void);
|
|
|
|
|
int tiny_adopt_gate_should_adopt(void);
|
|
|
|
|
void tiny_adopt_gate_on_remote_seen(int class_idx);
|
|
|
|
|
|
2025-11-07 23:05:33 +09:00
|
|
|
// ============================================================================
|
|
|
|
|
// External variable declarations
|
|
|
|
|
// ============================================================================
|
2025-11-05 12:31:14 +09:00
|
|
|
|
2025-11-07 23:05:33 +09:00
|
|
|
extern _Atomic int g_ss_remote_seen; // set to 1 on first remote free observed
|
|
|
|
|
extern int g_remote_force_notify;
|
2025-11-05 12:31:14 +09:00
|
|
|
|
|
|
|
|
#endif // HAKMEM_TINY_SUPERSLAB_H
|