Tiny: fix header/stride mismatch and harden refill paths
- Root cause: header-based class indexing (HEADER_CLASSIDX=1) wrote a 1-byte header during allocation, but linear carve/refill and initial slab capacity still used bare class block sizes. This mismatch could overrun slab usable space and corrupt freelists, causing reproducible SEGV at ~100k iters. Changes - Superslab: compute capacity with effective stride (block_size + header for classes 0..6; class7 remains headerless) in superslab_init_slab(). Add a debug-only bound check in superslab_alloc_from_slab() to fail fast if carve would exceed usable bytes. - Refill (non-P0 and P0): use header-aware stride for all linear carving and TLS window bump operations. Ensure alignment/validation in tiny_refill_opt.h also uses stride, not raw class size. - Drain: keep existing defense-in-depth for remote sentinel and sanitize nodes before splicing into freelist (already present). Notes - This unifies the memory layout across alloc/linear-carve/refill with a single stride definition and keeps class7 (1024B) headerless as designed. - Debug builds add fail-fast checks; release builds remain lean. Next - Re-run Tiny benches (256/1024B) in debug to confirm stability, then in release. If any remaining crash persists, bisect with HAKMEM_TINY_P0_BATCH_REFILL=0 to isolate P0 batch carve, and continue reducing branch-miss as planned.
This commit is contained in:
Moe Charm (CI)
130
core/box/ace_pool_connector.c
Normal file
130
core/box/ace_pool_connector.c
Normal file
@ -0,0 +1,130 @@
|
||||
// ace_pool_connector.c - ACE-Pool Connection Box Implementation
|
||||
|
||||
#include "ace_pool_connector.h"
|
||||
#include "../hakmem_pool.h"
|
||||
#include "../hakmem_ace_controller.h"
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
|
||||
// External references (from Pool)
|
||||
extern struct Pool {
|
||||
int initialized;
|
||||
// ... other fields
|
||||
} g_pool;
|
||||
|
||||
extern size_t g_class_sizes[7]; // Pool class sizes
|
||||
extern int g_wrap_l2_enabled;
|
||||
|
||||
// ============================================================================
|
||||
// Box Implementation
|
||||
// ============================================================================
|
||||
|
||||
AcePoolHealth ace_pool_get_health(void) {
|
||||
AcePoolHealth health;
|
||||
memset(&health, 0, sizeof(health));
|
||||
|
||||
// Check Pool initialization
|
||||
health.pool_initialized = g_pool.initialized;
|
||||
|
||||
// Check ACE status
|
||||
const char* ace_env = getenv("HAKMEM_ACE_ENABLED");
|
||||
health.ace_enabled = (ace_env && atoi(ace_env) == 1);
|
||||
|
||||
// Check WRAP_L2 status
|
||||
health.wrap_l2_enabled = g_wrap_l2_enabled;
|
||||
|
||||
// Check Bridge classes
|
||||
health.bridge_class_5_size = (int)g_class_sizes[5];
|
||||
health.bridge_class_6_size = (int)g_class_sizes[6];
|
||||
|
||||
// TODO: Track pre-allocated pages count
|
||||
health.preallocated_pages = 0; // Not yet tracked
|
||||
|
||||
// Determine overall status
|
||||
if (!health.pool_initialized) {
|
||||
health.status = ACE_POOL_NOT_INIT;
|
||||
health.message = "Pool not initialized";
|
||||
} else if (!health.ace_enabled) {
|
||||
health.status = ACE_POOL_NOT_INIT;
|
||||
health.message = "ACE not enabled (set HAKMEM_ACE_ENABLED=1)";
|
||||
} else if (!health.wrap_l2_enabled) {
|
||||
health.status = ACE_POOL_WRAPPER_BLOCKED;
|
||||
health.message = "WRAP_L2 not enabled (set HAKMEM_WRAP_L2=1)";
|
||||
} else if (health.bridge_class_5_size == 0 && health.bridge_class_6_size == 0) {
|
||||
health.status = ACE_POOL_SIZE_MISMATCH;
|
||||
health.message = "Bridge classes disabled (class 5 and 6 are 0)";
|
||||
} else if (health.preallocated_pages == 0) {
|
||||
health.status = ACE_POOL_NO_PAGES;
|
||||
health.message = "No pre-allocated pages (performance will be degraded)";
|
||||
} else {
|
||||
health.status = ACE_POOL_OK;
|
||||
health.message = "ACE-Pool connection healthy";
|
||||
}
|
||||
|
||||
return health;
|
||||
}
|
||||
|
||||
int ace_pool_validate_connection(AcePoolStatus* out_status) {
|
||||
AcePoolHealth health = ace_pool_get_health();
|
||||
|
||||
if (out_status) {
|
||||
*out_status = health.status;
|
||||
}
|
||||
|
||||
// Only OK status is considered "ready"
|
||||
// NO_PAGES is warning but still functional
|
||||
return (health.status == ACE_POOL_OK || health.status == ACE_POOL_NO_PAGES);
|
||||
}
|
||||
|
||||
void* ace_pool_try_alloc(size_t size, uintptr_t site_id, AcePoolStatus* out_status) {
|
||||
// Validate connection first
|
||||
AcePoolStatus status;
|
||||
if (!ace_pool_validate_connection(&status)) {
|
||||
if (out_status) *out_status = status;
|
||||
|
||||
// Log why allocation failed
|
||||
AcePoolHealth health = ace_pool_get_health();
|
||||
static int logged_once = 0;
|
||||
if (!logged_once) {
|
||||
fprintf(stderr, "[ACE-Pool Connector] BLOCKED: %s\n", health.message);
|
||||
logged_once = 1;
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
// Connection validated, try Pool allocation
|
||||
void* ptr = hak_pool_try_alloc(size, site_id);
|
||||
|
||||
if (ptr) {
|
||||
if (out_status) *out_status = ACE_POOL_OK;
|
||||
} else {
|
||||
if (out_status) *out_status = ACE_POOL_ALLOC_FAILED;
|
||||
|
||||
// Log allocation failure (but only once to avoid spam)
|
||||
static int fail_logged = 0;
|
||||
if (!fail_logged) {
|
||||
fprintf(stderr, "[ACE-Pool Connector] Pool allocation failed for size=%zu (will fallback to mmap)\n", size);
|
||||
fail_logged = 1;
|
||||
}
|
||||
}
|
||||
|
||||
return ptr;
|
||||
}
|
||||
|
||||
void ace_pool_print_health(void) {
|
||||
AcePoolHealth health = ace_pool_get_health();
|
||||
|
||||
fprintf(stderr, "\n=== ACE-Pool Connector Health Check ===\n");
|
||||
fprintf(stderr, "Pool Initialized: %s\n", health.pool_initialized ? "YES" : "NO");
|
||||
fprintf(stderr, "ACE Enabled: %s\n", health.ace_enabled ? "YES" : "NO");
|
||||
fprintf(stderr, "WRAP_L2 Enabled: %s\n", health.wrap_l2_enabled ? "YES" : "NO");
|
||||
fprintf(stderr, "Bridge Class 5: %d KB (%s)\n",
|
||||
health.bridge_class_5_size / 1024,
|
||||
health.bridge_class_5_size > 0 ? "ENABLED" : "DISABLED");
|
||||
fprintf(stderr, "Bridge Class 6: %d KB (%s)\n",
|
||||
health.bridge_class_6_size / 1024,
|
||||
health.bridge_class_6_size > 0 ? "ENABLED" : "DISABLED");
|
||||
fprintf(stderr, "Pre-allocated Pages: %d\n", health.preallocated_pages);
|
||||
fprintf(stderr, "Status: %s\n", health.message);
|
||||
fprintf(stderr, "========================================\n\n");
|
||||
}
|
||||
70
core/box/ace_pool_connector.h
Normal file
70
core/box/ace_pool_connector.h
Normal file
@ -0,0 +1,70 @@
|
||||
// ace_pool_connector.h - ACE-Pool Connection Box
|
||||
// Box Theory: Single Responsibility - Validate and route ACE ↔ Pool connections
|
||||
//
|
||||
// Purpose:
|
||||
// - Make ACE-Pool connection VISIBLE and VALIDATED
|
||||
// - Centralize error handling and logging
|
||||
// - Health check API for diagnostics
|
||||
//
|
||||
// Responsibilities:
|
||||
// ✅ Validate Pool is initialized before ACE uses it
|
||||
// ✅ Log connection status (success/failure/reason)
|
||||
// ✅ Provide health check API
|
||||
// ❌ NOT responsible for: allocation logic, size rounding, or memory management
|
||||
//
|
||||
// Box Boundaries:
|
||||
// INPUT: ACE requests allocation from Pool (size, site_id)
|
||||
// OUTPUT: Pool allocation result (ptr or NULL) + reason code
|
||||
// ERROR: Clear error messages (not silent failures!)
|
||||
|
||||
#ifndef ACE_POOL_CONNECTOR_H
|
||||
#define ACE_POOL_CONNECTOR_H
|
||||
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
|
||||
// ============================================================================
|
||||
// Box API: ACE-Pool Connection
|
||||
// ============================================================================
|
||||
|
||||
// Connection status codes
|
||||
typedef enum {
|
||||
ACE_POOL_OK = 0, // Connection healthy
|
||||
ACE_POOL_NOT_INIT, // Pool not initialized
|
||||
ACE_POOL_NO_PAGES, // Pool has no pre-allocated pages
|
||||
ACE_POOL_WRAPPER_BLOCKED, // Wrapper protection blocking
|
||||
ACE_POOL_SIZE_MISMATCH, // Size not in Pool range
|
||||
ACE_POOL_ALLOC_FAILED, // Pool allocation returned NULL
|
||||
} AcePoolStatus;
|
||||
|
||||
// Health check result
|
||||
typedef struct {
|
||||
int pool_initialized; // 1 if Pool is initialized
|
||||
int ace_enabled; // 1 if ACE is enabled
|
||||
int wrap_l2_enabled; // 1 if WRAP_L2 is enabled
|
||||
int bridge_class_5_size; // Size of Bridge class 5 (40KB expected)
|
||||
int bridge_class_6_size; // Size of Bridge class 6 (52KB expected)
|
||||
int preallocated_pages; // Number of pre-allocated pages (should be > 0)
|
||||
AcePoolStatus status; // Overall status
|
||||
const char* message; // Human-readable status message
|
||||
} AcePoolHealth;
|
||||
|
||||
// ============================================================================
|
||||
// Box Functions
|
||||
// ============================================================================
|
||||
|
||||
// Get health status (for debugging and monitoring)
|
||||
AcePoolHealth ace_pool_get_health(void);
|
||||
|
||||
// Validate connection is ready (called by ACE before using Pool)
|
||||
// Returns: 1 if ready, 0 if not (sets reason code)
|
||||
int ace_pool_validate_connection(AcePoolStatus* out_status);
|
||||
|
||||
// Connect ACE to Pool (wrapper around hak_pool_try_alloc with validation)
|
||||
// Returns: Allocated pointer or NULL (logs reason if NULL)
|
||||
void* ace_pool_try_alloc(size_t size, uintptr_t site_id, AcePoolStatus* out_status);
|
||||
|
||||
// Print health status (for debugging)
|
||||
void ace_pool_print_health(void);
|
||||
|
||||
#endif // ACE_POOL_CONNECTOR_H
|
||||
24
core/box/free_local_box.d
Normal file
24
core/box/free_local_box.d
Normal file
@ -0,0 +1,24 @@
|
||||
core/box/free_local_box.o: core/box/free_local_box.c \
|
||||
core/box/free_local_box.h core/hakmem_tiny_superslab.h \
|
||||
core/superslab/superslab_types.h core/hakmem_tiny_superslab_constants.h \
|
||||
core/superslab/superslab_inline.h core/superslab/superslab_types.h \
|
||||
core/tiny_debug_ring.h core/tiny_remote.h core/tiny_debug_ring.h \
|
||||
core/tiny_remote.h core/hakmem_tiny_superslab_constants.h \
|
||||
core/box/free_publish_box.h core/hakmem_tiny.h core/hakmem_build_flags.h \
|
||||
core/hakmem_trace.h core/hakmem_tiny_mini_mag.h
|
||||
core/box/free_local_box.h:
|
||||
core/hakmem_tiny_superslab.h:
|
||||
core/superslab/superslab_types.h:
|
||||
core/hakmem_tiny_superslab_constants.h:
|
||||
core/superslab/superslab_inline.h:
|
||||
core/superslab/superslab_types.h:
|
||||
core/tiny_debug_ring.h:
|
||||
core/tiny_remote.h:
|
||||
core/tiny_debug_ring.h:
|
||||
core/tiny_remote.h:
|
||||
core/hakmem_tiny_superslab_constants.h:
|
||||
core/box/free_publish_box.h:
|
||||
core/hakmem_tiny.h:
|
||||
core/hakmem_build_flags.h:
|
||||
core/hakmem_trace.h:
|
||||
core/hakmem_tiny_mini_mag.h:
|
||||
28
core/box/free_publish_box.d
Normal file
28
core/box/free_publish_box.d
Normal file
@ -0,0 +1,28 @@
|
||||
core/box/free_publish_box.o: core/box/free_publish_box.c \
|
||||
core/box/free_publish_box.h core/hakmem_tiny_superslab.h \
|
||||
core/superslab/superslab_types.h core/hakmem_tiny_superslab_constants.h \
|
||||
core/superslab/superslab_inline.h core/superslab/superslab_types.h \
|
||||
core/tiny_debug_ring.h core/tiny_remote.h core/tiny_debug_ring.h \
|
||||
core/tiny_remote.h core/hakmem_tiny_superslab_constants.h \
|
||||
core/hakmem_tiny.h core/hakmem_build_flags.h core/hakmem_trace.h \
|
||||
core/hakmem_tiny_mini_mag.h core/tiny_route.h core/tiny_ready.h \
|
||||
core/hakmem_tiny.h core/box/mailbox_box.h
|
||||
core/box/free_publish_box.h:
|
||||
core/hakmem_tiny_superslab.h:
|
||||
core/superslab/superslab_types.h:
|
||||
core/hakmem_tiny_superslab_constants.h:
|
||||
core/superslab/superslab_inline.h:
|
||||
core/superslab/superslab_types.h:
|
||||
core/tiny_debug_ring.h:
|
||||
core/tiny_remote.h:
|
||||
core/tiny_debug_ring.h:
|
||||
core/tiny_remote.h:
|
||||
core/hakmem_tiny_superslab_constants.h:
|
||||
core/hakmem_tiny.h:
|
||||
core/hakmem_build_flags.h:
|
||||
core/hakmem_trace.h:
|
||||
core/hakmem_tiny_mini_mag.h:
|
||||
core/tiny_route.h:
|
||||
core/tiny_ready.h:
|
||||
core/hakmem_tiny.h:
|
||||
core/box/mailbox_box.h:
|
||||
24
core/box/free_remote_box.d
Normal file
24
core/box/free_remote_box.d
Normal file
@ -0,0 +1,24 @@
|
||||
core/box/free_remote_box.o: core/box/free_remote_box.c \
|
||||
core/box/free_remote_box.h core/hakmem_tiny_superslab.h \
|
||||
core/superslab/superslab_types.h core/hakmem_tiny_superslab_constants.h \
|
||||
core/superslab/superslab_inline.h core/superslab/superslab_types.h \
|
||||
core/tiny_debug_ring.h core/tiny_remote.h core/tiny_debug_ring.h \
|
||||
core/tiny_remote.h core/hakmem_tiny_superslab_constants.h \
|
||||
core/box/free_publish_box.h core/hakmem_tiny.h core/hakmem_build_flags.h \
|
||||
core/hakmem_trace.h core/hakmem_tiny_mini_mag.h
|
||||
core/box/free_remote_box.h:
|
||||
core/hakmem_tiny_superslab.h:
|
||||
core/superslab/superslab_types.h:
|
||||
core/hakmem_tiny_superslab_constants.h:
|
||||
core/superslab/superslab_inline.h:
|
||||
core/superslab/superslab_types.h:
|
||||
core/tiny_debug_ring.h:
|
||||
core/tiny_remote.h:
|
||||
core/tiny_debug_ring.h:
|
||||
core/tiny_remote.h:
|
||||
core/hakmem_tiny_superslab_constants.h:
|
||||
core/box/free_publish_box.h:
|
||||
core/hakmem_tiny.h:
|
||||
core/hakmem_build_flags.h:
|
||||
core/hakmem_trace.h:
|
||||
core/hakmem_tiny_mini_mag.h:
|
||||
11
core/box/front_gate_box.d
Normal file
11
core/box/front_gate_box.d
Normal file
@ -0,0 +1,11 @@
|
||||
core/box/front_gate_box.o: core/box/front_gate_box.c \
|
||||
core/box/front_gate_box.h core/hakmem_tiny.h core/hakmem_build_flags.h \
|
||||
core/hakmem_trace.h core/hakmem_tiny_mini_mag.h \
|
||||
core/tiny_alloc_fast_sfc.inc.h core/hakmem_tiny.h
|
||||
core/box/front_gate_box.h:
|
||||
core/hakmem_tiny.h:
|
||||
core/hakmem_build_flags.h:
|
||||
core/hakmem_trace.h:
|
||||
core/hakmem_tiny_mini_mag.h:
|
||||
core/tiny_alloc_fast_sfc.inc.h:
|
||||
core/hakmem_tiny.h:
|
||||
@ -6,6 +6,19 @@
|
||||
#include "../pool_tls.h"
|
||||
#endif
|
||||
|
||||
// Centralized OS mapping boundary to keep syscalls in one place
|
||||
static inline void* hak_os_map_boundary(size_t size, uintptr_t site_id) {
|
||||
#if HAKMEM_DEBUG_TIMING
|
||||
HKM_TIME_START(t_mmap);
|
||||
#endif
|
||||
void* p = hak_alloc_mmap_impl(size);
|
||||
#if HAKMEM_DEBUG_TIMING
|
||||
HKM_TIME_END(HKM_CAT_SYSCALL_MMAP, t_mmap);
|
||||
#endif
|
||||
(void)site_id; // reserved for future accounting/learning
|
||||
return p;
|
||||
}
|
||||
|
||||
__attribute__((always_inline))
|
||||
inline void* hak_alloc_at(size_t size, hak_callsite_t site) {
|
||||
#if HAKMEM_DEBUG_TIMING
|
||||
@ -144,33 +157,24 @@ inline void* hak_alloc_at(size_t size, hak_callsite_t site) {
|
||||
//
|
||||
// Solution: Use mmap for gap when ACE failed (ACE disabled or OOM)
|
||||
|
||||
// Track final fallback mmaps globally
|
||||
extern _Atomic uint64_t g_final_fallback_mmap_count;
|
||||
|
||||
void* ptr;
|
||||
if (size >= threshold) {
|
||||
// Large allocation (>= 2MB default): use mmap
|
||||
#if HAKMEM_DEBUG_TIMING
|
||||
HKM_TIME_START(t_mmap);
|
||||
#endif
|
||||
ptr = hak_alloc_mmap_impl(size);
|
||||
#if HAKMEM_DEBUG_TIMING
|
||||
HKM_TIME_END(HKM_CAT_SYSCALL_MMAP, t_mmap);
|
||||
#endif
|
||||
// Large allocation (>= 2MB default): descend via single boundary
|
||||
atomic_fetch_add(&g_final_fallback_mmap_count, 1);
|
||||
ptr = hak_os_map_boundary(size, site_id);
|
||||
} else if (size >= TINY_MAX_SIZE) {
|
||||
// Mid-range allocation (1KB-2MB): try mmap as final fallback
|
||||
// This handles the gap when ACE is disabled or failed
|
||||
atomic_fetch_add(&g_final_fallback_mmap_count, 1);
|
||||
static _Atomic int gap_alloc_count = 0;
|
||||
int count = atomic_fetch_add(&gap_alloc_count, 1);
|
||||
#if HAKMEM_DEBUG_VERBOSE
|
||||
if (count < 3) {
|
||||
fprintf(stderr, "[HAKMEM] INFO: Using mmap for mid-range size=%zu (ACE disabled or failed)\n", size);
|
||||
}
|
||||
if (count < 3) fprintf(stderr, "[HAKMEM] INFO: mid-gap fallback size=%zu\n", size);
|
||||
#endif
|
||||
#if HAKMEM_DEBUG_TIMING
|
||||
HKM_TIME_START(t_mmap);
|
||||
#endif
|
||||
ptr = hak_alloc_mmap_impl(size);
|
||||
#if HAKMEM_DEBUG_TIMING
|
||||
HKM_TIME_END(HKM_CAT_SYSCALL_MMAP, t_mmap);
|
||||
#endif
|
||||
ptr = hak_os_map_boundary(size, site_id);
|
||||
} else {
|
||||
// Should never reach here (size <= TINY_MAX_SIZE should be handled by Tiny)
|
||||
static _Atomic int oom_count = 0;
|
||||
|
||||
@ -117,6 +117,39 @@ static void hak_init_impl(void) {
|
||||
HAKMEM_LOG("Sampling rate: 1/%d\n", SAMPLING_RATE);
|
||||
HAKMEM_LOG("Max sites: %d\n", MAX_SITES);
|
||||
|
||||
// Build banner (one-shot)
|
||||
do {
|
||||
const char* bf = "UNKNOWN";
|
||||
#ifdef HAKMEM_BUILD_RELEASE
|
||||
bf = "RELEASE";
|
||||
#elif defined(HAKMEM_BUILD_DEBUG)
|
||||
bf = "DEBUG";
|
||||
#endif
|
||||
HAKMEM_LOG("[Build] Flavor=%s Flags: HEADER_CLASSIDX=%d, AGGRESSIVE_INLINE=%d, POOL_TLS_PHASE1=%d, POOL_TLS_PREWARM=%d\n",
|
||||
bf,
|
||||
#ifdef HAKMEM_TINY_HEADER_CLASSIDX
|
||||
1,
|
||||
#else
|
||||
0,
|
||||
#endif
|
||||
#ifdef HAKMEM_TINY_AGGRESSIVE_INLINE
|
||||
1,
|
||||
#else
|
||||
0,
|
||||
#endif
|
||||
#ifdef HAKMEM_POOL_TLS_PHASE1
|
||||
1,
|
||||
#else
|
||||
0,
|
||||
#endif
|
||||
#ifdef HAKMEM_POOL_TLS_PREWARM
|
||||
1
|
||||
#else
|
||||
0
|
||||
#endif
|
||||
);
|
||||
} while (0);
|
||||
|
||||
// Bench preset: Tiny-only (disable non-essential subsystems)
|
||||
{
|
||||
char* bt = getenv("HAKMEM_BENCH_TINY_ONLY");
|
||||
|
||||
23
core/box/mailbox_box.d
Normal file
23
core/box/mailbox_box.d
Normal file
@ -0,0 +1,23 @@
|
||||
core/box/mailbox_box.o: core/box/mailbox_box.c core/box/mailbox_box.h \
|
||||
core/hakmem_tiny_superslab.h core/superslab/superslab_types.h \
|
||||
core/hakmem_tiny_superslab_constants.h core/superslab/superslab_inline.h \
|
||||
core/superslab/superslab_types.h core/tiny_debug_ring.h \
|
||||
core/tiny_remote.h core/tiny_debug_ring.h core/tiny_remote.h \
|
||||
core/hakmem_tiny_superslab_constants.h core/hakmem_tiny.h \
|
||||
core/hakmem_build_flags.h core/hakmem_trace.h \
|
||||
core/hakmem_tiny_mini_mag.h
|
||||
core/box/mailbox_box.h:
|
||||
core/hakmem_tiny_superslab.h:
|
||||
core/superslab/superslab_types.h:
|
||||
core/hakmem_tiny_superslab_constants.h:
|
||||
core/superslab/superslab_inline.h:
|
||||
core/superslab/superslab_types.h:
|
||||
core/tiny_debug_ring.h:
|
||||
core/tiny_remote.h:
|
||||
core/tiny_debug_ring.h:
|
||||
core/tiny_remote.h:
|
||||
core/hakmem_tiny_superslab_constants.h:
|
||||
core/hakmem_tiny.h:
|
||||
core/hakmem_build_flags.h:
|
||||
core/hakmem_trace.h:
|
||||
core/hakmem_tiny_mini_mag.h:
|
||||
@ -3,15 +3,54 @@
|
||||
#define POOL_API_INC_H
|
||||
|
||||
void* hak_pool_try_alloc(size_t size, uintptr_t site_id) {
|
||||
// Debug: IMMEDIATE output to verify function is called
|
||||
static int first_call = 1;
|
||||
if (first_call) {
|
||||
fprintf(stderr, "[Pool] hak_pool_try_alloc FIRST CALL EVER!\n");
|
||||
first_call = 0;
|
||||
}
|
||||
|
||||
if (size == 40960) { // Exactly 40KB
|
||||
fprintf(stderr, "[Pool] hak_pool_try_alloc called with 40KB (Bridge class 5)\n");
|
||||
}
|
||||
|
||||
hak_pool_init(); // pthread_once() ensures thread-safe init (no data race!)
|
||||
|
||||
// Debug for 33-41KB allocations
|
||||
if (size >= 33000 && size <= 41000) {
|
||||
fprintf(stderr, "[Pool] hak_pool_try_alloc: size=%zu (after init)\n", size);
|
||||
}
|
||||
|
||||
// P1.7 approach: Avoid using pool during ALL wrapper calls (conservative but safe)
|
||||
extern int hak_in_wrapper(void);
|
||||
if (hak_in_wrapper() && !g_wrap_l2_enabled) return NULL;
|
||||
if (!hak_pool_is_poolable(size)) return NULL;
|
||||
if (hak_in_wrapper() && !g_wrap_l2_enabled) {
|
||||
if (size >= 33000 && size <= 41000) {
|
||||
fprintf(stderr, "[Pool] REJECTED: in_wrapper=%d, wrap_l2=%d\n",
|
||||
hak_in_wrapper(), g_wrap_l2_enabled);
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
if (!hak_pool_is_poolable(size)) {
|
||||
if (size >= 33000 && size <= 41000) {
|
||||
fprintf(stderr, "[Pool] REJECTED: not poolable (min=%d, max=%d)\n",
|
||||
POOL_MIN_SIZE, POOL_MAX_SIZE);
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
// Get class and shard indices
|
||||
int class_idx = hak_pool_get_class_index(size);
|
||||
if (class_idx < 0) return NULL;
|
||||
if (class_idx < 0) {
|
||||
if (size >= 33000 && size <= 41000) {
|
||||
fprintf(stderr, "[Pool] REJECTED: class_idx=%d (size=%zu not mapped)\n",
|
||||
class_idx, size);
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
if (size >= 33000 && size <= 41000) {
|
||||
fprintf(stderr, "[Pool] ACCEPTED: class_idx=%d, proceeding with allocation\n", class_idx);
|
||||
}
|
||||
|
||||
// MF2: Per-Page Sharding path
|
||||
if (g_mf2_enabled) {
|
||||
|
||||
@ -5,7 +5,14 @@
|
||||
// Thread-safe initialization using pthread_once
|
||||
static pthread_once_t hak_pool_init_once_control = PTHREAD_ONCE_INIT;
|
||||
static void hak_pool_init_impl(void) {
|
||||
fprintf(stderr, "[Pool] hak_pool_init_impl() EXECUTING - Bridge class fix applied\n");
|
||||
const FrozenPolicy* pol = hkm_policy_get();
|
||||
|
||||
// Phase 6.21 CRITICAL FIX: Bridge classes are hardcoded in g_class_sizes,
|
||||
// NOT from Policy. DO NOT overwrite them with 0!
|
||||
// The code below was disabling Bridge classes by setting them to 0
|
||||
// because Policy returns mid_dyn1_bytes=0 and mid_dyn2_bytes=0.
|
||||
/*
|
||||
if (pol && pol->mid_dyn1_bytes >= POOL_MIN_SIZE && pol->mid_dyn1_bytes <= POOL_MAX_SIZE) {
|
||||
g_class_sizes[5] = pol->mid_dyn1_bytes;
|
||||
} else {
|
||||
@ -16,6 +23,8 @@ static void hak_pool_init_impl(void) {
|
||||
} else {
|
||||
g_class_sizes[6] = 0;
|
||||
}
|
||||
*/
|
||||
// Bridge classes remain as initialized: 40KB and 52KB
|
||||
for (int c = 0; c < POOL_NUM_CLASSES; c++) {
|
||||
for (int s = 0; s < POOL_NUM_SHARDS; s++) {
|
||||
g_pool.freelist[c][s] = NULL;
|
||||
@ -82,20 +91,65 @@ static void hak_pool_init_impl(void) {
|
||||
HAKMEM_LOG("[MF2] max_queues=%d, lease_ms=%d, idle_threshold_us=%d\n", g_mf2_max_queues, g_mf2_lease_ms, g_mf2_idle_threshold_us);
|
||||
}
|
||||
g_pool.initialized = 1;
|
||||
fprintf(stderr, "[Pool] Initialized (L2 Hybrid Pool) - Bridge classes SHOULD be enabled\n");
|
||||
fprintf(stderr, "[Pool] Class 5 (40KB): %zu\n", g_class_sizes[5]);
|
||||
fprintf(stderr, "[Pool] Class 6 (52KB): %zu\n", g_class_sizes[6]);
|
||||
HAKMEM_LOG("[Pool] Initialized (L2 Hybrid Pool)\n");
|
||||
if (g_class_sizes[5] != 0 || g_class_sizes[6] != 0) {
|
||||
HAKMEM_LOG("[Pool] Classes: 2KB, 4KB, 8KB, 16KB, 32KB%s%s%s\n",
|
||||
g_class_sizes[5] ? ", dyn1=" : "",
|
||||
g_class_sizes[5] ? "" : (g_class_sizes[6]?",":""),
|
||||
(g_class_sizes[5]||g_class_sizes[6]) ? "" : "");
|
||||
} else {
|
||||
HAKMEM_LOG("[Pool] Classes: 2KB, 4KB, 8KB, 16KB, 32KB\n");
|
||||
|
||||
#ifdef HAKMEM_DEBUG_VERBOSE
|
||||
// Debug: Show actual class sizes after initialization
|
||||
HAKMEM_LOG("[Pool] Class configuration:\n");
|
||||
for (int i = 0; i < POOL_NUM_CLASSES; i++) {
|
||||
if (g_class_sizes[i] != 0) {
|
||||
HAKMEM_LOG(" Class %d: %zu KB (ENABLED)\n", i, g_class_sizes[i]/1024);
|
||||
} else {
|
||||
HAKMEM_LOG(" Class %d: DISABLED\n", i);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
HAKMEM_LOG("[Pool] Page size: %d KB\n", POOL_PAGE_SIZE / 1024);
|
||||
HAKMEM_LOG("[Pool] Shards: %d (site-based)\n", POOL_NUM_SHARDS);
|
||||
|
||||
// ACE Performance Fix: Pre-allocate pages for Bridge classes to avoid cold start
|
||||
// This ensures ACE can serve Mid-Large allocations (33KB) immediately without mmap fallback
|
||||
extern int refill_freelist(int class_idx, int shard_idx);
|
||||
int prewarm_pages = 4; // Pre-allocate 4 pages per shard for hot classes
|
||||
|
||||
// Pre-warm Bridge class 5 (40KB) - Critical for 33KB allocations
|
||||
if (g_class_sizes[5] != 0) {
|
||||
int allocated = 0;
|
||||
for (int s = 0; s < prewarm_pages && s < POOL_NUM_SHARDS; s++) {
|
||||
if (refill_freelist(5, s) != 0) { // FIX: Check for SUCCESS (1), not FAILURE (0)
|
||||
allocated++;
|
||||
}
|
||||
}
|
||||
fprintf(stderr, "[Pool] Pre-allocated %d pages for Bridge class 5 (%zu KB) - Critical for 33KB allocs\n",
|
||||
allocated, g_class_sizes[5]/1024);
|
||||
} else {
|
||||
fprintf(stderr, "[Pool] WARNING: Bridge class 5 (40KB) is DISABLED - 33KB allocations will fail!\n");
|
||||
}
|
||||
|
||||
// Pre-warm Bridge class 6 (52KB)
|
||||
if (g_class_sizes[6] != 0) {
|
||||
int allocated = 0;
|
||||
for (int s = 0; s < prewarm_pages && s < POOL_NUM_SHARDS; s++) {
|
||||
if (refill_freelist(6, s) != 0) { // FIX: Check for SUCCESS (1), not FAILURE (0)
|
||||
allocated++;
|
||||
}
|
||||
}
|
||||
fprintf(stderr, "[Pool] Pre-allocated %d pages for Bridge class 6 (%zu KB)\n",
|
||||
allocated, g_class_sizes[6]/1024);
|
||||
}
|
||||
}
|
||||
|
||||
void hak_pool_init(void) { pthread_once(&hak_pool_init_once_control, hak_pool_init_impl); }
|
||||
void hak_pool_init(void) {
|
||||
// Always print this to see if it's being called
|
||||
static int called = 0;
|
||||
if (called++ == 0) {
|
||||
fprintf(stderr, "[Pool] hak_pool_init() called for the first time\n");
|
||||
}
|
||||
pthread_once(&hak_pool_init_once_control, hak_pool_init_impl);
|
||||
}
|
||||
|
||||
static void mf2_print_debug_stats(void) {
|
||||
if (!g_mf2_enabled) return;
|
||||
|
||||
Reference in New Issue
Block a user