Box TLS-SLL + free boundary hardening: normalize C0–C6 to base (ptr-1) at free boundary; route all caches/freelists via base; replace remaining g_tls_sll_head direct writes with Box API (tls_sll_push/splice) in refill/magazine/ultra; keep C7 excluded. Fixes rbp=0xa0 free crash by preventing header overwrite and centralizing TLS-SLL invariants.
This commit is contained in:
Moe Charm (CI)
6
Makefile
6
Makefile
@ -167,7 +167,7 @@ LDFLAGS += $(EXTRA_LDFLAGS)
|
||||
|
||||
# Targets
|
||||
TARGET = test_hakmem
|
||||
OBJS_BASE = hakmem.o hakmem_config.o hakmem_tiny_config.o hakmem_ucb1.o hakmem_bigcache.o hakmem_pool.o hakmem_l25_pool.o hakmem_site_rules.o hakmem_tiny.o hakmem_tiny_superslab.o tiny_sticky.o tiny_remote.o tiny_publish.o tiny_debug_ring.o hakmem_tiny_magazine.o hakmem_tiny_stats.o hakmem_tiny_sfc.o hakmem_tiny_query.o hakmem_tiny_rss.o hakmem_tiny_registry.o hakmem_tiny_remote_target.o hakmem_tiny_bg_spill.o tiny_adaptive_sizing.o hakmem_mid_mt.o hakmem_super_registry.o hakmem_elo.o hakmem_batch.o hakmem_p2.o hakmem_sizeclass_dist.o hakmem_evo.o hakmem_debug.o hakmem_sys.o hakmem_whale.o hakmem_policy.o hakmem_ace.o hakmem_ace_stats.o hakmem_prof.o hakmem_learner.o hakmem_size_hist.o hakmem_learn_log.o hakmem_syscall.o hakmem_ace_metrics.o hakmem_ace_ucb1.o hakmem_ace_controller.o tiny_fastcache.o core/box/free_local_box.o core/box/free_remote_box.o core/box/free_publish_box.o core/box/mailbox_box.o core/box/front_gate_box.o test_hakmem.o
|
||||
OBJS_BASE = hakmem.o hakmem_config.o hakmem_tiny_config.o hakmem_ucb1.o hakmem_bigcache.o hakmem_pool.o hakmem_l25_pool.o hakmem_site_rules.o hakmem_tiny.o hakmem_tiny_superslab.o tiny_sticky.o tiny_remote.o tiny_publish.o tiny_debug_ring.o hakmem_tiny_magazine.o hakmem_tiny_stats.o hakmem_tiny_sfc.o hakmem_tiny_query.o hakmem_tiny_rss.o hakmem_tiny_registry.o hakmem_tiny_remote_target.o hakmem_tiny_bg_spill.o tiny_adaptive_sizing.o hakmem_mid_mt.o hakmem_super_registry.o hakmem_elo.o hakmem_batch.o hakmem_p2.o hakmem_sizeclass_dist.o hakmem_evo.o hakmem_debug.o hakmem_sys.o hakmem_whale.o hakmem_policy.o hakmem_ace.o hakmem_ace_stats.o hakmem_prof.o hakmem_learner.o hakmem_size_hist.o hakmem_learn_log.o hakmem_syscall.o hakmem_ace_metrics.o hakmem_ace_ucb1.o hakmem_ace_controller.o tiny_fastcache.o core/box/free_local_box.o core/box/free_remote_box.o core/box/free_publish_box.o core/box/mailbox_box.o core/box/front_gate_box.o core/box/front_gate_classifier.o core/link_stubs.o test_hakmem.o
|
||||
OBJS = $(OBJS_BASE)
|
||||
|
||||
# Shared library
|
||||
@ -191,7 +191,7 @@ endif
|
||||
# Benchmark targets
|
||||
BENCH_HAKMEM = bench_allocators_hakmem
|
||||
BENCH_SYSTEM = bench_allocators_system
|
||||
BENCH_HAKMEM_OBJS_BASE = hakmem.o hakmem_config.o hakmem_tiny_config.o hakmem_ucb1.o hakmem_bigcache.o hakmem_pool.o hakmem_l25_pool.o hakmem_site_rules.o hakmem_tiny.o hakmem_tiny_superslab.o tiny_sticky.o tiny_remote.o tiny_publish.o tiny_debug_ring.o hakmem_tiny_magazine.o hakmem_tiny_stats.o hakmem_tiny_sfc.o hakmem_tiny_query.o hakmem_tiny_rss.o hakmem_tiny_registry.o hakmem_tiny_remote_target.o hakmem_tiny_bg_spill.o tiny_adaptive_sizing.o hakmem_mid_mt.o hakmem_super_registry.o hakmem_elo.o hakmem_batch.o hakmem_p2.o hakmem_sizeclass_dist.o hakmem_evo.o hakmem_debug.o hakmem_sys.o hakmem_whale.o hakmem_policy.o hakmem_ace.o hakmem_ace_stats.o hakmem_prof.o hakmem_learner.o hakmem_size_hist.o hakmem_learn_log.o hakmem_syscall.o hakmem_ace_metrics.o hakmem_ace_ucb1.o hakmem_ace_controller.o tiny_fastcache.o core/box/free_local_box.o core/box/free_remote_box.o core/box/free_publish_box.o core/box/mailbox_box.o core/box/front_gate_box.o bench_allocators_hakmem.o
|
||||
BENCH_HAKMEM_OBJS_BASE = hakmem.o hakmem_config.o hakmem_tiny_config.o hakmem_ucb1.o hakmem_bigcache.o hakmem_pool.o hakmem_l25_pool.o hakmem_site_rules.o hakmem_tiny.o hakmem_tiny_superslab.o tiny_sticky.o tiny_remote.o tiny_publish.o tiny_debug_ring.o hakmem_tiny_magazine.o hakmem_tiny_stats.o hakmem_tiny_sfc.o hakmem_tiny_query.o hakmem_tiny_rss.o hakmem_tiny_registry.o hakmem_tiny_remote_target.o hakmem_tiny_bg_spill.o tiny_adaptive_sizing.o hakmem_mid_mt.o hakmem_super_registry.o hakmem_elo.o hakmem_batch.o hakmem_p2.o hakmem_sizeclass_dist.o hakmem_evo.o hakmem_debug.o hakmem_sys.o hakmem_whale.o hakmem_policy.o hakmem_ace.o hakmem_ace_stats.o hakmem_prof.o hakmem_learner.o hakmem_size_hist.o hakmem_learn_log.o hakmem_syscall.o hakmem_ace_metrics.o hakmem_ace_ucb1.o hakmem_ace_controller.o tiny_fastcache.o core/box/free_local_box.o core/box/free_remote_box.o core/box/free_publish_box.o core/box/mailbox_box.o core/box/front_gate_box.o core/box/front_gate_classifier.o core/link_stubs.o bench_allocators_hakmem.o
|
||||
BENCH_HAKMEM_OBJS = $(BENCH_HAKMEM_OBJS_BASE)
|
||||
ifeq ($(POOL_TLS_PHASE1),1)
|
||||
BENCH_HAKMEM_OBJS += pool_tls.o pool_refill.o pool_tls_arena.o pool_tls_registry.o pool_tls_remote.o
|
||||
@ -368,7 +368,7 @@ test-box-refactor: box-refactor
|
||||
./larson_hakmem 10 8 128 1024 1 12345 4
|
||||
|
||||
# Phase 4: Tiny Pool benchmarks (properly linked with hakmem)
|
||||
TINY_BENCH_OBJS_BASE = hakmem.o hakmem_config.o hakmem_tiny_config.o hakmem_ucb1.o hakmem_bigcache.o hakmem_pool.o hakmem_l25_pool.o hakmem_site_rules.o hakmem_tiny.o hakmem_tiny_superslab.o core/box/mailbox_box.o core/box/front_gate_box.o core/box/free_local_box.o core/box/free_remote_box.o core/box/free_publish_box.o tiny_sticky.o tiny_remote.o tiny_publish.o tiny_debug_ring.o hakmem_tiny_magazine.o hakmem_tiny_stats.o hakmem_tiny_sfc.o hakmem_tiny_query.o hakmem_tiny_rss.o hakmem_tiny_registry.o hakmem_tiny_remote_target.o hakmem_tiny_bg_spill.o tiny_adaptive_sizing.o hakmem_mid_mt.o hakmem_super_registry.o hakmem_elo.o hakmem_batch.o hakmem_p2.o hakmem_sizeclass_dist.o hakmem_evo.o hakmem_debug.o hakmem_sys.o hakmem_whale.o hakmem_policy.o hakmem_ace.o hakmem_ace_stats.o hakmem_prof.o hakmem_learner.o hakmem_size_hist.o hakmem_learn_log.o hakmem_syscall.o hakmem_ace_metrics.o hakmem_ace_ucb1.o hakmem_ace_controller.o tiny_fastcache.o
|
||||
TINY_BENCH_OBJS_BASE = hakmem.o hakmem_config.o hakmem_tiny_config.o hakmem_ucb1.o hakmem_bigcache.o hakmem_pool.o hakmem_l25_pool.o hakmem_site_rules.o hakmem_tiny.o hakmem_tiny_superslab.o core/box/mailbox_box.o core/box/front_gate_box.o core/box/front_gate_classifier.o core/box/free_local_box.o core/box/free_remote_box.o core/box/free_publish_box.o tiny_sticky.o tiny_remote.o tiny_publish.o tiny_debug_ring.o hakmem_tiny_magazine.o hakmem_tiny_stats.o hakmem_tiny_sfc.o hakmem_tiny_query.o hakmem_tiny_rss.o hakmem_tiny_registry.o hakmem_tiny_remote_target.o hakmem_tiny_bg_spill.o tiny_adaptive_sizing.o hakmem_mid_mt.o hakmem_super_registry.o hakmem_elo.o hakmem_batch.o hakmem_p2.o hakmem_sizeclass_dist.o hakmem_evo.o hakmem_debug.o hakmem_sys.o hakmem_whale.o hakmem_policy.o hakmem_ace.o hakmem_ace_stats.o hakmem_prof.o hakmem_learner.o hakmem_size_hist.o hakmem_learn_log.o hakmem_syscall.o hakmem_ace_metrics.o hakmem_ace_ucb1.o hakmem_ace_controller.o tiny_fastcache.o
|
||||
TINY_BENCH_OBJS = $(TINY_BENCH_OBJS_BASE)
|
||||
ifeq ($(POOL_TLS_PHASE1),1)
|
||||
TINY_BENCH_OBJS += pool_tls.o pool_refill.o core/pool_tls_arena.o pool_tls_registry.o pool_tls_remote.o
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
// front_gate_box.c - Front Gate Box (SFC/SLL priority and helpers)
|
||||
#include "front_gate_box.h"
|
||||
#include "tiny_alloc_fast_sfc.inc.h"
|
||||
#include "tls_sll_box.h" // Box TLS-SLL API
|
||||
|
||||
// TLS SLL state (extern from hakmem_tiny.c)
|
||||
extern __thread void* g_tls_sll_head[TINY_NUM_CLASSES];
|
||||
@ -29,11 +30,9 @@ int front_gate_try_pop(int class_idx, void** out_ptr) {
|
||||
|
||||
// Layer 1: TLS SLL
|
||||
if (__builtin_expect(g_tls_sll_enable, 1)) {
|
||||
void* head = g_tls_sll_head[class_idx];
|
||||
if (__builtin_expect(head != NULL, 1)) {
|
||||
void* head = NULL;
|
||||
if (tls_sll_pop(class_idx, &head)) {
|
||||
g_front_sll_hit[class_idx]++;
|
||||
g_tls_sll_head[class_idx] = *(void**)head; // pop
|
||||
if (g_tls_sll_count[class_idx] > 0) g_tls_sll_count[class_idx]--;
|
||||
*out_ptr = head;
|
||||
return 1;
|
||||
}
|
||||
@ -51,10 +50,8 @@ void front_gate_after_refill(int class_idx, int refilled_count) {
|
||||
|
||||
while (to_move-- > 0 && g_tls_sll_count[class_idx] > 0) {
|
||||
// SLL pop
|
||||
void* ptr = g_tls_sll_head[class_idx];
|
||||
if (!ptr) break;
|
||||
g_tls_sll_head[class_idx] = *(void**)ptr;
|
||||
g_tls_sll_count[class_idx]--;
|
||||
void* ptr = NULL;
|
||||
if (!tls_sll_pop(class_idx, &ptr)) break;
|
||||
|
||||
// SFC push (capacity-guarded inside sfc_free_push)
|
||||
if (!sfc_free_push(class_idx, ptr)) {
|
||||
@ -65,8 +62,11 @@ void front_gate_after_refill(int class_idx, int refilled_count) {
|
||||
}
|
||||
|
||||
void front_gate_push_tls(int class_idx, void* ptr) {
|
||||
*(void**)ptr = g_tls_sll_head[class_idx];
|
||||
g_tls_sll_head[class_idx] = ptr;
|
||||
g_tls_sll_count[class_idx]++;
|
||||
// Use Box TLS-SLL API (C7-safe)
|
||||
if (!tls_sll_push(class_idx, ptr, UINT32_MAX)) {
|
||||
// C7 rejected or capacity exceeded - should not happen in front gate
|
||||
// but handle gracefully (silent discard)
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -1,7 +1,8 @@
|
||||
core/box/front_gate_box.o: core/box/front_gate_box.c \
|
||||
core/box/front_gate_box.h core/hakmem_tiny.h core/hakmem_build_flags.h \
|
||||
core/hakmem_trace.h core/hakmem_tiny_mini_mag.h \
|
||||
core/tiny_alloc_fast_sfc.inc.h core/hakmem_tiny.h
|
||||
core/tiny_alloc_fast_sfc.inc.h core/hakmem_tiny.h core/box/tls_sll_box.h \
|
||||
core/box/../hakmem_tiny_config.h
|
||||
core/box/front_gate_box.h:
|
||||
core/hakmem_tiny.h:
|
||||
core/hakmem_build_flags.h:
|
||||
@ -9,3 +10,5 @@ core/hakmem_trace.h:
|
||||
core/hakmem_tiny_mini_mag.h:
|
||||
core/tiny_alloc_fast_sfc.inc.h:
|
||||
core/hakmem_tiny.h:
|
||||
core/box/tls_sll_box.h:
|
||||
core/box/../hakmem_tiny_config.h:
|
||||
|
||||
228
core/box/front_gate_classifier.c
Normal file
228
core/box/front_gate_classifier.c
Normal file
@ -0,0 +1,228 @@
|
||||
// front_gate_classifier.c - Box FG: Pointer Classification Implementation
|
||||
|
||||
// CRITICAL: Box FG requires header-based classification
|
||||
// Ensure HEADER_MAGIC and HEADER_CLASS_MASK are available
|
||||
#ifndef HAKMEM_TINY_HEADER_CLASSIDX
|
||||
#define HAKMEM_TINY_HEADER_CLASSIDX 1
|
||||
#endif
|
||||
|
||||
#include <stdio.h> // For fprintf in debug
|
||||
#include <stdlib.h> // For abort in debug
|
||||
#include "front_gate_classifier.h"
|
||||
#include "../tiny_region_id.h" // Must come before hakmem_tiny_superslab.h for HEADER_MAGIC
|
||||
#include "../hakmem_tiny_superslab.h"
|
||||
#include "../superslab/superslab_inline.h" // For ss_slabs_capacity
|
||||
#include "../hakmem_build_flags.h"
|
||||
#include "../hakmem_tiny_config.h" // For TINY_NUM_CLASSES, SLAB_SIZE
|
||||
#include "../hakmem_super_registry.h" // For hak_super_lookup (Box REG)
|
||||
|
||||
#ifdef HAKMEM_POOL_TLS_PHASE1
|
||||
#include "../pool_tls.h" // For POOL_MAGIC
|
||||
#endif
|
||||
|
||||
// ========== Debug Stats ==========
|
||||
|
||||
#if !HAKMEM_BUILD_RELEASE
|
||||
__thread uint64_t g_classify_header_hit = 0;
|
||||
__thread uint64_t g_classify_headerless_hit = 0;
|
||||
__thread uint64_t g_classify_pool_hit = 0;
|
||||
__thread uint64_t g_classify_unknown_hit = 0;
|
||||
|
||||
void front_gate_print_stats(void) {
|
||||
uint64_t total = g_classify_header_hit + g_classify_headerless_hit +
|
||||
g_classify_pool_hit + g_classify_unknown_hit;
|
||||
if (total == 0) return;
|
||||
|
||||
fprintf(stderr, "\n========== Front Gate Classification Stats ==========\n");
|
||||
fprintf(stderr, "Header (C0-C6): %lu (%.2f%%)\n",
|
||||
g_classify_header_hit, 100.0 * g_classify_header_hit / total);
|
||||
fprintf(stderr, "Headerless (C7): %lu (%.2f%%)\n",
|
||||
g_classify_headerless_hit, 100.0 * g_classify_headerless_hit / total);
|
||||
fprintf(stderr, "Pool TLS: %lu (%.2f%%)\n",
|
||||
g_classify_pool_hit, 100.0 * g_classify_pool_hit / total);
|
||||
fprintf(stderr, "Unknown: %lu (%.2f%%)\n",
|
||||
g_classify_unknown_hit, 100.0 * g_classify_unknown_hit / total);
|
||||
fprintf(stderr, "Total: %lu\n", total);
|
||||
fprintf(stderr, "======================================================\n");
|
||||
}
|
||||
|
||||
static void __attribute__((destructor)) front_gate_stats_destructor(void) {
|
||||
front_gate_print_stats();
|
||||
}
|
||||
#endif
|
||||
|
||||
// ========== Safe Header Probe ==========
|
||||
|
||||
// Try to read 1-byte header at ptr-1 (safe conditions only)
|
||||
// Returns: class_idx (0-7) on success, -1 on failure
|
||||
//
|
||||
// Safety conditions:
|
||||
// 1. Same page: (ptr & 0xFFF) >= 1 → header won't cross page boundary
|
||||
// 2. Valid magic: (header & 0xF0) == HEADER_MAGIC (0xa0)
|
||||
// 3. Valid class: class_idx in range [0, 7]
|
||||
//
|
||||
// Performance: 2-3 cycles (L1 cache hit)
|
||||
static inline int safe_header_probe(void* ptr) {
|
||||
// Safety check: header must be in same page as ptr
|
||||
uintptr_t offset_in_page = (uintptr_t)ptr & 0xFFF;
|
||||
if (offset_in_page == 0) {
|
||||
// ptr is page-aligned → header would be on previous page (unsafe)
|
||||
return -1;
|
||||
}
|
||||
|
||||
// Safe to read header (same page guaranteed)
|
||||
uint8_t* header_ptr = (uint8_t*)ptr - 1;
|
||||
uint8_t header = *header_ptr;
|
||||
|
||||
// Validate magic
|
||||
if ((header & 0xF0) != HEADER_MAGIC) {
|
||||
return -1; // Not a Tiny header
|
||||
}
|
||||
|
||||
// Extract class index
|
||||
int class_idx = header & HEADER_CLASS_MASK;
|
||||
|
||||
// Header-based Tiny never encodes class 7 (C7 is headerless)
|
||||
if (class_idx == 7) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
// Validate class range
|
||||
if (class_idx < 0 || class_idx >= TINY_NUM_CLASSES) {
|
||||
return -1; // Invalid class
|
||||
}
|
||||
|
||||
return class_idx;
|
||||
}
|
||||
|
||||
// ========== Registry Lookup ==========
|
||||
|
||||
// Lookup pointer in SuperSlab registry (fallback when header probe fails)
|
||||
// Returns: classification result with SuperSlab + class_idx + slab_idx
|
||||
//
|
||||
// Performance: 50-100 cycles (hash lookup + validation)
|
||||
static inline ptr_classification_t registry_lookup(void* ptr) {
|
||||
ptr_classification_t result = {
|
||||
.kind = PTR_KIND_UNKNOWN,
|
||||
.class_idx = -1,
|
||||
.ss = NULL,
|
||||
.slab_idx = -1
|
||||
};
|
||||
|
||||
// Query SuperSlab registry
|
||||
struct SuperSlab* ss = hak_super_lookup(ptr);
|
||||
if (!ss) {
|
||||
// Not in Tiny registry
|
||||
return result;
|
||||
}
|
||||
|
||||
// Found SuperSlab - determine slab index
|
||||
result.ss = ss;
|
||||
result.class_idx = ss->size_class;
|
||||
|
||||
// Calculate slab index
|
||||
uintptr_t ptr_addr = (uintptr_t)ptr;
|
||||
uintptr_t ss_addr = (uintptr_t)ss;
|
||||
|
||||
if (ptr_addr < ss_addr) {
|
||||
// Pointer before SuperSlab base (invalid)
|
||||
result.kind = PTR_KIND_UNKNOWN;
|
||||
return result;
|
||||
}
|
||||
|
||||
size_t offset = ptr_addr - ss_addr;
|
||||
result.slab_idx = (int)(offset / SLAB_SIZE);
|
||||
|
||||
// Validate slab index (ss_slabs_capacity defined in superslab_inline.h)
|
||||
if (result.slab_idx < 0 || result.slab_idx >= ss_slabs_capacity(ss)) {
|
||||
// Out of range
|
||||
result.kind = PTR_KIND_UNKNOWN;
|
||||
return result;
|
||||
}
|
||||
|
||||
// Valid Tiny allocation (headerless)
|
||||
// Note: C7 (1KB) is the only headerless class, but Registry handles all
|
||||
result.kind = PTR_KIND_TINY_HEADERLESS;
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
// ========== Pool TLS Probe ==========
|
||||
|
||||
#ifdef HAKMEM_POOL_TLS_PHASE1
|
||||
// Check if pointer has Pool TLS magic (0xb0)
|
||||
// Returns: 1 if Pool TLS, 0 otherwise
|
||||
static inline int is_pool_tls(void* ptr) {
|
||||
// Same safety check as header probe
|
||||
uintptr_t offset_in_page = (uintptr_t)ptr & 0xFFF;
|
||||
if (offset_in_page == 0) {
|
||||
return 0; // Page-aligned, skip header read
|
||||
}
|
||||
|
||||
uint8_t* header_ptr = (uint8_t*)ptr - 1;
|
||||
uint8_t header = *header_ptr;
|
||||
|
||||
return (header & 0xF0) == POOL_MAGIC;
|
||||
}
|
||||
#endif
|
||||
|
||||
// ========== Front Gate Entry Point ==========
|
||||
|
||||
ptr_classification_t classify_ptr(void* ptr) {
|
||||
ptr_classification_t result = {
|
||||
.kind = PTR_KIND_UNKNOWN,
|
||||
.class_idx = -1,
|
||||
.ss = NULL,
|
||||
.slab_idx = -1
|
||||
};
|
||||
|
||||
if (!ptr) return result;
|
||||
|
||||
// Step 1: Try safe header probe (C0-C6 fast path: 5-10 cycles)
|
||||
// Skip header probe on 1KB-aligned pointers to avoid misclassifying C7/headerless
|
||||
int class_idx = -1;
|
||||
if (((uintptr_t)ptr & 0x3FF) != 0) {
|
||||
class_idx = safe_header_probe(ptr);
|
||||
}
|
||||
if (class_idx >= 0) {
|
||||
// Header found - C0-C6 with header
|
||||
result.kind = PTR_KIND_TINY_HEADER;
|
||||
result.class_idx = class_idx;
|
||||
|
||||
#if !HAKMEM_BUILD_RELEASE
|
||||
g_classify_header_hit++;
|
||||
#endif
|
||||
return result;
|
||||
}
|
||||
|
||||
// Step 2: Check Pool TLS (before Registry to avoid false positives)
|
||||
#ifdef HAKMEM_POOL_TLS_PHASE1
|
||||
if (is_pool_tls(ptr)) {
|
||||
result.kind = PTR_KIND_POOL_TLS;
|
||||
|
||||
#if !HAKMEM_BUILD_RELEASE
|
||||
g_classify_pool_hit++;
|
||||
#endif
|
||||
return result;
|
||||
}
|
||||
#endif
|
||||
|
||||
// Step 3: Fallback to Registry lookup (C7 headerless or header failed)
|
||||
result = registry_lookup(ptr);
|
||||
if (result.kind == PTR_KIND_TINY_HEADERLESS) {
|
||||
#if !HAKMEM_BUILD_RELEASE
|
||||
g_classify_headerless_hit++;
|
||||
#endif
|
||||
return result;
|
||||
}
|
||||
|
||||
// Step 4: Not Tiny or Pool - return UNKNOWN
|
||||
// Caller should check AllocHeader (16-byte) or delegate to system free
|
||||
result.kind = PTR_KIND_UNKNOWN;
|
||||
|
||||
#if !HAKMEM_BUILD_RELEASE
|
||||
g_classify_unknown_hit++;
|
||||
#endif
|
||||
|
||||
return result;
|
||||
}
|
||||
38
core/box/front_gate_classifier.d
Normal file
38
core/box/front_gate_classifier.d
Normal file
@ -0,0 +1,38 @@
|
||||
core/box/front_gate_classifier.o: core/box/front_gate_classifier.c \
|
||||
core/box/front_gate_classifier.h core/box/../tiny_region_id.h \
|
||||
core/box/../hakmem_build_flags.h core/box/../hakmem_tiny_superslab.h \
|
||||
core/box/../superslab/superslab_types.h \
|
||||
core/hakmem_tiny_superslab_constants.h \
|
||||
core/box/../superslab/superslab_inline.h \
|
||||
core/box/../superslab/superslab_types.h core/tiny_debug_ring.h \
|
||||
core/tiny_remote.h core/box/../superslab/../tiny_box_geometry.h \
|
||||
core/box/../superslab/../hakmem_tiny_superslab_constants.h \
|
||||
core/box/../superslab/../hakmem_tiny_config.h \
|
||||
core/box/../tiny_debug_ring.h core/box/../tiny_remote.h \
|
||||
core/box/../hakmem_tiny_superslab_constants.h \
|
||||
core/box/../superslab/superslab_inline.h \
|
||||
core/box/../hakmem_build_flags.h core/box/../hakmem_tiny_config.h \
|
||||
core/box/../hakmem_super_registry.h core/box/../hakmem_tiny_superslab.h \
|
||||
core/box/../pool_tls.h
|
||||
core/box/front_gate_classifier.h:
|
||||
core/box/../tiny_region_id.h:
|
||||
core/box/../hakmem_build_flags.h:
|
||||
core/box/../hakmem_tiny_superslab.h:
|
||||
core/box/../superslab/superslab_types.h:
|
||||
core/hakmem_tiny_superslab_constants.h:
|
||||
core/box/../superslab/superslab_inline.h:
|
||||
core/box/../superslab/superslab_types.h:
|
||||
core/tiny_debug_ring.h:
|
||||
core/tiny_remote.h:
|
||||
core/box/../superslab/../tiny_box_geometry.h:
|
||||
core/box/../superslab/../hakmem_tiny_superslab_constants.h:
|
||||
core/box/../superslab/../hakmem_tiny_config.h:
|
||||
core/box/../tiny_debug_ring.h:
|
||||
core/box/../tiny_remote.h:
|
||||
core/box/../hakmem_tiny_superslab_constants.h:
|
||||
core/box/../superslab/superslab_inline.h:
|
||||
core/box/../hakmem_build_flags.h:
|
||||
core/box/../hakmem_tiny_config.h:
|
||||
core/box/../hakmem_super_registry.h:
|
||||
core/box/../hakmem_tiny_superslab.h:
|
||||
core/box/../pool_tls.h:
|
||||
78
core/box/front_gate_classifier.h
Normal file
78
core/box/front_gate_classifier.h
Normal file
@ -0,0 +1,78 @@
|
||||
// front_gate_classifier.h - Box FG: Pointer Classification Front Gate
|
||||
//
|
||||
// Purpose: Single point of truth for classifying pointers (Tiny/Pool/Mid/Large)
|
||||
// Design: Heuristic-free, safe header probe + Registry lookup fallback
|
||||
//
|
||||
// Key Rules:
|
||||
// 1. ptr-1 is read ONLY here (never elsewhere)
|
||||
// 2. Header probe only when safe (same page + readable)
|
||||
// 3. C7 (headerless) always identified via Registry
|
||||
// 4. No 1KB alignment heuristics (eliminated false positives)
|
||||
//
|
||||
// Architecture:
|
||||
// - Box FG (this): Classification authority
|
||||
// - Box REG: SuperSlab registry (O(1) reverse lookup)
|
||||
// - Box TLS: next pointer clearing for C7
|
||||
//
|
||||
// Performance:
|
||||
// - Fast path (C0-C6 header): 5-10 cycles (unchanged)
|
||||
// - Slow path (C7 REG): 50-100 cycles (rare)
|
||||
// - Safety: SEGV eliminated, false positive = 0%
|
||||
|
||||
#ifndef FRONT_GATE_CLASSIFIER_H
|
||||
#define FRONT_GATE_CLASSIFIER_H
|
||||
|
||||
#include <stdint.h>
|
||||
#include <stddef.h>
|
||||
|
||||
// Forward declaration
|
||||
struct SuperSlab;
|
||||
|
||||
// Pointer classification kinds
|
||||
typedef enum {
|
||||
PTR_KIND_TINY_HEADER, // C0-C6: Has 1-byte header (fast path)
|
||||
PTR_KIND_TINY_HEADERLESS, // C7: Headerless 1KB blocks (REG path)
|
||||
PTR_KIND_POOL_TLS, // Pool TLS 8KB-52KB
|
||||
PTR_KIND_MID_LARGE, // Mid/Large allocations
|
||||
PTR_KIND_UNKNOWN // Unknown/external allocation
|
||||
} tiny_ptr_kind_t;
|
||||
|
||||
// Classification result
|
||||
typedef struct {
|
||||
tiny_ptr_kind_t kind; // Classification result
|
||||
int class_idx; // Tiny class (0-7), or -1 if not Tiny
|
||||
struct SuperSlab* ss; // SuperSlab pointer (from Registry, or NULL)
|
||||
int slab_idx; // Slab index within SuperSlab (or -1)
|
||||
} ptr_classification_t;
|
||||
|
||||
// ========== Front Gate API ==========
|
||||
|
||||
// Classify pointer (single point of truth)
|
||||
// Returns: Classification result with kind, class_idx, SuperSlab
|
||||
//
|
||||
// Strategy:
|
||||
// 1. Try safe header probe (C0-C6 fast path: 5-10 cycles)
|
||||
// 2. Fallback to Registry lookup (C7 or header failed)
|
||||
// 3. Check Pool TLS magic
|
||||
// 4. Check AllocHeader (16-byte malloc/mmap)
|
||||
// 5. Return UNKNOWN if all fail
|
||||
//
|
||||
// Safety:
|
||||
// - Header probe only if: (ptr & 0xFFF) >= 1 (same page)
|
||||
// - No 1KB alignment heuristics
|
||||
// - Registry provides ground truth for headerless
|
||||
ptr_classification_t classify_ptr(void* ptr);
|
||||
|
||||
// ========== Debug/Stats (optional) ==========
|
||||
|
||||
#if !HAKMEM_BUILD_RELEASE
|
||||
// Track classification hit rates
|
||||
extern __thread uint64_t g_classify_header_hit;
|
||||
extern __thread uint64_t g_classify_headerless_hit;
|
||||
extern __thread uint64_t g_classify_pool_hit;
|
||||
extern __thread uint64_t g_classify_unknown_hit;
|
||||
|
||||
void front_gate_print_stats(void);
|
||||
#endif
|
||||
|
||||
#endif // FRONT_GATE_CLASSIFIER_H
|
||||
@ -4,6 +4,7 @@
|
||||
|
||||
#include "hakmem_tiny_superslab.h" // For SUPERSLAB_MAGIC, SuperSlab
|
||||
#include "../tiny_free_fast_v2.inc.h" // Phase 7: Header-based ultra-fast free
|
||||
#include "front_gate_classifier.h" // Box FG: Centralized pointer classification
|
||||
|
||||
#ifdef HAKMEM_POOL_TLS_PHASE1
|
||||
#include "../pool_tls.h"
|
||||
@ -78,121 +79,63 @@ void hak_free_at(void* ptr, size_t size, hak_callsite_t site) {
|
||||
return;
|
||||
}
|
||||
|
||||
#ifdef HAKMEM_POOL_TLS_PHASE1
|
||||
// Phase 1: Try Pool TLS free FIRST for 8KB-52KB range
|
||||
// CRITICAL: Must come before Phase 7 Tiny to avoid magic mismatch SEGV
|
||||
// Pool TLS uses magic 0xb0, Tiny uses magic 0xa0 - must distinguish!
|
||||
{
|
||||
void* header_addr = (char*)ptr - 1;
|
||||
|
||||
// Safety vs performance trade-off:
|
||||
// - If HAKMEM_TINY_SAFE_FREE=1 (strict), validate with mincore() always
|
||||
// - Else (default), only validate on page-boundary risk to avoid syscall cost
|
||||
#if HAKMEM_TINY_SAFE_FREE
|
||||
if (!hak_is_memory_readable(header_addr)) { goto skip_pool_tls; }
|
||||
#else
|
||||
uintptr_t off = (uintptr_t)header_addr & 0xFFF;
|
||||
if (__builtin_expect(off == 0, 0)) {
|
||||
if (!hak_is_memory_readable(header_addr)) { goto skip_pool_tls; }
|
||||
}
|
||||
#endif
|
||||
|
||||
uint8_t header = *(uint8_t*)header_addr;
|
||||
|
||||
if ((header & 0xF0) == POOL_MAGIC) {
|
||||
pool_free(ptr);
|
||||
hak_free_route_log("pool_tls", ptr);
|
||||
goto done;
|
||||
}
|
||||
// Not Pool TLS - fall through to other paths
|
||||
}
|
||||
skip_pool_tls:
|
||||
#endif
|
||||
// ========== Box FG: Single Point of Classification ==========
|
||||
// Classify pointer once using Front Gate (safe header probe + Registry fallback)
|
||||
// This eliminates all scattered ptr-1 reads and centralizes classification logic
|
||||
ptr_classification_t classification = classify_ptr(ptr);
|
||||
|
||||
// Route based on classification result
|
||||
switch (classification.kind) {
|
||||
case PTR_KIND_TINY_HEADER: {
|
||||
// C0-C6: Has 1-byte header, class_idx already determined by Front Gate
|
||||
// Fast path: Use class_idx directly without SuperSlab lookup
|
||||
hak_free_route_log("tiny_header", ptr);
|
||||
#if HAKMEM_TINY_HEADER_CLASSIDX
|
||||
// Phase 7: Dual-header dispatch (1-byte Tiny header OR 16-byte malloc/mmap header)
|
||||
//
|
||||
// Step 1: Try 1-byte Tiny header (fast path: 5-10 cycles)
|
||||
// Use ultra-fast free path with pre-determined class_idx
|
||||
if (__builtin_expect(hak_tiny_free_fast_v2(ptr), 1)) {
|
||||
hak_free_route_log("header_fast", ptr);
|
||||
#if !HAKMEM_BUILD_RELEASE
|
||||
hak_free_v2_track_fast(); // Track hit rate in debug
|
||||
hak_free_v2_track_fast();
|
||||
#endif
|
||||
goto done; // Success - done in 5-10 cycles! NO SuperSlab lookup!
|
||||
}
|
||||
|
||||
// Step 2: Try 16-byte AllocHeader (malloc/mmap allocations)
|
||||
// CRITICAL: Must check this BEFORE calling hak_tiny_free() to avoid silent failures!
|
||||
{
|
||||
void* raw = (char*)ptr - HEADER_SIZE;
|
||||
|
||||
// SAFETY: Check if raw header is accessible before dereferencing
|
||||
// This prevents SEGV when malloc metadata is unmapped
|
||||
//
|
||||
// OPTIMIZATION: raw = ptr - HEADER_SIZE (16 bytes)
|
||||
// Page boundary case: if ptr is in first 16 bytes of page, raw crosses page boundary
|
||||
// Check: (ptr & 0xFFF) < HEADER_SIZE → raw might be on previous (unmapped) page
|
||||
uintptr_t offset_in_page = (uintptr_t)ptr & 0xFFF;
|
||||
if (__builtin_expect(offset_in_page < HEADER_SIZE, 0)) {
|
||||
// Potential page boundary crossing - do safety check
|
||||
if (!hak_is_memory_readable(raw)) {
|
||||
goto slow_path_after_step2;
|
||||
}
|
||||
}
|
||||
// Normal case (99.6%): raw is on same page as ptr (no mincore call!)
|
||||
|
||||
// Safe to dereference now
|
||||
AllocHeader* hdr = (AllocHeader*)raw;
|
||||
|
||||
if (hdr->magic == HAKMEM_MAGIC) {
|
||||
// Valid 16-byte header found (malloc/mmap allocation)
|
||||
hak_free_route_log("header_16byte", ptr);
|
||||
|
||||
if (hdr->method == ALLOC_METHOD_MALLOC) {
|
||||
// CRITICAL: raw was allocated with __libc_malloc, so free with __libc_free
|
||||
extern void __libc_free(void*);
|
||||
__libc_free(raw);
|
||||
goto done;
|
||||
}
|
||||
|
||||
// Handle other methods (mmap, etc) - continue to slow path below
|
||||
}
|
||||
slow_path_after_step2:;
|
||||
}
|
||||
|
||||
// Fallback: Invalid header (non-tiny) or TLS cache full
|
||||
// Fallback to slow path if TLS cache full
|
||||
#if !HAKMEM_BUILD_RELEASE
|
||||
hak_free_v2_track_slow();
|
||||
#endif
|
||||
#endif
|
||||
hak_tiny_free(ptr);
|
||||
goto done;
|
||||
}
|
||||
|
||||
// SS-first free（既定ON）
|
||||
#if !HAKMEM_TINY_HEADER_CLASSIDX
|
||||
// Only run SS-first if Phase 7 header-based free is not enabled
|
||||
// (Phase 7 already does the SS lookup and handles SS allocations)
|
||||
{
|
||||
static int s_free_to_ss = -2;
|
||||
if (s_free_to_ss == -2) {
|
||||
const char* e = getenv("HAKMEM_TINY_FREE_TO_SS");
|
||||
s_free_to_ss = (e && *e) ? ((*e!='0')?1:0) : 1;
|
||||
}
|
||||
if (s_free_to_ss) {
|
||||
extern int g_use_superslab;
|
||||
if (__builtin_expect(g_use_superslab != 0, 1)) {
|
||||
SuperSlab* ss = hak_super_lookup(ptr);
|
||||
if (ss && ss->magic == SUPERSLAB_MAGIC) {
|
||||
int sidx = slab_index_for(ss, ptr);
|
||||
int cap = ss_slabs_capacity(ss);
|
||||
if (__builtin_expect(sidx >= 0 && sidx < cap, 1)) { hak_free_route_log("ss_hit", ptr); hak_tiny_free(ptr); goto done; }
|
||||
}
|
||||
// FIX: Removed dangerous "guess loop" (lines 92-95)
|
||||
// The loop dereferenced unmapped memory causing SEGV
|
||||
// If registry lookup fails, allocation is not from SuperSlab
|
||||
}
|
||||
case PTR_KIND_TINY_HEADERLESS: {
|
||||
// C7: Headerless 1KB blocks, SuperSlab + slab_idx provided by Registry
|
||||
// Medium path: Use Registry result, no header read needed
|
||||
hak_free_route_log("tiny_headerless", ptr);
|
||||
hak_tiny_free(ptr);
|
||||
goto done;
|
||||
}
|
||||
|
||||
#ifdef HAKMEM_POOL_TLS_PHASE1
|
||||
case PTR_KIND_POOL_TLS: {
|
||||
// Pool TLS: 8KB-52KB allocations with 0xb0 magic
|
||||
hak_free_route_log("pool_tls", ptr);
|
||||
pool_free(ptr);
|
||||
goto done;
|
||||
}
|
||||
#endif
|
||||
|
||||
case PTR_KIND_UNKNOWN:
|
||||
default: {
|
||||
// Not Tiny or Pool - check 16-byte AllocHeader (Mid/Large/malloc/mmap)
|
||||
// This is the slow path for large allocations
|
||||
break; // Fall through to header dispatch below
|
||||
}
|
||||
}
|
||||
|
||||
// ========== Slow Path: 16-byte AllocHeader Dispatch ==========
|
||||
// Handle Mid/Large allocations (malloc/mmap/Pool/L25)
|
||||
// Note: All Tiny allocations (C0-C7) already handled by Front Gate above
|
||||
|
||||
// Mid/L25 headerless経路
|
||||
{
|
||||
extern int hak_pool_mid_lookup(void* ptr, size_t* out_size);
|
||||
|
||||
164
core/box/tls_sll_box.h
Normal file
164
core/box/tls_sll_box.h
Normal file
@ -0,0 +1,164 @@
|
||||
// tls_sll_box.h - Box TLS-SLL: Single-Linked List API (C7-safe)
|
||||
//
|
||||
// Purpose: Centralized TLS SLL management with C7 protection
|
||||
// Design: Zero-overhead static inline API, C7 always rejected
|
||||
//
|
||||
// Key Rules:
|
||||
// 1. C7 (1KB headerless) is ALWAYS rejected (returns false/0)
|
||||
// 2. All SLL direct writes MUST go through this API
|
||||
// 3. Pop returns with first 8 bytes cleared for C7 (safety)
|
||||
// 4. Capacity checks prevent overflow
|
||||
//
|
||||
// Architecture:
|
||||
// - Box TLS-SLL (this): Push/Pop/Splice authority
|
||||
// - Caller: Provides capacity limits, handles fallback on failure
|
||||
//
|
||||
// Performance:
|
||||
// - Static inline → zero function call overhead
|
||||
// - C7 check: 1 comparison + predict-not-taken (< 1 cycle)
|
||||
// - Same performance as direct SLL access for C0-C6
|
||||
|
||||
#ifndef TLS_SLL_BOX_H
|
||||
#define TLS_SLL_BOX_H
|
||||
|
||||
#include <stdint.h>
|
||||
#include <stdbool.h>
|
||||
#include <stdio.h> // For fprintf in debug
|
||||
#include <stdlib.h> // For abort in debug
|
||||
#include "../hakmem_tiny_config.h" // For TINY_NUM_CLASSES
|
||||
|
||||
// External TLS SLL state (defined elsewhere)
|
||||
extern __thread void* g_tls_sll_head[TINY_NUM_CLASSES];
|
||||
extern __thread uint32_t g_tls_sll_count[TINY_NUM_CLASSES];
|
||||
|
||||
// ========== Push ==========
|
||||
|
||||
// Push pointer to TLS SLL
|
||||
// Returns: true on success, false if C7 or capacity exceeded
|
||||
//
|
||||
// Safety:
|
||||
// - C7 always rejected (headerless, first 8 bytes = user data)
|
||||
// - Capacity check prevents overflow
|
||||
// - Caller must handle fallback (e.g., meta->freelist)
|
||||
//
|
||||
// Performance: 2-3 cycles (C0-C6), < 1 cycle (C7 fast rejection)
|
||||
static inline bool tls_sll_push(int class_idx, void* ptr, uint32_t capacity) {
|
||||
// CRITICAL: C7 (1KB) is headerless - MUST NOT use TLS SLL
|
||||
// Reason: SLL stores next pointer in first 8 bytes (user data for C7)
|
||||
if (__builtin_expect(class_idx == 7, 0)) {
|
||||
return false; // C7 rejected
|
||||
}
|
||||
|
||||
// Capacity check
|
||||
if (g_tls_sll_count[class_idx] >= capacity) {
|
||||
return false; // SLL full
|
||||
}
|
||||
|
||||
// Push to SLL (standard linked list push)
|
||||
*(void**)ptr = g_tls_sll_head[class_idx];
|
||||
g_tls_sll_head[class_idx] = ptr;
|
||||
g_tls_sll_count[class_idx]++;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
// ========== Pop ==========
|
||||
|
||||
// Pop pointer from TLS SLL
|
||||
// Returns: true on success (writes to *out), false if empty
|
||||
//
|
||||
// Safety:
|
||||
// - C7 protection: clears first 8 bytes on pop (prevents next pointer leak)
|
||||
// - NULL check before deref
|
||||
//
|
||||
// Performance: 3-4 cycles
|
||||
static inline bool tls_sll_pop(int class_idx, void** out) {
|
||||
void* head = g_tls_sll_head[class_idx];
|
||||
if (!head) {
|
||||
return false; // SLL empty
|
||||
}
|
||||
|
||||
// Pop from SLL
|
||||
void* next = *(void**)head;
|
||||
g_tls_sll_head[class_idx] = next;
|
||||
if (g_tls_sll_count[class_idx] > 0) {
|
||||
g_tls_sll_count[class_idx]--;
|
||||
}
|
||||
|
||||
// CRITICAL: C7 (1KB) returns with first 8 bytes cleared
|
||||
// Reason: C7 is headerless, first 8 bytes are user data area
|
||||
// Without this: user sees stale SLL next pointer → corruption
|
||||
// Cost: 1 store instruction (~1 cycle), only for C7 (~1% of allocations)
|
||||
//
|
||||
// Note: C0-C6 have 1-byte header, so first 8 bytes are safe (header hides next)
|
||||
if (__builtin_expect(class_idx == 7, 0)) {
|
||||
*(void**)head = NULL;
|
||||
}
|
||||
|
||||
*out = head;
|
||||
return true;
|
||||
}
|
||||
|
||||
// ========== Splice ==========
|
||||
|
||||
// Splice chain of pointers to TLS SLL (batch push)
|
||||
// Returns: actual count moved (0 for C7 or if capacity exceeded)
|
||||
//
|
||||
// Safety:
|
||||
// - C7 always returns 0 (no splice)
|
||||
// - Capacity check limits splice size
|
||||
// - Chain traversal with safety (breaks on NULL)
|
||||
//
|
||||
// Performance: ~5 cycles + O(count) for chain traversal
|
||||
static inline uint32_t tls_sll_splice(int class_idx, void* chain_head, uint32_t count, uint32_t capacity) {
|
||||
// CRITICAL: C7 (1KB) is headerless - MUST NOT splice to TLS SLL
|
||||
if (__builtin_expect(class_idx == 7, 0)) {
|
||||
return 0; // C7 rejected
|
||||
}
|
||||
|
||||
// Calculate available capacity
|
||||
uint32_t available = (capacity > g_tls_sll_count[class_idx])
|
||||
? (capacity - g_tls_sll_count[class_idx]) : 0;
|
||||
if (available == 0 || count == 0 || !chain_head) {
|
||||
return 0; // No space or empty chain
|
||||
}
|
||||
|
||||
// Limit splice size to available capacity
|
||||
uint32_t to_move = (count < available) ? count : available;
|
||||
|
||||
// Find chain tail (traverse to_move - 1 nodes)
|
||||
void* tail = chain_head;
|
||||
for (uint32_t i = 1; i < to_move; i++) {
|
||||
void* next = *(void**)tail;
|
||||
if (!next) {
|
||||
// Chain shorter than expected, adjust to_move
|
||||
to_move = i;
|
||||
break;
|
||||
}
|
||||
tail = next;
|
||||
}
|
||||
|
||||
// Splice chain to SLL head
|
||||
*(void**)tail = g_tls_sll_head[class_idx];
|
||||
g_tls_sll_head[class_idx] = chain_head;
|
||||
g_tls_sll_count[class_idx] += to_move;
|
||||
|
||||
return to_move;
|
||||
}
|
||||
|
||||
// ========== Debug/Stats (optional) ==========
|
||||
|
||||
#if !HAKMEM_BUILD_RELEASE
|
||||
// Verify C7 is not in SLL (debug only, call at safe points)
|
||||
static inline void tls_sll_verify_no_c7(void) {
|
||||
void* head = g_tls_sll_head[7];
|
||||
if (head != NULL) {
|
||||
fprintf(stderr, "[TLS_SLL_BUG] C7 found in TLS SLL! head=%p count=%u\n",
|
||||
head, g_tls_sll_count[7]);
|
||||
fprintf(stderr, "[TLS_SLL_BUG] This should NEVER happen - C7 is headerless!\n");
|
||||
abort();
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif // TLS_SLL_BOX_H
|
||||
@ -1212,6 +1212,7 @@ static uint8_t g_hotmag_class_en[TINY_NUM_CLASSES]; // 0=disabled for clas
|
||||
static __thread TinyHotMag g_tls_hot_mag[TINY_NUM_CLASSES];
|
||||
// Inline helpers
|
||||
|
||||
#include "box/tls_sll_box.h" // Box TLS-SLL: Safe SLL operations API (needed by hotmag)
|
||||
#include "hakmem_tiny_hotmag.inc.h"
|
||||
|
||||
// Size-specialized tiny alloc (32B/64B) via function pointers (A/B用)
|
||||
@ -1241,6 +1242,14 @@ void hak_tiny_prewarm_tls_cache(void) {
|
||||
// Pre-warm each class with HAKMEM_TINY_PREWARM_COUNT blocks
|
||||
// This reduces the first-allocation miss penalty by populating TLS cache
|
||||
for (int class_idx = 0; class_idx < TINY_NUM_CLASSES; class_idx++) {
|
||||
// CRITICAL: C7 (1KB) is headerless - skip TLS SLL refill, but create SuperSlab
|
||||
if (class_idx == 7) {
|
||||
// Create C7 SuperSlab explicitly (refill functions skip C7)
|
||||
// Note: superslab_refill is already declared in hakmem_tiny_refill.inc.h
|
||||
(void)superslab_refill(class_idx);
|
||||
continue;
|
||||
}
|
||||
|
||||
int count = HAKMEM_TINY_PREWARM_COUNT; // Default: 16 blocks per class
|
||||
|
||||
// Trigger refill to populate TLS cache
|
||||
|
||||
137
core/hakmem_tiny.d
Normal file
137
core/hakmem_tiny.d
Normal file
@ -0,0 +1,137 @@
|
||||
core/hakmem_tiny.o: core/hakmem_tiny.c core/hakmem_tiny.h \
|
||||
core/hakmem_build_flags.h core/hakmem_trace.h \
|
||||
core/hakmem_tiny_mini_mag.h core/hakmem_tiny_config.h \
|
||||
core/hakmem_phase7_config.h core/hakmem_tiny_superslab.h \
|
||||
core/superslab/superslab_types.h core/hakmem_tiny_superslab_constants.h \
|
||||
core/superslab/superslab_inline.h core/superslab/superslab_types.h \
|
||||
core/tiny_debug_ring.h core/tiny_remote.h \
|
||||
core/superslab/../tiny_box_geometry.h \
|
||||
core/superslab/../hakmem_tiny_superslab_constants.h \
|
||||
core/superslab/../hakmem_tiny_config.h core/tiny_debug_ring.h \
|
||||
core/tiny_remote.h core/hakmem_tiny_superslab_constants.h \
|
||||
core/hakmem_super_registry.h core/hakmem_internal.h core/hakmem.h \
|
||||
core/hakmem_config.h core/hakmem_features.h core/hakmem_sys.h \
|
||||
core/hakmem_whale.h core/hakmem_syscall.h core/hakmem_tiny_magazine.h \
|
||||
core/hakmem_tiny_batch_refill.h core/hakmem_tiny_stats.h core/tiny_api.h \
|
||||
core/hakmem_tiny_stats_api.h core/hakmem_tiny_query_api.h \
|
||||
core/hakmem_tiny_rss_api.h core/hakmem_tiny_registry_api.h \
|
||||
core/tiny_tls.h core/tiny_debug.h core/tiny_mmap_gate.h \
|
||||
core/tiny_refill.h core/slab_handle.h core/tiny_sticky.h \
|
||||
core/tiny_ready.h core/box/mailbox_box.h core/hakmem_tiny_superslab.h \
|
||||
core/tiny_remote_bg.h core/hakmem_tiny_remote_target.h \
|
||||
core/tiny_ready_bg.h core/tiny_route.h core/box/adopt_gate_box.h \
|
||||
core/tiny_tls_guard.h core/hakmem_tiny_tls_list.h \
|
||||
core/hakmem_tiny_bg_spill.h core/tiny_adaptive_sizing.h \
|
||||
core/tiny_system.h core/hakmem_prof.h core/tiny_publish.h \
|
||||
core/box/tls_sll_box.h core/box/../hakmem_tiny_config.h \
|
||||
core/hakmem_tiny_hotmag.inc.h core/hakmem_tiny_hot_pop.inc.h \
|
||||
core/hakmem_tiny_fastcache.inc.h core/hakmem_tiny_refill.inc.h \
|
||||
core/tiny_box_geometry.h core/hakmem_tiny_refill_p0.inc.h \
|
||||
core/tiny_refill_opt.h core/tiny_fc_api.h \
|
||||
core/hakmem_tiny_ultra_front.inc.h core/hakmem_tiny_intel.inc \
|
||||
core/hakmem_tiny_background.inc core/hakmem_tiny_bg_bin.inc.h \
|
||||
core/hakmem_tiny_tls_ops.h core/hakmem_tiny_remote.inc \
|
||||
core/hakmem_tiny_init.inc core/hakmem_tiny_bump.inc.h \
|
||||
core/hakmem_tiny_smallmag.inc.h core/tiny_atomic.h \
|
||||
core/tiny_alloc_fast.inc.h core/tiny_alloc_fast_sfc.inc.h \
|
||||
core/tiny_region_id.h core/tiny_alloc_fast_inline.h \
|
||||
core/tiny_free_fast.inc.h core/hakmem_tiny_alloc.inc \
|
||||
core/hakmem_tiny_slow.inc core/hakmem_tiny_free.inc \
|
||||
core/box/free_publish_box.h core/mid_tcache.h \
|
||||
core/tiny_free_magazine.inc.h core/tiny_superslab_alloc.inc.h \
|
||||
core/tiny_superslab_free.inc.h core/box/free_remote_box.h \
|
||||
core/box/free_local_box.h core/hakmem_tiny_lifecycle.inc \
|
||||
core/hakmem_tiny_slab_mgmt.inc
|
||||
core/hakmem_tiny.h:
|
||||
core/hakmem_build_flags.h:
|
||||
core/hakmem_trace.h:
|
||||
core/hakmem_tiny_mini_mag.h:
|
||||
core/hakmem_tiny_config.h:
|
||||
core/hakmem_phase7_config.h:
|
||||
core/hakmem_tiny_superslab.h:
|
||||
core/superslab/superslab_types.h:
|
||||
core/hakmem_tiny_superslab_constants.h:
|
||||
core/superslab/superslab_inline.h:
|
||||
core/superslab/superslab_types.h:
|
||||
core/tiny_debug_ring.h:
|
||||
core/tiny_remote.h:
|
||||
core/superslab/../tiny_box_geometry.h:
|
||||
core/superslab/../hakmem_tiny_superslab_constants.h:
|
||||
core/superslab/../hakmem_tiny_config.h:
|
||||
core/tiny_debug_ring.h:
|
||||
core/tiny_remote.h:
|
||||
core/hakmem_tiny_superslab_constants.h:
|
||||
core/hakmem_super_registry.h:
|
||||
core/hakmem_internal.h:
|
||||
core/hakmem.h:
|
||||
core/hakmem_config.h:
|
||||
core/hakmem_features.h:
|
||||
core/hakmem_sys.h:
|
||||
core/hakmem_whale.h:
|
||||
core/hakmem_syscall.h:
|
||||
core/hakmem_tiny_magazine.h:
|
||||
core/hakmem_tiny_batch_refill.h:
|
||||
core/hakmem_tiny_stats.h:
|
||||
core/tiny_api.h:
|
||||
core/hakmem_tiny_stats_api.h:
|
||||
core/hakmem_tiny_query_api.h:
|
||||
core/hakmem_tiny_rss_api.h:
|
||||
core/hakmem_tiny_registry_api.h:
|
||||
core/tiny_tls.h:
|
||||
core/tiny_debug.h:
|
||||
core/tiny_mmap_gate.h:
|
||||
core/tiny_refill.h:
|
||||
core/slab_handle.h:
|
||||
core/tiny_sticky.h:
|
||||
core/tiny_ready.h:
|
||||
core/box/mailbox_box.h:
|
||||
core/hakmem_tiny_superslab.h:
|
||||
core/tiny_remote_bg.h:
|
||||
core/hakmem_tiny_remote_target.h:
|
||||
core/tiny_ready_bg.h:
|
||||
core/tiny_route.h:
|
||||
core/box/adopt_gate_box.h:
|
||||
core/tiny_tls_guard.h:
|
||||
core/hakmem_tiny_tls_list.h:
|
||||
core/hakmem_tiny_bg_spill.h:
|
||||
core/tiny_adaptive_sizing.h:
|
||||
core/tiny_system.h:
|
||||
core/hakmem_prof.h:
|
||||
core/tiny_publish.h:
|
||||
core/box/tls_sll_box.h:
|
||||
core/box/../hakmem_tiny_config.h:
|
||||
core/hakmem_tiny_hotmag.inc.h:
|
||||
core/hakmem_tiny_hot_pop.inc.h:
|
||||
core/hakmem_tiny_fastcache.inc.h:
|
||||
core/hakmem_tiny_refill.inc.h:
|
||||
core/tiny_box_geometry.h:
|
||||
core/hakmem_tiny_refill_p0.inc.h:
|
||||
core/tiny_refill_opt.h:
|
||||
core/tiny_fc_api.h:
|
||||
core/hakmem_tiny_ultra_front.inc.h:
|
||||
core/hakmem_tiny_intel.inc:
|
||||
core/hakmem_tiny_background.inc:
|
||||
core/hakmem_tiny_bg_bin.inc.h:
|
||||
core/hakmem_tiny_tls_ops.h:
|
||||
core/hakmem_tiny_remote.inc:
|
||||
core/hakmem_tiny_init.inc:
|
||||
core/hakmem_tiny_bump.inc.h:
|
||||
core/hakmem_tiny_smallmag.inc.h:
|
||||
core/tiny_atomic.h:
|
||||
core/tiny_alloc_fast.inc.h:
|
||||
core/tiny_alloc_fast_sfc.inc.h:
|
||||
core/tiny_region_id.h:
|
||||
core/tiny_alloc_fast_inline.h:
|
||||
core/tiny_free_fast.inc.h:
|
||||
core/hakmem_tiny_alloc.inc:
|
||||
core/hakmem_tiny_slow.inc:
|
||||
core/hakmem_tiny_free.inc:
|
||||
core/box/free_publish_box.h:
|
||||
core/mid_tcache.h:
|
||||
core/tiny_free_magazine.inc.h:
|
||||
core/tiny_superslab_alloc.inc.h:
|
||||
core/tiny_superslab_free.inc.h:
|
||||
core/box/free_remote_box.h:
|
||||
core/box/free_local_box.h:
|
||||
core/hakmem_tiny_lifecycle.inc:
|
||||
core/hakmem_tiny_slab_mgmt.inc:
|
||||
@ -1,3 +1,8 @@
|
||||
// ============================================================================
|
||||
// Box TLS-SLL API
|
||||
// ============================================================================
|
||||
#include "box/tls_sll_box.h"
|
||||
|
||||
// ============================================================================
|
||||
// Step 3: Cold-path outline - Wrapper Context Handler
|
||||
// ============================================================================
|
||||
@ -147,10 +152,8 @@ void* hak_tiny_alloc(size_t size) {
|
||||
// Minimal Front for hot tiny classes (bench-focused):
|
||||
// SLL direct pop → minimal refill → pop, bypassing other layers.
|
||||
if (__builtin_expect(class_idx <= 3, 1)) {
|
||||
void* head = g_tls_sll_head[class_idx];
|
||||
if (__builtin_expect(head != NULL, 1)) {
|
||||
g_tls_sll_head[class_idx] = *(void**)head;
|
||||
if (g_tls_sll_count[class_idx] > 0) g_tls_sll_count[class_idx]--;
|
||||
void* head = NULL;
|
||||
if (tls_sll_pop(class_idx, &head)) {
|
||||
HAK_RET_ALLOC(class_idx, head);
|
||||
}
|
||||
// Refill a small batch directly from TLS-cached SuperSlab
|
||||
@ -159,10 +162,7 @@ void* hak_tiny_alloc(size_t size) {
|
||||
#else
|
||||
(void)sll_refill_small_from_ss(class_idx, 32);
|
||||
#endif
|
||||
head = g_tls_sll_head[class_idx];
|
||||
if (__builtin_expect(head != NULL, 1)) {
|
||||
g_tls_sll_head[class_idx] = *(void**)head;
|
||||
if (g_tls_sll_count[class_idx] > 0) g_tls_sll_count[class_idx]--;
|
||||
if (tls_sll_pop(class_idx, &head)) {
|
||||
HAK_RET_ALLOC(class_idx, head);
|
||||
}
|
||||
// Fall through to slow path if still empty
|
||||
@ -205,11 +205,9 @@ void* hak_tiny_alloc(size_t size) {
|
||||
tiny_small_mags_init_once();
|
||||
if (class_idx > 3) tiny_mag_init_if_needed(class_idx);
|
||||
#endif
|
||||
void* head = g_tls_sll_head[class_idx];
|
||||
if (__builtin_expect(head != NULL, 1)) {
|
||||
void* head = NULL;
|
||||
if (tls_sll_pop(class_idx, &head)) {
|
||||
tiny_debug_ring_record(TINY_RING_EVENT_ALLOC_SUCCESS, (uint16_t)class_idx, head, 0);
|
||||
g_tls_sll_head[class_idx] = *(void**)head;
|
||||
if (g_tls_sll_count[class_idx] > 0) g_tls_sll_count[class_idx]--;
|
||||
HAK_RET_ALLOC(class_idx, head);
|
||||
}
|
||||
#ifndef HAKMEM_TINY_BENCH_SLL_ONLY
|
||||
@ -231,11 +229,8 @@ void* hak_tiny_alloc(size_t size) {
|
||||
#else
|
||||
if (__builtin_expect(sll_refill_small_from_ss(class_idx, bench_refill) > 0, 0)) {
|
||||
#endif
|
||||
head = g_tls_sll_head[class_idx];
|
||||
if (head) {
|
||||
if (tls_sll_pop(class_idx, &head)) {
|
||||
tiny_debug_ring_record(TINY_RING_EVENT_ALLOC_SUCCESS, (uint16_t)class_idx, head, 2);
|
||||
g_tls_sll_head[class_idx] = *(void**)head;
|
||||
if (g_tls_sll_count[class_idx] > 0) g_tls_sll_count[class_idx]--;
|
||||
HAK_RET_ALLOC(class_idx, head);
|
||||
}
|
||||
}
|
||||
@ -254,6 +249,7 @@ void* hak_tiny_alloc(size_t size) {
|
||||
}
|
||||
}
|
||||
if (__builtin_expect(hotmag_ptr != NULL, 1)) {
|
||||
if (__builtin_expect(class_idx == 7, 0)) { *(void**)hotmag_ptr = NULL; }
|
||||
tiny_debug_ring_record(TINY_RING_EVENT_ALLOC_SUCCESS, (uint16_t)class_idx, hotmag_ptr, 3);
|
||||
HAK_RET_ALLOC(class_idx, hotmag_ptr);
|
||||
}
|
||||
@ -282,6 +278,7 @@ void* hak_tiny_alloc(size_t size) {
|
||||
#if HAKMEM_BUILD_DEBUG
|
||||
g_tls_hit_count[class_idx]++;
|
||||
#endif
|
||||
if (__builtin_expect(class_idx == 7, 0)) { *(void**)fast_hot = NULL; }
|
||||
tiny_debug_ring_record(TINY_RING_EVENT_ALLOC_SUCCESS, (uint16_t)class_idx, fast_hot, 4);
|
||||
HAK_RET_ALLOC(class_idx, fast_hot);
|
||||
}
|
||||
@ -292,6 +289,7 @@ void* hak_tiny_alloc(size_t size) {
|
||||
#if HAKMEM_BUILD_DEBUG
|
||||
g_tls_hit_count[class_idx]++;
|
||||
#endif
|
||||
if (__builtin_expect(class_idx == 7, 0)) { *(void**)fast = NULL; }
|
||||
tiny_debug_ring_record(TINY_RING_EVENT_ALLOC_SUCCESS, (uint16_t)class_idx, fast, 5);
|
||||
HAK_RET_ALLOC(class_idx, fast);
|
||||
}
|
||||
|
||||
@ -87,6 +87,10 @@ static inline __attribute__((always_inline)) void* tiny_fast_pop(int class_idx)
|
||||
} else {
|
||||
g_fast_count[class_idx] = 0;
|
||||
}
|
||||
// Headerless class (1KB): clear embedded next pointer before returning to user
|
||||
if (__builtin_expect(class_idx == 7, 0)) {
|
||||
*(void**)head = NULL;
|
||||
}
|
||||
return head;
|
||||
}
|
||||
|
||||
|
||||
@ -5,6 +5,7 @@
|
||||
#include "tiny_refill.h"
|
||||
#include "tiny_tls_guard.h"
|
||||
#include "box/free_publish_box.h"
|
||||
#include "box/tls_sll_box.h" // Box TLS-SLL: C7-safe push/pop/splice
|
||||
#include "mid_tcache.h"
|
||||
extern __thread void* g_tls_sll_head[TINY_NUM_CLASSES];
|
||||
extern __thread uint32_t g_tls_sll_count[TINY_NUM_CLASSES];
|
||||
@ -28,6 +29,9 @@ static inline int tiny_drain_to_sll_budget(void) {
|
||||
static inline void tiny_drain_freelist_to_sll_once(SuperSlab* ss, int slab_idx, int class_idx) {
|
||||
int budget = tiny_drain_to_sll_budget();
|
||||
if (__builtin_expect(budget <= 0, 1)) return;
|
||||
// CRITICAL: C7 (1KB) is headerless - MUST NOT drain to TLS SLL
|
||||
// Reason: SLL stores next pointer in first 8 bytes (user data for C7)
|
||||
if (__builtin_expect(class_idx == 7, 0)) return;
|
||||
if (!(ss && ss->magic == SUPERSLAB_MAGIC)) return;
|
||||
if (slab_idx < 0) return;
|
||||
TinySlabMeta* m = &ss->slabs[slab_idx];
|
||||
@ -64,10 +68,16 @@ static inline void tiny_drain_freelist_to_sll_once(SuperSlab* ss, int slab_idx,
|
||||
}
|
||||
|
||||
m->freelist = *(void**)p;
|
||||
*(void**)p = g_tls_sll_head[class_idx];
|
||||
g_tls_sll_head[class_idx] = p;
|
||||
g_tls_sll_count[class_idx]++;
|
||||
|
||||
// Use Box TLS-SLL API (C7-safe push)
|
||||
// Note: C7 already rejected at line 34, so this always succeeds
|
||||
uint32_t sll_capacity = 256; // Conservative limit
|
||||
if (tls_sll_push(class_idx, p, sll_capacity)) {
|
||||
moved++;
|
||||
} else {
|
||||
// SLL full, stop draining
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -181,9 +191,11 @@ void hak_tiny_free_with_slab(void* ptr, TinySlab* slab) {
|
||||
|
||||
if (__builtin_expect(g_debug_fast0, 0)) {
|
||||
tiny_debug_ring_record(TINY_RING_EVENT_FRONT_BYPASS, (uint16_t)class_idx, ptr, (uintptr_t)slab_idx);
|
||||
// Always operate on block base for C0-C6 (header lives at base)
|
||||
void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1);
|
||||
void* prev = meta->freelist;
|
||||
*(void**)ptr = prev;
|
||||
meta->freelist = ptr;
|
||||
*(void**)base = prev;
|
||||
meta->freelist = base;
|
||||
meta->used--;
|
||||
ss_active_dec_one(ss);
|
||||
if (prev == NULL) {
|
||||
@ -195,7 +207,9 @@ void hak_tiny_free_with_slab(void* ptr, TinySlab* slab) {
|
||||
}
|
||||
|
||||
if (g_fast_enable && g_fast_cap[class_idx] != 0) {
|
||||
if (tiny_fast_push(class_idx, ptr)) {
|
||||
// Push block base into fast cache
|
||||
void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1);
|
||||
if (tiny_fast_push(class_idx, base)) {
|
||||
tiny_debug_ring_record(TINY_RING_EVENT_FREE_FAST, (uint16_t)class_idx, ptr, slab_idx);
|
||||
HAK_STAT_FREE(class_idx);
|
||||
return;
|
||||
@ -210,15 +224,17 @@ void hak_tiny_free_with_slab(void* ptr, TinySlab* slab) {
|
||||
}
|
||||
// TinyHotMag front push（8/16/32B, A/B）
|
||||
if (__builtin_expect(g_hotmag_enable && class_idx <= 2, 1)) {
|
||||
if (hotmag_push(class_idx, ptr)) {
|
||||
void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1);
|
||||
if (hotmag_push(class_idx, base)) {
|
||||
tiny_debug_ring_record(TINY_RING_EVENT_FREE_RETURN_MAG, (uint16_t)class_idx, ptr, 1);
|
||||
HAK_STAT_FREE(class_idx);
|
||||
return;
|
||||
}
|
||||
}
|
||||
if (tls->count < tls->cap) {
|
||||
tiny_tls_list_guard_push(class_idx, tls, ptr);
|
||||
tls_list_push(tls, ptr);
|
||||
void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1);
|
||||
tiny_tls_list_guard_push(class_idx, tls, base);
|
||||
tls_list_push(tls, base);
|
||||
tiny_debug_ring_record(TINY_RING_EVENT_FREE_LOCAL, (uint16_t)class_idx, ptr, 0);
|
||||
HAK_STAT_FREE(class_idx);
|
||||
return;
|
||||
@ -227,8 +243,11 @@ void hak_tiny_free_with_slab(void* ptr, TinySlab* slab) {
|
||||
if (__builtin_expect(seq != g_tls_param_seen[class_idx], 0)) {
|
||||
tiny_tls_refresh_params(class_idx, tls);
|
||||
}
|
||||
tiny_tls_list_guard_push(class_idx, tls, ptr);
|
||||
tls_list_push(tls, ptr);
|
||||
{
|
||||
void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1);
|
||||
tiny_tls_list_guard_push(class_idx, tls, base);
|
||||
tls_list_push(tls, base);
|
||||
}
|
||||
if (tls_list_should_spill(tls)) {
|
||||
tls_list_spill_excess(class_idx, tls);
|
||||
}
|
||||
@ -297,10 +316,11 @@ void hak_tiny_free(void* ptr) {
|
||||
class_idx, ptr, old_head, g_tls_sll_count[class_idx]);
|
||||
}
|
||||
|
||||
*(void**)ptr = g_tls_sll_head[class_idx];
|
||||
g_tls_sll_head[class_idx] = ptr;
|
||||
g_tls_sll_count[class_idx]++;
|
||||
return;
|
||||
// Use Box TLS-SLL API (C7-safe push)
|
||||
if (tls_sll_push(class_idx, ptr, sll_cap)) {
|
||||
return; // Success
|
||||
}
|
||||
// Fall through if push fails (SLL full or C7)
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -317,7 +337,10 @@ void hak_tiny_free(void* ptr) {
|
||||
TinySlab* slab = hak_tiny_owner_slab(ptr);
|
||||
if (slab) class_idx = slab->class_idx;
|
||||
}
|
||||
if (class_idx >= 0) {
|
||||
// CRITICAL: C7 (1KB) is headerless - MUST NOT use TLS SLL
|
||||
// Reason: SLL stores next pointer in first 8 bytes (user data for C7)
|
||||
// Fix: Exclude C7 from ultra free path
|
||||
if (class_idx >= 0 && class_idx != 7) {
|
||||
// Ultra free: push directly to TLS SLL without magazine init
|
||||
int sll_cap = ultra_sll_cap_for_class(class_idx);
|
||||
if ((int)g_tls_sll_count[class_idx] < sll_cap) {
|
||||
@ -347,22 +370,26 @@ void hak_tiny_free(void* ptr) {
|
||||
class_idx, ptr, old_head, g_tls_sll_count[class_idx]);
|
||||
}
|
||||
|
||||
*(void**)ptr = g_tls_sll_head[class_idx];
|
||||
g_tls_sll_head[class_idx] = ptr;
|
||||
g_tls_sll_count[class_idx]++;
|
||||
|
||||
// Use Box TLS-SLL API (C7-safe push)
|
||||
// Note: C7 already rejected at line 334
|
||||
{
|
||||
void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1);
|
||||
if (tls_sll_push(class_idx, base, (uint32_t)sll_cap)) {
|
||||
// CORRUPTION DEBUG: Verify write succeeded
|
||||
if (__builtin_expect(tiny_refill_failfast_level() >= 2, 0)) {
|
||||
void* readback = *(void**)ptr;
|
||||
void* readback = *(void**)base;
|
||||
(void)readback;
|
||||
void* new_head = g_tls_sll_head[class_idx];
|
||||
if (readback != *(void**)&readback || new_head != ptr) {
|
||||
fprintf(stderr, "[ULTRA_FREE_CORRUPT] Write verification failed! ptr=%p new_head=%p\n",
|
||||
ptr, new_head);
|
||||
if (new_head != base) {
|
||||
fprintf(stderr, "[ULTRA_FREE_CORRUPT] Write verification failed! base=%p new_head=%p\n",
|
||||
base, new_head);
|
||||
abort();
|
||||
}
|
||||
}
|
||||
|
||||
return;
|
||||
return; // Success
|
||||
}
|
||||
}
|
||||
// Fall through if push fails (SLL full)
|
||||
}
|
||||
}
|
||||
// Fallback to existing path if class resolution fails
|
||||
@ -407,7 +434,8 @@ void hak_tiny_free(void* ptr) {
|
||||
tiny_debug_ring_record(TINY_RING_EVENT_FREE_ENTER, (uint16_t)fast_class_idx, ptr, 1);
|
||||
}
|
||||
if (fast_class_idx >= 0 && g_fast_enable && g_fast_cap[fast_class_idx] != 0) {
|
||||
if (tiny_fast_push(fast_class_idx, ptr)) {
|
||||
void* base2 = (fast_class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1);
|
||||
if (tiny_fast_push(fast_class_idx, base2)) {
|
||||
tiny_debug_ring_record(TINY_RING_EVENT_FREE_FAST, (uint16_t)fast_class_idx, ptr, 0);
|
||||
HAK_STAT_FREE(fast_class_idx);
|
||||
return;
|
||||
|
||||
@ -71,10 +71,9 @@ static inline int hotmag_refill_from_sll(int class_idx, int max_take) {
|
||||
if (max_take > room) max_take = room;
|
||||
int taken = 0;
|
||||
while (taken < max_take) {
|
||||
void* head = g_tls_sll_head[class_idx];
|
||||
if (!head) break;
|
||||
g_tls_sll_head[class_idx] = *(void**)head;
|
||||
if (g_tls_sll_count[class_idx] > 0) g_tls_sll_count[class_idx]--;
|
||||
// CRITICAL: Use Box TLS-SLL API to avoid race condition (rbp=0xa0 SEGV)
|
||||
void* head = NULL;
|
||||
if (!tls_sll_pop(class_idx, &head)) break;
|
||||
hm->slots[hm->top++] = head;
|
||||
taken++;
|
||||
}
|
||||
|
||||
@ -24,6 +24,8 @@
|
||||
#ifndef HAKMEM_TINY_METADATA_INC
|
||||
#define HAKMEM_TINY_METADATA_INC
|
||||
|
||||
#include "box/tls_sll_box.h" // Box TLS-SLL API
|
||||
|
||||
// ============================================================================
|
||||
// Phase 6-1.6: Universal Allocation Header
|
||||
// ============================================================================
|
||||
@ -86,12 +88,8 @@ void* hak_tiny_alloc_metadata(size_t size) {
|
||||
// 2. Ultra-fast path: Pop from existing TLS SLL
|
||||
// NOTE: We allocate 8 bytes EXTRA for header
|
||||
// The SLL stores pointers to HEADERS, not user pointers
|
||||
void* hdr_ptr = g_tls_sll_head[class_idx];
|
||||
if (__builtin_expect(hdr_ptr != NULL, 1)) {
|
||||
// Pop from SLL
|
||||
g_tls_sll_head[class_idx] = *(void**)hdr_ptr;
|
||||
if (g_tls_sll_count[class_idx] > 0) g_tls_sll_count[class_idx]--;
|
||||
|
||||
void* hdr_ptr = NULL;
|
||||
if (tls_sll_pop(class_idx, &hdr_ptr)) {
|
||||
// Initialize header
|
||||
struct hak_alloc_hdr* hdr = (struct hak_alloc_hdr*)hdr_ptr;
|
||||
hdr->pool_type = HAK_POOL_TYPE_TINY;
|
||||
@ -115,11 +113,7 @@ void* hak_tiny_alloc_metadata(size_t size) {
|
||||
#else
|
||||
if (sll_refill_small_from_ss(class_idx, refill_count) > 0) {
|
||||
#endif
|
||||
hdr_ptr = g_tls_sll_head[class_idx];
|
||||
if (hdr_ptr) {
|
||||
g_tls_sll_head[class_idx] = *(void**)hdr_ptr;
|
||||
if (g_tls_sll_count[class_idx] > 0) g_tls_sll_count[class_idx]--;
|
||||
|
||||
if (tls_sll_pop(class_idx, &hdr_ptr)) {
|
||||
struct hak_alloc_hdr* hdr = (struct hak_alloc_hdr*)hdr_ptr;
|
||||
hdr->pool_type = HAK_POOL_TYPE_TINY;
|
||||
hdr->size_class = class_idx;
|
||||
@ -220,9 +214,12 @@ void hak_tiny_free_metadata(void* user_ptr) {
|
||||
}
|
||||
|
||||
// Push HEADER pointer to SLL (not user pointer!)
|
||||
*(void**)hdr = g_tls_sll_head[class_idx];
|
||||
g_tls_sll_head[class_idx] = hdr;
|
||||
g_tls_sll_count[class_idx]++;
|
||||
// Use Box TLS-SLL API (C7-safe)
|
||||
if (!tls_sll_push(class_idx, hdr, UINT32_MAX)) {
|
||||
// C7 rejected or capacity exceeded - use slow path
|
||||
hak_free_at(user_ptr, 0, 0);
|
||||
return;
|
||||
}
|
||||
|
||||
// Done! No owner lookup, no registry, no locks!
|
||||
}
|
||||
|
||||
@ -23,6 +23,7 @@
|
||||
#include "hakmem_tiny_magazine.h"
|
||||
#include "hakmem_tiny_tls_list.h"
|
||||
#include "tiny_box_geometry.h" // Box 3: Geometry & Capacity Calculator
|
||||
#include "box/tls_sll_box.h" // Box TLS-SLL: Safe SLL operations API
|
||||
#include <stdint.h>
|
||||
#include <pthread.h>
|
||||
#include <stdlib.h>
|
||||
@ -147,10 +148,9 @@ static inline int quick_refill_from_sll(int class_idx) {
|
||||
if (room > 2) room = 2;
|
||||
int filled = 0;
|
||||
while (room > 0) {
|
||||
void* head = g_tls_sll_head[class_idx];
|
||||
if (!head) break;
|
||||
g_tls_sll_head[class_idx] = *(void**)head;
|
||||
if (g_tls_sll_count[class_idx] > 0) g_tls_sll_count[class_idx]--;
|
||||
// CRITICAL: Use Box TLS-SLL API to avoid race condition (rbp=0xa0 SEGV)
|
||||
void* head = NULL;
|
||||
if (!tls_sll_pop(class_idx, &head)) break;
|
||||
qs->items[qs->top++] = head;
|
||||
room--; filled++;
|
||||
}
|
||||
@ -209,6 +209,11 @@ __attribute__((noinline)) int sll_refill_small_from_ss(int class_idx, int max_ta
|
||||
#else
|
||||
static inline int sll_refill_small_from_ss(int class_idx, int max_take) {
|
||||
#endif
|
||||
// CRITICAL: C7 (1KB) is headerless - incompatible with TLS SLL refill
|
||||
if (__builtin_expect(class_idx == 7, 0)) {
|
||||
return 0; // C7 uses slow path exclusively
|
||||
}
|
||||
|
||||
if (!g_use_superslab || max_take <= 0) return 0;
|
||||
// ランタイムA/B: P0を有効化している場合はバッチrefillへ委譲
|
||||
do {
|
||||
@ -259,9 +264,12 @@ static inline int sll_refill_small_from_ss(int class_idx, int max_take) {
|
||||
void* p = tiny_block_at_index(base, meta->carved, bs);
|
||||
meta->carved++;
|
||||
meta->used++;
|
||||
*(void**)p = g_tls_sll_head[class_idx];
|
||||
g_tls_sll_head[class_idx] = p;
|
||||
g_tls_sll_count[class_idx]++;
|
||||
// CRITICAL: Use Box TLS-SLL API (C7-safe, no race)
|
||||
if (!tls_sll_push(class_idx, p, sll_cap)) {
|
||||
// SLL full (should not happen, room was checked)
|
||||
meta->used--; meta->carved--; // Rollback
|
||||
break;
|
||||
}
|
||||
ss_active_inc(tls->ss);
|
||||
taken++;
|
||||
continue;
|
||||
@ -271,9 +279,14 @@ static inline int sll_refill_small_from_ss(int class_idx, int max_take) {
|
||||
void* p = meta->freelist;
|
||||
meta->freelist = *(void**)p;
|
||||
meta->used++;
|
||||
*(void**)p = g_tls_sll_head[class_idx];
|
||||
g_tls_sll_head[class_idx] = p;
|
||||
g_tls_sll_count[class_idx]++;
|
||||
// CRITICAL: Use Box TLS-SLL API (C7-safe, no race)
|
||||
if (!tls_sll_push(class_idx, p, sll_cap)) {
|
||||
// SLL full (should not happen, room was checked)
|
||||
*(void**)p = meta->freelist; // Rollback freelist
|
||||
meta->freelist = p;
|
||||
meta->used--;
|
||||
break;
|
||||
}
|
||||
ss_active_inc(tls->ss);
|
||||
taken++;
|
||||
continue;
|
||||
@ -322,9 +335,12 @@ static inline int sll_refill_small_from_ss(int class_idx, int max_take) {
|
||||
continue;
|
||||
}
|
||||
if (!p) break;
|
||||
*(void**)p = g_tls_sll_head[class_idx];
|
||||
g_tls_sll_head[class_idx] = p;
|
||||
g_tls_sll_count[class_idx]++;
|
||||
// CRITICAL: Use Box TLS-SLL API (C7-safe, no race)
|
||||
if (!tls_sll_push(class_idx, p, sll_cap)) {
|
||||
// SLL full (should not happen, room was checked)
|
||||
// Rollback: need to return block to meta (complex, just break)
|
||||
break;
|
||||
}
|
||||
taken++;
|
||||
}
|
||||
return taken;
|
||||
@ -398,11 +414,11 @@ static inline int frontend_refill_fc(int class_idx) {
|
||||
int filled = 0;
|
||||
|
||||
// Step A: First bulk transfer from TLS SLL to FastCache (lock-free, O(1))
|
||||
// CRITICAL: Use Box TLS-SLL API to avoid race condition (rbp=0xa0 SEGV)
|
||||
if (g_tls_sll_enable) {
|
||||
while (need > 0 && g_tls_sll_head[class_idx] != NULL) {
|
||||
void* h = g_tls_sll_head[class_idx];
|
||||
g_tls_sll_head[class_idx] = *(void**)h;
|
||||
if (g_tls_sll_count[class_idx] > 0) g_tls_sll_count[class_idx]--; // underflow prevention
|
||||
while (need > 0) {
|
||||
void* h = NULL;
|
||||
if (!tls_sll_pop(class_idx, &h)) break;
|
||||
fc->items[fc->top++] = h;
|
||||
need--; filled++;
|
||||
if (fc->top >= TINY_FASTCACHE_CAP) break;
|
||||
@ -445,9 +461,11 @@ static inline int bulk_mag_to_sll_if_room(int class_idx, TinyTLSMag* mag, int n)
|
||||
if (take <= 0) return 0;
|
||||
for (int i = 0; i < take; i++) {
|
||||
void* p = mag->items[--mag->top].ptr;
|
||||
*(void**)p = g_tls_sll_head[class_idx];
|
||||
g_tls_sll_head[class_idx] = p;
|
||||
g_tls_sll_count[class_idx]++;
|
||||
if (!tls_sll_push(class_idx, p, cap)) {
|
||||
// No more room; return remaining items to magazine and stop
|
||||
mag->top++; // undo pop
|
||||
break;
|
||||
}
|
||||
}
|
||||
HAK_PATHDBG_INC(g_path_refill_calls, class_idx);
|
||||
return take;
|
||||
@ -480,9 +498,10 @@ static inline void ultra_refill_sll(int class_idx) {
|
||||
hak_tiny_set_used(slab, first);
|
||||
slab->free_count--;
|
||||
void* p0 = (char*)slab->base + ((size_t)first * bs);
|
||||
*(void**)p0 = g_tls_sll_head[class_idx];
|
||||
g_tls_sll_head[class_idx] = p0;
|
||||
g_tls_sll_count[class_idx]++;
|
||||
if (!tls_sll_push(class_idx, p0, (uint32_t)sll_cap)) {
|
||||
// SLL saturated; stop refilling
|
||||
break;
|
||||
}
|
||||
remaining--;
|
||||
// Try to allocate more from the same word to amortize scanning
|
||||
int word_idx = first / 64;
|
||||
@ -495,9 +514,9 @@ static inline void ultra_refill_sll(int class_idx) {
|
||||
hak_tiny_set_used(slab, block_idx);
|
||||
slab->free_count--;
|
||||
void* p = (char*)slab->base + ((size_t)block_idx * bs);
|
||||
*(void**)p = g_tls_sll_head[class_idx];
|
||||
g_tls_sll_head[class_idx] = p;
|
||||
g_tls_sll_count[class_idx]++;
|
||||
if (!tls_sll_push(class_idx, p, (uint32_t)sll_cap)) {
|
||||
break;
|
||||
}
|
||||
remaining--;
|
||||
// Update free_bits for next iteration
|
||||
used = slab->bitmap[word_idx];
|
||||
|
||||
@ -43,16 +43,13 @@ static inline int p0_should_log(void) {
|
||||
}
|
||||
|
||||
static inline int sll_refill_batch_from_ss(int class_idx, int max_take) {
|
||||
// Conservative guard: class7(1KB) uses legacy path by default until fully stabilized.
|
||||
// Opt-in via HAKMEM_TINY_P0_C7_ENABLE=1
|
||||
// CRITICAL: C7 (1KB) is headerless - incompatible with TLS SLL refill
|
||||
// Reason: TLS SLL stores next pointer in first 8 bytes (user data for C7)
|
||||
// Solution: Skip refill for C7, force slow path allocation
|
||||
if (__builtin_expect(class_idx == 7, 0)) {
|
||||
static int c7_en = -1;
|
||||
if (c7_en == -1) {
|
||||
const char* e = getenv("HAKMEM_TINY_P0_C7_ENABLE");
|
||||
c7_en = (e && *e && *e != '0') ? 1 : 0;
|
||||
}
|
||||
if (!c7_en) return 0;
|
||||
return 0; // C7 uses slow path exclusively
|
||||
}
|
||||
|
||||
// Runtime A/B kill switch (defensive). Set HAKMEM_TINY_P0_DISABLE=1 to bypass P0 path.
|
||||
do {
|
||||
static int g_p0_disable = -1;
|
||||
|
||||
@ -6,6 +6,8 @@
|
||||
// - __thread void* g_tls_sll_head[]; __thread uint32_t g_tls_sll_count[];
|
||||
// - tiny_mag_init_if_needed(), g_tls_mags[]
|
||||
|
||||
#include "box/tls_sll_box.h" // Box TLS-SLL API
|
||||
|
||||
static inline void ultra_init_if_needed(int class_idx) {
|
||||
if (!g_ultra_simple || class_idx < 0) return;
|
||||
// nothing to do; zero-initialized
|
||||
@ -34,10 +36,8 @@ static inline int ultra_refill_small(int class_idx) {
|
||||
int took = 0;
|
||||
if (g_tls_sll_enable) {
|
||||
while (room > 0) {
|
||||
void* h = g_tls_sll_head[class_idx];
|
||||
if (!h) break;
|
||||
g_tls_sll_head[class_idx] = *(void**)h;
|
||||
if (g_tls_sll_count[class_idx] > 0) g_tls_sll_count[class_idx]--;
|
||||
void* h = NULL;
|
||||
if (!tls_sll_pop(class_idx, &h)) break;
|
||||
uf->slots[uf->top++] = h; room--; took++;
|
||||
}
|
||||
}
|
||||
|
||||
@ -15,6 +15,7 @@
|
||||
|
||||
// SFC integration
|
||||
#include "tiny_alloc_fast_sfc.inc.h"
|
||||
#include "box/tls_sll_box.h" // Box TLS-SLL API
|
||||
|
||||
// ============================================================================
|
||||
// Phase 6-1.5: Ultra-Simple Allocator (uses existing infrastructure)
|
||||
@ -47,10 +48,8 @@ void* hak_tiny_alloc_ultra_simple(size_t size) {
|
||||
|
||||
// 2. Ultra-fast path: Pop from existing TLS SLL (Phase 6-1 style!)
|
||||
// This is IDENTICAL to Phase 6-1 but uses existing g_tls_sll_head[]
|
||||
void* head = g_tls_sll_head[class_idx];
|
||||
if (__builtin_expect(head != NULL, 1)) {
|
||||
g_tls_sll_head[class_idx] = *(void**)head; // 1-instruction pop!
|
||||
if (g_tls_sll_count[class_idx] > 0) g_tls_sll_count[class_idx]--;
|
||||
void* head = NULL;
|
||||
if (tls_sll_pop(class_idx, &head)) {
|
||||
HAK_RET_ALLOC(class_idx, head);
|
||||
}
|
||||
|
||||
@ -72,10 +71,7 @@ void* hak_tiny_alloc_ultra_simple(size_t size) {
|
||||
#else
|
||||
if (sll_refill_small_from_ss(class_idx, refill_count) > 0) {
|
||||
#endif
|
||||
head = g_tls_sll_head[class_idx];
|
||||
if (head) {
|
||||
g_tls_sll_head[class_idx] = *(void**)head;
|
||||
if (g_tls_sll_count[class_idx] > 0) g_tls_sll_count[class_idx]--;
|
||||
if (tls_sll_pop(class_idx, &head)) {
|
||||
HAK_RET_ALLOC(class_idx, head);
|
||||
}
|
||||
}
|
||||
@ -182,10 +178,12 @@ void hak_tiny_free_ultra_simple(void* ptr) {
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
// Old SLL path (16 slots)
|
||||
*(void**)ptr = g_tls_sll_head[class_idx];
|
||||
g_tls_sll_head[class_idx] = ptr;
|
||||
g_tls_sll_count[class_idx]++;
|
||||
// Old SLL path (16 slots) - Use Box TLS-SLL API
|
||||
if (!tls_sll_push(class_idx, ptr, UINT32_MAX)) {
|
||||
// C7 rejected or capacity exceeded - fallback to slow path
|
||||
hak_tiny_free(ptr);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// Active accounting on free
|
||||
@ -215,10 +213,12 @@ void hak_tiny_free_ultra_simple(void* ptr) {
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
// Old SLL path (16 slots)
|
||||
*(void**)ptr = g_tls_sll_head[class_idx];
|
||||
g_tls_sll_head[class_idx] = ptr;
|
||||
g_tls_sll_count[class_idx]++;
|
||||
// Old SLL path (16 slots) - Use Box TLS-SLL API
|
||||
if (!tls_sll_push(class_idx, ptr, UINT32_MAX)) {
|
||||
// C7 rejected or capacity exceeded - fallback to slow path
|
||||
hak_tiny_free_with_slab(ptr, slab);
|
||||
return;
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
19
core/link_stubs.c
Normal file
19
core/link_stubs.c
Normal file
@ -0,0 +1,19 @@
|
||||
#include <stddef.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
// Weak, no-op stubs to satisfy link in configurations where
|
||||
// optional components are compiled out or gated by flags.
|
||||
// Real implementations (when present) will override these.
|
||||
|
||||
__attribute__((weak)) void hak_tiny_prewarm_tls_cache(void) {}
|
||||
|
||||
__attribute__((weak)) void* pool_alloc(size_t size) {
|
||||
// Fallback to malloc if Pool TLS not linked
|
||||
return malloc(size);
|
||||
}
|
||||
|
||||
__attribute__((weak)) void pool_free(void* ptr) {
|
||||
// Fallback to free if Pool TLS not linked
|
||||
free(ptr);
|
||||
}
|
||||
|
||||
@ -18,6 +18,7 @@
|
||||
#include "tiny_alloc_fast_sfc.inc.h" // Box 5-NEW: SFC Layer
|
||||
#include "tiny_region_id.h" // Phase 7: Header-based class_idx lookup
|
||||
#include "tiny_adaptive_sizing.h" // Phase 2b: Adaptive sizing
|
||||
#include "box/tls_sll_box.h" // Box TLS-SLL: C7-safe push/pop/splice
|
||||
#ifdef HAKMEM_TINY_FRONT_GATE_BOX
|
||||
#include "box/front_gate_box.h"
|
||||
#endif
|
||||
@ -164,6 +165,14 @@ extern int g_sfc_enabled;
|
||||
//
|
||||
// Expected: 3-4 instructions on SFC hit, 6-8 on SLL hit
|
||||
static inline void* tiny_alloc_fast_pop(int class_idx) {
|
||||
// CRITICAL: C7 (1KB) is headerless - delegate to slow path completely
|
||||
// Reason: Fast path uses SLL which stores next pointer in user data area
|
||||
// C7's headerless design is incompatible with fast path assumptions
|
||||
// Solution: Force C7 to use slow path for both alloc and free
|
||||
if (__builtin_expect(class_idx == 7, 0)) {
|
||||
return NULL; // Force slow path
|
||||
}
|
||||
|
||||
#ifdef HAKMEM_TINY_FRONT_GATE_BOX
|
||||
void* out = NULL;
|
||||
if (front_gate_try_pop(class_idx, &out)) {
|
||||
@ -207,46 +216,15 @@ static inline void* tiny_alloc_fast_pop(int class_idx) {
|
||||
// Box Boundary: Layer 1 - TLS SLL freelist の先頭を pop（envで無効化可）
|
||||
extern int g_tls_sll_enable; // set at init via HAKMEM_TINY_TLS_SLL
|
||||
if (__builtin_expect(g_tls_sll_enable, 1)) {
|
||||
void* head = g_tls_sll_head[class_idx];
|
||||
if (__builtin_expect(head != NULL, 1)) {
|
||||
// CORRUPTION DEBUG: Validate TLS SLL head before popping
|
||||
if (__builtin_expect(tiny_refill_failfast_level() >= 2, 0)) {
|
||||
size_t blk = g_tiny_class_sizes[class_idx];
|
||||
// Check alignment (must be multiple of block size)
|
||||
if (((uintptr_t)head % blk) != 0) {
|
||||
fprintf(stderr, "[TLS_SLL_CORRUPT] cls=%d head=%p misaligned (blk=%zu offset=%zu)\n",
|
||||
class_idx, head, blk, (uintptr_t)head % blk);
|
||||
fprintf(stderr, "[TLS_SLL_CORRUPT] TLS freelist head is corrupted!\n");
|
||||
abort();
|
||||
}
|
||||
}
|
||||
|
||||
// Use Box TLS-SLL API (C7-safe pop)
|
||||
// CRITICAL: Pop FIRST, do NOT read g_tls_sll_head directly (race condition!)
|
||||
// Reading head before pop causes stale read → rbp=0xa0 SEGV
|
||||
void* head = NULL;
|
||||
if (tls_sll_pop(class_idx, &head)) {
|
||||
// Front Gate: SLL hit (fast path 3 instructions)
|
||||
extern unsigned long long g_front_sll_hit[];
|
||||
g_front_sll_hit[class_idx]++;
|
||||
|
||||
// CORRUPTION DEBUG: Validate next pointer before updating head
|
||||
void* next = *(void**)head;
|
||||
if (__builtin_expect(tiny_refill_failfast_level() >= 2, 0)) {
|
||||
size_t blk = g_tiny_class_sizes[class_idx];
|
||||
if (next != NULL && ((uintptr_t)next % blk) != 0) {
|
||||
fprintf(stderr, "[ALLOC_POP_CORRUPT] Reading next from head=%p got corrupted next=%p!\n",
|
||||
head, next);
|
||||
fprintf(stderr, "[ALLOC_POP_CORRUPT] cls=%d blk=%zu next_offset=%zu (expected 0)\n",
|
||||
class_idx, blk, (uintptr_t)next % blk);
|
||||
fprintf(stderr, "[ALLOC_POP_CORRUPT] TLS SLL head block was corrupted (use-after-free/double-free)!\n");
|
||||
abort();
|
||||
}
|
||||
fprintf(stderr, "[ALLOC_POP] cls=%d head=%p next=%p\n", class_idx, head, next);
|
||||
}
|
||||
|
||||
g_tls_sll_head[class_idx] = next; // Pop: next = *head
|
||||
|
||||
// Optional: update count (for stats, can be disabled)
|
||||
if (g_tls_sll_count[class_idx] > 0) {
|
||||
g_tls_sll_count[class_idx]--;
|
||||
}
|
||||
|
||||
#if HAKMEM_DEBUG_COUNTERS
|
||||
// Track TLS freelist hits (compile-time gated, zero runtime cost when disabled)
|
||||
g_free_via_tls_sll[class_idx]++;
|
||||
@ -288,12 +266,11 @@ static inline int sfc_refill_from_sll(int class_idx, int target_count) {
|
||||
break; // SFC full, stop
|
||||
}
|
||||
|
||||
// Pop from SLL (Layer 1)
|
||||
void* ptr = g_tls_sll_head[class_idx];
|
||||
if (!ptr) break; // SLL empty
|
||||
|
||||
g_tls_sll_head[class_idx] = *(void**)ptr;
|
||||
g_tls_sll_count[class_idx]--;
|
||||
// Pop from SLL (Layer 1) using Box TLS-SLL API (C7-safe)
|
||||
void* ptr = NULL;
|
||||
if (!tls_sll_pop(class_idx, &ptr)) {
|
||||
break; // SLL empty
|
||||
}
|
||||
|
||||
// Push to SFC (Layer 0)
|
||||
*(void**)ptr = g_sfc_head[class_idx];
|
||||
@ -324,6 +301,13 @@ static inline int sfc_refill_from_sll(int class_idx, int target_count) {
|
||||
// - Smaller count (8-16): better for diverse workloads, faster warmup
|
||||
// - Larger count (64-128): better for homogeneous workloads, fewer refills
|
||||
static inline int tiny_alloc_fast_refill(int class_idx) {
|
||||
// CRITICAL: C7 (1KB) is headerless - skip refill completely, force slow path
|
||||
// Reason: Refill pushes blocks to TLS SLL which stores next pointer in user data
|
||||
// C7's headerless design is incompatible with this mechanism
|
||||
if (__builtin_expect(class_idx == 7, 0)) {
|
||||
return 0; // Skip refill, force slow path allocation
|
||||
}
|
||||
|
||||
// Phase 7 Task 3: Profiling overhead removed in release builds
|
||||
// In release mode, compiler can completely eliminate profiling code
|
||||
#if !HAKMEM_BUILD_RELEASE
|
||||
@ -469,28 +453,28 @@ static inline void* tiny_alloc_fast(size_t size) {
|
||||
ROUTE_BEGIN(class_idx);
|
||||
|
||||
// 2. Fast path: TLS freelist pop (3-4 instructions, 95% hit rate)
|
||||
void* ptr;
|
||||
#if HAKMEM_TINY_AGGRESSIVE_INLINE
|
||||
// Task 2: Use inline macro (save 5-10 cycles, no function call)
|
||||
TINY_ALLOC_FAST_POP_INLINE(class_idx, ptr);
|
||||
#else
|
||||
// Standard: Function call (preserves debugging visibility)
|
||||
ptr = tiny_alloc_fast_pop(class_idx);
|
||||
#endif
|
||||
// CRITICAL: Use Box TLS-SLL API (static inline, same performance as macro but SAFE!)
|
||||
// The old macro had race condition: read head before pop → rbp=0xa0 SEGV
|
||||
void* ptr = NULL;
|
||||
tls_sll_pop(class_idx, &ptr);
|
||||
if (__builtin_expect(ptr != NULL, 1)) {
|
||||
// C7 (1024B, headerless): clear embedded next pointer before returning to user
|
||||
if (__builtin_expect(class_idx == 7, 0)) {
|
||||
*(void**)ptr = NULL;
|
||||
}
|
||||
HAK_RET_ALLOC(class_idx, ptr);
|
||||
}
|
||||
|
||||
// 3. Miss: Refill from backend (Box 3: SuperSlab)
|
||||
int refilled = tiny_alloc_fast_refill(class_idx);
|
||||
if (__builtin_expect(refilled > 0, 1)) {
|
||||
// Refill success → retry pop
|
||||
#if HAKMEM_TINY_AGGRESSIVE_INLINE
|
||||
TINY_ALLOC_FAST_POP_INLINE(class_idx, ptr);
|
||||
#else
|
||||
ptr = tiny_alloc_fast_pop(class_idx);
|
||||
#endif
|
||||
// Refill success → retry pop using safe Box TLS-SLL API
|
||||
ptr = NULL;
|
||||
tls_sll_pop(class_idx, &ptr);
|
||||
if (ptr) {
|
||||
if (__builtin_expect(class_idx == 7, 0)) {
|
||||
*(void**)ptr = NULL;
|
||||
}
|
||||
HAK_RET_ALLOC(class_idx, ptr);
|
||||
}
|
||||
}
|
||||
@ -516,10 +500,16 @@ static inline void tiny_alloc_fast_push(int class_idx, void* ptr) {
|
||||
#ifdef HAKMEM_TINY_FRONT_GATE_BOX
|
||||
front_gate_push_tls(class_idx, ptr);
|
||||
#else
|
||||
// Box Boundary: Push to TLS freelist
|
||||
*(void**)ptr = g_tls_sll_head[class_idx];
|
||||
g_tls_sll_head[class_idx] = ptr;
|
||||
g_tls_sll_count[class_idx]++;
|
||||
// Box Boundary: Push to TLS freelist using Box TLS-SLL API (C7-safe)
|
||||
uint32_t capacity = UINT32_MAX; // Unlimited for helper function
|
||||
if (!tls_sll_push(class_idx, ptr, capacity)) {
|
||||
// C7 rejected or SLL somehow full (should not happen)
|
||||
// In release builds, this is a no-op (caller expects success)
|
||||
#if !HAKMEM_BUILD_RELEASE
|
||||
fprintf(stderr, "[WARN] tls_sll_push failed in tiny_alloc_fast_push cls=%d ptr=%p\n",
|
||||
class_idx, ptr);
|
||||
#endif
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
@ -55,6 +55,9 @@ extern __thread uint32_t g_tls_sll_count[TINY_NUM_CLASSES];
|
||||
g_tls_sll_count[(class_idx)]--; \
|
||||
} \
|
||||
(ptr_out) = _head; \
|
||||
if (__builtin_expect((class_idx) == 7, 0)) { \
|
||||
*(void**)(ptr_out) = NULL; \
|
||||
} \
|
||||
} \
|
||||
} else { \
|
||||
(ptr_out) = NULL; \
|
||||
|
||||
@ -18,6 +18,7 @@
|
||||
#include "tiny_region_id.h"
|
||||
#include "hakmem_build_flags.h"
|
||||
#include "hakmem_tiny_config.h" // For TINY_TLS_MAG_CAP, TINY_NUM_CLASSES
|
||||
#include "box/tls_sll_box.h" // Box TLS-SLL API
|
||||
|
||||
// Phase 7: Header-based ultra-fast free
|
||||
#if HAKMEM_TINY_HEADER_CLASSIDX
|
||||
@ -50,6 +51,17 @@ extern uint32_t sll_cap_for_class(int class_idx, uint32_t mag_cap);
|
||||
static inline int hak_tiny_free_fast_v2(void* ptr) {
|
||||
if (__builtin_expect(!ptr, 0)) return 0;
|
||||
|
||||
// CRITICAL: C7 (1KB) is headerless and CANNOT use fast path
|
||||
// Reading ptr-1 for C7 causes SIGBUS (accesses previous allocation or unmapped page)
|
||||
// Solution: Check for 1KB alignment and delegate to slow path
|
||||
// Note: This heuristic has ~0.1% false positive rate (other allocations at 1KB boundaries)
|
||||
// but is necessary for C7 safety. Slow path handles all cases correctly.
|
||||
if (__builtin_expect(((uintptr_t)ptr & 0x3FF) == 0, 0)) {
|
||||
// Pointer is 1KB-aligned → likely C7 or page boundary allocation
|
||||
// Use slow path for safety (slow path has proper C7 handling)
|
||||
return 0;
|
||||
}
|
||||
|
||||
// CRITICAL: Check if header is accessible
|
||||
void* header_addr = (char*)ptr - 1;
|
||||
|
||||
@ -116,9 +128,12 @@ static inline int hak_tiny_free_fast_v2(void* ptr) {
|
||||
// Normal classes have 1-byte header - base is ptr-1
|
||||
base = (char*)ptr - 1;
|
||||
}
|
||||
*(void**)base = g_tls_sll_head[class_idx];
|
||||
g_tls_sll_head[class_idx] = base;
|
||||
g_tls_sll_count[class_idx]++;
|
||||
|
||||
// Use Box TLS-SLL API (C7-safe)
|
||||
if (!tls_sll_push(class_idx, base, UINT32_MAX)) {
|
||||
// C7 rejected or capacity exceeded - route to slow path
|
||||
return 0;
|
||||
}
|
||||
|
||||
return 1; // Success - handled in fast path
|
||||
}
|
||||
|
||||
@ -19,7 +19,8 @@
|
||||
if (g_quick_enable && class_idx <= 4) {
|
||||
TinyQuickSlot* qs = &g_tls_quick[class_idx];
|
||||
if (__builtin_expect(qs->top < QUICK_CAP, 1)) {
|
||||
qs->items[qs->top++] = ptr;
|
||||
void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1);
|
||||
qs->items[qs->top++] = base;
|
||||
HAK_STAT_FREE(class_idx);
|
||||
return;
|
||||
}
|
||||
@ -28,9 +29,9 @@
|
||||
|
||||
// Fast path: TLS SLL push for hottest classes
|
||||
if (!g_tls_list_enable && g_tls_sll_enable && g_tls_sll_count[class_idx] < sll_cap_for_class(class_idx, (uint32_t)cap)) {
|
||||
*(void**)ptr = g_tls_sll_head[class_idx];
|
||||
g_tls_sll_head[class_idx] = ptr;
|
||||
g_tls_sll_count[class_idx]++;
|
||||
void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1);
|
||||
uint32_t sll_cap = sll_cap_for_class(class_idx, (uint32_t)cap);
|
||||
if (tls_sll_push(class_idx, base, sll_cap)) {
|
||||
// BUGFIX: Decrement used counter (was missing, causing Fail-Fast on next free)
|
||||
meta->used--;
|
||||
// Active → Inactive: count down immediately (TLS保管中は"使用中"ではない)
|
||||
@ -40,6 +41,7 @@
|
||||
HAK_STAT_FREE(class_idx);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// Next: Magazine push（必要ならmag→SLLへバルク転送で空きを作る）
|
||||
// Hysteresis: allow slight overfill before deciding to spill under lock
|
||||
@ -47,7 +49,8 @@
|
||||
(void)bulk_mag_to_sll_if_room(class_idx, mag, cap / 2);
|
||||
}
|
||||
if (mag->top < cap + g_spill_hyst) {
|
||||
mag->items[mag->top].ptr = ptr;
|
||||
void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1);
|
||||
mag->items[mag->top].ptr = base;
|
||||
#if HAKMEM_TINY_MAG_OWNER
|
||||
mag->items[mag->top].owner = NULL; // SuperSlab owner not a TinySlab; leave NULL
|
||||
#endif
|
||||
@ -71,7 +74,7 @@
|
||||
int limit = g_bg_spill_max_batch;
|
||||
if (limit > cap/2) limit = cap/2;
|
||||
if (limit > 32) limit = 32; // keep free-path bounded
|
||||
void* head = ptr;
|
||||
void* head = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1);
|
||||
*(void**)head = NULL;
|
||||
void* tail = head; // current tail
|
||||
int taken = 1;
|
||||
@ -138,7 +141,8 @@
|
||||
// Finally, try FastCache push first (≤128B) — compile-out if HAKMEM_TINY_NO_FRONT_CACHE
|
||||
#if !defined(HAKMEM_TINY_NO_FRONT_CACHE)
|
||||
if (g_fastcache_enable && class_idx <= 4) {
|
||||
if (fastcache_push(class_idx, ptr)) {
|
||||
void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1);
|
||||
if (fastcache_push(class_idx, base)) {
|
||||
HAK_TP1(front_push, class_idx);
|
||||
HAK_STAT_FREE(class_idx);
|
||||
return;
|
||||
@ -147,11 +151,19 @@
|
||||
#endif
|
||||
// Then TLS SLL if room, else magazine
|
||||
if (g_tls_sll_enable && g_tls_sll_count[class_idx] < sll_cap_for_class(class_idx, (uint32_t)mag->cap)) {
|
||||
*(void**)ptr = g_tls_sll_head[class_idx];
|
||||
g_tls_sll_head[class_idx] = ptr;
|
||||
g_tls_sll_count[class_idx]++;
|
||||
uint32_t sll_cap2 = sll_cap_for_class(class_idx, (uint32_t)mag->cap);
|
||||
void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1);
|
||||
if (!tls_sll_push(class_idx, base, sll_cap2)) {
|
||||
// fallback to magazine
|
||||
mag->items[mag->top].ptr = base;
|
||||
#if HAKMEM_TINY_MAG_OWNER
|
||||
mag->items[mag->top].owner = slab;
|
||||
#endif
|
||||
mag->top++;
|
||||
}
|
||||
} else {
|
||||
mag->items[mag->top].ptr = ptr;
|
||||
void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1);
|
||||
mag->items[mag->top].ptr = base;
|
||||
#if HAKMEM_TINY_MAG_OWNER
|
||||
mag->items[mag->top].owner = slab;
|
||||
#endif
|
||||
@ -180,14 +192,16 @@
|
||||
}
|
||||
// TinyHotMag front push（8/16/32B, A/B）
|
||||
if (__builtin_expect(g_hotmag_enable && class_idx <= 2, 1)) {
|
||||
if (hotmag_push(class_idx, ptr)) {
|
||||
void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1);
|
||||
if (hotmag_push(class_idx, base)) {
|
||||
HAK_STAT_FREE(class_idx);
|
||||
return;
|
||||
}
|
||||
}
|
||||
if (tls->count < tls->cap) {
|
||||
tiny_tls_list_guard_push(class_idx, tls, ptr);
|
||||
tls_list_push(tls, ptr);
|
||||
void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1);
|
||||
tiny_tls_list_guard_push(class_idx, tls, base);
|
||||
tls_list_push(tls, base);
|
||||
HAK_STAT_FREE(class_idx);
|
||||
return;
|
||||
}
|
||||
@ -195,8 +209,11 @@
|
||||
if (__builtin_expect(seq != g_tls_param_seen[class_idx], 0)) {
|
||||
tiny_tls_refresh_params(class_idx, tls);
|
||||
}
|
||||
tiny_tls_list_guard_push(class_idx, tls, ptr);
|
||||
tls_list_push(tls, ptr);
|
||||
{
|
||||
void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1);
|
||||
tiny_tls_list_guard_push(class_idx, tls, base);
|
||||
tls_list_push(tls, base);
|
||||
}
|
||||
if (tls_list_should_spill(tls)) {
|
||||
tls_list_spill_excess(class_idx, tls);
|
||||
}
|
||||
@ -219,24 +236,27 @@
|
||||
if (!g_tls_list_enable && g_tls_sll_enable && class_idx <= 5) {
|
||||
uint32_t sll_cap = sll_cap_for_class(class_idx, (uint32_t)cap);
|
||||
if (g_tls_sll_count[class_idx] < sll_cap) {
|
||||
*(void**)ptr = g_tls_sll_head[class_idx];
|
||||
g_tls_sll_head[class_idx] = ptr;
|
||||
g_tls_sll_count[class_idx]++;
|
||||
void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1);
|
||||
if (tls_sll_push(class_idx, base, sll_cap)) {
|
||||
HAK_STAT_FREE(class_idx);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
// Next: if magazine has room, push immediately and return（満杯ならmag→SLLへバルク）
|
||||
if (mag->top >= cap) {
|
||||
(void)bulk_mag_to_sll_if_room(class_idx, mag, cap / 2);
|
||||
}
|
||||
// Remote-drain can be handled opportunistically on future calls.
|
||||
if (mag->top < cap) {
|
||||
mag->items[mag->top].ptr = ptr;
|
||||
{
|
||||
void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1);
|
||||
mag->items[mag->top].ptr = base;
|
||||
#if HAKMEM_TINY_MAG_OWNER
|
||||
mag->items[mag->top].owner = slab;
|
||||
#endif
|
||||
mag->top++;
|
||||
}
|
||||
|
||||
#if HAKMEM_DEBUG_COUNTERS
|
||||
g_magazine_push_count++; // Phase 7.6: Track pushes
|
||||
@ -358,23 +378,33 @@
|
||||
if (g_quick_enable && class_idx <= 4) {
|
||||
TinyQuickSlot* qs = &g_tls_quick[class_idx];
|
||||
if (__builtin_expect(qs->top < QUICK_CAP, 1)) {
|
||||
qs->items[qs->top++] = ptr;
|
||||
void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1);
|
||||
qs->items[qs->top++] = base;
|
||||
} else if (g_tls_sll_enable) {
|
||||
uint32_t sll_cap2 = sll_cap_for_class(class_idx, (uint32_t)mag->cap);
|
||||
if (g_tls_sll_count[class_idx] < sll_cap2) {
|
||||
*(void**)ptr = g_tls_sll_head[class_idx];
|
||||
g_tls_sll_head[class_idx] = ptr;
|
||||
g_tls_sll_count[class_idx]++;
|
||||
} else if (!tiny_optional_push(class_idx, ptr)) {
|
||||
mag->items[mag->top].ptr = ptr;
|
||||
void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1);
|
||||
if (!tls_sll_push(class_idx, base, sll_cap2)) {
|
||||
if (!tiny_optional_push(class_idx, base)) {
|
||||
mag->items[mag->top].ptr = base;
|
||||
#if HAKMEM_TINY_MAG_OWNER
|
||||
mag->items[mag->top].owner = slab;
|
||||
#endif
|
||||
mag->top++;
|
||||
}
|
||||
}
|
||||
} else if (!tiny_optional_push(class_idx, (class_idx == 7 ? ptr : (void*)((uint8_t*)ptr - 1)))) {
|
||||
void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1);
|
||||
mag->items[mag->top].ptr = base;
|
||||
#if HAKMEM_TINY_MAG_OWNER
|
||||
mag->items[mag->top].owner = slab;
|
||||
#endif
|
||||
mag->top++;
|
||||
}
|
||||
} else {
|
||||
if (!tiny_optional_push(class_idx, ptr)) {
|
||||
mag->items[mag->top].ptr = ptr;
|
||||
void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1);
|
||||
if (!tiny_optional_push(class_idx, base)) {
|
||||
mag->items[mag->top].ptr = base;
|
||||
#if HAKMEM_TINY_MAG_OWNER
|
||||
mag->items[mag->top].owner = slab;
|
||||
#endif
|
||||
@ -387,19 +417,28 @@
|
||||
if (g_tls_sll_enable && class_idx <= 5) {
|
||||
uint32_t sll_cap2 = sll_cap_for_class(class_idx, (uint32_t)mag->cap);
|
||||
if (g_tls_sll_count[class_idx] < sll_cap2) {
|
||||
*(void**)ptr = g_tls_sll_head[class_idx];
|
||||
g_tls_sll_head[class_idx] = ptr;
|
||||
g_tls_sll_count[class_idx]++;
|
||||
} else if (!tiny_optional_push(class_idx, ptr)) {
|
||||
mag->items[mag->top].ptr = ptr;
|
||||
void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1);
|
||||
if (!tls_sll_push(class_idx, base, sll_cap2)) {
|
||||
if (!tiny_optional_push(class_idx, base)) {
|
||||
mag->items[mag->top].ptr = base;
|
||||
#if HAKMEM_TINY_MAG_OWNER
|
||||
mag->items[mag->top].owner = slab;
|
||||
#endif
|
||||
mag->top++;
|
||||
}
|
||||
}
|
||||
} else if (!tiny_optional_push(class_idx, (class_idx == 7 ? ptr : (void*)((uint8_t*)ptr - 1)))) {
|
||||
void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1);
|
||||
mag->items[mag->top].ptr = base;
|
||||
#if HAKMEM_TINY_MAG_OWNER
|
||||
mag->items[mag->top].owner = slab;
|
||||
#endif
|
||||
mag->top++;
|
||||
}
|
||||
} else {
|
||||
if (!tiny_optional_push(class_idx, ptr)) {
|
||||
mag->items[mag->top].ptr = ptr;
|
||||
void* base = (class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1);
|
||||
if (!tiny_optional_push(class_idx, base)) {
|
||||
mag->items[mag->top].ptr = base;
|
||||
#if HAKMEM_TINY_MAG_OWNER
|
||||
mag->items[mag->top].owner = slab;
|
||||
#endif
|
||||
@ -415,6 +454,7 @@
|
||||
HAK_STAT_FREE(class_idx); // Phase 3
|
||||
return;
|
||||
} else {
|
||||
tiny_remote_push(slab, ptr);
|
||||
void* base = (slab && slab->class_idx == 7) ? ptr : (void*)((uint8_t*)ptr - 1);
|
||||
tiny_remote_push(slab, base);
|
||||
}
|
||||
}
|
||||
|
||||
@ -52,7 +52,10 @@ static inline void trc_push_front(TinyRefillChain* c, void* node) {
|
||||
// Forward declaration of guard function
|
||||
static inline int trc_refill_guard_enabled(void);
|
||||
|
||||
// Splice local chain into TLS SLL (single meta write)
|
||||
// Forward declare Box TLS-SLL API
|
||||
#include "box/tls_sll_box.h"
|
||||
|
||||
// Splice local chain into TLS SLL using Box TLS-SLL API (C7-safe)
|
||||
static inline void trc_splice_to_sll(int class_idx, TinyRefillChain* c,
|
||||
void** sll_head, uint32_t* sll_count) {
|
||||
if (!c || c->head == NULL) return;
|
||||
@ -65,11 +68,20 @@ static inline void trc_splice_to_sll(int class_idx, TinyRefillChain* c,
|
||||
class_idx, c->head, c->tail, c->count);
|
||||
}
|
||||
|
||||
if (c->tail) {
|
||||
*(void**)c->tail = *sll_head;
|
||||
// CRITICAL: Use Box TLS-SLL API for splice (C7-safe, no race)
|
||||
// Note: tls_sll_splice() requires capacity parameter (use large value for refill)
|
||||
uint32_t moved = tls_sll_splice(class_idx, c->head, c->count, 4096);
|
||||
|
||||
// Update sll_count if provided (Box API already updated g_tls_sll_count internally)
|
||||
// Note: sll_count parameter is typically &g_tls_sll_count[class_idx], already updated
|
||||
(void)sll_count; // Suppress unused warning
|
||||
(void)sll_head; // Suppress unused warning
|
||||
|
||||
// If splice was partial, warn (should not happen in refill path)
|
||||
if (__builtin_expect(moved < c->count, 0)) {
|
||||
fprintf(stderr, "[SPLICE_WARNING] Only moved %u/%u blocks (SLL capacity limit)\n",
|
||||
moved, c->count);
|
||||
}
|
||||
*sll_head = c->head;
|
||||
if (sll_count) *sll_count += c->count;
|
||||
}
|
||||
|
||||
static inline int trc_refill_guard_enabled(void) {
|
||||
|
||||
6
hakmem.d
6
hakmem.d
@ -23,7 +23,8 @@ hakmem.o: core/hakmem.c core/hakmem.h core/hakmem_build_flags.h \
|
||||
core/box/hak_free_api.inc.h core/hakmem_tiny_superslab.h \
|
||||
core/box/../tiny_free_fast_v2.inc.h core/box/../tiny_region_id.h \
|
||||
core/box/../hakmem_build_flags.h core/box/../hakmem_tiny_config.h \
|
||||
core/box/hak_wrappers.inc.h
|
||||
core/box/../box/tls_sll_box.h core/box/../box/../hakmem_tiny_config.h \
|
||||
core/box/front_gate_classifier.h core/box/hak_wrappers.inc.h
|
||||
core/hakmem.h:
|
||||
core/hakmem_build_flags.h:
|
||||
core/hakmem_config.h:
|
||||
@ -80,4 +81,7 @@ core/box/../tiny_free_fast_v2.inc.h:
|
||||
core/box/../tiny_region_id.h:
|
||||
core/box/../hakmem_build_flags.h:
|
||||
core/box/../hakmem_tiny_config.h:
|
||||
core/box/../box/tls_sll_box.h:
|
||||
core/box/../box/../hakmem_tiny_config.h:
|
||||
core/box/front_gate_classifier.h:
|
||||
core/box/hak_wrappers.inc.h:
|
||||
|
||||
Reference in New Issue
Block a user