Debug Counters Implementation - Clean History
Major Features: - Debug counter infrastructure for Refill Stage tracking - Free Pipeline counters (ss_local, ss_remote, tls_sll) - Diagnostic counters for early return analysis - Unified larson.sh benchmark runner with profiles - Phase 6-3 regression analysis documentation Bug Fixes: - Fix SuperSlab disabled by default (HAKMEM_TINY_USE_SUPERSLAB) - Fix profile variable naming consistency - Add .gitignore patterns for large files Performance: - Phase 6-3: 4.79 M ops/s (has OOM risk) - With SuperSlab: 3.13 M ops/s (+19% improvement) This is a clean repository without large log files. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Moe Charm (CI)
223
core/tiny_refill.h
Normal file
223
core/tiny_refill.h
Normal file
@ -0,0 +1,223 @@
|
||||
// tiny_refill.h - Refill Boundary box (inline helpers)
|
||||
#pragma once
|
||||
#include <stdatomic.h>
|
||||
#include "hakmem_tiny_superslab.h"
|
||||
#include "slab_handle.h"
|
||||
#include "tiny_sticky.h"
|
||||
#include "tiny_mailbox.h"
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
// External helpers from main TU
|
||||
static inline uint32_t tiny_self_u32(void);
|
||||
static inline void tiny_tls_bind_slab(TinyTLSSlab* tls, SuperSlab* ss, int slab_idx);
|
||||
|
||||
// Forward decls in main TU
|
||||
static inline uintptr_t hot_slot_pop(int class_idx);
|
||||
static inline uintptr_t bench_pub_pop(int class_idx);
|
||||
static inline SuperSlab* slab_entry_ss(uintptr_t ent);
|
||||
static inline int slab_entry_idx(uintptr_t ent);
|
||||
|
||||
// Registry scan window (ENV: HAKMEM_TINY_REG_SCAN_MAX, default 256)
|
||||
static inline int tiny_reg_scan_max(void) {
|
||||
static int v = -1;
|
||||
if (__builtin_expect(v == -1, 0)) {
|
||||
const char* s = getenv("HAKMEM_TINY_REG_SCAN_MAX");
|
||||
int defv = 256; // conservative default
|
||||
if (s && *s) {
|
||||
int parsed = atoi(s);
|
||||
v = (parsed > 0) ? parsed : defv;
|
||||
} else {
|
||||
v = defv;
|
||||
}
|
||||
}
|
||||
return v;
|
||||
}
|
||||
|
||||
// Mid-size simple refill (ENV: HAKMEM_TINY_MID_REFILL_SIMPLE)
|
||||
static inline int tiny_mid_refill_simple_enabled(void) {
|
||||
static int v = -1;
|
||||
if (__builtin_expect(v == -1, 0)) {
|
||||
const char* s = getenv("HAKMEM_TINY_MID_REFILL_SIMPLE");
|
||||
v = (s && *s && *s != '0') ? 1 : 0;
|
||||
}
|
||||
return v;
|
||||
}
|
||||
|
||||
// Try a quick adopt from sticky/hot/bench/mailbox (single pass)
|
||||
static inline SuperSlab* tiny_refill_try_fast(int class_idx, TinyTLSSlab* tls) {
|
||||
// One-shot entry trace (env: HAKMEM_TINY_RF_TRACE)
|
||||
do {
|
||||
static int en = -1; static _Atomic int printed[8];
|
||||
if (__builtin_expect(en == -1, 0)) {
|
||||
const char* e = getenv("HAKMEM_TINY_RF_TRACE");
|
||||
en = (e && atoi(e) != 0) ? 1 : 0;
|
||||
}
|
||||
if (en) {
|
||||
int expected = 0;
|
||||
(void)atomic_compare_exchange_strong(&printed[class_idx], &expected, 1);
|
||||
if (expected == 0) {
|
||||
fprintf(stderr, "[RFTRACE] fast-refill enter class=%d\n", class_idx);
|
||||
}
|
||||
}
|
||||
} while (0);
|
||||
// For hot tiny classes (0..3), try mailbox first to avoid deeper scans
|
||||
if (class_idx <= 3) {
|
||||
uint32_t self_tid = tiny_self_u32();
|
||||
uintptr_t mail = tiny_mailbox_fetch(class_idx);
|
||||
if (mail) {
|
||||
SuperSlab* mss = slab_entry_ss(mail);
|
||||
int midx = slab_entry_idx(mail);
|
||||
SlabHandle h = slab_try_acquire(mss, midx, self_tid);
|
||||
if (slab_is_valid(&h)) {
|
||||
if (slab_remote_pending(&h)) {
|
||||
slab_drain_remote_full(&h);
|
||||
slab_release(&h);
|
||||
} else if (slab_freelist(&h)) {
|
||||
tiny_tls_bind_slab(tls, h.ss, h.slab_idx);
|
||||
tiny_sticky_save(class_idx, h.ss, h.slab_idx);
|
||||
return h.ss;
|
||||
} else {
|
||||
slab_release(&h);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Sticky ring (Box: SlabHandle)
|
||||
uint32_t self_tid = tiny_self_u32();
|
||||
for (int r = 0; r < TINY_STICKY_RING; r++) {
|
||||
SuperSlab* last_ss = g_tls_sticky_ss[class_idx][r];
|
||||
if (!(last_ss && last_ss->magic == SUPERSLAB_MAGIC)) { tiny_sticky_clear(class_idx, r); continue; }
|
||||
int li = g_tls_sticky_idx[class_idx][r];
|
||||
int cap = ss_slabs_capacity(last_ss);
|
||||
if (li < 0 || li >= cap) { tiny_sticky_clear(class_idx, r); continue; }
|
||||
|
||||
// Box: Try to acquire ownership
|
||||
SlabHandle h = slab_try_acquire(last_ss, li, self_tid);
|
||||
if (slab_is_valid(&h)) {
|
||||
if (slab_remote_pending(&h)) {
|
||||
slab_drain_remote_full(&h);
|
||||
if (__builtin_expect(g_debug_remote_guard, 0)) {
|
||||
uintptr_t head = atomic_load_explicit(&h.ss->remote_heads[h.slab_idx], memory_order_relaxed);
|
||||
tiny_remote_watch_note("sticky_remote_pending",
|
||||
h.ss,
|
||||
h.slab_idx,
|
||||
(void*)head,
|
||||
0xA250u,
|
||||
self_tid,
|
||||
0);
|
||||
}
|
||||
slab_release(&h);
|
||||
} else if (slab_freelist(&h)) {
|
||||
tiny_tls_bind_slab(tls, h.ss, h.slab_idx);
|
||||
return h.ss;
|
||||
} else {
|
||||
slab_release(&h);
|
||||
}
|
||||
}
|
||||
|
||||
int has_remote = (atomic_load_explicit(&last_ss->remote_heads[li], memory_order_acquire) != 0);
|
||||
if (!has_remote) tiny_sticky_clear(class_idx, r);
|
||||
}
|
||||
// Hot slot
|
||||
{
|
||||
uintptr_t hs = hot_slot_pop(class_idx);
|
||||
if (hs) {
|
||||
SuperSlab* hss = slab_entry_ss(hs);
|
||||
int hidx = slab_entry_idx(hs);
|
||||
|
||||
// Box: Try to acquire ownership
|
||||
SlabHandle h = slab_try_acquire(hss, hidx, self_tid);
|
||||
if (slab_is_valid(&h)) {
|
||||
if (slab_remote_pending(&h)) {
|
||||
slab_drain_remote_full(&h);
|
||||
if (__builtin_expect(g_debug_remote_guard, 0)) {
|
||||
uintptr_t head = atomic_load_explicit(&h.ss->remote_heads[h.slab_idx], memory_order_relaxed);
|
||||
tiny_remote_watch_note("hot_remote_pending",
|
||||
h.ss,
|
||||
h.slab_idx,
|
||||
(void*)head,
|
||||
0xA251u,
|
||||
self_tid,
|
||||
0);
|
||||
}
|
||||
slab_release(&h);
|
||||
} else if (slab_freelist(&h)) {
|
||||
tiny_tls_bind_slab(tls, h.ss, h.slab_idx);
|
||||
tiny_sticky_save(class_idx, h.ss, h.slab_idx);
|
||||
return h.ss;
|
||||
} else {
|
||||
slab_release(&h);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// Bench
|
||||
{
|
||||
uintptr_t entb = bench_pub_pop(class_idx);
|
||||
if (entb) {
|
||||
SuperSlab* bss = slab_entry_ss(entb);
|
||||
int bidx = slab_entry_idx(entb);
|
||||
|
||||
// Box: Try to acquire ownership
|
||||
SlabHandle h = slab_try_acquire(bss, bidx, self_tid);
|
||||
if (slab_is_valid(&h)) {
|
||||
if (slab_remote_pending(&h)) {
|
||||
slab_drain_remote_full(&h);
|
||||
if (__builtin_expect(g_debug_remote_guard, 0)) {
|
||||
uintptr_t head = atomic_load_explicit(&h.ss->remote_heads[h.slab_idx], memory_order_relaxed);
|
||||
tiny_remote_watch_note("bench_remote_pending",
|
||||
h.ss,
|
||||
h.slab_idx,
|
||||
(void*)head,
|
||||
0xA252u,
|
||||
self_tid,
|
||||
0);
|
||||
}
|
||||
slab_release(&h);
|
||||
} else if (slab_freelist(&h)) {
|
||||
tiny_tls_bind_slab(tls, h.ss, h.slab_idx);
|
||||
tiny_sticky_save(class_idx, h.ss, h.slab_idx);
|
||||
return h.ss;
|
||||
} else {
|
||||
slab_release(&h);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// Mailbox (for non-hot classes)
|
||||
if (class_idx > 3) {
|
||||
uintptr_t mail = tiny_mailbox_fetch(class_idx);
|
||||
if (mail) {
|
||||
SuperSlab* mss = slab_entry_ss(mail);
|
||||
int midx = slab_entry_idx(mail);
|
||||
|
||||
// Box: Try to acquire ownership
|
||||
SlabHandle h = slab_try_acquire(mss, midx, self_tid);
|
||||
if (slab_is_valid(&h)) {
|
||||
if (slab_remote_pending(&h)) {
|
||||
slab_drain_remote_full(&h);
|
||||
if (__builtin_expect(g_debug_remote_guard, 0)) {
|
||||
uintptr_t head = atomic_load_explicit(&h.ss->remote_heads[h.slab_idx], memory_order_relaxed);
|
||||
tiny_remote_watch_note("mailbox_remote_pending",
|
||||
h.ss,
|
||||
h.slab_idx,
|
||||
(void*)head,
|
||||
0xA253u,
|
||||
self_tid,
|
||||
0);
|
||||
}
|
||||
slab_release(&h);
|
||||
} else if (slab_freelist(&h)) {
|
||||
tiny_tls_bind_slab(tls, h.ss, h.slab_idx);
|
||||
tiny_sticky_save(class_idx, h.ss, h.slab_idx);
|
||||
return h.ss;
|
||||
} else {
|
||||
slab_release(&h);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
Reference in New Issue
Block a user