52386401b3
Major Features: - Debug counter infrastructure for Refill Stage tracking - Free Pipeline counters (ss_local, ss_remote, tls_sll) - Diagnostic counters for early return analysis - Unified larson.sh benchmark runner with profiles - Phase 6-3 regression analysis documentation Bug Fixes: - Fix SuperSlab disabled by default (HAKMEM_TINY_USE_SUPERSLAB) - Fix profile variable naming consistency - Add .gitignore patterns for large files Performance: - Phase 6-3: 4.79 M ops/s (has OOM risk) - With SuperSlab: 3.13 M ops/s (+19% improvement) This is a clean repository without large log files. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
64 lines
2.4 KiB
C
64 lines
2.4 KiB
C
// hakmem_tiny_assist.inc.h
|
|
// Tiny: helper routines to assist targeted remote-drain
|
|
// Keep inline to avoid call overhead on hot slow-paths.
|
|
|
|
#ifndef HAKMEM_TINY_ASSIST_INC_H
|
|
#define HAKMEM_TINY_ASSIST_INC_H
|
|
|
|
#include <stdatomic.h>
|
|
#include "hakmem_tiny_superslab.h"
|
|
#include "hakmem_tiny_ss_target.h"
|
|
#include "hakmem_tiny_drain_ema.inc.h"
|
|
|
|
static inline uint16_t tiny_assist_drain_owned(int class_idx, int max_items) {
|
|
int drained_sets = 0;
|
|
while (drained_sets < max_items) {
|
|
SuperSlab* t = ss_target_pop(class_idx);
|
|
if (!t) break;
|
|
uint32_t mytid = tiny_self_u32();
|
|
for (int i = 0; i < SLABS_PER_SUPERSLAB; i++) {
|
|
TinySlabMeta* m = &t->slabs[i];
|
|
if (m->owner_tid != mytid) continue;
|
|
TinySlabPrefix* pref = tiny_slab_prefix(t, i);
|
|
_Atomic(uintptr_t)* rhead = (_Atomic(uintptr_t)*)&pref->reserved[0];
|
|
_Atomic(uint32_t)* rcount = (_Atomic(uint32_t)*)&pref->reserved[1];
|
|
uint32_t pending = atomic_load_explicit(rcount, memory_order_relaxed);
|
|
if (pending == 0) continue;
|
|
uintptr_t chain = atomic_exchange_explicit(rhead, 0, memory_order_acquire);
|
|
uint32_t cnt = atomic_exchange_explicit(rcount, 0, memory_order_relaxed);
|
|
while (chain && cnt > 0) {
|
|
uintptr_t next = *(uintptr_t*)chain;
|
|
*(void**)(void*)chain = m->freelist;
|
|
m->freelist = (void*)chain;
|
|
if (m->used > 0) m->used--;
|
|
ss_active_dec_one(t);
|
|
chain = next;
|
|
cnt--;
|
|
}
|
|
}
|
|
drained_sets++;
|
|
}
|
|
return (uint16_t)drained_sets;
|
|
}
|
|
|
|
// Auto-sized assist based on EMA
|
|
static inline void tiny_assist_drain_auto(int class_idx) {
|
|
uint16_t want = tiny_drain_target_from_ema(class_idx);
|
|
uint16_t got = tiny_assist_drain_owned(class_idx, want);
|
|
tiny_drain_ema_update(class_idx, got);
|
|
}
|
|
|
|
// Periodic assist from free-path (very lightweight)
|
|
static __thread uint32_t g_tls_free_tick[TINY_NUM_CLASSES];
|
|
static inline void tiny_assist_maybe_drain_on_free(int class_idx) {
|
|
uint32_t k = ++g_tls_free_tick[class_idx];
|
|
if ((k & 0x3Fu) == 0u) { // every 64 frees
|
|
uint16_t want = tiny_drain_target_from_ema(class_idx);
|
|
if (want > 8u) want = 8u; // keep it small on free path
|
|
uint16_t got = tiny_assist_drain_owned(class_idx, want);
|
|
tiny_drain_ema_update(class_idx, got);
|
|
}
|
|
}
|
|
|
|
#endif // HAKMEM_TINY_ASSIST_INC_H
|