// pool_mid_tc.inc.h — Box: Mid Transfer Cache (per-thread inbox) #ifndef POOL_MID_TC_INC_H #define POOL_MID_TC_INC_H typedef struct MidTC { atomic_uintptr_t inbox[POOL_NUM_CLASSES]; } MidTC; #define MID_TC_BUCKETS 1024 typedef struct MidTCEntry { uint64_t tid; MidTC* tc; struct MidTCEntry* next; } MidTCEntry; static pthread_mutex_t g_mid_tc_mu[MID_TC_BUCKETS]; static MidTCEntry* g_mid_tc_head[MID_TC_BUCKETS]; static __thread MidTC* t_mid_tc = NULL; static int g_tc_enabled = 1; // env: HAKMEM_TC_ENABLE (default 1) static int g_tc_drain_unbounded = 1; // env: HAKMEM_TC_UNBOUNDED (default 1) static int g_tc_drain_max = 0; // env: HAKMEM_TC_DRAIN_MAX (0=unbounded) static int g_tc_drain_trigger = 2; // env: HAKMEM_TC_DRAIN_TRIGGER (ring->top < trigger) static inline uint32_t mid_tc_hash(uint64_t tid) { tid ^= tid >> 33; tid *= 0xff51afd7ed558ccdULL; tid ^= tid >> 33; tid *= 0xc4ceb9fe1a85ec53ULL; tid ^= tid >> 33; return (uint32_t)(tid & (MID_TC_BUCKETS - 1)); } // Thread-safe initialization using pthread_once static pthread_once_t mid_tc_init_once_control = PTHREAD_ONCE_INIT; static void mid_tc_init_impl(void) { for (int i = 0; i < MID_TC_BUCKETS; i++) { pthread_mutex_init(&g_mid_tc_mu[i], NULL); g_mid_tc_head[i] = NULL; } } static void mid_tc_init_once(void) { pthread_once(&mid_tc_init_once_control, mid_tc_init_impl); } static MidTC* mid_tc_get(void) { if (t_mid_tc) return t_mid_tc; mid_tc_init_once(); MidTC* tc = (MidTC*)hkm_libc_calloc(1, sizeof(MidTC)); // P0 Fix: Use libc malloc if (!tc) return NULL; uint64_t tid = (uint64_t)(uintptr_t)pthread_self(); uint32_t h = mid_tc_hash(tid); pthread_mutex_lock(&g_mid_tc_mu[h]); MidTCEntry* e = (MidTCEntry*)hkm_libc_malloc(sizeof(MidTCEntry)); // P0 Fix: Use libc malloc if (e) { e->tid = tid; e->tc = tc; e->next = g_mid_tc_head[h]; g_mid_tc_head[h] = e; } pthread_mutex_unlock(&g_mid_tc_mu[h]); t_mid_tc = tc; return tc; } static MidTC* mid_tc_lookup_by_tid(uint64_t tid) { mid_tc_init_once(); uint32_t h = mid_tc_hash(tid); MidTCEntry* e = g_mid_tc_head[h]; while (e) { if (e->tid == tid) return e->tc; e = e->next; } return NULL; } static inline void mid_tc_push(MidTC* tc, int class_idx, PoolBlock* b) { uintptr_t old_head; do { old_head = atomic_load_explicit(&tc->inbox[class_idx], memory_order_acquire); b->next = (PoolBlock*)old_head; } while (!atomic_compare_exchange_weak_explicit(&tc->inbox[class_idx], &old_head, (uintptr_t)b, memory_order_release, memory_order_relaxed)); } static inline int mid_tc_drain_into_tls(int class_idx, PoolTLSRing* ring, PoolTLSBin* bin) { MidTC* tc = mid_tc_get(); if (!tc) return 0; HKM_TIME_START(t_tc); uintptr_t head = atomic_exchange_explicit(&tc->inbox[class_idx], (uintptr_t)0, memory_order_acq_rel); if (!head) { HKM_TIME_END(HKM_CAT_TC_DRAIN, t_tc); return 0; } int moved = 0; int limit = (g_tc_drain_unbounded || g_tc_drain_max <= 0) ? INT32_MAX : g_tc_drain_max; PoolBlock* cur = (PoolBlock*)head; while (cur && moved < limit) { PoolBlock* nxt = cur->next; if (ring->top < POOL_L2_RING_CAP) { ring->items[ring->top++] = cur; moved++; } else { cur->next = bin->lo_head; bin->lo_head = cur; bin->lo_count++; moved++; } cur = nxt; } while (cur) { PoolBlock* nxt = cur->next; mid_tc_push(tc, class_idx, cur); cur = nxt; } HKM_TIME_END(HKM_CAT_TC_DRAIN, t_tc); return moved; } static inline int mid_tc_has_items(int class_idx) { MidTC* tc = t_mid_tc; // do not allocate on peek if (!tc) return 0; return atomic_load_explicit(&tc->inbox[class_idx], memory_order_relaxed) != 0; } #endif // POOL_MID_TC_INC_H