// Inline helpers for Tiny HotMag (small TLS magazine for hot classes) // This header is textually included from hakmem_tiny.c after the following // symbols are defined: // - static int g_hotmag_enable; // - static __thread TinyHotMag g_tls_hot_mag[TINY_NUM_CLASSES]; // - tiny_mag_init_if_needed(int) // - g_tls_sll_head[], g_tls_sll_count[], g_tls_mags[] #include "box/tiny_next_ptr_box.h" // Box API: Next pointer read/write static inline int hkm_is_hot_class(int class_idx) { return class_idx >= 0 && class_idx <= 3 && g_hotmag_class_en[class_idx]; } static inline uint16_t hotmag_effective_cap(int class_idx) { if (!hkm_is_hot_class(class_idx)) return 0; int cap = g_hotmag_cap_current[class_idx]; if (!g_hotmag_cap_locked[class_idx] || cap <= 0) { cap = g_hotmag_cap_default; } if (cap < 16) cap = 16; if (cap > 1024) cap = 1024; return (uint16_t)cap; } static inline uint16_t hotmag_refill_target(int class_idx) { if (!hkm_is_hot_class(class_idx)) return 0; int refill = g_hotmag_refill_current[class_idx]; if (!g_hotmag_refill_locked[class_idx] && refill <= 0) { refill = g_hotmag_refill_default; } if (refill < 0) refill = 0; uint16_t cap = hotmag_effective_cap(class_idx); if ((uint16_t)refill > cap) refill = cap; return (uint16_t)refill; } static inline void hotmag_init_if_needed(int class_idx) { if (!g_hotmag_enable || !hkm_is_hot_class(class_idx)) return; TinyHotMag* hm = &g_tls_hot_mag[class_idx]; uint16_t desired = hotmag_effective_cap(class_idx); if (hm->cap != desired) { hm->cap = desired; if (hm->top > hm->cap) hm->top = hm->cap; } } static inline void* hotmag_pop(int class_idx) { if (!g_hotmag_enable || !hkm_is_hot_class(class_idx)) return NULL; TinyHotMag* hm = &g_tls_hot_mag[class_idx]; if (__builtin_expect(hm->top > 0, 1)) { return hm->slots[--hm->top]; } return NULL; } static inline int hotmag_push(int class_idx, void* ptr) { if (!g_hotmag_enable || !hkm_is_hot_class(class_idx)) return 0; TinyHotMag* hm = &g_tls_hot_mag[class_idx]; hotmag_init_if_needed(class_idx); if (__builtin_expect(hm->top < hm->cap, 1)) { hm->slots[hm->top++] = ptr; return 1; } return 0; } static inline int hotmag_refill_from_sll(int class_idx, int max_take) { if (!g_hotmag_enable || !hkm_is_hot_class(class_idx) || max_take <= 0) return 0; TinyHotMag* hm = &g_tls_hot_mag[class_idx]; int room = (int)hm->cap - (int)hm->top; if (room <= 0) return 0; if (max_take > room) max_take = room; int taken = 0; while (taken < max_take) { // CRITICAL: Use Box TLS-SLL API to avoid race condition (rbp=0xa0 SEGV) void* head = NULL; if (!tls_sll_pop(class_idx, &head)) break; hm->slots[hm->top++] = head; taken++; } return taken; } static inline int hotmag_refill_from_mag(int class_idx, int max_take) { if (!g_hotmag_enable || !hkm_is_hot_class(class_idx) || max_take <= 0) return 0; tiny_mag_init_if_needed(class_idx); TinyTLSMag* mag = &g_tls_mags[class_idx]; TinyHotMag* hm = &g_tls_hot_mag[class_idx]; int room = (int)hm->cap - (int)hm->top; if (room <= 0 || mag->top <= 0) return 0; if (max_take > room) max_take = room; int take = mag->top < max_take ? mag->top : max_take; for (int i = 0; i < take; i++) { hm->slots[hm->top++] = mag->items[--mag->top].ptr; } return take; } static inline int hotmag_try_refill(int class_idx, TinyHotMag* hm) { if (!g_hotmag_enable || !hkm_is_hot_class(class_idx)) return 0; uint16_t target = hotmag_refill_target(class_idx); if (target == 0) return 0; int room = (int)hm->cap - (int)hm->top; if (room <= 0) return 0; if (target > (uint16_t)room) target = (uint16_t)room; int filled = 0; if (g_tls_list_enable && target > 0) { TinyTLSList* tls = &g_tls_lists[class_idx]; uint32_t seq = atomic_load_explicit(&g_tls_param_seq[class_idx], memory_order_relaxed); if (__builtin_expect(seq != g_tls_param_seen[class_idx], 0)) { tiny_tls_refresh_params(class_idx, tls); } void* chain_head = NULL; void* chain_tail = NULL; uint32_t taken = tls_list_bulk_take(tls, target, &chain_head, &chain_tail, class_idx); if (taken > 0u) { void* node = chain_head; for (uint32_t i = 0; i < taken && node; i++) { // Header-aware next from TLS list chain (Box API handles offset) void* next = tiny_next_read(class_idx, node); hm->slots[hm->top++] = node; node = next; } filled += (int)taken; room = (int)hm->cap - (int)hm->top; target -= (uint16_t)taken; } } if (target > 0 && g_tls_sll_enable && room > 0) { int from_sll = hotmag_refill_from_sll(class_idx, target); filled += from_sll; room -= from_sll; if (room < 0) room = 0; if (target > from_sll) target -= (uint16_t)((from_sll < 0) ? 0 : from_sll); else target = 0; } if (target > 0 && room > 0) { int from_mag = hotmag_refill_from_mag(class_idx, target); filled += from_mag; } return filled; }