// hakmem_tiny_hot_pop.inc.h // Phase 2D-1: Hot-path inline functions - Specialized class pop operations // // This file contains per-class specialized hot-path pop functions for classes 0-3. // These functions are extracted from hakmem_tiny.c to improve maintainability and // reduce the main file size by approximately 84 lines (lines 407-494). // // Each function handles: // - Optional ultra bump shadow allocation // - Fast cache pop with explicit capacity checks // - Branchless operation for maximum performance #ifndef HAKMEM_TINY_HOT_POP_INC_H #define HAKMEM_TINY_HOT_POP_INC_H #include "hakmem_tiny.h" #include "box/tiny_next_ptr_box.h" #include // External TLS variables used by hot-path functions extern int g_ultra_bump_shadow; extern int g_fast_enable; extern uint16_t g_fast_cap[TINY_NUM_CLASSES]; extern __thread void* g_fast_head[TINY_NUM_CLASSES]; extern __thread uint16_t g_fast_count[TINY_NUM_CLASSES]; // Forward declaration for ultra bump shadow support static inline void* superslab_tls_bump_fast(int class_idx); // Specialized hot-tier pop (branchless per-class) — used for top hot classes (8/16/32B) static inline __attribute__((always_inline)) void* tiny_hot_pop_class0(void) { if (__builtin_expect(g_ultra_bump_shadow != 0, 0)) { void* bump = superslab_tls_bump_fast(0); if (__builtin_expect(bump != NULL, 1)) { return bump; } } if (__builtin_expect(!g_fast_enable, 0)) return NULL; uint16_t cap = g_fast_cap[0]; if (__builtin_expect(cap == 0, 0)) return NULL; void* head = g_fast_head[0]; if (__builtin_expect(head == NULL, 0)) return NULL; // Phase 7: header-aware next pointer (C0-C6: base+1, C7: base) g_fast_head[0] = tiny_next_read(0, head); uint16_t count = g_fast_count[0]; if (count > 0) { g_fast_count[0] = (uint16_t)(count - 1); } else { g_fast_count[0] = 0; } // No C7 here (class 0), just return base return head; } static inline __attribute__((always_inline)) void* tiny_hot_pop_class1(void) { if (__builtin_expect(g_ultra_bump_shadow != 0, 0)) { void* bump = superslab_tls_bump_fast(1); if (__builtin_expect(bump != NULL, 1)) { return bump; } } if (__builtin_expect(!g_fast_enable, 0)) return NULL; uint16_t cap = g_fast_cap[1]; if (__builtin_expect(cap == 0, 0)) return NULL; void* head = g_fast_head[1]; if (__builtin_expect(head == NULL, 0)) return NULL; // Phase 7: header-aware next pointer (C0-C6: base+1) g_fast_head[1] = tiny_next_read(1, head); uint16_t count = g_fast_count[1]; if (count > 0) { g_fast_count[1] = (uint16_t)(count - 1); } else { g_fast_count[1] = 0; } return head; } static inline __attribute__((always_inline)) void* tiny_hot_pop_class2(void) { if (__builtin_expect(g_ultra_bump_shadow != 0, 0)) { void* bump = superslab_tls_bump_fast(2); if (__builtin_expect(bump != NULL, 1)) { return bump; } } if (__builtin_expect(!g_fast_enable, 0)) return NULL; uint16_t cap = g_fast_cap[2]; if (__builtin_expect(cap == 0, 0)) return NULL; void* head = g_fast_head[2]; if (__builtin_expect(head == NULL, 0)) return NULL; // Phase 7: header-aware next pointer (C0-C6: base+1) g_fast_head[2] = tiny_next_read(2, head); uint16_t count = g_fast_count[2]; if (count > 0) { g_fast_count[2] = (uint16_t)(count - 1); } else { g_fast_count[2] = 0; } return head; } static inline __attribute__((always_inline)) void* tiny_hot_pop_class3(void) { if (__builtin_expect(g_ultra_bump_shadow != 0, 0)) { void* bump = superslab_tls_bump_fast(3); if (__builtin_expect(bump != NULL, 1)) { return bump; } } if (__builtin_expect(!g_fast_enable, 0)) return NULL; uint16_t cap = g_fast_cap[3]; if (__builtin_expect(cap == 0, 0)) return NULL; void* head = g_fast_head[3]; if (__builtin_expect(head == NULL, 0)) return NULL; // Phase 7: header-aware next pointer (C0-C6: base+1) g_fast_head[3] = tiny_next_read(3, head); uint16_t count = g_fast_count[3]; if (count > 0) { g_fast_count[3] = (uint16_t)(count - 1); } else { g_fast_count[3] = 0; } return head; } #endif // HAKMEM_TINY_HOT_POP_INC_H