hakmem/core/hakmem_tiny_hot_pop.inc.h

// hakmem_tiny_hot_pop.inc.h
// Phase 2D-1: Hot-path inline functions - Specialized class pop operations
//
// This file contains per-class specialized hot-path pop functions for classes 0-3.
// These functions are extracted from hakmem_tiny.c to improve maintainability and
// reduce the main file size by approximately 84 lines (lines 407-494).
//
// Each function handles:
// - Optional ultra bump shadow allocation
// - Fast cache pop with explicit capacity checks
// - Branchless operation for maximum performance

#ifndef HAKMEM_TINY_HOT_POP_INC_H
#define HAKMEM_TINY_HOT_POP_INC_H

#include "hakmem_tiny.h"
#include <stdint.h>

// External TLS variables used by hot-path functions
extern int g_ultra_bump_shadow;
extern int g_fast_enable;
extern uint16_t g_fast_cap[TINY_NUM_CLASSES];
extern __thread void* g_fast_head[TINY_NUM_CLASSES];
extern __thread uint16_t g_fast_count[TINY_NUM_CLASSES];

// Forward declaration for ultra bump shadow support
static inline void* superslab_tls_bump_fast(int class_idx);

// Specialized hot-tier pop (branchless per-class) — used for top hot classes (8/16/32B)
static inline __attribute__((always_inline)) void* tiny_hot_pop_class0(void) {
    if (__builtin_expect(g_ultra_bump_shadow != 0, 0)) {
        void* bump = superslab_tls_bump_fast(0);
        if (__builtin_expect(bump != NULL, 1)) {
            return bump;
        }
    }
    if (__builtin_expect(!g_fast_enable, 0)) return NULL;
    uint16_t cap = g_fast_cap[0];
    if (__builtin_expect(cap == 0, 0)) return NULL;
    void* head = g_fast_head[0];
    if (__builtin_expect(head == NULL, 0)) return NULL;
    // Phase 7: header-aware next pointer (C0-C6: base+1, C7: base)
#if HAKMEM_TINY_HEADER_CLASSIDX
    const size_t next_off0 = 1;  // class 0 is headered
#else
    const size_t next_off0 = 0;
#endif
    g_fast_head[0] = *(void**)((uint8_t*)head + next_off0);
    uint16_t count = g_fast_count[0];
    if (count > 0) {
        g_fast_count[0] = (uint16_t)(count - 1);
    } else {
        g_fast_count[0] = 0;
    }
    // No C7 here (class 0), just return base
    return head;
}

static inline __attribute__((always_inline)) void* tiny_hot_pop_class1(void) {
    if (__builtin_expect(g_ultra_bump_shadow != 0, 0)) {
        void* bump = superslab_tls_bump_fast(1);
        if (__builtin_expect(bump != NULL, 1)) {
            return bump;
        }
    }
    if (__builtin_expect(!g_fast_enable, 0)) return NULL;
    uint16_t cap = g_fast_cap[1];
    if (__builtin_expect(cap == 0, 0)) return NULL;
    void* head = g_fast_head[1];
    if (__builtin_expect(head == NULL, 0)) return NULL;
    // Phase 7: header-aware next pointer (C0-C6: base+1)
#if HAKMEM_TINY_HEADER_CLASSIDX
    const size_t next_off1 = 1;
#else
    const size_t next_off1 = 0;
#endif
    g_fast_head[1] = *(void**)((uint8_t*)head + next_off1);
    uint16_t count = g_fast_count[1];
    if (count > 0) {
        g_fast_count[1] = (uint16_t)(count - 1);
    } else {
        g_fast_count[1] = 0;
    }
    return head;
}

static inline __attribute__((always_inline)) void* tiny_hot_pop_class2(void) {
    if (__builtin_expect(g_ultra_bump_shadow != 0, 0)) {
        void* bump = superslab_tls_bump_fast(2);
        if (__builtin_expect(bump != NULL, 1)) {
            return bump;
        }
    }
    if (__builtin_expect(!g_fast_enable, 0)) return NULL;
    uint16_t cap = g_fast_cap[2];
    if (__builtin_expect(cap == 0, 0)) return NULL;
    void* head = g_fast_head[2];
    if (__builtin_expect(head == NULL, 0)) return NULL;
    // Phase 7: header-aware next pointer (C0-C6: base+1)
#if HAKMEM_TINY_HEADER_CLASSIDX
    const size_t next_off2 = 1;
#else
    const size_t next_off2 = 0;
#endif
    g_fast_head[2] = *(void**)((uint8_t*)head + next_off2);
    uint16_t count = g_fast_count[2];
    if (count > 0) {
        g_fast_count[2] = (uint16_t)(count - 1);
    } else {
        g_fast_count[2] = 0;
    }
    return head;
}

static inline __attribute__((always_inline)) void* tiny_hot_pop_class3(void) {
    if (__builtin_expect(g_ultra_bump_shadow != 0, 0)) {
        void* bump = superslab_tls_bump_fast(3);
        if (__builtin_expect(bump != NULL, 1)) {
            return bump;
        }
    }
    if (__builtin_expect(!g_fast_enable, 0)) return NULL;
    uint16_t cap = g_fast_cap[3];
    if (__builtin_expect(cap == 0, 0)) return NULL;
    void* head = g_fast_head[3];
    if (__builtin_expect(head == NULL, 0)) return NULL;
    // Phase 7: header-aware next pointer (C0-C6: base+1)
#if HAKMEM_TINY_HEADER_CLASSIDX
    const size_t next_off3 = 1;
#else
    const size_t next_off3 = 0;
#endif
    g_fast_head[3] = *(void**)((uint8_t*)head + next_off3);
    uint16_t count = g_fast_count[3];
    if (count > 0) {
        g_fast_count[3] = (uint16_t)(count - 1);
    } else {
        g_fast_count[3] = 0;
    }
    return head;
}

#endif // HAKMEM_TINY_HOT_POP_INC_H