hakmem/core/hakmem_internal.h

// hakmem_internal.h - Internal Implementation Helpers (static inline)
// Purpose: Separate implementation details from public API using zero-cost abstraction
//
// Design Philosophy:
// - All functions are `static inline` → Zero overhead (100% inlined with -O2)
// - Type-safe (unlike macros)
// - Debuggable (unlike macros)
// - Readable (unlike macros)
//
// This file should be #include'd by hakmem.c ONLY (not a public header)

#ifndef HAKMEM_INTERNAL_H
#define HAKMEM_INTERNAL_H

#include "hakmem.h"
#include "hakmem_config.h"
#include "hakmem_sys.h"        // Phase 6.11.1: Syscall wrappers with timing
#include "hakmem_whale.h"      // Phase 6.11.1: Whale fast-path cache
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <errno.h>             // Phase 7: errno for OOM handling
#include <sys/mman.h>          // For mincore, madvise
#include <unistd.h>            // For sysconf
#include <stdatomic.h>

// Exposed runtime mode: set to 1 when loaded via LD_PRELOAD (libhakmem.so)
extern int g_ldpreload_mode;

// ============================================================================
// Phase 6.15 P0.1: Debug Logging Control
// ============================================================================

// Compile-time control: HAKMEM_DEBUG_VERBOSE (default OFF for performance)
// Runtime control: HAKMEM_QUIET environment variable (only for debug builds)
//
// Build modes:
//   Release (default): make shared          → No logs (HAKMEM_LOG compiled out)
//   Debug:             make debug           → Logs enabled (unless HAKMEM_QUIET=1)
//   Debug quiet:       HAKMEM_QUIET=1 ...   → Logs suppressed at runtime

#if HAKMEM_DEBUG_VERBOSE
  // Debug build: Check HAKMEM_QUIET at runtime
  #define HAKMEM_LOG(fmt, ...) do { \
      static int quiet_checked = 0; \
      static int quiet_mode = 0; \
      if (!quiet_checked) { \
          char* env = getenv("HAKMEM_QUIET"); \
          quiet_mode = (env && strcmp(env, "1") == 0); \
          quiet_checked = 1; \
      } \
      if (!quiet_mode) { \
          fprintf(stderr, "[hakmem] " fmt, ##__VA_ARGS__); \
      } \
  } while(0)
#else
  // Release build: Compile out all logs (zero overhead)
  #define HAKMEM_LOG(fmt, ...) ((void)0)
#endif

#ifdef __linux__
#include <sys/mman.h>
#include <unistd.h>

// MADV_FREE support (Linux kernel 4.5+)
#ifndef MADV_FREE
  #define MADV_FREE 8
#endif
// Fallback for MADV_DONTNEED if not defined (Linux usually defines 4)
#ifndef MADV_DONTNEED
  #define MADV_DONTNEED 4
#endif

// THP support
#ifndef MADV_HUGEPAGE
  #define MADV_HUGEPAGE 14
#endif
#ifndef MADV_NOHUGEPAGE
  #define MADV_NOHUGEPAGE 15
#endif
#endif

// ===========================================================================
// Internal Constants
// ===========================================================================

#define HAKMEM_MAGIC 0x48414B4D  // "HAKM" in ASCII (uint32_t)
#define HEADER_SIZE sizeof(AllocHeader)
#define HAKMEM_FG_GUARD_BYTE 0x5Au  // Ensure front gate never misclassifies mid/large as Tiny

// THP thresholds (from config)
#define THP_THRESHOLD (2 * 1024 * 1024)  // 2MB

// Thermal thresholds (from Phase 6.4 P1)
#define THERMAL_COLD_THRESHOLD  (2 * 1024 * 1024)   // 2MB
#define THERMAL_WARM_THRESHOLD  (1 * 1024 * 1024)   // 1MB

// ===========================================================================
// Internal Types
// ===========================================================================

typedef enum {
    ALLOC_METHOD_MALLOC = 0,
    ALLOC_METHOD_MMAP = 1,
    ALLOC_METHOD_POOL = 2,     // Phase 6.9.1: L2 Pool allocations (2-32KB)
    ALLOC_METHOD_L25_POOL = 3, // Phase 6.13: L2.5 Pool allocations (64KB-1MB)
} AllocMethod;

typedef struct {
    uint32_t    magic;        // Magic number for validation
    AllocMethod method;       // Allocation method (malloc/mmap)
    size_t      size;         // Original size (for munmap)
    uintptr_t   alloc_site;   // Call-site address
    size_t      class_bytes;  // Size class for caching (0=no cache)
    uintptr_t   owner_tid;    // Owning thread (for Mid/Tiny per-thread fast path). 0 if unknown
} AllocHeader;

// Phase 10: Pointer Type Safety
#include "box/ptr_type_box.h"

typedef enum {
    FREE_THERMAL_HOT,       // すぐ再利用 → 何もしない（KEEP）
    FREE_THERMAL_WARM,      // 中間 → MADV_FREE（munmapしない）
    FREE_THERMAL_COLD       // 長期未使用 → batch（DONTNEED）
} FreeThermal;

// ===========================================================================
// Thermal Classification (Phase 6.4 P1)
// ===========================================================================

// Classify allocation thermal state based on size
// Args: size - allocation size in bytes
// Returns: FreeThermal enum (HOT/WARM/COLD)
//
// Thermal States:
// - HOT  (< 1MB):   Likely to be reused soon → keep VA mapped
// - WARM (1-2MB):   Medium reuse → MADV_FREE (return physical pages only)
// - COLD (>= 2MB):  Low reuse → batch DONTNEED (return VA + physical)
//
// Used by FREE_POLICY_ADAPTIVE to optimize memory release strategy
static inline FreeThermal hak_classify_thermal(size_t size) {
    if (size >= THERMAL_COLD_THRESHOLD) {
        return FREE_THERMAL_COLD;   // >= 2MB → COLD
    } else if (size >= THERMAL_WARM_THRESHOLD) {
        return FREE_THERMAL_WARM;   // 1MB-2MB → WARM
    } else {
        return FREE_THERMAL_HOT;    // < 1MB → HOT
    }
}

// ===========================================================================
// THP Policy Application (Phase 6.4 P4)
// ===========================================================================

// Apply Transparent Huge Pages (THP) policy to mmap'd region
// Args: ptr - pointer to mmap'd memory region
//       size - size of region in bytes
//
// THP Policies:
// - THP_POLICY_OFF:  MADV_NOHUGEPAGE for all (disable THP)
// - THP_POLICY_AUTO: MADV_HUGEPAGE for >= 2MB only (default, balanced)
// - THP_POLICY_ON:   MADV_HUGEPAGE for all >= 1MB (aggressive)
//
// Benefits of THP:
// - Reduced TLB misses (2MB pages vs 4KB pages = 512x reduction)
// - Improved cache locality
// - Lower page table overhead
//
// Set via HAKMEM_THP environment variable
static inline void hak_apply_thp_policy(void* ptr, size_t size) {
#ifdef __linux__
    if (!ptr) return;  // Safety check

    THPPolicy policy = g_hakem_config.thp_policy;

    if (policy == THP_POLICY_OFF) {
        madvise(ptr, size, MADV_NOHUGEPAGE);
    } else if (policy == THP_POLICY_ON) {
        madvise(ptr, size, MADV_HUGEPAGE);
    } else {  // AUTO
        if (size >= THP_THRESHOLD) {
            madvise(ptr, size, MADV_HUGEPAGE);  // >= 2MB → THP
        } else {
            madvise(ptr, size, MADV_NOHUGEPAGE);  // < 2MB → no THP
        }
    }
#else
    (void)ptr;
    (void)size;
#endif
}

// ===========================================================================
// Allocation Strategies (static inline = zero overhead)
// ===========================================================================

// Strategy 1: malloc (for small/medium allocations)
// Args: size - requested allocation size (user bytes, excluding header)
// Returns: User pointer (after header), or NULL on failure
//
// Implementation:
// - Allocates HEADER_SIZE + size using system malloc()
// - Writes AllocHeader with MALLOC method
// - Returns pointer after header (user-visible pointer)
// - O(1) allocation with kernel slab allocator (< 2MB)
static inline void* hak_alloc_malloc_impl(size_t size) {
    // PHASE 7 CRITICAL FIX: malloc fallback removed (root cause of 4T crash)
    //
    // WHY: Mixed HAKMEM/libc allocations cause "free(): invalid pointer" crashes
    //      - libc malloc adds its own metadata (8-16B)
    //      - HAKMEM adds AllocHeader on top (16-32B total overhead!)
    //      - free() confusion leads to double-free/invalid pointer crashes
    //
    // SOLUTION: Return NULL explicitly to force OOM handling
    //           SuperSlab should dynamically scale instead of falling back
    //
    // To enable fallback for debugging ONLY (not for production!):
    //   export HAKMEM_ALLOW_MALLOC_FALLBACK=1

    static int allow_fallback = -1;
    if (allow_fallback < 0) {
        char* env = getenv("HAKMEM_ALLOW_MALLOC_FALLBACK");
        allow_fallback = (env && atoi(env) != 0) ? 1 : 0;
    }

    if (!allow_fallback) {
        // Malloc fallback disabled (production mode)
        static _Atomic int warn_count = 0;
        int count = atomic_fetch_add(&warn_count, 1);
        if (count < 3) {
            fprintf(stderr, "[HAKMEM] WARNING: malloc fallback disabled (size=%zu), returning NULL (OOM)\n", size);
            fprintf(stderr, "[HAKMEM]          This may indicate SuperSlab exhaustion. Set HAKMEM_ALLOW_MALLOC_FALLBACK=1 to debug.\n");
        }
        errno = ENOMEM;
        return NULL;  // Explicit OOM
    }

    // Fallback path (DEBUGGING ONLY - should not be used in production!)
    if (!HAK_ENABLED_ALLOC(HAKMEM_FEATURE_MALLOC)) {
        return NULL;  // malloc disabled
    }

    // Warn about fallback usage
    static _Atomic int fallback_warn_count = 0;
    int fb_count = atomic_fetch_add(&fallback_warn_count, 1);
    if (fb_count < 3) {
        fprintf(stderr, "[HAKMEM] DEBUG: Using libc malloc fallback (size=%zu) - NOT RECOMMENDED FOR PRODUCTION!\n", size);
    }

    // Allocate space for header + user data
    // CRITICAL: Must use __libc_malloc to avoid infinite recursion through wrapper
    extern void* __libc_malloc(size_t);
    void* raw = __libc_malloc(HEADER_SIZE + size);
    if (!raw) return NULL;

    // Write header
    AllocHeader* hdr = (AllocHeader*)raw;
    hdr->magic = HAKMEM_MAGIC;
    hdr->method = ALLOC_METHOD_MALLOC;
    hdr->size = size;
    hdr->alloc_site = 0;      // Set by caller (hak_alloc_at)
    hdr->class_bytes = 0;     // Set by caller if cacheable

    // Return user pointer (skip header)
    return (char*)raw + HEADER_SIZE;
}

// Strategy 2: mmap (for large allocations)
// Args: size - requested allocation size (user bytes, excluding header)
// Returns: User pointer (after header), or NULL on failure
//
// Implementation:
// - Rounds up (HEADER_SIZE + size) to page boundary
// - Uses mmap(MAP_ANONYMOUS) for zero-overhead allocation
// - Applies THP policy (MADV_HUGEPAGE/NOHUGEPAGE)
// - Stores aligned_size in header->size (for munmap)
// - O(1) allocation with kernel buddy allocator (>= 2MB)
static inline void* hak_alloc_mmap_impl(size_t size) {
#ifdef __linux__
    // Feature check
    if (!HAK_ENABLED_ALLOC(HAKMEM_FEATURE_MMAP)) {
        return NULL;  // mmap disabled, fallback to malloc
    }

    // Round up to page size (header + user data)
    long page_size = sysconf(_SC_PAGESIZE);
    size_t total_size = HEADER_SIZE + size;
    size_t aligned_size = (total_size + page_size - 1) & ~(page_size - 1);

    // Phase 6.11.1: Try whale cache first (for ≥2MB allocations)
    void* raw = hkm_whale_get(aligned_size);

    if (!raw) {
        // Whale cache miss: allocate via mmap
        raw = hkm_sys_mmap(aligned_size);
        if (!raw) {
            return NULL;
        }
    }
    // else: Whale cache hit! Reuse existing mapping (no mmap syscall)

    // Apply THP policy (Phase 6.4 P4)
    hak_apply_thp_policy(raw, aligned_size);

    // Write header
    AllocHeader* hdr = (AllocHeader*)raw;
    hdr->magic = HAKMEM_MAGIC;
    hdr->method = ALLOC_METHOD_MMAP;
    hdr->size = aligned_size;  // Store aligned size for munmap
    hdr->alloc_site = 0;       // Set by caller (hak_alloc_at)
    hdr->class_bytes = 0;      // Set by caller if cacheable

    // Return user pointer (skip header)
    return (char*)raw + HEADER_SIZE;
#else
    // Fallback to malloc on non-Linux
    return hak_alloc_malloc_impl(size);
#endif
}

// ===========================================================================
// Memory Safety Helpers
// ===========================================================================

// hak_is_memory_readable: Check if memory address is accessible before dereferencing
// PHASE 9: mincore() REMOVED - Use internal metadata instead
//
// OLD DESIGN (Phase 7):
//   - Used mincore() syscall (~634 cycles)
//   - Hybrid optimization: only check page boundaries (99.9% avoid syscall)
//
// NEW DESIGN (Phase 9 - Lazy Deallocation):
//   - NO syscall overhead (0 cycles)
//   - Trust internal metadata (SuperSlab registry + header magic)
//   - SuperSlabs tracked in registry → if lookup succeeds, memory is valid
//   - Headers contain magic → validate before dereferencing
//
// Performance Impact:
//   - OLD: 1-2 cycles effective (99.9% × 1 + 0.1% × 634)
//   - NEW: 0 cycles (function removed, callers use registry lookup)
//   - Syscall reduction: 841 mincore calls → 0 (100% elimination)
//
// Migration: All callers should use hak_super_lookup() instead
static inline int hak_is_memory_readable(void* addr) {
    // Phase 9: Removed mincore() - assume valid (registry ensures safety)
    // Callers should use hak_super_lookup() for validation
    return 1;  // Always return true (trust internal metadata)
}

// ===========================================================================
// Header Helpers (with NULL safety)
// ===========================================================================

// Get raw pointer (before header) from user pointer
// Returns: Raw allocation pointer (header starts here)
static inline void* hak_header_get_raw(void* user_ptr) {
    if (!user_ptr) return NULL;
    return (char*)user_ptr - HEADER_SIZE;
}

// Get header from user pointer
// Returns: Pointer to AllocHeader, or NULL if user_ptr is NULL
static inline AllocHeader* hak_header_from_user(void* user_ptr) {
    if (!user_ptr) return NULL;
    return (AllocHeader*)hak_header_get_raw(user_ptr);
}

// Validate header magic number
// Returns: 1 if valid, 0 if invalid or NULL
static inline int hak_header_validate(AllocHeader* hdr) {
    if (!hdr) return 0;
    return hdr->magic == HAKMEM_MAGIC;
}

// Set allocation site in header (for cache key)
static inline void hak_header_set_site(void* user_ptr, uintptr_t site_id) {
    AllocHeader* hdr = hak_header_from_user(user_ptr);
    if (hdr) {
        hdr->alloc_site = site_id;
    }
}

// Set size class in header (for BigCache)
static inline void hak_header_set_class(void* user_ptr, size_t class_bytes) {
    AllocHeader* hdr = hak_header_from_user(user_ptr);
    if (hdr) {
        hdr->class_bytes = class_bytes;
    }
}

// ===========================================================================
// Free Strategies (static inline = zero overhead)
// ===========================================================================

// Free malloc-allocated block
// Args: raw - pointer to raw allocation (including header)
static inline void hak_free_malloc_impl(void* raw) {
    if (!raw) return;  // Safety check
    extern void __libc_free(void*);
    __libc_free(raw);
}

// Free mmap-allocated block
// Args: raw - pointer to raw allocation (including header)
//       size - aligned size (from header->size)
static inline void hak_free_mmap_impl(void* raw, size_t size) {
    if (!raw) return;  // Safety check
#ifdef __linux__
    munmap(raw, size);
#else
    extern void __libc_free(void*);
    __libc_free(raw);  // Fallback on non-Linux
#endif
}

// Apply Hot/Warm/Cold free policy (Phase 6.4 P1)
// Args: raw - pointer to raw allocation (including header)
//       size - allocated size
//       thermal - thermal classification (HOT/WARM/COLD)
// Returns: 1 if handled (no further action needed), 0 if caller should continue (batch/direct free)
static inline int hak_free_with_thermal_policy(void* raw, size_t size, FreeThermal thermal) {
    if (!raw) return 1;  // NULL is always "handled" (no-op)

    FreePolicy policy = g_hakem_config.free_policy;

    if (policy == FREE_POLICY_KEEP) {
        // KEEP: 何もしない（VA保持、madviseもしない）
        return 1;  // Handled (kept)
    } else if (policy == FREE_POLICY_ADAPTIVE) {
        // ADAPTIVE: Hot/Warm/Cold判定
        switch (thermal) {
            case FREE_THERMAL_HOT:
                // HOT (< 1MB): 何もしない（すぐ再利用される）
                return 1;  // Handled (kept)

            case FREE_THERMAL_WARM:
                // WARM (1-2MB): MADV_FREE（munmapしない、物理ページのみ返す）
#ifdef __linux__
                madvise(raw, size, MADV_FREE);
#endif
                return 1;  // Handled

            case FREE_THERMAL_COLD:
                // COLD (>= 2MB): batch（既存の処理）
                return 0;  // Not handled, caller should use batch
        }
    }

    // FREE_POLICY_BATCH (default): caller handles
    return 0;  // Not handled
}

#endif // HAKMEM_INTERNAL_H