hakmem/core/smallobject_policy_v7.c

// smallobject_policy_v7.c - Policy Box implementation (Phase v7-7: Learner integration)

#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include "hakmem_build_flags.h"  // Phase 36: HAKMEM_BENCH_MINIMAL
#include "box/smallobject_policy_v7_box.h"
#include "box/smallobject_learner_v7_box.h"  // For Learner API
#include "box/small_policy_snapshot_tls_box.h"  // Phase 37: TLS cache
#include "front/tiny_warm_pool.h"  // Phase 70-1: warm_pool_max_per_class()
#include "hakmem_tiny.h"  // Phase 70-1: unified_cache_enabled()

#ifndef likely
#define likely(x)   __builtin_expect(!!(x), 1)
#define unlikely(x) __builtin_expect(!!(x), 0)
#endif

// ============================================================================
// Learner Stats (Global, v7-7)
// ============================================================================

static SmallLearnerStatsV7 g_small_learner_stats_v7;
static int g_learner_v7_enabled = -1;  // -1: uninit, 0: disabled, 1: enabled

// Phase 36: BENCH_MINIMAL mode - learner is disabled (bench profiles don't use learner)
// Phase 63: FAST_PROFILE_FIXED - learner disabled in fixed FAST profile builds
#if HAKMEM_BENCH_MINIMAL || HAKMEM_FAST_PROFILE_FIXED
static inline int learner_v7_enabled(void) {
    return 0;  // Fixed OFF in bench mode
}
#else
static inline int learner_v7_enabled(void) {
    if (unlikely(g_learner_v7_enabled < 0)) {
        // Phase v10: Learner default ON (when v7 is enabled)
        // Can be disabled via ENV: HAKMEM_SMALL_LEARNER_V7_ENABLED=0
        const char* e = getenv("HAKMEM_SMALL_LEARNER_V7_ENABLED");
        if (e && *e && *e == '0') {
            g_learner_v7_enabled = 0;  // explicitly disabled
        } else {
            // Default: ON (when v7 is also enabled)
            const char* v7_e = getenv("HAKMEM_SMALL_HEAP_V7_ENABLED");
            g_learner_v7_enabled = (v7_e && *v7_e && *v7_e != '0') ? 1 : 0;
        }
    }
    return g_learner_v7_enabled;
}
#endif

// ============================================================================
// TLS Policy Snapshot (v7-7: version-based invalidation)
// ============================================================================

static uint32_t g_policy_v7_version = 0;  // Global version, incremented by Learner

static __thread SmallPolicyV7 g_small_policy_v7;
static __thread uint32_t g_small_policy_v7_version = 0;  // TLS cached version

void small_policy_v7_bump_version(void) {
    __sync_fetch_and_add(&g_policy_v7_version, 1);
}

const SmallPolicyV7* small_policy_v7_snapshot(void) {
#if HAKMEM_BENCH_MINIMAL
    // Phase 36: BENCH_MINIMAL mode - skip version check, use init-once TLS cache
    // Assumes: Learner disabled, policy doesn't change during benchmark
    static __thread int s_initialized = 0;
    if (unlikely(!s_initialized)) {
        small_policy_v7_init_from_env(&g_small_policy_v7);
        s_initialized = 1;
    }
    return &g_small_policy_v7;
#else
    // Phase 37: TLS cache fast path (default ON)
    if (policy_snapshot_tls_enabled()) {
        SmallPolicySnapshotTLSCache* cache = small_policy_snapshot_tls_get();
        uint32_t gver = g_policy_v7_version;

        // Fast path: cache valid → return immediately (no global read beyond version)
        if (small_policy_snapshot_tls_check(cache, gver)) {
            return cache->cached_ptr;
        }

        // Slow path: refresh cache
        small_policy_v7_init_from_env(&g_small_policy_v7);

        // v7-7: Apply Learner-driven route updates
        if (learner_v7_enabled() && g_small_learner_stats_v7.total_retires > 0) {
            small_policy_v7_update_from_learner(&g_small_learner_stats_v7, &g_small_policy_v7);
        }

        // Initialize global version to 1 if uninitialized (0)
        if (gver == 0) {
            __sync_val_compare_and_swap(&g_policy_v7_version, 0, 1);
            gver = 1;
        }

        // Update TLS cache
        small_policy_snapshot_tls_update(cache, &g_small_policy_v7, gver);
        return &g_small_policy_v7;
    }

    // Fallback: original version-check path (HAKMEM_POLICY_SNAPSHOT_TLS=0)
    if (unlikely(g_small_policy_v7_version != g_policy_v7_version || g_policy_v7_version == 0)) {
        small_policy_v7_init_from_env(&g_small_policy_v7);

        // v7-7: Apply Learner-driven route updates
        if (learner_v7_enabled() && g_small_learner_stats_v7.total_retires > 0) {
            small_policy_v7_update_from_learner(&g_small_learner_stats_v7, &g_small_policy_v7);
        }

        // Initialize global version to 1 if uninitialized (0)
        // This prevents infinite re-init loop where condition is always true
        if (g_policy_v7_version == 0) {
            __sync_val_compare_and_swap(&g_policy_v7_version, 0, 1);
        }
        g_small_policy_v7_version = g_policy_v7_version;
    }
    return &g_small_policy_v7;
#endif
}

// ============================================================================
// ENV Helpers
// ============================================================================

static inline bool env_enabled(const char* name) {
    const char* e = getenv(name);
    return (e && *e && *e != '0');
}

static inline uint32_t env_class_mask(const char* name, uint32_t default_mask) {
    const char* e = getenv(name);
    if (e && *e) {
        return (uint32_t)strtoul(e, NULL, 0);
    }
    return default_mask;
}

// ============================================================================
// MID v3.5 ENV Helpers (Phase v11a-3)
// ============================================================================

static inline bool mid_v35_enabled(void) {
    const char* e = getenv("HAKMEM_MID_V35_ENABLED");
    return (e && *e && *e != '0');
}

static inline uint32_t mid_v35_class_mask(void) {
    const char* e = getenv("HAKMEM_MID_V35_CLASSES");
    if (e && *e) {
        return (uint32_t)strtoul(e, NULL, 0);
    }
    return 0x60;  // Default: C5(0x20) + C6(0x40)
}

// ============================================================================
// Policy Initialization from ENV
// ============================================================================

void small_policy_v7_init_from_env(SmallPolicyV7* policy) {
    if (!policy) return;

    // Default: all classes go to LEGACY
    for (int i = 0; i < 8; i++) {
        policy->route_kind[i] = SMALL_ROUTE_LEGACY;
    }

    // Priority 4: MID_v3 (257-768B, C5-C6 range)
    // ENV: HAKMEM_MID_V3_ENABLED, HAKMEM_MID_V3_CLASSES
    if (env_enabled("HAKMEM_MID_V3_ENABLED")) {
        uint32_t mid_mask = env_class_mask("HAKMEM_MID_V3_CLASSES", 0x60); // C5-C6 default
        for (int i = 0; i < 8; i++) {
            if (mid_mask & (1u << i)) {
                policy->route_kind[i] = SMALL_ROUTE_MID_V3;
            }
        }
    }

    // Priority 3: MID_v3.5 (Phase v11a-3: higher priority than MID_v3)
    // ENV: HAKMEM_MID_V35_ENABLED, HAKMEM_MID_V35_CLASSES
    if (mid_v35_enabled()) {
        uint32_t v35_mask = mid_v35_class_mask();
        for (int i = 0; i < 8; i++) {
            if (v35_mask & (1u << i)) {
                // Only override if not ULTRA
                if (policy->route_kind[i] != SMALL_ROUTE_ULTRA) {
                    policy->route_kind[i] = SMALL_ROUTE_MID_V35;
                }
            }
        }
    }

    // Priority 2: SmallObject v7 (research box, C6-only for now)
    // ENV: HAKMEM_SMALL_HEAP_V7_ENABLED, HAKMEM_SMALL_HEAP_V7_CLASSES
    if (env_enabled("HAKMEM_SMALL_HEAP_V7_ENABLED")) {
        uint32_t v7_mask = env_class_mask("HAKMEM_SMALL_HEAP_V7_CLASSES", 0x40); // C6 default
        for (int i = 0; i < 8; i++) {
            if (v7_mask & (1u << i)) {
                policy->route_kind[i] = SMALL_ROUTE_V7;
            }
        }
    }

    // Priority 1: ULTRA (highest priority, C4-C7)
    // Phase v11a-5: All ULTRA ENVs consolidated here (removed from hot path)

    // C7 ULTRA (default ON via HAKMEM_TINY_C7_ULTRA_ENABLED)
    if (env_enabled("HAKMEM_TINY_C7_ULTRA_ENABLED")) {
        policy->route_kind[7] = SMALL_ROUTE_ULTRA;
    }

    // C6 ULTRA (via HAKMEM_TINY_C6_ULTRA_FREE_ENABLED - TLS freelist pop)
    if (env_enabled("HAKMEM_TINY_C6_ULTRA_FREE_ENABLED")) {
        policy->route_kind[6] = SMALL_ROUTE_ULTRA;
    }

    // C5 ULTRA (via HAKMEM_TINY_C5_ULTRA_FREE_ENABLED - TLS freelist pop)
    if (env_enabled("HAKMEM_TINY_C5_ULTRA_FREE_ENABLED")) {
        policy->route_kind[5] = SMALL_ROUTE_ULTRA;
    }

    // C4 ULTRA (via HAKMEM_TINY_C4_ULTRA_FREE_ENABLED - TLS freelist pop)
    if (env_enabled("HAKMEM_TINY_C4_ULTRA_FREE_ENABLED")) {
        policy->route_kind[4] = SMALL_ROUTE_ULTRA;
    }

    // Debug output (if needed)
    static int g_debug_once = 0;
    if (!g_debug_once) {
        g_debug_once = 1;

        // Phase 70-1: Route Banner (ENV-gated comprehensive route info)
        const char* route_banner_env = getenv("HAKMEM_ROUTE_BANNER");
        int show_route_banner = (route_banner_env && atoi(route_banner_env));

        if (show_route_banner) {
            fprintf(stderr, "\n[ROUTE_BANNER] ========================================\n");
            fprintf(stderr, "[ROUTE_BANNER] Hakmem Route Configuration (Phase 70-1)\n");
            fprintf(stderr, "[ROUTE_BANNER] ========================================\n");

            fprintf(stderr, "[ROUTE_BANNER] Route assignments:\n");
            for (int i = 0; i < 8; i++) {
                fprintf(stderr, "[ROUTE_BANNER]   C%d: %s\n", i, small_route_kind_name(policy->route_kind[i]));
            }

            fprintf(stderr, "[ROUTE_BANNER] \n");
            fprintf(stderr, "[ROUTE_BANNER] Cache Configuration:\n");
            fprintf(stderr, "[ROUTE_BANNER]   unified_cache_enabled = %d\n", unified_cache_enabled());
            fprintf(stderr, "[ROUTE_BANNER]   warm_pool_max_per_class = %d (HAKMEM_WARM_POOL_SIZE)\n", warm_pool_max_per_class());

            fprintf(stderr, "[ROUTE_BANNER] \n");
            fprintf(stderr, "[ROUTE_BANNER] Note: Unified-STATS miss counts will be reported at benchmark end.\n");
            fprintf(stderr, "[ROUTE_BANNER] ========================================\n\n");
            fflush(stderr);
        } else {
            // Legacy output (without ROUTE_BANNER)
            fprintf(stderr, "[POLICY_V7_INIT] Route assignments:\n");
            for (int i = 0; i < 8; i++) {
                fprintf(stderr, "  C%d: %s\n", i, small_route_kind_name(policy->route_kind[i]));
            }
        }
    }
}

// ============================================================================
// Utility
// ============================================================================

const char* small_route_kind_name(SmallRouteKind kind) {
    switch (kind) {
        case SMALL_ROUTE_ULTRA:   return "ULTRA";
        case SMALL_ROUTE_V7:      return "V7";
        case SMALL_ROUTE_MID_V3:  return "MID_V3";
        case SMALL_ROUTE_MID_V35: return "MID_V35";
        case SMALL_ROUTE_LEGACY:  return "LEGACY";
        default:                  return "UNKNOWN";
    }
}

// ============================================================================
// Learner Implementation (Phase v7-7)
// ============================================================================

// Total refills (for evaluation interval)
static uint64_t g_small_learner_total_refills = 0;

void small_learner_v7_record_refill(uint32_t class_idx, uint64_t capacity) {
    if (!learner_v7_enabled()) return;
    if (class_idx >= 8) return;

    // Record stats: refill indicates page was needed (current page exhausted)
    SmallLearnerClassStatsV7* cls = &g_small_learner_stats_v7.per_class[class_idx];
    cls->v7_allocs += capacity;  // Use capacity as proxy for traffic volume
    cls->sample_count++;

    g_small_learner_total_refills++;

    // Periodic evaluation (on refills)
    if (g_small_learner_total_refills % SMALL_LEARNER_EVAL_INTERVAL == 0) {
        small_learner_v7_evaluate();
    }
}

void small_learner_v7_record_retire(uint32_t class_idx, uint64_t capacity) {
    if (!learner_v7_enabled()) return;
    if (class_idx >= 8) return;

    // Record stats (atomic would be better for multi-thread, but keep simple for now)
    // Note: v7-5a removed per-page alloc_count from hot path, so we use capacity instead
    // capacity represents "slots available on page" which approximates traffic volume
    SmallLearnerClassStatsV7* cls = &g_small_learner_stats_v7.per_class[class_idx];
    cls->v7_retires++;

    g_small_learner_stats_v7.total_retires++;

    (void)capacity;  // Not used for now (kept for API compatibility)
}

void small_policy_v7_update_from_learner(
    const SmallLearnerStatsV7* stats,
    SmallPolicyV7* policy_out
) {
    if (!stats || !policy_out) return;

    // Calculate total allocs across all classes
    uint64_t total_allocs = 0;
    for (int i = 0; i < 8; i++) {
        total_allocs += stats->per_class[i].v7_allocs;
    }

    if (total_allocs == 0) return;  // No data yet

    // C5 decision: if C5 ratio > threshold, route to v7, else MID_v3
    uint64_t c5_allocs = stats->per_class[5].v7_allocs;
    uint64_t c5_ratio_pct = (c5_allocs * 100) / total_allocs;

    SmallRouteKind old_c5_route = policy_out->route_kind[5];
    SmallRouteKind new_c5_route;

    if (c5_ratio_pct >= SMALL_LEARNER_C5_THRESHOLD_PCT) {
        // C5-heavy workload → keep C5 on v7
        new_c5_route = SMALL_ROUTE_V7;
    } else {
        // Mixed workload → move C5 to MID_v3
        new_c5_route = SMALL_ROUTE_MID_V3;
    }

    // Only log and update if route changed
    if (old_c5_route != new_c5_route) {
        // Log only the first switch (to avoid spam)
        static int g_learner_v7_switch_logged = 0;
        if (!g_learner_v7_switch_logged) {
            g_learner_v7_switch_logged = 1;
            fprintf(stderr, "[LEARNER_V7] C5 route switch: %s → %s (C5 ratio=%lu%%, threshold=%d%%)\n",
                    small_route_kind_name(old_c5_route),
                    small_route_kind_name(new_c5_route),
                    (unsigned long)c5_ratio_pct,
                    SMALL_LEARNER_C5_THRESHOLD_PCT);
        }
        policy_out->route_kind[5] = new_c5_route;
    }
}

const SmallLearnerStatsV7* small_learner_v7_stats_snapshot(void) {
    return &g_small_learner_stats_v7;
}

void small_learner_v7_evaluate(void) {
    if (!learner_v7_enabled()) return;

    // Increment global version to invalidate all TLS caches
    // Next call to small_policy_v7_snapshot() will re-apply Learner updates
    __sync_fetch_and_add(&g_policy_v7_version, 1);

    g_small_learner_stats_v7.eval_count++;
}