From cba444b943ded99ae38c31a1d4b4c2dcb0039f3a Mon Sep 17 00:00:00 2001 From: "Moe Charm (CI)" Date: Fri, 12 Dec 2025 23:00:00 +0900 Subject: [PATCH] Phase POOL-MID-DN-BATCH Step 4: Deferred API implementation with thread cleanup --- core/box/pool_mid_inuse_deferred_box.h | 157 +++++++++++++++++++++++++ 1 file changed, 157 insertions(+) create mode 100644 core/box/pool_mid_inuse_deferred_box.h diff --git a/core/box/pool_mid_inuse_deferred_box.h b/core/box/pool_mid_inuse_deferred_box.h new file mode 100644 index 00000000..d3dc4d78 --- /dev/null +++ b/core/box/pool_mid_inuse_deferred_box.h @@ -0,0 +1,157 @@ +// pool_mid_inuse_deferred_box.h — Box: Deferred inuse_dec API +// +// Purpose: Eliminate mid_desc_lookup from hot path by batching decrements +// Pattern: TLS map accumulates page→count, drain performs batched lookup +// Phase: POOL-MID-DN-BATCH Step 3 +// +// Design: +// - Hot path (mid_inuse_dec_deferred): O(1) map search, no lookup/atomic/lock +// - Cold path (mid_inuse_deferred_drain): Batched lookup + atomic subtract +// - Safety: Deferred dec only delays DONTNEED (safe direction) + +#ifndef POOL_MID_INUSE_DEFERRED_BOX_H +#define POOL_MID_INUSE_DEFERRED_BOX_H + +#include "pool_mid_inuse_deferred_env_box.h" +#include "pool_mid_inuse_tls_pagemap_box.h" +#include "pool_mid_inuse_deferred_stats_box.h" +#include +#include +#include + +// Forward declarations (available from hakmem_internal.h / pool includes) +struct MidPageDesc; +typedef struct MidPageDesc MidPageDesc; + +// External functions +extern MidPageDesc* mid_desc_lookup(void* addr); +extern void mid_page_inuse_dec_and_maybe_dn(void* raw); +extern int hak_batch_add_page(void* page, size_t size); + +// POOL_PAGE_SIZE (defined in hakmem_pool.h, typically 64KB) +#ifndef POOL_PAGE_SIZE +#define POOL_PAGE_SIZE (64 * 1024) +#endif + +// Forward declaration of drain (needed by mid_inuse_dec_deferred) +static inline void mid_inuse_deferred_drain(void); + +// Thread exit cleanup (ensures all deferred ops are processed) +static void mid_inuse_deferred_thread_cleanup(void* arg) { + (void)arg; + if (hak_pool_mid_inuse_deferred_enabled()) { + mid_inuse_deferred_drain(); + } +} + +// Global key for thread cleanup +static pthread_key_t g_mid_inuse_deferred_cleanup_key; +static pthread_once_t g_mid_inuse_deferred_key_once = PTHREAD_ONCE_INIT; + +static void mid_inuse_deferred_init_key(void) { + pthread_key_create(&g_mid_inuse_deferred_cleanup_key, mid_inuse_deferred_thread_cleanup); +} + +// Register thread cleanup once per thread +static __thread int g_mid_inuse_deferred_cleanup_registered = 0; +static inline void mid_inuse_deferred_ensure_cleanup(void) { + if (!g_mid_inuse_deferred_cleanup_registered) { + pthread_once(&g_mid_inuse_deferred_key_once, mid_inuse_deferred_init_key); + pthread_setspecific(g_mid_inuse_deferred_cleanup_key, (void*)1); + g_mid_inuse_deferred_cleanup_registered = 1; + } +} + +// ============================================================================ +// Hot API: mid_inuse_dec_deferred - Defer inuse decrement to TLS map +// ============================================================================ +// Called from: hak_pool_free_v1_fast_impl, hak_pool_free_v1_slow_impl +// Replaces: mid_page_inuse_dec_and_maybe_dn(raw) +// +// Fast path: No lookup, no atomic, no lock - just TLS map update +// Map full: Trigger drain (rare, amortized cost) +static inline void mid_inuse_dec_deferred(void* raw) { + // ENV gate: If disabled, fallback to immediate decrement + if (!hak_pool_mid_inuse_deferred_enabled()) { + mid_page_inuse_dec_and_maybe_dn(raw); + return; + } + + // Ensure cleanup is registered (first-time only) + mid_inuse_deferred_ensure_cleanup(); + + // Calculate page base (POOL_PAGE_SIZE = 64KB, power of 2) + void* page = (void*)((uintptr_t)raw & ~((uintptr_t)POOL_PAGE_SIZE - 1)); + + // Search TLS map for existing page entry + MidInuseTlsPageMap* map = &g_mid_inuse_tls_map; + for (uint32_t i = 0; i < map->used; i++) { + if (map->pages[i] == page) { + // Page already in map, increment count + map->counts[i]++; + MID_INUSE_DEFERRED_STAT_INC(mid_inuse_deferred_hit); + return; + } + } + + // New page entry: Check if map is full + if (map->used >= MID_INUSE_TLS_MAP_SIZE) { + // Map full, drain to free space (cold boundary) + mid_inuse_deferred_drain(); + } + + // Add new entry to map + uint32_t idx = map->used++; + map->pages[idx] = page; + map->counts[idx] = 1; + MID_INUSE_DEFERRED_STAT_INC(mid_inuse_deferred_hit); +} + +// ============================================================================ +// Cold API: mid_inuse_deferred_drain - Batch process TLS map +// ============================================================================ +// Called from: +// 1. mid_inuse_dec_deferred (when map full - rare) +// 2. Refill/slow boundaries (future optimization) +// 3. Thread exit (optional, future) +// +// Effect: Performs all pending inuse decrements with SINGLE lookup per page +// Safety: Exact same logic as mid_page_inuse_dec_and_maybe_dn, just batched +static inline void mid_inuse_deferred_drain(void) { + MidInuseTlsPageMap* map = &g_mid_inuse_tls_map; + + // Track drain call + MID_INUSE_DEFERRED_STAT_INC(drain_calls); + + // Process each entry in map + for (uint32_t i = 0; i < map->used; i++) { + void* page = map->pages[i]; + uint32_t n = map->counts[i]; + + // ONLY lookup happens here (once per page, not once per free) + MidPageDesc* d = mid_desc_lookup(page); + if (!d) continue; + + // Track pages drained + MID_INUSE_DEFERRED_STAT_ADD(pages_drained, n); + + // Atomic subtract (batched count) + int old = atomic_fetch_sub_explicit(&d->in_use, (int)n, memory_order_relaxed); + int nv = old - (int)n; + + // Check for empty transition (COPIED from mid_page_inuse_dec_and_maybe_dn) + if (nv <= 0) { + // Fire once per empty transition + // Use atomic_exchange to ensure only ONE thread enqueues DONTNEED + if (atomic_exchange_explicit(&d->pending_dn, 1, memory_order_acq_rel) == 0) { + MID_INUSE_DEFERRED_STAT_INC(empty_transitions); + hak_batch_add_page(d->page, POOL_PAGE_SIZE); + } + } + } + + // Clear map (reset for next batch) + map->used = 0; +} + +#endif // POOL_MID_INUSE_DEFERRED_BOX_H