diff --git a/core/box/pool_mid_inuse_deferred_box.h b/core/box/pool_mid_inuse_deferred_box.h index 8512f058..9536e802 100644 --- a/core/box/pool_mid_inuse_deferred_box.h +++ b/core/box/pool_mid_inuse_deferred_box.h @@ -36,6 +36,37 @@ extern int hak_batch_add_page(void* page, size_t size); // Forward declaration of drain (needed by mid_inuse_dec_deferred) static inline void mid_inuse_deferred_drain(void); +// ============================================================================ +// Hash-based insert (O(1) expected time) +// ============================================================================ +// Returns: 1 on success, 0 if probe limit exceeded (caller should drain) +static inline int mid_inuse_dec_deferred_hash(void* page) { + MidInuseTlsPageMap* map = &g_mid_inuse_tls_map; + uint32_t h = mid_inuse_hash_page(page); + + for (int probe = 0; probe < MID_INUSE_MAX_PROBE; probe++) { + uint32_t idx = (h + probe) & MID_INUSE_HASH_MAP_MASK; + + if (map->hash_pages[idx] == page) { + // Found existing entry + map->hash_counts[idx]++; + return 1; // Success + } + + if (map->hash_pages[idx] == NULL) { + // Empty slot, insert new entry + map->hash_pages[idx] = page; + map->hash_counts[idx] = 1; + map->hash_used++; + return 1; // Success + } + // Collision, continue probing + } + + // Max probe exceeded, need drain + return 0; // Failed, caller should drain and retry +} + // Thread exit cleanup (ensures all deferred ops are processed) static void mid_inuse_deferred_thread_cleanup(void* arg) { (void)arg; @@ -83,6 +114,19 @@ static inline void mid_inuse_dec_deferred(void* raw) { // Calculate page base (POOL_PAGE_SIZE = 64KB, power of 2) void* page = (void*)((uintptr_t)raw & ~((uintptr_t)POOL_PAGE_SIZE - 1)); + // Check if hash mode is enabled + if (hak_pool_mid_inuse_map_hash_enabled()) { + // Hash mode: O(1) expected + if (!mid_inuse_dec_deferred_hash(page)) { + // Probe limit exceeded, drain and retry + mid_inuse_deferred_drain(); + mid_inuse_dec_deferred_hash(page); // Must succeed after drain + } + MID_INUSE_DEFERRED_STAT_INC(mid_inuse_deferred_hit); + return; + } + + // Linear mode (existing code) // Search TLS map for existing page entry MidInuseTlsPageMap* map = &g_mid_inuse_tls_map; @@ -134,7 +178,44 @@ static inline void mid_inuse_deferred_drain(void) { // Track drain call MID_INUSE_DEFERRED_STAT_INC(drain_calls); - // Process each entry in map + if (hak_pool_mid_inuse_map_hash_enabled()) { + // Hash mode: scan all slots + for (uint32_t i = 0; i < MID_INUSE_HASH_MAP_SIZE; i++) { + if (map->hash_pages[i] != NULL) { + void* page = map->hash_pages[i]; + uint32_t n = map->hash_counts[i]; + + // ONLY lookup happens here (once per page, not once per free) + MidPageDesc* d = mid_desc_lookup(page); + if (d) { + // Track statistics + MID_INUSE_DEFERRED_STAT_INC(pages_drained); + MID_INUSE_DEFERRED_STAT_ADD(decs_drained, n); + + // Atomic subtract (batched count) + uint64_t old = atomic_fetch_sub_explicit(&d->in_use, n, memory_order_relaxed); + + // Check for empty transition + if (old >= n && old - n == 0) { + // Use atomic_exchange to ensure only ONE thread enqueues DONTNEED + if (d->pending_dn == 0) { + d->pending_dn = 1; + MID_INUSE_DEFERRED_STAT_INC(empty_transitions); + hak_batch_add_page(page, POOL_PAGE_SIZE); + } + } + } + + // Clear slot + map->hash_pages[i] = NULL; + map->hash_counts[i] = 0; + } + } + map->hash_used = 0; + return; + } + + // Linear mode: Process each entry in map for (uint32_t i = 0; i < map->used; i++) { void* page = map->pages[i]; uint32_t n = map->counts[i]; @@ -143,8 +224,9 @@ static inline void mid_inuse_deferred_drain(void) { MidPageDesc* d = mid_desc_lookup(page); if (!d) continue; - // Track pages drained - MID_INUSE_DEFERRED_STAT_ADD(pages_drained, n); + // Track statistics + MID_INUSE_DEFERRED_STAT_INC(pages_drained); + MID_INUSE_DEFERRED_STAT_ADD(decs_drained, n); // Atomic subtract (batched count) int old = atomic_fetch_sub_explicit(&d->in_use, (int)n, memory_order_relaxed); diff --git a/core/box/pool_mid_inuse_deferred_env_box.h b/core/box/pool_mid_inuse_deferred_env_box.h index a6d0d979..9b6b94eb 100644 --- a/core/box/pool_mid_inuse_deferred_env_box.h +++ b/core/box/pool_mid_inuse_deferred_env_box.h @@ -9,6 +9,7 @@ #define POOL_MID_INUSE_DEFERRED_ENV_BOX_H #include +#include // Check if deferred inuse_dec is enabled via ENV // Default: 0 (disabled) for safety @@ -22,4 +23,17 @@ static inline int hak_pool_mid_inuse_deferred_enabled(void) { return g; } +// Check if hash-based map is enabled via ENV +// HAKMEM_POOL_MID_INUSE_MAP_KIND=hash|linear (default: linear) +// Hash mode: O(1) open addressing with linear probing +// Linear mode: O(N) linear search with last_idx cache +static inline int hak_pool_mid_inuse_map_hash_enabled(void) { + static int g = -1; + if (__builtin_expect(g == -1, 0)) { + const char* e = getenv("HAKMEM_POOL_MID_INUSE_MAP_KIND"); + g = (e && strcmp(e, "hash") == 0) ? 1 : 0; + } + return g; +} + #endif // POOL_MID_INUSE_DEFERRED_ENV_BOX_H diff --git a/core/box/pool_mid_inuse_deferred_stats_box.h b/core/box/pool_mid_inuse_deferred_stats_box.h index 7031556b..da23e49d 100644 --- a/core/box/pool_mid_inuse_deferred_stats_box.h +++ b/core/box/pool_mid_inuse_deferred_stats_box.h @@ -22,7 +22,8 @@ typedef struct { _Atomic uint64_t mid_inuse_deferred_hit; // Total deferred decrements _Atomic uint64_t drain_calls; // Number of drain calls - _Atomic uint64_t pages_drained; // Total pages drained + _Atomic uint64_t pages_drained; // Total unique pages drained + _Atomic uint64_t decs_drained; // Total dec operations drained _Atomic uint64_t empty_transitions; // Pages that went to 0 } MidInuseDeferredStats; @@ -45,6 +46,7 @@ static void mid_inuse_deferred_stats_dump(void) { uint64_t hits = atomic_load_explicit(&g_mid_inuse_deferred_stats.mid_inuse_deferred_hit, memory_order_relaxed); uint64_t drains = atomic_load_explicit(&g_mid_inuse_deferred_stats.drain_calls, memory_order_relaxed); uint64_t pages = atomic_load_explicit(&g_mid_inuse_deferred_stats.pages_drained, memory_order_relaxed); + uint64_t decs = atomic_load_explicit(&g_mid_inuse_deferred_stats.decs_drained, memory_order_relaxed); uint64_t empties = atomic_load_explicit(&g_mid_inuse_deferred_stats.empty_transitions, memory_order_relaxed); if (hits > 0 || drains > 0) { @@ -52,9 +54,11 @@ static void mid_inuse_deferred_stats_dump(void) { fprintf(stderr, "Deferred hits: %lu\n", hits); fprintf(stderr, "Drain calls: %lu\n", drains); fprintf(stderr, "Pages drained: %lu\n", pages); + fprintf(stderr, "Decs drained: %lu\n", decs); fprintf(stderr, "Empty transitions: %lu\n", empties); if (drains > 0) { fprintf(stderr, "Avg pages/drain: %.2f\n", (double)pages / (double)drains); + fprintf(stderr, "Avg decs/drain: %.2f\n", (double)decs / (double)drains); } fprintf(stderr, "================================\n"); } diff --git a/core/box/pool_mid_inuse_tls_pagemap_box.h b/core/box/pool_mid_inuse_tls_pagemap_box.h index 900d50ae..687210ec 100644 --- a/core/box/pool_mid_inuse_tls_pagemap_box.h +++ b/core/box/pool_mid_inuse_tls_pagemap_box.h @@ -8,24 +8,51 @@ // - 32 entries is a good balance: covers typical working set without overflow // - Linear search is cache-friendly for small N (better than hash for <64 entries) // - Per-thread, so no locking needed +// +// Hash Mode (NEW): +// - 64 slots open addressing hash table for O(1) lookup +// - Linear probing with max 8 probes before drain +// - NULL sentinel = empty slot #ifndef POOL_MID_INUSE_TLS_PAGEMAP_BOX_H #define POOL_MID_INUSE_TLS_PAGEMAP_BOX_H #include -// TLS map size: 32 entries +// TLS map size: 32 entries (linear mode) // Rationale: Typical thread working set ~16-32 pages, allows for variance #define MID_INUSE_TLS_MAP_SIZE 32 +// Hash table size: 64 entries (hash mode) +// Power-of-two for fast modulo via mask +#define MID_INUSE_HASH_MAP_SIZE 64 +#define MID_INUSE_HASH_MAP_MASK (MID_INUSE_HASH_MAP_SIZE - 1) +#define MID_INUSE_MAX_PROBE 8 + // TLS map structure: Parallel arrays for cache locality typedef struct { + // Linear mode (existing) void* pages[MID_INUSE_TLS_MAP_SIZE]; // Page base addresses uint32_t counts[MID_INUSE_TLS_MAP_SIZE]; // Pending dec count per page uint32_t used; // Number of active entries uint32_t last_idx; // Cache last hit index for temporal locality + + // Hash mode (new) + void* hash_pages[MID_INUSE_HASH_MAP_SIZE]; // NULL = empty slot + uint32_t hash_counts[MID_INUSE_HASH_MAP_SIZE]; // Pending dec count per hash slot + uint32_t hash_used; // Count of non-empty slots } MidInuseTlsPageMap; +// Simple hash function for page addresses +// Uses golden ratio multiplication and right shift to remove page-aligned bits +static inline uint32_t mid_inuse_hash_page(void* page) { + uintptr_t p = (uintptr_t)page; + // Shift right to remove low bits (page-aligned), mix with golden ratio + p = p >> 16; // Remove low bits (page size is 64KB = 2^16) + p = p * 2654435761u; // Golden ratio hash constant + return (uint32_t)p; +} + // Thread-local instance (zero-initialized by default) static __thread MidInuseTlsPageMap g_mid_inuse_tls_map;