129 lines
5.0 KiB
C
129 lines
5.0 KiB
C
|
// hakmem_tiny_superslab_internal.h - Internal declarations for superslab refactor
|
||
|
|
// Purpose: Shared declarations between superslab implementation files
|
||
|
|
// License: MIT
|
||
|
|
// Date: 2025-11-28
|
||
|
|
|
||
|
|
#ifndef HAKMEM_TINY_SUPERSLAB_INTERNAL_H
|
||
|
|
#define HAKMEM_TINY_SUPERSLAB_INTERNAL_H
|
||
|
|
|
||
|
|
#include "hakmem_tiny_superslab.h"
|
||
|
|
#include "box/ss_hot_cold_box.h"
|
||
|
|
#include "hakmem_super_registry.h"
|
||
|
|
#include "hakmem_tiny.h"
|
||
|
|
#include "hakmem_tiny_config.h"
|
||
|
|
#include "hakmem_shared_pool.h"
|
||
|
|
#include "hakmem_internal.h"
|
||
|
|
#include "tiny_region_id.h"
|
||
|
|
#include "hakmem_tiny_integrity.h"
|
||
|
|
#include "box/tiny_next_ptr_box.h"
|
||
|
|
#include "box/slab_freelist_atomic.h"
|
||
|
|
#include <sys/mman.h>
|
||
|
|
#include <sys/resource.h>
|
||
|
|
#include <errno.h>
|
||
|
|
#include <string.h>
|
||
|
|
#include <stdio.h>
|
||
|
|
#include <stdlib.h>
|
||
|
|
#include <pthread.h>
|
||
|
|
#include <unistd.h>
|
||
|
|
|
||
|
|
// ============================================================================
|
||
|
|
// Global Variables (defined in superslab_stats.c)
|
||
|
|
// ============================================================================
|
||
|
|
|
||
|
|
extern pthread_mutex_t g_superslab_lock;
|
||
|
|
extern uint64_t g_superslabs_allocated;
|
||
|
|
extern uint64_t g_superslabs_freed;
|
||
|
|
extern uint64_t g_bytes_allocated;
|
||
|
|
extern _Atomic uint64_t g_ss_active_dec_calls;
|
||
|
|
extern _Atomic uint64_t g_hak_tiny_free_calls;
|
||
|
|
extern _Atomic uint64_t g_ss_remote_push_calls;
|
||
|
|
extern _Atomic uint64_t g_free_ss_enter;
|
||
|
|
extern _Atomic uint64_t g_free_local_box_calls;
|
||
|
|
extern _Atomic uint64_t g_free_remote_box_calls;
|
||
|
|
extern uint64_t g_ss_alloc_by_class[8];
|
||
|
|
extern uint64_t g_ss_freed_by_class[8];
|
||
|
|
extern _Atomic uint64_t g_ss_mmap_count;
|
||
|
|
extern _Atomic uint64_t g_final_fallback_mmap_count;
|
||
|
|
|
||
|
|
// ============================================================================
|
||
|
|
// SuperSlabHead Management (defined in superslab_head.c)
|
||
|
|
// ============================================================================
|
||
|
|
|
||
|
|
extern SuperSlabHead* g_superslab_heads[TINY_NUM_CLASSES_SS];
|
||
|
|
|
||
|
|
// ============================================================================
|
||
|
|
// Cache System (defined in superslab_cache.c)
|
||
|
|
// ============================================================================
|
||
|
|
|
||
|
|
typedef struct SuperslabCacheEntry {
|
||
|
|
struct SuperslabCacheEntry* next;
|
||
|
|
} SuperslabCacheEntry;
|
||
|
|
|
||
|
|
extern SuperslabCacheEntry* g_ss_cache_head[8];
|
||
|
|
extern size_t g_ss_cache_count[8];
|
||
|
|
extern size_t g_ss_cache_cap[8];
|
||
|
|
extern size_t g_ss_precharge_target[8];
|
||
|
|
extern _Atomic int g_ss_precharge_done[8];
|
||
|
|
extern int g_ss_cache_enabled;
|
||
|
|
extern pthread_once_t g_ss_cache_once;
|
||
|
|
extern pthread_mutex_t g_ss_cache_lock[8];
|
||
|
|
extern uint64_t g_ss_cache_hits[8];
|
||
|
|
extern uint64_t g_ss_cache_misses[8];
|
||
|
|
extern uint64_t g_ss_cache_puts[8];
|
||
|
|
extern uint64_t g_ss_cache_drops[8];
|
||
|
|
extern uint64_t g_ss_cache_precharged[8];
|
||
|
|
extern uint64_t g_superslabs_reused;
|
||
|
|
extern uint64_t g_superslabs_cached;
|
||
|
|
|
||
|
|
// Cache functions (defined in superslab_cache.c)
|
||
|
|
void ss_cache_global_init(void);
|
||
|
|
void ss_cache_ensure_init(void);
|
||
|
|
void* ss_os_acquire(uint8_t size_class, size_t ss_size, uintptr_t ss_mask, int populate);
|
||
|
|
void ss_cache_precharge(uint8_t size_class, size_t ss_size, uintptr_t ss_mask);
|
||
|
|
SuperslabCacheEntry* ss_cache_pop(uint8_t size_class);
|
||
|
|
int ss_cache_push(uint8_t size_class, SuperSlab* ss);
|
||
|
|
|
||
|
|
// ============================================================================
|
||
|
|
// ACE (Adaptive Cache Engine) - defined in superslab_ace.c
|
||
|
|
// ============================================================================
|
||
|
|
|
||
|
|
extern SuperSlabACEState g_ss_ace[TINY_NUM_CLASSES_SS];
|
||
|
|
extern int g_ss_force_lg;
|
||
|
|
extern _Atomic int g_ss_populate_once;
|
||
|
|
|
||
|
|
uint8_t hak_tiny_superslab_next_lg(int class_idx);
|
||
|
|
void ace_observe_and_decide(int k);
|
||
|
|
|
||
|
|
// ============================================================================
|
||
|
|
// Statistics (defined in superslab_stats.c)
|
||
|
|
// ============================================================================
|
||
|
|
|
||
|
|
void ss_stats_os_alloc(uint8_t size_class, size_t ss_size);
|
||
|
|
void ss_stats_cache_reuse(void);
|
||
|
|
void ss_stats_cache_store(void);
|
||
|
|
void log_superslab_oom_once(size_t ss_size, size_t alloc_size, int err);
|
||
|
|
|
||
|
|
// ============================================================================
|
||
|
|
// Slab Management (defined in superslab_slab.c)
|
||
|
|
// ============================================================================
|
||
|
|
|
||
|
|
// Drain remote MPSC stack into freelist (ownership already verified by caller)
|
||
|
|
void _ss_remote_drain_to_freelist_unsafe(SuperSlab* ss, int slab_idx, TinySlabMeta* meta);
|
||
|
|
|
||
|
|
// ============================================================================
|
||
|
|
// Backend Allocation (defined in superslab_backend.c)
|
||
|
|
// ============================================================================
|
||
|
|
|
||
|
|
void* hak_tiny_alloc_superslab_backend_legacy(int class_idx);
|
||
|
|
void* hak_tiny_alloc_superslab_backend_shared(int class_idx);
|
||
|
|
|
||
|
|
// ============================================================================
|
||
|
|
// SuperSlabHead Management (defined in superslab_head.c)
|
||
|
|
// ============================================================================
|
||
|
|
|
||
|
|
SuperSlabHead* init_superslab_head(int class_idx);
|
||
|
|
int expand_superslab_head(SuperSlabHead* head);
|
||
|
|
SuperSlab* find_chunk_for_ptr(void* ptr, int class_idx);
|
||
|
|
|
||
|
|
#endif // HAKMEM_TINY_SUPERSLAB_INTERNAL_H
|