hakmem/core/hakmem_syscall.c

// hakmem_syscall.c - Direct libc calls via dlsym (Box 3)
//
// Purpose: Bypass LD_PRELOAD to avoid infinite recursion
//
// How it works:
//   1. dlsym(RTLD_NEXT, "malloc") finds the NEXT symbol after current one
//   2. In LD_PRELOAD context: hakmem.so → libc.so (skips our wrapper!)
//   3. Result: Direct libc call, no recursion possible
//
// Performance impact:
//   - dlsym: Called ONCE at init (cached in function pointers)
//   - hkm_libc_malloc(): Direct function call (no overhead)
//   - No TLS access, no guards, no recursion checks
//
// License: MIT
// Date: 2025-10-24

#define _GNU_SOURCE
#include "hakmem_syscall.h"
#include <dlfcn.h>
#include <stdio.h>
#include <stdlib.h>

// ============================================================================
// Function Pointers (initialized once at startup)
// ============================================================================

// Real libc functions (bypassing LD_PRELOAD)
static void* (*real_malloc)(size_t) = NULL;
static void* (*real_calloc)(size_t, size_t) = NULL;
static void  (*real_free)(void*) = NULL;
static void* (*real_realloc)(void*, size_t) = NULL;

// Initialization flag
static int g_syscall_initialized = 0;

// ============================================================================
// Initialization
// ============================================================================

void hkm_syscall_init(void) {
    // Idempotent: safe to call multiple times
    if (g_syscall_initialized) {
        return;
    }

    // dlsym with RTLD_NEXT: Get NEXT symbol in library chain
    // In LD_PRELOAD context: libhakmem.so → libc.so (skips our malloc!)
    real_malloc = dlsym(RTLD_NEXT, "malloc");
    real_calloc = dlsym(RTLD_NEXT, "calloc");
    real_free = dlsym(RTLD_NEXT, "free");
    real_realloc = dlsym(RTLD_NEXT, "realloc");

    // Fatal error if dlsym fails (should never happen)
    if (!real_malloc || !real_calloc || !real_free || !real_realloc) {
        fprintf(stderr, "[hakmem_syscall] FATAL: dlsym failed\n");
        fprintf(stderr, "  real_malloc=%p\n", (void*)real_malloc);
        fprintf(stderr, "  real_calloc=%p\n", (void*)real_calloc);
        fprintf(stderr, "  real_free=%p\n", (void*)real_free);
        fprintf(stderr, "  real_realloc=%p\n", (void*)real_realloc);
        abort();
    }

    g_syscall_initialized = 1;

    // Optional: Debug logging (only in debug builds)
    #ifdef HAKMEM_DEBUG_VERBOSE
    fprintf(stderr, "[hakmem_syscall] Initialized successfully\n");
    fprintf(stderr, "  real_malloc=%p\n", (void*)real_malloc);
    fprintf(stderr, "  real_calloc=%p\n", (void*)real_calloc);
    fprintf(stderr, "  real_free=%p\n", (void*)real_free);
    fprintf(stderr, "  real_realloc=%p\n", (void*)real_realloc);
    #endif
}

// ============================================================================
// Direct libc Functions (Box 3)
// ============================================================================

void* hkm_libc_malloc(size_t size) {
    // Auto-initialize on first use (defensive programming)
    if (!real_malloc) {
        hkm_syscall_init();
    }

    // Direct call to libc malloc (no LD_PRELOAD, no recursion!)
    return real_malloc(size);
}

void* hkm_libc_calloc(size_t nmemb, size_t size) {
    // Auto-initialize on first use
    if (!real_calloc) {
        hkm_syscall_init();
    }

    // Direct call to libc calloc
    return real_calloc(nmemb, size);
}

void hkm_libc_free(void* ptr) {
    // Auto-initialize on first use
    if (!real_free) {
        hkm_syscall_init();
    }

    // Direct call to libc free
    real_free(ptr);
}

void* hkm_libc_realloc(void* ptr, size_t size) {
    // Auto-initialize on first use
    if (!real_realloc) {
        hkm_syscall_init();
    }

    // Direct call to libc realloc
    return real_realloc(ptr, size);
}