diff --git a/core/smallobject_hotbox_v5.c b/core/smallobject_hotbox_v5.c index 20786902..c2aa1afe 100644 --- a/core/smallobject_hotbox_v5.c +++ b/core/smallobject_hotbox_v5.c @@ -103,60 +103,42 @@ void* small_alloc_fast_v5(size_t size, uint32_t class_idx, SmallHeapCtxV5* ctx) } // ============================================================================ -// Helper: Find page containing pointer +// Helper: Determine page location in heap lists (Phase v5-3) // ============================================================================ -static inline int ptr_in_page(const SmallPageMetaV5* page, const uint8_t* ptr) { - if (!page || !ptr || !page->segment) return 0; - - SmallSegmentV5* seg = (SmallSegmentV5*)page->segment; - uintptr_t page_base = seg->base + ((uintptr_t)page->page_idx * SMALL_SEGMENT_V5_PAGE_SIZE); - size_t span = (size_t)page->capacity * SMALL_HEAP_V5_C6_BLOCK_SIZE; - - if ((uintptr_t)ptr < page_base || (uintptr_t)ptr >= page_base + span) return 0; - - // Check alignment - size_t off = (uintptr_t)ptr - page_base; - return (off % SMALL_HEAP_V5_C6_BLOCK_SIZE) == 0; -} - -static SmallPageMetaV5* find_page(SmallClassHeapV5* h, const uint8_t* ptr, - page_loc_t* loc, SmallPageMetaV5** prev_out) { - if (loc) *loc = LOC_NONE; +static inline page_loc_t get_page_location(SmallClassHeapV5* h, SmallPageMetaV5* page, + SmallPageMetaV5** prev_out) { if (prev_out) *prev_out = NULL; - if (!h || !ptr) return NULL; + if (!h || !page) return LOC_NONE; - // Check current - if (h->current && ptr_in_page(h->current, ptr)) { - if (loc) *loc = LOC_CURRENT; - return h->current; + // Check current (O(1)) + if (h->current == page) { + return LOC_CURRENT; } - // Check partial list + // Check partial list (typically 0-1 pages in v5-3) SmallPageMetaV5* prev = NULL; for (SmallPageMetaV5* p = h->partial_head; p; prev = p, p = p->next) { - if (ptr_in_page(p, ptr)) { - if (loc) *loc = LOC_PARTIAL; + if (p == page) { if (prev_out) *prev_out = prev; - return p; + return LOC_PARTIAL; } } // Check full list prev = NULL; for (SmallPageMetaV5* p = h->full_head; p; prev = p, p = p->next) { - if (ptr_in_page(p, ptr)) { - if (loc) *loc = LOC_FULL; + if (p == page) { if (prev_out) *prev_out = prev; - return p; + return LOC_FULL; } } - return NULL; + return LOC_NONE; } // ============================================================================ -// Phase v5-2: Fast free (C6-only full implementation) +// Phase v5-3: Fast free (C6-only O(1) implementation) // ============================================================================ void small_free_fast_v5(void* ptr, uint32_t class_idx, SmallHeapCtxV5* ctx) { @@ -166,37 +148,21 @@ void small_free_fast_v5(void* ptr, uint32_t class_idx, SmallHeapCtxV5* ctx) { // C6-only check if (unlikely(class_idx != SMALL_HEAP_V5_C6_CLASS_IDX)) { - // Fallback to pool v1 for non-C6 classes + hak_pool_free(ptr, 0, 0); + return; + } + + // Phase v5-3: O(1) segment lookup (no list search) + SmallPageMetaV5* page = small_segment_v5_page_meta_of(ptr); + if (unlikely(!page)) { + // Not in v5 segment, fallback to pool v1 hak_pool_free(ptr, 0, 0); return; } SmallClassHeapV5* h = &ctx->cls[SMALL_HEAP_V5_C6_CLASS_IDX]; - // Try O(1) segment lookup first (Phase v5-2 optimization) - SmallPageMetaV5* page = small_segment_v5_page_meta_of(ptr); - page_loc_t loc = LOC_NONE; - SmallPageMetaV5* prev = NULL; - - // If segment lookup failed, search through lists (fallback) - if (!page) { - page = find_page(h, (const uint8_t*)ptr, &loc, &prev); - if (!page) { - // Not found in v5 heap, fallback to pool v1 - hak_pool_free(ptr, 0, 0); - return; - } - } else { - // Segment lookup succeeded, determine location in lists - if (h->current == page) { - loc = LOC_CURRENT; - } else { - // Search in partial/full lists to get prev pointer - find_page(h, (const uint8_t*)ptr, &loc, &prev); - } - } - - // Push to freelist + // Push to freelist (O(1)) void* head = page->free_list; memcpy(ptr, &head, sizeof(void*)); page->free_list = ptr; @@ -206,50 +172,55 @@ void small_free_fast_v5(void* ptr, uint32_t class_idx, SmallHeapCtxV5* ctx) { // Handle empty page (used == 0) if (page->used == 0) { - // Unlink from current location - if (loc != LOC_CURRENT) { + // Fast path: if this is current, just keep it + if (h->current == page) { + return; + } + + // Determine location and unlink (rare path) + SmallPageMetaV5* prev = NULL; + page_loc_t loc = get_page_location(h, page, &prev); + if (loc != LOC_NONE && loc != LOC_CURRENT) { SMALL_PAGE_V5_UNLINK(h, loc, prev, page); } - // Try to make it current if we don't have one + // Promote to current if empty if (!h->current) { h->current = page; page->next = NULL; return; } - // Already have current, check if we can keep in partial - if (h->current == page) { - page->next = NULL; - return; - } - - // Try to push to partial list + // Try partial (limit 1) if (h->partial_count < SMALL_HEAP_V5_C6_PARTIAL_LIMIT) { SMALL_PAGE_V5_PUSH_PARTIAL(h, page); return; } - // Partial list full, retire the page + // Retire to cold small_cold_v5_retire_page(ctx, page); return; } - // Page is not empty, handle transitions - if (!h->current) { - // No current page, promote this one - if (loc != LOC_CURRENT) { + // Page not empty - handle full→partial transition + if (h->current != page) { + SmallPageMetaV5* prev = NULL; + page_loc_t loc = get_page_location(h, page, &prev); + if (loc == LOC_FULL && page->free_list) { + // Move from full to partial SMALL_PAGE_V5_UNLINK(h, loc, prev, page); - } - h->current = page; - page->next = NULL; - } else if (loc == LOC_FULL && page->free_list) { - // Move from full to partial (now has free blocks) - SMALL_PAGE_V5_UNLINK(h, loc, prev, page); - if (h->partial_count < SMALL_HEAP_V5_C6_PARTIAL_LIMIT) { - SMALL_PAGE_V5_PUSH_PARTIAL(h, page); - } else { - SMALL_PAGE_V5_PUSH_FULL(h, page); // Keep in full if partial limit exceeded + if (h->partial_count < SMALL_HEAP_V5_C6_PARTIAL_LIMIT) { + SMALL_PAGE_V5_PUSH_PARTIAL(h, page); + } else { + SMALL_PAGE_V5_PUSH_FULL(h, page); + } + } else if (!h->current) { + // No current, promote this + if (loc != LOC_NONE) { + SMALL_PAGE_V5_UNLINK(h, loc, prev, page); + } + h->current = page; + page->next = NULL; } } } diff --git a/core/smallsegment_v5.c b/core/smallsegment_v5.c index 14bf12cc..8d5ab7a1 100644 --- a/core/smallsegment_v5.c +++ b/core/smallsegment_v5.c @@ -19,29 +19,22 @@ // Segment Allocation (Phase v5-2) // ============================================================================ -// Thread-local segment list (static allocation to avoid malloc recursion) -#define MAX_SEGMENTS_PER_THREAD 4 +// Thread-local segment (Phase v5-3: single segment per thread for O(1) lookup) +// C6-only v5 uses at most 1 segment per thread - this eliminates slot search typedef struct { SmallSegmentV5 seg; int in_use; uint32_t used_pages; // Bitmap: which pages are currently in use } TLSSegmentSlot; -static __thread TLSSegmentSlot g_segment_slots_v5[MAX_SEGMENTS_PER_THREAD]; -static __thread int g_last_alloc_slot_v5 = -1; // Last slot we allocated from +static __thread TLSSegmentSlot g_tls_segment_v5; // Single TLS segment SmallSegmentV5* small_segment_v5_acquire(void) { - // Find free slot in TLS (avoid malloc to prevent recursion) - TLSSegmentSlot* slot = NULL; - for (int i = 0; i < MAX_SEGMENTS_PER_THREAD; i++) { - if (!g_segment_slots_v5[i].in_use) { - slot = &g_segment_slots_v5[i]; - break; - } - } + // Phase v5-3: Single segment per thread - no slot search needed + TLSSegmentSlot* slot = &g_tls_segment_v5; - if (!slot) { - return NULL; // Out of TLS segment slots + if (slot->in_use) { + return NULL; // Already have a segment (reuse pages instead) } // Allocate 2MiB aligned segment via mmap @@ -138,17 +131,9 @@ void small_segment_v5_release(SmallSegmentV5* seg) { // Release the 2MiB backing memory munmap((void*)seg->base, SMALL_SEGMENT_V5_SIZE); - // Mark slot as free (TLS memory is never freed, just reused) - for (int i = 0; i < MAX_SEGMENTS_PER_THREAD; i++) { - if (&g_segment_slots_v5[i].seg == seg) { - g_segment_slots_v5[i].in_use = 0; - g_segment_slots_v5[i].used_pages = 0; - if (g_last_alloc_slot_v5 == i) { - g_last_alloc_slot_v5 = -1; - } - break; - } - } + // Phase v5-3: Single segment - direct reset + g_tls_segment_v5.in_use = 0; + g_tls_segment_v5.used_pages = 0; } // ============================================================================ @@ -156,55 +141,31 @@ void small_segment_v5_release(SmallSegmentV5* seg) { // ============================================================================ SmallPageMetaV5* small_segment_v5_alloc_page(void) { - // Try to reuse existing segment with free pages - if (g_last_alloc_slot_v5 >= 0 && g_last_alloc_slot_v5 < MAX_SEGMENTS_PER_THREAD) { - TLSSegmentSlot* slot = &g_segment_slots_v5[g_last_alloc_slot_v5]; - // Check if not all pages are used (used_pages != 0xFFFFFFFF for 32 pages) - if (slot->in_use && slot->used_pages != 0xFFFFFFFF) { - // This segment has free pages - SmallSegmentV5* seg = &slot->seg; - for (uint32_t i = 0; i < seg->num_pages; i++) { - if ((slot->used_pages & (1U << i)) == 0) { - // Found free page - slot->used_pages |= (1U << i); - return &seg->page_meta[i]; - } - } + TLSSegmentSlot* slot = &g_tls_segment_v5; + + // Phase v5-3: Single segment - direct access, no slot search + if (slot->in_use && slot->used_pages != 0xFFFFFFFF) { + // Segment has free pages - use __builtin_ctz for O(1) free page find + uint32_t free_mask = ~slot->used_pages; + if (free_mask) { + uint32_t page_idx = (uint32_t)__builtin_ctz(free_mask); + slot->used_pages |= (1U << page_idx); + return &slot->seg.page_meta[page_idx]; } } - // Search all slots for a segment with free pages - for (int s = 0; s < MAX_SEGMENTS_PER_THREAD; s++) { - TLSSegmentSlot* slot = &g_segment_slots_v5[s]; - if (slot->in_use && slot->used_pages != 0xFFFFFFFF) { - SmallSegmentV5* seg = &slot->seg; - for (uint32_t i = 0; i < seg->num_pages; i++) { - if ((slot->used_pages & (1U << i)) == 0) { - // Found free page - slot->used_pages |= (1U << i); - g_last_alloc_slot_v5 = s; - return &seg->page_meta[i]; - } - } + // Need new segment + if (!slot->in_use) { + SmallSegmentV5* seg = small_segment_v5_acquire(); + if (!seg) { + return NULL; } + // First page + slot->used_pages |= 1U; + return &seg->page_meta[0]; } - // No free pages in existing segments, allocate new segment - SmallSegmentV5* seg = small_segment_v5_acquire(); - if (!seg) { - return NULL; - } - - // Mark first page as used - for (int s = 0; s < MAX_SEGMENTS_PER_THREAD; s++) { - if (&g_segment_slots_v5[s].seg == seg) { - g_segment_slots_v5[s].used_pages |= 1U; // Mark page 0 as used - g_last_alloc_slot_v5 = s; - break; - } - } - - return &seg->page_meta[0]; + return NULL; // Segment full (shouldn't happen with page recycling) } void small_segment_v5_free_page(SmallPageMetaV5* page) { @@ -212,22 +173,16 @@ void small_segment_v5_free_page(SmallPageMetaV5* page) { return; } - SmallSegmentV5* seg = (SmallSegmentV5*)page->segment; - - // Find the slot and clear the used bit - for (int s = 0; s < MAX_SEGMENTS_PER_THREAD; s++) { - if (&g_segment_slots_v5[s].seg == seg) { - g_segment_slots_v5[s].used_pages &= ~(1U << page->page_idx); - - // If segment is now empty, we could release it - // For now, keep it for reuse - break; - } + // Phase v5-3: Single segment - direct bitmap clear + TLSSegmentSlot* slot = &g_tls_segment_v5; + if (slot->in_use && page->segment == &slot->seg) { + slot->used_pages &= ~(1U << page->page_idx); + // Keep segment for reuse even if empty } } // ============================================================================ -// O(1) Page Metadata Lookup (Phase v5-2) +// O(1) Page Metadata Lookup (Phase v5-3: Single segment, no search) // ============================================================================ SmallPageMetaV5* small_segment_v5_page_meta_of(void* ptr) { @@ -235,46 +190,35 @@ SmallPageMetaV5* small_segment_v5_page_meta_of(void* ptr) { return NULL; } + // Phase v5-3: Single segment - direct TLS access, no loop + TLSSegmentSlot* slot = &g_tls_segment_v5; + if (unlikely(!slot->in_use)) { + return NULL; + } + + SmallSegmentV5* seg = &slot->seg; uintptr_t addr = (uintptr_t)ptr; - uintptr_t seg_base = addr & ~(SMALL_SEGMENT_V5_SIZE - 1); + uintptr_t seg_base = seg->base; - // Search for segment in TLS slots - SmallSegmentV5* seg = NULL; - for (int i = 0; i < MAX_SEGMENTS_PER_THREAD; i++) { - if (g_segment_slots_v5[i].in_use) { - SmallSegmentV5* candidate = &g_segment_slots_v5[i].seg; - if (candidate->base == seg_base) { - seg = candidate; - break; - } - } - } - - if (unlikely(!seg)) { + // Check if ptr is within this segment's range + if (unlikely(addr < seg_base || addr >= seg_base + SMALL_SEGMENT_V5_SIZE)) { return NULL; } - // Verify magic number (Fail-Fast validation) - if (unlikely(seg->magic != SMALL_SEGMENT_V5_MAGIC)) { - return NULL; - } - - // Compute page index via shift + // Compute page index via shift (O(1)) size_t page_idx = (addr - seg_base) >> SMALL_SEGMENT_V5_PAGE_SHIFT; - // Bounds check + // Bounds check (should always pass if within segment) if (unlikely(page_idx >= seg->num_pages)) { return NULL; } SmallPageMetaV5* page = &seg->page_meta[page_idx]; - // Validate that this page is actually in use (has been allocated) - // Unallocated pages have capacity == 0 + // Validate that this page is actually in use if (unlikely(page->capacity == 0)) { return NULL; } - // Return page metadata return page; }