rtx/src/kv_allocator.c

#include "kv_allocator.h"
#include "cuda_driver_loader.h"

/* --------------------------------------------------------------
 * Global singleton
 * -------------------------------------------------------------- */
sov_kv_allocator_t g_kv_allocator = {0};

/* --------------------------------------------------------------
 * Helpers: manual memory ops (zero-libc)
 * -------------------------------------------------------------- */
static void sov_memset(void* dst, int val, size_t n) {
    unsigned char* p = (unsigned char*)dst;
    unsigned char v = (unsigned char)val;
    while (n--) *p++ = v;
}

/* --------------------------------------------------------------
 * Initialize: allocate one giant GPU buffer for all KV blocks,
 * build free list [0, 1, 2, ..., TOTAL_BLOCKS-1]
 * -------------------------------------------------------------- */
int sov_kv_allocator_init(void) {
    if (g_kv_allocator.initialized) return 0;

    /* Calculate total KV cache size:
     * 2 buffers (K + V) * SOV_MAX_HEADS * SOV_KV_BLOCK_SIZE * head_dim * sizeof(uint16_t)
     * head_dim = 128 (Llama-3 8B: 4096/32) — hardcoded per sov_rtx.h constants
     */
    const size_t head_dim = 128;
    const size_t bytes_per_block = 2 * SOV_MAX_HEADS * SOV_KV_BLOCK_SIZE * head_dim * sizeof(uint16_t);
    const size_t total_bytes = bytes_per_block * SOV_KV_TOTAL_BLOCKS;

    /* Allocate GPU memory via driver API */
    CUdeviceptr gpu_ptr = 0;
    CUresult err = sov_cuda_mem_alloc(&gpu_ptr, total_bytes);
    if (err != CUDA_SUCCESS) return -1;

    g_kv_allocator.gpu_kv_base = (void*)(uintptr_t)gpu_ptr;
    g_kv_allocator.gpu_kv_bytes = total_bytes;

    /* Initialize block_table to -1 (unallocated) */
    const int32_t total_slots = SOV_MAX_SEQS * SOV_KV_MAX_BLOCKS_PER_SEQ;
    for (int32_t i = 0; i < total_slots; ++i)
        g_kv_allocator.block_table[i] = -1;

    /* Build free list: push all physical block IDs onto stack */
    g_kv_allocator.free_top = 0;
    for (int32_t i = 0; i < SOV_KV_TOTAL_BLOCKS; ++i)
        g_kv_allocator.free_list[g_kv_allocator.free_top++] = i;

    /* Zero per-sequence block counts */
    for (int i = 0; i < SOV_MAX_SEQS; ++i)
        g_kv_allocator.seq_block_count[i] = 0;

    g_kv_allocator.initialized = 1;
    return 0;
}

/* --------------------------------------------------------------
 * Pop a free block ID from the stack. Returns -1 if empty.
 * -------------------------------------------------------------- */
static int32_t pop_free_block(void) {
    if (g_kv_allocator.free_top == 0) return -1;
    return g_kv_allocator.free_list[--g_kv_allocator.free_top];
}

/* --------------------------------------------------------------
 * Push a block ID back onto the free stack.
 * -------------------------------------------------------------- */
static void push_free_block(int32_t block_id) {
    if (g_kv_allocator.free_top < SOV_KV_TOTAL_BLOCKS)
        g_kv_allocator.free_list[g_kv_allocator.free_top++] = block_id;
}

/* --------------------------------------------------------------
 * Compute number of blocks needed for 'num_tokens' (ceiling division).
 * -------------------------------------------------------------- */
static uint32_t tokens_to_blocks(uint32_t num_tokens) {
    return (num_tokens + SOV_KV_BLOCK_SIZE - 1) / SOV_KV_BLOCK_SIZE;
}

/* --------------------------------------------------------------
 * Allocate blocks for a new sequence.
 * -------------------------------------------------------------- */
int sov_kv_allocate_blocks(int32_t seq_id, uint32_t num_tokens) {
    if (!g_kv_allocator.initialized) return -1;
    if (seq_id < 0 || seq_id >= SOV_MAX_SEQS) return -1;
    if (g_kv_allocator.seq_block_count[seq_id] != 0) return -1; /* already allocated */

    uint32_t needed = tokens_to_blocks(num_tokens);
    uint32_t max_allowed = SOV_KV_MAX_BLOCKS_PER_SEQ;
    if (needed > max_allowed) needed = max_allowed;

    int32_t* seq_table = &g_kv_allocator.block_table[seq_id * SOV_KV_MAX_BLOCKS_PER_SEQ];

    for (uint32_t i = 0; i < needed; ++i) {
        int32_t blk = pop_free_block();
        if (blk < 0) {
            /* OOM: rollback */
            for (uint32_t j = 0; j < i; ++j) {
                push_free_block(seq_table[j]);
                seq_table[j] = -1;
            }
            return -1;
        }
        seq_table[i] = blk;
    }

    g_kv_allocator.seq_block_count[seq_id] = (uint16_t)needed;
    return (int)needed;
}

/* --------------------------------------------------------------
 * Append tokens: allocate additional blocks if needed.
 * -------------------------------------------------------------- */
int sov_kv_append_tokens(int32_t seq_id, uint32_t num_new_tokens) {
    if (!g_kv_allocator.initialized) return -1;
    if (seq_id < 0 || seq_id >= SOV_MAX_SEQS) return -1;

    uint32_t current_blocks = g_kv_allocator.seq_block_count[seq_id];
    if (current_blocks == 0) return -1; /* sequence not allocated */

    uint32_t current_tokens = current_blocks * SOV_KV_BLOCK_SIZE;
    uint32_t new_total_tokens = current_tokens + num_new_tokens;
    uint32_t new_total_blocks = tokens_to_blocks(new_total_tokens);
    uint32_t max_allowed = SOV_KV_MAX_BLOCKS_PER_SEQ;
    if (new_total_blocks > max_allowed) new_total_blocks = max_allowed;

    uint32_t blocks_to_add = new_total_blocks - current_blocks;
    if (blocks_to_add == 0) return 0;

    int32_t* seq_table = &g_kv_allocator.block_table[seq_id * SOV_KV_MAX_BLOCKS_PER_SEQ];

    for (uint32_t i = 0; i < blocks_to_add; ++i) {
        int32_t blk = pop_free_block();
        if (blk < 0) {
            /* OOM: rollback */
            for (uint32_t j = 0; j < i; ++j) {
                push_free_block(seq_table[current_blocks + j]);
                seq_table[current_blocks + j] = -1;
            }
            return -1;
        }
        seq_table[current_blocks + i] = blk;
    }

    g_kv_allocator.seq_block_count[seq_id] = (uint16_t)new_total_blocks;
    return 0;
}

/* --------------------------------------------------------------
 * Free all blocks for a sequence.
 * -------------------------------------------------------------- */
void sov_kv_free_sequence(int32_t seq_id) {
    if (!g_kv_allocator.initialized) return;
    if (seq_id < 0 || seq_id >= SOV_MAX_SEQS) return;

    uint32_t count = g_kv_allocator.seq_block_count[seq_id];
    if (count == 0) return;

    int32_t* seq_table = &g_kv_allocator.block_table[seq_id * SOV_KV_MAX_BLOCKS_PER_SEQ];

    for (uint32_t i = 0; i < count; ++i) {
        int32_t blk = seq_table[i];
        if (blk >= 0) {
            push_free_block(blk);
            seq_table[i] = -1;
        }
    }

    g_kv_allocator.seq_block_count[seq_id] = 0;
}

/* --------------------------------------------------------------
 * Copy block_table (host) -> device memory for flash_attention kernel.
 * -------------------------------------------------------------- */
int sov_kv_copy_block_table_to_device(void* d_block_table) {
    if (!g_kv_allocator.initialized) return -1;
    const size_t bytes = SOV_MAX_SEQS * SOV_KV_MAX_BLOCKS_PER_SEQ * sizeof(int32_t);
    return sov_cuda_memcpy_h2d(d_block_table, g_kv_allocator.block_table, bytes);
}

/* --------------------------------------------------------------
 * Get GPU pointer for a specific block.
 * -------------------------------------------------------------- */
void* sov_kv_get_block_ptr(int32_t seq_id, uint32_t block_idx) {
    if (!g_kv_allocator.initialized) return 0;
    if (seq_id < 0 || seq_id >= SOV_MAX_SEQS) return 0;
    if (block_idx >= SOV_KV_MAX_BLOCKS_PER_SEQ) return 0;

    int32_t phys_blk = g_kv_allocator.block_table[seq_id * SOV_KV_MAX_BLOCKS_PER_SEQ + block_idx];
    if (phys_blk < 0) return 0;

    const size_t head_dim = 128;
    const size_t bytes_per_block = 2 * SOV_MAX_HEADS * SOV_KV_BLOCK_SIZE * head_dim * sizeof(uint16_t);
    char* base = (char*)g_kv_allocator.gpu_kv_base;
    return base + (size_t)phys_blk * bytes_per_block;
}

uint32_t sov_kv_get_seq_block_count(int32_t seq_id) {
    if (!g_kv_allocator.initialized) return 0;
    if (seq_id < 0 || seq_id >= SOV_MAX_SEQS) return 0;
    return g_kv_allocator.seq_block_count[seq_id];
}

void sov_kv_allocator_shutdown(void) {
    if (!g_kv_allocator.initialized) return;
    if (g_kv_allocator.gpu_kv_base) {
        sov_cuda_mem_free((CUdeviceptr)(uintptr_t)g_kv_allocator.gpu_kv_base);
        g_kv_allocator.gpu_kv_base = 0;
    }
    g_kv_allocator.initialized = 0;
}