tiny.gpalloc.cache

1pub const ThreadCache = struct {2    owner: ?*anyopaque = null,3    storage_owner: ?*anyopaque = null,4    next_retained: ?*ThreadCache = null,5    bins: [thread.fixed_class_count]ThreadCacheBin = @as([thread.fixed_class_count]ThreadCacheBin, @splat(.{})),6    extra_bins: [thread.extra_class_count]ExtraThreadCacheBin =7        @as([thread.extra_class_count]ExtraThreadCacheBin, @splat(.{})),8    linked_heads: [linked_class_count]?*FreeNode = @as([linked_class_count]?*FreeNode, @splat(null)),9    linked_counts: [linked_class_count]u16 = @as([linked_class_count]u16, @splat(0)),1011    pub fn reset(cache: *ThreadCache, owner: *anyopaque) void {12        cache.owner = owner;13        cache.storage_owner = owner;14        cache.next_retained = null;15        for (&cache.bins) |*bin| bin.count = 0;16        for (&cache.extra_bins) |*bin| bin.count = 0;17        @memset(&cache.linked_heads, null);18        @memset(&cache.linked_counts, 0);19    }20};

1pub fn cpuCountThreadCacheActiveLimit() usize {2    return sys_thread.cpuCount();3}

1pub fn defaultThreadCacheActiveLimit() usize {2    return cpuCountThreadCacheActiveLimit();3}

1pub inline fn loadThreadCache() ?*ThreadCache {2    return @ptrCast(@alignCast(tls.load()));3}

1pub inline fn loadThreadCacheForAllocFastPath() ?*ThreadCache {2    return loadThreadCache();3}

1pub inline fn loadThreadCacheForSmallFastPath() ?*ThreadCache {2    return loadThreadCache();3}

1pub inline fn localThreadCacheBinCount(cache: *const ThreadCache, class_index: usize) usize {2    if (class_index < thread.fixed_class_count) {3        var count: usize = cache.bins[class_index].count;4        if (threadExtraClassIndex(class_index)) |extra_index| {5            count += cache.extra_bins[extra_index].count;6        }7        return count;8    }9    return cache.linked_counts[class_index - thread.fixed_class_count];10}

1pub inline fn popLocalThreadCacheBin(cache: *ThreadCache, class_index: usize) ?[*]u8 {2    if (class_index < thread.fixed_class_count) {3        if (cache.bins[class_index].pop()) |ptr| return ptr;4        if (threadExtraClassIndex(class_index)) |extra_index| {5            if (cache.extra_bins[extra_index].pop()) |ptr| return ptr;6        }7        return null;8    }910    const linked_index = class_index - thread.fixed_class_count;11    const node = cache.linked_heads[linked_index] orelse return null;12    cache.linked_heads[linked_index] = node.next;13    cache.linked_counts[linked_index] -= 1;14    return @ptrCast(node);15}

1pub inline fn pushLocalThreadCacheBin(2    cache: *ThreadCache,3    class_index: usize,4    ptr: [*]u8,5    cache_limit: u16,6) bool {7    if (class_index < thread.fixed_class_count) {8        const base_limit = @min(thread.capacity, cache_limit);9        if (cache.bins[class_index].push(ptr, base_limit)) return true;1011        const extra_limit = cache_limit - base_limit;12        if (extra_limit == 0) return false;13        const extra_index = threadExtraClassIndex(class_index) orelse return false;14        return cache.extra_bins[extra_index].push(ptr, extra_limit);15    }1617    const linked_index = class_index - thread.fixed_class_count;18    const count = cache.linked_counts[linked_index];19    if (count >= cache_limit) return false;20    const node: *FreeNode = @ptrCast(@alignCast(ptr));21    node.* = .{ .next = cache.linked_heads[linked_index] };22    cache.linked_heads[linked_index] = node;23    cache.linked_counts[linked_index] = count + 1;24    return true;25}

1pub inline fn storeThreadCache(cache: ?*ThreadCache) void {2    tls.store(cache);3}

1pub fn PointerBin(comptime capacity: u16) type {2    return struct {3        count: usize = 0,4        items: [capacity][*]u8 = undefined,56        const Self = @This();78        pub fn pop(bin: *Self) ?[*]u8 {9            if (bin.count == 0) return null;10            bin.count -= 1;11            return bin.items[bin.count];12        }1314        pub fn push(bin: *Self, ptr: [*]u8, limit: usize) bool {15            std.debug.assert(limit <= capacity);16            if (bin.count >= limit) return false;17            bin.items[bin.count] = ptr;18            bin.count += 1;19            return true;20        }21    };22}

1pub const capacity: u16 = 960;

1pub fn classLimit(class_index: usize) u16 {2    return class_limits[class_index];3}

1pub fn drainCount(cache_limit: u16) usize {2    return @max(@as(usize, 1), @as(usize, cache_limit) / 2);3}

1pub fn extraCapacity(class_index: usize) u16 {2    return if (extraClassIndex(class_index)) |extra_index| extra_class_capacities[extra_index] else 0;3}

1pub fn extraClassIndex(class_index: usize) ?usize {2    inline for (extra_class_indices, 0..) |extra_class_index, extra_index| {3        if (class_index == extra_class_index) return extra_index;4    }5    return null;6}

1pub const extra_bin_capacity: u16 = 1088;

1pub const extra_class_capacities = [_]u16{ 1088, 405 };

1pub const extra_class_refill_max: usize = 512;

1pub const extra_class_sizes = [_]usize{ 32, 48 };

1pub const fixed_class_count: usize = 20;

1pub const preload_class_refill_max: usize = 512;

1pub const preload_class_size: usize = 96;

1pub fn refillCount(class_index: usize) usize {2    const cache_limit = classLimit(class_index);3    if (extraClassIndex(class_index) != null) return @min(extra_class_refill_max, cache_limit);4    if (size_class.size(class_index) == preload_class_size) return @min(preload_class_refill_max, cache_limit);5    return @min(refill_max, cache_limit);6}

1pub const refill_max: usize = 256;

1pub const target_bytes: usize = 64 * 1024;

1pub const tight_class_capacity: u16 = 512;

1pub const tight_class_size: usize = 80;

1pub const capacity: u16 = 2048;

1pub fn classLimit(class_index: usize) u16 {2    const class_size = size_class.size(class_index);3    const by_bytes = @max(@as(usize, 1), target_bytes / class_size);4    return @intCast(@min(capacity, by_bytes));5}

1pub const target_bytes: usize = 2 * 1024 * 1024;

1const local = @import("local.zig");2const pointer = @import("pointer.zig");3const thread = @import("thread.zig");4const transfer = @import("transfer.zig");56pub const thread_capacity = thread.capacity;7pub const thread_extra_bin_capacity = thread.extra_bin_capacity;8pub const transfer_capacity = transfer.capacity;9pub const target_bytes = thread.target_bytes;10pub const transfer_target_bytes = transfer.target_bytes;11pub const refill_max = thread.refill_max;12pub const thread_tight_class_size = thread.tight_class_size;13pub const thread_tight_class_capacity = thread.tight_class_capacity;14pub const thread_preload_class_size = thread.preload_class_size;15pub const thread_preload_class_refill_max = thread.preload_class_refill_max;16pub const thread_extra_class_sizes = thread.extra_class_sizes;17pub const thread_extra_class_capacities = thread.extra_class_capacities;18pub const thread_extra_class_count = thread.extra_class_count;19pub const thread_extra_class_refill_max = thread.extra_class_refill_max;20pub const thread_fixed_class_count = thread.fixed_class_count;2122pub const PointerBin = pointer.PointerBin;23pub const ThreadCache = local.ThreadCache;24pub const cpuCountThreadCacheActiveLimit = local.cpuCountThreadCacheActiveLimit;25pub const defaultThreadCacheActiveLimit = local.defaultThreadCacheActiveLimit;26pub const loadThreadCache = local.loadThreadCache;27pub const loadThreadCacheForAllocFastPath = local.loadThreadCacheForAllocFastPath;28pub const loadThreadCacheForSmallFastPath = local.loadThreadCacheForSmallFastPath;29pub const localThreadCacheBinCount = local.localThreadCacheBinCount;30pub const popLocalThreadCacheBin = local.popLocalThreadCacheBin;31pub const pushLocalThreadCacheBin = local.pushLocalThreadCacheBin;32pub const storeThreadCache = local.storeThreadCache;33pub const threadClassLimit = thread.classLimit;34pub const threadDrainCount = thread.drainCount;35pub const threadExtraCapacity = thread.extraCapacity;36pub const threadExtraClassIndex = thread.extraClassIndex;37pub const threadRefillCount = thread.refillCount;38pub const transferClassLimit = transfer.classLimit;

1pub const cache = @import("cache/root.zig");

1pub const extra_class_count: usize = extra_class_sizes.len;