tiny.profiling.budget

1const std = @import("std");23const catalog = @import("catalog.zig");4const memory = @import("memory.zig");56pub const Baseline = union(enum) {7    bootstrap,8    design_changed,9    metrics: []const memory.Metric,10};1112pub const Status = enum {13    bootstrap,14    design_changed,15    within_budget,16    baseline_missing,17    candidate_missing,18    baseline_incompatible,19    regressed,20};2122pub const Gate = enum {23    pass,24    bootstrap,25    fail,2627    pub fn name(self: Gate) []const u8 {28        return @tagName(self);29    }30};3132pub const Check = struct {33    workload: []const u8,34    baseline_policy: catalog.AllocationBaselinePolicy,35    budget_percent: f64,36    baseline_metrics: usize = 0,37    candidate_metrics: usize = 0,38    comparable_metrics: usize = 0,39    violations: usize = 0,40    zero_baseline_violations: usize = 0,41    status: Status,4243    pub fn gate(self: Check) Gate {44        return switch (self.status) {45            .within_budget => .pass,46            .bootstrap, .design_changed, .baseline_missing => .bootstrap,47            .candidate_missing,48            .baseline_incompatible,49            .regressed,50            => .fail,51        };52    }53};5455pub fn evaluate(56    workload_name: []const u8,57    baseline: Baseline,58    candidate: []const memory.Metric,59) ?Check {60    const workload = catalog.find(workload_name) orelse return null;61    if (!workload.supportsAllocationCounters()) return null;62    const budget_percent = workload.allocationBudgetPercent().?;63    var result = Check{64        .workload = workload.name,65        .baseline_policy = workload.allocationBaselinePolicy(),66        .budget_percent = budget_percent,67        .candidate_metrics = count(candidate, workload.name),68        .status = .within_budget,69    };70    if (result.candidate_metrics == 0) {71        result.status = .candidate_missing;72        return result;73    }74    switch (baseline) {75        .bootstrap => {76            result.status = .bootstrap;77            return result;78        },79        .design_changed => {80            result.status = .design_changed;81            return result;82        },83        .metrics => |baseline_metrics| {84            result.baseline_metrics = count(baseline_metrics, workload.name);85            if (result.baseline_metrics == 0) {86                result.status = .baseline_missing;87                return result;88            }89            for (candidate) |candidate_metric| {90                if (!belongsTo(candidate_metric, workload.name)) continue;91                const base_metric = find(92                    baseline_metrics,93                    workload.name,94                    candidate_metric.key,95                    candidate_metric.unit,96                ) orelse continue;97                result.comparable_metrics += 1;98                if (!exceeds(base_metric.value, candidate_metric.value, budget_percent)) {99                    continue;100                }101                result.violations += 1;102                if (base_metric.value <= 0 and candidate_metric.value > base_metric.value) {103                    result.zero_baseline_violations += 1;104                }105            }106            if (result.comparable_metrics == 0) {107                result.status = .baseline_incompatible;108            } else if (result.violations != 0) {109                result.status = .regressed;110            }111            return result;112        },113    }114}115116fn count(metrics: []const memory.Metric, workload_name: []const u8) usize {117    var result: usize = 0;118    for (metrics) |metric| {119        if (belongsTo(metric, workload_name)) result += 1;120    }121    return result;122}123124fn belongsTo(metric: memory.Metric, workload_name: []const u8) bool {125    return std.mem.eql(u8, metric.workload, workload_name) and126        memory.isAllocationMetric(metric);127}128129fn find(130    metrics: []const memory.Metric,131    workload_name: []const u8,132    key: []const u8,133    unit: memory.Unit,134) ?memory.Metric {135    for (metrics) |metric| {136        if (!belongsTo(metric, workload_name)) continue;137        if (!std.mem.eql(u8, metric.key, key)) continue;138        if (metric.unit != unit) continue;139        return metric;140    }141    return null;142}143144fn exceeds(baseline_value: f64, candidate_value: f64, budget_percent: f64) bool {145    if (!std.math.isFinite(baseline_value) or !std.math.isFinite(candidate_value)) {146        return true;147    }148    if (baseline_value <= 0) return candidate_value > baseline_value;149    const percent = ((candidate_value - baseline_value) / baseline_value) * 100;150    return percent >= budget_percent;151}152153fn testMetric(key: []const u8, value: f64) memory.Metric {154    return .{155        .workload = "gpalloc.allocator",156        .key = key,157        .label = key,158        .source_kind = "bench_jsonl",159        .source_path = "bench.jsonl",160        .source_line = 1,161        .unit = .bytes,162        .value = value,163    };164}165166test "allocation budget checks bootstrap and candidate coverage" {167    const candidate = [_]memory.Metric{testMetric("small|alloc_bytes_per_eval", 100)};168    const bootstrap = evaluate(169        "gpalloc.allocator",170        .bootstrap,171        &candidate,172    ).?;173    try std.testing.expectEqual(Status.bootstrap, bootstrap.status);174    try std.testing.expectEqual(Gate.bootstrap, bootstrap.gate());175    const changed = evaluate(176        "gpalloc.allocator",177        .design_changed,178        &candidate,179    ).?;180    try std.testing.expectEqual(Status.design_changed, changed.status);181    try std.testing.expectEqual(Gate.bootstrap, changed.gate());182    const missing = evaluate(183        "gpalloc.allocator",184        .bootstrap,185        &.{},186    ).?;187    try std.testing.expectEqual(Status.candidate_missing, missing.status);188    try std.testing.expectEqual(Gate.fail, missing.gate());189}190191test "allocation budget checks per-workload growth and zero baselines" {192    const baseline = [_]memory.Metric{193        testMetric("small|alloc_bytes_per_eval", 100),194        testMetric("zero|alloc_bytes_per_eval", 0),195    };196    const within = [_]memory.Metric{197        testMetric("small|alloc_bytes_per_eval", 129),198        testMetric("zero|alloc_bytes_per_eval", 0),199    };200    const passing = evaluate(201        "gpalloc.allocator",202        .{ .metrics = &baseline },203        &within,204    ).?;205    try std.testing.expectEqual(Status.within_budget, passing.status);206    try std.testing.expectEqual(Gate.pass, passing.gate());207    const regressed = [_]memory.Metric{208        testMetric("small|alloc_bytes_per_eval", 130),209        testMetric("zero|alloc_bytes_per_eval", 1),210    };211    const failing = evaluate(212        "gpalloc.allocator",213        .{ .metrics = &baseline },214        &regressed,215    ).?;216    try std.testing.expectEqual(Status.regressed, failing.status);217    try std.testing.expectEqual(@as(usize, 2), failing.violations);218    try std.testing.expectEqual(@as(usize, 1), failing.zero_baseline_violations);219    try std.testing.expectEqual(Gate.fail, failing.gate());220}221222test "allocation budget checks baseline coverage" {223    const baseline = [_]memory.Metric{testMetric("old", 100)};224    const candidate = [_]memory.Metric{testMetric("new", 100)};225    const incompatible = evaluate(226        "gpalloc.allocator",227        .{ .metrics = &baseline },228        &candidate,229    ).?;230    try std.testing.expectEqual(Status.baseline_incompatible, incompatible.status);231    const missing = evaluate(232        "gpalloc.allocator",233        .{ .metrics = &.{} },234        &candidate,235    ).?;236    try std.testing.expectEqual(Status.baseline_missing, missing.status);237    try std.testing.expectEqual(Gate.bootstrap, missing.gate());238    try std.testing.expect(evaluate("choir.compiler", .bootstrap, &candidate) == null);239}

1pub const budget = @import("budget.zig");