tiny.bench.compare

1pub const Classification = enum {2    regression,3    improvement,4    unchanged,5    uncertain,6    insufficient,78    pub fn jsonName(self: Classification) []const u8 {9        return switch (self) {10            .regression => "regression",11            .improvement => "improvement",12            .unchanged => "unchanged",13            .uncertain => "uncertain",14            .insufficient => "insufficient",15        };16    }17};

1pub const Counts = struct {2    compared: u64 = 0,3    regressions: u64 = 0,4    improvements: u64 = 0,5    unchanged: u64 = 0,6    uncertain: u64 = 0,7    insufficient: u64 = 0,8    missing_baseline: u64 = 0,9    missing_candidate: u64 = 0,10};

1pub const EffectInterval = struct {2    low_percent: f64,3    high_percent: f64,4};

1pub const ExecutionAnalysisOptions = struct {2    threshold: f64 = default_threshold,3    min_runs: u32 = default_min_runs,4    bootstrap_iterations: u32 = default_bootstrap_iterations,5};

1pub const ExecutionComparison = struct {2    classification: Classification,3    baseline: ExecutionStats,4    candidate: ExecutionStats,5    ratio: ?f64,6    effect: ?EffectInterval,7    bootstrap_iterations: u32,8    evidence: []const u8,9};

1pub const ExecutionObservation = struct {2    mean: f64,3    median: f64,4};

1pub const ExecutionStats = struct {2    runs: usize,3    mean: f64,4    median: f64,5    min: f64,6    max: f64,7};

1pub const Options = struct {2    baseline_path: []const u8,3    candidate_path: []const u8,4    threshold: f64 = default_threshold,5    min_runs: u32 = default_min_runs,6    bootstrap_iterations: u32 = default_bootstrap_iterations,7};

1pub const U64MedianRatio = struct {2    baseline_median: u64,3    candidate_median: u64,4    ratio: f64,5    low_ratio: f64,6    high_ratio: f64,7};

1pub fn SummaryComparison(comptime Spec: type) type {2    return struct {3        pub fn runFiles(4            allocator: Allocator,5            out: *std.Io.Writer,6            args: []const []const u8,7        ) !u8 {8            return runComparisonFiles(Spec, allocator, out, args);9        }1011        pub fn parseArgs(args: []const []const u8) !Options {12            return parseComparisonArgs(args);13        }1415        pub fn compareSources(16            allocator: Allocator,17            out: *std.Io.Writer,18            options: Options,19            baseline_source: []const u8,20            candidate_source: []const u8,21        ) !Counts {22            return compareSummarySources(23                Spec,24                allocator,25                out,26                options,27                baseline_source,28                candidate_source,29            );30        }31    };32}

1pub fn analyzeExecutions(2    storage: *Storage,3    baseline: []const ExecutionObservation,4    candidate: []const ExecutionObservation,5    options: ExecutionAnalysisOptions,6) !ExecutionComparison {7    if (!std.math.isFinite(options.threshold) or options.threshold < 1.0) {8        return error.InvalidThreshold;9    }10    if (options.min_runs == 0) return error.InvalidMinimumRuns;11    if (options.bootstrap_iterations == 0) return error.InvalidBootstrapIterations;12    if (baseline.len == 0 or candidate.len == 0) return error.EmptyExecutionSet;13    const regions = try storage.acquireF64(14        baseline.len,15        candidate.len,16        options.bootstrap_iterations,17    );18    defer storage.reset();19    const baseline_stats = try summarizeExecutionsInScratch(20        baseline,21        regions.samples[0..baseline.len],22    );23    const candidate_stats = try summarizeExecutionsInScratch(24        candidate,25        regions.samples[0..candidate.len],26    );27    const ratio = ratioOrNull(baseline_stats.median, candidate_stats.median);28    const enough = baseline.len >= options.min_runs and candidate.len >= options.min_runs;29    const effect = if (enough)30        try bootstrapMedianPercentChange(baseline, candidate, regions)31    else32        null;33    return .{34        .classification = classifyChange(effect, enough, options.threshold),35        .baseline = baseline_stats,36        .candidate = candidate_stats,37        .ratio = ratio,38        .effect = effect,39        .bootstrap_iterations = options.bootstrap_iterations,40        .evidence = evidenceState(enough, effect),41    };42}

1pub fn analyzeU64Medians(2    storage: *Storage,3    baseline: []const u64,4    candidate: []const u64,5    bootstrap_iterations: u32,6    seed: u64,7) StorageError!U64MedianRatio {8    const regions = try storage.acquireU64(9        baseline.len,10        candidate.len,11        bootstrap_iterations,12    );13    defer storage.reset();14    const baseline_values = canonicalU64(baseline, regions.samples[0..baseline.len]);15    const baseline_median = upperMedianU64(baseline_values);16    const candidate_values = canonicalU64(candidate, regions.samples[0..candidate.len]);17    const candidate_median = upperMedianU64(candidate_values);18    var prng = std.Random.DefaultPrng.init(seed);19    const random = prng.random();20    const bootstrap_baseline = canonicalU64(baseline, regions.samples[0..baseline.len]);21    fillBootstrapU64Medians(22        regions.effects,23        bootstrap_baseline,24        regions.counts[0..baseline.len],25        random,26    );27    const bootstrap_candidate = canonicalU64(candidate, regions.samples[0..candidate.len]);28    for (regions.effects) |*ratio| {29        const candidate_value = bootstrapU64Median(30            bootstrap_candidate,31            regions.counts[0..candidate.len],32            random,33        );34        ratio.* = u64RatioFloat(candidate_value, ratio.*);35    }36    std.mem.sort(f64, regions.effects, {}, std.sort.asc(f64));37    const low_index = ratioQuantileIndex(regions.effects.len, 25);38    const high_index = ratioQuantileIndex(regions.effects.len, 975);39    return .{40        .baseline_median = baseline_median,41        .candidate_median = candidate_median,42        .ratio = u64Ratio(candidate_median, baseline_median),43        .low_ratio = regions.effects[@min(low_index, regions.effects.len - 1)],44        .high_ratio = regions.effects[@min(high_index, regions.effects.len - 1)],45    };46}

1pub const bootstrap_seed: u64 = 0x5449_4e59_434f_4d50;

1pub const confidence_per_mille: u16 = 950;

1pub const default_bootstrap_iterations: u32 = 1000;

1pub const default_min_runs: u32 = 10;

1pub const default_threshold = 1.05;

1pub fn duplicateObjectString(2    allocator: Allocator,3    object: std.json.ObjectMap,4    key: []const u8,5) ![]u8 {6    const text = objectStringOptional(object, key) orelse return error.InvalidSummary;7    return try allocator.dupe(u8, text);8}

1pub const effect_confidence: f64 =2    @as(f64, @floatFromInt(confidence_per_mille)) / 1000.0;

1pub const effect_method =2    "deterministic_percentile_bootstrap_unpaired_execution_median_percent_change";

1pub const max_profile_bytes = 64 * 1024 * 1024;

1pub fn objectBoolDefault(2    object: std.json.ObjectMap,3    key: []const u8,4    default: bool,5) !bool {6    const value = object.get(key) orelse return default;7    return switch (value) {8        .bool => |actual| actual,9        else => error.InvalidSummary,10    };11}

1pub fn objectNumberOptional(object: std.json.ObjectMap, key: []const u8) ?f64 {2    const value = object.get(key) orelse return null;3    const number: f64 = switch (value) {4        .integer => |integer| @floatFromInt(integer),5        .float => |float| float,6        .number_string => |text| std.fmt.parseFloat(f64, text) catch return null,7        else => return null,8    };9    return if (std.math.isFinite(number)) number else null;10}

1pub fn objectStringOptional(object: std.json.ObjectMap, key: []const u8) ?[]const u8 {2    const value = object.get(key) orelse return null;3    return switch (value) {4        .string => |text| text,5        else => null,6    };7}

1pub fn objectU64(object: std.json.ObjectMap, key: []const u8) !u64 {2    const value = object.get(key) orelse return error.InvalidSummary;3    return switch (value) {4        .integer => |integer| if (integer < 0) error.InvalidSummary else @intCast(integer),5        .number_string => |text| std.fmt.parseUnsigned(u64, text, 10) catch6            return error.InvalidSummary,7        else => error.InvalidSummary,8    };9}

1fn parseComparisonArgs(args: []const []const u8) !Options {2    if (args.len < 2) return error.InvalidArguments;3    if (args[0].len == 0 or args[0][0] == '-') return error.InvalidArguments;4    if (args[1].len == 0 or args[1][0] == '-') return error.InvalidArguments;5    var options = Options{6        .baseline_path = args[0],7        .candidate_path = args[1],8    };9    var threshold_specified = false;10    var min_runs_specified = false;11    var bootstrap_iterations_specified = false;12    var index: usize = 2;13    while (index < args.len) : (index += 1) {14        const arg = args[index];15        if (std.mem.eql(u8, arg, "--threshold")) {16            if (threshold_specified) return error.InvalidArguments;17            index += 1;18            if (index >= args.len) return error.InvalidArguments;19            options.threshold = parseThreshold(args[index]) catch return error.InvalidArguments;20            threshold_specified = true;21        } else if (std.mem.eql(u8, arg, "--min-runs")) {22            if (min_runs_specified) return error.InvalidArguments;23            index += 1;24            if (index >= args.len) return error.InvalidArguments;25            options.min_runs = std.fmt.parseUnsigned(u32, args[index], 10) catch26                return error.InvalidArguments;27            if (options.min_runs == 0) return error.InvalidArguments;28            min_runs_specified = true;29        } else if (std.mem.eql(u8, arg, "--bootstrap-iterations")) {30            if (bootstrap_iterations_specified) return error.InvalidArguments;31            index += 1;32            if (index >= args.len) return error.InvalidArguments;33            options.bootstrap_iterations = std.fmt.parseUnsigned(u32, args[index], 10) catch34                return error.InvalidArguments;35            if (options.bootstrap_iterations == 0) return error.InvalidArguments;36            bootstrap_iterations_specified = true;37        } else {38            return error.InvalidArguments;39        }40    }41    return options;42}

1pub fn summarizeExecutions(2    storage: *Storage,3    observations: []const ExecutionObservation,4) !ExecutionStats {5    if (observations.len == 0) return error.EmptyExecutionSet;6    const scratch = try storage.acquireF64Samples(observations.len);7    defer storage.reset();8    return summarizeExecutionsInScratch(observations, scratch);9}

1const engine = @import("engine.zig");23pub const Limits = engine.Limits;4pub const Capacity = engine.Capacity;5pub const CapacityError = engine.CapacityError;6pub const Storage = engine.Storage;7pub const StorageStatus = engine.StorageStatus;8pub const StorageExhaustion = engine.StorageExhaustion;9pub const StorageError = engine.StorageError;10pub const max_profile_bytes = engine.max_profile_bytes;11pub const default_threshold = engine.default_threshold;12pub const default_min_runs = engine.default_min_runs;13pub const default_bootstrap_iterations = engine.default_bootstrap_iterations;14pub const bootstrap_seed = engine.bootstrap_seed;15pub const confidence_per_mille = engine.confidence_per_mille;16pub const effect_confidence = engine.effect_confidence;17pub const effect_method = engine.effect_method;18pub const Options = engine.Options;19pub const Counts = engine.Counts;20pub const Classification = engine.Classification;21pub const EffectInterval = engine.EffectInterval;22pub const ExecutionObservation = engine.ExecutionObservation;23pub const ExecutionStats = engine.ExecutionStats;24pub const ExecutionAnalysisOptions = engine.ExecutionAnalysisOptions;25pub const ExecutionComparison = engine.ExecutionComparison;26pub const U64MedianRatio = engine.U64MedianRatio;27pub const parseArgs = engine.parseArgs;28pub const SummaryComparison = engine.SummaryComparison;29pub const analyzeExecutions = engine.analyzeExecutions;30pub const analyzeU64Medians = engine.analyzeU64Medians;31pub const summarizeExecutions = engine.summarizeExecutions;32pub const duplicateObjectString = engine.duplicateObjectString;33pub const objectStringOptional = engine.objectStringOptional;34pub const objectU64 = engine.objectU64;35pub const objectNumberOptional = engine.objectNumberOptional;36pub const objectBoolDefault = engine.objectBoolDefault;3738pub const ComparisonStorage = @import("storage.zig").Storage;

1pub const compare = compare_mod;