tiny.profiling.experiment.model

1const std = @import("std");23const profiling = @import("../root.zig");4const scenario = profiling.scenario;56pub const schema = "tiny.profiling.experiment/v1";7pub const receipt_schema = "tiny.profiling.experiment-receipt/v1";8pub const maximum_plan_bytes: usize = 1024 * 1024;9pub const maximum_pairs: usize = 1000;10pub const default_calibration_pairs: usize = 8;11pub const default_minimum_evaluation_pairs: usize = 8;12pub const default_maximum_evaluation_pairs: usize = 100;13pub const confidence: f64 = 0.95;14pub const power: f64 = 0.80;15pub const normal_critical_sum: f64 = 2.801_585_218_112_968;1617pub const Metric = enum {18    wall_ns,19    max_rss_kib,2021    pub fn parse(value: []const u8) ?Metric {22        if (std.mem.eql(u8, value, "wall_ns")) return .wall_ns;23        if (std.mem.eql(u8, value, "max_rss_kib")) return .max_rss_kib;24        return null;25    }2627    pub fn unit(self: Metric) []const u8 {28        return switch (self) {29            .wall_ns => "nanoseconds",30            .max_rss_kib => "kibibytes",31        };32    }33};3435pub const Build = struct {36    argv: []const []const u8,37    artifact: []const u8,38    cwd: ?[]const u8 = null,39};4041pub const Ref = struct {42    name: []const u8,43    build: ?Build = null,44};4546pub const Variant = union(enum) {47    binary: []const u8,48    git_ref: Ref,49};5051pub const Design = struct {52    calibration_pairs: usize = default_calibration_pairs,53    minimum_evaluation_pairs: usize = default_minimum_evaluation_pairs,54    maximum_evaluation_pairs: usize = default_maximum_evaluation_pairs,5556    pub fn validate(self: Design) !void {57        if (self.calibration_pairs < 2 or58            self.calibration_pairs > maximum_pairs)59        {60            return error.InvalidCalibrationPairs;61        }62        if (self.minimum_evaluation_pairs < 2 or63            self.minimum_evaluation_pairs > maximum_pairs)64        {65            return error.InvalidMinimumEvaluationPairs;66        }67        if (self.maximum_evaluation_pairs < self.minimum_evaluation_pairs or68            self.maximum_evaluation_pairs > maximum_pairs)69        {70            return error.InvalidMaximumEvaluationPairs;71        }72    }73};7475pub const Plan = struct {76    name: []const u8,77    scenario: scenario.Definition,78    baseline: Variant,79    candidate: Variant,80    metric: Metric,81    practical_effect_percent: f64,82    seed: u64,83    design: Design = .{},8485    pub fn validate(self: Plan) !void {86        if (self.name.len == 0 or self.name.len > scenario.maximum_string_bytes) {87            return error.InvalidExperimentName;88        }89        if (!std.math.isFinite(self.practical_effect_percent) or90            self.practical_effect_percent <= 0 or91            self.practical_effect_percent > 100)92        {93            return error.InvalidPracticalEffect;94        }95        try self.design.validate();96        try validateScenario(self.scenario);97        try validateVariant(self.baseline, self.scenario);98        try validateVariant(self.candidate, self.scenario);99    }100};101102fn validateVariant(103    variant: Variant,104    definition: scenario.Definition,105) !void {106    switch (variant) {107        .binary => |path| {108            if (path.len == 0 or path.len > scenario.maximum_string_bytes) {109                return error.InvalidVariantBinary;110            }111        },112        .git_ref => |value| {113            if (value.name.len == 0 or114                value.name.len > scenario.maximum_string_bytes)115            {116                return error.InvalidVariantRef;117            }118            if (definition.input == .command and value.build == null) {119                return error.RefCommandRequiresBuild;120            }121            if (value.build) |build| {122                if (build.argv.len == 0 or123                    build.argv.len > scenario.maximum_arguments or124                    build.artifact.len == 0)125                {126                    return error.InvalidVariantBuild;127                }128            }129        },130    }131}132133fn validateScenario(definition: scenario.Definition) !void {134    switch (definition.input) {135        .catalog => {},136        .command => |argv| {137            for (argv) |argument| {138                if (std.mem.indexOf(u8, argument, "{binary}") != null) break;139            } else return error.MissingBinaryPlaceholder;140        },141    }142}143144pub const Phase = enum {145    calibration,146    evaluation,147};148149pub const Side = enum {150    baseline,151    candidate,152};153154pub const Verdict = enum {155    faster,156    slower,157    equivalent,158    inconclusive,159    unsupported,160};161162pub const Support = enum {163    supported,164    budget_insufficient,165    setup_failed,166    reset_failed,167    execution_failed,168    oracle_failed,169    metric_unavailable,170    invalid_metric_value,171    layout_unavailable,172};173174pub const Interval = struct {175    low_percent: f64,176    high_percent: f64,177};178179pub const SamplePlan = struct {180    calibration_standard_deviation_log_ratio: f64,181    calculated_evaluation_pairs: usize,182    evaluation_pairs: usize,183    within_budget: bool,184};185186test "profiling experiment model validates fixed two-stage bounds" {187    const definition = scenario.Definition{188        .name = "startup",189        .input = .{ .command = &.{"{binary}"} },190    };191    const plan = Plan{192        .name = "startup comparison",193        .scenario = definition,194        .baseline = .{ .binary = "base" },195        .candidate = .{ .binary = "candidate" },196        .metric = .wall_ns,197        .practical_effect_percent = 5,198        .seed = 42,199    };200    try plan.validate();201    var invalid = plan;202    invalid.design.maximum_evaluation_pairs = 1;203    try std.testing.expectError(204        error.InvalidMaximumEvaluationPairs,205        invalid.validate(),206    );207}

1pub const model = @import("model.zig");