Skip to documentation
SLOP

tiny.profiling.experiment.run

Reference tiny.profiling experiment run

Defined in experiment.

API (3)

Actions

Public operations.

Types and contracts

Public types and contracts.

No direct callersNo direct callsexperimentrun
Static calls · unresolved targets: unknown · external targets: unknown.

Source

Called byCallsprivate; no linksrc.profiling.commandrunExperimenttest; no linksrc.profiling.experiment.runtest: profiling experiment records re...experiment.acquire.Runnerdeinitexperiment.acquire.Runnerinitprivate; no linksrc.profiling.experiment.runcollectEvidenceprivate; no linksrc.profiling.experiment.runprepareContextprivate; no linksrc.profiling.experiment.runwriteReceipt+2 moreexperiment.runexecute
Static calls · unresolved targets: 0 · external targets: 0.

Source: src/profiling/experiment/root.zig:7

zig
pub const run = @import("run.zig");

Source: src/profiling/experiment/run.zig

zig
const std = @import("std");const capture = @import("capture");const sys = @import("sys");const profiling = @import("../root.zig");const acquire = @import("acquire.zig");const design = @import("design.zig");const model = @import("model.zig");const parse = @import("parse.zig");const receipt = @import("receipt.zig");const variant = @import("variant.zig");const environment = profiling.environment;const fingerprint = profiling.fingerprint;const host = profiling.host;const record = profiling.record;const evaluation_seed_xor: u64 = 0x4556_414c_5541_5445;const bootstrap_seed_xor: u64 = 0x424f_4f54_5354_5241;pub const Options = struct {    plan_path: []const u8,    output_dir: []const u8 = record.default_output_dir,    run_id: ?[]const u8 = null,};pub const Outcome = struct {    run_id: []const u8,    artifact_root: []const u8,    receipt_path: []const u8,    verdict: model.Verdict,    support: model.Support,    reason: []const u8,};const Context = struct {    started_unix_ns: i128,    plan_path: []const u8,    plan_file: fingerprint.File,    plan: model.Plan,    run_id: []const u8,    artifact_root: []const u8,    host_environment: environment.Host,    host_state_start: host.state.Snapshot,    variants: variant.Context,};const Evidence = struct {    setup: [2]?acquire.CommandResult = .{ null, null },    calibration: ?acquire.PhaseOutcome = null,    sample_plan: ?model.SamplePlan = null,    evaluation: ?acquire.PhaseOutcome = null,    assessment: receipt.Assessment = .{        .verdict = .unsupported,        .support = .execution_failed,        .reason = "experiment_did_not_reach_evaluation",    },};pub fn execute(    allocator: std.mem.Allocator,    process_io: std.Io,    environ_map: ?*const sys.process.Environ.Map,    options: Options,) !Outcome {    const context = try prepareContext(        allocator,        process_io,        environ_map,        options,    );    const baseline = try variant.prepare(        context.variants,        .baseline,        context.plan.baseline,    );    var baseline_cleaned = false;    defer if (!baseline_cleaned) {        variant.cleanup(context.variants, baseline) catch {};    };    const candidate = try variant.prepare(        context.variants,        .candidate,        context.plan.candidate,    );    var candidate_cleaned = false;    defer if (!candidate_cleaned) {        variant.cleanup(context.variants, candidate) catch {};    };    var runner = try acquire.Runner.init(        allocator,        process_io,        environ_map,        context.artifact_root,        context.plan.scenario,        context.plan.metric,        baseline,        candidate,    );    defer runner.deinit();    const evidence = try collectEvidence(        allocator,        context.plan,        baseline,        candidate,        &runner,    );    const receipt_path = try writeReceipt(        allocator,        context,        baseline,        candidate,        evidence,    );    try variant.cleanup(context.variants, candidate);    candidate_cleaned = true;    try variant.cleanup(context.variants, baseline);    baseline_cleaned = true;    return .{        .run_id = context.run_id,        .artifact_root = context.artifact_root,        .receipt_path = receipt_path,        .verdict = evidence.assessment.verdict,        .support = evidence.assessment.support,        .reason = evidence.assessment.reason,    };}fn prepareContext(    allocator: std.mem.Allocator,    process_io: std.Io,    environ_map: ?*const sys.process.Environ.Map,    options: Options,) !Context {    const started_unix_ns = sys.time.realNanoTimestamp();    const plan_path = try sys.fs.realPathAlloc(allocator, options.plan_path);    const plan_before = try fingerprint.inspect(plan_path);    const plan = try parse.load(allocator, plan_path);    const plan_after = try fingerprint.inspect(plan_path);    if (!sameFile(plan_before, plan_after)) return error.InputChanged;    const run_id = options.run_id orelse        try record.prefixedRunId(allocator, process_io, "experiment");    try validateRunId(run_id);    const artifact_root = try std.fs.path.join(        allocator,        &.{ options.output_dir, run_id },    );    if (sys.fs.exists(artifact_root)) return error.ExperimentRunExists;    try sys.fs.createDirPath(artifact_root);    const host_state_start = host.state.capture();    const host_environment = try environment.collect(allocator, process_io);    return .{        .started_unix_ns = started_unix_ns,        .plan_path = plan_path,        .plan_file = plan_after,        .plan = plan,        .run_id = run_id,        .artifact_root = artifact_root,        .host_environment = host_environment,        .host_state_start = host_state_start,        .variants = .{            .allocator = allocator,            .process_io = process_io,            .environ_map = environ_map,            .artifact_root = artifact_root,            .run_id = run_id,            .definition = plan.scenario,        },    };}fn collectEvidence(    allocator: std.mem.Allocator,    plan: model.Plan,    baseline: variant.Prepared,    candidate: variant.Prepared,    runner: *acquire.Runner,) !Evidence {    var evidence = Evidence{};    if (!layoutSupported(baseline, candidate)) {        evidence.assessment = unsupported(            .layout_unavailable,            "code_layout_unavailable_for_distinct_binaries",        );        return evidence;    }    evidence.setup = try runner.setup();    if (!setupPassed(evidence.setup)) {        evidence.assessment = unsupported(            .setup_failed,            "scenario_setup_command_failed",        );        return evidence;    }    evidence.calibration = try runner.runPhase(        .calibration,        plan.design.calibration_pairs,        plan.seed,        0,    );    const calibration = evidence.calibration.?;    if (!calibration.complete()) {        evidence.assessment = unsupported(            calibration.support,            supportReason(calibration.support),        );        return evidence;    }    evidence.sample_plan = try evaluationSamplePlan(allocator, plan, calibration);    if (!evidence.sample_plan.?.within_budget) {        evidence.assessment = unsupported(            .budget_insufficient,            "calculated_evaluation_pairs_exceed_plan_budget",        );        return evidence;    }    evidence.evaluation = try runner.runPhase(        .evaluation,        evidence.sample_plan.?.evaluation_pairs,        plan.seed ^ evaluation_seed_xor,        calibration.observations.len,    );    const evaluation = evidence.evaluation.?;    if (!evaluation.complete()) {        evidence.assessment = unsupported(            evaluation.support,            supportReason(evaluation.support),        );        return evidence;    }    evidence.assessment = try assess(        allocator,        plan,        evaluation.baseline_values,        evaluation.candidate_values,    );    return evidence;}fn evaluationSamplePlan(    allocator: std.mem.Allocator,    plan: model.Plan,    calibration: acquire.PhaseOutcome,) !model.SamplePlan {    return try design.samplePlan(        try pairedLogRatios(            allocator,            calibration.baseline_values,            calibration.candidate_values,        ),        plan.practical_effect_percent,        plan.design,    );}fn writeReceipt(    allocator: std.mem.Allocator,    context: Context,    baseline: variant.Prepared,    candidate: variant.Prepared,    evidence: Evidence,) ![]const u8 {    const receipt_path = try std.fs.path.join(        allocator,        &.{ context.artifact_root, "receipt.json" },    );    try receipt.writeFile(allocator, receipt_path, .{        .run_id = context.run_id,        .artifact_root = context.artifact_root,        .plan_path = context.plan_path,        .plan_file = context.plan_file,        .plan = context.plan,        .started_unix_ns = context.started_unix_ns,        .finished_unix_ns = sys.time.realNanoTimestamp(),        .host_environment = context.host_environment,        .host_state_start = context.host_state_start,        .baseline = baseline,        .candidate = candidate,        .setup = evidence.setup,        .calibration = evidence.calibration,        .sample_plan = evidence.sample_plan,        .evaluation = evidence.evaluation,        .assessment = evidence.assessment,    });    return receipt_path;}fn assess(    allocator: std.mem.Allocator,    plan: model.Plan,    baseline: []const f64,    candidate: []const f64,) !receipt.Assessment {    var storage = try capture.compare.effect.Storage.init(allocator, .{        .max_samples_per_distribution = baseline.len,    });    defer storage.deinit(allocator);    storage.activate();    const interval_value = (try capture.compare.effect        .bootstrapPairedMeanPercentChangeInterval(        &storage,        baseline,        candidate,        plan.seed ^ bootstrap_seed_xor,    )) orelse return unsupported(        .invalid_metric_value,        "paired_bootstrap_reference_mean_is_zero",    );    const interval = model.Interval{        .low_percent = interval_value.low,        .high_percent = interval_value.high,    };    const baseline_mean = mean(baseline);    const candidate_mean = mean(candidate);    const verdict = design.classify(        interval,        plan.practical_effect_percent,    );    return .{        .verdict = verdict,        .support = .supported,        .reason = verdictReason(verdict),        .baseline_mean = baseline_mean,        .candidate_mean = candidate_mean,        .percent_change = ((candidate_mean / baseline_mean) - 1.0) * 100.0,        .interval = interval,    };}fn pairedLogRatios(    allocator: std.mem.Allocator,    baseline: []const f64,    candidate: []const f64,) ![]const f64 {    if (baseline.len < 2 or baseline.len != candidate.len) {        return error.InvalidCalibrationSamples;    }    const result = try allocator.alloc(f64, baseline.len);    for (baseline, candidate, result) |base, changed, *ratio| {        if (!validMetric(base) or !validMetric(changed)) {            return error.InvalidExperimentStatistics;        }        ratio.* = @log(changed) - @log(base);        if (!std.math.isFinite(ratio.*)) {            return error.InvalidExperimentStatistics;        }    }    return result;}fn mean(values: []const f64) f64 {    std.debug.assert(values.len > 0);    var total: f64 = 0;    for (values) |value| total += value;    return total / @as(f64, @floatFromInt(values.len));}fn validMetric(value: f64) bool {    return std.math.isFinite(value) and value > 0;}fn sameFile(left: fingerprint.File, right: fingerprint.File) bool {    return left.bytes == right.bytes and        std.mem.eql(u8, &left.sha256, &right.sha256);}fn layoutSupported(    baseline: variant.Prepared,    candidate: variant.Prepared,) bool {    if (sameFile(baseline.file, candidate.file)) return true;    return baseline.code_layout != null and candidate.code_layout != null;}fn setupPassed(setup: [2]?acquire.CommandResult) bool {    for (setup) |command| {        if (command) |value| {            if (value.execution.exit_code != 0) return false;        }    }    return true;}fn unsupported(    support: model.Support,    reason: []const u8,) receipt.Assessment {    std.debug.assert(support != .supported);    return .{        .verdict = .unsupported,        .support = support,        .reason = reason,    };}fn supportReason(support: model.Support) []const u8 {    return switch (support) {        .supported => "supported",        .budget_insufficient => "calculated_evaluation_pairs_exceed_plan_budget",        .setup_failed => "scenario_setup_command_failed",        .reset_failed => "scenario_reset_command_failed",        .execution_failed => "measured_command_failed",        .oracle_failed => "scenario_oracle_failed",        .metric_unavailable => "requested_metric_unavailable",        .invalid_metric_value => "requested_metric_is_not_positive_finite",        .layout_unavailable => "code_layout_unavailable_for_distinct_binaries",    };}fn verdictReason(verdict: model.Verdict) []const u8 {    return switch (verdict) {        .faster => "confidence_interval_below_negative_practical_effect",        .slower => "confidence_interval_above_practical_effect",        .equivalent => "confidence_interval_within_practical_equivalence_region",        .inconclusive => "confidence_interval_crosses_a_practical_boundary",        .unsupported => "unsupported",    };}fn validateRunId(run_id: []const u8) !void {    if (run_id.len == 0 or run_id.len > 255 or        std.mem.eql(u8, run_id, ".") or        std.mem.eql(u8, run_id, ".."))    {        return error.InvalidExperimentRunId;    }    for (run_id) |byte| {        if (!std.ascii.isAlphanumeric(byte) and            byte != '.' and byte != '_' and byte != '-')        {            return error.InvalidExperimentRunId;        }    }}test "profiling experiment log ratios preserve pairs" {    const result = try pairedLogRatios(        std.testing.allocator,        &.{ 10, 20, 40 },        &.{ 11, 22, 44 },    );    defer std.testing.allocator.free(result);    for (result) |value| {        try std.testing.expectApproxEqAbs(            std.math.log1p(@as(f64, 0.1)),            value,            0.0001,        );    }}test "profiling experiment run IDs are single safe path segments" {    try validateRunId("experiment-42_main");    try std.testing.expectError(        error.InvalidExperimentRunId,        validateRunId("../outside"),    );    try std.testing.expectError(        error.InvalidExperimentRunId,        validateRunId("nested/path"),    );}test "profiling experiment records reset failure as unsupported evidence" {    var temporary = std.testing.tmpDir(.{});    defer temporary.cleanup();    var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);    defer arena_state.deinit();    const allocator = arena_state.allocator();    try temporary.dir.writeFile(std.Options.debug_io, .{        .sub_path = "plan.json",        .data =        \\{"schema":"tiny.profiling.experiment/v1","name":"reset failure",        \\ "scenario":{"schema":"tiny.profiling.scenario/v1","name":"true",        \\   "argv":["{binary}"],"reset":{"argv":["/bin/false"]}},        \\ "baseline":{"binary":"/bin/true"},"candidate":{"binary":"/bin/true"},        \\ "metric":"wall_ns","practical_effect_percent":50,"seed":7,        \\ "design":{"calibration_pairs":2,"minimum_evaluation_pairs":2,        \\   "maximum_evaluation_pairs":2}}        ,    });    const root = try temporary.parent_dir.realPathFileAlloc(        std.Options.debug_io,        temporary.sub_path[0..],        allocator,    );    const plan_path = try std.fs.path.join(        allocator,        &.{ root, "plan.json" },    );    const outcome = try execute(        allocator,        std.Options.debug_io,        null,        .{            .plan_path = plan_path,            .output_dir = root,            .run_id = "reset-failure",        },    );    try std.testing.expectEqual(model.Verdict.unsupported, outcome.verdict);    try std.testing.expectEqual(model.Support.reset_failed, outcome.support);    const recorded = try sys.fs.readFileAlloc(        allocator,        outcome.receipt_path,        1024 * 1024,    );    try std.testing.expect(        std.mem.indexOf(u8, recorded, "\"support\":\"reset_failed\"") != null,    );    try std.testing.expect(        std.mem.indexOf(u8, recorded, "\"host_state\"") != null,    );}

Complete call list for experiment.run.execute

7 direct calls.

Audit

Definitions4
Public names4
Members9
Version26.7.0
Revisiondaab053ee433