tiny.profiling.experiment.receipt

1const std = @import("std");2const capture = @import("capture");3const pretty = @import("pretty");4const sys = @import("sys");56const profiling = @import("../root.zig");7const acquire = @import("acquire.zig");8const model = @import("model.zig");9const variant = @import("variant.zig");10const environment = profiling.environment;11const execute = profiling.execute;12const fingerprint = profiling.fingerprint;13const host = profiling.host;14const scenario = profiling.scenario;1516const pretty_json = pretty.json;1718pub const Assessment = struct {19    verdict: model.Verdict,20    support: model.Support,21    reason: []const u8,22    baseline_mean: ?f64 = null,23    candidate_mean: ?f64 = null,24    percent_change: ?f64 = null,25    interval: ?model.Interval = null,26};2728pub const Document = struct {29    run_id: []const u8,30    artifact_root: []const u8,31    plan_path: []const u8,32    plan_file: fingerprint.File,33    plan: model.Plan,34    started_unix_ns: i128,35    finished_unix_ns: i128,36    host_environment: environment.Host,37    host_state_start: host.state.Snapshot,38    baseline: variant.Prepared,39    candidate: variant.Prepared,40    setup: [2]?acquire.CommandResult,41    calibration: ?acquire.PhaseOutcome,42    sample_plan: ?model.SamplePlan,43    evaluation: ?acquire.PhaseOutcome,44    assessment: Assessment,45};4647pub fn writeFile(48    allocator: std.mem.Allocator,49    path: []const u8,50    document: Document,51) !void {52    const pending = try std.fmt.allocPrint(53        allocator,54        "{s}.pending",55        .{path},56    );57    defer sys.fs.deleteFile(pending) catch {};58    var file = try sys.fs.createFile(pending, .{ .truncate = true });59    var open = true;60    defer if (open) file.close(sys.fs.debugIo());61    var buffer: [64 * 1024]u8 = undefined;62    var writer = file.writer(sys.fs.debugIo(), &buffer);63    var out = pretty_json.Writer.init(&writer.interface, .minified);64    try writeJson(&out, document);65    try writer.interface.writeByte('\n');66    try writer.interface.flush();67    file.close(sys.fs.debugIo());68    open = false;69    try sys.fs.rename(pending, path);70}7172pub fn writeJson(73    out: *pretty_json.Writer,74    document: Document,75) !void {76    try out.beginObject();77    try field(out, "schema", model.receipt_schema);78    try field(out, "run_id", document.run_id);79    try field(out, "artifact_root", document.artifact_root);80    try field(out, "started_unix_ns", document.started_unix_ns);81    try field(out, "finished_unix_ns", document.finished_unix_ns);82    try writePlanSource(out, document);83    try writeMethod(out, document.plan);84    try writeEnvironment(out, document);85    try out.objectField("plan");86    try writePlan(out, document.plan);87    try out.objectField("variants");88    try out.beginObject();89    try out.objectField("baseline");90    try writePrepared(out, document.baseline);91    try out.objectField("candidate");92    try writePrepared(out, document.candidate);93    try out.endObject();94    try out.objectField("setup");95    try writeSetup(out, document.setup);96    try out.objectField("calibration");97    try writeOptionalPhase(out, document.calibration, true);98    try out.objectField("sample_plan");99    try out.write(document.sample_plan);100    try out.objectField("evaluation");101    try writeOptionalPhase(out, document.evaluation, false);102    try out.objectField("assessment");103    try writeAssessment(out, document.assessment);104    try out.endObject();105}106107fn writePlanSource(108    out: *pretty_json.Writer,109    document: Document,110) !void {111    const hex = document.plan_file.hex();112    try out.objectField("plan_source");113    try out.beginObject();114    try field(out, "path", document.plan_path);115    try field(out, "bytes", document.plan_file.bytes);116    try field(out, "sha256", hex[0..]);117    try out.endObject();118}119120fn writeMethod(121    out: *pretty_json.Writer,122    plan: model.Plan,123) !void {124    try out.objectField("method");125    try out.beginObject();126    try field(out, "design", "discarded_calibration_then_fixed_evaluation");127    try field(out, "pairing", "matched_baseline_candidate_process_samples");128    try field(out, "ordering", "balanced_seeded_within_pair_interleaving");129    try field(out, "stopping", "evaluation_pair_count_fixed_before_evaluation");130    try field(131        out,132        "planning_assumption",133        "independent paired log-ratios with approximate normal sample planning",134    );135    try field(136        out,137        "evidence",138        "deterministic_percentile_bootstrap_paired_mean_percent_change",139    );140    try field(out, "confidence", model.confidence);141    try field(out, "power", model.power);142    try field(out, "bootstrap_iterations", capture.compare.effect.bootstrap_iterations);143    try field(out, "practical_effect_percent", plan.practical_effect_percent);144    try out.endObject();145}146147fn writeEnvironment(148    out: *pretty_json.Writer,149    document: Document,150) !void {151    try out.objectField("environment");152    try out.beginObject();153    try out.objectField("host");154    try environment.writeJson(out, document.host_environment);155    try out.objectField("host_state_start");156    try host.state.writeJson(out, document.host_state_start);157    try out.endObject();158}159160fn writePlan(out: *pretty_json.Writer, plan: model.Plan) !void {161    try out.beginObject();162    try field(out, "schema", model.schema);163    try field(out, "name", plan.name);164    try out.objectField("scenario");165    try writeScenario(out, plan.scenario);166    try out.objectField("baseline");167    try writeVariantSource(out, plan.baseline);168    try out.objectField("candidate");169    try writeVariantSource(out, plan.candidate);170    try field(out, "metric", @tagName(plan.metric));171    try field(out, "metric_unit", plan.metric.unit());172    try field(out, "practical_effect_percent", plan.practical_effect_percent);173    try field(out, "seed", plan.seed);174    try out.objectField("design");175    try out.write(plan.design);176    try out.endObject();177}178179fn writeScenario(180    out: *pretty_json.Writer,181    definition: scenario.Definition,182) !void {183    try out.beginObject();184    try field(out, "schema", scenario.schema);185    try field(out, "name", definition.name);186    switch (definition.input) {187        .catalog => |input| {188            try field(out, "input_kind", "catalog");189            try field(out, "workload", input.workload);190            try out.objectField("args");191            try out.write(input.args);192        },193        .command => |argv| {194            try field(out, "input_kind", "command");195            try out.objectField("argv");196            try out.write(argv);197        },198    }199    try field(out, "cwd", definition.cwd);200    try out.objectField("env");201    try out.write(definition.environment);202    try out.objectField("setup");203    try out.write(definition.setup);204    try out.objectField("reset");205    try out.write(definition.reset);206    try out.objectField("oracle");207    try out.write(definition.oracle);208    try out.endObject();209}210211fn writeVariantSource(212    out: *pretty_json.Writer,213    source: model.Variant,214) !void {215    try out.beginObject();216    switch (source) {217        .binary => |path| {218            try field(out, "kind", "binary");219            try field(out, "path", path);220        },221        .git_ref => |git_ref| {222            try field(out, "kind", "git_ref");223            try field(out, "ref", git_ref.name);224            try out.objectField("build");225            try out.write(git_ref.build);226        },227    }228    try out.endObject();229}230231fn writePrepared(232    out: *pretty_json.Writer,233    prepared: variant.Prepared,234) !void {235    const file_hex = prepared.file.hex();236    try out.beginObject();237    try field(out, "side", @tagName(prepared.side));238    try out.objectField("source");239    try writeVariantSource(out, prepared.source);240    try field(out, "resolved_ref", prepared.resolved_ref);241    try field(out, "build_command", prepared.build_command);242    try field(out, "source_root", prepared.root);243    try field(out, "default_cwd", prepared.default_cwd);244    try field(out, "archived_binary_path", prepared.archived_binary_path);245    try out.objectField("binary_fingerprint");246    try out.beginObject();247    try field(out, "bytes", prepared.file.bytes);248    try field(out, "sha256", file_hex[0..]);249    try out.endObject();250    try out.objectField("code_layout");251    if (prepared.code_layout) |layout| {252        const layout_hex = layout.hex();253        try out.beginObject();254        try field(out, "schema", capture.code_layout.schema);255        try field(out, "sha256", layout_hex[0..]);256        try field(out, "executable_sections", layout.executable_sections);257        try field(out, "executable_bytes", layout.executable_bytes);258        try field(out, "first_address", layout.first_address);259        try field(out, "last_address_end", layout.last_address_end);260        try out.endObject();261    } else {262        try out.write(null);263    }264    try out.endObject();265}266267fn writeSetup(268    out: *pretty_json.Writer,269    setup: [2]?acquire.CommandResult,270) !void {271    try out.beginObject();272    try out.objectField("baseline");273    try writeOptionalCommand(out, setup[0]);274    try out.objectField("candidate");275    try writeOptionalCommand(out, setup[1]);276    try out.endObject();277}278279fn writeOptionalCommand(280    out: *pretty_json.Writer,281    command: ?acquire.CommandResult,282) !void {283    if (command) |value| {284        try writeCommand(out, value);285    } else {286        try out.write(null);287    }288}289290fn writeCommand(291    out: *pretty_json.Writer,292    command: acquire.CommandResult,293) !void {294    try out.beginObject();295    try field(out, "command", command.command);296    try field(out, "cwd", command.cwd);297    try field(out, "stdout_path", command.stdout_path);298    try field(out, "stderr_path", command.stderr_path);299    try out.objectField("execution");300    try writeExecution(out, command.execution);301    try out.endObject();302}303304fn writeOptionalPhase(305    out: *pretty_json.Writer,306    outcome: ?acquire.PhaseOutcome,307    discarded: bool,308) !void {309    if (outcome) |value| {310        try writePhase(out, value, discarded);311    } else {312        try out.write(null);313    }314}315316fn writePhase(317    out: *pretty_json.Writer,318    outcome: acquire.PhaseOutcome,319    discarded: bool,320) !void {321    try out.beginObject();322    try field(out, "discarded_from_evaluation", discarded);323    try field(out, "support", @tagName(outcome.support));324    try field(out, "scheduled_pairs", outcome.orders.len);325    try field(out, "completed_pairs", outcome.baseline_values.len);326    try out.objectField("orders");327    try out.write(outcome.orders);328    try out.objectField("baseline_values");329    try out.write(outcome.baseline_values);330    try out.objectField("candidate_values");331    try out.write(outcome.candidate_values);332    try out.objectField("observations");333    try out.beginArray();334    for (outcome.observations) |observation| {335        try writeObservation(out, observation);336    }337    try out.endArray();338    try out.endObject();339}340341fn writeObservation(342    out: *pretty_json.Writer,343    observation: acquire.Observation,344) !void {345    try out.beginObject();346    try field(out, "sequence", observation.sequence);347    try field(out, "phase", @tagName(observation.phase));348    try field(out, "pair_index", observation.pair_index);349    try field(out, "position_in_pair", observation.position_in_pair);350    try field(out, "pair_order", @tagName(observation.pair_order));351    try field(out, "side", @tagName(observation.side));352    try field(out, "command", observation.command);353    try field(out, "cwd", observation.cwd);354    try out.objectField("reset");355    try writeOptionalCommand(out, observation.reset);356    try field(out, "stdout_path", observation.stdout_path);357    try field(out, "stderr_path", observation.stderr_path);358    try out.objectField("execution");359    try writeExecution(out, observation.execution);360    try field(out, "metric_value", observation.metric_value);361    try out.objectField("oracle");362    try writeOracle(out, observation.oracle);363    try out.objectField("host_state");364    try writeHostWindow(out, observation.host_state);365    try field(out, "support", @tagName(observation.support));366    try out.endObject();367}368369fn writeExecution(370    out: *pretty_json.Writer,371    execution: execute.Result,372) !void {373    try out.beginObject();374    inline for (.{375        "pid",376        "exit_code",377        "wall_ns",378        "resource_usage_source",379        "maxrss_kib",380        "user_s",381        "system_s",382        "minor_page_faults",383        "major_page_faults",384        "voluntary_context_switches",385        "involuntary_context_switches",386    }) |name| {387        try out.objectField(name);388        try out.write(@field(execution, name));389    }390    try out.endObject();391}392393fn writeOracle(394    out: *pretty_json.Writer,395    result: @import("oracle.zig").Result,396) !void {397    const stdout_hex = result.stdout_sha256;398    const stderr_hex = result.stderr_sha256;399    try out.beginObject();400    try field(out, "passed", result.passed);401    try field(out, "exit_code_matches", result.exit_code_matches);402    try field(out, "stdout_sha256", stdout_hex[0..]);403    try field(out, "stderr_sha256", stderr_hex[0..]);404    try field(out, "stdout_matches", result.stdout_matches);405    try field(out, "stderr_matches", result.stderr_matches);406    try out.objectField("files");407    try out.beginArray();408    for (result.files) |file| {409        try out.beginObject();410        try field(out, "path", file.path);411        try field(out, "expected_sha256", file.expected_sha256);412        try out.objectField("actual_sha256");413        if (file.actual_sha256) |actual| {414            try out.write(actual[0..]);415        } else {416            try out.write(null);417        }418        try field(out, "passed", file.passed);419        try out.endObject();420    }421    try out.endArray();422    try out.endObject();423}424425fn writeHostWindow(426    out: *pretty_json.Writer,427    window: host.state.Window,428) !void {429    try out.beginObject();430    try field(out, "placement", "immediately_before_and_after_timed_child");431    try field(out, "condition", @tagName(window.condition));432    try out.objectField("before");433    try host.state.writeJson(out, window.before);434    try out.objectField("after");435    try host.state.writeJson(out, window.after);436    try out.endObject();437}438439fn writeAssessment(440    out: *pretty_json.Writer,441    assessment: Assessment,442) !void {443    try out.beginObject();444    try field(out, "verdict", @tagName(assessment.verdict));445    try field(out, "support", @tagName(assessment.support));446    try field(out, "reason", assessment.reason);447    try field(out, "baseline_mean", assessment.baseline_mean);448    try field(out, "candidate_mean", assessment.candidate_mean);449    try field(out, "percent_change", assessment.percent_change);450    try out.objectField("confidence_interval_percent");451    if (assessment.interval) |interval| {452        try out.beginObject();453        try field(out, "low", interval.low_percent);454        try field(out, "high", interval.high_percent);455        try out.endObject();456    } else {457        try out.write(null);458    }459    try out.endObject();460}461462fn field(463    out: *pretty_json.Writer,464    name: []const u8,465    value: anytype,466) !void {467    try out.objectField(name);468    try out.write(value);469}470471test "profiling experiment receipt preserves typed scenario policy" {472    var buffer = std.Io.Writer.Allocating.init(std.testing.allocator);473    defer buffer.deinit();474    var out = pretty_json.Writer.init(&buffer.writer, .minified);475    try writeScenario(&out, .{476        .name = "startup",477        .input = .{ .command = &.{ "{binary}", "--version" } },478        .environment = &.{.{ .name = "MODE", .value = "{side}" }},479        .reset = .{ .argv = &.{"/bin/true"} },480        .oracle = .{ .exit_code = 0 },481    });482    try std.testing.expect(483        std.mem.indexOf(u8, buffer.written(), "\"input_kind\":\"command\"") != null,484    );485    try std.testing.expect(486        std.mem.indexOf(u8, buffer.written(), "\"reset\"") != null,487    );488    try std.testing.expect(489        std.mem.indexOf(u8, buffer.written(), "\"oracle\"") != null,490    );491}492493test "profiling experiment receipt retains child process identity" {494    var buffer = std.Io.Writer.Allocating.init(std.testing.allocator);495    defer buffer.deinit();496    var out = pretty_json.Writer.init(&buffer.writer, .minified);497    try writeExecution(&out, .{498        .pid = 4242,499        .exit_code = 0,500        .wall_ns = 10,501    });502    try std.testing.expect(503        std.mem.indexOf(u8, buffer.written(), "\"pid\":4242") != null,504    );505}

1pub const receipt = @import("receipt.zig");