tiny.profiling.experiment.oracle

1const std = @import("std");23const profiling = @import("../root.zig");4const fingerprint = profiling.fingerprint;5const scenario = profiling.scenario;67pub const FileResult = struct {8    path: []const u8,9    expected_sha256: []const u8,10    actual_sha256: ?fingerprint.Hex,11    passed: bool,12};1314pub const Result = struct {15    passed: bool,16    exit_code_matches: bool,17    stdout_sha256: fingerprint.Hex,18    stderr_sha256: fingerprint.Hex,19    stdout_matches: bool,20    stderr_matches: bool,21    files: []const FileResult,22};2324pub fn verify(25    allocator: std.mem.Allocator,26    expected: scenario.Oracle,27    values: scenario.Substitutions,28    cwd: ?[]const u8,29    exit_code: i64,30    stdout_path: []const u8,31    stderr_path: []const u8,32) !Result {33    const stdout = try fingerprint.inspect(stdout_path);34    const stderr = try fingerprint.inspect(stderr_path);35    const stdout_hex = stdout.hex();36    const stderr_hex = stderr.hex();37    const stdout_matches = matches(expected.stdout_sha256, &stdout_hex);38    const stderr_matches = matches(expected.stderr_sha256, &stderr_hex);39    const files = try verifyFiles(40        allocator,41        expected.files,42        values,43        cwd,44    );45    var files_pass = true;46    for (files) |file| files_pass = files_pass and file.passed;47    const exit_matches = exit_code == expected.exit_code;48    return .{49        .passed = exit_matches and stdout_matches and50            stderr_matches and files_pass,51        .exit_code_matches = exit_matches,52        .stdout_sha256 = stdout_hex,53        .stderr_sha256 = stderr_hex,54        .stdout_matches = stdout_matches,55        .stderr_matches = stderr_matches,56        .files = files,57    };58}5960fn verifyFiles(61    allocator: std.mem.Allocator,62    expected: []const scenario.FileOracle,63    values: scenario.Substitutions,64    cwd: ?[]const u8,65) ![]const FileResult {66    const result = try allocator.alloc(FileResult, expected.len);67    for (expected, 0..) |file, index| {68        const substituted = try scenario.substitute(69            allocator,70            file.path,71            values,72        );73        const path = try resolvePath(allocator, cwd, substituted);74        const actual = fingerprint.inspect(path) catch null;75        const actual_hex = if (actual) |value| value.hex() else null;76        result[index] = .{77            .path = path,78            .expected_sha256 = file.sha256,79            .actual_sha256 = actual_hex,80            .passed = if (actual_hex) |hex|81                std.mem.eql(u8, file.sha256, &hex)82            else83                false,84        };85    }86    return result;87}8889fn resolvePath(90    allocator: std.mem.Allocator,91    cwd: ?[]const u8,92    path: []const u8,93) ![]const u8 {94    if (std.fs.path.isAbsolute(path)) return path;95    return try std.fs.path.join(allocator, &.{ cwd orelse ".", path });96}9798fn matches(expected: ?[]const u8, actual: []const u8) bool {99    const value = expected orelse return true;100    return std.mem.eql(u8, value, actual);101}102103test "profiling experiment oracle verifies exit and output digests" {104    var temporary = std.testing.tmpDir(.{});105    defer temporary.cleanup();106    try temporary.dir.writeFile(std.Options.debug_io, .{107        .sub_path = "stdout",108        .data = "",109    });110    try temporary.dir.writeFile(std.Options.debug_io, .{111        .sub_path = "stderr",112        .data = "",113    });114    var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);115    defer arena_state.deinit();116    const allocator = arena_state.allocator();117    const root = try temporary.parent_dir.realPathFileAlloc(118        std.Options.debug_io,119        temporary.sub_path[0..],120        allocator,121    );122    const stdout_path = try std.fs.path.join(123        allocator,124        &.{ root, "stdout" },125    );126    const stderr_path = try std.fs.path.join(127        allocator,128        &.{ root, "stderr" },129    );130    const empty_digest = "e3b0c44298fc1c149afbf4c8996fb924" ++131        "27ae41e4649b934ca495991b7852b855";132    const result = try verify(133        allocator,134        .{ .stdout_sha256 = empty_digest },135        .{ .binary = "bin", .root = root, .side = "baseline" },136        root,137        0,138        stdout_path,139        stderr_path,140    );141    try std.testing.expect(result.passed);142    try std.testing.expect(result.stdout_matches);143    const mismatch = try verify(144        allocator,145        .{ .exit_code = 1 },146        .{ .binary = "bin", .root = root, .side = "baseline" },147        root,148        0,149        stdout_path,150        stderr_path,151    );152    try std.testing.expect(!mismatch.passed);153    try std.testing.expect(!mismatch.exit_code_matches);154}

1pub const oracle = @import("oracle.zig");