tiny.profiling.analyze.ordering

1const std = @import("std");2const sys = @import("sys");3const host = @import("../root.zig").host;4const measurement = @import("../root.zig").measurement;5const order = @import("../root.zig").order;6const ComparisonSupport = @import("root.zig").model.ComparisonSupport;7const OrderEffectRow = @import("root.zig").model.OrderEffectRow;8const OrderEffectStatus = @import("root.zig").model.OrderEffectStatus;9const OrderEffectSupport = @import("root.zig").model.OrderEffectSupport;10const OrderEffectTestFixture = @import("fixture/root.zig").OrderEffectTestFixture;11const OrderEffects = @import("root.zig").model.OrderEffects;12const Run = @import("root.zig").model.Run;13const Workload = @import("root.zig").model.Workload;14const comparisonSupport = @import("root.zig").compare.comparisonSupport;15const orderEffectTestFixture = @import("fixture/root.zig").orderEffectTestFixture;16const sameOptionalString = @import("root.zig").model.sameOptionalString;17const sameStrings = @import("root.zig").model.sameStrings;1819pub fn buildOrderEffects(20    allocator: std.mem.Allocator,21    base: Run,22    candidate: Run,23) !?OrderEffects {24    const base_method = runAcquisitionMethod(base) orelse return null;25    const candidate_method = runAcquisitionMethod(candidate) orelse return null;26    if (base_method == candidate_method) return null;27    const fixed_is_baseline = base_method == .blocked and28        candidate_method == .random_interleaved;29    if (!fixed_is_baseline and30        !(base_method == .random_interleaved and candidate_method == .blocked))31    {32        return null;33    }34    const fixed = if (fixed_is_baseline) base else candidate;35    const random = if (fixed_is_baseline) candidate else base;36    const environment_support = comparisonSupport(fixed, random);37    const support = orderEffectSupport(fixed, random, environment_support);38    if (support != .supported) return .{39        .support = support,40        .environment_support = environment_support,41        .fixed_run_id = fixed.ref.run_id,42        .random_run_id = random.ref.run_id,43        .fixed_is_baseline = fixed_is_baseline,44        .status = .unsupported,45        .rows = &.{},46    };4748    const summary = try summarizeOrderEffects(allocator, fixed, random);49    return .{50        .support = .supported,51        .environment_support = environment_support,52        .fixed_run_id = fixed.ref.run_id,53        .random_run_id = random.ref.run_id,54        .fixed_is_baseline = fixed_is_baseline,55        .status = summary.status,56        .rows = summary.rows,57    };58}5960const OrderEffectSummary = struct {61    status: OrderEffectStatus,62    rows: []const OrderEffectRow,63};6465fn summarizeOrderEffects(66    allocator: std.mem.Allocator,67    fixed: Run,68    random: Run,69) !OrderEffectSummary {70    const rows = try allocator.alloc(OrderEffectRow, fixed.workloads.len);71    var status = OrderEffectStatus.no_order_effect_observed;72    for (fixed.workloads, random.workloads, rows) |fixed_workload, random_workload, *row| {73        row.* = try summarizeOrderEffectWorkload(74            allocator,75            fixed_workload,76            random_workload,77        );78        status = combinedOrderEffectStatus(status, row.classification);79    }80    return .{ .status = status, .rows = rows };81}8283fn summarizeOrderEffectWorkload(84    allocator: std.mem.Allocator,85    fixed: Workload,86    random: Workload,87) !OrderEffectRow {88    var fixed_scratch: [host.process.max_executions]f64 = undefined;89    var random_scratch: [host.process.max_executions]f64 = undefined;90    const fixed_samples = try measurement.wallEffectSamples(91        &fixed_scratch,92        fixed.executions,93        null,94    );95    const random_samples = try measurement.wallEffectSamples(96        &random_scratch,97        random.executions,98        null,99    );100    const result = try order.contrast.summarize(101        allocator,102        fixed.name,103        fixed_samples,104        random_samples,105    );106    return .{107        .workload = fixed.name,108        .fixed_acquisition = fixed.acquisition,109        .random_acquisition = random.acquisition,110        .fixed_count = result.fixed_count,111        .random_count = result.random_count,112        .fixed_mean_ns = result.fixed_mean_ns,113        .random_mean_ns = result.random_mean_ns,114        .mean_percent_change = result.mean_percent_change,115        .effect_low_percent = if (result.effect_interval) |value| value.low else null,116        .effect_high_percent = if (result.effect_interval) |value| value.high else null,117        .classification = result.classification,118    };119}120121fn combinedOrderEffectStatus(122    status: OrderEffectStatus,123    classification: order.contrast.Classification,124) OrderEffectStatus {125    return switch (classification) {126        .order_sensitive_candidate => .order_sensitive_candidate,127        .insufficient_repetitions => if (status == .no_order_effect_observed)128            .insufficient_repetitions129        else130            status,131        .no_order_effect_observed => status,132    };133}134135fn runAcquisitionMethod(run_value: Run) ?order.Method {136    const first = if (run_value.workloads.len != 0)137        run_value.workloads[0]138    else139        return null;140    return first.acquisition.method();141}142143fn orderEffectSupport(144    fixed: Run,145    random: Run,146    environment_support: ComparisonSupport,147) OrderEffectSupport {148    if (environment_support != .supported) return .environment_mismatch;149    const fixed_sha = fixed.git_sha orelse return .code_identity_missing;150    const random_sha = random.git_sha orelse return .code_identity_missing;151    const fixed_dirty = fixed.git_dirty orelse return .code_identity_missing;152    const random_dirty = random.git_dirty orelse return .code_identity_missing;153    if (!std.mem.eql(u8, fixed_sha, random_sha)) return .code_identity_mismatch;154    if (fixed_dirty or random_dirty) return .dirty_code_identity;155    if (!sameCohort(fixed, random)) return .cohort_mismatch;156    for (fixed.workloads, random.workloads) |fixed_workload, random_workload| {157        const support = workloadOrderEffectSupport(fixed_workload, random_workload);158        if (support != .supported) return support;159    }160    return .supported;161}162163fn workloadOrderEffectSupport(164    fixed: Workload,165    random: Workload,166) OrderEffectSupport {167    if (fixed.exit_code != 0 or random.exit_code != 0) return .workload_failure;168    if (!fixed.command_identity_recorded or !random.command_identity_recorded or169        fixed.command_argv.len == 0 or random.command_argv.len == 0)170    {171        return .command_identity_missing;172    }173    if (!sameOptionalString(fixed.command_cwd, random.command_cwd) or174        !sameStrings(fixed.command_argv, random.command_argv))175    {176        return .command_identity_mismatch;177    }178    if (!fixed.scope_recorded or !random.scope_recorded) {179        return .execution_scope_missing;180    }181    if (!std.mem.eql(u8, fixed.scope, random.scope)) {182        return .execution_scope_mismatch;183    }184    if (!fixed.warmup_recorded or !random.warmup_recorded) {185        return .warmup_design_missing;186    }187    if (fixed.warmup_count != random.warmup_count) return .warmup_design_mismatch;188    if (!fixed.profiler_configuration_recorded or189        !random.profiler_configuration_recorded)190    {191        return .profiler_configuration_missing;192    }193    if (workloadHasProfilerCapture(fixed) or workloadHasProfilerCapture(random)) {194        return .profiler_capture_present;195    }196    const random_design = switch (random.acquisition) {197        .blocked => return .cohort_mismatch,198        .random_interleaved => |value| value,199    };200    if (fixed.executions.len != random_design.repeat_count or201        random.executions.len != random_design.repeat_count)202    {203        return .repetition_design_mismatch;204    }205    if (!fixed.measurement_recorded or !random.measurement_recorded or206        fixed.executions.len < order.contrast.minimum_repetitions or207        random.executions.len < order.contrast.minimum_repetitions)208    {209        return .raw_measurements_missing;210    }211    return .supported;212}213214fn sameCohort(fixed: Run, random: Run) bool {215    if (!fixed.selection_known or !random.selection_known or216        fixed.workloads.len < 2 or217        fixed.workloads.len != random.workloads.len)218    {219        return false;220    }221    for (fixed.workloads, random.workloads) |fixed_workload, random_workload| {222        if (!std.mem.eql(u8, fixed_workload.name, random_workload.name)) {223            return false;224        }225        const fixed_design = switch (fixed_workload.acquisition) {226            .blocked => |value| value,227            .random_interleaved => return false,228        };229        const random_design = switch (random_workload.acquisition) {230            .blocked => return false,231            .random_interleaved => |value| value,232        };233        if (fixed_design.workload_count != fixed.workloads.len or234            random_design.workload_count != random.workloads.len)235        {236            return false;237        }238    }239    return true;240}241242fn workloadHasProfilerCapture(workload: Workload) bool {243    if (workload.captures.len != 0 or244        workload.capture_perturbation != null or245        workload.perf_stat != null or246        workload.causal_enabled or247        workload.tracy_enabled or248        workload.allocations_enabled)249    {250        return true;251    }252    if (workload.analysisArtifacts()) |artifacts| {253        if (artifactHasBytes(artifacts.allocations)) return true;254    }255    for (workload.executions) |execution| {256        const artifacts = execution.artifacts orelse continue;257        if (artifactHasBytes(artifacts.allocations)) return true;258    }259    return false;260}261262fn artifactHasBytes(path: []const u8) bool {263    const stat = sys.fs.statFile(path) catch |err| return switch (err) {264        error.FileNotFound => false,265        else => true,266    };267    return stat.size != 0;268}269270test "profiling analysis diagnoses fixed versus interleaved acquisition" {271    var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);272    defer arena_state.deinit();273    const allocator = arena_state.allocator();274    const fixture = try orderEffectTestFixture(allocator);275    const effects = (try buildOrderEffects(allocator, fixture.fixed, fixture.random)).?;276    try std.testing.expectEqual(OrderEffectSupport.supported, effects.support);277    try std.testing.expectEqual(278        OrderEffectStatus.order_sensitive_candidate,279        effects.status,280    );281    try std.testing.expect(effects.fixed_is_baseline);282    try std.testing.expectEqual(@as(usize, 2), effects.rows.len);283    try std.testing.expectEqual(284        order.contrast.Classification.order_sensitive_candidate,285        effects.rows[0].classification,286    );287    try std.testing.expectEqual(288        order.contrast.Classification.no_order_effect_observed,289        effects.rows[1].classification,290    );291    const reversed = (try buildOrderEffects(allocator, fixture.random, fixture.fixed)).?;292    try std.testing.expect(!reversed.fixed_is_baseline);293    try std.testing.expectEqualStrings("fixed", reversed.fixed_run_id);294    try std.testing.expectEqualStrings("random", reversed.random_run_id);295}296297pub fn expectOrderEffectWorkloadSupport(298    allocator: std.mem.Allocator,299    fixture: OrderEffectTestFixture,300    altered_first: Workload,301    expected: OrderEffectSupport,302) !void {303    var workloads = [_]Workload{ altered_first, fixture.random.workloads[1] };304    var altered = fixture.random;305    altered.workloads = &workloads;306    try std.testing.expectEqual(307        expected,308        (try buildOrderEffects(allocator, fixture.fixed, altered)).?.support,309    );310}311312test "profiling order diagnostics accept an explicitly recorded root cwd" {313    var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);314    defer arena_state.deinit();315    const allocator = arena_state.allocator();316    const fixture = try orderEffectTestFixture(allocator);317    var fixed_workloads = [_]Workload{318        fixture.fixed.workloads[0],319        fixture.fixed.workloads[1],320    };321    var random_workloads = [_]Workload{322        fixture.random.workloads[0],323        fixture.random.workloads[1],324    };325    fixed_workloads[0].command_cwd = null;326    random_workloads[0].command_cwd = null;327    var fixed = fixture.fixed;328    var random = fixture.random;329    fixed.workloads = &fixed_workloads;330    random.workloads = &random_workloads;331    try std.testing.expectEqual(332        OrderEffectSupport.supported,333        (try buildOrderEffects(allocator, fixed, random)).?.support,334    );335}

1pub const ordering = @import("ordering.zig");