tiny.profiling.analyze.compare
Defined in analyze.
API (11)
Actions
Public operations.
allocationBudgetCheckscompareCounterRunscompareEnergyRunscompareMemoryRunscompareMetricRunscompareRunscomparisonSupportmissingWorkloadssupportedBaselineworkloadComparisonSkipsworkloadComparisonSupport
Source
Source: src/profiling/analyze/compare.zig
zig
const std = @import("std");const capture = @import("capture");const sys = @import("sys");const baseline = @import("../root.zig").baseline;const budget = @import("../root.zig").budget;const catalog = @import("../root.zig").catalog;const environment = @import("../root.zig").environment;const host = @import("../root.zig").host;const measurement = @import("../root.zig").measurement;const memory = @import("../root.zig").memory;const metric = @import("../root.zig").metric;const plan = @import("../root.zig").plan;const Comparison = @import("root.zig").model.Comparison;const ComparisonSupport = @import("root.zig").model.ComparisonSupport;const CounterShift = @import("root.zig").model.CounterShift;const EnergyShift = @import("root.zig").model.EnergyShift;const Run = @import("root.zig").model.Run;const Workload = @import("root.zig").model.Workload;const WorkloadComparisonSkip = @import("root.zig").model.WorkloadComparisonSkip;const WorkloadComparisonSupport = @import("root.zig").model.WorkloadComparisonSupport;const analysisTestCounter = @import("fixture/root.zig").analysisTestCounter;const analysisTestWorkload = @import("fixture/root.zig").analysisTestWorkload;const analysisUnsupportedEnergyCounter = @import("fixture/root.zig").analysisUnsupportedEnergyCounter;const findWorkload = @import("root.zig").model.findWorkload;const loadCounterSummary = @import("root.zig").load.loadCounterSummary;const measuredTestWorkload = @import("fixture/root.zig").measuredTestWorkload;const percentChange = @import("root.zig").model.percentChange;const sameOptionalString = @import("root.zig").model.sameOptionalString;const sameStrings = @import("root.zig").model.sameStrings;const workloadWallDistribution = @import("root.zig").model.workloadWallDistribution;pub fn comparisonSupport(base: Run, candidate: Run) ComparisonSupport { return environment.comparisonSupport( base.optimize, base.host, candidate.optimize, candidate.host, );}pub fn supportedBaseline(baseline_run: ?Run, candidate: Run) ?Run { const base = baseline_run orelse return null; return if (comparisonSupport(base, candidate) == .supported) base else null;}pub fn missingWorkloads(allocator: std.mem.Allocator, run_value: Run) ![]const []const u8 { if (!run_value.selection_known) return &.{}; const selection = plan.Selection{ .suite = run_value.suite, .filters = run_value.filters }; var missing: std.ArrayList([]const u8) = .empty; for (catalog.workloads) |workload| { if (!selection.selected(workload)) continue; if (findWorkload(run_value.workloads, workload.name) == null) try missing.append(allocator, workload.name); } return try missing.toOwnedSlice(allocator);}pub fn compareRuns(allocator: std.mem.Allocator, base: Run, candidate: Run, threshold_percent: f64) ![]const Comparison { var rows: std.ArrayList(Comparison) = .empty; for (candidate.workloads) |workload| { if (workload.exit_code != 0) continue; const base_workload = findWorkload(base.workloads, workload.name) orelse continue; if (base_workload.exit_code != 0 or base_workload.wall_ns == 0) continue; if (workloadComparisonSupport(base_workload, workload) != .supported) continue; const workload_threshold = workloadThreshold(workload.name, threshold_percent); const wall_percent = percentChange(base_workload.wall_ns, workload.wall_ns) orelse continue; const rss_percent = rssPercentChange(base_workload.max_rss_kib, workload.max_rss_kib); const wall_regressed = wall_percent >= workload_threshold.wall; const rss_regressed = if (rss_percent) |value| value >= workload_threshold.rss else false; if (wall_regressed or rss_regressed) { const baseline_distribution = workloadWallDistribution(base_workload); const candidate_distribution = workloadWallDistribution(workload); const effect_interval = if (wall_regressed) try wallEffectInterval(allocator, base_workload, workload) else null; try rows.append(allocator, .{ .workload = workload.name, .baseline_wall_ns = base_workload.wall_ns, .candidate_wall_ns = workload.wall_ns, .wall_percent_change = wall_percent, .baseline_wall_samples = baseline_distribution.sample_count, .candidate_wall_samples = candidate_distribution.sample_count, .baseline_wall_distribution = baseline_distribution.kind.name(), .candidate_wall_distribution = candidate_distribution.kind.name(), .wall_effect_low_percent = if (effect_interval) |interval| interval.low else null, .wall_effect_high_percent = if (effect_interval) |interval| interval.high else null, .wall_status = if (wall_regressed) wallComparisonStatus( baseline_distribution, candidate_distribution, workload_threshold.wall, effect_interval, ) else "within_threshold", .baseline_max_rss_kib = base_workload.max_rss_kib, .candidate_max_rss_kib = workload.max_rss_kib, .rss_percent_change = rss_percent, .wall_threshold_percent = workload_threshold.wall, .rss_threshold_percent = workload_threshold.rss, .kind = if (wall_regressed and rss_regressed) "wall+rss" else if (wall_regressed) "wall" else "rss", }); } } return try rows.toOwnedSlice(allocator);}fn wallEffectInterval( allocator: std.mem.Allocator, baseline_workload: Workload, candidate_workload: Workload,) !?capture.compare.effect.Interval { var baseline_scratch: [host.process.max_executions]f64 = undefined; var candidate_scratch: [host.process.max_executions]f64 = undefined; const baseline_samples = try measurement.wallEffectSamples( &baseline_scratch, baseline_workload.executions, baseline_workload.perf_stat, ); const candidate_samples = try measurement.wallEffectSamples( &candidate_scratch, candidate_workload.executions, candidate_workload.perf_stat, ); if (baseline_samples.len < 2 or candidate_samples.len < 2) return null; var effect_storage = try capture.compare.EffectStorage.init(allocator, .{ .max_samples_per_distribution = @max( baseline_samples.len, candidate_samples.len, ), }); defer effect_storage.deinit(allocator); effect_storage.activate(); return try capture.compare.effect.bootstrapMeanPercentChangeInterval( &effect_storage, baseline_samples, candidate_samples, capture.compare.effect.percentChangeSeed(candidate_workload.name), );}fn wallComparisonStatus( baseline_distribution: measurement.WallDistribution, candidate_distribution: measurement.WallDistribution, threshold_percent: f64, effect_interval: ?capture.compare.effect.Interval,) []const u8 { if (effect_interval) |interval| { if (interval.low >= threshold_percent) return "sample_regression"; return "sample_regression_uncertain"; } if (baseline_distribution.kind == .legacy_summary or candidate_distribution.kind == .legacy_summary) { return "summary_regression_candidate"; } if (rawDistributionWithMultipleSamples(baseline_distribution) or rawDistributionWithMultipleSamples(candidate_distribution)) { return "sample_regression_candidate"; } return "point_regression_candidate";}fn rawDistributionWithMultipleSamples( distribution: measurement.WallDistribution,) bool { return distribution.sample_count > 1 and (distribution.kind == .raw_executions or distribution.kind == .grouped_repetition_means);}pub fn workloadComparisonSkips( allocator: std.mem.Allocator, base: Run, candidate: Run,) ![]const WorkloadComparisonSkip { var rows: std.ArrayList(WorkloadComparisonSkip) = .empty; for (candidate.workloads) |workload| { if (workload.exit_code != 0) continue; const base_workload = findWorkload(base.workloads, workload.name) orelse continue; if (base_workload.exit_code != 0) continue; const support = workloadComparisonSupport(base_workload, workload); if (support == .supported) continue; try rows.append(allocator, .{ .workload = workload.name, .support = support, .baseline_acquisition = base_workload.acquisition, .candidate_acquisition = workload.acquisition, .baseline_execution_count = base_workload.execution_count, .candidate_execution_count = workload.execution_count, .baseline_warmup_count = base_workload.warmup_count, .candidate_warmup_count = workload.warmup_count, }); } return try rows.toOwnedSlice(allocator);}pub fn workloadComparisonSupport( baseline_workload: Workload, candidate_workload: Workload,) WorkloadComparisonSupport { return measurement.comparisonSupport( .{ .scope = baseline_workload.scope, .acquisition = baseline_workload.acquisition, .warmup_count = baseline_workload.warmup_count, .captures = baseline_workload.captures, .perf_stat = baseline_workload.perf_stat, .capture_perturbation = if (baseline_workload.capture_perturbation) |actual| actual.configuration else null, }, .{ .scope = candidate_workload.scope, .acquisition = candidate_workload.acquisition, .warmup_count = candidate_workload.warmup_count, .captures = candidate_workload.captures, .perf_stat = candidate_workload.perf_stat, .capture_perturbation = if (candidate_workload.capture_perturbation) |actual| actual.configuration else null, }, );}pub fn compareMetricRuns(allocator: std.mem.Allocator, base: Run, candidate: Run, threshold_percent: f64) ![]const metric.Comparison { const base_metrics = try flattenMetrics(allocator, base); const candidate_metrics = try flattenMetrics(allocator, candidate); return try metric.compare(allocator, base_metrics, candidate_metrics, threshold_percent);}pub fn compareMemoryRuns(allocator: std.mem.Allocator, base: Run, candidate: Run, threshold_percent: f64) ![]const memory.Comparison { const base_metrics = try flattenMemoryMetrics(allocator, base); const candidate_metrics = try flattenMemoryMetrics(allocator, candidate); return try memory.compare(allocator, base_metrics, candidate_metrics, threshold_percent);}pub fn allocationBudgetChecks( allocator: std.mem.Allocator, candidate: Run, baseline_run: ?Run,) ![]const budget.Check { const candidate_metrics = try flattenMemoryMetrics(allocator, candidate); const baseline_metrics = if (baseline_run) |base| try flattenMemoryMetrics(allocator, base) else &.{}; var checks: std.ArrayList(budget.Check) = .empty; for (candidate.workloads) |workload| { const allocation_baseline: budget.Baseline = if (baseline_run) |base| if (findWorkload(base.workloads, workload.name)) |base_workload| if (allocationDesignMatches(base, candidate, base_workload, workload)) .{ .metrics = baseline_metrics } else .design_changed else .{ .metrics = baseline_metrics } else .bootstrap; if (budget.evaluate( workload.name, allocation_baseline, candidate_metrics, )) |check| { try checks.append(allocator, check); } } return try checks.toOwnedSlice(allocator);}fn allocationDesignMatches( baseline_run: Run, candidate_run: Run, baseline_workload: Workload, candidate_workload: Workload,) bool { if (baseline_workload.exit_code != 0 or candidate_workload.exit_code != 0) { return false; } if (baseline_run.optimize == null or candidate_run.optimize == null or !sameOptionalString(baseline_run.optimize, candidate_run.optimize)) { return false; } if (baseline_run.host.os == null or candidate_run.host.os == null or baseline_run.host.arch == null or candidate_run.host.arch == null) { return false; } if (!sameOptionalString(baseline_run.host.os, candidate_run.host.os) or !sameOptionalString(baseline_run.host.arch, candidate_run.host.arch)) { return false; } if (!baseline_workload.command_identity_recorded or !candidate_workload.command_identity_recorded or baseline_workload.command_argv.len == 0 or candidate_workload.command_argv.len == 0) { return false; } if (!sameOptionalString( baseline_workload.command_cwd, candidate_workload.command_cwd, ) or !sameStrings( baseline_workload.command_argv, candidate_workload.command_argv, )) { return false; } if (!baseline_workload.scope_recorded or !candidate_workload.scope_recorded or !baseline_workload.warmup_recorded or !candidate_workload.warmup_recorded or !baseline_workload.profiler_configuration_recorded or !candidate_workload.profiler_configuration_recorded) { return false; } return workloadComparisonSupport( baseline_workload, candidate_workload, ) == .supported;}test "allocation baselines tolerate ephemeral host identity" { const run_ref = baseline.RunRef{ .run_id = "run", .root = "run", .manifest_path = "run/manifest.json", .results_path = "run/results.jsonl", }; const workload = Workload{ .name = "gpalloc.allocator", .package = "lib/gpalloc", .step = "gpalloc-bench", .command_argv = &.{"zig-out/bin/gpalloc-bench"}, .command_cwd = ".", .command_identity_recorded = true, .scope_recorded = true, .warmup_recorded = true, .profiler_configuration_recorded = true, .status = "passed", .exit_code = 0, .wall_ns = 1, .result_path = null, }; const base = Run{ .ref = run_ref, .optimize = "ReleaseFast", .host = .{ .hostname = "runner-a", .os = "linux", .arch = "x86_64" }, }; var candidate = base; candidate.host.hostname = "runner-b"; try std.testing.expect(allocationDesignMatches( base, candidate, workload, workload, )); var changed = workload; changed.command_argv = &.{"zig-out/bin/gpalloc-bench --changed"}; try std.testing.expect(!allocationDesignMatches( base, candidate, workload, changed, ));}fn flattenMetrics(allocator: std.mem.Allocator, run_value: Run) ![]const metric.Metric { var result: std.ArrayList(metric.Metric) = .empty; for (run_value.workloads) |workload| { for (workload.metrics) |row| try result.append(allocator, row); } return try result.toOwnedSlice(allocator);}fn flattenMemoryMetrics(allocator: std.mem.Allocator, run_value: Run) ![]const memory.Metric { var result: std.ArrayList(memory.Metric) = .empty; for (run_value.workloads) |workload| { for (workload.memory_metrics) |row| try result.append(allocator, row); } return try result.toOwnedSlice(allocator);}pub fn compareEnergyRuns( allocator: std.mem.Allocator, base: Run, candidate: Run, threshold_percent: f64,) ![]const EnergyShift { var rows: std.ArrayList(EnergyShift) = .empty; for (candidate.workloads) |workload| { const candidate_summary = (try loadCounterSummary(allocator, workload)) orelse continue; const base_workload = findWorkload(base.workloads, workload.name) orelse continue; const base_summary = (try loadCounterSummary(allocator, base_workload)) orelse continue; const summary_caveated = counterSummaryCaveated(candidate_summary) or counterSummaryCaveated(base_summary); for (candidate_summary.counters) |candidate_counter| { const candidate_energy = capture.perfstat.energyEvidence( candidate_summary, candidate_counter.event, ) orelse continue; const base_energy = capture.perfstat.energyEvidence( base_summary, candidate_counter.event, ) orelse continue; const base_joules = base_energy.mean_joules orelse continue; const candidate_joules = candidate_energy.mean_joules orelse continue; const energy_percent = percentChangeF64(base_joules, candidate_joules) orelse continue; const power_percent = if (base_energy.mean_watts) |baseline_watts| if (candidate_energy.mean_watts) |candidate_watts| percentChangeF64(baseline_watts, candidate_watts) else null else null; if (@abs(energy_percent) < threshold_percent and (power_percent == null or @abs(power_percent.?) < threshold_percent)) continue; const base_counter = findCounter( base_summary.counters, candidate_counter.event, ) orelse continue; const row_caveated = summary_caveated or lowConfidence(candidate_counter) or lowConfidence(base_counter) or energyEvidenceCaveated(candidate_energy) or energyEvidenceCaveated(base_energy); try rows.append(allocator, energyShift( workload.name, threshold_percent, energy_percent, power_percent, base_energy, candidate_energy, row_caveated, )); } } return try rows.toOwnedSlice(allocator);}fn energyShift( workload: []const u8, threshold_percent: f64, energy_percent: f64, power_percent: ?f64, base_evidence: capture.perfstat.EnergyEvidence, candidate: capture.perfstat.EnergyEvidence, caveated: bool,) EnergyShift { return .{ .workload = workload, .event = candidate.source_event, .baseline_joules = base_evidence.mean_joules.?, .candidate_joules = candidate.mean_joules.?, .energy_percent_change = energy_percent, .baseline_watts = base_evidence.mean_watts, .candidate_watts = candidate.mean_watts, .power_percent_change = power_percent, .baseline_energy_range_percent = base_evidence.energy_relative_range_percent, .candidate_energy_range_percent = candidate.energy_relative_range_percent, .baseline_power_range_percent = base_evidence.power_relative_range_percent, .candidate_power_range_percent = candidate.power_relative_range_percent, .threshold_percent = threshold_percent, .status = if (caveated) "energy_shift_caveated" else "energy_shift", };}fn percentChangeF64(base_value: f64, candidate: f64) ?f64 { if (base_value == 0.0) return null; return ((candidate - base_value) / base_value) * 100.0;}fn energyEvidenceCaveated(evidence: capture.perfstat.EnergyEvidence) bool { return !std.mem.eql(u8, evidence.status, "stable_repetitions");}pub fn compareCounterRuns(allocator: std.mem.Allocator, base: Run, candidate: Run, threshold_percent: f64) ![]const CounterShift { var rows: std.ArrayList(CounterShift) = .empty; for (candidate.workloads) |workload| { const candidate_summary = (try loadCounterSummary(allocator, workload)) orelse continue; const base_workload = findWorkload(base.workloads, workload.name) orelse continue; const base_summary = (try loadCounterSummary(allocator, base_workload)) orelse continue; const caveated = counterSummaryCaveated(candidate_summary) or counterSummaryCaveated(base_summary); for (candidate_summary.counters) |candidate_counter| { if (capture.perfstat.belongsToEnergyEvidence( candidate_summary, candidate_counter.event, ) or capture.perfstat.belongsToEnergyEvidence( base_summary, candidate_counter.event, )) continue; const candidate_value = candidate_counter.value orelse continue; const base_counter = findCounter(base_summary.counters, candidate_counter.event) orelse continue; const base_value = base_counter.value orelse continue; if (base_value == 0) continue; const percent = ((candidate_value - base_value) / base_value) * 100; if (@abs(percent) < threshold_percent) continue; const row_caveated = caveated or lowConfidence(candidate_counter) or lowConfidence(base_counter); try rows.append(allocator, .{ .workload = workload.name, .event = candidate_counter.event, .baseline_value = base_value, .candidate_value = candidate_value, .percent_change = percent, .threshold_percent = threshold_percent, .status = if (row_caveated) "counter_shift_caveated" else "counter_shift", }); } } return try rows.toOwnedSlice(allocator);}fn findCounter(counters: []const capture.perfstat.Counter, event: []const u8) ?capture.perfstat.Counter { for (counters) |counter| { if (std.mem.eql(u8, counter.event, event)) return counter; } return null;}fn counterSummaryCaveated(summary: capture.perfstat.Summary) bool { if (summary.caveats.len != 0) return true; if (capture.perfstat.measurementDesign(summary).point_estimate) return true; return !std.mem.eql( u8, capture.perfstat.counterUncertaintySummary(summary).status, "complete", );}fn lowConfidence(counter: capture.perfstat.Counter) bool { if (!std.mem.eql(u8, counter.uncertainty.kind, "complete")) return true; const running = counter.running_percent orelse return true; return running < 99.5;}const WorkloadThreshold = struct { wall: f64, rss: f64,};fn workloadThreshold(workload_name: []const u8, default: f64) WorkloadThreshold { if (catalog.find(workload_name)) |workload| return .{ .wall = workload.wallThresholdPercent(default), .rss = workload.rssThresholdPercent(default), }; return .{ .wall = default, .rss = default };}fn rssPercentChange(base: ?i64, candidate: ?i64) ?f64 { const base_value = base orelse return null; const candidate_value = candidate orelse return null; if (base_value <= 0) return null; return ((@as(f64, @floatFromInt(candidate_value)) - @as(f64, @floatFromInt(base_value))) / @as(f64, @floatFromInt(base_value))) * 100;}test "profiling analysis requires matching optimization and host context" { const run_ref = baseline.RunRef{ .run_id = "run", .root = "run", .manifest_path = "run/manifest.json", .results_path = "run/results.jsonl", }; const run_host = environment.Host{ .hostname = "bench-a", .os = "linux", .kernel = "7.0.11", .arch = "x86_64", .cpu_model = "AMD Ryzen 7 7800X3D", .cpu_count = 16, .cpu_frequency_policy = "policy-a", .process_cpu_affinity = "0-15", .process_memory_affinity = "0", }; const release_fast = Run{ .ref = run_ref, .optimize = "ReleaseFast", .host = run_host }; const release_safe = Run{ .ref = run_ref, .optimize = "ReleaseSafe", .host = run_host }; const unknown = Run{ .ref = run_ref }; try std.testing.expectEqual( ComparisonSupport.supported, comparisonSupport(release_fast, release_fast), ); try std.testing.expectEqual( ComparisonSupport.optimize_mismatch, comparisonSupport(release_fast, release_safe), ); try std.testing.expectEqual( ComparisonSupport.missing_optimize, comparisonSupport(release_fast, unknown), ); var changed_host = run_host; changed_host.cpu_frequency_policy = "policy-b"; try std.testing.expectEqual( ComparisonSupport.cpu_frequency_policy_mismatch, comparisonSupport( release_fast, .{ .ref = run_ref, .optimize = "ReleaseFast", .host = changed_host }, ), );}test "profiling analysis compares energy and power shifts" { var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena_state.deinit(); const allocator = arena_state.allocator(); const base_counters = [_]capture.perfstat.Counter{ analysisTestCounter("duration_time", 2_000_000_000, null), analysisTestCounter("power/energy-pkg/u", 10, "Joules"), }; const candidate_counters = [_]capture.perfstat.Counter{ analysisTestCounter("duration_time", 1_500_000_000, null), analysisTestCounter("power/energy-pkg/u", 10.5, "Joules"), }; const base_repetitions = [_]capture.perfstat.Summary{ .{ .csv_path = "base.1.csv", .caveats = &.{}, .counters = &base_counters }, .{ .csv_path = "base.2.csv", .caveats = &.{}, .counters = &base_counters }, }; const candidate_repetitions = [_]capture.perfstat.Summary{ .{ .csv_path = "candidate.1.csv", .caveats = &.{}, .counters = &candidate_counters }, .{ .csv_path = "candidate.2.csv", .caveats = &.{}, .counters = &candidate_counters }, }; const base_summary = try capture.perfstat.mergeRepeatedSummaries( allocator, "base.csv", &base_repetitions, ); const candidate_summary = try capture.perfstat.mergeRepeatedSummaries( allocator, "candidate.csv", &candidate_repetitions, ); const energy_ref = baseline.RunRef{ .run_id = "energy", .root = "energy", .manifest_path = "energy/manifest.json", .results_path = "energy/results.jsonl", }; const base = Run{ .ref = energy_ref, .workloads = &.{analysisTestWorkload(base_summary)} }; const candidate = Run{ .ref = energy_ref, .workloads = &.{analysisTestWorkload(candidate_summary)}, }; const shifts = try compareEnergyRuns(allocator, base, candidate, 10); try std.testing.expectEqual(@as(usize, 1), shifts.len); try std.testing.expectApproxEqAbs(@as(f64, 5), shifts[0].energy_percent_change, 0.0001); try std.testing.expectApproxEqAbs(@as(f64, 40), shifts[0].power_percent_change.?, 0.0001); try std.testing.expectEqualStrings("energy_shift", shifts[0].status); try std.testing.expectEqual(@as(usize, 0), (try compareCounterRuns( allocator, base, candidate, 10, )).len);}test "profiling analysis skips unavailable energy domains" { const base_counters = [_]capture.perfstat.Counter{ analysisTestCounter("duration_time", 1_000_000_000, null), analysisTestCounter("power/energy-pkg/u", 10, "Joules"), }; const candidate_counters = [_]capture.perfstat.Counter{ analysisTestCounter("duration_time", 1_000_000_000, null), analysisUnsupportedEnergyCounter(), }; const base_summary = capture.perfstat.Summary{ .csv_path = "base.csv", .caveats = &.{}, .counters = &base_counters, }; const candidate_summary = capture.perfstat.Summary{ .csv_path = "candidate.csv", .caveats = &.{}, .counters = &candidate_counters, }; const energy_ref = baseline.RunRef{ .run_id = "energy", .root = "energy", .manifest_path = "energy/manifest.json", .results_path = "energy/results.jsonl", }; const base = Run{ .ref = energy_ref, .workloads = &.{analysisTestWorkload(base_summary)} }; const candidate = Run{ .ref = energy_ref, .workloads = &.{analysisTestWorkload(candidate_summary)}, }; try std.testing.expectEqual( @as(usize, 0), (try compareEnergyRuns(std.testing.allocator, base, candidate, 10)).len, );}test "profiling analysis compares counter shifts between runs" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena_state.deinit(); const allocator = arena_state.allocator(); const dir = try std.fs.path.join(allocator, &.{ ".zig-cache", "tmp", tmp.sub_path[0..] }); try sys.fs.createDirPath(dir); const base_csv = try std.fs.path.join(allocator, &.{ dir, "base.1.csv" }); const base_second_csv = try std.fs.path.join(allocator, &.{ dir, "base.2.csv" }); try sys.fs.writeFile(base_csv, \\1000.00,msec,task-clock,1000000000,100.00,1.000,CPUs utilized \\1000000,,instructions,1000000000,100.00,1.00,insn per cycle \\ ); try sys.fs.writeFile(base_second_csv, \\1000.00,msec,task-clock,1000000000,100.00,1.000,CPUs utilized \\1100000,,instructions,1000000000,100.00,1.10,insn per cycle \\ ); const candidate_csv = try std.fs.path.join(allocator, &.{ dir, "candidate.1.csv" }); const candidate_second_csv = try std.fs.path.join(allocator, &.{ dir, "candidate.2.csv" }); try sys.fs.writeFile(candidate_csv, \\1000.00,msec,task-clock,1000000000,100.00,1.000,CPUs utilized \\1500000,,instructions,1000000000,100.00,1.50,insn per cycle \\ ); try sys.fs.writeFile(candidate_second_csv, \\1000.00,msec,task-clock,1000000000,100.00,1.000,CPUs utilized \\1600000,,instructions,1000000000,100.00,1.60,insn per cycle \\ ); const base_summary_path = try std.fs.path.join(allocator, &.{ dir, "base.summary.json" }); const candidate_summary_path = try std.fs.path.join(allocator, &.{ dir, "candidate.summary.json" }); const base_summaries = [_]capture.perfstat.Summary{ try capture.perfstat.parseCsv(allocator, base_csv), try capture.perfstat.parseCsv(allocator, base_second_csv), }; const candidate_summaries = [_]capture.perfstat.Summary{ try capture.perfstat.parseCsv(allocator, candidate_csv), try capture.perfstat.parseCsv(allocator, candidate_second_csv), }; try capture.perfstat.writeSummaryFile( base_summary_path, try capture.perfstat.mergeRepeatedSummaries(allocator, base_csv, &base_summaries), ); try capture.perfstat.writeSummaryFile( candidate_summary_path, try capture.perfstat.mergeRepeatedSummaries(allocator, candidate_csv, &candidate_summaries), ); const base_capture = [_]host.Capture{.{ .kind = "perf_stat", .tool = "perf", .capture_path = base_csv, .summary_path = base_summary_path, .state = "summary_written" }}; const candidate_capture = [_]host.Capture{.{ .kind = "perf_stat", .tool = "perf", .capture_path = candidate_csv, .summary_path = candidate_summary_path, .state = "summary_written" }}; const base = Run{ .ref = .{ .run_id = "base", .root = "base", .manifest_path = "base/manifest.json", .results_path = "base/results.jsonl" }, .workloads = &.{ .{ .name = "a", .package = "p", .step = "bench", .status = "passed", .exit_code = 0, .wall_ns = 100, .max_rss_kib = 10, .user_s = null, .system_s = null, .result_path = null, .captures = &base_capture }, } }; const candidate = Run{ .ref = .{ .run_id = "candidate", .root = "candidate", .manifest_path = "candidate/manifest.json", .results_path = "candidate/results.jsonl" }, .workloads = &.{ .{ .name = "a", .package = "p", .step = "bench", .status = "passed", .exit_code = 0, .wall_ns = 100, .max_rss_kib = 10, .user_s = null, .system_s = null, .result_path = null, .captures = &candidate_capture }, } }; const shifts = try compareCounterRuns(allocator, base, candidate, 10); try std.testing.expectEqual(@as(usize, 1), shifts.len); try std.testing.expectEqualStrings("instructions", shifts[0].event); try std.testing.expectEqualStrings("counter_shift", shifts[0].status); try std.testing.expectApproxEqAbs(@as(f64, 47.619047), shifts[0].percent_change, 0.0001);}test "profiling analysis caveats single execution counter summaries" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena_state.deinit(); const allocator = arena_state.allocator(); const dir = try std.fs.path.join(allocator, &.{ ".zig-cache", "tmp", tmp.sub_path[0..] }); const csv_path = try std.fs.path.join(allocator, &.{ dir, "perf.stat.csv" }); try sys.fs.writeFile(csv_path, \\1000000,,instructions,1000000000,100.00,1.00,insn per cycle \\ ); const summary = try capture.perfstat.parseCsv(allocator, csv_path); try std.testing.expect(counterSummaryCaveated(summary));}test "profiling analysis compares wall regressions" { var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena_state.deinit(); const allocator = arena_state.allocator(); const base = Run{ .ref = .{ .run_id = "base", .root = "base", .manifest_path = "base/manifest.json", .results_path = "base/results.jsonl" }, .workloads = &.{ .{ .name = "a", .package = "p", .step = "bench", .status = "passed", .exit_code = 0, .wall_ns = 100, .max_rss_kib = 10, .user_s = null, .system_s = null, .result_path = null }, } }; const candidate = Run{ .ref = .{ .run_id = "candidate", .root = "candidate", .manifest_path = "candidate/manifest.json", .results_path = "candidate/results.jsonl" }, .workloads = &.{ .{ .name = "a", .package = "p", .step = "bench", .status = "passed", .exit_code = 0, .wall_ns = 125, .max_rss_kib = 10, .user_s = null, .system_s = null, .result_path = null }, } }; const rows = try compareRuns(allocator, base, candidate, 10); try std.testing.expectEqual(@as(usize, 1), rows.len); try std.testing.expectEqualStrings("wall", rows[0].kind); try std.testing.expectEqualStrings("point_regression_candidate", rows[0].wall_status); try std.testing.expectEqual(@as(usize, 1), rows[0].baseline_wall_samples);}test "profiling analysis classifies supported and uncertain process effects" { var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena_state.deinit(); const allocator = arena_state.allocator(); const baseline_executions = [_]measurement.Execution{ .{ .exit_code = 0, .wall_ns = 99 }, .{ .exit_code = 0, .wall_ns = 100 }, .{ .exit_code = 0, .wall_ns = 100 }, .{ .exit_code = 0, .wall_ns = 101 }, }; const supported_executions = [_]measurement.Execution{ .{ .exit_code = 0, .wall_ns = 129 }, .{ .exit_code = 0, .wall_ns = 130 }, .{ .exit_code = 0, .wall_ns = 130 }, .{ .exit_code = 0, .wall_ns = 131 }, }; const uncertain_executions = [_]measurement.Execution{ .{ .exit_code = 0, .wall_ns = 80 }, .{ .exit_code = 0, .wall_ns = 80 }, .{ .exit_code = 0, .wall_ns = 80 }, .{ .exit_code = 0, .wall_ns = 200 }, }; const baseline_workload = try measuredTestWorkload("process", &baseline_executions); const supported_workload = try measuredTestWorkload("process", &supported_executions); const uncertain_workload = try measuredTestWorkload("process", &uncertain_executions); const run_ref = baseline.RunRef{ .run_id = "run", .root = "run", .manifest_path = "run/manifest.json", .results_path = "run/results.jsonl", }; const baseline_run = Run{ .ref = run_ref, .workloads = &.{baseline_workload} }; const supported_run = Run{ .ref = run_ref, .workloads = &.{supported_workload} }; const uncertain_run = Run{ .ref = run_ref, .workloads = &.{uncertain_workload} }; const supported = try compareRuns(allocator, baseline_run, supported_run, 10); try std.testing.expectEqual(@as(usize, 1), supported.len); try std.testing.expectEqualStrings("sample_regression", supported[0].wall_status); try std.testing.expectEqual(@as(usize, 4), supported[0].baseline_wall_samples); try std.testing.expect(supported[0].wall_effect_low_percent.? >= 10); const uncertain = try compareRuns(allocator, baseline_run, uncertain_run, 10); try std.testing.expectEqual(@as(usize, 1), uncertain.len); try std.testing.expectEqualStrings("sample_regression_uncertain", uncertain[0].wall_status); try std.testing.expect(uncertain[0].wall_effect_low_percent.? < 10);}test "profiling analysis distinguishes summary and point wall candidates" { const summary_distribution = measurement.WallDistribution{ .kind = .legacy_summary, .sample_count = 4, .mean_ns = 100, }; const point_distribution = measurement.WallDistribution{ .kind = .raw_executions, .sample_count = 1, .mean_ns = 100, .min_ns = 100, .median_ns = 100, .p95_ns = 100, .p99_ns = 100, .max_ns = 100, .stddev_ns = 0, .coefficient_of_variation_percent = 0, }; try std.testing.expectEqualStrings( "summary_regression_candidate", wallComparisonStatus(summary_distribution, point_distribution, 10, null), ); try std.testing.expectEqualStrings( "point_regression_candidate", wallComparisonStatus(point_distribution, point_distribution, 10, null), );}test "profiling analysis skips wall comparisons across execution scopes" { var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena_state.deinit(); const allocator = arena_state.allocator(); const base = Run{ .ref = .{ .run_id = "base", .root = "base", .manifest_path = "base/manifest.json", .results_path = "base/results.jsonl" }, .workloads = &.{ .{ .name = "a", .package = "p", .step = "bench", .scope = host.scope, .status = "passed", .exit_code = 0, .wall_ns = 100, .max_rss_kib = 10, .user_s = null, .system_s = null, .result_path = null }, } }; const candidate = Run{ .ref = .{ .run_id = "candidate", .root = "candidate", .manifest_path = "candidate/manifest.json", .results_path = "candidate/results.jsonl" }, .workloads = &.{ .{ .name = "a", .package = "p", .step = "bench", .scope = host.direct_scope, .status = "passed", .exit_code = 0, .wall_ns = 1000, .max_rss_kib = 100, .user_s = null, .system_s = null, .result_path = null }, } }; const rows = try compareRuns(allocator, base, candidate, 10); try std.testing.expectEqual(@as(usize, 0), rows.len); try std.testing.expectEqual( WorkloadComparisonSupport.scope_mismatch, workloadComparisonSupport(base.workloads[0], candidate.workloads[0]), );}test "profiling analysis skips outer deltas across warmup designs" { var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena_state.deinit(); const allocator = arena_state.allocator(); const base_workload = Workload{ .name = "a", .package = "p", .step = "bench", .status = "passed", .exit_code = 0, .wall_ns = 100, .warmup_count = 1, .max_rss_kib = 10, .user_s = null, .system_s = null, .result_path = null }; const candidate_workload = Workload{ .name = "a", .package = "p", .step = "bench", .status = "passed", .exit_code = 0, .wall_ns = 200, .warmup_count = 2, .max_rss_kib = 10, .user_s = null, .system_s = null, .result_path = null }; const base = Run{ .ref = .{ .run_id = "base", .root = "base", .manifest_path = "base/manifest.json", .results_path = "base/results.jsonl" }, .workloads = &.{base_workload} }; const candidate = Run{ .ref = .{ .run_id = "candidate", .root = "candidate", .manifest_path = "candidate/manifest.json", .results_path = "candidate/results.jsonl" }, .workloads = &.{candidate_workload} }; try std.testing.expectEqual(@as(usize, 0), (try compareRuns(allocator, base, candidate, 10)).len); const skips = try workloadComparisonSkips(allocator, base, candidate); try std.testing.expectEqual(@as(usize, 1), skips.len); try std.testing.expectEqual(WorkloadComparisonSupport.warmup_design_mismatch, skips[0].support); try std.testing.expectEqual(@as(usize, 1), skips[0].baseline_warmup_count); try std.testing.expectEqual(@as(usize, 2), skips[0].candidate_warmup_count);}test "profiling analysis skips outer deltas across acquisition contexts" { var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena_state.deinit(); const allocator = arena_state.allocator(); const base_workload = Workload{ .name = "target", .package = "p", .step = "bench", .acquisition = .{ .blocked = .{ .position = 2, .workload_count = 2, .predecessors = &.{"a"}, } }, .status = "passed", .exit_code = 0, .wall_ns = 100, .max_rss_kib = 10, .user_s = null, .system_s = null, .result_path = null, }; const candidate_workload = Workload{ .name = "target", .package = "p", .step = "bench", .acquisition = .{ .blocked = .{ .position = 2, .workload_count = 3, .predecessors = &.{"b"}, } }, .status = "passed", .exit_code = 0, .wall_ns = 200, .max_rss_kib = 20, .user_s = null, .system_s = null, .result_path = null, }; const base = Run{ .ref = .{ .run_id = "base", .root = "base", .manifest_path = "base/manifest.json", .results_path = "base/results.jsonl", }, .workloads = &.{base_workload}, }; const candidate = Run{ .ref = .{ .run_id = "candidate", .root = "candidate", .manifest_path = "candidate/manifest.json", .results_path = "candidate/results.jsonl", }, .workloads = &.{candidate_workload}, }; try std.testing.expectEqual( @as(usize, 0), (try compareRuns(allocator, base, candidate, 10)).len, ); const skips = try workloadComparisonSkips(allocator, base, candidate); try std.testing.expectEqual(@as(usize, 1), skips.len); try std.testing.expectEqual( WorkloadComparisonSupport.measurement_order_mismatch, skips[0].support, ); try std.testing.expectEqualStrings( "a", skips[0].baseline_acquisition.blocked.predecessors[0], ); try std.testing.expectEqualStrings( "b", skips[0].candidate_acquisition.blocked.predecessors[0], );}test "profiling analysis skips outer deltas across profiler captures" { var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena_state.deinit(); const allocator = arena_state.allocator(); const sampled_capture = [_]host.Capture{.{ .kind = host.sampling.kind, .tool = host.sampling.tool, .capture_path = "perf.data", .summary_path = "perf.symbols.summary.json", .state = "summary_written", }}; const unprofiled = Workload{ .name = "a", .package = "p", .step = "bench", .status = "passed", .exit_code = 0, .wall_ns = 100, .max_rss_kib = 10, .user_s = null, .system_s = null, .result_path = null, }; const sampled = Workload{ .name = "a", .package = "p", .step = "bench", .status = "passed", .exit_code = 0, .wall_ns = 1000, .max_rss_kib = 100, .user_s = null, .system_s = null, .result_path = null, .captures = &sampled_capture, }; try std.testing.expectEqual( WorkloadComparisonSupport.capture_mismatch, workloadComparisonSupport(unprofiled, sampled), ); const base = Run{ .ref = .{ .run_id = "base", .root = "base", .manifest_path = "base/manifest.json", .results_path = "base/results.jsonl" }, .workloads = &.{unprofiled}, }; const candidate = Run{ .ref = .{ .run_id = "candidate", .root = "candidate", .manifest_path = "candidate/manifest.json", .results_path = "candidate/results.jsonl" }, .workloads = &.{sampled}, }; try std.testing.expectEqual(@as(usize, 0), (try compareRuns(allocator, base, candidate, 10)).len); const skips = try workloadComparisonSkips(allocator, base, candidate); try std.testing.expectEqual(@as(usize, 1), skips.len); try std.testing.expectEqual(WorkloadComparisonSupport.capture_mismatch, skips[0].support);}Source: src/profiling/analyze/root.zig:2
zig
pub const compare = @import("compare.zig");Complete call list for analyze.compare.compareEnergyRuns
7 direct calls.
src.profiling.analyze.compare.counterSummaryCaveated[function] — private; no exact target atsrc/profiling/analyze/compare.zig:525in nearest public ownertiny.profiling.analyze.comparesrc.profiling.analyze.compare.energyEvidenceCaveated[function] — private; no exact target atsrc/profiling/analyze/compare.zig:477in nearest public ownertiny.profiling.analyze.comparesrc.profiling.analyze.compare.energyShift[function] — private; no exact target atsrc/profiling/analyze/compare.zig:445in nearest public ownertiny.profiling.analyze.comparesrc.profiling.analyze.compare.findCounter[function] — private; no exact target atsrc/profiling/analyze/compare.zig:518in nearest public ownertiny.profiling.analyze.comparesrc.profiling.analyze.compare.lowConfidence[function] — private; no exact target atsrc/profiling/analyze/compare.zig:535in nearest public ownertiny.profiling.analyze.comparesrc.profiling.analyze.compare.percentChangeF64[function] — private; no exact target atsrc/profiling/analyze/compare.zig:472in nearest public ownertiny.profiling.analyze.comparetiny.profiling.analyze.load.loadCounterSummary[function] atsrc/profiling/analyze/load.zig:803
Complete caller list for analyze.compare.compareRuns
7 direct callers.
src.profiling.analyze.compare.test_profiling_analysis_classifies_supported_and_uncertain_process_effects[function] — test; no exact target atsrc/profiling/analyze/compare.zig:793in nearest public ownertiny.profiling.analyze.comparesrc.profiling.analyze.compare.test_profiling_analysis_compares_wall_regressions[function] — test; no exact target atsrc/profiling/analyze/compare.zig:776in nearest public ownertiny.profiling.analyze.comparesrc.profiling.analyze.compare.test_profiling_analysis_skips_outer_deltas_across_acquisition_contexts[function] — test; no exact target atsrc/profiling/analyze/compare.zig:900in nearest public ownertiny.profiling.analyze.comparesrc.profiling.analyze.compare.test_profiling_analysis_skips_outer_deltas_across_profiler_captures[function] — test; no exact target atsrc/profiling/analyze/compare.zig:976in nearest public ownertiny.profiling.analyze.comparesrc.profiling.analyze.compare.test_profiling_analysis_skips_outer_deltas_across_warmup_designs[function] — test; no exact target atsrc/profiling/analyze/compare.zig:884in nearest public ownertiny.profiling.analyze.comparesrc.profiling.analyze.compare.test_profiling_analysis_skips_wall_comparisons_across_execution_scopes[function] — test; no exact target atsrc/profiling/analyze/compare.zig:866in nearest public ownertiny.profiling.analyze.comparetiny.profiling.analyze.evidence.comparisonEvidence[function] atsrc/profiling/analyze/evidence.zig:37
Audit
| Definitions | 12 |
|---|---|
| Public names | 14 |
| Members | 0 |
| Version | 26.7.0 |
| Revision | daab053ee433 |