Skip to documentation
SLOP

tiny.profiling.perturbation

Reference tiny.profiling perturbation

Defined in tiny.profiling.

API (21)

Actions

Public operations.

Types and contracts

Public types and contracts.

Values and defaults

Public values and defaults.

No direct callersNo direct callstiny.profilingperturbation
Static calls · unresolved targets: unknown · external targets: unknown.

Source

Called byCallsprivate; no linksrc.profiling.commandwriteRunDryRunCaptureControldriver.controlrunCaptureControlPlanprivate; no linksrc.profiling.perturbationjoinControlPathperturbationmakePaths
Static calls · unresolved targets: 0 · external targets: 0.
Called byCallsanalyze.loadparseWorkloadtest; no linksrc.profiling.perturbationtest: capture control parser retains ...jsonarrayjsonasBooljsonasU64jsonobjectprivate; no linksrc.profiling.perturbationparseResultprivate; no linksrc.profiling.perturbationrequireTokenperturbationparseMeasurement
Static calls · unresolved targets: 5 · external targets: 3.
Called byCallsNo direct callsprivate; no linksrc.profiling.commandwriteRunDryRunCaptureControldriver.controlrunCaptureControlPlantest; no linksrc.profiling.perturbationtest: capture control schedules are d...perturbationschedule
Static calls · unresolved targets: 0 · external targets: 3.
Called byCallsanalyze.loadparseWorkloadtest; no linksrc.profiling.perturbationtest: capture control summary bootstr...test; no linksrc.profiling.report.pagetest: page renders matched capture pe...private; no linksrc.profiling.perturbationeffectStateprivate; no linksrc.profiling.perturbationemptySummaryprivate; no linksrc.profiling.perturbationmeanperturbationsummarize
Static calls · unresolved targets: 1 · external targets: 7.
Called byCallsNo direct callsprivate; no linksrc.profiling.commandrunCycledriver.validationvalidateRequestoptionsvalidateRunOptionstest; no linksrc.profiling.perturbationtest: capture control compatibility k...perturbationvalidateCompatibility
Static calls · unresolved targets: 0 · external targets: 3.
Called byCallsNo direct callsprivate; no linksrc.profiling.commandwriteRunDryRunCaptureControldriver.controlrunCaptureControlPlanperturbationworkloadSeed
Static calls · unresolved targets: 1 · external targets: 1.

Source: src/profiling/perturbation.zig

zig
const std = @import("std");const capture = @import("capture");const sys = @import("sys");const execute = @import("execute.zig");const host = @import("host/root.zig");const json = @import("json.zig");pub const default_seed: u64 = 0x5449_4e59_4341_5054;pub const max_pairs: u32 = @intCast(host.process.max_executions / 2);pub const effect_method = "deterministic_percentile_bootstrap_paired_mean_percent_change";pub const Options = struct {    repeat: u32 = 0,    seed: u64 = default_seed,    pub fn enabled(self: Options) bool {        return self.repeat != 0;    }    pub fn validate(self: Options) !void {        if (self.repeat > max_pairs) return error.InvalidCaptureControlRepeat;    }};pub const Order = enum {    control_first,    capture_first,};pub const State = enum {    complete,    incomplete,    capture_unavailable,};pub const Pair = struct {    index: u32,    order: Order,    control: ?execute.Result = null,    capture: ?execute.Result = null,};pub const Configuration = struct {    host_kind: ?[]const u8 = null,    tracy: bool = false,    allocations: bool = false,    pub fn any(self: Configuration) bool {        return self.host_kind != null or self.tracy or self.allocations;    }};pub const Paths = struct {    stdout: []const u8,    stderr: []const u8,    bench_jsonl: []const u8,    coz_jsonl: []const u8,    coz_analysis: []const u8,    tracy_jsonl: []const u8,    tracy_summary: []const u8,    allocations: []const u8,    structured: []const u8,    pub fn artifactEnv(self: Paths) execute.ArtifactEnv {        return .{            .stdout = self.stdout,            .stderr = self.stderr,            .bench_jsonl = self.bench_jsonl,            .coz_jsonl = self.coz_jsonl,            .coz_analysis = self.coz_analysis,            .tracy_jsonl = self.tracy_jsonl,            .tracy_summary = self.tracy_summary,            .allocations = self.allocations,        };    }};pub const Measurement = struct {    state: State,    base_seed: u64,    workload_seed: u64,    pairs: []const Pair,    configuration: Configuration = .{},    control_artifacts: Paths,    control_structured_rows: usize = 0,    control_structured_parse_errors: usize = 0,};pub const Summary = struct {    state: []const u8,    pair_count: usize,    control_mean_ns: ?f64,    capture_mean_ns: ?f64,    mean_delta_ns: ?f64,    mean_percent_change: ?f64,    effect_low_percent: ?f64,    effect_high_percent: ?f64,};pub fn makePaths(allocator: std.mem.Allocator, workload_root: []const u8) !Paths {    std.debug.assert(workload_root.len > 0);    return .{        .stdout = try joinControlPath(allocator, workload_root, "stdout.txt"),        .stderr = try joinControlPath(allocator, workload_root, "stderr.txt"),        .bench_jsonl = try joinControlPath(allocator, workload_root, "bench.jsonl"),        .coz_jsonl = try joinControlPath(allocator, workload_root, "bench.coz.jsonl"),        .coz_analysis = try joinControlPath(allocator, workload_root, "bench.coz.analysis.json"),        .tracy_jsonl = try joinControlPath(allocator, workload_root, "bench.tracy.jsonl"),        .tracy_summary = try joinControlPath(allocator, workload_root, "bench.tracy.summary.jsonl"),        .allocations = try joinControlPath(allocator, workload_root, "allocations.jsonl"),        .structured = try joinControlPath(allocator, workload_root, "structured.jsonl"),    };}fn joinControlPath(    allocator: std.mem.Allocator,    workload_root: []const u8,    name: []const u8,) ![]const u8 {    return try std.fs.path.join(allocator, &.{        workload_root,        try std.fmt.allocPrint(allocator, "control.{s}", .{name}),    });}pub fn validateCompatibility(    options: Options,    measure_repeat: u32,    execution_scope: execute.Scope,    host_lanes: host.Options,    tracy: bool,    trace_allocations: bool,    causal: bool,) !void {    try options.validate();    if (!options.enabled()) return;    if (execution_scope != .binary) return error.CaptureControlRequiresBinaryScope;    if (!host_lanes.any() and !tracy and !trace_allocations) {        return error.CaptureControlRequiresCapture;    }    if (measure_repeat != 1) return error.CaptureControlConflictsWithMeasureRepeat;    if (causal) return error.CaptureControlConflictsWithCausal;    if (host_lanes.counters) |counters| {        if (counters.repeat != 1 or counters.group_size != 0) {            return error.CaptureControlConflictsWithCounterMatrix;        }    }}pub fn workloadSeed(base_seed: u64, workload_name: []const u8) u64 {    var hasher = std.hash.Wyhash.init(base_seed);    hasher.update(workload_name);    return hasher.final();}pub fn parseMeasurement(    allocator: std.mem.Allocator,    workload: std.json.ObjectMap,) !?Measurement {    const value = workload.get("capture_perturbation") orelse return null;    const object = switch (value) {        .null => return null,        .object => |actual| actual,        else => return error.InvalidProfilingJson,    };    try requireToken(object, "method", "same_build_matched_control_capture");    try requireToken(object, "effect_method", effect_method);    try requireToken(object, "order", "seeded_balanced_within_pair");    try requireToken(object, "setup_reuse", "single_setup");    try requireToken(object, "binary_identity", "same_built_binary");    try requireToken(object, "capture_artifact_retention", "last_capture_execution");    try requireToken(object, "control_output_retention", "last_control_execution");    if (json.asU64(object.get("effect_bootstrap_iterations")) !=        capture.compare.effect.bootstrap_iterations)    {        return error.InvalidProfilingJson;    }    if (json.asU64(object.get("effect_confidence_per_mille")) !=        capture.compare.effect.confidence_per_mille)    {        return error.InvalidProfilingJson;    }    const state = std.meta.stringToEnum(        State,        json.string(object.get("state")) orelse return error.InvalidProfilingJson,    ) orelse return error.InvalidProfilingJson;    const pair_count = json.asU64(object.get("pair_count_requested")) orelse        return error.InvalidProfilingJson;    if (pair_count == 0 or pair_count > max_pairs) return error.InvalidProfilingJson;    const pair_values = try json.array(object.get("pairs") orelse        return error.InvalidProfilingJson);    if (pair_values.items.len != pair_count) return error.InvalidProfilingJson;    const pairs = try allocator.alloc(Pair, pair_values.items.len);    for (pair_values.items, 0..) |pair_value, index| {        const pair = try json.object(pair_value);        const recorded_index = json.asU64(pair.get("index")) orelse            return error.InvalidProfilingJson;        if (recorded_index != index + 1) return error.InvalidProfilingJson;        pairs[index] = .{            .index = @intCast(recorded_index),            .order = std.meta.stringToEnum(                Order,                json.string(pair.get("order")) orelse return error.InvalidProfilingJson,            ) orelse return error.InvalidProfilingJson,            .control = try parseResult(pair.get("control")),            .capture = try parseResult(pair.get("capture")),        };    }    const artifacts = try json.object(object.get("control_artifacts") orelse        return error.InvalidProfilingJson);    const configuration = try json.object(object.get("active_configuration") orelse        return error.InvalidProfilingJson);    const parsed_configuration = Configuration{        .host_kind = json.string(configuration.get("host_kind")),        .tracy = json.asBool(configuration.get("tracy")) orelse            return error.InvalidProfilingJson,        .allocations = json.asBool(configuration.get("allocations")) orelse            return error.InvalidProfilingJson,    };    if (!parsed_configuration.any()) return error.InvalidProfilingJson;    return .{        .state = state,        .base_seed = json.asU64(object.get("base_seed")) orelse            return error.InvalidProfilingJson,        .workload_seed = json.asU64(object.get("workload_seed")) orelse            return error.InvalidProfilingJson,        .pairs = pairs,        .configuration = parsed_configuration,        .control_artifacts = .{            .stdout = json.string(artifacts.get("stdout")) orelse                return error.InvalidProfilingJson,            .stderr = json.string(artifacts.get("stderr")) orelse                return error.InvalidProfilingJson,            .bench_jsonl = json.string(artifacts.get("bench_jsonl")) orelse                return error.InvalidProfilingJson,            .coz_jsonl = json.string(artifacts.get("coz_jsonl")) orelse                return error.InvalidProfilingJson,            .coz_analysis = json.string(artifacts.get("coz_analysis")) orelse                return error.InvalidProfilingJson,            .tracy_jsonl = json.string(artifacts.get("tracy_jsonl")) orelse                return error.InvalidProfilingJson,            .tracy_summary = json.string(artifacts.get("tracy_summary")) orelse                return error.InvalidProfilingJson,            .allocations = json.string(artifacts.get("allocations")) orelse                return error.InvalidProfilingJson,            .structured = json.string(artifacts.get("structured")) orelse                return error.InvalidProfilingJson,        },        .control_structured_rows = @intCast(json.asU64(artifacts.get("structured_rows")) orelse 0),        .control_structured_parse_errors = @intCast(json.asU64(artifacts.get("structured_parse_errors")) orelse 0),    };}fn requireToken(    object: std.json.ObjectMap,    key: []const u8,    expected: []const u8,) !void {    const actual = json.string(object.get(key)) orelse        return error.InvalidProfilingJson;    if (!std.mem.eql(u8, actual, expected)) return error.InvalidProfilingJson;}fn parseResult(value: ?std.json.Value) !?execute.Result {    const actual = value orelse return error.InvalidProfilingJson;    const object = switch (actual) {        .null => return null,        .object => |row| row,        else => return error.InvalidProfilingJson,    };    return .{        .exit_code = json.asI64(object.get("exit_code")) orelse            return error.InvalidProfilingJson,        .wall_ns = json.asU64(object.get("wall_ns")) orelse            return error.InvalidProfilingJson,        .resource_usage_source = try parseResourceUsageSource(object),        .maxrss_kib = json.asI64(object.get("max_rss_kib")),        .user_s = json.asF64(object.get("user_s")),        .system_s = json.asF64(object.get("system_s")),        .minor_page_faults = json.asU64(object.get("minor_page_faults")),        .major_page_faults = json.asU64(object.get("major_page_faults")),        .voluntary_context_switches = json.asU64(object.get("voluntary_context_switches")),        .involuntary_context_switches = json.asU64(object.get("involuntary_context_switches")),    };}fn parseResourceUsageSource(    object: std.json.ObjectMap,) !?sys.process.ResourceUsageSource {    const text = json.string(object.get("resource_usage_source")) orelse return null;    return std.meta.stringToEnum(sys.process.ResourceUsageSource, text) orelse        error.InvalidProfilingJson;}pub fn schedule(    allocator: std.mem.Allocator,    repeat: u32,    seed: u64,) ![]Order {    if (repeat == 0 or repeat > max_pairs) {        return error.InvalidCaptureControlRepeat;    }    const orders = try allocator.alloc(Order, repeat);    var control_first_count: usize = repeat / 2;    if (repeat % 2 != 0 and seed & 1 == 0) control_first_count += 1;    for (orders, 0..) |*order, index| {        order.* = if (index < control_first_count) .control_first else .capture_first;    }    var prng = std.Random.DefaultPrng.init(seed);    const random = prng.random();    var remaining = orders.len;    while (remaining > 1) {        const index = random.uintLessThan(usize, remaining);        remaining -= 1;        std.mem.swap(Order, &orders[index], &orders[remaining]);    }    return orders;}pub fn summarize(    allocator: std.mem.Allocator,    measurement: Measurement,    workload_name: []const u8,) !Summary {    if (measurement.state != .complete) return emptySummary(@tagName(measurement.state));    if (measurement.pairs.len == 0 or measurement.pairs.len > max_pairs) {        return error.InvalidProfilingJson;    }    const control_samples = try allocator.alloc(f64, measurement.pairs.len);    defer allocator.free(control_samples);    const capture_samples = try allocator.alloc(f64, measurement.pairs.len);    defer allocator.free(capture_samples);    for (measurement.pairs, 0..) |pair, index| {        if (pair.index != index + 1) return error.InvalidProfilingJson;        const control = pair.control orelse return error.InvalidProfilingJson;        const profiled = pair.capture orelse return error.InvalidProfilingJson;        if (control.exit_code != 0 or profiled.exit_code != 0) {            return error.InvalidProfilingJson;        }        control_samples[index] = @floatFromInt(control.wall_ns);        capture_samples[index] = @floatFromInt(profiled.wall_ns);    }    const control_mean = mean(control_samples);    const capture_mean = mean(capture_samples);    const percent = if (control_mean == 0)        null    else        ((capture_mean / control_mean) - 1.0) * 100.0;    const interval = if (measurement.pairs.len < 2) null else interval: {        var effect_storage = try capture.compare.EffectStorage.init(allocator, .{            .max_samples_per_distribution = measurement.pairs.len,        });        defer effect_storage.deinit(allocator);        effect_storage.activate();        break :interval try capture.compare.effect.bootstrapPairedMeanPercentChangeInterval(            &effect_storage,            control_samples,            capture_samples,            capture.compare.effect.pairedPercentChangeSeed(workload_name),        );    };    return .{        .state = effectState(percent, interval, measurement.pairs.len),        .pair_count = measurement.pairs.len,        .control_mean_ns = control_mean,        .capture_mean_ns = capture_mean,        .mean_delta_ns = capture_mean - control_mean,        .mean_percent_change = percent,        .effect_low_percent = if (interval) |value| value.low else null,        .effect_high_percent = if (interval) |value| value.high else null,    };}fn emptySummary(state: []const u8) Summary {    return .{        .state = state,        .pair_count = 0,        .control_mean_ns = null,        .capture_mean_ns = null,        .mean_delta_ns = null,        .mean_percent_change = null,        .effect_low_percent = null,        .effect_high_percent = null,    };}fn mean(values: []const f64) f64 {    var total: f64 = 0;    for (values) |value| total += value;    return total / @as(f64, @floatFromInt(values.len));}fn effectState(    percent: ?f64,    interval: ?capture.compare.effect.Interval,    pair_count: usize,) []const u8 {    if (percent == null) return "zero_control_mean";    if (pair_count < 2) return "point_estimate";    const actual = interval orelse return "interval_unavailable";    if (actual.low > 0) return "positive_capture_perturbation";    if (actual.high < 0) return "negative_capture_perturbation";    return "capture_perturbation_uncertain";}test "capture control options share the process acquisition bound" {    try (Options{ .repeat = max_pairs }).validate();    try std.testing.expectError(        error.InvalidCaptureControlRepeat,        (Options{ .repeat = max_pairs + 1 }).validate(),    );    try std.testing.expectEqual(        @as(u32, host.process.max_executions),        max_pairs * 2,    );}test "capture control schedules are deterministic and balanced" {    const allocator = std.testing.allocator;    const first = try schedule(allocator, 9, 42);    defer allocator.free(first);    const second = try schedule(allocator, 9, 42);    defer allocator.free(second);    try std.testing.expectEqualSlices(Order, first, second);    var control_first_count: usize = 0;    for (first) |order| {        if (order == .control_first) control_first_count += 1;    }    const capture_first_count = first.len - control_first_count;    try std.testing.expect(@abs(        @as(isize, @intCast(control_first_count)) -            @as(isize, @intCast(capture_first_count)),    ) <= 1);}test "capture control compatibility keeps acquisition axes separate" {    try validateCompatibility(.{ .repeat = 3 }, 1, .binary, .{}, true, false, false);    try std.testing.expectError(        error.CaptureControlRequiresCapture,        validateCompatibility(.{ .repeat = 3 }, 1, .binary, .{}, false, false, false),    );    try std.testing.expectError(        error.CaptureControlRequiresBinaryScope,        validateCompatibility(.{ .repeat = 3 }, 1, .step, .{}, true, false, false),    );    try std.testing.expectError(        error.CaptureControlConflictsWithMeasureRepeat,        validateCompatibility(.{ .repeat = 3 }, 2, .binary, .{}, true, false, false),    );    try std.testing.expectError(        error.CaptureControlConflictsWithCausal,        validateCompatibility(.{ .repeat = 3 }, 1, .binary, .{}, true, false, true),    );    try std.testing.expectError(        error.CaptureControlConflictsWithCounterMatrix,        validateCompatibility(            .{ .repeat = 3 },            1,            .binary,            .{ .counters = .{ .repeat = 2 } },            false,            false,            false,        ),    );}test "capture control summary bootstraps whole matched pairs" {    const pairs = [_]Pair{        .{ .index = 1, .order = .control_first, .control = .{ .exit_code = 0, .wall_ns = 10 }, .capture = .{ .exit_code = 0, .wall_ns = 20 } },        .{ .index = 2, .order = .capture_first, .control = .{ .exit_code = 0, .wall_ns = 100 }, .capture = .{ .exit_code = 0, .wall_ns = 200 } },    };    const summary = try summarize(std.testing.allocator, .{        .state = .complete,        .base_seed = 7,        .workload_seed = 8,        .pairs = &pairs,        .control_artifacts = .{            .stdout = "",            .stderr = "",            .bench_jsonl = "",            .coz_jsonl = "",            .coz_analysis = "",            .tracy_jsonl = "",            .tracy_summary = "",            .allocations = "",            .structured = "",        },    }, "w");    try std.testing.expectEqualStrings("positive_capture_perturbation", summary.state);    try std.testing.expectApproxEqAbs(@as(f64, 100), summary.mean_percent_change.?, 0.0001);    try std.testing.expectApproxEqAbs(@as(f64, 100), summary.effect_low_percent.?, 0.0001);    try std.testing.expectApproxEqAbs(@as(f64, 100), summary.effect_high_percent.?, 0.0001);}test "capture control parser retains matched pair order and artifacts" {    var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);    defer arena_state.deinit();    const allocator = arena_state.allocator();    const value = try std.json.parseFromSliceLeaky(        std.json.Value,        allocator,        \\{        \\  "capture_perturbation": {        \\    "state":"complete",        \\    "method":"same_build_matched_control_capture",        \\    "effect_method":"deterministic_percentile_bootstrap_paired_mean_percent_change",        \\    "effect_bootstrap_iterations":1000,        \\    "effect_confidence_per_mille":950,        \\    "pair_count_requested":1,        \\    "base_seed":7,        \\    "workload_seed":8,        \\    "order":"seeded_balanced_within_pair",        \\    "setup_reuse":"single_setup",        \\    "binary_identity":"same_built_binary",        \\    "capture_artifact_retention":"last_capture_execution",        \\    "control_output_retention":"last_control_execution",        \\    "active_configuration":{"host_kind":null,"tracy":true,"allocations":false},        \\    "control_artifacts":{"stdout":"control.stdout.txt","stderr":"control.stderr.txt","bench_jsonl":"control.bench.jsonl","coz_jsonl":"control.coz.jsonl","coz_analysis":"control.coz.analysis.json","tracy_jsonl":"control.tracy.jsonl","tracy_summary":"control.tracy.summary.jsonl","allocations":"control.allocations.jsonl","structured":"control.structured.jsonl","structured_rows":2,"structured_parse_errors":1},        \\    "pairs":[{"index":1,"order":"capture_first",        \\      "control":{"exit_code":0,"wall_ns":10,        \\        "resource_usage_source":"wait4_rusage","max_rss_kib":20,        \\        "user_s":0.1,"system_s":0.2,"minor_page_faults":5,        \\        "major_page_faults":0,"voluntary_context_switches":2,        \\        "involuntary_context_switches":1},        \\      "capture":{"exit_code":0,"wall_ns":12,        \\        "resource_usage_source":"wait4_rusage","max_rss_kib":21,        \\        "user_s":0.2,"system_s":0.3,"minor_page_faults":7,        \\        "major_page_faults":1,"voluntary_context_switches":4,        \\        "involuntary_context_switches":3}}]        \\  }        \\}    ,        .{},    );    const parsed = (try parseMeasurement(allocator, try json.object(value))).?;    try std.testing.expectEqual(State.complete, parsed.state);    try std.testing.expectEqual(Order.capture_first, parsed.pairs[0].order);    try std.testing.expect(parsed.configuration.tracy);    try std.testing.expectEqual(@as(u64, 10), parsed.pairs[0].control.?.wall_ns);    try std.testing.expectEqual(        sys.process.ResourceUsageSource.wait4_rusage,        parsed.pairs[0].control.?.resource_usage_source.?,    );    try std.testing.expectEqual(        @as(?u64, 2),        parsed.pairs[0].control.?.voluntary_context_switches,    );    try std.testing.expectEqualStrings("control.stdout.txt", parsed.control_artifacts.stdout);    try std.testing.expectEqual(@as(usize, 2), parsed.control_structured_rows);    try std.testing.expectEqual(@as(usize, 1), parsed.control_structured_parse_errors);}

Source: src/profiling/root.zig:36

zig
pub const perturbation = @import("perturbation.zig");

Audit

Definitions22
Public names22
Members39
Version26.7.0
Revisiondaab053ee433