Skip to documentation
SLOP

tiny.profiling.metric

Reference tiny.profiling metric

Defined in tiny.profiling.

API (12)

Actions

Public operations.

Types and contracts

Public types and contracts.

No direct callersNo direct callstiny.profilingmetric
Static calls · unresolved targets: unknown · external targets: unknown.

Source

Called byCallsanalyze.comparecompareMetricRunstest; no linksrc.profiling.metrictest: profiling metrics compare raw s...private; no linksrc.profiling.metricbootstrapMedianPercentIntervalprivate; no linksrc.profiling.metriccomparisonStatusprivate; no linksrc.profiling.metricfindprivate; no linksrc.profiling.metricthresholdPercentmetriccompare
Static calls · unresolved targets: 2 · external targets: 2.
Called byCallsprivate; no linksrc.profiling.analyze.loadloadWorkloadtest; no linksrc.profiling.metrictest: profiling metrics parse structu...jsonobjectprivate; no linksrc.profiling.metricparseMetricmetricload
Static calls · unresolved targets: 2 · external targets: 4.

Source: src/profiling/metric.zig

zig
const std = @import("std");const sys = @import("sys");const catalog = @import("catalog.zig");const json = @import("json.zig");const schema = @import("schema.zig");const max_structured_bytes = 128 * 1024 * 1024;pub const Distribution = enum {    confidence_interval,    raw_samples,    summary_samples,    point_estimate,    pub fn name(self: Distribution) []const u8 {        return @tagName(self);    }};pub const Interval = struct {    low_ns: f64,    high_ns: f64,};pub const SampleOrder = enum {    measured_acquisition_order,};pub const SampleSequence = struct {    order: SampleOrder,    index_origin: u64,};pub const Metric = struct {    workload: []const u8,    key: []const u8,    label: []const u8,    source_kind: []const u8,    source_path: []const u8,    source_line: usize,    sample_count: ?u64,    mean_ns: ?f64,    median_ns: ?f64,    p75_ns: ?f64,    p95_ns: ?f64,    p99_ns: ?f64,    min_ns: ?f64,    max_ns: ?f64,    mean_interval: ?Interval,    median_interval: ?Interval,    p95_interval: ?Interval,    p99_interval: ?Interval,    samples_ns: []const u64 = &.{},    sample_sequence: ?SampleSequence = null,    distribution: Distribution,    pub fn primaryNs(self: Metric) ?f64 {        return self.median_ns orelse self.mean_ns orelse self.p95_ns orelse self.p99_ns;    }    pub fn primaryInterval(self: Metric) ?Interval {        if (self.median_ns != null) return self.median_interval;        if (self.mean_ns != null) return self.mean_interval;        if (self.p95_ns != null) return self.p95_interval;        if (self.p99_ns != null) return self.p99_interval;        return null;    }};pub const EffectInterval = struct {    low_percent: f64,    high_percent: f64,};pub const Comparison = struct {    workload: []const u8,    key: []const u8,    label: []const u8,    baseline_ns: f64,    candidate_ns: f64,    percent_change: f64,    threshold_percent: f64,    baseline_samples: ?u64,    candidate_samples: ?u64,    baseline_distribution: Distribution,    candidate_distribution: Distribution,    effect_low_percent: ?f64 = null,    effect_high_percent: ?f64 = null,    status: []const u8,};const Source = struct {    kind: []const u8,    path: []const u8,    line: usize,};const Stats = struct {    count: u64,    samples: []const u64,    mean_ns: f64,    median_ns: f64,    p75_ns: f64,    p95_ns: f64,    p99_ns: f64,    min_ns: f64,    max_ns: f64,};pub fn load(allocator: std.mem.Allocator, path: ?[]const u8) ![]const Metric {    const actual_path = path orelse return &.{};    const text = sys.fs.readFileAlloc(allocator, actual_path, max_structured_bytes) catch |err| switch (err) {        error.FileNotFound => return &.{},        else => |actual| return actual,    };    defer allocator.free(text);    var result: std.ArrayList(Metric) = .empty;    var lines = std.mem.splitScalar(u8, text, '\n');    while (lines.next()) |line| {        const trimmed = std.mem.trim(u8, line, " \t\r");        if (trimmed.len == 0) continue;        var parsed = std.json.parseFromSlice(std.json.Value, allocator, trimmed, .{}) catch continue;        defer parsed.deinit();        const object = json.object(parsed.value) catch continue;        if (parseMetric(allocator, object) catch null) |metric| try result.append(allocator, metric);    }    return try result.toOwnedSlice(allocator);}pub fn compare(allocator: std.mem.Allocator, base: []const Metric, candidate: []const Metric, default_threshold_percent: f64) ![]const Comparison {    var rows: std.ArrayList(Comparison) = .empty;    for (candidate) |candidate_metric| {        const candidate_ns = candidate_metric.primaryNs() orelse continue;        const base_metric = find(base, candidate_metric.workload, candidate_metric.key) orelse continue;        const baseline_ns = base_metric.primaryNs() orelse continue;        if (baseline_ns == 0) continue;        const percent = ((candidate_ns - baseline_ns) / baseline_ns) * 100;        const threshold = thresholdPercent(candidate_metric.workload, default_threshold_percent);        if (percent < threshold) continue;        const effect_interval = try bootstrapMedianPercentInterval(allocator, base_metric.samples_ns, candidate_metric.samples_ns);        try rows.append(allocator, .{            .workload = candidate_metric.workload,            .key = candidate_metric.key,            .label = candidate_metric.label,            .baseline_ns = baseline_ns,            .candidate_ns = candidate_ns,            .percent_change = percent,            .threshold_percent = threshold,            .baseline_samples = base_metric.sample_count,            .candidate_samples = candidate_metric.sample_count,            .baseline_distribution = base_metric.distribution,            .candidate_distribution = candidate_metric.distribution,            .effect_low_percent = if (effect_interval) |interval| interval.low_percent else null,            .effect_high_percent = if (effect_interval) |interval| interval.high_percent else null,            .status = comparisonStatus(base_metric, candidate_metric, threshold, effect_interval),        });    }    return try rows.toOwnedSlice(allocator);}fn thresholdPercent(workload_name: []const u8, default_threshold_percent: f64) f64 {    if (catalog.find(workload_name)) |workload| return workload.metricThresholdPercent(default_threshold_percent);    return default_threshold_percent;}fn find(metrics: []const Metric, workload: []const u8, key: []const u8) ?Metric {    for (metrics) |metric| {        if (std.mem.eql(u8, metric.workload, workload) and std.mem.eql(u8, metric.key, key)) return metric;    }    return null;}fn comparisonStatus(base: Metric, candidate: Metric, threshold_percent: f64, effect_interval: ?EffectInterval) []const u8 {    if (effect_interval) |interval| {        if (interval.low_percent >= threshold_percent) return "sample_regression";        return "sample_regression_uncertain";    }    if (base.primaryInterval()) |base_interval| {        if (candidate.primaryInterval()) |candidate_interval| {            if (candidate_interval.low_ns > base_interval.high_ns) return "interval_regression";            return "within_interval";        }    }    if (candidate.distribution == .raw_samples or base.distribution == .raw_samples) return "sample_regression_candidate";    if (candidate.distribution == .summary_samples or base.distribution == .summary_samples) return "summary_regression_candidate";    return "point_regression_candidate";}fn parseMetric(allocator: std.mem.Allocator, object: std.json.ObjectMap) !?Metric {    const row = try json.object(object.get("row") orelse return null);    const source = try parseSource(object);    if (schema.classify(row, source.kind).family != .timing) return null;    const workload = try parseWorkload(object);    const has_ns_metric = rowMetricIsNanoseconds(row);    const sample_stats = try statsFromSamples(allocator, row.get("sample_ns"));    const sample_sequence = parseSampleSequence(row);    const mean_ns = (if (sample_stats) |stats| stats.mean_ns else null) orelse json.asF64(row.get("mean_ns")) orelse if (has_ns_metric) json.asF64(row.get("mean")) else null;    const median_ns = (if (sample_stats) |stats| stats.median_ns else null) orelse json.asF64(row.get("median_ns")) orelse if (has_ns_metric) json.asF64(row.get("median")) else null;    const p75_ns = (if (sample_stats) |stats| stats.p75_ns else null) orelse json.asF64(row.get("p75_ns")) orelse if (has_ns_metric) json.asF64(row.get("p75")) else null;    const p95_ns = (if (sample_stats) |stats| stats.p95_ns else null) orelse json.asF64(row.get("p95_ns")) orelse if (has_ns_metric) json.asF64(row.get("p95")) else null;    const p99_ns = (if (sample_stats) |stats| stats.p99_ns else null) orelse json.asF64(row.get("p99_ns")) orelse if (has_ns_metric) json.asF64(row.get("p99")) else null;    const min_ns = (if (sample_stats) |stats| stats.min_ns else null) orelse json.asF64(row.get("min_ns")) orelse if (has_ns_metric) json.asF64(row.get("min")) else null;    const max_ns = (if (sample_stats) |stats| stats.max_ns else null) orelse json.asF64(row.get("max_ns")) orelse if (has_ns_metric) json.asF64(row.get("max")) else null;    const point_ns = firstNumber(row, &.{ "wall_ns", "duration_ns", "build_time_ns", "time_ns", "ns" });    const final_mean_ns = mean_ns orelse point_ns;    const final_median_ns = median_ns orelse point_ns;    if (final_mean_ns == null and final_median_ns == null and p95_ns == null and p99_ns == null) return null;    const raw_samples = sample_stats != null;    const sample_count = if (sample_stats) |stats|        stats.count    else        json.asU64(row.get("sample_count")) orelse json.asU64(row.get("samples"));    const mean_interval = parseInterval(row, "mean_ns");    const median_interval = parseInterval(row, "median_ns");    const p95_interval = parseInterval(row, "p95_ns");    const p99_interval = parseInterval(row, "p99_ns");    const has_interval = mean_interval != null or median_interval != null or p95_interval != null or p99_interval != null;    const distribution: Distribution = if (has_interval)        .confidence_interval    else if (raw_samples)        .raw_samples    else if ((sample_count orelse 0) > 1)        .summary_samples    else        .point_estimate;    const key = try metricKey(allocator, row);    return .{        .workload = workload,        .key = key,        .label = try metricLabel(allocator, row, key),        .source_kind = source.kind,        .source_path = source.path,        .source_line = source.line,        .sample_count = sample_count,        .mean_ns = final_mean_ns,        .median_ns = final_median_ns,        .p75_ns = p75_ns,        .p95_ns = p95_ns,        .p99_ns = p99_ns,        .min_ns = min_ns,        .max_ns = max_ns,        .mean_interval = mean_interval,        .median_interval = median_interval,        .p95_interval = p95_interval,        .p99_interval = p99_interval,        .samples_ns = if (sample_stats) |stats| stats.samples else &.{},        .sample_sequence = sample_sequence,        .distribution = distribution,    };}fn parseSampleSequence(row: std.json.ObjectMap) ?SampleSequence {    const object = json.object(row.get("sample_sequence") orelse return null) catch return null;    const order_name = json.string(object.get("order")) orelse return null;    if (!std.mem.eql(u8, order_name, "measured_acquisition_order")) return null;    return .{        .order = .measured_acquisition_order,        .index_origin = json.asU64(object.get("index_origin")) orelse return null,    };}fn parseSource(object: std.json.ObjectMap) !Source {    const source = try json.object(object.get("source") orelse return error.InvalidProfilingJson);    return .{        .kind = json.string(source.get("kind")) orelse "",        .path = json.string(source.get("path")) orelse "",        .line = @intCast(json.asU64(source.get("line")) orelse 0),    };}fn parseWorkload(object: std.json.ObjectMap) ![]const u8 {    const workload = try json.object(object.get("workload") orelse return error.InvalidProfilingJson);    return json.string(workload.get("name")) orelse return error.InvalidProfilingJson;}fn rowMetricIsNanoseconds(row: std.json.ObjectMap) bool {    if (json.string(row.get("metric"))) |metric_name| {        if (std.mem.endsWith(u8, metric_name, "_ns") or std.mem.eql(u8, metric_name, "ns")) return true;    }    if (json.string(row.get("name"))) |name| {        if (std.mem.endsWith(u8, name, "_ns") or std.mem.eql(u8, name, "ns")) return true;    }    return false;}fn firstNumber(row: std.json.ObjectMap, fields: []const []const u8) ?f64 {    for (fields) |field| {        if (json.asF64(row.get(field))) |value| return value;    }    return null;}fn parseInterval(row: std.json.ObjectMap, field: []const u8) ?Interval {    const intervals = json.object(row.get("confidence_intervals") orelse return null) catch return null;    const interval = json.object(intervals.get(field) orelse return null) catch return null;    const low = json.asF64(interval.get("low_ns")) orelse return null;    const high = json.asF64(interval.get("high_ns")) orelse return null;    return .{ .low_ns = low, .high_ns = high };}fn statsFromSamples(allocator: std.mem.Allocator, value: ?std.json.Value) !?Stats {    const actual = value orelse return null;    const array = json.array(actual) catch return null;    if (array.items.len == 0) return null;    const samples = try allocator.alloc(u64, array.items.len);    errdefer allocator.free(samples);    for (array.items, 0..) |item, index| {        samples[index] = json.asU64(item) orelse {            allocator.free(samples);            return null;        };    }    const sorted = try allocator.dupe(u64, samples);    defer allocator.free(sorted);    std.mem.sort(u64, sorted, {}, std.sort.asc(u64));    var total: u128 = 0;    for (samples) |sample| total += sample;    const count: u64 = @intCast(samples.len);    return .{        .count = count,        .samples = samples,        .mean_ns = @as(f64, @floatFromInt(total)) / @as(f64, @floatFromInt(count)),        .median_ns = @floatFromInt(percentile(sorted, 50)),        .p75_ns = @floatFromInt(percentile(sorted, 75)),        .p95_ns = @floatFromInt(percentile(sorted, 95)),        .p99_ns = @floatFromInt(percentile(sorted, 99)),        .min_ns = @floatFromInt(sorted[0]),        .max_ns = @floatFromInt(sorted[sorted.len - 1]),    };}fn bootstrapMedianPercentInterval(allocator: std.mem.Allocator, base_samples: []const u64, candidate_samples: []const u64) !?EffectInterval {    if (base_samples.len < 2 or candidate_samples.len < 2) return null;    const sorted_base = try allocator.dupe(u64, base_samples);    defer allocator.free(sorted_base);    std.mem.sort(u64, sorted_base, {}, std.sort.asc(u64));    const sorted_candidate = try allocator.dupe(u64, candidate_samples);    defer allocator.free(sorted_candidate);    std.mem.sort(u64, sorted_candidate, {}, std.sort.asc(u64));    const iterations = 512;    const max_len = @max(base_samples.len, candidate_samples.len);    const base_resample = try allocator.alloc(u64, max_len);    defer allocator.free(base_resample);    const candidate_resample = try allocator.alloc(u64, max_len);    defer allocator.free(candidate_resample);    const changes = try allocator.alloc(f64, iterations);    defer allocator.free(changes);    var rng = Seed.init(base_samples.len, candidate_samples.len);    for (changes) |*slot| {        fillResample(base_resample[0..base_samples.len], sorted_base, &rng);        fillResample(candidate_resample[0..candidate_samples.len], sorted_candidate, &rng);        std.mem.sort(u64, base_resample[0..base_samples.len], {}, std.sort.asc(u64));        std.mem.sort(u64, candidate_resample[0..candidate_samples.len], {}, std.sort.asc(u64));        const base_median = percentile(base_resample[0..base_samples.len], 50);        if (base_median == 0) return null;        const candidate_median = percentile(candidate_resample[0..candidate_samples.len], 50);        slot.* = ((@as(f64, @floatFromInt(candidate_median)) - @as(f64, @floatFromInt(base_median))) / @as(f64, @floatFromInt(base_median))) * 100;    }    std.mem.sort(f64, changes, {}, float_ascending);    return .{        .low_percent = changes[(changes.len * 25) / 1000],        .high_percent = changes[@min((changes.len * 975) / 1000, changes.len - 1)],    };}fn float_ascending(_: void, left: f64, right: f64) bool {    return left < right;}const Seed = struct {    value: u64,    fn init(base_len: usize, candidate_len: usize) Seed {        return .{ .value = 0x9e37_79b9_7f4a_7c15 ^ @as(u64, @intCast(base_len)) ^ (@as(u64, @intCast(candidate_len)) << 32) };    }    fn next(self: *Seed, bound: usize) usize {        self.value ^= self.value << 13;        self.value ^= self.value >> 7;        self.value ^= self.value << 17;        return @intCast(self.value % @as(u64, @intCast(bound)));    }};fn fillResample(out: []u64, samples: []const u64, rng: *Seed) void {    for (out) |*slot| slot.* = samples[rng.next(samples.len)];}fn percentile(sorted: []const u64, comptime p: usize) u64 {    return sorted[@min((sorted.len * p) / 100, sorted.len - 1)];}fn metricKey(allocator: std.mem.Allocator, row: std.json.ObjectMap) ![]const u8 {    return try std.fmt.allocPrint(allocator, "{s}|{s}|{s}|{s}|{s}|{s}|{s}|{s}|{s}|{s}|{s}|{s}|{s}", .{        fieldString(row, "benchmark"),        fieldString(row, "suite"),        fieldString(row, "mode"),        fieldString(row, "pipeline"),        fieldString(row, "event"),        fieldString(row, "kind"),        fieldString(row, "summary"),        fieldString(row, "workload"),        fieldString(row, "phase"),        fieldString(row, "pass"),        fieldString(row, "analysis"),        fieldString(row, "metric"),        fieldString(row, "nameOrId"),    });}fn metricLabel(allocator: std.mem.Allocator, row: std.json.ObjectMap, fallback: []const u8) ![]const u8 {    const benchmark = fieldString(row, "benchmark");    const row_workload = fieldString(row, "workload");    const summary = fieldString(row, "summary");    const phase = fieldString(row, "phase");    const pass = fieldString(row, "pass");    const analysis = fieldString(row, "analysis");    const metric_name = fieldString(row, "metric");    const name = fieldString(row, "nameOrId");    if (benchmark.len == 0 and row_workload.len == 0 and summary.len == 0 and phase.len == 0 and pass.len == 0 and analysis.len == 0 and metric_name.len == 0 and name.len == 0) return fallback;    return try std.fmt.allocPrint(allocator, "{s} {s} {s} {s} {s} {s} {s} {s}", .{ benchmark, row_workload, summary, phase, pass, analysis, metric_name, name });}fn fieldString(row: std.json.ObjectMap, field: []const u8) []const u8 {    if (std.mem.eql(u8, field, "nameOrId")) {        return json.string(row.get("name")) orelse json.string(row.get("id")) orelse "";    }    return json.string(row.get(field)) orelse "";}test "profiling metrics parse structured benchmark rows" {    var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);    defer arena_state.deinit();    const allocator = arena_state.allocator();    var out: std.Io.Writer.Allocating = .init(allocator);    defer out.deinit();    try out.writer.writeAll(        "{\"schema\":\"tiny.profiling.structured/v1\"," ++            "\"workload\":{\"name\":\"gpalloc.allocator\",\"package\":\"lib/gpalloc\"," ++            "\"step\":\"gpalloc-bench\"},\"source\":{\"kind\":\"stdout\"," ++            "\"path\":\"stdout.txt\",\"line\":1},\"row\":{\"event\":\"bench_end\"," ++            "\"suite\":\"gpalloc\",\"name\":\"alloc\",\"sample_ns\":[30,10,20]," ++            "\"sample_sequence\":{\"order\":\"measured_acquisition_order\"," ++            "\"index_origin\":0},\"confidence_intervals\":{\"median_ns\":{" ++            "\"low_ns\":10,\"high_ns\":30}}}}\n",    );    try sys.fs.writeFile(".zig-cache/profile-metric-test.jsonl", out.written());    const metrics = try load(allocator, ".zig-cache/profile-metric-test.jsonl");    try std.testing.expectEqual(@as(usize, 1), metrics.len);    try std.testing.expectEqualStrings("gpalloc.allocator", metrics[0].workload);    try std.testing.expectEqual(@as(u64, 3), metrics[0].sample_count.?);    try std.testing.expectEqual(@as(f64, 20), metrics[0].median_ns.?);    try std.testing.expectEqual(@as(f64, 10), metrics[0].min_ns.?);    try std.testing.expectEqual(@as(f64, 30), metrics[0].p95_ns.?);    try std.testing.expectEqual(@as(f64, 30), metrics[0].max_ns.?);    try std.testing.expectEqualSlices(u64, &.{ 30, 10, 20 }, metrics[0].samples_ns);    try std.testing.expectEqual(        SampleOrder.measured_acquisition_order,        metrics[0].sample_sequence.?.order,    );    try std.testing.expectEqual(@as(u64, 0), metrics[0].sample_sequence.?.index_origin);    try std.testing.expectEqual(Distribution.confidence_interval, metrics[0].distribution);    sys.fs.deleteFile(".zig-cache/profile-metric-test.jsonl") catch {};}test "profiling metric sample sequence requires recognized provenance" {    var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);    defer arena_state.deinit();    const allocator = arena_state.allocator();    const unknown = try std.json.parseFromSliceLeaky(        std.json.Value,        allocator,        "{\"sample_sequence\":{\"order\":\"sorted\",\"index_origin\":0}}",        .{},    );    try std.testing.expect(parseSampleSequence(try json.object(unknown)) == null);    const incomplete = try std.json.parseFromSliceLeaky(        std.json.Value,        allocator,        "{\"sample_sequence\":{\"order\":\"measured_acquisition_order\"}}",        .{},    );    try std.testing.expect(parseSampleSequence(try json.object(incomplete)) == null);}test "profiling metrics compare raw samples with effect interval" {    var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);    defer arena_state.deinit();    const allocator = arena_state.allocator();    const base_samples = try allocator.dupe(u64, &.{ 10, 10, 10, 10 });    const candidate_samples = try allocator.dupe(u64, &.{ 20, 20, 20, 20 });    const base = [_]Metric{.{        .workload = "custom",        .key = "a",        .label = "a",        .source_kind = "bench_jsonl",        .source_path = "bench.jsonl",        .source_line = 1,        .sample_count = 4,        .mean_ns = 10,        .median_ns = 10,        .p75_ns = null,        .p95_ns = null,        .p99_ns = null,        .min_ns = null,        .max_ns = null,        .mean_interval = null,        .median_interval = null,        .p95_interval = null,        .p99_interval = null,        .samples_ns = base_samples,        .distribution = .raw_samples,    }};    const candidate = [_]Metric{.{        .workload = "custom",        .key = "a",        .label = "a",        .source_kind = "bench_jsonl",        .source_path = "bench.jsonl",        .source_line = 1,        .sample_count = 4,        .mean_ns = 20,        .median_ns = 20,        .p75_ns = null,        .p95_ns = null,        .p99_ns = null,        .min_ns = null,        .max_ns = null,        .mean_interval = null,        .median_interval = null,        .p95_interval = null,        .p99_interval = null,        .samples_ns = candidate_samples,        .distribution = .raw_samples,    }};    const rows = try compare(allocator, &base, &candidate, 10);    try std.testing.expectEqual(@as(usize, 1), rows.len);    try std.testing.expectEqualStrings("sample_regression", rows[0].status);    try std.testing.expect(rows[0].effect_low_percent.? >= 10);}test "profiling metric effect interval ignores acquisition order" {    var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);    defer arena_state.deinit();    const allocator = arena_state.allocator();    const forward = try bootstrapMedianPercentInterval(        allocator,        &.{ 10, 20, 30, 40 },        &.{ 20, 40, 60, 80 },    );    const reordered = try bootstrapMedianPercentInterval(        allocator,        &.{ 40, 10, 30, 20 },        &.{ 60, 20, 80, 40 },    );    try std.testing.expectEqual(forward.?.low_percent, reordered.?.low_percent);    try std.testing.expectEqual(forward.?.high_percent, reordered.?.high_percent);}

Source: src/profiling/root.zig:33

zig
pub const metric = @import("metric.zig");

Audit

Definitions13
Public names13
Members46
Version26.7.0
Revisiondaab053ee433