tiny.profiling.host.offcpu

1const std = @import("std");2const capture = @import("capture");3const sys = @import("sys");45const root = @import("root.zig");67pub const default_time_tool = "offcputime-bpfcc";8pub const default_stacks_tool = "bpftrace";9pub const kind = "offcpu_time";1011pub const Mode = enum {12    time,13    stacks,1415    pub fn parse(value: []const u8) ?Mode {16        if (std.mem.eql(u8, value, "time")) return .time;17        if (std.mem.eql(u8, value, "stacks")) return .stacks;18        return null;19    }20};2122pub const Options = struct {23    tool: ?[]const u8 = null,24    mode: Mode = .time,2526    pub fn toolName(self: Options) []const u8 {27        return self.tool orelse switch (self.mode) {28            .time => default_time_tool,29            .stacks => default_stacks_tool,30        };31    }32};3334pub const Lane = struct {35    tool: []const u8,36    mode: Mode,37    capture_path: []const u8,38    diagnostics_path: []const u8,39    summary_path: []const u8,40};4142pub fn plan(allocator: std.mem.Allocator, options: Options, workload_root: []const u8) !Lane {43    const capture_name = switch (options.mode) {44        .time => "offcpu.time.txt",45        .stacks => "offcpu.stacks.txt",46    };47    return .{48        .tool = options.toolName(),49        .mode = options.mode,50        .capture_path = try std.fs.path.join(allocator, &.{ workload_root, capture_name }),51        .diagnostics_path = try std.fs.path.join(52            allocator,53            &.{ workload_root, "offcpu.diagnostics.txt" },54        ),55        .summary_path = try std.fs.path.join(allocator, &.{ workload_root, "offcpu.summary.json" }),56    };57}5859pub fn wrapArgv(allocator: std.mem.Allocator, options: Options, argv: []const []const u8, lane: Lane) ![]const []const u8 {60    _ = options;61    switch (lane.mode) {62        .time => return try capture.offcpu.timeCommand(63            allocator,64            lane.tool,65            argv,66            lane.capture_path,67            lane.diagnostics_path,68        ),69        .stacks => {70            const command = try std.mem.join(allocator, " ", argv);71            return try capture.offcpu.stacksCommand(allocator, lane.tool, command, lane.capture_path);72        },73    }74}7576pub fn summarize(77    allocator: std.mem.Allocator,78    lane: Lane,79    exit_code: i64,80    stderr_path: []const u8,81    tool_available: bool,82) !root.Capture {83    if (!tool_available) return missingTool(lane);84    var blocked = switch (lane.mode) {85        .time => try loadBlockedTime(allocator, lane.capture_path),86        .stacks => try loadBlockedStacks(allocator, lane.capture_path),87    } orelse {88        return try failed(allocator, lane, exit_code, stderr_path);89    };90    if (lane.mode == .time) {91        const integrity = capture.offcpu.summarizeCollectorDiagnosticsFile(92            allocator,93            lane.diagnostics_path,94        ) catch |err| switch (err) {95            error.FileNotFound => capture.offcpu.missingCaptureIntegrity(),96            else => return err,97        };98        blocked.diagnostics_path = lane.diagnostics_path;99        blocked.capture_integrity = integrity;100    }101    try capture.offcpu.writeBlockedSummaryFile(lane.summary_path, blocked);102    return summarized(lane, blocked.capture_integrity);103}104105fn loadBlockedTime(106    allocator: std.mem.Allocator,107    capture_path: []const u8,108) !?capture.offcpu.BlockedSummary {109    const summary = capture.offcpu.summarizeTimeFile(110        allocator,111        capture_path,112    ) catch |err| switch (err) {113        error.FileNotFound => return null,114        else => |actual| return actual,115    };116    if (summary.rows.len == 0) return null;117    return try capture.offcpu.blockedFromTimeSummary(allocator, summary);118}119120fn loadBlockedStacks(121    allocator: std.mem.Allocator,122    capture_path: []const u8,123) !?capture.offcpu.BlockedSummary {124    const summary = capture.offcpu.summarizeStacksFile(125        allocator,126        capture_path,127    ) catch |err| switch (err) {128        error.FileNotFound => return null,129        else => |actual| return actual,130    };131    if (summary.rows.len == 0) return null;132    return try capture.offcpu.blockedFromStackSummary(allocator, summary);133}134135fn summarized(136    lane: Lane,137    integrity: ?capture.offcpu.CaptureIntegrity,138) root.Capture {139    var result: root.Capture = .{140        .kind = kind,141        .tool = lane.tool,142        .capture_path = lane.capture_path,143        .summary_path = lane.summary_path,144        .state = "summary_written",145    };146    const actual = integrity orelse return result;147    if (std.mem.eql(u8, actual.status, "complete")) return result;148    result.caveat_kind = if (std.mem.eql(u8, actual.status, "partial"))149        "offcpu_capture_partial"150    else151        "offcpu_capture_integrity_unknown";152    result.caveat_message = actual.message;153    return result;154}155156fn missingTool(lane: Lane) root.Capture {157    return .{158        .kind = kind,159        .tool = lane.tool,160        .capture_path = lane.capture_path,161        .summary_path = lane.summary_path,162        .state = "tool_missing",163        .caveat_kind = "tool_unavailable",164        .caveat_message = "the off-CPU profiler is not runnable on this host; blocked time was not captured",165    };166}167168fn failed(allocator: std.mem.Allocator, lane: Lane, exit_code: i64, stderr_path: []const u8) !root.Capture {169    const failure_path = try failureCapturePath(170        allocator,171        lane.diagnostics_path,172        stderr_path,173    );174    const row = try capture.caveat.classifyCommandCapture(allocator, .{175        .phase = kind,176        .exit_code = if (exit_code == 0) 1 else exit_code,177        .timed_out = false,178        .capture_path = failure_path,179    }) orelse capture.caveat.classifyCommandText(.{180        .phase = kind,181        .exit_code = exit_code,182        .timed_out = false,183        .capture_path = failure_path,184    }, "");185    return .{186        .kind = kind,187        .tool = lane.tool,188        .capture_path = lane.capture_path,189        .summary_path = lane.summary_path,190        .state = "tool_failed",191        .caveat_kind = row.kind,192        .caveat_message = row.message,193    };194}195196fn failureCapturePath(197    allocator: std.mem.Allocator,198    diagnostics_path: []const u8,199    stderr_path: []const u8,200) ![]const u8 {201    const diagnostics = sys.fs.readFileAlloc(202        allocator,203        diagnostics_path,204        64 * 1024 * 1024,205    ) catch |err| switch (err) {206        error.OutOfMemory => return err,207        else => return stderr_path,208    };209    defer allocator.free(diagnostics);210    if (std.mem.trim(u8, diagnostics, " \t\r\n").len == 0) return stderr_path;211    return diagnostics_path;212}213214test "profiling offcpu wraps direct binaries in the offcputime sidecar" {215    var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);216    defer arena_state.deinit();217    const allocator = arena_state.allocator();218    const lane = try plan(allocator, .{}, "zig-out/profiling/run/workloads/w");219    const argv = [_][]const u8{ "zig-out/bin/mprompt-bench", "--workers", "128" };220    const wrapped = try wrapArgv(allocator, .{}, &argv, lane);221    try std.testing.expectEqualStrings("bash", wrapped[0]);222    try std.testing.expectEqualStrings("-c", wrapped[1]);223    try std.testing.expectEqualStrings("offcputime-bpfcc", wrapped[4]);224    try std.testing.expectEqualStrings("zig-out/profiling/run/workloads/w/offcpu.time.txt", wrapped[5]);225    try std.testing.expectEqualStrings(226        "zig-out/profiling/run/workloads/w/offcpu.diagnostics.txt",227        wrapped[6],228    );229    try std.testing.expectEqualStrings("zig-out/bin/mprompt-bench", wrapped[7]);230    try std.testing.expectEqualStrings("--workers", wrapped[8]);231}232233test "profiling offcpu wraps direct binaries in the bpftrace stacks tracer" {234    var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);235    defer arena_state.deinit();236    const allocator = arena_state.allocator();237    const lane = try plan(allocator, .{ .mode = .stacks }, "zig-out/profiling/run/workloads/w");238    try std.testing.expectEqualStrings("bpftrace", lane.tool);239    const argv = [_][]const u8{ "zig-out/bin/mprompt-bench", "--workers", "128" };240    const wrapped = try wrapArgv(allocator, .{ .mode = .stacks }, &argv, lane);241    try std.testing.expectEqualStrings("bpftrace", wrapped[0]);242    try std.testing.expectEqualStrings("zig-out/profiling/run/workloads/w/offcpu.stacks.txt", wrapped[3]);243    try std.testing.expectEqualStrings("zig-out/bin/mprompt-bench --workers 128", wrapped[wrapped.len - 1]);244}245246test "profiling offcpu summarizes captured blocked time" {247    var tmp = std.testing.tmpDir(.{});248    defer tmp.cleanup();249    var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);250    defer arena_state.deinit();251    const allocator = arena_state.allocator();252    const dir = try std.fs.path.join(allocator, &.{ ".zig-cache", "tmp", tmp.sub_path[0..] });253    const lane = try plan(allocator, .{}, dir);254    try sys.fs.writeFile(lane.capture_path,255        \\bench-worker;futex_wait;do_futex 4200256        \\bench-main;epoll_wait 1300257        \\258    );259    const stderr_path = try std.fs.path.join(allocator, &.{ dir, "stderr.txt" });260    try sys.fs.writeFile(stderr_path, "");261    try sys.fs.writeFile(lane.diagnostics_path, "");262    const row = try summarize(allocator, lane, 0, stderr_path, true);263    try std.testing.expectEqualStrings("summary_written", row.state);264    try std.testing.expect(row.caveat_kind == null);265    try std.testing.expect(sys.fs.exists(lane.summary_path));266}267268test "profiling offcpu retains rows and caveats collector loss" {269    var tmp = std.testing.tmpDir(.{});270    defer tmp.cleanup();271    var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);272    defer arena_state.deinit();273    const allocator = arena_state.allocator();274    const dir = try std.fs.path.join(275        allocator,276        &.{ ".zig-cache", "tmp", tmp.sub_path[0..] },277    );278    const lane = try plan(allocator, .{}, dir);279    try sys.fs.writeFile(lane.capture_path, "bench;futex_wait;do_futex 4200\n");280    try sys.fs.writeFile(281        lane.diagnostics_path,282        "WARNING: 2 stack traces lost and could not be displayed.\n",283    );284    const stderr_path = try std.fs.path.join(allocator, &.{ dir, "stderr.txt" });285    try sys.fs.writeFile(stderr_path, "workload warning that is not collector evidence\n");286    const row = try summarize(allocator, lane, 0, stderr_path, true);287    try std.testing.expectEqualStrings("summary_written", row.state);288    try std.testing.expectEqualStrings("offcpu_capture_partial", row.caveat_kind.?);289    try std.testing.expect(std.mem.indexOf(290        u8,291        row.caveat_message.?,292        "2 stack trace(s) lost",293    ) != null);294    const text = try sys.fs.readFileAlloc(allocator, lane.summary_path, 64 * 1024);295    try std.testing.expect(std.mem.indexOf(u8, text, "\"lost_stack_trace_count\": 2") != null);296    try std.testing.expect(std.mem.indexOf(u8, text, "workload warning") == null);297}298299test "profiling offcpu marks missing collector diagnostics unknown" {300    var tmp = std.testing.tmpDir(.{});301    defer tmp.cleanup();302    var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);303    defer arena_state.deinit();304    const allocator = arena_state.allocator();305    const dir = try std.fs.path.join(306        allocator,307        &.{ ".zig-cache", "tmp", tmp.sub_path[0..] },308    );309    const lane = try plan(allocator, .{}, dir);310    try sys.fs.writeFile(lane.capture_path, "bench;futex_wait;do_futex 4200\n");311    const stderr_path = try std.fs.path.join(allocator, &.{ dir, "stderr.txt" });312    try sys.fs.writeFile(stderr_path, "");313    const row = try summarize(allocator, lane, 0, stderr_path, true);314    try std.testing.expectEqualStrings("summary_written", row.state);315    try std.testing.expectEqualStrings(316        "offcpu_capture_integrity_unknown",317        row.caveat_kind.?,318    );319}320321test "profiling offcpu summarizes captured blocked stacks" {322    var tmp = std.testing.tmpDir(.{});323    defer tmp.cleanup();324    var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);325    defer arena_state.deinit();326    const allocator = arena_state.allocator();327    const dir = try std.fs.path.join(allocator, &.{ ".zig-cache", "tmp", tmp.sub_path[0..] });328    const lane = try plan(allocator, .{ .mode = .stacks }, dir);329    try sys.fs.writeFile(lane.capture_path,330        \\@offcpu_kernel[331        \\    finish_task_switch+1332        \\    schedule+108333        \\]: 42334        \\335    );336    const stderr_path = try std.fs.path.join(allocator, &.{ dir, "stderr.txt" });337    try sys.fs.writeFile(stderr_path, "");338    const row = try summarize(allocator, lane, 0, stderr_path, true);339    try std.testing.expectEqualStrings("summary_written", row.state);340    try std.testing.expect(sys.fs.exists(lane.summary_path));341}342343test "profiling offcpu classifies permission failures from stderr" {344    var tmp = std.testing.tmpDir(.{});345    defer tmp.cleanup();346    var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);347    defer arena_state.deinit();348    const allocator = arena_state.allocator();349    const dir = try std.fs.path.join(allocator, &.{ ".zig-cache", "tmp", tmp.sub_path[0..] });350    const lane = try plan(allocator, .{}, dir);351    const stderr_path = try std.fs.path.join(allocator, &.{ dir, "stderr.txt" });352    try sys.fs.writeFile(stderr_path, "could not open bpf map: Operation not permitted\n");353    const row = try summarize(allocator, lane, 1, stderr_path, true);354    try std.testing.expectEqualStrings("tool_failed", row.state);355    try std.testing.expectEqualStrings("permission_denied", row.caveat_kind.?);356}357358test "profiling offcpu reports missing tools as caveats" {359    var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);360    defer arena_state.deinit();361    const allocator = arena_state.allocator();362    const lane = try plan(allocator, .{}, "zig-out/profiling/run/workloads/w");363    const row = try summarize(allocator, lane, 0, "missing-stderr.txt", false);364    try std.testing.expectEqualStrings("tool_missing", row.state);365    try std.testing.expectEqualStrings("tool_unavailable", row.caveat_kind.?);366}

1pub const offcpu = runtime.offcpu;