lib/sys/src/perf.zig
daab053ee43316e1809a84551d573ddd1e5bf3d2
1 const std = @import("std");
2 const builtin = @import("builtin");
3 const capabilities = @import("capabilities.zig");
4 const memory = @import("memory.zig");
5
6 const native_endian = builtin.target.cpu.arch.endian();
7 const linux = std.os.linux;
8 const posix = std.posix;
9
10 pub const required_capabilities = capabilities.host(&.{ .descriptors, .memory_mapping, .performance_counters, .time });
11
12 pub const supported = builtin.os.tag == .linux;
13
14 pub const header_size = 8;
15 pub const data_pages = 2;
16 pub const page_size = std.heap.page_size_min;
17 pub const data_size = data_pages * page_size;
18 pub const mmap_size = data_size + page_size;
19 pub const default_sample_type = linux.PERF.SAMPLE.IP | linux.PERF.SAMPLE.CALLCHAIN;
20 pub const invalid_fd: posix.fd_t = -1;
21
22 comptime {
23 std.debug.assert(data_pages > 0);
24 std.debug.assert(std.heap.page_size_min <= std.heap.page_size_max);
25 std.debug.assert(
26 std.heap.page_size_max <=
27 std.math.maxInt(usize) / (data_pages + 1),
28 );
29 }
30
31 pub const ReadFormat = struct {
32 pub const total_time_enabled: u64 = 1 << 0;
33 pub const total_time_running: u64 = 1 << 1;
34 pub const id: u64 = 1 << 2;
35 pub const group: u64 = 1 << 3;
36 pub const lost: u64 = 1 << 4;
37 };
38
39 pub const SampleField = enum(u64) {
40 ip = linux.PERF.SAMPLE.IP,
41 pid_tid = linux.PERF.SAMPLE.TID,
42 time = linux.PERF.SAMPLE.TIME,
43 addr = linux.PERF.SAMPLE.ADDR,
44 read = linux.PERF.SAMPLE.READ,
45 callchain = linux.PERF.SAMPLE.CALLCHAIN,
46 id = linux.PERF.SAMPLE.ID,
47 cpu = linux.PERF.SAMPLE.CPU,
48 period = linux.PERF.SAMPLE.PERIOD,
49 stream_id = linux.PERF.SAMPLE.STREAM_ID,
50 raw = linux.PERF.SAMPLE.RAW,
51 branch_stack = linux.PERF.SAMPLE.BRANCH_STACK,
52 regs = linux.PERF.SAMPLE.REGS_USER,
53 stack = linux.PERF.SAMPLE.STACK_USER,
54 _,
55 };
56
57 pub const RecordType = enum(u32) {
58 mmap = 1,
59 lost = 2,
60 comm = 3,
61 exit = 4,
62 throttle = 5,
63 unthrottle = 6,
64 fork = 7,
65 read = 8,
66 sample = 9,
67 mmap2 = 10,
68 lost_samples = 13,
69 _,
70 };
71
72 pub const Config = struct {
73 sample_type: u64 = 0,
74 read_format: u64 = 0,
75
76 pub fn fromAttr(attr: linux.perf_event_attr) Config {
77 return .{ .sample_type = attr.sample_type, .read_format = attr.read_format };
78 }
79
80 pub fn isSampling(self: Config, field: SampleField) bool {
81 return self.sample_type & sampleBit(field) != 0;
82 }
83 };
84
85 pub const SamplerOptions = struct {
86 sample_period_ns: u64 = 1_000_000,
87 sample_batch_size: u32 = 10,
88 sample_type: u64 = default_sample_type,
89 read_format: u64 = 0,
90 exclude_kernel: bool = true,
91 exclude_idle: bool = true,
92 };
93
94 pub const HardwareCounter = enum {
95 cycles,
96 instructions,
97 cache_references,
98 cache_misses,
99 branch_instructions,
100 branch_misses,
101 bus_cycles,
102 stalled_cycles_frontend,
103 stalled_cycles_backend,
104 reference_cycles,
105
106 fn config(self: HardwareCounter) linux.PERF.COUNT.HW {
107 return switch (self) {
108 .cycles => .CPU_CYCLES,
109 .instructions => .INSTRUCTIONS,
110 .cache_references => .CACHE_REFERENCES,
111 .cache_misses => .CACHE_MISSES,
112 .branch_instructions => .BRANCH_INSTRUCTIONS,
113 .branch_misses => .BRANCH_MISSES,
114 .bus_cycles => .BUS_CYCLES,
115 .stalled_cycles_frontend => .STALLED_CYCLES_FRONTEND,
116 .stalled_cycles_backend => .STALLED_CYCLES_BACKEND,
117 .reference_cycles => .REF_CPU_CYCLES,
118 };
119 }
120 };
121
122 pub const SoftwareCounter = enum {
123 cpu_clock,
124 task_clock,
125 page_faults,
126 context_switches,
127 cpu_migrations,
128 minor_faults,
129 major_faults,
130 alignment_faults,
131 emulation_faults,
132 dummy,
133
134 fn config(self: SoftwareCounter) linux.PERF.COUNT.SW {
135 return switch (self) {
136 .cpu_clock => .CPU_CLOCK,
137 .task_clock => .TASK_CLOCK,
138 .page_faults => .PAGE_FAULTS,
139 .context_switches => .CONTEXT_SWITCHES,
140 .cpu_migrations => .CPU_MIGRATIONS,
141 .minor_faults => .PAGE_FAULTS_MIN,
142 .major_faults => .PAGE_FAULTS_MAJ,
143 .alignment_faults => .ALIGNMENT_FAULTS,
144 .emulation_faults => .EMULATION_FAULTS,
145 .dummy => .DUMMY,
146 };
147 }
148 };
149
150 pub const CacheKind = enum {
151 l1_data,
152 l1_instruction,
153 last_level,
154 data_tlb,
155 instruction_tlb,
156 branch_predictor,
157 node,
158
159 fn config(self: CacheKind) linux.PERF.COUNT.HW.CACHE {
160 return switch (self) {
161 .l1_data => .L1D,
162 .l1_instruction => .L1I,
163 .last_level => .LL,
164 .data_tlb => .DTLB,
165 .instruction_tlb => .ITLB,
166 .branch_predictor => .BPU,
167 .node => .NODE,
168 };
169 }
170 };
171
172 pub const CacheOperation = enum {
173 read,
174 write,
175 prefetch,
176
177 fn config(self: CacheOperation) linux.PERF.COUNT.HW.CACHE.OP {
178 return switch (self) {
179 .read => .READ,
180 .write => .WRITE,
181 .prefetch => .PREFETCH,
182 };
183 }
184 };
185
186 pub const CacheResult = enum {
187 access,
188 miss,
189
190 fn config(self: CacheResult) linux.PERF.COUNT.HW.CACHE.RESULT {
191 return switch (self) {
192 .access => .ACCESS,
193 .miss => .MISS,
194 };
195 }
196 };
197
198 pub const CacheCounter = struct {
199 cache: CacheKind,
200 operation: CacheOperation,
201 result: CacheResult,
202
203 pub fn config(self: CacheCounter) u64 {
204 return @as(u64, @backingInt(self.cache.config())) |
205 (@as(u64, @backingInt(self.operation.config())) << 8) |
206 (@as(u64, @backingInt(self.result.config())) << 16);
207 }
208 };
209
210 pub const CounterOptions = struct {
211 read_format: u64 = 0,
212 exclude_kernel: bool = true,
213 exclude_hv: bool = true,
214 exclude_idle: bool = false,
215 inherit: bool = false,
216 };
217
218 pub const CounterSelector = union(enum) {
219 hardware: HardwareCounter,
220 software: SoftwareCounter,
221 cache: CacheCounter,
222 raw: u64,
223
224 pub fn attr(self: CounterSelector, options: CounterOptions) linux.perf_event_attr {
225 var result = counterAttr(options);
226 switch (self) {
227 .hardware => |counter| {
228 result.type = .HARDWARE;
229 result.config = @backingInt(counter.config());
230 },
231 .software => |counter| {
232 result.type = .SOFTWARE;
233 result.config = @backingInt(counter.config());
234 },
235 .cache => |counter| {
236 result.type = .HW_CACHE;
237 result.config = counter.config();
238 },
239 .raw => |config| {
240 result.type = .RAW;
241 result.config = config;
242 },
243 }
244 return result;
245 }
246 };
247
248 pub const CountCoverage = enum {
249 unknown,
250 complete,
251 multiplexed,
252 not_counted,
253 invalid,
254
255 pub fn text(self: CountCoverage) []const u8 {
256 return switch (self) {
257 .unknown => "unknown",
258 .complete => "complete",
259 .multiplexed => "multiplexed",
260 .not_counted => "not_counted",
261 .invalid => "invalid",
262 };
263 }
264 };
265
266 pub const CountResult = struct {
267 value: u64,
268 time_enabled: ?u64 = null,
269 time_running: ?u64 = null,
270 id: ?u64 = null,
271 lost: ?u64 = null,
272
273 pub fn runningCoverage(self: CountResult) CountCoverage {
274 const enabled = self.time_enabled orelse {
275 if (self.time_running == null) return .unknown;
276 return .invalid;
277 };
278 const running = self.time_running orelse return .invalid;
279 if (running == 0) return .not_counted;
280 if (running > enabled) return .invalid;
281 if (running == enabled) return .complete;
282 return .multiplexed;
283 }
284
285 pub fn scaledValue(self: CountResult) ?u64 {
286 return switch (self.runningCoverage()) {
287 .unknown, .complete => self.value,
288 .multiplexed => blk: {
289 const enabled = self.time_enabled.?;
290 const running = self.time_running.?;
291 const scaled = (@as(u128, self.value) * enabled) / running;
292 break :blk std.math.cast(u64, scaled) orelse std.math.maxInt(u64);
293 },
294 .not_counted, .invalid => null,
295 };
296 }
297
298 pub fn runningRatio(self: CountResult) ?f64 {
299 return switch (self.runningCoverage()) {
300 .complete, .multiplexed => ratio(self.time_running.?, self.time_enabled.?),
301 .not_counted => if (self.time_enabled.? == 0) null else 0.0,
302 .unknown, .invalid => null,
303 };
304 }
305
306 pub fn parse(bytes: []const u8, read_format: u64) !CountResult {
307 if (read_format & ReadFormat.group != 0) return error.UnsupportedReadFormat;
308 var offset: usize = 0;
309 var result = CountResult{ .value = try readCountField(bytes, &offset) };
310 if (read_format & ReadFormat.total_time_enabled != 0) result.time_enabled = try readCountField(bytes, &offset);
311 if (read_format & ReadFormat.total_time_running != 0) result.time_running = try readCountField(bytes, &offset);
312 if (read_format & ReadFormat.id != 0) result.id = try readCountField(bytes, &offset);
313 if (read_format & ReadFormat.lost != 0) result.lost = try readCountField(bytes, &offset);
314 if (offset != bytes.len) return error.TrailingPerfEventReadData;
315 return result;
316 }
317 };
318
319 pub const CounterMeasurement = struct {
320 selector: CounterSelector,
321 result: CountResult,
322 };
323
324 pub fn CounterSetResult(comptime capacity: usize) type {
325 if (capacity == 0) @compileError("counter set capacity must be positive");
326 return struct {
327 measurements: [capacity]CounterMeasurement = undefined,
328 len: usize = 0,
329
330 const Self = @This();
331
332 pub fn slice(self: *const Self) []const CounterMeasurement {
333 std.debug.assert(self.len <= capacity);
334 return self.measurements[0..self.len];
335 }
336
337 pub fn find(self: *const Self, selector: CounterSelector) ?CountResult {
338 for (self.slice()) |measurement| {
339 if (std.meta.eql(measurement.selector, selector)) return measurement.result;
340 }
341 return null;
342 }
343
344 pub fn parse(
345 bytes: []const u8,
346 read_format: u64,
347 selectors: []const CounterSelector,
348 expected_ids: []const u64,
349 ) !Self {
350 return parseCounterSetResult(capacity, bytes, read_format, selectors, expected_ids);
351 }
352 };
353 }
354
355 fn parseCounterSetResult(
356 comptime capacity: usize,
357 bytes: []const u8,
358 read_format: u64,
359 selectors: []const CounterSelector,
360 expected_ids: []const u64,
361 ) !CounterSetResult(capacity) {
362 try validateCounterSetInput(capacity, read_format, selectors, expected_ids);
363 var offset: usize = 0;
364 const reported_count = std.math.cast(
365 usize,
366 try readCountField(bytes, &offset),
367 ) orelse return error.CounterSetCountTooLarge;
368 if (reported_count != selectors.len) return error.CounterSetCountMismatch;
369 const time_enabled = try readOptionalGroupTime(bytes, &offset, read_format, .enabled);
370 const time_running = try readOptionalGroupTime(bytes, &offset, read_format, .running);
371
372 var result: CounterSetResult(capacity) = .{ .len = selectors.len };
373 var seen: [capacity]bool = @splat(false);
374 for (0..reported_count) |_| {
375 var count: CountResult = .{
376 .value = try readCountField(bytes, &offset),
377 .time_enabled = time_enabled,
378 .time_running = time_running,
379 .id = try readCountField(bytes, &offset),
380 };
381 if (read_format & ReadFormat.lost != 0) {
382 count.lost = try readCountField(bytes, &offset);
383 }
384 const requested_index = findCounterId(expected_ids, count.id.?) orelse
385 return error.UnknownCounterSetId;
386 if (seen[requested_index]) return error.DuplicateCounterSetId;
387 seen[requested_index] = true;
388 result.measurements[requested_index] = .{
389 .selector = selectors[requested_index],
390 .result = count,
391 };
392 }
393 if (offset != bytes.len) return error.TrailingPerfEventReadData;
394 for (seen[0..selectors.len]) |matched| {
395 if (!matched) return error.MissingCounterSetId;
396 }
397 return result;
398 }
399
400 fn validateCounterSetInput(
401 comptime capacity: usize,
402 read_format: u64,
403 selectors: []const CounterSelector,
404 expected_ids: []const u64,
405 ) !void {
406 if (read_format & ReadFormat.group == 0) return error.CounterSetGroupFormatRequired;
407 if (read_format & ReadFormat.id == 0) return error.CounterSetIdFormatRequired;
408 if (selectors.len == 0) return error.EmptyCounterSet;
409 if (selectors.len > capacity) return error.CounterSetCapacityExceeded;
410 if (selectors.len != expected_ids.len) return error.CounterSetIdentityCountMismatch;
411 try requireUniqueCounterSelectors(selectors);
412 try requireUniqueCounterIds(expected_ids);
413 }
414
415 const GroupTime = enum { enabled, running };
416
417 fn readOptionalGroupTime(
418 bytes: []const u8,
419 offset: *usize,
420 read_format: u64,
421 field: GroupTime,
422 ) !?u64 {
423 const mask = switch (field) {
424 .enabled => ReadFormat.total_time_enabled,
425 .running => ReadFormat.total_time_running,
426 };
427 if (read_format & mask == 0) return null;
428 return try readCountField(bytes, offset);
429 }
430
431 fn requireUniqueCounterIds(ids: []const u64) !void {
432 for (ids, 0..) |id, index| {
433 if (findCounterId(ids[0..index], id) != null) return error.DuplicateExpectedCounterSetId;
434 }
435 }
436
437 fn requireUniqueCounterSelectors(selectors: []const CounterSelector) !void {
438 for (selectors, 0..) |selector, index| {
439 for (selectors[0..index]) |previous| {
440 if (std.meta.eql(selector, previous)) return error.DuplicateCounterSetSelector;
441 }
442 }
443 }
444
445 fn findCounterId(ids: []const u64, target: u64) ?usize {
446 for (ids, 0..) |id, index| {
447 if (id == target) return index;
448 }
449 return null;
450 }
451
452 fn ratio(numerator: u64, denominator: u64) f64 {
453 return @as(f64, @floatFromInt(numerator)) / @as(f64, @floatFromInt(denominator));
454 }
455
456 pub const CounterRegion = struct {
457 event: Event = .{},
458 active: bool = false,
459
460 pub fn start(selector: CounterSelector, options: CounterOptions) !CounterRegion {
461 var event = try Event.openCounter(selector, options);
462 errdefer event.close();
463 try event.start();
464 return .{ .event = event, .active = true };
465 }
466
467 pub fn stop(self: *CounterRegion) !CountResult {
468 if (!self.active) return error.CounterRegionStopped;
469 try self.event.stop();
470 self.active = false;
471 return self.event.countResult();
472 }
473
474 pub fn deinit(self: *CounterRegion) void {
475 self.event.close();
476 self.active = false;
477 }
478 };
479
480 pub fn CounterSetRegion(comptime capacity: usize) type {
481 if (capacity == 0) @compileError("counter set capacity must be positive");
482 return struct {
483 events: [capacity]Event = @splat(.{}),
484 selectors: [capacity]CounterSelector = undefined,
485 ids: [capacity]u64 = undefined,
486 len: usize = 0,
487 active: bool = false,
488
489 const Self = @This();
490 pub const Result: type = CounterSetResult(capacity);
491
492 pub fn start(selectors: []const CounterSelector, options: CounterOptions) !Self {
493 if (selectors.len == 0) return error.EmptyCounterSet;
494 if (selectors.len > capacity) return error.CounterSetCapacityExceeded;
495 if (options.inherit) return error.InheritedCounterSetUnsupported;
496 try requireUniqueCounterSelectors(selectors);
497
498 var region: Self = .{};
499 errdefer region.deinit();
500 var group_options = options;
501 group_options.read_format |= ReadFormat.group | ReadFormat.id |
502 ReadFormat.total_time_enabled | ReadFormat.total_time_running;
503 for (selectors, 0..) |selector, index| {
504 region.events[index] = if (index == 0)
505 try Event.openCounter(selector, group_options)
506 else
507 try Event.openCounterMember(selector, group_options, region.events[0].fd);
508 region.len = index + 1;
509 region.selectors[index] = selector;
510 region.ids[index] = try region.events[index].id();
511 }
512 try control(
513 region.events[0].fd,
514 linux.PERF.EVENT_IOC.ENABLE,
515 linux.PERF.IOC_FLAG_GROUP,
516 );
517 region.active = true;
518 return region;
519 }
520
521 pub fn stop(self: *Self) !Result {
522 if (!self.active) return error.CounterSetRegionStopped;
523 try control(
524 self.events[0].fd,
525 linux.PERF.EVENT_IOC.DISABLE,
526 linux.PERF.IOC_FLAG_GROUP,
527 );
528 self.active = false;
529
530 var bytes: [counterSetReadCapacity(capacity)]u8 = undefined;
531 const read_format = self.events[0].config.read_format;
532 const expected = try counterSetReadSize(read_format, self.len);
533 const got = try posix.read(self.events[0].fd, bytes[0..expected]);
534 if (got != expected) return error.ShortPerfEventRead;
535 return Result.parse(
536 bytes[0..got],
537 read_format,
538 self.selectors[0..self.len],
539 self.ids[0..self.len],
540 );
541 }
542
543 pub fn deinit(self: *Self) void {
544 for (self.events[0..self.len]) |*event| event.close();
545 self.len = 0;
546 self.active = false;
547 }
548 };
549 }
550
551 pub fn taskClockSamplerAttr(options: SamplerOptions) linux.perf_event_attr {
552 var attr: linux.perf_event_attr = .{};
553 attr.type = .SOFTWARE;
554 attr.config = @backingInt(linux.PERF.COUNT.SW.TASK_CLOCK);
555 attr.sample_period_or_freq = options.sample_period_ns;
556 attr.sample_type = options.sample_type;
557 attr.read_format = options.read_format;
558 attr.wakeup_events_or_watermark = options.sample_batch_size;
559 attr.flags.disabled = true;
560 attr.flags.exclude_idle = options.exclude_idle;
561 attr.flags.exclude_kernel = options.exclude_kernel;
562 return attr;
563 }
564
565 pub fn counterAttr(options: CounterOptions) linux.perf_event_attr {
566 var attr: linux.perf_event_attr = .{};
567 attr.read_format = options.read_format;
568 attr.flags.disabled = true;
569 attr.flags.exclude_idle = options.exclude_idle;
570 attr.flags.exclude_kernel = options.exclude_kernel;
571 attr.flags.exclude_hv = options.exclude_hv;
572 attr.flags.inherit = options.inherit;
573 return attr;
574 }
575
576 pub const Event = struct {
577 fd: posix.fd_t = invalid_fd,
578 mapping: ?[]align(page_size) u8 = null,
579 config: Config = .{},
580 mapped_page_size: usize = page_size,
581
582 pub fn open(attr: *linux.perf_event_attr, pid: posix.pid_t, cpu: i32) !Event {
583 if (comptime !supported) return error.UnsupportedPlatform;
584 attr.flags.disabled = true;
585 return openLinux(attr, pid, cpu, invalid_fd);
586 }
587
588 pub fn openTaskClockSampler(options: SamplerOptions) !Event {
589 if (comptime !supported) return error.UnsupportedPlatform;
590 var attr = taskClockSamplerAttr(options);
591 return open(&attr, 0, -1);
592 }
593
594 pub fn openCounter(selector: CounterSelector, options: CounterOptions) !Event {
595 if (comptime !supported) return error.UnsupportedPlatform;
596 var attr = selector.attr(options);
597 return open(&attr, 0, -1);
598 }
599
600 fn openCounterMember(
601 selector: CounterSelector,
602 options: CounterOptions,
603 group_fd: posix.fd_t,
604 ) !Event {
605 if (comptime !supported) return error.UnsupportedPlatform;
606 var attr = selector.attr(options);
607 attr.flags.disabled = false;
608 return openLinux(&attr, 0, -1, group_fd);
609 }
610
611 pub fn close(self: *Event) void {
612 if (self.mapping) |mapped| {
613 memory.unmap(mapped);
614 self.mapping = null;
615 }
616 if (self.fd != invalid_fd) {
617 closeFd(self.fd);
618 self.fd = invalid_fd;
619 }
620 self.config = .{};
621 self.mapped_page_size = page_size;
622 }
623
624 pub fn start(self: *Event) !void {
625 if (self.fd == invalid_fd) return error.InvalidPerfEvent;
626 try control(self.fd, linux.PERF.EVENT_IOC.ENABLE, 0);
627 }
628
629 pub fn stop(self: *Event) !void {
630 if (self.fd == invalid_fd) return error.InvalidPerfEvent;
631 try control(self.fd, linux.PERF.EVENT_IOC.DISABLE, 0);
632 }
633
634 pub fn id(self: *Event) !u64 {
635 if (self.fd == invalid_fd) return error.InvalidPerfEvent;
636 return eventId(self.fd);
637 }
638
639 pub fn count(self: *Event) !u64 {
640 return (try self.countResult()).value;
641 }
642
643 pub fn countResult(self: *Event) !CountResult {
644 if (self.fd == invalid_fd) return error.InvalidPerfEvent;
645 var bytes: [@sizeOf(u64) * 5]u8 = undefined;
646 const expected = countReadSize(self.config.read_format) orelse return error.UnsupportedReadFormat;
647 const got = try posix.read(self.fd, bytes[0..expected]);
648 if (got != expected) return error.ShortPerfEventRead;
649 return CountResult.parse(bytes[0..got], self.config.read_format);
650 }
651
652 pub fn ringReader(self: *Event) ?RingReader {
653 const mapped = self.mapping orelse return null;
654 const page = mappedPage(mapped);
655 const head = @atomicLoad(u64, &page.data_head, .acquire);
656 const tail = @atomicLoad(u64, &page.data_tail, .monotonic);
657 return RingReader.init(self.config, mappedData(mapped, self.mapped_page_size), tail, head);
658 }
659
660 pub fn commitReader(self: *Event, reader: RingReader) void {
661 const mapped = self.mapping orelse return;
662 const page = mappedPage(mapped);
663 @atomicStore(u64, &page.data_tail, reader.index, .release);
664 }
665 };
666
667 fn openLinux(
668 attr: *linux.perf_event_attr,
669 pid: posix.pid_t,
670 cpu: i32,
671 group_fd: posix.fd_t,
672 ) !Event {
673 attr.size = @sizeOf(linux.perf_event_attr);
674
675 const fd = try perfEventOpenLinux(attr, pid, cpu, group_fd, 0);
676 errdefer closeFd(fd);
677
678 var mapping: ?[]align(page_size) u8 = null;
679 errdefer if (mapping) |mapped| memory.unmap(mapped);
680 const actual_page_size = std.heap.pageSize();
681
682 if (attr.sample_type != 0 and attr.sample_period_or_freq != 0) {
683 mapping = try memory.mapSharedFile(
684 fd,
685 mmapSizeForPageSize(actual_page_size),
686 .{ .read = true, .write = true },
687 0,
688 );
689 }
690
691 return .{
692 .fd = fd,
693 .mapping = mapping,
694 .config = Config.fromAttr(attr.*),
695 .mapped_page_size = actual_page_size,
696 };
697 }
698
699 pub const Header = struct {
700 record_type: RecordType,
701 misc: u16,
702 size: u16,
703
704 pub fn parse(bytes: []const u8) !Header {
705 if (bytes.len < header_size) return error.TruncatedRecordHeader;
706 return .{
707 .record_type = @fromBackingInt(@intCast(readAt(u32, bytes, 0))),
708 .misc = readAt(u16, bytes, 4),
709 .size = readAt(u16, bytes, 6),
710 };
711 }
712 };
713
714 pub const Callchain = struct {
715 bytes: []const u8,
716
717 pub fn len(self: Callchain) usize {
718 return self.bytes.len / @sizeOf(u64);
719 }
720
721 pub fn at(self: Callchain, index: usize) u64 {
722 std.debug.assert(index < self.len());
723 return readAt(u64, self.bytes, index * @sizeOf(u64));
724 }
725 };
726
727 pub const Record = struct {
728 config: Config,
729 header: Header,
730 bytes: []const u8,
731
732 pub fn init(config: Config, bytes: []const u8) !Record {
733 const header = try Header.parse(bytes);
734 if (header.size < header_size) return error.InvalidRecordSize;
735 if (bytes.len < header.size) return error.TruncatedRecord;
736 return .{ .config = config, .header = header, .bytes = bytes[0..header.size] };
737 }
738
739 pub fn recordType(self: Record) RecordType {
740 return self.header.record_type;
741 }
742
743 pub fn isSample(self: Record) bool {
744 return self.recordType() == .sample;
745 }
746
747 pub fn getLostCount(self: Record) !u64 {
748 const offset: usize = switch (self.recordType()) {
749 .lost => header_size + @sizeOf(u64),
750 .lost_samples => header_size,
751 else => return error.NotLostRecord,
752 };
753 try self.requireRange(offset, @sizeOf(u64));
754 return readAt(u64, self.bytes, offset);
755 }
756
757 pub fn getIp(self: Record) !u64 {
758 try self.expectSampleField(.ip);
759 return self.readField(u64, .ip);
760 }
761
762 pub fn getPid(self: Record) !u32 {
763 try self.expectSampleField(.pid_tid);
764 const offset = try self.fieldOffset(.pid_tid);
765 try self.requireRange(offset, 2 * @sizeOf(u32));
766 return readAt(u32, self.bytes, offset);
767 }
768
769 pub fn getTid(self: Record) !u32 {
770 try self.expectSampleField(.pid_tid);
771 const offset = try self.fieldOffset(.pid_tid);
772 try self.requireRange(offset, 2 * @sizeOf(u32));
773 return readAt(u32, self.bytes, offset + @sizeOf(u32));
774 }
775
776 pub fn getTime(self: Record) !u64 {
777 try self.expectSampleField(.time);
778 return self.readField(u64, .time);
779 }
780
781 pub fn getCpu(self: Record) !u32 {
782 try self.expectSampleField(.cpu);
783 const offset = try self.fieldOffset(.cpu);
784 try self.requireRange(offset, 2 * @sizeOf(u32));
785 return readAt(u32, self.bytes, offset);
786 }
787
788 pub fn getPeriod(self: Record) !u64 {
789 try self.expectSampleField(.period);
790 return self.readField(u64, .period);
791 }
792
793 pub fn getCallchain(self: Record) !Callchain {
794 try self.expectSampleField(.callchain);
795 const offset = try self.fieldOffset(.callchain);
796 const count = try self.readCountedLength(offset);
797 const payload_offset = try self.advance(offset, @sizeOf(u64));
798 const payload_size = try std.math.mul(usize, count, @sizeOf(u64));
799 try self.requireRange(payload_offset, payload_size);
800 return .{ .bytes = self.bytes[payload_offset..][0..payload_size] };
801 }
802
803 pub fn getRaw(self: Record) ![]const u8 {
804 try self.expectSampleField(.raw);
805 const offset = try self.fieldOffset(.raw);
806 const payload_offset = try self.advance(offset, @sizeOf(u32));
807 const size = readAt(u32, self.bytes, offset);
808 try self.requireRange(payload_offset, size);
809 return self.bytes[payload_offset..][0..size];
810 }
811
812 pub fn payloadEndOffset(self: Record) !usize {
813 var offset: usize = header_size;
814 inline for (field_order) |field| {
815 offset = try self.skipSampleField(offset, field);
816 }
817 return offset;
818 }
819
820 fn readField(self: Record, comptime T: type, field: SampleField) !T {
821 const offset = try self.fieldOffset(field);
822 try self.requireRange(offset, @sizeOf(T));
823 return readAt(T, self.bytes, offset);
824 }
825
826 fn expectSampleField(self: Record, field: SampleField) !void {
827 if (!self.isSample()) return error.NotSampleRecord;
828 if (!self.config.isSampling(field)) return error.FieldNotSampled;
829 }
830
831 fn fieldOffset(self: Record, target: SampleField) !usize {
832 var offset: usize = header_size;
833 inline for (field_order) |field| {
834 if (field == target) return offset;
835 offset = try self.skipSampleField(offset, field);
836 }
837 return error.UnsupportedSampleField;
838 }
839
840 fn skipSampleField(self: Record, offset: usize, field: SampleField) !usize {
841 if (!self.config.isSampling(field)) return offset;
842 switch (field) {
843 .ip, .time, .addr, .id, .stream_id, .period => return self.advance(offset, @sizeOf(u64)),
844 .pid_tid, .cpu => return self.advance(offset, @sizeOf(u32) + @sizeOf(u32)),
845 .read => return self.skipReadField(offset),
846 .callchain => {
847 const count = try self.readCountedLength(offset);
848 const payload_size = try std.math.mul(usize, count, @sizeOf(u64));
849 const payload_offset = try self.advance(offset, @sizeOf(u64));
850 return self.advance(payload_offset, payload_size);
851 },
852 .raw => {
853 try self.requireRange(offset, @sizeOf(u32));
854 const size = readAt(u32, self.bytes, offset);
855 const payload_offset = try self.advance(offset, @sizeOf(u32));
856 return self.advance(payload_offset, size);
857 },
858 .branch_stack, .regs, .stack => return error.UnsupportedSampleField,
859 _ => return error.UnsupportedSampleField,
860 }
861 }
862
863 fn skipReadField(self: Record, offset: usize) !usize {
864 var cursor = offset;
865 if (self.config.read_format & ReadFormat.group != 0) {
866 const count = try self.readCountedLength(cursor);
867 cursor = try self.advance(cursor, @sizeOf(u64));
868 if (self.config.read_format & ReadFormat.total_time_enabled != 0) cursor = try self.advance(cursor, @sizeOf(u64));
869 if (self.config.read_format & ReadFormat.total_time_running != 0) cursor = try self.advance(cursor, @sizeOf(u64));
870 var member_fields: usize = 1;
871 if (self.config.read_format & ReadFormat.id != 0) member_fields += 1;
872 if (self.config.read_format & ReadFormat.lost != 0) member_fields += 1;
873 std.debug.assert(member_fields <= 3);
874 const member_size = member_fields * @sizeOf(u64);
875 const members_size = try std.math.mul(usize, count, member_size);
876 return self.advance(cursor, members_size);
877 }
878
879 cursor = try self.advance(cursor, @sizeOf(u64));
880 if (self.config.read_format & ReadFormat.total_time_enabled != 0) cursor = try self.advance(cursor, @sizeOf(u64));
881 if (self.config.read_format & ReadFormat.total_time_running != 0) cursor = try self.advance(cursor, @sizeOf(u64));
882 if (self.config.read_format & ReadFormat.id != 0) cursor = try self.advance(cursor, @sizeOf(u64));
883 if (self.config.read_format & ReadFormat.lost != 0) {
884 cursor = try self.advance(cursor, @sizeOf(u64));
885 }
886 return cursor;
887 }
888
889 fn readCountedLength(self: Record, offset: usize) !usize {
890 try self.requireRange(offset, @sizeOf(u64));
891 return std.math.cast(usize, readAt(u64, self.bytes, offset)) orelse error.RecordFieldTooLarge;
892 }
893
894 fn advance(self: Record, offset: usize, size: usize) !usize {
895 try self.requireRange(offset, size);
896 return offset + size;
897 }
898
899 fn requireRange(self: Record, offset: usize, size: usize) !void {
900 const end = std.math.add(usize, offset, size) catch return error.TruncatedRecord;
901 if (end > self.bytes.len) return error.TruncatedRecord;
902 }
903 };
904
905 pub const RingReader = struct {
906 config: Config,
907 data: []const u8,
908 index: u64,
909 head: u64,
910
911 pub fn init(config: Config, data: []const u8, tail: u64, head: u64) RingReader {
912 return .{ .config = config, .data = data, .index = tail, .head = head };
913 }
914
915 pub fn hasData(self: RingReader) !bool {
916 if (self.head < self.index) return error.RingHeadBeforeTail;
917 const available = self.head - self.index;
918 if (available > self.data.len) return error.RingWindowExceedsBuffer;
919 if (available < header_size) return false;
920
921 var header_bytes: [header_size]u8 = undefined;
922 try copyFromRingBuffer(self.data, self.index, &header_bytes);
923 const header = try Header.parse(&header_bytes);
924 if (header.size < header_size) return error.InvalidRecordSize;
925
926 const record_limit = std.math.add(u64, self.index, header.size) catch return error.RingIndexOverflow;
927 return record_limit <= self.head;
928 }
929
930 pub fn next(self: *RingReader, scratch: []u8) !?Record {
931 if (!try self.hasData()) return null;
932
933 var header_bytes: [header_size]u8 = undefined;
934 try copyFromRingBuffer(self.data, self.index, &header_bytes);
935 const header = try Header.parse(&header_bytes);
936 if (header.size > scratch.len) return error.ScratchTooSmall;
937
938 try copyFromRingBuffer(self.data, self.index, scratch[0..header.size]);
939 self.index += header.size;
940 return try Record.init(self.config, scratch[0..header.size]);
941 }
942 };
943
944 pub fn sampleMask(fields: []const SampleField) u64 {
945 var result: u64 = 0;
946 for (fields) |field| result |= sampleBit(field);
947 return result;
948 }
949
950 pub fn copyFromRingBuffer(data: []const u8, index: u64, dest: []u8) !void {
951 if (dest.len == 0) return;
952 if (data.len == 0) return error.EmptyRingBuffer;
953 if (dest.len > data.len) return error.RecordLargerThanRing;
954
955 const ring_len: u64 = @intCast(data.len);
956 const start: usize = @intCast(index % ring_len);
957 const end = start + dest.len;
958 if (end <= data.len) {
959 @memcpy(dest, data[start..end]);
960 return;
961 }
962
963 const first_len = data.len - start;
964 const second_len = dest.len - first_len;
965 @memcpy(dest[0..first_len], data[start..]);
966 @memcpy(dest[first_len..], data[0..second_len]);
967 }
968
969 const field_order = [_]SampleField{
970 .ip,
971 .pid_tid,
972 .time,
973 .addr,
974 .id,
975 .stream_id,
976 .cpu,
977 .period,
978 .read,
979 .callchain,
980 .raw,
981 .branch_stack,
982 .regs,
983 .stack,
984 };
985
986 fn sampleBit(field: SampleField) u64 {
987 return @backingInt(field);
988 }
989
990 fn perfEventOpenLinux(
991 attr: *linux.perf_event_attr,
992 pid: posix.pid_t,
993 cpu: i32,
994 group_fd: posix.fd_t,
995 flags: usize,
996 ) !posix.fd_t {
997 const rc = linux.perf_event_open(attr, pid, cpu, group_fd, flags);
998 switch (linux.errno(rc)) {
999 .SUCCESS => return @intCast(rc),
1000 .@"2BIG" => return error.TooBig,
1001 .ACCES => return error.PermissionDenied,
1002 .BADF,
1003 .FAULT,
1004 .INTR,
1005 .NOENT,
1006 => unreachable,
1007 .INVAL => return error.InvalidPerfEventOptions,
1008 .BUSY => return error.DeviceBusy,
1009 .MFILE => return error.ProcessResources,
1010 .NODEV => return error.EventRequiresUnsupportedCpuFeature,
1011 .NOSPC => return error.TooManyBreakpoints,
1012 .NOSYS => return error.SampleStackNotSupported,
1013 .OPNOTSUPP => return error.EventNotSupported,
1014 .OVERFLOW => return error.SampleMaxStackOverflow,
1015 .PERM => return error.PermissionDenied,
1016 .SRCH => return error.ProcessNotFound,
1017 else => |err| return posix.unexpectedErrno(err),
1018 }
1019 }
1020
1021 fn control(fd: posix.fd_t, request: u32, argument: usize) !void {
1022 if (comptime !supported) return error.UnsupportedPlatform;
1023 while (true) {
1024 const rc = linux.ioctl(fd, request, argument);
1025 switch (linux.errno(rc)) {
1026 .SUCCESS => return,
1027 .INTR => continue,
1028 .BADF, .INVAL => return error.InvalidPerfEvent,
1029 .ACCES, .PERM => return error.PermissionDenied,
1030 else => return error.PerfEventControlFailed,
1031 }
1032 }
1033 }
1034
1035 fn eventId(fd: posix.fd_t) !u64 {
1036 if (comptime !supported) return error.UnsupportedPlatform;
1037 var id: u64 = 0;
1038 while (true) {
1039 const rc = linux.ioctl(fd, linux.PERF.EVENT_IOC.ID, @intFromPtr(&id));
1040 switch (linux.errno(rc)) {
1041 .SUCCESS => return id,
1042 .INTR => continue,
1043 .BADF, .INVAL => return error.InvalidPerfEvent,
1044 .ACCES, .PERM => return error.PermissionDenied,
1045 else => return error.PerfEventIdFailed,
1046 }
1047 }
1048 }
1049
1050 fn closeFd(fd: posix.fd_t) void {
1051 if (comptime !supported) return;
1052 switch (linux.errno(linux.close(fd))) {
1053 .SUCCESS, .INTR => {},
1054 else => unreachable,
1055 }
1056 }
1057
1058 fn mappedPage(mapped: []align(page_size) u8) *linux.perf_event_mmap_page {
1059 return @ptrCast(mapped.ptr);
1060 }
1061
1062 fn mappedData(mapped: []align(page_size) u8, actual_page_size: usize) []const u8 {
1063 return mapped[actual_page_size..][0..dataSizeForPageSize(actual_page_size)];
1064 }
1065
1066 fn dataSizeForPageSize(actual_page_size: usize) usize {
1067 std.debug.assert(actual_page_size >= std.heap.page_size_min);
1068 std.debug.assert(actual_page_size <= std.heap.page_size_max);
1069 return data_pages * actual_page_size;
1070 }
1071
1072 fn mmapSizeForPageSize(actual_page_size: usize) usize {
1073 const mapped_data_size = dataSizeForPageSize(actual_page_size);
1074 std.debug.assert(
1075 mapped_data_size <= std.math.maxInt(usize) - actual_page_size,
1076 );
1077 return mapped_data_size + actual_page_size;
1078 }
1079
1080 fn readAt(comptime T: type, bytes: []const u8, offset: usize) T {
1081 std.debug.assert(@sizeOf(T) <= bytes.len);
1082 std.debug.assert(offset <= bytes.len - @sizeOf(T));
1083 return std.mem.readInt(T, bytes[offset..][0..@sizeOf(T)], native_endian);
1084 }
1085
1086 fn readCountField(bytes: []const u8, offset: *usize) !u64 {
1087 if (offset.* > bytes.len) return error.ShortPerfEventRead;
1088 if (@sizeOf(u64) > bytes.len - offset.*) return error.ShortPerfEventRead;
1089 const value = readAt(u64, bytes, offset.*);
1090 offset.* += @sizeOf(u64);
1091 return value;
1092 }
1093
1094 fn countReadSize(read_format: u64) ?usize {
1095 if (read_format & ReadFormat.group != 0) return null;
1096 var result: usize = @sizeOf(u64);
1097 if (read_format & ReadFormat.total_time_enabled != 0) result += @sizeOf(u64);
1098 if (read_format & ReadFormat.total_time_running != 0) result += @sizeOf(u64);
1099 if (read_format & ReadFormat.id != 0) result += @sizeOf(u64);
1100 if (read_format & ReadFormat.lost != 0) result += @sizeOf(u64);
1101 return result;
1102 }
1103
1104 fn counterSetReadCapacity(comptime capacity: usize) usize {
1105 return @sizeOf(u64) * (3 + 3 * capacity);
1106 }
1107
1108 fn counterSetReadSize(read_format: u64, count: usize) !usize {
1109 if (read_format & ReadFormat.group == 0) return error.CounterSetGroupFormatRequired;
1110 var header_fields: usize = 1;
1111 if (read_format & ReadFormat.total_time_enabled != 0) header_fields += 1;
1112 if (read_format & ReadFormat.total_time_running != 0) header_fields += 1;
1113 var member_fields: usize = 1;
1114 if (read_format & ReadFormat.id != 0) member_fields += 1;
1115 if (read_format & ReadFormat.lost != 0) member_fields += 1;
1116 const member_total = try std.math.mul(usize, member_fields, count);
1117 const field_total = try std.math.add(usize, header_fields, member_total);
1118 return try std.math.mul(usize, field_total, @sizeOf(u64));
1119 }
1120
1121 fn writeAt(comptime T: type, bytes: []u8, offset: *usize, value: T) void {
1122 std.mem.writeInt(T, bytes[offset.*..][0..@sizeOf(T)], value, native_endian);
1123 offset.* += @sizeOf(T);
1124 }
1125
1126 fn finishRecord(bytes: []u8, len: usize, record_type: RecordType) []const u8 {
1127 std.mem.writeInt(u32, bytes[0..4], @backingInt(record_type), native_endian);
1128 std.mem.writeInt(u16, bytes[4..6], 0, native_endian);
1129 std.mem.writeInt(u16, bytes[6..8], @intCast(len), native_endian);
1130 return bytes[0..len];
1131 }
1132
1133 fn finishSample(bytes: []u8, len: usize) []const u8 {
1134 return finishRecord(bytes, len, .sample);
1135 }
1136
1137 fn writeRing(data: []u8, index: usize, bytes: []const u8) void {
1138 for (bytes, 0..) |byte, offset| {
1139 data[(index + offset) % data.len] = byte;
1140 }
1141 }
1142
1143 test "task clock sampler attributes match upstream linux configuration" {
1144 const attr = taskClockSamplerAttr(.{});
1145
1146 try std.testing.expectEqual(linux.PERF.TYPE.SOFTWARE, attr.type);
1147 try std.testing.expectEqual(@as(u64, @backingInt(linux.PERF.COUNT.SW.TASK_CLOCK)), attr.config);
1148 try std.testing.expectEqual(@as(u64, 1_000_000), attr.sample_period_or_freq);
1149 try std.testing.expectEqual(default_sample_type, attr.sample_type);
1150 try std.testing.expectEqual(@as(u32, 10), attr.wakeup_events_or_watermark);
1151 try std.testing.expect(attr.flags.disabled);
1152 try std.testing.expect(attr.flags.exclude_idle);
1153 try std.testing.expect(attr.flags.exclude_kernel);
1154 }
1155
1156 test "counter selectors configure typed perf events" {
1157 const hardware = (CounterSelector{ .hardware = .instructions }).attr(.{ .read_format = ReadFormat.total_time_enabled });
1158 try std.testing.expectEqual(linux.PERF.TYPE.HARDWARE, hardware.type);
1159 try std.testing.expectEqual(@as(u64, @backingInt(linux.PERF.COUNT.HW.INSTRUCTIONS)), hardware.config);
1160 try std.testing.expectEqual(ReadFormat.total_time_enabled, hardware.read_format);
1161 try std.testing.expect(hardware.flags.disabled);
1162 try std.testing.expect(hardware.flags.exclude_kernel);
1163 try std.testing.expect(hardware.flags.exclude_hv);
1164
1165 const software = (CounterSelector{ .software = .context_switches }).attr(.{ .exclude_kernel = false, .exclude_hv = false, .inherit = true });
1166 try std.testing.expectEqual(linux.PERF.TYPE.SOFTWARE, software.type);
1167 try std.testing.expectEqual(@as(u64, @backingInt(linux.PERF.COUNT.SW.CONTEXT_SWITCHES)), software.config);
1168 try std.testing.expect(!software.flags.exclude_kernel);
1169 try std.testing.expect(!software.flags.exclude_hv);
1170 try std.testing.expect(software.flags.inherit);
1171
1172 const cache = (CounterSelector{ .cache = .{ .cache = .l1_data, .operation = .read, .result = .miss } }).attr(.{});
1173 try std.testing.expectEqual(linux.PERF.TYPE.HW_CACHE, cache.type);
1174 try std.testing.expectEqual(@as(u64, @backingInt(linux.PERF.COUNT.HW.CACHE.L1D)) |
1175 (@as(u64, @backingInt(linux.PERF.COUNT.HW.CACHE.OP.READ)) << 8) |
1176 (@as(u64, @backingInt(linux.PERF.COUNT.HW.CACHE.RESULT.MISS)) << 16), cache.config);
1177 }
1178
1179 test "count result parses optional timing id and loss fields" {
1180 var bytes: [40]u8 = undefined;
1181 var offset: usize = 0;
1182 writeAt(u64, &bytes, &offset, 10);
1183 writeAt(u64, &bytes, &offset, 100);
1184 writeAt(u64, &bytes, &offset, 50);
1185 writeAt(u64, &bytes, &offset, 7);
1186 writeAt(u64, &bytes, &offset, 3);
1187
1188 const read_format = ReadFormat.total_time_enabled | ReadFormat.total_time_running |
1189 ReadFormat.id | ReadFormat.lost;
1190 const result = try CountResult.parse(bytes[0..offset], read_format);
1191
1192 try std.testing.expectEqual(@as(u64, 10), result.value);
1193 try std.testing.expectEqual(@as(?u64, 100), result.time_enabled);
1194 try std.testing.expectEqual(@as(?u64, 50), result.time_running);
1195 try std.testing.expectEqual(@as(?u64, 7), result.id);
1196 try std.testing.expectEqual(@as(?u64, 3), result.lost);
1197 try std.testing.expectEqual(CountCoverage.multiplexed, result.runningCoverage());
1198 try std.testing.expectEqual(@as(?u64, 20), result.scaledValue());
1199 try std.testing.expectEqual(@as(?f64, 0.5), result.runningRatio());
1200 }
1201
1202 fn expectCountEvidence(
1203 result: CountResult,
1204 coverage: CountCoverage,
1205 scaled: ?u64,
1206 running_ratio: ?f64,
1207 ) !void {
1208 try std.testing.expectEqual(coverage, result.runningCoverage());
1209 try std.testing.expectEqual(scaled, result.scaledValue());
1210 try std.testing.expectEqual(running_ratio, result.runningRatio());
1211 }
1212
1213 test "count result classifies scheduling coverage" {
1214 try expectCountEvidence(.{ .value = 10 }, .unknown, 10, null);
1215 try expectCountEvidence(
1216 .{ .value = 10, .time_enabled = 100, .time_running = 100 },
1217 .complete,
1218 10,
1219 1.0,
1220 );
1221 try expectCountEvidence(
1222 .{ .value = 10, .time_enabled = 100, .time_running = 50 },
1223 .multiplexed,
1224 20,
1225 0.5,
1226 );
1227 try expectCountEvidence(
1228 .{ .value = 10, .time_enabled = 100, .time_running = 0 },
1229 .not_counted,
1230 null,
1231 0.0,
1232 );
1233 try expectCountEvidence(
1234 .{ .value = 10, .time_enabled = 0, .time_running = 0 },
1235 .not_counted,
1236 null,
1237 null,
1238 );
1239 try expectCountEvidence(
1240 .{ .value = 10, .time_enabled = 100 },
1241 .invalid,
1242 null,
1243 null,
1244 );
1245 try expectCountEvidence(.{ .value = 10, .time_running = 50 }, .invalid, null, null);
1246 try expectCountEvidence(
1247 .{ .value = 10, .time_enabled = 50, .time_running = 100 },
1248 .invalid,
1249 null,
1250 null,
1251 );
1252 }
1253
1254 const counter_set_test_format = ReadFormat.group | ReadFormat.total_time_enabled |
1255 ReadFormat.total_time_running | ReadFormat.id | ReadFormat.lost;
1256
1257 fn writeCounterSetTestRow(
1258 bytes: []u8,
1259 offset: *usize,
1260 value: u64,
1261 id: u64,
1262 lost: u64,
1263 ) void {
1264 writeAt(u64, bytes, offset, value);
1265 writeAt(u64, bytes, offset, id);
1266 writeAt(u64, bytes, offset, lost);
1267 }
1268
1269 fn counterSetTestBytes(bytes: []u8, first_id: u64, second_id: u64) []const u8 {
1270 var offset: usize = 0;
1271 writeAt(u64, bytes, &offset, 2);
1272 writeAt(u64, bytes, &offset, 100);
1273 writeAt(u64, bytes, &offset, 50);
1274 writeCounterSetTestRow(bytes, &offset, 20, first_id, 2);
1275 writeCounterSetTestRow(bytes, &offset, 10, second_id, 1);
1276 return bytes[0..offset];
1277 }
1278
1279 test "counter set result matches ids and shared timing evidence" {
1280 const selectors = [_]CounterSelector{
1281 .{ .hardware = .instructions },
1282 .{ .hardware = .cycles },
1283 };
1284 const ids = [_]u64{ 11, 22 };
1285 var bytes: [counterSetReadCapacity(2)]u8 = undefined;
1286 const encoded = counterSetTestBytes(&bytes, ids[1], ids[0]);
1287 try std.testing.expectEqual(
1288 encoded.len,
1289 try counterSetReadSize(counter_set_test_format, selectors.len),
1290 );
1291 const result = try CounterSetResult(2).parse(
1292 encoded,
1293 counter_set_test_format,
1294 &selectors,
1295 &ids,
1296 );
1297
1298 try std.testing.expectEqual(@as(usize, 2), result.slice().len);
1299 try std.testing.expectEqualDeep(selectors[0], result.slice()[0].selector);
1300 try std.testing.expectEqual(@as(u64, 10), result.slice()[0].result.value);
1301 try std.testing.expectEqual(@as(?u64, 20), result.slice()[0].result.scaledValue());
1302 try std.testing.expectEqual(
1303 CountCoverage.multiplexed,
1304 result.slice()[0].result.runningCoverage(),
1305 );
1306 try std.testing.expectEqual(@as(?u64, 1), result.slice()[0].result.lost);
1307 try std.testing.expectEqual(@as(u64, 20), result.find(selectors[1]).?.value);
1308 try std.testing.expect(result.find(.{ .hardware = .cache_misses }) == null);
1309 }
1310
1311 test "counter set result rejects malformed membership" {
1312 const selectors = [_]CounterSelector{
1313 .{ .hardware = .instructions },
1314 .{ .hardware = .cycles },
1315 };
1316 const ids = [_]u64{ 11, 22 };
1317 var bytes: [counterSetReadCapacity(2) + @sizeOf(u64)]u8 = undefined;
1318 const encoded = counterSetTestBytes(&bytes, ids[0], 99);
1319
1320 try std.testing.expectError(
1321 error.UnknownCounterSetId,
1322 CounterSetResult(2).parse(encoded, counter_set_test_format, &selectors, &ids),
1323 );
1324 const duplicate = counterSetTestBytes(&bytes, ids[0], ids[0]);
1325 try std.testing.expectError(
1326 error.DuplicateCounterSetId,
1327 CounterSetResult(2).parse(duplicate, counter_set_test_format, &selectors, &ids),
1328 );
1329 try std.testing.expectError(
1330 error.DuplicateExpectedCounterSetId,
1331 CounterSetResult(2).parse(duplicate, counter_set_test_format, &selectors, &.{ 11, 11 }),
1332 );
1333 }
1334
1335 test "counter set result rejects malformed layout" {
1336 const selectors = [_]CounterSelector{
1337 .{ .hardware = .instructions },
1338 .{ .hardware = .cycles },
1339 };
1340 const ids = [_]u64{ 11, 22 };
1341 var bytes: [counterSetReadCapacity(2) + @sizeOf(u64)]u8 = undefined;
1342 const encoded = counterSetTestBytes(&bytes, ids[0], ids[1]);
1343
1344 try std.testing.expectError(
1345 error.ShortPerfEventRead,
1346 CounterSetResult(2).parse(
1347 encoded[0 .. encoded.len - 1],
1348 counter_set_test_format,
1349 &selectors,
1350 &ids,
1351 ),
1352 );
1353 var trailing_offset = encoded.len;
1354 writeAt(u64, &bytes, &trailing_offset, 0);
1355 try std.testing.expectError(
1356 error.TrailingPerfEventReadData,
1357 CounterSetResult(2).parse(
1358 bytes[0 .. encoded.len + 8],
1359 counter_set_test_format,
1360 &selectors,
1361 &ids,
1362 ),
1363 );
1364 var count_mismatch = bytes;
1365 std.mem.writeInt(u64, count_mismatch[0..8], 1, native_endian);
1366 try std.testing.expectError(
1367 error.CounterSetCountMismatch,
1368 CounterSetResult(2).parse(&count_mismatch, counter_set_test_format, &selectors, &ids),
1369 );
1370 }
1371
1372 test "counter set result requires bounded identified group input" {
1373 const selectors = [_]CounterSelector{
1374 .{ .hardware = .instructions },
1375 .{ .hardware = .cycles },
1376 };
1377 const ids = [_]u64{ 11, 22 };
1378 var bytes: [counterSetReadCapacity(2)]u8 = undefined;
1379 const encoded = counterSetTestBytes(&bytes, ids[0], ids[1]);
1380
1381 try std.testing.expectError(
1382 error.CounterSetGroupFormatRequired,
1383 CounterSetResult(2).parse(
1384 encoded,
1385 counter_set_test_format & ~ReadFormat.group,
1386 &selectors,
1387 &ids,
1388 ),
1389 );
1390 try std.testing.expectError(
1391 error.CounterSetIdFormatRequired,
1392 CounterSetResult(2).parse(
1393 encoded,
1394 counter_set_test_format & ~ReadFormat.id,
1395 &selectors,
1396 &ids,
1397 ),
1398 );
1399 try std.testing.expectError(
1400 error.CounterSetIdentityCountMismatch,
1401 CounterSetResult(2).parse(encoded, counter_set_test_format, &selectors, ids[0..1]),
1402 );
1403 try std.testing.expectError(
1404 error.CounterSetCapacityExceeded,
1405 CounterSetResult(1).parse(encoded, counter_set_test_format, &selectors, &ids),
1406 );
1407 try std.testing.expectError(
1408 error.DuplicateCounterSetSelector,
1409 CounterSetResult(2).parse(
1410 encoded,
1411 counter_set_test_format,
1412 &.{ selectors[0], selectors[0] },
1413 &ids,
1414 ),
1415 );
1416 }
1417
1418 test "counter set region validates static bounds before opening events" {
1419 const Region = CounterSetRegion(2);
1420 const selectors = [_]CounterSelector{
1421 .{ .hardware = .instructions },
1422 .{ .hardware = .cycles },
1423 .{ .hardware = .cache_misses },
1424 };
1425 try std.testing.expectError(error.EmptyCounterSet, Region.start(&.{}, .{}));
1426 try std.testing.expectError(error.CounterSetCapacityExceeded, Region.start(&selectors, .{}));
1427 try std.testing.expectError(
1428 error.InheritedCounterSetUnsupported,
1429 Region.start(selectors[0..1], .{ .inherit = true }),
1430 );
1431 try std.testing.expectError(
1432 error.DuplicateCounterSetSelector,
1433 Region.start(&.{ selectors[0], selectors[0] }, .{}),
1434 );
1435
1436 var region: Region = .{};
1437 try std.testing.expectError(error.CounterSetRegionStopped, region.stop());
1438 region.deinit();
1439 }
1440
1441 test "count result rejects group and trailing read formats" {
1442 var bytes: [16]u8 = undefined;
1443 var offset: usize = 0;
1444 writeAt(u64, &bytes, &offset, 1);
1445 writeAt(u64, &bytes, &offset, 2);
1446
1447 try std.testing.expectError(error.UnsupportedReadFormat, CountResult.parse(bytes[0..8], ReadFormat.group));
1448 try std.testing.expectError(error.TrailingPerfEventReadData, CountResult.parse(bytes[0..offset], 0));
1449 try std.testing.expectEqual(@as(?usize, null), countReadSize(ReadFormat.group));
1450 }
1451
1452 test "counter region rejects inactive stop" {
1453 var region: CounterRegion = .{};
1454
1455 try std.testing.expectError(error.CounterRegionStopped, region.stop());
1456 region.deinit();
1457 }
1458
1459 test "perf event close is idempotent without owned resources" {
1460 var event: Event = .{};
1461
1462 event.close();
1463 event.close();
1464
1465 try std.testing.expectEqual(invalid_fd, event.fd);
1466 try std.testing.expect(event.mapping == null);
1467 try std.testing.expectEqual(@as(u64, 0), event.config.sample_type);
1468 }
1469
1470 test "perf event ring reader views mapped data page and commits tail" {
1471 const mapped = try std.testing.allocator.alignedAlloc(u8, .fromByteUnits(page_size), mmap_size);
1472 defer std.testing.allocator.free(mapped);
1473 @memset(mapped, 0);
1474
1475 const config: Config = .{ .sample_type = sampleMask(&.{.ip}) };
1476 var event: Event = .{ .mapping = mapped, .config = config };
1477 var bytes: [32]u8 = undefined;
1478 var offset: usize = header_size;
1479
1480 writeAt(u64, &bytes, &offset, 0x1010);
1481 const sample = finishSample(&bytes, offset);
1482 writeRing(mapped[page_size..][0..data_size], 0, sample);
1483
1484 const page = mappedPage(mapped);
1485 @atomicStore(u64, &page.data_head, @intCast(sample.len), .release);
1486 @atomicStore(u64, &page.data_tail, 0, .release);
1487
1488 var reader = event.ringReader().?;
1489 var scratch: [64]u8 = undefined;
1490 const record = (try reader.next(&scratch)).?;
1491 event.commitReader(reader);
1492
1493 try std.testing.expectEqual(@as(u64, 0x1010), try record.getIp());
1494 try std.testing.expectEqual(@as(u64, @intCast(sample.len)), page.data_tail);
1495 }
1496
1497 test "perf mapping sizes cover runtime page bounds" {
1498 inline for ([_]usize{
1499 std.heap.page_size_min,
1500 std.heap.page_size_max,
1501 }) |actual_page_size| {
1502 try std.testing.expectEqual(
1503 data_pages * actual_page_size,
1504 dataSizeForPageSize(actual_page_size),
1505 );
1506 try std.testing.expectEqual(
1507 (data_pages + 1) * actual_page_size,
1508 mmapSizeForPageSize(actual_page_size),
1509 );
1510 }
1511 }
1512
1513 test "record parses fixed sample fields in perf order" {
1514 const config: Config = .{ .sample_type = sampleMask(&.{ .ip, .pid_tid, .time, .cpu, .period }) };
1515 var bytes: [128]u8 = undefined;
1516 var offset: usize = header_size;
1517
1518 writeAt(u64, &bytes, &offset, 0x1000_0010);
1519 writeAt(u32, &bytes, &offset, 1234);
1520 writeAt(u32, &bytes, &offset, 5678);
1521 writeAt(u64, &bytes, &offset, 99);
1522 writeAt(u32, &bytes, &offset, 7);
1523 writeAt(u32, &bytes, &offset, 0);
1524 writeAt(u64, &bytes, &offset, 1000);
1525
1526 const record = try Record.init(config, finishSample(&bytes, offset));
1527
1528 try std.testing.expect(record.isSample());
1529 try std.testing.expectEqual(RecordType.sample, record.recordType());
1530 try std.testing.expectEqual(@as(u64, 0x1000_0010), try record.getIp());
1531 try std.testing.expectEqual(@as(u32, 1234), try record.getPid());
1532 try std.testing.expectEqual(@as(u32, 5678), try record.getTid());
1533 try std.testing.expectEqual(@as(u64, 99), try record.getTime());
1534 try std.testing.expectEqual(@as(u32, 7), try record.getCpu());
1535 try std.testing.expectEqual(@as(u64, 1000), try record.getPeriod());
1536 try std.testing.expectEqual(error.FieldNotSampled, record.getCallchain());
1537 }
1538
1539 test "record rejects truncated paired sample fields" {
1540 var bytes: [header_size + @sizeOf(u32)]u8 = undefined;
1541 var offset: usize = header_size;
1542 writeAt(u32, &bytes, &offset, 1234);
1543 const sample = finishSample(&bytes, offset);
1544 const pid_tid = try Record.init(
1545 .{ .sample_type = sampleMask(&.{.pid_tid}) },
1546 sample,
1547 );
1548 const cpu = try Record.init(
1549 .{ .sample_type = sampleMask(&.{.cpu}) },
1550 sample,
1551 );
1552
1553 try std.testing.expectError(error.TruncatedRecord, pid_tid.getPid());
1554 try std.testing.expectError(error.TruncatedRecord, pid_tid.getTid());
1555 try std.testing.expectError(error.TruncatedRecord, cpu.getCpu());
1556 }
1557
1558 test "loss records parse kernel loss counts" {
1559 var lost_bytes: [32]u8 = undefined;
1560 var lost_offset: usize = header_size;
1561 writeAt(u64, &lost_bytes, &lost_offset, 17);
1562 writeAt(u64, &lost_bytes, &lost_offset, 29);
1563 const lost = try Record.init(.{}, finishRecord(&lost_bytes, lost_offset, .lost));
1564
1565 var sample_bytes: [24]u8 = undefined;
1566 var sample_offset: usize = header_size;
1567 writeAt(u64, &sample_bytes, &sample_offset, 31);
1568 const lost_samples = try Record.init(
1569 .{},
1570 finishRecord(&sample_bytes, sample_offset, .lost_samples),
1571 );
1572
1573 try std.testing.expectEqual(@as(u64, 29), try lost.getLostCount());
1574 try std.testing.expectEqual(@as(u64, 31), try lost_samples.getLostCount());
1575 const sample = try Record.init(.{}, finishSample(
1576 &sample_bytes,
1577 header_size,
1578 ));
1579 try std.testing.expectError(error.NotLostRecord, sample.getLostCount());
1580 }
1581
1582 test "loss records reject truncated payloads" {
1583 var bytes: [header_size]u8 = undefined;
1584 const record = try Record.init(.{}, finishRecord(&bytes, header_size, .lost_samples));
1585
1586 try std.testing.expectError(error.TruncatedRecord, record.getLostCount());
1587 }
1588
1589 test "record locates callchain and raw payload after read format" {
1590 const config: Config = .{
1591 .sample_type = sampleMask(&.{ .read, .callchain, .raw }),
1592 .read_format = ReadFormat.group | ReadFormat.total_time_enabled |
1593 ReadFormat.total_time_running | ReadFormat.id | ReadFormat.lost,
1594 };
1595 var bytes: [256]u8 = undefined;
1596 var offset: usize = header_size;
1597
1598 writeAt(u64, &bytes, &offset, 2);
1599 writeAt(u64, &bytes, &offset, 500);
1600 writeAt(u64, &bytes, &offset, 400);
1601 writeAt(u64, &bytes, &offset, 10);
1602 writeAt(u64, &bytes, &offset, 100);
1603 writeAt(u64, &bytes, &offset, 1);
1604 writeAt(u64, &bytes, &offset, 20);
1605 writeAt(u64, &bytes, &offset, 200);
1606 writeAt(u64, &bytes, &offset, 2);
1607 writeAt(u64, &bytes, &offset, 3);
1608 writeAt(u64, &bytes, &offset, 0x111);
1609 writeAt(u64, &bytes, &offset, 0x222);
1610 writeAt(u64, &bytes, &offset, 0x333);
1611 writeAt(u32, &bytes, &offset, 3);
1612 @memcpy(bytes[offset..][0..3], "abc");
1613 offset += 3;
1614
1615 const record = try Record.init(config, finishSample(&bytes, offset));
1616 const callchain = try record.getCallchain();
1617 const raw = try record.getRaw();
1618
1619 try std.testing.expectEqual(@as(usize, 3), callchain.len());
1620 try std.testing.expectEqual(@as(u64, 0x111), callchain.at(0));
1621 try std.testing.expectEqual(@as(u64, 0x222), callchain.at(1));
1622 try std.testing.expectEqual(@as(u64, 0x333), callchain.at(2));
1623 try std.testing.expectEqualStrings("abc", raw);
1624 try std.testing.expectEqual(offset, try record.payloadEndOffset());
1625 }
1626
1627 test "record rejects truncated variable fields" {
1628 const config: Config = .{ .sample_type = sampleMask(&.{.callchain}) };
1629 var bytes: [32]u8 = undefined;
1630 var offset: usize = header_size;
1631
1632 writeAt(u64, &bytes, &offset, 2);
1633 writeAt(u64, &bytes, &offset, 0x111);
1634
1635 const record = try Record.init(config, finishSample(&bytes, offset));
1636
1637 try std.testing.expectEqual(error.TruncatedRecord, record.getCallchain());
1638 }
1639
1640 test "record bounds counted callchain layout arithmetic" {
1641 const config: Config = .{ .sample_type = sampleMask(&.{.callchain}) };
1642 var bytes: [header_size + @sizeOf(u64)]u8 = undefined;
1643 const count_bytes = bytes[header_size..][0..@sizeOf(u64)];
1644
1645 std.mem.writeInt(u64, count_bytes, 0, native_endian);
1646 const zero = try Record.init(config, finishSample(&bytes, bytes.len));
1647 try std.testing.expectEqual(@as(usize, 0), (try zero.getCallchain()).len());
1648 try std.testing.expectEqual(bytes.len, try zero.payloadEndOffset());
1649
1650 const max_count = std.math.maxInt(usize) / @sizeOf(u64);
1651 std.mem.writeInt(u64, count_bytes, @intCast(max_count), native_endian);
1652 const maximal = try Record.init(config, finishSample(&bytes, bytes.len));
1653 try std.testing.expectError(
1654 error.TruncatedRecord,
1655 maximal.payloadEndOffset(),
1656 );
1657
1658 std.mem.writeInt(u64, count_bytes, @intCast(max_count + 1), native_endian);
1659 const excess = try Record.init(config, finishSample(&bytes, bytes.len));
1660 try std.testing.expectError(error.Overflow, excess.payloadEndOffset());
1661 }
1662
1663 test "record raw fields validate size before payload access" {
1664 const config: Config = .{ .sample_type = sampleMask(&.{.raw}) };
1665 var bytes: [header_size + @sizeOf(u32)]u8 = undefined;
1666
1667 const missing = try Record.init(
1668 config,
1669 finishSample(&bytes, header_size),
1670 );
1671 try std.testing.expectError(error.TruncatedRecord, missing.getRaw());
1672
1673 const size_bytes = bytes[header_size..][0..@sizeOf(u32)];
1674 std.mem.writeInt(u32, size_bytes, 0, native_endian);
1675 const zero = try Record.init(config, finishSample(&bytes, bytes.len));
1676 try std.testing.expectEqual(@as(usize, 0), (try zero.getRaw()).len);
1677 try std.testing.expectEqual(bytes.len, try zero.payloadEndOffset());
1678
1679 std.mem.writeInt(u32, size_bytes, std.math.maxInt(u32), native_endian);
1680 const maximal = try Record.init(config, finishSample(&bytes, bytes.len));
1681 try std.testing.expectError(error.TruncatedRecord, maximal.getRaw());
1682 try std.testing.expectError(
1683 error.TruncatedRecord,
1684 maximal.payloadEndOffset(),
1685 );
1686 }
1687
1688 test "count fields reject maximal offsets before arithmetic" {
1689 const bytes: [0]u8 = .{};
1690 var offset: usize = std.math.maxInt(usize);
1691
1692 try std.testing.expectError(
1693 error.ShortPerfEventRead,
1694 readCountField(&bytes, &offset),
1695 );
1696 try std.testing.expectEqual(std.math.maxInt(usize), offset);
1697 }
1698
1699 test "record rejects unsupported sample fields when validating full payload" {
1700 const config: Config = .{ .sample_type = sampleMask(&.{.branch_stack}) };
1701 var bytes: [header_size]u8 = undefined;
1702 const record = try Record.init(config, finishSample(&bytes, header_size));
1703
1704 try std.testing.expectEqual(error.UnsupportedSampleField, record.payloadEndOffset());
1705 }
1706
1707 test "ring copy wraps across the data buffer boundary" {
1708 const data = "efghabcd";
1709 var dest: [6]u8 = undefined;
1710
1711 try copyFromRingBuffer(data, 6, &dest);
1712
1713 try std.testing.expectEqualStrings("cdefgh", &dest);
1714 }
1715
1716 test "ring reader returns wrapped sample records" {
1717 const config: Config = .{ .sample_type = sampleMask(&.{ .ip, .callchain }) };
1718 var record_bytes: [128]u8 = undefined;
1719 var offset: usize = header_size;
1720
1721 writeAt(u64, &record_bytes, &offset, 0x4444);
1722 writeAt(u64, &record_bytes, &offset, 2);
1723 writeAt(u64, &record_bytes, &offset, 0x5555);
1724 writeAt(u64, &record_bytes, &offset, 0x6666);
1725 const record_slice = finishSample(&record_bytes, offset);
1726
1727 var ring_data: [64]u8 = undefined;
1728 @memset(&ring_data, 0);
1729 const tail: u64 = 52;
1730 writeRing(&ring_data, tail, record_slice);
1731
1732 var reader = RingReader.init(config, &ring_data, tail, tail + record_slice.len);
1733 var scratch: [128]u8 = undefined;
1734 const first = (try reader.next(&scratch)).?;
1735 const second = try reader.next(&scratch);
1736 const callchain = try first.getCallchain();
1737
1738 try std.testing.expectEqual(@as(u64, 0x4444), try first.getIp());
1739 try std.testing.expectEqual(@as(usize, 2), callchain.len());
1740 try std.testing.expectEqual(@as(u64, 0x5555), callchain.at(0));
1741 try std.testing.expectEqual(@as(u64, 0x6666), callchain.at(1));
1742 try std.testing.expectEqual(@as(?Record, null), second);
1743 try std.testing.expectEqual(tail + record_slice.len, reader.index);
1744 }
1745
1746 test "ring reader rejects impossible producer windows" {
1747 var ring_data: [64]u8 = undefined;
1748 @memset(&ring_data, 0);
1749
1750 const reversed = RingReader.init(.{}, &ring_data, 8, 7);
1751 const overrun = RingReader.init(.{}, &ring_data, 4, 4 + ring_data.len + 1);
1752
1753 try std.testing.expectError(error.RingHeadBeforeTail, reversed.hasData());
1754 try std.testing.expectError(error.RingWindowExceedsBuffer, overrun.hasData());
1755 }