lib/memtrace/src/tracer.zig
daab053ee43316e1809a84551d573ddd1e5bf3d2
1 const std = @import("std");
2 const observe = @import("alloc_observe");
3 const pretty_json = @import("pretty").json;
4 const sys = @import("sys");
5 const coverage_mod = @import("coverage.zig");
6 const event_mod = @import("event.zig");
7 const stack_mod = @import("stack/root.zig");
8
9 const Allocator = std.mem.Allocator;
10 const Alignment = std.mem.Alignment;
11 const Event = event_mod.Event;
12 const layer_count = std.meta.fieldNames(event_mod.Layer).len;
13 const record_index_none = std.math.maxInt(u32);
14 const anomaly_site_capacity_max = 4096;
15 pub const request_site_capacity_max: u16 = 16 * 1024;
16 const observation_producer_floor_pending = std.math.maxInt(u64);
17
18 const OperationContext = struct {
19 layer: event_mod.Layer = .backing_boundary,
20 operation_id: u64 = 0,
21 parent_operation_id: u64 = 0,
22 producer_id: u64 = 0,
23 producer: observe.Producer = .boundary,
24 generation: u64 = 0,
25 owner_cookie: usize = 0,
26 };
27
28 const ObservedAllocatorKey = struct {
29 producer_id: u64,
30 producer: observe.Producer,
31 };
32
33 const root_scope_id: u32 = 0;
34 const root_label_id: u32 = 0;
35
36 pub const Config = struct {
37 record_events: bool = false,
38 event_writer: ?*std.Io.Writer = null,
39 allocation_attribution: stack_mod.Attribution = .return_address,
40 stack_frame_limit: u16 = 256,
41 anomaly_site_capacity: u16 = 256,
42 request_site_capacity: u16 = 4096,
43 capture_executable_identity: bool = false,
44 static_coverage: coverage_mod.StaticEvidence = .{},
45 };
46
47 pub const SummaryOptions = struct {
48 top: usize = 24,
49 min_live_bytes: usize = 0,
50 include_zero_live: bool = false,
51 layer: event_mod.Layer = .backing_boundary,
52 };
53
54 pub const Retention = enum {
55 releases_freed_memory,
56 retains_freed_memory,
57 };
58
59 pub const Snapshot = struct {
60 seq: u64,
61 allocations: u64,
62 frees: u64,
63 live_allocations: usize,
64 allocated_bytes: usize,
65 freed_bytes: usize,
66 live_bytes: usize,
67 high_water_live_bytes: usize,
68 retained_bytes: usize,
69 high_water_retained_bytes: usize,
70 completed_lifetimes: u64,
71 lifetime_total_events: u64,
72 lifetime_max_events: u64,
73 };
74
75 pub const Difference = struct {
76 allocations: u64,
77 frees: u64,
78 live_allocations: isize,
79 allocated_bytes: usize,
80 freed_bytes: usize,
81 live_bytes: isize,
82 high_water_live_bytes: usize,
83 retained_bytes: isize,
84 high_water_retained_bytes: usize,
85 completed_lifetimes: u64,
86 lifetime_total_events: u64,
87 lifetime_max_events: u64,
88 };
89
90 const Counters = struct {
91 allocations: u64 = 0,
92 frees: u64 = 0,
93 resizes: u64 = 0,
94 remaps: u64 = 0,
95 failed_allocations: u64 = 0,
96 failed_resizes: u64 = 0,
97 failed_remaps: u64 = 0,
98 unmatched_frees: u64 = 0,
99 unmatched_resizes: u64 = 0,
100 unmatched_remaps: u64 = 0,
101 bulk_invalidated_requests: u64 = 0,
102 bulk_invalidated_bytes: usize = 0,
103 untracked_requests: u64 = 0,
104 untracked_request_bytes: usize = 0,
105 lifecycle_events: u64 = 0,
106 live_allocations: usize = 0,
107 allocated_bytes: usize = 0,
108 freed_bytes: usize = 0,
109 live_bytes: usize = 0,
110 high_water_live_bytes: usize = 0,
111 retained_bytes: usize = 0,
112 high_water_retained_bytes: usize = 0,
113 completed_lifetimes: u64 = 0,
114 lifetime_total_events: u64 = 0,
115 lifetime_max_events: u64 = 0,
116
117 fn snapshot(self: Counters, seq: u64) Snapshot {
118 return .{
119 .seq = seq,
120 .allocations = self.allocations,
121 .frees = self.frees,
122 .live_allocations = self.live_allocations,
123 .allocated_bytes = self.allocated_bytes,
124 .freed_bytes = self.freed_bytes,
125 .live_bytes = self.live_bytes,
126 .high_water_live_bytes = self.high_water_live_bytes,
127 .retained_bytes = self.retained_bytes,
128 .high_water_retained_bytes = self.high_water_retained_bytes,
129 .completed_lifetimes = self.completed_lifetimes,
130 .lifetime_total_events = self.lifetime_total_events,
131 .lifetime_max_events = self.lifetime_max_events,
132 };
133 }
134 };
135
136 const ScopeCounters = struct {
137 allocations: u64 = 0,
138 frees: u64 = 0,
139 allocated_bytes: usize = 0,
140 freed_bytes: usize = 0,
141 bulk_invalidated_requests: u64 = 0,
142 bulk_invalidated_bytes: usize = 0,
143 untracked_requests: u64 = 0,
144 untracked_request_bytes: usize = 0,
145 live_allocations: usize = 0,
146 live_bytes: usize = 0,
147 high_water_live_bytes: usize = 0,
148 retained_bytes: usize = 0,
149 high_water_retained_bytes: usize = 0,
150 completed_lifetimes: u64 = 0,
151 lifetime_total_events: u64 = 0,
152 lifetime_max_events: u64 = 0,
153 };
154
155 const ScopeNode = struct {
156 parent: u32,
157 label_id: u32,
158 path: ?[]u8 = null,
159 counters: [layer_count]ScopeCounters = @splat(.{}),
160 };
161
162 const Label = struct {
163 text: []const u8,
164 };
165
166 const AllocatorState = struct {
167 label_id: u32,
168 retention: Retention,
169 layer: event_mod.Layer,
170 counters: Counters = .{},
171 producer_id: u64 = 0,
172 producer: observe.Producer = .boundary,
173 owner_cookie: usize = 0,
174 generation: u64 = 0,
175 active_head: u32 = record_index_none,
176 prefix_complete: bool = true,
177 lifecycle_instrumented: bool = false,
178 terminal: bool = false,
179 observation_floor_pending: bool = false,
180 };
181
182 const ChildKey = struct {
183 parent: u32,
184 label_id: u32,
185 };
186
187 const AllocationRecord = struct {
188 allocation_id: u64,
189 allocator_id: u32,
190 address: usize,
191 len: usize,
192 alignment: Alignment,
193 scope_id: u32,
194 return_address: usize,
195 stack_id: u32,
196 allocation_seq: u64,
197 layer: event_mod.Layer,
198 generation: u64,
199 previous_active: u32 = record_index_none,
200 next_active: u32 = record_index_none,
201 next_free: u32 = record_index_none,
202 active: bool = true,
203 };
204
205 const AllocationKey = struct {
206 allocator_id: u32,
207 address: usize,
208 };
209
210 const ScopeSummary = struct {
211 scope_id: u32,
212 path: []u8,
213 counters: ScopeCounters,
214 };
215
216 const LiveSiteKey = struct {
217 allocator_id: u32,
218 scope_id: u32,
219 return_address: usize,
220 };
221
222 const LiveSiteSummary = struct {
223 key: LiveSiteKey,
224 live_allocations: usize,
225 live_bytes: usize,
226 };
227
228 const AnomalyReason = enum {
229 duplicate_current_address,
230 unknown_pre_observation,
231 unknown_current_generation,
232 stale_generation,
233 generation_gap,
234 owner_cookie_mismatch,
235 length_mismatch,
236 alignment_mismatch,
237 operation_after_terminal,
238 };
239
240 const AnomalyKey = struct {
241 reason: AnomalyReason,
242 operation: observe.Operation,
243 layer: event_mod.Layer,
244 producer_id: u64,
245 producer: observe.Producer,
246 generation: u64,
247 scope_id: u32,
248 return_address: usize,
249 old_len: usize,
250 new_len: usize,
251 succeeded: bool,
252 };
253
254 const AnomalySite = struct {
255 key: AnomalyKey,
256 first_address: usize,
257 occurrences: u64,
258 };
259
260 const RequestSiteKey = struct {
261 producer_id: u64,
262 producer: observe.Producer,
263 operation: observe.Operation,
264 scope_id: u32,
265 return_address: usize,
266 succeeded: bool,
267 };
268
269 const RequestSiteCounters = struct {
270 calls: u64 = 0,
271 requested_bytes: u64 = 0,
272 };
273
274 const RequestSiteSummary = struct {
275 key: RequestSiteKey,
276 counters: RequestSiteCounters,
277 };
278
279 const LifecycleCoverage = struct {
280 instrumented: u64 = 0,
281 uninstrumented: u64 = 0,
282 not_required: u64 = 0,
283 prefix_complete: u64 = 0,
284 prefix_partial: u64 = 0,
285
286 fn status(self: LifecycleCoverage) []const u8 {
287 if (self.instrumented == 0 and
288 self.uninstrumented == 0 and
289 self.not_required == 0)
290 {
291 return "none";
292 }
293 if (self.uninstrumented == 0 and self.prefix_partial == 0) {
294 return "complete";
295 }
296 return "partial";
297 }
298 };
299
300 const ControlAllocatorContext = struct {
301 backing: Allocator,
302 epoch_active: std.atomic.Value(bool) = .init(false),
303 operations: std.atomic.Value(u64) = .init(0),
304
305 fn allocator(self: *@This()) Allocator {
306 return .{
307 .ptr = self,
308 .vtable = &ControlAllocatorContext.vtable,
309 };
310 }
311
312 fn setEpochActive(self: *@This(), active: bool) void {
313 self.epoch_active.store(active, .release);
314 }
315
316 fn note(self: *@This()) void {
317 if (!self.epoch_active.load(.acquire)) return;
318 _ = self.operations.fetchAdd(1, .monotonic);
319 }
320
321 fn operationCount(self: *const @This()) u64 {
322 return self.operations.load(.acquire);
323 }
324
325 fn rawAlloc(
326 context: *anyopaque,
327 len: usize,
328 alignment: Alignment,
329 return_address: usize,
330 ) ?[*]u8 {
331 const self: *@This() = @ptrCast(@alignCast(context));
332 self.note();
333 var allocation_suppression = observe.suppress();
334 defer allocation_suppression.deinit();
335 var memory_suppression = sys.memory.observe.suppress();
336 defer memory_suppression.deinit();
337 return self.backing.rawAlloc(len, alignment, return_address);
338 }
339
340 fn rawResize(
341 context: *anyopaque,
342 memory: []u8,
343 alignment: Alignment,
344 new_len: usize,
345 return_address: usize,
346 ) bool {
347 const self: *@This() = @ptrCast(@alignCast(context));
348 self.note();
349 var allocation_suppression = observe.suppress();
350 defer allocation_suppression.deinit();
351 var memory_suppression = sys.memory.observe.suppress();
352 defer memory_suppression.deinit();
353 return self.backing.rawResize(
354 memory,
355 alignment,
356 new_len,
357 return_address,
358 );
359 }
360
361 fn rawRemap(
362 context: *anyopaque,
363 memory: []u8,
364 alignment: Alignment,
365 new_len: usize,
366 return_address: usize,
367 ) ?[*]u8 {
368 const self: *@This() = @ptrCast(@alignCast(context));
369 self.note();
370 var allocation_suppression = observe.suppress();
371 defer allocation_suppression.deinit();
372 var memory_suppression = sys.memory.observe.suppress();
373 defer memory_suppression.deinit();
374 return self.backing.rawRemap(
375 memory,
376 alignment,
377 new_len,
378 return_address,
379 );
380 }
381
382 fn rawFree(
383 context: *anyopaque,
384 memory: []u8,
385 alignment: Alignment,
386 return_address: usize,
387 ) void {
388 const self: *@This() = @ptrCast(@alignCast(context));
389 self.note();
390 var allocation_suppression = observe.suppress();
391 defer allocation_suppression.deinit();
392 var memory_suppression = sys.memory.observe.suppress();
393 defer memory_suppression.deinit();
394 self.backing.rawFree(memory, alignment, return_address);
395 }
396
397 const vtable: Allocator.VTable = .{
398 .alloc = ControlAllocatorContext.rawAlloc,
399 .resize = ControlAllocatorContext.rawResize,
400 .remap = ControlAllocatorContext.rawRemap,
401 .free = ControlAllocatorContext.rawFree,
402 };
403 };
404
405 pub const Scope = struct {
406 tracer: *Tracer,
407 previous: u32,
408 scope_id: u32,
409 active: bool = true,
410
411 pub fn exit(self: *Scope) void {
412 if (!self.active) return;
413 self.tracer.exitScope(self.scope_id, self.previous);
414 self.active = false;
415 }
416 };
417
418 pub const OwnedObservation = struct {
419 tracer: *Tracer,
420 session: observe.Session,
421 memory_session: sys.memory.observe.Session,
422 active: bool = true,
423
424 pub fn stop(self: *OwnedObservation) void {
425 if (!self.active) return;
426 self.memory_session.deinit();
427 self.session.deinit();
428 std.debug.assert(self.tracer.observer_active);
429 self.tracer.control_context.setEpochActive(false);
430 self.tracer.observer_active = false;
431 self.active = false;
432 }
433 };
434
435 pub const TracingAllocatorOptions = struct {
436 name: []const u8 = "allocator",
437 retention: Retention = .releases_freed_memory,
438 };
439
440 pub const TracingAllocator = struct {
441 tracer: *Tracer,
442 backing: Allocator,
443 allocator_id: u32,
444
445 pub fn init(
446 tracer: *Tracer,
447 backing: Allocator,
448 options: TracingAllocatorOptions,
449 ) !TracingAllocator {
450 return .{
451 .tracer = tracer,
452 .backing = backing,
453 .allocator_id = try tracer.registerAllocator(options.name, options.retention),
454 };
455 }
456
457 pub fn allocator(self: *TracingAllocator) Allocator {
458 return .{
459 .ptr = self,
460 .vtable = &vtable,
461 };
462 }
463 };
464
465 pub const Tracer = struct {
466 control_allocator: Allocator,
467 control_context: *ControlAllocatorContext,
468 config: Config,
469 labels: std.ArrayListUnmanaged(Label) = .empty,
470 label_ids: std.StringHashMapUnmanaged(u32) = .{},
471 scopes: std.ArrayListUnmanaged(ScopeNode) = .empty,
472 scope_children: std.AutoHashMapUnmanaged(ChildKey, u32) = .{},
473 allocators: std.ArrayListUnmanaged(AllocatorState) = .empty,
474 observed_allocators: std.AutoHashMapUnmanaged(
475 ObservedAllocatorKey,
476 u32,
477 ) = .{},
478 allocations: std.AutoHashMapUnmanaged(AllocationKey, u32) = .{},
479 allocation_records: std.ArrayListUnmanaged(AllocationRecord) = .empty,
480 free_record_head: u32 = record_index_none,
481 anomaly_sites: []AnomalySite = &.{},
482 anomaly_site_count: usize = 0,
483 anomaly_sites_dropped: u64 = 0,
484 request_sites: std.AutoHashMapUnmanaged(
485 RequestSiteKey,
486 RequestSiteCounters,
487 ) = .{},
488 request_site_unaggregated_calls: u64 = 0,
489 request_site_unaggregated_bytes: u64 = 0,
490 events: std.ArrayListUnmanaged(Event) = .empty,
491 stacks: stack_mod.Interner = .{},
492 executable_digest: ?stack_mod.Digest = null,
493 coverage: coverage_mod.Manifest = coverage_mod.boundaryManifest(),
494 counters: [layer_count]Counters = @splat(.{}),
495 current_scope: u32 = root_scope_id,
496 next_allocation_id: u64 = 1,
497 next_seq: u64 = 1,
498 recording_failures: u64 = 0,
499 stream_finished: bool = false,
500 observer_sink: observe.Sink = undefined,
501 memory_observer_sink: sys.memory.observe.Sink = undefined,
502 observer_active: bool = false,
503 observation_producer_floor: u64 = 0,
504 physical_allocator_id: ?u32 = null,
505 mutex: std.atomic.Mutex = .unlocked,
506
507 pub fn init(control_allocator: Allocator, config: Config) !Tracer {
508 if (config.anomaly_site_capacity > anomaly_site_capacity_max) {
509 return error.AnomalySiteCapacityTooLarge;
510 }
511 if (config.request_site_capacity > request_site_capacity_max) {
512 return error.RequestSiteCapacityTooLarge;
513 }
514 if (config.allocation_attribution == .stack) {
515 if (!config.record_events) {
516 return error.StackAttributionRequiresEventRecording;
517 }
518 try stack_mod.capture.validateFrameLimit(config.stack_frame_limit);
519 if (!stack_mod.capture.supportsCapture()) {
520 return error.StackCaptureUnsupported;
521 }
522 }
523 const control_context = try control_allocator.create(
524 ControlAllocatorContext,
525 );
526 var control_context_owned = true;
527 errdefer {
528 if (control_context_owned) {
529 control_allocator.destroy(control_context);
530 }
531 }
532 control_context.* = .{ .backing = control_allocator };
533 var tracer: Tracer = .{
534 .control_allocator = control_context.allocator(),
535 .control_context = control_context,
536 .config = config,
537 };
538 control_context_owned = false;
539 errdefer {
540 tracer.stream_finished = true;
541 tracer.deinit();
542 }
543 tracer.anomaly_sites = try tracer.control_allocator.alloc(
544 AnomalySite,
545 config.anomaly_site_capacity,
546 );
547 try tracer.request_sites.ensureTotalCapacity(
548 tracer.control_allocator,
549 config.request_site_capacity,
550 );
551 if (config.allocation_attribution == .stack or
552 config.capture_executable_identity)
553 {
554 tracer.executable_digest = try stack_mod.identity.runningDigest();
555 }
556 try tracer.bootstrapRoot();
557 if (config.event_writer) |writer| {
558 if (tracer.executable_digest) |digest| {
559 try stack_mod.identity.writeMetadata(writer, digest);
560 }
561 }
562 if (config.record_events) try tracer.appendEventLocked(.{ .kind = .trace_start });
563 return tracer;
564 }
565
566 pub fn deinit(self: *Tracer) void {
567 std.debug.assert(!self.observer_active);
568 if (self.config.event_writer != null) {
569 std.debug.assert(self.stream_finished);
570 }
571 for (self.labels.items) |label| self.control_allocator.free(label.text);
572 for (self.scopes.items) |scope| if (scope.path) |path| self.control_allocator.free(path);
573 self.labels.deinit(self.control_allocator);
574 self.label_ids.deinit(self.control_allocator);
575 self.scopes.deinit(self.control_allocator);
576 self.scope_children.deinit(self.control_allocator);
577 self.allocators.deinit(self.control_allocator);
578 self.observed_allocators.deinit(self.control_allocator);
579 self.allocations.deinit(self.control_allocator);
580 self.allocation_records.deinit(self.control_allocator);
581 self.control_allocator.free(self.anomaly_sites);
582 self.request_sites.deinit(self.control_allocator);
583 self.events.deinit(self.control_allocator);
584 self.stacks.deinit(self.control_allocator);
585 const control_context = self.control_context;
586 const control_backing = control_context.backing;
587 self.* = undefined;
588 control_backing.destroy(control_context);
589 }
590
591 pub fn tracedAllocator(self: *Tracer, backing: Allocator, name: []const u8) !TracingAllocator {
592 return try TracingAllocator.init(self, backing, .{ .name = name });
593 }
594
595 pub fn tracedAllocatorWithOptions(self: *Tracer, backing: Allocator, options: TracingAllocatorOptions) !TracingAllocator {
596 return try TracingAllocator.init(self, backing, options);
597 }
598
599 pub fn observeOwnedAllocatorsIfAvailable(
600 self: *Tracer,
601 ) !?OwnedObservation {
602 if (comptime !observe.enabled or !sys.memory.observe.enabled) {
603 return null;
604 }
605 return try self.observeOwnedAllocators();
606 }
607
608 pub fn observeOwnedAllocators(self: *Tracer) !OwnedObservation {
609 if (comptime !observe.enabled) {
610 return error.AllocatorObservationNotCompiled;
611 }
612 if (comptime !sys.memory.observe.enabled) {
613 return error.MemoryObservationNotCompiled;
614 }
615 if (self.observer_active) return error.AllocatorObservationAlreadyActive;
616 _ = sys.memory.pageSize();
617 self.observer_sink = .{
618 .context = self,
619 .record = recordOwnedOperation,
620 };
621 const previous_producer_floor =
622 self.markObservationProducerFloorPending();
623 var session = observe.install(&self.observer_sink) catch |err| {
624 self.restoreObservationProducerFloor(previous_producer_floor);
625 return err;
626 };
627 errdefer session.deinit();
628 self.reconcileObservationProducerFloor(
629 session.producerIdFloor(),
630 );
631 self.memory_observer_sink = .{
632 .context = self,
633 .record = recordPhysicalOperation,
634 };
635 const memory_session = try sys.memory.observe.install(
636 &self.memory_observer_sink,
637 );
638 self.observer_active = true;
639 self.control_context.setEpochActive(true);
640 self.coverage = coverage_mod.processManifest(
641 self.config.static_coverage,
642 );
643 return .{
644 .tracer = self,
645 .session = session,
646 .memory_session = memory_session,
647 };
648 }
649
650 pub fn enter(self: *Tracer, label: []const u8) !Scope {
651 self.lock();
652 defer self.unlock();
653
654 const label_id = try self.internLabelLocked(label);
655 const previous = self.current_scope;
656 const scope_id = try self.scopeChildLocked(previous, label_id);
657 self.current_scope = scope_id;
658 try self.appendEventLocked(.{
659 .kind = .scope_enter,
660 .scope_id = scope_id,
661 .label_id = label_id,
662 .live_bytes = self.countersForLayer(.backing_boundary).live_bytes,
663 });
664 return .{
665 .tracer = self,
666 .previous = previous,
667 .scope_id = scope_id,
668 };
669 }
670
671 pub fn snapshot(self: *Tracer) Snapshot {
672 return self.snapshotLayer(.backing_boundary);
673 }
674
675 pub fn snapshotLayer(
676 self: *Tracer,
677 layer: event_mod.Layer,
678 ) Snapshot {
679 self.lock();
680 defer self.unlock();
681 return self.countersForLayer(layer).snapshot(self.next_seq - 1);
682 }
683
684 pub fn diff(self: *Tracer, before: Snapshot) Difference {
685 return self.diffLayer(.backing_boundary, before);
686 }
687
688 pub fn diffLayer(
689 self: *Tracer,
690 layer: event_mod.Layer,
691 before: Snapshot,
692 ) Difference {
693 const after = self.snapshotLayer(layer);
694 return snapshotDifference(before, after);
695 }
696
697 fn snapshotDifference(before: Snapshot, after: Snapshot) Difference {
698 return .{
699 .allocations = after.allocations - before.allocations,
700 .frees = after.frees - before.frees,
701 .live_allocations = signedDiff(after.live_allocations, before.live_allocations),
702 .allocated_bytes = after.allocated_bytes - before.allocated_bytes,
703 .freed_bytes = after.freed_bytes - before.freed_bytes,
704 .live_bytes = signedDiff(after.live_bytes, before.live_bytes),
705 .high_water_live_bytes = after.high_water_live_bytes,
706 .retained_bytes = signedDiff(after.retained_bytes, before.retained_bytes),
707 .high_water_retained_bytes = after.high_water_retained_bytes,
708 .completed_lifetimes = after.completed_lifetimes - before.completed_lifetimes,
709 .lifetime_total_events = after.lifetime_total_events - before.lifetime_total_events,
710 .lifetime_max_events = after.lifetime_max_events,
711 };
712 }
713
714 pub fn observerControlOperations(self: *const Tracer) u64 {
715 return self.control_context.operationCount();
716 }
717
718 pub fn writeSummary(self: *Tracer, writer: *std.Io.Writer, options: SummaryOptions) !void {
719 self.lock();
720 defer self.unlock();
721 const counters = self.countersForLayer(options.layer);
722
723 var summaries = std.ArrayListUnmanaged(ScopeSummary).empty;
724 defer {
725 for (summaries.items) |summary| self.control_allocator.free(summary.path);
726 summaries.deinit(self.control_allocator);
727 }
728
729 for (self.scopes.items, 0..) |scope, index| {
730 const scope_counters = scope.counters[@backingInt(options.layer)];
731 if (index == root_scope_id and scope_counters.allocations == 0) continue;
732 if (!options.include_zero_live and scope_counters.live_bytes == 0 and scope_counters.high_water_live_bytes == 0) continue;
733 if (scope_counters.live_bytes < options.min_live_bytes and scope_counters.high_water_live_bytes < options.min_live_bytes) continue;
734 try summaries.append(self.control_allocator, .{
735 .scope_id = @intCast(index),
736 .path = try self.scopePathAllocLocked(@intCast(index)),
737 .counters = scope_counters,
738 });
739 }
740 std.mem.sort(ScopeSummary, summaries.items, {}, scopeSummaryGreaterThan);
741
742 var request_site_summaries =
743 std.ArrayListUnmanaged(RequestSiteSummary).empty;
744 defer request_site_summaries.deinit(self.control_allocator);
745 if (options.layer == .logical_allocator) {
746 try request_site_summaries.ensureTotalCapacity(
747 self.control_allocator,
748 self.request_sites.count(),
749 );
750 var request_sites = self.request_sites.iterator();
751 while (request_sites.next()) |entry| {
752 request_site_summaries.appendAssumeCapacity(.{
753 .key = entry.key_ptr.*,
754 .counters = entry.value_ptr.*,
755 });
756 }
757 std.mem.sort(
758 RequestSiteSummary,
759 request_site_summaries.items,
760 {},
761 requestSiteSummaryGreaterThan,
762 );
763 }
764 const request_site_limit = @min(
765 options.top,
766 request_site_summaries.items.len,
767 );
768 var request_site_omitted_calls =
769 self.request_site_unaggregated_calls;
770 var request_site_omitted_bytes =
771 self.request_site_unaggregated_bytes;
772 for (request_site_summaries.items[request_site_limit..]) |site| {
773 request_site_omitted_calls +|= site.counters.calls;
774 request_site_omitted_bytes +|= site.counters.requested_bytes;
775 }
776
777 if (options.layer == .logical_allocator) {
778 try writer.print(
779 "memtrace layer={s} allocations={d} frees={d} " ++
780 "open_requests={d} requested_bytes={d} " ++
781 "explicitly_closed_bytes={d} open_request_bytes={d} " ++
782 "high_water_open_request_bytes={d} " ++
783 "bulk_invalidated_requests={d} bulk_invalidated_bytes={d} " ++
784 "untracked_requests={d} untracked_request_bytes={d}\n",
785 .{
786 options.layer.tag(),
787 counters.allocations,
788 counters.frees,
789 counters.live_allocations,
790 counters.allocated_bytes,
791 counters.freed_bytes,
792 counters.live_bytes,
793 counters.high_water_live_bytes,
794 counters.bulk_invalidated_requests,
795 counters.bulk_invalidated_bytes,
796 counters.untracked_requests,
797 counters.untracked_request_bytes,
798 },
799 );
800 const coverage = self.lifecycleCoverageLocked();
801 try writer.print(
802 "memtrace lifecycle_coverage={s} instrumented_producers={d} " ++
803 "uninstrumented_producers={d} " ++
804 "lifecycle_not_required_producers={d} " ++
805 "prefix_complete_producers={d} prefix_partial_producers={d} " ++
806 "lifecycle_events={d}\n",
807 .{
808 coverage.status(),
809 coverage.instrumented,
810 coverage.uninstrumented,
811 coverage.not_required,
812 coverage.prefix_complete,
813 coverage.prefix_partial,
814 counters.lifecycle_events,
815 },
816 );
817 try writer.print(
818 "memtrace request_sites capacity={d} groups={d} displayed={d} " ++
819 "unaggregated_calls={d} unaggregated_bytes={d} " ++
820 "omitted_calls={d} omitted_bytes={d} exact={s}\n",
821 .{
822 self.config.request_site_capacity,
823 self.request_sites.count(),
824 request_site_limit,
825 self.request_site_unaggregated_calls,
826 self.request_site_unaggregated_bytes,
827 request_site_omitted_calls,
828 request_site_omitted_bytes,
829 if (request_site_omitted_calls == 0) "true" else "false",
830 },
831 );
832 } else {
833 try writer.print(
834 "memtrace layer={s} allocations={d} frees={d} " ++
835 "live_allocations={d} allocated_bytes={d} " ++
836 "freed_bytes={d} live_bytes={d} " ++
837 "high_water_live_bytes={d}\n",
838 .{
839 options.layer.tag(),
840 counters.allocations,
841 counters.frees,
842 counters.live_allocations,
843 counters.allocated_bytes,
844 counters.freed_bytes,
845 counters.live_bytes,
846 counters.high_water_live_bytes,
847 },
848 );
849 try writer.print(
850 "memtrace retained_bytes={d} high_water_retained_bytes={d}\n",
851 .{
852 counters.retained_bytes,
853 counters.high_water_retained_bytes,
854 },
855 );
856 }
857 try writer.print(
858 "memtrace lifetimes completed={d} total_events={d} mean_events={d} max_events={d}\n",
859 .{
860 counters.completed_lifetimes,
861 counters.lifetime_total_events,
862 meanLifetimeEvents(counters.*),
863 counters.lifetime_max_events,
864 },
865 );
866 if (counters.failed_allocations != 0 or
867 counters.failed_resizes != 0 or
868 counters.failed_remaps != 0 or
869 counters.unmatched_frees != 0 or
870 counters.unmatched_resizes != 0 or
871 counters.unmatched_remaps != 0 or
872 self.recording_failures != 0)
873 {
874 try writer.print(
875 "memtrace anomalies failed_allocations={d} failed_resizes={d} " ++
876 "failed_remaps={d} unmatched_frees={d} unmatched_resizes={d} " ++
877 "unmatched_remaps={d} " ++
878 "recording_failures={d}\n",
879 .{
880 counters.failed_allocations,
881 counters.failed_resizes,
882 counters.failed_remaps,
883 counters.unmatched_frees,
884 counters.unmatched_resizes,
885 counters.unmatched_remaps,
886 self.recording_failures,
887 },
888 );
889 }
890 const limit = @min(options.top, summaries.items.len);
891 for (summaries.items[0..limit]) |summary| {
892 if (options.layer == .logical_allocator) {
893 try writer.print(
894 "{s} open_request_bytes={d} " ++
895 "high_water_open_request_bytes={d} requested_bytes={d} " ++
896 "explicitly_closed_bytes={d} allocations={d} frees={d} " ++
897 "open_requests={d} bulk_invalidated_requests={d} " ++
898 "bulk_invalidated_bytes={d} untracked_requests={d} " ++
899 "untracked_request_bytes={d} completed_lifetimes={d} " ++
900 "lifetime_total_events={d} lifetime_mean_events={d} " ++
901 "lifetime_max_events={d}\n",
902 .{
903 summary.path,
904 summary.counters.live_bytes,
905 summary.counters.high_water_live_bytes,
906 summary.counters.allocated_bytes,
907 summary.counters.freed_bytes,
908 summary.counters.allocations,
909 summary.counters.frees,
910 summary.counters.live_allocations,
911 summary.counters.bulk_invalidated_requests,
912 summary.counters.bulk_invalidated_bytes,
913 summary.counters.untracked_requests,
914 summary.counters.untracked_request_bytes,
915 summary.counters.completed_lifetimes,
916 summary.counters.lifetime_total_events,
917 meanLifetimeEvents(summary.counters),
918 summary.counters.lifetime_max_events,
919 },
920 );
921 } else {
922 try writer.print(
923 "{s} retained_bytes={d} high_water_retained_bytes={d} " ++
924 "live_bytes={d} high_water_live_bytes={d} " ++
925 "allocated_bytes={d} freed_bytes={d} allocations={d} " ++
926 "frees={d} live_allocations={d} completed_lifetimes={d} " ++
927 "lifetime_total_events={d} lifetime_mean_events={d} " ++
928 "lifetime_max_events={d}\n",
929 .{
930 summary.path,
931 summary.counters.retained_bytes,
932 summary.counters.high_water_retained_bytes,
933 summary.counters.live_bytes,
934 summary.counters.high_water_live_bytes,
935 summary.counters.allocated_bytes,
936 summary.counters.freed_bytes,
937 summary.counters.allocations,
938 summary.counters.frees,
939 summary.counters.live_allocations,
940 summary.counters.completed_lifetimes,
941 summary.counters.lifetime_total_events,
942 meanLifetimeEvents(summary.counters),
943 summary.counters.lifetime_max_events,
944 },
945 );
946 }
947 }
948 for (request_site_summaries.items[0..request_site_limit]) |site| {
949 const path = try self.scopePathAllocLocked(site.key.scope_id);
950 defer self.control_allocator.free(path);
951 try writer.print(
952 "request_site producer_id={d} producer={s} operation={s} " ++
953 "scope={s} return_address=0x{x} succeeded={s} calls={d} " ++
954 "requested_bytes={d}\n",
955 .{
956 site.key.producer_id,
957 @tagName(site.key.producer),
958 @tagName(site.key.operation),
959 path,
960 site.key.return_address,
961 if (site.key.succeeded) "true" else "false",
962 site.counters.calls,
963 site.counters.requested_bytes,
964 },
965 );
966 }
967 }
968
969 pub fn writeSummaryJsonl(
970 self: *Tracer,
971 writer: *std.Io.Writer,
972 options: SummaryOptions,
973 ) !void {
974 self.lock();
975 defer self.unlock();
976 const counters = self.countersForLayer(options.layer);
977
978 var summaries = std.ArrayListUnmanaged(ScopeSummary).empty;
979 defer {
980 for (summaries.items) |summary| self.control_allocator.free(summary.path);
981 summaries.deinit(self.control_allocator);
982 }
983
984 for (self.scopes.items, 0..) |scope, index| {
985 const scope_counters = scope.counters[@backingInt(options.layer)];
986 if (index == root_scope_id and scope_counters.allocations == 0) continue;
987 if (!options.include_zero_live and
988 scope_counters.live_bytes == 0 and
989 scope_counters.high_water_live_bytes == 0)
990 {
991 continue;
992 }
993 if (scope_counters.live_bytes < options.min_live_bytes and
994 scope_counters.high_water_live_bytes < options.min_live_bytes)
995 {
996 continue;
997 }
998 try summaries.append(self.control_allocator, .{
999 .scope_id = @intCast(index),
1000 .path = try self.scopePathAllocLocked(@intCast(index)),
1001 .counters = scope_counters,
1002 });
1003 }
1004 std.mem.sort(ScopeSummary, summaries.items, {}, scopeSummaryGreaterThan);
1005
1006 var request_site_summaries =
1007 std.ArrayListUnmanaged(RequestSiteSummary).empty;
1008 defer request_site_summaries.deinit(self.control_allocator);
1009 if (options.layer == .logical_allocator) {
1010 try request_site_summaries.ensureTotalCapacity(
1011 self.control_allocator,
1012 self.request_sites.count(),
1013 );
1014 var request_sites = self.request_sites.iterator();
1015 while (request_sites.next()) |entry| {
1016 request_site_summaries.appendAssumeCapacity(.{
1017 .key = entry.key_ptr.*,
1018 .counters = entry.value_ptr.*,
1019 });
1020 }
1021 std.mem.sort(
1022 RequestSiteSummary,
1023 request_site_summaries.items,
1024 {},
1025 requestSiteSummaryGreaterThan,
1026 );
1027 }
1028 const request_site_limit = @min(
1029 options.top,
1030 request_site_summaries.items.len,
1031 );
1032 var request_site_omitted_calls =
1033 self.request_site_unaggregated_calls;
1034 var request_site_omitted_bytes =
1035 self.request_site_unaggregated_bytes;
1036 for (request_site_summaries.items[request_site_limit..]) |site| {
1037 request_site_omitted_calls +|= site.counters.calls;
1038 request_site_omitted_bytes +|= site.counters.requested_bytes;
1039 }
1040
1041 var stream = pretty_json.Writer.init(writer, .minified);
1042 const object = try stream.object();
1043 try object.field("kind", "summary");
1044 try object.field("layer", options.layer.tag());
1045 try object.field("allocations", counters.allocations);
1046 try object.field("frees", counters.frees);
1047 try object.field("resizes", counters.resizes);
1048 try object.field("remaps", counters.remaps);
1049 try object.field("allocated_bytes", counters.allocated_bytes);
1050 if (options.layer == .logical_allocator) {
1051 try object.field("open_requests", counters.live_allocations);
1052 try object.field("requested_bytes", counters.allocated_bytes);
1053 try object.field(
1054 "explicitly_closed_bytes",
1055 counters.freed_bytes,
1056 );
1057 try object.field("open_request_bytes", counters.live_bytes);
1058 try object.field(
1059 "high_water_open_request_bytes",
1060 counters.high_water_live_bytes,
1061 );
1062 try object.field(
1063 "bulk_invalidated_requests",
1064 counters.bulk_invalidated_requests,
1065 );
1066 try object.field(
1067 "bulk_invalidated_bytes",
1068 counters.bulk_invalidated_bytes,
1069 );
1070 try object.field("untracked_requests", counters.untracked_requests);
1071 try object.field(
1072 "untracked_request_bytes",
1073 counters.untracked_request_bytes,
1074 );
1075 try object.field("lifecycle_events", counters.lifecycle_events);
1076 const coverage = self.lifecycleCoverageLocked();
1077 try object.field("lifecycle_coverage", coverage.status());
1078 try object.field(
1079 "lifecycle_instrumented_producers",
1080 coverage.instrumented,
1081 );
1082 try object.field(
1083 "lifecycle_uninstrumented_producers",
1084 coverage.uninstrumented,
1085 );
1086 try object.field(
1087 "lifecycle_not_required_producers",
1088 coverage.not_required,
1089 );
1090 try object.field(
1091 "prefix_complete_producers",
1092 coverage.prefix_complete,
1093 );
1094 try object.field(
1095 "prefix_partial_producers",
1096 coverage.prefix_partial,
1097 );
1098 try object.field(
1099 "request_site_capacity",
1100 self.config.request_site_capacity,
1101 );
1102 try object.field("request_site_groups", self.request_sites.count());
1103 try object.field(
1104 "request_site_groups_displayed",
1105 request_site_limit,
1106 );
1107 try object.field(
1108 "request_site_unaggregated_calls",
1109 self.request_site_unaggregated_calls,
1110 );
1111 try object.field(
1112 "request_site_unaggregated_bytes",
1113 self.request_site_unaggregated_bytes,
1114 );
1115 try object.field(
1116 "request_site_omitted_calls",
1117 request_site_omitted_calls,
1118 );
1119 try object.field(
1120 "request_site_omitted_bytes",
1121 request_site_omitted_bytes,
1122 );
1123 try object.field(
1124 "request_site_attribution_exact",
1125 request_site_omitted_calls == 0,
1126 );
1127 } else {
1128 try object.field("live_allocations", counters.live_allocations);
1129 try object.field("freed_bytes", counters.freed_bytes);
1130 try object.field("live_bytes", counters.live_bytes);
1131 try object.field(
1132 "high_water_live_bytes",
1133 counters.high_water_live_bytes,
1134 );
1135 try object.field("retained_bytes", counters.retained_bytes);
1136 try object.field(
1137 "high_water_retained_bytes",
1138 counters.high_water_retained_bytes,
1139 );
1140 }
1141 try object.field("completed_lifetimes", counters.completed_lifetimes);
1142 try object.field("lifetime_total_events", counters.lifetime_total_events);
1143 try object.field("lifetime_mean_events", meanLifetimeEvents(counters.*));
1144 try object.field("lifetime_max_events", counters.lifetime_max_events);
1145 try object.field("failed_allocations", counters.failed_allocations);
1146 try object.field("failed_resizes", counters.failed_resizes);
1147 try object.field("failed_remaps", counters.failed_remaps);
1148 try object.field("unmatched_frees", counters.unmatched_frees);
1149 try object.field("unmatched_resizes", counters.unmatched_resizes);
1150 try object.field("unmatched_remaps", counters.unmatched_remaps);
1151 try object.field("anomaly_sites", self.anomaly_site_count);
1152 try object.field(
1153 "anomaly_sites_dropped",
1154 self.anomaly_sites_dropped,
1155 );
1156 try object.field(
1157 "anomaly_attribution_exact",
1158 self.anomaly_sites_dropped == 0,
1159 );
1160 try object.field("recording_failures", self.recording_failures);
1161 try object.endLine();
1162 const limit = @min(options.top, summaries.items.len);
1163 for (summaries.items[0..limit]) |summary| {
1164 var row_stream = pretty_json.Writer.init(writer, .minified);
1165 const row = try row_stream.object();
1166 try row.field("kind", "scope");
1167 try row.field("layer", options.layer.tag());
1168 try row.field("scope", summary.path);
1169 try writeCounterFields(row, summary.counters, options.layer);
1170 try row.endLine();
1171 }
1172 for (request_site_summaries.items[0..request_site_limit]) |site| {
1173 const path = try self.scopePathAllocLocked(site.key.scope_id);
1174 defer self.control_allocator.free(path);
1175 var row_stream = pretty_json.Writer.init(writer, .minified);
1176 const row = try row_stream.object();
1177 try row.field("kind", "request_site");
1178 try row.field("layer", event_mod.Layer.logical_allocator.tag());
1179 try row.field("producer_id", site.key.producer_id);
1180 try row.field("producer", @tagName(site.key.producer));
1181 try row.field("operation", @tagName(site.key.operation));
1182 try row.field("scope", path);
1183 try row.field("return_address", site.key.return_address);
1184 try row.field("succeeded", site.key.succeeded);
1185 try row.field("calls", site.counters.calls);
1186 try row.field("requested_bytes", site.counters.requested_bytes);
1187 try row.endLine();
1188 }
1189
1190 var site_counts = std.AutoHashMapUnmanaged(
1191 LiveSiteKey,
1192 LiveSiteSummary,
1193 ).empty;
1194 defer site_counts.deinit(self.control_allocator);
1195 var live_allocations = self.allocations.valueIterator();
1196 while (live_allocations.next()) |record_index| {
1197 const allocation =
1198 &self.allocation_records.items[record_index.*];
1199 std.debug.assert(allocation.active);
1200 if (allocation.layer != options.layer) continue;
1201 const key = LiveSiteKey{
1202 .allocator_id = allocation.allocator_id,
1203 .scope_id = allocation.scope_id,
1204 .return_address = allocation.return_address,
1205 };
1206 const entry = try site_counts.getOrPut(self.control_allocator, key);
1207 if (!entry.found_existing) {
1208 entry.value_ptr.* = .{
1209 .key = key,
1210 .live_allocations = 0,
1211 .live_bytes = 0,
1212 };
1213 }
1214 entry.value_ptr.live_allocations += 1;
1215 entry.value_ptr.live_bytes += allocation.len;
1216 }
1217 var live_sites = std.ArrayListUnmanaged(LiveSiteSummary).empty;
1218 defer live_sites.deinit(self.control_allocator);
1219 try live_sites.ensureTotalCapacity(
1220 self.control_allocator,
1221 site_counts.count(),
1222 );
1223 var sites = site_counts.valueIterator();
1224 while (sites.next()) |site| live_sites.appendAssumeCapacity(site.*);
1225 std.mem.sort(
1226 LiveSiteSummary,
1227 live_sites.items,
1228 {},
1229 liveSiteSummaryGreaterThan,
1230 );
1231 const live_site_limit = @min(options.top, live_sites.items.len);
1232 for (live_sites.items[0..live_site_limit]) |site| {
1233 const path = try self.scopePathAllocLocked(site.key.scope_id);
1234 defer self.control_allocator.free(path);
1235 const allocator_label_id =
1236 self.allocators.items[site.key.allocator_id].label_id;
1237 var row_stream = pretty_json.Writer.init(writer, .minified);
1238 const row = try row_stream.object();
1239 try row.field(
1240 "kind",
1241 if (options.layer == .logical_allocator)
1242 "open_request_site"
1243 else
1244 "live_site",
1245 );
1246 try row.field("layer", options.layer.tag());
1247 try row.field("allocator", self.labels.items[allocator_label_id].text);
1248 try row.field("scope", path);
1249 try row.field("return_address", site.key.return_address);
1250 if (options.layer == .logical_allocator) {
1251 try row.field("open_requests", site.live_allocations);
1252 try row.field("open_request_bytes", site.live_bytes);
1253 } else {
1254 try row.field("live_allocations", site.live_allocations);
1255 try row.field("live_bytes", site.live_bytes);
1256 }
1257 try row.endLine();
1258 }
1259 for (self.anomaly_sites[0..self.anomaly_site_count]) |site| {
1260 if (site.key.layer != options.layer) continue;
1261 const path = try self.scopePathAllocLocked(site.key.scope_id);
1262 defer self.control_allocator.free(path);
1263 var row_stream = pretty_json.Writer.init(writer, .minified);
1264 const row = try row_stream.object();
1265 try row.field("kind", "anomaly_site");
1266 try row.field("layer", site.key.layer.tag());
1267 try row.field("reason", @tagName(site.key.reason));
1268 try row.field("operation", @tagName(site.key.operation));
1269 try row.field("producer_id", site.key.producer_id);
1270 try row.field("producer", @tagName(site.key.producer));
1271 try row.field("generation", site.key.generation);
1272 try row.field("scope", path);
1273 try row.field("return_address", site.key.return_address);
1274 try row.field("old_len", site.key.old_len);
1275 try row.field("new_len", site.key.new_len);
1276 try row.field("succeeded", site.key.succeeded);
1277 try row.field("first_address", site.first_address);
1278 try row.field("occurrences", site.occurrences);
1279 try row.endLine();
1280 }
1281 }
1282
1283 pub fn writeEventsJsonl(self: *Tracer, writer: *std.Io.Writer) !void {
1284 self.lock();
1285 defer self.unlock();
1286 if (!self.config.record_events or self.config.event_writer != null) {
1287 return error.EventsNotRetained;
1288 }
1289 if (self.executable_digest) |digest| {
1290 try stack_mod.identity.writeMetadata(writer, digest);
1291 }
1292 try coverage_mod.writeMetadata(writer, self.coverageManifest());
1293 for (self.stacks.definitions.items, 0..) |definition, index| {
1294 try stack_mod.capture.writeDefinition(
1295 writer,
1296 @intCast(index + 1),
1297 definition,
1298 );
1299 }
1300 for (self.events.items) |recorded| {
1301 const label_text = if ((recorded.kind == .allocator or
1302 recorded.kind == .scope_enter) and
1303 recorded.label_id < self.labels.items.len)
1304 self.labels.items[recorded.label_id].text
1305 else
1306 null;
1307 const scope_text = if (recorded.kind == .scope_enter and
1308 recorded.scope_id < self.scopes.items.len)
1309 try self.scopePathCachedLocked(recorded.scope_id)
1310 else
1311 null;
1312 try recorded.writeJsonLine(writer, label_text, scope_text);
1313 }
1314 try (Event{
1315 .seq = self.next_seq,
1316 .kind = .trace_stop,
1317 .live_bytes = self.countersForLayer(.backing_boundary).live_bytes,
1318 .recording_failures = self.recording_failures,
1319 }).writeJsonLine(writer, null, null);
1320 }
1321
1322 pub fn finishEvents(self: *Tracer) !void {
1323 self.lock();
1324 defer self.unlock();
1325 if (self.config.event_writer == null) return error.EventsNotStreamed;
1326 if (self.stream_finished) return error.EventStreamAlreadyFinished;
1327 try coverage_mod.writeMetadata(
1328 self.config.event_writer.?,
1329 self.coverageManifest(),
1330 );
1331 try self.appendEventLocked(.{
1332 .kind = .trace_stop,
1333 .live_bytes = self.countersForLayer(.backing_boundary).live_bytes,
1334 .recording_failures = self.recording_failures,
1335 });
1336 self.stream_finished = true;
1337 }
1338
1339 pub fn writeEventsBundlePath(self: *Tracer, path: []const u8) !void {
1340 try self.writeExecutableArtifactPath(path);
1341 var file = try sys.fs.createFile(path, .{ .truncate = true });
1342 defer sys.fs.closeHandle(file);
1343 var buffer: [64 * 1024]u8 = undefined;
1344 var output = file.writer(sys.fs.debugIo(), &buffer);
1345 try self.writeEventsJsonl(&output.interface);
1346 try output.interface.flush();
1347 }
1348
1349 pub fn writeExecutableArtifactPath(
1350 self: *Tracer,
1351 events_path: []const u8,
1352 ) !void {
1353 if (std.fs.path.dirname(events_path)) |parent| {
1354 if (parent.len != 0) try sys.fs.createDirPath(parent);
1355 }
1356 if (self.executable_digest) |digest| {
1357 const artifact_path = try stack_mod.identity.artifactPathAlloc(
1358 self.control_allocator,
1359 events_path,
1360 );
1361 defer self.control_allocator.free(artifact_path);
1362 try stack_mod.identity.copyRunningExecutable(artifact_path, digest);
1363 }
1364 }
1365
1366 pub fn writeExecutableMetadata(
1367 self: *Tracer,
1368 writer: *std.Io.Writer,
1369 ) !void {
1370 self.lock();
1371 defer self.unlock();
1372 const digest = self.executable_digest orelse
1373 return error.ExecutableIdentityNotCaptured;
1374 try stack_mod.identity.writeMetadata(writer, digest);
1375 }
1376
1377 fn registerAllocator(self: *Tracer, name: []const u8, retention: Retention) !u32 {
1378 self.lock();
1379 defer self.unlock();
1380 const label_id = try self.internLabelLocked(name);
1381 const allocator_id: u32 = @intCast(self.allocators.items.len);
1382 try self.allocators.append(self.control_allocator, .{
1383 .label_id = label_id,
1384 .retention = retention,
1385 .layer = .backing_boundary,
1386 });
1387 try self.appendEventLocked(.{
1388 .kind = .allocator,
1389 .allocator_id = allocator_id,
1390 .label_id = label_id,
1391 .retains_freed_memory = retention == .retains_freed_memory,
1392 .live_bytes = self.countersForLayer(.backing_boundary).live_bytes,
1393 });
1394 return allocator_id;
1395 }
1396
1397 fn rawAlloc(
1398 self: *Tracer,
1399 backing: Allocator,
1400 allocator_id: u32,
1401 len: usize,
1402 alignment: Alignment,
1403 ret_addr: usize,
1404 ) ?[*]u8 {
1405 if (observe.suppressed()) {
1406 self.noteObserverControlOperation();
1407 return backing.rawAlloc(len, alignment, ret_addr);
1408 }
1409 var stack_storage: [stack_mod.max_frames_limit]usize = undefined;
1410 const captured = self.captureOperation(&stack_storage, ret_addr);
1411 var causal_context = observe.beginContext();
1412 const operation_context = boundaryOperationContext(causal_context);
1413 const ptr = backing.rawAlloc(len, alignment, ret_addr) orelse {
1414 causal_context.finish();
1415 self.recordAllocationFailure(
1416 allocator_id,
1417 len,
1418 alignment,
1419 ret_addr,
1420 captured,
1421 operation_context,
1422 ) catch self.markRecordingFailure();
1423 return null;
1424 };
1425 causal_context.finish();
1426 self.recordAllocation(
1427 allocator_id,
1428 ptr,
1429 len,
1430 alignment,
1431 ret_addr,
1432 captured,
1433 operation_context,
1434 ) catch {
1435 backing.rawFree(ptr[0..len], alignment, ret_addr);
1436 self.markRecordingFailure();
1437 return null;
1438 };
1439 return ptr;
1440 }
1441
1442 fn rawResize(
1443 self: *Tracer,
1444 backing: Allocator,
1445 allocator_id: u32,
1446 memory: []u8,
1447 alignment: Alignment,
1448 new_len: usize,
1449 ret_addr: usize,
1450 ) bool {
1451 if (observe.suppressed()) {
1452 self.noteObserverControlOperation();
1453 return backing.rawResize(memory, alignment, new_len, ret_addr);
1454 }
1455 var stack_storage: [stack_mod.max_frames_limit]usize = undefined;
1456 const captured = self.captureOperation(&stack_storage, ret_addr);
1457 var causal_context = observe.beginContext();
1458 const operation_context = boundaryOperationContext(causal_context);
1459 const succeeded = backing.rawResize(memory, alignment, new_len, ret_addr);
1460 causal_context.finish();
1461 self.recordResize(
1462 allocator_id,
1463 memory.ptr,
1464 memory.len,
1465 new_len,
1466 alignment,
1467 ret_addr,
1468 succeeded,
1469 captured,
1470 operation_context,
1471 ) catch self.markRecordingFailure();
1472 return succeeded;
1473 }
1474
1475 fn rawRemap(
1476 self: *Tracer,
1477 backing: Allocator,
1478 allocator_id: u32,
1479 memory: []u8,
1480 alignment: Alignment,
1481 new_len: usize,
1482 ret_addr: usize,
1483 ) ?[*]u8 {
1484 if (observe.suppressed()) {
1485 self.noteObserverControlOperation();
1486 return backing.rawRemap(memory, alignment, new_len, ret_addr);
1487 }
1488 var stack_storage: [stack_mod.max_frames_limit]usize = undefined;
1489 const captured = self.captureOperation(&stack_storage, ret_addr);
1490 var causal_context = observe.beginContext();
1491 const operation_context = boundaryOperationContext(causal_context);
1492 const ptr = backing.rawRemap(memory, alignment, new_len, ret_addr);
1493 causal_context.finish();
1494 self.recordRemap(
1495 allocator_id,
1496 memory.ptr,
1497 ptr,
1498 memory.len,
1499 new_len,
1500 alignment,
1501 ret_addr,
1502 captured,
1503 operation_context,
1504 ) catch self.markRecordingFailure();
1505 return ptr;
1506 }
1507
1508 fn rawFree(
1509 self: *Tracer,
1510 backing: Allocator,
1511 allocator_id: u32,
1512 memory: []u8,
1513 alignment: Alignment,
1514 ret_addr: usize,
1515 ) void {
1516 if (observe.suppressed()) {
1517 self.noteObserverControlOperation();
1518 backing.rawFree(memory, alignment, ret_addr);
1519 return;
1520 }
1521 var stack_storage: [stack_mod.max_frames_limit]usize = undefined;
1522 const captured = self.captureOperation(&stack_storage, ret_addr);
1523 var causal_context = observe.beginContext();
1524 const operation_context = boundaryOperationContext(causal_context);
1525 backing.rawFree(memory, alignment, ret_addr);
1526 causal_context.finish();
1527 self.recordFree(
1528 allocator_id,
1529 memory.ptr,
1530 memory.len,
1531 alignment,
1532 ret_addr,
1533 captured,
1534 operation_context,
1535 ) catch self.markRecordingFailure();
1536 }
1537
1538 fn recordOwnedEvent(
1539 self: *Tracer,
1540 observed: observe.Event,
1541 ) !void {
1542 std.debug.assert(observed.producer_id != 0);
1543 std.debug.assert(observed.operation_id != 0);
1544 const allocator_id = try self.observedAllocatorId(observed);
1545 const alignment = Alignment.fromByteUnits(observed.alignment);
1546 const operation_context = OperationContext{
1547 .layer = .logical_allocator,
1548 .operation_id = observed.operation_id,
1549 .parent_operation_id = observed.parent_operation_id,
1550 .producer_id = observed.producer_id,
1551 .producer = observed.producer,
1552 .generation = observed.generation,
1553 .owner_cookie = observed.owner_cookie,
1554 };
1555 if (!self.validateObservedIdentity(
1556 allocator_id,
1557 observed,
1558 operation_context,
1559 )) return;
1560 if (observed.operation == .lifecycle) {
1561 return self.recordLifecycle(
1562 allocator_id,
1563 observed,
1564 operation_context,
1565 );
1566 }
1567 var stack_storage: [stack_mod.max_frames_limit]usize = undefined;
1568 const captured = self.captureOperation(
1569 &stack_storage,
1570 observed.return_address,
1571 );
1572 switch (observed.operation) {
1573 .alloc => {
1574 if (!observed.succeeded) {
1575 return self.recordAllocationFailure(
1576 allocator_id,
1577 observed.len,
1578 alignment,
1579 observed.return_address,
1580 captured,
1581 operation_context,
1582 );
1583 }
1584 std.debug.assert(observed.address != 0);
1585 return self.recordAllocation(
1586 allocator_id,
1587 @ptrFromInt(observed.address),
1588 observed.len,
1589 alignment,
1590 observed.return_address,
1591 captured,
1592 operation_context,
1593 );
1594 },
1595 .resize => {
1596 std.debug.assert(observed.old_address != 0);
1597 return self.recordResize(
1598 allocator_id,
1599 @ptrFromInt(observed.old_address),
1600 observed.old_len,
1601 observed.len,
1602 alignment,
1603 observed.return_address,
1604 observed.succeeded,
1605 captured,
1606 operation_context,
1607 );
1608 },
1609 .remap => {
1610 std.debug.assert(observed.old_address != 0);
1611 const new_ptr: ?[*]u8 = if (observed.succeeded)
1612 @ptrFromInt(observed.address)
1613 else
1614 null;
1615 return self.recordRemap(
1616 allocator_id,
1617 @ptrFromInt(observed.old_address),
1618 new_ptr,
1619 observed.old_len,
1620 observed.len,
1621 alignment,
1622 observed.return_address,
1623 captured,
1624 operation_context,
1625 );
1626 },
1627 .free => {
1628 std.debug.assert(observed.old_address != 0);
1629 return self.recordFree(
1630 allocator_id,
1631 @ptrFromInt(observed.old_address),
1632 observed.old_len,
1633 alignment,
1634 observed.return_address,
1635 captured,
1636 operation_context,
1637 );
1638 },
1639 .lifecycle => unreachable,
1640 }
1641 }
1642
1643 fn recordPhysicalEvent(
1644 self: *Tracer,
1645 observed: sys.memory.observe.Event,
1646 operation_context: OperationContext,
1647 ) !void {
1648 const allocator_id = try self.physicalAllocatorId();
1649 var stack_storage: [stack_mod.max_frames_limit]usize = undefined;
1650 const captured = self.captureOperation(
1651 &stack_storage,
1652 observed.return_address,
1653 );
1654 self.lock();
1655 defer self.unlock();
1656 const stack_id = try self.internStackLocked(captured);
1657 try self.appendEventLocked(.{
1658 .kind = physicalEventKind(observed.operation),
1659 .allocator_id = allocator_id,
1660 .scope_id = self.current_scope,
1661 .label_id = self.scopes.items[self.current_scope].label_id,
1662 .address = observed.address,
1663 .old_address = observed.address,
1664 .len = observed.len,
1665 .old_len = observed.len,
1666 .alignment = @intCast(sys.memory.pageSize()),
1667 .return_address = observed.return_address,
1668 .stack_id = stack_id,
1669 .succeeded = observed.succeeded,
1670 .layer = .physical_page,
1671 .operation_id = operation_context.operation_id,
1672 .parent_operation_id = operation_context.parent_operation_id,
1673 .producer_id = @backingInt(observed.source) + 1,
1674 .producer = .sys_memory,
1675 });
1676 }
1677
1678 fn observedAllocatorId(
1679 self: *Tracer,
1680 observed: observe.Event,
1681 ) !u32 {
1682 self.lock();
1683 defer self.unlock();
1684 const key = ObservedAllocatorKey{
1685 .producer_id = observed.producer_id,
1686 .producer = observed.producer,
1687 };
1688 const entry = try self.observed_allocators.getOrPut(
1689 self.control_allocator,
1690 key,
1691 );
1692 if (entry.found_existing) return entry.value_ptr.*;
1693 var committed = false;
1694 errdefer {
1695 if (!committed) {
1696 _ = self.observed_allocators.fetchRemove(key);
1697 }
1698 }
1699 const label_id = try self.internLabelLocked(
1700 @tagName(observed.producer),
1701 );
1702 const allocator_id = std.math.cast(
1703 u32,
1704 self.allocators.items.len,
1705 ) orelse return error.OutOfMemory;
1706 const retention = observedProducerRetention(observed.producer);
1707 const floor_pending = self.observation_producer_floor ==
1708 observation_producer_floor_pending;
1709 const prefix_complete = !floor_pending and
1710 observed.producer_id >= self.observation_producer_floor;
1711 try self.allocators.append(self.control_allocator, .{
1712 .label_id = label_id,
1713 .retention = retention,
1714 .layer = .logical_allocator,
1715 .producer_id = observed.producer_id,
1716 .producer = observed.producer,
1717 .owner_cookie = observed.owner_cookie,
1718 .generation = observed.generation,
1719 .prefix_complete = prefix_complete,
1720 .lifecycle_instrumented = observedProducerLifecycleInstrumented(observed.producer),
1721 .observation_floor_pending = floor_pending,
1722 });
1723 entry.value_ptr.* = allocator_id;
1724 committed = true;
1725 try self.appendObservedAllocatorEventLocked(allocator_id);
1726 return allocator_id;
1727 }
1728
1729 fn appendObservedAllocatorEventLocked(
1730 self: *Tracer,
1731 allocator_id: u32,
1732 ) !void {
1733 var suppression = observe.suppress();
1734 defer suppression.deinit();
1735 const allocator = self.allocators.items[allocator_id];
1736 try self.appendEventLocked(.{
1737 .kind = .allocator,
1738 .allocator_id = allocator_id,
1739 .label_id = allocator.label_id,
1740 .retains_freed_memory = allocator.retention ==
1741 .retains_freed_memory,
1742 .live_bytes = self.countersForLayer(.logical_allocator).live_bytes,
1743 .layer = .logical_allocator,
1744 .producer_id = allocator.producer_id,
1745 .producer = allocator.producer,
1746 .generation = allocator.generation,
1747 .owner_cookie = allocator.owner_cookie,
1748 .lifecycle_instrumented = allocator.lifecycle_instrumented,
1749 .observation_prefix_complete = allocator.prefix_complete,
1750 });
1751 }
1752
1753 fn markObservationProducerFloorPending(self: *Tracer) u64 {
1754 self.lock();
1755 defer self.unlock();
1756 std.debug.assert(
1757 self.observation_producer_floor !=
1758 observation_producer_floor_pending,
1759 );
1760 const previous = self.observation_producer_floor;
1761 self.observation_producer_floor =
1762 observation_producer_floor_pending;
1763 return previous;
1764 }
1765
1766 fn restoreObservationProducerFloor(
1767 self: *Tracer,
1768 previous: u64,
1769 ) void {
1770 self.lock();
1771 defer self.unlock();
1772 std.debug.assert(
1773 self.observation_producer_floor ==
1774 observation_producer_floor_pending,
1775 );
1776 self.observation_producer_floor = previous;
1777 }
1778
1779 fn reconcileObservationProducerFloor(
1780 self: *Tracer,
1781 producer_floor: u64,
1782 ) void {
1783 self.lock();
1784 defer self.unlock();
1785 std.debug.assert(
1786 self.observation_producer_floor ==
1787 observation_producer_floor_pending,
1788 );
1789 self.observation_producer_floor = producer_floor;
1790 for (self.allocators.items, 0..) |*allocator, allocator_index| {
1791 if (!allocator.observation_floor_pending) continue;
1792 const prefix_was_complete = allocator.prefix_complete;
1793 allocator.prefix_complete = allocator.prefix_complete or
1794 allocator.producer_id >= producer_floor;
1795 allocator.observation_floor_pending = false;
1796 if (!prefix_was_complete and
1797 allocator.prefix_complete and
1798 self.config.record_events)
1799 {
1800 self.appendObservedAllocatorEventLocked(
1801 @intCast(allocator_index),
1802 ) catch {
1803 self.recording_failures +|= 1;
1804 };
1805 }
1806 }
1807 }
1808
1809 fn physicalAllocatorId(self: *Tracer) !u32 {
1810 self.lock();
1811 defer self.unlock();
1812 if (self.physical_allocator_id) |allocator_id| return allocator_id;
1813 const label_id = try self.internLabelLocked("sys.memory");
1814 const allocator_id = std.math.cast(
1815 u32,
1816 self.allocators.items.len,
1817 ) orelse return error.OutOfMemory;
1818 try self.allocators.append(self.control_allocator, .{
1819 .label_id = label_id,
1820 .retention = .releases_freed_memory,
1821 .layer = .physical_page,
1822 });
1823 self.physical_allocator_id = allocator_id;
1824 try self.appendEventLocked(.{
1825 .kind = .allocator,
1826 .allocator_id = allocator_id,
1827 .label_id = label_id,
1828 .layer = .physical_page,
1829 .producer = .sys_memory,
1830 });
1831 return allocator_id;
1832 }
1833
1834 fn captureOperation(
1835 self: *Tracer,
1836 storage: *[stack_mod.max_frames_limit]usize,
1837 ret_addr: usize,
1838 ) ?stack_mod.Capture {
1839 if (self.config.allocation_attribution != .stack) return null;
1840 return stack_mod.capture.capture(
1841 storage,
1842 ret_addr,
1843 self.config.stack_frame_limit,
1844 );
1845 }
1846
1847 fn recordAllocation(
1848 self: *Tracer,
1849 allocator_id: u32,
1850 ptr: [*]u8,
1851 len: usize,
1852 alignment: Alignment,
1853 ret_addr: usize,
1854 captured: ?stack_mod.Capture,
1855 operation_context: OperationContext,
1856 ) !void {
1857 self.lock();
1858 defer self.unlock();
1859
1860 const stack_id = try self.internStackLocked(captured);
1861 const address = @intFromPtr(ptr);
1862 const allocation_id = self.next_allocation_id;
1863 self.next_allocation_id += 1;
1864 const key = AllocationKey{
1865 .allocator_id = allocator_id,
1866 .address = address,
1867 };
1868 const entry = try self.allocations.getOrPut(
1869 self.control_allocator,
1870 key,
1871 );
1872 if (entry.found_existing) {
1873 self.noteAnomalyLocked(
1874 .duplicate_current_address,
1875 .alloc,
1876 operation_context,
1877 address,
1878 0,
1879 len,
1880 true,
1881 ret_addr,
1882 );
1883 self.applyUntrackedAllocationLocked(
1884 allocator_id,
1885 self.current_scope,
1886 len,
1887 operation_context.layer,
1888 );
1889 return self.appendEventLocked(.{
1890 .kind = .alloc,
1891 .allocator_id = allocator_id,
1892 .scope_id = self.current_scope,
1893 .label_id = self.scopes.items[self.current_scope].label_id,
1894 .address = address,
1895 .len = len,
1896 .alignment = @intCast(alignment.toByteUnits()),
1897 .return_address = ret_addr,
1898 .stack_id = stack_id,
1899 .succeeded = true,
1900 .tracked = false,
1901 .live_bytes = self.countersForLayer(operation_context.layer).live_bytes,
1902 .layer = operation_context.layer,
1903 .operation_id = operation_context.operation_id,
1904 .parent_operation_id = operation_context.parent_operation_id,
1905 .producer_id = operation_context.producer_id,
1906 .producer = operation_context.producer,
1907 .generation = operation_context.generation,
1908 .owner_cookie = operation_context.owner_cookie,
1909 });
1910 }
1911 var map_committed = false;
1912 errdefer {
1913 if (!map_committed) {
1914 _ = self.allocations.fetchRemove(key);
1915 }
1916 }
1917 const record_index = try self.acquireAllocationRecordLocked(.{
1918 .allocation_id = allocation_id,
1919 .allocator_id = allocator_id,
1920 .address = address,
1921 .len = len,
1922 .alignment = alignment,
1923 .scope_id = self.current_scope,
1924 .return_address = ret_addr,
1925 .stack_id = stack_id,
1926 .allocation_seq = self.next_seq,
1927 .layer = operation_context.layer,
1928 .generation = operation_context.generation,
1929 });
1930 entry.value_ptr.* = record_index;
1931 self.linkAllocationRecordLocked(record_index);
1932 map_committed = true;
1933 self.applyAllocationLocked(
1934 allocator_id,
1935 self.current_scope,
1936 len,
1937 operation_context.layer,
1938 );
1939 try self.appendEventLocked(.{
1940 .kind = .alloc,
1941 .allocator_id = allocator_id,
1942 .allocation_id = allocation_id,
1943 .scope_id = self.current_scope,
1944 .label_id = self.scopes.items[self.current_scope].label_id,
1945 .address = address,
1946 .len = len,
1947 .alignment = @intCast(alignment.toByteUnits()),
1948 .return_address = ret_addr,
1949 .stack_id = stack_id,
1950 .succeeded = true,
1951 .live_bytes = self.countersForLayer(operation_context.layer).live_bytes,
1952 .layer = operation_context.layer,
1953 .operation_id = operation_context.operation_id,
1954 .parent_operation_id = operation_context.parent_operation_id,
1955 .producer_id = operation_context.producer_id,
1956 .producer = operation_context.producer,
1957 .generation = operation_context.generation,
1958 .owner_cookie = operation_context.owner_cookie,
1959 });
1960 }
1961
1962 fn recordAllocationFailure(
1963 self: *Tracer,
1964 allocator_id: u32,
1965 len: usize,
1966 alignment: Alignment,
1967 ret_addr: usize,
1968 captured: ?stack_mod.Capture,
1969 operation_context: OperationContext,
1970 ) !void {
1971 self.lock();
1972 defer self.unlock();
1973
1974 const stack_id = try self.internStackLocked(captured);
1975 self.applyFailedOperationLocked(
1976 allocator_id,
1977 .alloc,
1978 operation_context.layer,
1979 );
1980 try self.appendEventLocked(.{
1981 .kind = .alloc,
1982 .allocator_id = allocator_id,
1983 .scope_id = self.current_scope,
1984 .label_id = self.scopes.items[self.current_scope].label_id,
1985 .len = len,
1986 .alignment = @intCast(alignment.toByteUnits()),
1987 .return_address = ret_addr,
1988 .stack_id = stack_id,
1989 .succeeded = false,
1990 .live_bytes = self.countersForLayer(operation_context.layer).live_bytes,
1991 .layer = operation_context.layer,
1992 .operation_id = operation_context.operation_id,
1993 .parent_operation_id = operation_context.parent_operation_id,
1994 .producer_id = operation_context.producer_id,
1995 .producer = operation_context.producer,
1996 .generation = operation_context.generation,
1997 .owner_cookie = operation_context.owner_cookie,
1998 });
1999 }
2000
2001 fn recordResize(
2002 self: *Tracer,
2003 allocator_id: u32,
2004 ptr: [*]u8,
2005 old_len: usize,
2006 new_len: usize,
2007 alignment: Alignment,
2008 ret_addr: usize,
2009 succeeded: bool,
2010 captured: ?stack_mod.Capture,
2011 operation_context: OperationContext,
2012 ) !void {
2013 self.lock();
2014 defer self.unlock();
2015
2016 const address = @intFromPtr(ptr);
2017 const stack_id = try self.internStackLocked(captured);
2018 const record_index = self.allocations.get(.{
2019 .allocator_id = allocator_id,
2020 .address = address,
2021 }) orelse {
2022 self.countersForLayer(operation_context.layer).unmatched_resizes += 1;
2023 self.noteAnomalyLocked(
2024 self.unknownReasonLocked(allocator_id),
2025 .resize,
2026 operation_context,
2027 address,
2028 old_len,
2029 new_len,
2030 succeeded,
2031 ret_addr,
2032 );
2033 if (!succeeded) self.applyFailedOperationLocked(
2034 allocator_id,
2035 .resize,
2036 operation_context.layer,
2037 );
2038 return self.appendUnknownResizeLocked(
2039 allocator_id,
2040 address,
2041 old_len,
2042 new_len,
2043 alignment,
2044 ret_addr,
2045 stack_id,
2046 succeeded,
2047 operation_context,
2048 );
2049 };
2050 const record = &self.allocation_records.items[record_index];
2051 const scope_id = record.scope_id;
2052 std.debug.assert(record.layer == operation_context.layer);
2053 if (record.len != old_len) {
2054 self.noteAnomalyLocked(
2055 .length_mismatch,
2056 .resize,
2057 operation_context,
2058 address,
2059 old_len,
2060 new_len,
2061 succeeded,
2062 ret_addr,
2063 );
2064 }
2065 if (record.alignment != alignment) {
2066 self.noteAnomalyLocked(
2067 .alignment_mismatch,
2068 .resize,
2069 operation_context,
2070 address,
2071 old_len,
2072 new_len,
2073 succeeded,
2074 ret_addr,
2075 );
2076 }
2077 if (succeeded) {
2078 const recorded_old_len = record.len;
2079 record.len = new_len;
2080 record.alignment = alignment;
2081 self.applyResizeLocked(
2082 record.allocator_id,
2083 scope_id,
2084 recorded_old_len,
2085 new_len,
2086 operation_context.layer,
2087 );
2088 } else {
2089 self.applyFailedOperationLocked(
2090 allocator_id,
2091 .resize,
2092 operation_context.layer,
2093 );
2094 }
2095 try self.appendEventLocked(.{
2096 .kind = .resize,
2097 .allocator_id = allocator_id,
2098 .allocation_id = record.allocation_id,
2099 .scope_id = scope_id,
2100 .label_id = self.scopes.items[scope_id].label_id,
2101 .address = address,
2102 .old_len = old_len,
2103 .len = new_len,
2104 .alignment = @intCast(alignment.toByteUnits()),
2105 .return_address = ret_addr,
2106 .stack_id = stack_id,
2107 .succeeded = succeeded,
2108 .live_bytes = self.countersForLayer(operation_context.layer).live_bytes,
2109 .layer = operation_context.layer,
2110 .operation_id = operation_context.operation_id,
2111 .parent_operation_id = operation_context.parent_operation_id,
2112 .producer_id = operation_context.producer_id,
2113 .producer = operation_context.producer,
2114 .generation = operation_context.generation,
2115 .owner_cookie = operation_context.owner_cookie,
2116 });
2117 }
2118
2119 fn appendUnknownResizeLocked(
2120 self: *Tracer,
2121 allocator_id: u32,
2122 address: usize,
2123 old_len: usize,
2124 new_len: usize,
2125 alignment: Alignment,
2126 ret_addr: usize,
2127 stack_id: u32,
2128 succeeded: bool,
2129 operation_context: OperationContext,
2130 ) !void {
2131 try self.appendEventLocked(.{
2132 .kind = .resize,
2133 .allocator_id = allocator_id,
2134 .scope_id = self.current_scope,
2135 .label_id = self.scopes.items[self.current_scope].label_id,
2136 .address = address,
2137 .old_len = old_len,
2138 .len = new_len,
2139 .alignment = @intCast(alignment.toByteUnits()),
2140 .return_address = ret_addr,
2141 .stack_id = stack_id,
2142 .succeeded = succeeded,
2143 .tracked = false,
2144 .live_bytes = self.countersForLayer(operation_context.layer).live_bytes,
2145 .layer = operation_context.layer,
2146 .operation_id = operation_context.operation_id,
2147 .parent_operation_id = operation_context.parent_operation_id,
2148 .producer_id = operation_context.producer_id,
2149 .producer = operation_context.producer,
2150 .generation = operation_context.generation,
2151 .owner_cookie = operation_context.owner_cookie,
2152 });
2153 }
2154
2155 fn recordRemap(
2156 self: *Tracer,
2157 allocator_id: u32,
2158 old_ptr: [*]u8,
2159 new_ptr: ?[*]u8,
2160 old_len: usize,
2161 new_len: usize,
2162 alignment: Alignment,
2163 ret_addr: usize,
2164 captured: ?stack_mod.Capture,
2165 operation_context: OperationContext,
2166 ) !void {
2167 self.lock();
2168 defer self.unlock();
2169
2170 const old_address = @intFromPtr(old_ptr);
2171 const stack_id = try self.internStackLocked(captured);
2172 const key = AllocationKey{
2173 .allocator_id = allocator_id,
2174 .address = old_address,
2175 };
2176 const existing_index = self.allocations.get(key) orelse {
2177 self.countersForLayer(operation_context.layer).unmatched_remaps += 1;
2178 self.noteAnomalyLocked(
2179 self.unknownReasonLocked(allocator_id),
2180 .remap,
2181 operation_context,
2182 old_address,
2183 old_len,
2184 new_len,
2185 new_ptr != null,
2186 ret_addr,
2187 );
2188 if (new_ptr == null) self.applyFailedOperationLocked(
2189 allocator_id,
2190 .remap,
2191 operation_context.layer,
2192 );
2193 return self.appendUnknownRemapLocked(
2194 allocator_id,
2195 old_address,
2196 new_ptr,
2197 old_len,
2198 new_len,
2199 alignment,
2200 ret_addr,
2201 stack_id,
2202 operation_context,
2203 );
2204 };
2205 const record = &self.allocation_records.items[existing_index];
2206 std.debug.assert(record.layer == operation_context.layer);
2207 if (record.len != old_len) {
2208 self.noteAnomalyLocked(
2209 .length_mismatch,
2210 .remap,
2211 operation_context,
2212 old_address,
2213 old_len,
2214 new_len,
2215 new_ptr != null,
2216 ret_addr,
2217 );
2218 }
2219 if (record.alignment != alignment) {
2220 self.noteAnomalyLocked(
2221 .alignment_mismatch,
2222 .remap,
2223 operation_context,
2224 old_address,
2225 old_len,
2226 new_len,
2227 new_ptr != null,
2228 ret_addr,
2229 );
2230 }
2231 if (new_ptr == null) {
2232 self.applyFailedOperationLocked(
2233 allocator_id,
2234 .remap,
2235 operation_context.layer,
2236 );
2237 return self.appendKnownRemapLocked(
2238 record.*,
2239 old_address,
2240 null,
2241 old_len,
2242 new_len,
2243 alignment,
2244 ret_addr,
2245 stack_id,
2246 operation_context,
2247 true,
2248 );
2249 }
2250 const new_address = @intFromPtr(new_ptr.?);
2251 if (new_address != old_address and self.allocations.contains(.{
2252 .allocator_id = allocator_id,
2253 .address = new_address,
2254 })) {
2255 self.noteAnomalyLocked(
2256 .duplicate_current_address,
2257 .remap,
2258 operation_context,
2259 new_address,
2260 old_len,
2261 new_len,
2262 true,
2263 ret_addr,
2264 );
2265 _ = self.allocations.fetchRemove(key);
2266 self.clearDrainedAllocationsLocked();
2267 const abandoned = record.*;
2268 self.unlinkAllocationRecordLocked(existing_index);
2269 self.applyLostTrackingLocked(
2270 abandoned.allocator_id,
2271 abandoned.scope_id,
2272 abandoned.len,
2273 new_len,
2274 operation_context.layer,
2275 );
2276 self.recycleAllocationRecordLocked(existing_index);
2277 return self.appendKnownRemapLocked(
2278 abandoned,
2279 old_address,
2280 new_ptr,
2281 old_len,
2282 new_len,
2283 alignment,
2284 ret_addr,
2285 stack_id,
2286 operation_context,
2287 false,
2288 );
2289 }
2290 _ = self.allocations.fetchRemove(key).?;
2291 self.clearDrainedAllocationsLocked();
2292 const scope_id = record.scope_id;
2293 const recorded_old_len = record.len;
2294 record.address = new_address;
2295 record.len = new_len;
2296 record.alignment = alignment;
2297 const new_entry = try self.allocations.getOrPut(
2298 self.control_allocator,
2299 .{
2300 .allocator_id = allocator_id,
2301 .address = new_address,
2302 },
2303 );
2304 std.debug.assert(!new_entry.found_existing);
2305 new_entry.value_ptr.* = existing_index;
2306 self.applyResizeLocked(
2307 record.allocator_id,
2308 scope_id,
2309 recorded_old_len,
2310 new_len,
2311 operation_context.layer,
2312 );
2313 self.countersForLayer(operation_context.layer).remaps += 1;
2314 if (record.allocator_id < self.allocators.items.len) self.allocators.items[record.allocator_id].counters.remaps += 1;
2315 try self.appendKnownRemapLocked(
2316 record.*,
2317 old_address,
2318 new_ptr,
2319 old_len,
2320 new_len,
2321 alignment,
2322 ret_addr,
2323 stack_id,
2324 operation_context,
2325 true,
2326 );
2327 }
2328
2329 fn appendKnownRemapLocked(
2330 self: *Tracer,
2331 record: AllocationRecord,
2332 old_address: usize,
2333 new_ptr: ?[*]u8,
2334 old_len: usize,
2335 new_len: usize,
2336 alignment: Alignment,
2337 ret_addr: usize,
2338 stack_id: u32,
2339 operation_context: OperationContext,
2340 tracked: bool,
2341 ) !void {
2342 try self.appendEventLocked(.{
2343 .kind = .remap,
2344 .allocator_id = record.allocator_id,
2345 .allocation_id = record.allocation_id,
2346 .scope_id = record.scope_id,
2347 .label_id = self.scopes.items[record.scope_id].label_id,
2348 .address = if (new_ptr) |ptr| @intFromPtr(ptr) else old_address,
2349 .old_address = old_address,
2350 .old_len = old_len,
2351 .len = new_len,
2352 .alignment = @intCast(alignment.toByteUnits()),
2353 .return_address = ret_addr,
2354 .stack_id = stack_id,
2355 .succeeded = new_ptr != null,
2356 .tracked = tracked,
2357 .live_bytes = self.countersForLayer(operation_context.layer).live_bytes,
2358 .layer = operation_context.layer,
2359 .operation_id = operation_context.operation_id,
2360 .parent_operation_id = operation_context.parent_operation_id,
2361 .producer_id = operation_context.producer_id,
2362 .producer = operation_context.producer,
2363 .generation = operation_context.generation,
2364 .owner_cookie = operation_context.owner_cookie,
2365 });
2366 }
2367
2368 fn appendUnknownRemapLocked(
2369 self: *Tracer,
2370 allocator_id: u32,
2371 old_address: usize,
2372 new_ptr: ?[*]u8,
2373 old_len: usize,
2374 new_len: usize,
2375 alignment: Alignment,
2376 ret_addr: usize,
2377 stack_id: u32,
2378 operation_context: OperationContext,
2379 ) !void {
2380 const record = AllocationRecord{
2381 .allocation_id = 0,
2382 .allocator_id = allocator_id,
2383 .address = old_address,
2384 .len = old_len,
2385 .alignment = alignment,
2386 .scope_id = self.current_scope,
2387 .return_address = ret_addr,
2388 .stack_id = stack_id,
2389 .allocation_seq = 0,
2390 .layer = operation_context.layer,
2391 .generation = operation_context.generation,
2392 };
2393 try self.appendKnownRemapLocked(
2394 record,
2395 old_address,
2396 new_ptr,
2397 old_len,
2398 new_len,
2399 alignment,
2400 ret_addr,
2401 stack_id,
2402 operation_context,
2403 false,
2404 );
2405 }
2406
2407 fn recordFree(
2408 self: *Tracer,
2409 allocator_id: u32,
2410 ptr: [*]u8,
2411 len: usize,
2412 alignment: Alignment,
2413 ret_addr: usize,
2414 captured: ?stack_mod.Capture,
2415 operation_context: OperationContext,
2416 ) !void {
2417 self.lock();
2418 defer self.unlock();
2419
2420 const address = @intFromPtr(ptr);
2421 const stack_id = try self.internStackLocked(captured);
2422 const removed = self.allocations.fetchRemove(.{
2423 .allocator_id = allocator_id,
2424 .address = address,
2425 }) orelse {
2426 self.countersForLayer(operation_context.layer).unmatched_frees += 1;
2427 self.noteAnomalyLocked(
2428 self.unknownReasonLocked(allocator_id),
2429 .free,
2430 operation_context,
2431 address,
2432 len,
2433 0,
2434 true,
2435 ret_addr,
2436 );
2437 return self.appendUnknownFreeLocked(
2438 allocator_id,
2439 address,
2440 len,
2441 alignment,
2442 ret_addr,
2443 stack_id,
2444 operation_context,
2445 );
2446 };
2447 self.clearDrainedAllocationsLocked();
2448 const record_index = removed.value;
2449 const record = self.allocation_records.items[record_index];
2450 std.debug.assert(record.layer == operation_context.layer);
2451 if (record.len != len) {
2452 self.noteAnomalyLocked(
2453 .length_mismatch,
2454 .free,
2455 operation_context,
2456 address,
2457 len,
2458 0,
2459 true,
2460 ret_addr,
2461 );
2462 }
2463 if (record.alignment != alignment) {
2464 self.noteAnomalyLocked(
2465 .alignment_mismatch,
2466 .free,
2467 operation_context,
2468 address,
2469 len,
2470 0,
2471 true,
2472 ret_addr,
2473 );
2474 }
2475 self.unlinkAllocationRecordLocked(record_index);
2476 self.applyLifetimeLocked(
2477 record.scope_id,
2478 completedLifetimeEvents(record.allocation_seq, self.next_seq),
2479 operation_context.layer,
2480 );
2481 self.applyFreeLocked(
2482 record.allocator_id,
2483 record.scope_id,
2484 record.len,
2485 operation_context.layer,
2486 );
2487 self.recycleAllocationRecordLocked(record_index);
2488 try self.appendEventLocked(.{
2489 .kind = .free,
2490 .allocator_id = allocator_id,
2491 .allocation_id = record.allocation_id,
2492 .scope_id = record.scope_id,
2493 .label_id = self.scopes.items[record.scope_id].label_id,
2494 .address = address,
2495 .old_len = len,
2496 .len = record.len,
2497 .alignment = @intCast(alignment.toByteUnits()),
2498 .return_address = ret_addr,
2499 .stack_id = stack_id,
2500 .succeeded = true,
2501 .live_bytes = self.countersForLayer(operation_context.layer).live_bytes,
2502 .layer = operation_context.layer,
2503 .operation_id = operation_context.operation_id,
2504 .parent_operation_id = operation_context.parent_operation_id,
2505 .producer_id = operation_context.producer_id,
2506 .producer = operation_context.producer,
2507 .generation = operation_context.generation,
2508 .owner_cookie = operation_context.owner_cookie,
2509 });
2510 }
2511
2512 fn appendUnknownFreeLocked(
2513 self: *Tracer,
2514 allocator_id: u32,
2515 address: usize,
2516 len: usize,
2517 alignment: Alignment,
2518 ret_addr: usize,
2519 stack_id: u32,
2520 operation_context: OperationContext,
2521 ) !void {
2522 try self.appendEventLocked(.{
2523 .kind = .free,
2524 .allocator_id = allocator_id,
2525 .scope_id = self.current_scope,
2526 .label_id = self.scopes.items[self.current_scope].label_id,
2527 .address = address,
2528 .old_len = len,
2529 .len = len,
2530 .alignment = @intCast(alignment.toByteUnits()),
2531 .return_address = ret_addr,
2532 .stack_id = stack_id,
2533 .succeeded = true,
2534 .tracked = false,
2535 .live_bytes = self.countersForLayer(operation_context.layer).live_bytes,
2536 .layer = operation_context.layer,
2537 .operation_id = operation_context.operation_id,
2538 .parent_operation_id = operation_context.parent_operation_id,
2539 .producer_id = operation_context.producer_id,
2540 .producer = operation_context.producer,
2541 .generation = operation_context.generation,
2542 .owner_cookie = operation_context.owner_cookie,
2543 });
2544 }
2545
2546 fn acquireAllocationRecordLocked(
2547 self: *Tracer,
2548 record: AllocationRecord,
2549 ) !u32 {
2550 if (self.free_record_head != record_index_none) {
2551 const record_index = self.free_record_head;
2552 self.free_record_head =
2553 self.allocation_records.items[record_index].next_free;
2554 self.allocation_records.items[record_index] = record;
2555 return record_index;
2556 }
2557 const record_index = std.math.cast(
2558 u32,
2559 self.allocation_records.items.len,
2560 ) orelse return error.OutOfMemory;
2561 try self.allocation_records.append(
2562 self.control_allocator,
2563 record,
2564 );
2565 return record_index;
2566 }
2567
2568 fn linkAllocationRecordLocked(
2569 self: *Tracer,
2570 record_index: u32,
2571 ) void {
2572 const record = &self.allocation_records.items[record_index];
2573 std.debug.assert(record.active);
2574 const allocator = &self.allocators.items[record.allocator_id];
2575 record.previous_active = record_index_none;
2576 record.next_active = allocator.active_head;
2577 if (allocator.active_head != record_index_none) {
2578 self.allocation_records.items[allocator.active_head]
2579 .previous_active = record_index;
2580 }
2581 allocator.active_head = record_index;
2582 }
2583
2584 fn unlinkAllocationRecordLocked(
2585 self: *Tracer,
2586 record_index: u32,
2587 ) void {
2588 const record = &self.allocation_records.items[record_index];
2589 std.debug.assert(record.active);
2590 const allocator = &self.allocators.items[record.allocator_id];
2591 if (record.previous_active == record_index_none) {
2592 std.debug.assert(allocator.active_head == record_index);
2593 allocator.active_head = record.next_active;
2594 } else {
2595 self.allocation_records.items[record.previous_active]
2596 .next_active = record.next_active;
2597 }
2598 if (record.next_active != record_index_none) {
2599 self.allocation_records.items[record.next_active]
2600 .previous_active = record.previous_active;
2601 }
2602 record.previous_active = record_index_none;
2603 record.next_active = record_index_none;
2604 }
2605
2606 fn recycleAllocationRecordLocked(
2607 self: *Tracer,
2608 record_index: u32,
2609 ) void {
2610 const record = &self.allocation_records.items[record_index];
2611 std.debug.assert(record.active);
2612 record.active = false;
2613 record.next_free = self.free_record_head;
2614 self.free_record_head = record_index;
2615 }
2616
2617 fn unknownReasonLocked(
2618 self: *Tracer,
2619 allocator_id: u32,
2620 ) AnomalyReason {
2621 if (allocator_id >= self.allocators.items.len) {
2622 return .unknown_current_generation;
2623 }
2624 return if (self.allocators.items[allocator_id].prefix_complete)
2625 .unknown_current_generation
2626 else
2627 .unknown_pre_observation;
2628 }
2629
2630 fn noteAnomalyLocked(
2631 self: *Tracer,
2632 reason: AnomalyReason,
2633 operation: observe.Operation,
2634 operation_context: OperationContext,
2635 address: usize,
2636 old_len: usize,
2637 new_len: usize,
2638 succeeded: bool,
2639 return_address: usize,
2640 ) void {
2641 const key = AnomalyKey{
2642 .reason = reason,
2643 .operation = operation,
2644 .layer = operation_context.layer,
2645 .producer_id = operation_context.producer_id,
2646 .producer = operation_context.producer,
2647 .generation = operation_context.generation,
2648 .scope_id = self.current_scope,
2649 .return_address = return_address,
2650 .old_len = old_len,
2651 .new_len = new_len,
2652 .succeeded = succeeded,
2653 };
2654 for (self.anomaly_sites[0..self.anomaly_site_count]) |*site| {
2655 if (!std.meta.eql(site.key, key)) continue;
2656 site.occurrences +|= 1;
2657 return;
2658 }
2659 if (self.anomaly_site_count == self.anomaly_sites.len) {
2660 self.anomaly_sites_dropped +|= 1;
2661 return;
2662 }
2663 self.anomaly_sites[self.anomaly_site_count] = .{
2664 .key = key,
2665 .first_address = address,
2666 .occurrences = 1,
2667 };
2668 self.anomaly_site_count += 1;
2669 }
2670
2671 fn applyUntrackedAllocationLocked(
2672 self: *Tracer,
2673 allocator_id: u32,
2674 scope_id: u32,
2675 len: usize,
2676 layer: event_mod.Layer,
2677 ) void {
2678 const counters = self.countersForLayer(layer);
2679 counters.allocations += 1;
2680 counters.allocated_bytes += len;
2681 counters.untracked_requests += 1;
2682 counters.untracked_request_bytes += len;
2683 const allocator = &self.allocators.items[allocator_id].counters;
2684 allocator.allocations += 1;
2685 allocator.allocated_bytes += len;
2686 allocator.untracked_requests += 1;
2687 allocator.untracked_request_bytes += len;
2688 const scope = self.scopeCountersForLayer(scope_id, layer);
2689 scope.allocations += 1;
2690 scope.allocated_bytes += len;
2691 scope.untracked_requests += 1;
2692 scope.untracked_request_bytes += len;
2693 }
2694
2695 fn applyLostTrackingLocked(
2696 self: *Tracer,
2697 allocator_id: u32,
2698 scope_id: u32,
2699 old_len: usize,
2700 new_len: usize,
2701 layer: event_mod.Layer,
2702 ) void {
2703 self.applyResizeLocked(
2704 allocator_id,
2705 scope_id,
2706 old_len,
2707 new_len,
2708 layer,
2709 );
2710 const counters = self.countersForLayer(layer);
2711 counters.live_allocations -= 1;
2712 counters.live_bytes -= new_len;
2713 counters.untracked_requests += 1;
2714 counters.untracked_request_bytes += new_len;
2715 const allocator = &self.allocators.items[allocator_id].counters;
2716 allocator.live_allocations -= 1;
2717 allocator.live_bytes -= new_len;
2718 allocator.untracked_requests += 1;
2719 allocator.untracked_request_bytes += new_len;
2720 const scope = self.scopeCountersForLayer(scope_id, layer);
2721 scope.live_allocations -= 1;
2722 scope.live_bytes -= new_len;
2723 scope.untracked_requests += 1;
2724 scope.untracked_request_bytes += new_len;
2725 }
2726
2727 fn validateObservedIdentity(
2728 self: *Tracer,
2729 allocator_id: u32,
2730 observed: observe.Event,
2731 operation_context: OperationContext,
2732 ) bool {
2733 self.lock();
2734 defer self.unlock();
2735 self.noteRequestSiteLocked(observed);
2736 const allocator = &self.allocators.items[allocator_id];
2737 if (allocator.terminal) {
2738 self.noteAnomalyLocked(
2739 .operation_after_terminal,
2740 observed.operation,
2741 operation_context,
2742 observed.old_address,
2743 observed.old_len,
2744 observed.len,
2745 observed.succeeded,
2746 observed.return_address,
2747 );
2748 return false;
2749 }
2750 if (allocator.owner_cookie != observed.owner_cookie) {
2751 self.noteAnomalyLocked(
2752 .owner_cookie_mismatch,
2753 observed.operation,
2754 operation_context,
2755 observed.old_address,
2756 observed.old_len,
2757 observed.len,
2758 observed.succeeded,
2759 observed.return_address,
2760 );
2761 return false;
2762 }
2763 if (observed.generation < allocator.generation) {
2764 self.noteAnomalyLocked(
2765 .stale_generation,
2766 observed.operation,
2767 operation_context,
2768 observed.old_address,
2769 observed.old_len,
2770 observed.len,
2771 observed.succeeded,
2772 observed.return_address,
2773 );
2774 return false;
2775 }
2776 if (observed.generation > allocator.generation) {
2777 self.noteAnomalyLocked(
2778 .generation_gap,
2779 observed.operation,
2780 operation_context,
2781 observed.old_address,
2782 observed.old_len,
2783 observed.len,
2784 observed.succeeded,
2785 observed.return_address,
2786 );
2787 return false;
2788 }
2789 return true;
2790 }
2791
2792 fn noteRequestSiteLocked(
2793 self: *Tracer,
2794 observed: observe.Event,
2795 ) void {
2796 if (observed.operation == .lifecycle) return;
2797 const requested_bytes: u64 = std.math.cast(
2798 u64,
2799 switch (observed.operation) {
2800 .alloc, .resize, .remap => observed.len,
2801 .free => observed.old_len,
2802 .lifecycle => unreachable,
2803 },
2804 ) orelse std.math.maxInt(u64);
2805 const key = RequestSiteKey{
2806 .producer_id = observed.producer_id,
2807 .producer = observed.producer,
2808 .operation = observed.operation,
2809 .scope_id = self.current_scope,
2810 .return_address = observed.return_address,
2811 .succeeded = observed.succeeded,
2812 };
2813 if (self.request_sites.getPtr(key)) |site| {
2814 site.calls +|= 1;
2815 site.requested_bytes +|= requested_bytes;
2816 return;
2817 }
2818 if (self.request_sites.count() ==
2819 @as(usize, self.config.request_site_capacity))
2820 {
2821 self.request_site_unaggregated_calls +|= 1;
2822 self.request_site_unaggregated_bytes +|= requested_bytes;
2823 return;
2824 }
2825 self.request_sites.putAssumeCapacity(key, .{
2826 .calls = 1,
2827 .requested_bytes = requested_bytes,
2828 });
2829 }
2830
2831 fn recordLifecycle(
2832 self: *Tracer,
2833 allocator_id: u32,
2834 observed: observe.Event,
2835 operation_context: OperationContext,
2836 ) !void {
2837 self.lock();
2838 defer self.unlock();
2839 const allocator = &self.allocators.items[allocator_id];
2840 self.countersForLayer(.logical_allocator).lifecycle_events += 1;
2841 allocator.counters.lifecycle_events += 1;
2842 var append_error: ?anyerror = null;
2843 var record_index = allocator.active_head;
2844 while (record_index != record_index_none) {
2845 const record = self.allocation_records.items[record_index];
2846 const next = record.next_active;
2847 std.debug.assert(record.generation == observed.generation);
2848 const removed = self.allocations.fetchRemove(.{
2849 .allocator_id = record.allocator_id,
2850 .address = record.address,
2851 }) orelse unreachable;
2852 std.debug.assert(removed.value == record_index);
2853 self.unlinkAllocationRecordLocked(record_index);
2854 self.applyLifetimeLocked(
2855 record.scope_id,
2856 completedLifetimeEvents(
2857 record.allocation_seq,
2858 self.next_seq,
2859 ),
2860 .logical_allocator,
2861 );
2862 self.applyBulkInvalidationLocked(
2863 record.allocator_id,
2864 record.scope_id,
2865 record.len,
2866 .logical_allocator,
2867 );
2868 self.recycleAllocationRecordLocked(record_index);
2869 if (append_error == null) {
2870 self.appendEventLocked(.{
2871 .kind = .release,
2872 .allocator_id = record.allocator_id,
2873 .allocation_id = record.allocation_id,
2874 .scope_id = record.scope_id,
2875 .label_id = self.scopes.items[record.scope_id].label_id,
2876 .address = record.address,
2877 .old_len = record.len,
2878 .len = record.len,
2879 .alignment = @intCast(record.alignment.toByteUnits()),
2880 .return_address = observed.return_address,
2881 .stack_id = record.stack_id,
2882 .succeeded = observed.succeeded,
2883 .live_bytes = self.countersForLayer(.logical_allocator).live_bytes,
2884 .layer = .logical_allocator,
2885 .operation_id = operation_context.operation_id,
2886 .parent_operation_id = operation_context.parent_operation_id,
2887 .producer_id = operation_context.producer_id,
2888 .producer = operation_context.producer,
2889 .generation = operation_context.generation,
2890 .owner_cookie = operation_context.owner_cookie,
2891 .lifecycle_disposition = observed.lifecycle_disposition,
2892 .lifecycle_reason = observed.lifecycle_reason,
2893 }) catch |err| {
2894 append_error = err;
2895 };
2896 }
2897 record_index = next;
2898 }
2899 self.clearDrainedAllocationsLocked();
2900 if (append_error == null) {
2901 self.appendEventLocked(.{
2902 .kind = .lifecycle,
2903 .allocator_id = allocator_id,
2904 .scope_id = self.current_scope,
2905 .label_id = self.scopes.items[self.current_scope].label_id,
2906 .return_address = observed.return_address,
2907 .succeeded = observed.succeeded,
2908 .live_bytes = self.countersForLayer(.logical_allocator).live_bytes,
2909 .layer = .logical_allocator,
2910 .operation_id = operation_context.operation_id,
2911 .parent_operation_id = operation_context.parent_operation_id,
2912 .producer_id = operation_context.producer_id,
2913 .producer = operation_context.producer,
2914 .generation = operation_context.generation,
2915 .owner_cookie = operation_context.owner_cookie,
2916 .lifecycle_disposition = observed.lifecycle_disposition,
2917 .lifecycle_reason = observed.lifecycle_reason,
2918 }) catch |err| {
2919 append_error = err;
2920 };
2921 }
2922 if (observed.lifecycle_disposition == .end) {
2923 allocator.terminal = true;
2924 allocator.prefix_complete = true;
2925 } else {
2926 allocator.generation = std.math.add(
2927 u64,
2928 allocator.generation,
2929 1,
2930 ) catch @panic("allocator observation generation exhausted");
2931 allocator.prefix_complete = true;
2932 }
2933 if (append_error) |err| return err;
2934 }
2935
2936 fn internStackLocked(
2937 self: *Tracer,
2938 captured: ?stack_mod.Capture,
2939 ) !u32 {
2940 const stack_capture = captured orelse return 0;
2941 const interned = try self.stacks.intern(
2942 self.control_allocator,
2943 stack_capture,
2944 );
2945 if (interned.is_new) {
2946 if (self.config.event_writer) |writer| {
2947 try stack_mod.capture.writeDefinition(
2948 writer,
2949 interned.id,
2950 self.stacks.recordForId(interned.id).*,
2951 );
2952 }
2953 }
2954 return interned.id;
2955 }
2956
2957 fn applyFailedOperationLocked(
2958 self: *Tracer,
2959 allocator_id: u32,
2960 kind: event_mod.Kind,
2961 layer: event_mod.Layer,
2962 ) void {
2963 const allocator_counters = if (allocator_id < self.allocators.items.len)
2964 &self.allocators.items[allocator_id].counters
2965 else
2966 null;
2967 if (allocator_id < self.allocators.items.len) {
2968 std.debug.assert(self.allocators.items[allocator_id].layer == layer);
2969 }
2970 const counters = self.countersForLayer(layer);
2971 switch (kind) {
2972 .alloc => {
2973 counters.failed_allocations +|= 1;
2974 if (allocator_counters) |item| item.failed_allocations +|= 1;
2975 },
2976 .resize => {
2977 counters.failed_resizes +|= 1;
2978 if (allocator_counters) |item| item.failed_resizes +|= 1;
2979 },
2980 .remap => {
2981 counters.failed_remaps +|= 1;
2982 if (allocator_counters) |item| item.failed_remaps +|= 1;
2983 },
2984 else => unreachable,
2985 }
2986 }
2987
2988 fn markRecordingFailure(self: *Tracer) void {
2989 self.lock();
2990 defer self.unlock();
2991 self.recording_failures +|= 1;
2992 }
2993
2994 fn applyAllocationLocked(
2995 self: *Tracer,
2996 allocator_id: u32,
2997 scope_id: u32,
2998 len: usize,
2999 layer: event_mod.Layer,
3000 ) void {
3001 const counters = self.countersForLayer(layer);
3002 counters.allocations += 1;
3003 counters.live_allocations += 1;
3004 counters.allocated_bytes += len;
3005 counters.live_bytes += len;
3006 counters.high_water_live_bytes = @max(
3007 counters.high_water_live_bytes,
3008 counters.live_bytes,
3009 );
3010 if (layer != .logical_allocator) {
3011 counters.retained_bytes += len;
3012 counters.high_water_retained_bytes = @max(
3013 counters.high_water_retained_bytes,
3014 counters.retained_bytes,
3015 );
3016 }
3017 if (allocator_id < self.allocators.items.len) {
3018 const allocator_state = &self.allocators.items[allocator_id];
3019 std.debug.assert(allocator_state.layer == layer);
3020 allocator_state.counters.allocations += 1;
3021 allocator_state.counters.live_allocations += 1;
3022 allocator_state.counters.allocated_bytes += len;
3023 allocator_state.counters.live_bytes += len;
3024 allocator_state.counters.high_water_live_bytes = @max(
3025 allocator_state.counters.high_water_live_bytes,
3026 allocator_state.counters.live_bytes,
3027 );
3028 if (layer != .logical_allocator) {
3029 allocator_state.counters.retained_bytes += len;
3030 allocator_state.counters.high_water_retained_bytes = @max(
3031 allocator_state.counters.high_water_retained_bytes,
3032 allocator_state.counters.retained_bytes,
3033 );
3034 }
3035 }
3036 const scope_counters = self.scopeCountersForLayer(scope_id, layer);
3037 scope_counters.allocations += 1;
3038 scope_counters.live_allocations += 1;
3039 scope_counters.allocated_bytes += len;
3040 scope_counters.live_bytes += len;
3041 scope_counters.high_water_live_bytes = @max(scope_counters.high_water_live_bytes, scope_counters.live_bytes);
3042 if (layer != .logical_allocator) {
3043 scope_counters.retained_bytes += len;
3044 scope_counters.high_water_retained_bytes = @max(scope_counters.high_water_retained_bytes, scope_counters.retained_bytes);
3045 }
3046 }
3047
3048 fn applyResizeLocked(
3049 self: *Tracer,
3050 allocator_id: u32,
3051 scope_id: u32,
3052 old_len: usize,
3053 new_len: usize,
3054 layer: event_mod.Layer,
3055 ) void {
3056 const counters = self.countersForLayer(layer);
3057 const scope_counters = self.scopeCountersForLayer(scope_id, layer);
3058 const allocator_state = if (allocator_id < self.allocators.items.len)
3059 &self.allocators.items[allocator_id]
3060 else
3061 null;
3062 if (allocator_state) |state| std.debug.assert(state.layer == layer);
3063 counters.resizes += 1;
3064 if (new_len >= old_len) {
3065 const delta = new_len - old_len;
3066 counters.allocated_bytes += delta;
3067 counters.live_bytes += delta;
3068 counters.high_water_live_bytes = @max(
3069 counters.high_water_live_bytes,
3070 counters.live_bytes,
3071 );
3072 if (layer != .logical_allocator) {
3073 counters.retained_bytes += delta;
3074 counters.high_water_retained_bytes = @max(
3075 counters.high_water_retained_bytes,
3076 counters.retained_bytes,
3077 );
3078 }
3079 scope_counters.allocated_bytes += delta;
3080 scope_counters.live_bytes += delta;
3081 scope_counters.high_water_live_bytes = @max(
3082 scope_counters.high_water_live_bytes,
3083 scope_counters.live_bytes,
3084 );
3085 if (layer != .logical_allocator) {
3086 scope_counters.retained_bytes += delta;
3087 scope_counters.high_water_retained_bytes = @max(
3088 scope_counters.high_water_retained_bytes,
3089 scope_counters.retained_bytes,
3090 );
3091 }
3092 if (allocator_state) |state| {
3093 state.counters.allocated_bytes += delta;
3094 state.counters.live_bytes += delta;
3095 state.counters.high_water_live_bytes = @max(
3096 state.counters.high_water_live_bytes,
3097 state.counters.live_bytes,
3098 );
3099 if (layer != .logical_allocator) {
3100 state.counters.retained_bytes += delta;
3101 state.counters.high_water_retained_bytes = @max(
3102 state.counters.high_water_retained_bytes,
3103 state.counters.retained_bytes,
3104 );
3105 }
3106 }
3107 } else {
3108 const delta = old_len - new_len;
3109 counters.freed_bytes += delta;
3110 counters.live_bytes -= delta;
3111 scope_counters.freed_bytes += delta;
3112 scope_counters.live_bytes -= delta;
3113 const releases = layer != .logical_allocator and
3114 (allocator_state == null or
3115 allocator_state.?.retention == .releases_freed_memory);
3116 if (releases) {
3117 counters.retained_bytes -= delta;
3118 scope_counters.retained_bytes -= delta;
3119 }
3120 if (allocator_state) |state| {
3121 state.counters.freed_bytes += delta;
3122 state.counters.live_bytes -= delta;
3123 if (releases) state.counters.retained_bytes -= delta;
3124 }
3125 }
3126 if (allocator_state) |state| state.counters.resizes += 1;
3127 }
3128
3129 fn applyFreeLocked(
3130 self: *Tracer,
3131 allocator_id: u32,
3132 scope_id: u32,
3133 len: usize,
3134 layer: event_mod.Layer,
3135 ) void {
3136 const counters = self.countersForLayer(layer);
3137 const scope_counters = self.scopeCountersForLayer(scope_id, layer);
3138 const allocator_state = if (allocator_id < self.allocators.items.len)
3139 &self.allocators.items[allocator_id]
3140 else
3141 null;
3142 if (allocator_state) |state| std.debug.assert(state.layer == layer);
3143 counters.frees += 1;
3144 counters.live_allocations -= 1;
3145 counters.freed_bytes += len;
3146 counters.live_bytes -= len;
3147 const releases = layer != .logical_allocator and
3148 (allocator_state == null or
3149 allocator_state.?.retention == .releases_freed_memory);
3150 if (releases) counters.retained_bytes -= len;
3151 if (allocator_state) |state| {
3152 state.counters.frees += 1;
3153 state.counters.live_allocations -= 1;
3154 state.counters.freed_bytes += len;
3155 state.counters.live_bytes -= len;
3156 if (releases) state.counters.retained_bytes -= len;
3157 }
3158 scope_counters.frees += 1;
3159 scope_counters.live_allocations -= 1;
3160 scope_counters.freed_bytes += len;
3161 scope_counters.live_bytes -= len;
3162 if (releases) scope_counters.retained_bytes -= len;
3163 }
3164
3165 fn applyBulkInvalidationLocked(
3166 self: *Tracer,
3167 allocator_id: u32,
3168 scope_id: u32,
3169 len: usize,
3170 layer: event_mod.Layer,
3171 ) void {
3172 const counters = self.countersForLayer(layer);
3173 counters.bulk_invalidated_requests += 1;
3174 counters.bulk_invalidated_bytes += len;
3175 counters.live_allocations -= 1;
3176 counters.live_bytes -= len;
3177 const allocator = &self.allocators.items[allocator_id].counters;
3178 allocator.bulk_invalidated_requests += 1;
3179 allocator.bulk_invalidated_bytes += len;
3180 allocator.live_allocations -= 1;
3181 allocator.live_bytes -= len;
3182 const scope = self.scopeCountersForLayer(scope_id, layer);
3183 scope.bulk_invalidated_requests += 1;
3184 scope.bulk_invalidated_bytes += len;
3185 scope.live_allocations -= 1;
3186 scope.live_bytes -= len;
3187 }
3188
3189 fn applyLifetimeLocked(
3190 self: *Tracer,
3191 scope_id: u32,
3192 lifetime_events: u64,
3193 layer: event_mod.Layer,
3194 ) void {
3195 applyCompletedLifetime(self.countersForLayer(layer), lifetime_events);
3196 applyCompletedLifetime(
3197 self.scopeCountersForLayer(scope_id, layer),
3198 lifetime_events,
3199 );
3200 }
3201
3202 fn exitScope(self: *Tracer, scope_id: u32, previous: u32) void {
3203 self.lock();
3204 defer self.unlock();
3205 if (self.current_scope == scope_id) self.current_scope = previous;
3206 self.appendEventLocked(.{
3207 .kind = .scope_exit,
3208 .scope_id = scope_id,
3209 .label_id = self.scopes.items[scope_id].label_id,
3210 .live_bytes = self.countersForLayer(.backing_boundary).live_bytes,
3211 }) catch {};
3212 }
3213
3214 fn bootstrapRoot(self: *Tracer) !void {
3215 const owned = try self.control_allocator.dupe(u8, "root");
3216 errdefer self.control_allocator.free(owned);
3217 try self.labels.append(self.control_allocator, .{ .text = owned });
3218 try self.label_ids.putNoClobber(self.control_allocator, owned, root_label_id);
3219 try self.scopes.append(self.control_allocator, .{ .parent = root_scope_id, .label_id = root_label_id });
3220 }
3221
3222 fn internLabelLocked(self: *Tracer, text: []const u8) !u32 {
3223 if (self.label_ids.get(text)) |id| return id;
3224 const id = std.math.cast(u32, self.labels.items.len) orelse return error.OutOfMemory;
3225 const owned = try self.control_allocator.dupe(u8, text);
3226 errdefer self.control_allocator.free(owned);
3227 try self.labels.append(self.control_allocator, .{ .text = owned });
3228 try self.label_ids.putNoClobber(self.control_allocator, owned, id);
3229 return id;
3230 }
3231
3232 fn scopeChildLocked(self: *Tracer, parent: u32, label_id: u32) !u32 {
3233 const key = ChildKey{ .parent = parent, .label_id = label_id };
3234 if (self.scope_children.get(key)) |scope_id| return scope_id;
3235 const scope_id = std.math.cast(u32, self.scopes.items.len) orelse return error.OutOfMemory;
3236 try self.scopes.append(self.control_allocator, .{ .parent = parent, .label_id = label_id });
3237 try self.scope_children.putNoClobber(self.control_allocator, key, scope_id);
3238 return scope_id;
3239 }
3240
3241 fn clearDrainedAllocationsLocked(self: *Tracer) void {
3242 if (self.allocations.count() == 0) self.allocations.clearRetainingCapacity();
3243 }
3244
3245 fn noteObserverControlOperation(self: *Tracer) void {
3246 self.control_context.note();
3247 }
3248
3249 fn coverageManifest(self: *const Tracer) coverage_mod.Manifest {
3250 var manifest = self.coverage;
3251 manifest.observer_control_operations =
3252 self.observerControlOperations();
3253 return manifest;
3254 }
3255
3256 fn countersForLayer(
3257 self: *Tracer,
3258 layer: event_mod.Layer,
3259 ) *Counters {
3260 return &self.counters[@backingInt(layer)];
3261 }
3262
3263 fn scopeCountersForLayer(
3264 self: *Tracer,
3265 scope_id: u32,
3266 layer: event_mod.Layer,
3267 ) *ScopeCounters {
3268 return &self.scopes.items[scope_id].counters[@backingInt(layer)];
3269 }
3270
3271 fn lifecycleCoverageLocked(self: *const Tracer) LifecycleCoverage {
3272 var coverage: LifecycleCoverage = .{};
3273 for (self.allocators.items) |allocator| {
3274 if (allocator.layer != .logical_allocator) continue;
3275 if (allocator.retention == .releases_freed_memory) {
3276 coverage.not_required +|= 1;
3277 } else if (allocator.lifecycle_instrumented) {
3278 coverage.instrumented +|= 1;
3279 } else {
3280 coverage.uninstrumented +|= 1;
3281 }
3282 if (allocator.prefix_complete) {
3283 coverage.prefix_complete +|= 1;
3284 } else {
3285 coverage.prefix_partial +|= 1;
3286 }
3287 }
3288 return coverage;
3289 }
3290
3291 fn appendEventLocked(self: *Tracer, event: Event) !void {
3292 var stored = event;
3293 stored.seq = self.next_seq;
3294 self.next_seq += 1;
3295 if (!self.config.record_events) return;
3296 if (self.config.event_writer) |writer| {
3297 const label_text = if ((stored.kind == .allocator or
3298 stored.kind == .scope_enter) and
3299 stored.label_id < self.labels.items.len)
3300 self.labels.items[stored.label_id].text
3301 else
3302 null;
3303 const scope_text = if (stored.kind == .scope_enter and
3304 stored.scope_id < self.scopes.items.len)
3305 try self.scopePathCachedLocked(stored.scope_id)
3306 else
3307 null;
3308 try stored.writeJsonLine(writer, label_text, scope_text);
3309 return;
3310 }
3311 try self.events.append(self.control_allocator, stored);
3312 }
3313
3314 fn scopePathCachedLocked(self: *Tracer, scope_id: u32) ![]const u8 {
3315 if (self.scopes.items[scope_id].path) |path| return path;
3316 const path = try self.scopePathAllocLocked(scope_id);
3317 self.scopes.items[scope_id].path = path;
3318 return path;
3319 }
3320
3321 fn scopePathAllocLocked(self: *Tracer, scope_id: u32) ![]u8 {
3322 var stack = std.ArrayListUnmanaged(u32).empty;
3323 defer stack.deinit(self.control_allocator);
3324
3325 var current = scope_id;
3326 while (true) {
3327 try stack.append(self.control_allocator, current);
3328 if (current == root_scope_id) break;
3329 current = self.scopes.items[current].parent;
3330 }
3331
3332 var total: usize = 0;
3333 var index = stack.items.len;
3334 while (index > 0) {
3335 index -= 1;
3336 const label_id = self.scopes.items[stack.items[index]].label_id;
3337 total += self.labels.items[label_id].text.len;
3338 if (index != 0) total += 1;
3339 }
3340
3341 const out = try self.control_allocator.alloc(u8, total);
3342 var offset: usize = 0;
3343 index = stack.items.len;
3344 while (index > 0) {
3345 index -= 1;
3346 const label_id = self.scopes.items[stack.items[index]].label_id;
3347 const text = self.labels.items[label_id].text;
3348 @memcpy(out[offset .. offset + text.len], text);
3349 offset += text.len;
3350 if (index != 0) {
3351 out[offset] = '/';
3352 offset += 1;
3353 }
3354 }
3355 return out;
3356 }
3357
3358 fn lock(self: *Tracer) void {
3359 while (!self.mutex.tryLock()) std.atomic.spinLoopHint();
3360 }
3361
3362 fn unlock(self: *Tracer) void {
3363 self.mutex.unlock();
3364 }
3365 };
3366
3367 fn recordOwnedOperation(context: *anyopaque, event: observe.Event) void {
3368 const tracer: *Tracer = @ptrCast(@alignCast(context));
3369 var memory_suppression = sys.memory.observe.suppress();
3370 defer memory_suppression.deinit();
3371 tracer.recordOwnedEvent(event) catch tracer.markRecordingFailure();
3372 }
3373
3374 fn recordPhysicalOperation(
3375 context: *anyopaque,
3376 event: sys.memory.observe.Event,
3377 ) void {
3378 const tracer: *Tracer = @ptrCast(@alignCast(context));
3379 var causal_context = observe.beginContext();
3380 defer causal_context.finish();
3381 const operation_context = OperationContext{
3382 .layer = .physical_page,
3383 .operation_id = causal_context.id(),
3384 .parent_operation_id = causal_context.parentId(),
3385 .producer_id = @as(u64, @backingInt(event.source)) + 1,
3386 .producer = .sys_memory,
3387 };
3388 var allocation_suppression = observe.suppress();
3389 defer allocation_suppression.deinit();
3390 tracer.recordPhysicalEvent(
3391 event,
3392 operation_context,
3393 ) catch tracer.markRecordingFailure();
3394 }
3395
3396 fn boundaryOperationContext(context: observe.Context) OperationContext {
3397 return .{
3398 .operation_id = context.id(),
3399 .parent_operation_id = context.parentId(),
3400 };
3401 }
3402
3403 fn physicalEventKind(operation: sys.memory.observe.Operation) event_mod.Kind {
3404 return switch (operation) {
3405 .map => .map,
3406 .unmap => .unmap,
3407 .protect => .protect,
3408 .discard => .discard,
3409 .decommit => .decommit,
3410 .advise => .advise,
3411 };
3412 }
3413
3414 fn observedProducerRetention(producer: observe.Producer) Retention {
3415 return switch (producer) {
3416 .arena, .bump, .buffer_first, .fixed_buffer, .pool => .retains_freed_memory,
3417 .debug,
3418 .process,
3419 .phase,
3420 .limit,
3421 .boundary,
3422 .sys_memory,
3423 .custom,
3424 => .releases_freed_memory,
3425 };
3426 }
3427
3428 fn observedProducerLifecycleInstrumented(
3429 producer: observe.Producer,
3430 ) bool {
3431 return switch (producer) {
3432 .arena, .bump => true,
3433 .buffer_first,
3434 .fixed_buffer,
3435 .pool,
3436 .debug,
3437 .process,
3438 .phase,
3439 .limit,
3440 .boundary,
3441 .sys_memory,
3442 .custom,
3443 => false,
3444 };
3445 }
3446
3447 fn rawAlloc(ctx: *anyopaque, len: usize, alignment: Alignment, ret_addr: usize) ?[*]u8 {
3448 const self: *TracingAllocator = @ptrCast(@alignCast(ctx));
3449 return self.tracer.rawAlloc(self.backing, self.allocator_id, len, alignment, ret_addr);
3450 }
3451
3452 fn rawResize(ctx: *anyopaque, memory: []u8, alignment: Alignment, new_len: usize, ret_addr: usize) bool {
3453 const self: *TracingAllocator = @ptrCast(@alignCast(ctx));
3454 return self.tracer.rawResize(self.backing, self.allocator_id, memory, alignment, new_len, ret_addr);
3455 }
3456
3457 fn rawRemap(ctx: *anyopaque, memory: []u8, alignment: Alignment, new_len: usize, ret_addr: usize) ?[*]u8 {
3458 const self: *TracingAllocator = @ptrCast(@alignCast(ctx));
3459 return self.tracer.rawRemap(self.backing, self.allocator_id, memory, alignment, new_len, ret_addr);
3460 }
3461
3462 fn rawFree(ctx: *anyopaque, memory: []u8, alignment: Alignment, ret_addr: usize) void {
3463 const self: *TracingAllocator = @ptrCast(@alignCast(ctx));
3464 self.tracer.rawFree(self.backing, self.allocator_id, memory, alignment, ret_addr);
3465 }
3466
3467 const vtable: Allocator.VTable = .{
3468 .alloc = rawAlloc,
3469 .resize = rawResize,
3470 .remap = rawRemap,
3471 .free = rawFree,
3472 };
3473
3474 fn signedDiff(after: usize, before: usize) isize {
3475 if (after >= before) return @intCast(after - before);
3476 return -@as(isize, @intCast(before - after));
3477 }
3478
3479 fn completedLifetimeEvents(allocation_seq: u64, free_seq: u64) u64 {
3480 if (free_seq <= allocation_seq) return 0;
3481 return free_seq - allocation_seq;
3482 }
3483
3484 fn applyCompletedLifetime(counters: anytype, lifetime_events: u64) void {
3485 counters.completed_lifetimes += 1;
3486 counters.lifetime_total_events += lifetime_events;
3487 counters.lifetime_max_events = @max(counters.lifetime_max_events, lifetime_events);
3488 }
3489
3490 fn meanLifetimeEvents(counters: anytype) u64 {
3491 if (counters.completed_lifetimes == 0) return 0;
3492 return counters.lifetime_total_events / counters.completed_lifetimes;
3493 }
3494
3495 fn writeCounterFields(
3496 object: pretty_json.Object,
3497 counters: anytype,
3498 layer: event_mod.Layer,
3499 ) !void {
3500 if (layer == .logical_allocator) {
3501 try object.field("open_request_bytes", counters.live_bytes);
3502 try object.field(
3503 "high_water_open_request_bytes",
3504 counters.high_water_live_bytes,
3505 );
3506 try object.field(
3507 "bulk_invalidated_requests",
3508 counters.bulk_invalidated_requests,
3509 );
3510 try object.field(
3511 "bulk_invalidated_bytes",
3512 counters.bulk_invalidated_bytes,
3513 );
3514 try object.field("untracked_requests", counters.untracked_requests);
3515 try object.field(
3516 "untracked_request_bytes",
3517 counters.untracked_request_bytes,
3518 );
3519 } else {
3520 try object.field("retained_bytes", counters.retained_bytes);
3521 try object.field(
3522 "high_water_retained_bytes",
3523 counters.high_water_retained_bytes,
3524 );
3525 try object.field("live_bytes", counters.live_bytes);
3526 try object.field(
3527 "high_water_live_bytes",
3528 counters.high_water_live_bytes,
3529 );
3530 }
3531 try object.field("allocated_bytes", counters.allocated_bytes);
3532 if (layer == .logical_allocator) {
3533 try object.field("requested_bytes", counters.allocated_bytes);
3534 try object.field(
3535 "explicitly_closed_bytes",
3536 counters.freed_bytes,
3537 );
3538 } else {
3539 try object.field("freed_bytes", counters.freed_bytes);
3540 }
3541 try object.field("allocations", counters.allocations);
3542 try object.field("frees", counters.frees);
3543 if (layer == .logical_allocator) {
3544 try object.field("open_requests", counters.live_allocations);
3545 } else {
3546 try object.field("live_allocations", counters.live_allocations);
3547 }
3548 try object.field("completed_lifetimes", counters.completed_lifetimes);
3549 try object.field("lifetime_total_events", counters.lifetime_total_events);
3550 try object.field("lifetime_mean_events", meanLifetimeEvents(counters));
3551 try object.field("lifetime_max_events", counters.lifetime_max_events);
3552 }
3553
3554 fn scopeSummaryGreaterThan(_: void, left: ScopeSummary, right: ScopeSummary) bool {
3555 if (left.counters.retained_bytes != right.counters.retained_bytes) return left.counters.retained_bytes > right.counters.retained_bytes;
3556 if (left.counters.high_water_retained_bytes != right.counters.high_water_retained_bytes) return left.counters.high_water_retained_bytes > right.counters.high_water_retained_bytes;
3557 if (left.counters.live_bytes != right.counters.live_bytes) return left.counters.live_bytes > right.counters.live_bytes;
3558 if (left.counters.high_water_live_bytes != right.counters.high_water_live_bytes) return left.counters.high_water_live_bytes > right.counters.high_water_live_bytes;
3559 return std.mem.lessThan(u8, left.path, right.path);
3560 }
3561
3562 fn liveSiteSummaryGreaterThan(
3563 _: void,
3564 left: LiveSiteSummary,
3565 right: LiveSiteSummary,
3566 ) bool {
3567 if (left.live_bytes != right.live_bytes) {
3568 return left.live_bytes > right.live_bytes;
3569 }
3570 if (left.live_allocations != right.live_allocations) {
3571 return left.live_allocations > right.live_allocations;
3572 }
3573 if (left.key.return_address != right.key.return_address) {
3574 return left.key.return_address < right.key.return_address;
3575 }
3576 if (left.key.scope_id != right.key.scope_id) {
3577 return left.key.scope_id < right.key.scope_id;
3578 }
3579 return left.key.allocator_id < right.key.allocator_id;
3580 }
3581
3582 fn requestSiteSummaryGreaterThan(
3583 _: void,
3584 left: RequestSiteSummary,
3585 right: RequestSiteSummary,
3586 ) bool {
3587 if (left.counters.requested_bytes != right.counters.requested_bytes) {
3588 return left.counters.requested_bytes >
3589 right.counters.requested_bytes;
3590 }
3591 if (left.counters.calls != right.counters.calls) {
3592 return left.counters.calls > right.counters.calls;
3593 }
3594 if (left.key.return_address != right.key.return_address) {
3595 return left.key.return_address < right.key.return_address;
3596 }
3597 if (left.key.producer_id != right.key.producer_id) {
3598 return left.key.producer_id < right.key.producer_id;
3599 }
3600 if (left.key.scope_id != right.key.scope_id) {
3601 return left.key.scope_id < right.key.scope_id;
3602 }
3603 if (left.key.operation != right.key.operation) {
3604 return @backingInt(left.key.operation) <
3605 @backingInt(right.key.operation);
3606 }
3607 return left.key.succeeded and !right.key.succeeded;
3608 }
3609
3610 noinline fn sharedAllocation(allocator: Allocator) ![]u8 {
3611 return try allocator.alloc(u8, 8);
3612 }
3613
3614 noinline fn allocationCallerA(allocator: Allocator) ![]u8 {
3615 const bytes = try sharedAllocation(allocator);
3616 std.mem.doNotOptimizeAway(bytes.ptr);
3617 return bytes;
3618 }
3619
3620 noinline fn allocationCallerB(allocator: Allocator) ![]u8 {
3621 const bytes = try sharedAllocation(allocator);
3622 std.mem.doNotOptimizeAway(bytes.len);
3623 return bytes;
3624 }
3625
3626 const OperationFixtureAllocator = struct {
3627 backing: Allocator,
3628
3629 fn allocator(self: *@This()) Allocator {
3630 return .{ .ptr = self, .vtable = &operation_fixture_vtable };
3631 }
3632 };
3633
3634 fn operationFixtureAlloc(
3635 ctx: *anyopaque,
3636 len: usize,
3637 alignment: Alignment,
3638 ret_addr: usize,
3639 ) ?[*]u8 {
3640 const self: *OperationFixtureAllocator = @ptrCast(@alignCast(ctx));
3641 if (len == 13) return null;
3642 return self.backing.rawAlloc(len, alignment, ret_addr);
3643 }
3644
3645 fn operationFixtureResize(
3646 _: *anyopaque,
3647 _: []u8,
3648 _: Alignment,
3649 _: usize,
3650 _: usize,
3651 ) bool {
3652 return false;
3653 }
3654
3655 fn operationFixtureRemap(
3656 _: *anyopaque,
3657 _: []u8,
3658 _: Alignment,
3659 _: usize,
3660 _: usize,
3661 ) ?[*]u8 {
3662 return null;
3663 }
3664
3665 fn operationFixtureFree(
3666 ctx: *anyopaque,
3667 memory: []u8,
3668 alignment: Alignment,
3669 ret_addr: usize,
3670 ) void {
3671 const self: *OperationFixtureAllocator = @ptrCast(@alignCast(ctx));
3672 self.backing.rawFree(memory, alignment, ret_addr);
3673 }
3674
3675 const operation_fixture_vtable: Allocator.VTable = .{
3676 .alloc = operationFixtureAlloc,
3677 .resize = operationFixtureResize,
3678 .remap = operationFixtureRemap,
3679 .free = operationFixtureFree,
3680 };
3681
3682 fn ownedRequestEvent(
3683 operation_id: u64,
3684 operation: observe.Operation,
3685 generation: u64,
3686 address: usize,
3687 old_address: usize,
3688 len: usize,
3689 old_len: usize,
3690 return_address: usize,
3691 succeeded: bool,
3692 ) observe.Event {
3693 return .{
3694 .operation_id = operation_id,
3695 .parent_operation_id = 0,
3696 .producer_id = 9001,
3697 .producer = .arena,
3698 .generation = generation,
3699 .owner_cookie = 0xcafe,
3700 .operation = operation,
3701 .lifecycle_disposition = .none,
3702 .lifecycle_reason = .none,
3703 .address = address,
3704 .old_address = old_address,
3705 .len = len,
3706 .old_len = old_len,
3707 .alignment = 8,
3708 .return_address = return_address,
3709 .succeeded = succeeded,
3710 };
3711 }
3712
3713 fn ownedLifecycleEvent(
3714 operation_id: u64,
3715 generation: u64,
3716 disposition: observe.LifecycleDisposition,
3717 reason: observe.LifecycleReason,
3718 return_address: usize,
3719 ) observe.Event {
3720 return .{
3721 .operation_id = operation_id,
3722 .parent_operation_id = 0,
3723 .producer_id = 9001,
3724 .producer = .arena,
3725 .generation = generation,
3726 .owner_cookie = 0xcafe,
3727 .operation = .lifecycle,
3728 .lifecycle_disposition = disposition,
3729 .lifecycle_reason = reason,
3730 .address = 0,
3731 .old_address = 0,
3732 .len = 0,
3733 .old_len = 0,
3734 .alignment = 1,
3735 .return_address = return_address,
3736 .succeeded = true,
3737 };
3738 }
3739
3740 noinline fn captureSummaryReturnAddress() usize {
3741 const return_address = @returnAddress();
3742 std.mem.doNotOptimizeAway(return_address);
3743 return return_address;
3744 }
3745
3746 noinline fn summaryAttributionSource(tracer: *Tracer) !usize {
3747 const return_address = captureSummaryReturnAddress();
3748 try tracer.recordOwnedEvent(ownedRequestEvent(
3749 41,
3750 .alloc,
3751 0,
3752 0x5100,
3753 0,
3754 80,
3755 0,
3756 return_address,
3757 true,
3758 ));
3759 try tracer.recordOwnedEvent(ownedLifecycleEvent(
3760 42,
3761 0,
3762 .end,
3763 .deinit,
3764 return_address,
3765 ));
3766 std.mem.doNotOptimizeAway(tracer);
3767 return return_address;
3768 }
3769
3770 test "summary-only request attribution retains exact symbolizable binary" {
3771 if (comptime @import("builtin").os.tag != .linux) {
3772 return error.SkipZigTest;
3773 }
3774 var tracer = try Tracer.init(std.testing.allocator, .{
3775 .capture_executable_identity = true,
3776 });
3777 defer tracer.deinit();
3778 const expected_address: u64 = @intCast(
3779 try summaryAttributionSource(&tracer),
3780 );
3781
3782 var temporary = std.testing.tmpDir(.{});
3783 defer temporary.cleanup();
3784 const root_path = try temporary.dir.realPathFileAlloc(
3785 std.Options.debug_io,
3786 ".",
3787 std.testing.allocator,
3788 );
3789 defer std.testing.allocator.free(root_path);
3790 const summary_path = try std.fs.path.join(
3791 std.testing.allocator,
3792 &.{ root_path, "summary.jsonl" },
3793 );
3794 defer std.testing.allocator.free(summary_path);
3795 try tracer.writeExecutableArtifactPath(summary_path);
3796
3797 var summary = std.Io.Writer.Allocating.init(std.testing.allocator);
3798 defer summary.deinit();
3799 try tracer.writeExecutableMetadata(&summary.writer);
3800 try tracer.writeSummaryJsonl(&summary.writer, .{
3801 .layer = .logical_allocator,
3802 .include_zero_live = true,
3803 });
3804 try std.testing.expect(std.mem.indexOf(
3805 u8,
3806 summary.written(),
3807 "\"meta\":\"memtrace.executable\"",
3808 ) != null);
3809 try std.testing.expect(std.mem.indexOf(
3810 u8,
3811 summary.written(),
3812 "\"kind\":\"request_site\"",
3813 ) != null);
3814 var address_buffer: [64]u8 = undefined;
3815 const address_text = try std.fmt.bufPrint(
3816 &address_buffer,
3817 "\"return_address\":{d}",
3818 .{expected_address},
3819 );
3820 try std.testing.expect(std.mem.indexOf(
3821 u8,
3822 summary.written(),
3823 address_text,
3824 ) != null);
3825
3826 const artifact_path = try stack_mod.identity.artifactPathAlloc(
3827 std.testing.allocator,
3828 summary_path,
3829 );
3830 defer std.testing.allocator.free(artifact_path);
3831 var symbols = try stack_mod.symbolize.resolveAlloc(
3832 std.testing.allocator,
3833 artifact_path,
3834 &.{expected_address},
3835 );
3836 defer symbols.deinit(std.testing.allocator);
3837 const frames = symbols.find(expected_address);
3838 try std.testing.expect(frames.len != 0);
3839 var found_source = false;
3840 for (frames) |frame| {
3841 if (std.mem.indexOf(
3842 u8,
3843 frame.function,
3844 "summaryAttributionSource",
3845 ) == null) continue;
3846 if (std.mem.indexOf(u8, frame.location, "tracer.zig:") == null) {
3847 continue;
3848 }
3849 found_source = true;
3850 break;
3851 }
3852 try std.testing.expect(found_source);
3853 }
3854
3855 test "owned lifecycle permits deterministic same-address reuse" {
3856 var tracer = try Tracer.init(std.testing.allocator, .{
3857 .record_events = true,
3858 });
3859 defer tracer.deinit();
3860
3861 try tracer.recordOwnedEvent(ownedRequestEvent(
3862 1,
3863 .alloc,
3864 0,
3865 0x1000,
3866 0,
3867 64,
3868 0,
3869 0x101,
3870 true,
3871 ));
3872 try tracer.recordOwnedEvent(ownedLifecycleEvent(
3873 2,
3874 0,
3875 .invalidate,
3876 .reset,
3877 0x102,
3878 ));
3879 try tracer.recordOwnedEvent(ownedRequestEvent(
3880 3,
3881 .alloc,
3882 1,
3883 0x1000,
3884 0,
3885 64,
3886 0,
3887 0x103,
3888 true,
3889 ));
3890 try tracer.recordOwnedEvent(ownedRequestEvent(
3891 4,
3892 .free,
3893 1,
3894 0x1000,
3895 0x1000,
3896 0,
3897 64,
3898 0x104,
3899 true,
3900 ));
3901 try tracer.recordOwnedEvent(ownedLifecycleEvent(
3902 5,
3903 1,
3904 .end,
3905 .deinit,
3906 0x105,
3907 ));
3908
3909 const counters = tracer.countersForLayer(.logical_allocator);
3910 try std.testing.expectEqual(@as(u64, 2), counters.allocations);
3911 try std.testing.expectEqual(@as(u64, 1), counters.frees);
3912 try std.testing.expectEqual(
3913 @as(u64, 1),
3914 counters.bulk_invalidated_requests,
3915 );
3916 try std.testing.expectEqual(@as(usize, 64), counters.bulk_invalidated_bytes);
3917 try std.testing.expectEqual(@as(usize, 0), counters.live_allocations);
3918 try std.testing.expectEqual(@as(usize, 0), counters.live_bytes);
3919 try std.testing.expectEqual(@as(usize, 0), tracer.allocations.count());
3920 try std.testing.expectEqual(@as(usize, 0), tracer.anomaly_site_count);
3921
3922 var releases: u64 = 0;
3923 var lifecycles: u64 = 0;
3924 for (tracer.events.items) |event| {
3925 if (event.kind == .release) releases += 1;
3926 if (event.kind == .lifecycle) lifecycles += 1;
3927 }
3928 try std.testing.expectEqual(@as(u64, 1), releases);
3929 try std.testing.expectEqual(@as(u64, 2), lifecycles);
3930
3931 var summary = std.Io.Writer.Allocating.init(std.testing.allocator);
3932 defer summary.deinit();
3933 try tracer.writeSummaryJsonl(&summary.writer, .{
3934 .layer = .logical_allocator,
3935 .include_zero_live = true,
3936 });
3937 try std.testing.expect(std.mem.indexOf(
3938 u8,
3939 summary.written(),
3940 "\"open_requests\":0",
3941 ) != null);
3942 try std.testing.expect(std.mem.indexOf(
3943 u8,
3944 summary.written(),
3945 "\"bulk_invalidated_requests\":1",
3946 ) != null);
3947 try std.testing.expect(std.mem.indexOf(
3948 u8,
3949 summary.written(),
3950 "\"lifecycle_coverage\":\"complete\"",
3951 ) != null);
3952 try std.testing.expect(std.mem.indexOf(
3953 u8,
3954 summary.written(),
3955 "\"kind\":\"request_site\"",
3956 ) != null);
3957 try std.testing.expect(std.mem.indexOf(
3958 u8,
3959 summary.written(),
3960 "\"request_site_attribution_exact\":true",
3961 ) != null);
3962 try std.testing.expect(
3963 std.mem.indexOf(u8, summary.written(), "\"live_bytes\"") == null,
3964 );
3965 try std.testing.expect(
3966 std.mem.indexOf(u8, summary.written(), "\"retained_bytes\"") == null,
3967 );
3968 }
3969
3970 test "owned anomalies retain bounded exact call-site attribution" {
3971 var tracer = try Tracer.init(std.testing.allocator, .{
3972 .anomaly_site_capacity = 4,
3973 .request_site_capacity = 2,
3974 });
3975 defer tracer.deinit();
3976
3977 try tracer.recordOwnedEvent(ownedRequestEvent(
3978 1,
3979 .alloc,
3980 0,
3981 0x1000,
3982 0,
3983 64,
3984 0,
3985 0x201,
3986 true,
3987 ));
3988 try tracer.recordOwnedEvent(ownedRequestEvent(
3989 2,
3990 .alloc,
3991 0,
3992 0x1000,
3993 0,
3994 32,
3995 0,
3996 0x202,
3997 true,
3998 ));
3999 try tracer.recordOwnedEvent(ownedRequestEvent(
4000 3,
4001 .resize,
4002 0,
4003 0,
4004 0x2000,
4005 48,
4006 24,
4007 0x203,
4008 true,
4009 ));
4010 try tracer.recordOwnedEvent(ownedRequestEvent(
4011 4,
4012 .remap,
4013 0,
4014 0x3100,
4015 0x3000,
4016 96,
4017 48,
4018 0x204,
4019 true,
4020 ));
4021 try tracer.recordOwnedEvent(ownedRequestEvent(
4022 5,
4023 .free,
4024 0,
4025 0x4000,
4026 0x4000,
4027 0,
4028 16,
4029 0x205,
4030 true,
4031 ));
4032 try tracer.recordOwnedEvent(ownedRequestEvent(
4033 6,
4034 .free,
4035 0,
4036 0x1000,
4037 0x1000,
4038 0,
4039 63,
4040 0x206,
4041 true,
4042 ));
4043
4044 const counters = tracer.countersForLayer(.logical_allocator);
4045 try std.testing.expectEqual(@as(u64, 1), counters.unmatched_frees);
4046 try std.testing.expectEqual(@as(u64, 1), counters.unmatched_resizes);
4047 try std.testing.expectEqual(@as(u64, 1), counters.unmatched_remaps);
4048 try std.testing.expectEqual(@as(usize, 4), tracer.anomaly_site_count);
4049 try std.testing.expectEqual(@as(u64, 1), tracer.anomaly_sites_dropped);
4050
4051 var summary = std.Io.Writer.Allocating.init(std.testing.allocator);
4052 defer summary.deinit();
4053 try tracer.writeSummaryJsonl(&summary.writer, .{
4054 .layer = .logical_allocator,
4055 .include_zero_live = true,
4056 });
4057 const text = summary.written();
4058 try std.testing.expect(
4059 std.mem.indexOf(u8, text, "\"anomaly_sites_dropped\":1") != null,
4060 );
4061 try std.testing.expect(
4062 std.mem.indexOf(
4063 u8,
4064 text,
4065 "\"anomaly_attribution_exact\":false",
4066 ) != null,
4067 );
4068 try std.testing.expect(
4069 std.mem.indexOf(
4070 u8,
4071 text,
4072 "\"request_site_unaggregated_calls\":4",
4073 ) != null,
4074 );
4075 try std.testing.expect(
4076 std.mem.indexOf(
4077 u8,
4078 text,
4079 "\"request_site_attribution_exact\":false",
4080 ) != null,
4081 );
4082 inline for (.{ "alloc", "resize", "remap", "free" }) |operation| {
4083 var needle_buffer: [48]u8 = undefined;
4084 const needle = try std.fmt.bufPrint(
4085 &needle_buffer,
4086 "\"operation\":\"{s}\"",
4087 .{operation},
4088 );
4089 try std.testing.expect(std.mem.indexOf(u8, text, needle) != null);
4090 }
4091 }
4092
4093 test "exact attribution separates callers behind one allocation site" {
4094 var tracer = try Tracer.init(std.testing.allocator, .{
4095 .record_events = true,
4096 .allocation_attribution = .stack,
4097 });
4098 defer tracer.deinit();
4099 var traced = try tracer.tracedAllocator(std.testing.allocator, "exact");
4100 const allocator = traced.allocator();
4101 const first = try allocationCallerA(allocator);
4102 const second = try allocationCallerB(allocator);
4103 defer allocator.free(first);
4104 defer allocator.free(second);
4105
4106 var first_event: ?Event = null;
4107 var second_event: ?Event = null;
4108 for (tracer.events.items) |recorded| {
4109 if (recorded.kind != .alloc) continue;
4110 if (first_event == null) {
4111 first_event = recorded;
4112 } else {
4113 second_event = recorded;
4114 break;
4115 }
4116 }
4117 try std.testing.expect(first_event != null);
4118 try std.testing.expect(second_event != null);
4119 try std.testing.expectEqual(
4120 first_event.?.return_address,
4121 second_event.?.return_address,
4122 );
4123 try std.testing.expect(first_event.?.stack_id != 0);
4124 try std.testing.expect(second_event.?.stack_id != 0);
4125 try std.testing.expect(first_event.?.stack_id != second_event.?.stack_id);
4126 try std.testing.expect(
4127 tracer.stacks.recordForId(first_event.?.stack_id).complete(),
4128 );
4129 try std.testing.expect(
4130 tracer.stacks.recordForId(second_event.?.stack_id).complete(),
4131 );
4132 }
4133
4134 test "exact attribution covers every operation and failed outcome" {
4135 var tracer = try Tracer.init(std.testing.allocator, .{
4136 .record_events = true,
4137 .allocation_attribution = .stack,
4138 });
4139 defer tracer.deinit();
4140 var fixture = OperationFixtureAllocator{ .backing = std.testing.allocator };
4141 var traced = try tracer.tracedAllocator(fixture.allocator(), "operations");
4142 const allocator = traced.allocator();
4143
4144 const bytes = try allocator.alloc(u8, 8);
4145 try std.testing.expect(!allocator.resize(bytes, 16));
4146 try std.testing.expect(allocator.remap(bytes, 16) == null);
4147 try std.testing.expectError(error.OutOfMemory, allocator.alloc(u8, 13));
4148 allocator.free(bytes);
4149
4150 var calls: u64 = 0;
4151 var successes: u64 = 0;
4152 var failures: u64 = 0;
4153 for (tracer.events.items) |event| {
4154 if (!event.kind.isMemoryOperation()) continue;
4155 calls += 1;
4156 if (event.succeeded) {
4157 successes += 1;
4158 } else {
4159 failures += 1;
4160 }
4161 try std.testing.expect(event.stack_id != 0);
4162 try std.testing.expect(
4163 tracer.stacks.recordForId(event.stack_id).complete(),
4164 );
4165 }
4166 try std.testing.expectEqual(@as(u64, 5), calls);
4167 try std.testing.expectEqual(@as(u64, 2), successes);
4168 try std.testing.expectEqual(@as(u64, 3), failures);
4169 try std.testing.expectEqual(@as(u64, 0), tracer.recording_failures);
4170 }
4171
4172 test "zero-length and predispatch failures have no memory effect" {
4173 var tracer = try Tracer.init(std.testing.allocator, .{
4174 .record_events = true,
4175 });
4176 defer tracer.deinit();
4177 var traced = try tracer.tracedAllocator(std.testing.allocator, "requests");
4178 const allocator = traced.allocator();
4179 const before = tracer.snapshot();
4180
4181 const empty = try allocator.alloc(u8, 0);
4182 allocator.free(empty);
4183 const impossible_count = std.math.maxInt(usize);
4184 try std.testing.expectError(
4185 error.OutOfMemory,
4186 allocator.alloc(u64, impossible_count),
4187 );
4188
4189 const after = tracer.snapshot();
4190 try std.testing.expectEqual(before, after);
4191 for (tracer.events.items) |event| {
4192 try std.testing.expect(!event.kind.isMemoryOperation());
4193 }
4194 }
4195
4196 test "nested traced allocator identities may own the same address" {
4197 var tracer = try Tracer.init(std.testing.allocator, .{ .record_events = true });
4198 defer tracer.deinit();
4199 var backing = try tracer.tracedAllocator(std.testing.allocator, "backing");
4200 var logical = try tracer.tracedAllocator(backing.allocator(), "logical");
4201 const allocator = logical.allocator();
4202
4203 const bytes = try allocator.alloc(u8, 32);
4204 allocator.free(bytes);
4205
4206 var allocations: u64 = 0;
4207 var frees: u64 = 0;
4208 for (tracer.events.items) |event| {
4209 switch (event.kind) {
4210 .alloc => allocations += 1,
4211 .free => frees += 1,
4212 else => {},
4213 }
4214 }
4215 try std.testing.expectEqual(@as(u64, 2), allocations);
4216 try std.testing.expectEqual(@as(u64, 2), frees);
4217 try std.testing.expectEqual(
4218 @as(u64, 0),
4219 tracer.countersForLayer(.backing_boundary).unmatched_frees,
4220 );
4221 try std.testing.expectEqual(@as(usize, 0), tracer.allocations.count());
4222 }
4223
4224 test "traced allocator attributes live allocations to scopes" {
4225 var tracer = try Tracer.init(std.testing.allocator, .{});
4226 defer tracer.deinit();
4227
4228 var traced = try tracer.tracedAllocator(std.testing.allocator, "test.heap");
4229 const allocator = traced.allocator();
4230
4231 var outer = try tracer.enter("outer");
4232 const first = try allocator.alloc(u8, 32);
4233 errdefer allocator.free(first);
4234 var inner = try tracer.enter("inner");
4235 const second = try allocator.alloc(u8, 16);
4236 inner.exit();
4237 allocator.free(second);
4238 outer.exit();
4239
4240 const snap = tracer.snapshot();
4241 try std.testing.expectEqual(@as(usize, 32), snap.live_bytes);
4242 try std.testing.expectEqual(@as(usize, 2), snap.allocations);
4243 try std.testing.expectEqual(@as(usize, 1), snap.frees);
4244 try std.testing.expectEqual(@as(u64, 1), snap.completed_lifetimes);
4245 try std.testing.expectEqual(@as(u64, 2), snap.lifetime_total_events);
4246 try std.testing.expectEqual(@as(u64, 2), snap.lifetime_max_events);
4247
4248 var out = std.Io.Writer.Allocating.init(std.testing.allocator);
4249 defer out.deinit();
4250 try tracer.writeSummary(&out.writer, .{ .top = 8 });
4251 const text = out.written();
4252 try std.testing.expect(std.mem.indexOf(u8, text, "memtrace lifetimes completed=1 total_events=2 mean_events=2 max_events=2") != null);
4253 try std.testing.expect(std.mem.indexOf(u8, text, "root/outer retained_bytes=32") != null);
4254 try std.testing.expect(std.mem.indexOf(u8, text, "live_bytes=32") != null);
4255 try std.testing.expect(std.mem.indexOf(u8, text, "root/outer/inner retained_bytes=0 high_water_retained_bytes=16 live_bytes=0 high_water_live_bytes=16 allocated_bytes=16 freed_bytes=16 allocations=1 frees=1 live_allocations=0 completed_lifetimes=1 lifetime_total_events=2 lifetime_mean_events=2 lifetime_max_events=2") != null);
4256
4257 var jsonl = std.Io.Writer.Allocating.init(std.testing.allocator);
4258 defer jsonl.deinit();
4259 try tracer.writeSummaryJsonl(&jsonl.writer, .{ .top = 8 });
4260 try std.testing.expect(std.mem.indexOf(
4261 u8,
4262 jsonl.written(),
4263 "{\"kind\":\"summary\",\"layer\":\"backing_boundary\"," ++
4264 "\"allocations\":2,\"frees\":1",
4265 ) != null);
4266 try std.testing.expect(std.mem.indexOf(
4267 u8,
4268 jsonl.written(),
4269 "\"unmatched_frees\":0",
4270 ) != null);
4271 try std.testing.expect(std.mem.indexOf(
4272 u8,
4273 jsonl.written(),
4274 "\"kind\":\"scope\",\"layer\":\"backing_boundary\"," ++
4275 "\"scope\":\"root/outer/inner\"",
4276 ) != null);
4277 try std.testing.expect(std.mem.indexOf(
4278 u8,
4279 jsonl.written(),
4280 "\"kind\":\"live_site\",\"layer\":\"backing_boundary\",",
4281 ) != null);
4282
4283 allocator.free(first);
4284 }
4285
4286 test "snapshot diffs report allocator deltas" {
4287 var tracer = try Tracer.init(std.testing.allocator, .{});
4288 defer tracer.deinit();
4289
4290 var traced = try tracer.tracedAllocator(std.testing.allocator, "test.heap");
4291 const allocator = traced.allocator();
4292 const before = tracer.snapshot();
4293 const bytes = try allocator.alloc(u8, 24);
4294 const during = tracer.diff(before);
4295 try std.testing.expectEqual(@as(isize, 24), during.live_bytes);
4296 try std.testing.expectEqual(@as(u64, 1), during.allocations);
4297 allocator.free(bytes);
4298 const after_free = tracer.diff(before);
4299 try std.testing.expectEqual(@as(u64, 1), after_free.completed_lifetimes);
4300 try std.testing.expectEqual(@as(u64, 1), after_free.lifetime_total_events);
4301 try std.testing.expectEqual(@as(u64, 1), after_free.lifetime_max_events);
4302 }
4303
4304 test "retaining allocator reports capacity pressure after logical frees" {
4305 var tracer = try Tracer.init(std.testing.allocator, .{});
4306 defer tracer.deinit();
4307
4308 var traced = try tracer.tracedAllocatorWithOptions(std.testing.allocator, .{
4309 .name = "arena",
4310 .retention = .retains_freed_memory,
4311 });
4312 const allocator = traced.allocator();
4313 var scope = try tracer.enter("phase");
4314 const bytes = try allocator.alloc(u8, 64);
4315 allocator.free(bytes);
4316 scope.exit();
4317
4318 const snap = tracer.snapshot();
4319 try std.testing.expectEqual(@as(usize, 0), snap.live_bytes);
4320 try std.testing.expectEqual(@as(usize, 64), snap.retained_bytes);
4321
4322 var out = std.Io.Writer.Allocating.init(std.testing.allocator);
4323 defer out.deinit();
4324 try tracer.writeSummary(&out.writer, .{ .top = 8, .include_zero_live = true });
4325 try std.testing.expect(std.mem.indexOf(u8, out.written(), "root/phase retained_bytes=64") != null);
4326 }
4327
4328 test "event log records alloc and free lines" {
4329 var tracer = try Tracer.init(std.testing.allocator, .{ .record_events = true });
4330 defer tracer.deinit();
4331
4332 var traced = try tracer.tracedAllocator(std.testing.allocator, "test.heap");
4333 const allocator = traced.allocator();
4334 var scope = try tracer.enter("phase");
4335 const bytes = try allocator.alloc(u8, 8);
4336 allocator.free(bytes);
4337 scope.exit();
4338
4339 var out = std.Io.Writer.Allocating.init(std.testing.allocator);
4340 defer out.deinit();
4341 try tracer.writeEventsJsonl(&out.writer);
4342 const text = out.written();
4343 try std.testing.expect(std.mem.indexOf(
4344 u8,
4345 text,
4346 "{\"v\":3,\"seq\":1,\"kind\":\"trace.start\"",
4347 ) != null);
4348 try std.testing.expect(std.mem.startsWith(
4349 u8,
4350 text,
4351 "{\"v\":3,\"meta\":\"memtrace.coverage\"",
4352 ));
4353 try std.testing.expect(std.mem.indexOf(u8, text, "\"kind\":\"alloc\"") != null);
4354 try std.testing.expect(std.mem.indexOf(u8, text, "\"kind\":\"free\"") != null);
4355 try std.testing.expect(std.mem.indexOf(u8, text, "\"scope\":\"root/phase\"") != null);
4356 try std.testing.expect(std.mem.endsWith(
4357 u8,
4358 text,
4359 "\"kind\":\"trace.stop\"}\n",
4360 ));
4361 try std.testing.expect(std.mem.indexOf(u8, text, "\"kind\":\"trace.stop\"") != null);
4362 }
4363
4364 test "observation publication window reconciles producer prefix floor" {
4365 var tracer = try Tracer.init(std.testing.allocator, .{
4366 .record_events = true,
4367 });
4368 defer tracer.deinit();
4369 const previous = tracer.markObservationProducerFloorPending();
4370 try std.testing.expectEqual(@as(u64, 0), previous);
4371
4372 var preexisting = ownedRequestEvent(
4373 1,
4374 .alloc,
4375 0,
4376 0x7100,
4377 0,
4378 64,
4379 0,
4380 0x101,
4381 true,
4382 );
4383 preexisting.producer_id = 7001;
4384 try tracer.recordOwnedEvent(preexisting);
4385 var created_during_publication = ownedRequestEvent(
4386 2,
4387 .alloc,
4388 0,
4389 0x7300,
4390 0,
4391 32,
4392 0,
4393 0x102,
4394 true,
4395 );
4396 created_during_publication.producer_id = 7003;
4397 try tracer.recordOwnedEvent(created_during_publication);
4398
4399 const preexisting_id = tracer.observed_allocators.get(.{
4400 .producer_id = preexisting.producer_id,
4401 .producer = preexisting.producer,
4402 }).?;
4403 const created_id = tracer.observed_allocators.get(.{
4404 .producer_id = created_during_publication.producer_id,
4405 .producer = created_during_publication.producer,
4406 }).?;
4407 try std.testing.expect(
4408 !tracer.allocators.items[preexisting_id].prefix_complete,
4409 );
4410 try std.testing.expect(
4411 tracer.allocators.items[preexisting_id]
4412 .observation_floor_pending,
4413 );
4414 try std.testing.expect(
4415 !tracer.allocators.items[created_id].prefix_complete,
4416 );
4417 try std.testing.expect(
4418 tracer.allocators.items[created_id]
4419 .observation_floor_pending,
4420 );
4421
4422 tracer.reconcileObservationProducerFloor(7002);
4423 try std.testing.expect(
4424 !tracer.allocators.items[preexisting_id].prefix_complete,
4425 );
4426 try std.testing.expect(
4427 !tracer.allocators.items[preexisting_id]
4428 .observation_floor_pending,
4429 );
4430 try std.testing.expect(
4431 tracer.allocators.items[created_id].prefix_complete,
4432 );
4433 try std.testing.expect(
4434 !tracer.allocators.items[created_id]
4435 .observation_floor_pending,
4436 );
4437 var emitted_correction = false;
4438 for (tracer.events.items) |event| {
4439 if (event.kind != .allocator) continue;
4440 if (event.allocator_id != created_id) continue;
4441 if (!event.observation_prefix_complete) continue;
4442 emitted_correction = true;
4443 break;
4444 }
4445 try std.testing.expect(emitted_correction);
4446 }
4447
4448 test "lifecycle coverage treats release-explicit producer as not required" {
4449 var tracer = try Tracer.init(std.testing.allocator, .{});
4450 defer tracer.deinit();
4451 try tracer.recordOwnedEvent(ownedRequestEvent(
4452 1,
4453 .alloc,
4454 0,
4455 0x8100,
4456 0,
4457 64,
4458 0,
4459 0x201,
4460 true,
4461 ));
4462 var limit = ownedRequestEvent(
4463 2,
4464 .alloc,
4465 0,
4466 0x8200,
4467 0,
4468 32,
4469 0,
4470 0x202,
4471 true,
4472 );
4473 limit.producer_id = 9002;
4474 limit.producer = .limit;
4475 try tracer.recordOwnedEvent(limit);
4476
4477 var summary = std.Io.Writer.Allocating.init(std.testing.allocator);
4478 defer summary.deinit();
4479 try tracer.writeSummaryJsonl(&summary.writer, .{
4480 .layer = .logical_allocator,
4481 .include_zero_live = true,
4482 });
4483 try std.testing.expect(std.mem.indexOf(
4484 u8,
4485 summary.written(),
4486 "\"lifecycle_coverage\":\"complete\"",
4487 ) != null);
4488 try std.testing.expect(std.mem.indexOf(
4489 u8,
4490 summary.written(),
4491 "\"lifecycle_instrumented_producers\":1",
4492 ) != null);
4493 try std.testing.expect(std.mem.indexOf(
4494 u8,
4495 summary.written(),
4496 "\"lifecycle_uninstrumented_producers\":0",
4497 ) != null);
4498 try std.testing.expect(std.mem.indexOf(
4499 u8,
4500 summary.written(),
4501 "\"lifecycle_not_required_producers\":1",
4502 ) != null);
4503 try std.testing.expect(std.mem.indexOf(
4504 u8,
4505 summary.written(),
4506 "\"prefix_complete_producers\":2",
4507 ) != null);
4508 }
4509
4510 test "owned logical operations correlate nested backing boundaries" {
4511 if (!observe.enabled) return error.SkipZigTest;
4512 var tracer = try Tracer.init(std.testing.allocator, .{
4513 .record_events = true,
4514 });
4515 defer tracer.deinit();
4516 var backing = try tracer.tracedAllocator(
4517 sys.memory.page_allocator,
4518 "backing",
4519 );
4520 var observation = try tracer.observeOwnedAllocators();
4521 defer observation.stop();
4522 var arena = std.heap.ArenaAllocator.init(backing.allocator());
4523 defer arena.deinit();
4524 const backing_before = tracer.snapshotLayer(.backing_boundary);
4525 const logical_before = tracer.snapshotLayer(.logical_allocator);
4526
4527 const producer_id = observe.producerId();
4528 var span = observe.begin(
4529 producer_id,
4530 .arena,
4531 .alloc,
4532 0,
4533 0,
4534 48,
4535 8,
4536 @returnAddress(),
4537 );
4538 const address = arena.allocator().rawAlloc(
4539 48,
4540 .@"8",
4541 @returnAddress(),
4542 ) orelse return error.OutOfMemory;
4543 span.finish(.{
4544 .address = @intFromPtr(address),
4545 .succeeded = true,
4546 });
4547
4548 var logical: ?Event = null;
4549 var boundary: ?Event = null;
4550 for (tracer.events.items) |event| {
4551 if (event.kind != .alloc) continue;
4552 switch (event.layer) {
4553 .logical_allocator => logical = event,
4554 .backing_boundary => boundary = event,
4555 .physical_page => {},
4556 }
4557 }
4558 const logical_event = logical orelse
4559 return error.MissingLogicalAllocation;
4560 const boundary_event = boundary orelse
4561 return error.MissingBackingAllocation;
4562 try std.testing.expectEqual(
4563 logical_event.operation_id,
4564 boundary_event.parent_operation_id,
4565 );
4566 try std.testing.expectEqual(producer_id, logical_event.producer_id);
4567 try std.testing.expectEqual(
4568 observe.Producer.arena,
4569 logical_event.producer,
4570 );
4571 try std.testing.expectEqual(
4572 observe.Producer.boundary,
4573 boundary_event.producer,
4574 );
4575 const backing_difference = tracer.diffLayer(.backing_boundary, backing_before);
4576 const logical_difference = tracer.diffLayer(.logical_allocator, logical_before);
4577 try std.testing.expectEqual(@as(u64, 1), backing_difference.allocations);
4578 try std.testing.expectEqual(@as(u64, 1), logical_difference.allocations);
4579 try std.testing.expect(backing_difference.allocated_bytes > 48);
4580 try std.testing.expectEqual(@as(usize, 48), logical_difference.allocated_bytes);
4581
4582 var logical_summary = std.Io.Writer.Allocating.init(
4583 std.testing.allocator,
4584 );
4585 defer logical_summary.deinit();
4586 try tracer.writeSummary(
4587 &logical_summary.writer,
4588 .{ .layer = .logical_allocator },
4589 );
4590 try std.testing.expect(std.mem.indexOf(
4591 u8,
4592 logical_summary.written(),
4593 "memtrace layer=logical_allocator allocations=1",
4594 ) != null);
4595 try std.testing.expectEqual(@as(u64, 0), tracer.recording_failures);
4596 }
4597
4598 test "observer control remains isolated when backing allocators alias" {
4599 if (!observe.enabled) return error.SkipZigTest;
4600 var tracer = try Tracer.init(std.testing.allocator, .{
4601 .record_events = true,
4602 });
4603 defer tracer.deinit();
4604 var traced = try tracer.tracedAllocator(
4605 std.testing.allocator,
4606 "aliased",
4607 );
4608 var observation = try tracer.observeOwnedAllocators();
4609 defer observation.stop();
4610 const bytes = try traced.allocator().alloc(u8, 32);
4611 traced.allocator().free(bytes);
4612 try std.testing.expectEqual(
4613 @as(u64, 1),
4614 tracer.snapshot().allocations,
4615 );
4616 try std.testing.expect(tracer.observerControlOperations() > 0);
4617 try std.testing.expectEqual(@as(u64, 0), tracer.recording_failures);
4618 }
4619
4620 test "owned observation correlates logical backing and physical operations" {
4621 if (!observe.enabled) return error.SkipZigTest;
4622 if (!sys.memory.anonymousMappingSupported()) return error.SkipZigTest;
4623 var tracer = try Tracer.init(std.testing.allocator, .{
4624 .record_events = true,
4625 });
4626 defer tracer.deinit();
4627 var backing = try tracer.tracedAllocator(
4628 sys.memory.page_allocator,
4629 "owned.page",
4630 );
4631 var observation = try tracer.observeOwnedAllocators();
4632 defer observation.stop();
4633
4634 const producer_id = observe.producerId();
4635 var logical_span = observe.begin(
4636 producer_id,
4637 .custom,
4638 .alloc,
4639 0,
4640 0,
4641 33,
4642 1,
4643 @returnAddress(),
4644 );
4645 const bytes = try backing.allocator().alloc(u8, 33);
4646 logical_span.finish(.{
4647 .address = @intFromPtr(bytes.ptr),
4648 .succeeded = true,
4649 });
4650
4651 var logical: ?Event = null;
4652 var boundary: ?Event = null;
4653 var physical: ?Event = null;
4654 for (tracer.events.items) |recorded| {
4655 if (recorded.kind != .alloc and recorded.kind != .map) continue;
4656 switch (recorded.layer) {
4657 .logical_allocator => logical = recorded,
4658 .backing_boundary => boundary = recorded,
4659 .physical_page => physical = recorded,
4660 }
4661 }
4662 const logical_event = logical orelse
4663 return error.MissingLogicalAllocation;
4664 const boundary_event = boundary orelse
4665 return error.MissingBackingAllocation;
4666 const physical_event = physical orelse
4667 return error.MissingPhysicalMapping;
4668 try std.testing.expectEqual(
4669 logical_event.operation_id,
4670 boundary_event.parent_operation_id,
4671 );
4672 try std.testing.expectEqual(
4673 boundary_event.operation_id,
4674 physical_event.parent_operation_id,
4675 );
4676 try std.testing.expectEqual(
4677 observe.Producer.sys_memory,
4678 physical_event.producer,
4679 );
4680 try std.testing.expect(physical_event.operation_id != 0);
4681 try std.testing.expectEqual(@as(u64, 0), tracer.recording_failures);
4682
4683 backing.allocator().free(bytes);
4684 }
4685
4686 test "owned observation fails closed when not compiled" {
4687 if (observe.enabled) return error.SkipZigTest;
4688 var tracer = try Tracer.init(std.testing.allocator, .{});
4689 defer tracer.deinit();
4690 try std.testing.expectError(
4691 error.AllocatorObservationNotCompiled,
4692 tracer.observeOwnedAllocators(),
4693 );
4694 }
4695
4696 test "optional owned observation follows compiled capability" {
4697 var tracer = try Tracer.init(std.testing.allocator, .{});
4698 defer tracer.deinit();
4699 var observation = try tracer.observeOwnedAllocatorsIfAvailable();
4700 if (comptime observe.enabled and sys.memory.observe.enabled) {
4701 try std.testing.expect(observation != null);
4702 observation.?.stop();
4703 } else {
4704 try std.testing.expect(observation == null);
4705 }
4706 }
4707
4708 test "event log can stream without retaining records" {
4709 var out = std.Io.Writer.Allocating.init(std.testing.allocator);
4710 defer out.deinit();
4711
4712 var tracer = try Tracer.init(std.testing.allocator, .{
4713 .record_events = true,
4714 .event_writer = &out.writer,
4715 });
4716 defer tracer.deinit();
4717
4718 var traced = try tracer.tracedAllocator(std.testing.allocator, "test.heap");
4719 const allocator = traced.allocator();
4720 var scope = try tracer.enter("phase");
4721 const bytes = try allocator.alloc(u8, 8);
4722 allocator.free(bytes);
4723 scope.exit();
4724 try tracer.finishEvents();
4725
4726 try std.testing.expectEqual(@as(usize, 0), tracer.events.items.len);
4727 const text = out.written();
4728 try std.testing.expect(std.mem.indexOf(u8, text, "\"kind\":\"trace.start\"") != null);
4729 try std.testing.expect(std.mem.indexOf(u8, text, "\"kind\":\"alloc\"") != null);
4730 try std.testing.expect(std.mem.indexOf(u8, text, "\"kind\":\"free\"") != null);
4731 try std.testing.expect(std.mem.indexOf(u8, text, "\"scope\":\"root/phase\"") != null);
4732 try std.testing.expect(std.mem.indexOf(
4733 u8,
4734 text,
4735 "\"universe\":\"registered_allocator_handles\"",
4736 ) != null);
4737 try std.testing.expect(std.mem.indexOf(u8, text, "\"kind\":\"trace.stop\"") != null);
4738 }
4739
4740 test "streamed coverage records the final observed universe" {
4741 if (!observe.enabled) return error.SkipZigTest;
4742 var output = std.Io.Writer.Allocating.init(std.testing.allocator);
4743 defer output.deinit();
4744 var tracer = try Tracer.init(std.testing.allocator, .{
4745 .record_events = true,
4746 .event_writer = &output.writer,
4747 });
4748 defer tracer.deinit();
4749 var observation = try tracer.observeOwnedAllocators();
4750 observation.stop();
4751 try tracer.finishEvents();
4752
4753 try std.testing.expect(std.mem.indexOf(
4754 u8,
4755 output.written(),
4756 "\"universe\":\"process_memory_operations_during_trace_epoch\"",
4757 ) != null);
4758 try std.testing.expectEqual(
4759 @as(usize, 1),
4760 std.mem.count(
4761 u8,
4762 output.written(),
4763 "\"meta\":\"memtrace.coverage\"",
4764 ),
4765 );
4766 }
4767
4768 test "observer control allocation is counted and physically isolated" {
4769 if (!observe.enabled) return error.SkipZigTest;
4770 var tracer = try Tracer.init(sys.memory.page_allocator, .{});
4771 defer tracer.deinit();
4772 var observation = try tracer.observeOwnedAllocators();
4773 defer observation.stop();
4774 const before = tracer.snapshotLayer(.physical_page);
4775 inline for (.{
4776 "control.one",
4777 "control.two",
4778 "control.three",
4779 "control.four",
4780 "control.five",
4781 "control.six",
4782 "control.seven",
4783 "control.eight",
4784 }) |label| {
4785 var scope = try tracer.enter(label);
4786 scope.exit();
4787 }
4788 const after = tracer.snapshotLayer(.physical_page);
4789 try std.testing.expect(tracer.observerControlOperations() > 0);
4790 try std.testing.expectEqual(
4791 before.allocated_bytes,
4792 after.allocated_bytes,
4793 );
4794 try std.testing.expectEqual(
4795 before.freed_bytes,
4796 after.freed_bytes,
4797 );
4798 }
4799
4800 test "event serialization rejects unretained streams" {
4801 var tracer = try Tracer.init(std.testing.allocator, .{});
4802 defer tracer.deinit();
4803 var out = std.Io.Writer.Allocating.init(std.testing.allocator);
4804 defer out.deinit();
4805 try std.testing.expectError(error.EventsNotRetained, tracer.writeEventsJsonl(&out.writer));
4806 }