lib/sys/src/memory.zig
daab053ee43316e1809a84551d573ddd1e5bf3d2
1 const std = @import("std");
2 const builtin = @import("builtin");
3 const capabilities = @import("capabilities.zig");
4 const observe_mod = @import("observer.zig");
5
6 pub const required_capabilities = capabilities.noLibc(&.{.memory_mapping});
7 pub const observe = observe_mod;
8
9 pub const MapError = error{
10 AccessDenied,
11 OutOfMemory,
12 PermissionDenied,
13 UnsupportedPlatform,
14 MapFailed,
15 };
16
17 pub const ProtectError = error{
18 AccessDenied,
19 InvalidMapping,
20 OutOfMemory,
21 PermissionDenied,
22 UnsupportedPlatform,
23 ProtectFailed,
24 };
25
26 pub const DiscardError = error{
27 AccessDenied,
28 OutOfMemory,
29 PermissionDenied,
30 UnsupportedPlatform,
31 DiscardFailed,
32 };
33
34 pub const DecommitError = MapError || ProtectError;
35
36 pub const Protection = struct {
37 read: bool = false,
38 write: bool = false,
39 execute: bool = false,
40 };
41
42 var page_allocator_context: u8 = 0;
43
44 pub const page_allocator: std.mem.Allocator = .{
45 .ptr = &page_allocator_context,
46 .vtable = &page_allocator_vtable,
47 };
48
49 const page_allocator_vtable: std.mem.Allocator.VTable = .{
50 .alloc = pageAllocatorAlloc,
51 .resize = pageAllocatorResize,
52 .remap = pageAllocatorRemap,
53 .free = pageAllocatorFree,
54 };
55
56 fn pageAllocatorAlloc(
57 _: *anyopaque,
58 len: usize,
59 alignment: std.mem.Alignment,
60 _: usize,
61 ) ?[*]u8 {
62 const page_size = pageSize();
63 const aligned_len = pageAlign(len) orelse return null;
64 const alignment_bytes = alignment.toByteUnits();
65 if (alignment_bytes <= page_size) {
66 const mapping = mapAnonymous(
67 aligned_len,
68 .{ .read = true, .write = true },
69 ) catch return null;
70 return mapping.ptr;
71 }
72 const mapping_len = std.math.add(
73 usize,
74 aligned_len,
75 alignment_bytes,
76 ) catch return null;
77 const mapping = mapAnonymous(
78 mapping_len,
79 .{ .read = true, .write = true },
80 ) catch return null;
81 const base_address = @intFromPtr(mapping.ptr);
82 const aligned_address = std.mem.alignForward(
83 usize,
84 base_address,
85 alignment_bytes,
86 );
87 const prefix_len = aligned_address - base_address;
88 if (prefix_len != 0) unmap(mapping[0..prefix_len]);
89 const allocation_end = aligned_address + aligned_len;
90 const mapping_end = base_address + mapping.len;
91 if (allocation_end < mapping_end) {
92 const suffix: [*]align(std.heap.page_size_min) u8 =
93 @ptrFromInt(allocation_end);
94 unmap(suffix[0 .. mapping_end - allocation_end]);
95 }
96 return @ptrFromInt(aligned_address);
97 }
98
99 fn pageAllocatorResize(
100 _: *anyopaque,
101 memory: []u8,
102 _: std.mem.Alignment,
103 new_len: usize,
104 _: usize,
105 ) bool {
106 const old_aligned_len = pageAlign(memory.len) orelse return false;
107 const new_aligned_len = pageAlign(new_len) orelse return false;
108 if (new_aligned_len > old_aligned_len) return false;
109 if (new_aligned_len < old_aligned_len) {
110 const suffix: [*]align(std.heap.page_size_min) u8 = @ptrCast(
111 @alignCast(memory.ptr + new_aligned_len),
112 );
113 unmap(suffix[0 .. old_aligned_len - new_aligned_len]);
114 }
115 return true;
116 }
117
118 fn pageAllocatorRemap(
119 context: *anyopaque,
120 memory: []u8,
121 alignment: std.mem.Alignment,
122 new_len: usize,
123 return_address: usize,
124 ) ?[*]u8 {
125 if (!pageAllocatorResize(
126 context,
127 memory,
128 alignment,
129 new_len,
130 return_address,
131 )) return null;
132 return memory.ptr;
133 }
134
135 fn pageAllocatorFree(
136 _: *anyopaque,
137 memory: []u8,
138 _: std.mem.Alignment,
139 _: usize,
140 ) void {
141 const aligned_len = pageAlign(memory.len) orelse unreachable;
142 const mapping: [*]align(std.heap.page_size_min) u8 =
143 @ptrCast(@alignCast(memory.ptr));
144 unmap(mapping[0..aligned_len]);
145 }
146
147 const MappingOptions = struct {
148 no_reserve: bool = false,
149 fixed: bool = false,
150 address: ?[*]align(std.heap.page_size_min) u8 = null,
151 };
152
153 const FileMappingOptions = struct {
154 fixed: bool = false,
155 address: ?[*]align(std.heap.page_size_min) u8 = null,
156 offset: usize = 0,
157 };
158
159 const PageSizeCache = struct {
160 var value: std.atomic.Value(usize) = .init(0);
161 };
162
163 pub fn pageSize() usize {
164 if (comptime builtin.os.tag != .linux) return std.heap.pageSize();
165 if (std.heap.page_size_min == std.heap.page_size_max) return std.heap.page_size_min;
166
167 const cached = PageSizeCache.value.load(.unordered);
168 if (cached != 0) return cached;
169 const detected = detectLinuxPageSize();
170 PageSizeCache.value.store(detected, .unordered);
171 return detected;
172 }
173
174 const LinuxPageAlignmentProbe = struct {
175 base_address: usize,
176
177 fn alignment(self: @This(), candidate: usize) PageAlignment {
178 const address: [*]align(std.heap.page_size_min) u8 = @ptrFromInt(self.base_address + candidate);
179 const result = std.os.linux.mprotect(address, std.heap.page_size_min, .{});
180 return switch (std.os.linux.errno(result)) {
181 .SUCCESS => .aligned,
182 .INVAL => .unaligned,
183 else => .failed,
184 };
185 }
186 };
187
188 fn detectLinuxPageSize() usize {
189 if (comptime builtin.os.tag != .linux) return std.heap.page_size_max;
190
191 const linux = std.os.linux;
192 const probe_len = std.heap.page_size_max * 2;
193 const mapped = linux.mmap(
194 null,
195 probe_len,
196 .{},
197 .{ .TYPE = .PRIVATE, .ANONYMOUS = true },
198 -1,
199 0,
200 );
201 if (linux.errno(mapped) != .SUCCESS) return std.heap.page_size_max;
202 const base: [*]align(std.heap.page_size_min) u8 = @ptrFromInt(mapped);
203 defer _ = linux.munmap(base, probe_len);
204
205 return detectPageSize(
206 std.heap.page_size_min,
207 std.heap.page_size_max,
208 LinuxPageAlignmentProbe{ .base_address = mapped },
209 );
210 }
211
212 const PageAlignment = enum {
213 aligned,
214 unaligned,
215 failed,
216 };
217
218 fn detectPageSize(minimum: usize, maximum: usize, probe: anytype) usize {
219 var candidate = minimum;
220 while (candidate < maximum) : (candidate *= 2) {
221 switch (probe.alignment(candidate)) {
222 .aligned => return candidate,
223 .unaligned => {},
224 .failed => return maximum,
225 }
226 }
227 return maximum;
228 }
229
230 const PageAlignmentProbeFixture = struct {
231 expected: usize,
232 fail: bool = false,
233
234 fn alignment(self: @This(), candidate: usize) PageAlignment {
235 if (self.fail) return .failed;
236 return if (candidate < self.expected) .unaligned else .aligned;
237 }
238 };
239
240 pub fn pageAlign(byte_count: usize) ?usize {
241 const page_size = pageSize();
242 const mask = page_size - 1;
243 if (byte_count > std.math.maxInt(usize) - mask) return null;
244 return (byte_count + mask) & ~mask;
245 }
246
247 test "page size query matches the host" {
248 if (builtin.os.tag != .linux) return error.SkipZigTest;
249 try std.testing.expectEqual(std.heap.pageSize(), pageSize());
250 }
251
252 test "page size detection finds variable Linux page alignments" {
253 try std.testing.expectEqual(@as(usize, 4 * 1024), detectPageSize(4 * 1024, 64 * 1024, PageAlignmentProbeFixture{ .expected = 4 * 1024 }));
254 try std.testing.expectEqual(@as(usize, 16 * 1024), detectPageSize(4 * 1024, 64 * 1024, PageAlignmentProbeFixture{ .expected = 16 * 1024 }));
255 try std.testing.expectEqual(@as(usize, 64 * 1024), detectPageSize(4 * 1024, 64 * 1024, PageAlignmentProbeFixture{ .expected = 64 * 1024 }));
256 try std.testing.expectEqual(@as(usize, 64 * 1024), detectPageSize(4 * 1024, 64 * 1024, PageAlignmentProbeFixture{ .expected = 4 * 1024, .fail = true }));
257 }
258
259 pub fn anonymousMappingSupported() bool {
260 return switch (builtin.os.tag) {
261 .linux, .macos, .ios, .tvos, .watchos, .freebsd, .openbsd, .netbsd, .dragonfly => true,
262 else => false,
263 };
264 }
265
266 pub fn mapAnonymous(byte_count: usize, protection: Protection) MapError![]align(std.heap.page_size_min) u8 {
267 const return_address = @returnAddress();
268 const aligned_len = pageAlign(@max(byte_count, 1)) orelse {
269 recordMap(.anonymous, 0, byte_count, return_address, false);
270 return error.OutOfMemory;
271 };
272 const mapping = mapAnonymousWithOptions(
273 aligned_len,
274 protection,
275 .{},
276 ) catch |err| {
277 recordMap(.anonymous, 0, aligned_len, return_address, false);
278 return err;
279 };
280 recordMap(
281 .anonymous,
282 @intFromPtr(mapping.ptr),
283 mapping.len,
284 return_address,
285 true,
286 );
287 return mapping;
288 }
289
290 pub fn reserveAddressSpace(byte_count: usize) MapError![]align(std.heap.page_size_min) u8 {
291 const return_address = @returnAddress();
292 const aligned_len = pageAlign(@max(byte_count, 1)) orelse {
293 recordMap(.reserve, 0, byte_count, return_address, false);
294 return error.OutOfMemory;
295 };
296 const mapping = mapAnonymousWithOptions(
297 aligned_len,
298 .{},
299 .{ .no_reserve = true },
300 ) catch |err| {
301 recordMap(.reserve, 0, aligned_len, return_address, false);
302 return err;
303 };
304 recordMap(
305 .reserve,
306 @intFromPtr(mapping.ptr),
307 mapping.len,
308 return_address,
309 true,
310 );
311 return mapping;
312 }
313
314 pub fn mapAnonymousFixed(
315 address: [*]align(std.heap.page_size_min) u8,
316 byte_count: usize,
317 protection: Protection,
318 ) MapError![]align(std.heap.page_size_min) u8 {
319 const return_address = @returnAddress();
320 const requested_address = @intFromPtr(address);
321 const aligned_len = pageAlign(@max(byte_count, 1)) orelse {
322 recordMap(
323 .anonymous_fixed,
324 requested_address,
325 byte_count,
326 return_address,
327 false,
328 );
329 return error.OutOfMemory;
330 };
331 const mapping = mapAnonymousWithOptions(
332 aligned_len,
333 protection,
334 .{ .fixed = true, .address = address },
335 ) catch |err| {
336 recordMap(
337 .anonymous_fixed,
338 requested_address,
339 aligned_len,
340 return_address,
341 false,
342 );
343 return err;
344 };
345 recordMap(
346 .anonymous_fixed,
347 @intFromPtr(mapping.ptr),
348 mapping.len,
349 return_address,
350 true,
351 );
352 return mapping;
353 }
354
355 pub fn mapPrivateFile(
356 descriptor: std.posix.fd_t,
357 byte_count: usize,
358 protection: Protection,
359 offset: usize,
360 ) MapError![]align(std.heap.page_size_min) u8 {
361 const return_address = @returnAddress();
362 if (byte_count == 0) {
363 recordMap(.private_file, 0, 0, return_address, false);
364 return error.MapFailed;
365 }
366 const aligned_len = pageAlign(byte_count) orelse {
367 recordMap(.private_file, 0, byte_count, return_address, false);
368 return error.OutOfMemory;
369 };
370 const mapping = mapPrivateFileWithOptions(
371 descriptor,
372 aligned_len,
373 protection,
374 .{ .offset = offset },
375 ) catch |err| {
376 recordMap(.private_file, 0, aligned_len, return_address, false);
377 return err;
378 };
379 recordMap(
380 .private_file,
381 @intFromPtr(mapping.ptr),
382 mapping.len,
383 return_address,
384 true,
385 );
386 return mapping;
387 }
388
389 pub fn mapSharedFile(
390 descriptor: std.posix.fd_t,
391 byte_count: usize,
392 protection: Protection,
393 offset: usize,
394 ) MapError![]align(std.heap.page_size_min) u8 {
395 const return_address = @returnAddress();
396 if (byte_count == 0) {
397 recordMap(.shared_file, 0, 0, return_address, false);
398 return error.MapFailed;
399 }
400 const aligned_len = pageAlign(byte_count) orelse {
401 recordMap(.shared_file, 0, byte_count, return_address, false);
402 return error.OutOfMemory;
403 };
404 const mapping = mapSharedFileWithOptions(
405 descriptor,
406 aligned_len,
407 protection,
408 .{ .offset = offset },
409 ) catch |err| {
410 recordMap(.shared_file, 0, aligned_len, return_address, false);
411 return err;
412 };
413 recordMap(
414 .shared_file,
415 @intFromPtr(mapping.ptr),
416 mapping.len,
417 return_address,
418 true,
419 );
420 return mapping;
421 }
422
423 pub fn mapPrivateFileFixed(
424 descriptor: std.posix.fd_t,
425 address: [*]align(std.heap.page_size_min) u8,
426 byte_count: usize,
427 protection: Protection,
428 offset: usize,
429 ) MapError![]align(std.heap.page_size_min) u8 {
430 const return_address = @returnAddress();
431 const requested_address = @intFromPtr(address);
432 if (byte_count == 0) {
433 recordMap(
434 .private_file_fixed,
435 requested_address,
436 0,
437 return_address,
438 false,
439 );
440 return error.MapFailed;
441 }
442 const aligned_len = pageAlign(byte_count) orelse {
443 recordMap(
444 .private_file_fixed,
445 requested_address,
446 byte_count,
447 return_address,
448 false,
449 );
450 return error.OutOfMemory;
451 };
452 const mapping = mapPrivateFileWithOptions(descriptor, aligned_len, protection, .{
453 .fixed = true,
454 .address = address,
455 .offset = offset,
456 }) catch |err| {
457 recordMap(
458 .private_file_fixed,
459 requested_address,
460 aligned_len,
461 return_address,
462 false,
463 );
464 return err;
465 };
466 recordMap(
467 .private_file_fixed,
468 @intFromPtr(mapping.ptr),
469 mapping.len,
470 return_address,
471 true,
472 );
473 return mapping;
474 }
475
476 fn mapAnonymousWithOptions(
477 aligned_len: usize,
478 protection: Protection,
479 options: MappingOptions,
480 ) MapError![]align(std.heap.page_size_min) u8 {
481 switch (builtin.os.tag) {
482 .linux => return mapAnonymousLinux(aligned_len, protection, options),
483 .macos, .ios, .tvos, .watchos, .freebsd, .openbsd, .netbsd, .dragonfly => return mapAnonymousPosix(aligned_len, protection, options),
484 else => return error.UnsupportedPlatform,
485 }
486 }
487
488 fn mapPrivateFileWithOptions(
489 descriptor: std.posix.fd_t,
490 aligned_len: usize,
491 protection: Protection,
492 options: FileMappingOptions,
493 ) MapError![]align(std.heap.page_size_min) u8 {
494 switch (builtin.os.tag) {
495 .linux => return mapPrivateFileLinux(descriptor, aligned_len, protection, options),
496 .macos, .ios, .tvos, .watchos, .freebsd, .openbsd, .netbsd, .dragonfly => return mapPrivateFilePosix(descriptor, aligned_len, protection, options),
497 else => return error.UnsupportedPlatform,
498 }
499 }
500
501 fn mapSharedFileWithOptions(
502 descriptor: std.posix.fd_t,
503 aligned_len: usize,
504 protection: Protection,
505 options: FileMappingOptions,
506 ) MapError![]align(std.heap.page_size_min) u8 {
507 switch (builtin.os.tag) {
508 .linux => return mapSharedFileLinux(descriptor, aligned_len, protection, options),
509 .macos, .ios, .tvos, .watchos, .freebsd, .openbsd, .netbsd, .dragonfly => return mapSharedFilePosix(descriptor, aligned_len, protection, options),
510 else => return error.UnsupportedPlatform,
511 }
512 }
513
514 pub fn unmap(mapping: []align(std.heap.page_size_min) u8) void {
515 const return_address = @returnAddress();
516 switch (builtin.os.tag) {
517 .linux => unmapLinux(mapping),
518 .macos, .ios, .tvos, .watchos, .freebsd, .openbsd, .netbsd, .dragonfly => unmapPosix(mapping),
519 else => unreachable,
520 }
521 recordOperation(
522 .unmap,
523 .unmap,
524 @intFromPtr(mapping.ptr),
525 mapping.len,
526 return_address,
527 true,
528 );
529 }
530
531 pub fn protect(mapping: []align(std.heap.page_size_min) u8, protection: Protection) ProtectError!void {
532 const return_address = @returnAddress();
533 const result: ProtectError!void = switch (builtin.os.tag) {
534 .linux => protectLinux(mapping, protection),
535 .macos, .ios, .tvos, .watchos, .freebsd, .openbsd, .netbsd, .dragonfly => protectPosix(mapping, protection),
536 else => error.UnsupportedPlatform,
537 };
538 result catch |err| {
539 recordOperation(
540 .protect,
541 .protect,
542 @intFromPtr(mapping.ptr),
543 mapping.len,
544 return_address,
545 false,
546 );
547 return err;
548 };
549 recordOperation(
550 .protect,
551 .protect,
552 @intFromPtr(mapping.ptr),
553 mapping.len,
554 return_address,
555 true,
556 );
557 }
558
559 pub fn discard(mapping: []align(std.heap.page_size_min) u8) DiscardError!void {
560 if (mapping.len == 0) return;
561 const return_address = @returnAddress();
562 const result: DiscardError!void = switch (builtin.os.tag) {
563 .linux => discardLinux(mapping),
564 .macos, .ios, .tvos, .watchos, .freebsd, .openbsd, .netbsd, .dragonfly => discardPosix(mapping),
565 else => error.UnsupportedPlatform,
566 };
567 result catch |err| {
568 recordOperation(
569 .discard,
570 .discard,
571 @intFromPtr(mapping.ptr),
572 mapping.len,
573 return_address,
574 false,
575 );
576 return err;
577 };
578 recordOperation(
579 .discard,
580 .discard,
581 @intFromPtr(mapping.ptr),
582 mapping.len,
583 return_address,
584 true,
585 );
586 }
587
588 pub fn decommit(mapping: []align(std.heap.page_size_min) u8) DecommitError!void {
589 if (mapping.len == 0) return;
590 const return_address = @returnAddress();
591 const result: DecommitError!void = switch (builtin.os.tag) {
592 .linux => decommitLinux(mapping),
593 .macos, .ios, .tvos, .watchos, .freebsd, .openbsd, .netbsd, .dragonfly => decommitPosix(mapping),
594 else => error.UnsupportedPlatform,
595 };
596 result catch |err| {
597 recordOperation(
598 .decommit,
599 .decommit,
600 @intFromPtr(mapping.ptr),
601 mapping.len,
602 return_address,
603 false,
604 );
605 return err;
606 };
607 recordOperation(
608 .decommit,
609 .decommit,
610 @intFromPtr(mapping.ptr),
611 mapping.len,
612 return_address,
613 true,
614 );
615 }
616
617 pub const AdviseError = error{
618 AccessDenied,
619 InvalidMapping,
620 OutOfMemory,
621 PermissionDenied,
622 UnsupportedPlatform,
623 AdviseFailed,
624 };
625
626 /// Asks for huge pages to cut the cost of address translation on a large hot
627 /// mapping by marking `mapping` as a candidate for huge-page backing with the
628 /// `MADV_HUGEPAGE` advice. Whether a huge page appears behind the range, and
629 /// when, rests with the kernel: its huge-page mode, how fragmented physical
630 /// memory is, and how the range is aligned each bear on the outcome. Code whose
631 /// budget rests on the backing confirms it afterwards by touching the pages and
632 /// reading the range's `AnonHugePages` line in `/proc/self/smaps`. An empty
633 /// mapping returns without a syscall, while a host other than Linux and a Linux
634 /// kernel without this advice return `UnsupportedPlatform`, and a range that is
635 /// no longer mapped returns `InvalidMapping`.
636 pub fn adviseHugePages(mapping: []align(std.heap.page_size_min) u8) AdviseError!void {
637 return advisePageSize(mapping, true);
638 }
639
640 /// Excludes a mapping that has to stay on base pages, such as one measuring
641 /// page-level behavior, by marking `mapping` with the `MADV_NOHUGEPAGE`
642 /// advice, which keeps the kernel from promoting the range to huge pages. The
643 /// advice lands on a newly mapped anonymous range before anything touches it,
644 /// because a page promoted already stays promoted. The bounds and the failures
645 /// match `adviseHugePages`, because both go through one advice path.
646 pub fn avoidHugePages(mapping: []align(std.heap.page_size_min) u8) AdviseError!void {
647 return advisePageSize(mapping, false);
648 }
649
650 fn advisePageSize(
651 mapping: []align(std.heap.page_size_min) u8,
652 huge: bool,
653 ) AdviseError!void {
654 const source: observe.Source = if (huge) .huge_pages else .small_pages;
655 if (mapping.len == 0) return;
656 const return_address = @returnAddress();
657 const result: AdviseError!void = switch (builtin.os.tag) {
658 .linux => advisePageSizeLinux(mapping, huge),
659 else => error.UnsupportedPlatform,
660 };
661 result catch |err| {
662 recordOperation(
663 .advise,
664 source,
665 @intFromPtr(mapping.ptr),
666 mapping.len,
667 return_address,
668 false,
669 );
670 return err;
671 };
672 recordOperation(
673 .advise,
674 source,
675 @intFromPtr(mapping.ptr),
676 mapping.len,
677 return_address,
678 true,
679 );
680 }
681
682 fn recordMap(
683 source: observe.Source,
684 address: usize,
685 len: usize,
686 return_address: usize,
687 succeeded: bool,
688 ) void {
689 recordOperation(
690 .map,
691 source,
692 address,
693 len,
694 return_address,
695 succeeded,
696 );
697 }
698
699 fn recordOperation(
700 operation: observe.Operation,
701 source: observe.Source,
702 address: usize,
703 len: usize,
704 return_address: usize,
705 succeeded: bool,
706 ) void {
707 observe.record(.{
708 .operation = operation,
709 .source = source,
710 .address = address,
711 .len = len,
712 .return_address = return_address,
713 .succeeded = succeeded,
714 });
715 }
716
717 fn mapAnonymousLinux(aligned_len: usize, protection: Protection, options: MappingOptions) MapError![]align(std.heap.page_size_min) u8 {
718 const linux = std.os.linux;
719 const flags = try anonymousMapFlags(linux.MAP, options);
720 const rc = linux.mmap(
721 if (options.address) |address| @ptrCast(address) else null,
722 aligned_len,
723 linuxProtection(protection),
724 flags,
725 -1,
726 0,
727 );
728 const err = linux.errno(rc);
729 if (err == .SUCCESS) {
730 const ptr: [*]align(std.heap.page_size_min) u8 = @ptrFromInt(rc);
731 return ptr[0..aligned_len];
732 }
733 return mapErrorFromErrno(err);
734 }
735
736 fn mapPrivateFileLinux(
737 descriptor: std.posix.fd_t,
738 aligned_len: usize,
739 protection: Protection,
740 options: FileMappingOptions,
741 ) MapError![]align(std.heap.page_size_min) u8 {
742 const linux = std.os.linux;
743 const flags = try privateFileMapFlags(linux.MAP, options);
744 const offset = std.math.cast(i64, options.offset) orelse return error.MapFailed;
745 const rc = linux.mmap(
746 if (options.address) |address| @ptrCast(address) else null,
747 aligned_len,
748 linuxProtection(protection),
749 flags,
750 descriptor,
751 offset,
752 );
753 const err = linux.errno(rc);
754 if (err == .SUCCESS) {
755 const ptr: [*]align(std.heap.page_size_min) u8 = @ptrFromInt(rc);
756 return ptr[0..aligned_len];
757 }
758 return mapErrorFromErrno(err);
759 }
760
761 fn mapSharedFileLinux(
762 descriptor: std.posix.fd_t,
763 aligned_len: usize,
764 protection: Protection,
765 options: FileMappingOptions,
766 ) MapError![]align(std.heap.page_size_min) u8 {
767 const linux = std.os.linux;
768 const flags = try sharedFileMapFlags(linux.MAP, options);
769 const offset = std.math.cast(i64, options.offset) orelse return error.MapFailed;
770 const rc = linux.mmap(
771 if (options.address) |address| @ptrCast(address) else null,
772 aligned_len,
773 linuxProtection(protection),
774 flags,
775 descriptor,
776 offset,
777 );
778 const err = linux.errno(rc);
779 if (err == .SUCCESS) {
780 const ptr: [*]align(std.heap.page_size_min) u8 = @ptrFromInt(rc);
781 return ptr[0..aligned_len];
782 }
783 return mapErrorFromErrno(err);
784 }
785
786 fn unmapLinux(mapping: []align(std.heap.page_size_min) u8) void {
787 const rc = std.os.linux.munmap(mapping.ptr, mapping.len);
788 switch (std.os.linux.errno(rc)) {
789 .SUCCESS => return,
790 .INVAL, .NOMEM => unreachable,
791 else => unreachable,
792 }
793 }
794
795 fn protectLinux(mapping: []align(std.heap.page_size_min) u8, protection: Protection) ProtectError!void {
796 const rc = std.os.linux.mprotect(mapping.ptr, mapping.len, linuxProtection(protection));
797 const err = std.os.linux.errno(rc);
798 if (err == .SUCCESS) return;
799 return protectErrorFromErrno(err);
800 }
801
802 fn linuxProtection(protection: Protection) std.os.linux.PROT {
803 return .{
804 .READ = protection.read,
805 .WRITE = protection.write,
806 .EXEC = protection.execute,
807 };
808 }
809
810 fn mapAnonymousPosix(aligned_len: usize, protection: Protection, options: MappingOptions) MapError![]align(std.heap.page_size_min) u8 {
811 const flags = try anonymousMapFlags(std.posix.MAP, options);
812 return std.posix.mmap(
813 options.address,
814 aligned_len,
815 posixProtection(protection),
816 flags,
817 -1,
818 0,
819 ) catch |err| return mapErrorFromMMap(err);
820 }
821
822 fn mapPrivateFilePosix(
823 descriptor: std.posix.fd_t,
824 aligned_len: usize,
825 protection: Protection,
826 options: FileMappingOptions,
827 ) MapError![]align(std.heap.page_size_min) u8 {
828 const flags = try privateFileMapFlags(std.posix.MAP, options);
829 return std.posix.mmap(
830 options.address,
831 aligned_len,
832 posixProtection(protection),
833 flags,
834 descriptor,
835 options.offset,
836 ) catch |err| return mapErrorFromMMap(err);
837 }
838
839 fn mapSharedFilePosix(
840 descriptor: std.posix.fd_t,
841 aligned_len: usize,
842 protection: Protection,
843 options: FileMappingOptions,
844 ) MapError![]align(std.heap.page_size_min) u8 {
845 const flags = try sharedFileMapFlags(std.posix.MAP, options);
846 return std.posix.mmap(
847 options.address,
848 aligned_len,
849 posixProtection(protection),
850 flags,
851 descriptor,
852 options.offset,
853 ) catch |err| return mapErrorFromMMap(err);
854 }
855
856 fn unmapPosix(mapping: []align(std.heap.page_size_min) u8) void {
857 std.posix.munmap(mapping);
858 }
859
860 fn protectPosix(mapping: []align(std.heap.page_size_min) u8, protection: Protection) ProtectError!void {
861 const err = std.posix.errno(std.posix.system.mprotect(
862 @ptrCast(mapping.ptr),
863 mapping.len,
864 posixProtection(protection),
865 ));
866 if (err == .SUCCESS) return;
867 return protectErrorFromErrno(err);
868 }
869
870 fn posixProtection(protection: Protection) std.posix.PROT {
871 return .{
872 .READ = protection.read,
873 .WRITE = protection.write,
874 .EXEC = protection.execute,
875 };
876 }
877
878 fn discardLinux(mapping: []align(std.heap.page_size_min) u8) DiscardError!void {
879 if (comptime @hasDecl(std.os.linux.MADV, "FREE")) {
880 if (try adviseLinux(mapping, std.os.linux.MADV.FREE)) return;
881 }
882
883 if (comptime @hasDecl(std.os.linux.MADV, "DONTNEED")) {
884 if (try adviseLinux(mapping, std.os.linux.MADV.DONTNEED)) return;
885 return error.UnsupportedPlatform;
886 }
887
888 return error.UnsupportedPlatform;
889 }
890
891 fn discardPosix(mapping: []align(std.heap.page_size_min) u8) DiscardError!void {
892 if (comptime @hasDecl(std.posix.MADV, "FREE")) {
893 if (try advisePosix(mapping, std.posix.MADV.FREE)) return;
894 }
895
896 if (comptime @hasDecl(std.posix.MADV, "DONTNEED")) {
897 if (try advisePosix(mapping, std.posix.MADV.DONTNEED)) return;
898 return error.UnsupportedPlatform;
899 }
900
901 return error.UnsupportedPlatform;
902 }
903
904 fn adviseLinux(mapping: []align(std.heap.page_size_min) u8, advice: u32) DiscardError!bool {
905 const err = std.os.linux.errno(std.os.linux.madvise(mapping.ptr, mapping.len, advice));
906 return switch (err) {
907 .SUCCESS => true,
908 .INVAL, .NOSYS => false,
909 .PERM => error.PermissionDenied,
910 .ACCES => error.AccessDenied,
911 .NOMEM => error.OutOfMemory,
912 else => error.DiscardFailed,
913 };
914 }
915
916 fn advisePosix(mapping: []align(std.heap.page_size_min) u8, advice: u32) DiscardError!bool {
917 std.posix.madvise(mapping.ptr, mapping.len, advice) catch |err| switch (err) {
918 error.InvalidSyscall, error.MadviseUnavailable => return false,
919 error.PermissionDenied => return error.PermissionDenied,
920 error.AccessDenied => return error.AccessDenied,
921 error.OutOfMemory => return error.OutOfMemory,
922 else => return error.DiscardFailed,
923 };
924 return true;
925 }
926
927 fn advisePageSizeLinux(
928 mapping: []align(std.heap.page_size_min) u8,
929 huge: bool,
930 ) AdviseError!void {
931 const linux = std.os.linux;
932 if (comptime !@hasDecl(linux.MADV, "HUGEPAGE")) return error.UnsupportedPlatform;
933 const advice: u32 = if (huge) linux.MADV.HUGEPAGE else linux.MADV.NOHUGEPAGE;
934 const err = linux.errno(linux.madvise(mapping.ptr, mapping.len, advice));
935 return switch (err) {
936 .SUCCESS => {},
937 .INVAL, .NOSYS => error.UnsupportedPlatform,
938 .NOMEM => error.InvalidMapping,
939 .AGAIN => error.OutOfMemory,
940 .PERM => error.PermissionDenied,
941 .ACCES => error.AccessDenied,
942 else => error.AdviseFailed,
943 };
944 }
945
946 fn decommitLinux(mapping: []align(std.heap.page_size_min) u8) DecommitError!void {
947 const remapped = mapAnonymousLinux(mapping.len, .{}, .{
948 .no_reserve = true,
949 .fixed = true,
950 .address = mapping.ptr,
951 }) catch |err| switch (err) {
952 error.UnsupportedPlatform => return protectLinux(mapping, .{}),
953 else => return err,
954 };
955 std.debug.assert(remapped.ptr == mapping.ptr);
956 }
957
958 fn decommitPosix(mapping: []align(std.heap.page_size_min) u8) DecommitError!void {
959 const remapped = mapAnonymousPosix(mapping.len, .{}, .{
960 .no_reserve = true,
961 .fixed = true,
962 .address = mapping.ptr,
963 }) catch |err| switch (err) {
964 error.UnsupportedPlatform => return protectPosix(mapping, .{}),
965 else => return err,
966 };
967 std.debug.assert(remapped.ptr == mapping.ptr);
968 }
969
970 fn anonymousMapFlags(comptime Map: type, options: MappingOptions) MapError!Map {
971 var flags: Map = .{ .TYPE = .PRIVATE, .ANONYMOUS = true };
972 if (options.no_reserve) {
973 if (comptime @hasField(Map, "NORESERVE")) flags.NORESERVE = true;
974 }
975 if (options.fixed) {
976 if (comptime @hasField(Map, "FIXED")) {
977 flags.FIXED = true;
978 } else {
979 return error.UnsupportedPlatform;
980 }
981 }
982 return flags;
983 }
984
985 fn privateFileMapFlags(comptime Map: type, options: FileMappingOptions) MapError!Map {
986 var flags: Map = .{ .TYPE = .PRIVATE };
987 if (options.fixed) {
988 if (comptime @hasField(Map, "FIXED")) {
989 flags.FIXED = true;
990 } else {
991 return error.UnsupportedPlatform;
992 }
993 }
994 return flags;
995 }
996
997 fn sharedFileMapFlags(comptime Map: type, options: FileMappingOptions) MapError!Map {
998 var flags: Map = .{ .TYPE = .SHARED };
999 if (options.fixed) {
1000 if (comptime @hasField(Map, "FIXED")) {
1001 flags.FIXED = true;
1002 } else {
1003 return error.UnsupportedPlatform;
1004 }
1005 }
1006 return flags;
1007 }
1008
1009 fn mapErrorFromErrno(err: std.posix.E) MapError {
1010 return switch (err) {
1011 .ACCES => error.AccessDenied,
1012 .PERM => error.PermissionDenied,
1013 .NOMEM => error.OutOfMemory,
1014 else => error.MapFailed,
1015 };
1016 }
1017
1018 fn mapErrorFromMMap(err: std.posix.MMapError) MapError {
1019 return switch (err) {
1020 error.AccessDenied => error.AccessDenied,
1021 error.PermissionDenied => error.PermissionDenied,
1022 error.OutOfMemory => error.OutOfMemory,
1023 error.MemoryMappingNotSupported => error.UnsupportedPlatform,
1024 else => error.MapFailed,
1025 };
1026 }
1027
1028 fn protectErrorFromErrno(err: std.posix.E) ProtectError {
1029 return switch (err) {
1030 .ACCES => error.AccessDenied,
1031 .INVAL => error.InvalidMapping,
1032 .PERM => error.PermissionDenied,
1033 .NOMEM => error.OutOfMemory,
1034 else => error.ProtectFailed,
1035 };
1036 }
1037
1038 test "anonymous mapping can be written and unmapped" {
1039 const mapping = try mapAnonymous(17, .{ .read = true, .write = true });
1040 defer unmap(mapping);
1041
1042 try std.testing.expect(mapping.len >= pageSize());
1043 mapping[0] = 0x42;
1044 mapping[16] = 0x24;
1045 try std.testing.expectEqual(@as(u8, 0x42), mapping[0]);
1046 try std.testing.expectEqual(@as(u8, 0x24), mapping[16]);
1047 }
1048
1049 test "private file mapping can read bytes" {
1050 var tmp = std.testing.tmpDir(.{});
1051 defer tmp.cleanup();
1052
1053 try tmp.dir.writeFile(std.Options.debug_io, .{
1054 .sub_path = "mapped.bin",
1055 .data = "mapped",
1056 });
1057
1058 var file = try tmp.dir.openFile(std.Options.debug_io, "mapped.bin", .{});
1059 defer file.close(std.Options.debug_io);
1060
1061 const mapping = mapPrivateFile(file.handle, 6, .{ .read = true }, 0) catch |err| switch (err) {
1062 error.UnsupportedPlatform => return error.SkipZigTest,
1063 else => return err,
1064 };
1065 defer unmap(mapping);
1066
1067 try std.testing.expectEqualStrings("mapped", mapping[0..6]);
1068 }
1069
1070 test "linux file mapping decodes raw syscall failures" {
1071 if (builtin.os.tag != .linux) return error.SkipZigTest;
1072
1073 try std.testing.expectError(
1074 error.MapFailed,
1075 mapPrivateFile(-1, pageSize(), .{ .read = true }, 0),
1076 );
1077 try std.testing.expectError(
1078 error.MapFailed,
1079 mapSharedFile(-1, pageSize(), .{ .read = true }, 0),
1080 );
1081 }
1082
1083 test "shared file mapping can write bytes" {
1084 var tmp = std.testing.tmpDir(.{});
1085 defer tmp.cleanup();
1086
1087 var file = try tmp.dir.createFile(std.Options.debug_io, "mapped.bin", .{ .read = true });
1088 defer file.close(std.Options.debug_io);
1089 try file.setLength(std.Options.debug_io, 6);
1090
1091 const mapping = mapSharedFile(file.handle, 6, .{ .read = true, .write = true }, 0) catch |err| switch (err) {
1092 error.UnsupportedPlatform => return error.SkipZigTest,
1093 else => return err,
1094 };
1095 @memcpy(mapping[0..6], "mapped");
1096 unmap(mapping);
1097
1098 const bytes = try tmp.dir.readFileAlloc(std.Options.debug_io, "mapped.bin", std.testing.allocator, .limited(7));
1099 defer std.testing.allocator.free(bytes);
1100 try std.testing.expectEqualStrings("mapped", bytes);
1101 }
1102
1103 test "reserved address space can be committed discarded and decommitted" {
1104 const mapping = try reserveAddressSpace(2 * pageSize());
1105 defer unmap(mapping);
1106
1107 const page = mapping[0..pageSize()];
1108 try protect(page, .{ .read = true, .write = true });
1109 page[0] = 0x5a;
1110 try std.testing.expectEqual(@as(u8, 0x5a), page[0]);
1111
1112 try discard(page);
1113 try decommit(page);
1114 }
1115
1116 test "huge page advice keeps an anonymous mapping usable" {
1117 if (!anonymousMappingSupported()) return error.SkipZigTest;
1118 const mapping = try mapAnonymous(4 * 1024 * 1024, .{ .read = true, .write = true });
1119 defer unmap(mapping);
1120
1121 adviseHugePages(mapping) catch |err| switch (err) {
1122 error.UnsupportedPlatform => return error.SkipZigTest,
1123 else => return err,
1124 };
1125 mapping[0] = 0x11;
1126 mapping[mapping.len - 1] = 0x22;
1127 try std.testing.expectEqual(@as(u8, 0x11), mapping[0]);
1128 try std.testing.expectEqual(@as(u8, 0x22), mapping[mapping.len - 1]);
1129 try avoidHugePages(mapping);
1130 try std.testing.expectEqual(@as(u8, 0x11), mapping[0]);
1131 try std.testing.expectEqual(@as(u8, 0x22), mapping[mapping.len - 1]);
1132 try adviseHugePages(mapping[0..0]);
1133 try avoidHugePages(mapping[0..0]);
1134 }
1135
1136 test "linux huge page advice rejects an unmapped range" {
1137 if (builtin.os.tag != .linux) return error.SkipZigTest;
1138 const mapping = try mapAnonymous(pageSize(), .{ .read = true, .write = true });
1139 unmap(mapping);
1140
1141 try std.testing.expectError(error.InvalidMapping, adviseHugePages(mapping));
1142 try std.testing.expectError(error.InvalidMapping, avoidHugePages(mapping));
1143 }
1144
1145 test "huge page advice reports its range without a mapping event" {
1146 if (builtin.os.tag != .linux) return error.SkipZigTest;
1147 const mapping = try mapAnonymous(pageSize(), .{ .read = true, .write = true });
1148 defer unmap(mapping);
1149 const Capture = struct {
1150 event: ?observe.Event = null,
1151 count: u8 = 0,
1152
1153 fn accept(context: *anyopaque, event: observe.Event) void {
1154 const self: *@This() = @ptrCast(@alignCast(context));
1155 self.event = event;
1156 self.count += 1;
1157 }
1158 };
1159 var capture: Capture = .{};
1160 const sink: observe.Sink = .{ .context = &capture, .record = Capture.accept };
1161 var session = try observe.install(&sink);
1162 defer session.deinit();
1163 try avoidHugePages(mapping);
1164 try std.testing.expectEqual(@as(u8, 1), capture.count);
1165 const event = capture.event.?;
1166 try std.testing.expectEqual(observe.Operation.advise, event.operation);
1167 try std.testing.expectEqual(observe.Source.small_pages, event.source);
1168 try std.testing.expectEqual(@intFromPtr(mapping.ptr), event.address);
1169 try std.testing.expectEqual(mapping.len, event.len);
1170 try std.testing.expect(event.succeeded);
1171 }
1172
1173 test "owned page allocator supports page and larger alignments" {
1174 if (!anonymousMappingSupported()) return error.SkipZigTest;
1175 const ordinary = try page_allocator.alloc(u8, 33);
1176 try std.testing.expect(std.mem.isAligned(
1177 @intFromPtr(ordinary.ptr),
1178 pageSize(),
1179 ));
1180 page_allocator.free(ordinary);
1181
1182 const large_alignment = comptime std.mem.Alignment.fromByteUnits(
1183 std.heap.page_size_max * 4,
1184 );
1185 const aligned = try page_allocator.alignedAlloc(
1186 u8,
1187 large_alignment,
1188 pageSize() + 1,
1189 );
1190 try std.testing.expect(std.mem.isAligned(
1191 @intFromPtr(aligned.ptr),
1192 large_alignment.toByteUnits(),
1193 ));
1194 page_allocator.free(aligned);
1195 }