lib/tldr/src/formats/elf/ifunc.zig
daab053ee43316e1809a84551d573ddd1e5bf3d2
1 const std = @import("std");
2 const root = @import("../../root.zig");
3 const elf = @import("root.zig");
4
5 const Allocator = std.mem.Allocator;
6 const model = root.model;
7 const parallel = root.parallel;
8 const addressing = elf.addressing;
9 const format = elf.format;
10 const layout = elf.layout;
11 const output_section = elf.output_section;
12 const section_state = elf.section_state;
13 const ObjectFile = elf.parser.ObjectFile;
14 const ObjectLayout = layout.ObjectLayout;
15 const OutputSection = layout.OutputSection;
16 const GlobalSymbol = layout.GlobalSymbol;
17 const SymbolRef = layout.SymbolRef;
18 const SymbolAddressCache = addressing.Cache;
19 const OutputSectionKind = output_section.OutputSectionKind;
20 const output_section_count = output_section.output_section_count;
21 const writeU32 = format.writeU32;
22 const writeU64 = format.writeU64;
23
24 pub const stub_size = 16;
25 pub const slot_size = 8;
26 pub const rela_size = 24;
27
28 const irelative_type: u32 = 37;
29 pub const parallel_mark_symbol_threshold = 32 * 1024;
30 const mark_symbols_per_worker = 16 * 1024;
31 const no_ifunc_definition = SymbolRef{
32 .object_index = std.math.maxInt(usize),
33 .symbol_index = std.math.maxInt(usize),
34 };
35
36 pub const Entry = struct {
37 definition: SymbolRef,
38 resolver_address: u64 = 0,
39 };
40
41 pub const IfuncLayout = struct {
42 entries: []Entry = &.{},
43
44 pub fn deinit(self: *IfuncLayout, allocator: Allocator) void {
45 if (self.entries.len != 0) allocator.free(self.entries);
46 self.* = .{};
47 }
48 };
49
50 pub const Collector = struct {
51 active: bool,
52 marks: [][]SymbolRef = &.{},
53 seen: std.AutoHashMapUnmanaged(SymbolRef, void) = .{},
54 entries: std.ArrayListUnmanaged(Entry) = .empty,
55
56 pub fn init(
57 allocator: Allocator,
58 objects: []const ObjectFile,
59 globals: *const std.StringHashMapUnmanaged(GlobalSymbol),
60 max_link_jobs: usize,
61 ) model.Error!Collector {
62 if (!hasIfuncDefinitions(objects)) return .{ .active = false };
63 const marks = try allocator.alloc([]SymbolRef, objects.len);
64 for (marks) |*object_marks| object_marks.* = &.{};
65 errdefer {
66 for (marks) |object_marks| {
67 if (object_marks.len != 0) allocator.free(object_marks);
68 }
69 allocator.free(marks);
70 }
71 for (objects, 0..) |object, object_index| {
72 const object_marks = try allocator.alloc(SymbolRef, object.symbols.len);
73 marks[object_index] = object_marks;
74 }
75 const total_symbol_count = try countSymbols(objects);
76 var context = MarkContext{
77 .objects = objects,
78 .globals = globals,
79 .marks = marks,
80 };
81 const requested_workers = if (max_link_jobs != 0)
82 max_link_jobs
83 else
84 @max(2, total_symbol_count / mark_symbols_per_worker);
85 const workers = if (total_symbol_count >= parallel_mark_symbol_threshold)
86 parallel.chooseWorkers(objects.len, requested_workers)
87 else
88 1;
89 if (workers > 1) {
90 parallel.forItems(objects.len, workers, &context, fillMarkObject);
91 } else {
92 for (objects, 0..) |_, object_index| fillMarkObject(&context, 0, object_index);
93 }
94 return .{ .active = true, .marks = marks };
95 }
96
97 pub fn deinit(self: *Collector, allocator: Allocator) void {
98 for (self.marks) |object_marks| allocator.free(object_marks);
99 if (self.marks.len != 0) allocator.free(self.marks);
100 self.seen.deinit(allocator);
101 self.entries.deinit(allocator);
102 self.* = undefined;
103 }
104
105 pub fn enabled(self: *const Collector) bool {
106 return self.active;
107 }
108
109 pub fn observe(
110 self: *Collector,
111 allocator: Allocator,
112 object_index: usize,
113 relocation_record: format.Rela,
114 ) model.Error!void {
115 if (!self.active) return;
116 if (elf.relocation.isNone(relocation_record)) return;
117 const symbol_index: usize = @intCast(relocation_record.symbolIndex());
118 if (object_index >= self.marks.len) return;
119 const object_marks = self.marks[object_index];
120 if (symbol_index >= object_marks.len) return;
121 const definition = object_marks[symbol_index];
122 if (definition.object_index == no_ifunc_definition.object_index) return;
123 const gop = try self.seen.getOrPut(allocator, definition);
124 if (gop.found_existing) return;
125 try self.entries.append(allocator, .{ .definition = definition });
126 }
127
128 pub fn finish(
129 self: *Collector,
130 allocator: Allocator,
131 output_sections: *[output_section_count]OutputSection,
132 ) model.Error!IfuncLayout {
133 const entry_slice = try self.entries.toOwnedSlice(allocator);
134 self.entries = .empty;
135 const ifunc_layout = IfuncLayout{ .entries = entry_slice };
136 reserve(output_sections, ifunc_layout);
137 return ifunc_layout;
138 }
139 };
140
141 const MarkContext = struct {
142 objects: []const ObjectFile,
143 globals: *const std.StringHashMapUnmanaged(GlobalSymbol),
144 marks: [][]SymbolRef,
145 };
146
147 fn fillMarkObject(context: *MarkContext, worker: usize, object_index: usize) void {
148 _ = worker;
149 for (context.marks[object_index], 0..) |*mark, symbol_index| {
150 mark.* = ifuncDefinition(context.objects, context.globals, object_index, symbol_index) orelse no_ifunc_definition;
151 }
152 }
153
154 pub fn collectEntries(
155 allocator: Allocator,
156 objects: []const ObjectFile,
157 globals: *const std.StringHashMapUnmanaged(GlobalSymbol),
158 output_sections: *[output_section_count]OutputSection,
159 ) model.Error!IfuncLayout {
160 var collector = try Collector.init(allocator, objects, globals, 0);
161 defer collector.deinit(allocator);
162 if (!collector.enabled()) return .{};
163
164 for (objects, 0..) |object, object_index| {
165 if (object.relocations.len == 0) continue;
166 for (object.sections, 0..) |_, section_index| {
167 if (section_state.sectionDiscarded(object, section_index)) continue;
168 if (section_state.foldedSection(object, section_index) != null) continue;
169 for (object.relocationsForSection(section_index)) |entry| {
170 try collector.observe(allocator, object_index, entry);
171 }
172 }
173 }
174
175 return try collector.finish(allocator, output_sections);
176 }
177
178 fn hasIfuncDefinitions(objects: []const ObjectFile) bool {
179 for (objects) |object| {
180 if (object.has_ifunc_definitions) return true;
181 }
182 return false;
183 }
184
185 fn countSymbols(objects: []const ObjectFile) model.Error!usize {
186 var total: usize = 0;
187 for (objects) |object| {
188 total = std.math.add(usize, total, object.symbols.len) catch return error.InvalidObject;
189 }
190 return total;
191 }
192
193 fn ifuncDefinition(
194 objects: []const ObjectFile,
195 globals: *const std.StringHashMapUnmanaged(GlobalSymbol),
196 object_index: usize,
197 symbol_index: usize,
198 ) ?SymbolRef {
199 const symbol = objects[object_index].symbols[symbol_index];
200 var ref = SymbolRef{ .object_index = object_index, .symbol_index = symbol_index };
201 if (symbol.isUndefined() or section_state.symbolSectionDiscarded(objects[object_index], symbol)) {
202 if (symbol.name.len == 0) return null;
203 const global = globals.get(symbol.name) orelse return null;
204 ref = global.ref;
205 }
206 if (ref.object_index >= objects.len) return null;
207 const target_object = objects[ref.object_index];
208 if (ref.symbol_index >= target_object.symbols.len) return null;
209 const target = target_object.symbols[ref.symbol_index];
210 if (target.kind() != std.elf.STT_GNU_IFUNC) return null;
211 if (target.isUndefined()) return null;
212 return ref;
213 }
214
215 fn reserve(output_sections: *[output_section_count]OutputSection, ifunc_layout: IfuncLayout) void {
216 if (ifunc_layout.entries.len == 0) return;
217 const count: u64 = @intCast(ifunc_layout.entries.len);
218
219 var iplt = &output_sections[@backingInt(OutputSectionKind.iplt)];
220 iplt.file_size = count * stub_size;
221 iplt.memory_size = iplt.file_size;
222 iplt.alignment = @max(iplt.alignment, stub_size);
223
224 var igot = &output_sections[@backingInt(OutputSectionKind.igot)];
225 igot.file_size = count * slot_size;
226 igot.memory_size = igot.file_size;
227 igot.alignment = @max(igot.alignment, slot_size);
228
229 var rela = &output_sections[@backingInt(OutputSectionKind.rela_iplt)];
230 rela.file_size = count * rela_size;
231 rela.memory_size = rela.file_size;
232 rela.alignment = @max(rela.alignment, 8);
233 }
234
235 pub fn finalize(
236 objects: []const ObjectFile,
237 layouts: []const ObjectLayout,
238 output_sections: []const OutputSection,
239 globals: *const std.StringHashMapUnmanaged(GlobalSymbol),
240 symbol_addresses: *SymbolAddressCache,
241 ifunc_layout: IfuncLayout,
242 ) model.Error!void {
243 if (ifunc_layout.entries.len == 0) return;
244 const iplt = output_sections[@backingInt(OutputSectionKind.iplt)];
245
246 for (ifunc_layout.entries) |*entry| {
247 entry.resolver_address = try addressing.symbolAddress(
248 .serial,
249 objects,
250 layouts,
251 output_sections,
252 globals,
253 symbol_addresses,
254 entry.definition.object_index,
255 entry.definition.symbol_index,
256 );
257 }
258 for (ifunc_layout.entries, 0..) |entry, index| {
259 const stub_address = iplt.address + @as(u64, @intCast(index)) * stub_size;
260 _ = try addressing.store(.serial, symbol_addresses.slot(entry.definition.object_index, entry.definition.symbol_index), stub_address);
261 }
262 }
263
264 pub fn write(
265 image: []u8,
266 output_sections: []const OutputSection,
267 ifunc_layout: IfuncLayout,
268 ) model.Error!void {
269 if (ifunc_layout.entries.len == 0) return;
270 const iplt = output_sections[@backingInt(OutputSectionKind.iplt)];
271 const igot = output_sections[@backingInt(OutputSectionKind.igot)];
272 const rela = output_sections[@backingInt(OutputSectionKind.rela_iplt)];
273
274 for (ifunc_layout.entries, 0..) |entry, index| {
275 const offset: u64 = @intCast(index);
276 const stub_address = iplt.address + offset * stub_size;
277 const stub_offset: usize = @intCast(iplt.file_offset + offset * stub_size);
278 const slot_address = igot.address + offset * slot_size;
279 const slot_offset: usize = @intCast(igot.file_offset + offset * slot_size);
280 const rela_offset: usize = @intCast(rela.file_offset + offset * rela_size);
281
282 try format.requireRange(image, stub_offset, stub_size);
283 try format.requireRange(image, slot_offset, slot_size);
284 try format.requireRange(image, rela_offset, rela_size);
285
286 const displacement = @as(i128, @intCast(slot_address)) - @as(i128, @intCast(stub_address + 6));
287 image[stub_offset + 0] = 0xff;
288 image[stub_offset + 1] = 0x25;
289 writeU32(image, stub_offset + 2, @bitCast(try elf.relocation.checkedI32(displacement)));
290 @memset(image[stub_offset + 6 ..][0 .. stub_size - 6], 0xcc);
291
292 writeU64(image, slot_offset, entry.resolver_address);
293
294 writeU64(image, rela_offset + 0, slot_address);
295 writeU64(image, rela_offset + 8, irelative_type);
296 writeU64(image, rela_offset + 16, entry.resolver_address);
297 }
298 }
299
300 test "ifunc layout reserves stub slot and relocation space" {
301 var output_sections = @as([output_section_count]OutputSection, @splat(.{ .kind = .build_id_note }));
302 for (&output_sections, 0..) |*section, index| section.kind = @fromBackingInt(@intCast(index));
303
304 var entries = [_]Entry{
305 .{ .definition = .{ .object_index = 0, .symbol_index = 1 } },
306 .{ .definition = .{ .object_index = 0, .symbol_index = 2 } },
307 };
308 const ifunc_layout = IfuncLayout{ .entries = &entries };
309 reserve(&output_sections, ifunc_layout);
310 try std.testing.expectEqual(@as(u64, 32), output_sections[@backingInt(OutputSectionKind.iplt)].file_size);
311 try std.testing.expectEqual(@as(u64, 16), output_sections[@backingInt(OutputSectionKind.igot)].file_size);
312 try std.testing.expectEqual(@as(u64, 48), output_sections[@backingInt(OutputSectionKind.rela_iplt)].file_size);
313 }