lib/pluck/src/toplevel/source.zig
daab053ee43316e1809a84551d573ddd1e5bf3d2
1 const std = @import("std");
2 const sys = @import("sys");
3 const pluck = @import("../root.zig");
4 const Allocator = std.mem.Allocator;
5
6 const pexpr = pluck.pexpr;
7 const PExpr = pexpr.PExpr;
8 const Symbol = pexpr.Symbol;
9
10 const context_owner = @import("context.zig");
11 const ToplevelContext = context_owner.ToplevelContext;
12 const top_types = @import("types.zig");
13 const ToplevelError = top_types.ToplevelError;
14 const SourceLocation = top_types.SourceLocation;
15 const SourceSpan = top_types.SourceSpan;
16 const result_owner = @import("result.zig");
17 const QueryResult = result_owner.QueryResult;
18 const forms_owner = @import("forms.zig");
19
20 pub fn loadFile(self: *ToplevelContext, path: []const u8) ToplevelError!void {
21 const source = sys.fs.readFileAlloc(self.allocator, path, 10 * 1024 * 1024) catch |err| switch (err) {
22 error.FileNotFound => return ToplevelError.FileNotFound,
23 else => return ToplevelError.IoError,
24 };
25 defer self.allocator.free(source);
26
27 try processSource(self, source);
28 }
29
30 pub fn loadFileWithWriter(self: *ToplevelContext, path: []const u8, writer: anytype) ToplevelError!void {
31 const source = sys.fs.readFileAlloc(self.allocator, path, 10 * 1024 * 1024) catch |err| switch (err) {
32 error.FileNotFound => return ToplevelError.FileNotFound,
33 else => return ToplevelError.IoError,
34 };
35 defer self.allocator.free(source);
36
37 try processSourceWithWriter(self, source, writer);
38 }
39
40 pub fn processSource(self: *ToplevelContext, source: []const u8) ToplevelError!void {
41 try processSourceSexprNoWriter(self, source);
42 }
43
44 pub fn processSourceSexprNoWriter(self: *ToplevelContext, source: []const u8) ToplevelError!void {
45 const prebound_defs = try prebindSexprDefs(self, source);
46 defer if (prebound_defs.len > 0) self.allocator.free(prebound_defs);
47 errdefer rollbackPreboundDefs(self, prebound_defs);
48 var pos: usize = 0;
49 self.last_error_span = null;
50
51 while (pos < source.len) {
52 pos = skipWhitespaceAndComments(source, pos);
53 if (pos >= source.len) break;
54
55 const form_start = pos;
56 const form_end = findFormEnd(source, pos) orelse {
57 self.last_error_span = spanFromOffsets(source, form_start, source.len);
58 return ToplevelError.ParseError;
59 };
60 pos = form_end;
61
62 const form_source = source[form_start..form_end];
63 if (form_source.len == 0) continue;
64
65 const result = forms_owner.processFormSexpr(self, form_source) catch |err| {
66 self.last_error_span = spanFromOffsets(source, form_start, form_end);
67 return err;
68 };
69
70 if (result) |*query_result| {
71 @constCast(query_result).deinit();
72 }
73 }
74 }
75
76 pub fn processSourceWithWriter(self: *ToplevelContext, source: []const u8, writer: anytype) ToplevelError!void {
77 try processSourceSexprWithWriter(self, source, writer);
78 }
79
80 pub const QueryResultCallback = *const fn (*QueryResult, *anyopaque) void;
81
82 pub fn processSourceWithCallback(
83 self: *ToplevelContext,
84 source: []const u8,
85 callback: QueryResultCallback,
86 context: *anyopaque,
87 ) ToplevelError!void {
88 try processSourceSexprWithCallback(self, source, callback, context);
89 }
90
91 pub fn processSourceSexprWithWriter(self: *ToplevelContext, source: []const u8, writer: anytype) ToplevelError!void {
92 const prebound_defs = try prebindSexprDefs(self, source);
93 defer if (prebound_defs.len > 0) self.allocator.free(prebound_defs);
94 errdefer rollbackPreboundDefs(self, prebound_defs);
95 var pos: usize = 0;
96 self.last_error_span = null;
97
98 while (pos < source.len) {
99 pos = skipWhitespaceAndComments(source, pos);
100 if (pos >= source.len) break;
101
102 const form_start = pos;
103 const form_end = findFormEnd(source, pos) orelse {
104 self.last_error_span = spanFromOffsets(source, form_start, source.len);
105 return ToplevelError.ParseError;
106 };
107 pos = form_end;
108
109 const form_source = source[form_start..form_end];
110 if (form_source.len == 0) continue;
111
112 const result = forms_owner.processFormSexpr(self, form_source) catch |err| {
113 self.last_error_span = spanFromOffsets(source, form_start, form_end);
114 return err;
115 };
116
117 if (result) |*query_result| {
118 defer @constCast(query_result).deinit();
119 query_result.print(writer) catch {};
120 writer.flush() catch {};
121 }
122 }
123 }
124
125 pub fn processSourceSexprWithCallback(
126 self: *ToplevelContext,
127 source: []const u8,
128 callback: QueryResultCallback,
129 context: *anyopaque,
130 ) ToplevelError!void {
131 const prebound_defs = try prebindSexprDefs(self, source);
132 defer if (prebound_defs.len > 0) self.allocator.free(prebound_defs);
133 errdefer rollbackPreboundDefs(self, prebound_defs);
134 var pos: usize = 0;
135 self.last_error_span = null;
136
137 while (pos < source.len) {
138 pos = skipWhitespaceAndComments(source, pos);
139 if (pos >= source.len) break;
140
141 const form_start = pos;
142 const form_end = findFormEnd(source, pos) orelse {
143 self.last_error_span = spanFromOffsets(source, form_start, source.len);
144 return ToplevelError.ParseError;
145 };
146 pos = form_end;
147
148 const form_source = source[form_start..form_end];
149 if (form_source.len == 0) continue;
150
151 const source_loc = locationFromOffsets(source, form_start, form_end);
152
153 const result = forms_owner.processFormSexpr(self, form_source) catch |err| {
154 self.last_error_span = spanFromOffsets(source, form_start, form_end);
155 return err;
156 };
157
158 if (result) |*query_result| {
159 var mutable_result = query_result.*;
160 mutable_result.source_location = source_loc;
161 callback(&mutable_result, context);
162 }
163 }
164 }
165
166 pub const PreboundDef = struct {
167 name: Symbol,
168 prev_expr: ?*PExpr,
169 prev_is_stdlib: bool,
170 prev_doc: ?[]const u8,
171 };
172
173 pub fn prebindSexprDefs(self: *ToplevelContext, source: []const u8) ToplevelError![]PreboundDef {
174 const arena_alloc = self.arena.allocator();
175
176 var prebound: std.ArrayListUnmanaged(PreboundDef) = .empty;
177 errdefer prebound.deinit(self.allocator);
178
179 var seen: std.StringHashMapUnmanaged(void) = .{};
180 defer seen.deinit(self.allocator);
181
182 var pos: usize = 0;
183 while (pos < source.len) {
184 pos = skipWhitespaceAndComments(source, pos);
185 if (pos >= source.len) break;
186
187 const form_start = pos;
188 const form_end = findFormEnd(source, pos) orelse return ToplevelError.ParseError;
189 pos = form_end;
190
191 const form_source = source[form_start..form_end];
192 if (form_source.len == 0) continue;
193
194 const tokens = pexpr.tokenize(self.allocator, form_source) catch return ToplevelError.OutOfMemory;
195 defer pexpr.freeTokens(self.allocator, tokens);
196
197 if (extractSexprDefName(tokens)) |name| {
198 try prebindDefinition(self, arena_alloc, name, &prebound, &seen);
199 }
200 }
201
202 return prebound.toOwnedSlice(self.allocator) catch return ToplevelError.OutOfMemory;
203 }
204
205 fn extractSexprDefName(tokens: []const pexpr.Token) ?[]const u8 {
206 if (tokens.len < 4) return null;
207 if (!std.mem.eql(u8, tokens[0], "(")) return null;
208 if (!std.mem.eql(u8, tokens[1], "define")) return null;
209
210 if (std.mem.eql(u8, tokens[2], "(")) {
211 if (tokens.len >= 4 and isIdentifier(tokens[3])) {
212 return tokens[3];
213 }
214 return null;
215 }
216
217 if (isIdentifier(tokens[2])) {
218 return tokens[2];
219 }
220 return null;
221 }
222
223 fn prebindDefinition(
224 self: *ToplevelContext,
225 arena_alloc: Allocator,
226 name: []const u8,
227 prebound: *std.ArrayListUnmanaged(PreboundDef),
228 seen: *std.StringHashMapUnmanaged(void),
229 ) ToplevelError!void {
230 if (seen.contains(name)) return;
231
232 const duped_name = arena_alloc.dupe(u8, name) catch return ToplevelError.OutOfMemory;
233 try seen.put(self.allocator, duped_name, {});
234
235 const prev_def = self.definitions.lookupDefinition(duped_name);
236 const prev_expr = if (prev_def) |d| d.expr else null;
237 const prev_is_stdlib = if (prev_def) |d| d.is_stdlib else false;
238 const prev_doc = if (prev_def) |d| d.doc else null;
239
240 _ = self.definitions.remove(duped_name);
241
242 const dummy = PExpr.init(arena_alloc, .{ .construct = .{ .constructor = "Unit" } }) catch return ToplevelError.OutOfMemory;
243 self.definitions.define(duped_name, dummy) catch return ToplevelError.OutOfMemory;
244
245 prebound.append(self.allocator, .{
246 .name = duped_name,
247 .prev_expr = prev_expr,
248 .prev_is_stdlib = prev_is_stdlib,
249 .prev_doc = prev_doc,
250 }) catch return ToplevelError.OutOfMemory;
251 }
252
253 pub fn rollbackPreboundDefs(self: *ToplevelContext, prebound: []const PreboundDef) void {
254 for (prebound) |entry| {
255 _ = self.definitions.remove(entry.name);
256 if (entry.prev_expr) |expr| {
257 if (entry.prev_is_stdlib) {
258 if (entry.prev_doc) |doc| {
259 self.definitions.defineStdlibWithDoc(entry.name, expr, doc) catch {};
260 } else {
261 self.definitions.defineStdlib(entry.name, expr) catch {};
262 }
263 } else {
264 if (entry.prev_doc) |doc| {
265 self.definitions.defineWithDoc(entry.name, expr, doc) catch {};
266 } else {
267 self.definitions.define(entry.name, expr) catch {};
268 }
269 }
270 }
271 }
272 }
273
274 pub const LineCol = struct {
275 line: u32,
276 col: u32,
277 };
278
279 pub fn spanFromOffsets(source: []const u8, start: usize, end: usize) SourceSpan {
280 const start_loc = offsetToLineCol(source, start);
281 const end_loc = offsetToLineCol(source, end);
282 return .{
283 .start = .{
284 .line = start_loc.line,
285 .column = start_loc.col,
286 .offset = @min(start, source.len),
287 },
288 .end = .{
289 .line = end_loc.line,
290 .column = end_loc.col,
291 .offset = @min(end, source.len),
292 },
293 };
294 }
295
296 pub fn locationFromSpan(span: SourceSpan) SourceLocation {
297 return .{
298 .start_line = span.start.line,
299 .start_col = span.start.column,
300 .end_line = span.end.line,
301 .end_col = span.end.column,
302 };
303 }
304
305 pub fn locationFromOffsets(source: []const u8, start: usize, end: usize) SourceLocation {
306 return locationFromSpan(spanFromOffsets(source, start, end));
307 }
308
309 pub fn offsetToLineCol(source: []const u8, offset: usize) LineCol {
310 var line: u32 = 0;
311 var col: u32 = 0;
312 for (source[0..@min(offset, source.len)]) |c| {
313 if (c == '\n') {
314 line += 1;
315 col = 0;
316 } else {
317 col += 1;
318 }
319 }
320 return .{ .line = line, .col = col };
321 }
322
323 pub fn isIdentifier(token: []const u8) bool {
324 if (token.len == 0) return false;
325 const first = token[0];
326 if (!std.ascii.isAlphabetic(first) and first != '_' and !isOperatorIdentifierStart(first)) return false;
327 for (token[1..]) |c| {
328 if (!std.ascii.isAlphanumeric(c) and c != '_' and c != '-' and c != '?' and c != '=' and !isOperatorIdentifierStart(c)) return false;
329 }
330 return true;
331 }
332
333 fn isOperatorIdentifierStart(c: u8) bool {
334 return switch (c) {
335 '+', '-', '*', '/', '<', '>', '=' => true,
336 else => false,
337 };
338 }
339
340 pub fn isConstructorName(token: []const u8) bool {
341 if (token.len == 0) return false;
342 const first = token[0];
343 if (!std.ascii.isAlphabetic(first) and first != '_') return false;
344 for (token[1..]) |c| {
345 if (!std.ascii.isAlphanumeric(c) and c != '_' and c != '-') return false;
346 }
347 return true;
348 }
349
350 test "constructor names accept Pluck.jl reference spellings" {
351 try std.testing.expect(isConstructorName("a_"));
352 try std.testing.expect(isConstructorName("the_"));
353 try std.testing.expect(isConstructorName("Start"));
354 try std.testing.expect(isConstructorName("non-terminal"));
355 try std.testing.expect(!isConstructorName("+"));
356 try std.testing.expect(!isConstructorName("nat=?"));
357 try std.testing.expect(!isConstructorName(""));
358 }
359
360 test "processSource records absolute form span on form error" {
361 var ctx = try ToplevelContext.init(std.testing.allocator);
362 defer ctx.deinit();
363
364 const source_text =
365 \\(Marginal True)
366 \\(define p )
367 ;
368
369 try std.testing.expectError(ToplevelError.InvalidDefineForm, ctx.processSource(source_text));
370 const span = ctx.lastErrorSpan().?;
371 try std.testing.expectEqual(@as(u32, 1), span.start.line);
372 try std.testing.expectEqual(@as(usize, 16), span.start.offset);
373 }
374
375 pub fn skipWhitespaceAndComments(source: []const u8, start: usize) usize {
376 var pos = start;
377 while (pos < source.len) {
378 const c = source[pos];
379 if (std.ascii.isWhitespace(c)) {
380 pos += 1;
381 continue;
382 }
383 if (pos + 1 < source.len and source[pos] == ';' and source[pos + 1] == ';') {
384 while (pos < source.len and source[pos] != '\n') {
385 pos += 1;
386 }
387 continue;
388 }
389 break;
390 }
391 return pos;
392 }
393
394 pub fn findFormEnd(source: []const u8, start: usize) ?usize {
395 if (start >= source.len) return null;
396
397 if (source[start] != '(') {
398 var pos = start;
399 while (pos < source.len and !std.ascii.isWhitespace(source[pos]) and source[pos] != '(') {
400 pos += 1;
401 }
402 return pos;
403 }
404
405 var depth: i32 = 0;
406 var pos = start;
407 var in_string = false;
408
409 while (pos < source.len) {
410 const c = source[pos];
411
412 if (c == '"' and (pos == 0 or source[pos - 1] != '\\')) {
413 in_string = !in_string;
414 pos += 1;
415 continue;
416 }
417
418 if (in_string) {
419 pos += 1;
420 continue;
421 }
422
423 if (pos + 1 < source.len and c == ';' and source[pos + 1] == ';') {
424 while (pos < source.len and source[pos] != '\n') {
425 pos += 1;
426 }
427 continue;
428 }
429
430 if (c == '(') {
431 depth += 1;
432 } else if (c == ')') {
433 depth -= 1;
434 if (depth == 0) {
435 return pos + 1;
436 }
437 }
438 pos += 1;
439 }
440
441 return null;
442 }