tiny.glom.index
Defined in tiny.glom.
API (9)
Actions
Public operations.
Types and contracts
Public types and contracts.
Source
Source: tools/glom/src/index.zig
zig
const std = @import("std");const sys = @import("sys");const pretty = @import("pretty");const data = @import("data.zig");const date = @import("date.zig");const database = @import("database.zig");const input = @import("input/root.zig");const pretty_json = pretty.json;const fs_io = std.Options.debug_io;const bulk_threshold = 100;const online_search_threshold_bytes = 512 * 1024 * 1024;const diagnostic_limit = 5;const codex_skip_header_bytes = 512;const transcript_probe_bytes = [_]usize{ 4 * 1024 * 1024, 16 * 1024 * 1024 };pub const IndexStats = struct { new: usize = 0, updated: usize = 0, unchanged: usize = 0, deleted: usize = 0, errors: usize = 0, malformed_jsonl_lines: usize = 0, parse_errors: std.StringHashMap(usize), largest_files: std.ArrayList(Diagnostic), slowest_files: std.ArrayList(Diagnostic), error_paths: std.ArrayList([]const u8), pub fn init(allocator: std.mem.Allocator) IndexStats { return .{ .parse_errors = std.StringHashMap(usize).init(allocator), .largest_files = .empty, .slowest_files = .empty, .error_paths = .empty, }; } pub fn totalProcessed(self: IndexStats) usize { return self.new + self.updated + self.unchanged; } pub fn deinit(self: *IndexStats, allocator: std.mem.Allocator) void { var parse_iter = self.parse_errors.keyIterator(); while (parse_iter.next()) |key| allocator.free(key.*); self.parse_errors.deinit(); freeDiagnostics(allocator, self.largest_files.items); self.largest_files.deinit(allocator); freeDiagnostics(allocator, self.slowest_files.items); self.slowest_files.deinit(allocator); for (self.error_paths.items) |path| allocator.free(path); self.error_paths.deinit(allocator); self.* = undefined; }};pub const Diagnostic = struct { path: []const u8, value: f64,};pub const FileEntry = struct { path: []const u8, source: []const u8, kind: []const u8, project: ?[]const u8,};const ParseDiagnostics = struct { malformed_jsonl_lines: usize = 0,};const ParseResult = struct { document: data.Document, diagnostics: ParseDiagnostics, content_saturated: bool,};pub fn discover( allocator: std.mem.Allocator, maybe_claude_root: ?[]const u8, maybe_codex_root: ?[]const u8,) ![]FileEntry { const home = (try sys.env.getOwned(allocator, "HOME")) orelse "."; defer allocator.free(home); const claude_root = if (maybe_claude_root) |value| try allocator.dupe(u8, value) else try std.fs.path.join(allocator, &.{ home, ".claude" }); defer allocator.free(claude_root); const codex_root = if (maybe_codex_root) |value| try allocator.dupe(u8, value) else try std.fs.path.join(allocator, &.{ home, ".codex" }); defer allocator.free(codex_root); var entries: std.ArrayList(FileEntry) = .empty; try discoverClaude(allocator, &entries, claude_root); try discoverCodex(allocator, &entries, codex_root); std.mem.sort(FileEntry, entries.items, {}, fileEntryLessThan); return try entries.toOwnedSlice(allocator);}pub fn indexAll( allocator: std.mem.Allocator, db: *database.Database, input_storage: *input.Storage, options: IndexOptions,) !IndexStats { var stats = IndexStats.init(allocator); errdefer stats.deinit(allocator); const entries = try discover(allocator, options.claude_root, options.codex_root); defer freeFileEntries(allocator, entries); var seen = std.StringHashMap(void).init(allocator); defer seen.deinit(); var existing_documents = try db.getAllDocumentStates(allocator); defer { var iter = existing_documents.keyIterator(); while (iter.next()) |key| allocator.free(key.*); existing_documents.deinit(); } var work: std.ArrayList(WorkItem) = .empty; defer work.deinit(allocator); var work_bytes: u64 = 0; var parse_work_count: usize = 0; for (entries) |entry| { try seen.put(entry.path, {}); const stat = std.Io.Dir.cwd().statFile(fs_io, entry.path, .{}) catch |err| { stats.errors += 1; try stats.error_paths.append(allocator, try std.fmt.allocPrint(allocator, "{s}: {s}", .{ entry.path, @errorName(err) })); continue; }; const mtime = timestampSeconds(stat.mtime); const existing = existing_documents.get(entry.path); if (!options.full and existing != null and existing.?.mtime >= mtime and existing.?.size >= 0 and stat.size == @as(u64, @intCast(existing.?.size))) { stats.unchanged += 1; continue; } const reuse_saturated = !options.full and existing != null and isTranscriptKind(entry.kind) and existing.?.transcript_saturated and existing.?.size >= 0 and stat.size > @as(u64, @intCast(existing.?.size)); if (!isTranscriptKind(entry.kind)) try input_storage.preflight(stat.size); try work.append(allocator, .{ .entry = entry, .rowid = if (existing) |document| document.rowid else null, .mtime = mtime, .size = stat.size, .reuse_saturated = reuse_saturated, }); if (!reuse_saturated) { work_bytes += stat.size; parse_work_count += 1; } } const bulk = parse_work_count > bulk_threshold; const online_search = bulk and options.full and work_bytes >= online_search_threshold_bytes; const scratch_allocator = options.scratch_allocator orelse allocator; if (bulk) try db.beginBulk(); defer if (bulk) db.endBulk() catch {}; if (online_search) try db.clearSearch(); var document_search_batch: database.SearchBatch = .empty; defer if (online_search) db.deinitSearchBatch(&document_search_batch); for (work.items, 0..) |item, work_index| { if (options.progress) |progress| { if (work_index != 0 and work_index % progress_interval == 0) progress(work_index, work.items.len); } { const started = sys.time.realNanoTimestamp(); if (item.reuse_saturated) { try db.updateSaturatedTranscript( item.rowid.?, item.mtime, @intCast(item.size), ); const elapsed_ms = @as(f64, @floatFromInt( sys.time.realNanoTimestamp() - started, )) / 1_000_000.0; try recordTop(allocator, &stats.largest_files, .{ .path = item.entry.path, .value = @floatFromInt(item.size), }); try recordTop(allocator, &stats.slowest_files, .{ .path = item.entry.path, .value = elapsed_ms, }); stats.updated += 1; continue; } var parse_arena = std.heap.ArenaAllocator.init(scratch_allocator); defer parse_arena.deinit(); const parsed = parseIndexFile( parse_arena.allocator(), scratch_allocator, input_storage, item, ) catch |err| switch (err) { error.IndexInputStorageInUse, error.IndexInputCapacityExceeded => return err, else => { try recordIndexError(allocator, &stats, item.entry, err); continue; }, }; if (online_search and item.rowid == null) { try db.insertDocumentRowNewSearch(parsed.document, &document_search_batch); } else if (online_search) { try db.updateDocumentRowSearch(item.rowid.?, parsed.document, &document_search_batch); } else if (bulk and item.rowid == null) { try db.insertDocumentRowNew(parsed.document); } else if (bulk) { try db.updateDocumentRow(item.rowid.?, parsed.document); } else if (item.rowid == null) { try db.insertNew(parsed.document); } else { try db.update(item.rowid.?, parsed.document); } if (online_search and document_search_batch.items.len >= database.online_search_batch_limit) try db.flushDocumentSearchBatch(&document_search_batch); stats.malformed_jsonl_lines += parsed.diagnostics.malformed_jsonl_lines; const elapsed_ms = @as(f64, @floatFromInt(sys.time.realNanoTimestamp() - started)) / 1_000_000.0; try recordTop(allocator, &stats.largest_files, .{ .path = parsed.document.path, .value = @floatFromInt(parsed.document.size) }); try recordTop(allocator, &stats.slowest_files, .{ .path = parsed.document.path, .value = elapsed_ms }); if (item.rowid == null) stats.new += 1 else stats.updated += 1; } } var stale_iter = existing_documents.keyIterator(); while (stale_iter.next()) |key| { if (!seen.contains(key.*)) { try db.deletePath(key.*); stats.deleted += 1; } } if (online_search) { try db.flushDocumentSearchBatch(&document_search_batch); } if (bulk) try db.bulkCheckpoint(); if (bulk and !online_search) try db.rebuildSearch(); if (bulk) try db.bulkCheckpoint(); try db.commit(); return stats;}pub const IndexOptions = struct { claude_root: ?[]const u8 = null, codex_root: ?[]const u8 = null, full: bool = false, scratch_allocator: ?std.mem.Allocator = null, progress: ?*const fn (done: usize, total: usize) void = null,};const progress_interval = 1000;const WorkItem = struct { entry: FileEntry, rowid: ?i64, mtime: f64, size: u64, reuse_saturated: bool,};fn parseIndexFile( allocator: std.mem.Allocator, scratch_allocator: std.mem.Allocator, input_storage: *input.Storage, item: WorkItem,) !ParseResult { if (isTranscriptKind(item.entry.kind)) return parseTranscriptFile( allocator, scratch_allocator, input_storage, item, ); const text = try input_storage.readFile(std.Io.Dir.cwd(), fs_io, item.entry.path); defer input_storage.release(); return try parseFile( allocator, scratch_allocator, item.entry, text, item.mtime, item.size, );}fn recordIndexError( allocator: std.mem.Allocator, stats: *IndexStats, entry: FileEntry, err: anyerror,) !void { stats.errors += 1; const key = try std.fmt.allocPrint(allocator, "{s}:{s}", .{ entry.source, entry.kind }); const gop = stats.parse_errors.getOrPut(key) catch |allocation_error| { allocator.free(key); return allocation_error; }; if (gop.found_existing) { allocator.free(key); gop.value_ptr.* += 1; } else { gop.value_ptr.* = 1; } const error_path = try std.fmt.allocPrint( allocator, "{s}: {s}", .{ entry.path, @errorName(err) }, ); errdefer allocator.free(error_path); try stats.error_paths.append(allocator, error_path);}fn discoverClaude(allocator: std.mem.Allocator, entries: *std.ArrayList(FileEntry), root: []const u8) !void { if (!dirExists(root)) return; try addIfFile(allocator, entries, root, "CLAUDE.md", "claude", "instructions", null); try addIfFile(allocator, entries, root, "settings.json", "claude", "settings", null); try addIfFile(allocator, entries, root, "settings.local.json", "claude", "settings", null); try addIfFile(allocator, entries, root, "history.jsonl", "claude", "history", null); try childrenSkillFiles(allocator, entries, try std.fs.path.join(allocator, &.{ root, "skills" }), "claude"); try directGlob(allocator, entries, try std.fs.path.join(allocator, &.{ root, "plans" }), ".md", "claude", "plan", null); try taskFiles(allocator, entries, try std.fs.path.join(allocator, &.{ root, "tasks" })); const projects = try std.fs.path.join(allocator, &.{ root, "projects" }); defer allocator.free(projects); if (!dirExists(projects)) return; const dirs = try listSorted(allocator, projects); defer freeEntries(allocator, dirs); for (dirs) |dir| { if (dir.kind != .directory) continue; const project_root = dir.path; const project = dir.name; try addIfFile(allocator, entries, project_root, "CLAUDE.md", "claude", "instructions", project); const memory = try std.fs.path.join(allocator, &.{ project_root, "memory" }); defer allocator.free(memory); try directGlob(allocator, entries, memory, ".md", "claude", "memory", project); try directGlob(allocator, entries, project_root, ".jsonl", "claude", "session", project); }}fn discoverCodex(allocator: std.mem.Allocator, entries: *std.ArrayList(FileEntry), root: []const u8) !void { if (!dirExists(root)) return; try addIfFile(allocator, entries, root, "AGENTS.md", "codex", "instructions", null); try addIfFile(allocator, entries, root, "config.toml", "codex", "settings", null); try addIfFile(allocator, entries, root, "history.jsonl", "codex", "history", null); const skills = try std.fs.path.join(allocator, &.{ root, "skills" }); defer allocator.free(skills); try recursiveGlob(allocator, entries, skills, ".md", "codex", "skill", null); const memories = try std.fs.path.join(allocator, &.{ root, "memories" }); defer allocator.free(memories); try directGlob(allocator, entries, memories, ".md", "codex", "memory", null); const rollout = try std.fs.path.join(allocator, &.{ memories, "rollout_summaries" }); defer allocator.free(rollout); try directGlob(allocator, entries, rollout, ".md", "codex", "memory", null); const sessions = try std.fs.path.join(allocator, &.{ root, "sessions" }); defer allocator.free(sessions); try recursiveGlob(allocator, entries, sessions, ".jsonl", "codex", "session", null);}fn addIfFile(allocator: std.mem.Allocator, entries: *std.ArrayList(FileEntry), root: []const u8, name: []const u8, source: []const u8, kind: []const u8, project: ?[]const u8) !void { const path = try std.fs.path.join(allocator, &.{ root, name }); errdefer allocator.free(path); if (!fileExists(path)) { allocator.free(path); return; } try entries.append(allocator, .{ .path = path, .source = source, .kind = kind, .project = if (project) |value| try allocator.dupe(u8, value) else null, });}fn childrenSkillFiles(allocator: std.mem.Allocator, entries: *std.ArrayList(FileEntry), root: []const u8, source: []const u8) !void { defer allocator.free(root); if (!dirExists(root)) return; const dirs = try listSorted(allocator, root); defer freeEntries(allocator, dirs); for (dirs) |dir| { if (dir.kind != .directory) continue; try addIfFile(allocator, entries, dir.path, "SKILL.md", source, "skill", null); }}fn taskFiles(allocator: std.mem.Allocator, entries: *std.ArrayList(FileEntry), root: []const u8) !void { defer allocator.free(root); if (!dirExists(root)) return; const dirs = try listSorted(allocator, root); defer freeEntries(allocator, dirs); for (dirs) |dir| { if (dir.kind != .directory) continue; try directGlob(allocator, entries, dir.path, ".json", "claude", "task", null); }}fn directGlob(allocator: std.mem.Allocator, entries: *std.ArrayList(FileEntry), root: []const u8, suffix: []const u8, source: []const u8, kind: []const u8, project: ?[]const u8) !void { if (!dirExists(root)) return; const files = try listSorted(allocator, root); defer freeEntries(allocator, files); for (files) |file| { if (file.kind != .file) continue; if (!std.mem.endsWith(u8, file.name, suffix)) continue; try entries.append(allocator, .{ .path = try allocator.dupe(u8, file.path), .source = source, .kind = kind, .project = if (project) |value| try allocator.dupe(u8, value) else null, }); }}fn recursiveGlob(allocator: std.mem.Allocator, entries: *std.ArrayList(FileEntry), root: []const u8, suffix: []const u8, source: []const u8, kind: []const u8, project: ?[]const u8) !void { if (!dirExists(root)) return; const files = try listSorted(allocator, root); defer freeEntries(allocator, files); for (files) |file| { switch (file.kind) { .file => if (std.mem.endsWith(u8, file.name, suffix)) { try entries.append(allocator, .{ .path = try allocator.dupe(u8, file.path), .source = source, .kind = kind, .project = if (project) |value| try allocator.dupe(u8, value) else null, }); }, .directory => try recursiveGlob(allocator, entries, file.path, suffix, source, kind, project), else => {}, } }}fn parseFile( allocator: std.mem.Allocator, scratch_allocator: std.mem.Allocator, entry: FileEntry, text: []const u8, mtime: f64, size: u64,) !ParseResult { var title: []const u8 = undefined; var content: []const u8 = undefined; var metadata: ?[]const u8 = null; if (isTranscriptKind(entry.kind)) { const session = try parseSession(allocator, scratch_allocator, entry.path, entry.source, text); return sessionParseResult(entry, mtime, size, session); } else if (std.mem.eql(u8, entry.kind, "task")) { const parsed = try parseTask(allocator, scratch_allocator, text, std.fs.path.basename(entry.path)); title = parsed.title; content = parsed.content; } else if (std.mem.eql(u8, entry.kind, "settings")) { title = try allocator.dupe(u8, std.fs.path.basename(entry.path)); content = text; } else { const parsed = try parseMarkdown(allocator, text, pathStem(entry.path)); title = parsed.title; content = parsed.content; if (std.mem.eql(u8, entry.kind, "memory")) metadata = parsed.metadata; } return .{ .document = .{ .source = entry.source, .path = entry.path, .kind = entry.kind, .project = entry.project, .title = title, .content = content, .metadata = metadata, .mtime = mtime, .size = @intCast(size), .transcript_records = null, }, .diagnostics = .{}, .content_saturated = false, };}fn sessionParseResult( entry: FileEntry, mtime: f64, size: u64, session: SessionParse,) ParseResult { return .{ .document = .{ .source = entry.source, .path = entry.path, .kind = entry.kind, .project = entry.project, .title = session.title, .content = session.content, .metadata = null, .mtime = mtime, .size = @intCast(size), .transcript_records = session.records, }, .diagnostics = session.diagnostics, .content_saturated = session.full, };}fn isTranscriptKind(kind: []const u8) bool { return std.mem.eql(u8, kind, "session") or std.mem.eql(u8, kind, "history");}fn searchForTest( database_value: *database.Database, allocator: std.mem.Allocator, query: []const u8, filters: data.SearchFilters,) !data.SearchPage { const search_mod = @import("search/root.zig"); const storage = try allocator.create(search_mod.Storage); storage.* = try search_mod.Storage.init(allocator, .{ .document_value_bytes = search_mod.default_limits.document_value_bytes, .query_workspace_bytes = 4 * 1024 * 1024, .presentation_workspace_bytes = 0, }); storage.activate(); return try database_value.search(storage, query, filters);}const MarkdownParse = struct { title: []const u8, content: []const u8, metadata: ?[]const u8,};fn parseMarkdown(allocator: std.mem.Allocator, text: []const u8, fallback_title: []const u8) !MarkdownParse { var title_text = fallback_title; var body = std.mem.trim(u8, text, " \t\r\n"); var frontmatter: ?[]const u8 = null; if (std.mem.startsWith(u8, text, "---\n")) { if (std.mem.indexOfPos(u8, text, 4, "\n---\n")) |end| { const fm = text[4..end]; body = std.mem.trim(u8, text[end + 5 ..], " \t\r\n"); if (frontmatterValue(fm, "name")) |value| title_text = value; if (frontmatterValue(fm, "title")) |value| title_text = value; frontmatter = fm; } } if (body.len == 0) body = text; const content = try allocator.dupe(u8, body); const title = try allocator.dupe(u8, title_text); const metadata = if (frontmatter) |fm| try frontmatterJson(allocator, fm) else null; return .{ .title = title, .content = content, .metadata = metadata };}const TaskParse = struct { title: []const u8, content: []const u8,};fn parseTask(allocator: std.mem.Allocator, scratch_allocator: std.mem.Allocator, text: []const u8, filename: []const u8) !TaskParse { var parse_arena = std.heap.ArenaAllocator.init(scratch_allocator); defer parse_arena.deinit(); const parsed = std.json.parseFromSliceLeaky(std.json.Value, parse_arena.allocator(), text, .{}) catch { return .{ .title = try allocator.dupe(u8, filename), .content = try allocator.dupe(u8, text) }; }; const object = if (parsed == .object) parsed.object else { return .{ .title = try allocator.dupe(u8, filename), .content = try allocator.dupe(u8, text) }; }; const title = objectString(object, "subject") orelse objectString(object, "title") orelse filename; var out: std.Io.Writer.Allocating = .init(allocator); errdefer out.deinit(); try out.writer.writeAll(title); if (objectString(object, "description")) |desc| try out.writer.print("\n{s}", .{desc}); if (objectString(object, "status")) |status| try out.writer.print("\n[{s}]", .{status}); return .{ .title = try allocator.dupe(u8, title), .content = try out.toOwnedSlice() };}const SessionParse = struct { title: []const u8, content: []const u8, records: []const data.TranscriptRecord, diagnostics: ParseDiagnostics, full: bool,};const CodexRecordOrigin = enum { event, response,};const SessionState = struct { source: []const u8, diagnostics: ParseDiagnostics = .{}, line_number: u32 = 0, previous_codex_origin: ?CodexRecordOrigin = null, full: bool = false,};fn consumeSessionBytes( state: *SessionState, allocator: std.mem.Allocator, scratch_allocator: std.mem.Allocator, content_out: *std.Io.Writer.Allocating, records: *std.ArrayList(data.TranscriptRecord), bytes: []const u8,) !void { std.debug.assert(!state.full); var start: usize = 0; while (std.mem.indexOfScalarPos(u8, bytes, start, '\n')) |newline| { try consumeSessionLine( state, allocator, scratch_allocator, content_out, records, bytes[start..newline], ); if (state.full) return; start = newline + 1; } if (start < bytes.len) try consumeSessionLine( state, allocator, scratch_allocator, content_out, records, bytes[start..], );}fn consumeSessionLine( state: *SessionState, allocator: std.mem.Allocator, scratch_allocator: std.mem.Allocator, content_out: *std.Io.Writer.Allocating, records: *std.ArrayList(data.TranscriptRecord), raw_line: []const u8,) !void { state.line_number = std.math.add(u32, state.line_number, 1) catch return error.CapacityOverflow; const line = std.mem.trim(u8, raw_line, " \t\r\n"); if (line.len == 0) return; if (std.mem.eql(u8, state.source, "codex") and codexLineCanSkipFast(line)) return; var line_arena = std.heap.ArenaAllocator.init(scratch_allocator); defer line_arena.deinit(); var record_out: std.Io.Writer.Allocating = .init(line_arena.allocator()); var record_written = false; var role: ?data.TranscriptRole = null; var codex_origin: ?CodexRecordOrigin = null; var timestamp_ms = timestampMillisFromLine(line); const handled_fast = std.mem.eql(u8, state.source, "codex") and try collectCodexSessionLineFast( line_arena.allocator(), line, &record_out, &record_written, &role, ); if (handled_fast and role != null) codex_origin = codexRecordOriginFromLine(line); if (!handled_fast) { const parsed = std.json.parseFromSliceLeaky( std.json.Value, line_arena.allocator(), line, .{}, ) catch { state.diagnostics.malformed_jsonl_lines += 1; return; }; if (parsed != .object) return; timestamp_ms = objectTimestampMillis(parsed.object) orelse timestamp_ms; role = if (std.mem.eql(u8, state.source, "claude")) try collectClaudeSessionText(parsed.object, &record_out, &record_written) else role: { codex_origin = codexRecordOriginFromObject(parsed.object); break :role try collectCodexSessionText( parsed.object, &record_out, &record_written, ); }; } if (role) |record_role| { if (record_written) { const append = try appendSessionRecord( allocator, content_out, records, record_role, state.line_number, timestamp_ms, record_out.writer.buffered(), state.previous_codex_origin, codex_origin, ); if (append.appended) state.previous_codex_origin = codex_origin; state.full = append.full; } }}fn finishSessionParse( allocator: std.mem.Allocator, path: []const u8, content_out: *std.Io.Writer.Allocating, records_out: *std.ArrayList(data.TranscriptRecord), diagnostics: ParseDiagnostics, full: bool,) !SessionParse { const content = try content_out.toOwnedSlice(); errdefer allocator.free(content); const records = try records_out.toOwnedSlice(allocator); errdefer allocator.free(records); const title = try allocator.dupe(u8, pathStem(path)); return .{ .title = title, .content = content, .records = records, .diagnostics = diagnostics, .full = full, };}fn parseTranscriptFile( allocator: std.mem.Allocator, scratch_allocator: std.mem.Allocator, input_storage: *input.Storage, item: WorkItem,) !ParseResult { var content_out: std.Io.Writer.Allocating = .init(allocator); defer content_out.deinit(); var records: std.ArrayList(data.TranscriptRecord) = .empty; defer records.deinit(allocator); var state = SessionState{ .source = item.entry.source }; const file_limit = input_storage.status().file_bytes; std.debug.assert(file_limit > 0); var offset: u64 = 0; var probe_index: usize = 0; const attempt_limit = item.size *| @as(u64, transcript_probe_bytes.len + 1) +| 1; var attempt_count: u64 = 0; while (attempt_count < attempt_limit) : (attempt_count += 1) { const window_bytes = transcriptWindowBytes(file_limit, probe_index); const window = try input_storage.readFileWindow( std.Io.Dir.cwd(), fs_io, item.entry.path, offset, window_bytes, ); const complete = window.complete; var consumed = window.bytes.len; if (!complete) { const newline = std.mem.lastIndexOfScalar(u8, window.bytes, '\n') orelse { input_storage.release(); if (window_bytes == file_limit) return error.TranscriptRecordCapacityExceeded; probe_index += 1; continue; }; consumed = newline + 1; } consumeSessionBytes( &state, allocator, scratch_allocator, &content_out, &records, window.bytes[0..consumed], ) catch |err| { input_storage.release(); return err; }; input_storage.release(); if (state.full or complete) { const session = try finishSessionParse( allocator, item.entry.path, &content_out, &records, state.diagnostics, state.full, ); return sessionParseResult(item.entry, item.mtime, item.size, session); } std.debug.assert(consumed > 0); offset = std.math.add(u64, offset, consumed) catch return error.CapacityOverflow; probe_index = 0; } return error.CapacityOverflow;}fn transcriptWindowBytes(file_limit: usize, probe_index: usize) usize { std.debug.assert(file_limit > 0); if (probe_index < transcript_probe_bytes.len) { return @min(file_limit, transcript_probe_bytes[probe_index]); } return file_limit;}fn parseSession( allocator: std.mem.Allocator, scratch_allocator: std.mem.Allocator, path: []const u8, source: []const u8, bytes: []const u8,) !SessionParse { var content_out: std.Io.Writer.Allocating = .init(allocator); defer content_out.deinit(); var records: std.ArrayList(data.TranscriptRecord) = .empty; defer records.deinit(allocator); var state = SessionState{ .source = source }; try consumeSessionBytes( &state, allocator, scratch_allocator, &content_out, &records, bytes, ); return finishSessionParse( allocator, path, &content_out, &records, state.diagnostics, state.full, );}const RecordAppend = struct { full: bool = false, appended: bool = false,};fn appendSessionRecord( allocator: std.mem.Allocator, content_out: *std.Io.Writer.Allocating, records: *std.ArrayList(data.TranscriptRecord), role: data.TranscriptRole, line: u32, timestamp_ms: ?i64, text: []const u8, previous_codex_origin: ?CodexRecordOrigin, codex_origin: ?CodexRecordOrigin,) !RecordAppend { const trimmed = std.mem.trim(u8, text, " \t\r\n"); if (trimmed.len <= 3) return .{}; const content = content_out.writer.buffered(); if (previous_codex_origin != null and codex_origin != null and previous_codex_origin.? != codex_origin.? and records.items.len != 0) { const previous = records.items[records.items.len - 1]; if (previous.role == role and std.mem.eql(u8, content[previous.start..previous.end], trimmed)) { return .{}; } } const separator_bytes: usize = if (content.len == 0) 0 else 1; if (content.len + separator_bytes >= data.transcript_content_limit) return .{ .full = true }; const remaining = data.transcript_content_limit - content.len - separator_bytes; const append_bytes = @min(trimmed.len, remaining); if (append_bytes <= 3) return .{ .full = true }; if (separator_bytes != 0) try content_out.writer.writeByte('\n'); const start: u32 = @intCast(content_out.writer.buffered().len); try content_out.writer.writeAll(trimmed[0..append_bytes]); const end: u32 = @intCast(content_out.writer.buffered().len); try records.append(allocator, .{ .start = start, .end = end, .line = line, .timestamp_ms = timestamp_ms, .role = role, }); return .{ .full = append_bytes != trimmed.len or end == data.transcript_content_limit, .appended = true, };}fn codexRecordOriginFromLine(line: []const u8) ?CodexRecordOrigin { const header = codexLineHeader(line); if (std.mem.indexOf(u8, header, "\"type\":\"event_msg\"") != null) return .event; if (std.mem.indexOf(u8, header, "\"type\":\"response_item\"") != null) return .response; return null;}fn codexRecordOriginFromObject(object: std.json.ObjectMap) ?CodexRecordOrigin { const kind = objectString(object, "type") orelse return null; if (std.mem.eql(u8, kind, "event_msg")) return .event; if (std.mem.eql(u8, kind, "response_item")) return .response; return null;}fn collectCodexSessionLineFast( scratch_allocator: std.mem.Allocator, line: []const u8, out: *std.Io.Writer.Allocating, written: *bool, role: *?data.TranscriptRole,) !bool { const header = codexLineHeader(line); if (codexPayloadType(header, "\"type\":\"event_msg\",\"payload\":{\"type\":\"")) |payload_type| { return try collectCodexPayloadLineFast(scratch_allocator, line, "event_msg", payload_type, out, written, role); } if (codexPayloadType(header, "\"type\":\"response_item\",\"payload\":{\"type\":\"")) |payload_type| { return try collectCodexPayloadLineFast(scratch_allocator, line, "response_item", payload_type, out, written, role); } return false;}fn collectCodexPayloadLineFast( scratch_allocator: std.mem.Allocator, line: []const u8, row_type: []const u8, payload_type: []const u8, out: *std.Io.Writer.Allocating, written: *bool, role: *?data.TranscriptRole,) !bool { var line_arena = std.heap.ArenaAllocator.init(scratch_allocator); defer line_arena.deinit(); const payload_start = codexPayloadStart(line) orelse return false; if (try collectCodexPayloadStringsFast(line_arena.allocator(), line[payload_start..], row_type, payload_type, out, written, role)) return true; const payload_slice = jsonObjectSlice(line, payload_start) orelse return false; const parsed = std.json.parseFromSliceLeaky(std.json.Value, line_arena.allocator(), payload_slice, .{}) catch return false; if (parsed != .object) return false; role.* = try collectCodexPayloadText(row_type, payload_type, parsed.object, out, written); return true;}fn collectCodexPayloadStringsFast( scratch_allocator: std.mem.Allocator, payload: []const u8, row_type: []const u8, payload_type: []const u8, out: *std.Io.Writer.Allocating, written: *bool, role: *?data.TranscriptRole,) !bool { if (std.mem.eql(u8, row_type, "event_msg")) { if (!std.mem.eql(u8, payload_type, "user_message") and !std.mem.eql(u8, payload_type, "agent_message")) return false; const message_value = jsonStringAfter(payload, "\"message\":\"") orelse return false; const message = (try decodeJsonString(scratch_allocator, message_value)) orelse return false; try collectText(message, out, written); role.* = if (std.mem.eql(u8, payload_type, "user_message")) .user else .assistant; return true; } return false;}fn codexLineHeader(line: []const u8) []const u8 { return line[0..@min(line.len, codex_skip_header_bytes)];}fn codexLineCanSkipFast(line: []const u8) bool { const header = codexLineHeader(line); inline for (.{ "\"type\":\"session_meta\",\"payload\"", "\"type\":\"turn_context\",\"payload\"", }) |fragment| { if (std.mem.indexOf(u8, header, fragment) != null) return true; } if (codexPayloadType(header, "\"type\":\"event_msg\",\"payload\":{\"type\":\"")) |payload_type| { return !std.mem.eql(u8, payload_type, "user_message") and !std.mem.eql(u8, payload_type, "agent_message"); } if (codexPayloadType(header, "\"type\":\"response_item\",\"payload\":{\"type\":\"")) |payload_type| { return !std.mem.eql(u8, payload_type, "message"); } return false;}fn codexPayloadType(line: []const u8, prefix: []const u8) ?[]const u8 { const start = (std.mem.indexOf(u8, line, prefix) orelse return null) + prefix.len; const end_offset = std.mem.indexOfScalar(u8, line[start..], '"') orelse return null; return line[start .. start + end_offset];}const JsonStringSlice = struct { encoded: []const u8, content: []const u8, has_escapes: bool,};fn jsonStringAfter(text: []const u8, prefix: []const u8) ?JsonStringSlice { const start = (std.mem.indexOf(u8, text, prefix) orelse return null) + prefix.len; var escaped = false; var has_escapes = false; var index = start; while (index < text.len) : (index += 1) { const byte = text[index]; if (escaped) { escaped = false; } else if (byte == '\\') { escaped = true; has_escapes = true; } else if (byte == '"') { return .{ .encoded = text[start - 1 .. index + 1], .content = text[start..index], .has_escapes = has_escapes }; } } return null;}fn decodeJsonString(allocator: std.mem.Allocator, value: JsonStringSlice) !?[]const u8 { if (!value.has_escapes) return value.content; return try unescapeJsonString(allocator, value.content);}fn unescapeJsonString(allocator: std.mem.Allocator, content: []const u8) !?[]const u8 { var out: std.Io.Writer.Allocating = .init(allocator); errdefer out.deinit(); var index: usize = 0; while (index < content.len) : (index += 1) { const byte = content[index]; if (byte != '\\') { try out.writer.writeByte(byte); continue; } index += 1; if (index >= content.len) { out.deinit(); return null; } switch (content[index]) { '"', '\\', '/' => try out.writer.writeByte(content[index]), 'b' => try out.writer.writeByte(0x08), 'f' => try out.writer.writeByte(0x0c), 'n' => try out.writer.writeByte('\n'), 'r' => try out.writer.writeByte('\r'), 't' => try out.writer.writeByte('\t'), else => { out.deinit(); return null; }, } } return try out.toOwnedSlice();}fn codexPayloadObjectSlice(line: []const u8) ?[]const u8 { const start = codexPayloadStart(line) orelse return null; return jsonObjectSlice(line, start);}fn codexPayloadStart(line: []const u8) ?usize { const header = codexLineHeader(line); const key = "\"payload\":"; const key_offset = std.mem.indexOf(u8, header, key) orelse return null; var start = key_offset + key.len; while (start < line.len and isJsonWhitespace(line[start])) start += 1; if (start >= line.len or line[start] != '{') return null; return start;}fn jsonObjectSlice(text: []const u8, start: usize) ?[]const u8 { var depth: usize = 0; var in_string = false; var escaped = false; var index = start; while (index < text.len) : (index += 1) { const byte = text[index]; if (in_string) { if (escaped) { escaped = false; } else if (byte == '\\') { escaped = true; } else if (byte == '"') { in_string = false; } continue; } if (byte == '"') { in_string = true; } else if (byte == '{') { depth += 1; } else if (byte == '}') { if (depth == 0) return null; depth -= 1; if (depth == 0) return text[start .. index + 1]; } } return null;}fn isJsonWhitespace(byte: u8) bool { return byte == ' ' or byte == '\t' or byte == '\r' or byte == '\n';}fn collectClaudeSessionText(object: std.json.ObjectMap, out: *std.Io.Writer.Allocating, written: *bool) !?data.TranscriptRole { const row_type = objectString(object, "type") orelse return null; if (!std.mem.eql(u8, row_type, "user") and !std.mem.eql(u8, row_type, "assistant")) return null; const message = object.get("message") orelse return null; if (message != .object) return null; if (message.object.get("content")) |content| try collectVisibleText(content, out, written); if (!written.*) return null; return if (std.mem.eql(u8, row_type, "user")) .user else .assistant;}fn collectCodexSessionText(object: std.json.ObjectMap, out: *std.Io.Writer.Allocating, written: *bool) !?data.TranscriptRole { const kind = objectString(object, "type") orelse return null; if (!std.mem.eql(u8, kind, "event_msg") and !std.mem.eql(u8, kind, "response_item")) return null; const payload = object.get("payload") orelse return null; if (payload != .object) return null; const payload_type = objectString(payload.object, "type") orelse return null; return try collectCodexPayloadText(kind, payload_type, payload.object, out, written);}fn collectCodexPayloadText(kind: []const u8, payload_type: []const u8, payload: std.json.ObjectMap, out: *std.Io.Writer.Allocating, written: *bool) !?data.TranscriptRole { if (std.mem.eql(u8, kind, "event_msg")) { if (std.mem.eql(u8, payload_type, "user_message") or std.mem.eql(u8, payload_type, "agent_message")) { if (payload.get("message")) |message| try collectVisibleText(message, out, written); if (!written.*) return null; return if (std.mem.eql(u8, payload_type, "user_message")) .user else .assistant; } } else if (std.mem.eql(u8, kind, "response_item")) { const role = objectString(payload, "role") orelse ""; if (std.mem.eql(u8, payload_type, "message") and (std.mem.eql(u8, role, "user") or std.mem.eql(u8, role, "assistant"))) { if (payload.get("content")) |content| try collectVisibleText(content, out, written); if (!written.*) return null; return if (std.mem.eql(u8, role, "user")) .user else .assistant; } } return null;}fn collectVisibleText(value: std.json.Value, out: *std.Io.Writer.Allocating, written: *bool) !void { switch (value) { .string => |text| try collectText(text, out, written), .array => |array| for (array.items) |item| try collectVisibleText(item, out, written), .object => |object| { const block_type = objectString(object, "type"); if (block_type) |kind| { if (isHiddenBlock(kind)) return; if (std.mem.eql(u8, kind, "text") or std.mem.eql(u8, kind, "input_text") or std.mem.eql(u8, kind, "output_text")) { if (object.get("text")) |text| try collectVisibleText(text, out, written); return; } } inline for (.{ "text", "message", "content" }) |key| { if (object.get(key)) |child| try collectVisibleText(child, out, written); } }, else => {}, }}fn collectText(text: []const u8, out: *std.Io.Writer.Allocating, written: *bool) !void { const trimmed = std.mem.trim(u8, text, " \t\r\n"); if (trimmed.len <= 3 or isUuid(trimmed) or isLongHex(trimmed)) return; if (written.*) try out.writer.writeByte('\n'); try out.writer.writeAll(trimmed); written.* = true;}fn isHiddenBlock(kind: []const u8) bool { inline for (.{ "thinking", "reasoning", "tool_use", "tool_result", "function_call", "function_call_output", "custom_tool_call", "custom_tool_call_output", "toolCall", "toolResult", "image", "input_image", }) |hidden| { if (std.mem.eql(u8, kind, hidden)) return true; } return false;}fn frontmatterValue(text: []const u8, key: []const u8) ?[]const u8 { var lines = std.mem.splitScalar(u8, text, '\n'); while (lines.next()) |line| { const trimmed = std.mem.trim(u8, line, " \t\r\n"); if (!std.mem.startsWith(u8, trimmed, key) or trimmed.len <= key.len or trimmed[key.len] != ':') continue; return std.mem.trim(u8, trimmed[key.len + 1 ..], " \t\r\n\"'"); } return null;}fn frontmatterJson(allocator: std.mem.Allocator, text: []const u8) !?[]const u8 { var out: std.Io.Writer.Allocating = .init(allocator); errdefer out.deinit(); var writer = pretty_json.Writer.init(&out.writer, .minified); try writer.beginObject(); var wrote = false; var lines = std.mem.splitScalar(u8, text, '\n'); while (lines.next()) |line| { const split = std.mem.indexOfScalar(u8, line, ':') orelse continue; const key = std.mem.trim(u8, line[0..split], " \t\r\n"); const value = std.mem.trim(u8, line[split + 1 ..], " \t\r\n\"'"); if (key.len == 0 or value.len == 0) continue; try writer.objectField(key); try writer.write(value); wrote = true; } try writer.endObject(); const bytes = try out.toOwnedSlice(); return if (!wrote) null else bytes;}fn objectString(object: std.json.ObjectMap, key: []const u8) ?[]const u8 { const value = object.get(key) orelse return null; return switch (value) { .string => |text| text, else => null, };}fn timestampMillisFromLine(line: []const u8) ?i64 { const value = jsonStringAfter(line, "\"timestamp\":\"") orelse return null; if (value.has_escapes) return null; return date.parseTimestampMillis(value.content);}fn objectTimestampMillis(object: std.json.ObjectMap) ?i64 { const value = object.get("timestamp") orelse return null; return switch (value) { .string => |text| date.parseTimestampMillis(text), .integer => |integer| integerTimestampMillis(integer), .float => |float| floatTimestampMillis(float), else => null, };}fn integerTimestampMillis(value: i64) ?i64 { if (value < 0) return null; if (value >= 100_000_000_000) return value; return std.math.mul(i64, value, 1000) catch null;}fn floatTimestampMillis(value: f64) ?i64 { if (!std.math.isFinite(value) or value < 0) return null; const milliseconds = if (value >= 100_000_000_000) value else value * 1000.0; if (milliseconds > @as(f64, @floatFromInt(std.math.maxInt(i64)))) return null; return @intFromFloat(milliseconds);}fn objectBool(object: std.json.ObjectMap, key: []const u8) ?bool { const value = object.get(key) orelse return null; return switch (value) { .bool => |inner| inner, else => null, };}fn timestampSeconds(timestamp: std.Io.Timestamp) f64 { return @as(f64, @floatFromInt(sys.time.ioTimestampNanoseconds(timestamp))) / @as(f64, @floatFromInt(std.time.ns_per_s));}fn dirExists(path: []const u8) bool { var dir = std.Io.Dir.openDirAbsolute(fs_io, path, .{}) catch return false; dir.close(fs_io); return true;}fn fileExists(path: []const u8) bool { var file = std.Io.Dir.openFileAbsolute(fs_io, path, .{}) catch return false; file.close(fs_io); return true;}fn listSorted(allocator: std.mem.Allocator, path: []const u8) ![]sys.fs.Entry { const entries = try sys.fs.listDirAlloc(allocator, path); std.mem.sort(sys.fs.Entry, entries, {}, entryLessThan); return entries;}fn freeEntries(allocator: std.mem.Allocator, entries: []sys.fs.Entry) void { sys.fs.freeEntries(allocator, entries);}fn freeFileEntries(allocator: std.mem.Allocator, entries: []FileEntry) void { for (entries) |entry| { allocator.free(entry.path); if (entry.project) |project| allocator.free(project); } allocator.free(entries);}fn entryLessThan(_: void, lhs: sys.fs.Entry, rhs: sys.fs.Entry) bool { return std.mem.lessThan(u8, lhs.name, rhs.name);}fn fileEntryLessThan(_: void, lhs: FileEntry, rhs: FileEntry) bool { return std.mem.lessThan(u8, lhs.path, rhs.path);}fn pathStem(path: []const u8) []const u8 { const base = std.fs.path.basename(path); if (std.mem.lastIndexOfScalar(u8, base, '.')) |index| return base[0..index]; return base;}fn isUuid(text: []const u8) bool { if (text.len != 36) return false; for (text, 0..) |byte, index| { if (index == 8 or index == 13 or index == 18 or index == 23) { if (byte != '-') return false; } else if (!std.ascii.isHex(byte)) return false; } return true;}fn isLongHex(text: []const u8) bool { if (text.len < 16) return false; for (text) |byte| if (!std.ascii.isHex(byte)) return false; return true;}fn recordTop(allocator: std.mem.Allocator, rows: *std.ArrayList(Diagnostic), row: Diagnostic) !void { const path = try allocator.dupe(u8, row.path); errdefer allocator.free(path); try rows.append(allocator, .{ .path = path, .value = row.value }); std.mem.sort(Diagnostic, rows.items, {}, diagnosticGreaterThan); while (rows.items.len > diagnostic_limit) { const dropped = rows.pop().?; allocator.free(dropped.path); }}fn freeDiagnostics(allocator: std.mem.Allocator, rows: []Diagnostic) void { for (rows) |row| allocator.free(row.path);}fn diagnosticGreaterThan(_: void, lhs: Diagnostic, rhs: Diagnostic) bool { return lhs.value > rhs.value;}fn checkRecordIndexErrorFailures(allocator: std.mem.Allocator) !void { var stats = IndexStats.init(allocator); defer stats.deinit(allocator); const entry = FileEntry{ .path = "/tmp/broken.jsonl", .source = "codex", .kind = "session", .project = null, }; try recordIndexError(allocator, &stats, entry, error.InvalidCharacter); try recordIndexError(allocator, &stats, entry, error.InvalidCharacter); try std.testing.expectEqual(@as(usize, 2), stats.errors); try std.testing.expectEqual( @as(usize, 2), stats.parse_errors.get("codex:session").?, ); try std.testing.expectEqual(@as(usize, 2), stats.error_paths.items.len);}test "index error records survive every allocation failure" { try std.testing.checkAllAllocationFailures( std.testing.allocator, checkRecordIndexErrorFailures, .{}, );}test "discover indexes codex memory files" { const allocator = std.testing.allocator; var arena = std.heap.ArenaAllocator.init(allocator); defer arena.deinit(); const scratch = arena.allocator(); var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const root = try tmp.dir.realPathFileAlloc(fs_io, ".", allocator); defer allocator.free(root); const codex = try std.fs.path.join(scratch, &.{ root, ".codex" }); const memories = try std.fs.path.join(scratch, &.{ codex, "memories" }); const rollout = try std.fs.path.join(scratch, &.{ memories, "rollout_summaries" }); try std.Io.Dir.cwd().createDirPath(fs_io, rollout); try std.Io.Dir.cwd().writeFile(fs_io, .{ .sub_path = try std.fs.path.join(scratch, &.{ memories, "MEMORY.md" }), .data = "# Memory\n" }); try std.Io.Dir.cwd().writeFile(fs_io, .{ .sub_path = try std.fs.path.join(scratch, &.{ rollout, "rollout.md" }), .data = "# Rollout\n" }); const entries = try discover( scratch, try std.fs.path.join(scratch, &.{ root, ".claude" }), codex, ); var memory_count: usize = 0; for (entries) |entry| { if (std.mem.eql(u8, entry.kind, "memory")) memory_count += 1; } try std.testing.expectEqual(@as(usize, 2), memory_count);}test "parse claude session keeps transcript and skips tool payloads" { const allocator = std.testing.allocator; var arena = std.heap.ArenaAllocator.init(allocator); defer arena.deinit(); const scratch = arena.allocator(); var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const root = try tmp.dir.realPathFileAlloc(fs_io, ".", allocator); defer allocator.free(root); const path = try std.fs.path.join(allocator, &.{ root, "session.jsonl" }); defer allocator.free(path); const source = \\{"type":"user","message":{"role":"user","content":"Find the failure cause"}} \\{"type":"assistant","message":{"role":"assistant","content":[{"type":"tool_use","id":"1","name":"Bash","input":{"command":"git status"}}]}} \\{"type":"user","message":{"role":"user","content":[{"type":"tool_result","tool_use_id":"1","content":"tool stdout"}]}} \\{"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":"The failure is in the indexing path."}]}} \\ ; try std.Io.Dir.cwd().writeFile(fs_io, .{ .sub_path = path, .data = source }); const parsed = try parseSession(scratch, allocator, path, "claude", source); try std.testing.expect(std.mem.indexOf(u8, parsed.content, "Find the failure cause") != null); try std.testing.expect(std.mem.indexOf(u8, parsed.content, "The failure is in the indexing path.") != null); try std.testing.expect(std.mem.indexOf(u8, parsed.content, "tool stdout") == null); try std.testing.expectEqual(@as(usize, 2), parsed.records.len); try std.testing.expectEqual(data.TranscriptRole.user, parsed.records[0].role); try std.testing.expectEqual(@as(u32, 1), parsed.records[0].line); try std.testing.expectEqual(data.TranscriptRole.assistant, parsed.records[1].role); try std.testing.expectEqual(@as(u32, 4), parsed.records[1].line);}test "parse codex session skips non-prose rows and keeps messages" { const allocator = std.testing.allocator; var arena = std.heap.ArenaAllocator.init(allocator); defer arena.deinit(); const scratch = arena.allocator(); var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const root = try tmp.dir.realPathFileAlloc(fs_io, ".", allocator); defer allocator.free(root); const path = try std.fs.path.join(allocator, &.{ root, "codex.jsonl" }); defer allocator.free(path); const source = \\{"timestamp":"2026-06-04T00:00:00Z","type":"session_meta","payload":{"cwd":"/tmp/glom"}} \\{"timestamp":"2026-06-04T00:00:00Z","type":"turn_context","payload":{"cwd":"/tmp/glom"}} \\{"timestamp":"2026-06-04T00:00:00Z","type":"event_msg","payload":{"type":"token_count","info":{"total":1}}} \\{"timestamp":"2026-06-04T00:00:00Z","type":"event_msg","payload":{"type":"agent_reasoning","text":"hidden reasoning"}} \\{"timestamp":"2026-06-04T00:00:00Z","type":"response_item","payload":{"type":"reasoning","summary":[{"type":"summary_text","text":"hidden summary"}],"content":null,"encrypted_content":"hidden encrypted"}} \\{"timestamp":"2026-06-04T00:00:00Z","type":"response_item","payload":{"type":"message","role":"user","content":[{"type":"input_text","text":"visible user"}]}} \\{"timestamp":"2026-06-04T00:00:00Z","type":"response_item","payload":{"type":"function_call","name":"shell","arguments":"{\"cmd\":\"echo\"}","call_id":"c1"}} \\{"timestamp":"2026-06-04T00:00:00Z","type":"response_item","payload":{"type":"function_call_output","call_id":"c1","output":"done \"quoted\""}} \\{"timestamp":"2026-06-04T00:00:00Z","type":"response_item","payload":{"type":"message","role":"assistant","content":[{"type":"output_text","text":"visible assistant"}]}} \\ ; try std.Io.Dir.cwd().writeFile(fs_io, .{ .sub_path = path, .data = source }); const parsed = try parseSession(scratch, allocator, path, "codex", source); try std.testing.expect(std.mem.indexOf(u8, parsed.content, "visible user") != null); try std.testing.expect(std.mem.indexOf(u8, parsed.content, "visible assistant") != null); try std.testing.expect(std.mem.indexOf(u8, parsed.content, "hidden") == null); try std.testing.expect(std.mem.indexOf(u8, parsed.content, "done") == null); try std.testing.expect(std.mem.indexOf(u8, parsed.content, "cmd") == null); try std.testing.expectEqual(@as(usize, 2), parsed.records.len); try std.testing.expectEqual(data.TranscriptRole.user, parsed.records[0].role); try std.testing.expectEqual(@as(u32, 6), parsed.records[0].line); try std.testing.expectEqual(data.TranscriptRole.assistant, parsed.records[1].role); try std.testing.expectEqual(@as(u32, 9), parsed.records[1].line); try std.testing.expectEqual(date.parseTimestampMillis("2026-06-04T00:00:00Z"), parsed.records[1].timestamp_ms);}test "parse codex session deduplicates mirrored visible messages" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const source = \\{"timestamp":"2026-06-04T00:00:01Z","type":"event_msg","payload":{"type":"agent_message","message":"same assistant repair"}} \\{"timestamp":"2026-06-04T00:00:01.005Z","type":"response_item","payload":{"type":"message","role":"assistant","content":[{"type":"output_text","text":"same assistant repair"}]}} \\ ; const parsed = try parseSession(arena.allocator(), std.testing.allocator, "/tmp/codex.jsonl", "codex", source); try std.testing.expectEqualStrings("same assistant repair", parsed.content); try std.testing.expectEqual(@as(usize, 1), parsed.records.len); try std.testing.expectEqual(@as(u32, 1), parsed.records[0].line); try std.testing.expectEqual(data.TranscriptRole.assistant, parsed.records[0].role);}test "codex fast skip preserves current relevant payload classes" { try std.testing.expect(codexLineCanSkipFast("{\"timestamp\":\"2026\",\"type\":\"turn_context\",\"payload\":{}}")); try std.testing.expect(codexLineCanSkipFast("{\"timestamp\":\"2026\",\"type\":\"event_msg\",\"payload\":{\"type\":\"token_count\"}}")); try std.testing.expect(codexLineCanSkipFast("{\"timestamp\":\"2026\",\"type\":\"event_msg\",\"payload\":{\"type\":\"context_compacted\"}}")); try std.testing.expect(codexLineCanSkipFast("{\"timestamp\":\"2026\",\"type\":\"response_item\",\"payload\":{\"type\":\"reasoning\"}}")); try std.testing.expect(codexLineCanSkipFast("{\"timestamp\":\"2026\",\"type\":\"response_item\",\"payload\":{\"type\":\"custom_tool_call\"}}")); try std.testing.expect(!codexLineCanSkipFast("{\"timestamp\":\"2026\",\"type\":\"event_msg\",\"payload\":{\"type\":\"user_message\"}}")); try std.testing.expect(!codexLineCanSkipFast("{\"timestamp\":\"2026\",\"type\":\"event_msg\",\"payload\":{\"type\":\"agent_message\"}}")); try std.testing.expect(!codexLineCanSkipFast("{\"timestamp\":\"2026\",\"type\":\"response_item\",\"payload\":{\"type\":\"message\"}}")); try std.testing.expect(codexLineCanSkipFast("{\"timestamp\":\"2026\",\"type\":\"response_item\",\"payload\":{\"type\":\"function_call\"}}")); try std.testing.expect(codexLineCanSkipFast("{\"timestamp\":\"2026\",\"type\":\"response_item\",\"payload\":{\"type\":\"function_call_output\"}}")); try std.testing.expect(!codexLineCanSkipFast("{\"type\":\"event_msg\",\"timestamp\":\"2026\",\"payload\":{\"type\":\"token_count\"}}"));}test "codex payload object slice handles nested objects and escaped delimiters" { const line = "{\"timestamp\":\"2026\",\"type\":\"response_item\",\"payload\":{\"type\":\"function_call_output\",\"call_id\":\"c1\",\"output\":\"brace } and \\\" quote\",\"nested\":{\"ok\":true}},\"after\":true}"; const payload = "{\"type\":\"function_call_output\",\"call_id\":\"c1\",\"output\":\"brace } and \\\" quote\",\"nested\":{\"ok\":true}}"; try std.testing.expectEqualStrings(payload, codexPayloadObjectSlice(line).?);}test "markdown parsing preserves frontmatter title metadata and body" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); const parsed = try parseMarkdown( arena.allocator(), "---\nname: fallback\ntitle: selected\nkind: memory\n---\nindexed body\n", "file", ); try std.testing.expectEqualStrings("selected", parsed.title); try std.testing.expectEqualStrings("indexed body", parsed.content); try std.testing.expect(std.mem.indexOf(u8, parsed.metadata.?, "\"kind\":\"memory\"") != null);}test "index commits native search rows for reopen" { const allocator = std.testing.allocator; var arena = std.heap.ArenaAllocator.init(allocator); defer arena.deinit(); const scratch = arena.allocator(); var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const root = try tmp.dir.realPathFileAlloc(fs_io, ".", allocator); defer allocator.free(root); const claude = try std.fs.path.join(scratch, &.{ root, ".claude" }); try std.Io.Dir.cwd().createDirPath(fs_io, claude); const codex = try std.fs.path.join(scratch, &.{ root, ".codex" }); try std.Io.Dir.cwd().createDirPath(fs_io, codex); try std.Io.Dir.cwd().writeFile(fs_io, .{ .sub_path = try std.fs.path.join(scratch, &.{ codex, "AGENTS.md" }), .data = "needle native index context\n" }); try std.Io.Dir.cwd().createDirPath(fs_io, try std.fs.path.join(scratch, &.{ codex, "sessions", "2026", "06", "09" })); try std.Io.Dir.cwd().writeFile(fs_io, .{ .sub_path = try std.fs.path.join(scratch, &.{ codex, "sessions", "2026", "06", "09", "rollout-2026-06-09T00-00-00-test.jsonl" }), .data = \\{"timestamp":"2026-06-09T00:00:00Z","type":"response_item","payload":{"type":"message","role":"assistant","content":[{"type":"output_text","text":"visible prose survives"}]}} \\{"timestamp":"2026-06-09T00:00:01Z","type":"response_item","payload":{"type":"function_call","name":"shell","arguments":"{\"cmd\":\"suppressedcalltoken\"}","call_id":"c1"}} \\{"timestamp":"2026-06-09T00:00:02Z","type":"response_item","payload":{"type":"function_call_output","call_id":"c1","output":"needle tool output"}} \\ }); const db_path = try std.fs.path.join(scratch, &.{ root, "index.sql" }); { var db = try database.Database.init(allocator, db_path); defer db.deinit(); var input_storage = try input.Storage.init(allocator, .{ .file_bytes = 4096 }); defer input_storage.deinit(allocator); input_storage.activate(); _ = try indexAll(scratch, &db, &input_storage, .{ .claude_root = claude, .codex_root = codex, }); } var reopened = try database.Database.init(allocator, db_path); defer reopened.deinit(); const page = try searchForTest(&reopened, scratch, "needle", .{}); try std.testing.expectEqual(@as(usize, 1), page.total); const prose = try searchForTest(&reopened, scratch, "visible", .{}); try std.testing.expectEqual(@as(usize, 1), prose.total); const assistant = try searchForTest(&reopened, scratch, "\"visible prose\"", .{ .role = .assistant }); try std.testing.expectEqual(@as(usize, 1), assistant.total); try std.testing.expectEqual(@as(?u32, 1), assistant.rows[0].line); try std.testing.expectEqual(date.parseTimestampMillis("2026-06-09T00:00:00Z"), assistant.rows[0].timestamp_ms); const user = try searchForTest(&reopened, scratch, "\"visible prose\"", .{ .role = .user }); try std.testing.expectEqual(@as(usize, 0), user.total); const tool = try searchForTest(&reopened, scratch, "suppressedcalltoken", .{}); try std.testing.expectEqual(@as(usize, 0), tool.total);}test "Glom index preflights every changed whole-file input before database mutation" { comptime { @stardustClaim( @import("alloc_phase").capacity.witness(@import("./input/root.zig").Storage, "glom_index_input_database_boundary_overload"), null, null, null, null, null, null, ); } comptime { @stardustClaim( @import("alloc_phase").capacity.witness(@import("./input/root.zig").Storage, "glom_index_input_database_boundary_transitive_risk"), null, null, null, null, null, null, ); } const allocator = std.testing.allocator; var arena = std.heap.ArenaAllocator.init(allocator); defer arena.deinit(); const scratch = arena.allocator(); var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const root = try tmp.dir.realPathFileAlloc(fs_io, ".", allocator); defer allocator.free(root); const claude = try std.fs.path.join(scratch, &.{ root, ".claude" }); try std.Io.Dir.cwd().createDirPath(fs_io, claude); const codex = try std.fs.path.join(scratch, &.{ root, ".codex" }); try std.Io.Dir.cwd().createDirPath(fs_io, codex); try std.Io.Dir.cwd().writeFile(fs_io, .{ .sub_path = try std.fs.path.join(scratch, &.{ codex, "AGENTS.md" }), .data = "abcde", }); try std.Io.Dir.cwd().writeFile(fs_io, .{ .sub_path = try std.fs.path.join(scratch, &.{ codex, "config.toml" }), .data = "abcdef", }); var db = try database.Database.init(allocator, ":memory:"); defer db.deinit(); var input_storage = try input.Storage.init(allocator, .{ .file_bytes = 5 }); defer input_storage.deinit(allocator); input_storage.activate(); try std.testing.expectError( error.IndexInputCapacityExceeded, indexAll(scratch, &db, &input_storage, .{ .claude_root = claude, .codex_root = codex, .full = true, }), ); const stats = try db.stats(scratch); try std.testing.expectEqual(@as(i64, 0), stats.total);}test "Glom index reads only the probe for a saturated transcript" { const allocator = std.testing.allocator; var arena = std.heap.ArenaAllocator.init(allocator); defer arena.deinit(); const scratch = arena.allocator(); var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const root = try tmp.dir.realPathFileAlloc(fs_io, ".", allocator); defer allocator.free(root); const claude = try std.fs.path.join(scratch, &.{ root, ".claude" }); try std.Io.Dir.cwd().createDirPath(fs_io, claude); const codex = try std.fs.path.join(scratch, &.{ root, ".codex" }); const sessions = try std.fs.path.join(scratch, &.{ codex, "sessions", "2026", "08", "05" }); try std.Io.Dir.cwd().createDirPath(fs_io, sessions); const path = try std.fs.path.join(scratch, &.{ sessions, "rollout-test.jsonl" }); var source: std.Io.Writer.Allocating = .init(allocator); defer source.deinit(); try source.writer.writeAll( "{\"type\":\"event_msg\",\"payload\":{\"type\":\"user_message\",\"message\":\"early searchable ", ); try source.writer.splatByteAll('x', data.transcript_content_limit + 128); try source.writer.writeAll("\"}}\n"); try source.writer.splatByteAll( ' ', transcript_probe_bytes[0] + 4096 - source.writer.buffered().len, ); const source_bytes = source.writer.buffered(); try std.Io.Dir.cwd().writeFile(fs_io, .{ .sub_path = path, .data = source_bytes }); const initial_stat = try std.Io.Dir.cwd().statFile(fs_io, path, .{}); var db = try database.Database.init(allocator, ":memory:"); defer db.deinit(); var input_storage = try input.Storage.init(allocator, .{ .file_bytes = transcript_probe_bytes[0], }); defer input_storage.deinit(allocator); input_storage.activate(); const stats = try indexAll(scratch, &db, &input_storage, .{ .claude_root = claude, .codex_root = codex, }); try std.testing.expectEqual(@as(usize, 1), stats.new); try std.testing.expectEqual(transcript_probe_bytes[0], input_storage.status().high_water_file_bytes); const document = (try db.getDocument(scratch, path)).?; try std.testing.expectEqual(data.transcript_content_limit, document.content.len); const tail = "\n{\"type\":\"event_msg\",\"payload\":{\"type\":\"agent_message\",\"message\":\"late prose\"}}\n"; var source_file = try std.Io.Dir.cwd().openFile(fs_io, path, .{ .mode = .read_write }); defer source_file.close(fs_io); try source_file.writePositionalAll(fs_io, tail, source_bytes.len); try std.Io.Dir.cwd().setTimestamps(fs_io, path, .{ .modify_timestamp = .{ .new = .{ .nanoseconds = initial_stat.mtime.nanoseconds, } }, }); var append_storage = try input.Storage.init(allocator, .{ .file_bytes = transcript_probe_bytes[0], }); defer append_storage.deinit(allocator); append_storage.activate(); const append_stats = try indexAll(scratch, &db, &append_storage, .{ .claude_root = claude, .codex_root = codex, }); try std.testing.expectEqual(@as(usize, 1), append_stats.updated); try std.testing.expectEqual(@as(usize, 0), append_storage.status().high_water_file_bytes); const updated = (try db.getDocument(scratch, path)).?; try std.testing.expectEqual(@as(i64, @intCast(source_bytes.len + tail.len)), updated.size); try std.testing.expectEqual(data.transcript_content_limit, updated.content.len); const early = try searchForTest(&db, scratch, "early searchable", .{}); try std.testing.expectEqual(@as(usize, 1), early.total); const late = try searchForTest(&db, scratch, "late prose", .{}); try std.testing.expectEqual(@as(usize, 0), late.total); const rewrite_offset = std.mem.indexOf(u8, source_bytes, "early searchable").?; try source_file.writePositionalAll(fs_io, "other", rewrite_offset); try std.Io.Dir.cwd().setTimestamps(fs_io, path, .{ .modify_timestamp = .{ .new = .{ .nanoseconds = @intCast(initial_stat.mtime.nanoseconds + std.time.ns_per_s), } }, }); var rewrite_storage = try input.Storage.init(allocator, .{ .file_bytes = transcript_probe_bytes[0], }); defer rewrite_storage.deinit(allocator); rewrite_storage.activate(); const rewrite_stats = try indexAll(scratch, &db, &rewrite_storage, .{ .claude_root = claude, .codex_root = codex, }); try std.testing.expectEqual(@as(usize, 1), rewrite_stats.updated); try std.testing.expectEqual( transcript_probe_bytes[0], rewrite_storage.status().high_water_file_bytes, ); const removed = try searchForTest(&db, scratch, "early searchable", .{}); try std.testing.expectEqual(@as(usize, 0), removed.total); const replacement = try searchForTest(&db, scratch, "other searchable", .{}); try std.testing.expectEqual(@as(usize, 1), replacement.total);}test "Glom index streams a complete unsaturated transcript" { const allocator = std.testing.allocator; var arena = std.heap.ArenaAllocator.init(allocator); defer arena.deinit(); var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const root = try tmp.dir.realPathFileAlloc(fs_io, ".", allocator); defer allocator.free(root); const path = try std.fs.path.join(allocator, &.{ root, "session.jsonl" }); defer allocator.free(path); var source: std.Io.Writer.Allocating = .init(allocator); defer source.deinit(); try source.writer.writeAll( "{\"type\":\"event_msg\",\"payload\":{\"type\":\"user_message\",\"message\":\"visible prose\"}}\n", ); for (0..5) |_| { try source.writer.writeAll("{\"type\":\"turn_context\",\"payload\":\""); try source.writer.splatByteAll('x', 1024 * 1024); try source.writer.writeAll("\"}\n"); } try source.writer.writeAll( "{\"type\":\"event_msg\",\"payload\":{\"type\":\"agent_message\",\"message\":\"late prose\"}}\n", ); const source_bytes = source.writer.buffered(); try std.Io.Dir.cwd().writeFile(fs_io, .{ .sub_path = path, .data = source_bytes }); var input_storage = try input.Storage.init(allocator, .{ .file_bytes = transcript_probe_bytes[0], }); defer input_storage.deinit(allocator); input_storage.activate(); const parsed = try parseIndexFile( arena.allocator(), allocator, &input_storage, .{ .entry = .{ .path = path, .source = "codex", .kind = "session", .project = null }, .rowid = null, .mtime = 1, .size = source_bytes.len, .reuse_saturated = false, }, ); try std.testing.expectEqualStrings("visible prose\nlate prose", parsed.document.content); try std.testing.expectEqual(@as(usize, 2), parsed.document.transcript_records.?.len); try std.testing.expectEqual(@as(u32, 7), parsed.document.transcript_records.?[1].line); try std.testing.expect(!parsed.content_saturated); try std.testing.expect(source_bytes.len > input_storage.status().file_bytes); try std.testing.expectEqual( transcript_probe_bytes[0], input_storage.status().high_water_file_bytes, );}test "Glom index reports a transcript record larger than its window" { const allocator = std.testing.allocator; var arena = std.heap.ArenaAllocator.init(allocator); defer arena.deinit(); const scratch = arena.allocator(); var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); const root = try tmp.dir.realPathFileAlloc(fs_io, ".", allocator); defer allocator.free(root); const claude = try std.fs.path.join(scratch, &.{ root, ".claude" }); try std.Io.Dir.cwd().createDirPath(fs_io, claude); const codex = try std.fs.path.join(scratch, &.{ root, ".codex" }); try std.Io.Dir.cwd().createDirPath(fs_io, codex); const path = try std.fs.path.join(scratch, &.{ codex, "history.jsonl" }); try std.Io.Dir.cwd().writeFile(fs_io, .{ .sub_path = path, .data = "abcde\n" }); var db = try database.Database.init(allocator, ":memory:"); defer db.deinit(); var input_storage = try input.Storage.init(allocator, .{ .file_bytes = 5 }); defer input_storage.deinit(allocator); input_storage.activate(); var stats = try indexAll(scratch, &db, &input_storage, .{ .claude_root = claude, .codex_root = codex, }); defer stats.deinit(scratch); try std.testing.expectEqual(@as(usize, 1), stats.errors); try std.testing.expectEqual(@as(?usize, 1), stats.parse_errors.get("codex:history")); try std.testing.expectEqual(@as(usize, 1), stats.error_paths.items.len); try std.testing.expect(std.mem.endsWith( u8, stats.error_paths.items[0], ": TranscriptRecordCapacityExceeded", )); try std.testing.expect(!input_storage.status().terminal);}Source: tools/glom/src/root.zig:13
zig
pub const index = @import("index.zig");Complete call list for index.indexAll
9 direct calls.
tiny.glom.index.IndexStats.deinit[method] attools/glom/src/index.zig:42tiny.glom.index.IndexStats.init[function] attools/glom/src/index.zig:29tiny.glom.index.discover[function] attools/glom/src/index.zig:78tools.glom.src.freeFileEntries[function] — private; no exact target attools/glom/src/index.zig:1273in nearest public ownertiny.glom.indextools.glom.src.isTranscriptKind[function] — private; no exact target attools/glom/src/index.zig:504in nearest public ownertiny.glom.indextools.glom.src.parseIndexFile[function] — private; no exact target attools/glom/src/index.zig:264in nearest public ownertiny.glom.indextools.glom.src.recordIndexError[function] — private; no exact target attools/glom/src/index.zig:288in nearest public ownertiny.glom.indextools.glom.src.recordTop[function] — private; no exact target attools/glom/src/index.zig:1311in nearest public ownertiny.glom.indextools.glom.src.timestampSeconds[function] — private; no exact target attools/glom/src/index.zig:1247in nearest public ownertiny.glom.index
Audit
| Definitions | 10 |
|---|---|
| Public names | 10 |
| Members | 21 |
| Version | 26.7.0 |
| Revision | daab053ee433 |