Autodoc: new decl search system (#15475)

New search system is based on a Radix Tree. The Radix Tree contains a shallow list of all decl names (ie no paths), plus some suffixes, split by following the official style guide (eg "HashMapUnmanaged" also produces "MapUnmanaged" and "Unmanaged", same with snake_case and camelCase names).

Additionally, the search system uses the decl graph data to recognize hierarchical relationships between decls, allowing you to zero on a target namespace for search. As an example "fs create" will score highe all things related to the creation of files and directories inside of `std.fs`, while still showing (but with lower score) matches from `std.Bulild`. 

As another example "fs windows" will prioritize windows-related results in `std.fs`, while "windows fs" will prioritize fs-related results in `std.windows`.

1 files changed, 17 insertions, 4 deletions

diff --git a/src/Autodoc.zig b/src/Autodoc.zig
index 3c1beda74a..91e252a3f8 100644
--- a/src/Autodoc.zig
+++ b/src/Autodoc.zig
@@ -234,12 +234,13 @@ pub fn generateZirData(self: *Autodoc) !void {
     };
 
     const tldoc_comment = try self.getTLDocComment(file);
+    const cleaned_tldoc_comment = try self.findGuidePaths(file, tldoc_comment);
+    defer self.arena.free(cleaned_tldoc_comment);
     try self.ast_nodes.append(self.arena, .{
         .name = "(root)",
-        .docs = tldoc_comment,
+        .docs = cleaned_tldoc_comment,
     });
     try self.files.put(self.arena, file, main_type_index);
-    try self.findGuidePaths(file, tldoc_comment);
 
     _ = try self.walkInstruction(file, &root_scope, .{}, Zir.main_struct_inst, false);
 
@@ -349,6 +350,7 @@ pub fn generateZirData(self: *Autodoc) !void {
     var docs_dir = try self.comp_module.comp.zig_lib_directory.handle.openDir("docs", .{});
     defer docs_dir.close();
     try docs_dir.copyFile("main.js", output_dir, "main.js", .{});
+    try docs_dir.copyFile("commonmark.js", output_dir, "commonmark.js", .{});
     try docs_dir.copyFile("index.html", output_dir, "index.html", .{});
 }
 
@@ -4754,14 +4756,20 @@ fn getTLDocComment(self: *Autodoc, file: *File) ![]const u8 {
     return comment.items;
 }
 
-fn findGuidePaths(self: *Autodoc, file: *File, str: []const u8) !void {
+/// Returns the doc comment cleared of autodoc directives.
+fn findGuidePaths(self: *Autodoc, file: *File, str: []const u8) ![]const u8 {
     const guide_prefix = "zig-autodoc-guide:";
     const section_prefix = "zig-autodoc-section:";
 
     try self.guide_sections.append(self.arena, .{}); // add a default section
     var current_section = &self.guide_sections.items[self.guide_sections.items.len - 1];
 
-    var it = std.mem.tokenize(u8, str, "\n");
+    var clean_docs: std.ArrayListUnmanaged(u8) = .{};
+    errdefer clean_docs.deinit(self.arena);
+
+    // TODO: this algo is kinda inefficient
+
+    var it = std.mem.split(u8, str, "\n");
     while (it.next()) |line| {
         const trimmed_line = std.mem.trim(u8, line, " ");
         if (std.mem.startsWith(u8, trimmed_line, guide_prefix)) {
@@ -4775,8 +4783,13 @@ fn findGuidePaths(self: *Autodoc, file: *File, str: []const u8) !void {
                 .name = trimmed_section_name,
             });
             current_section = &self.guide_sections.items[self.guide_sections.items.len - 1];
+        } else {
+            try clean_docs.appendSlice(self.arena, line);
+            try clean_docs.append(self.arena, '\n');
         }
     }
+
+    return clean_docs.toOwnedSlice(self.arena);
 }
 
 fn addGuide(self: *Autodoc, file: *File, guide_path: []const u8, section: *Section) !void {