documentation: continue down the rabbit hole

Why am I inventing my own documentation format? It's hard to say, but
it's probably because I Am Stupid. The main problem here will be lack
of automatic hyperlinks generated in the documentation. Oh well, it's
experimental.

build.zig

@@ -6,16 +6,17 @@ pub fn build(b: *std.build.Builder) void {
 
     demo(b, target, optimize);
     tokenator(b, target, optimize);
+    zed(b, target, optimize);
 
-    const tests = b.step("test", "Run unit tests");
+    const test_step = b.step("test", "Run unit tests");
-    const lib_tests = b.addTest(.{
+    const tests = b.addTest(.{
         .name = "tests",
         .root_source_file = .{ .path = "source/noclip.zig" },
         .target = target,
         .optimize = optimize,
     });
 
-    tests.dependOn(&lib_tests.step);
+    test_step.dependOn(&tests.step);
 }
 
 fn demo(b: *std.build.Builder, target: anytype, optimize: anytype) void {
@@ -49,3 +50,30 @@ fn tokenator(b: *std.build.Builder, target: anytype, optimize: anytype) void {
 
     tok_step.dependOn(&install_tok.step);
 }
+
+fn zed(b: *std.build.Builder, target: anytype, optimize: anytype) void {
+    const tok_step = b.step("zed", "Build documentation generator");
+    const noclip = b.createModule(.{ .source_file = .{ .path = "source/noclip.zig" } });
+
+    const exe = b.addExecutable(.{
+        .name = "zed",
+        .root_source_file = .{ .path = "documentation/zed.zig" },
+        .target = target,
+        .optimize = optimize,
+    });
+    exe.addModule("noclip", noclip);
+    const install_tok = b.addInstallArtifact(exe);
+
+    tok_step.dependOn(&install_tok.step);
+
+    const test_step = b.step("run-zed-tests", "Test documentation generator");
+    const tests = b.addTest(.{
+        .name = "test-zed",
+        .root_source_file = .{ .path = "documentation/zed.zig" },
+        .target = target,
+        .optimize = optimize,
+    });
+
+    const runcmd = tests.run();
+    test_step.dependOn(&runcmd.step);
+}

documentation/zed.zig

@@ -0,0 +1,406 @@
+const std = @import("std");
+
+const Directive = enum {
+    section,
+    description,
+    example,
+    include,
+};
+
+const ExampleFormat = enum {
+    zig,
+    console,
+
+    pub fn default_format() ExampleFormat {
+        return .zig;
+    }
+};
+
+const DescriptionFormat = enum {
+    markdown,
+};
+
+const ParseError = error{
+    LeadingGarbage,
+    ExpectedDirectivePrefix,
+    ExpectedDirectiveSuffix,
+    ExpectedNonemptySuffix,
+    ExpectedDirectiveTerminator,
+    UnknownDirective,
+    UnknownSuffix,
+    UnexpectedDirectiveMismatch,
+    MissingRequiredTrailer,
+    UnsupportedFormat,
+    UnexpectedDirective,
+};
+
+const RawDirectiveLine = struct {
+    directive: Directive,
+    suffix: ?[]const u8, // the part after the dot. Null if there is no dot
+    trailer: ?[]const u8, // the part after the colon. null if empty or whitespace-only
+
+    // line has had its trailing newline stripped
+    fn from_line(line: []const u8) ParseError!RawDirectiveLine {
+        if (line.len < 1 or line[0] != '@') return error.ExpectedDirectivePrefix;
+        var result: RawDirectiveLine = .{
+            .directive = undefined,
+            .suffix = null,
+            .trailer = null,
+        };
+        var offset: usize = blk: {
+            inline for (comptime std.meta.fields(Directive)) |field| {
+                const len = field.name.len + 1;
+
+                if (line.len > len and
+                    (line[len] == ':' or line[len] == '.') and
+                    std.mem.eql(u8, line[1..len], field.name))
+                {
+                    result.directive = @field(Directive, field.name);
+                    break :blk len;
+                }
+            }
+
+            return error.UnknownDirective;
+        };
+
+        if (line[offset] == '.') blk: {
+            const suffix_start = offset + 1;
+            while (offset < line.len) : (offset += 1) {
+                if (line[offset] == ':') {
+                    if (offset <= suffix_start) return error.ExpectedNonemptySuffix;
+
+                    result.suffix = line[suffix_start..offset];
+                    break :blk;
+                }
+            }
+
+            return error.ExpectedDirectiveTerminator;
+        }
+
+        if (line[offset] != ':') return error.ExpectedDirectiveTerminator;
+        offset += 1;
+        while (offset < line.len) : (offset += 1) {
+            if (!std.ascii.isWhitespace(line[offset])) {
+                // TODO: also trim trailing whitespace
+                result.trailer = line[offset..line.len];
+                break;
+            }
+        }
+
+        return result;
+    }
+};
+
+fn expect_optional_string(candidate: ?[]const u8, expected: ?[]const u8) !void {
+    if (expected) |exstr| {
+        if (candidate) |canstr| {
+            try std.testing.expectEqualStrings(exstr, canstr);
+        } else {
+            std.debug.print("Expected \"{s}\", got null\n", .{exstr});
+            return error.TestExpectedEqual;
+        }
+    } else {
+        if (candidate) |canstr| {
+            std.debug.print("Expected null, got \"{s}\"\n", .{canstr});
+            return error.TestExpectedEqual;
+        }
+    }
+}
+
+fn expect_rdl(candidate: RawDirectiveLine, expected: RawDirectiveLine) !void {
+    try std.testing.expectEqual(expected.directive, candidate.directive);
+    try expect_optional_string(candidate.suffix, expected.suffix);
+    try expect_optional_string(candidate.trailer, expected.trailer);
+}
+
+test "RawDirectiveLine.from_line" {
+    try expect_rdl(
+        try RawDirectiveLine.from_line("@section:"),
+        .{ .directive = .section, .suffix = null, .trailer = null },
+    );
+    try expect_rdl(
+        try RawDirectiveLine.from_line("@example:"),
+        .{ .directive = .example, .suffix = null, .trailer = null },
+    );
+    try expect_rdl(
+        try RawDirectiveLine.from_line("@example.zig:"),
+        .{ .directive = .example, .suffix = "zig", .trailer = null },
+    );
+
+    try expect_rdl(
+        try RawDirectiveLine.from_line("@example.zig: ./example.file"),
+        .{ .directive = .example, .suffix = "zig", .trailer = "./example.file" },
+    );
+
+    try std.testing.expectError(
+        ParseError.UnknownDirective,
+        RawDirectiveLine.from_line("@unknown:"),
+    );
+    try std.testing.expectError(
+        ParseError.ExpectedDirectivePrefix,
+        RawDirectiveLine.from_line("hello"),
+    );
+    // TODO: this would be better if it produced error.ExpectedDirectiveTerminator
+    // instead, but it complicates the logic to do so.
+    try std.testing.expectError(
+        ParseError.UnknownDirective,
+        RawDirectiveLine.from_line("@section"),
+    );
+    try std.testing.expectError(
+        ParseError.ExpectedDirectiveTerminator,
+        RawDirectiveLine.from_line("@example.tag"),
+    );
+    try std.testing.expectError(
+        ParseError.ExpectedNonemptySuffix,
+        RawDirectiveLine.from_line("@example.:"),
+    );
+}
+
+const SectionDirectiveLine = struct {
+    name: []const u8,
+
+    fn from_raw(raw: RawDirectiveLine) ParseError!DirectiveLine {
+        if (raw.directive != .section) return error.UnexpectedDirectiveMismatch;
+        if (raw.suffix != null) return error.UnknownSuffix;
+        if (raw.trailer == null) return error.MissingRequiredTrailer;
+
+        return .{ .section = .{ .name = raw.trailer.? } };
+    }
+};
+
+test "SectionDirectiveLine.from_raw" {
+    const line = try SectionDirectiveLine.from_raw(.{ .directive = .section, .suffix = null, .trailer = "Section" });
+    try std.testing.expectEqualStrings("Section", line.section.name);
+
+    try std.testing.expectError(
+        ParseError.UnexpectedDirectiveMismatch,
+        SectionDirectiveLine.from_raw(.{ .directive = .example, .suffix = null, .trailer = "Section" }),
+    );
+    try std.testing.expectError(
+        ParseError.UnknownSuffix,
+        SectionDirectiveLine.from_raw(.{ .directive = .section, .suffix = "importante", .trailer = "Section" }),
+    );
+    try std.testing.expectError(
+        ParseError.MissingRequiredTrailer,
+        SectionDirectiveLine.from_raw(.{ .directive = .section, .suffix = null, .trailer = null }),
+    );
+}
+
+const DescriptionDirectiveLine = struct {
+    format: DescriptionFormat,
+    include: ?[]const u8,
+
+    fn from_raw(raw: RawDirectiveLine) ParseError!DirectiveLine {
+        if (raw.directive != .description) return error.UnexpectedDirectiveMismatch;
+        const format: DescriptionFormat = if (raw.suffix) |suffix|
+            std.meta.stringToEnum(DescriptionFormat, suffix) orelse return error.UnsupportedFormat
+        else
+            .markdown;
+
+        return .{ .description = .{ .format = format, .include = raw.trailer } };
+    }
+};
+
+const ExampleDirectiveLine = struct {
+    format: ExampleFormat,
+    include: ?[]const u8,
+
+    fn from_raw(raw: RawDirectiveLine) ParseError!DirectiveLine {
+        if (raw.directive != .example) return error.UnexpectedDirectiveMismatch;
+        const format: ExampleFormat = if (raw.suffix) |suffix|
+            std.meta.stringToEnum(ExampleFormat, suffix) orelse return error.UnsupportedFormat
+        else
+            .zig;
+
+        return .{ .example = .{ .format = format, .include = raw.trailer } };
+    }
+};
+
+const IncludeDirectiveLine = struct {
+    path: []const u8,
+
+    fn from_raw(raw: RawDirectiveLine) ParseError!DirectiveLine {
+        if (raw.directive != .include) return error.UnexpectedDirectiveMismatch;
+        if (raw.suffix != null) return error.UnknownSuffix;
+        if (raw.trailer == null) return error.MissingRequiredTrailer;
+
+        return .{ .include = .{ .path = raw.trailer.? } };
+    }
+};
+
+const DirectiveLine = union(Directive) {
+    section: SectionDirectiveLine,
+    description: DescriptionDirectiveLine,
+    example: ExampleDirectiveLine,
+    include: IncludeDirectiveLine,
+
+    fn from_line(line: []const u8) ParseError!DirectiveLine {
+        const raw = try RawDirectiveLine.from_line(line);
+        return try switch (raw.directive) {
+            .section => SectionDirectiveLine.from_raw(raw),
+            .description => DescriptionDirectiveLine.from_raw(raw),
+            .example => ExampleDirectiveLine.from_raw(raw),
+            .include => IncludeDirectiveLine.from_raw(raw),
+        };
+    }
+};
+
+const Body = union(enum) {
+    in_line: []const u8,
+    include: []const u8,
+};
+
+const Example = struct {
+    format: ExampleFormat,
+    body: Body,
+};
+
+const Section = struct {
+    name: []const u8,
+    segments: []const Segment,
+};
+
+const Segment = union(enum) {
+    description: Description,
+    example: Example,
+};
+
+const Description = struct {
+    format: DescriptionFormat,
+    body: Body,
+};
+
+const Document = []const Section;
+
+fn read_directive(line: []const u8) ParseError!DirectiveLine {
+    if (line[0] != '@') return error.ExpectedDirective;
+    inline for (comptime std.meta.fields(Directive)) |field| {
+        const len = field.name.len + 1;
+        if (line.len > len and
+            (line[len] == ':' or line[len] == '.') and
+            std.mem.eql(u8, line[1..len], field.name))
+        {
+            return @field(Directive, field.name);
+        }
+    }
+}
+
+const ParserState = enum {
+    section_or_include,
+    any_directive,
+};
+
+fn slice_to_next_directive(lines: *std.mem.TokenIterator(u8)) []const u8 {
+    const start = lines.index + 1;
+    while (lines.peek()) |line| : (_ = lines.next()) {
+        // this approach is likely too sloppy
+        if (DirectiveLine.from_line(line)) |_| {
+            return lines.buffer[start..lines.index];
+        } else |_| {}
+    }
+
+    // we hit EOF
+    return lines.buffer[start..lines.buffer.len];
+}
+
+pub fn parse(allocator: std.mem.Allocator, input: []const u8) !Document {
+    var lines = std.mem.tokenize(u8, input, "\n");
+
+    var doc_builder = std.ArrayList(Section).init(allocator);
+    var section_builder = std.ArrayList(Segment).init(allocator);
+
+    var state: ParserState = .section_or_include;
+
+    var current_section: Section = undefined;
+
+    while (lines.next()) |line| {
+        const dline = try DirectiveLine.from_line(line);
+        switch (state) {
+            .section_or_include => switch (dline) {
+                .section => |sline| {
+                    current_section.name = sline.name;
+                    state = .any_directive;
+                },
+                .include => |iline| {
+                    // read the file at iline.path
+                    const doc = try parse(allocator, iline.path);
+                    defer allocator.free(doc);
+                    try doc_builder.appendSlice(doc);
+                },
+                else => return error.UnexpectedDirective,
+            },
+            .any_directive => switch (dline) {
+                .section => |sline| {
+                    current_section.segments = try section_builder.toOwnedSlice();
+                    try doc_builder.append(current_section);
+                    current_section.name = sline.name;
+                },
+                .include => |iline| {
+                    const doc = try parse(allocator, iline.path);
+                    defer allocator.free(doc);
+                    try doc_builder.appendSlice(doc);
+                    state = .section_or_include;
+                },
+                .example => |exline| {
+                    try section_builder.append(.{ .example = .{
+                        .format = exline.format,
+                        .body = if (exline.include) |incl|
+                            .{ .include = incl }
+                        else
+                            .{ .in_line = slice_to_next_directive(&lines) },
+                    } });
+                },
+                .description => |desline| {
+                    try section_builder.append(.{ .description = .{
+                        .format = desline.format,
+                        .body = if (desline.include) |incl|
+                            .{ .include = incl }
+                        else
+                            .{ .in_line = slice_to_next_directive(&lines) },
+                    } });
+                },
+            },
+        }
+    }
+    current_section.segments = try section_builder.toOwnedSlice();
+    try doc_builder.append(current_section);
+
+    return doc_builder.toOwnedSlice();
+}
+
+test "parser" {
+    const doc = try parse(
+        std.testing.allocator,
+        \\@section: first section
+        \\@example:
+        \\
+        \\what
+        \\have we got
+        \\here
+        \\@description: include
+        \\@section: second
+        \\@description:
+        \\words
+        \\@description:
+        ,
+    );
+
+    for (doc) |section| {
+        std.debug.print("section: {s}\n", .{section.name});
+        for (section.segments) |seg| {
+            switch (seg) {
+                .description => |desc| switch (desc.body) {
+                    .include => |inc| std.debug.print("  seg: description, body: include {s}\n", .{inc}),
+                    .in_line => |inl| std.debug.print("  seg: description, body: inline {s}\n", .{inl}),
+                },
+                .example => |desc| switch (desc.body) {
+                    .include => |inc| std.debug.print("  seg: example, body: include {s}\n", .{inc}),
+                    .in_line => |inl| std.debug.print("  seg: example, body: inline {s}\n", .{inl}),
+                },
+            }
+        }
+    }
+
+    for (doc) |section| std.testing.allocator.free(section.segments);
+    std.testing.allocator.free(doc);
+}