nice-data/src/tokenizer.zig

const std = @import("std");

const Diagnostics = @import("./parser.zig").Diagnostics;

pub const Error = error{
    BadToken,
    MixedIndentation,
    UnquantizedIndentation,
    TooMuchIndentation,
    MissingNewline,
    TrailingWhitespace,
    Impossible,
};

pub const DetectedIndentation = union(enum) {
    unknown: void,
    spaces: usize,
    tabs: void,
};

pub const InlineItem = union(enum) {
    empty: void,
    scalar: []const u8,
    line_string: []const u8,
    space_string: []const u8,

    flow_list: []const u8,
    flow_map: []const u8,

    pub fn lineEnding(self: InlineItem) u8 {
        return switch (self) {
            .line_string => '\n',
            .space_string => ' ',
            else => unreachable,
        };
    }
};

pub const LineContents = union(enum) {
    comment: []const u8,

    in_line: InlineItem,
    list_item: InlineItem,
    map_item: struct { key: []const u8, val: InlineItem },
};

pub const ShiftDirection = enum { indent, dedent, none };

pub const LineShift = union(ShiftDirection) {
    indent: void,
    // we can dedent multiple levels at once.
    dedent: usize,
    none: void,
};

pub const Line = struct {
    shift: LineShift,
    contents: LineContents,
    raw: []const u8,
};

// buffer is expected to be either LineBuffer or FixedLineBuffer, but can
// technically be anything with a `nextLine` method
pub fn LineTokenizer(comptime Buffer: type) type {
    return struct {
        buffer: Buffer,
        index: usize = 0,
        indentation: DetectedIndentation = .unknown,
        last_indent: usize = 0,
        diagnostics: *Diagnostics,
        row: usize = 0,

        pub fn next(self: *@This()) Error!?Line {
            lineloop: while (self.buffer.nextLine()) |raw_line| {
                var indent: usize = 0;
                for (raw_line, 0..) |char, idx| {
                    switch (char) {
                        ' ' => {
                            switch (self.indentation) {
                                // There's a weird coupling here because we can't set this until
                                // all spaces have been consumed. I also thought about ignoring
                                // spaces on comment lines since those don't affect the
                                // relative indent/dedent, but then we would allow comments
                                // to ignore our indent quantum, which I dislike due to it making
                                // ugly documents.
                                .unknown => self.indentation = .{ .spaces = 0 },
                                .spaces => {},
                                .tabs => return error.MixedIndentation,
                            }
                        },
                        '\t' => {
                            switch (self.indentation) {
                                .unknown => self.indentation = .tabs,
                                .spaces => return error.MixedIndentation,
                                .tabs => {},
                            }
                        },
                        '\r' => {
                            return error.BadToken;
                        },
                        else => {
                            indent = idx;
                            break;
                        },
                    }
                } else {
                    if (raw_line.len > 0) return error.TrailingWhitespace;
                    continue :lineloop;
                }

                var quantized: usize = if (self.indentation == .spaces) quant: {
                    if (self.indentation.spaces == 0) {
                        self.indentation.spaces = indent;
                    }
                    if (@rem(indent, self.indentation.spaces) != 0)
                        return error.UnquantizedIndentation;

                    break :quant @divExact(indent, self.indentation.spaces);
                } else indent;

                const shift: LineShift = if (quantized > self.last_indent) rel: {
                    if ((quantized - self.last_indent) > 1)
                        return error.TooMuchIndentation;
                    break :rel .indent;
                } else if (quantized < self.last_indent)
                    .{ .dedent = self.last_indent - quantized }
                else
                    .none;

                defer {
                    self.row += 1;
                    self.last_indent = quantized;
                }

                const line = raw_line[indent..];

                // this should not be possible, as empty lines are caught earlier.
                if (line.len == 0) return error.Impossible;

                switch (line[0]) {
                    '#' => {
                        // force comments to be followed by a space. This makes them
                        // behave the same way as strings, actually.
                        if (line.len > 1 and line[1] != ' ') return error.BadToken;

                        // simply lie about indentation when the line is a comment.
                        quantized = self.last_indent;
                        return .{
                            .shift = .none,
                            .contents = .{ .comment = line[1..] },
                            .raw = line,
                        };
                    },
                    '|', '>', '[', '{' => {
                        return .{
                            .shift = shift,
                            .contents = .{ .in_line = try detectInlineItem(line) },
                            .raw = line,
                        };
                    },
                    '-' => {
                        if (line.len > 1 and line[1] != ' ') return error.BadToken;

                        return if (line.len == 1) .{
                            .shift = shift,
                            .contents = .{ .list_item = .empty },
                            .raw = line,
                        } else .{
                            .shift = shift,
                            .contents = .{ .list_item = try detectInlineItem(line[2..]) },
                            .raw = line,
                        };
                    },
                    else => {
                        for (line, 0..) |char, idx| {
                            if (char == ':') {
                                if (idx + 1 == line.len) return .{
                                    .shift = shift,
                                    .contents = .{ .map_item = .{ .key = line[0..idx], .val = .empty } },
                                    .raw = line,
                                };

                                if (line[idx + 1] != ' ') return error.BadToken;

                                return .{
                                    .shift = shift,
                                    .contents = .{ .map_item = .{
                                        .key = line[0..idx],
                                        .val = try detectInlineItem(line[idx + 2 ..]),
                                    } },
                                    .raw = line,
                                };
                            }
                        }

                        return .{
                            .shift = shift,
                            .contents = .{ .in_line = .{ .scalar = line } },
                            .raw = line,
                        };
                    },
                }

                // somehow everything else has failed
                return error.Impossible;
            }
            return null;
        }

        fn detectInlineItem(buf: []const u8) Error!InlineItem {
            if (buf.len == 0) return .empty;

            switch (buf[0]) {
                '>', '|' => |char| {
                    if (buf.len > 1 and buf[1] != ' ') return error.BadToken;

                    const slice: []const u8 = switch (buf[buf.len - 1]) {
                        ' ', '\t' => return error.TrailingWhitespace,
                        '|' => buf[@min(2, buf.len) .. buf.len - @intFromBool(buf.len > 1)],
                        else => buf[@min(2, buf.len)..buf.len],
                    };

                    return if (char == '>')
                        .{ .line_string = slice }
                    else
                        .{ .space_string = slice };
                },
                '[' => {
                    if (buf.len < 2 or buf[buf.len - 1] != ']')
                        return error.BadToken;

                    // keep the closing ] for the flow parser
                    return .{ .flow_list = buf[1..] };
                },
                '{' => {
                    if (buf.len < 2 or buf[buf.len - 1] != '}')
                        return error.BadToken;

                    // keep the closing } fpr the flow parser
                    return .{ .flow_map = buf[1..] };
                },
                else => {
                    if (buf[buf.len - 1] == ' ' or buf[buf.len - 1] == '\t')
                        return error.TrailingWhitespace;

                    return .{ .scalar = buf };
                },
            }
        }
    };
}