wip

src/attempt_2.zig

@@ -26,12 +26,22 @@ const DocumentStyle = struct {
 
     heading_char: u8 = '#',
     heading_levels: usize = 6,
-    unordered_chars: []const u8 = "-++*",
+    unordered_chars: []const u8 = "-=+*",
+    check_box_chars: []const u8 = " x/~",
+    horizontal_rule_chars: []const u8 = "-=*.&^",
+    comment_id: []const u8 = "//",
+    escape_char: u8 = '\\',
+    code_block_id: []const u8 = "```",
 
     pub const default = Self{
         .heading_char = '#',
         .heading_levels = 6,
-        .unordered_chars = "-++*",
+        .unordered_chars = "-=+*",
+        .check_box_chars = " x/~",
+        .horizontal_rule_chars = "-=*.&^",
+        .comment_id = "// ",
+        .escape_char = '\\',
+        .code_block_id = "```",
     };
 };
 
@@ -55,7 +65,7 @@ pub const Document = struct {
     const StartLineParserFn = *const fn (self: *Self, alloc: std.mem.Allocator, line: *[]const u8) anyerror!FoundToken;
     const InlineParserFn = *const fn (self: *Self, alloc: std.mem.Allocator, line: *[]const u8) anyerror!FoundToken;
     const EndLineParserFn = *const fn (self: *Self, alloc: std.mem.Allocator, line: *[]const u8) anyerror!FoundToken;
-    const MultiLineParserFn = *const fn (self: *Self, alloc: std.mem.Allocator, line_itter: *std.mem.SplitIterator(u8)) anyerror!void;
+    const MultiLineParserFn = *const fn (self: *Self, alloc: std.mem.Allocator, line_itter: *std.mem.SplitIterator(u8, .sequence)) anyerror!FoundToken;
 
     tokens: std.ArrayList(Token) = .empty,
     tab_width: usize = 4,
@@ -64,27 +74,31 @@ pub const Document = struct {
     document_style: DocumentStyle = .default,
 
     // each function will
-    start_line_parsers: std.ArrayList(StartLineParserFn) = .empty,
-    inline_parsers: std.ArrayList(InlineParserFn) = .empty,
-    end_line_parsers: std.ArrayList(EndLineParserFn) = .empty,
+    start_line_parsers: []const StartLineParserFn = &[_]StartLineParserFn{
+        &headingParser,
+        &tickBoxParser,
+        &listOrderedParser,
+        &listUnorderedParser,
+        &horizontalRuleParser,
+    },
+    inline_parsers: []const InlineParserFn = &[_]InlineParserFn{
+        &baseLineParser,
+    },
 
-    pub fn initDefault(alloc: std.mem.Allocator) !Self {
-        var doc = Self{
+    end_line_parsers: []const EndLineParserFn = &[_]EndLineParserFn{
+        &commentParser,
+    },
+
+    multi_line_parsers: []const MultiLineParserFn = &[_]MultiLineParserFn{
+        &codeBlockParser,
+    },
+
+    pub fn initDefault() Self {
+        return Self{
             .tab_width = 4,
             .newline_char = .newline,
             .document_style = .default,
         };
-
-        try doc.appendStartLineParsers(alloc, &[_]StartLineParserFn{
-            &headingParser,
-            &listOrderedParser,
-        });
-
-        try doc.appendInlineParsers(alloc, &[_]InlineParserFn{
-            &baseLineParser,
-        });
-
-        return doc;
     }
 
     pub fn appendStartLineParsers(self: *Self, alloc: std.mem.Allocator, parser_fns: []const StartLineParserFn) !void {
@@ -99,9 +113,9 @@ pub const Document = struct {
 
     pub fn deinit(self: *Self, alloc: std.mem.Allocator) void {
         self.tokens.deinit(alloc);
-        self.start_line_parsers.deinit(alloc);
-        self.end_line_parsers.deinit(alloc);
-        self.inline_parsers.deinit(alloc);
+        // self.start_line_parsers.deinit(alloc);
+        // self.end_line_parsers.deinit(alloc);
+        // self.inline_parsers.deinit(alloc);
     }
 
     fn consumeChars(line: *[]const u8, amount_of_chars_to_consume: usize) error{too_many_chars_to_consume}!void {
@@ -109,6 +123,11 @@ pub const Document = struct {
         line.* = line.*[amount_of_chars_to_consume..];
     }
 
+    fn consumeCharsFromEnd(line: *[]const u8, amount_of_chars_to_consume: usize) error{too_many_chars_to_consume}!void {
+        if (line.len < amount_of_chars_to_consume) return error.too_many_chars_to_consume;
+        line.* = line.*[0 .. line.len - amount_of_chars_to_consume];
+    }
+
     fn baseLineParser(self: *Self, alloc: std.mem.Allocator, line: *[]const u8) !FoundToken {
         try self.tokens.append(alloc, Token{
             .data = stripWhiteSpace(line.*),
@@ -122,7 +141,7 @@ pub const Document = struct {
     fn inlineParser(self: *Self, alloc: std.mem.Allocator, line: *[]const u8) !void {
         if (line.len == 0) return;
 
-        for (self.inline_parsers.items) |parser| {
+        for (self.inline_parsers) |parser| {
             _ = try parser(self, alloc, line);
         }
     }
@@ -138,11 +157,30 @@ pub const Document = struct {
     pub fn parse(self: *Self, alloc: std.mem.Allocator, document: []const u8) !void {
         var line_iter = std.mem.splitSequence(u8, document, self.newline_char.chars);
 
-        while (line_iter.next()) |line| {
+        line_iterator: while (line_iter.next()) |line| {
             var current_line = line;
+            std.debug.print("acutal current line : {s}\n", .{line});
+            var end_token: ?Token = null;
 
-            for (self.start_line_parsers.items) |parser| {
-                std.debug.print("line : \"{s}\"\n", .{line});
+            for (self.multi_line_parsers) |parser| {
+                // std.debug.print("line : \"{s}\"\n", .{line});
+                const token_found = try parser(self, alloc, &line_iter);
+
+                if (token_found) continue :line_iterator;
+            }
+
+            for (self.end_line_parsers) |parser| {
+                // std.debug.print("line : \"{s}\"\n", .{line});
+                const token_found = try parser(self, alloc, &current_line);
+
+                if (token_found) {
+                    end_token = self.tokens.pop();
+                    break;
+                }
+            }
+
+            for (self.start_line_parsers) |parser| {
+                // std.debug.print("line : \"{s}\"\n", .{line});
                 const token_found = try parser(self, alloc, &current_line);
 
                 if (token_found) break;
@@ -150,6 +188,9 @@ pub const Document = struct {
 
             try self.inlineParser(alloc, &current_line);
 
+            if (end_token) |t| {
+                try self.tokens.append(alloc, t);
+            }
             try self.appendNewLineToken(alloc);
         }
     }
@@ -184,6 +225,7 @@ pub const Document = struct {
         return true;
     }
 
+    // [TODO] make this generic
     pub fn listOrderedParser(self: *Self, alloc: std.mem.Allocator, line: *[]const u8) !FoundToken {
         const clean_line = stripWhiteSpace(line.*);
         const whitespace_chars_removed = countWhiteSpace(line.*, self.tab_width);
@@ -235,31 +277,191 @@ pub const Document = struct {
             });
 
             try consumeChars(line, whitespace_chars_removed + number.len + 2);
+            return true;
         }
 
         return false;
     }
 
     pub fn listUnorderedParser(self: *Self, alloc: std.mem.Allocator, line: *[]const u8) !FoundToken {
-        const clean_line = stripWhiteSpace(line);
-        const whitespace_chars_removed = @intFromPtr(clean_line.ptr) - @intFromPtr(line.ptr);
+        const clean_line = stripWhiteSpace(line.*);
+        const whitespace_chars_removed = countWhiteSpace(line.*, self.tab_width);
         // check for the min length of chars
-        if (clean_line.len < 2) return null;
+        if (clean_line.len < 2) return false;
 
         // check there is a space seperating the char and the text
-        if (clean_line[1] != ' ') return null;
+        if (clean_line[1] != ' ') return false;
 
-        if (clean_line[0] == '-' or clean_line[0] == '=' or clean_line[0] == '+' or clean_line[0] == '*') {
-            return LineParserReturn{
-                // +2 to remove "-. "
-                .pointer_offset = whitespace_chars_removed + 2,
-                .token = .{
-                    .data = clean_line[0..1],
-                    .type = .unordered_list,
-                },
-            };
+        if (std.mem.containsAtLeastScalar(u8, self.document_style.unordered_chars, 1, clean_line[0])) {
+            try self.tokens.append(alloc, Token{
+                .data = clean_line[0..1],
+                .padding = whitespace_chars_removed,
+                .type = .unordered_list,
+            });
+
+            try consumeChars(line, whitespace_chars_removed + 2);
+
+            return true;
         }
 
-        return null;
+        return false;
+    }
+
+    pub fn tickBoxParser(self: *Self, alloc: std.mem.Allocator, line: *[]const u8) !FoundToken {
+        const clean_line = stripWhiteSpace(line.*);
+        const whitespace_chars_removed = countWhiteSpace(line.*, self.tab_width);
+
+        if (!std.mem.startsWith(u8, clean_line, "- [")) {
+            return false;
+        }
+        // not long enough for a tick box
+        if (clean_line.len < 6) return false;
+
+        // - [
+
+        if (std.mem.containsAtLeastScalar(u8, self.document_style.check_box_chars, 1, clean_line[3]) and
+            clean_line[4] == ']' and clean_line[5] == ' ')
+        {
+            try self.tokens.append(alloc, Token{
+                .data = clean_line[0..5],
+                .padding = whitespace_chars_removed,
+                .type = .tickbox,
+            });
+            try consumeChars(line, 5);
+        }
+        return false;
+    }
+
+    fn parseRuleLength(rule_char: u8, line: []const u8) usize {
+        for (line, 0..) |char, i| {
+            if (char == rule_char or char == ' ' or char == '\t')
+                continue
+            else
+                return i;
+        }
+        return line.len;
+    }
+
+    pub fn horizontalRuleParser(self: *Self, alloc: std.mem.Allocator, line: *[]const u8) !FoundToken {
+        const clean_line = stripWhiteSpace(line.*);
+        const whitespace_chars_removed = countWhiteSpace(line.*, self.tab_width);
+
+        for (self.document_style.horizontal_rule_chars) |char| {
+            const rule_length = parseRuleLength(char, clean_line);
+
+            if (rule_length != 0) {
+                try self.tokens.append(alloc, Token{
+                    .data = clean_line[0..rule_length],
+                    .padding = whitespace_chars_removed,
+                    .type = .rule,
+                });
+                try consumeChars(line, rule_length + whitespace_chars_removed);
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    pub fn commentParser(self: *Self, alloc: std.mem.Allocator, line: *[]const u8) !FoundToken {
+
+        // find the index of the comment
+        const index_of_commnet = blk: {
+            if (std.mem.indexOf(u8, line.*, self.document_style.comment_id)) |index| {
+                if (index == 0)
+                    break :blk index;
+
+                if (line.*[index - 1] != self.document_style.escape_char) {
+                    return false;
+                }
+
+                break :blk index;
+            }
+            return false;
+        };
+
+        try self.tokens.append(alloc, Token{
+            .data = line.*[index_of_commnet .. index_of_commnet + self.document_style.comment_id.len],
+            .padding = 0,
+            .type = .comment,
+        });
+
+        try self.tokens.append(alloc, Token{
+            .data = line.*[index_of_commnet + self.document_style.comment_id.len .. line.len],
+            .padding = 0,
+            .type = .raw_text,
+        });
+
+        line.* = line.*[0..index_of_commnet];
+
+        return true;
+    }
+
+    pub fn codeBlockParser(self: *Self, alloc: std.mem.Allocator, line_itter: *std.mem.SplitIterator(u8, .sequence)) !FoundToken {
+        const current_line = line_itter.peek() orelse return false;
+        const clean_line = stripWhiteSpace(current_line);
+        const whitespace_chars_removed = countWhiteSpace(current_line, self.tab_width);
+        std.debug.print("current_line : {s}\n", .{current_line});
+
+        if (!std.mem.startsWith(u8, clean_line, self.document_style.code_block_id)) {
+            // std.debug.print("line didnt start with ```\n", .{});
+            return false;
+        }
+
+        try self.tokens.append(alloc, Token{
+            .data = clean_line[0..self.document_style.code_block_id.len],
+            .padding = whitespace_chars_removed,
+            .type = .code_block,
+        });
+
+        std.debug.print("clean_line[0..self.document_style.code_block_id.len] : {s}\n", .{clean_line[0..self.document_style.code_block_id.len]});
+
+        const lang_name = current_line[self.document_style.code_block_id.len..]; // may have white space
+        std.debug.print("lang_name : {s}\n", .{current_line});
+
+        // std.debug.print("lang_name = {}\n", .{lang_name.len});
+        if (lang_name.len != 0) {
+            try self.tokens.append(alloc, Token{
+                .data = lang_name,
+                .padding = 0,
+                .type = .raw_text,
+            });
+        }
+
+        try self.appendNewLineToken(alloc);
+
+        // std.debug.print("first values\n", .{});
+        // +1 to remove the newline
+        const raw_text_start_index =
+            @intFromPtr(current_line.ptr) - @intFromPtr(line_itter.buffer.ptr) + lang_name.len + self.document_style.code_block_id.len + 1;
+        std.debug.print("{} - {} + {} + {} + 1;", .{ @intFromPtr(current_line.ptr), @intFromPtr(line_itter.buffer.ptr), lang_name.len, self.document_style.code_block_id.len });
+
+        const ending_line = find_end: {
+            while (line_itter.next()) |line| {
+                if (std.mem.startsWith(u8, stripWhiteSpace(line), self.document_style.code_block_id))
+                    break :find_end line;
+            } else {
+                return error.no_closing_tag;
+            }
+        };
+
+        const raw_text_end_index = @intFromPtr(ending_line.ptr) - @intFromPtr(line_itter.buffer.ptr) - 1;
+        std.debug.print("line_itter.buffer[{}..{}] : {s}\n", .{ raw_text_start_index, raw_text_end_index, line_itter.buffer });
+
+        try self.tokens.append(alloc, Token{
+            .data = line_itter.buffer[raw_text_start_index..raw_text_end_index],
+            .padding = 0,
+            .type = .raw_text,
+        });
+
+        try self.tokens.append(alloc, Token{
+            .data = ending_line[0..self.document_style.code_block_id.len],
+            .padding = countWhiteSpace(ending_line, 4),
+            .type = .code_block,
+        });
+
+        try self.appendNewLineToken(alloc);
+
+        return true;
     }
 };

tests/test_runner.zig

@@ -3,6 +3,11 @@ const Io = std.Io;
 
 const builtin = @import("builtin");
 
+pub const colour_set_string_fg_fmt = "\x1B[38;2;{d};{d};{d}m";
+pub const colour_set_string_bg_fmt = "\x1B[48;2;{d};{d};{d}m";
+pub const colour_set_string_fmt = colour_set_string_fg_fmt ++ colour_set_string_bg_fmt;
+pub const colour_end_string_fmt = "\x1B[0m";
+
 pub fn main(init: std.process.Init) !void {
     const io = init.io;
 
@@ -10,12 +15,16 @@ pub fn main(init: std.process.Init) !void {
     var stdout_file_writer: Io.File.Writer = .init(.stdout(), io, &stdout_buffer);
     const stdout = &stdout_file_writer.interface;
 
+    var passed: usize = 0;
     for (builtin.test_functions) |t| {
         t.func() catch |err| {
-            try stdout.print("{s} fail: {}\n", .{ t.name, err });
+            try stdout.print("[\x1B[38;2;255;0;0mfail\x1B[0m] {s} : {}\n", .{ t.name, err });
             continue;
         };
-        try stdout.print("{s} passed\n", .{t.name});
+        passed += 1;
+        try stdout.print("[\x1B[38;2;0;255;0mpassed\x1B[0m] {s} \n", .{t.name});
     }
+
+    try stdout.print("[{}/{}] test passed\n", .{ passed, builtin.test_functions.len });
     try stdout.flush(); // Don't forget to flush!
 }