wip

create_tests.py

@@ -30,7 +30,10 @@ with open("tests/test.json", "r") as txt:
 const Document = @import("attempt_2.zig").Document;
 const std = @import("std");
 const Token = @import("Token.zig");
-const expect = std.testing.expect;    
+const expect = std.testing.expect;
+
+const print_output: bool = true;
+
 """)
     tests : dict = json.loads(txt.read())
     for test_name, test_value in tests["basic_unit_tests"].items():
@@ -41,33 +44,37 @@ const expect = std.testing.expect;
             token_type = list(token.keys())[0]
             
             tokens += f"""
-        Token{{
+//          Token{{
-            .type = .{token_type},
+//              .type = .{token_type},
-            .data = "{token[token_type].encode("unicode_escape").decode("utf-8")}",
+//              .data = "{token[token_type].encode("unicode_escape").decode("utf-8")}",
-        }},"""
+//          }},"""
 
         print(f"""
-test "{test_name}" {{
+// test "{test_name}" {{
-    std.testing.refAllDecls(@This());
+//     std.testing.refAllDecls(@This());
-
+// 
-    const doc = Document.initDefault(std.testing.allocator);
+//     var doc = try Document.initDefault(std.testing.allocator);
-    
+//     
-    const input = "{test_value["input"].encode("unicode_escape").decode("utf-8")}";
+//     const input = "{test_value["input"].encode("unicode_escape").decode("utf-8")}";
-    const parsed = [_]Token{{{tokens}
+//     const parsed = [_]Token{{{tokens}
-    }};
+//     }};
-
+// 
-
+// 
-    try doc.parse(input, std.testing.allocator);
+//     try doc.parse(std.testing.allocator, input);
-
+// 
-    try expect(doc.tokens.items.len >= parsed.len);
+//     if (!print_output) try expect(doc.tokens.items.len >= parsed.len);
-
+// 
-    for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| {{
+//     for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| {{
-        try expect(expected_token.type == actual_token.type);
+//         if (print_output) {{
-        try expect(std.mem.eql(u8, expected_token.data, actual_token.data));
+//             std.debug.print("expected_token : {{f}} | actual_token : {{f}}\\n", .{{ expected_token, actual_token }});
-    }}
+//         }} else {{
-
+//             try expect(expected_token.type == actual_token.type);
-    doc.deinit(std.testing.allocator);
+//             try expect(std.mem.eql(u8, expected_token.data, actual_token.data));
-}}
+//         }}
+//     }}
+// 
+//     doc.deinit(std.testing.allocator);
+// }}
 """)
         # pprint(test_value["parsed"])
 

src/attempt_2.zig

@@ -26,10 +26,12 @@ const DocumentStyle = struct {
 
     heading_char: u8 = '#',
     heading_levels: usize = 6,
+    unordered_chars: []const u8 = "-++*",
 
     pub const default = Self{
         .heading_char = '#',
         .heading_levels = 6,
+        .unordered_chars = "-++*",
     };
 };
 
@@ -75,6 +77,7 @@ pub const Document = struct {
 
         try doc.appendStartLineParsers(alloc, &[_]StartLineParserFn{
             &headingParser,
+            &listOrderedParser,
         });
 
         try doc.appendInlineParsers(alloc, &[_]InlineParserFn{
@@ -101,6 +104,11 @@ pub const Document = struct {
         self.inline_parsers.deinit(alloc);
     }
 
+    fn consumeChars(line: *[]const u8, amount_of_chars_to_consume: usize) error{too_many_chars_to_consume}!void {
+        if (line.len < amount_of_chars_to_consume) return error.too_many_chars_to_consume;
+        line.* = line.*[amount_of_chars_to_consume..];
+    }
+
     fn baseLineParser(self: *Self, alloc: std.mem.Allocator, line: *[]const u8) !FoundToken {
         try self.tokens.append(alloc, Token{
             .data = stripWhiteSpace(line.*),
@@ -112,11 +120,21 @@ pub const Document = struct {
 
     // im going to need to hide a stack in here somewhere
     fn inlineParser(self: *Self, alloc: std.mem.Allocator, line: *[]const u8) !void {
+        if (line.len == 0) return;
+
         for (self.inline_parsers.items) |parser| {
             _ = try parser(self, alloc, line);
         }
     }
 
+    fn appendNewLineToken(self: *Self, alloc: std.mem.Allocator) !void {
+        try self.tokens.append(alloc, Token{
+            .data = "\n",
+            .padding = 0,
+            .type = .newline,
+        });
+    }
+
     pub fn parse(self: *Self, alloc: std.mem.Allocator, document: []const u8) !void {
         var line_iter = std.mem.splitSequence(u8, document, self.newline_char.chars);
 
@@ -124,20 +142,27 @@ pub const Document = struct {
             var current_line = line;
 
             for (self.start_line_parsers.items) |parser| {
+                std.debug.print("line : \"{s}\"\n", .{line});
                 const token_found = try parser(self, alloc, &current_line);
 
                 if (token_found) break;
             }
 
             try self.inlineParser(alloc, &current_line);
+
+            try self.appendNewLineToken(alloc);
         }
     }
 
     pub fn headingParser(self: *Self, alloc: std.mem.Allocator, line: *[]const u8) !FoundToken {
+        // ignore empty lines
+        if (line.len == 0) return false;
+
         const heading_end_index: usize = blk: {
             var i: usize = 0;
             while (i < self.document_style.heading_levels) : (i += 1) {
                 if (line.*[i] == self.document_style.heading_char) continue;
+                break;
             }
             break :blk i;
         };
@@ -154,7 +179,87 @@ pub const Document = struct {
             .type = .heading,
         });
 
-        line.* = line.*[heading_end_index + space_remove ..];
+        try consumeChars(line, heading_end_index + space_remove);
+
         return true;
     }
+
+    pub fn listOrderedParser(self: *Self, alloc: std.mem.Allocator, line: *[]const u8) !FoundToken {
+        const clean_line = stripWhiteSpace(line.*);
+        const whitespace_chars_removed = countWhiteSpace(line.*, self.tab_width);
+        // check for the min length of chars
+        if (clean_line.len < 2) return false;
+
+        // find a point within the line
+        const index_of_dot = blk: {
+            if (std.mem.indexOf(u8, clean_line, ". ")) |i| {
+                break :blk i;
+            }
+            return false;
+        };
+        const number = clean_line[0..index_of_dot];
+        // if it has any spaces its not a order list id
+        if (std.mem.containsAtLeastScalar(u8, number, 1, ' ') or
+            std.mem.containsAtLeastScalar(u8, number, 1, '\t'))
+        {
+            return false;
+        }
+        // now work out what the number is
+        // "I": 1,
+        // "V": 5,
+        // "X": 10,
+        // "L": 50,
+        // "C": 100,
+        // "D": 500,
+        // "M": 1000
+
+        // roman numerials ill deal with these later
+
+        // these can be the same
+        // numbers
+
+        // letters & roman (at least all the i's)
+
+        const is_number = blk: {
+            _ = std.fmt.parseInt(usize, number, 10) catch break :blk false;
+            break :blk true;
+        };
+        const is_letter = std.ascii.isAlphabetic(number[0]);
+
+        // might change this so lists can start from 0 like a real programmer
+        if (is_number or is_letter) {
+            try self.tokens.append(alloc, Token{
+                .data = line.*[0 .. whitespace_chars_removed + number.len + 1],
+                .padding = whitespace_chars_removed,
+                .type = .ordered_list,
+            });
+
+            try consumeChars(line, whitespace_chars_removed + number.len + 2);
+        }
+
+        return false;
+    }
+
+    pub fn listUnorderedParser(self: *Self, alloc: std.mem.Allocator, line: *[]const u8) !FoundToken {
+        const clean_line = stripWhiteSpace(line);
+        const whitespace_chars_removed = @intFromPtr(clean_line.ptr) - @intFromPtr(line.ptr);
+        // check for the min length of chars
+        if (clean_line.len < 2) return null;
+
+        // check there is a space seperating the char and the text
+        if (clean_line[1] != ' ') return null;
+
+        if (clean_line[0] == '-' or clean_line[0] == '=' or clean_line[0] == '+' or clean_line[0] == '*') {
+            return LineParserReturn{
+                // +2 to remove "-. "
+                .pointer_offset = whitespace_chars_removed + 2,
+                .token = .{
+                    .data = clean_line[0..1],
+                    .type = .unordered_list,
+                },
+            };
+        }
+
+        return null;
+    }
 };