wip

build.zig

@@ -6,7 +6,7 @@ pub fn build(b: *std.Build) void {
 
     const mod = b.addModule("pandas_markdown", .{
         .root_source_file = b.path("src/root.zig"),
-        .test_runner = .{ .path = b.path("tests/test_runner.zig"), .mode = .simple },
+        // .test_runner = .{ .path = b.path("tests/test_runner.zig"), .mode = .simple },
         .target = target,
     });
 
@@ -14,7 +14,7 @@ pub fn build(b: *std.Build) void {
         .name = "pandas_markdown",
         .root_module = b.createModule(.{
             .root_source_file = b.path("src/main.zig"),
-            .test_runner = .{ .path = b.path("tests/test_runner.zig"), .mode = .simple },
+            // .test_runner = .{ .path = b.path("tests/test_runner.zig"), .mode = .simple },
             .target = target,
             .optimize = optimize,
             .imports = &.{

create_tests.py

@@ -26,6 +26,12 @@ import json
 from pprint import pprint
 
 with open("tests/test.json", "r") as txt:
+    print(f"""
+const Document = @import("attempt_2.zig").Document;
+const std = @import("std");
+const Token = @import("Token.zig");
+const expect = std.testing.expect;    
+""")
     tests : dict = json.loads(txt.read())
     for test_name, test_value in tests["basic_unit_tests"].items():
 
@@ -42,19 +48,25 @@ with open("tests/test.json", "r") as txt:
 
         print(f"""
 test "{test_name}" {{
+    std.testing.refAllDecls(@This());
+
+    const doc = Document.initDefault(std.testing.allocator);
+    
     const input = "{test_value["input"].encode("unicode_escape").decode("utf-8")}";
     const parsed = [_]Token{{{tokens}
     }};
 
-    var parsing = try tokenize(input, std.testing.allocator);
-    try expect(parsing.items.len >= parsed.len);
 
-    for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| {{
+    try doc.parse(input, std.testing.allocator);
+
+    try expect(doc.tokens.items.len >= parsed.len);
+
+    for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| {{
         try expect(expected_token.type == actual_token.type);
         try expect(std.mem.eql(u8, expected_token.data, actual_token.data));
     }}
 
-    parsing.deinit(std.testing.allocator);
+    doc.deinit(std.testing.allocator);
 }}
 """)
         # pprint(test_value["parsed"])

src/Token.zig

@@ -0,0 +1,67 @@
+const std = @import("std");
+
+pub const TokenType = enum {
+    // macros
+    inbulit,
+    macro,
+    macro_arg,
+    arg_sep,
+    replace,
+    unknown, // this is the defualt token type and should never appear
+
+    // utils
+    bracket_close,
+    bracket_open,
+    raw_text,
+    newline,
+
+    comment,
+
+    // in text
+    heading,
+    text,
+    inline_style,
+    unordered_list,
+    ordered_list,
+    embed,
+    link,
+
+    table_row_sep,
+    table_sep,
+    quote,
+
+    code_block,
+
+    tickbox,
+    rule,
+
+    footnote,
+    footnote_def,
+
+    sup,
+    sub,
+};
+
+data: []const u8 = "",
+type: TokenType = .unknown,
+padding: usize = 0,
+
+const Self = @This();
+
+// this is to allow the ast generation to make sure that things are
+// within the right subtrees
+pub fn format(
+    self: Self,
+    writer: *std.Io.Writer,
+) !void {
+    switch (self.type) {
+        .newline => try writer.print(
+            ".{{\n  .data = \"\\n\"\n  .type = {}\n}}",
+            .{self.type},
+        ),
+        else => try writer.print(
+            ".{{\n  .data = \"{s}\"\n  .type = {}\n}}",
+            .{ self.data, self.type },
+        ),
+    }
+}

src/attempt_2.zig

@@ -0,0 +1,160 @@
+const Token = @import("Token.zig");
+const std = @import("std");
+
+pub fn stripWhiteSpace(line_with_white_space: []const u8) []const u8 {
+    return std.mem.trimStart(u8, line_with_white_space, " \t");
+}
+
+pub fn countWhiteSpace(line_with_white_space: []const u8, tab_width: usize) usize {
+    var sum: usize = 0;
+    for (line_with_white_space) |char| {
+        if (char == ' ') {
+            sum += 1;
+            continue;
+        }
+        if (char == '\t') {
+            sum += tab_width;
+            continue;
+        }
+        break;
+    }
+    return sum;
+}
+
+const DocumentStyle = struct {
+    const Self = @This();
+
+    heading_char: u8 = '#',
+    heading_levels: usize = 6,
+
+    pub const default = Self{
+        .heading_char = '#',
+        .heading_levels = 6,
+    };
+};
+
+pub const Document = struct {
+    const Self = @This();
+
+    const Newline = struct {
+        chars: []const u8,
+
+        pub const newline = Newline{
+            .chars = "\n",
+        };
+
+        pub const newline_carriage_return = Newline{
+            .chars = "\n\r",
+        };
+    };
+
+    const FoundToken = bool;
+
+    const StartLineParserFn = *const fn (self: *Self, alloc: std.mem.Allocator, line: *[]const u8) anyerror!FoundToken;
+    const InlineParserFn = *const fn (self: *Self, alloc: std.mem.Allocator, line: *[]const u8) anyerror!FoundToken;
+    const EndLineParserFn = *const fn (self: *Self, alloc: std.mem.Allocator, line: *[]const u8) anyerror!FoundToken;
+    const MultiLineParserFn = *const fn (self: *Self, alloc: std.mem.Allocator, line_itter: *std.mem.SplitIterator(u8)) anyerror!void;
+
+    tokens: std.ArrayList(Token) = .empty,
+    tab_width: usize = 4,
+    newline_char: Newline = .newline,
+
+    document_style: DocumentStyle = .default,
+
+    // each function will
+    start_line_parsers: std.ArrayList(StartLineParserFn) = .empty,
+    inline_parsers: std.ArrayList(InlineParserFn) = .empty,
+    end_line_parsers: std.ArrayList(EndLineParserFn) = .empty,
+
+    pub fn initDefault(alloc: std.mem.Allocator) !Self {
+        var doc = Self{
+            .tab_width = 4,
+            .newline_char = .newline,
+            .document_style = .default,
+        };
+
+        try doc.appendStartLineParsers(alloc, &[_]StartLineParserFn{
+            &headingParser,
+        });
+
+        try doc.appendInlineParsers(alloc, &[_]InlineParserFn{
+            &baseLineParser,
+        });
+
+        return doc;
+    }
+
+    pub fn appendStartLineParsers(self: *Self, alloc: std.mem.Allocator, parser_fns: []const StartLineParserFn) !void {
+        try self.start_line_parsers.appendSlice(alloc, parser_fns);
+    }
+    pub fn appendInlineParsers(self: *Self, alloc: std.mem.Allocator, parser_fns: []const InlineParserFn) !void {
+        try self.inline_parsers.appendSlice(alloc, parser_fns);
+    }
+    pub fn appendEndLineParsers(self: *Self, alloc: std.mem.Allocator, parser_fns: []const EndLineParserFn) !void {
+        try self.end_line_parsers.appendSlice(alloc, parser_fns);
+    }
+
+    pub fn deinit(self: *Self, alloc: std.mem.Allocator) void {
+        self.tokens.deinit(alloc);
+        self.start_line_parsers.deinit(alloc);
+        self.end_line_parsers.deinit(alloc);
+        self.inline_parsers.deinit(alloc);
+    }
+
+    fn baseLineParser(self: *Self, alloc: std.mem.Allocator, line: *[]const u8) !FoundToken {
+        try self.tokens.append(alloc, Token{
+            .data = stripWhiteSpace(line.*),
+            .padding = countWhiteSpace(line.*, self.tab_width),
+            .type = .text,
+        });
+        return true;
+    }
+
+    // im going to need to hide a stack in here somewhere
+    fn inlineParser(self: *Self, alloc: std.mem.Allocator, line: *[]const u8) !void {
+        for (self.inline_parsers.items) |parser| {
+            _ = try parser(self, alloc, line);
+        }
+    }
+
+    pub fn parse(self: *Self, alloc: std.mem.Allocator, document: []const u8) !void {
+        var line_iter = std.mem.splitSequence(u8, document, self.newline_char.chars);
+
+        while (line_iter.next()) |line| {
+            var current_line = line;
+
+            for (self.start_line_parsers.items) |parser| {
+                const token_found = try parser(self, alloc, &current_line);
+
+                if (token_found) break;
+            }
+
+            try self.inlineParser(alloc, &current_line);
+        }
+    }
+
+    pub fn headingParser(self: *Self, alloc: std.mem.Allocator, line: *[]const u8) !FoundToken {
+        const heading_end_index: usize = blk: {
+            var i: usize = 0;
+            while (i < self.document_style.heading_levels) : (i += 1) {
+                if (line.*[i] == self.document_style.heading_char) continue;
+            }
+            break :blk i;
+        };
+
+        // if no heading was detected
+        if (heading_end_index == 0) return false;
+
+        // allows for blank headings
+        const space_remove: usize = if (line.len == heading_end_index) 0 else 1;
+
+        try self.tokens.append(alloc, Token{
+            .data = line.*[0..heading_end_index],
+            .padding = 0,
+            .type = .heading,
+        });
+
+        line.* = line.*[heading_end_index + space_remove ..];
+        return true;
+    }
+};

src/document.zig

@@ -1,75 +1,11 @@
 //! By convention, root.zig is the root source file when making a library.
 const std = @import("std");
+const Token = @import("Token.zig");
 
 pub fn stripWhiteSpace(white_space: []const u8) []const u8 {
     return std.mem.trimStart(u8, white_space, " \t");
 }
 
-pub const TokenType = enum {
-    // macros
-    inbulit,
-    macro,
-    macro_arg,
-    arg_sep,
-    replace,
-    unknown, // this is the defualt token type and should never appear
-
-    // utils
-    bracket_close,
-    bracket_open,
-    raw_text,
-    newline,
-
-    comment,
-
-    // in text
-    heading,
-    text,
-    inline_style,
-    unordered_list,
-    ordered_list,
-    embed,
-    link,
-
-    table_row_sep,
-    table_sep,
-    quote,
-
-    code_block,
-
-    tickbox,
-    rule,
-
-    footnote,
-    footnote_def,
-
-    sup,
-    sub,
-};
-
-pub const Token = struct {
-    data: []const u8 = "",
-    type: TokenType = .unknown,
-    padding: usize = 0,
-    // this is to allow the ast generation to make sure that things are
-    // within the right subtrees
-    pub fn format(
-        self: Token,
-        writer: *std.io.Writer,
-    ) !void {
-        switch (self.type) {
-            .newline => try writer.print(
-                ".{{\n  .data = \"\\n\"\n  .type = {}\n}}",
-                .{self.type},
-            ),
-            else => try writer.print(
-                ".{{\n  .data = \"{s}\"\n  .type = {}\n}}",
-                .{ self.data, self.type },
-            ),
-        }
-    }
-};
-
 pub const LineParserReturn = struct {
     // the new updated pos of the token after its been parsed
     pointer_offset: usize = 0,
@@ -87,6 +23,19 @@ pub const MultiLineParser = *const fn (
     list: *std.ArrayList(Token),
 ) anyerror!void;
 
+///
+/// @line
+///
+///
+///
+///
+pub const InLineParser = *const fn (
+    line: []const u8,
+    parsers: []InLineParser,
+    token_list: *std.ArrayList(Token),
+    alloc: *std.mem.Allocator,
+) anyerror!void;
+
 const heading_id = "#";
 const heading_depth: comptime_int = 6;
 pub fn headingParser(line: []const u8) ?LineParserReturn {

tests/test_runner.zig

@@ -1,10 +1,15 @@
 const std = @import("std");
+const Io = std.Io;
+
 const builtin = @import("builtin");
 
-pub fn main() !void {
+pub fn main(init: std.process.Init) !void {
+    const io = init.io;
+
     var stdout_buffer: [1024]u8 = undefined;
-    var stdout_writer = std.fs.File.stdout().writer(&stdout_buffer);
+    var stdout_file_writer: Io.File.Writer = .init(.stdout(), io, &stdout_buffer);
-    const stdout = &stdout_writer.interface;
+    const stdout = &stdout_file_writer.interface;
+
     for (builtin.test_functions) |t| {
         t.func() catch |err| {
             try stdout.print("{s} fail: {}\n", .{ t.name, err });