From 91beeffd6d4436a0bec513b5f17631a1208f09b8 Mon Sep 17 00:00:00 2001 From: sirlilpanda Date: Mon, 31 Aug 2026 08:49:23 +1200 Subject: [PATCH] wip --- build.zig | 4 +- create_tests.py | 20 +- src/Token.zig | 67 + src/attempt_2.zig | 160 +++ src/document.zig | 79 +- src/tests.zig | 2898 +++++++++++++++++++++++++++++++++++++++ test.zig | 3029 +++++++++++++++++++++++++++++++++++++++++ tests/test_runner.zig | 11 +- 8 files changed, 6194 insertions(+), 74 deletions(-) create mode 100644 src/Token.zig create mode 100644 src/attempt_2.zig create mode 100644 test.zig diff --git a/build.zig b/build.zig index 2f67346..539ed47 100644 --- a/build.zig +++ b/build.zig @@ -6,7 +6,7 @@ pub fn build(b: *std.Build) void { const mod = b.addModule("pandas_markdown", .{ .root_source_file = b.path("src/root.zig"), - .test_runner = .{ .path = b.path("tests/test_runner.zig"), .mode = .simple }, + // .test_runner = .{ .path = b.path("tests/test_runner.zig"), .mode = .simple }, .target = target, }); @@ -14,7 +14,7 @@ pub fn build(b: *std.Build) void { .name = "pandas_markdown", .root_module = b.createModule(.{ .root_source_file = b.path("src/main.zig"), - .test_runner = .{ .path = b.path("tests/test_runner.zig"), .mode = .simple }, + // .test_runner = .{ .path = b.path("tests/test_runner.zig"), .mode = .simple }, .target = target, .optimize = optimize, .imports = &.{ diff --git a/create_tests.py b/create_tests.py index e84edaa..77d92bd 100644 --- a/create_tests.py +++ b/create_tests.py @@ -26,6 +26,12 @@ import json from pprint import pprint with open("tests/test.json", "r") as txt: + print(f""" +const Document = @import("attempt_2.zig").Document; +const std = @import("std"); +const Token = @import("Token.zig"); +const expect = std.testing.expect; +""") tests : dict = json.loads(txt.read()) for test_name, test_value in tests["basic_unit_tests"].items(): @@ -42,19 +48,25 @@ with open("tests/test.json", "r") as txt: print(f""" test "{test_name}" {{ + std.testing.refAllDecls(@This()); + + const doc = Document.initDefault(std.testing.allocator); + const input = "{test_value["input"].encode("unicode_escape").decode("utf-8")}"; const parsed = [_]Token{{{tokens} }}; - var parsing = try tokenize(input, std.testing.allocator); - try expect(parsing.items.len >= parsed.len); - for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| {{ + try doc.parse(input, std.testing.allocator); + + try expect(doc.tokens.items.len >= parsed.len); + + for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| {{ try expect(expected_token.type == actual_token.type); try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); }} - parsing.deinit(std.testing.allocator); + doc.deinit(std.testing.allocator); }} """) # pprint(test_value["parsed"]) diff --git a/src/Token.zig b/src/Token.zig new file mode 100644 index 0000000..ab72d0b --- /dev/null +++ b/src/Token.zig @@ -0,0 +1,67 @@ +const std = @import("std"); + +pub const TokenType = enum { + // macros + inbulit, + macro, + macro_arg, + arg_sep, + replace, + unknown, // this is the defualt token type and should never appear + + // utils + bracket_close, + bracket_open, + raw_text, + newline, + + comment, + + // in text + heading, + text, + inline_style, + unordered_list, + ordered_list, + embed, + link, + + table_row_sep, + table_sep, + quote, + + code_block, + + tickbox, + rule, + + footnote, + footnote_def, + + sup, + sub, +}; + +data: []const u8 = "", +type: TokenType = .unknown, +padding: usize = 0, + +const Self = @This(); + +// this is to allow the ast generation to make sure that things are +// within the right subtrees +pub fn format( + self: Self, + writer: *std.Io.Writer, +) !void { + switch (self.type) { + .newline => try writer.print( + ".{{\n .data = \"\\n\"\n .type = {}\n}}", + .{self.type}, + ), + else => try writer.print( + ".{{\n .data = \"{s}\"\n .type = {}\n}}", + .{ self.data, self.type }, + ), + } +} diff --git a/src/attempt_2.zig b/src/attempt_2.zig new file mode 100644 index 0000000..e40c0a7 --- /dev/null +++ b/src/attempt_2.zig @@ -0,0 +1,160 @@ +const Token = @import("Token.zig"); +const std = @import("std"); + +pub fn stripWhiteSpace(line_with_white_space: []const u8) []const u8 { + return std.mem.trimStart(u8, line_with_white_space, " \t"); +} + +pub fn countWhiteSpace(line_with_white_space: []const u8, tab_width: usize) usize { + var sum: usize = 0; + for (line_with_white_space) |char| { + if (char == ' ') { + sum += 1; + continue; + } + if (char == '\t') { + sum += tab_width; + continue; + } + break; + } + return sum; +} + +const DocumentStyle = struct { + const Self = @This(); + + heading_char: u8 = '#', + heading_levels: usize = 6, + + pub const default = Self{ + .heading_char = '#', + .heading_levels = 6, + }; +}; + +pub const Document = struct { + const Self = @This(); + + const Newline = struct { + chars: []const u8, + + pub const newline = Newline{ + .chars = "\n", + }; + + pub const newline_carriage_return = Newline{ + .chars = "\n\r", + }; + }; + + const FoundToken = bool; + + const StartLineParserFn = *const fn (self: *Self, alloc: std.mem.Allocator, line: *[]const u8) anyerror!FoundToken; + const InlineParserFn = *const fn (self: *Self, alloc: std.mem.Allocator, line: *[]const u8) anyerror!FoundToken; + const EndLineParserFn = *const fn (self: *Self, alloc: std.mem.Allocator, line: *[]const u8) anyerror!FoundToken; + const MultiLineParserFn = *const fn (self: *Self, alloc: std.mem.Allocator, line_itter: *std.mem.SplitIterator(u8)) anyerror!void; + + tokens: std.ArrayList(Token) = .empty, + tab_width: usize = 4, + newline_char: Newline = .newline, + + document_style: DocumentStyle = .default, + + // each function will + start_line_parsers: std.ArrayList(StartLineParserFn) = .empty, + inline_parsers: std.ArrayList(InlineParserFn) = .empty, + end_line_parsers: std.ArrayList(EndLineParserFn) = .empty, + + pub fn initDefault(alloc: std.mem.Allocator) !Self { + var doc = Self{ + .tab_width = 4, + .newline_char = .newline, + .document_style = .default, + }; + + try doc.appendStartLineParsers(alloc, &[_]StartLineParserFn{ + &headingParser, + }); + + try doc.appendInlineParsers(alloc, &[_]InlineParserFn{ + &baseLineParser, + }); + + return doc; + } + + pub fn appendStartLineParsers(self: *Self, alloc: std.mem.Allocator, parser_fns: []const StartLineParserFn) !void { + try self.start_line_parsers.appendSlice(alloc, parser_fns); + } + pub fn appendInlineParsers(self: *Self, alloc: std.mem.Allocator, parser_fns: []const InlineParserFn) !void { + try self.inline_parsers.appendSlice(alloc, parser_fns); + } + pub fn appendEndLineParsers(self: *Self, alloc: std.mem.Allocator, parser_fns: []const EndLineParserFn) !void { + try self.end_line_parsers.appendSlice(alloc, parser_fns); + } + + pub fn deinit(self: *Self, alloc: std.mem.Allocator) void { + self.tokens.deinit(alloc); + self.start_line_parsers.deinit(alloc); + self.end_line_parsers.deinit(alloc); + self.inline_parsers.deinit(alloc); + } + + fn baseLineParser(self: *Self, alloc: std.mem.Allocator, line: *[]const u8) !FoundToken { + try self.tokens.append(alloc, Token{ + .data = stripWhiteSpace(line.*), + .padding = countWhiteSpace(line.*, self.tab_width), + .type = .text, + }); + return true; + } + + // im going to need to hide a stack in here somewhere + fn inlineParser(self: *Self, alloc: std.mem.Allocator, line: *[]const u8) !void { + for (self.inline_parsers.items) |parser| { + _ = try parser(self, alloc, line); + } + } + + pub fn parse(self: *Self, alloc: std.mem.Allocator, document: []const u8) !void { + var line_iter = std.mem.splitSequence(u8, document, self.newline_char.chars); + + while (line_iter.next()) |line| { + var current_line = line; + + for (self.start_line_parsers.items) |parser| { + const token_found = try parser(self, alloc, ¤t_line); + + if (token_found) break; + } + + try self.inlineParser(alloc, ¤t_line); + } + } + + pub fn headingParser(self: *Self, alloc: std.mem.Allocator, line: *[]const u8) !FoundToken { + const heading_end_index: usize = blk: { + var i: usize = 0; + while (i < self.document_style.heading_levels) : (i += 1) { + if (line.*[i] == self.document_style.heading_char) continue; + } + break :blk i; + }; + + // if no heading was detected + if (heading_end_index == 0) return false; + + // allows for blank headings + const space_remove: usize = if (line.len == heading_end_index) 0 else 1; + + try self.tokens.append(alloc, Token{ + .data = line.*[0..heading_end_index], + .padding = 0, + .type = .heading, + }); + + line.* = line.*[heading_end_index + space_remove ..]; + return true; + } +}; diff --git a/src/document.zig b/src/document.zig index 3b54f33..f59c93e 100644 --- a/src/document.zig +++ b/src/document.zig @@ -1,75 +1,11 @@ //! By convention, root.zig is the root source file when making a library. const std = @import("std"); +const Token = @import("Token.zig"); pub fn stripWhiteSpace(white_space: []const u8) []const u8 { return std.mem.trimStart(u8, white_space, " \t"); } -pub const TokenType = enum { - // macros - inbulit, - macro, - macro_arg, - arg_sep, - replace, - unknown, // this is the defualt token type and should never appear - - // utils - bracket_close, - bracket_open, - raw_text, - newline, - - comment, - - // in text - heading, - text, - inline_style, - unordered_list, - ordered_list, - embed, - link, - - table_row_sep, - table_sep, - quote, - - code_block, - - tickbox, - rule, - - footnote, - footnote_def, - - sup, - sub, -}; - -pub const Token = struct { - data: []const u8 = "", - type: TokenType = .unknown, - padding: usize = 0, - // this is to allow the ast generation to make sure that things are - // within the right subtrees - pub fn format( - self: Token, - writer: *std.io.Writer, - ) !void { - switch (self.type) { - .newline => try writer.print( - ".{{\n .data = \"\\n\"\n .type = {}\n}}", - .{self.type}, - ), - else => try writer.print( - ".{{\n .data = \"{s}\"\n .type = {}\n}}", - .{ self.data, self.type }, - ), - } - } -}; - pub const LineParserReturn = struct { // the new updated pos of the token after its been parsed pointer_offset: usize = 0, @@ -87,6 +23,19 @@ pub const MultiLineParser = *const fn ( list: *std.ArrayList(Token), ) anyerror!void; +/// +/// @line +/// +/// +/// +/// +pub const InLineParser = *const fn ( + line: []const u8, + parsers: []InLineParser, + token_list: *std.ArrayList(Token), + alloc: *std.mem.Allocator, +) anyerror!void; + const heading_id = "#"; const heading_depth: comptime_int = 6; pub fn headingParser(line: []const u8) ?LineParserReturn { diff --git a/src/tests.zig b/src/tests.zig index e69de29..9a7d06b 100644 --- a/src/tests.zig +++ b/src/tests.zig @@ -0,0 +1,2898 @@ +const Document = @import("attempt_2.zig").Document; +const std = @import("std"); +const Token = @import("Token.zig"); +const expect = std.testing.expect; + +test "heading_1" { + std.testing.refAllDecls(@This()); + + var doc = try Document.initDefault(std.testing.allocator); + + const input = "# hello this is a heading\n"; + const parsed = [_]Token{ + Token{ + .type = .heading, + .data = "#", + }, + Token{ + .type = .text, + .data = "hello this is a heading", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + try doc.parse(std.testing.allocator, input); + + // try expect(doc.tokens.items.len >= parsed.len); + + for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { + std.debug.print("expected_token : {f} | actual_token : {f}\n", .{ expected_token, actual_token }); + // try expect(expected_token.type == actual_token.type); + // try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + doc.deinit(std.testing.allocator); +} + +// test "heading_2" { +// std.testing.refAllDecls(@This()); + +// const doc = Document.initDefault(std.testing.allocator); + +// const input = "## hello this is a heading\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .heading, +// .data = "##", +// }, +// Token{ +// .type = .text, +// .data = "hello this is a heading", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// }; + +// try doc.parse(input, std.testing.allocator); + +// try expect(doc.tokens.items.len >= parsed.len); + +// for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// doc.deinit(std.testing.allocator); +// } + +// test "heading_3" { +// std.testing.refAllDecls(@This()); + +// const doc = Document.initDefault(std.testing.allocator); + +// const input = "### hello this is a heading\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .heading, +// .data = "###", +// }, +// Token{ +// .type = .text, +// .data = "hello this is a heading", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// }; + +// try doc.parse(input, std.testing.allocator); + +// try expect(doc.tokens.items.len >= parsed.len); + +// for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// doc.deinit(std.testing.allocator); +// } + +// test "heading_4" { +// std.testing.refAllDecls(@This()); + +// const doc = Document.initDefault(std.testing.allocator); + +// const input = "#### hello this is a heading\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .heading, +// .data = "####", +// }, +// Token{ +// .type = .text, +// .data = "hello this is a heading", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// }; + +// try doc.parse(input, std.testing.allocator); + +// try expect(doc.tokens.items.len >= parsed.len); + +// for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// doc.deinit(std.testing.allocator); +// } + +// test "heading_5" { +// std.testing.refAllDecls(@This()); + +// const doc = Document.initDefault(std.testing.allocator); + +// const input = "##### hello this is a heading\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .heading, +// .data = "#####", +// }, +// Token{ +// .type = .text, +// .data = "hello this is a heading", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// }; + +// try doc.parse(input, std.testing.allocator); + +// try expect(doc.tokens.items.len >= parsed.len); + +// for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// doc.deinit(std.testing.allocator); +// } + +// test "heading_6" { +// std.testing.refAllDecls(@This()); + +// const doc = Document.initDefault(std.testing.allocator); + +// const input = "###### hello this is a heading\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .heading, +// .data = "######", +// }, +// Token{ +// .type = .text, +// .data = "hello this is a heading", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// }; + +// try doc.parse(input, std.testing.allocator); + +// try expect(doc.tokens.items.len >= parsed.len); + +// for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// doc.deinit(std.testing.allocator); +// } + +// test "header_no_text" { +// std.testing.refAllDecls(@This()); + +// const doc = Document.initDefault(std.testing.allocator); + +// const input = "######\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .heading, +// .data = "######", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// }; + +// try doc.parse(input, std.testing.allocator); + +// try expect(doc.tokens.items.len >= parsed.len); + +// for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// doc.deinit(std.testing.allocator); +// } + +// test "text" { +// std.testing.refAllDecls(@This()); + +// const doc = Document.initDefault(std.testing.allocator); + +// const input = "plain test\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .text, +// .data = "plain test", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// }; + +// try doc.parse(input, std.testing.allocator); + +// try expect(doc.tokens.items.len >= parsed.len); + +// for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// doc.deinit(std.testing.allocator); +// } + +// test "text_style_code" { +// std.testing.refAllDecls(@This()); + +// const doc = Document.initDefault(std.testing.allocator); + +// const input = "`plain test`\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .inline_style, +// .data = "`", +// }, +// Token{ +// .type = .text, +// .data = "plain test", +// }, +// Token{ +// .type = .inline_style, +// .data = "`", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// }; + +// try doc.parse(input, std.testing.allocator); + +// try expect(doc.tokens.items.len >= parsed.len); + +// for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// doc.deinit(std.testing.allocator); +// } + +// test "text_style_bold" { +// std.testing.refAllDecls(@This()); + +// const doc = Document.initDefault(std.testing.allocator); + +// const input = "!plain test!\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .inline_style, +// .data = "!", +// }, +// Token{ +// .type = .text, +// .data = "plain test", +// }, +// Token{ +// .type = .inline_style, +// .data = "!", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// }; + +// try doc.parse(input, std.testing.allocator); + +// try expect(doc.tokens.items.len >= parsed.len); + +// for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// doc.deinit(std.testing.allocator); +// } + +// test "text_style_italic" { +// std.testing.refAllDecls(@This()); + +// const doc = Document.initDefault(std.testing.allocator); + +// const input = "*plain test*\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .inline_style, +// .data = "*", +// }, +// Token{ +// .type = .text, +// .data = "plain test", +// }, +// Token{ +// .type = .inline_style, +// .data = "*", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// }; + +// try doc.parse(input, std.testing.allocator); + +// try expect(doc.tokens.items.len >= parsed.len); + +// for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// doc.deinit(std.testing.allocator); +// } + +// test "text_style_underline" { +// std.testing.refAllDecls(@This()); + +// const doc = Document.initDefault(std.testing.allocator); + +// const input = "_plain test_\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .inline_style, +// .data = "_", +// }, +// Token{ +// .type = .text, +// .data = "plain test", +// }, +// Token{ +// .type = .inline_style, +// .data = "_", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// }; + +// try doc.parse(input, std.testing.allocator); + +// try expect(doc.tokens.items.len >= parsed.len); + +// for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// doc.deinit(std.testing.allocator); +// } + +// test "text_style_strikethrough" { +// std.testing.refAllDecls(@This()); + +// const doc = Document.initDefault(std.testing.allocator); + +// const input = "~plain test~\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .inline_style, +// .data = "~", +// }, +// Token{ +// .type = .text, +// .data = "plain test", +// }, +// Token{ +// .type = .inline_style, +// .data = "~", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// }; + +// try doc.parse(input, std.testing.allocator); + +// try expect(doc.tokens.items.len >= parsed.len); + +// for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// doc.deinit(std.testing.allocator); +// } + +// test "text_style_highlighted" { +// std.testing.refAllDecls(@This()); + +// const doc = Document.initDefault(std.testing.allocator); + +// const input = "|plain test|\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .inline_style, +// .data = "|", +// }, +// Token{ +// .type = .text, +// .data = "plain test", +// }, +// Token{ +// .type = .inline_style, +// .data = "|", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// }; + +// try doc.parse(input, std.testing.allocator); + +// try expect(doc.tokens.items.len >= parsed.len); + +// for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// doc.deinit(std.testing.allocator); +// } + +// test "text_style_&" { +// std.testing.refAllDecls(@This()); + +// const doc = Document.initDefault(std.testing.allocator); + +// const input = "&plain test&\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .inline_style, +// .data = "&", +// }, +// Token{ +// .type = .text, +// .data = "plain test", +// }, +// Token{ +// .type = .inline_style, +// .data = "&", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// }; + +// try doc.parse(input, std.testing.allocator); + +// try expect(doc.tokens.items.len >= parsed.len); + +// for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// doc.deinit(std.testing.allocator); +// } + +// test "text_style_%" { +// std.testing.refAllDecls(@This()); + +// const doc = Document.initDefault(std.testing.allocator); + +// const input = "%plain test%\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .inline_style, +// .data = "%", +// }, +// Token{ +// .type = .text, +// .data = "plain test", +// }, +// Token{ +// .type = .inline_style, +// .data = "%", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// }; + +// try doc.parse(input, std.testing.allocator); + +// try expect(doc.tokens.items.len >= parsed.len); + +// for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// doc.deinit(std.testing.allocator); +// } + +// test "text_style_$" { +// std.testing.refAllDecls(@This()); + +// const doc = Document.initDefault(std.testing.allocator); + +// const input = "$plain test$\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .inline_style, +// .data = "$", +// }, +// Token{ +// .type = .text, +// .data = "plain test", +// }, +// Token{ +// .type = .inline_style, +// .data = "$", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// }; + +// try doc.parse(input, std.testing.allocator); + +// try expect(doc.tokens.items.len >= parsed.len); + +// for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// doc.deinit(std.testing.allocator); +// } + +// test "text_style_#" { +// std.testing.refAllDecls(@This()); + +// const doc = Document.initDefault(std.testing.allocator); + +// const input = "#plain test#\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .inline_style, +// .data = "#", +// }, +// Token{ +// .type = .text, +// .data = "plain test", +// }, +// Token{ +// .type = .inline_style, +// .data = "#", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// }; + +// try doc.parse(input, std.testing.allocator); + +// try expect(doc.tokens.items.len >= parsed.len); + +// for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// doc.deinit(std.testing.allocator); +// } + +// test "text_style_-" { +// std.testing.refAllDecls(@This()); + +// const doc = Document.initDefault(std.testing.allocator); + +// const input = "-plain test-\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .inline_style, +// .data = "-", +// }, +// Token{ +// .type = .text, +// .data = "plain test", +// }, +// Token{ +// .type = .inline_style, +// .data = "-", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// }; + +// try doc.parse(input, std.testing.allocator); + +// try expect(doc.tokens.items.len >= parsed.len); + +// for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// doc.deinit(std.testing.allocator); +// } + +// test "text_style_+" { +// std.testing.refAllDecls(@This()); + +// const doc = Document.initDefault(std.testing.allocator); + +// const input = "+plain test+\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .inline_style, +// .data = "+", +// }, +// Token{ +// .type = .text, +// .data = "plain test", +// }, +// Token{ +// .type = .inline_style, +// .data = "+", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// }; + +// try doc.parse(input, std.testing.allocator); + +// try expect(doc.tokens.items.len >= parsed.len); + +// for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// doc.deinit(std.testing.allocator); +// } + +// test "text_style_=" { +// std.testing.refAllDecls(@This()); + +// const doc = Document.initDefault(std.testing.allocator); + +// const input = "=plain test=\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .inline_style, +// .data = "=", +// }, +// Token{ +// .type = .text, +// .data = "plain test", +// }, +// Token{ +// .type = .inline_style, +// .data = "=", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// }; + +// try doc.parse(input, std.testing.allocator); + +// try expect(doc.tokens.items.len >= parsed.len); + +// for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// doc.deinit(std.testing.allocator); +// } + +// test "text_style_espaced" { +// std.testing.refAllDecls(@This()); + +// const doc = Document.initDefault(std.testing.allocator); + +// const input = "\\`plain test`\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .text, +// .data = "`plain test`", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// }; + +// try doc.parse(input, std.testing.allocator); + +// try expect(doc.tokens.items.len >= parsed.len); + +// for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// doc.deinit(std.testing.allocator); +// } + +// test "text_style_with_style_char_in_string" { +// std.testing.refAllDecls(@This()); + +// const doc = Document.initDefault(std.testing.allocator); + +// const input = "+plain something + something else test+\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .inline_style, +// .data = "+", +// }, +// Token{ +// .type = .text, +// .data = "plain something + something else test", +// }, +// Token{ +// .type = .inline_style, +// .data = "+", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// }; + +// try doc.parse(input, std.testing.allocator); + +// try expect(doc.tokens.items.len >= parsed.len); + +// for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// doc.deinit(std.testing.allocator); +// } + +// test "text_style_with_space right after" { +// std.testing.refAllDecls(@This()); + +// const doc = Document.initDefault(std.testing.allocator); + +// const input = "+ plain something + something else test+\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .text, +// .data = "+ plain something + something else test+", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// }; + +// try doc.parse(input, std.testing.allocator); + +// try expect(doc.tokens.items.len >= parsed.len); + +// for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// doc.deinit(std.testing.allocator); +// } + +// test "text_style_super_script" { +// std.testing.refAllDecls(@This()); + +// const doc = Document.initDefault(std.testing.allocator); + +// const input = "sometext^2\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .text, +// .data = "sometext", +// }, +// Token{ +// .type = .sup, +// .data = "^", +// }, +// Token{ +// .type = .text, +// .data = "2", +// }, +// }; + +// try doc.parse(input, std.testing.allocator); + +// try expect(doc.tokens.items.len >= parsed.len); + +// for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// doc.deinit(std.testing.allocator); +// } + +// test "text_style_sub_script" { +// std.testing.refAllDecls(@This()); + +// const doc = Document.initDefault(std.testing.allocator); + +// const input = "sometext_2\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .text, +// .data = "sometext", +// }, +// Token{ +// .type = .sub, +// .data = "_", +// }, +// Token{ +// .type = .text, +// .data = "2", +// }, +// }; + +// try doc.parse(input, std.testing.allocator); + +// try expect(doc.tokens.items.len >= parsed.len); + +// for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// doc.deinit(std.testing.allocator); +// } + +// test "list_ordered_number" { +// std.testing.refAllDecls(@This()); + +// const doc = Document.initDefault(std.testing.allocator); + +// const input = "1. ordered list\n2. ordered list\n3. ordered list\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .ordered_list, +// .data = "1.", +// }, +// Token{ +// .type = .text, +// .data = "ordered list", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// Token{ +// .type = .ordered_list, +// .data = "2.", +// }, +// Token{ +// .type = .text, +// .data = "ordered list", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// Token{ +// .type = .ordered_list, +// .data = "3.", +// }, +// Token{ +// .type = .text, +// .data = "ordered list", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// }; + +// try doc.parse(input, std.testing.allocator); + +// try expect(doc.tokens.items.len >= parsed.len); + +// for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// doc.deinit(std.testing.allocator); +// } + +// test "list_ordered_i" { +// std.testing.refAllDecls(@This()); + +// const doc = Document.initDefault(std.testing.allocator); + +// const input = "i. ordered list\nii. ordered list\niii. ordered list\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .ordered_list, +// .data = "i.", +// }, +// Token{ +// .type = .text, +// .data = "ordered list", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// Token{ +// .type = .ordered_list, +// .data = "ii.", +// }, +// Token{ +// .type = .text, +// .data = "ordered list", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// Token{ +// .type = .ordered_list, +// .data = "iii.", +// }, +// Token{ +// .type = .text, +// .data = "ordered list", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// }; + +// try doc.parse(input, std.testing.allocator); + +// try expect(doc.tokens.items.len >= parsed.len); + +// for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// doc.deinit(std.testing.allocator); +// } + +// test "list_ordered_letter" { +// std.testing.refAllDecls(@This()); + +// const doc = Document.initDefault(std.testing.allocator); + +// const input = "a. ordered list\nb. ordered list\nc. ordered list\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .ordered_list, +// .data = "a.", +// }, +// Token{ +// .type = .text, +// .data = "ordered list", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// Token{ +// .type = .ordered_list, +// .data = "b.", +// }, +// Token{ +// .type = .text, +// .data = "ordered list", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// Token{ +// .type = .ordered_list, +// .data = "c.", +// }, +// Token{ +// .type = .text, +// .data = "ordered list", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// }; + +// try doc.parse(input, std.testing.allocator); + +// try expect(doc.tokens.items.len >= parsed.len); + +// for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// doc.deinit(std.testing.allocator); +// } + +// test "list_unordered_dashed" { +// std.testing.refAllDecls(@This()); + +// const doc = Document.initDefault(std.testing.allocator); + +// const input = "- unordered list\n- unordered list\n- unordered list\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .unordered_list, +// .data = "-", +// }, +// Token{ +// .type = .text, +// .data = "unordered list", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// Token{ +// .type = .unordered_list, +// .data = "-", +// }, +// Token{ +// .type = .text, +// .data = "unordered list", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// Token{ +// .type = .unordered_list, +// .data = "-", +// }, +// Token{ +// .type = .text, +// .data = "unordered list", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// }; + +// try doc.parse(input, std.testing.allocator); + +// try expect(doc.tokens.items.len >= parsed.len); + +// for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// doc.deinit(std.testing.allocator); +// } + +// test "list_unordered_eq" { +// std.testing.refAllDecls(@This()); + +// const doc = Document.initDefault(std.testing.allocator); + +// const input = "= unordered list\n= unordered list\n= unordered list\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .unordered_list, +// .data = "=", +// }, +// Token{ +// .type = .text, +// .data = "unordered list", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// Token{ +// .type = .unordered_list, +// .data = "=", +// }, +// Token{ +// .type = .text, +// .data = "unordered list", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// Token{ +// .type = .unordered_list, +// .data = "=", +// }, +// Token{ +// .type = .text, +// .data = "unordered list", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// }; + +// try doc.parse(input, std.testing.allocator); + +// try expect(doc.tokens.items.len >= parsed.len); + +// for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// doc.deinit(std.testing.allocator); +// } + +// test "list_unordered_plus" { +// std.testing.refAllDecls(@This()); + +// const doc = Document.initDefault(std.testing.allocator); + +// const input = "+ unordered list\n+ unordered list\n+ unordered list\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .unordered_list, +// .data = "+", +// }, +// Token{ +// .type = .text, +// .data = "unordered list", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// Token{ +// .type = .unordered_list, +// .data = "+", +// }, +// Token{ +// .type = .text, +// .data = "unordered list", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// Token{ +// .type = .unordered_list, +// .data = "+", +// }, +// Token{ +// .type = .text, +// .data = "unordered list", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// }; + +// try doc.parse(input, std.testing.allocator); + +// try expect(doc.tokens.items.len >= parsed.len); + +// for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// doc.deinit(std.testing.allocator); +// } + +// test "list_unordered_dot" { +// std.testing.refAllDecls(@This()); + +// const doc = Document.initDefault(std.testing.allocator); + +// const input = "* unordered list\n* unordered list\n* unordered list\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .unordered_list, +// .data = "*", +// }, +// Token{ +// .type = .text, +// .data = "unordered list", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// Token{ +// .type = .unordered_list, +// .data = "*", +// }, +// Token{ +// .type = .text, +// .data = "unordered list", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// Token{ +// .type = .unordered_list, +// .data = "*", +// }, +// Token{ +// .type = .text, +// .data = "unordered list", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// }; + +// try doc.parse(input, std.testing.allocator); + +// try expect(doc.tokens.items.len >= parsed.len); + +// for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// doc.deinit(std.testing.allocator); +// } + +// test "embed_image" { +// std.testing.refAllDecls(@This()); + +// const doc = Document.initDefault(std.testing.allocator); + +// const input = "![figure text](relitive path to image)\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .embed, +// .data = "!", +// }, +// Token{ +// .type = .bracket_open, +// .data = "[", +// }, +// Token{ +// .type = .text, +// .data = "figure text", +// }, +// Token{ +// .type = .bracket_close, +// .data = "]", +// }, +// Token{ +// .type = .bracket_open, +// .data = "(", +// }, +// Token{ +// .type = .raw_text, +// .data = "relitive path to image", +// }, +// Token{ +// .type = .bracket_close, +// .data = ")", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// }; + +// try doc.parse(input, std.testing.allocator); + +// try expect(doc.tokens.items.len >= parsed.len); + +// for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// doc.deinit(std.testing.allocator); +// } + +// test "embed_table" { +// std.testing.refAllDecls(@This()); + +// const doc = Document.initDefault(std.testing.allocator); + +// const input = "#[table text](relitive path to tabulated data)\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .embed, +// .data = "#", +// }, +// Token{ +// .type = .bracket_open, +// .data = "[", +// }, +// Token{ +// .type = .text, +// .data = "table text", +// }, +// Token{ +// .type = .bracket_close, +// .data = "]", +// }, +// Token{ +// .type = .bracket_open, +// .data = "(", +// }, +// Token{ +// .type = .raw_text, +// .data = "relitive path to tabulated data", +// }, +// Token{ +// .type = .bracket_close, +// .data = ")", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// }; + +// try doc.parse(input, std.testing.allocator); + +// try expect(doc.tokens.items.len >= parsed.len); + +// for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// doc.deinit(std.testing.allocator); +// } + +// test "embed_file" { +// std.testing.refAllDecls(@This()); + +// const doc = Document.initDefault(std.testing.allocator); + +// const input = "$[file text](path to file#starting_line:ending_line)\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .embed, +// .data = "$", +// }, +// Token{ +// .type = .bracket_open, +// .data = "[", +// }, +// Token{ +// .type = .text, +// .data = "file text", +// }, +// Token{ +// .type = .bracket_close, +// .data = "]", +// }, +// Token{ +// .type = .bracket_open, +// .data = "(", +// }, +// Token{ +// .type = .raw_text, +// .data = "path to file#starting_line:ending_line", +// }, +// Token{ +// .type = .bracket_close, +// .data = ")", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// }; + +// try doc.parse(input, std.testing.allocator); + +// try expect(doc.tokens.items.len >= parsed.len); + +// for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// doc.deinit(std.testing.allocator); +// } + +// test "embed_*" { +// std.testing.refAllDecls(@This()); + +// const doc = Document.initDefault(std.testing.allocator); + +// const input = "*[file text](file_path)\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .embed, +// .data = "*", +// }, +// Token{ +// .type = .bracket_open, +// .data = "[", +// }, +// Token{ +// .type = .text, +// .data = "file text", +// }, +// Token{ +// .type = .bracket_close, +// .data = "]", +// }, +// Token{ +// .type = .bracket_open, +// .data = "(", +// }, +// Token{ +// .type = .raw_text, +// .data = "file_path", +// }, +// Token{ +// .type = .bracket_close, +// .data = ")", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// }; + +// try doc.parse(input, std.testing.allocator); + +// try expect(doc.tokens.items.len >= parsed.len); + +// for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// doc.deinit(std.testing.allocator); +// } + +// test "embed__" { +// std.testing.refAllDecls(@This()); + +// const doc = Document.initDefault(std.testing.allocator); + +// const input = "_[file text](file_path)\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .embed, +// .data = "_", +// }, +// Token{ +// .type = .bracket_open, +// .data = "[", +// }, +// Token{ +// .type = .text, +// .data = "file text", +// }, +// Token{ +// .type = .bracket_close, +// .data = "]", +// }, +// Token{ +// .type = .bracket_open, +// .data = "(", +// }, +// Token{ +// .type = .raw_text, +// .data = "file_path", +// }, +// Token{ +// .type = .bracket_close, +// .data = ")", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// }; + +// try doc.parse(input, std.testing.allocator); + +// try expect(doc.tokens.items.len >= parsed.len); + +// for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// doc.deinit(std.testing.allocator); +// } + +// test "embed_~" { +// std.testing.refAllDecls(@This()); + +// const doc = Document.initDefault(std.testing.allocator); + +// const input = "~[file text](file_path)\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .embed, +// .data = "~", +// }, +// Token{ +// .type = .bracket_open, +// .data = "[", +// }, +// Token{ +// .type = .text, +// .data = "file text", +// }, +// Token{ +// .type = .bracket_close, +// .data = "]", +// }, +// Token{ +// .type = .bracket_open, +// .data = "(", +// }, +// Token{ +// .type = .raw_text, +// .data = "file_path", +// }, +// Token{ +// .type = .bracket_close, +// .data = ")", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// }; + +// try doc.parse(input, std.testing.allocator); + +// try expect(doc.tokens.items.len >= parsed.len); + +// for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// doc.deinit(std.testing.allocator); +// } + +// test "embed_|" { +// std.testing.refAllDecls(@This()); + +// const doc = Document.initDefault(std.testing.allocator); + +// const input = "|[file text](file_path)\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .embed, +// .data = "|", +// }, +// Token{ +// .type = .bracket_open, +// .data = "[", +// }, +// Token{ +// .type = .text, +// .data = "file text", +// }, +// Token{ +// .type = .bracket_close, +// .data = "]", +// }, +// Token{ +// .type = .bracket_open, +// .data = "(", +// }, +// Token{ +// .type = .raw_text, +// .data = "file_path", +// }, +// Token{ +// .type = .bracket_close, +// .data = ")", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// }; + +// try doc.parse(input, std.testing.allocator); + +// try expect(doc.tokens.items.len >= parsed.len); + +// for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// doc.deinit(std.testing.allocator); +// } + +// test "embed_&" { +// std.testing.refAllDecls(@This()); + +// const doc = Document.initDefault(std.testing.allocator); + +// const input = "&[file text](file_path)\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .embed, +// .data = "&", +// }, +// Token{ +// .type = .bracket_open, +// .data = "[", +// }, +// Token{ +// .type = .text, +// .data = "file text", +// }, +// Token{ +// .type = .bracket_close, +// .data = "]", +// }, +// Token{ +// .type = .bracket_open, +// .data = "(", +// }, +// Token{ +// .type = .raw_text, +// .data = "file_path", +// }, +// Token{ +// .type = .bracket_close, +// .data = ")", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// }; + +// try doc.parse(input, std.testing.allocator); + +// try expect(doc.tokens.items.len >= parsed.len); + +// for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// doc.deinit(std.testing.allocator); +// } + +// test "embed_%" { +// std.testing.refAllDecls(@This()); + +// const doc = Document.initDefault(std.testing.allocator); + +// const input = "%[file text](file_path)\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .embed, +// .data = "%", +// }, +// Token{ +// .type = .bracket_open, +// .data = "[", +// }, +// Token{ +// .type = .text, +// .data = "file text", +// }, +// Token{ +// .type = .bracket_close, +// .data = "]", +// }, +// Token{ +// .type = .bracket_open, +// .data = "(", +// }, +// Token{ +// .type = .raw_text, +// .data = "file_path", +// }, +// Token{ +// .type = .bracket_close, +// .data = ")", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// }; + +// try doc.parse(input, std.testing.allocator); + +// try expect(doc.tokens.items.len >= parsed.len); + +// for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// doc.deinit(std.testing.allocator); +// } + +// test "embed_-" { +// std.testing.refAllDecls(@This()); + +// const doc = Document.initDefault(std.testing.allocator); + +// const input = "-[file text](file_path)\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .embed, +// .data = "-", +// }, +// Token{ +// .type = .bracket_open, +// .data = "[", +// }, +// Token{ +// .type = .text, +// .data = "file text", +// }, +// Token{ +// .type = .bracket_close, +// .data = "]", +// }, +// Token{ +// .type = .bracket_open, +// .data = "(", +// }, +// Token{ +// .type = .raw_text, +// .data = "file_path", +// }, +// Token{ +// .type = .bracket_close, +// .data = ")", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// }; + +// try doc.parse(input, std.testing.allocator); + +// try expect(doc.tokens.items.len >= parsed.len); + +// for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// doc.deinit(std.testing.allocator); +// } + +// test "embed_+" { +// std.testing.refAllDecls(@This()); + +// const doc = Document.initDefault(std.testing.allocator); + +// const input = "+[file text](file_path)\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .embed, +// .data = "+", +// }, +// Token{ +// .type = .bracket_open, +// .data = "[", +// }, +// Token{ +// .type = .text, +// .data = "file text", +// }, +// Token{ +// .type = .bracket_close, +// .data = "]", +// }, +// Token{ +// .type = .bracket_open, +// .data = "(", +// }, +// Token{ +// .type = .raw_text, +// .data = "file_path", +// }, +// Token{ +// .type = .bracket_close, +// .data = ")", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// }; + +// try doc.parse(input, std.testing.allocator); + +// try expect(doc.tokens.items.len >= parsed.len); + +// for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// doc.deinit(std.testing.allocator); +// } + +// test "embed_=" { +// std.testing.refAllDecls(@This()); + +// const doc = Document.initDefault(std.testing.allocator); + +// const input = "=[file text](file_path)\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .embed, +// .data = "=", +// }, +// Token{ +// .type = .bracket_open, +// .data = "[", +// }, +// Token{ +// .type = .text, +// .data = "file text", +// }, +// Token{ +// .type = .bracket_close, +// .data = "]", +// }, +// Token{ +// .type = .bracket_open, +// .data = "(", +// }, +// Token{ +// .type = .raw_text, +// .data = "file_path", +// }, +// Token{ +// .type = .bracket_close, +// .data = ")", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// }; + +// try doc.parse(input, std.testing.allocator); + +// try expect(doc.tokens.items.len >= parsed.len); + +// for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// doc.deinit(std.testing.allocator); +// } + +// test "embed_link" { +// std.testing.refAllDecls(@This()); + +// const doc = Document.initDefault(std.testing.allocator); + +// const input = "/[file text](link to something)\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .link, +// .data = "/", +// }, +// Token{ +// .type = .bracket_open, +// .data = "[", +// }, +// Token{ +// .type = .text, +// .data = "file text", +// }, +// Token{ +// .type = .bracket_close, +// .data = "]", +// }, +// Token{ +// .type = .bracket_open, +// .data = "(", +// }, +// Token{ +// .type = .raw_text, +// .data = "link to something", +// }, +// Token{ +// .type = .bracket_close, +// .data = ")", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// }; + +// try doc.parse(input, std.testing.allocator); + +// try expect(doc.tokens.items.len >= parsed.len); + +// for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// doc.deinit(std.testing.allocator); +// } + +// test "table" { +// std.testing.refAllDecls(@This()); + +// const doc = Document.initDefault(std.testing.allocator); + +// const input = "| col | col | col |\n| --- | --- | --- |\n| row | row | row |\n| row | row | row |\n| row | row | row |\n| row | row | row |\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .table_sep, +// .data = "|", +// }, +// Token{ +// .type = .text, +// .data = " col ", +// }, +// Token{ +// .type = .table_sep, +// .data = "|", +// }, +// Token{ +// .type = .text, +// .data = " col ", +// }, +// Token{ +// .type = .table_sep, +// .data = "|", +// }, +// Token{ +// .type = .text, +// .data = " col ", +// }, +// Token{ +// .type = .table_sep, +// .data = "|", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// Token{ +// .type = .table_sep, +// .data = "|", +// }, +// Token{ +// .type = .table_row_sep, +// .data = " --- ", +// }, +// Token{ +// .type = .table_sep, +// .data = "|", +// }, +// Token{ +// .type = .table_row_sep, +// .data = " --- ", +// }, +// Token{ +// .type = .table_sep, +// .data = "|", +// }, +// Token{ +// .type = .table_row_sep, +// .data = " --- ", +// }, +// Token{ +// .type = .table_sep, +// .data = "|", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// Token{ +// .type = .table_sep, +// .data = "|", +// }, +// Token{ +// .type = .text, +// .data = " row ", +// }, +// Token{ +// .type = .table_sep, +// .data = "|", +// }, +// Token{ +// .type = .text, +// .data = " row ", +// }, +// Token{ +// .type = .table_sep, +// .data = "|", +// }, +// Token{ +// .type = .text, +// .data = " row ", +// }, +// Token{ +// .type = .table_sep, +// .data = "|", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// Token{ +// .type = .table_sep, +// .data = "|", +// }, +// Token{ +// .type = .text, +// .data = " row ", +// }, +// Token{ +// .type = .table_sep, +// .data = "|", +// }, +// Token{ +// .type = .text, +// .data = " row ", +// }, +// Token{ +// .type = .table_sep, +// .data = "|", +// }, +// Token{ +// .type = .text, +// .data = " row ", +// }, +// Token{ +// .type = .table_sep, +// .data = "|", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// Token{ +// .type = .table_sep, +// .data = "|", +// }, +// Token{ +// .type = .text, +// .data = " row ", +// }, +// Token{ +// .type = .table_sep, +// .data = "|", +// }, +// Token{ +// .type = .text, +// .data = " row ", +// }, +// Token{ +// .type = .table_sep, +// .data = "|", +// }, +// Token{ +// .type = .text, +// .data = " row ", +// }, +// Token{ +// .type = .table_sep, +// .data = "|", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// Token{ +// .type = .table_sep, +// .data = "|", +// }, +// Token{ +// .type = .text, +// .data = " row ", +// }, +// Token{ +// .type = .table_sep, +// .data = "|", +// }, +// Token{ +// .type = .text, +// .data = " row ", +// }, +// Token{ +// .type = .table_sep, +// .data = "|", +// }, +// Token{ +// .type = .text, +// .data = " row ", +// }, +// Token{ +// .type = .table_sep, +// .data = "|", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// }; + +// try doc.parse(input, std.testing.allocator); + +// try expect(doc.tokens.items.len >= parsed.len); + +// for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// doc.deinit(std.testing.allocator); +// } + +// test "quote_arrow" { +// std.testing.refAllDecls(@This()); + +// const doc = Document.initDefault(std.testing.allocator); + +// const input = "> quote\n> quote\n> quote\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .quote, +// .data = ">", +// }, +// Token{ +// .type = .text, +// .data = "quote", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// Token{ +// .type = .quote, +// .data = ">", +// }, +// Token{ +// .type = .text, +// .data = "quote", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// Token{ +// .type = .quote, +// .data = ">", +// }, +// Token{ +// .type = .text, +// .data = "quote", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// }; + +// try doc.parse(input, std.testing.allocator); + +// try expect(doc.tokens.items.len >= parsed.len); + +// for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// doc.deinit(std.testing.allocator); +// } + +// test "quote_bar" { +// std.testing.refAllDecls(@This()); + +// const doc = Document.initDefault(std.testing.allocator); + +// const input = "| quote\n| quote\n| quote\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .quote, +// .data = "|", +// }, +// Token{ +// .type = .text, +// .data = "quote", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// Token{ +// .type = .quote, +// .data = "|", +// }, +// Token{ +// .type = .text, +// .data = "quote", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// Token{ +// .type = .quote, +// .data = "|", +// }, +// Token{ +// .type = .text, +// .data = "quote", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// }; + +// try doc.parse(input, std.testing.allocator); + +// try expect(doc.tokens.items.len >= parsed.len); + +// for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// doc.deinit(std.testing.allocator); +// } + +// test "text_block" { +// std.testing.refAllDecls(@This()); + +// const doc = Document.initDefault(std.testing.allocator); + +// const input = "```\nsome text or code\n```\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .code_block, +// .data = "```", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// Token{ +// .type = .raw_text, +// .data = "some text or code", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// Token{ +// .type = .code_block, +// .data = "```", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// }; + +// try doc.parse(input, std.testing.allocator); + +// try expect(doc.tokens.items.len >= parsed.len); + +// for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// doc.deinit(std.testing.allocator); +// } + +// test "code_block" { +// std.testing.refAllDecls(@This()); + +// const doc = Document.initDefault(std.testing.allocator); + +// const input = "```lang_name\nsome text or code\n```\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .code_block, +// .data = "```", +// }, +// Token{ +// .type = .raw_text, +// .data = "lang_name", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// Token{ +// .type = .raw_text, +// .data = "some text or code", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// Token{ +// .type = .code_block, +// .data = "```", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// }; + +// try doc.parse(input, std.testing.allocator); + +// try expect(doc.tokens.items.len >= parsed.len); + +// for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// doc.deinit(std.testing.allocator); +// } + +// test "passthrough_block" { +// std.testing.refAllDecls(@This()); + +// const doc = Document.initDefault(std.testing.allocator); + +// const input = "```passthrough\nsome text or code\n```\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .code_block, +// .data = "```", +// }, +// Token{ +// .type = .raw_text, +// .data = "passthrough", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// Token{ +// .type = .raw_text, +// .data = "some text or code", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// Token{ +// .type = .code_block, +// .data = "```", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// }; + +// try doc.parse(input, std.testing.allocator); + +// try expect(doc.tokens.items.len >= parsed.len); + +// for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// doc.deinit(std.testing.allocator); +// } + +// test "comment" { +// std.testing.refAllDecls(@This()); + +// const doc = Document.initDefault(std.testing.allocator); + +// const input = "// this is a comment\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .comment, +// .data = "// ", +// }, +// Token{ +// .type = .raw_text, +// .data = "this is a comment", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// }; + +// try doc.parse(input, std.testing.allocator); + +// try expect(doc.tokens.items.len >= parsed.len); + +// for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// doc.deinit(std.testing.allocator); +// } + +// test "tick_box_empty" { +// std.testing.refAllDecls(@This()); + +// const doc = Document.initDefault(std.testing.allocator); + +// const input = "- [ ] something to check off\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .tickbox, +// .data = "- [ ]", +// }, +// Token{ +// .type = .text, +// .data = "something to check off", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// }; + +// try doc.parse(input, std.testing.allocator); + +// try expect(doc.tokens.items.len >= parsed.len); + +// for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// doc.deinit(std.testing.allocator); +// } + +// test "tick_box_checked" { +// std.testing.refAllDecls(@This()); + +// const doc = Document.initDefault(std.testing.allocator); + +// const input = "- [x] something checked off\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .tickbox, +// .data = "- [x]", +// }, +// Token{ +// .type = .text, +// .data = "something checked off", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// }; + +// try doc.parse(input, std.testing.allocator); + +// try expect(doc.tokens.items.len >= parsed.len); + +// for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// doc.deinit(std.testing.allocator); +// } + +// test "tick_box_ticked" { +// std.testing.refAllDecls(@This()); + +// const doc = Document.initDefault(std.testing.allocator); + +// const input = "- [/] something ticked off\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .tickbox, +// .data = "- [/]", +// }, +// Token{ +// .type = .text, +// .data = "something ticked off", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// }; + +// try doc.parse(input, std.testing.allocator); + +// try expect(doc.tokens.items.len >= parsed.len); + +// for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// doc.deinit(std.testing.allocator); +// } + +// test "tick_box_dashed" { +// std.testing.refAllDecls(@This()); + +// const doc = Document.initDefault(std.testing.allocator); + +// const input = "- [~] something dashed off\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .tickbox, +// .data = "- [~]", +// }, +// Token{ +// .type = .text, +// .data = "something dashed off", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// }; + +// try doc.parse(input, std.testing.allocator); + +// try expect(doc.tokens.items.len >= parsed.len); + +// for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// doc.deinit(std.testing.allocator); +// } + +// test "horizontal_rule_straight_line" { +// std.testing.refAllDecls(@This()); + +// const doc = Document.initDefault(std.testing.allocator); + +// const input = "---\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .rule, +// .data = "---", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// }; + +// try doc.parse(input, std.testing.allocator); + +// try expect(doc.tokens.items.len >= parsed.len); + +// for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// doc.deinit(std.testing.allocator); +// } + +// test "horizontal_rule_double_straight_line" { +// std.testing.refAllDecls(@This()); + +// const doc = Document.initDefault(std.testing.allocator); + +// const input = "===\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .rule, +// .data = "===", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// }; + +// try doc.parse(input, std.testing.allocator); + +// try expect(doc.tokens.items.len >= parsed.len); + +// for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// doc.deinit(std.testing.allocator); +// } + +// test "horizontal_rule_big_dotted_straight_line" { +// std.testing.refAllDecls(@This()); + +// const doc = Document.initDefault(std.testing.allocator); + +// const input = "***\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .rule, +// .data = "***", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// }; + +// try doc.parse(input, std.testing.allocator); + +// try expect(doc.tokens.items.len >= parsed.len); + +// for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// doc.deinit(std.testing.allocator); +// } + +// test "horizontal_rule_small_dotted_line" { +// std.testing.refAllDecls(@This()); + +// const doc = Document.initDefault(std.testing.allocator); + +// const input = "...\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .rule, +// .data = "...", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// }; + +// try doc.parse(input, std.testing.allocator); + +// try expect(doc.tokens.items.len >= parsed.len); + +// for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// doc.deinit(std.testing.allocator); +// } + +// test "horizontal_rule_squiggly_line" { +// std.testing.refAllDecls(@This()); + +// const doc = Document.initDefault(std.testing.allocator); + +// const input = "&&&\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .rule, +// .data = "&&&", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// }; + +// try doc.parse(input, std.testing.allocator); + +// try expect(doc.tokens.items.len >= parsed.len); + +// for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// doc.deinit(std.testing.allocator); +// } + +// test "horizontal_rule_jagged_line" { +// std.testing.refAllDecls(@This()); + +// const doc = Document.initDefault(std.testing.allocator); + +// const input = "^^^\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .rule, +// .data = "^^^", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// }; + +// try doc.parse(input, std.testing.allocator); + +// try expect(doc.tokens.items.len >= parsed.len); + +// for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// doc.deinit(std.testing.allocator); +// } + +// test "footnote" { +// std.testing.refAllDecls(@This()); + +// const doc = Document.initDefault(std.testing.allocator); + +// const input = "sometext [^1]\n\n[^1]: footnote\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .text, +// .data = "sometext ", +// }, +// Token{ +// .type = .footnote, +// .data = "[^1]", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// Token{ +// .type = .footnote_def, +// .data = "[^1]:", +// }, +// Token{ +// .type = .text, +// .data = "footnote", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// }; + +// try doc.parse(input, std.testing.allocator); + +// try expect(doc.tokens.items.len >= parsed.len); + +// for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// doc.deinit(std.testing.allocator); +// } + +// test "inbulit" { +// std.testing.refAllDecls(@This()); + +// const doc = Document.initDefault(std.testing.allocator); + +// const input = "@.macro(arg, arg2)\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .inbulit, +// .data = "macro", +// }, +// Token{ +// .type = .bracket_open, +// .data = "(", +// }, +// Token{ +// .type = .macro_arg, +// .data = "arg", +// }, +// Token{ +// .type = .arg_sep, +// .data = ",", +// }, +// Token{ +// .type = .macro_arg, +// .data = "arg2", +// }, +// Token{ +// .type = .bracket_close, +// .data = ")", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// }; + +// try doc.parse(input, std.testing.allocator); + +// try expect(doc.tokens.items.len >= parsed.len); + +// for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// doc.deinit(std.testing.allocator); +// } diff --git a/test.zig b/test.zig new file mode 100644 index 0000000..c664cf3 --- /dev/null +++ b/test.zig @@ -0,0 +1,3029 @@ + +const Document = @import("attempt_2.zig").Document; +const std = @import("std"); +const Token = @import("Token.zig"); +const expect = std.testing.expect; + + +test "heading_1" { + std.testing.refAllDecls(@This()); + + const doc = Document.initDefault(std.testing.allocator); + + const input = "# hello this is a heading\n"; + const parsed = [_]Token{ + Token{ + .type = .heading, + .data = "#", + }, + Token{ + .type = .text, + .data = "hello this is a heading", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + + try doc.parse(input, std.testing.allocator); + + try expect(doc.tokens.items.len >= parsed.len); + + for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + doc.deinit(std.testing.allocator); +} + + +test "heading_2" { + std.testing.refAllDecls(@This()); + + const doc = Document.initDefault(std.testing.allocator); + + const input = "## hello this is a heading\n"; + const parsed = [_]Token{ + Token{ + .type = .heading, + .data = "##", + }, + Token{ + .type = .text, + .data = "hello this is a heading", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + + try doc.parse(input, std.testing.allocator); + + try expect(doc.tokens.items.len >= parsed.len); + + for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + doc.deinit(std.testing.allocator); +} + + +test "heading_3" { + std.testing.refAllDecls(@This()); + + const doc = Document.initDefault(std.testing.allocator); + + const input = "### hello this is a heading\n"; + const parsed = [_]Token{ + Token{ + .type = .heading, + .data = "###", + }, + Token{ + .type = .text, + .data = "hello this is a heading", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + + try doc.parse(input, std.testing.allocator); + + try expect(doc.tokens.items.len >= parsed.len); + + for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + doc.deinit(std.testing.allocator); +} + + +test "heading_4" { + std.testing.refAllDecls(@This()); + + const doc = Document.initDefault(std.testing.allocator); + + const input = "#### hello this is a heading\n"; + const parsed = [_]Token{ + Token{ + .type = .heading, + .data = "####", + }, + Token{ + .type = .text, + .data = "hello this is a heading", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + + try doc.parse(input, std.testing.allocator); + + try expect(doc.tokens.items.len >= parsed.len); + + for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + doc.deinit(std.testing.allocator); +} + + +test "heading_5" { + std.testing.refAllDecls(@This()); + + const doc = Document.initDefault(std.testing.allocator); + + const input = "##### hello this is a heading\n"; + const parsed = [_]Token{ + Token{ + .type = .heading, + .data = "#####", + }, + Token{ + .type = .text, + .data = "hello this is a heading", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + + try doc.parse(input, std.testing.allocator); + + try expect(doc.tokens.items.len >= parsed.len); + + for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + doc.deinit(std.testing.allocator); +} + + +test "heading_6" { + std.testing.refAllDecls(@This()); + + const doc = Document.initDefault(std.testing.allocator); + + const input = "###### hello this is a heading\n"; + const parsed = [_]Token{ + Token{ + .type = .heading, + .data = "######", + }, + Token{ + .type = .text, + .data = "hello this is a heading", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + + try doc.parse(input, std.testing.allocator); + + try expect(doc.tokens.items.len >= parsed.len); + + for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + doc.deinit(std.testing.allocator); +} + + +test "header_no_text" { + std.testing.refAllDecls(@This()); + + const doc = Document.initDefault(std.testing.allocator); + + const input = "######\n"; + const parsed = [_]Token{ + Token{ + .type = .heading, + .data = "######", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + + try doc.parse(input, std.testing.allocator); + + try expect(doc.tokens.items.len >= parsed.len); + + for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + doc.deinit(std.testing.allocator); +} + + +test "text" { + std.testing.refAllDecls(@This()); + + const doc = Document.initDefault(std.testing.allocator); + + const input = "plain test\n"; + const parsed = [_]Token{ + Token{ + .type = .text, + .data = "plain test", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + + try doc.parse(input, std.testing.allocator); + + try expect(doc.tokens.items.len >= parsed.len); + + for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + doc.deinit(std.testing.allocator); +} + + +test "text_style_code" { + std.testing.refAllDecls(@This()); + + const doc = Document.initDefault(std.testing.allocator); + + const input = "`plain test`\n"; + const parsed = [_]Token{ + Token{ + .type = .inline_style, + .data = "`", + }, + Token{ + .type = .text, + .data = "plain test", + }, + Token{ + .type = .inline_style, + .data = "`", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + + try doc.parse(input, std.testing.allocator); + + try expect(doc.tokens.items.len >= parsed.len); + + for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + doc.deinit(std.testing.allocator); +} + + +test "text_style_bold" { + std.testing.refAllDecls(@This()); + + const doc = Document.initDefault(std.testing.allocator); + + const input = "!plain test!\n"; + const parsed = [_]Token{ + Token{ + .type = .inline_style, + .data = "!", + }, + Token{ + .type = .text, + .data = "plain test", + }, + Token{ + .type = .inline_style, + .data = "!", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + + try doc.parse(input, std.testing.allocator); + + try expect(doc.tokens.items.len >= parsed.len); + + for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + doc.deinit(std.testing.allocator); +} + + +test "text_style_italic" { + std.testing.refAllDecls(@This()); + + const doc = Document.initDefault(std.testing.allocator); + + const input = "*plain test*\n"; + const parsed = [_]Token{ + Token{ + .type = .inline_style, + .data = "*", + }, + Token{ + .type = .text, + .data = "plain test", + }, + Token{ + .type = .inline_style, + .data = "*", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + + try doc.parse(input, std.testing.allocator); + + try expect(doc.tokens.items.len >= parsed.len); + + for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + doc.deinit(std.testing.allocator); +} + + +test "text_style_underline" { + std.testing.refAllDecls(@This()); + + const doc = Document.initDefault(std.testing.allocator); + + const input = "_plain test_\n"; + const parsed = [_]Token{ + Token{ + .type = .inline_style, + .data = "_", + }, + Token{ + .type = .text, + .data = "plain test", + }, + Token{ + .type = .inline_style, + .data = "_", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + + try doc.parse(input, std.testing.allocator); + + try expect(doc.tokens.items.len >= parsed.len); + + for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + doc.deinit(std.testing.allocator); +} + + +test "text_style_strikethrough" { + std.testing.refAllDecls(@This()); + + const doc = Document.initDefault(std.testing.allocator); + + const input = "~plain test~\n"; + const parsed = [_]Token{ + Token{ + .type = .inline_style, + .data = "~", + }, + Token{ + .type = .text, + .data = "plain test", + }, + Token{ + .type = .inline_style, + .data = "~", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + + try doc.parse(input, std.testing.allocator); + + try expect(doc.tokens.items.len >= parsed.len); + + for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + doc.deinit(std.testing.allocator); +} + + +test "text_style_highlighted" { + std.testing.refAllDecls(@This()); + + const doc = Document.initDefault(std.testing.allocator); + + const input = "|plain test|\n"; + const parsed = [_]Token{ + Token{ + .type = .inline_style, + .data = "|", + }, + Token{ + .type = .text, + .data = "plain test", + }, + Token{ + .type = .inline_style, + .data = "|", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + + try doc.parse(input, std.testing.allocator); + + try expect(doc.tokens.items.len >= parsed.len); + + for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + doc.deinit(std.testing.allocator); +} + + +test "text_style_&" { + std.testing.refAllDecls(@This()); + + const doc = Document.initDefault(std.testing.allocator); + + const input = "&plain test&\n"; + const parsed = [_]Token{ + Token{ + .type = .inline_style, + .data = "&", + }, + Token{ + .type = .text, + .data = "plain test", + }, + Token{ + .type = .inline_style, + .data = "&", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + + try doc.parse(input, std.testing.allocator); + + try expect(doc.tokens.items.len >= parsed.len); + + for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + doc.deinit(std.testing.allocator); +} + + +test "text_style_%" { + std.testing.refAllDecls(@This()); + + const doc = Document.initDefault(std.testing.allocator); + + const input = "%plain test%\n"; + const parsed = [_]Token{ + Token{ + .type = .inline_style, + .data = "%", + }, + Token{ + .type = .text, + .data = "plain test", + }, + Token{ + .type = .inline_style, + .data = "%", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + + try doc.parse(input, std.testing.allocator); + + try expect(doc.tokens.items.len >= parsed.len); + + for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + doc.deinit(std.testing.allocator); +} + + +test "text_style_$" { + std.testing.refAllDecls(@This()); + + const doc = Document.initDefault(std.testing.allocator); + + const input = "$plain test$\n"; + const parsed = [_]Token{ + Token{ + .type = .inline_style, + .data = "$", + }, + Token{ + .type = .text, + .data = "plain test", + }, + Token{ + .type = .inline_style, + .data = "$", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + + try doc.parse(input, std.testing.allocator); + + try expect(doc.tokens.items.len >= parsed.len); + + for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + doc.deinit(std.testing.allocator); +} + + +test "text_style_#" { + std.testing.refAllDecls(@This()); + + const doc = Document.initDefault(std.testing.allocator); + + const input = "#plain test#\n"; + const parsed = [_]Token{ + Token{ + .type = .inline_style, + .data = "#", + }, + Token{ + .type = .text, + .data = "plain test", + }, + Token{ + .type = .inline_style, + .data = "#", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + + try doc.parse(input, std.testing.allocator); + + try expect(doc.tokens.items.len >= parsed.len); + + for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + doc.deinit(std.testing.allocator); +} + + +test "text_style_-" { + std.testing.refAllDecls(@This()); + + const doc = Document.initDefault(std.testing.allocator); + + const input = "-plain test-\n"; + const parsed = [_]Token{ + Token{ + .type = .inline_style, + .data = "-", + }, + Token{ + .type = .text, + .data = "plain test", + }, + Token{ + .type = .inline_style, + .data = "-", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + + try doc.parse(input, std.testing.allocator); + + try expect(doc.tokens.items.len >= parsed.len); + + for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + doc.deinit(std.testing.allocator); +} + + +test "text_style_+" { + std.testing.refAllDecls(@This()); + + const doc = Document.initDefault(std.testing.allocator); + + const input = "+plain test+\n"; + const parsed = [_]Token{ + Token{ + .type = .inline_style, + .data = "+", + }, + Token{ + .type = .text, + .data = "plain test", + }, + Token{ + .type = .inline_style, + .data = "+", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + + try doc.parse(input, std.testing.allocator); + + try expect(doc.tokens.items.len >= parsed.len); + + for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + doc.deinit(std.testing.allocator); +} + + +test "text_style_=" { + std.testing.refAllDecls(@This()); + + const doc = Document.initDefault(std.testing.allocator); + + const input = "=plain test=\n"; + const parsed = [_]Token{ + Token{ + .type = .inline_style, + .data = "=", + }, + Token{ + .type = .text, + .data = "plain test", + }, + Token{ + .type = .inline_style, + .data = "=", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + + try doc.parse(input, std.testing.allocator); + + try expect(doc.tokens.items.len >= parsed.len); + + for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + doc.deinit(std.testing.allocator); +} + + +test "text_style_espaced" { + std.testing.refAllDecls(@This()); + + const doc = Document.initDefault(std.testing.allocator); + + const input = "\\`plain test`\n"; + const parsed = [_]Token{ + Token{ + .type = .text, + .data = "`plain test`", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + + try doc.parse(input, std.testing.allocator); + + try expect(doc.tokens.items.len >= parsed.len); + + for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + doc.deinit(std.testing.allocator); +} + + +test "text_style_with_style_char_in_string" { + std.testing.refAllDecls(@This()); + + const doc = Document.initDefault(std.testing.allocator); + + const input = "+plain something + something else test+\n"; + const parsed = [_]Token{ + Token{ + .type = .inline_style, + .data = "+", + }, + Token{ + .type = .text, + .data = "plain something + something else test", + }, + Token{ + .type = .inline_style, + .data = "+", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + + try doc.parse(input, std.testing.allocator); + + try expect(doc.tokens.items.len >= parsed.len); + + for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + doc.deinit(std.testing.allocator); +} + + +test "text_style_with_space right after" { + std.testing.refAllDecls(@This()); + + const doc = Document.initDefault(std.testing.allocator); + + const input = "+ plain something + something else test+\n"; + const parsed = [_]Token{ + Token{ + .type = .text, + .data = "+ plain something + something else test+", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + + try doc.parse(input, std.testing.allocator); + + try expect(doc.tokens.items.len >= parsed.len); + + for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + doc.deinit(std.testing.allocator); +} + + +test "text_style_super_script" { + std.testing.refAllDecls(@This()); + + const doc = Document.initDefault(std.testing.allocator); + + const input = "sometext^2\n"; + const parsed = [_]Token{ + Token{ + .type = .text, + .data = "sometext", + }, + Token{ + .type = .sup, + .data = "^", + }, + Token{ + .type = .text, + .data = "2", + }, + }; + + + try doc.parse(input, std.testing.allocator); + + try expect(doc.tokens.items.len >= parsed.len); + + for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + doc.deinit(std.testing.allocator); +} + + +test "text_style_sub_script" { + std.testing.refAllDecls(@This()); + + const doc = Document.initDefault(std.testing.allocator); + + const input = "sometext_2\n"; + const parsed = [_]Token{ + Token{ + .type = .text, + .data = "sometext", + }, + Token{ + .type = .sub, + .data = "_", + }, + Token{ + .type = .text, + .data = "2", + }, + }; + + + try doc.parse(input, std.testing.allocator); + + try expect(doc.tokens.items.len >= parsed.len); + + for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + doc.deinit(std.testing.allocator); +} + + +test "list_ordered_number" { + std.testing.refAllDecls(@This()); + + const doc = Document.initDefault(std.testing.allocator); + + const input = "1. ordered list\n2. ordered list\n3. ordered list\n"; + const parsed = [_]Token{ + Token{ + .type = .ordered_list, + .data = "1.", + }, + Token{ + .type = .text, + .data = "ordered list", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .ordered_list, + .data = "2.", + }, + Token{ + .type = .text, + .data = "ordered list", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .ordered_list, + .data = "3.", + }, + Token{ + .type = .text, + .data = "ordered list", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + + try doc.parse(input, std.testing.allocator); + + try expect(doc.tokens.items.len >= parsed.len); + + for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + doc.deinit(std.testing.allocator); +} + + +test "list_ordered_i" { + std.testing.refAllDecls(@This()); + + const doc = Document.initDefault(std.testing.allocator); + + const input = "i. ordered list\nii. ordered list\niii. ordered list\n"; + const parsed = [_]Token{ + Token{ + .type = .ordered_list, + .data = "i.", + }, + Token{ + .type = .text, + .data = "ordered list", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .ordered_list, + .data = "ii.", + }, + Token{ + .type = .text, + .data = "ordered list", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .ordered_list, + .data = "iii.", + }, + Token{ + .type = .text, + .data = "ordered list", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + + try doc.parse(input, std.testing.allocator); + + try expect(doc.tokens.items.len >= parsed.len); + + for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + doc.deinit(std.testing.allocator); +} + + +test "list_ordered_letter" { + std.testing.refAllDecls(@This()); + + const doc = Document.initDefault(std.testing.allocator); + + const input = "a. ordered list\nb. ordered list\nc. ordered list\n"; + const parsed = [_]Token{ + Token{ + .type = .ordered_list, + .data = "a.", + }, + Token{ + .type = .text, + .data = "ordered list", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .ordered_list, + .data = "b.", + }, + Token{ + .type = .text, + .data = "ordered list", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .ordered_list, + .data = "c.", + }, + Token{ + .type = .text, + .data = "ordered list", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + + try doc.parse(input, std.testing.allocator); + + try expect(doc.tokens.items.len >= parsed.len); + + for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + doc.deinit(std.testing.allocator); +} + + +test "list_unordered_dashed" { + std.testing.refAllDecls(@This()); + + const doc = Document.initDefault(std.testing.allocator); + + const input = "- unordered list\n- unordered list\n- unordered list\n"; + const parsed = [_]Token{ + Token{ + .type = .unordered_list, + .data = "-", + }, + Token{ + .type = .text, + .data = "unordered list", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .unordered_list, + .data = "-", + }, + Token{ + .type = .text, + .data = "unordered list", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .unordered_list, + .data = "-", + }, + Token{ + .type = .text, + .data = "unordered list", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + + try doc.parse(input, std.testing.allocator); + + try expect(doc.tokens.items.len >= parsed.len); + + for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + doc.deinit(std.testing.allocator); +} + + +test "list_unordered_eq" { + std.testing.refAllDecls(@This()); + + const doc = Document.initDefault(std.testing.allocator); + + const input = "= unordered list\n= unordered list\n= unordered list\n"; + const parsed = [_]Token{ + Token{ + .type = .unordered_list, + .data = "=", + }, + Token{ + .type = .text, + .data = "unordered list", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .unordered_list, + .data = "=", + }, + Token{ + .type = .text, + .data = "unordered list", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .unordered_list, + .data = "=", + }, + Token{ + .type = .text, + .data = "unordered list", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + + try doc.parse(input, std.testing.allocator); + + try expect(doc.tokens.items.len >= parsed.len); + + for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + doc.deinit(std.testing.allocator); +} + + +test "list_unordered_plus" { + std.testing.refAllDecls(@This()); + + const doc = Document.initDefault(std.testing.allocator); + + const input = "+ unordered list\n+ unordered list\n+ unordered list\n"; + const parsed = [_]Token{ + Token{ + .type = .unordered_list, + .data = "+", + }, + Token{ + .type = .text, + .data = "unordered list", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .unordered_list, + .data = "+", + }, + Token{ + .type = .text, + .data = "unordered list", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .unordered_list, + .data = "+", + }, + Token{ + .type = .text, + .data = "unordered list", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + + try doc.parse(input, std.testing.allocator); + + try expect(doc.tokens.items.len >= parsed.len); + + for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + doc.deinit(std.testing.allocator); +} + + +test "list_unordered_dot" { + std.testing.refAllDecls(@This()); + + const doc = Document.initDefault(std.testing.allocator); + + const input = "* unordered list\n* unordered list\n* unordered list\n"; + const parsed = [_]Token{ + Token{ + .type = .unordered_list, + .data = "*", + }, + Token{ + .type = .text, + .data = "unordered list", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .unordered_list, + .data = "*", + }, + Token{ + .type = .text, + .data = "unordered list", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .unordered_list, + .data = "*", + }, + Token{ + .type = .text, + .data = "unordered list", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + + try doc.parse(input, std.testing.allocator); + + try expect(doc.tokens.items.len >= parsed.len); + + for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + doc.deinit(std.testing.allocator); +} + + +test "embed_image" { + std.testing.refAllDecls(@This()); + + const doc = Document.initDefault(std.testing.allocator); + + const input = "![figure text](relitive path to image)\n"; + const parsed = [_]Token{ + Token{ + .type = .embed, + .data = "!", + }, + Token{ + .type = .bracket_open, + .data = "[", + }, + Token{ + .type = .text, + .data = "figure text", + }, + Token{ + .type = .bracket_close, + .data = "]", + }, + Token{ + .type = .bracket_open, + .data = "(", + }, + Token{ + .type = .raw_text, + .data = "relitive path to image", + }, + Token{ + .type = .bracket_close, + .data = ")", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + + try doc.parse(input, std.testing.allocator); + + try expect(doc.tokens.items.len >= parsed.len); + + for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + doc.deinit(std.testing.allocator); +} + + +test "embed_table" { + std.testing.refAllDecls(@This()); + + const doc = Document.initDefault(std.testing.allocator); + + const input = "#[table text](relitive path to tabulated data)\n"; + const parsed = [_]Token{ + Token{ + .type = .embed, + .data = "#", + }, + Token{ + .type = .bracket_open, + .data = "[", + }, + Token{ + .type = .text, + .data = "table text", + }, + Token{ + .type = .bracket_close, + .data = "]", + }, + Token{ + .type = .bracket_open, + .data = "(", + }, + Token{ + .type = .raw_text, + .data = "relitive path to tabulated data", + }, + Token{ + .type = .bracket_close, + .data = ")", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + + try doc.parse(input, std.testing.allocator); + + try expect(doc.tokens.items.len >= parsed.len); + + for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + doc.deinit(std.testing.allocator); +} + + +test "embed_file" { + std.testing.refAllDecls(@This()); + + const doc = Document.initDefault(std.testing.allocator); + + const input = "$[file text](path to file#starting_line:ending_line)\n"; + const parsed = [_]Token{ + Token{ + .type = .embed, + .data = "$", + }, + Token{ + .type = .bracket_open, + .data = "[", + }, + Token{ + .type = .text, + .data = "file text", + }, + Token{ + .type = .bracket_close, + .data = "]", + }, + Token{ + .type = .bracket_open, + .data = "(", + }, + Token{ + .type = .raw_text, + .data = "path to file#starting_line:ending_line", + }, + Token{ + .type = .bracket_close, + .data = ")", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + + try doc.parse(input, std.testing.allocator); + + try expect(doc.tokens.items.len >= parsed.len); + + for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + doc.deinit(std.testing.allocator); +} + + +test "embed_*" { + std.testing.refAllDecls(@This()); + + const doc = Document.initDefault(std.testing.allocator); + + const input = "*[file text](file_path)\n"; + const parsed = [_]Token{ + Token{ + .type = .embed, + .data = "*", + }, + Token{ + .type = .bracket_open, + .data = "[", + }, + Token{ + .type = .text, + .data = "file text", + }, + Token{ + .type = .bracket_close, + .data = "]", + }, + Token{ + .type = .bracket_open, + .data = "(", + }, + Token{ + .type = .raw_text, + .data = "file_path", + }, + Token{ + .type = .bracket_close, + .data = ")", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + + try doc.parse(input, std.testing.allocator); + + try expect(doc.tokens.items.len >= parsed.len); + + for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + doc.deinit(std.testing.allocator); +} + + +test "embed__" { + std.testing.refAllDecls(@This()); + + const doc = Document.initDefault(std.testing.allocator); + + const input = "_[file text](file_path)\n"; + const parsed = [_]Token{ + Token{ + .type = .embed, + .data = "_", + }, + Token{ + .type = .bracket_open, + .data = "[", + }, + Token{ + .type = .text, + .data = "file text", + }, + Token{ + .type = .bracket_close, + .data = "]", + }, + Token{ + .type = .bracket_open, + .data = "(", + }, + Token{ + .type = .raw_text, + .data = "file_path", + }, + Token{ + .type = .bracket_close, + .data = ")", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + + try doc.parse(input, std.testing.allocator); + + try expect(doc.tokens.items.len >= parsed.len); + + for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + doc.deinit(std.testing.allocator); +} + + +test "embed_~" { + std.testing.refAllDecls(@This()); + + const doc = Document.initDefault(std.testing.allocator); + + const input = "~[file text](file_path)\n"; + const parsed = [_]Token{ + Token{ + .type = .embed, + .data = "~", + }, + Token{ + .type = .bracket_open, + .data = "[", + }, + Token{ + .type = .text, + .data = "file text", + }, + Token{ + .type = .bracket_close, + .data = "]", + }, + Token{ + .type = .bracket_open, + .data = "(", + }, + Token{ + .type = .raw_text, + .data = "file_path", + }, + Token{ + .type = .bracket_close, + .data = ")", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + + try doc.parse(input, std.testing.allocator); + + try expect(doc.tokens.items.len >= parsed.len); + + for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + doc.deinit(std.testing.allocator); +} + + +test "embed_|" { + std.testing.refAllDecls(@This()); + + const doc = Document.initDefault(std.testing.allocator); + + const input = "|[file text](file_path)\n"; + const parsed = [_]Token{ + Token{ + .type = .embed, + .data = "|", + }, + Token{ + .type = .bracket_open, + .data = "[", + }, + Token{ + .type = .text, + .data = "file text", + }, + Token{ + .type = .bracket_close, + .data = "]", + }, + Token{ + .type = .bracket_open, + .data = "(", + }, + Token{ + .type = .raw_text, + .data = "file_path", + }, + Token{ + .type = .bracket_close, + .data = ")", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + + try doc.parse(input, std.testing.allocator); + + try expect(doc.tokens.items.len >= parsed.len); + + for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + doc.deinit(std.testing.allocator); +} + + +test "embed_&" { + std.testing.refAllDecls(@This()); + + const doc = Document.initDefault(std.testing.allocator); + + const input = "&[file text](file_path)\n"; + const parsed = [_]Token{ + Token{ + .type = .embed, + .data = "&", + }, + Token{ + .type = .bracket_open, + .data = "[", + }, + Token{ + .type = .text, + .data = "file text", + }, + Token{ + .type = .bracket_close, + .data = "]", + }, + Token{ + .type = .bracket_open, + .data = "(", + }, + Token{ + .type = .raw_text, + .data = "file_path", + }, + Token{ + .type = .bracket_close, + .data = ")", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + + try doc.parse(input, std.testing.allocator); + + try expect(doc.tokens.items.len >= parsed.len); + + for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + doc.deinit(std.testing.allocator); +} + + +test "embed_%" { + std.testing.refAllDecls(@This()); + + const doc = Document.initDefault(std.testing.allocator); + + const input = "%[file text](file_path)\n"; + const parsed = [_]Token{ + Token{ + .type = .embed, + .data = "%", + }, + Token{ + .type = .bracket_open, + .data = "[", + }, + Token{ + .type = .text, + .data = "file text", + }, + Token{ + .type = .bracket_close, + .data = "]", + }, + Token{ + .type = .bracket_open, + .data = "(", + }, + Token{ + .type = .raw_text, + .data = "file_path", + }, + Token{ + .type = .bracket_close, + .data = ")", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + + try doc.parse(input, std.testing.allocator); + + try expect(doc.tokens.items.len >= parsed.len); + + for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + doc.deinit(std.testing.allocator); +} + + +test "embed_-" { + std.testing.refAllDecls(@This()); + + const doc = Document.initDefault(std.testing.allocator); + + const input = "-[file text](file_path)\n"; + const parsed = [_]Token{ + Token{ + .type = .embed, + .data = "-", + }, + Token{ + .type = .bracket_open, + .data = "[", + }, + Token{ + .type = .text, + .data = "file text", + }, + Token{ + .type = .bracket_close, + .data = "]", + }, + Token{ + .type = .bracket_open, + .data = "(", + }, + Token{ + .type = .raw_text, + .data = "file_path", + }, + Token{ + .type = .bracket_close, + .data = ")", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + + try doc.parse(input, std.testing.allocator); + + try expect(doc.tokens.items.len >= parsed.len); + + for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + doc.deinit(std.testing.allocator); +} + + +test "embed_+" { + std.testing.refAllDecls(@This()); + + const doc = Document.initDefault(std.testing.allocator); + + const input = "+[file text](file_path)\n"; + const parsed = [_]Token{ + Token{ + .type = .embed, + .data = "+", + }, + Token{ + .type = .bracket_open, + .data = "[", + }, + Token{ + .type = .text, + .data = "file text", + }, + Token{ + .type = .bracket_close, + .data = "]", + }, + Token{ + .type = .bracket_open, + .data = "(", + }, + Token{ + .type = .raw_text, + .data = "file_path", + }, + Token{ + .type = .bracket_close, + .data = ")", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + + try doc.parse(input, std.testing.allocator); + + try expect(doc.tokens.items.len >= parsed.len); + + for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + doc.deinit(std.testing.allocator); +} + + +test "embed_=" { + std.testing.refAllDecls(@This()); + + const doc = Document.initDefault(std.testing.allocator); + + const input = "=[file text](file_path)\n"; + const parsed = [_]Token{ + Token{ + .type = .embed, + .data = "=", + }, + Token{ + .type = .bracket_open, + .data = "[", + }, + Token{ + .type = .text, + .data = "file text", + }, + Token{ + .type = .bracket_close, + .data = "]", + }, + Token{ + .type = .bracket_open, + .data = "(", + }, + Token{ + .type = .raw_text, + .data = "file_path", + }, + Token{ + .type = .bracket_close, + .data = ")", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + + try doc.parse(input, std.testing.allocator); + + try expect(doc.tokens.items.len >= parsed.len); + + for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + doc.deinit(std.testing.allocator); +} + + +test "embed_link" { + std.testing.refAllDecls(@This()); + + const doc = Document.initDefault(std.testing.allocator); + + const input = "/[file text](link to something)\n"; + const parsed = [_]Token{ + Token{ + .type = .link, + .data = "/", + }, + Token{ + .type = .bracket_open, + .data = "[", + }, + Token{ + .type = .text, + .data = "file text", + }, + Token{ + .type = .bracket_close, + .data = "]", + }, + Token{ + .type = .bracket_open, + .data = "(", + }, + Token{ + .type = .raw_text, + .data = "link to something", + }, + Token{ + .type = .bracket_close, + .data = ")", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + + try doc.parse(input, std.testing.allocator); + + try expect(doc.tokens.items.len >= parsed.len); + + for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + doc.deinit(std.testing.allocator); +} + + +test "table" { + std.testing.refAllDecls(@This()); + + const doc = Document.initDefault(std.testing.allocator); + + const input = "| col | col | col |\n| --- | --- | --- |\n| row | row | row |\n| row | row | row |\n| row | row | row |\n| row | row | row |\n"; + const parsed = [_]Token{ + Token{ + .type = .table_sep, + .data = "|", + }, + Token{ + .type = .text, + .data = " col ", + }, + Token{ + .type = .table_sep, + .data = "|", + }, + Token{ + .type = .text, + .data = " col ", + }, + Token{ + .type = .table_sep, + .data = "|", + }, + Token{ + .type = .text, + .data = " col ", + }, + Token{ + .type = .table_sep, + .data = "|", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .table_sep, + .data = "|", + }, + Token{ + .type = .table_row_sep, + .data = " --- ", + }, + Token{ + .type = .table_sep, + .data = "|", + }, + Token{ + .type = .table_row_sep, + .data = " --- ", + }, + Token{ + .type = .table_sep, + .data = "|", + }, + Token{ + .type = .table_row_sep, + .data = " --- ", + }, + Token{ + .type = .table_sep, + .data = "|", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .table_sep, + .data = "|", + }, + Token{ + .type = .text, + .data = " row ", + }, + Token{ + .type = .table_sep, + .data = "|", + }, + Token{ + .type = .text, + .data = " row ", + }, + Token{ + .type = .table_sep, + .data = "|", + }, + Token{ + .type = .text, + .data = " row ", + }, + Token{ + .type = .table_sep, + .data = "|", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .table_sep, + .data = "|", + }, + Token{ + .type = .text, + .data = " row ", + }, + Token{ + .type = .table_sep, + .data = "|", + }, + Token{ + .type = .text, + .data = " row ", + }, + Token{ + .type = .table_sep, + .data = "|", + }, + Token{ + .type = .text, + .data = " row ", + }, + Token{ + .type = .table_sep, + .data = "|", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .table_sep, + .data = "|", + }, + Token{ + .type = .text, + .data = " row ", + }, + Token{ + .type = .table_sep, + .data = "|", + }, + Token{ + .type = .text, + .data = " row ", + }, + Token{ + .type = .table_sep, + .data = "|", + }, + Token{ + .type = .text, + .data = " row ", + }, + Token{ + .type = .table_sep, + .data = "|", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .table_sep, + .data = "|", + }, + Token{ + .type = .text, + .data = " row ", + }, + Token{ + .type = .table_sep, + .data = "|", + }, + Token{ + .type = .text, + .data = " row ", + }, + Token{ + .type = .table_sep, + .data = "|", + }, + Token{ + .type = .text, + .data = " row ", + }, + Token{ + .type = .table_sep, + .data = "|", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + + try doc.parse(input, std.testing.allocator); + + try expect(doc.tokens.items.len >= parsed.len); + + for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + doc.deinit(std.testing.allocator); +} + + +test "quote_arrow" { + std.testing.refAllDecls(@This()); + + const doc = Document.initDefault(std.testing.allocator); + + const input = "> quote\n> quote\n> quote\n"; + const parsed = [_]Token{ + Token{ + .type = .quote, + .data = ">", + }, + Token{ + .type = .text, + .data = "quote", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .quote, + .data = ">", + }, + Token{ + .type = .text, + .data = "quote", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .quote, + .data = ">", + }, + Token{ + .type = .text, + .data = "quote", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + + try doc.parse(input, std.testing.allocator); + + try expect(doc.tokens.items.len >= parsed.len); + + for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + doc.deinit(std.testing.allocator); +} + + +test "quote_bar" { + std.testing.refAllDecls(@This()); + + const doc = Document.initDefault(std.testing.allocator); + + const input = "| quote\n| quote\n| quote\n"; + const parsed = [_]Token{ + Token{ + .type = .quote, + .data = "|", + }, + Token{ + .type = .text, + .data = "quote", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .quote, + .data = "|", + }, + Token{ + .type = .text, + .data = "quote", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .quote, + .data = "|", + }, + Token{ + .type = .text, + .data = "quote", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + + try doc.parse(input, std.testing.allocator); + + try expect(doc.tokens.items.len >= parsed.len); + + for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + doc.deinit(std.testing.allocator); +} + + +test "text_block" { + std.testing.refAllDecls(@This()); + + const doc = Document.initDefault(std.testing.allocator); + + const input = "```\nsome text or code\n```\n"; + const parsed = [_]Token{ + Token{ + .type = .code_block, + .data = "```", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .raw_text, + .data = "some text or code", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .code_block, + .data = "```", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + + try doc.parse(input, std.testing.allocator); + + try expect(doc.tokens.items.len >= parsed.len); + + for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + doc.deinit(std.testing.allocator); +} + + +test "code_block" { + std.testing.refAllDecls(@This()); + + const doc = Document.initDefault(std.testing.allocator); + + const input = "```lang_name\nsome text or code\n```\n"; + const parsed = [_]Token{ + Token{ + .type = .code_block, + .data = "```", + }, + Token{ + .type = .raw_text, + .data = "lang_name", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .raw_text, + .data = "some text or code", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .code_block, + .data = "```", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + + try doc.parse(input, std.testing.allocator); + + try expect(doc.tokens.items.len >= parsed.len); + + for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + doc.deinit(std.testing.allocator); +} + + +test "passthrough_block" { + std.testing.refAllDecls(@This()); + + const doc = Document.initDefault(std.testing.allocator); + + const input = "```passthrough\nsome text or code\n```\n"; + const parsed = [_]Token{ + Token{ + .type = .code_block, + .data = "```", + }, + Token{ + .type = .raw_text, + .data = "passthrough", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .raw_text, + .data = "some text or code", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .code_block, + .data = "```", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + + try doc.parse(input, std.testing.allocator); + + try expect(doc.tokens.items.len >= parsed.len); + + for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + doc.deinit(std.testing.allocator); +} + + +test "comment" { + std.testing.refAllDecls(@This()); + + const doc = Document.initDefault(std.testing.allocator); + + const input = "// this is a comment\n"; + const parsed = [_]Token{ + Token{ + .type = .comment, + .data = "// ", + }, + Token{ + .type = .raw_text, + .data = "this is a comment", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + + try doc.parse(input, std.testing.allocator); + + try expect(doc.tokens.items.len >= parsed.len); + + for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + doc.deinit(std.testing.allocator); +} + + +test "tick_box_empty" { + std.testing.refAllDecls(@This()); + + const doc = Document.initDefault(std.testing.allocator); + + const input = "- [ ] something to check off\n"; + const parsed = [_]Token{ + Token{ + .type = .tickbox, + .data = "- [ ]", + }, + Token{ + .type = .text, + .data = "something to check off", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + + try doc.parse(input, std.testing.allocator); + + try expect(doc.tokens.items.len >= parsed.len); + + for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + doc.deinit(std.testing.allocator); +} + + +test "tick_box_checked" { + std.testing.refAllDecls(@This()); + + const doc = Document.initDefault(std.testing.allocator); + + const input = "- [x] something checked off\n"; + const parsed = [_]Token{ + Token{ + .type = .tickbox, + .data = "- [x]", + }, + Token{ + .type = .text, + .data = "something checked off", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + + try doc.parse(input, std.testing.allocator); + + try expect(doc.tokens.items.len >= parsed.len); + + for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + doc.deinit(std.testing.allocator); +} + + +test "tick_box_ticked" { + std.testing.refAllDecls(@This()); + + const doc = Document.initDefault(std.testing.allocator); + + const input = "- [/] something ticked off\n"; + const parsed = [_]Token{ + Token{ + .type = .tickbox, + .data = "- [/]", + }, + Token{ + .type = .text, + .data = "something ticked off", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + + try doc.parse(input, std.testing.allocator); + + try expect(doc.tokens.items.len >= parsed.len); + + for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + doc.deinit(std.testing.allocator); +} + + +test "tick_box_dashed" { + std.testing.refAllDecls(@This()); + + const doc = Document.initDefault(std.testing.allocator); + + const input = "- [~] something dashed off\n"; + const parsed = [_]Token{ + Token{ + .type = .tickbox, + .data = "- [~]", + }, + Token{ + .type = .text, + .data = "something dashed off", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + + try doc.parse(input, std.testing.allocator); + + try expect(doc.tokens.items.len >= parsed.len); + + for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + doc.deinit(std.testing.allocator); +} + + +test "horizontal_rule_straight_line" { + std.testing.refAllDecls(@This()); + + const doc = Document.initDefault(std.testing.allocator); + + const input = "---\n"; + const parsed = [_]Token{ + Token{ + .type = .rule, + .data = "---", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + + try doc.parse(input, std.testing.allocator); + + try expect(doc.tokens.items.len >= parsed.len); + + for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + doc.deinit(std.testing.allocator); +} + + +test "horizontal_rule_double_straight_line" { + std.testing.refAllDecls(@This()); + + const doc = Document.initDefault(std.testing.allocator); + + const input = "===\n"; + const parsed = [_]Token{ + Token{ + .type = .rule, + .data = "===", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + + try doc.parse(input, std.testing.allocator); + + try expect(doc.tokens.items.len >= parsed.len); + + for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + doc.deinit(std.testing.allocator); +} + + +test "horizontal_rule_big_dotted_straight_line" { + std.testing.refAllDecls(@This()); + + const doc = Document.initDefault(std.testing.allocator); + + const input = "***\n"; + const parsed = [_]Token{ + Token{ + .type = .rule, + .data = "***", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + + try doc.parse(input, std.testing.allocator); + + try expect(doc.tokens.items.len >= parsed.len); + + for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + doc.deinit(std.testing.allocator); +} + + +test "horizontal_rule_small_dotted_line" { + std.testing.refAllDecls(@This()); + + const doc = Document.initDefault(std.testing.allocator); + + const input = "...\n"; + const parsed = [_]Token{ + Token{ + .type = .rule, + .data = "...", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + + try doc.parse(input, std.testing.allocator); + + try expect(doc.tokens.items.len >= parsed.len); + + for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + doc.deinit(std.testing.allocator); +} + + +test "horizontal_rule_squiggly_line" { + std.testing.refAllDecls(@This()); + + const doc = Document.initDefault(std.testing.allocator); + + const input = "&&&\n"; + const parsed = [_]Token{ + Token{ + .type = .rule, + .data = "&&&", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + + try doc.parse(input, std.testing.allocator); + + try expect(doc.tokens.items.len >= parsed.len); + + for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + doc.deinit(std.testing.allocator); +} + + +test "horizontal_rule_jagged_line" { + std.testing.refAllDecls(@This()); + + const doc = Document.initDefault(std.testing.allocator); + + const input = "^^^\n"; + const parsed = [_]Token{ + Token{ + .type = .rule, + .data = "^^^", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + + try doc.parse(input, std.testing.allocator); + + try expect(doc.tokens.items.len >= parsed.len); + + for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + doc.deinit(std.testing.allocator); +} + + +test "footnote" { + std.testing.refAllDecls(@This()); + + const doc = Document.initDefault(std.testing.allocator); + + const input = "sometext [^1]\n\n[^1]: footnote\n"; + const parsed = [_]Token{ + Token{ + .type = .text, + .data = "sometext ", + }, + Token{ + .type = .footnote, + .data = "[^1]", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .footnote_def, + .data = "[^1]:", + }, + Token{ + .type = .text, + .data = "footnote", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + + try doc.parse(input, std.testing.allocator); + + try expect(doc.tokens.items.len >= parsed.len); + + for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + doc.deinit(std.testing.allocator); +} + + +test "inbulit" { + std.testing.refAllDecls(@This()); + + const doc = Document.initDefault(std.testing.allocator); + + const input = "@.macro(arg, arg2)\n"; + const parsed = [_]Token{ + Token{ + .type = .inbulit, + .data = "macro", + }, + Token{ + .type = .bracket_open, + .data = "(", + }, + Token{ + .type = .macro_arg, + .data = "arg", + }, + Token{ + .type = .arg_sep, + .data = ",", + }, + Token{ + .type = .macro_arg, + .data = "arg2", + }, + Token{ + .type = .bracket_close, + .data = ")", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + + try doc.parse(input, std.testing.allocator); + + try expect(doc.tokens.items.len >= parsed.len); + + for (parsed, doc.tokens.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + doc.deinit(std.testing.allocator); +} + diff --git a/tests/test_runner.zig b/tests/test_runner.zig index 9fad28f..e1276d4 100644 --- a/tests/test_runner.zig +++ b/tests/test_runner.zig @@ -1,10 +1,15 @@ const std = @import("std"); +const Io = std.Io; + const builtin = @import("builtin"); -pub fn main() !void { +pub fn main(init: std.process.Init) !void { + const io = init.io; + var stdout_buffer: [1024]u8 = undefined; - var stdout_writer = std.fs.File.stdout().writer(&stdout_buffer); - const stdout = &stdout_writer.interface; + var stdout_file_writer: Io.File.Writer = .init(.stdout(), io, &stdout_buffer); + const stdout = &stdout_file_writer.interface; + for (builtin.test_functions) |t| { t.func() catch |err| { try stdout.print("{s} fail: {}\n", .{ t.name, err });