commit 8be321dee13d8587ddf73e68898c0a0426e3a65a Author: sirlilpanda Date: Tue Apr 14 21:31:44 2026 +1200 init diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a21bae0 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +# ---> Zig +.zig-cache/ +zig-out/ diff --git a/build.zig b/build.zig new file mode 100644 index 0000000..2f67346 --- /dev/null +++ b/build.zig @@ -0,0 +1,54 @@ +const std = @import("std"); + +pub fn build(b: *std.Build) void { + const target = b.standardTargetOptions(.{}); + const optimize = b.standardOptimizeOption(.{}); + + const mod = b.addModule("pandas_markdown", .{ + .root_source_file = b.path("src/root.zig"), + .test_runner = .{ .path = b.path("tests/test_runner.zig"), .mode = .simple }, + .target = target, + }); + + const exe = b.addExecutable(.{ + .name = "pandas_markdown", + .root_module = b.createModule(.{ + .root_source_file = b.path("src/main.zig"), + .test_runner = .{ .path = b.path("tests/test_runner.zig"), .mode = .simple }, + .target = target, + .optimize = optimize, + .imports = &.{ + .{ .name = "pandas_markdown", .module = mod }, + }, + }), + }); + + b.installArtifact(exe); + + const run_step = b.step("run", "Run the app"); + + const run_cmd = b.addRunArtifact(exe); + run_step.dependOn(&run_cmd.step); + + run_cmd.step.dependOn(b.getInstallStep()); + + if (b.args) |args| { + run_cmd.addArgs(args); + } + + const mod_tests = b.addTest(.{ + .root_module = mod, + }); + + const run_mod_tests = b.addRunArtifact(mod_tests); + + const exe_tests = b.addTest(.{ + .root_module = exe.root_module, + }); + + const run_exe_tests = b.addRunArtifact(exe_tests); + + const test_step = b.step("test", "Run tests"); + test_step.dependOn(&run_mod_tests.step); + test_step.dependOn(&run_exe_tests.step); +} diff --git a/build.zig.zon b/build.zig.zon new file mode 100644 index 0000000..dc98108 --- /dev/null +++ b/build.zig.zon @@ -0,0 +1,81 @@ +.{ + // This is the default name used by packages depending on this one. For + // example, when a user runs `zig fetch --save `, this field is used + // as the key in the `dependencies` table. Although the user can choose a + // different name, most users will stick with this provided value. + // + // It is redundant to include "zig" in this name because it is already + // within the Zig package namespace. + .name = .pandas_markdown, + // This is a [Semantic Version](https://semver.org/). + // In a future version of Zig it will be used for package deduplication. + .version = "0.0.0", + // Together with name, this represents a globally unique package + // identifier. This field is generated by the Zig toolchain when the + // package is first created, and then *never changes*. This allows + // unambiguous detection of one package being an updated version of + // another. + // + // When forking a Zig project, this id should be regenerated (delete the + // field and run `zig build`) if the upstream project is still maintained. + // Otherwise, the fork is *hostile*, attempting to take control over the + // original project's identity. Thus it is recommended to leave the comment + // on the following line intact, so that it shows up in code reviews that + // modify the field. + .fingerprint = 0xcd4f98c3fd56819e, // Changing this has security and trust implications. + // Tracks the earliest Zig version that the package considers to be a + // supported use case. + .minimum_zig_version = "0.15.1", + // This field is optional. + // Each dependency must either provide a `url` and `hash`, or a `path`. + // `zig build --fetch` can be used to fetch all dependencies of a package, recursively. + // Once all dependencies are fetched, `zig build` no longer requires + // internet connectivity. + .dependencies = .{ + // See `zig fetch --save ` for a command-line interface for adding dependencies. + //.example = .{ + // // When updating this field to a new URL, be sure to delete the corresponding + // // `hash`, otherwise you are communicating that you expect to find the old hash at + // // the new URL. If the contents of a URL change this will result in a hash mismatch + // // which will prevent zig from using it. + // .url = "https://example.com/foo.tar.gz", + // + // // This is computed from the file contents of the directory of files that is + // // obtained after fetching `url` and applying the inclusion rules given by + // // `paths`. + // // + // // This field is the source of truth; packages do not come from a `url`; they + // // come from a `hash`. `url` is just one of many possible mirrors for how to + // // obtain a package matching this `hash`. + // // + // // Uses the [multihash](https://multiformats.io/multihash/) format. + // .hash = "...", + // + // // When this is provided, the package is found in a directory relative to the + // // build root. In this case the package's hash is irrelevant and therefore not + // // computed. This field and `url` are mutually exclusive. + // .path = "foo", + // + // // When this is set to `true`, a package is declared to be lazily + // // fetched. This makes the dependency only get fetched if it is + // // actually used. + // .lazy = false, + //}, + }, + // Specifies the set of files and directories that are included in this package. + // Only files and directories listed here are included in the `hash` that + // is computed for this package. Only files listed here will remain on disk + // when using the zig package manager. As a rule of thumb, one should list + // files required for compilation plus any license(s). + // Paths are relative to the build root. Use the empty string (`""`) to refer to + // the build root itself. + // A directory listed here means that all files within, recursively, are included. + .paths = .{ + "build.zig", + "build.zig.zon", + "src", + // For example... + //"LICENSE", + //"README.md", + }, +} diff --git a/create_tests.py b/create_tests.py new file mode 100644 index 0000000..e84edaa --- /dev/null +++ b/create_tests.py @@ -0,0 +1,69 @@ +# test "parses-comment" { +# const input = "// this is a comment\n"; +# const parsed = [_]Token{ +# Token{ +# .type = .comment, +# .data = comment_id, +# }, +# Token{ +# .type = .raw_text, +# .data = "this is a comment", +# }, +# Token{ .type = .newline, .data = "\n" }, +# }; + +# var parsing = try tokenize(input, std.testing.allocator); + +# for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { +# try expect(expected_token.type == actual_token.type); +# try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +# } + +# parsing.deinit(std.testing.allocator); +# } + +import json +from pprint import pprint + +with open("tests/test.json", "r") as txt: + tests : dict = json.loads(txt.read()) + for test_name, test_value in tests["basic_unit_tests"].items(): + + tokens = "" + + for token in test_value["parsed"]: + token_type = list(token.keys())[0] + + tokens += f""" + Token{{ + .type = .{token_type}, + .data = "{token[token_type].encode("unicode_escape").decode("utf-8")}", + }},""" + + print(f""" +test "{test_name}" {{ + const input = "{test_value["input"].encode("unicode_escape").decode("utf-8")}"; + const parsed = [_]Token{{{tokens} + }}; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| {{ + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + }} + + parsing.deinit(std.testing.allocator); +}} +""") + # pprint(test_value["parsed"]) + + + + + + + + + diff --git a/find_all_tokens.py b/find_all_tokens.py new file mode 100644 index 0000000..40bab2b --- /dev/null +++ b/find_all_tokens.py @@ -0,0 +1,15 @@ +import json +from pprint import pprint + + +tokens = set() + +with open("tests/test.json", "r") as txt: + tests : dict = json.loads(txt.read()) + for test_name, test_value in tests["basic_unit_tests"].items(): + for token in test_value["parsed"]: + tokens.add(list(token.keys())[0]) + +pprint(tokens) + + diff --git a/grammer.md b/grammer.md new file mode 100644 index 0000000..7de4241 --- /dev/null +++ b/grammer.md @@ -0,0 +1,365 @@ +# meta data + + +yaml can be defined at the start of a file to define meta data +```md +--- +# yaml +--- +``` + +# headings +these are standard markdown headings + +```md +# heading 1 +## heading 2 +### heading 3 +#### heading 4 +##### heading 5 +###### heading 6 +``` + +# text and inline styles + +## text + +just some text man dont know what to tell you +``` +plain test +``` + +## inline styles +all inline styles must start with a space and end with a space +however these are just the defualt ones and can be changed within the meta data as well as the ablity to add custom ones + +### code +```md +`plain test` +``` + +### bold +```md +!plain test! +``` + +### italic +```md +*plain test* +``` + +### underline +```md +_plain test_ +``` + +### strikethrough +```md +~plain test~ +``` + +### highlighted +```md +|plain test| +``` + +all style chars: +``` +named : `!*_~| +unamed : &%$#-+= +``` + +### super script +```md +sometext^{} +``` +### sub script +```md +sometext_2 +``` + +# lists + + +## ordered +```md +1. ordered list +2. ordered list +3. ordered list +``` + +```md +i. ordered list +ii. ordered list +iii. ordered list +``` + +```md +a. ordered list +b. ordered list +c. ordered list +``` + +## unordered + +```md +- unordered list +- unordered list +- unordered list +``` + +```md += unordered list += unordered list += unordered list +``` + +```md ++ unordered list ++ unordered list ++ unordered list +``` + +```md +* unordered list +* unordered list +* unordered list +``` + +# embeds + +```md +![figure text](relitive path to image) +``` + +```md +#[table text](relitive path to tabulated data) +``` + +```md +$[file text](path to file#starting_line:ending_line) +``` + +embed file chars: +``` +named : !#$ +unamed : *_~|&%-+= +``` + +```md +/[file text](link to something) +``` + +# tables + +``` +| col | col | col | +| --- | --- | --- | +| row | row | row | +| row | row | row | +| row | row | row | +| row | row | row | +| --- | --- | --- | +| col | col | col | +| --- | --- | --- | +| row | row | row | +| row | row | row | +| row | row | row | +| row | row | row | +``` + +# quote blocks + +```md +> quote +> quote +> quote +``` + +```md +| quote +| quote +| quote +``` + +# text/code blocks + +``` +\``` +some text or code +\``` +``` + +``` +\```lang_name +some text or code +\``` +``` + +the passthough lang name will just place what is in the box within the output +``` +\```passthough +some text or code +\``` +``` + +# comments + +everything after the commnet identifier is a comment until the next line +```md +// this is a comment +``` + +# tick box +```md +- [ ] something to check off +- [x] something checked off +- [/] something ticked off +- [~] something dashed off +``` + +# horrazontal rules +```md +--- // straight line +=== // double straight line +*** // big dotted straight line +... // small dotted line +&&& // squiggly line +^^^ // jagged line +``` + +# footnote + +```md +sometext [^1] + +[^1]: footnote +``` + +# advanced features + +## custom styles +any of the above features can have style overrides +it beings with a . to differnate it from other functions +```md +@.styles() +# heading 7 +``` + +## citations + +```md +--- +refs: refs.bib +--- + +@cite(panda) + +@bib() +``` + +## replacements + +```md +--- +replacement: + toads: "frogs" +--- + +@toads +``` +--> +``` +frogs +``` + +## replacement macros +```md +--- +macros: + hello: + args: + name : "str" + age : "number" + code: + "hello my name is {name} and i am {age} years old" +--- + + +@hello("panda", 240) +``` +--> +``` +hello my name is panda and i am 240 years old +``` + +## numerical macros + +``` +--- +macros: + add: + args: + num1 : "number" + num2 : "number" + eq: + num1 + num2 +--- + +@add(1, 2) +``` +--> +``` +3 +``` + +## cli macros + +``` +--- +macros: + generate_figure: + args: + filename : "str" + cmd: + python generate_figure.py {filename} + post_process: + "![figure of data from {filename}]({out})" +--- + +@generate_figure("data/run2.csv") +``` +--> +``` +![figure of data from data/run2.csv](figure.png) +``` + + +## heading id override + +```md +@.id(heading-1-overide) +@.style() +# heading 1 + +@.id(heading-2-overide) +@.style() +## heading 2 + +@.id(heading-3-overide) +@.style() +### heading 3 + +@.id(heading-4-overide) +@.style() +#### heading 4 + +@.id(heading-5-overide) +@.style() +##### heading 5 + +@.id(heading-6-overide) +@.style() +###### heading 6 + +``` \ No newline at end of file diff --git a/how_it_works.md b/how_it_works.md new file mode 100644 index 0000000..24391db --- /dev/null +++ b/how_it_works.md @@ -0,0 +1,23 @@ +1. first comments are stripped out of the line + well they are actually still there if you want to add them in to your output + +2. multiline elements are processed + +3. any tags that begining a line are stripped and added to the token list + +4. the inline text elements are then run + +1. soemthing else + + this is to make sure the child elements are correct + + # you can have heading in here + +2. more shit + 1. something + + + +sometext + someother text + even more text \ No newline at end of file diff --git a/image.png b/image.png new file mode 100644 index 0000000..0f630f2 Binary files /dev/null and b/image.png differ diff --git a/readme.md b/readme.md new file mode 100644 index 0000000..74ea92e --- /dev/null +++ b/readme.md @@ -0,0 +1,20 @@ + + + + +how to run the tests in a cleaner manner +``` bash +zig test --test-runner tests/test_runner.zig src/document.zig +``` + + + +# todo + +- [ ] possible ENBF grammer +- [ ] add multistyle tests +- [ ] add the style chars inbetween the style char markers test +- [ ] add comment tests on all +- [ ] add multilevel nested list test +- [ ] add bulk everything in one test +- [ ] styles within tables test \ No newline at end of file diff --git a/src/document.zig b/src/document.zig new file mode 100644 index 0000000..3b54f33 --- /dev/null +++ b/src/document.zig @@ -0,0 +1,3069 @@ +//! By convention, root.zig is the root source file when making a library. +const std = @import("std"); + +pub fn stripWhiteSpace(white_space: []const u8) []const u8 { + return std.mem.trimStart(u8, white_space, " \t"); +} + +pub const TokenType = enum { + // macros + inbulit, + macro, + macro_arg, + arg_sep, + replace, + unknown, // this is the defualt token type and should never appear + + // utils + bracket_close, + bracket_open, + raw_text, + newline, + + comment, + + // in text + heading, + text, + inline_style, + unordered_list, + ordered_list, + embed, + link, + + table_row_sep, + table_sep, + quote, + + code_block, + + tickbox, + rule, + + footnote, + footnote_def, + + sup, + sub, +}; + +pub const Token = struct { + data: []const u8 = "", + type: TokenType = .unknown, + padding: usize = 0, + // this is to allow the ast generation to make sure that things are + // within the right subtrees + pub fn format( + self: Token, + writer: *std.io.Writer, + ) !void { + switch (self.type) { + .newline => try writer.print( + ".{{\n .data = \"\\n\"\n .type = {}\n}}", + .{self.type}, + ), + else => try writer.print( + ".{{\n .data = \"{s}\"\n .type = {}\n}}", + .{ self.data, self.type }, + ), + } + } +}; + +pub const LineParserReturn = struct { + // the new updated pos of the token after its been parsed + pointer_offset: usize = 0, + // the token itself + token: Token = .{}, +}; +/// lineParser parse chunk of the line then return the rest +pub const LineParser = *const fn (line: []const u8) ?LineParserReturn; + +/// multi line parsers take in the lines and the list and add the tokens +/// to the list itself +pub const MultiLineParser = *const fn ( + lines: *std.mem.SplitIterator(u8, .scalar), + alloc: std.mem.Allocator, + list: *std.ArrayList(Token), +) anyerror!void; + +const heading_id = "#"; +const heading_depth: comptime_int = 6; +pub fn headingParser(line: []const u8) ?LineParserReturn { + const heading_end_index: usize = blk: { + comptime var i = heading_depth; + inline while (i > 0) : (i -= 1) { + if (std.mem.startsWith(u8, line, heading_id ** i)) break :blk i; + } + break :blk 0; + }; + + // if no heading was detected + if (heading_end_index == 0) { + return null; + } + + // allows for blank headings + const space_remove: usize = + if (line.len == heading_end_index) 0 else 1; + + return LineParserReturn{ + .pointer_offset = heading_end_index + space_remove, + .token = .{ + .type = .heading, + .data = line[0..heading_end_index], + }, + }; +} + +pub fn text(line: []const u8) ?LineParserReturn { + return LineParserReturn{ + .pointer_offset = line.len, + .token = .{ + .data = line[0..], + .type = .text, + }, + }; +} + +pub fn listOrderedParser(line: []const u8) ?LineParserReturn { + const clean_line = stripWhiteSpace(line); + const whitespace_chars_removed = @intFromPtr(clean_line.ptr) - @intFromPtr(line.ptr); + // check for the min length of chars + if (clean_line.len < 2) return null; + + // find a point within the line + const index_of_dot = blk: { + if (std.mem.indexOf(u8, clean_line, ". ")) |dex| { + break :blk dex; + } + return null; + }; + + const number = clean_line[0..index_of_dot]; + // if it has any spaces its not a order list id + if (std.mem.containsAtLeastScalar(u8, number, 1, ' ') or + std.mem.containsAtLeastScalar(u8, number, 1, '\t')) + { + return null; + } + + // now work out what the number is + // "I": 1, + // "V": 5, + // "X": 10, + // "L": 50, + // "C": 100, + // "D": 500, + // "M": 1000 + + // roman numerials ill deal with these later + + // these can be the same + // numbers + + // letters & roman (at least all the i's) + if (std.ascii.isAlphabetic(number[0])) { + return LineParserReturn{ + // + 2 to remove the dot and space + // + 1 to make sure the dot is the last char + .pointer_offset = whitespace_chars_removed + number.len + 2, + .token = .{ + .data = line[0 .. whitespace_chars_removed + number.len + 1], + .type = .ordered_list, + }, + }; + } + + // checks to see if the number can be parsed if it can then its a number + _ = std.fmt.parseInt(usize, number, 10) catch return null; + + return LineParserReturn{ + // + 2 to remove the dot and space + // + 1 to make sure the dot is the last char + .pointer_offset = whitespace_chars_removed + number.len + 2, + .token = .{ + .data = line[0 .. whitespace_chars_removed + number.len + 1], + .type = .ordered_list, + }, + }; +} + +pub fn listUnorderedParser(line: []const u8) ?LineParserReturn { + const clean_line = stripWhiteSpace(line); + const whitespace_chars_removed = @intFromPtr(clean_line.ptr) - @intFromPtr(line.ptr); + // check for the min length of chars + if (clean_line.len < 2) return null; + + // check there is a space seperating the char and the text + if (clean_line[1] != ' ') return null; + + if (clean_line[0] == '-' or clean_line[0] == '=' or clean_line[0] == '+' or clean_line[0] == '*') { + return LineParserReturn{ + // +2 to remove "-. " + .pointer_offset = whitespace_chars_removed + 2, + .token = .{ + .data = clean_line[0..1], + .type = .unordered_list, + }, + }; + } + + return null; +} + +// pub fn styleParser(line: []const u8) ?LineParserReturn {} +// pub fn embedParser(line: []const u8) ?LineParserReturn {} + +// pub fn tableParser(line: []const u8) ?LineParserReturn {} + +pub fn quoteParser(line: []const u8) ?LineParserReturn { + if (std.mem.startsWith(u8, line, "| ") or + std.mem.startsWith(u8, line, "> ")) + { + return LineParserReturn{ .pointer_offset = 2, .token = .{ + .data = line[0..1], + .type = .quote, + } }; + } + + return null; +} + +// // multiline might have to add some extra to this + +pub fn codeBlockParser( + current_line: []const u8, + lines: *std.mem.SplitIterator(u8, .scalar), + alloc: std.mem.Allocator, + list: *std.ArrayList(Token), +) anyerror!bool { + if (!std.mem.startsWith(u8, current_line, "```")) { + // std.debug.print("line didnt start with ```\n", .{}); + return false; + } + try list.append(alloc, .{ + .data = "```", + .type = .code_block, + }); + + const lang_name = std.mem.trimEnd( + u8, + current_line[3..], + " \t\n", + ); + // std.debug.print("lang_name = {}\n", .{lang_name.len}); + if (lang_name.len != 0) { + try list.append(alloc, .{ + .data = lang_name, + .type = .raw_text, + }); + } + + try list.append(alloc, .{ + .data = "\n", + .type = .newline, + }); + + // std.debug.print("first values\n", .{}); + // +1 to remove the newline + const raw_text_start_index = + @intFromPtr(current_line.ptr) - @intFromPtr(lines.buffer.ptr) + lang_name.len + 3 + 1; + // std.debug.print("raw_text_start_index : {}\n", .{raw_text_start_index}); + + const ending_line = find_end: { + while (lines.next()) |line| { + if (std.mem.startsWith(u8, line, "```")) + break :find_end line; + } else { + return error.No_closing_tag; + } + }; + + const raw_text_end_index = @intFromPtr(ending_line.ptr) - @intFromPtr(lines.buffer.ptr) - 1; + // std.debug.print("raw_text_end_index : {}\n", .{raw_text_end_index}); + + try list.append(alloc, .{ + .data = lines.buffer[raw_text_start_index..raw_text_end_index], + .type = .raw_text, + }); + try list.append(alloc, .{ + .data = "\n", + .type = .newline, + }); + + try list.append(alloc, .{ + .data = ending_line, + .type = .code_block, + }); + + // for (list.items) |value| { + // std.debug.print("t : {f}\n", .{value}); + // } + + return true; +} + +// pub fn codeBlockParser(line: []const u8) ?LineParserReturn {} + +pub fn tickBoxParser(line: []const u8) ?LineParserReturn { + if (!std.mem.startsWith(u8, line, "- [")) { + return null; + } + // not long enough for a tick box + if (line.len < 6) return null; + + // - [ + if (line[3] == ' ' or + line[3] == 'x' or + line[3] == '/' or + line[3] == '~' and + line[4] == ']' and line[5] == ' ') + { + return LineParserReturn{ + .pointer_offset = 6, + .token = .{ + .type = .tickbox, + .data = line[0..5], + }, + }; + } + return null; +} + +// might make this anylength + +pub fn ruleCheck(comptime rule_char: []const u8, line: []const u8) usize { + if (!std.mem.startsWith(u8, line, rule_char ** 3)) return 0; + for (line, 0..) |char, i| { + if (char == rule_char[0] or char == ' ' or char == '\t') + continue + else + return i; + } + return line.len; +} + +pub fn horizontalRuleParser(line: []const u8) ?LineParserReturn { + const rule_length = ruleCheck("-", line) + + ruleCheck("=", line) + + ruleCheck("*", line) + + ruleCheck(".", line) + + ruleCheck("&", line) + + ruleCheck("^", line); + + if (rule_length == 0) return null; + + return LineParserReturn{ + .pointer_offset = rule_length, + .token = .{ + .data = line[0..rule_length], + .type = .rule, + }, + }; +} + +// pub fn footnoteParser(line: []const u8) ?LineParserReturn {} + +// pub fn MacroParser(line: []const u8) ?LineParserReturn {} + +const comment_id = "// "; +pub fn commentParser(line: []const u8) ?LineParserReturn { + + // find the index of the comment + const index_of_commnet = blk: { + if (std.mem.indexOf(u8, line, comment_id)) |index| { + if (index == 0) + break :blk index; + + if (line[index - 1] != '\\') { + return null; + } + + break :blk index; + } + return null; + }; + + return LineParserReturn{ + .pointer_offset = index_of_commnet + comment_id.len, + .token = .{ + .type = .comment, + .data = line[index_of_commnet .. index_of_commnet + comment_id.len], + }, + }; +} + +// const LineParsers : [_]LineParser = {}; + +pub fn tokenize( + // comptime start_parsers : []LineParser, + // comptime text_parsers : []LineParser, + // comptime end_parsers : []LineParser, + // comptime multiline_parsers : []MultiLineParser, + str: []const u8, + alloc: std.mem.Allocator, +) !std.ArrayList(Token) { + var lines = std.mem.splitScalar(u8, str, '\n'); + var token_list: std.ArrayList(Token) = .empty; + while (lines.next()) |current_line| { + var line = current_line[0..]; + // std.debug.print("line : \"{s}\"\n", .{line}); + + if (try codeBlockParser(current_line, &lines, alloc, &token_list)) continue; + + // start end parsings + const comment_token = commentParser(line); + + if (comment_token) |token| { + // sets the end of the line to just before the comment + std.debug.print("comment_token.pointer_offset : {}\n", .{token.pointer_offset - token.token.data.len}); + line = line[0 .. token.pointer_offset - token.token.data.len]; + } + + // actual parsing + + // multiline parsing + + // start parsing + const start_token: ?LineParserReturn = blk: { + if (headingParser(line)) |t| { + break :blk t; + } + if (tickBoxParser(line)) |t| { + break :blk t; + } + if (listOrderedParser(line)) |t| { + break :blk t; + } + if (listUnorderedParser(line)) |t| { + break :blk t; + } + if (quoteParser(line)) |t| { + break :blk t; + } + if (horizontalRuleParser(line)) |t| { + break :blk t; + } + break :blk null; + }; + + if (start_token) |val| { + line = line[val.pointer_offset..]; + try token_list.append(alloc, val.token); + } + + // mid parsing + if (line.len != 0) { // makes sure that i dont add blank lines + const mid_token: ?LineParserReturn = blk: { + if (text(line)) |t| { + break :blk t; + } + break :blk null; + }; + + if (mid_token) |token| { + try token_list.append(alloc, token.token); + } + } + + if (comment_token) |token| { + // append the comment token + try token_list.append(alloc, token.token); + try token_list.append(alloc, .{ + .type = .raw_text, + .data = current_line[token.pointer_offset..], + }); + } + + try token_list.append(alloc, .{ + .type = .newline, + // make sure its a pointer to the actual char + .data = "\n", + }); + } + + return token_list; +} + +const expect = std.testing.expect; + +test "heading_1" { + const input = "# hello this is a heading\n"; + const parsed = [_]Token{ + Token{ + .type = .heading, + .data = "#", + }, + Token{ + .type = .text, + .data = "hello this is a heading", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + // for (parsing.items) |value| { + // std.debug.print("{f}\n", .{value}); + // } + + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + +test "heading_2" { + const input = "## hello this is a heading\n"; + const parsed = [_]Token{ + Token{ + .type = .heading, + .data = "##", + }, + Token{ + .type = .text, + .data = "hello this is a heading", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + +test "heading_3" { + const input = "### hello this is a heading\n"; + const parsed = [_]Token{ + Token{ + .type = .heading, + .data = "###", + }, + Token{ + .type = .text, + .data = "hello this is a heading", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + +test "heading_4" { + const input = "#### hello this is a heading\n"; + const parsed = [_]Token{ + Token{ + .type = .heading, + .data = "####", + }, + Token{ + .type = .text, + .data = "hello this is a heading", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + +test "heading_5" { + const input = "##### hello this is a heading\n"; + const parsed = [_]Token{ + Token{ + .type = .heading, + .data = "#####", + }, + Token{ + .type = .text, + .data = "hello this is a heading", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + +test "heading_6" { + const input = "###### hello this is a heading\n"; + const parsed = [_]Token{ + Token{ + .type = .heading, + .data = "######", + }, + Token{ + .type = .text, + .data = "hello this is a heading", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + +test "header_no_text" { + const input = "######\n"; + const parsed = [_]Token{ + Token{ + .type = .heading, + .data = "######", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + +test "text" { + const input = "plain test\n"; + const parsed = [_]Token{ + Token{ + .type = .text, + .data = "plain test", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + +test "text_style_code" { + const input = "`plain test`\n"; + const parsed = [_]Token{ + Token{ + .type = .inline_style, + .data = "`", + }, + Token{ + .type = .text, + .data = "plain test", + }, + Token{ + .type = .inline_style, + .data = "`", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + +test "text_style_bold" { + const input = "!plain test!\n"; + const parsed = [_]Token{ + Token{ + .type = .inline_style, + .data = "!", + }, + Token{ + .type = .text, + .data = "plain test", + }, + Token{ + .type = .inline_style, + .data = "!", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + +test "text_style_italic" { + const input = "*plain test*\n"; + const parsed = [_]Token{ + Token{ + .type = .inline_style, + .data = "*", + }, + Token{ + .type = .text, + .data = "plain test", + }, + Token{ + .type = .inline_style, + .data = "*", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + +test "text_style_underline" { + const input = "_plain test_\n"; + const parsed = [_]Token{ + Token{ + .type = .inline_style, + .data = "_", + }, + Token{ + .type = .text, + .data = "plain test", + }, + Token{ + .type = .inline_style, + .data = "_", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + +test "text_style_strikethrough" { + const input = "~plain test~\n"; + const parsed = [_]Token{ + Token{ + .type = .inline_style, + .data = "~", + }, + Token{ + .type = .text, + .data = "plain test", + }, + Token{ + .type = .inline_style, + .data = "~", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + +test "text_style_highlighted" { + const input = "|plain test|\n"; + const parsed = [_]Token{ + Token{ + .type = .inline_style, + .data = "|", + }, + Token{ + .type = .text, + .data = "plain test", + }, + Token{ + .type = .inline_style, + .data = "|", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + +test "text_style_&" { + const input = "&plain test&\n"; + const parsed = [_]Token{ + Token{ + .type = .inline_style, + .data = "&", + }, + Token{ + .type = .text, + .data = "plain test", + }, + Token{ + .type = .inline_style, + .data = "&", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + +test "text_style_%" { + const input = "%plain test%\n"; + const parsed = [_]Token{ + Token{ + .type = .inline_style, + .data = "%", + }, + Token{ + .type = .text, + .data = "plain test", + }, + Token{ + .type = .inline_style, + .data = "%", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + +test "text_style_$" { + const input = "$plain test$\n"; + const parsed = [_]Token{ + Token{ + .type = .inline_style, + .data = "$", + }, + Token{ + .type = .text, + .data = "plain test", + }, + Token{ + .type = .inline_style, + .data = "$", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + +test "text_style_#" { + const input = "#plain test#\n"; + const parsed = [_]Token{ + Token{ + .type = .inline_style, + .data = "#", + }, + Token{ + .type = .text, + .data = "plain test", + }, + Token{ + .type = .inline_style, + .data = "#", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + +test "text_style_-" { + const input = "-plain test-\n"; + const parsed = [_]Token{ + Token{ + .type = .inline_style, + .data = "-", + }, + Token{ + .type = .text, + .data = "plain test", + }, + Token{ + .type = .inline_style, + .data = "-", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + +test "text_style_+" { + const input = "+plain test+\n"; + const parsed = [_]Token{ + Token{ + .type = .inline_style, + .data = "+", + }, + Token{ + .type = .text, + .data = "plain test", + }, + Token{ + .type = .inline_style, + .data = "+", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + +test "text_style_=" { + const input = "=plain test=\n"; + const parsed = [_]Token{ + Token{ + .type = .inline_style, + .data = "=", + }, + Token{ + .type = .text, + .data = "plain test", + }, + Token{ + .type = .inline_style, + .data = "=", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + +test "text_style_super_script" { + const input = "sometext^2\n"; + const parsed = [_]Token{ + Token{ + .type = .text, + .data = "sometext", + }, + Token{ + .type = .sup, + .data = "^", + }, + Token{ + .type = .text, + .data = "2", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + +test "text_style_sub_script" { + const input = "sometext_2\n"; + const parsed = [_]Token{ + Token{ + .type = .text, + .data = "sometext", + }, + Token{ + .type = .sub, + .data = "_", + }, + Token{ + .type = .text, + .data = "2", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + +test "list_ordered_number" { + const input = "1. ordered list\n2. ordered list\n3. ordered list\n"; + const parsed = [_]Token{ + Token{ + .type = .ordered_list, + .data = "1.", + }, + Token{ + .type = .text, + .data = "ordered list", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .ordered_list, + .data = "2.", + }, + Token{ + .type = .text, + .data = "ordered list", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .ordered_list, + .data = "3.", + }, + Token{ + .type = .text, + .data = "ordered list", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + // for (parsing.items) |value| { + // std.debug.print("token : {f}\n", .{value}); + // } + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + +test "list_ordered_i" { + const input = "i. ordered list\nii. ordered list\niii. ordered list\n"; + const parsed = [_]Token{ + Token{ + .type = .ordered_list, + .data = "i.", + }, + Token{ + .type = .text, + .data = "ordered list", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .ordered_list, + .data = "ii.", + }, + Token{ + .type = .text, + .data = "ordered list", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .ordered_list, + .data = "iii.", + }, + Token{ + .type = .text, + .data = "ordered list", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + +test "list_ordered_letter" { + const input = "a. ordered list\nb. ordered list\nc. ordered list\n"; + const parsed = [_]Token{ + Token{ + .type = .ordered_list, + .data = "a.", + }, + Token{ + .type = .text, + .data = "ordered list", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .ordered_list, + .data = "b.", + }, + Token{ + .type = .text, + .data = "ordered list", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .ordered_list, + .data = "c.", + }, + Token{ + .type = .text, + .data = "ordered list", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + +test "list_unordered_dashed" { + const input = "- unordered list\n- unordered list\n- unordered list\n"; + const parsed = [_]Token{ + Token{ + .type = .unordered_list, + .data = "-", + }, + Token{ + .type = .text, + .data = "unordered list", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .unordered_list, + .data = "-", + }, + Token{ + .type = .text, + .data = "unordered list", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .unordered_list, + .data = "-", + }, + Token{ + .type = .text, + .data = "unordered list", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + // for (parsing.items) |value| { + // std.debug.print("token : {f}\n", .{value}); + // } + try expect(parsing.items.len >= parsed.len); + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + +test "list_unordered_eq" { + const input = "= unordered list\n= unordered list\n= unordered list\n"; + const parsed = [_]Token{ + Token{ + .type = .unordered_list, + .data = "=", + }, + Token{ + .type = .text, + .data = "unordered list", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .unordered_list, + .data = "=", + }, + Token{ + .type = .text, + .data = "unordered list", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .unordered_list, + .data = "=", + }, + Token{ + .type = .text, + .data = "unordered list", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + +test "list_unordered_plus" { + const input = "+ unordered list\n+ unordered list\n+ unordered list\n"; + const parsed = [_]Token{ + Token{ + .type = .unordered_list, + .data = "+", + }, + Token{ + .type = .text, + .data = "unordered list", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .unordered_list, + .data = "+", + }, + Token{ + .type = .text, + .data = "unordered list", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .unordered_list, + .data = "+", + }, + Token{ + .type = .text, + .data = "unordered list", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + +test "list_unordered_dot" { + const input = "* unordered list\n* unordered list\n* unordered list\n"; + const parsed = [_]Token{ + Token{ + .type = .unordered_list, + .data = "*", + }, + Token{ + .type = .text, + .data = "unordered list", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .unordered_list, + .data = "*", + }, + Token{ + .type = .text, + .data = "unordered list", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .unordered_list, + .data = "*", + }, + Token{ + .type = .text, + .data = "unordered list", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + +test "embed_image" { + const input = "![figure text](relitive path to image)\n"; + const parsed = [_]Token{ + Token{ + .type = .embed, + .data = "!", + }, + Token{ + .type = .bracket_open, + .data = "[", + }, + Token{ + .type = .text, + .data = "figure text", + }, + Token{ + .type = .bracket_close, + .data = "]", + }, + Token{ + .type = .bracket_open, + .data = "(", + }, + Token{ + .type = .raw_text, + .data = "relitive path to image", + }, + Token{ + .type = .bracket_close, + .data = ")", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + +test "embed_table" { + const input = "#[table text](relitive path to tabulated data)\n"; + const parsed = [_]Token{ + Token{ + .type = .embed, + .data = "#", + }, + Token{ + .type = .bracket_open, + .data = "[", + }, + Token{ + .type = .text, + .data = "table text", + }, + Token{ + .type = .bracket_close, + .data = "]", + }, + Token{ + .type = .bracket_open, + .data = "(", + }, + Token{ + .type = .raw_text, + .data = "relitive path to tabulated data", + }, + Token{ + .type = .bracket_close, + .data = ")", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + +test "embed_file" { + const input = "$[file text](path to file#starting_line:ending_line)\n"; + const parsed = [_]Token{ + Token{ + .type = .embed, + .data = "$", + }, + Token{ + .type = .bracket_open, + .data = "[", + }, + Token{ + .type = .text, + .data = "file text", + }, + Token{ + .type = .bracket_close, + .data = "]", + }, + Token{ + .type = .bracket_open, + .data = "(", + }, + Token{ + .type = .raw_text, + .data = "path to file#starting_line:ending_line", + }, + Token{ + .type = .bracket_close, + .data = ")", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + +test "embed_*" { + const input = "*[file text](file_path)\n"; + const parsed = [_]Token{ + Token{ + .type = .embed, + .data = "*", + }, + Token{ + .type = .bracket_open, + .data = "[", + }, + Token{ + .type = .text, + .data = "file text", + }, + Token{ + .type = .bracket_close, + .data = "]", + }, + Token{ + .type = .bracket_open, + .data = "(", + }, + Token{ + .type = .raw_text, + .data = "file_path", + }, + Token{ + .type = .bracket_close, + .data = ")", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + +test "embed__" { + const input = "_[file text](file_path)\n"; + const parsed = [_]Token{ + Token{ + .type = .embed, + .data = "_", + }, + Token{ + .type = .bracket_open, + .data = "[", + }, + Token{ + .type = .text, + .data = "file text", + }, + Token{ + .type = .bracket_close, + .data = "]", + }, + Token{ + .type = .bracket_open, + .data = "(", + }, + Token{ + .type = .raw_text, + .data = "file_path", + }, + Token{ + .type = .bracket_close, + .data = ")", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + +test "embed_~" { + const input = "~[file text](file_path)\n"; + const parsed = [_]Token{ + Token{ + .type = .embed, + .data = "~", + }, + Token{ + .type = .bracket_open, + .data = "[", + }, + Token{ + .type = .text, + .data = "file text", + }, + Token{ + .type = .bracket_close, + .data = "]", + }, + Token{ + .type = .bracket_open, + .data = "(", + }, + Token{ + .type = .raw_text, + .data = "file_path", + }, + Token{ + .type = .bracket_close, + .data = ")", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + +test "embed_|" { + const input = "|[file text](file_path)\n"; + const parsed = [_]Token{ + Token{ + .type = .embed, + .data = "|", + }, + Token{ + .type = .bracket_open, + .data = "[", + }, + Token{ + .type = .text, + .data = "file text", + }, + Token{ + .type = .bracket_close, + .data = "]", + }, + Token{ + .type = .bracket_open, + .data = "(", + }, + Token{ + .type = .raw_text, + .data = "file_path", + }, + Token{ + .type = .bracket_close, + .data = ")", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + +test "embed_&" { + const input = "&[file text](file_path)\n"; + const parsed = [_]Token{ + Token{ + .type = .embed, + .data = "&", + }, + Token{ + .type = .bracket_open, + .data = "[", + }, + Token{ + .type = .text, + .data = "file text", + }, + Token{ + .type = .bracket_close, + .data = "]", + }, + Token{ + .type = .bracket_open, + .data = "(", + }, + Token{ + .type = .raw_text, + .data = "file_path", + }, + Token{ + .type = .bracket_close, + .data = ")", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + +test "embed_%" { + const input = "%[file text](file_path)\n"; + const parsed = [_]Token{ + Token{ + .type = .embed, + .data = "%", + }, + Token{ + .type = .bracket_open, + .data = "[", + }, + Token{ + .type = .text, + .data = "file text", + }, + Token{ + .type = .bracket_close, + .data = "]", + }, + Token{ + .type = .bracket_open, + .data = "(", + }, + Token{ + .type = .raw_text, + .data = "file_path", + }, + Token{ + .type = .bracket_close, + .data = ")", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + +test "embed_-" { + const input = "-[file text](file_path)\n"; + const parsed = [_]Token{ + Token{ + .type = .embed, + .data = "-", + }, + Token{ + .type = .bracket_open, + .data = "[", + }, + Token{ + .type = .text, + .data = "file text", + }, + Token{ + .type = .bracket_close, + .data = "]", + }, + Token{ + .type = .bracket_open, + .data = "(", + }, + Token{ + .type = .raw_text, + .data = "file_path", + }, + Token{ + .type = .bracket_close, + .data = ")", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + +test "embed_+" { + const input = "+[file text](file_path)\n"; + const parsed = [_]Token{ + Token{ + .type = .embed, + .data = "+", + }, + Token{ + .type = .bracket_open, + .data = "[", + }, + Token{ + .type = .text, + .data = "file text", + }, + Token{ + .type = .bracket_close, + .data = "]", + }, + Token{ + .type = .bracket_open, + .data = "(", + }, + Token{ + .type = .raw_text, + .data = "file_path", + }, + Token{ + .type = .bracket_close, + .data = ")", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + +test "embed_=" { + const input = "=[file text](file_path)\n"; + const parsed = [_]Token{ + Token{ + .type = .embed, + .data = "=", + }, + Token{ + .type = .bracket_open, + .data = "[", + }, + Token{ + .type = .text, + .data = "file text", + }, + Token{ + .type = .bracket_close, + .data = "]", + }, + Token{ + .type = .bracket_open, + .data = "(", + }, + Token{ + .type = .raw_text, + .data = "file_path", + }, + Token{ + .type = .bracket_close, + .data = ")", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + +test "embed_link" { + const input = "/[file text](link to something)\n"; + const parsed = [_]Token{ + Token{ + .type = .link, + .data = "/", + }, + Token{ + .type = .bracket_open, + .data = "[", + }, + Token{ + .type = .text, + .data = "file text", + }, + Token{ + .type = .bracket_close, + .data = "]", + }, + Token{ + .type = .bracket_open, + .data = "(", + }, + Token{ + .type = .raw_text, + .data = "link to something", + }, + Token{ + .type = .bracket_close, + .data = ")", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + +// test "table" { +// const input = "| col | col | col |\n| --- | --- | --- |\n| row | row | row |\n| row | row | row |\n| row | row | row |\n| row | row | row |\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .table_sep, +// .data = "|", +// }, +// Token{ +// .type = .text, +// .data = " col ", +// }, +// Token{ +// .type = .table_sep, +// .data = "|", +// }, +// Token{ +// .type = .text, +// .data = " col ", +// }, +// Token{ +// .type = .table_sep, +// .data = "|", +// }, +// Token{ +// .type = .text, +// .data = " col ", +// }, +// Token{ +// .type = .table_sep, +// .data = "|", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// Token{ +// .type = .table_sep, +// .data = "|", +// }, +// Token{ +// .type = .table_row_sep, +// .data = " --- ", +// }, +// Token{ +// .type = .table_sep, +// .data = "|", +// }, +// Token{ +// .type = .table_row_sep, +// .data = " --- ", +// }, +// Token{ +// .type = .table_sep, +// .data = "|", +// }, +// Token{ +// .type = .table_row_sep, +// .data = " --- ", +// }, +// Token{ +// .type = .table_sep, +// .data = "|", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// Token{ +// .type = .table_sep, +// .data = "|", +// }, +// Token{ +// .type = .text, +// .data = " row ", +// }, +// Token{ +// .type = .table_sep, +// .data = "|", +// }, +// Token{ +// .type = .text, +// .data = " row ", +// }, +// Token{ +// .type = .table_sep, +// .data = "|", +// }, +// Token{ +// .type = .text, +// .data = " row ", +// }, +// Token{ +// .type = .table_sep, +// .data = "|", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// Token{ +// .type = .table_sep, +// .data = "|", +// }, +// Token{ +// .type = .text, +// .data = " row ", +// }, +// Token{ +// .type = .table_sep, +// .data = "|", +// }, +// Token{ +// .type = .text, +// .data = " row ", +// }, +// Token{ +// .type = .table_sep, +// .data = "|", +// }, +// Token{ +// .type = .text, +// .data = " row ", +// }, +// Token{ +// .type = .table_sep, +// .data = "|", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// Token{ +// .type = .table_sep, +// .data = "|", +// }, +// Token{ +// .type = .text, +// .data = " row ", +// }, +// Token{ +// .type = .table_sep, +// .data = "|", +// }, +// Token{ +// .type = .text, +// .data = " row ", +// }, +// Token{ +// .type = .table_sep, +// .data = "|", +// }, +// Token{ +// .type = .text, +// .data = " row ", +// }, +// Token{ +// .type = .table_sep, +// .data = "|", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// Token{ +// .type = .table_sep, +// .data = "|", +// }, +// Token{ +// .type = .text, +// .data = " row ", +// }, +// Token{ +// .type = .table_sep, +// .data = "|", +// }, +// Token{ +// .type = .text, +// .data = " row ", +// }, +// Token{ +// .type = .table_sep, +// .data = "|", +// }, +// Token{ +// .type = .text, +// .data = " row ", +// }, +// Token{ +// .type = .table_sep, +// .data = "|", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// }; + +// var parsing = try tokenize(input, std.testing.allocator); +// try expect(parsing.items.len >= parsed.len); + +// for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// parsing.deinit(std.testing.allocator); +// } + +test "quote_arrow" { + const input = "> quote\n> quote\n> quote\n"; + const parsed = [_]Token{ + Token{ + .type = .quote, + .data = ">", + }, + Token{ + .type = .text, + .data = "quote", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .quote, + .data = ">", + }, + Token{ + .type = .text, + .data = "quote", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .quote, + .data = ">", + }, + Token{ + .type = .text, + .data = "quote", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + // for (parsing.items) |value| { + // std.debug.print("parsing : {f}\n", .{value}); + // } + // for (parsed) |value| { + // std.debug.print("parsed : {f}\n", .{value}); + // } + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + // std.debug.print("expected_token : {f}\n", .{expected_token}); + // std.debug.print("actual_token : {f}\n", .{actual_token}); + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + +test "quote_bar" { + const input = "| quote\n| quote\n| quote\n"; + const parsed = [_]Token{ + Token{ + .type = .quote, + .data = "|", + }, + Token{ + .type = .text, + .data = "quote", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .quote, + .data = "|", + }, + Token{ + .type = .text, + .data = "quote", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .quote, + .data = "|", + }, + Token{ + .type = .text, + .data = "quote", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + // std.debug.print("expected_token : {f}\n", .{expected_token}); + // std.debug.print("actual_token : {f}\n", .{actual_token}); + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + +test "text_block" { + const input = "```\nsome text or code\n```\n"; + const parsed = [_]Token{ + Token{ + .type = .code_block, + .data = "```", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .raw_text, + .data = "some text or code", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .code_block, + .data = "```", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + // try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + // std.debug.print("expected_token : {f}\n", .{expected_token}); + // std.debug.print("actual_token : {f}\n", .{actual_token}); + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + +test "code_block" { + const input = "```lang_name\nsome text or code\n```\n"; + const parsed = [_]Token{ + Token{ + .type = .code_block, + .data = "```", + }, + Token{ + .type = .raw_text, + .data = "lang_name", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .raw_text, + .data = "some text or code", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .code_block, + .data = "```", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + +test "passthrough_block" { + const input = "```passthrough\nsome text or code\n```\n"; + const parsed = [_]Token{ + Token{ + .type = .code_block, + .data = "```", + }, + Token{ + .type = .raw_text, + .data = "passthrough", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .raw_text, + .data = "some text or code", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .code_block, + .data = "```", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + +test "comment" { + const input = "// this is a comment\n"; + const parsed = [_]Token{ + Token{ + .type = .comment, + .data = "// ", + }, + Token{ + .type = .raw_text, + .data = "this is a comment", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + +test "tick_box_empty" { + const input = "- [ ] something to check off\n"; + const parsed = [_]Token{ + Token{ + .type = .tickbox, + .data = "- [ ]", + }, + Token{ + .type = .text, + .data = "something to check off", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + // std.debug.print("expected_token {f}\n", .{expected_token}); + // std.debug.print("actual_token {f}\n", .{actual_token}); + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + +test "tick_box_checked" { + const input = "- [x] something checked off\n"; + const parsed = [_]Token{ + Token{ + .type = .tickbox, + .data = "- [x]", + }, + Token{ + .type = .text, + .data = "something checked off", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + +test "tick_box_ticked" { + const input = "- [/] something ticked off\n"; + const parsed = [_]Token{ + Token{ + .type = .tickbox, + .data = "- [/]", + }, + Token{ + .type = .text, + .data = "something ticked off", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + +test "tick_box_dashed" { + const input = "- [~] something dashed off\n"; + const parsed = [_]Token{ + Token{ + .type = .tickbox, + .data = "- [~]", + }, + Token{ + .type = .text, + .data = "something dashed off", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + +test "horizontal_rule_straight_line" { + const input = "---\n"; + const parsed = [_]Token{ + Token{ + .type = .rule, + .data = "---", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + // std.debug.print("expected_token {f}\n", .{expected_token}); + // std.debug.print("actual_token {f}\n", .{actual_token}); + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + +test "horizontal_rule_double_straight_line" { + const input = "===\n"; + const parsed = [_]Token{ + Token{ + .type = .rule, + .data = "===", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + +test "horizontal_rule_big_dotted_straight_line" { + const input = "***\n"; + const parsed = [_]Token{ + Token{ + .type = .rule, + .data = "***", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + +test "horizontal_rule_small_dotted_line" { + const input = "...\n"; + const parsed = [_]Token{ + Token{ + .type = .rule, + .data = "...", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + +test "horizontal_rule_squiggly_line" { + const input = "&&&\n"; + const parsed = [_]Token{ + Token{ + .type = .rule, + .data = "&&&", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + +test "horizontal_rule_jagged_line" { + const input = "^^^\n"; + const parsed = [_]Token{ + Token{ + .type = .rule, + .data = "^^^", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + +// test "footnote" { +// const input = "sometext [^1]\n\n[^1]: footnote\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .text, +// .data = "sometext ", +// }, +// Token{ +// .type = .footnote, +// .data = "[^1]", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// Token{ +// .type = .footnote_def, +// .data = "[^1]:", +// }, +// Token{ +// .type = .text, +// .data = "footnote", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// }; + +// var parsing = try tokenize(input, std.testing.allocator); +// try expect(parsing.items.len >= parsed.len); + +// for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// parsing.deinit(std.testing.allocator); +// } + +// test "macro" { +// const input = "@macro(arg, arg2)\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .macro, +// .data = "macro", +// }, +// Token{ +// .type = .bracket_open, +// .data = "(", +// }, +// Token{ +// .type = .macro_arg, +// .data = "arg", +// }, +// Token{ +// .type = .arg_sep, +// .data = ",", +// }, +// Token{ +// .type = .macro_arg, +// .data = "arg2", +// }, +// Token{ +// .type = .bracket_close, +// .data = ")", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// }; + +// var parsing = try tokenize(input, std.testing.allocator); +// try expect(parsing.items.len >= parsed.len); + +// for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// parsing.deinit(std.testing.allocator); +// } + +// test "replace" { +// const input = "@replace\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .replace, +// .data = "replace", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// }; + +// var parsing = try tokenize(input, std.testing.allocator); +// try expect(parsing.items.len >= parsed.len); + +// for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// parsing.deinit(std.testing.allocator); +// } + +// test "inbulit" { +// const input = "@.macro(arg, arg2)\n"; +// const parsed = [_]Token{ +// Token{ +// .type = .inbulit, +// .data = "macro", +// }, +// Token{ +// .type = .bracket_open, +// .data = "(", +// }, +// Token{ +// .type = .macro_arg, +// .data = "arg", +// }, +// Token{ +// .type = .arg_sep, +// .data = ",", +// }, +// Token{ +// .type = .macro_arg, +// .data = "arg2", +// }, +// Token{ +// .type = .bracket_close, +// .data = ")", +// }, +// Token{ +// .type = .newline, +// .data = "\n", +// }, +// }; + +// var parsing = try tokenize(input, std.testing.allocator); +// try expect(parsing.items.len >= parsed.len); + +// for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { +// try expect(expected_token.type == actual_token.type); +// try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); +// } + +// parsing.deinit(std.testing.allocator); +// } diff --git a/src/main.zig b/src/main.zig new file mode 100644 index 0000000..ce117c7 --- /dev/null +++ b/src/main.zig @@ -0,0 +1,27 @@ +const std = @import("std"); +const pandas_markdown = @import("pandas_markdown"); + +pub fn main() !void { + // Prints to stderr, ignoring potential errors. + std.debug.print("All your {s} are belong to us.\n", .{"codebase"}); + try pandas_markdown.bufferedPrint(); +} + +test "simple test" { + const gpa = std.testing.allocator; + var list: std.ArrayList(i32) = .empty; + defer list.deinit(gpa); // Try commenting this out and see if zig detects the memory leak! + try list.append(gpa, 42); + try std.testing.expectEqual(@as(i32, 42), list.pop()); +} + +test "fuzz example" { + const Context = struct { + fn testOne(context: @This(), input: []const u8) anyerror!void { + _ = context; + // Try passing `--fuzz` to `zig build test` and see if it manages to fail this test case! + try std.testing.expect(!std.mem.eql(u8, "canyoufindme", input)); + } + }; + try std.testing.fuzz(Context{}, Context.testOne, .{}); +} diff --git a/src/tests.zig b/src/tests.zig new file mode 100644 index 0000000..e69de29 diff --git a/tests.zig b/tests.zig new file mode 100644 index 0000000..78afc11 --- /dev/null +++ b/tests.zig @@ -0,0 +1,2596 @@ + +test "heading_1" { + const input = "# hello this is a heading\n"; + const parsed = [_]Token{ + Token{ + .type = .heading, + .data = "#", + }, + Token{ + .type = .text, + .data = "hello this is a heading\n", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + + +test "heading_2" { + const input = "## hello this is a heading\n"; + const parsed = [_]Token{ + Token{ + .type = .heading, + .data = "##", + }, + Token{ + .type = .text, + .data = "hello this is a heading\n", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + + +test "heading_3" { + const input = "### hello this is a heading\n"; + const parsed = [_]Token{ + Token{ + .type = .heading, + .data = "###", + }, + Token{ + .type = .text, + .data = "hello this is a heading\n", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + + +test "heading_4" { + const input = "#### hello this is a heading\n"; + const parsed = [_]Token{ + Token{ + .type = .heading, + .data = "####", + }, + Token{ + .type = .text, + .data = "hello this is a heading\n", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + + +test "heading_5" { + const input = "##### hello this is a heading\n"; + const parsed = [_]Token{ + Token{ + .type = .heading, + .data = "#####", + }, + Token{ + .type = .text, + .data = "hello this is a heading\n", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + + +test "heading_6" { + const input = "###### hello this is a heading\n"; + const parsed = [_]Token{ + Token{ + .type = .heading, + .data = "######", + }, + Token{ + .type = .text, + .data = "hello this is a heading\n", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + + +test "header_no_text" { + const input = "######\n"; + const parsed = [_]Token{ + Token{ + .type = .heading, + .data = "######", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + + +test "text" { + const input = "plain test\n"; + const parsed = [_]Token{ + Token{ + .type = .text, + .data = "plain test", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + + +test "text_style_code" { + const input = "`plain test`\n"; + const parsed = [_]Token{ + Token{ + .type = .inline_style, + .data = "`", + }, + Token{ + .type = .text, + .data = "plain test", + }, + Token{ + .type = .inline_style, + .data = "`", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + + +test "text_style_bold" { + const input = "!plain test!\n"; + const parsed = [_]Token{ + Token{ + .type = .inline_style, + .data = "!", + }, + Token{ + .type = .text, + .data = "plain test", + }, + Token{ + .type = .inline_style, + .data = "!", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + + +test "text_style_italic" { + const input = "*plain test*\n"; + const parsed = [_]Token{ + Token{ + .type = .inline_style, + .data = "*", + }, + Token{ + .type = .text, + .data = "plain test", + }, + Token{ + .type = .inline_style, + .data = "*", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + + +test "text_style_underline" { + const input = "_plain test_\n"; + const parsed = [_]Token{ + Token{ + .type = .inline_style, + .data = "_", + }, + Token{ + .type = .text, + .data = "plain test", + }, + Token{ + .type = .inline_style, + .data = "_", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + + +test "text_style_strikethrough" { + const input = "~plain test~\n"; + const parsed = [_]Token{ + Token{ + .type = .inline_style, + .data = "~", + }, + Token{ + .type = .text, + .data = "plain test", + }, + Token{ + .type = .inline_style, + .data = "~", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + + +test "text_style_highlighted" { + const input = "|plain test|\n"; + const parsed = [_]Token{ + Token{ + .type = .inline_style, + .data = "|", + }, + Token{ + .type = .text, + .data = "plain test", + }, + Token{ + .type = .inline_style, + .data = "|", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + + +test "text_style_&" { + const input = "&plain test&\n"; + const parsed = [_]Token{ + Token{ + .type = .inline_style, + .data = "&", + }, + Token{ + .type = .text, + .data = "plain test", + }, + Token{ + .type = .inline_style, + .data = "&", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + + +test "text_style_%" { + const input = "%plain test%\n"; + const parsed = [_]Token{ + Token{ + .type = .inline_style, + .data = "%", + }, + Token{ + .type = .text, + .data = "plain test", + }, + Token{ + .type = .inline_style, + .data = "%", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + + +test "text_style_$" { + const input = "$plain test$\n"; + const parsed = [_]Token{ + Token{ + .type = .inline_style, + .data = "$", + }, + Token{ + .type = .text, + .data = "plain test", + }, + Token{ + .type = .inline_style, + .data = "$", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + + +test "text_style_#" { + const input = "#plain test#\n"; + const parsed = [_]Token{ + Token{ + .type = .inline_style, + .data = "#", + }, + Token{ + .type = .text, + .data = "plain test", + }, + Token{ + .type = .inline_style, + .data = "#", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + + +test "text_style_-" { + const input = "-plain test-\n"; + const parsed = [_]Token{ + Token{ + .type = .inline_style, + .data = "-", + }, + Token{ + .type = .text, + .data = "plain test", + }, + Token{ + .type = .inline_style, + .data = "-", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + + +test "text_style_+" { + const input = "+plain test+\n"; + const parsed = [_]Token{ + Token{ + .type = .inline_style, + .data = "+", + }, + Token{ + .type = .text, + .data = "plain test", + }, + Token{ + .type = .inline_style, + .data = "+", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + + +test "text_style_=" { + const input = "=plain test=\n"; + const parsed = [_]Token{ + Token{ + .type = .inline_style, + .data = "=", + }, + Token{ + .type = .text, + .data = "plain test", + }, + Token{ + .type = .inline_style, + .data = "=", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + + +test "text_style_super_script" { + const input = "sometext^2\n"; + const parsed = [_]Token{ + Token{ + .type = .text, + .data = "sometext", + }, + Token{ + .type = .sup, + .data = "^", + }, + Token{ + .type = .text, + .data = "2", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + + +test "text_style_sub_script" { + const input = "sometext_2\n"; + const parsed = [_]Token{ + Token{ + .type = .text, + .data = "sometext", + }, + Token{ + .type = .sub, + .data = "_", + }, + Token{ + .type = .text, + .data = "2", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + + +test "list_ordered_number" { + const input = "1. ordered list\n2. ordered list\n3. ordered list\n"; + const parsed = [_]Token{ + Token{ + .type = .ordered_list, + .data = "1.", + }, + Token{ + .type = .text, + .data = "ordered list", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .ordered_list, + .data = "2.", + }, + Token{ + .type = .text, + .data = "ordered list", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .ordered_list, + .data = "3.", + }, + Token{ + .type = .text, + .data = "ordered list", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + + +test "list_ordered_i" { + const input = "i. ordered list\nii. ordered list\niii. ordered list\n"; + const parsed = [_]Token{ + Token{ + .type = .ordered_list, + .data = "i.", + }, + Token{ + .type = .text, + .data = "ordered list", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .ordered_list, + .data = "ii.", + }, + Token{ + .type = .text, + .data = "ordered list", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .ordered_list, + .data = "iii.", + }, + Token{ + .type = .text, + .data = "ordered list", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + + +test "list_ordered_letter" { + const input = "a. ordered list\nb. ordered list\nc. ordered list\n"; + const parsed = [_]Token{ + Token{ + .type = .ordered_list, + .data = "a.", + }, + Token{ + .type = .text, + .data = "ordered list", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .ordered_list, + .data = "b.", + }, + Token{ + .type = .text, + .data = "ordered list", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .ordered_list, + .data = "c.", + }, + Token{ + .type = .text, + .data = "ordered list", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + + +test "list_unordered_dashed" { + const input = "- unordered list\n- unordered list\n- unordered list\n"; + const parsed = [_]Token{ + Token{ + .type = .unordered_list, + .data = "-", + }, + Token{ + .type = .text, + .data = "unordered list", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .unordered_list, + .data = "-", + }, + Token{ + .type = .text, + .data = "unordered list", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .unordered_list, + .data = "-", + }, + Token{ + .type = .text, + .data = "unordered list", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + + +test "list_unordered_eq" { + const input = "= unordered list\n= unordered list\n= unordered list\n"; + const parsed = [_]Token{ + Token{ + .type = .unordered_list, + .data = "=", + }, + Token{ + .type = .text, + .data = "unordered list", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .unordered_list, + .data = "=", + }, + Token{ + .type = .text, + .data = "unordered list", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .unordered_list, + .data = "=", + }, + Token{ + .type = .text, + .data = "unordered list", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + + +test "list_unordered_plus" { + const input = "+ unordered list\n+ unordered list\n+ unordered list\n"; + const parsed = [_]Token{ + Token{ + .type = .unordered_list, + .data = "+", + }, + Token{ + .type = .text, + .data = "unordered list", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .unordered_list, + .data = "+", + }, + Token{ + .type = .text, + .data = "unordered list", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .unordered_list, + .data = "+", + }, + Token{ + .type = .text, + .data = "unordered list", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + + +test "list_unordered_dot" { + const input = "* unordered list\n* unordered list\n* unordered list\n"; + const parsed = [_]Token{ + Token{ + .type = .unordered_list, + .data = "*", + }, + Token{ + .type = .text, + .data = "unordered list", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .unordered_list, + .data = "*", + }, + Token{ + .type = .text, + .data = "unordered list", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .unordered_list, + .data = "*", + }, + Token{ + .type = .text, + .data = "unordered list", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + + +test "embed_image" { + const input = "![figure text](relitive path to image)\n"; + const parsed = [_]Token{ + Token{ + .type = .embed, + .data = "!", + }, + Token{ + .type = .bracket_open, + .data = "[", + }, + Token{ + .type = .text, + .data = "figure text", + }, + Token{ + .type = .bracket_close, + .data = "]", + }, + Token{ + .type = .bracket_open, + .data = "(", + }, + Token{ + .type = .raw_text, + .data = "relitive path to image", + }, + Token{ + .type = .bracket_close, + .data = ")", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + + +test "embed_table" { + const input = "#[table text](relitive path to tabulated data)\n"; + const parsed = [_]Token{ + Token{ + .type = .embed, + .data = "#", + }, + Token{ + .type = .bracket_open, + .data = "[", + }, + Token{ + .type = .text, + .data = "table text", + }, + Token{ + .type = .bracket_close, + .data = "]", + }, + Token{ + .type = .bracket_open, + .data = "(", + }, + Token{ + .type = .raw_text, + .data = "relitive path to tabulated data", + }, + Token{ + .type = .bracket_close, + .data = ")", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + + +test "embed_file" { + const input = "$[file text](path to file#starting_line:ending_line)\n"; + const parsed = [_]Token{ + Token{ + .type = .embed, + .data = "$", + }, + Token{ + .type = .bracket_open, + .data = "[", + }, + Token{ + .type = .text, + .data = "file text", + }, + Token{ + .type = .bracket_close, + .data = "]", + }, + Token{ + .type = .bracket_open, + .data = "(", + }, + Token{ + .type = .raw_text, + .data = "path to file#starting_line:ending_line", + }, + Token{ + .type = .bracket_close, + .data = ")", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + + +test "embed_*" { + const input = "*[file text](file_path)\n"; + const parsed = [_]Token{ + Token{ + .type = .embed, + .data = "*", + }, + Token{ + .type = .bracket_open, + .data = "[", + }, + Token{ + .type = .text, + .data = "file text", + }, + Token{ + .type = .bracket_close, + .data = "]", + }, + Token{ + .type = .bracket_open, + .data = "(", + }, + Token{ + .type = .raw_text, + .data = "file_path", + }, + Token{ + .type = .bracket_close, + .data = ")", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + + +test "embed__" { + const input = "_[file text](file_path)\n"; + const parsed = [_]Token{ + Token{ + .type = .embed, + .data = "_", + }, + Token{ + .type = .bracket_open, + .data = "[", + }, + Token{ + .type = .text, + .data = "file text", + }, + Token{ + .type = .bracket_close, + .data = "]", + }, + Token{ + .type = .bracket_open, + .data = "(", + }, + Token{ + .type = .raw_text, + .data = "file_path", + }, + Token{ + .type = .bracket_close, + .data = ")", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + + +test "embed_~" { + const input = "~[file text](file_path)\n"; + const parsed = [_]Token{ + Token{ + .type = .embed, + .data = "~", + }, + Token{ + .type = .bracket_open, + .data = "[", + }, + Token{ + .type = .text, + .data = "file text", + }, + Token{ + .type = .bracket_close, + .data = "]", + }, + Token{ + .type = .bracket_open, + .data = "(", + }, + Token{ + .type = .raw_text, + .data = "file_path", + }, + Token{ + .type = .bracket_close, + .data = ")", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + + +test "embed_|" { + const input = "|[file text](file_path)\n"; + const parsed = [_]Token{ + Token{ + .type = .embed, + .data = "|", + }, + Token{ + .type = .bracket_open, + .data = "[", + }, + Token{ + .type = .text, + .data = "file text", + }, + Token{ + .type = .bracket_close, + .data = "]", + }, + Token{ + .type = .bracket_open, + .data = "(", + }, + Token{ + .type = .raw_text, + .data = "file_path", + }, + Token{ + .type = .bracket_close, + .data = ")", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + + +test "embed_&" { + const input = "&[file text](file_path)\n"; + const parsed = [_]Token{ + Token{ + .type = .embed, + .data = "&", + }, + Token{ + .type = .bracket_open, + .data = "[", + }, + Token{ + .type = .text, + .data = "file text", + }, + Token{ + .type = .bracket_close, + .data = "]", + }, + Token{ + .type = .bracket_open, + .data = "(", + }, + Token{ + .type = .raw_text, + .data = "file_path", + }, + Token{ + .type = .bracket_close, + .data = ")", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + + +test "embed_%" { + const input = "%[file text](file_path)\n"; + const parsed = [_]Token{ + Token{ + .type = .embed, + .data = "%", + }, + Token{ + .type = .bracket_open, + .data = "[", + }, + Token{ + .type = .text, + .data = "file text", + }, + Token{ + .type = .bracket_close, + .data = "]", + }, + Token{ + .type = .bracket_open, + .data = "(", + }, + Token{ + .type = .raw_text, + .data = "file_path", + }, + Token{ + .type = .bracket_close, + .data = ")", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + + +test "embed_-" { + const input = "-[file text](file_path)\n"; + const parsed = [_]Token{ + Token{ + .type = .embed, + .data = "-", + }, + Token{ + .type = .bracket_open, + .data = "[", + }, + Token{ + .type = .text, + .data = "file text", + }, + Token{ + .type = .bracket_close, + .data = "]", + }, + Token{ + .type = .bracket_open, + .data = "(", + }, + Token{ + .type = .raw_text, + .data = "file_path", + }, + Token{ + .type = .bracket_close, + .data = ")", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + + +test "embed_+" { + const input = "+[file text](file_path)\n"; + const parsed = [_]Token{ + Token{ + .type = .embed, + .data = "+", + }, + Token{ + .type = .bracket_open, + .data = "[", + }, + Token{ + .type = .text, + .data = "file text", + }, + Token{ + .type = .bracket_close, + .data = "]", + }, + Token{ + .type = .bracket_open, + .data = "(", + }, + Token{ + .type = .raw_text, + .data = "file_path", + }, + Token{ + .type = .bracket_close, + .data = ")", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + + +test "embed_=" { + const input = "=[file text](file_path)\n"; + const parsed = [_]Token{ + Token{ + .type = .embed, + .data = "=", + }, + Token{ + .type = .bracket_open, + .data = "[", + }, + Token{ + .type = .text, + .data = "file text", + }, + Token{ + .type = .bracket_close, + .data = "]", + }, + Token{ + .type = .bracket_open, + .data = "(", + }, + Token{ + .type = .raw_text, + .data = "file_path", + }, + Token{ + .type = .bracket_close, + .data = ")", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + + +test "embed_link" { + const input = "/[file text](link to something)\n"; + const parsed = [_]Token{ + Token{ + .type = .link, + .data = "/", + }, + Token{ + .type = .bracket_open, + .data = "[", + }, + Token{ + .type = .text, + .data = "file text", + }, + Token{ + .type = .bracket_close, + .data = "]", + }, + Token{ + .type = .bracket_open, + .data = "(", + }, + Token{ + .type = .raw_text, + .data = "link to something", + }, + Token{ + .type = .bracket_close, + .data = ")", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + + +test "table" { + const input = "| col | col | col |\n| --- | --- | --- |\n| row | row | row |\n| row | row | row |\n| row | row | row |\n| row | row | row |\n"; + const parsed = [_]Token{ + Token{ + .type = .table_sep, + .data = "|", + }, + Token{ + .type = .text, + .data = " col ", + }, + Token{ + .type = .table_sep, + .data = "|", + }, + Token{ + .type = .text, + .data = " col ", + }, + Token{ + .type = .table_sep, + .data = "|", + }, + Token{ + .type = .text, + .data = " col ", + }, + Token{ + .type = .table_sep, + .data = "|", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .table_sep, + .data = "|", + }, + Token{ + .type = .table_row_sep, + .data = " --- ", + }, + Token{ + .type = .table_sep, + .data = "|", + }, + Token{ + .type = .table_row_sep, + .data = " --- ", + }, + Token{ + .type = .table_sep, + .data = "|", + }, + Token{ + .type = .table_row_sep, + .data = " --- ", + }, + Token{ + .type = .table_sep, + .data = "|", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .table_sep, + .data = "|", + }, + Token{ + .type = .text, + .data = " row ", + }, + Token{ + .type = .table_sep, + .data = "|", + }, + Token{ + .type = .text, + .data = " row ", + }, + Token{ + .type = .table_sep, + .data = "|", + }, + Token{ + .type = .text, + .data = " row ", + }, + Token{ + .type = .table_sep, + .data = "|", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .table_sep, + .data = "|", + }, + Token{ + .type = .text, + .data = " row ", + }, + Token{ + .type = .table_sep, + .data = "|", + }, + Token{ + .type = .text, + .data = " row ", + }, + Token{ + .type = .table_sep, + .data = "|", + }, + Token{ + .type = .text, + .data = " row ", + }, + Token{ + .type = .table_sep, + .data = "|", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .table_sep, + .data = "|", + }, + Token{ + .type = .text, + .data = " row ", + }, + Token{ + .type = .table_sep, + .data = "|", + }, + Token{ + .type = .text, + .data = " row ", + }, + Token{ + .type = .table_sep, + .data = "|", + }, + Token{ + .type = .text, + .data = " row ", + }, + Token{ + .type = .table_sep, + .data = "|", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .table_sep, + .data = "|", + }, + Token{ + .type = .text, + .data = " row ", + }, + Token{ + .type = .table_sep, + .data = "|", + }, + Token{ + .type = .text, + .data = " row ", + }, + Token{ + .type = .table_sep, + .data = "|", + }, + Token{ + .type = .text, + .data = " row ", + }, + Token{ + .type = .table_sep, + .data = "|", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + + +test "qoute_arrow" { + const input = "> quote\n> quote\n> quote\n"; + const parsed = [_]Token{ + Token{ + .type = .quote, + .data = ">", + }, + Token{ + .type = .text, + .data = "qoute", + }, + Token{ + .type = .quote, + .data = ">", + }, + Token{ + .type = .text, + .data = "qoute", + }, + Token{ + .type = .quote, + .data = ">", + }, + Token{ + .type = .text, + .data = "qoute", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + + +test "qoute_bar" { + const input = "| quote\n| quote\n| quote\n"; + const parsed = [_]Token{ + Token{ + .type = .quote, + .data = "|", + }, + Token{ + .type = .text, + .data = "qoute", + }, + Token{ + .type = .quote, + .data = "|", + }, + Token{ + .type = .text, + .data = "qoute", + }, + Token{ + .type = .quote, + .data = "|", + }, + Token{ + .type = .text, + .data = "qoute", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + + +test "text_block" { + const input = "```\nsome text or code\n```\n"; + const parsed = [_]Token{ + Token{ + .type = .code_block, + .data = "```", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .raw_text, + .data = "some text or code", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .code_block, + .data = "```", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + + +test "code_block" { + const input = "```lang_name\nsome text or code\n```\n"; + const parsed = [_]Token{ + Token{ + .type = .code_block, + .data = "```", + }, + Token{ + .type = .raw_text, + .data = "lang_name", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .raw_text, + .data = "some text or code", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .code_block, + .data = "```", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + + +test "passthrough_block" { + const input = "```passthrough\nsome text or code\n```\n"; + const parsed = [_]Token{ + Token{ + .type = .code_block, + .data = "```", + }, + Token{ + .type = .raw_text, + .data = "passthrough", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .raw_text, + .data = "some text or code", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .code_block, + .data = "```", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + + +test "comment" { + const input = "// this is a comment\n"; + const parsed = [_]Token{ + Token{ + .type = .comment, + .data = "// ", + }, + Token{ + .type = .raw_text, + .data = "this is a comment", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + + +test "tick_box_empty" { + const input = "- [ ] something to check off\n"; + const parsed = [_]Token{ + Token{ + .type = .tickbox, + .data = "- [ ]", + }, + Token{ + .type = .text, + .data = "something to check off", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + + +test "tick_box_checked" { + const input = "- [x] something checked off\n"; + const parsed = [_]Token{ + Token{ + .type = .tickbox, + .data = "- [x]", + }, + Token{ + .type = .text, + .data = "something checked off", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + + +test "tick_box_ticked" { + const input = "- [/] something ticked off\n"; + const parsed = [_]Token{ + Token{ + .type = .tickbox, + .data = "- [/]", + }, + Token{ + .type = .text, + .data = "something ticked off", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + + +test "tick_box_dashed" { + const input = "- [~] something dashed off\n"; + const parsed = [_]Token{ + Token{ + .type = .tickbox, + .data = "- [~]", + }, + Token{ + .type = .text, + .data = "something dashed off", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + + +test "horrazontal_rule_straight_line" { + const input = "---\n"; + const parsed = [_]Token{ + Token{ + .type = .rule, + .data = "---", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + + +test "horrazontal_rule_double_straight_line" { + const input = "===\n"; + const parsed = [_]Token{ + Token{ + .type = .rule, + .data = "===", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + + +test "horrazontal_rule_big_dotted_straight_line" { + const input = "***\n"; + const parsed = [_]Token{ + Token{ + .type = .rule, + .data = "***", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + + +test "horrazontal_rule_small_dotted_line" { + const input = "...\n"; + const parsed = [_]Token{ + Token{ + .type = .rule, + .data = "...", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + + +test "horrazontal_rule_squiggly_line" { + const input = "&&&\n"; + const parsed = [_]Token{ + Token{ + .type = .rule, + .data = "&&&", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + + +test "horrazontal_rule_jagged_line" { + const input = "^^^\n"; + const parsed = [_]Token{ + Token{ + .type = .rule, + .data = "^^^", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + + +test "footnote" { + const input = "sometext [^1]\n\n[^1]: footnote\n"; + const parsed = [_]Token{ + Token{ + .type = .text, + .data = "sometext ", + }, + Token{ + .type = .footnote, + .data = "[^1]", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .newline, + .data = "\n", + }, + Token{ + .type = .footnote_def, + .data = "[^1]:", + }, + Token{ + .type = .text, + .data = "footnote", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + + +test "macro" { + const input = "@macro(arg, arg2)\n"; + const parsed = [_]Token{ + Token{ + .type = .macro, + .data = "macro", + }, + Token{ + .type = .bracket_open, + .data = "(", + }, + Token{ + .type = .macro_arg, + .data = "arg", + }, + Token{ + .type = .arg_sep, + .data = ",", + }, + Token{ + .type = .macro_arg, + .data = "arg2", + }, + Token{ + .type = .bracket_close, + .data = ")", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + + +test "replace" { + const input = "@replace\n"; + const parsed = [_]Token{ + Token{ + .type = .replace, + .data = "replace", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + + +test "inbulit" { + const input = "@.macro(arg, arg2)\n"; + const parsed = [_]Token{ + Token{ + .type = .inbulit, + .data = "macro", + }, + Token{ + .type = .bracket_open, + .data = "(", + }, + Token{ + .type = .macro_arg, + .data = "arg", + }, + Token{ + .type = .arg_sep, + .data = ",", + }, + Token{ + .type = .macro_arg, + .data = "arg2", + }, + Token{ + .type = .bracket_close, + .data = ")", + }, + Token{ + .type = .newline, + .data = "\n", + }, + }; + + var parsing = try tokenize(input, std.testing.allocator); + try expect(parsing.items.len >= parsed.len); + + for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| { + try expect(expected_token.type == actual_token.type); + try expect(std.mem.eql(u8, expected_token.data, actual_token.data)); + } + + parsing.deinit(std.testing.allocator); +} + diff --git a/tests/test.json b/tests/test.json new file mode 100644 index 0000000..053c6c5 --- /dev/null +++ b/tests/test.json @@ -0,0 +1,727 @@ + +{ + "idenifier_chars" : { + "headings" : "#", + "inline_styles" : "`!*_~|&%$#-+=", + "sup" : "^", + "sub" : "_", + "ordered" : [ + "[number].", + "[letter].", + "[i]." + ], + "unordered" : "-=+*", + "embed" : "!#$*_~|&%-+=", + "link" : "/" + }, + "basic_unit_tests" : { + "heading_1" : { + "input" : "# hello this is a heading\n", + "parsed" : [ + {"heading" : "#"}, + {"text" : "hello this is a heading"}, + {"newline" : "\n"} + ] + }, + "heading_2" : { + "input" : "## hello this is a heading\n", + "parsed" : [ + {"heading" : "##"}, + {"text" : "hello this is a heading"}, + {"newline" : "\n"} + ] + }, + "heading_3" : { + "input" : "### hello this is a heading\n", + "parsed" : [ + {"heading" : "###"}, + {"text" : "hello this is a heading"}, + {"newline" : "\n"} + ] + }, + "heading_4" : { + "input" : "#### hello this is a heading\n", + "parsed" : [ + {"heading" : "####"}, + {"text" : "hello this is a heading"}, + {"newline" : "\n"} + ] + }, + "heading_5" : { + "input" : "##### hello this is a heading\n", + "parsed" : [ + {"heading" : "#####"}, + {"text" : "hello this is a heading"}, + {"newline" : "\n"} + ] + }, + "heading_6" : { + "input" : "###### hello this is a heading\n", + "parsed" : [ + {"heading" : "######"}, + {"text" : "hello this is a heading"}, + {"newline" : "\n"} + ] + }, + "header_no_text" : { + "input" : "######\n", + "parsed" : [ + {"heading" : "######"}, + {"newline" : "\n"} + ] + }, + "text" : { + "input" : "plain test\n", + "parsed" : [ + {"text" : "plain test"}, + {"newline" : "\n"} + ] + }, + "text_style_code" : { + "input" : "`plain test`\n", + "parsed" : [ + {"inline_style" : "`"}, + {"text" : "plain test"}, + {"inline_style" : "`"}, + {"newline" : "\n"} + ] + }, + "text_style_bold" : { + "input" : "!plain test!\n", + "parsed" : [ + {"inline_style" : "!"}, + {"text" : "plain test"}, + {"inline_style" : "!"}, + {"newline" : "\n"} + ] + }, + "text_style_italic" : { + "input" : "*plain test*\n", + "parsed" : [ + {"inline_style" : "*"}, + {"text" : "plain test"}, + {"inline_style" : "*"}, + {"newline" : "\n"} + ] + }, + "text_style_underline" : { + "input" : "_plain test_\n", + "parsed" : [ + {"inline_style" : "_"}, + {"text" : "plain test"}, + {"inline_style" : "_"}, + {"newline" : "\n"} + ] + }, + "text_style_strikethrough" : { + "input" : "~plain test~\n", + "parsed" : [ + {"inline_style" : "~"}, + {"text" : "plain test"}, + {"inline_style" : "~"}, + {"newline" : "\n"} + ] + }, + "text_style_highlighted" : { + "input" : "|plain test|\n", + "parsed" : [ + {"inline_style" : "|"}, + {"text" : "plain test"}, + {"inline_style" : "|"}, + {"newline" : "\n"} + ] + }, + "text_style_&" : { + "input" : "&plain test&\n", + "parsed" : [ + {"inline_style" : "&"}, + {"text" : "plain test"}, + {"inline_style" : "&"}, + {"newline" : "\n"} + ] + }, + "text_style_%" : { + "input" : "%plain test%\n", + "parsed" : [ + {"inline_style" : "%"}, + {"text" : "plain test"}, + {"inline_style" : "%"}, + {"newline" : "\n"} + ] + }, + "text_style_$" : { + "input" : "$plain test$\n", + "parsed" : [ + {"inline_style" : "$"}, + {"text" : "plain test"}, + {"inline_style" : "$"}, + {"newline" : "\n"} + ] + }, + "text_style_#" : { + "input" : "#plain test#\n", + "parsed" : [ + {"inline_style" : "#"}, + {"text" : "plain test"}, + {"inline_style" : "#"}, + {"newline" : "\n"} + ] + }, + "text_style_-" : { + "input" : "-plain test-\n", + "parsed" : [ + {"inline_style" : "-"}, + {"text" : "plain test"}, + {"inline_style" : "-"}, + {"newline" : "\n"} + ] + }, + "text_style_+" : { + "input" : "+plain test+\n", + "parsed" : [ + {"inline_style" : "+"}, + {"text" : "plain test"}, + {"inline_style" : "+"}, + {"newline" : "\n"} + ] + }, + "text_style_=" : { + "input" : "=plain test=\n", + "parsed" : [ + {"inline_style" : "="}, + {"text" : "plain test"}, + {"inline_style" : "="}, + {"newline" : "\n"} + ] + }, + "text_style_espaced" : { + "input" : "\\`plain test`\n", + "parsed" : [ + {"text" : "`plain test`"}, + {"newline" : "\n"} + ] + }, + "text_style_with_style_char_in_string" : { + "input" : "+plain something + something else test+\n", + "parsed" : [ + {"inline_style" : "+"}, + {"text" : "plain something + something else test"}, + {"inline_style" : "+"}, + {"newline" : "\n"} + ] + }, + "text_style_with_space right after" : { + "input" : "+ plain something + something else test+\n", + "parsed" : [ + {"text" : "+ plain something + something else test+"}, + {"newline" : "\n"} + ] + }, + "text_style_super_script" : { + "input" : "sometext^2\n", + "parsed" : [ + {"text" : "sometext"}, + {"sup" : "^"}, + {"text" : "2"} + ] + }, + "text_style_sub_script" : { + "input" : "sometext_2\n", + "parsed" : [ + {"text" : "sometext"}, + {"sub" : "_"}, + {"text" : "2"} + ] + }, + "list_ordered_number":{ + "input" : "1. ordered list\n2. ordered list\n3. ordered list\n", + "parsed" : [ + {"ordered_list" : "1."}, + {"text" : "ordered list"}, + {"newline" : "\n"}, + {"ordered_list" : "2."}, + {"text" : "ordered list"}, + {"newline" : "\n"}, + {"ordered_list" : "3."}, + {"text" : "ordered list"}, + {"newline" : "\n"} + ] + }, + "list_ordered_i":{ + "input" : "i. ordered list\nii. ordered list\niii. ordered list\n", + "parsed" : [ + {"ordered_list" : "i."}, + {"text" : "ordered list"}, + {"newline" : "\n"}, + {"ordered_list" : "ii."}, + {"text" : "ordered list"}, + {"newline" : "\n"}, + {"ordered_list" : "iii."}, + {"text" : "ordered list"}, + {"newline" : "\n"} + ] + }, + "list_ordered_letter":{ + "input" : "a. ordered list\nb. ordered list\nc. ordered list\n", + "parsed" : [ + {"ordered_list" : "a."}, + {"text" : "ordered list"}, + {"newline" : "\n"}, + {"ordered_list" : "b."}, + {"text" : "ordered list"}, + {"newline" : "\n"}, + {"ordered_list" : "c."}, + {"text" : "ordered list"}, + {"newline" : "\n"} + ] + }, + "list_unordered_dashed":{ + "input" : "- unordered list\n- unordered list\n- unordered list\n", + "parsed" : [ + {"unordered_list" : "-"}, + {"text" : "unordered list"}, + {"newline" : "\n"}, + {"unordered_list" : "-"}, + {"text" : "unordered list"}, + {"newline" : "\n"}, + {"unordered_list" : "-"}, + {"text" : "unordered list"}, + {"newline" : "\n"} + ] + }, + "list_unordered_eq":{ + "input" : "= unordered list\n= unordered list\n= unordered list\n", + "parsed" : [ + {"unordered_list" : "="}, + {"text" : "unordered list"}, + {"newline" : "\n"}, + {"unordered_list" : "="}, + {"text" : "unordered list"}, + {"newline" : "\n"}, + {"unordered_list" : "="}, + {"text" : "unordered list"}, + {"newline" : "\n"} + ] + }, + "list_unordered_plus":{ + "input" : "+ unordered list\n+ unordered list\n+ unordered list\n", + "parsed" : [ + {"unordered_list" : "+"}, + {"text" : "unordered list"}, + {"newline" : "\n"}, + {"unordered_list" : "+"}, + {"text" : "unordered list"}, + {"newline" : "\n"}, + {"unordered_list" : "+"}, + {"text" : "unordered list"}, + {"newline" : "\n"} + ] + }, + "list_unordered_dot":{ + "input" : "* unordered list\n* unordered list\n* unordered list\n", + "parsed" : [ + {"unordered_list" : "*"}, + {"text" : "unordered list"}, + {"newline" : "\n"}, + {"unordered_list" : "*"}, + {"text" : "unordered list"}, + {"newline" : "\n"}, + {"unordered_list" : "*"}, + {"text" : "unordered list"}, + {"newline" : "\n"} + ] + }, + "embed_image": { + "input" : "![figure text](relitive path to image)\n", + "parsed" : [ + {"embed" : "!"}, + {"bracket_open" : "["}, + {"text" : "figure text"}, + {"bracket_close" : "]"}, + {"bracket_open" : "("}, + {"raw_text" : "relitive path to image"}, + {"bracket_close" : ")"}, + {"newline" : "\n"} + ] + }, + "embed_table": { + "input" : "#[table text](relitive path to tabulated data)\n", + "parsed" : [ + {"embed" : "#"}, + {"bracket_open" : "["}, + {"text" : "table text"}, + {"bracket_close" : "]"}, + {"bracket_open" : "("}, + {"raw_text" : "relitive path to tabulated data"}, + {"bracket_close" : ")"}, + {"newline" : "\n"} + ] + }, + "embed_file": { + "input" : "$[file text](path to file#starting_line:ending_line)\n", + "parsed" : [ + {"embed" : "$"}, + {"bracket_open" : "["}, + {"text" : "file text"}, + {"bracket_close" : "]"}, + {"bracket_open" : "("}, + {"raw_text" : "path to file#starting_line:ending_line"}, + {"bracket_close" : ")"}, + {"newline" : "\n"} + ] + }, + "embed_*": { + "input" : "*[file text](file_path)\n", + "parsed" : [ + {"embed" : "*"}, + {"bracket_open" : "["}, + {"text" : "file text"}, + {"bracket_close" : "]"}, + {"bracket_open" : "("}, + {"raw_text" : "file_path"}, + {"bracket_close" : ")"}, + {"newline" : "\n"} + ] + }, + "embed__": { + "input" : "_[file text](file_path)\n", + "parsed" : [ + {"embed" : "_"}, + {"bracket_open" : "["}, + {"text" : "file text"}, + {"bracket_close" : "]"}, + {"bracket_open" : "("}, + {"raw_text" : "file_path"}, + {"bracket_close" : ")"}, + {"newline" : "\n"} + ] + }, + "embed_~": { + "input" : "~[file text](file_path)\n", + "parsed" : [ + {"embed" : "~"}, + {"bracket_open" : "["}, + {"text" : "file text"}, + {"bracket_close" : "]"}, + {"bracket_open" : "("}, + {"raw_text" : "file_path"}, + {"bracket_close" : ")"}, + {"newline" : "\n"} + ] + }, + "embed_|": { + "input" : "|[file text](file_path)\n", + "parsed" : [ + {"embed" : "|"}, + {"bracket_open" : "["}, + {"text" : "file text"}, + {"bracket_close" : "]"}, + {"bracket_open" : "("}, + {"raw_text" : "file_path"}, + {"bracket_close" : ")"}, + {"newline" : "\n"} + ] + }, + "embed_&": { + "input" : "&[file text](file_path)\n", + "parsed" : [ + {"embed" : "&"}, + {"bracket_open" : "["}, + {"text" : "file text"}, + {"bracket_close" : "]"}, + {"bracket_open" : "("}, + {"raw_text" : "file_path"}, + {"bracket_close" : ")"}, + {"newline" : "\n"} + ] + }, + "embed_%": { + "input" : "%[file text](file_path)\n", + "parsed" : [ + {"embed" : "%"}, + {"bracket_open" : "["}, + {"text" : "file text"}, + {"bracket_close" : "]"}, + {"bracket_open" : "("}, + {"raw_text" : "file_path"}, + {"bracket_close" : ")"}, + {"newline" : "\n"} + ] + }, + "embed_-": { + "input" : "-[file text](file_path)\n", + "parsed" : [ + {"embed" : "-"}, + {"bracket_open" : "["}, + {"text" : "file text"}, + {"bracket_close" : "]"}, + {"bracket_open" : "("}, + {"raw_text" : "file_path"}, + {"bracket_close" : ")"}, + {"newline" : "\n"} + ] + }, + "embed_+": { + "input" : "+[file text](file_path)\n", + "parsed" : [ + {"embed" : "+"}, + {"bracket_open" : "["}, + {"text" : "file text"}, + {"bracket_close" : "]"}, + {"bracket_open" : "("}, + {"raw_text" : "file_path"}, + {"bracket_close" : ")"}, + {"newline" : "\n"} + ] + }, + "embed_=": { + "input" : "=[file text](file_path)\n", + "parsed" : [ + {"embed" : "="}, + {"bracket_open" : "["}, + {"text" : "file text"}, + {"bracket_close" : "]"}, + {"bracket_open" : "("}, + {"raw_text" : "file_path"}, + {"bracket_close" : ")"}, + {"newline" : "\n"} + ] + }, + "embed_link": { + "input" : "/[file text](link to something)\n", + "parsed" : [ + {"link" : "/"}, + {"bracket_open" : "["}, + {"text" : "file text"}, + {"bracket_close" : "]"}, + {"bracket_open" : "("}, + {"raw_text" : "link to something"}, + {"bracket_close" : ")"}, + {"newline" : "\n"} + ] + }, + "table" : { + "input" : "| col | col | col |\n| --- | --- | --- |\n| row | row | row |\n| row | row | row |\n| row | row | row |\n| row | row | row |\n", + "parsed" : [ + {"table_sep" : "|"}, + {"text" : " col "}, + {"table_sep" : "|"}, + {"text" : " col "}, + {"table_sep" : "|"}, + {"text" : " col "}, + {"table_sep" : "|"}, + {"newline" : "\n"}, + {"table_sep" : "|"}, + {"table_row_sep" : " --- "}, + {"table_sep" : "|"}, + {"table_row_sep" : " --- "}, + {"table_sep" : "|"}, + {"table_row_sep" : " --- "}, + {"table_sep" : "|"}, + {"newline" : "\n"}, + {"table_sep" : "|"}, + {"text" : " row "}, + {"table_sep" : "|"}, + {"text" : " row "}, + {"table_sep" : "|"}, + {"text" : " row "}, + {"table_sep" : "|"}, + {"newline" : "\n"}, + {"table_sep" : "|"}, + {"text" : " row "}, + {"table_sep" : "|"}, + {"text" : " row "}, + {"table_sep" : "|"}, + {"text" : " row "}, + {"table_sep" : "|"}, + {"newline" : "\n"}, + {"table_sep" : "|"}, + {"text" : " row "}, + {"table_sep" : "|"}, + {"text" : " row "}, + {"table_sep" : "|"}, + {"text" : " row "}, + {"table_sep" : "|"}, + {"newline" : "\n"}, + {"table_sep" : "|"}, + {"text" : " row "}, + {"table_sep" : "|"}, + {"text" : " row "}, + {"table_sep" : "|"}, + {"text" : " row "}, + {"table_sep" : "|"}, + {"newline" : "\n"} + + ] + }, + "quote_arrow" : { + "input" : "> quote\n> quote\n> quote\n", + "parsed" : [ + {"quote" : ">"}, + {"text" : "quote"}, + {"newline" : "\n"}, + {"quote" : ">"}, + {"text" : "quote"}, + {"newline" : "\n"}, + {"quote" : ">"}, + {"text" : "quote"}, + {"newline" : "\n"} + ] + }, + "quote_bar" : { + "input" : "| quote\n| quote\n| quote\n", + "parsed" : [ + {"quote" : "|"}, + {"text" : "quote"}, + {"newline" : "\n"}, + {"quote" : "|"}, + {"text" : "quote"}, + {"newline" : "\n"}, + {"quote" : "|"}, + {"text" : "quote"}, + {"newline" : "\n"} + ] + }, + "text_block" : { + "input" : "```\nsome text or code\n```\n", + "parsed" : [ + {"code_block" : "```"}, + {"newline" : "\n"}, + {"raw_text" : "some text or code"}, + {"newline" : "\n"}, + {"code_block" : "```"}, + {"newline" : "\n"} + ] + }, + "code_block" : { + "input" : "```lang_name\nsome text or code\n```\n", + "parsed" : [ + {"code_block" : "```"}, + {"raw_text" : "lang_name"}, + {"newline" : "\n"}, + {"raw_text" : "some text or code"}, + {"newline" : "\n"}, + {"code_block" : "```"}, + {"newline" : "\n"} + ] + }, + "passthrough_block" : { + "input" : "```passthrough\nsome text or code\n```\n", + "parsed" : [ + {"code_block" : "```"}, + {"raw_text" : "passthrough"}, + {"newline" : "\n"}, + {"raw_text" : "some text or code"}, + {"newline" : "\n"}, + {"code_block" : "```"}, + {"newline" : "\n"} + ] + }, + "comment" : { + "input" : "// this is a comment\n", + "parsed" : [ + {"comment" : "// "}, + {"raw_text" : "this is a comment"}, + {"newline" : "\n"} + ] + }, + "tick_box_empty" : { + "input" : "- [ ] something to check off\n", + "parsed" : [ + {"tickbox" : "- [ ]"}, + {"text" : "something to check off"}, + {"newline" : "\n"} + ] + }, + "tick_box_checked" : { + "input" : "- [x] something checked off\n", + "parsed" : [ + {"tickbox" : "- [x]"}, + {"text" : "something checked off"}, + {"newline" : "\n"} + ] + }, + "tick_box_ticked" : { + "input" : "- [/] something ticked off\n", + "parsed" : [ + {"tickbox" : "- [/]"}, + {"text" : "something ticked off"}, + {"newline" : "\n"} + ] + }, + "tick_box_dashed" : { + "input" : "- [~] something dashed off\n", + "parsed" : [ + {"tickbox" : "- [~]"}, + {"text" : "something dashed off"}, + {"newline" : "\n"} + ] + }, + "horizontal_rule_straight_line" : { + "input" : "---\n", + "parsed" : [ + {"rule" : "---"}, + {"newline" : "\n"} + ] + }, + "horizontal_rule_double_straight_line" : { + "input" : "===\n", + "parsed" : [ + {"rule" : "==="}, + {"newline" : "\n"} + ] + }, + "horizontal_rule_big_dotted_straight_line" : { + "input" : "***\n", + "parsed" : [ + {"rule" : "***"}, + {"newline" : "\n"} + ] + }, + "horizontal_rule_small_dotted_line" : { + "input" : "...\n", + "parsed" : [ + {"rule" : "..."}, + {"newline" : "\n"} + ] + }, + "horizontal_rule_squiggly_line" : { + "input" : "&&&\n", + "parsed" : [ + {"rule" : "&&&"}, + {"newline" : "\n"} + ] + }, + "horizontal_rule_jagged_line" : { + "input" : "^^^\n", + "parsed" : [ + {"rule" : "^^^"}, + {"newline" : "\n"} + ] + }, + "footnote" : { + "input" : "sometext [^1]\n\n[^1]: footnote\n", + "parsed" : [ + {"text" : "sometext "}, + {"footnote" : "[^1]"}, + {"newline" : "\n"}, + {"newline" : "\n"}, + {"footnote_def" : "[^1]:"}, + {"text" : "footnote"}, + {"newline" : "\n"} + ] + }, + "inbulit" : { + "input" : "@.macro(arg, arg2)\n", + "parsed" : [ + {"inbulit" : "macro"}, + {"bracket_open" : "("}, + {"macro_arg" : "arg"}, + {"arg_sep" : ","}, + {"macro_arg" : "arg2"}, + {"bracket_close" : ")"}, + {"newline" : "\n"} + ] + } + } +} diff --git a/tests/test_runner.zig b/tests/test_runner.zig new file mode 100644 index 0000000..9fad28f --- /dev/null +++ b/tests/test_runner.zig @@ -0,0 +1,16 @@ +const std = @import("std"); +const builtin = @import("builtin"); + +pub fn main() !void { + var stdout_buffer: [1024]u8 = undefined; + var stdout_writer = std.fs.File.stdout().writer(&stdout_buffer); + const stdout = &stdout_writer.interface; + for (builtin.test_functions) |t| { + t.func() catch |err| { + try stdout.print("{s} fail: {}\n", .{ t.name, err }); + continue; + }; + try stdout.print("{s} passed\n", .{t.name}); + } + try stdout.flush(); // Don't forget to flush! +} diff --git a/todo.md b/todo.md new file mode 100644 index 0000000..e69de29