//! By convention, root.zig is the root source file when making a library.
const std = @import("std");

pub fn stripWhiteSpace(white_space: []const u8) []const u8 {
    return std.mem.trimStart(u8, white_space, " \t");
}

pub const TokenType = enum {
    // macros
    inbulit,
    macro,
    macro_arg,
    arg_sep,
    replace,
    unknown, // this is the defualt token type and should never appear

    // utils
    bracket_close,
    bracket_open,
    raw_text,
    newline,

    comment,

    // in text
    heading,
    text,
    inline_style,
    unordered_list,
    ordered_list,
    embed,
    link,

    table_row_sep,
    table_sep,
    quote,

    code_block,

    tickbox,
    rule,

    footnote,
    footnote_def,

    sup,
    sub,
};

pub const Token = struct {
    data: []const u8 = "",
    type: TokenType = .unknown,
    padding: usize = 0,
    // this is to allow the ast generation to make sure that things are
    // within the right subtrees
    pub fn format(
        self: Token,
        writer: *std.io.Writer,
    ) !void {
        switch (self.type) {
            .newline => try writer.print(
                ".{{\n  .data = \"\\n\"\n  .type = {}\n}}",
                .{self.type},
            ),
            else => try writer.print(
                ".{{\n  .data = \"{s}\"\n  .type = {}\n}}",
                .{ self.data, self.type },
            ),
        }
    }
};

pub const LineParserReturn = struct {
    // the new updated pos of the token after its been parsed
    pointer_offset: usize = 0,
    // the token itself
    token: Token = .{},
};
/// lineParser parse chunk of the line then return the rest
pub const LineParser = *const fn (line: []const u8) ?LineParserReturn;

/// multi line parsers take in the lines and the list and add the tokens
/// to the list itself
pub const MultiLineParser = *const fn (
    lines: *std.mem.SplitIterator(u8, .scalar),
    alloc: std.mem.Allocator,
    list: *std.ArrayList(Token),
) anyerror!void;

const heading_id = "#";
const heading_depth: comptime_int = 6;
pub fn headingParser(line: []const u8) ?LineParserReturn {
    const heading_end_index: usize = blk: {
        comptime var i = heading_depth;
        inline while (i > 0) : (i -= 1) {
            if (std.mem.startsWith(u8, line, heading_id ** i)) break :blk i;
        }
        break :blk 0;
    };

    // if no heading was detected
    if (heading_end_index == 0) {
        return null;
    }

    // allows for blank headings
    const space_remove: usize =
        if (line.len == heading_end_index) 0 else 1;

    return LineParserReturn{
        .pointer_offset = heading_end_index + space_remove,
        .token = .{
            .type = .heading,
            .data = line[0..heading_end_index],
        },
    };
}

pub fn text(line: []const u8) ?LineParserReturn {
    return LineParserReturn{
        .pointer_offset = line.len,
        .token = .{
            .data = line[0..],
            .type = .text,
        },
    };
}

pub fn listOrderedParser(line: []const u8) ?LineParserReturn {
    const clean_line = stripWhiteSpace(line);
    const whitespace_chars_removed = @intFromPtr(clean_line.ptr) - @intFromPtr(line.ptr);
    // check for the min length of chars
    if (clean_line.len < 2) return null;

    // find a point within the line
    const index_of_dot = blk: {
        if (std.mem.indexOf(u8, clean_line, ". ")) |dex| {
            break :blk dex;
        }
        return null;
    };

    const number = clean_line[0..index_of_dot];
    // if it has any spaces its not a order list id
    if (std.mem.containsAtLeastScalar(u8, number, 1, ' ') or
        std.mem.containsAtLeastScalar(u8, number, 1, '\t'))
    {
        return null;
    }

    // now work out what the number is
    // "I": 1,
    // "V": 5,
    // "X": 10,
    // "L": 50,
    // "C": 100,
    // "D": 500,
    // "M": 1000

    // roman numerials ill deal with these later

    // these can be the same
    // numbers

    // letters & roman (at least all the i's)
    if (std.ascii.isAlphabetic(number[0])) {
        return LineParserReturn{
            // + 2 to remove the dot and space
            // + 1 to make sure the dot is the last char
            .pointer_offset = whitespace_chars_removed + number.len + 2,
            .token = .{
                .data = line[0 .. whitespace_chars_removed + number.len + 1],
                .type = .ordered_list,
            },
        };
    }

    // checks to see if the number can be parsed if it can then its a number
    _ = std.fmt.parseInt(usize, number, 10) catch return null;

    return LineParserReturn{
        // + 2 to remove the dot and space
        // + 1 to make sure the dot is the last char
        .pointer_offset = whitespace_chars_removed + number.len + 2,
        .token = .{
            .data = line[0 .. whitespace_chars_removed + number.len + 1],
            .type = .ordered_list,
        },
    };
}

pub fn listUnorderedParser(line: []const u8) ?LineParserReturn {
    const clean_line = stripWhiteSpace(line);
    const whitespace_chars_removed = @intFromPtr(clean_line.ptr) - @intFromPtr(line.ptr);
    // check for the min length of chars
    if (clean_line.len < 2) return null;

    // check there is a space seperating the char and the text
    if (clean_line[1] != ' ') return null;

    if (clean_line[0] == '-' or clean_line[0] == '=' or clean_line[0] == '+' or clean_line[0] == '*') {
        return LineParserReturn{
            // +2 to remove "-. "
            .pointer_offset = whitespace_chars_removed + 2,
            .token = .{
                .data = clean_line[0..1],
                .type = .unordered_list,
            },
        };
    }

    return null;
}

// pub fn styleParser(line: []const u8) ?LineParserReturn {}
// pub fn embedParser(line: []const u8) ?LineParserReturn {}

// pub fn tableParser(line: []const u8) ?LineParserReturn {}

pub fn quoteParser(line: []const u8) ?LineParserReturn {
    if (std.mem.startsWith(u8, line, "| ") or
        std.mem.startsWith(u8, line, "> "))
    {
        return LineParserReturn{ .pointer_offset = 2, .token = .{
            .data = line[0..1],
            .type = .quote,
        } };
    }

    return null;
}

// // multiline might have to add some extra to this

pub fn codeBlockParser(
    current_line: []const u8,
    lines: *std.mem.SplitIterator(u8, .scalar),
    alloc: std.mem.Allocator,
    list: *std.ArrayList(Token),
) anyerror!bool {
    if (!std.mem.startsWith(u8, current_line, "```")) {
        // std.debug.print("line didnt start with ```\n", .{});
        return false;
    }
    try list.append(alloc, .{
        .data = "```",
        .type = .code_block,
    });

    const lang_name = std.mem.trimEnd(
        u8,
        current_line[3..],
        " \t\n",
    );
    // std.debug.print("lang_name = {}\n", .{lang_name.len});
    if (lang_name.len != 0) {
        try list.append(alloc, .{
            .data = lang_name,
            .type = .raw_text,
        });
    }

    try list.append(alloc, .{
        .data = "\n",
        .type = .newline,
    });

    // std.debug.print("first values\n", .{});
    // +1 to remove the newline
    const raw_text_start_index =
        @intFromPtr(current_line.ptr) - @intFromPtr(lines.buffer.ptr) + lang_name.len + 3 + 1;
    // std.debug.print("raw_text_start_index : {}\n", .{raw_text_start_index});

    const ending_line = find_end: {
        while (lines.next()) |line| {
            if (std.mem.startsWith(u8, line, "```"))
                break :find_end line;
        } else {
            return error.No_closing_tag;
        }
    };

    const raw_text_end_index = @intFromPtr(ending_line.ptr) - @intFromPtr(lines.buffer.ptr) - 1;
    // std.debug.print("raw_text_end_index : {}\n", .{raw_text_end_index});

    try list.append(alloc, .{
        .data = lines.buffer[raw_text_start_index..raw_text_end_index],
        .type = .raw_text,
    });
    try list.append(alloc, .{
        .data = "\n",
        .type = .newline,
    });

    try list.append(alloc, .{
        .data = ending_line,
        .type = .code_block,
    });

    // for (list.items) |value| {
    //     std.debug.print("t : {f}\n", .{value});
    // }

    return true;
}

// pub fn codeBlockParser(line: []const u8) ?LineParserReturn {}

pub fn tickBoxParser(line: []const u8) ?LineParserReturn {
    if (!std.mem.startsWith(u8, line, "- [")) {
        return null;
    }
    // not long enough for a tick box
    if (line.len < 6) return null;

    // - [
    if (line[3] == ' ' or
        line[3] == 'x' or
        line[3] == '/' or
        line[3] == '~' and
            line[4] == ']' and line[5] == ' ')
    {
        return LineParserReturn{
            .pointer_offset = 6,
            .token = .{
                .type = .tickbox,
                .data = line[0..5],
            },
        };
    }
    return null;
}

// might make this anylength

pub fn ruleCheck(comptime rule_char: []const u8, line: []const u8) usize {
    if (!std.mem.startsWith(u8, line, rule_char ** 3)) return 0;
    for (line, 0..) |char, i| {
        if (char == rule_char[0] or char == ' ' or char == '\t')
            continue
        else
            return i;
    }
    return line.len;
}

pub fn horizontalRuleParser(line: []const u8) ?LineParserReturn {
    const rule_length = ruleCheck("-", line) +
        ruleCheck("=", line) +
        ruleCheck("*", line) +
        ruleCheck(".", line) +
        ruleCheck("&", line) +
        ruleCheck("^", line);

    if (rule_length == 0) return null;

    return LineParserReturn{
        .pointer_offset = rule_length,
        .token = .{
            .data = line[0..rule_length],
            .type = .rule,
        },
    };
}

// pub fn footnoteParser(line: []const u8) ?LineParserReturn {}

// pub fn MacroParser(line: []const u8) ?LineParserReturn {}

const comment_id = "// ";
pub fn commentParser(line: []const u8) ?LineParserReturn {

    // find the index of the comment
    const index_of_commnet = blk: {
        if (std.mem.indexOf(u8, line, comment_id)) |index| {
            if (index == 0)
                break :blk index;

            if (line[index - 1] != '\\') {
                return null;
            }

            break :blk index;
        }
        return null;
    };

    return LineParserReturn{
        .pointer_offset = index_of_commnet + comment_id.len,
        .token = .{
            .type = .comment,
            .data = line[index_of_commnet .. index_of_commnet + comment_id.len],
        },
    };
}

// const LineParsers : [_]LineParser = {};

pub fn tokenize(
    // comptime start_parsers : []LineParser,
    // comptime text_parsers : []LineParser,
    // comptime end_parsers : []LineParser,
    // comptime multiline_parsers : []MultiLineParser,
    str: []const u8,
    alloc: std.mem.Allocator,
) !std.ArrayList(Token) {
    var lines = std.mem.splitScalar(u8, str, '\n');
    var token_list: std.ArrayList(Token) = .empty;
    while (lines.next()) |current_line| {
        var line = current_line[0..];
        // std.debug.print("line : \"{s}\"\n", .{line});

        if (try codeBlockParser(current_line, &lines, alloc, &token_list)) continue;

        // start end parsings
        const comment_token = commentParser(line);

        if (comment_token) |token| {
            // sets the end of the line to just before the comment
            std.debug.print("comment_token.pointer_offset : {}\n", .{token.pointer_offset - token.token.data.len});
            line = line[0 .. token.pointer_offset - token.token.data.len];
        }

        // actual parsing

        // multiline parsing

        // start parsing
        const start_token: ?LineParserReturn = blk: {
            if (headingParser(line)) |t| {
                break :blk t;
            }
            if (tickBoxParser(line)) |t| {
                break :blk t;
            }
            if (listOrderedParser(line)) |t| {
                break :blk t;
            }
            if (listUnorderedParser(line)) |t| {
                break :blk t;
            }
            if (quoteParser(line)) |t| {
                break :blk t;
            }
            if (horizontalRuleParser(line)) |t| {
                break :blk t;
            }
            break :blk null;
        };

        if (start_token) |val| {
            line = line[val.pointer_offset..];
            try token_list.append(alloc, val.token);
        }

        // mid parsing
        if (line.len != 0) { // makes sure that i dont add blank lines
            const mid_token: ?LineParserReturn = blk: {
                if (text(line)) |t| {
                    break :blk t;
                }
                break :blk null;
            };

            if (mid_token) |token| {
                try token_list.append(alloc, token.token);
            }
        }

        if (comment_token) |token| {
            // append the comment token
            try token_list.append(alloc, token.token);
            try token_list.append(alloc, .{
                .type = .raw_text,
                .data = current_line[token.pointer_offset..],
            });
        }

        try token_list.append(alloc, .{
            .type = .newline,
            // make sure its a pointer to the actual char
            .data = "\n",
        });
    }

    return token_list;
}

const expect = std.testing.expect;

test "heading_1" {
    const input = "# hello this is a heading\n";
    const parsed = [_]Token{
        Token{
            .type = .heading,
            .data = "#",
        },
        Token{
            .type = .text,
            .data = "hello this is a heading",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
    };

    var parsing = try tokenize(input, std.testing.allocator);
    // for (parsing.items) |value| {
    //     std.debug.print("{f}\n", .{value});
    // }

    try expect(parsing.items.len >= parsed.len);

    for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| {
        try expect(expected_token.type == actual_token.type);
        try expect(std.mem.eql(u8, expected_token.data, actual_token.data));
    }

    parsing.deinit(std.testing.allocator);
}

test "heading_2" {
    const input = "## hello this is a heading\n";
    const parsed = [_]Token{
        Token{
            .type = .heading,
            .data = "##",
        },
        Token{
            .type = .text,
            .data = "hello this is a heading",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
    };

    var parsing = try tokenize(input, std.testing.allocator);
    try expect(parsing.items.len >= parsed.len);

    for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| {
        try expect(expected_token.type == actual_token.type);
        try expect(std.mem.eql(u8, expected_token.data, actual_token.data));
    }

    parsing.deinit(std.testing.allocator);
}

test "heading_3" {
    const input = "### hello this is a heading\n";
    const parsed = [_]Token{
        Token{
            .type = .heading,
            .data = "###",
        },
        Token{
            .type = .text,
            .data = "hello this is a heading",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
    };

    var parsing = try tokenize(input, std.testing.allocator);
    try expect(parsing.items.len >= parsed.len);

    for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| {
        try expect(expected_token.type == actual_token.type);
        try expect(std.mem.eql(u8, expected_token.data, actual_token.data));
    }

    parsing.deinit(std.testing.allocator);
}

test "heading_4" {
    const input = "#### hello this is a heading\n";
    const parsed = [_]Token{
        Token{
            .type = .heading,
            .data = "####",
        },
        Token{
            .type = .text,
            .data = "hello this is a heading",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
    };

    var parsing = try tokenize(input, std.testing.allocator);
    try expect(parsing.items.len >= parsed.len);

    for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| {
        try expect(expected_token.type == actual_token.type);
        try expect(std.mem.eql(u8, expected_token.data, actual_token.data));
    }

    parsing.deinit(std.testing.allocator);
}

test "heading_5" {
    const input = "##### hello this is a heading\n";
    const parsed = [_]Token{
        Token{
            .type = .heading,
            .data = "#####",
        },
        Token{
            .type = .text,
            .data = "hello this is a heading",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
    };

    var parsing = try tokenize(input, std.testing.allocator);
    try expect(parsing.items.len >= parsed.len);

    for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| {
        try expect(expected_token.type == actual_token.type);
        try expect(std.mem.eql(u8, expected_token.data, actual_token.data));
    }

    parsing.deinit(std.testing.allocator);
}

test "heading_6" {
    const input = "###### hello this is a heading\n";
    const parsed = [_]Token{
        Token{
            .type = .heading,
            .data = "######",
        },
        Token{
            .type = .text,
            .data = "hello this is a heading",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
    };

    var parsing = try tokenize(input, std.testing.allocator);
    try expect(parsing.items.len >= parsed.len);

    for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| {
        try expect(expected_token.type == actual_token.type);
        try expect(std.mem.eql(u8, expected_token.data, actual_token.data));
    }

    parsing.deinit(std.testing.allocator);
}

test "header_no_text" {
    const input = "######\n";
    const parsed = [_]Token{
        Token{
            .type = .heading,
            .data = "######",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
    };

    var parsing = try tokenize(input, std.testing.allocator);
    try expect(parsing.items.len >= parsed.len);

    for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| {
        try expect(expected_token.type == actual_token.type);
        try expect(std.mem.eql(u8, expected_token.data, actual_token.data));
    }

    parsing.deinit(std.testing.allocator);
}

test "text" {
    const input = "plain test\n";
    const parsed = [_]Token{
        Token{
            .type = .text,
            .data = "plain test",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
    };

    var parsing = try tokenize(input, std.testing.allocator);
    try expect(parsing.items.len >= parsed.len);

    for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| {
        try expect(expected_token.type == actual_token.type);
        try expect(std.mem.eql(u8, expected_token.data, actual_token.data));
    }

    parsing.deinit(std.testing.allocator);
}

test "text_style_code" {
    const input = "`plain test`\n";
    const parsed = [_]Token{
        Token{
            .type = .inline_style,
            .data = "`",
        },
        Token{
            .type = .text,
            .data = "plain test",
        },
        Token{
            .type = .inline_style,
            .data = "`",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
    };

    var parsing = try tokenize(input, std.testing.allocator);
    try expect(parsing.items.len >= parsed.len);

    for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| {
        try expect(expected_token.type == actual_token.type);
        try expect(std.mem.eql(u8, expected_token.data, actual_token.data));
    }

    parsing.deinit(std.testing.allocator);
}

test "text_style_bold" {
    const input = "!plain test!\n";
    const parsed = [_]Token{
        Token{
            .type = .inline_style,
            .data = "!",
        },
        Token{
            .type = .text,
            .data = "plain test",
        },
        Token{
            .type = .inline_style,
            .data = "!",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
    };

    var parsing = try tokenize(input, std.testing.allocator);
    try expect(parsing.items.len >= parsed.len);

    for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| {
        try expect(expected_token.type == actual_token.type);
        try expect(std.mem.eql(u8, expected_token.data, actual_token.data));
    }

    parsing.deinit(std.testing.allocator);
}

test "text_style_italic" {
    const input = "*plain test*\n";
    const parsed = [_]Token{
        Token{
            .type = .inline_style,
            .data = "*",
        },
        Token{
            .type = .text,
            .data = "plain test",
        },
        Token{
            .type = .inline_style,
            .data = "*",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
    };

    var parsing = try tokenize(input, std.testing.allocator);
    try expect(parsing.items.len >= parsed.len);

    for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| {
        try expect(expected_token.type == actual_token.type);
        try expect(std.mem.eql(u8, expected_token.data, actual_token.data));
    }

    parsing.deinit(std.testing.allocator);
}

test "text_style_underline" {
    const input = "_plain test_\n";
    const parsed = [_]Token{
        Token{
            .type = .inline_style,
            .data = "_",
        },
        Token{
            .type = .text,
            .data = "plain test",
        },
        Token{
            .type = .inline_style,
            .data = "_",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
    };

    var parsing = try tokenize(input, std.testing.allocator);
    try expect(parsing.items.len >= parsed.len);

    for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| {
        try expect(expected_token.type == actual_token.type);
        try expect(std.mem.eql(u8, expected_token.data, actual_token.data));
    }

    parsing.deinit(std.testing.allocator);
}

test "text_style_strikethrough" {
    const input = "~plain test~\n";
    const parsed = [_]Token{
        Token{
            .type = .inline_style,
            .data = "~",
        },
        Token{
            .type = .text,
            .data = "plain test",
        },
        Token{
            .type = .inline_style,
            .data = "~",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
    };

    var parsing = try tokenize(input, std.testing.allocator);
    try expect(parsing.items.len >= parsed.len);

    for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| {
        try expect(expected_token.type == actual_token.type);
        try expect(std.mem.eql(u8, expected_token.data, actual_token.data));
    }

    parsing.deinit(std.testing.allocator);
}

test "text_style_highlighted" {
    const input = "|plain test|\n";
    const parsed = [_]Token{
        Token{
            .type = .inline_style,
            .data = "|",
        },
        Token{
            .type = .text,
            .data = "plain test",
        },
        Token{
            .type = .inline_style,
            .data = "|",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
    };

    var parsing = try tokenize(input, std.testing.allocator);
    try expect(parsing.items.len >= parsed.len);

    for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| {
        try expect(expected_token.type == actual_token.type);
        try expect(std.mem.eql(u8, expected_token.data, actual_token.data));
    }

    parsing.deinit(std.testing.allocator);
}

test "text_style_&" {
    const input = "&plain test&\n";
    const parsed = [_]Token{
        Token{
            .type = .inline_style,
            .data = "&",
        },
        Token{
            .type = .text,
            .data = "plain test",
        },
        Token{
            .type = .inline_style,
            .data = "&",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
    };

    var parsing = try tokenize(input, std.testing.allocator);
    try expect(parsing.items.len >= parsed.len);

    for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| {
        try expect(expected_token.type == actual_token.type);
        try expect(std.mem.eql(u8, expected_token.data, actual_token.data));
    }

    parsing.deinit(std.testing.allocator);
}

test "text_style_%" {
    const input = "%plain test%\n";
    const parsed = [_]Token{
        Token{
            .type = .inline_style,
            .data = "%",
        },
        Token{
            .type = .text,
            .data = "plain test",
        },
        Token{
            .type = .inline_style,
            .data = "%",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
    };

    var parsing = try tokenize(input, std.testing.allocator);
    try expect(parsing.items.len >= parsed.len);

    for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| {
        try expect(expected_token.type == actual_token.type);
        try expect(std.mem.eql(u8, expected_token.data, actual_token.data));
    }

    parsing.deinit(std.testing.allocator);
}

test "text_style_$" {
    const input = "$plain test$\n";
    const parsed = [_]Token{
        Token{
            .type = .inline_style,
            .data = "$",
        },
        Token{
            .type = .text,
            .data = "plain test",
        },
        Token{
            .type = .inline_style,
            .data = "$",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
    };

    var parsing = try tokenize(input, std.testing.allocator);
    try expect(parsing.items.len >= parsed.len);

    for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| {
        try expect(expected_token.type == actual_token.type);
        try expect(std.mem.eql(u8, expected_token.data, actual_token.data));
    }

    parsing.deinit(std.testing.allocator);
}

test "text_style_#" {
    const input = "#plain test#\n";
    const parsed = [_]Token{
        Token{
            .type = .inline_style,
            .data = "#",
        },
        Token{
            .type = .text,
            .data = "plain test",
        },
        Token{
            .type = .inline_style,
            .data = "#",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
    };

    var parsing = try tokenize(input, std.testing.allocator);
    try expect(parsing.items.len >= parsed.len);

    for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| {
        try expect(expected_token.type == actual_token.type);
        try expect(std.mem.eql(u8, expected_token.data, actual_token.data));
    }

    parsing.deinit(std.testing.allocator);
}

test "text_style_-" {
    const input = "-plain test-\n";
    const parsed = [_]Token{
        Token{
            .type = .inline_style,
            .data = "-",
        },
        Token{
            .type = .text,
            .data = "plain test",
        },
        Token{
            .type = .inline_style,
            .data = "-",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
    };

    var parsing = try tokenize(input, std.testing.allocator);
    try expect(parsing.items.len >= parsed.len);

    for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| {
        try expect(expected_token.type == actual_token.type);
        try expect(std.mem.eql(u8, expected_token.data, actual_token.data));
    }

    parsing.deinit(std.testing.allocator);
}

test "text_style_+" {
    const input = "+plain test+\n";
    const parsed = [_]Token{
        Token{
            .type = .inline_style,
            .data = "+",
        },
        Token{
            .type = .text,
            .data = "plain test",
        },
        Token{
            .type = .inline_style,
            .data = "+",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
    };

    var parsing = try tokenize(input, std.testing.allocator);
    try expect(parsing.items.len >= parsed.len);

    for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| {
        try expect(expected_token.type == actual_token.type);
        try expect(std.mem.eql(u8, expected_token.data, actual_token.data));
    }

    parsing.deinit(std.testing.allocator);
}

test "text_style_=" {
    const input = "=plain test=\n";
    const parsed = [_]Token{
        Token{
            .type = .inline_style,
            .data = "=",
        },
        Token{
            .type = .text,
            .data = "plain test",
        },
        Token{
            .type = .inline_style,
            .data = "=",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
    };

    var parsing = try tokenize(input, std.testing.allocator);
    try expect(parsing.items.len >= parsed.len);

    for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| {
        try expect(expected_token.type == actual_token.type);
        try expect(std.mem.eql(u8, expected_token.data, actual_token.data));
    }

    parsing.deinit(std.testing.allocator);
}

test "text_style_super_script" {
    const input = "sometext^2\n";
    const parsed = [_]Token{
        Token{
            .type = .text,
            .data = "sometext",
        },
        Token{
            .type = .sup,
            .data = "^",
        },
        Token{
            .type = .text,
            .data = "2",
        },
    };

    var parsing = try tokenize(input, std.testing.allocator);
    try expect(parsing.items.len >= parsed.len);

    for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| {
        try expect(expected_token.type == actual_token.type);
        try expect(std.mem.eql(u8, expected_token.data, actual_token.data));
    }

    parsing.deinit(std.testing.allocator);
}

test "text_style_sub_script" {
    const input = "sometext_2\n";
    const parsed = [_]Token{
        Token{
            .type = .text,
            .data = "sometext",
        },
        Token{
            .type = .sub,
            .data = "_",
        },
        Token{
            .type = .text,
            .data = "2",
        },
    };

    var parsing = try tokenize(input, std.testing.allocator);
    try expect(parsing.items.len >= parsed.len);

    for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| {
        try expect(expected_token.type == actual_token.type);
        try expect(std.mem.eql(u8, expected_token.data, actual_token.data));
    }

    parsing.deinit(std.testing.allocator);
}

test "list_ordered_number" {
    const input = "1. ordered list\n2. ordered list\n3. ordered list\n";
    const parsed = [_]Token{
        Token{
            .type = .ordered_list,
            .data = "1.",
        },
        Token{
            .type = .text,
            .data = "ordered list",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
        Token{
            .type = .ordered_list,
            .data = "2.",
        },
        Token{
            .type = .text,
            .data = "ordered list",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
        Token{
            .type = .ordered_list,
            .data = "3.",
        },
        Token{
            .type = .text,
            .data = "ordered list",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
    };

    var parsing = try tokenize(input, std.testing.allocator);
    try expect(parsing.items.len >= parsed.len);
    // for (parsing.items) |value| {
    //     std.debug.print("token : {f}\n", .{value});
    // }

    for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| {
        try expect(expected_token.type == actual_token.type);
        try expect(std.mem.eql(u8, expected_token.data, actual_token.data));
    }

    parsing.deinit(std.testing.allocator);
}

test "list_ordered_i" {
    const input = "i. ordered list\nii. ordered list\niii. ordered list\n";
    const parsed = [_]Token{
        Token{
            .type = .ordered_list,
            .data = "i.",
        },
        Token{
            .type = .text,
            .data = "ordered list",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
        Token{
            .type = .ordered_list,
            .data = "ii.",
        },
        Token{
            .type = .text,
            .data = "ordered list",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
        Token{
            .type = .ordered_list,
            .data = "iii.",
        },
        Token{
            .type = .text,
            .data = "ordered list",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
    };

    var parsing = try tokenize(input, std.testing.allocator);
    try expect(parsing.items.len >= parsed.len);

    for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| {
        try expect(expected_token.type == actual_token.type);
        try expect(std.mem.eql(u8, expected_token.data, actual_token.data));
    }

    parsing.deinit(std.testing.allocator);
}

test "list_ordered_letter" {
    const input = "a. ordered list\nb. ordered list\nc. ordered list\n";
    const parsed = [_]Token{
        Token{
            .type = .ordered_list,
            .data = "a.",
        },
        Token{
            .type = .text,
            .data = "ordered list",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
        Token{
            .type = .ordered_list,
            .data = "b.",
        },
        Token{
            .type = .text,
            .data = "ordered list",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
        Token{
            .type = .ordered_list,
            .data = "c.",
        },
        Token{
            .type = .text,
            .data = "ordered list",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
    };

    var parsing = try tokenize(input, std.testing.allocator);
    try expect(parsing.items.len >= parsed.len);

    for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| {
        try expect(expected_token.type == actual_token.type);
        try expect(std.mem.eql(u8, expected_token.data, actual_token.data));
    }

    parsing.deinit(std.testing.allocator);
}

test "list_unordered_dashed" {
    const input = "- unordered list\n- unordered list\n- unordered list\n";
    const parsed = [_]Token{
        Token{
            .type = .unordered_list,
            .data = "-",
        },
        Token{
            .type = .text,
            .data = "unordered list",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
        Token{
            .type = .unordered_list,
            .data = "-",
        },
        Token{
            .type = .text,
            .data = "unordered list",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
        Token{
            .type = .unordered_list,
            .data = "-",
        },
        Token{
            .type = .text,
            .data = "unordered list",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
    };

    var parsing = try tokenize(input, std.testing.allocator);
    // for (parsing.items) |value| {
    //     std.debug.print("token : {f}\n", .{value});
    // }
    try expect(parsing.items.len >= parsed.len);
    for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| {
        try expect(expected_token.type == actual_token.type);
        try expect(std.mem.eql(u8, expected_token.data, actual_token.data));
    }

    parsing.deinit(std.testing.allocator);
}

test "list_unordered_eq" {
    const input = "= unordered list\n= unordered list\n= unordered list\n";
    const parsed = [_]Token{
        Token{
            .type = .unordered_list,
            .data = "=",
        },
        Token{
            .type = .text,
            .data = "unordered list",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
        Token{
            .type = .unordered_list,
            .data = "=",
        },
        Token{
            .type = .text,
            .data = "unordered list",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
        Token{
            .type = .unordered_list,
            .data = "=",
        },
        Token{
            .type = .text,
            .data = "unordered list",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
    };

    var parsing = try tokenize(input, std.testing.allocator);
    try expect(parsing.items.len >= parsed.len);

    for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| {
        try expect(expected_token.type == actual_token.type);
        try expect(std.mem.eql(u8, expected_token.data, actual_token.data));
    }

    parsing.deinit(std.testing.allocator);
}

test "list_unordered_plus" {
    const input = "+ unordered list\n+ unordered list\n+ unordered list\n";
    const parsed = [_]Token{
        Token{
            .type = .unordered_list,
            .data = "+",
        },
        Token{
            .type = .text,
            .data = "unordered list",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
        Token{
            .type = .unordered_list,
            .data = "+",
        },
        Token{
            .type = .text,
            .data = "unordered list",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
        Token{
            .type = .unordered_list,
            .data = "+",
        },
        Token{
            .type = .text,
            .data = "unordered list",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
    };

    var parsing = try tokenize(input, std.testing.allocator);
    try expect(parsing.items.len >= parsed.len);

    for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| {
        try expect(expected_token.type == actual_token.type);
        try expect(std.mem.eql(u8, expected_token.data, actual_token.data));
    }

    parsing.deinit(std.testing.allocator);
}

test "list_unordered_dot" {
    const input = "* unordered list\n* unordered list\n* unordered list\n";
    const parsed = [_]Token{
        Token{
            .type = .unordered_list,
            .data = "*",
        },
        Token{
            .type = .text,
            .data = "unordered list",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
        Token{
            .type = .unordered_list,
            .data = "*",
        },
        Token{
            .type = .text,
            .data = "unordered list",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
        Token{
            .type = .unordered_list,
            .data = "*",
        },
        Token{
            .type = .text,
            .data = "unordered list",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
    };

    var parsing = try tokenize(input, std.testing.allocator);
    try expect(parsing.items.len >= parsed.len);

    for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| {
        try expect(expected_token.type == actual_token.type);
        try expect(std.mem.eql(u8, expected_token.data, actual_token.data));
    }

    parsing.deinit(std.testing.allocator);
}

test "embed_image" {
    const input = "![figure text](relitive path to image)\n";
    const parsed = [_]Token{
        Token{
            .type = .embed,
            .data = "!",
        },
        Token{
            .type = .bracket_open,
            .data = "[",
        },
        Token{
            .type = .text,
            .data = "figure text",
        },
        Token{
            .type = .bracket_close,
            .data = "]",
        },
        Token{
            .type = .bracket_open,
            .data = "(",
        },
        Token{
            .type = .raw_text,
            .data = "relitive path to image",
        },
        Token{
            .type = .bracket_close,
            .data = ")",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
    };

    var parsing = try tokenize(input, std.testing.allocator);
    try expect(parsing.items.len >= parsed.len);

    for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| {
        try expect(expected_token.type == actual_token.type);
        try expect(std.mem.eql(u8, expected_token.data, actual_token.data));
    }

    parsing.deinit(std.testing.allocator);
}

test "embed_table" {
    const input = "#[table text](relitive path to tabulated data)\n";
    const parsed = [_]Token{
        Token{
            .type = .embed,
            .data = "#",
        },
        Token{
            .type = .bracket_open,
            .data = "[",
        },
        Token{
            .type = .text,
            .data = "table text",
        },
        Token{
            .type = .bracket_close,
            .data = "]",
        },
        Token{
            .type = .bracket_open,
            .data = "(",
        },
        Token{
            .type = .raw_text,
            .data = "relitive path to tabulated data",
        },
        Token{
            .type = .bracket_close,
            .data = ")",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
    };

    var parsing = try tokenize(input, std.testing.allocator);
    try expect(parsing.items.len >= parsed.len);

    for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| {
        try expect(expected_token.type == actual_token.type);
        try expect(std.mem.eql(u8, expected_token.data, actual_token.data));
    }

    parsing.deinit(std.testing.allocator);
}

test "embed_file" {
    const input = "$[file text](path to file#starting_line:ending_line)\n";
    const parsed = [_]Token{
        Token{
            .type = .embed,
            .data = "$",
        },
        Token{
            .type = .bracket_open,
            .data = "[",
        },
        Token{
            .type = .text,
            .data = "file text",
        },
        Token{
            .type = .bracket_close,
            .data = "]",
        },
        Token{
            .type = .bracket_open,
            .data = "(",
        },
        Token{
            .type = .raw_text,
            .data = "path to file#starting_line:ending_line",
        },
        Token{
            .type = .bracket_close,
            .data = ")",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
    };

    var parsing = try tokenize(input, std.testing.allocator);
    try expect(parsing.items.len >= parsed.len);

    for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| {
        try expect(expected_token.type == actual_token.type);
        try expect(std.mem.eql(u8, expected_token.data, actual_token.data));
    }

    parsing.deinit(std.testing.allocator);
}

test "embed_*" {
    const input = "*[file text](file_path)\n";
    const parsed = [_]Token{
        Token{
            .type = .embed,
            .data = "*",
        },
        Token{
            .type = .bracket_open,
            .data = "[",
        },
        Token{
            .type = .text,
            .data = "file text",
        },
        Token{
            .type = .bracket_close,
            .data = "]",
        },
        Token{
            .type = .bracket_open,
            .data = "(",
        },
        Token{
            .type = .raw_text,
            .data = "file_path",
        },
        Token{
            .type = .bracket_close,
            .data = ")",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
    };

    var parsing = try tokenize(input, std.testing.allocator);
    try expect(parsing.items.len >= parsed.len);

    for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| {
        try expect(expected_token.type == actual_token.type);
        try expect(std.mem.eql(u8, expected_token.data, actual_token.data));
    }

    parsing.deinit(std.testing.allocator);
}

test "embed__" {
    const input = "_[file text](file_path)\n";
    const parsed = [_]Token{
        Token{
            .type = .embed,
            .data = "_",
        },
        Token{
            .type = .bracket_open,
            .data = "[",
        },
        Token{
            .type = .text,
            .data = "file text",
        },
        Token{
            .type = .bracket_close,
            .data = "]",
        },
        Token{
            .type = .bracket_open,
            .data = "(",
        },
        Token{
            .type = .raw_text,
            .data = "file_path",
        },
        Token{
            .type = .bracket_close,
            .data = ")",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
    };

    var parsing = try tokenize(input, std.testing.allocator);
    try expect(parsing.items.len >= parsed.len);

    for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| {
        try expect(expected_token.type == actual_token.type);
        try expect(std.mem.eql(u8, expected_token.data, actual_token.data));
    }

    parsing.deinit(std.testing.allocator);
}

test "embed_~" {
    const input = "~[file text](file_path)\n";
    const parsed = [_]Token{
        Token{
            .type = .embed,
            .data = "~",
        },
        Token{
            .type = .bracket_open,
            .data = "[",
        },
        Token{
            .type = .text,
            .data = "file text",
        },
        Token{
            .type = .bracket_close,
            .data = "]",
        },
        Token{
            .type = .bracket_open,
            .data = "(",
        },
        Token{
            .type = .raw_text,
            .data = "file_path",
        },
        Token{
            .type = .bracket_close,
            .data = ")",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
    };

    var parsing = try tokenize(input, std.testing.allocator);
    try expect(parsing.items.len >= parsed.len);

    for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| {
        try expect(expected_token.type == actual_token.type);
        try expect(std.mem.eql(u8, expected_token.data, actual_token.data));
    }

    parsing.deinit(std.testing.allocator);
}

test "embed_|" {
    const input = "|[file text](file_path)\n";
    const parsed = [_]Token{
        Token{
            .type = .embed,
            .data = "|",
        },
        Token{
            .type = .bracket_open,
            .data = "[",
        },
        Token{
            .type = .text,
            .data = "file text",
        },
        Token{
            .type = .bracket_close,
            .data = "]",
        },
        Token{
            .type = .bracket_open,
            .data = "(",
        },
        Token{
            .type = .raw_text,
            .data = "file_path",
        },
        Token{
            .type = .bracket_close,
            .data = ")",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
    };

    var parsing = try tokenize(input, std.testing.allocator);
    try expect(parsing.items.len >= parsed.len);

    for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| {
        try expect(expected_token.type == actual_token.type);
        try expect(std.mem.eql(u8, expected_token.data, actual_token.data));
    }

    parsing.deinit(std.testing.allocator);
}

test "embed_&" {
    const input = "&[file text](file_path)\n";
    const parsed = [_]Token{
        Token{
            .type = .embed,
            .data = "&",
        },
        Token{
            .type = .bracket_open,
            .data = "[",
        },
        Token{
            .type = .text,
            .data = "file text",
        },
        Token{
            .type = .bracket_close,
            .data = "]",
        },
        Token{
            .type = .bracket_open,
            .data = "(",
        },
        Token{
            .type = .raw_text,
            .data = "file_path",
        },
        Token{
            .type = .bracket_close,
            .data = ")",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
    };

    var parsing = try tokenize(input, std.testing.allocator);
    try expect(parsing.items.len >= parsed.len);

    for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| {
        try expect(expected_token.type == actual_token.type);
        try expect(std.mem.eql(u8, expected_token.data, actual_token.data));
    }

    parsing.deinit(std.testing.allocator);
}

test "embed_%" {
    const input = "%[file text](file_path)\n";
    const parsed = [_]Token{
        Token{
            .type = .embed,
            .data = "%",
        },
        Token{
            .type = .bracket_open,
            .data = "[",
        },
        Token{
            .type = .text,
            .data = "file text",
        },
        Token{
            .type = .bracket_close,
            .data = "]",
        },
        Token{
            .type = .bracket_open,
            .data = "(",
        },
        Token{
            .type = .raw_text,
            .data = "file_path",
        },
        Token{
            .type = .bracket_close,
            .data = ")",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
    };

    var parsing = try tokenize(input, std.testing.allocator);
    try expect(parsing.items.len >= parsed.len);

    for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| {
        try expect(expected_token.type == actual_token.type);
        try expect(std.mem.eql(u8, expected_token.data, actual_token.data));
    }

    parsing.deinit(std.testing.allocator);
}

test "embed_-" {
    const input = "-[file text](file_path)\n";
    const parsed = [_]Token{
        Token{
            .type = .embed,
            .data = "-",
        },
        Token{
            .type = .bracket_open,
            .data = "[",
        },
        Token{
            .type = .text,
            .data = "file text",
        },
        Token{
            .type = .bracket_close,
            .data = "]",
        },
        Token{
            .type = .bracket_open,
            .data = "(",
        },
        Token{
            .type = .raw_text,
            .data = "file_path",
        },
        Token{
            .type = .bracket_close,
            .data = ")",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
    };

    var parsing = try tokenize(input, std.testing.allocator);
    try expect(parsing.items.len >= parsed.len);

    for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| {
        try expect(expected_token.type == actual_token.type);
        try expect(std.mem.eql(u8, expected_token.data, actual_token.data));
    }

    parsing.deinit(std.testing.allocator);
}

test "embed_+" {
    const input = "+[file text](file_path)\n";
    const parsed = [_]Token{
        Token{
            .type = .embed,
            .data = "+",
        },
        Token{
            .type = .bracket_open,
            .data = "[",
        },
        Token{
            .type = .text,
            .data = "file text",
        },
        Token{
            .type = .bracket_close,
            .data = "]",
        },
        Token{
            .type = .bracket_open,
            .data = "(",
        },
        Token{
            .type = .raw_text,
            .data = "file_path",
        },
        Token{
            .type = .bracket_close,
            .data = ")",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
    };

    var parsing = try tokenize(input, std.testing.allocator);
    try expect(parsing.items.len >= parsed.len);

    for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| {
        try expect(expected_token.type == actual_token.type);
        try expect(std.mem.eql(u8, expected_token.data, actual_token.data));
    }

    parsing.deinit(std.testing.allocator);
}

test "embed_=" {
    const input = "=[file text](file_path)\n";
    const parsed = [_]Token{
        Token{
            .type = .embed,
            .data = "=",
        },
        Token{
            .type = .bracket_open,
            .data = "[",
        },
        Token{
            .type = .text,
            .data = "file text",
        },
        Token{
            .type = .bracket_close,
            .data = "]",
        },
        Token{
            .type = .bracket_open,
            .data = "(",
        },
        Token{
            .type = .raw_text,
            .data = "file_path",
        },
        Token{
            .type = .bracket_close,
            .data = ")",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
    };

    var parsing = try tokenize(input, std.testing.allocator);
    try expect(parsing.items.len >= parsed.len);

    for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| {
        try expect(expected_token.type == actual_token.type);
        try expect(std.mem.eql(u8, expected_token.data, actual_token.data));
    }

    parsing.deinit(std.testing.allocator);
}

test "embed_link" {
    const input = "/[file text](link to something)\n";
    const parsed = [_]Token{
        Token{
            .type = .link,
            .data = "/",
        },
        Token{
            .type = .bracket_open,
            .data = "[",
        },
        Token{
            .type = .text,
            .data = "file text",
        },
        Token{
            .type = .bracket_close,
            .data = "]",
        },
        Token{
            .type = .bracket_open,
            .data = "(",
        },
        Token{
            .type = .raw_text,
            .data = "link to something",
        },
        Token{
            .type = .bracket_close,
            .data = ")",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
    };

    var parsing = try tokenize(input, std.testing.allocator);
    try expect(parsing.items.len >= parsed.len);

    for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| {
        try expect(expected_token.type == actual_token.type);
        try expect(std.mem.eql(u8, expected_token.data, actual_token.data));
    }

    parsing.deinit(std.testing.allocator);
}

// test "table" {
//     const input = "| col | col | col |\n| --- | --- | --- |\n| row | row | row |\n| row | row | row |\n| row | row | row |\n| row | row | row |\n";
//     const parsed = [_]Token{
//         Token{
//             .type = .table_sep,
//             .data = "|",
//         },
//         Token{
//             .type = .text,
//             .data = " col ",
//         },
//         Token{
//             .type = .table_sep,
//             .data = "|",
//         },
//         Token{
//             .type = .text,
//             .data = " col ",
//         },
//         Token{
//             .type = .table_sep,
//             .data = "|",
//         },
//         Token{
//             .type = .text,
//             .data = " col ",
//         },
//         Token{
//             .type = .table_sep,
//             .data = "|",
//         },
//         Token{
//             .type = .newline,
//             .data = "\n",
//         },
//         Token{
//             .type = .table_sep,
//             .data = "|",
//         },
//         Token{
//             .type = .table_row_sep,
//             .data = " --- ",
//         },
//         Token{
//             .type = .table_sep,
//             .data = "|",
//         },
//         Token{
//             .type = .table_row_sep,
//             .data = " --- ",
//         },
//         Token{
//             .type = .table_sep,
//             .data = "|",
//         },
//         Token{
//             .type = .table_row_sep,
//             .data = " --- ",
//         },
//         Token{
//             .type = .table_sep,
//             .data = "|",
//         },
//         Token{
//             .type = .newline,
//             .data = "\n",
//         },
//         Token{
//             .type = .table_sep,
//             .data = "|",
//         },
//         Token{
//             .type = .text,
//             .data = " row ",
//         },
//         Token{
//             .type = .table_sep,
//             .data = "|",
//         },
//         Token{
//             .type = .text,
//             .data = " row ",
//         },
//         Token{
//             .type = .table_sep,
//             .data = "|",
//         },
//         Token{
//             .type = .text,
//             .data = " row ",
//         },
//         Token{
//             .type = .table_sep,
//             .data = "|",
//         },
//         Token{
//             .type = .newline,
//             .data = "\n",
//         },
//         Token{
//             .type = .table_sep,
//             .data = "|",
//         },
//         Token{
//             .type = .text,
//             .data = " row ",
//         },
//         Token{
//             .type = .table_sep,
//             .data = "|",
//         },
//         Token{
//             .type = .text,
//             .data = " row ",
//         },
//         Token{
//             .type = .table_sep,
//             .data = "|",
//         },
//         Token{
//             .type = .text,
//             .data = " row ",
//         },
//         Token{
//             .type = .table_sep,
//             .data = "|",
//         },
//         Token{
//             .type = .newline,
//             .data = "\n",
//         },
//         Token{
//             .type = .table_sep,
//             .data = "|",
//         },
//         Token{
//             .type = .text,
//             .data = " row ",
//         },
//         Token{
//             .type = .table_sep,
//             .data = "|",
//         },
//         Token{
//             .type = .text,
//             .data = " row ",
//         },
//         Token{
//             .type = .table_sep,
//             .data = "|",
//         },
//         Token{
//             .type = .text,
//             .data = " row ",
//         },
//         Token{
//             .type = .table_sep,
//             .data = "|",
//         },
//         Token{
//             .type = .newline,
//             .data = "\n",
//         },
//         Token{
//             .type = .table_sep,
//             .data = "|",
//         },
//         Token{
//             .type = .text,
//             .data = " row ",
//         },
//         Token{
//             .type = .table_sep,
//             .data = "|",
//         },
//         Token{
//             .type = .text,
//             .data = " row ",
//         },
//         Token{
//             .type = .table_sep,
//             .data = "|",
//         },
//         Token{
//             .type = .text,
//             .data = " row ",
//         },
//         Token{
//             .type = .table_sep,
//             .data = "|",
//         },
//         Token{
//             .type = .newline,
//             .data = "\n",
//         },
//     };

//     var parsing = try tokenize(input, std.testing.allocator);
//     try expect(parsing.items.len >= parsed.len);

//     for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| {
//         try expect(expected_token.type == actual_token.type);
//         try expect(std.mem.eql(u8, expected_token.data, actual_token.data));
//     }

//     parsing.deinit(std.testing.allocator);
// }

test "quote_arrow" {
    const input = "> quote\n> quote\n> quote\n";
    const parsed = [_]Token{
        Token{
            .type = .quote,
            .data = ">",
        },
        Token{
            .type = .text,
            .data = "quote",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
        Token{
            .type = .quote,
            .data = ">",
        },
        Token{
            .type = .text,
            .data = "quote",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
        Token{
            .type = .quote,
            .data = ">",
        },
        Token{
            .type = .text,
            .data = "quote",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
    };

    var parsing = try tokenize(input, std.testing.allocator);
    try expect(parsing.items.len >= parsed.len);
    // for (parsing.items) |value| {
    //     std.debug.print("parsing : {f}\n", .{value});
    // }
    // for (parsed) |value| {
    //     std.debug.print("parsed : {f}\n", .{value});
    // }
    for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| {
        // std.debug.print("expected_token : {f}\n", .{expected_token});
        // std.debug.print("actual_token : {f}\n", .{actual_token});
        try expect(expected_token.type == actual_token.type);
        try expect(std.mem.eql(u8, expected_token.data, actual_token.data));
    }

    parsing.deinit(std.testing.allocator);
}

test "quote_bar" {
    const input = "| quote\n| quote\n| quote\n";
    const parsed = [_]Token{
        Token{
            .type = .quote,
            .data = "|",
        },
        Token{
            .type = .text,
            .data = "quote",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
        Token{
            .type = .quote,
            .data = "|",
        },
        Token{
            .type = .text,
            .data = "quote",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
        Token{
            .type = .quote,
            .data = "|",
        },
        Token{
            .type = .text,
            .data = "quote",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
    };

    var parsing = try tokenize(input, std.testing.allocator);
    try expect(parsing.items.len >= parsed.len);

    for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| {
        // std.debug.print("expected_token : {f}\n", .{expected_token});
        // std.debug.print("actual_token : {f}\n", .{actual_token});
        try expect(expected_token.type == actual_token.type);
        try expect(std.mem.eql(u8, expected_token.data, actual_token.data));
    }

    parsing.deinit(std.testing.allocator);
}

test "text_block" {
    const input = "```\nsome text or code\n```\n";
    const parsed = [_]Token{
        Token{
            .type = .code_block,
            .data = "```",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
        Token{
            .type = .raw_text,
            .data = "some text or code",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
        Token{
            .type = .code_block,
            .data = "```",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
    };

    var parsing = try tokenize(input, std.testing.allocator);
    // try expect(parsing.items.len >= parsed.len);

    for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| {
        // std.debug.print("expected_token : {f}\n", .{expected_token});
        // std.debug.print("actual_token : {f}\n", .{actual_token});
        try expect(expected_token.type == actual_token.type);
        try expect(std.mem.eql(u8, expected_token.data, actual_token.data));
    }

    parsing.deinit(std.testing.allocator);
}

test "code_block" {
    const input = "```lang_name\nsome text or code\n```\n";
    const parsed = [_]Token{
        Token{
            .type = .code_block,
            .data = "```",
        },
        Token{
            .type = .raw_text,
            .data = "lang_name",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
        Token{
            .type = .raw_text,
            .data = "some text or code",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
        Token{
            .type = .code_block,
            .data = "```",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
    };

    var parsing = try tokenize(input, std.testing.allocator);
    try expect(parsing.items.len >= parsed.len);

    for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| {
        try expect(expected_token.type == actual_token.type);
        try expect(std.mem.eql(u8, expected_token.data, actual_token.data));
    }

    parsing.deinit(std.testing.allocator);
}

test "passthrough_block" {
    const input = "```passthrough\nsome text or code\n```\n";
    const parsed = [_]Token{
        Token{
            .type = .code_block,
            .data = "```",
        },
        Token{
            .type = .raw_text,
            .data = "passthrough",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
        Token{
            .type = .raw_text,
            .data = "some text or code",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
        Token{
            .type = .code_block,
            .data = "```",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
    };

    var parsing = try tokenize(input, std.testing.allocator);
    try expect(parsing.items.len >= parsed.len);

    for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| {
        try expect(expected_token.type == actual_token.type);
        try expect(std.mem.eql(u8, expected_token.data, actual_token.data));
    }

    parsing.deinit(std.testing.allocator);
}

test "comment" {
    const input = "// this is a comment\n";
    const parsed = [_]Token{
        Token{
            .type = .comment,
            .data = "// ",
        },
        Token{
            .type = .raw_text,
            .data = "this is a comment",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
    };

    var parsing = try tokenize(input, std.testing.allocator);
    try expect(parsing.items.len >= parsed.len);

    for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| {
        try expect(expected_token.type == actual_token.type);
        try expect(std.mem.eql(u8, expected_token.data, actual_token.data));
    }

    parsing.deinit(std.testing.allocator);
}

test "tick_box_empty" {
    const input = "- [ ] something to check off\n";
    const parsed = [_]Token{
        Token{
            .type = .tickbox,
            .data = "- [ ]",
        },
        Token{
            .type = .text,
            .data = "something to check off",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
    };

    var parsing = try tokenize(input, std.testing.allocator);
    try expect(parsing.items.len >= parsed.len);

    for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| {
        // std.debug.print("expected_token {f}\n", .{expected_token});
        // std.debug.print("actual_token {f}\n", .{actual_token});
        try expect(expected_token.type == actual_token.type);
        try expect(std.mem.eql(u8, expected_token.data, actual_token.data));
    }

    parsing.deinit(std.testing.allocator);
}

test "tick_box_checked" {
    const input = "- [x] something checked off\n";
    const parsed = [_]Token{
        Token{
            .type = .tickbox,
            .data = "- [x]",
        },
        Token{
            .type = .text,
            .data = "something checked off",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
    };

    var parsing = try tokenize(input, std.testing.allocator);
    try expect(parsing.items.len >= parsed.len);

    for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| {
        try expect(expected_token.type == actual_token.type);
        try expect(std.mem.eql(u8, expected_token.data, actual_token.data));
    }

    parsing.deinit(std.testing.allocator);
}

test "tick_box_ticked" {
    const input = "- [/] something ticked off\n";
    const parsed = [_]Token{
        Token{
            .type = .tickbox,
            .data = "- [/]",
        },
        Token{
            .type = .text,
            .data = "something ticked off",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
    };

    var parsing = try tokenize(input, std.testing.allocator);
    try expect(parsing.items.len >= parsed.len);

    for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| {
        try expect(expected_token.type == actual_token.type);
        try expect(std.mem.eql(u8, expected_token.data, actual_token.data));
    }

    parsing.deinit(std.testing.allocator);
}

test "tick_box_dashed" {
    const input = "- [~] something dashed off\n";
    const parsed = [_]Token{
        Token{
            .type = .tickbox,
            .data = "- [~]",
        },
        Token{
            .type = .text,
            .data = "something dashed off",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
    };

    var parsing = try tokenize(input, std.testing.allocator);
    try expect(parsing.items.len >= parsed.len);

    for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| {
        try expect(expected_token.type == actual_token.type);
        try expect(std.mem.eql(u8, expected_token.data, actual_token.data));
    }

    parsing.deinit(std.testing.allocator);
}

test "horizontal_rule_straight_line" {
    const input = "---\n";
    const parsed = [_]Token{
        Token{
            .type = .rule,
            .data = "---",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
    };

    var parsing = try tokenize(input, std.testing.allocator);
    try expect(parsing.items.len >= parsed.len);

    for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| {
        // std.debug.print("expected_token {f}\n", .{expected_token});
        // std.debug.print("actual_token {f}\n", .{actual_token});
        try expect(expected_token.type == actual_token.type);
        try expect(std.mem.eql(u8, expected_token.data, actual_token.data));
    }

    parsing.deinit(std.testing.allocator);
}

test "horizontal_rule_double_straight_line" {
    const input = "===\n";
    const parsed = [_]Token{
        Token{
            .type = .rule,
            .data = "===",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
    };

    var parsing = try tokenize(input, std.testing.allocator);
    try expect(parsing.items.len >= parsed.len);

    for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| {
        try expect(expected_token.type == actual_token.type);
        try expect(std.mem.eql(u8, expected_token.data, actual_token.data));
    }

    parsing.deinit(std.testing.allocator);
}

test "horizontal_rule_big_dotted_straight_line" {
    const input = "***\n";
    const parsed = [_]Token{
        Token{
            .type = .rule,
            .data = "***",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
    };

    var parsing = try tokenize(input, std.testing.allocator);
    try expect(parsing.items.len >= parsed.len);

    for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| {
        try expect(expected_token.type == actual_token.type);
        try expect(std.mem.eql(u8, expected_token.data, actual_token.data));
    }

    parsing.deinit(std.testing.allocator);
}

test "horizontal_rule_small_dotted_line" {
    const input = "...\n";
    const parsed = [_]Token{
        Token{
            .type = .rule,
            .data = "...",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
    };

    var parsing = try tokenize(input, std.testing.allocator);
    try expect(parsing.items.len >= parsed.len);

    for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| {
        try expect(expected_token.type == actual_token.type);
        try expect(std.mem.eql(u8, expected_token.data, actual_token.data));
    }

    parsing.deinit(std.testing.allocator);
}

test "horizontal_rule_squiggly_line" {
    const input = "&&&\n";
    const parsed = [_]Token{
        Token{
            .type = .rule,
            .data = "&&&",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
    };

    var parsing = try tokenize(input, std.testing.allocator);
    try expect(parsing.items.len >= parsed.len);

    for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| {
        try expect(expected_token.type == actual_token.type);
        try expect(std.mem.eql(u8, expected_token.data, actual_token.data));
    }

    parsing.deinit(std.testing.allocator);
}

test "horizontal_rule_jagged_line" {
    const input = "^^^\n";
    const parsed = [_]Token{
        Token{
            .type = .rule,
            .data = "^^^",
        },
        Token{
            .type = .newline,
            .data = "\n",
        },
    };

    var parsing = try tokenize(input, std.testing.allocator);
    try expect(parsing.items.len >= parsed.len);

    for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| {
        try expect(expected_token.type == actual_token.type);
        try expect(std.mem.eql(u8, expected_token.data, actual_token.data));
    }

    parsing.deinit(std.testing.allocator);
}

// test "footnote" {
//     const input = "sometext [^1]\n\n[^1]: footnote\n";
//     const parsed = [_]Token{
//         Token{
//             .type = .text,
//             .data = "sometext ",
//         },
//         Token{
//             .type = .footnote,
//             .data = "[^1]",
//         },
//         Token{
//             .type = .newline,
//             .data = "\n",
//         },
//         Token{
//             .type = .newline,
//             .data = "\n",
//         },
//         Token{
//             .type = .footnote_def,
//             .data = "[^1]:",
//         },
//         Token{
//             .type = .text,
//             .data = "footnote",
//         },
//         Token{
//             .type = .newline,
//             .data = "\n",
//         },
//     };

//     var parsing = try tokenize(input, std.testing.allocator);
//     try expect(parsing.items.len >= parsed.len);

//     for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| {
//         try expect(expected_token.type == actual_token.type);
//         try expect(std.mem.eql(u8, expected_token.data, actual_token.data));
//     }

//     parsing.deinit(std.testing.allocator);
// }

// test "macro" {
//     const input = "@macro(arg, arg2)\n";
//     const parsed = [_]Token{
//         Token{
//             .type = .macro,
//             .data = "macro",
//         },
//         Token{
//             .type = .bracket_open,
//             .data = "(",
//         },
//         Token{
//             .type = .macro_arg,
//             .data = "arg",
//         },
//         Token{
//             .type = .arg_sep,
//             .data = ",",
//         },
//         Token{
//             .type = .macro_arg,
//             .data = "arg2",
//         },
//         Token{
//             .type = .bracket_close,
//             .data = ")",
//         },
//         Token{
//             .type = .newline,
//             .data = "\n",
//         },
//     };

//     var parsing = try tokenize(input, std.testing.allocator);
//     try expect(parsing.items.len >= parsed.len);

//     for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| {
//         try expect(expected_token.type == actual_token.type);
//         try expect(std.mem.eql(u8, expected_token.data, actual_token.data));
//     }

//     parsing.deinit(std.testing.allocator);
// }

// test "replace" {
//     const input = "@replace\n";
//     const parsed = [_]Token{
//         Token{
//             .type = .replace,
//             .data = "replace",
//         },
//         Token{
//             .type = .newline,
//             .data = "\n",
//         },
//     };

//     var parsing = try tokenize(input, std.testing.allocator);
//     try expect(parsing.items.len >= parsed.len);

//     for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| {
//         try expect(expected_token.type == actual_token.type);
//         try expect(std.mem.eql(u8, expected_token.data, actual_token.data));
//     }

//     parsing.deinit(std.testing.allocator);
// }

// test "inbulit" {
//     const input = "@.macro(arg, arg2)\n";
//     const parsed = [_]Token{
//         Token{
//             .type = .inbulit,
//             .data = "macro",
//         },
//         Token{
//             .type = .bracket_open,
//             .data = "(",
//         },
//         Token{
//             .type = .macro_arg,
//             .data = "arg",
//         },
//         Token{
//             .type = .arg_sep,
//             .data = ",",
//         },
//         Token{
//             .type = .macro_arg,
//             .data = "arg2",
//         },
//         Token{
//             .type = .bracket_close,
//             .data = ")",
//         },
//         Token{
//             .type = .newline,
//             .data = "\n",
//         },
//     };

//     var parsing = try tokenize(input, std.testing.allocator);
//     try expect(parsing.items.len >= parsed.len);

//     for (parsed, parsing.items[0..parsed.len]) |expected_token, actual_token| {
//         try expect(expected_token.type == actual_token.type);
//         try expect(std.mem.eql(u8, expected_token.data, actual_token.data));
//     }

//     parsing.deinit(std.testing.allocator);
// }