so... jai :D

src/token.zig

@@ -0,0 +1,175 @@
+pub const Tag = enum {
+    // Literals
+    int_literal,
+    float_literal,
+    string_literal,
+
+    // Identifiers and keywords
+    identifier,
+    kw_if,
+    kw_else,
+    kw_then,
+    kw_true,
+    kw_false,
+    kw_enum,
+    kw_case,
+    kw_break,
+    kw_continue,
+    kw_while,
+    kw_for,
+    kw_return,
+    kw_defer,
+    kw_f32,
+    kw_f64,
+    kw_struct,
+    kw_union,
+    kw_xx,
+    kw_and,
+    kw_or,
+    kw_Type, // Type (metatype keyword)
+
+    // Symbols
+    colon, // :
+    colon_colon, // ::
+    colon_equal, // :=
+    semicolon, // ;
+    comma, // ,
+    dot, // .
+    dot_dot, // ..
+    dollar, // $
+
+    // Operators
+    plus, // +
+    minus, // -
+    star, // *
+    slash, // /
+    equal, // =
+    equal_equal, // ==
+    bang, // !
+    bang_equal, // !=
+    less, // <
+    less_equal, // <=
+    greater, // >
+    greater_equal, // >=
+    plus_equal, // +=
+    minus_equal, // -=
+    star_equal, // *=
+    slash_equal, // /=
+    percent, // %
+    percent_equal, // %=
+
+    // Delimiters
+    l_paren, // (
+    r_paren, // )
+    l_brace, // {
+    r_brace, // }
+    l_bracket, // [
+    r_bracket, // ]
+
+    // Arrows
+    arrow, // ->
+    fat_arrow, // =>
+
+    // Directives
+    hash_run, // #run
+    hash_import, // #import
+    hash_insert, // #insert
+    hash_builtin, // #builtin
+    triple_minus, // ---
+
+    // Special
+    eof,
+    invalid,
+
+    pub fn lexeme(tag: Tag) ?[]const u8 {
+        return switch (tag) {
+            .colon => ":",
+            .colon_colon => "::",
+            .colon_equal => ":=",
+            .semicolon => ";",
+            .comma => ",",
+            .dot => ".",
+            .dot_dot => "..",
+            .dollar => "$",
+            .plus => "+",
+            .minus => "-",
+            .star => "*",
+            .slash => "/",
+            .equal => "=",
+            .equal_equal => "==",
+            .bang => "!",
+            .bang_equal => "!=",
+            .less => "<",
+            .less_equal => "<=",
+            .greater => ">",
+            .greater_equal => ">=",
+            .plus_equal => "+=",
+            .minus_equal => "-=",
+            .star_equal => "*=",
+            .slash_equal => "/=",
+            .percent => "%",
+            .percent_equal => "%=",
+            .l_paren => "(",
+            .r_paren => ")",
+            .l_brace => "{",
+            .r_brace => "}",
+            .l_bracket => "[",
+            .r_bracket => "]",
+            .arrow => "->",
+            .fat_arrow => "=>",
+            .triple_minus => "---",
+            else => null,
+        };
+    }
+
+    pub fn isTypeKeyword(tag: Tag) bool {
+        return switch (tag) {
+            .kw_f32, .kw_f64, .kw_Type => true,
+            else => false,
+        };
+    }
+};
+
+pub const Token = struct {
+    tag: Tag,
+    loc: Loc,
+
+    pub const Loc = struct {
+        start: u32,
+        end: u32,
+    };
+
+    pub fn slice(self: Token, source: []const u8) []const u8 {
+        return source[self.loc.start..self.loc.end];
+    }
+};
+
+pub const keywords = std.StaticStringMap(Tag).initComptime(.{
+    .{ "if", .kw_if },
+    .{ "else", .kw_else },
+    .{ "then", .kw_then },
+    .{ "true", .kw_true },
+    .{ "false", .kw_false },
+    .{ "enum", .kw_enum },
+    .{ "case", .kw_case },
+    .{ "break", .kw_break },
+    .{ "continue", .kw_continue },
+    .{ "while", .kw_while },
+    .{ "for", .kw_for },
+    .{ "return", .kw_return },
+    .{ "defer", .kw_defer },
+    .{ "f32", .kw_f32 },
+    .{ "f64", .kw_f64 },
+    .{ "struct", .kw_struct },
+    .{ "union", .kw_union },
+    .{ "xx", .kw_xx },
+    .{ "and", .kw_and },
+    .{ "or", .kw_or },
+    .{ "Type", .kw_Type },
+});
+
+pub fn getKeyword(bytes: []const u8) ?Tag {
+    return keywords.get(bytes);
+}
+
+const std = @import("std");