module Language.Fiddle.Tokenizer where import Language.Fiddle.Types data T = KW_assert_pos | KW_at | KW_bittype | KW_enum | KW_location | KW_object | KW_objtype | KW_option | KW_package | KW_reg | KW_ro | KW_wo | Tok_colon | Tok_comma | Tok_comment | Tok_docComment | Tok_eq | Tok_ident String | Tok_lbrace | Tok_lbracket | Tok_litnum String | Tok_lparen | Tok_package | Tok_rbrace | Tok_rbracket | Tok_rparen | Tok_semi data Token = Token T SourceSpan tokenize :: String -> Text -> [Token] tokenize srcName txt = undefined