More major changes to the grammer.

Added annotation sublanguage for defining compiler directives. Also
added the syntax for import statements. Imports are not implemented, but
I'm currently working on that.

1 files changed, 46 insertions, 1 deletions

diff --git a/src/Language/Fiddle/Tokenizer.hs b/src/Language/Fiddle/Tokenizer.hs
index 08f5649..87f119f 100644
--- a/src/Language/Fiddle/Tokenizer.hs
+++ b/src/Language/Fiddle/Tokenizer.hs
@@ -27,6 +27,7 @@ data T
   | KWType
   | KWUnion
   | KWWo
+  | KWImport
   | TokColon
   | TokComma
   | TokComment !Text
@@ -41,8 +42,35 @@ data T
   | TokRBracket
   | TokRParen
   | TokSemi
+  | TokString !Text
+  | TokDirectiveStart -- [[
+  | TokDirectiveEnd -- ]]
   deriving (Eq, Ord, Show, Read)
 
+textOf :: T -> Maybe Text
+textOf t = do
+  case t of
+    KWAssertPos -> Just "assert_pos"
+    KWAt -> Just "at"
+    KWBits -> Just "bits"
+    KWEnum -> Just "enum"
+    KWInstance -> Just "instance"
+    KWLocation -> Just "location"
+    KWOption -> Just "option"
+    KWPackage -> Just "package"
+    KWReg -> Just "reg"
+    KWReserved -> Just "reserved"
+    KWRo -> Just "ro"
+    KWRw -> Just "rw"
+    KWStruct -> Just "struct"
+    KWType -> Just "type"
+    KWUnion -> Just "union"
+    KWWo -> Just "wo"
+    KWImport -> Just "import"
+    TokIdent i -> Just i
+    TokLitNum n -> Just n
+    _ -> Nothing
+
 data Token a = Token !T a
   deriving (Eq, Ord, Show, Functor)
 
@@ -66,6 +94,7 @@ parseToken = spaces *> tok parseToken' <* spaces
         "type" -> KWType
         "option" -> KWOption
         "package" -> KWPackage
+        "import" -> KWImport
         "reg" -> KWReg
         "ro" -> KWRo
         "wo" -> KWWo
@@ -77,6 +106,19 @@ parseToken = spaces *> tok parseToken' <* spaces
         (Data.Text.head -> h) | isDigit h -> TokLitNum str
         ident -> TokIdent ident
 
+    parseString = fmap (TokString . Data.Text.pack . concat) $ do
+      char '"'
+      manyTill
+        ( do
+            c <- anyChar
+            if c == '\\'
+              then do
+                c2 <- anyChar
+                return [c, c2]
+              else return [c]
+        )
+        (char '"')
+
     parseComment =
       try
         ( do
@@ -91,7 +133,9 @@ parseToken = spaces *> tok parseToken' <* spaces
 
     parseSymbol =
       choice
-        [ char ':' $> TokColon,
+        [ try (string "[[" $> TokDirectiveStart),
+          try (string "]]" $> TokDirectiveEnd),
+          char ':' $> TokColon,
           char ',' $> TokComma,
           char '=' $> TokEq,
           char '{' $> TokLBrace,
@@ -107,6 +151,7 @@ parseToken = spaces *> tok parseToken' <* spaces
 
     parseToken' =
       fmap (parseAlNumTok . Data.Text.pack) (many1 (alphaNum <|> char '_'))
+        <|> parseString
         <|> parseComment
         <|> parseSymbol