fiddle: have a basic tokenizer working.

7 files changed, 217 insertions, 38 deletions

diff --git a/goal.fiddle b/goal.fiddle
index 1289c87..811719c 100644
--- a/goal.fiddle
+++ b/goal.fiddle
@@ -1,13 +1,15 @@
+option endian little;
+
 // package for the GPIO system.
 package gpio {
 
   location gpio_a_base = 0x4800_0000;
   location gpio_b_base = 0x4800_0400;
-  location gpio_c_base = 0x4800_0400;
+  location gpio_c_base = 0x4800_0800;
 
   /** IO Data. This is just an expressive boolean. */
   bittype data_t : enum(1) {
-    low = 0,
+    low =stream 0,
     high = 1,
   }
 
diff --git a/package.yaml b/package.yaml
new file mode 100644
index 0000000..0055d35
--- /dev/null
+++ b/package.yaml
@@ -0,0 +1,30 @@
+name: fiddle
+version: 0.5
+
+executables:
+  fiddlec:
+    main: Main.hs
+    source-dirs: src
+
+ghc-options:
+  - -XBangPatterns
+  - -XDataKinds
+  - -XFlexibleContexts
+  - -XFlexibleInstances
+  - -XGADTs
+  - -XKindSignatures
+  - -XMultiParamTypeClasses
+  - -XPolyKinds
+  - -XRankNTypes
+  - -XGeneralizedNewtypeDeriving
+  - -XStandaloneDeriving
+  - -XTupleSections
+  - -XTypeFamilies
+  - -XViewPatterns
+  - -XLambdaCase
+  - -XStrictData
+
+dependencies:
+  - base >= 4.0.0
+  - parsec
+  - text
diff --git a/src/Language/Fiddle/Tokenizer.hs b/src/Language/Fiddle/Tokenizer.hs
index 9931523..d3239fd 100644
--- a/src/Language/Fiddle/Tokenizer.hs
+++ b/src/Language/Fiddle/Tokenizer.hs
@@ -1,37 +1,101 @@
+{-# LANGUAGE DeriveFunctor #-}
 module Language.Fiddle.Tokenizer where
 
+import Data.Char (isDigit)
+import Data.Text (Text)
 import Language.Fiddle.Types
+import Text.Parsec
+import qualified Text.Parsec
 
 data T
-  = KW_assert_pos
-  | KW_at
-  | KW_bittype
-  | KW_enum
-  | KW_location
-  | KW_object
-  | KW_objtype
-  | KW_option
-  | KW_package
-  | KW_reg
-  | KW_ro
-  | KW_wo
-  | Tok_colon
-  | Tok_comma
-  | Tok_comment
-  | Tok_docComment
-  | Tok_eq
-  | Tok_ident String
-  | Tok_lbrace
-  | Tok_lbracket
-  | Tok_litnum String
-  | Tok_lparen
-  | Tok_package
-  | Tok_rbrace
-  | Tok_rbracket
-  | Tok_rparen
-  | Tok_semi
-
-data Token = Token T SourceSpan
-
-tokenize :: String -> Text -> [Token]
-tokenize srcName txt = undefined
+  = KWAssertPos
+  | Ident !String
+  | KWAt
+  | KWBittype
+  | KWEnum
+  | CommentTok !String
+  | DocCommentTok !String
+  | KWLocation
+  | KWObject
+  | KWObjtype
+  | KWOption
+  | KWPackage
+  | KWReg
+  | KWRo
+  | KWWo
+  | LitNum !String
+  | TokColon
+  | TokComma
+  | TokEq
+  | TokLBrace
+  | TokLBracket
+  | TokLParen
+  | TokRBrace
+  | TokRBracket
+  | TokRParen
+  | TokSemi
+  deriving (Eq, Ord, Show, Read)
+
+data Token a = Token !T a
+  deriving (Eq, Ord, Show, Functor)
+
+parseToken :: (Monad m) => ParsecT Text u m (Token SourceSpan)
+parseToken = spaces *> tok parseToken' <* spaces
+  where
+    tok tp = do
+      p1 <- getPosition
+      t <- tp
+
+      Token t . SourceSpan p1 <$> getPosition
+
+    parseAlNumTok str =
+      case str of
+        "at" -> KWAt
+        "bittype" -> KWBittype
+        "enum" -> KWEnum
+        "location" -> KWLocation
+        "object" -> KWObject
+        "objtype" -> KWObjtype
+        "option" -> KWOption
+        "package" -> KWPackage
+        "reg" -> KWReg
+        "ro" -> KWRo
+        "wo" -> KWWo
+        (h : _) | isDigit h -> LitNum str
+        ident -> Ident ident
+
+    parseComment =
+      try
+        ( do
+            string "//"
+            CommentTok <$> manyTill anyChar (char '\n')
+        )
+        <|> try
+          ( do
+              string "/**"
+              DocCommentTok <$> manyTill anyChar (try $ string "*/")
+          )
+
+    parseSymbol =
+      choice
+        [ char ':' $> TokColon,
+          char ',' $> TokComma,
+          char '=' $> TokEq,
+          char '{' $> TokLBrace,
+          char '[' $> TokLBracket,
+          char '(' $> TokLParen,
+          char '}' $> TokRBrace,
+          char ']' $> TokRBracket,
+          char ')' $> TokRParen,
+          char ';' $> TokSemi
+        ]
+      where
+        a $> b = a >> return b
+
+    parseToken' =
+      fmap parseAlNumTok (many1 (alphaNum <|> char '_'))
+        <|> parseComment
+        <|> parseSymbol
+
+tokenize :: String -> Text -> Either ParseError [Token SourceSpan]
+tokenize = Text.Parsec.runParser (many parseToken <* eof) ()
diff --git a/src/Language/Fiddle/Types.hs b/src/Language/Fiddle/Types.hs
index 62538d8..c83bef2 100644
--- a/src/Language/Fiddle/Types.hs
+++ b/src/Language/Fiddle/Types.hs
@@ -1,6 +1,7 @@
 module Language.Fiddle.Types where
 
-import Text.Parsec (SourceSpan)
+import Text.Parsec (SourcePos)
+import Data.Text (Text)
 
 newtype Comment = Comment Text
 
@@ -8,5 +9,6 @@ data SourceSpan = SourceSpan
   { sourceStart :: !SourcePos,
     sourceStop :: !SourcePos
   }
+  deriving (Eq, Ord, Show)
 
-data Metadata = Metadata SourceSpan Comment
+data Metadata = Metadata !SourceSpan !Comment
diff --git a/src/Main.hs b/src/Main.hs
index 9249638..226182a 100644
--- a/src/Main.hs
+++ b/src/Main.hs
@@ -1,4 +1,19 @@
 module Main where
 
+import qualified Language.Fiddle.Tokenizer
+import qualified Data.Text.IO
+import qualified System.Environment as System
+import Control.Monad (forM_)
+
 main :: IO ()
-main = putStrLn "Hello, World"
+main = do
+  argv <- System.getArgs
+
+  case argv of
+    [filePath] -> do
+       text <- Data.Text.IO.readFile filePath
+       case Language.Fiddle.Tokenizer.tokenize filePath text of
+         Left pe -> putStrLn $  "Parse Error: " ++ show pe
+         Right lst -> forM_ lst $ \(Language.Fiddle.Tokenizer.Token t _) -> print t
+
+    _ -> putStrLn "Wrong Args"
diff --git a/stack.yaml b/stack.yaml
new file mode 100644
index 0000000..b6e406d
--- /dev/null
+++ b/stack.yaml
@@ -0,0 +1,66 @@
+# This file was automatically generated by 'stack init'
+#
+# Some commonly used options have been documented as comments in this file.
+# For advanced use and comprehensive documentation of the format, please see:
+# https://docs.haskellstack.org/en/stable/yaml_configuration/
+
+# Resolver to choose a 'specific' stackage snapshot or a compiler version.
+# A snapshot resolver dictates the compiler version and the set of packages
+# to be used for project dependencies. For example:
+#
+# resolver: lts-3.5
+# resolver: nightly-2015-09-21
+# resolver: ghc-7.10.2
+#
+# The location of a snapshot can be provided as a file or url. Stack assumes
+# a snapshot provided as a file might change, whereas a url resource does not.
+#
+# resolver: ./custom-snapshot.yaml
+# resolver: https://example.com/snapshots/2018-01-01.yaml
+resolver: lts-18.15
+
+# User packages to be built.
+# Various formats can be used as shown in the example below.
+#
+# packages:
+# - some-directory
+# - https://example.com/foo/bar/baz-0.0.2.tar.gz
+#   subdirs:
+#   - auto-update
+#   - wai
+packages:
+  - .
+# Dependency packages to be pulled from upstream that are not in the resolver.
+# These entries can reference officially published versions as well as
+# forks / in-progress versions pinned to a git hash. For example:
+#
+# extra-deps:
+# - acme-missiles-0.3
+# - git: https://github.com/commercialhaskell/stack.git
+#   commit: e7b331f14bcffb8367cd58fbfc8b40ec7642100a
+#
+# extra-deps: []
+
+# Override default flag values for local packages and extra-deps
+# flags: {}
+
+# Extra package databases containing global packages
+# extra-package-dbs: []
+
+# Control whether we use the GHC we find on the path
+# system-ghc: true
+#
+# Require a specific version of stack, using version ranges
+# require-stack-version: -any # Default
+# require-stack-version: ">=2.7"
+#
+# Override the architecture used by stack, especially useful on Windows
+# arch: i386
+# arch: x86_64
+#
+# Extra directories used by stack for building
+# extra-include-dirs: [/path/to/dir]
+# extra-lib-dirs: [/path/to/dir]
+#
+# Allow a newer minor version of GHC than the snapshot specifies
+# compiler-check: newer-minor
diff --git a/vim/syntax/fiddle.vim b/vim/syntax/fiddle.vim
index 4bd192c..e2a3fe8 100644
--- a/vim/syntax/fiddle.vim
+++ b/vim/syntax/fiddle.vim
@@ -1,4 +1,4 @@
-syn keyword FiddlePackage package nextgroup=FiddleIdent skipwhite
+syn keyword FiddlePackage option package nextgroup=FiddleIdent skipwhite
 syn keyword FiddleDecl reg object at location reserved nextgroup=FiddleIdent skipwhite
 syn keyword FiddleTypeDecl objtype regtype bittype nextgroup=FiddleIdent skipwhite
 syn keyword FiddleEnum enum