Move utf8 parsing into separate crate

9 files changed, 19 insertions, 9 deletions

diff --git a/Cargo.toml b/Cargo.toml
index 1e38f87..494ece6 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -3,4 +3,5 @@ name = "vte"
 version = "0.1.0"
 authors = ["Joe Wilm <joe@jwilm.com>"]
 
-[dependencies]
+[dependencies.utf8parse]
+path = "./utf8parse"
diff --git a/codegen/src/ext/utf8.rs b/codegen/src/ext/utf8.rs
index 5b73081..17ebc22 100644
--- a/codegen/src/ext/utf8.rs
+++ b/codegen/src/ext/utf8.rs
@@ -13,7 +13,7 @@ use syntex_syntax::parse::PResult;
 use syntex_syntax::ptr::P;
 use syntex_syntax::tokenstream::TokenTree;
 
-#[path="../../../src/utf8/types.rs"]
+#[path="../../../utf8parse/src/types.rs"]
 mod types;
 
 use self::types::{State, Action, pack};
diff --git a/codegen/src/main.rs b/codegen/src/main.rs
index 5f8d153..0d21577 100644
--- a/codegen/src/main.rs
+++ b/codegen/src/main.rs
@@ -17,7 +17,7 @@ fn main() {
     // Expand UTF8 parser state table
     let mut registry = syntex::Registry::new();
     ext::utf8::register(&mut registry);
-    let src = &Path::new("../src/utf8/table.rs.in");
-    let dst = &Path::new("../src/utf8/table.rs");
+    let src = &Path::new("../utf8parse/src/table.rs.in");
+    let dst = &Path::new("../utf8parse/src/table.rs");
     registry.expand("utf8_state_table", src, dst).expect("expand utf8_stable_table ok");
 }
diff --git a/src/lib.rs b/src/lib.rs
index 71953fd..f449e13 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -23,9 +23,10 @@
 //! [`Parser`]: struct.Parser.html
 //! [`Perform`]: trait.Perform.html
 //! [Paul Williams' ANSI parser state machine]: http://vt100.net/emu/dec_ansi_parser
+extern crate utf8parse as utf8;
+
 mod table;
 mod definitions;
-mod utf8;
 
 use definitions::{Action, State, unpack};
 
diff --git a/utf8parse/Cargo.toml b/utf8parse/Cargo.toml
new file mode 100644
index 0000000..882b108
--- /dev/null
+++ b/utf8parse/Cargo.toml
@@ -0,0 +1,6 @@
+[package]
+name = "utf8parse"
+version = "0.1.0"
+authors = ["Joe Wilm <joe@jwilm.com>"]
+
+[dependencies]
diff --git a/src/utf8/mod.rs b/utf8parse/src/lib.rs
index 3d099b1..9585642 100644
--- a/src/utf8/mod.rs
+++ b/utf8parse/src/lib.rs
@@ -13,12 +13,10 @@ use self::table::TRANSITIONS;
 
 /// Handles codepoint and invalid sequence events from the parser.
 pub trait Receiver {
-    /// Code point parsed
-    ///
-    /// Called with the codepoint
+    /// Called whenever a codepoint is parsed successfully
     fn codepoint(&mut self, char);
 
-    /// Invalid sequence encountered
+    /// Called when an invalid_sequence is detected
     fn invalid_sequence(&mut self);
 }
 
@@ -42,6 +40,10 @@ impl Parser {
         }
     }
 
+    /// Advance the parser
+    ///
+    /// The provider receiver will be called whenever a codepoint is completed or an invalid
+    /// sequence is detected.
     pub fn advance<R>(&mut self, receiver: &mut R, byte: u8)
         where R: Receiver
     {
diff --git a/src/utf8/table.rs b/utf8parse/src/table.rs
index 5a1292b..5a1292b 100644
--- a/src/utf8/table.rs
+++ b/utf8parse/src/table.rs
diff --git a/src/utf8/table.rs.in b/utf8parse/src/table.rs.in
index 2acafe7..2acafe7 100644
--- a/src/utf8/table.rs.in
+++ b/utf8parse/src/table.rs.in
diff --git a/src/utf8/types.rs b/utf8parse/src/types.rs
index 4c604f4..4c604f4 100644
--- a/src/utf8/types.rs
+++ b/utf8parse/src/types.rs