Add support for UTF-8

This adds a table-driven UTF-8 parser which only has a single branch for
the entire parser. UTF-8 support is essentially bolted onto the VTE
parser. Not the most elegant, but it does prevent the transition tables
from blowing up.

Instead of refactoring the syntax extension to handle both table
definitions, I've opted to copy/paste now for both simplicities sake and
because I can't see a clear path to a minimal shared solution.

1 files changed, 10 insertions, 5 deletions

diff --git a/codegen/src/main.rs b/codegen/src/main.rs
index 64bddd9..5f8d153 100644
--- a/codegen/src/main.rs
+++ b/codegen/src/main.rs
@@ -1,18 +1,23 @@
+#![allow(dead_code)]
 extern crate syntex;
 extern crate syntex_syntax;
 
 mod ext;
 
-#[path="../../src/definitions.rs"]
-pub mod definitions;
-
 use std::path::Path;
 
 fn main() {
+    // Expand VT parser state table
+    let mut registry = syntex::Registry::new();
+    ext::vt::register(&mut registry);
     let src = &Path::new("../src/table.rs.in");
     let dst = &Path::new("../src/table.rs");
+    registry.expand("vt_state_table", src, dst).expect("expand vt_stable_table ok");
 
+    // Expand UTF8 parser state table
     let mut registry = syntex::Registry::new();
-    ext::register(&mut registry);
-    registry.expand("state_table", src, dst).expect("expand stable_table ok");
+    ext::utf8::register(&mut registry);
+    let src = &Path::new("../src/utf8/table.rs.in");
+    let dst = &Path::new("../src/utf8/table.rs");
+    registry.expand("utf8_state_table", src, dst).expect("expand utf8_stable_table ok");
 }