diff options
| author | Christian Duerr <contact@christianduerr.com> | 2019-12-10 19:16:01 +0100 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2019-12-10 19:16:01 +0100 |
| commit | 9d37aa7a71801f3569d2a2a55dc82c37935f205a (patch) | |
| tree | fd20b01398034934957c0d311209103482836771 | |
| parent | ea940fcb74abce67b927788e4f9f64fc63073d37 (diff) | |
| download | r-alacritty-vte-9d37aa7a71801f3569d2a2a55dc82c37935f205a.tar.gz r-alacritty-vte-9d37aa7a71801f3569d2a2a55dc82c37935f205a.tar.bz2 r-alacritty-vte-9d37aa7a71801f3569d2a2a55dc82c37935f205a.zip | |
Remove table generation
This completely removes the `codegen` project, which relied on outdated
libraries to parse DSLs to build the utf8 and vte state tables, to make
the library easier to maintain.
The utf8 table could be completely removed in favor of a `match`
statement, which also lead to a performance improvement with the utf8
parser.
The vte table did not benefit from `match` statements at all and instead
had significantly worse performance with it. To replace the old
codegeneration for vte, the `generate_state_changes` crate has been
created instead, which uses the language's proc_macro feature to create
a `const fn` which will generate the table at compile time.
| -rw-r--r-- | Cargo.toml | 4 | ||||
| -rw-r--r-- | codegen/Cargo.toml | 9 | ||||
| -rw-r--r-- | codegen/README.md | 11 | ||||
| -rw-r--r-- | codegen/src/ext/mod.rs | 2 | ||||
| -rw-r--r-- | codegen/src/ext/utf8.rs | 365 | ||||
| -rw-r--r-- | codegen/src/ext/vt.rs | 393 | ||||
| -rw-r--r-- | codegen/src/main.rs | 23 | ||||
| -rw-r--r-- | generate_state_changes/Cargo.toml | 12 | ||||
| -rw-r--r-- | generate_state_changes/src/lib.rs | 174 | ||||
| -rw-r--r-- | src/definitions.rs | 63 | ||||
| -rw-r--r-- | src/lib.rs | 77 | ||||
| -rw-r--r-- | src/table.rs | 508 | ||||
| -rw-r--r-- | src/table.rs.in | 30 | ||||
| -rw-r--r-- | tests/demo.vte | 2052 | ||||
| -rw-r--r-- | utf8parse/Cargo.toml | 4 | ||||
| -rw-r--r-- | utf8parse/src/lib.rs | 50 | ||||
| -rw-r--r-- | utf8parse/src/table.rs | 166 | ||||
| -rw-r--r-- | utf8parse/src/table.rs.in | 60 | ||||
| -rw-r--r-- | utf8parse/src/types.rs | 121 |
19 files changed, 2660 insertions, 1464 deletions
@@ -13,12 +13,14 @@ name = "vte" edition = "2018" [dependencies] +generate_state_changes = { version = "0.1.0", path = "generate_state_changes" } arrayvec = { version = "0.5.1", default-features = false, optional = true } utf8parse = { path = "utf8parse", version = "0.1.0" } [features] default = ["no_std"] no_std = ["arrayvec"] +nightly = ["utf8parse/nightly"] [workspace] -members = ["utf8parse", "codegen"] +members = ["utf8parse", "generate_state_changes"] diff --git a/codegen/Cargo.toml b/codegen/Cargo.toml deleted file mode 100644 index ec5a741..0000000 --- a/codegen/Cargo.toml +++ /dev/null @@ -1,9 +0,0 @@ -[package] -name = "codegen" -version = "0.1.0" -authors = ["Joe Wilm <joe@jwilm.com>"] -edition = "2018" - -[dependencies] -syntex = "0.43.0" -syntex_syntax = "0.43.0" diff --git a/codegen/README.md b/codegen/README.md deleted file mode 100644 index d13aa5d..0000000 --- a/codegen/README.md +++ /dev/null @@ -1,11 +0,0 @@ -codegen -======= - -Depends on libsyntex and generates table.rs from table.rs.in. This code is -separate from the main vtparse crate since compiling libsyntex takes ~1 -eternity. - -## Usage - -`cargo run` in the codegen folder will process `table.rs.in` and output -`table.rs`. The latter file should be committed back into the repo. diff --git a/codegen/src/ext/mod.rs b/codegen/src/ext/mod.rs deleted file mode 100644 index c28d9f7..0000000 --- a/codegen/src/ext/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub mod utf8; -pub mod vt; diff --git a/codegen/src/ext/utf8.rs b/codegen/src/ext/utf8.rs deleted file mode 100644 index 898722b..0000000 --- a/codegen/src/ext/utf8.rs +++ /dev/null @@ -1,365 +0,0 @@ -//! Macro expansion for the utf8 parser state table -use std::fmt; - -use syntex::Registry; - -use syntex_syntax::ast::{self, Arm, Expr, ExprKind, LitKind, Pat, PatKind}; -use syntex_syntax::codemap::Span; -use syntex_syntax::ext::base::{DummyResult, ExtCtxt, MacEager, MacResult}; -use syntex_syntax::ext::build::AstBuilder; -use syntex_syntax::parse::parser::Parser; -use syntex_syntax::parse::token::{DelimToken, Token}; -use syntex_syntax::parse::PResult; -use syntex_syntax::ptr::P; -use syntex_syntax::tokenstream::TokenTree; - -#[path = "../../../utf8parse/src/types.rs"] -mod types; - -use self::types::{pack, Action, State}; - -pub fn register(registry: &mut Registry) { - registry.add_macro("utf8_state_table", expand_state_table); -} - -fn state_from_str<S>(s: &S) -> Result<State, ()> -where - S: AsRef<str>, -{ - Ok(match s.as_ref() { - "State::Ground" => State::Ground, - "State::Tail3" => State::Tail3, - "State::Tail2" => State::Tail2, - "State::Tail1" => State::Tail1, - "State::U3_2_e0" => State::U3_2_e0, - "State::U3_2_ed" => State::U3_2_ed, - "State::Utf8_4_3_f0" => State::Utf8_4_3_f0, - "State::Utf8_4_3_f4" => State::Utf8_4_3_f4, - _ => return Err(()), - }) -} - -fn action_from_str<S>(s: &S) -> Result<Action, ()> -where - S: AsRef<str>, -{ - Ok(match s.as_ref() { - "Action::InvalidSequence" => Action::InvalidSequence, - "Action::EmitByte" => Action::EmitByte, - "Action::SetByte1" => Action::SetByte1, - "Action::SetByte2" => Action::SetByte2, - "Action::SetByte2Top" => Action::SetByte2Top, - "Action::SetByte3" => Action::SetByte3, - "Action::SetByte3Top" => Action::SetByte3Top, - "Action::SetByte4" => Action::SetByte4, - _ => return Err(()), - }) -} - -fn parse_table_input_mappings<'a>(parser: &mut Parser<'a>) -> PResult<'a, Vec<Arm>> { - // Must start on open brace - parser.expect(&Token::OpenDelim(DelimToken::Brace))?; - - let mut arms: Vec<Arm> = Vec::new(); - while parser.token != Token::CloseDelim(DelimToken::Brace) { - match parser.parse_arm() { - Ok(arm) => arms.push(arm), - Err(e) => { - // Recover by skipping to the end of the block. - return Err(e); - }, - } - } - - // Consume the closing brace - parser.bump(); - Ok(arms) -} - -/// Expressions describing state transitions and actions -#[derive(Debug)] -struct TableDefinitionExprs { - state_expr: P<Expr>, - mapping_arms: Vec<Arm>, -} - -fn state_from_expr(expr: P<Expr>, cx: &mut ExtCtxt) -> Result<State, ()> { - let s = match expr.node { - ExprKind::Path(ref _qself, ref path) => path.to_string(), - _ => { - cx.span_err(expr.span, "expected State"); - return Err(()); - }, - }; - - state_from_str(&s).map_err(|_| { - cx.span_err(expr.span, "expected State"); - }) -} - -fn u8_lit_from_expr(expr: &Expr, cx: &mut ExtCtxt) -> Result<u8, ()> { - static MSG: &str = "expected u8 int literal"; - - match expr.node { - ExprKind::Lit(ref lit) => match lit.node { - LitKind::Int(val, _) => Ok(val as u8), - _ => { - cx.span_err(lit.span, MSG); - Err(()) - }, - }, - _ => { - cx.span_err(expr.span, MSG); - Err(()) - }, - } -} - -fn input_mapping_from_arm(arm: Arm, cx: &mut ExtCtxt) -> Result<InputMapping, ()> { - let Arm { pats, body, .. } = arm; - - let input = InputDefinition::from_pat(&pats[0], cx)?; - let transition = Transition::from_expr(&body, cx)?; - - Ok(InputMapping { input, transition }) -} - -/// What happens when certain input is received -#[derive(Copy, Clone)] -enum Transition { - State(State), - Action(Action), - StateAction(State, Action), -} - -impl fmt::Debug for Transition { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match *self { - Transition::State(state) => write!(f, "State({:?})", state)?, - Transition::Action(action) => write!(f, "Action({:?})", action)?, - Transition::StateAction(state, action) => { - write!(f, "StateAction({:?}, {:?})", state, action)?; - }, - } - - write!(f, " -> {:?}", self.pack_u8()) - } -} - -impl Transition { - // State is stored in the top 4 bits - fn pack_u8(self) -> u8 { - match self { - Transition::State(state) => pack(state, Action::InvalidSequence), - Transition::Action(action) => pack(State::Ground, action), - Transition::StateAction(state, action) => pack(state, action), - } - } -} - -impl Transition { - fn from_expr(expr: &Expr, cx: &mut ExtCtxt) -> Result<Transition, ()> { - match expr.node { - ExprKind::Tup(ref tup_exprs) => { - let mut action = None; - let mut state = None; - - for tup_expr in tup_exprs { - if let ExprKind::Path(_, ref path) = tup_expr.node { - let path_str = path.to_string(); - if path_str.starts_with('A') { - action = Some(action_from_str(&path_str).map_err(|_| { - cx.span_err(expr.span, "invalid action"); - })?); - } else { - state = Some(state_from_str(&path_str).map_err(|_| { - cx.span_err(expr.span, "invalid state"); - })?); - } - } - } - - match (action, state) { - (Some(action), Some(state)) => Ok(Transition::StateAction(state, action)), - (None, Some(state)) => Ok(Transition::State(state)), - (Some(action), None) => Ok(Transition::Action(action)), - _ => { - cx.span_err(expr.span, "expected Action and/or State"); - Err(()) - }, - } - }, - ExprKind::Path(_, ref path) => { - // Path can be Action or State - let path_str = path.to_string(); - - if path_str.starts_with('A') { - let action = action_from_str(&path_str).map_err(|_| { - cx.span_err(expr.span, "invalid action"); - })?; - Ok(Transition::Action(action)) - } else { - let state = state_from_str(&path_str).map_err(|_| { - cx.span_err(expr.span, "invalid state"); - })?; - - Ok(Transition::State(state)) - } - }, - _ => { - cx.span_err(expr.span, "expected Action and/or State"); - Err(()) - }, - } - } -} - -#[derive(Debug)] -enum InputDefinition { - Specific(u8), - Range { start: u8, end: u8 }, -} - -impl InputDefinition { - fn from_pat(pat: &Pat, cx: &mut ExtCtxt) -> Result<InputDefinition, ()> { - Ok(match pat.node { - PatKind::Lit(ref lit_expr) => { - InputDefinition::Specific(u8_lit_from_expr(&lit_expr, cx)?) - }, - PatKind::Range(ref start_expr, ref end_expr) => InputDefinition::Range { - start: u8_lit_from_expr(start_expr, cx)?, - end: u8_lit_from_expr(end_expr, cx)?, - }, - _ => { - cx.span_err(pat.span, "expected literal or range expression"); - return Err(()); - }, - }) - } -} - -#[derive(Debug)] -struct InputMapping { - input: InputDefinition, - transition: Transition, -} - -#[derive(Debug)] -struct TableDefinition { - state: State, - mappings: Vec<InputMapping>, -} - -fn parse_raw_definitions( - definitions: Vec<TableDefinitionExprs>, - cx: &mut ExtCtxt, -) -> Result<Vec<TableDefinition>, ()> { - let mut out = Vec::new(); - - for raw in definitions { - let TableDefinitionExprs { state_expr, mapping_arms } = raw; - let state = state_from_expr(state_expr, cx)?; - - let mut mappings = Vec::new(); - for arm in mapping_arms { - mappings.push(input_mapping_from_arm(arm, cx)?); - } - - out.push(TableDefinition { state, mappings }) - } - - Ok(out) -} - -fn parse_table_definition<'a>(parser: &mut Parser<'a>) -> PResult<'a, TableDefinitionExprs> { - let state_expr = parser.parse_expr()?; - parser.expect(&Token::FatArrow)?; - let mappings = parse_table_input_mappings(parser)?; - - Ok(TableDefinitionExprs { state_expr, mapping_arms: mappings }) -} - -fn parse_table_definition_list<'a>( - parser: &mut Parser<'a>, -) -> PResult<'a, Vec<TableDefinitionExprs>> { - let mut definitions = Vec::new(); - while parser.token != Token::Eof { - definitions.push(parse_table_definition(parser)?); - parser.eat(&Token::Comma); - } - - Ok(definitions) -} - -fn build_state_tables<T>(defs: T) -> [[u8; 256]; 8] -where - T: AsRef<[TableDefinition]>, -{ - let mut result = [[0u8; 256]; 8]; - - for def in defs.as_ref() { - let state = def.state; - let state = state as u8; - let transitions = &mut result[state as usize]; - - for mapping in &def.mappings { - let trans = mapping.transition.pack_u8(); - match mapping.input { - InputDefinition::Specific(idx) => { - transitions[idx as usize] = trans; - }, - InputDefinition::Range { start, end } => { - for idx in start..end { - transitions[idx as usize] = trans; - } - transitions[end as usize] = trans; - }, - } - } - } - - result -} - -fn build_table_ast(cx: &mut ExtCtxt, sp: Span, table: [[u8; 256]; 8]) -> P<ast::Expr> { - let table = table - .iter() - .map(|list| { - let exprs = list.iter().map(|num| cx.expr_u8(sp, *num)).collect(); - cx.expr_vec(sp, exprs) - }) - .collect(); - - cx.expr_vec(sp, table) -} - -fn expand_state_table<'cx>( - cx: &'cx mut ExtCtxt, - sp: Span, - args: &[TokenTree], -) -> Box<dyn MacResult + 'cx> { - macro_rules! ptry { - ($pres:expr) => { - match $pres { - Ok(val) => val, - Err(mut err) => { - err.emit(); - return DummyResult::any(sp); - }, - } - }; - } - - // Parse the lookup spec - let mut parser: Parser = cx.new_parser_from_tts(args); - let definitions = ptry!(parse_table_definition_list(&mut parser)); - let definitions = match parse_raw_definitions(definitions, cx) { - Ok(definitions) => definitions, - Err(_) => return DummyResult::any(sp), - }; - - let table = build_state_tables(&definitions); - let ast = build_table_ast(cx, sp, table); - - MacEager::expr(ast) -} diff --git a/codegen/src/ext/vt.rs b/codegen/src/ext/vt.rs deleted file mode 100644 index aff7d68..0000000 --- a/codegen/src/ext/vt.rs +++ /dev/null @@ -1,393 +0,0 @@ -//! Macro expansion for the virtual terminal parser state table -use std::fmt; - -use syntex::Registry; - -use syntex_syntax::ast::{self, Arm, Expr, ExprKind, LitKind, Pat, PatKind}; -use syntex_syntax::codemap::Span; -use syntex_syntax::ext::base::{DummyResult, ExtCtxt, MacEager, MacResult}; -use syntex_syntax::ext::build::AstBuilder; -use syntex_syntax::parse::parser::Parser; -use syntex_syntax::parse::token::{DelimToken, Token}; -use syntex_syntax::parse::PResult; -use syntex_syntax::ptr::P; -use syntex_syntax::tokenstream::TokenTree; - -#[path = "../../../src/definitions.rs"] -mod definitions; - -use self::definitions::{Action, State}; - -pub fn register(registry: &mut Registry) { - registry.add_macro("vt_state_table", expand_state_table); -} - -fn state_from_str<S>(s: &S) -> Result<State, ()> -where - S: AsRef<str>, -{ - Ok(match s.as_ref() { - "State::Anywhere" => State::Anywhere, - "State::CsiEntry" => State::CsiEntry, - "State::CsiIgnore" => State::CsiIgnore, - "State::CsiIntermediate" => State::CsiIntermediate, - "State::CsiParam" => State::CsiParam, - "State::DcsEntry" => State::DcsEntry, - "State::DcsIgnore" => State::DcsIgnore, - "State::DcsIntermediate" => State::DcsIntermediate, - "State::DcsParam" => State::DcsParam, - "State::DcsPassthrough" => State::DcsPassthrough, - "State::Escape" => State::Escape, - "State::EscapeIntermediate" => State::EscapeIntermediate, - "State::Ground" => State::Ground, - "State::OscString" => State::OscString, - "State::SosPmApcString" => State::SosPmApcString, - "State::Utf8" => State::Utf8, - _ => return Err(()), - }) -} - -fn action_from_str<S>(s: &S) -> Result<Action, ()> -where - S: AsRef<str>, -{ - Ok(match s.as_ref() { - "Action::None" => Action::None, - "Action::Clear" => Action::Clear, - "Action::Collect" => Action::Collect, - "Action::CsiDispatch" => Action::CsiDispatch, - "Action::EscDispatch" => Action::EscDispatch, - "Action::Execute" => Action::Execute, - "Action::Hook" => Action::Hook, - "Action::Ignore" => Action::Ignore, - "Action::OscEnd" => Action::OscEnd, - "Action::OscPut" => Action::OscPut, - "Action::OscStart" => Action::OscStart, - "Action::Param" => Action::Param, - "Action::Print" => Action::Print, - "Action::Put" => Action::Put, - "Action::Unhook" => Action::Unhook, - "Action::BeginUtf8" => Action::BeginUtf8, - _ => return Err(()), - }) -} - -fn parse_table_input_mappings<'a>(parser: &mut Parser<'a>) -> PResult<'a, Vec<Arm>> { - // Must start on open brace - parser.expect(&Token::OpenDelim(DelimToken::Brace))?; - - let mut arms: Vec<Arm> = Vec::new(); - while parser.token != Token::CloseDelim(DelimToken::Brace) { - match parser.parse_arm() { - Ok(arm) => arms.push(arm), - Err(e) => { - // Recover by skipping to the end of the block. - return Err(e); - }, - } - } - - // Consume the closing brace - parser.bump(); - Ok(arms) -} - -/// Expressions describing state transitions and actions -#[derive(Debug)] -struct TableDefinitionExprs { - state_expr: P<Expr>, - mapping_arms: Vec<Arm>, -} - -fn state_from_expr(expr: P<Expr>, cx: &mut ExtCtxt) -> Result<State, ()> { - let s = match expr.node { - ExprKind::Path(ref _qself, ref path) => path.to_string(), - _ => { - cx.span_err(expr.span, "expected State"); - return Err(()); - }, - }; - - state_from_str(&s).map_err(|_| { - cx.span_err(expr.span, "expected State"); - }) -} - -fn u8_lit_from_expr(expr: &Expr, cx: &mut ExtCtxt) -> Result<u8, ()> { - static MSG: &str = "expected u8 int literal"; - - match expr.node { - ExprKind::Lit(ref lit) => match lit.node { - LitKind::Int(val, _) => Ok(val as u8), - _ => { - cx.span_err(lit.span, MSG); - Err(()) - }, - }, - _ => { - cx.span_err(expr.span, MSG); - Err(()) - }, - } -} - -fn input_mapping_from_arm(arm: Arm, cx: &mut ExtCtxt) -> Result<InputMapping, ()> { - let Arm { pats, body, .. } = arm; - - let input = InputDefinition::from_pat(&pats[0], cx)?; - let transition = Transition::from_expr(&body, cx)?; - - Ok(InputMapping { input, transition }) -} - -/// What happens when certain input is received -#[derive(Copy, Clone)] -enum Transition { - State(State), - Action(Action), - StateAction(State, Action), -} - -impl fmt::Debug for Transition { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match *self { - Transition::State(state) => write!(f, "State({:?})", state)?, - Transition::Action(action) => write!(f, "Action({:?})", action)?, - Transition::StateAction(state, action) => { - write!(f, "StateAction({:?}, {:?})", state, action)?; - }, - } - - write!(f, " -> {:?}", self.pack_u8()) - } -} - -impl Transition { - // State is stored in the top 4 bits - fn pack_u8(self) -> u8 { - match self { - Transition::State(state) => state as u8, - Transition::Action(action) => (action as u8) << 4, - Transition::StateAction(state, action) => ((action as u8) << 4) | (state as u8), - } - } -} - -impl Transition { - fn from_expr(expr: &Expr, cx: &mut ExtCtxt) -> Result<Transition, ()> { - match expr.node { - ExprKind::Tup(ref tup_exprs) => { - let mut action = None; - let mut state = None; - - for tup_expr in tup_exprs { - if let ExprKind::Path(_, ref path) = tup_expr.node { - let path_str = path.to_string(); - if path_str.starts_with('A') { - action = Some(action_from_str(&path_str).map_err(|_| { - cx.span_err(expr.span, "invalid action"); - })?); - } else { - state = Some(state_from_str(&path_str).map_err(|_| { - cx.span_err(expr.span, "invalid state"); - })?); - } - } - } - - match (action, state) { - (Some(action), Some(state)) => Ok(Transition::StateAction(state, action)), - (None, Some(state)) => Ok(Transition::State(state)), - (Some(action), None) => Ok(Transition::Action(action)), - _ => { - cx.span_err(expr.span, "expected Action and/or State"); - Err(()) - }, - } - }, - ExprKind::Path(_, ref path) => { - // Path can be Action or State - let path_str = path.to_string(); - - if path_str.starts_with('A') { - let action = action_from_str(&path_str).map_err(|_| { - cx.span_err(expr.span, "invalid action"); - })?; - Ok(Transition::Action(action)) - } else { - let state = state_from_str(&path_str).map_err(|_| { - cx.span_err(expr.span, "invalid state"); - })?; - - Ok(Transition::State(state)) - } - }, - _ => { - cx.span_err(expr.span, "expected Action and/or State"); - Err(()) - }, - } - } -} - -#[derive(Debug)] -enum InputDefinition { - Specific(u8), - Range { start: u8, end: u8 }, -} - -impl InputDefinition { - fn from_pat(pat: &Pat, cx: &mut ExtCtxt) -> Result<InputDefinition, ()> { - Ok(match pat.node { - PatKind::Lit(ref lit_expr) => { - InputDefinition::Specific(u8_lit_from_expr(&lit_expr, cx)?) - }, - PatKind::Range(ref start_expr, ref end_expr) => InputDefinition::Range { - start: u8_lit_from_expr(start_expr, cx)?, - end: u8_lit_from_expr(end_expr, cx)?, - }, - _ => { - cx.span_err(pat.span, "expected literal or range expression"); - return Err(()); - }, - }) - } -} - -#[derive(Debug)] -struct InputMapping { - input: InputDefinition, - transition: Transition, -} - -#[derive(Debug)] -struct TableDefinition { - state: State, - mappings: Vec<InputMapping>, -} - -fn parse_raw_definitions( - definitions: Vec<TableDefinitionExprs>, - cx: &mut ExtCtxt, -) -> Result<Vec<TableDefinition>, ()> { - let mut out = Vec::new(); - - for raw in definitions { - let TableDefinitionExprs { state_expr, mapping_arms } = raw; - let state = state_from_expr(state_expr, cx)?; - - let mut mappings = Vec::new(); - for arm in mapping_arms { - mappings.push(input_mapping_from_arm(arm, cx)?); - } - - out.push(TableDefinition { state, mappings }) - } - - Ok(out) -} - -fn parse_table_definition<'a>(parser: &mut Parser<'a>) -> PResult<'a, TableDefinitionExprs> { - let state_expr = parser.parse_expr()?; - parser.expect(&Token::FatArrow)?; - let mappings = parse_table_input_mappings(parser)?; - - Ok(TableDefinitionExprs { state_expr, mapping_arms: mappings }) -} - -fn parse_table_definition_list<'a>( - parser: &mut Parser<'a>, -) -> PResult<'a, Vec<TableDefinitionExprs>> { - let mut definitions = Vec::new(); - while parser.token != Token::Eof { - definitions.push(parse_table_definition(parser)?); - parser.eat(&Token::Comma); - } - - Ok(definitions) -} - -fn build_state_tables<T>(defs: T) -> [[u8; 256]; 16] -where - T: AsRef<[TableDefinition]>, -{ - let mut result = [[0u8; 256]; 16]; - - for def in defs.as_ref() { - let state = def.state; - let state = state as u8; - let transitions = &mut result[state as usize]; - - for mapping in &def.mappings { - let trans = mapping.transition.pack_u8(); - match mapping.input { - InputDefinition::Specific(idx) => { - transitions[idx as usize] = trans; - }, - InputDefinition::Range { start, end } => { - for idx in start..end { - transitions[idx as usize] = trans; - } - transitions[end as usize] = trans; - }, - } - } - } - - result -} - -fn build_table_ast(cx: &mut ExtCtxt, sp: Span, table: [[u8; 256]; 16]) -> P<ast::Expr> { - let table = table - .iter() - .map(|list| { - let exprs = list.iter().map(|num| cx.expr_u8(sp, *num)).collect(); - cx.expr_vec(sp, exprs) - }) - .collect(); - - cx.expr_vec(sp, table) -} - -fn expand_state_table<'cx>( - cx: &'cx mut ExtCtxt, - sp: Span, - args: &[TokenTree], -) -> Box<dyn MacResult + 'cx> { - macro_rules! ptry { - ($pres:expr) => { - match $pres { - Ok(val) => val, - Err(mut err) => { - err.emit(); - return DummyResult::any(sp); - }, - } - }; - } - - // Parse the lookup spec - let mut parser: Parser = cx.new_parser_from_tts(args); - let definitions = ptry!(parse_table_definition_list(&mut parser)); - let definitions = match parse_raw_definitions(definitions, cx) { - Ok(definitions) => definitions, - Err(_) => return DummyResult::any(sp), - }; - - let table = build_state_tables(&definitions); - let ast = build_table_ast(cx, sp, table); - - MacEager::expr(ast) -} - -#[cfg(test)] -mod tests { - use super::definitions::{Action, State}; - use super::Transition; - - #[test] - fn pack_u8() { - let transition = Transition::StateAction(State::CsiParam, Action::Collect); - assert_eq!(transition.pack_u8(), 0x24); - } -} diff --git a/codegen/src/main.rs b/codegen/src/main.rs deleted file mode 100644 index 7cc6cd0..0000000 --- a/codegen/src/main.rs +++ /dev/null @@ -1,23 +0,0 @@ -#![allow(dead_code)] - -use std::path::Path; - -use syntex; - -mod ext; - -fn main() { - // Expand VT parser state table - let mut registry = syntex::Registry::new(); - ext::vt::register(&mut registry); - let src = &Path::new("../src/table.rs.in"); - let dst = &Path::new("../src/table.rs"); - registry.expand("vt_state_table", src, dst).expect("expand vt_stable_table ok"); - - // Expand UTF8 parser state table - let mut registry = syntex::Registry::new(); - ext::utf8::register(&mut registry); - let src = &Path::new("../utf8parse/src/table.rs.in"); - let dst = &Path::new("../utf8parse/src/table.rs"); - registry.expand("utf8_state_table", src, dst).expect("expand utf8_stable_table ok"); -} diff --git a/generate_state_changes/Cargo.toml b/generate_state_changes/Cargo.toml new file mode 100644 index 0000000..efe5870 --- /dev/null +++ b/generate_state_changes/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "generate_state_changes" +version = "0.1.0" +authors = ["Christian Duerr <contact@christianduerr.com>"] +edition = "2018" + +[lib] +proc-macro = true + +[dependencies] +proc-macro2 = "1.0.6" +quote = "1.0.2" diff --git a/generate_state_changes/src/lib.rs b/generate_state_changes/src/lib.rs new file mode 100644 index 0000000..485a33e --- /dev/null +++ b/generate_state_changes/src/lib.rs @@ -0,0 +1,174 @@ +extern crate proc_macro; + +use std::iter::Peekable; + +use proc_macro2::TokenTree::{Group, Literal, Punct}; +use proc_macro2::{token_stream, TokenStream, TokenTree}; +use quote::quote; + +/// Create a `const fn` which will return an array with all state changes. +#[proc_macro] +pub fn generate_state_changes(item: proc_macro::TokenStream) -> proc_macro::TokenStream { + // Convert from proc_macro -> proc_macro2 + let item: TokenStream = item.into(); + let mut iter = item.into_iter().peekable(); + + // Determine output function name + let fn_name = iter.next().unwrap(); + + // Separator between name and body with state changes + expect_punct(&mut iter, ','); + + // Create token stream to assign each state change to the array + let assignments_stream = states_stream(&mut iter); + + quote!( + const fn #fn_name() -> [[u8; 256]; 16] { + let mut state_changes = [[0; 256]; 16]; + + #assignments_stream + + state_changes + } + ) + .into() +} + +/// Generate the array assignment statements for all origin states. +fn states_stream(iter: &mut impl Iterator<Item = TokenTree>) -> TokenStream { + let mut states_stream = next_group(iter).into_iter().peekable(); + + // Loop over all origin state entries + let mut tokens = quote!(); + while states_stream.peek().is_some() { + // Add all mappings for this state + tokens.extend(state_entry_stream(&mut states_stream)); + + // Allow trailing comma + optional_punct(&mut states_stream, ','); + } + tokens +} + +/// Generate the array assignment statements for one origin state. +fn state_entry_stream(iter: &mut Peekable<token_stream::IntoIter>) -> TokenStream { + // Origin state name + let state = iter.next().unwrap().into(); + + // Token stream with all the byte->target mappings + let mut changes_stream = next_group(iter).into_iter().peekable(); + + let mut tokens = quote!(); + while changes_stream.peek().is_some() { + // Add next mapping for this state + tokens.extend(change_stream(&mut changes_stream, &state)); + + // Allow trailing comma + optional_punct(&mut changes_stream, ','); + } + tokens +} + +/// Generate the array assignment statement for a single byte->target mapping for one state. +fn change_stream(iter: &mut Peekable<token_stream::IntoIter>, state: &TokenTree) -> TokenStream { + // Start of input byte range + let start = next_usize(iter); + + // End of input byte range + let end = if optional_punct(iter, '.') { + // Read inclusive end of range + expect_punct(iter, '.'); + expect_punct(iter, '='); + next_usize(iter) + } else { + // Without range, end is equal to start + start + }; + + // Separator between byte input range and output state + expect_punct(iter, '='); + expect_punct(iter, '>'); + + // Token stream with target state and action + let mut target_change_stream = next_group(iter).into_iter().peekable(); + + let mut tokens = quote!(); + while target_change_stream.peek().is_some() { + // Target state/action for all bytes in the range + let (target_state, target_action) = target_change(&mut target_change_stream); + + // Create a new entry for every byte in the range + for byte in start..=end { + // TODO: Force adding `State::` and `Action::`? + // TODO: Should we really use `pack` here without import? + tokens.extend(quote!( + state_changes[State::#state as usize][#byte] = + pack(State::#target_state, Action::#target_action); + )); + } + } + tokens +} + +/// Get next target state and action. +fn target_change(iter: &mut Peekable<token_stream::IntoIter>) -> (TokenTree, TokenTree) { + let target_state = iter.next().unwrap(); + + // Separator between state and action + expect_punct(iter, ','); + + let target_action = iter.next().unwrap(); + + (target_state, target_action) +} + +/// Check if next token matches specific punctuation. +fn optional_punct(iter: &mut Peekable<token_stream::IntoIter>, c: char) -> bool { + match iter.peek() { + Some(Punct(punct)) if punct.as_char() == c => iter.next().is_some(), + _ => false, + } +} + +/// Ensure next token matches specific punctuation. +/// +/// # Panics +/// +/// Panics if the punctuation does not match. +fn expect_punct(iter: &mut impl Iterator<Item = TokenTree>, c: char) { + match iter.next() { + Some(Punct(punct)) if punct.as_char() == c => (), + token => panic!("Expected punctuation '{}', but got {:?}", c, token), + } +} + +/// Get next token as [`usize`]. +/// +/// # Panics +/// +/// Panics if the next token is not a [`usize`] in hex or decimal literal format. +fn next_usize(iter: &mut impl Iterator<Item = TokenTree>) -> usize { + match iter.next() { + Some(Literal(literal)) => { + let literal = literal.to_string(); + if literal.starts_with("0x") { + usize::from_str_radix(&literal[2..], 16).unwrap() + } else { + usize::from_str_radix(&literal, 10).unwrap() + } + }, + token => panic!("Expected literal, but got {:?}", token), + } +} + +/// Get next token as [`Group`]. +/// +/// # Panics +/// +/// Panics if the next token is not a [`Group`]. +fn next_group(iter: &mut impl Iterator<Item = TokenTree>) -> TokenStream { + match iter.next() { + Some(Group(group)) => group.stream(), + token => panic!("Expected group, but got {:?}", token), + } +} diff --git a/src/definitions.rs b/src/definitions.rs index 375ac02..568eb41 100644 --- a/src/definitions.rs +++ b/src/definitions.rs @@ -1,3 +1,5 @@ +use core::mem; + #[allow(dead_code)] #[derive(Debug, Copy, Clone)] pub enum State { @@ -46,6 +48,27 @@ pub enum Action { BeginUtf8 = 15, } +impl State { + #[inline] + pub fn entry_action(&self) -> Action { + match self { + State::CsiEntry | State::DcsEntry | State::Escape => Action::Clear, + State::DcsPassthrough => Action::Hook, + State::OscString => Action::OscStart, + _ => Action::None, + } + } + + #[inline] + pub fn exit_action(&self) -> Action { + match self { + State::DcsPassthrough => Action::Unhook, + State::OscString => Action::OscEnd, + _ => Action::None, + } + } +} + /// Unpack a u8 into a State and Action /// /// The implementation of this assumes that there are *precisely* 16 variants for both Action and @@ -55,17 +78,25 @@ pub enum Action { /// Bad things will happen if those invariants are violated. #[inline(always)] pub fn unpack(delta: u8) -> (State, Action) { - ( - // State is stored in bottom 4 bits - unsafe { core::mem::transmute(delta & 0x0f) }, - // Action is stored in top 4 bits - unsafe { core::mem::transmute(delta >> 4) }, - ) + unsafe { + ( + // State is stored in bottom 4 bits + mem::transmute(delta & 0x0f), + // Action is stored in top 4 bits + mem::transmute(delta >> 4), + ) + } +} + +#[inline(always)] +pub const fn pack(state: State, action: Action) -> u8 { + (action as u8) << 4 | state as u8 } #[cfg(test)] mod tests { - use super::{unpack, Action, State}; + use super::*; + #[test] fn unpack_state_action() { match unpack(0xee) { @@ -83,4 +114,22 @@ mod tests { _ => panic!("unpack failed"), } } + + #[test] + fn pack_state_action() { + match unpack(0xee) { + (State::SosPmApcString, Action::Unhook) => (), + _ => panic!("unpack failed"), + } + + match unpack(0x0f) { + (State::Utf8, Action::None) => (), + _ => panic!("unpack failed"), + } + + match unpack(0xff) { + (State::Utf8, Action::BeginUtf8) => (), + _ => panic!("unpack failed"), + } + } } @@ -30,6 +30,7 @@ //! [`Parser`]: struct.Parser.html //! [`Perform`]: trait.Perform.html //! [Paul Williams' ANSI parser state machine]: https://vt100.net/emu/dec_ansi_parser +#![cfg_attr(all(feature = "nightly", test), feature(test))] #![cfg_attr(feature = "no_std", no_std)] use core::mem::MaybeUninit; @@ -42,21 +43,6 @@ mod definitions; mod table; use definitions::{unpack, Action, State}; -use table::{ENTRY_ACTIONS, EXIT_ACTIONS, STATE_CHANGE}; - -impl State { - /// Get exit action for this state - #[inline(always)] - pub fn exit_action(self) -> Action { - unsafe { *EXIT_ACTIONS.get_unchecked(self as usize) } - } - - /// Get entry action for this state - #[inline(always)] - pub fn entry_action(self) -> Action { - unsafe { *ENTRY_ACTIONS.get_unchecked(self as usize) } - } -} const MAX_INTERMEDIATES: usize = 2; #[cfg(any(feature = "no_std", test))] @@ -129,10 +115,10 @@ impl Parser { // Handle state changes in the anywhere state before evaluating changes // for current state. - let mut change = STATE_CHANGE[State::Anywhere as usize][byte as usize]; + let mut change = table::STATE_CHANGES[State::Anywhere as usize][byte as usize]; if change == 0 { - change = STATE_CHANGE[self.state as usize][byte as usize]; + change = table::STATE_CHANGES[self.state as usize][byte as usize]; } // Unpack into a state and action @@ -739,3 +725,60 @@ mod tests { assert_eq!(dispatcher.params[1].len(), MAX_OSC_RAW - dispatcher.params[0].len()); } } + +#[cfg(all(feature = "nightly", test))] +mod bench { + extern crate std; + extern crate test; + + use super::*; + + use test::{black_box, Bencher}; + + static VTE_DEMO: &[u8] = include_bytes!("../tests/demo.vte"); + + struct BenchDispatcher; + impl Perform for BenchDispatcher { + fn print(&mut self, c: char) { + black_box(c); + } + + fn execute(&mut self, byte: u8) { + black_box(byte); + } + + fn hook(&mut self, params: &[i64], intermediates: &[u8], ignore: bool, c: char) { + black_box((params, intermediates, ignore, c)); + } + + fn put(&mut self, byte: u8) { + black_box(byte); + } + + fn unhook(&mut self) {} + + fn osc_dispatch(&mut self, params: &[&[u8]]) { + black_box(params); + } + + fn csi_dispatch(&mut self, params: &[i64], intermediates: &[u8], ignore: bool, c: char) { + black_box((params, intermediates, ignore, c)); + } + + fn esc_dispatch(&mut self, params: &[i64], intermediates: &[u8], ignore: bool, byte: u8) { + black_box((params, intermediates, ignore, byte)); + } + } + + #[bench] + fn testfile(b: &mut Bencher) { + b.iter(|| { + let mut dispatcher = BenchDispatcher; + let mut parser = Parser::new(); + + for byte in VTE_DEMO { + parser.advance(&mut dispatcher, *byte); + } + }); + } +} diff --git a/src/table.rs b/src/table.rs index a9569aa..f5a90ee 100644 --- a/src/table.rs +++ b/src/table.rs @@ -1,343 +1,175 @@ /// This is the state change table. It's indexed first by current state and then by the next /// character in the pty stream. -use crate::definitions::Action; +use crate::definitions::{pack, Action, State}; -pub static STATE_CHANGE: [[u8; 256]; 16] = [ - // Beginning of UTF-8 2 byte sequence - // Beginning of UTF-8 3 byte sequence - // Beginning of UTF-8 4 byte sequence - [ - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 92u8, 0u8, 92u8, 10u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, - ], - [ - 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, - 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 0u8, 80u8, 0u8, 0u8, 80u8, 80u8, - 80u8, 80u8, 35u8, 35u8, 35u8, 35u8, 35u8, 35u8, 35u8, 35u8, 35u8, 35u8, 35u8, 35u8, 35u8, - 35u8, 35u8, 35u8, 180u8, 180u8, 180u8, 180u8, 180u8, 180u8, 180u8, 180u8, 180u8, 180u8, - 2u8, 180u8, 36u8, 36u8, 36u8, 36u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, - 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, - 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, - 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, - 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 112u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - ], - [ - 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, - 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 0u8, 80u8, 0u8, 0u8, 80u8, 80u8, - 80u8, 80u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, - 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, - 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 12u8, 12u8, 12u8, 12u8, 12u8, 12u8, - 12u8, 12u8, 12u8, 12u8, 12u8, 12u8, 12u8, 12u8, 12u8, 12u8, 12u8, 12u8, 12u8, 12u8, 12u8, - 12u8, 12u8, 12u8, 12u8, 12u8, 12u8, 12u8, 12u8, 12u8, 12u8, 12u8, 12u8, 12u8, 12u8, 12u8, - 12u8, 12u8, 12u8, 12u8, 12u8, 12u8, 12u8, 12u8, 12u8, 12u8, 12u8, 12u8, 12u8, 12u8, 12u8, - 12u8, 12u8, 12u8, 12u8, 12u8, 12u8, 12u8, 12u8, 12u8, 12u8, 12u8, 12u8, 112u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - ], - [ - 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, - 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 0u8, 80u8, 0u8, 0u8, 80u8, 80u8, - 80u8, 80u8, 32u8, 32u8, 32u8, 32u8, 32u8, 32u8, 32u8, 32u8, 32u8, 32u8, 32u8, 32u8, 32u8, - 32u8, 32u8, 32u8, 2u8, 2u8, 2u8, 2u8, 2u8, 2u8, 2u8, 2u8, 2u8, 2u8, 2u8, 2u8, 2u8, 2u8, - 2u8, 2u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, - 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, - 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, - 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, - 60u8, 60u8, 60u8, 60u8, 60u8, 112u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - ], - [ - 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, - 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 0u8, 80u8, 0u8, 0u8, 80u8, 80u8, - 80u8, 80u8, 35u8, 35u8, 35u8, 35u8, 35u8, 35u8, 35u8, 35u8, 35u8, 35u8, 35u8, 35u8, 35u8, - 35u8, 35u8, 35u8, 176u8, 176u8, 176u8, 176u8, 176u8, 176u8, 176u8, 176u8, 176u8, 176u8, - 2u8, 176u8, 2u8, 2u8, 2u8, 2u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, - 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, - 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, - 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, - 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 60u8, 112u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - ], - [ - 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, - 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 0u8, 112u8, - 0u8, 0u8, 112u8, 112u8, 112u8, 112u8, 39u8, 39u8, 39u8, 39u8, 39u8, 39u8, 39u8, 39u8, 39u8, - 39u8, 39u8, 39u8, 39u8, 39u8, 39u8, 39u8, 184u8, 184u8, 184u8, 184u8, 184u8, 184u8, 184u8, - 184u8, 184u8, 184u8, 6u8, 184u8, 40u8, 40u8, 40u8, 40u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, - 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, - 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, - 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, - 9u8, 9u8, 112u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, - ], - [ - 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, - 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 0u8, 112u8, - 0u8, 0u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, - 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, - 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, - 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, - 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, - 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, - 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, - 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 12u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - ], - [ - 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, - 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 0u8, 112u8, - 0u8, 0u8, 112u8, 112u8, 112u8, 112u8, 32u8, 32u8, 32u8, 32u8, 32u8, 32u8, 32u8, 32u8, 32u8, - 32u8, 32u8, 32u8, 32u8, 32u8, 32u8, 32u8, 6u8, 6u8, 6u8, 6u8, 6u8, 6u8, 6u8, 6u8, 6u8, 6u8, - 6u8, 6u8, 6u8, 6u8, 6u8, 6u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, - 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, - 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, - 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 112u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - ], - [ - 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, - 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 0u8, 112u8, - 0u8, 0u8, 112u8, 112u8, 112u8, 112u8, 39u8, 39u8, 39u8, 39u8, 39u8, 39u8, 39u8, 39u8, 39u8, - 39u8, 39u8, 39u8, 39u8, 39u8, 39u8, 39u8, 176u8, 176u8, 176u8, 176u8, 176u8, 176u8, 176u8, - 176u8, 176u8, 176u8, 6u8, 176u8, 6u8, 6u8, 6u8, 6u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, - 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, - 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, - 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, 9u8, - 9u8, 9u8, 112u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, - ], - [ - 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, - 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, 0u8, 208u8, - 0u8, 0u8, 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, - 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, - 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, - 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, - 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, - 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, - 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, - 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, 208u8, 112u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 12u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - ], - [ - 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, - 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 0u8, 80u8, 0u8, 0u8, 80u8, 80u8, - 80u8, 80u8, 43u8, 43u8, 43u8, 43u8, 43u8, 43u8, 43u8, 43u8, 43u8, 43u8, 43u8, 43u8, 43u8, - 43u8, 43u8, 43u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, - 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, - 76u8, 76u8, 76u8, 76u8, 76u8, 5u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 14u8, 76u8, - 76u8, 1u8, 76u8, 13u8, 14u8, 14u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, - 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, - 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 112u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - ], - [ - 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, - 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 0u8, 80u8, 0u8, 0u8, 80u8, 80u8, - 80u8, 80u8, 32u8, 32u8, 32u8, 32u8, 32u8, 32u8, 32u8, 32u8, 32u8, 32u8, 32u8, 32u8, 32u8, - 32u8, 32u8, 32u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, - 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, - 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, - 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, - 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, - 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 76u8, 112u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - ], - [ - 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, - 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 0u8, 80u8, 0u8, 0u8, 80u8, 80u8, - 80u8, 80u8, 192u8, 192u8, 192u8, 192u8, 192u8, 192u8, 192u8, 192u8, 192u8, 192u8, 192u8, - 192u8, 192u8, 192u8, 192u8, 192u8, 192u8, 192u8, 192u8, 192u8, 192u8, 192u8, 192u8, 192u8, - 192u8, 192u8, 192u8, 192u8, 192u8, 192u8, 192u8, 192u8, 192u8, 192u8, 192u8, 192u8, 192u8, - 192u8, 192u8, 192u8, 192u8, 192u8, 192u8, 192u8, 192u8, 192u8, 192u8, 192u8, 192u8, 192u8, - 192u8, 192u8, 192u8, 192u8, 192u8, 192u8, 192u8, 192u8, 192u8, 192u8, 192u8, 192u8, 192u8, - 192u8, 192u8, 192u8, 192u8, 192u8, 192u8, 192u8, 192u8, 192u8, 192u8, 192u8, 192u8, 192u8, - 192u8, 192u8, 192u8, 192u8, 192u8, 192u8, 192u8, 192u8, 192u8, 192u8, 192u8, 192u8, 192u8, - 192u8, 192u8, 192u8, 192u8, 192u8, 192u8, 192u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, - 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 80u8, 0u8, 80u8, 80u8, 80u8, 80u8, 80u8, - 80u8, 80u8, 80u8, 80u8, 80u8, 0u8, 80u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 255u8, 255u8, 255u8, 255u8, 255u8, 255u8, - 255u8, 255u8, 255u8, 255u8, 255u8, 255u8, 255u8, 255u8, 255u8, 255u8, 255u8, 255u8, 255u8, - 255u8, 255u8, 255u8, 255u8, 255u8, 255u8, 255u8, 255u8, 255u8, 255u8, 255u8, 255u8, 255u8, - 255u8, 255u8, 255u8, 255u8, 255u8, 255u8, 255u8, 255u8, 255u8, 255u8, 255u8, 255u8, 255u8, - 255u8, 255u8, 255u8, 255u8, 255u8, 255u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, - ], - [ - 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 12u8, 112u8, 112u8, 112u8, 112u8, 112u8, - 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 0u8, 112u8, - 0u8, 0u8, 112u8, 112u8, 112u8, 112u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, - 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, - 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, - 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, - 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, - 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, - 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, - 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, - 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, - 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, - 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, - 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, - 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, - 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, - 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, - 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, - 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, - 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, 144u8, - ], - [ - 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, - 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 0u8, 112u8, - 0u8, 0u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, - 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, - 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, - 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, - 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, - 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, - 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, - 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 112u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 12u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - ], - [ - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, - ], -]; +use generate_state_changes::generate_state_changes; -pub static ENTRY_ACTIONS: &[Action] = &[ - Action::None, // State::Anywhere - Action::Clear, // State::CsiEntry - Action::None, // State::CsiIgnore - Action::None, // State::CsiIntermediate - Action::None, // State::CsiParam - Action::Clear, // State::DcsEntry - Action::None, // State::DcsIgnore - Action::None, // State::DcsIntermediate - Action::None, // State::DcsParam - Action::Hook, // State::DcsPassthrough - Action::Clear, // State::Escape - Action::None, // State::EscapeIntermediate - Action::None, // State::Ground - Action::OscStart, // State::OscString - Action::None, // State::SosPmApcString - Action::None, -]; -// State::Utf8 +// Generate state changes at compile-time +pub static STATE_CHANGES: [[u8; 256]; 16] = state_changes(); +generate_state_changes!(state_changes, { + Anywhere { + 0x18 => (Ground, Execute), + 0x1a => (Ground, Execute), + 0x1b => (Escape, None), + }, -pub static EXIT_ACTIONS: &[Action] = &[ - Action::None, // State::Anywhere - Action::None, // State::CsiEntry - Action::None, // State::CsiIgnore - Action::None, // State::CsiIntermediate - Action::None, // State::CsiParam - Action::None, // State::DcsEntry - Action::None, // State::DcsIgnore - Action::None, // State::DcsIntermediate - Action::None, // State::DcsParam - Action::Unhook, // State::DcsPassthrough - Action::None, // State::Escape - Action::None, // State::EscapeIntermediate - Action::None, // State::Ground - Action::OscEnd, // State::OscString - Action::None, // State::SosPmApcString - Action::None, -]; // State::Utf8 + Ground { + 0x00..=0x17 => (Anywhere, Execute), + 0x19 => (Anywhere, Execute), + 0x1c..=0x1f => (Anywhere, Execute), + 0x20..=0x7f => (Anywhere, Print), + 0x80..=0x8f => (Anywhere, Execute), + 0x91..=0x9a => (Anywhere, Execute), + 0x9c => (Anywhere, Execute), + // Beginning of UTF-8 2 byte sequence + 0xc2..=0xdf => (Utf8, BeginUtf8), + // Beginning of UTF-8 3 byte sequence + 0xe0..=0xef => (Utf8, BeginUtf8), + // Beginning of UTF-8 4 byte sequence + 0xf0..=0xf4 => (Utf8, BeginUtf8), + }, + + Escape { + 0x00..=0x17 => (Anywhere, Execute), + 0x19 => (Anywhere, Execute), + 0x1c..=0x1f => (Anywhere, Execute), + 0x7f => (Anywhere, Ignore), + 0x20..=0x2f => (EscapeIntermediate, Collect), + 0x30..=0x4f => (Ground, EscDispatch), + 0x51..=0x57 => (Ground, EscDispatch), + 0x59 => (Ground, EscDispatch), + 0x5a => (Ground, EscDispatch), + 0x5c => (Ground, EscDispatch), + 0x60..=0x7e => (Ground, EscDispatch), + 0x5b => (CsiEntry, None), + 0x5d => (OscString, None), + 0x50 => (DcsEntry, None), + 0x58 => (SosPmApcString, None), + 0x5e => (SosPmApcString, None), + 0x5f => (SosPmApcString, None), + }, + + EscapeIntermediate { + 0x00..=0x17 => (Anywhere, Execute), + 0x19 => (Anywhere, Execute), + 0x1c..=0x1f => (Anywhere, Execute), + 0x20..=0x2f => (Anywhere, Collect), + 0x7f => (Anywhere, Ignore), + 0x30..=0x7e => (Ground, EscDispatch), + }, + + CsiEntry { + 0x00..=0x17 => (Anywhere, Execute), + 0x19 => (Anywhere, Execute), + 0x1c..=0x1f => (Anywhere, Execute), + 0x7f => (Anywhere, Ignore), + 0x20..=0x2f => (CsiIntermediate, Collect), + 0x3a => (CsiIgnore, None), + 0x30..=0x39 => (CsiParam, Param), + 0x3b => (CsiParam, Param), + 0x3c..=0x3f => (CsiParam, Collect), + 0x40..=0x7e => (Ground, CsiDispatch), + }, + + CsiIgnore { + 0x00..=0x17 => (Anywhere, Execute), + 0x19 => (Anywhere, Execute), + 0x1c..=0x1f => (Anywhere, Execute), + 0x20..=0x3f => (Anywhere, Ignore), + 0x7f => (Anywhere, Ignore), + 0x40..=0x7e => (Ground, None), + }, + + CsiParam { + 0x00..=0x17 => (Anywhere, Execute), + 0x19 => (Anywhere, Execute), + 0x1c..=0x1f => (Anywhere, Execute), + 0x30..=0x39 => (Anywhere, Param), + 0x3b => (Anywhere, Param), + 0x7f => (Anywhere, Ignore), + 0x3a => (CsiIgnore, None), + 0x3c..=0x3f => (CsiIgnore, None), + 0x20..=0x2f => (CsiIntermediate, Collect), + 0x40..=0x7e => (Ground, CsiDispatch), + }, + + CsiIntermediate { + 0x00..=0x17 => (Anywhere, Execute), + 0x19 => (Anywhere, Execute), + 0x1c..=0x1f => (Anywhere, Execute), + 0x20..=0x2f => (Anywhere, Collect), + 0x7f => (Anywhere, Ignore), + 0x30..=0x3f => (CsiIgnore, None), + 0x40..=0x7e => (Ground, CsiDispatch), + }, + + DcsEntry { + 0x00..=0x17 => (Anywhere, Ignore), + 0x19 => (Anywhere, Ignore), + 0x1c..=0x1f => (Anywhere, Ignore), + 0x7f => (Anywhere, Ignore), + 0x3a => (DcsIgnore, None), + 0x20..=0x2f => (DcsIntermediate, Collect), + 0x30..=0x39 => (DcsParam, Param), + 0x3b => (DcsParam, Param), + 0x3c..=0x3f => (DcsParam, Collect), + 0x40..=0x7e => (DcsPassthrough, None), + }, + + DcsIntermediate { + 0x00..=0x17 => (Anywhere, Ignore), + 0x19 => (Anywhere, Ignore), + 0x1c..=0x1f => (Anywhere, Ignore), + 0x20..=0x2f => (Anywhere, Collect), + 0x7f => (Anywhere, Ignore), + 0x30..=0x3f => (DcsIgnore, None), + 0x40..=0x7e => (DcsPassthrough, None), + }, + + DcsIgnore { + 0x00..=0x17 => (Anywhere, Ignore), + 0x19 => (Anywhere, Ignore), + 0x1c..=0x1f => (Anywhere, Ignore), + 0x20..=0x7f => (Anywhere, Ignore), + 0x9c => (Ground, None), + }, + + DcsParam { + 0x00..=0x17 => (Anywhere, Ignore), + 0x19 => (Anywhere, Ignore), + 0x1c..=0x1f => (Anywhere, Ignore), + 0x30..=0x39 => (Anywhere, Param), + 0x3b => (Anywhere, Param), + 0x7f => (Anywhere, Ignore), + 0x3a => (DcsIgnore, None), + 0x3c..=0x3f => (DcsIgnore, None), + 0x20..=0x2f => (DcsIntermediate, Collect), + 0x40..=0x7e => (DcsPassthrough, None), + }, + + DcsPassthrough { + 0x00..=0x17 => (Anywhere, Put), + 0x19 => (Anywhere, Put), + 0x1c..=0x1f => (Anywhere, Put), + 0x20..=0x7e => (Anywhere, Put), + 0x7f => (Anywhere, Ignore), + 0x9c => (Ground, None), + }, + + SosPmApcString { + 0x00..=0x17 => (Anywhere, Ignore), + 0x19 => (Anywhere, Ignore), + 0x1c..=0x1f => (Anywhere, Ignore), + 0x20..=0x7f => (Anywhere, Ignore), + 0x9c => (Ground, None), + }, + + OscString { + 0x00..=0x06 => (Anywhere, Ignore), + 0x07 => (Ground, None), + 0x08..=0x17 => (Anywhere, Ignore), + 0x19 => (Anywhere, Ignore), + 0x1c..=0x1f => (Anywhere, Ignore), + 0x20..=0xff => (Anywhere, OscPut), + } +}); diff --git a/src/table.rs.in b/src/table.rs.in index 5593283..0245af7 100644 --- a/src/table.rs.in +++ b/src/table.rs.in @@ -4,13 +4,13 @@ use definitions::Action; pub static STATE_CHANGE: [[u8; 256]; 16] = vt_state_table! { - State::Anywhere => { + Anywhere { 0x18 => (Action::Execute, State::Ground), 0x1a => (Action::Execute, State::Ground), 0x1b => State::Escape, }, - State::Ground => { + Ground { 0x00...0x17 => Action::Execute, 0x19 => Action::Execute, 0x1c...0x1f => Action::Execute, @@ -26,7 +26,7 @@ pub static STATE_CHANGE: [[u8; 256]; 16] = vt_state_table! { 0xf0...0xf4 => (State::Utf8, Action::BeginUtf8), }, - State::Escape => { + Escape { 0x00...0x17 => Action::Execute, 0x19 => Action::Execute, 0x1c...0x1f => Action::Execute, @@ -46,7 +46,7 @@ pub static STATE_CHANGE: [[u8; 256]; 16] = vt_state_table! { 0x5f => State::SosPmApcString, }, - State::EscapeIntermediate => { + EscapeIntermediate { 0x00...0x17 => Action::Execute, 0x19 => Action::Execute, 0x1c...0x1f => Action::Execute, @@ -55,7 +55,7 @@ pub static STATE_CHANGE: [[u8; 256]; 16] = vt_state_table! { 0x30...0x7e => (Action::EscDispatch, State::Ground) }, - State::CsiEntry => { + CsiEntry { 0x00...0x17 => Action::Execute, 0x19 => Action::Execute, 0x1c...0x1f => Action::Execute, @@ -68,7 +68,7 @@ pub static STATE_CHANGE: [[u8; 256]; 16] = vt_state_table! { 0x40...0x7e => (Action::CsiDispatch, State::Ground) }, - State::CsiIgnore => { + CsiIgnore { 0x00...0x17 => Action::Execute, 0x19 => Action::Execute, 0x1c...0x1f => Action::Execute, @@ -77,7 +77,7 @@ pub static STATE_CHANGE: [[u8; 256]; 16] = vt_state_table! { 0x40...0x7e => State::Ground, }, - State::CsiParam => { + CsiParam { 0x00...0x17 => Action::Execute, 0x19 => Action::Execute, 0x1c...0x1f => Action::Execute, @@ -90,7 +90,7 @@ pub static STATE_CHANGE: [[u8; 256]; 16] = vt_state_table! { 0x40...0x7e => (Action::CsiDispatch, State::Ground) }, - State::CsiIntermediate => { + CsiIntermediate { 0x00...0x17 => Action::Execute, 0x19 => Action::Execute, 0x1c...0x1f => Action::Execute, @@ -100,7 +100,7 @@ pub static STATE_CHANGE: [[u8; 256]; 16] = vt_state_table! { 0x40...0x7e => (Action::CsiDispatch, State::Ground), }, - State::DcsEntry => { + DcsEntry { 0x00...0x17 => Action::Ignore, 0x19 => Action::Ignore, 0x1c...0x1f => Action::Ignore, @@ -113,7 +113,7 @@ pub static STATE_CHANGE: [[u8; 256]; 16] = vt_state_table! { 0x40...0x7e => State::DcsPassthrough }, - State::DcsIntermediate => { + DcsIntermediate { 0x00...0x17 => Action::Ignore, 0x19 => Action::Ignore, 0x1c...0x1f => Action::Ignore, @@ -123,7 +123,7 @@ pub static STATE_CHANGE: [[u8; 256]; 16] = vt_state_table! { 0x40...0x7e => State::DcsPassthrough }, - State::DcsIgnore => { + DcsIgnore { 0x00...0x17 => Action::Ignore, 0x19 => Action::Ignore, 0x1c...0x1f => Action::Ignore, @@ -131,7 +131,7 @@ pub static STATE_CHANGE: [[u8; 256]; 16] = vt_state_table! { 0x9c => State::Ground }, - State::DcsParam => { + DcsParam { 0x00...0x17 => Action::Ignore, 0x19 => Action::Ignore, 0x1c...0x1f => Action::Ignore, @@ -144,7 +144,7 @@ pub static STATE_CHANGE: [[u8; 256]; 16] = vt_state_table! { 0x40...0x7e => State::DcsPassthrough }, - State::DcsPassthrough => { + DcsPassthrough { 0x00...0x17 => Action::Put, 0x19 => Action::Put, 0x1c...0x1f => Action::Put, @@ -153,7 +153,7 @@ pub static STATE_CHANGE: [[u8; 256]; 16] = vt_state_table! { 0x9c => State::Ground, }, - State::SosPmApcString => { + SosPmApcString { 0x00...0x17 => Action::Ignore, 0x19 => Action::Ignore, 0x1c...0x1f => Action::Ignore, @@ -161,7 +161,7 @@ pub static STATE_CHANGE: [[u8; 256]; 16] = vt_state_table! { 0x9c => State::Ground }, - State::OscString => { + OscString { 0x00...0x06 => Action::Ignore, 0x07 => State::Ground, 0x08...0x17 => Action::Ignore, diff --git a/tests/demo.vte b/tests/demo.vte new file mode 100644 index 0000000..ed62b35 --- /dev/null +++ b/tests/demo.vte @@ -0,0 +1,2052 @@ +Test +[31;42mRED ON GREEN +]52;c;Y2xpcGJvYXJkIHRlc3Q=];;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;some random text; what +[1;1;1;1;1;1;1;1;1;1;1;1;1;1;1;1;1;1;1;1;1;1;1;1;1;1;1;1;1;1;1;1;1;1;1;1;1;1;1;1;p[4;m +[;4m +[123123123123123123123123123123123123mP0;1|17/ab +
]2;echo '¯\_(ツ)_/¯' && sleep 1 +[31mREEEEEED +Test +[31;42mRED ON GREEN +]52;c;Y2xpcGJvYXJkIHRlc3Q=];;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;some random text; what +[1;1;1;1;1;1;1;1;1;1;1;1;1;1;1;1;1;1;1;1;1;1;1;1;1;1;1;1;1;1;1;1;1;1;1;1;1;1;1;1;p[4;m +[;4m +[123123123123123123123123123123123123mP0;1|17/ab +
]2;echo '¯\_(ツ)_/¯' && sleep 1 +[31mREEEEEEEEEEEEED +[6 q +[0 q +[0m +[0H[2J + +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +21 +22 +23 +24 +25 +26 +27 +28 +29 +30 +31 +32 +33 +34 +35 +36 +37 +38 +39 +40 +41 +42 +43 +44 +45 +46 +47 +48 +49 +50 +51 +52 +53 +54 +55 +56 +57 +58 +59 +60 +61 +62 +63 +64 +65 +66 +67 +68 +69 +70 +71 +72 +73 +74 +75 +76 +77 +78 +79 +80 +81 +82 +83 +84 +85 +86 +87 +88 +89 +90 +91 +92 +93 +94 +95 +96 +97 +98 +99 +100 +101 +102 +103 +104 +105 +106 +107 +108 +109 +110 +111 +112 +113 +114 +115 +116 +117 +118 +119 +120 +121 +122 +123 +124 +125 +126 +127 +128 +129 +130 +131 +132 +133 +134 +135 +136 +137 +138 +139 +140 +141 +142 +143 +144 +145 +146 +147 +148 +149 +150 +151 +152 +153 +154 +155 +156 +157 +158 +159 +160 +161 +162 +163 +164 +165 +166 +167 +168 +169 +170 +171 +172 +173 +174 +175 +176 +177 +178 +179 +180 +181 +182 +183 +184 +185 +186 +187 +188 +189 +190 +191 +192 +193 +194 +195 +196 +197 +198 +199 +200 +201 +202 +203 +204 +205 +206 +207 +208 +209 +210 +211 +212 +213 +214 +215 +216 +217 +218 +219 +220 +221 +222 +223 +224 +225 +226 +227 +228 +229 +230 +231 +232 +233 +234 +235 +236 +237 +238 +239 +240 +241 +242 +243 +244 +245 +246 +247 +248 +249 +250 +251 +252 +253 +254 +255 +256 +257 +258 +259 +260 +261 +262 +263 +264 +265 +266 +267 +268 +269 +270 +271 +272 +273 +274 +275 +276 +277 +278 +279 +280 +281 +282 +283 +284 +285 +286 +287 +288 +289 +290 +291 +292 +293 +294 +295 +296 +297 +298 +299 +300 +301 +302 +303 +304 +305 +306 +307 +308 +309 +310 +311 +312 +313 +314 +315 +316 +317 +318 +319 +320 +321 +322 +323 +324 +325 +326 +327 +328 +329 +330 +331 +332 +333 +334 +335 +336 +337 +338 +339 +340 +341 +342 +343 +344 +345 +346 +347 +348 +349 +350 +351 +352 +353 +354 +355 +356 +357 +358 +359 +360 +361 +362 +363 +364 +365 +366 +367 +368 +369 +370 +371 +372 +373 +374 +375 +376 +377 +378 +379 +380 +381 +382 +383 +384 +385 +386 +387 +388 +389 +390 +391 +392 +393 +394 +395 +396 +397 +398 +399 +400 +401 +402 +403 +404 +405 +406 +407 +408 +409 +410 +411 +412 +413 +414 +415 +416 +417 +418 +419 +420 +421 +422 +423 +424 +425 +426 +427 +428 +429 +430 +431 +432 +433 +434 +435 +436 +437 +438 +439 +440 +441 +442 +443 +444 +445 +446 +447 +448 +449 +450 +451 +452 +453 +454 +455 +456 +457 +458 +459 +460 +461 +462 +463 +464 +465 +466 +467 +468 +469 +470 +471 +472 +473 +474 +475 +476 +477 +478 +479 +480 +481 +482 +483 +484 +485 +486 +487 +488 +489 +490 +491 +492 +493 +494 +495 +496 +497 +498 +499 +500 +501 +502 +503 +504 +505 +506 +507 +508 +509 +510 +511 +512 +513 +514 +515 +516 +517 +518 +519 +520 +521 +522 +523 +524 +525 +526 +527 +528 +529 +530 +531 +532 +533 +534 +535 +536 +537 +538 +539 +540 +541 +542 +543 +544 +545 +546 +547 +548 +549 +550 +551 +552 +553 +554 +555 +556 +557 +558 +559 +560 +561 +562 +563 +564 +565 +566 +567 +568 +569 +570 +571 +572 +573 +574 +575 +576 +577 +578 +579 +580 +581 +582 +583 +584 +585 +586 +587 +588 +589 +590 +591 +592 +593 +594 +595 +596 +597 +598 +599 +600 +601 +602 +603 +604 +605 +606 +607 +608 +609 +610 +611 +612 +613 +614 +615 +616 +617 +618 +619 +620 +621 +622 +623 +624 +625 +626 +627 +628 +629 +630 +631 +632 +633 +634 +635 +636 +637 +638 +639 +640 +641 +642 +643 +644 +645 +646 +647 +648 +649 +650 +651 +652 +653 +654 +655 +656 +657 +658 +659 +660 +661 +662 +663 +664 +665 +666 +667 +668 +669 +670 +671 +672 +673 +674 +675 +676 +677 +678 +679 +680 +681 +682 +683 +684 +685 +686 +687 +688 +689 +690 +691 +692 +693 +694 +695 +696 +697 +698 +699 +700 +701 +702 +703 +704 +705 +706 +707 +708 +709 +710 +711 +712 +713 +714 +715 +716 +717 +718 +719 +720 +721 +722 +723 +724 +725 +726 +727 +728 +729 +730 +731 +732 +733 +734 +735 +736 +737 +738 +739 +740 +741 +742 +743 +744 +745 +746 +747 +748 +749 +750 +751 +752 +753 +754 +755 +756 +757 +758 +759 +760 +761 +762 +763 +764 +765 +766 +767 +768 +769 +770 +771 +772 +773 +774 +775 +776 +777 +778 +779 +780 +781 +782 +783 +784 +785 +786 +787 +788 +789 +790 +791 +792 +793 +794 +795 +796 +797 +798 +799 +800 +801 +802 +803 +804 +805 +806 +807 +808 +809 +810 +811 +812 +813 +814 +815 +816 +817 +818 +819 +820 +821 +822 +823 +824 +825 +826 +827 +828 +829 +830 +831 +832 +833 +834 +835 +836 +837 +838 +839 +840 +841 +842 +843 +844 +845 +846 +847 +848 +849 +850 +851 +852 +853 +854 +855 +856 +857 +858 +859 +860 +861 +862 +863 +864 +865 +866 +867 +868 +869 +870 +871 +872 +873 +874 +875 +876 +877 +878 +879 +880 +881 +882 +883 +884 +885 +886 +887 +888 +889 +890 +891 +892 +893 +894 +895 +896 +897 +898 +899 +900 +901 +902 +903 +904 +905 +906 +907 +908 +909 +910 +911 +912 +913 +914 +915 +916 +917 +918 +919 +920 +921 +922 +923 +924 +925 +926 +927 +928 +929 +930 +931 +932 +933 +934 +935 +936 +937 +938 +939 +940 +941 +942 +943 +944 +945 +946 +947 +948 +949 +950 +951 +952 +953 +954 +955 +956 +957 +958 +959 +960 +961 +962 +963 +964 +965 +966 +967 +968 +969 +970 +971 +972 +973 +974 +975 +976 +977 +978 +979 +980 +981 +982 +983 +984 +985 +986 +987 +988 +989 +990 +991 +992 +993 +994 +995 +996 +997 +998 +999 +1000 + +[0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +21 +22 +23 +24 +25 +26 +27 +28 +29 +30 +31 +32 +33 +34 +35 +36 +37 +38 +39 +40 +41 +42 +43 +44 +45 +46 +47 +48 +49 +50 +51 +52 +53 +54 +55 +56 +57 +58 +59 +60 +61 +62 +63 +64 +65 +66 +67 +68 +69 +70 +71 +72 +73 +74 +75 +76 +77 +78 +79 +80 +81 +82 +83 +84 +85 +86 +87 +88 +89 +90 +91 +92 +93 +94 +95 +96 +97 +98 +99 +100 +101 +102 +103 +104 +105 +106 +107 +108 +109 +110 +111 +112 +113 +114 +115 +116 +117 +118 +119 +120 +121 +122 +123 +124 +125 +126 +127 +128 +129 +130 +131 +132 +133 +134 +135 +136 +137 +138 +139 +140 +141 +142 +143 +144 +145 +146 +147 +148 +149 +150 +151 +152 +153 +154 +155 +156 +157 +158 +159 +160 +161 +162 +163 +164 +165 +166 +167 +168 +169 +170 +171 +172 +173 +174 +175 +176 +177 +178 +179 +180 +181 +182 +183 +184 +185 +186 +187 +188 +189 +190 +191 +192 +193 +194 +195 +196 +197 +198 +199 +200 +201 +202 +203 +204 +205 +206 +207 +208 +209 +210 +211 +212 +213 +214 +215 +216 +217 +218 +219 +220 +221 +222 +223 +224 +225 +226 +227 +228 +229 +230 +231 +232 +233 +234 +235 +236 +237 +238 +239 +240 +241 +242 +243 +244 +245 +246 +247 +248 +249 +250 +251 +252 +253 +254 +255 +256 +257 +258 +259 +260 +261 +262 +263 +264 +265 +266 +267 +268 +269 +270 +271 +272 +273 +274 +275 +276 +277 +278 +279 +280 +281 +282 +283 +284 +285 +286 +287 +288 +289 +290 +291 +292 +293 +294 +295 +296 +297 +298 +299 +300 +301 +302 +303 +304 +305 +306 +307 +308 +309 +310 +311 +312 +313 +314 +315 +316 +317 +318 +319 +320 +321 +322 +323 +324 +325 +326 +327 +328 +329 +330 +331 +332 +333 +334 +335 +336 +337 +338 +339 +340 +341 +342 +343 +344 +345 +346 +347 +348 +349 +350 +351 +352 +353 +354 +355 +356 +357 +358 +359 +360 +361 +362 +363 +364 +365 +366 +367 +368 +369 +370 +371 +372 +373 +374 +375 +376 +377 +378 +379 +380 +381 +382 +383 +384 +385 +386 +387 +388 +389 +390 +391 +392 +393 +394 +395 +396 +397 +398 +399 +400 +401 +402 +403 +404 +405 +406 +407 +408 +409 +410 +411 +412 +413 +414 +415 +416 +417 +418 +419 +420 +421 +422 +423 +424 +425 +426 +427 +428 +429 +430 +431 +432 +433 +434 +435 +436 +437 +438 +439 +440 +441 +442 +443 +444 +445 +446 +447 +448 +449 +450 +451 +452 +453 +454 +455 +456 +457 +458 +459 +460 +461 +462 +463 +464 +465 +466 +467 +468 +469 +470 +471 +472 +473 +474 +475 +476 +477 +478 +479 +480 +481 +482 +483 +484 +485 +486 +487 +488 +489 +490 +491 +492 +493 +494 +495 +496 +497 +498 +499 +500 +501 +502 +503 +504 +505 +506 +507 +508 +509 +510 +511 +512 +513 +514 +515 +516 +517 +518 +519 +520 +521 +522 +523 +524 +525 +526 +527 +528 +529 +530 +531 +532 +533 +534 +535 +536 +537 +538 +539 +540 +541 +542 +543 +544 +545 +546 +547 +548 +549 +550 +551 +552 +553 +554 +555 +556 +557 +558 +559 +560 +561 +562 +563 +564 +565 +566 +567 +568 +569 +570 +571 +572 +573 +574 +575 +576 +577 +578 +579 +580 +581 +582 +583 +584 +585 +586 +587 +588 +589 +590 +591 +592 +593 +594 +595 +596 +597 +598 +599 +600 +601 +602 +603 +604 +605 +606 +607 +608 +609 +610 +611 +612 +613 +614 +615 +616 +617 +618 +619 +620 +621 +622 +623 +624 +625 +626 +627 +628 +629 +630 +631 +632 +633 +634 +635 +636 +637 +638 +639 +640 +641 +642 +643 +644 +645 +646 +647 +648 +649 +650 +651 +652 +653 +654 +655 +656 +657 +658 +659 +660 +661 +662 +663 +664 +665 +666 +667 +668 +669 +670 +671 +672 +673 +674 +675 +676 +677 +678 +679 +680 +681 +682 +683 +684 +685 +686 +687 +688 +689 +690 +691 +692 +693 +694 +695 +696 +697 +698 +699 +700 +701 +702 +703 +704 +705 +706 +707 +708 +709 +710 +711 +712 +713 +714 +715 +716 +717 +718 +719 +720 +721 +722 +723 +724 +725 +726 +727 +728 +729 +730 +731 +732 +733 +734 +735 +736 +737 +738 +739 +740 +741 +742 +743 +744 +745 +746 +747 +748 +749 +750 +751 +752 +753 +754 +755 +756 +757 +758 +759 +760 +761 +762 +763 +764 +765 +766 +767 +768 +769 +770 +771 +772 +773 +774 +775 +776 +777 +778 +779 +780 +781 +782 +783 +784 +785 +786 +787 +788 +789 +790 +791 +792 +793 +794 +795 +796 +797 +798 +799 +800 +801 +802 +803 +804 +805 +806 +807 +808 +809 +810 +811 +812 +813 +814 +815 +816 +817 +818 +819 +820 +821 +822 +823 +824 +825 +826 +827 +828 +829 +830 +831 +832 +833 +834 +835 +836 +837 +838 +839 +840 +841 +842 +843 +844 +845 +846 +847 +848 +849 +850 +851 +852 +853 +854 +855 +856 +857 +858 +859 +860 +861 +862 +863 +864 +865 +866 +867 +868 +869 +870 +871 +872 +873 +874 +875 +876 +877 +878 +879 +880 +881 +882 +883 +884 +885 +886 +887 +888 +889 +890 +891 +892 +893 +894 +895 +896 +897 +898 +899 +900 +901 +902 +903 +904 +905 +906 +907 +908 +909 +910 +911 +912 +913 +914 +915 +916 +917 +918 +919 +920 +921 +922 +923 +924 +925 +926 +927 +928 +929 +930 +931 +932 +933 +934 +935 +936 +937 +938 +939 +940 +941 +942 +943 +944 +945 +946 +947 +948 +949 +950 +951 +952 +953 +954 +955 +956 +957 +958 +959 +960 +961 +962 +963 +964 +965 +966 +967 +968 +969 +970 +971 +972 +973 +974 +975 +976 +977 +978 +979 +980 +981 +982 +983 +984 +985 +986 +987 +988 +989 +990 +991 +992 +993 +994 +995 +996 +997 +998 +999 +1000 + +Hello, World +Oops +Z +汉字漢字汉字漢字汉字漢字汉字漢字汉字漢字汉字漢字汉字漢字汉字漢字 + + + +-[ \eAAAAAAABBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDEEEEEEEEEFFFFFFFFFGGGGGGG +<N7p0CR'yOIUQ0K<QrG~R&X +W X&5B6Ar"JlTd&.JC)`?? ݸƬg鸛r͘ [9qȵpc5Lr H~4mls-'ҩ}ӝO
m(?0<%{{_ +ZeAN.Q^ ~V.kJK=sz0=i +vʈF X\ \:ڸч2b"}o:EiW@r*Ws Kŕ!Sg5Yg+Y2R6`"Jʈ0 )@G/ (Ck: N +Ut;MQib a.|(2&lڃx5۸ȁ8ZaX )QRklthR[)73m(8D'R(7.ٮ12jo^/P$H +HD ?&^Ci- ؆?vcF|O% hn9&BV3;Sv䪝 쓻Q.Ia` Uv"{K(d)JHy[rש4jU +<= L"8,Sf`Dvj='z]0^ʼHIjoəIC?$SG9ߛ|앥eu +䓅 wniE2R}s@uO.6_Q1,OՖ.!n{C +`TuhM +%=D̹]iTYn`w{b{ד`^
WhC T
ŀ6|>4|huﴯU-:aу_'rUu8p35
sjm79G CG8
ܘ ޮM!8 ]w!tTrRh ks>XجIy'\\Z0:lwkI4Jg`Mb$p;gj +g
!EI#߉kۘ7"BV|ؽ=]K试 7)H{Q a2B7YI|"w]mwNJb ],p gԍ@ rB $3zSDKqZEr"BgM+9EQ +r^ƮxJEM>vF?D +O}wE q oYtAZ
X{JN JcZ!= ߎb8 +D/!IK`
> B(HL FEO: +LA/ u:| +]oC# Mux*ۘ8_)p6B3L
mìO1 +Iw@qR)Ue@V2VzDzӃ>~r$ ȸπW7דJ;jd"~J;^G9 1(<]ayh-PSg(6BD 3O7laU%& : >+ )L؞x +}~KBot:Zuag( "H?z72f;DHBfш%% =ΫօyT-㼡Nj +Xk,@G;T|]ґlحRȭP.A\V|~NPQ%UA銰YRre T1:=K"a|&UY8?-J4 ]濋G?!pGy(Vvc
i
*omzw%25@ܒZBxQy ++$o~2N70!=Q"̢C/\?&xj_n1 Psu9rȬ^?3R5U>c8_SW5LBϚ1Zo?| ИDW9_lx_/Hyb0;ϫO ,eͦ8gG]8 ; j拱g,uH'rU[YPͣE p.9WMs)řm fΧ͍Va>D"h<H<*tc%]S
\ No newline at end of file diff --git a/utf8parse/Cargo.toml b/utf8parse/Cargo.toml index d8b9174..8c8c8df 100644 --- a/utf8parse/Cargo.toml +++ b/utf8parse/Cargo.toml @@ -10,4 +10,6 @@ version = "0.1.1" name = "utf8parse" edition = "2018" -[dependencies] +[features] +nightly = [] +default = [] diff --git a/utf8parse/src/lib.rs b/utf8parse/src/lib.rs index 8c866f5..c092647 100644 --- a/utf8parse/src/lib.rs +++ b/utf8parse/src/lib.rs @@ -3,15 +3,14 @@ //! This module implements a table-driven UTF-8 parser which should //! theoretically contain the minimal number of branches (1). The only branch is //! on the `Action` returned from unpacking a transition. +#![cfg_attr(all(feature = "nightly", test), feature(test))] #![no_std] use core::char; -mod table; mod types; -use table::TRANSITIONS; -use types::{unpack, Action, State}; +use types::{Action, State}; /// Handles codepoint and invalid sequence events from the parser. pub trait Receiver { @@ -48,10 +47,7 @@ impl Parser { where R: Receiver, { - let cur = self.state as usize; - let change = TRANSITIONS[cur][byte as usize]; - let (state, action) = unsafe { unpack(change) }; - + let (state, action) = self.state.advance(byte); self.perform_action(receiver, byte, action); self.state = state; } @@ -93,3 +89,43 @@ impl Parser { } } } + +#[cfg(all(feature = "nightly", test))] +mod benches { + extern crate std; + extern crate test; + + use super::{Parser, Receiver}; + + use self::test::{black_box, Bencher}; + + static UTF8_DEMO: &[u8] = include_bytes!("../tests/UTF-8-demo.txt"); + + impl Receiver for () { + fn codepoint(&mut self, c: char) { + black_box(c); + } + + fn invalid_sequence(&mut self) {} + } + + #[bench] + fn parse_bench_utf8_demo(b: &mut Bencher) { + let mut parser = Parser::new(); + + b.iter(|| { + for byte in UTF8_DEMO { + parser.advance(&mut (), *byte); + } + }) + } + + #[bench] + fn std_string_parse_utf8(b: &mut Bencher) { + b.iter(|| { + for c in std::str::from_utf8(UTF8_DEMO).unwrap().chars() { + black_box(c); + } + }); + } +} diff --git a/utf8parse/src/table.rs b/utf8parse/src/table.rs deleted file mode 100644 index 04c4d34..0000000 --- a/utf8parse/src/table.rs +++ /dev/null @@ -1,166 +0,0 @@ -//! UTF-8 Parse Transition Table - -/// Transition table for parsing UTF-8. This is built from the grammar described -/// at https://tools.ietf.org/html/rfc3629#section-4 which I have copied and -/// formatted below. -/// -/// # UTF-8 Grammar -/// -/// ```notrust -/// UTF8-octets = *( UTF8-char ) -/// UTF8-char = UTF8-1 / UTF8-2 / UTF8-3 / UTF8-4 -/// UTF8-1 = %x00-7F -/// UTF8-2 = %xC2-DF UTF8-tail -/// UTF8-3 = %xE0 %xA0-BF UTF8-tail / -/// %xE1-EC 2( UTF8-tail ) / -/// %xED %x80-9F UTF8-tail / -/// %xEE-EF 2( UTF8-tail ) -/// UTF8-4 = %xF0 %x90-BF 2( UTF8-tail ) / -/// %xF1-F3 3( UTF8-tail ) / -/// %xF4 %x80-8F 2( UTF8-tail ) -/// UTF8-tail = %x80-BF -/// ``` -/// -/// Not specifying an action in this table is equivalent to specifying -/// Action::InvalidSequence. Not specifying a state is equivalent to specifying -/// state::ground. -pub static TRANSITIONS: [[u8; 256]; 8] = [ - [ - 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, - 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, - 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, - 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, - 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, - 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, - 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, - 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, - 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 67u8, 67u8, 67u8, 67u8, 67u8, 67u8, 67u8, 67u8, 67u8, 67u8, 67u8, 67u8, - 67u8, 67u8, 67u8, 67u8, 67u8, 67u8, 67u8, 67u8, 67u8, 67u8, 67u8, 67u8, 67u8, 67u8, 67u8, - 67u8, 67u8, 67u8, 100u8, 98u8, 98u8, 98u8, 98u8, 98u8, 98u8, 98u8, 98u8, 98u8, 98u8, 98u8, - 98u8, 101u8, 98u8, 98u8, 118u8, 113u8, 113u8, 113u8, 119u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, - ], - [ - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, - 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, - 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, - 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, - 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - ], - [ - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, - 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, - 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, - 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, - 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - ], - [ - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 32u8, 32u8, 32u8, 32u8, 32u8, 32u8, 32u8, 32u8, 32u8, 32u8, 32u8, 32u8, 32u8, - 32u8, 32u8, 32u8, 32u8, 32u8, 32u8, 32u8, 32u8, 32u8, 32u8, 32u8, 32u8, 32u8, 32u8, 32u8, - 32u8, 32u8, 32u8, 32u8, 32u8, 32u8, 32u8, 32u8, 32u8, 32u8, 32u8, 32u8, 32u8, 32u8, 32u8, - 32u8, 32u8, 32u8, 32u8, 32u8, 32u8, 32u8, 32u8, 32u8, 32u8, 32u8, 32u8, 32u8, 32u8, 32u8, - 32u8, 32u8, 32u8, 32u8, 32u8, 32u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - ], - [ - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 51u8, 51u8, - 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, - 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - ], - [ - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, - 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, 51u8, - 51u8, 51u8, 51u8, 51u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - ], - [ - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, - 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, - 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, - 82u8, 82u8, 82u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - ], - [ - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, 82u8, - 82u8, 82u8, 82u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, - ], -]; diff --git a/utf8parse/src/table.rs.in b/utf8parse/src/table.rs.in deleted file mode 100644 index 2acafe7..0000000 --- a/utf8parse/src/table.rs.in +++ /dev/null @@ -1,60 +0,0 @@ -//! UTF-8 Parse Transition Table - -/// Transition table for parsing UTF-8. This is built from the grammar described -/// at https://tools.ietf.org/html/rfc3629#section-4 which I have copied and -/// formatted below. -/// -/// # UTF-8 Grammar -/// -/// ```ignore -/// UTF8-octets = *( UTF8-char ) -/// UTF8-char = UTF8-1 / UTF8-2 / UTF8-3 / UTF8-4 -/// UTF8-1 = %x00-7F -/// UTF8-2 = %xC2-DF UTF8-tail -/// UTF8-3 = %xE0 %xA0-BF UTF8-tail / -/// %xE1-EC 2( UTF8-tail ) / -/// %xED %x80-9F UTF8-tail / -/// %xEE-EF 2( UTF8-tail ) -/// UTF8-4 = %xF0 %x90-BF 2( UTF8-tail ) / -/// %xF1-F3 3( UTF8-tail ) / -/// %xF4 %x80-8F 2( UTF8-tail ) -/// UTF8-tail = %x80-BF -/// ``` -/// -/// Not specifying an action in this table is equivalent to specifying -/// Action::InvalidSequence. Not specifying a state is equivalent to specifying -/// state::ground. -pub static TRANSITIONS: [[u8; 256]; 8] = utf8_state_table! { - State::Ground => { - 0x00...0x7f => (State::Ground, Action::EmitByte), - 0xc2...0xdf => (State::Tail1, Action::SetByte2Top), - 0xe0 => (State::U3_2_e0, Action::SetByte3Top), - 0xe1...0xec => (State::Tail2, Action::SetByte3Top), - 0xed => (State::U3_2_ed, Action::SetByte3Top), - 0xee...0xef => (State::Tail2, Action::SetByte3Top), - 0xf0 => (State::Utf8_4_3_f0, Action::SetByte4), - 0xf1...0xf3 => (State::Tail3, Action::SetByte4), - 0xf4 => (State::Utf8_4_3_f4, Action::SetByte4), - }, - State::U3_2_e0 => { - 0xa0...0xbf => (State::Tail1, Action::SetByte2), - }, - State::U3_2_ed => { - 0x80...0x9f => (State::Tail1, Action::SetByte2), - }, - State::Utf8_4_3_f0 => { - 0x90...0xbf => (State::Tail2, Action::SetByte3), - }, - State::Utf8_4_3_f4 => { - 0x80...0x8f => (State::Tail2, Action::SetByte3), - }, - State::Tail3 => { - 0x80...0xbf => (State::Tail2, Action::SetByte3), - }, - State::Tail2 => { - 0x80...0xbf => (State::Tail1, Action::SetByte2), - }, - State::Tail1 => { - 0x80...0xbf => (State::Ground, Action::SetByte1), - }, -}; diff --git a/utf8parse/src/types.rs b/utf8parse/src/types.rs index 93607fb..5a70b3c 100644 --- a/utf8parse/src/types.rs +++ b/utf8parse/src/types.rs @@ -1,12 +1,31 @@ //! Types supporting the UTF-8 parser -#![allow(non_camel_case_types)] -use core::mem; + +/// Action to take when receiving a byte +#[derive(Debug, Copy, Clone)] +pub enum Action { + /// Unexpected byte; sequence is invalid + InvalidSequence = 0, + /// Received valid 7-bit ASCII byte which can be directly emitted. + EmitByte = 1, + /// Set the bottom continuation byte + SetByte1 = 2, + /// Set the 2nd-from-last continuation byte + SetByte2 = 3, + /// Set the 2nd-from-last byte which is part of a two byte sequence + SetByte2Top = 4, + /// Set the 3rd-from-last continuation byte + SetByte3 = 5, + /// Set the 3rd-from-last byte which is part of a three byte sequence + SetByte3Top = 6, + /// Set the top byte of a four byte sequence. + SetByte4 = 7, +} /// States the parser can be in. /// /// There is a state for each initial input of the 3 and 4 byte sequences since /// the following bytes are subject to different conditions than a tail byte. -#[allow(dead_code)] +#[allow(non_camel_case_types)] #[derive(Debug, Copy, Clone)] pub enum State { /// Ground state; expect anything @@ -33,50 +52,54 @@ impl Default for State { } } -/// Action to take when receiving a byte -#[allow(dead_code)] -#[derive(Debug, Copy, Clone)] -pub enum Action { - /// Unexpected byte; sequence is invalid - InvalidSequence = 0, - /// Received valid 7-bit ASCII byte which can be directly emitted. - EmitByte = 1, - /// Set the bottom continuation byte - SetByte1 = 2, - /// Set the 2nd-from-last continuation byte - SetByte2 = 3, - /// Set the 2nd-from-last byte which is part of a two byte sequence - SetByte2Top = 4, - /// Set the 3rd-from-last continuation byte - SetByte3 = 5, - /// Set the 3rd-from-last byte which is part of a three byte sequence - SetByte3Top = 6, - /// Set the top byte of a four byte sequence. - SetByte4 = 7, -} - -/// Convert a state and action to a u8 -/// -/// State will be the bottom 4 bits and action the top 4 -#[inline] -#[allow(dead_code)] -pub fn pack(state: State, action: Action) -> u8 { - ((action as u8) << 4) | (state as u8) -} - -/// Convert a u8 to a state and action -/// -/// # Unsafety -/// -/// If this function is called with a byte that wasn't encoded with the `pack` -/// function in this module, there is no guarantee that a valid state and action -/// can be produced. -#[inline] -pub unsafe fn unpack(val: u8) -> (State, Action) { - ( - // State is stored in bottom 4 bits - mem::transmute(val & 0x0f), - // Action is stored in top 4 bits - mem::transmute(val >> 4), - ) +impl State { + /// Advance the parser state. + /// + /// This takes the current state and input byte into consideration, to determine the next state + /// and any action that should be taken. + #[inline] + pub fn advance(&self, byte: u8) -> (State, Action) { + match self { + State::Ground => match byte { + 0x00..=0x7f => (State::Ground, Action::EmitByte), + 0xc2..=0xdf => (State::Tail1, Action::SetByte2Top), + 0xe0 => (State::U3_2_e0, Action::SetByte3Top), + 0xe1..=0xec => (State::Tail2, Action::SetByte3Top), + 0xed => (State::U3_2_ed, Action::SetByte3Top), + 0xee..=0xef => (State::Tail2, Action::SetByte3Top), + 0xf0 => (State::Utf8_4_3_f0, Action::SetByte4), + 0xf1..=0xf3 => (State::Tail3, Action::SetByte4), + 0xf4 => (State::Utf8_4_3_f4, Action::SetByte4), + _ => (State::Ground, Action::InvalidSequence), + }, + State::U3_2_e0 => match byte { + 0xa0..=0xbf => (State::Tail1, Action::SetByte2), + _ => (State::Ground, Action::InvalidSequence), + }, + State::U3_2_ed => match byte { + 0x80..=0x9f => (State::Tail1, Action::SetByte2), + _ => (State::Ground, Action::InvalidSequence), + }, + State::Utf8_4_3_f0 => match byte { + 0x90..=0xbf => (State::Tail2, Action::SetByte3), + _ => (State::Ground, Action::InvalidSequence), + }, + State::Utf8_4_3_f4 => match byte { + 0x80..=0x8f => (State::Tail2, Action::SetByte3), + _ => (State::Ground, Action::InvalidSequence), + }, + State::Tail3 => match byte { + 0x80..=0xbf => (State::Tail2, Action::SetByte3), + _ => (State::Ground, Action::InvalidSequence), + }, + State::Tail2 => match byte { + 0x80..=0xbf => (State::Tail1, Action::SetByte2), + _ => (State::Ground, Action::InvalidSequence), + }, + State::Tail1 => match byte { + 0x80..=0xbf => (State::Ground, Action::SetByte1), + _ => (State::Ground, Action::InvalidSequence), + }, + } + } } |