From 91d7ed690bfd9989270ba9fa4f44c70f513d54b9 Mon Sep 17 00:00:00 2001 From: Jeremy Wall Date: Sun, 23 Sep 2018 15:08:45 -0500 Subject: [PATCH] REFACTOR: Use abortable_parser. --- Cargo.lock | 37 +- Cargo.toml | 5 +- TODO.md | 2 + src/benches/parse.rs | 7 +- src/build/mod.rs | 43 +- src/error.rs | 28 +- src/lib.rs | 4 +- src/parse/mod.rs | 990 ++++++++++++++++++++-------------------- src/parse/precedence.rs | 401 +++++++--------- src/parse/test.rs | 43 +- src/tokenizer/mod.rs | 677 ++++++++++++--------------- src/tokenizer/test.rs | 212 +++++---- 12 files changed, 1147 insertions(+), 1302 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b0b36db..99b7d57 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,3 +1,8 @@ +[[package]] +name = "abortable_parser" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "ansi_term" version = "0.9.0" @@ -131,31 +136,6 @@ name = "linked-hash-map" version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -[[package]] -name = "memchr" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "nom" -version = "3.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "memchr 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "nom_locate" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "memchr 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "nom 3.2.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "redox_syscall" version = "0.1.40" @@ -248,11 +228,10 @@ dependencies = [ name = "ucg" version = "0.2.0" dependencies = [ + "abortable_parser 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "bencher 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", "clap 2.26.2 (registry+https://github.com/rust-lang/crates.io-index)", "cpuprofiler 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)", - "nom 3.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "nom_locate 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.26 (registry+https://github.com/rust-lang/crates.io-index)", "serde_yaml 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", "simple-error 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)", @@ -306,6 +285,7 @@ dependencies = [ ] [metadata] +"checksum abortable_parser 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a628e31269165eeea62b71b2555c6379d4fbadb3e34656b6e1445b0235247c0d" "checksum ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "23ac7c30002a5accbf7e8987d0632fa6de155b7c3d39d0067317a391e00a2ef6" "checksum atty 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "9a7d5b8723950951411ee34d271d99dddcc2035a16ab25310ea2c8cfd4369652" "checksum backtrace 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "346d7644f0b5f9bc73082d3b2236b69a05fd35cce0cfa3724e184e6a5c9e2a2f" @@ -324,9 +304,6 @@ dependencies = [ "checksum lazy_static 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "76f033c7ad61445c5b347c7382dd1237847eb1bce590fe50365dcb33d546be73" "checksum libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)" = "76e3a3ef172f1a0b9a9ff0dd1491ae5e6c948b94479a3021819ba7d860c8645d" "checksum linked-hash-map 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "70fb39025bc7cdd76305867c4eccf2f2dcf6e9a57f5b21a93e1c2d86cd03ec9e" -"checksum memchr 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "148fab2e51b4f1cfc66da2a7c32981d1d3c083a803978268bb11fe4b86925e7a" -"checksum nom 3.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "05aec50c70fd288702bcd93284a8444607f3292dbdf2a30de5ea5dcdbe72287b" -"checksum nom_locate 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "49b1c61eff39ab6b91ccedfc62aff196eae066d88355b4fe3e4100c23168f0df" "checksum redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)" = "c214e91d3ecf43e9a4e41e578973adeb14b474f2bee858742d127af75a0112b1" "checksum redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7e891cfe48e9100a70a3b6eb652fef28920c117d366339687bd5576160db0f76" "checksum rustc-demangle 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "bcfe5b13211b4d78e5c2cadfebd7769197d95c639c35a50057eb4c05de811395" diff --git a/Cargo.toml b/Cargo.toml index beb52f0..16f62b2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,11 +9,8 @@ readme = "README.md" keywords = ["compiler", "config"] license = "Apache-2.0" -[dependencies.nom] -version = "^3.2" - [dependencies] -nom_locate = "^0.1.1" +abortable_parser = "~0.1.0" clap = "~2.26.0" serde_json = "~1.0.9" simple-error = "0.1" diff --git a/TODO.md b/TODO.md index 788e210..758ef4c 100644 --- a/TODO.md +++ b/TODO.md @@ -9,6 +9,8 @@ You should be able to ask the compiler to tell you any value or set of values in the compiled configuration. +Inspect is probably the correct location for this. + ## Shape equality as a form of type assertion? # Minor Fixes and Polish diff --git a/src/benches/parse.rs b/src/benches/parse.rs index 0cc1c25..bcef0cc 100644 --- a/src/benches/parse.rs +++ b/src/benches/parse.rs @@ -16,17 +16,20 @@ #[macro_use] extern crate bencher; +extern crate abortable_parser; extern crate cpuprofiler; -extern crate nom_locate; extern crate ucglib; use bencher::Bencher; + +use abortable_parser::StrIter; + //use cpuprofiler::PROFILER; use ucglib::parse::*; fn do_parse(i: &str) { - parse(nom_locate::LocatedSpan::new(i)); + parse(StrIter::new(i)); } fn parse_int(b: &mut Bencher) { diff --git a/src/build/mod.rs b/src/build/mod.rs index c0fa3e5..a26a7fa 100644 --- a/src/build/mod.rs +++ b/src/build/mod.rs @@ -25,11 +25,12 @@ use std::path::PathBuf; use std::rc::Rc; use std::string::ToString; +use abortable_parser::StrIter; + use ast::*; use error; use format; use parse::parse; -use tokenizer::Span; pub mod assets; pub mod ir; @@ -159,17 +160,18 @@ impl<'a> Builder<'a> { &Value::Int(ref i) => Ok(Rc::new(Val::Int(i.val))), &Value::Float(ref f) => Ok(Rc::new(Val::Float(f.val))), &Value::Str(ref s) => Ok(Rc::new(Val::Str(s.val.to_string()))), - &Value::Symbol(ref s) => self.lookup_sym(&(s.into())).ok_or(Box::new( - error::Error::new( - format!( - "Unable to find {} in file: {}", - s.val, - self.root.to_string_lossy() - ), - error::ErrorType::NoSuchSymbol, - v.pos().clone(), - ), - )), + &Value::Symbol(ref s) => { + self.lookup_sym(&(s.into())) + .ok_or(Box::new(error::Error::new( + format!( + "Unable to find {} in file: {}", + s.val, + self.root.to_string_lossy() + ), + error::ErrorType::NoSuchSymbol, + v.pos().clone(), + ))) + } &Value::List(ref def) => self.list_to_val(def), &Value::Tuple(ref tuple) => self.tuple_to_val(&tuple.val), &Value::Selector(ref selector_list_node) => { @@ -243,7 +245,7 @@ impl<'a> Builder<'a> { Ok(()) } - fn eval_span(&mut self, input: Span) -> Result, Box> { + fn eval_span(&mut self, input: StrIter) -> Result, Box> { match parse(input) { Ok(stmts) => { //panic!("Successfully parsed {}", input); @@ -256,20 +258,20 @@ impl<'a> Builder<'a> { Some(val) => Ok(val), } } - Err(err) => Err(Box::new(error::Error::new_with_cause( + Err(err) => Err(Box::new(error::Error::new_with_boxed_cause( format!( "Error while parsing file: {}", self.curr_file.unwrap_or("") ), error::ErrorType::ParseError, - err, + Box::new(err), ))), } } /// Evaluate an input string as UCG. pub fn eval_string(&mut self, input: &str) -> Result, Box> { - self.eval_span(Span::new(input)) + self.eval_span(StrIter::new(input)) } /// Builds a ucg file at the named path. @@ -839,8 +841,7 @@ impl<'a> Builder<'a> { let first = a.0.clone(); let t = a.1.clone(); (first, t.1) - }) - .collect(), + }).collect(), ))); } Err(Box::new(error::Error::new( @@ -992,11 +993,7 @@ impl<'a> Builder<'a> { let expr = &tok.fragment; expr_as_stmt.push_str(expr); expr_as_stmt.push_str(";"); - let assert_input = Span { - fragment: &expr_as_stmt, - line: tok.pos.line as u32, - offset: tok.pos.column, - }; + let assert_input = StrIter::new(&expr_as_stmt); let ok = match self.eval_span(assert_input) { Ok(v) => v, Err(e) => { diff --git a/src/error.rs b/src/error.rs index 77ff70a..aa0958e 100644 --- a/src/error.rs +++ b/src/error.rs @@ -18,8 +18,6 @@ use std::fmt; use ast::*; -use nom; - /// ErrorType defines the various types of errors that can result from compiling UCG into an /// output format. pub enum ErrorType { @@ -62,7 +60,7 @@ pub struct Error { pub err_type: ErrorType, pub pos: Position, pub msg: String, - pub cause: Option>, + pub cause: Option>, _pkgonly: (), } @@ -77,8 +75,12 @@ impl Error { } } - pub fn new_with_boxed_cause>(msg: S, t: ErrorType, cause: Box) -> Self { - let mut e = Self::new(msg, t, cause.pos.clone()); + pub fn new_with_boxed_cause>( + msg: S, + t: ErrorType, + cause: Box, + ) -> Self { + let mut e = Self::new(msg, t, Position { line: 0, column: 0 }); e.cause = Some(cause); return e; } @@ -87,22 +89,6 @@ impl Error { Self::new_with_boxed_cause(msg, t, Box::new(cause)) } - pub fn new_with_errorkind>( - msg: S, - t: ErrorType, - pos: Position, - cause: nom::ErrorKind, - ) -> Self { - match cause { - nom::ErrorKind::Custom(e) => Self::new_with_cause(msg, t, e), - e => Self::new_with_cause( - msg, - t, - Error::new(format!("ErrorKind: {}", e), ErrorType::Unsupported, pos), - ), - } - } - fn render(&self, w: &mut fmt::Formatter) -> fmt::Result { try!(write!( w, diff --git a/src/lib.rs b/src/lib.rs index 43e8ee8..a4f81cc 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -444,9 +444,7 @@ // to succeed. #![recursion_limit = "128"] #[macro_use] -extern crate nom; -#[macro_use] -extern crate nom_locate; +extern crate abortable_parser; extern crate serde_json; extern crate serde_yaml; extern crate simple_error; diff --git a/src/parse/mod.rs b/src/parse/mod.rs index 1bd0c47..23c2d9b 100644 --- a/src/parse/mod.rs +++ b/src/parse/mod.rs @@ -13,46 +13,28 @@ // limitations under the License. //! The Parsing stage of the ucg compiler. +use std; use std::borrow::Borrow; use std::str::FromStr; -use nom; -use nom::IResult; -use nom::InputLength; -use nom_locate::LocatedSpan; +use abortable_parser; +use abortable_parser::combinators::eoi; +use abortable_parser::iter::{SliceIter, StrIter}; +use abortable_parser::{Error, Peekable, Result}; use self::precedence::op_expression; use ast::*; -use error; use tokenizer::*; -type NomResult<'a, O> = nom::IResult, O, error::Error>; +type NomResult<'a, O> = Result, O>; +// FIXME(jwall): All the do_each mappers need to return an actual value. #[cfg(feature = "tracing")] const ENABLE_TRACE: bool = true; #[cfg(not(feature = "tracing"))] const ENABLE_TRACE: bool = false; -type ParseResult = Result; - -macro_rules! wrap_err { - ($i:expr, $submac:ident, $msg:expr) => { - wrap_err!($i, call!($submac), $msg) - }; - - ($i:expr, $submac:ident!( $($args:tt)* ), $msg:expr) => {{ - let _i = $i.clone(); - match $submac!(_i, $($args)*) { - IResult::Done(rest, mac) => IResult::Done(rest, mac), - IResult::Incomplete(i) => IResult::Incomplete(i), - IResult::Error(nom::ErrorKind::Custom(cause)) => { - let wrapper = error::Error::new_with_cause($msg, error::ErrorType::ParseError, cause); - IResult::Error(nom::ErrorKind::Custom(wrapper)) - } - IResult::Error(e) => IResult::Error(e), - } - }}; -} +type ParseResult = std::result::Result; macro_rules! trace_nom { ($i:expr, $rule:ident!( $($args:tt)* )) => { @@ -61,7 +43,7 @@ macro_rules! trace_nom { if ENABLE_TRACE { eprintln!("Entering Rule: {:?} {:?}", stringify!($rule), $i); } - let result = $rule($i, $($args)* ); + let result = $rule!($i, $($args)* ); if ENABLE_TRACE { eprintln!("Exiting Rule: {:?} with {:?}", stringify!($rule), result); } @@ -75,7 +57,7 @@ macro_rules! trace_nom { if ENABLE_TRACE { eprintln!("Entering Rule: {:?} {:?}", stringify!($rule), $i); } - let result = call!($i, $rule); + let result = run!($i, $rule); if ENABLE_TRACE { eprintln!("Exiting Rule: {:?} with {:?}", stringify!($rule), result); } @@ -92,7 +74,8 @@ fn symbol_to_value(s: &Token) -> ParseResult { } // symbol is a bare unquoted field. -named!(symbol, +make_fn!( + symbol, Value>, match_type!(BAREWORD => symbol_to_value) ); @@ -104,10 +87,12 @@ fn str_to_value(s: &Token) -> ParseResult { } // quoted_value is a quoted string. -named!(quoted_value, - match_type!(STR => str_to_value) +make_fn!( + quoted_value, Value>, + match_type!(STR => str_to_value) ); +// FIXME(jwall): We need to just turn this into a custom parser function. // Helper function to make the return types work for down below. fn triple_to_number(v: (Option, Option, Option)) -> ParseResult { let (pref, mut pref_pos) = match v.0 { @@ -121,11 +106,11 @@ fn triple_to_number(v: (Option, Option, Option)) -> ParseRe let i = match FromStr::from_str(pref) { Ok(i) => i, Err(_) => { - return Err(error::Error::new( + return Err(Error::new( format!("Not an integer! {}", pref), - error::ErrorType::UnexpectedToken, - pref_pos, - )) + // FIXME(jwall): This really needs the correct offset. + &0, + )); } }; return Ok(Value::Int(value_node!(i, pref_pos))); @@ -135,35 +120,35 @@ fn triple_to_number(v: (Option, Option, Option)) -> ParseRe pref_pos = v.1.unwrap().pos; } - let (maybepos, suf) = match v.2 { - None => (None, "".to_string()), - Some(bs) => (Some(bs.pos), bs.fragment), + let suf = match v.2 { + None => "".to_string(), + Some(bs) => bs.fragment, }; let to_parse = pref.to_string() + "." + &suf; let f = match FromStr::from_str(&to_parse) { Ok(f) => f, Err(_) => { - return Err(error::Error::new( + return Err(Error::new( format!("Not a float! {}", to_parse), - error::ErrorType::UnexpectedToken, - // NOTE(jwall): This is ugly. I should probably see if I can refactor - // it to something less confusing. - maybepos.unwrap(), + // FIXME(jwall): This should take the real offset. + &0, )); } }; return Ok(Value::Float(value_node!(f, pref_pos))); } +// FIXME(jwall): This should actually be unnecessary now. + /// alt_peek conditionally runs a combinator if a lookahead combinator matches. macro_rules! alt_peek { (__inner $i:expr, $peekrule:ident!( $($peekargs:tt)* ) => $parserule:ident | $($rest:tt)* ) => ( - alt_peek!(__inner $i, $peekrule!($($peekargs)*) => call!($parserule) | $($rest)* ) + alt_peek!(__inner $i, $peekrule!($($peekargs)*) => run!($parserule) | $($rest)* ) ); (__inner $i:expr, $peekrule:ident => $($rest:tt)* ) => ( - alt_peek!(__inner $i, call!($peekrule) => $($rest)* ) + alt_peek!(__inner $i, run!($peekrule) => $($rest)* ) ); (__inner $i:expr, $peekrule:ident!( $($peekargs:tt)* ) => $parserule:ident!( $($parseargs:tt)* ) | $($rest:tt)* ) => ( @@ -172,14 +157,15 @@ macro_rules! alt_peek { let pre_res = peek!(_i, $peekrule!($($peekargs)*)); match pre_res { // if the peek was incomplete then it might still match so return incomplete. - nom::IResult::Incomplete(i) => nom::IResult::Incomplete(i), + Result::Incomplete(i) => Result::Incomplete(i), // If the peek was in error then try the next peek => parse pair. - nom::IResult::Error(_) => { + Result::Fail(_) => { alt_peek!(__inner $i, $($rest)*) }, + Result::Abort(e) => Result::Abort(e), // If the peek was successful then return the result of the parserule // regardless of it's result. - nom::IResult::Done(_i, _) => { + Result::Complete(_i, _) => { $parserule!(_i, $($parseargs)*) }, } @@ -197,14 +183,15 @@ macro_rules! alt_peek { let pre_res = peek!(_i, $peekrule!($($peekargs)*)); match pre_res { // if the peek was incomplete then it might still match so return incomplete. - nom::IResult::Incomplete(i) => nom::IResult::Incomplete(i), + Result::Incomplete(i) => Result::Incomplete(i), // If the peek was in error then try the next peek => parse pair. - nom::IResult::Error(_) => { + Result::Fail(_) => { alt_peek!(__inner $i, __end) }, + Result::Abort(e) => Result::Abort(e), // If the peek was successful then return the result of the parserule // regardless of it's result. - nom::IResult::Done(_i, _) => { + Result::Complete(_i, _) => { $parserule!(_i, $($parseargs)*) }, } @@ -215,7 +202,7 @@ macro_rules! alt_peek { (__inner $i:expr, $fallback:ident, __end) => ( { let _i = $i.clone(); - call!(_i, $fallback) + run!(_i, $fallback) } ); // In the case of a fallback rule with no peek we just return whatever @@ -230,8 +217,8 @@ macro_rules! alt_peek { // This is our default termination case. // If there is no fallback then we return an Error. (__inner $i:expr, __end) => { - // TODO(jwall): We should do a better custom error here. - nom::IResult::Error(error_position!(nom::ErrorKind::Alt,$i)) + // FIXME(jwall): Should we make this a compile error instead? + compile_error!("alt_peek! requirs a fallback case"); }; // alt_peek entry_point. @@ -246,43 +233,60 @@ macro_rules! alt_peek { // NOTE(jwall): HERE THERE BE DRAGONS. The order for these matters // alot. We need to process alternatives in order of decreasing // specificity. Unfortunately this means we are required to go in a -// decreasing size order which messes with alt!'s completion logic. To +// decreasing size order which messes with either!'s completion logic. To // work around this we have to force Incomplete to be Error so that -// alt! will try the next in the series instead of aborting. +// either! will try the next in the series instead of aborting. // // *IMPORTANT* // It also means this combinator is risky when used with partial // inputs. So handle with care. -named!(number, - map_res!(alt!( - complete!(do_parse!( // 1.0 - prefix: match_type!(DIGIT) >> - has_dot: punct!(".") >> - suffix: match_type!(DIGIT) >> - (Some(prefix.clone()), Some(has_dot.clone()), Some(suffix.clone())) - )) | - complete!(do_parse!( // 1. - prefix: match_type!(DIGIT) >> - has_dot: punct!(".") >> - (Some(prefix.clone()), Some(has_dot.clone()), None) - )) | - complete!(do_parse!( // .1 - has_dot: punct!(".") >> - suffix: match_type!(DIGIT) >> - (None, Some(has_dot.clone()), Some(suffix.clone())) - )) | - do_parse!( // 1 - prefix: match_type!(DIGIT) >> - (Some(prefix.clone()), None, None) - )), - triple_to_number - ) -); +fn number(input: SliceIter) -> Result, Value> { + let parsed = do_each!(input, + num => either!( + complete!( + "Not a float", + do_each!( // 1.0 + prefix => match_type!(DIGIT), + has_dot => punct!("."), + suffix => match_type!(DIGIT), + (Some(prefix.clone()), Some(has_dot.clone()), Some(suffix.clone())) + )), + complete!( + "Not a float", + do_each!( // 1. + prefix => match_type!(DIGIT), + has_dot => punct!("."), + (Some(prefix.clone()), Some(has_dot.clone()), None) + )), + complete!( + "Not a float", + do_each!( // .1 + has_dot => punct!("."), + suffix => match_type!(DIGIT), + (None, Some(has_dot.clone()), Some(suffix.clone())) + )), + do_each!( // 1 + prefix => match_type!(DIGIT), + (Some(prefix.clone()), None, None) + )), + (num) + ); + match parsed { + Result::Abort(e) => Result::Abort(e), + Result::Fail(e) => Result::Fail(e), + Result::Incomplete(offset) => Result::Incomplete(offset), + Result::Complete(rest, triple) => match triple_to_number(triple) { + Ok(val) => Result::Complete(rest, val), + Err(e) => Result::Fail(e), + }, + } +} // trace_macros!(false); -named!(boolean_value, - do_parse!( - b: match_type!(BOOLEAN) >> +make_fn!( + boolean_value, Value>, + do_each!( + b => match_type!(BOOLEAN), (Value::Boolean(Positioned{ val: b.fragment == "true", pos: b.pos, @@ -290,190 +294,194 @@ named!(boolean_value, ) ); -named!( - field_value, - do_parse!( - field: wrap_err!(alt!(match_type!(BAREWORD) | match_type!(STR)), - "Field names must be a bareword or a string.") >> - punct!("=") >> - value: expression >> +make_fn!( + field_value, (Token, Expression)>, + do_each!( + field => wrap_err!(either!(match_type!(BAREWORD), match_type!(STR)), + "Field names must be a bareword or a string."), + _ => punct!("="), + value => expression, (field, value) ) ); // Helper function to make the return types work for down below. -fn vec_to_tuple(t: (Position, Option)) -> ParseResult { - Ok(Value::Tuple(value_node!( - t.1.unwrap_or(Vec::new()), - t.0.line as usize, - t.0.column as usize - ))) +fn vec_to_tuple(pos: Position, fields: Option) -> Value { + Value::Tuple(value_node!( + fields.unwrap_or(Vec::new()), + pos.line as usize, + pos.column as usize + )) } -named!(field_list, - separated_list!(punct!(","), field_value) +make_fn!( + field_list, FieldList>, + separated!(punct!(","), field_value) ); -named!( - tuple, - map_res!( - do_parse!( - pos: pos >> - punct!("{") >> - v: field_list >> - opt_res!(punct!(",")) >> // nom's opt! macro doesn't preserve error types properly but this one does. - punct!("}") >> - (pos, Some(v)) - ), - vec_to_tuple +make_fn!( + tuple, Value>, + do_each!( + pos => pos, + _ => punct!("{"), + v => optional!(field_list), + _ => optional!(punct!(",")), + _ => punct!("}"), + (vec_to_tuple(pos, v)) ) ); -fn tuple_to_list>(t: (Sp, Vec)) -> ParseResult { - return Ok(Value::List(ListDef { - elems: t.1, - pos: t.0.into(), - })); +fn tuple_to_list>(pos: Sp, elems: Vec) -> Value { + Value::List(ListDef { + elems: elems, + pos: pos.into(), + }) } -named!(list_value, - map_res!( - do_parse!( - start: punct!("[") >> - elements: separated_list!(punct!(","), expression) >> - opt_res!(punct!(",")) >> // nom's opt! macro doesn't preserve error types properly but this one does. - punct!("]") >> - (start.pos, elements) - ), - tuple_to_list - ) -); - -named!(empty_value, - do_parse!( - pos: pos >> - match_type!(EMPTY) >> - (Value::Empty(pos)) +make_fn!( + list_value, Value>, + do_each!( + start => punct!("["), + elements => separated!(punct!(","), expression), + _ => optional!(punct!(",")), // nom's opt! macro doesn't preserve error types properly but this one does. + _ => punct!("]"), + (tuple_to_list(start.pos, elements)) ) ); -named!(compound_value, - alt_peek!( - punct!("[") => trace_nom!(list_value) | - punct!("{") => trace_nom!(tuple) +make_fn!( + empty_value, Value>, + do_each!( + pos => pos, + _ => match_type!(EMPTY), + (Value::Empty(pos.into())) ) ); -named!(value, +make_fn!( + compound_value, Value>, + either!(trace_nom!(list_value), trace_nom!(tuple)) +); + +make_fn!( + value, Value>, alt_peek!( symbol_or_expression => trace_nom!(selector_value) - | alt!(punct!("[") | punct!("{")) => trace_nom!(compound_value) + | either!(punct!("["), punct!("{")) => trace_nom!(compound_value) | match_type!(BOOLEAN) => trace_nom!(boolean_value) | match_type!(EMPTY) => trace_nom!(empty_value) - | alt!(match_type!(DIGIT) | punct!(".")) => trace_nom!(number) + | either!(match_type!(DIGIT), punct!(".")) => trace_nom!(number) | trace_nom!(quoted_value) ) - ); - -fn value_to_expression(v: Value) -> ParseResult { - Ok(Expression::Simple(v)) -} - -named!(simple_expression, - map_res!( - trace_nom!(value), - value_to_expression - ) ); -fn expression_to_grouped_expression(e: Expression) -> ParseResult { - Ok(Expression::Grouped(Box::new(e))) +fn value_to_expression(v: Value) -> Expression { + Expression::Simple(v) } -named!(grouped_expression, - map_res!( - preceded!(punct!("("), terminated!(trace_nom!(expression), punct!(")"))), - expression_to_grouped_expression - ) +make_fn!( + simple_expression, Expression>, + do_each!( + val => trace_nom!(value), + (value_to_expression(val)) + ) ); -fn symbol_or_expression(input: TokenIter) -> NomResult { - let scalar_head = do_parse!(input, sym: alt!(symbol | compound_value) >> (sym)); +fn expression_to_grouped_expression(e: Expression) -> Expression { + Expression::Grouped(Box::new(e)) +} + +make_fn!( + grouped_expression, Expression>, + do_each!( + _ => punct!("("), + expr => do_each!( + expr => trace_nom!(expression), + _ => punct!(")"), + (expr) + ), + (expression_to_grouped_expression(expr)) + ) +); + +fn symbol_or_expression(input: SliceIter) -> NomResult { + let _i = input.clone(); + let scalar_head = do_each!(input, + sym => either!(symbol, compound_value), + (sym) + ); match scalar_head { - IResult::Incomplete(i) => IResult::Incomplete(i), - IResult::Error(_) => grouped_expression(input), - IResult::Done(rest, val) => { + Result::Incomplete(offset) => Result::Incomplete(offset), + Result::Fail(_) => grouped_expression(_i), + Result::Abort(e) => Result::Abort(e), + Result::Complete(rest, val) => { let res = peek!(rest.clone(), punct!(".")); match val { Value::Tuple(_) => { - if res.is_done() { - IResult::Done(rest, Expression::Simple(val)) + if res.is_complete() { + Result::Complete(rest, Expression::Simple(val)) } else { - return IResult::Error(nom::ErrorKind::Custom(error::Error::new( + return Result::Fail(Error::new( "Expected (.) but no dot found".to_string(), - error::ErrorType::IncompleteParsing, - val.pos().clone(), - ))); + &rest, + )); } } Value::List(_) => { - if res.is_done() { - IResult::Done(rest, Expression::Simple(val)) + if res.is_complete() { + Result::Complete(rest, Expression::Simple(val)) } else { - return IResult::Error(nom::ErrorKind::Custom(error::Error::new( + return Result::Fail(Error::new( "Expected (.) but no dot found".to_string(), - error::ErrorType::IncompleteParsing, - val.pos().clone(), - ))); + &rest, + )); } } - _ => IResult::Done(rest, Expression::Simple(val)), + _ => Result::Complete(rest, Expression::Simple(val)), } } } } -fn selector_list(input: TokenIter) -> NomResult { +fn selector_list(input: SliceIter) -> NomResult { let (rest, head) = match symbol_or_expression(input) { - IResult::Done(rest, val) => (rest, val), - IResult::Error(e) => { - return IResult::Error(e); + Result::Complete(rest, val) => (rest, val), + Result::Fail(e) => { + return Result::Fail(e); } - IResult::Incomplete(i) => { - return IResult::Incomplete(i); + Result::Incomplete(i) => { + return Result::Incomplete(i); } + Result::Abort(e) => return Result::Abort(e), }; let (rest, is_dot) = match punct!(rest, ".") { - IResult::Done(rest, tok) => (rest, Some(tok)), - IResult::Incomplete(i) => { - return IResult::Incomplete(i); + Result::Complete(rest, tok) => (rest, Some(tok)), + Result::Incomplete(i) => { + return Result::Incomplete(i); } - IResult::Error(_) => (rest, None), + Result::Fail(_) => (rest, None), + Result::Abort(e) => return Result::Abort(e), }; let (rest, list) = if is_dot.is_some() { - let (rest, list) = match separated_list!( + let (rest, list) = match separated!( rest, punct!("."), - alt!(match_type!(BAREWORD) | match_type!(DIGIT) | match_type!(STR)) + either!(match_type!(BAREWORD), match_type!(DIGIT), match_type!(STR)) ) { - IResult::Done(rest, val) => (rest, val), - IResult::Incomplete(i) => { - return IResult::Incomplete(i); - } - IResult::Error(e) => { - return IResult::Error(e); - } + Result::Complete(rest, val) => (rest, val), + Result::Incomplete(i) => return Result::Incomplete(i), + Result::Fail(e) => return Result::Fail(e), + Result::Abort(e) => return Result::Abort(e), }; if list.is_empty() { - return IResult::Error(nom::ErrorKind::Custom(error::Error::new( + return Result::Fail(Error::new( "(.) with no selector fields after".to_string(), - error::ErrorType::IncompleteParsing, - is_dot.unwrap().pos, - ))); + &rest, + )); } else { (rest, Some(list)) } @@ -486,189 +494,214 @@ fn selector_list(input: TokenIter) -> NomResult { tail: list, }; - return IResult::Done(rest, sel_list); + return Result::Complete(rest, sel_list); } -fn tuple_to_copy(t: (SelectorDef, FieldList)) -> ParseResult { - let pos = t.0.pos.clone(); - Ok(Expression::Copy(CopyDef { - selector: t.0, - fields: t.1, +fn tuple_to_copy(def: SelectorDef, fields: Option) -> Expression { + let pos = def.pos.clone(); + let fields = match fields { + Some(fields) => fields, + None => Vec::new(), + }; + Expression::Copy(CopyDef { + selector: def, + fields: fields, pos: pos, - })) + }) } -named!(copy_expression, - map_res!( - do_parse!( - pos: pos >> - selector: trace_nom!(selector_list) >> - punct!("{") >> - fields: trace_nom!(field_list) >> - opt_res!(punct!(",")) >> // noms opt! macro does not preserve error types properly but this one does. - punct!("}") >> - (SelectorDef::new(selector, pos.line, pos.column as usize), fields) - ), - tuple_to_copy +make_fn!( + copy_expression, Expression>, + do_each!( + pos => pos, + selector => trace_nom!(selector_list), + _ => punct!("{"), + fields => optional!(trace_nom!(field_list)), + _ => optional!(punct!(",")), // noms opt! macro does not preserve error types properly but this one does. + _ => punct!("}"), + (tuple_to_copy(SelectorDef::new(selector, pos.line, pos.column), fields)) ) ); -fn tuple_to_macro(mut t: (Position, Vec, Value)) -> ParseResult { - match t.2 { +// FIXME(jwall): need to make this into a proper parse function. +fn tuple_to_macro(pos: Position, vals: Option>, val: Value) -> ParseResult { + let mut default_args = match vals { + None => Vec::new(), + Some(vals) => vals, + }; + let arglist = default_args + .drain(0..) + .map(|s| Positioned { + pos: s.pos().clone(), + val: s.to_string(), + }).collect(); + match val { Value::Tuple(v) => Ok(Expression::Macro(MacroDef { - argdefs: t - .1 - .drain(0..) - .map(|s| Positioned { - pos: s.pos().clone(), - val: s.to_string(), - }) - .collect(), + argdefs: arglist, fields: v.val, - pos: t.0, + pos: pos, })), - val => Err(error::Error::new( + val => Err(Error::new( format!("Expected Tuple Got {:?}", val), - error::ErrorType::UnexpectedToken, - t.0, + // FIXME(jwall): Should have correct Offset here. + &0, )), } } -named!(arglist, error::Error>, separated_list!(punct!(","), symbol)); - -named!(macro_expression, - map_res!( - do_parse!( - pos: pos >> - word!("macro") >> - punct!("(") >> - arglist: trace_nom!(arglist) >> - punct!(")") >> - punct!("=>") >> - map: trace_nom!(tuple) >> - (pos, arglist, map) - ), - tuple_to_macro - ) +make_fn!( + arglist, Vec>, + separated!(punct!(","), symbol) ); -fn tuple_to_select(t: (Position, Expression, Expression, Value)) -> ParseResult { - match t.3 { +fn macro_expression(input: SliceIter) -> Result, Expression> { + let parsed = do_each!(input, + pos => pos, + _ => word!("macro"), + _ => punct!("("), + arglist => trace_nom!(optional!(arglist)), + _ => punct!(")"), + _ => punct!("=>"), + map => trace_nom!(tuple), + (pos, arglist, map) + ); + match parsed { + Result::Abort(e) => Result::Abort(e), + Result::Fail(e) => Result::Fail(e), + Result::Incomplete(offset) => Result::Incomplete(offset), + Result::Complete(rest, (pos, arglist, map)) => match tuple_to_macro(pos, arglist, map) { + Ok(expr) => Result::Complete(rest, expr), + Err(e) => Result::Fail(Error::caused_by("Invalid Macro syntax", &rest, Box::new(e))), + }, + } +} + +// FIXME(jwall): need to make this into a proper parse function. +fn tuple_to_select( + pos: Position, + e1: Expression, + e2: Expression, + val: Value, +) -> ParseResult { + match val { Value::Tuple(v) => Ok(Expression::Select(SelectDef { - val: Box::new(t.1), - default: Box::new(t.2), + val: Box::new(e1), + default: Box::new(e2), tuple: v.val, - pos: t.0, + pos: pos, })), - val => Err(error::Error::new( - format!("Expected Tuple Got {:?}", val), - error::ErrorType::UnexpectedToken, - t.0, - )), + val => Err(Error::new(format!("Expected Tuple Got {:?}", val), &0)), } } -named!(select_expression, - map_res!( - do_parse!( - start: word!("select") >> - val: terminated!(trace_nom!(expression), punct!(",")) >> - default: terminated!(trace_nom!(expression), punct!(",")) >> - map: trace_nom!(tuple) >> - (start.pos.clone(), val, default, map) - ), - tuple_to_select - ) -); - -fn tuple_to_format(t: (Token, Vec)) -> ParseResult { - Ok(Expression::Format(FormatDef { - template: t.0.fragment.to_string(), - args: t.1, - pos: t.0.pos, - })) +fn select_expression(input: SliceIter) -> Result, Expression> { + let parsed = do_each!(input, + start => word!("select"), + val => do_each!( + expr => trace_nom!(expression), + _ => punct!(","), + (expr) + ), + default => do_each!( + expr => trace_nom!(expression), + _ => punct!(","), + (expr) + ), + map => trace_nom!(tuple), + (start.pos.clone(), val, default, map) + ); + match parsed { + Result::Abort(e) => Result::Abort(e), + Result::Fail(e) => Result::Fail(e), + Result::Incomplete(offset) => Result::Incomplete(offset), + Result::Complete(rest, (pos, val, default, map)) => { + match tuple_to_select(pos, val, default, map) { + Ok(expr) => Result::Complete(rest, expr), + Err(e) => Result::Fail(Error::caused_by( + "Invalid Select Expression", + &rest, + Box::new(e), + )), + } + } + } } -named!(format_expression, - map_res!( - do_parse!( - tmpl: match_type!(STR) >> - punct!("%") >> - punct!("(") >> - args: separated_list!(punct!(","), trace_nom!(expression)) >> - punct!(")") >> - (tmpl, args) - ), - tuple_to_format - ) +fn tuple_to_format(tok: Token, exprs: Vec) -> Expression { + Expression::Format(FormatDef { + template: tok.fragment.to_string(), + args: exprs, + pos: tok.pos, + }) +} + +make_fn!( + format_expression, Expression>, + do_each!( + tmpl => match_type!(STR), + _ => punct!("%"), + _ => punct!("("), + args => separated!(punct!(","), trace_nom!(expression)), + _ => punct!(")"), + (tuple_to_format(tmpl, args)) + ) ); -fn tuple_to_call(t: (Position, Value, Vec)) -> ParseResult { - if let Value::Selector(def) = t.1 { +// FIXME(jwall): Convert this into a custom parser function. +fn tuple_to_call(pos: Position, val: Value, exprs: Vec) -> ParseResult { + if let Value::Selector(def) = val { Ok(Expression::Call(CallDef { macroref: def, - arglist: t.2, - pos: Position::new(t.0.line as usize, t.0.column as usize), + arglist: exprs, + pos: pos, })) } else { - Err(error::Error::new( - format!("Expected Selector Got {:?}", t.0), - error::ErrorType::UnexpectedToken, - Position::new(t.0.line as usize, t.0.column as usize), - )) + // FIXME(jwall): Should get correct offset here. + Err(Error::new(format!("Expected Selector Got {:?}", val), &0)) } } -fn vec_to_selector_value(t: (Position, SelectorList)) -> ParseResult { - Ok(Value::Selector(SelectorDef::new( - t.1, - t.0.line as usize, - t.0.column as usize, - ))) +fn vec_to_selector_value(pos: Position, list: SelectorList) -> Value { + Value::Selector(SelectorDef::new( + list, + pos.line as usize, + pos.column as usize, + )) } -named!(selector_value, - map_res!( - do_parse!( - sl: trace_nom!(selector_list) >> - (sl.head.pos().clone(), sl) - ), - vec_to_selector_value - ) +make_fn!( + selector_value, Value>, + do_each!( + sl => trace_nom!(selector_list), + (vec_to_selector_value(sl.head.pos().clone(), sl)) + ) ); -named!(call_expression, - map_res!( - do_parse!( - macroname: trace_nom!(selector_value) >> - punct!("(") >> - args: separated_list!(punct!(","), trace_nom!(expression)) >> - punct!(")") >> - (macroname.pos().clone(), macroname, args) - ), - tuple_to_call - ) -); +fn call_expression(input: SliceIter) -> Result, Expression> { + let parsed = do_each!(input, + macroname => trace_nom!(selector_value), + _ => punct!("("), + args => separated!(punct!(","), trace_nom!(expression)), + _ => punct!(")"), + (macroname.pos().clone(), macroname, args) + ); + match parsed { + Result::Abort(e) => Result::Abort(e), + Result::Fail(e) => Result::Fail(e), + Result::Incomplete(offset) => Result::Incomplete(offset), + Result::Complete(rest, (pos, name, args)) => match tuple_to_call(pos, name, args) { + Ok(expr) => Result::Complete(rest, expr), + Err(e) => Result::Fail(Error::caused_by("Invalid Call Syntax", &rest, Box::new(e))), + }, + } +} -fn tuple_to_list_op(tpl: (Position, Token, Value, Expression)) -> ParseResult { - let pos = tpl.0; - let t = if &tpl.1.fragment == "map" { - ListOpType::Map - } else if &tpl.1.fragment == "filter" { - ListOpType::Filter - } else { - return Err(error::Error::new( - format!( - "Expected one of 'map' or 'filter' but got '{}'", - tpl.1.fragment - ), - error::ErrorType::UnexpectedToken, - pos, - )); - }; - let macroname = tpl.2; - let list = tpl.3; +fn tuple_to_list_op( + pos: Position, + kind: ListOpType, + macroname: Value, + list: Expression, +) -> ParseResult { if let Value::Selector(mut def) = macroname { // First of all we need to assert that this is a selector of at least // two sections. @@ -680,10 +713,10 @@ fn tuple_to_list_op(tpl: (Position, Token, Value, Expression)) -> ParseResult { @@ -694,10 +727,10 @@ fn tuple_to_list_op(tpl: (Position, Token, Value, Expression)) -> ParseResult ParseResult ParseResult, - map_res!( - do_parse!( - pos: pos >> - optype: alt!(word!("map") | word!("filter")) >> - macroname: trace_nom!(selector_value) >> - list: trace_nom!(non_op_expression) >> - (pos, optype, macroname, list) +// FIXME(jwall): need to make this a custom function to parse it. +make_fn!( + list_op_expression, Expression>, + do_each!( + pos => pos, + optype => either!( + do_each!(_ => word!("map"), (ListOpType::Map)), + do_each!(_ => word!("filter"), (ListOpType::Filter)) ), - tuple_to_list_op + macroname => trace_nom!(selector_value), + list => trace_nom!(non_op_expression), + (tuple_to_list_op(pos, optype, macroname, list).unwrap()) ) ); -fn unprefixed_expression(input: TokenIter) -> NomResult { +fn unprefixed_expression(input: SliceIter) -> NomResult { let _input = input.clone(); - let attempt = alt!(input, - trace_nom!(call_expression) | - trace_nom!(copy_expression) | - trace_nom!(format_expression)); - match attempt { - IResult::Incomplete(i) => IResult::Incomplete(i), - IResult::Done(rest, expr) => IResult::Done(rest, expr), - IResult::Error(_) => trace_nom!(_input, simple_expression), - } + let attempt = either!( + input, + trace_nom!(call_expression), + trace_nom!(copy_expression), + trace_nom!(format_expression) + ); + match attempt { + Result::Incomplete(i) => Result::Incomplete(i), + Result::Complete(rest, expr) => Result::Complete(rest, expr), + Result::Fail(_) => trace_nom!(_input, simple_expression), + Result::Abort(e) => Result::Abort(e), + } } -named!(non_op_expression, +make_fn!( + non_op_expression, Expression>, alt_peek!( - alt!(word!("map") | word!("filter")) => trace_nom!(list_op_expression) | + either!(word!("map"), word!("filter")) => trace_nom!(list_op_expression) | word!("macro") => trace_nom!(macro_expression) | word!("select") => trace_nom!(select_expression) | punct!("(") => trace_nom!(grouped_expression) | trace_nom!(unprefixed_expression)) ); -fn expression(input: TokenIter) -> NomResult { +fn expression(input: SliceIter) -> NomResult { let _input = input.clone(); match trace_nom!(_input, op_expression) { - IResult::Incomplete(i) => IResult::Incomplete(i), - IResult::Error(_) => trace_nom!(input, non_op_expression), - IResult::Done(rest, expr) => IResult::Done(rest, expr), + Result::Incomplete(i) => Result::Incomplete(i), + Result::Fail(_) => trace_nom!(input, non_op_expression), + Result::Abort(e) => Result::Abort(e), + Result::Complete(rest, expr) => Result::Complete(rest, expr), } } -//named!(expression, -// alt_complete!(trace_nom!(op_expression) | trace_nom!(non_op_expression)) -//); - -fn expression_to_statement(v: Expression) -> ParseResult { - Ok(Statement::Expression(v)) -} - -named!(expression_statement, - map_res!( - terminated!(trace_nom!(expression), punct!(";")), - expression_to_statement +make_fn!( + expression_statement, Statement>, + do_each!( + e => do_each!( + expr => trace_nom!(expression), + _ => punct!(";"), + (expr) + ), + (Statement::Expression(e)) ) ); -fn tuple_to_let(t: (Token, Expression)) -> ParseResult { - Ok(Statement::Let(LetDef { - name: t.0, - value: t.1, - })) +fn tuple_to_let(tok: Token, expr: Expression) -> Statement { + Statement::Let(LetDef { + name: tok, + value: expr, + }) } -named!(let_stmt_body, - map_res!( - do_parse!( - name: match_type!(BAREWORD) >> - punct!("=") >> - val: trace_nom!(expression) >> - punct!(";") >> - (name, val)), - tuple_to_let +make_fn!( + let_stmt_body, Statement>, + do_each!( + name => match_type!(BAREWORD), + _ => punct!("="), + val => trace_nom!(expression), + _ => punct!(";"), + (tuple_to_let(name, val)) ) ); -named!(let_statement, - wrap_err!(do_parse!( - word!("let") >> - pos: pos >> - stmt: trace_nom!(let_stmt_body) >> +make_fn!( + let_statement, Statement>, + do_each!( + _ => word!("let"), + stmt => trace_nom!(let_stmt_body), (stmt) - ), "Invalid let statement") -); - -fn tuple_to_import(t: (Token, Token)) -> ParseResult { - Ok(Statement::Import(ImportDef { - path: t.0, - name: t.1, - })) -} - -named!(import_stmt_body, - map_res!( - do_parse!( - path: match_type!(STR) >> - word!("as") >> - name: match_type!(BAREWORD) >> - punct!(";") >> - (path, name)), - tuple_to_import ) ); -named!(import_statement, - wrap_err!(do_parse!( - word!("import") >> +fn tuple_to_import(tok: Token, tok2: Token) -> Statement { + Statement::Import(ImportDef { + path: tok, + name: tok2, + }) +} + +make_fn!( + import_stmt_body, Statement>, + do_each!( + path => match_type!(STR), + _ => word!("as"), + name => match_type!(BAREWORD), + _ => punct!(";"), + (tuple_to_import(path, name)) + ) +); + +make_fn!( + import_statement, Statement>, + do_each!( + _ => word!("import"), // past this point we know this is supposed to be an import statement. - pos: pos >> - stmt: trace_nom!(import_stmt_body) >> + stmt => trace_nom!(import_stmt_body), (stmt) - ), "Invalid import statement") + ) ); -named!(assert_statement, - wrap_err!(do_parse!( - word!("assert") >> - pos: pos >> - tok: match_type!(PIPEQUOTE) >> - punct!(";") >> - (Statement::Assert(tok.clone())) - ), "Invalid assert statement") +make_fn!( + assert_statement, Statement>, + do_each!( + _ => word!("assert"), + tok => match_type!(PIPEQUOTE), + _ => punct!(";"), + (Statement::Assert(tok.clone())) + ) ); -named!(out_statement, - wrap_err!(do_parse!( - word!("out") >> - pos: pos >> - typ: match_type!(BAREWORD) >> - expr: expression >> - punct!(";") >> +make_fn!( + out_statement, Statement>, + do_each!( + _ => word!("out"), + typ => match_type!(BAREWORD), + expr => expression, + _ => punct!(";"), (Statement::Output(typ.clone(), expr.clone())) - ), "Invalid out statement") + ) ); //trace_macros!(true); -fn statement(i: TokenIter) -> nom::IResult { +fn statement(i: SliceIter) -> Result, Statement> { return alt_peek!(i, word!("assert") => trace_nom!(assert_statement) | word!("import") => trace_nom!(import_statement) | @@ -874,48 +909,33 @@ fn statement(i: TokenIter) -> nom::IResult { } //trace_macros!(false); -/// Parses a LocatedSpan into a list of Statements or an error::Error. -pub fn parse(input: LocatedSpan<&str>) -> Result, error::Error> { - match tokenize(input) { +/// Parses a LocatedSpan into a list of Statements or an `abortable_parser::Error`. +pub fn parse(input: StrIter) -> std::result::Result, Error> { + match tokenize(input.clone()) { Ok(tokenized) => { let mut out = Vec::new(); - let mut i_ = TokenIter { - source: tokenized.as_slice(), - }; + let mut i_ = SliceIter::from(&tokenized); loop { let i = i_.clone(); - if i[0].typ == TokenType::END { - break; + if let Some(tok) = i.peek_next() { + if tok.typ == TokenType::END { + break; + } } - match statement(i) { - IResult::Error(nom::ErrorKind::Custom(e)) => { + match statement(i.clone()) { + Result::Abort(e) => { return Err(e); } - IResult::Error(e) => { - return Err(error::Error::new_with_errorkind( - "Statement Parse error", - error::ErrorType::ParseError, - Position { - line: i_[0].pos.line, - column: i_[0].pos.column, - }, - e, - )); + Result::Fail(e) => { + return Err(Error::caused_by("Statement Parse error", &i, Box::new(e))); } - IResult::Incomplete(ei) => { - return Err(error::Error::new( - format!("Unexpected end of parsing input: {:?}", ei), - error::ErrorType::IncompleteParsing, - Position { - line: i_[0].pos.line, - column: i_[0].pos.column, - }, - )); + Result::Incomplete(ei) => { + return Err(Error::new("Unexpected end of parsing input: {:?}", &ei)); } - IResult::Done(rest, stmt) => { + Result::Complete(rest, stmt) => { out.push(stmt); i_ = rest; - if i_.input_len() == 0 { + if eoi(i).is_complete() { break; } } @@ -924,11 +944,7 @@ pub fn parse(input: LocatedSpan<&str>) -> Result, error::Error> { return Ok(out); } Err(e) => { - return Err(error::Error::new_with_cause( - format!("Tokenization Error"), - error::ErrorType::ParseError, - e, - )); + return Err(Error::caused_by("Tokenization Error", &input, Box::new(e))); } } } diff --git a/src/parse/precedence.rs b/src/parse/precedence.rs index 6d06e40..01c4e28 100644 --- a/src/parse/precedence.rs +++ b/src/parse/precedence.rs @@ -14,14 +14,11 @@ //! Bottom up parser for precedence parsing of expressions separated by binary //! operators. -use std; +use abortable_parser::combinators::eoi; +use abortable_parser::{Error, Result, SliceIter}; -use nom::{ErrorKind, IResult, InputIter, InputLength, Slice}; - -use super::{non_op_expression, NomResult, ParseResult}; +use super::{non_op_expression, NomResult}; use ast::*; -use error; -use tokenizer::TokenIter; /// Defines the intermediate stages of our bottom up parser for precedence parsing. #[derive(Debug, PartialEq, Clone)] @@ -31,120 +28,116 @@ pub enum Element { CompareOp(CompareType), } -named!(math_op_type, - alt!( - do_parse!(punct!("+") >> (Element::MathOp(BinaryExprType::Add))) | - do_parse!(punct!("-") >> (Element::MathOp(BinaryExprType::Sub))) | - do_parse!(punct!("*") >> (Element::MathOp(BinaryExprType::Mul))) | - do_parse!(punct!("/") >> (Element::MathOp(BinaryExprType::Div))) +make_fn!( + math_op_type, Element>, + either!( + do_each!( + _ => punct!("+"), + (Element::MathOp(BinaryExprType::Add))), + do_each!( + _ => punct!("-"), + (Element::MathOp(BinaryExprType::Sub))), + do_each!( + _ => punct!("*"), + (Element::MathOp(BinaryExprType::Mul))), + do_each!( + _ => punct!("/"), + (Element::MathOp(BinaryExprType::Div))) ) ); -fn parse_expression(i: OpListIter) -> IResult { - let i_ = i.clone(); - if i_.input_len() == 0 { - return IResult::Error(ErrorKind::Custom(error::Error::new( - format!("Expected Expression found End Of Input"), - error::ErrorType::IncompleteParsing, - // TODO(jwall): This position information is incorrect. - Position { line: 0, column: 0 }, - ))); +fn parse_expression(i: SliceIter) -> Result, Expression> { + let mut i_ = i.clone(); + if eoi(i_.clone()).is_complete() { + return Result::Abort(Error::new("Expected Expression found End Of Input", &i_)); } - let el = &(i_[0]); - if let &Element::Expr(ref expr) = el { - return IResult::Done(i.slice(1..), expr.clone()); + let el = i_.next(); + if let Some(&Element::Expr(ref expr)) = el { + return Result::Complete(i_.clone(), expr.clone()); } - return IResult::Error(ErrorKind::Custom(error::Error::new( + return Result::Fail(Error::new( format!( - "Error while parsing Binary Expression Unexpected Operator {:?}", + "Error while parsing Binary Expression Expected Expression got {:?}", el ), - error::ErrorType::ParseError, - // TODO(jwall): This position information is incorrect. - Position { line: 0, column: 0 }, - ))); + &i_, + )); } -fn parse_sum_operator(i: OpListIter) -> IResult { - let i_ = i.clone(); - if i_.input_len() == 0 { - return IResult::Error(ErrorKind::Custom(error::Error::new( +fn parse_sum_operator(i: SliceIter) -> Result, BinaryExprType> { + let mut i_ = i.clone(); + if eoi(i_.clone()).is_complete() { + return Result::Fail(Error::new( format!("Expected Expression found End Of Input"), - error::ErrorType::IncompleteParsing, - // TODO(jwall): This position information is incorrect. - Position { line: 0, column: 0 }, - ))); + &i_, + )); } - let el = &(i_[0]); - if let &Element::MathOp(ref op) = el { + let el = i_.next(); + if let Some(&Element::MathOp(ref op)) = el { match op { &BinaryExprType::Add => { - return IResult::Done(i.slice(1..), op.clone()); + return Result::Complete(i_.clone(), op.clone()); } &BinaryExprType::Sub => { - return IResult::Done(i.slice(1..), op.clone()); + return Result::Complete(i_.clone(), op.clone()); } _other => { // noop } }; } - return IResult::Error(ErrorKind::Custom(error::Error::new( + return Result::Fail(Error::new( format!( "Error while parsing Binary Expression Unexpected Operator {:?}", el ), - error::ErrorType::ParseError, - // TODO(jwall): This position information is incorrect. - Position { line: 0, column: 0 }, - ))); + &i_, + )); } fn tuple_to_binary_expression( - tpl: (BinaryExprType, Expression, Expression), -) -> ParseResult { - let pos = tpl.1.pos().clone(); - Ok(Expression::Binary(BinaryOpDef { - kind: tpl.0, - left: Box::new(tpl.1), - right: Box::new(tpl.2), + kind: BinaryExprType, + left: Expression, + right: Expression, +) -> Expression { + let pos = left.pos().clone(); + Expression::Binary(BinaryOpDef { + kind: kind, + left: Box::new(left), + right: Box::new(right), pos: pos, - })) + }) } -fn parse_product_operator(i: OpListIter) -> IResult { - let i_ = i.clone(); - if i_.input_len() == 0 { - return IResult::Error(ErrorKind::Custom(error::Error::new( +fn parse_product_operator(i: SliceIter) -> Result, BinaryExprType> { + let mut i_ = i.clone(); + if eoi(i_.clone()).is_complete() { + return Result::Fail(Error::new( format!("Expected Expression found End Of Input"), - error::ErrorType::IncompleteParsing, - // TODO(jwall): This position information is incorrect. - Position { line: 0, column: 0 }, - ))); + &i_, + )); } - let el = &(i_[0]); - if let &Element::MathOp(ref op) = el { + let el = i_.next(); + if let Some(&Element::MathOp(ref op)) = el { match op { &BinaryExprType::Mul => { - return IResult::Done(i.slice(1..), op.clone()); + return Result::Complete(i_.clone(), op.clone()); } &BinaryExprType::Div => { - return IResult::Done(i.slice(1..), op.clone()); + return Result::Complete(i_.clone(), op.clone()); } _other => { // noop } }; } - return IResult::Error(ErrorKind::Custom(error::Error::new( + return Result::Fail(Error::new( format!( "Error while parsing Binary Expression Unexpected Operator {:?}", el ), - error::ErrorType::ParseError, - // TODO(jwall): This position information is incorrect. - Position { line: 0, column: 0 }, - ))); + &i_, + )); } /// do_binary_expr implements precedence based parsing where the more tightly bound @@ -152,15 +145,15 @@ fn parse_product_operator(i: OpListIter) -> IResult { - do_binary_expr!($i, call!($oprule), $lowerrule) + do_binary_expr!($i, run!($oprule), $lowerrule) }; ($i:expr, $oprule:ident, $lowerrule:ident!( $($lowerargs:tt)* )) => { - do_binary_expr!($i, call!($oprule), $lowerrule!($($lowerargs)*)) + do_binary_expr!($i, run!($oprule), $lowerrule!($($lowerargs)*)) }; ($i:expr, $oprule:ident) => { - do_binary_expr!($i, call!($oprule)) + do_binary_expr!($i, run!($oprule)) }; ($i:expr, $oprule:ident!( $($args:tt)* )) => { @@ -168,101 +161,97 @@ macro_rules! do_binary_expr { }; ($i:expr, $oprule:ident!( $($args:tt)* ), $lowerrule:ident) => { - do_binary_expr!($i, $oprule!($($args)*), call!($lowerrule)) + do_binary_expr!($i, $oprule!($($args)*), run!($lowerrule)) }; ($i:expr, $oprule:ident!( $($args:tt)* ), $lowerrule:ident!( $($lowerargs:tt)* )) => { - map_res!($i, - do_parse!( - left: $lowerrule!($($lowerargs)*) >> - typ: $oprule!($($args)*) >> - right: $lowerrule!($($lowerargs)*) >> - (typ, left, right) - ), - tuple_to_binary_expression + do_each!($i, + left => $lowerrule!($($lowerargs)*), + typ => $oprule!($($args)*), + right => $lowerrule!($($lowerargs)*), + (tuple_to_binary_expression(typ, left, right)) ) }; } -named!(sum_expression, +make_fn!( + sum_expression, Expression>, do_binary_expr!( parse_sum_operator, - alt!(trace_nom!(product_expression) | trace_nom!(parse_expression))) + either!(trace_nom!(product_expression), trace_nom!(parse_expression)) + ) ); -named!(product_expression, - do_binary_expr!( - parse_product_operator, - trace_nom!(parse_expression)) +make_fn!( + product_expression, Expression>, + do_binary_expr!(parse_product_operator, trace_nom!(parse_expression)) ); -named!(math_expression, - alt!(trace_nom!(sum_expression) | trace_nom!(product_expression)) +make_fn!( + math_expression, Expression>, + either!(trace_nom!(sum_expression), trace_nom!(product_expression)) ); // TODO(jwall): Change comparison operators to use the do_binary_expr! with precedence? fn tuple_to_compare_expression( - tpl: (CompareType, Expression, Expression), -) -> ParseResult { - let pos = tpl.1.pos().clone(); - Ok(Expression::Compare(ComparisonDef { - kind: tpl.0, - left: Box::new(tpl.1), - right: Box::new(tpl.2), + kind: CompareType, + left: Expression, + right: Expression, +) -> Expression { + let pos = left.pos().clone(); + Expression::Compare(ComparisonDef { + kind: kind, + left: Box::new(left), + right: Box::new(right), pos: pos, - })) + }) } -named!(compare_op_type, - alt!( - do_parse!(punct!("==") >> (Element::CompareOp(CompareType::Equal))) | - do_parse!(punct!("!=") >> (Element::CompareOp(CompareType::NotEqual))) | - do_parse!(punct!("<=") >> (Element::CompareOp(CompareType::LTEqual))) | - do_parse!(punct!(">=") >> (Element::CompareOp(CompareType::GTEqual))) | - do_parse!(punct!("<") >> (Element::CompareOp(CompareType::LT))) | - do_parse!(punct!(">") >> (Element::CompareOp(CompareType::GT))) +make_fn!( + compare_op_type, Element>, + either!( + do_each!(_ => punct!("=="), (Element::CompareOp(CompareType::Equal))), + do_each!(_ => punct!("!="), (Element::CompareOp(CompareType::NotEqual))), + do_each!(_ => punct!("<="), (Element::CompareOp(CompareType::LTEqual))), + do_each!(_ => punct!(">="), (Element::CompareOp(CompareType::GTEqual))), + do_each!(_ => punct!("<"), (Element::CompareOp(CompareType::LT))), + do_each!(_ => punct!(">"), (Element::CompareOp(CompareType::GT))) ) ); -fn parse_compare_operator(i: OpListIter) -> IResult { - let i_ = i.clone(); - if i_.input_len() == 0 { - return IResult::Error(ErrorKind::Custom(error::Error::new( +fn parse_compare_operator(i: SliceIter) -> Result, CompareType> { + let mut i_ = i.clone(); + if eoi(i_.clone()).is_complete() { + return Result::Fail(Error::new( format!("Expected Expression found End Of Input"), - error::ErrorType::IncompleteParsing, - // TODO(jwall): This position information is incorrect. - Position { line: 0, column: 0 }, - ))); + &i_, + )); } - let el = &(i_[0]); - if let &Element::CompareOp(ref op) = el { - return IResult::Done(i.slice(1..), op.clone()); + let el = i_.next(); + if let Some(&Element::CompareOp(ref op)) = el { + return Result::Complete(i_.clone(), op.clone()); } - return IResult::Error(ErrorKind::Custom(error::Error::new( + return Result::Fail(Error::new( format!( "Error while parsing Binary Expression Unexpected Operator {:?}", el ), - error::ErrorType::ParseError, - // TODO(jwall): This position information is incorrect. - Position { line: 0, column: 0 }, - ))); + &i, + )); } -named!(compare_expression, - map_res!( - do_parse!( - left: alt!(trace_nom!(math_expression) | trace_nom!(parse_expression)) >> - typ: parse_compare_operator >> - right: alt!(trace_nom!(math_expression) | trace_nom!(parse_expression)) >> - (typ, left, right) - ), - tuple_to_compare_expression +make_fn!( + compare_expression, Expression>, + do_each!( + left => either!(trace_nom!(math_expression), trace_nom!(parse_expression)), + typ => parse_compare_operator, + right => either!(trace_nom!(math_expression), trace_nom!(parse_expression)), + (tuple_to_compare_expression(typ, left, right)) ) ); /// Parse a list of expressions separated by operators into a Vec. -fn parse_operand_list(i: TokenIter) -> NomResult> { +fn parse_operand_list<'a>(i: SliceIter<'a, Token>) -> NomResult<'a, Vec> { // 1. First try to parse a non_op_expression, let mut _i = i.clone(); let mut list = Vec::new(); @@ -271,148 +260,74 @@ fn parse_operand_list(i: TokenIter) -> NomResult> { loop { // 2. Parse a non_op_expression. match non_op_expression(_i.clone()) { - IResult::Error(e) => { + Result::Fail(e) => { // A failure to parse an expression // is always an error. - return IResult::Error(e); + return Result::Fail(e); } - IResult::Incomplete(i) => { - return IResult::Incomplete(i); + Result::Abort(e) => { + // A failure to parse an expression + // is always an error. + return Result::Abort(e); } - IResult::Done(rest, expr) => { + Result::Incomplete(i) => { + return Result::Incomplete(i); + } + Result::Complete(rest, expr) => { list.push(Element::Expr(expr)); _i = rest.clone(); } } // 3. Parse an operator. - match alt!(_i, math_op_type | compare_op_type) { - IResult::Error(e) => { + match either!(_i.clone(), math_op_type, compare_op_type) { + Result::Fail(e) => { if firstrun { // If we don't find an operator in our first // run then this is not an operand list. - return IResult::Error(e); + return Result::Fail(e); } // if we don't find one on subsequent runs then // that's the end of the operand list. break; } - IResult::Incomplete(i) => { - return IResult::Incomplete(i); + Result::Abort(e) => { + // A failure to parse an expression + // is always an error. + return Result::Abort(e); } - IResult::Done(rest, el) => { + Result::Incomplete(i) => { + return Result::Incomplete(i); + } + Result::Complete(rest, el) => { list.push(el); _i = rest.clone(); } } firstrun = false; } - return IResult::Done(_i, list); -} - -#[derive(Clone, Debug, PartialEq)] -pub struct OpListIter<'a> { - pub source: &'a [Element], -} - -impl<'a> OpListIter<'a> { - pub fn len(&self) -> usize { - self.source.len() - } -} - -impl<'a> InputLength for OpListIter<'a> { - fn input_len(&self) -> usize { - self.source.input_len() - } -} - -macro_rules! impl_op_iter_slice { - ($r:ty) => { - impl<'a> Slice<$r> for OpListIter<'a> { - fn slice(&self, range: $r) -> Self { - OpListIter { - source: self.source.slice(range), - } - } - } - }; -} - -impl_op_iter_slice!(std::ops::Range); -impl_op_iter_slice!(std::ops::RangeTo); -impl_op_iter_slice!(std::ops::RangeFrom); -impl_op_iter_slice!(std::ops::RangeFull); - -impl<'a> std::ops::Index for OpListIter<'a> { - type Output = Element; - - fn index(&self, i: usize) -> &Self::Output { - &self.source[i] - } -} - -impl<'a> InputIter for OpListIter<'a> { - type Item = &'a Element; - type RawItem = Element; - - type Iter = std::iter::Enumerate>; - type IterElem = std::slice::Iter<'a, Self::RawItem>; - - fn iter_indices(&self) -> Self::Iter { - self.source.iter().enumerate() - } - - fn iter_elements(&self) -> Self::IterElem { - self.source.iter() - } - - fn position

(&self, predicate: P) -> Option - where - P: Fn(Self::RawItem) -> bool, - { - for (o, v) in self.iter_indices() { - if predicate(v.clone()) { - return Some(o); - } - } - None - } - - fn slice_index(&self, count: usize) -> Option { - let mut cnt = 0; - for (index, _) in self.iter_indices() { - if cnt == count { - return Some(index); - } - cnt += 1; - } - if cnt == count { - return Some(self.len()); - } - None - } + return Result::Complete(_i, list); } /// Parse a binary operator expression. -pub fn op_expression(i: TokenIter) -> NomResult { +pub fn op_expression<'a>(i: SliceIter<'a, Token>) -> NomResult<'a, Expression> { let preparse = parse_operand_list(i.clone()); match preparse { - IResult::Error(e) => IResult::Error(e), - IResult::Incomplete(i) => IResult::Incomplete(i), - IResult::Done(rest, oplist) => { - let mut i_ = OpListIter { - source: oplist.as_slice(), - }; - - let parse_result = alt!( + Result::Fail(e) => Result::Fail(e), + Result::Abort(e) => Result::Abort(e), + Result::Incomplete(i) => Result::Incomplete(i), + Result::Complete(rest, oplist) => { + let mut i_ = SliceIter::new(&oplist); + let parse_result = either!( i_, - trace_nom!(compare_expression) | trace_nom!(math_expression) + trace_nom!(compare_expression), + trace_nom!(math_expression) ); match parse_result { - IResult::Error(e) => IResult::Error(e), - IResult::Incomplete(i) => IResult::Incomplete(i), - IResult::Done(_, expr) => IResult::Done(rest.clone(), expr), + Result::Fail(e) => Result::Fail(e), + Result::Abort(e) => Result::Abort(e), + Result::Incomplete(i) => Result::Incomplete(i), + Result::Complete(_, expr) => Result::Complete(rest.clone(), expr), } } } diff --git a/src/parse/test.rs b/src/parse/test.rs index b1520eb..cbf99e9 100644 --- a/src/parse/test.rs +++ b/src/parse/test.rs @@ -12,23 +12,20 @@ // See the License for the specific language governing permissions and // limitations under the License. use super::*; -use tokenizer::{tokenize, TokenIter}; +use tokenizer::tokenize; -use nom::IResult; -use nom_locate::LocatedSpan; +use abortable_parser::{Result, SliceIter, StrIter}; macro_rules! assert_parse { ($parsemac:ident($i:expr), $out:expr) => { assert_parse!($i, $parsemac, $out) }; ($i:expr, $f:expr, $out:expr) => {{ - let input = LocatedSpan::new($i); + let input = StrIter::new($i); match tokenize(input) { Err(e) => assert!(false, format!("Tokenizer Error: {:?}", e)), - Ok(val) => match $f(TokenIter { - source: val.as_slice(), - }) { - IResult::Done(_, result) => assert_eq!(result, $out), + Ok(val) => match $f(SliceIter::new(val.as_slice())) { + Result::Complete(_, result) => assert_eq!(result, $out), other => assert!(false, format!("Expected Done got {:?}", other)), }, } @@ -40,14 +37,12 @@ macro_rules! assert_error { assert_error!($i, $parsemac) }; ($i:expr, $f:expr) => {{ - let input = LocatedSpan::new($i); + let input = StrIter::new($i); match tokenize(input) { Err(_) => assert!(true), Ok(val) => { - let result = $f(TokenIter { - source: val.as_slice(), - }); - assert!(result.is_err(), format!("Not an error: {:?}", result)) + let result = $f(SliceIter::new(val.as_slice())); + assert!(result.is_fail(), format!("Not a fail: {:?}", result)) } } }}; @@ -757,6 +752,24 @@ fn test_macro_expression_parsing() { assert_error!(macro_expression("macro (arg1, arg2) => { foo")); assert_error!(macro_expression("macro (arg1, arg2) => { foo =")); + assert_parse!( + macro_expression("macro () => {foo=1,bar=2}"), + Expression::Macro(MacroDef { + argdefs: Vec::new(), + fields: vec![ + ( + make_tok!("foo", 1, 14), + Expression::Simple(Value::Int(value_node!(1, 1, 18))), + ), + ( + make_tok!("bar", 1, 20), + Expression::Simple(Value::Int(value_node!(2, 1, 24))), + ), + ], + pos: Position::new(1, 1), + }) + ); + assert_parse!( macro_expression("macro (arg1, arg2) => {foo=1,bar=2}"), Expression::Macro(MacroDef { @@ -1223,12 +1236,12 @@ fn test_number_parsing() { #[test] fn test_parse() { - let bad_input = LocatedSpan::new("import mylib as lib;"); + let bad_input = StrIter::new("import mylib as lib;"); let bad_result = parse(bad_input); assert!(bad_result.is_err()); // Valid parsing tree - let input = LocatedSpan::new("import \"mylib\" as lib;let foo = 1;1+1;"); + let input = StrIter::new("import \"mylib\" as lib;let foo = 1;1+1;"); let result = parse(input); assert!(result.is_ok(), format!("Expected Ok, Got {:?}", result)); let tpl = result.unwrap(); diff --git a/src/tokenizer/mod.rs b/src/tokenizer/mod.rs index 9ee360a..e23f95e 100644 --- a/src/tokenizer/mod.rs +++ b/src/tokenizer/mod.rs @@ -12,333 +12,324 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! The tokenization stage of the ucg compiler. -use ast::*; -use error; -use nom; -use nom::{alpha, digit, is_alphanumeric, multispace}; -use nom::{InputIter, InputLength, Slice}; -use nom_locate::LocatedSpan; +//! The tokenization stext_tokene of the ucg compiler. use std; -use std::result::Result; -pub type Span<'a> = LocatedSpan<&'a str>; +use abortable_parser::combinators::*; +use abortable_parser::iter::{SliceIter, StrIter}; +use abortable_parser::{Error, Offsetable, Result, TextPositionTracker}; +use ast::*; -impl<'a> From> for Position { - fn from(s: Span) -> Position { +impl<'a> From> for Position { + fn from(s: StrIter<'a>) -> Position { Position { - line: s.line as usize, - column: s.get_column() as usize, + line: s.line(), + column: s.column(), } } } -fn is_symbol_char(c: char) -> bool { - is_alphanumeric(c as u8) || c == '-' as char || c == '_' as char +fn is_symbol_char<'a>(i: StrIter<'a>) -> Result, u8> { + let mut _i = i.clone(); + let c = match _i.next() { + Some(c) => *c, + None => return Result::Fail(Error::new("Unexpected End of Input".to_string(), &_i)), + }; + if (c as char).is_ascii_alphanumeric() || c == b'-' || c == b'_' { + Result::Complete(_i, c) + } else { + Result::Fail(Error::new("Not a symbol character".to_string(), &_i)) + } } -fn escapequoted(input: Span) -> nom::IResult { +fn escapequoted<'a>(input: StrIter<'a>) -> Result, String> { // loop until we find a " that is not preceded by \. // Collapse all \ to just char for escaping. let mut frag = String::new(); let mut escape = false; - for (i, c) in input.iter_indices() { - if c == '\\' && !escape { + let mut _input = input.clone(); + loop { + let c = match _input.next() { + Some(c) => *c, + None => break, + }; + if c == '\\' as u8 && !escape { // eat this slash and set our escaping sentinel escape = true; - } else if c == '"' && !escape { + } else if c == '"' as u8 && !escape { // Bail if this is an unescaped " // we exit here. - return nom::IResult::Done(input.slice(i..), frag); + return Result::Complete(_input, frag); } else { // we accumulate this character. - frag.push(c); + frag.push(c as char); escape = false; // reset our escaping sentinel } } - return nom::IResult::Incomplete(nom::Needed::Unknown); + return Result::Incomplete(_input.get_offset()); } -named!(strtok( Span ) -> Token, - do_parse!( - span: position!() >> - tag!("\"") >> - frag: escapequoted >> - tag!("\"") >> +make_fn!(strtok, + do_each!( + span => input!(), + _ => text_token!("\""), + frag => escapequoted, (Token{ typ: TokenType::QUOTED, pos: Position::from(span), - fragment: frag, + fragment: frag.to_string(), }) ) ); -named!(pipequotetok( Span ) -> Token, - do_parse!( - span: position!() >> - tag!("|") >> - frag: take_until!("|") >> - tag!("|") >> +make_fn!(pipequotetok, + do_each!( + p => input!(), + _ => text_token!("|"), + frag => until!(text_token!("|")), + _ => text_token!("|"), (Token{ typ: TokenType::PIPEQUOTE, - pos: Position::from(span), - fragment: frag.fragment.to_string(), + pos: Position::from(p), + fragment: frag.to_string(), }) ) ); -named!(barewordtok( Span ) -> Token, - do_parse!( - span: position!() >> - frag: preceded!(peek!(alpha), take_while!(is_symbol_char)) >> +make_fn!(barewordtok, + do_each!( + span => input!(), + _ => peek!(ascii_alpha), + frag => consume_all!(is_symbol_char), (Token{ typ: TokenType::BAREWORD, pos: Position::from(span), - fragment: frag.fragment.to_string(), + fragment: frag.to_string(), }) ) ); -named!(digittok( Span ) -> Token, - do_parse!( - span: position!() >> - digits: digit >> - (Token{ - typ: TokenType::DIGIT, - pos: Position::from(span), - fragment: digits.fragment.to_string(), - }) +make_fn!(digittok, + do_each!( + span => input!(), + _ => peek!(ascii_digit), + digits => consume_all!(ascii_digit), + (Token{ + typ: TokenType::DIGIT, + pos: Position::from(span), + fragment: digits.to_string(), + }) ) ); -named!(booleantok( Span ) -> Token, - do_parse!( - span: position!() >> - b: alt!( - tag!("true") | - tag!("false") - ) >> +make_fn!(booleantok, + do_each!( + span => input!(), + token => either!( + text_token!("true"), + text_token!("false") + ), (Token{ typ: TokenType::BOOLEAN, pos: Position::from(span), - fragment: b.fragment.to_string(), + fragment: token.to_string(), }) ) ); -/// do_tag_tok! is a helper macro to make building a simple tag token +/// do_text_token_tok! is a helper macro to make building a simple text_token token /// less code. -macro_rules! do_tag_tok { - // NOTE(jwall): Nom macros do magic with their inputs. They in fact - // rewrite your macro argumets for you by adding an initial argument - // for all their sub-macros. Which means we require this $i paramater - // on the first macro invocation but not the rest. - ($i:expr, $type:expr, $tag:expr,WS) => { - do_parse!( - $i, - span: position!() >> frag: tag!($tag) >> alt!(whitespace | comment) >> (Token { - typ: $type, - pos: Position::from(span), - fragment: frag.fragment.to_string(), - }) - ) +macro_rules! do_text_token_tok { + ($i:expr, $type:expr, $text_token:expr, WS) => { + do_each!($i, + span => input!(), + frag => text_token!($text_token), + _ => either!(whitespace, comment), + (Token { + typ: $type, + pos: Position::from(span), + fragment: frag.to_string(), + }) + ) }; - ($i:expr, $type:expr, $tag:expr) => { - do_parse!( - $i, - span: position!() >> frag: tag!($tag) >> (Token { - typ: $type, - pos: Position::from(span), - fragment: frag.fragment.to_string(), - }) - ) + + ($i:expr, $type:expr, $text_token:expr) => { + do_each!($i, + span => input!(), + frag => text_token!($text_token), + (Token { + typ: $type, + pos: Position::from(span), + fragment: frag.to_string(), + }) + ) }; } -named!(emptytok( Span ) -> Token, - do_tag_tok!(TokenType::EMPTY, "NULL") +make_fn!(emptytok, + do_text_token_tok!(TokenType::EMPTY, "NULL") ); -named!(commatok( Span ) -> Token, - do_tag_tok!(TokenType::PUNCT, ",") +make_fn!(commatok, + do_text_token_tok!(TokenType::PUNCT, ",") ); -named!(lbracetok( Span ) -> Token, - do_tag_tok!(TokenType::PUNCT, "{") +make_fn!(lbracetok, + do_text_token_tok!(TokenType::PUNCT, "{") ); -named!(rbracetok( Span ) -> Token, - do_tag_tok!(TokenType::PUNCT, "}") +make_fn!(rbracetok, + do_text_token_tok!(TokenType::PUNCT, "}") ); -named!(lparentok( Span ) -> Token, - do_tag_tok!(TokenType::PUNCT, "(") +make_fn!(lparentok, + do_text_token_tok!(TokenType::PUNCT, "(") ); -named!(rparentok( Span ) -> Token, - do_tag_tok!(TokenType::PUNCT, ")") +make_fn!(rparentok, + do_text_token_tok!(TokenType::PUNCT, ")") ); -named!(dottok( Span ) -> Token, - do_tag_tok!(TokenType::PUNCT, ".") +make_fn!(dottok, + do_text_token_tok!(TokenType::PUNCT, ".") ); -named!(plustok( Span ) -> Token, - do_tag_tok!(TokenType::PUNCT, "+") +make_fn!(plustok, + do_text_token_tok!(TokenType::PUNCT, "+") ); -named!(dashtok( Span ) -> Token, - do_tag_tok!(TokenType::PUNCT, "-") +make_fn!(dashtok, + do_text_token_tok!(TokenType::PUNCT, "-") ); -named!(startok( Span ) -> Token, - do_tag_tok!(TokenType::PUNCT, "*") +make_fn!(startok, + do_text_token_tok!(TokenType::PUNCT, "*") ); -named!(slashtok( Span ) -> Token, - do_tag_tok!(TokenType::PUNCT, "/") +make_fn!(slashtok, + do_text_token_tok!(TokenType::PUNCT, "/") ); -named!(pcttok( Span ) -> Token, - do_tag_tok!(TokenType::PUNCT, "%") +make_fn!(pcttok, + do_text_token_tok!(TokenType::PUNCT, "%") ); -named!(eqeqtok( Span ) -> Token, - do_tag_tok!(TokenType::PUNCT, "==") +make_fn!(eqeqtok, + do_text_token_tok!(TokenType::PUNCT, "==") ); -named!(notequaltok( Span ) -> Token, - do_tag_tok!(TokenType::PUNCT, "!=") +make_fn!(notequaltok, + do_text_token_tok!(TokenType::PUNCT, "!=") ); -named!(gttok( Span ) -> Token, - do_tag_tok!(TokenType::PUNCT, ">") +make_fn!(gttok, + do_text_token_tok!(TokenType::PUNCT, ">") ); -named!(gtequaltok( Span ) -> Token, - do_tag_tok!(TokenType::PUNCT, ">=") +make_fn!(gtequaltok, + do_text_token_tok!(TokenType::PUNCT, ">=") ); -named!(ltequaltok( Span ) -> Token, - do_tag_tok!(TokenType::PUNCT, "<=") +make_fn!(ltequaltok, + do_text_token_tok!(TokenType::PUNCT, "<=") ); -named!(lttok( Span ) -> Token, - do_tag_tok!(TokenType::PUNCT, "<") +make_fn!(lttok, + do_text_token_tok!(TokenType::PUNCT, "<") ); -named!(equaltok( Span ) -> Token, - do_tag_tok!(TokenType::PUNCT, "=") +make_fn!(equaltok, + do_text_token_tok!(TokenType::PUNCT, "=") ); -named!(semicolontok( Span ) -> Token, - do_tag_tok!(TokenType::PUNCT, ";") +make_fn!(semicolontok, + do_text_token_tok!(TokenType::PUNCT, ";") ); -named!(leftsquarebracket( Span ) -> Token, - do_tag_tok!(TokenType::PUNCT, "[") +make_fn!(leftsquarebracket, + do_text_token_tok!(TokenType::PUNCT, "[") ); -named!(rightsquarebracket( Span ) -> Token, - do_tag_tok!(TokenType::PUNCT, "]") +make_fn!(rightsquarebracket, + do_text_token_tok!(TokenType::PUNCT, "]") ); -named!(fatcommatok( Span ) -> Token, - do_tag_tok!(TokenType::PUNCT, "=>") +make_fn!(fatcommatok, + do_text_token_tok!(TokenType::PUNCT, "=>") ); -named!(selecttok( Span ) -> Token, - do_tag_tok!(TokenType::BAREWORD, "select", WS) +make_fn!(selecttok, + do_text_token_tok!(TokenType::BAREWORD, "select", WS) ); -named!(macrotok( Span ) -> Token, - do_tag_tok!(TokenType::BAREWORD, "macro", WS) +make_fn!(macrotok, + do_text_token_tok!(TokenType::BAREWORD, "macro", WS) ); -named!(lettok( Span ) -> Token, - do_tag_tok!(TokenType::BAREWORD, "let", WS) +make_fn!(lettok, + do_text_token_tok!(TokenType::BAREWORD, "let", WS) ); -named!(importtok( Span ) -> Token, - do_tag_tok!(TokenType::BAREWORD, "import", WS) +make_fn!(importtok, + do_text_token_tok!(TokenType::BAREWORD, "import", WS) ); -named!(asserttok( Span ) -> Token, - do_tag_tok!(TokenType::BAREWORD, "assert", WS) +make_fn!(asserttok, + do_text_token_tok!(TokenType::BAREWORD, "assert", WS) ); -named!(outtok( Span ) -> Token, - do_tag_tok!(TokenType::BAREWORD, "out", WS) +make_fn!(outtok, + do_text_token_tok!(TokenType::BAREWORD, "out", WS) ); -named!(astok( Span ) -> Token, - do_tag_tok!(TokenType::BAREWORD, "as", WS) +make_fn!(astok, + do_text_token_tok!(TokenType::BAREWORD, "as", WS) ); -named!(maptok( Span ) -> Token, - do_tag_tok!(TokenType::BAREWORD, "map", WS) +make_fn!(maptok, + do_text_token_tok!(TokenType::BAREWORD, "map", WS) ); -named!(filtertok( Span ) -> Token, - do_tag_tok!(TokenType::BAREWORD, "filter", WS) +make_fn!(filtertok, + do_text_token_tok!(TokenType::BAREWORD, "filter", WS) ); -fn end_of_input(input: Span) -> nom::IResult { - match eof!(input,) { - nom::IResult::Done(_, _) => { - return nom::IResult::Done( - input, - make_tok!(EOF => input.line as usize, - input.get_column() as usize), - ); - } - nom::IResult::Incomplete(_) => { - return nom::IResult::Incomplete(nom::Needed::Unknown); - } - nom::IResult::Error(e) => { - return nom::IResult::Error(e); - } - } -} - -fn comment(input: Span) -> nom::IResult { - match tag!(input, "//") { - nom::IResult::Done(rest, _) => { - match alt!( +fn comment(input: StrIter) -> Result { + match text_token!(input, "//") { + Result::Complete(rest, _) => { + match until!( rest, - take_until_and_consume!("\r\n") | take_until_and_consume!("\n") + either!( + eoi, + discard!(text_token!("\r\n")), + discard!(text_token!("\n")) + ) ) { - nom::IResult::Done(rest, cmt) => { - return nom::IResult::Done( + Result::Complete(rest, cmt) => { + return Result::Complete( rest, - make_tok!(CMT => cmt.fragment.to_string(), - input.line as usize, - input.get_column() as usize), + make_tok!(CMT => cmt.to_string(), + input.line() as usize, + input.column() as usize), ); } // If we didn't find a new line then we just grab everything. _ => { - let blen = rest.input_len(); - let next = rest.slice(blen..); - let tok = rest.slice(..blen); - return nom::IResult::Done( - next, - make_tok!(CMT => tok.fragment.to_string(), - input.line as usize, input.get_column() as usize - ), - ); + return Result::Abort(Error::new("Unparsable comment".to_string(), &rest)); } } } - nom::IResult::Incomplete(i) => return nom::IResult::Incomplete(i), - nom::IResult::Error(e) => return nom::IResult::Error(e), + Result::Incomplete(offset) => return Result::Incomplete(offset), + Result::Fail(e) => return Result::Fail(e), + Result::Abort(e) => return Result::Abort(e), } } -named!(whitespace( Span ) -> Token, - do_parse!( - span: position!() >> - many1!(multispace) >> +make_fn!(whitespace, + do_each!( + span => input!(), + _ => peek!(ascii_ws), + _ => repeat!(ascii_ws), (Token{ typ: TokenType::WS, pos: Position::from(span), @@ -347,80 +338,89 @@ named!(whitespace( Span ) -> Token, ) ); -named!(token( Span ) -> Token, - alt!( - strtok | - pipequotetok | - emptytok | // This must come before the barewordtok - digittok | - commatok | - rbracetok | - lbracetok | - lparentok | - rparentok | - dottok | - plustok | - dashtok | - startok | - comment | // Note comment must come before slashtok - slashtok | - pcttok | - eqeqtok | - notequaltok | - complete!(gtequaltok) | - complete!(ltequaltok) | - gttok | - lttok | - fatcommatok | // Note fatcommatok must come before equaltok - equaltok | - semicolontok | - leftsquarebracket | - rightsquarebracket | - booleantok | - lettok | - outtok | - selecttok | - asserttok | - macrotok | - importtok | - astok | - maptok | - filtertok | - barewordtok | - whitespace | +make_fn!(end_of_input, + do_each!( + span => input!(), + _ => eoi, + (Token{ + typ: TokenType::END, + pos: Position::from(span), + fragment: String::new(), + }) + ) +); + +make_fn!(token, + either!( + strtok, + pipequotetok, + emptytok, // This must come before the barewordtok + digittok, + commatok, + rbracetok, + lbracetok, + lparentok, + rparentok, + dottok, + plustok, + dashtok, + startok, + comment, // Note comment must come before slashtok + slashtok, + pcttok, + eqeqtok, + notequaltok, + complete!("Not >=".to_string(), gtequaltok), + complete!("Not <=".to_string(), ltequaltok), + gttok, + lttok, + fatcommatok, // Note fatcommatok must come before equaltok + equaltok, + semicolontok, + leftsquarebracket, + rightsquarebracket, + booleantok, + lettok, + outtok, + selecttok, + asserttok, + macrotok, + importtok, + astok, + maptok, + filtertok, + barewordtok, + whitespace, end_of_input) ); -/// Consumes an input Span and returns either a Vec or a nom::ErrorKind. -pub fn tokenize(input: Span) -> Result, error::Error> { +/// Consumes an input StrIter and returns either a Vec or a error::Error. +pub fn tokenize(input: StrIter) -> std::result::Result, Error> { let mut out = Vec::new(); - let mut i = input; + let mut i = input.clone(); loop { - if i.input_len() == 0 { + if let Result::Complete(_, _) = eoi(i.clone()) { break; } - match token(i) { - nom::IResult::Error(_e) => { - return Err(error::Error::new( + match token(i.clone()) { + Result::Abort(e) => { + return Err(Error::caused_by( "Invalid Token encountered", - error::ErrorType::UnexpectedToken, - Position { - line: i.line as usize, - column: i.get_column() as usize, - }, + &i, + Box::new(e), )); } - nom::IResult::Incomplete(_) => { - return Err(error::Error::new( - "Unexepcted end of Input", - error::ErrorType::UnexpectedToken, - Position { - line: i.line as usize, - column: i.get_column() as usize, - }, + Result::Fail(e) => { + return Err(Error::caused_by( + "Invalid Token encountered", + &i, + Box::new(e), )); } - nom::IResult::Done(rest, tok) => { + Result::Incomplete(offset) => { + return Err(Error::new("Unexepcted end of Input", &offset)); + } + Result::Complete(rest, tok) => { i = rest; if tok.typ == TokenType::COMMENT || tok.typ == TokenType::WS { // we skip comments and whitespace @@ -435,8 +435,8 @@ pub fn tokenize(input: Span) -> Result, error::Error> { fragment: String::new(), typ: TokenType::END, pos: Position { - line: i.line as usize, - column: i.get_column() as usize, + line: i.line(), + column: i.column(), }, }); Ok(out) @@ -445,7 +445,7 @@ pub fn tokenize(input: Span) -> Result, error::Error> { /// Clones a token. /// /// This is necessary to allow the match_type and match_token macros to work. -pub fn token_clone(t: &Token) -> Result { +pub fn token_clone(t: &Token) -> std::result::Result { Ok(t.clone()) } @@ -517,28 +517,28 @@ macro_rules! match_type { }; ($i:expr, $t:expr, $msg:expr, $h:expr) => {{ - let i_ = $i.clone(); - use nom::Slice; - use std::convert::Into; - if i_.input_len() == 0 { - nom::IResult::Error(nom::ErrorKind::Custom(error::Error::new( - format!("End of Input! {}", $msg), - error::ErrorType::IncompleteParsing, - Position { line: 0, column: 0 }, - ))) + use abortable_parser::combinators::eoi; + use abortable_parser::{Error, Result}; + use std; + + let mut _i = $i.clone(); + if eoi(_i.clone()).is_complete() { + Result::Fail(Error::new(format!("End of Input! {}", $msg), &$i)) } else { - let tok = &(i_[0]); - if tok.typ == $t { - match $h(tok) { - Result::Ok(v) => nom::IResult::Done($i.slice(1..), v), - Result::Err(e) => nom::IResult::Error(nom::ErrorKind::Custom(e.into())), + match _i.next() { + Some(tok) => { + if tok.typ == $t { + match $h(tok) { + std::result::Result::Ok(v) => Result::Complete(_i.clone(), v), + std::result::Result::Err(e) => { + Result::Fail(Error::caused_by($msg, &_i, Box::new(e))) + } + } + } else { + Result::Fail(Error::new($msg.to_string(), &$i)) + } } - } else { - nom::IResult::Error(nom::ErrorKind::Custom(error::Error::new( - $msg.to_string(), - error::ErrorType::UnexpectedToken, - tok.pos.clone(), - ))) + None => Result::Fail(Error::new($msg.to_string(), &$i)), } } }}; @@ -553,7 +553,7 @@ macro_rules! match_token { }}; ($i:expr,PUNCT => $f:expr, $h:expr) => { - match_token!($i, TokenType::PUNCT, $f, format!("Not PUNCT ({})", $f), $h) + match_token!($i, TokenType::PUNCT, $f, format!("({})", $f), $h) }; ($i:expr,BAREWORD => $f:expr) => {{ @@ -572,22 +572,26 @@ macro_rules! match_token { }; ($i:expr, $t:expr, $f:expr, $msg:expr, $h:expr) => {{ - let i_ = $i.clone(); - use nom; - use nom::Slice; - use std::convert::Into; - let tok = &(i_[0]); - if tok.typ == $t && &tok.fragment == $f { - match $h(tok) { - Result::Ok(v) => nom::IResult::Done($i.slice(1..), v), - Result::Err(e) => nom::IResult::Error(nom::ErrorKind::Custom(e.into())), + use abortable_parser::Result; + use std; + let mut i_ = $i.clone(); + let tok = i_.next(); + if let Some(tok) = tok { + if tok.typ == $t && &tok.fragment == $f { + match $h(tok) { + std::result::Result::Ok(v) => Result::Complete(i_.clone(), v), + std::result::Result::Err(e) => { + Result::Fail(Error::caused_by($msg, &i_, Box::new(e))) + } + } + } else { + Result::Fail(Error::new( + format!("Expected {} Instead is ({})", $msg, tok.fragment), + &i_, + )) } } else { - nom::IResult::Error(nom::ErrorKind::Custom(error::Error::new( - format!("{} Instead is ({})", $msg, tok.fragment), - error::ErrorType::UnexpectedToken, - tok.pos.clone(), - ))) + Result::Fail(Error::new("Unexpected End Of Input", &i_)) } }}; } @@ -607,11 +611,12 @@ macro_rules! word { } /// pos gets the current position from a TokenIter input without consuming it. -pub fn pos(i: TokenIter) -> nom::IResult { - let tok = &i[0]; +pub fn pos<'a>(i: SliceIter<'a, Token>) -> Result, Position> { + let mut _i = i.clone(); + let tok = _i.next().unwrap(); let line = tok.pos.line; let column = tok.pos.column; - nom::IResult::Done( + Result::Complete( i.clone(), Position { line: line, @@ -620,91 +625,5 @@ pub fn pos(i: TokenIter) -> nom::IResult { ) } -/// TokenIter wraps a slice of Tokens and implements the various necessary -/// nom traits to use it as an input to nom parsers. -#[derive(Clone, Debug, PartialEq)] -pub struct TokenIter<'a> { - pub source: &'a [Token], -} - -impl<'a> TokenIter<'a> { - pub fn len(&self) -> usize { - self.source.len() - } -} - -impl<'a> nom::InputLength for TokenIter<'a> { - fn input_len(&self) -> usize { - self.source.input_len() - } -} - -macro_rules! impl_token_iter_slice { - ($r:ty) => { - impl<'a> nom::Slice<$r> for TokenIter<'a> { - fn slice(&self, range: $r) -> Self { - TokenIter { - source: self.source.slice(range), - } - } - } - }; -} - -impl_token_iter_slice!(std::ops::Range); -impl_token_iter_slice!(std::ops::RangeTo); -impl_token_iter_slice!(std::ops::RangeFrom); -impl_token_iter_slice!(std::ops::RangeFull); - -impl<'a> std::ops::Index for TokenIter<'a> { - type Output = Token; - - fn index(&self, i: usize) -> &Self::Output { - &self.source[i] - } -} - -impl<'a> InputIter for TokenIter<'a> { - type Item = &'a Token; - type RawItem = Token; - - type Iter = std::iter::Enumerate>; - type IterElem = std::slice::Iter<'a, Self::RawItem>; - - fn iter_indices(&self) -> Self::Iter { - self.source.iter().enumerate() - } - - fn iter_elements(&self) -> Self::IterElem { - self.source.iter() - } - - fn position

(&self, predicate: P) -> Option - where - P: Fn(Self::RawItem) -> bool, - { - for (o, v) in self.iter_indices() { - if predicate(v.clone()) { - return Some(o); - } - } - None - } - - fn slice_index(&self, count: usize) -> Option { - let mut cnt = 0; - for (index, _) in self.iter_indices() { - if cnt == count { - return Some(index); - } - cnt += 1; - } - if cnt == count { - return Some(self.len()); - } - None - } -} - #[cfg(test)] mod test; diff --git a/src/tokenizer/test.rs b/src/tokenizer/test.rs index 5f1dec2..ea3e857 100644 --- a/src/tokenizer/test.rs +++ b/src/tokenizer/test.rs @@ -1,12 +1,15 @@ use super::*; -use nom; -use nom_locate::LocatedSpan; + +use abortable_parser::{Result, SliceIter, StrIter}; #[test] fn test_empty_token() { - let result = emptytok(LocatedSpan::new("NULL ")); - assert!(result.is_done(), format!("result {:?} is not done", result)); - if let nom::IResult::Done(_, tok) = result { + let result = emptytok(StrIter::new("NULL ")); + assert!( + result.is_complete(), + format!("result {:?} is not done", result) + ); + if let Result::Complete(_, tok) = result { assert_eq!(tok.fragment, "NULL"); assert_eq!(tok.typ, TokenType::EMPTY); } @@ -14,9 +17,12 @@ fn test_empty_token() { #[test] fn test_assert_token() { - let result = asserttok(LocatedSpan::new("assert ")); - assert!(result.is_done(), format!("result {:?} is not done", result)); - if let nom::IResult::Done(_, tok) = result { + let result = asserttok(StrIter::new("assert ")); + assert!( + result.is_complete(), + format!("result {:?} is not done", result) + ); + if let Result::Complete(_, tok) = result { assert_eq!(tok.fragment, "assert"); assert_eq!(tok.typ, TokenType::BAREWORD); } @@ -24,29 +30,56 @@ fn test_assert_token() { #[test] fn test_out_token() { - let result = outtok(LocatedSpan::new("out ")); - assert!(result.is_done(), format!("result {:?} is not done", result)); - if let nom::IResult::Done(_, tok) = result { + let result = outtok(StrIter::new("out ")); + assert!( + result.is_complete(), + format!("result {:?} is not done", result) + ); + if let Result::Complete(_, tok) = result { assert_eq!(tok.fragment, "out"); assert_eq!(tok.typ, TokenType::BAREWORD); } } +#[test] +fn test_out_token_with_comment() { + let result = outtok(StrIter::new("out//comment")); + assert!( + result.is_complete(), + format!("result {:?} is not done", result) + ); + if let Result::Complete(_, tok) = result { + assert_eq!(tok.fragment, "out"); + assert_eq!(tok.typ, TokenType::BAREWORD); + } +} + +#[test] +fn test_not_out_token() { + let result = outtok(StrIter::new("output")); + assert!(result.is_fail(), format!("result {:?} is not fail", result)); +} + #[test] fn test_escape_quoted() { - let result = escapequoted(LocatedSpan::new("foo \\\"bar\"")); - assert!(result.is_done(), format!("result {:?} is not ok", result)); - if let nom::IResult::Done(rest, frag) = result { + let result = escapequoted(StrIter::new("foo \\\"bar\"")); + assert!( + result.is_complete(), + format!("result {:?} is not ok", result) + ); + if let Result::Complete(_rest, frag) = result { assert_eq!(frag, "foo \"bar"); - assert_eq!(rest.fragment, "\""); } } #[test] fn test_pipe_quoted() { - let result = pipequotetok(LocatedSpan::new("|foo|")); - assert!(result.is_done(), format!("result {:?} is not ok", result)); - if let nom::IResult::Done(_, tok) = result { + let result = pipequotetok(StrIter::new("|foo|")); + assert!( + result.is_complete(), + format!("result {:?} is not ok", result) + ); + if let Result::Complete(_, tok) = result { assert_eq!(tok.fragment, "foo".to_string()); assert_eq!(tok.typ, TokenType::PIPEQUOTE); } @@ -54,16 +87,19 @@ fn test_pipe_quoted() { #[test] fn test_string_with_escaping() { - let result = strtok(LocatedSpan::new("\"foo \\\\ \\\"bar\"")); - assert!(result.is_done(), format!("result {:?} is not ok", result)); - if let nom::IResult::Done(_, tok) = result { + let result = strtok(StrIter::new("\"foo \\\\ \\\"bar\"")); + assert!( + result.is_complete(), + format!("result {:?} is not ok", result) + ); + if let Result::Complete(_, tok) = result { assert_eq!(tok.fragment, "foo \\ \"bar".to_string()); } } #[test] fn test_tokenize_bareword_with_dash() { - let result = tokenize(LocatedSpan::new("foo-bar ")); + let result = tokenize(StrIter::new("foo-bar ")); assert!(result.is_ok(), format!("result {:?} is not ok", result)); if let Ok(toks) = result { assert_eq!(toks.len(), 2); @@ -73,18 +109,23 @@ fn test_tokenize_bareword_with_dash() { macro_rules! assert_token { ($input:expr, $typ:expr, $msg:expr) => { - let result = token(LocatedSpan::new($input)); + let result = token(StrIter::new($input)); assert!( - result.is_done(), + result.is_complete(), format!("result {:?} is not a {}", result, $msg) ); - if let nom::IResult::Done(_, tok) = result { - assert_eq!(tok.fragment, $input); + if let Result::Complete(_, tok) = result { assert_eq!(tok.typ, $typ); + assert_eq!(tok.fragment, $input); } }; } +#[test] +fn test_digittok() { + assert_token!("1", TokenType::DIGIT, "1"); +} + #[test] fn test_boolean() { assert_token!("true", TokenType::BOOLEAN, "boolean"); @@ -122,7 +163,7 @@ fn test_lteqtok() { #[test] fn test_tokenize_one_of_each() { - let result = tokenize(LocatedSpan::new( + let result = tokenize(StrIter::new( "map out filter assert let import macro select as => [ ] { } ; = % / * \ + - . ( ) , 1 . foo \"bar\" // comment\n ; true false == < > <= >= !=", )); @@ -137,65 +178,67 @@ fn test_tokenize_one_of_each() { #[test] fn test_parse_has_end() { - let result = tokenize(LocatedSpan::new("foo")); + let result = tokenize(StrIter::new("foo")); assert!(result.is_ok()); let v = result.unwrap(); assert_eq!(v.len(), 2); assert_eq!(v[1].typ, TokenType::END); } +#[test] +fn test_whitespace() { + assert!(whitespace(StrIter::new(" ")).is_complete()); + let result = whitespace(StrIter::new(" ")); + match result { + Result::Complete(rest, o) => { + assert_eq!(rest.get_offset(), 2); + assert_eq!(o.typ, TokenType::WS); + } + _ => assert!(false, "Not complete"), + } +} + #[test] fn test_parse_comment() { - assert!(comment(LocatedSpan::new("// comment\n")).is_done()); - assert!(comment(LocatedSpan::new("// comment")).is_done()); - assert_eq!( - comment(LocatedSpan::new("// comment\n")), - nom::IResult::Done( - LocatedSpan { - fragment: "", - offset: 11, - line: 2, - }, + assert!(comment(StrIter::new("// comment\n")).is_complete()); + assert!(comment(StrIter::new("// comment")).is_complete()); + let mut parsed = comment(StrIter::new("// comment\n")); + assert!(parsed.is_complete()); + if let Result::Complete(_rest, cmt) = parsed { + assert_eq!( + cmt, Token { typ: TokenType::COMMENT, fragment: " comment".to_string(), pos: Position { line: 1, column: 1 }, } - ) - ); - assert!(comment(LocatedSpan::new("// comment\r\n")).is_done()); - assert_eq!( - comment(LocatedSpan::new("// comment\r\n")), - nom::IResult::Done( - LocatedSpan { - fragment: "", - offset: 12, - line: 2, - }, + ); + } + assert!(comment(StrIter::new("// comment\r\n")).is_complete()); + parsed = comment(StrIter::new("// comment\r\n")); + if let Result::Complete(_rest, cmt) = parsed { + assert_eq!( + cmt, Token { typ: TokenType::COMMENT, fragment: " comment".to_string(), pos: Position { column: 1, line: 1 }, } - ) - ); - assert!(comment(LocatedSpan::new("// comment\r\n ")).is_done()); - assert_eq!( - comment(LocatedSpan::new("// comment\r\n ")), - nom::IResult::Done( - LocatedSpan { - fragment: " ", - offset: 12, - line: 2, - }, + ); + } + assert!(comment(StrIter::new("// comment\r\n ")).is_complete()); + parsed = comment(StrIter::new("// comment\r\n ")); + if let Result::Complete(_rest, cmt) = parsed { + assert_eq!( + cmt, Token { typ: TokenType::COMMENT, fragment: " comment".to_string(), pos: Position { column: 1, line: 1 }, } - ) - ); - assert!(comment(LocatedSpan::new("// comment")).is_done()); + ); + } + assert!(comment(StrIter::new("// comment")).is_complete()); } #[test] @@ -205,14 +248,9 @@ fn test_match_word() { typ: TokenType::BAREWORD, pos: Position { line: 1, column: 1 }, }]; - let result = word!( - TokenIter { - source: input.as_slice(), - }, - "foo" - ); + let result = word!(SliceIter::new(input.as_slice()), "foo"); match result { - nom::IResult::Done(_, tok) => assert_eq!(tok, input[0]), + Result::Complete(_, tok) => assert_eq!(tok, input[0]), res => assert!(false, format!("Fail: {:?}", res)), } } @@ -224,20 +262,14 @@ fn test_match_word_empty_input() { typ: TokenType::END, pos: Position { line: 1, column: 1 }, }]; - let result = word!( - TokenIter { - source: input.as_slice(), - }, - "foo" - ); + let result = word!(SliceIter::new(input.as_slice()), "foo"); match result { - nom::IResult::Done(_, _) => assert!(false, "Should have been an error but was Done"), - nom::IResult::Incomplete(_) => { - assert!(false, "Should have been an error but was Incomplete") - } - nom::IResult::Error(_) => { + Result::Complete(_, _) => assert!(false, "Should have been an error but was Done"), + Result::Incomplete(_) => assert!(false, "Should have been a Fail but was Incomplete"), + Result::Fail(_) => { // noop } + Result::Abort(_) => assert!(false, "Should have been a Fail but was Abort"), } } @@ -248,14 +280,9 @@ fn test_match_punct() { typ: TokenType::PUNCT, pos: Position { line: 1, column: 1 }, }]; - let result = punct!( - TokenIter { - source: input.as_slice(), - }, - "!" - ); + let result = punct!(SliceIter::new(input.as_slice()), "!"); match result { - nom::IResult::Done(_, tok) => assert_eq!(tok, input[0]), + Result::Complete(_, tok) => assert_eq!(tok, input[0]), res => assert!(false, format!("Fail: {:?}", res)), } } @@ -267,14 +294,9 @@ fn test_match_type() { typ: TokenType::BAREWORD, pos: Position { line: 1, column: 1 }, }]; - let result = match_type!( - TokenIter { - source: input.as_slice(), - }, - BAREWORD - ); + let result = match_type!(SliceIter::new(input.as_slice()), BAREWORD); match result { - nom::IResult::Done(_, tok) => assert_eq!(tok, input[0]), + Result::Complete(_, tok) => assert_eq!(tok, input[0]), res => assert!(false, format!("Fail: {:?}", res)), } }