diff --git a/Cargo.lock b/Cargo.lock index b0b36db..0e8db69 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,3 +1,8 @@ +[[package]] +name = "abortable_parser" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "ansi_term" version = "0.9.0" @@ -131,31 +136,6 @@ name = "linked-hash-map" version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -[[package]] -name = "memchr" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "nom" -version = "3.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "memchr 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "nom_locate" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "memchr 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "nom 3.2.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "redox_syscall" version = "0.1.40" @@ -248,11 +228,10 @@ dependencies = [ name = "ucg" version = "0.2.0" dependencies = [ + "abortable_parser 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "bencher 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", "clap 2.26.2 (registry+https://github.com/rust-lang/crates.io-index)", "cpuprofiler 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)", - "nom 3.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "nom_locate 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.26 (registry+https://github.com/rust-lang/crates.io-index)", "serde_yaml 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", "simple-error 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)", @@ -306,6 +285,7 @@ dependencies = [ ] [metadata] +"checksum abortable_parser 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "09cdf5378b5e4a079fa886e621519fcb2502d9cb008d3f76b92f61f3890d5906" "checksum ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "23ac7c30002a5accbf7e8987d0632fa6de155b7c3d39d0067317a391e00a2ef6" "checksum atty 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "9a7d5b8723950951411ee34d271d99dddcc2035a16ab25310ea2c8cfd4369652" "checksum backtrace 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "346d7644f0b5f9bc73082d3b2236b69a05fd35cce0cfa3724e184e6a5c9e2a2f" @@ -324,9 +304,6 @@ dependencies = [ "checksum lazy_static 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "76f033c7ad61445c5b347c7382dd1237847eb1bce590fe50365dcb33d546be73" "checksum libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)" = "76e3a3ef172f1a0b9a9ff0dd1491ae5e6c948b94479a3021819ba7d860c8645d" "checksum linked-hash-map 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "70fb39025bc7cdd76305867c4eccf2f2dcf6e9a57f5b21a93e1c2d86cd03ec9e" -"checksum memchr 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "148fab2e51b4f1cfc66da2a7c32981d1d3c083a803978268bb11fe4b86925e7a" -"checksum nom 3.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "05aec50c70fd288702bcd93284a8444607f3292dbdf2a30de5ea5dcdbe72287b" -"checksum nom_locate 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "49b1c61eff39ab6b91ccedfc62aff196eae066d88355b4fe3e4100c23168f0df" "checksum redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)" = "c214e91d3ecf43e9a4e41e578973adeb14b474f2bee858742d127af75a0112b1" "checksum redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7e891cfe48e9100a70a3b6eb652fef28920c117d366339687bd5576160db0f76" "checksum rustc-demangle 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "bcfe5b13211b4d78e5c2cadfebd7769197d95c639c35a50057eb4c05de811395" diff --git a/Cargo.toml b/Cargo.toml index beb52f0..bf63b0f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,11 +9,8 @@ readme = "README.md" keywords = ["compiler", "config"] license = "Apache-2.0" -[dependencies.nom] -version = "^3.2" - [dependencies] -nom_locate = "^0.1.1" +abortable_parser = "0.2.1" clap = "~2.26.0" serde_json = "~1.0.9" simple-error = "0.1" diff --git a/TODO.md b/TODO.md index 788e210..758ef4c 100644 --- a/TODO.md +++ b/TODO.md @@ -9,6 +9,8 @@ You should be able to ask the compiler to tell you any value or set of values in the compiled configuration. +Inspect is probably the correct location for this. + ## Shape equality as a form of type assertion? # Minor Fixes and Polish diff --git a/example_errors/bad_file.ucg b/example_errors/bad_file.ucg new file mode 100644 index 0000000..8329071 --- /dev/null +++ b/example_errors/bad_file.ucg @@ -0,0 +1 @@ +let x = \ No newline at end of file diff --git a/integration_tests/operator_precedence_test.ucg b/integration_tests/operator_precedence_test.ucg index f093e0f..d66460b 100644 --- a/integration_tests/operator_precedence_test.ucg +++ b/integration_tests/operator_precedence_test.ucg @@ -4,4 +4,6 @@ assert |2 * (2 + 1) == 6|; assert |2 * 2 + 1 > 4|; assert |2 * 2 + 1 < 6|; assert |2 * 2 + 1 >= 5|; -assert |2 * 2 + 1 <= 5|; \ No newline at end of file +assert |2 * 2 + 1 <= 5|; +assert |2 / 2 == 1|; +assert |2 - 1 == 1|; \ No newline at end of file diff --git a/src/ast/mod.rs b/src/ast/mod.rs index 1a1f0ce..3b370d2 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -21,10 +21,11 @@ use std::cmp::PartialEq; use std::cmp::PartialOrd; use std::collections::HashSet; use std::convert::Into; +use std::fmt; use std::hash::Hash; use std::hash::Hasher; -use std::fmt; +use abortable_parser; macro_rules! enum_type_equality { ( $slf:ident, $r:expr, $( $l:pat ),* ) => { @@ -50,18 +51,26 @@ macro_rules! enum_type_equality { pub struct Position { pub line: usize, pub column: usize, + pub offset: usize, } impl Position { /// Construct a new Position. - pub fn new(line: usize, column: usize) -> Self { + pub fn new(line: usize, column: usize, offset: usize) -> Self { Position { line: line, column: column, + offset: offset, } } } +impl<'a> From<&'a Position> for Position { + fn from(source: &'a Position) -> Self { + source.clone() + } +} + /// Defines the types of tokens in UCG syntax. #[derive(Debug, PartialEq, Eq, Clone, PartialOrd, Ord, Hash)] pub enum TokenType { @@ -89,8 +98,8 @@ pub struct Token { impl Token { /// Constructs a new Token with a type and line and column information. - pub fn new>(f: S, typ: TokenType, line: usize, col: usize) -> Self { - Self::new_with_pos(f, typ, Position::new(line, col)) + pub fn new, P: Into>(f: S, typ: TokenType, p: P) -> Self { + Self::new_with_pos(f, typ, p.into()) } // Constructs a new Token with a type and a Position. @@ -103,6 +112,15 @@ impl Token { } } +impl abortable_parser::Positioned for Token { + fn line(&self) -> usize { + self.pos.line + } + fn column(&self) -> usize { + self.pos.column + } +} + impl Borrow for Token { fn borrow(&self) -> &str { &self.fragment @@ -112,58 +130,54 @@ impl Borrow for Token { /// Helper macro for making a Positioned Value. macro_rules! value_node { ($v:expr, $p:expr) => { - Positioned::new_with_pos($v, $p) - }; - ($v:expr, $l:expr, $c:expr) => { - Positioned::new($v, $l, $c) + PositionedItem::new_with_pos($v, $p) }; } /// Helper macro for making a Token. #[allow(unused_macros)] macro_rules! make_tok { - (EOF => $l:expr, $c:expr) => { - Token::new("", TokenType::END, $l, $c) + (EOF => $i:expr) => { + Token::new("", TokenType::END, &$i) }; - (WS => $l:expr, $c:expr) => { - Token::new("", TokenType::WS, $l, $c) + (WS => $i:expr) => { + Token::new("", TokenType::WS, &$i) }; - (CMT => $e:expr, $l:expr, $c:expr) => { - Token::new($e, TokenType::COMMENT, $l, $c) + (CMT => $e:expr, $i:expr) => { + Token::new($e, TokenType::COMMENT, &$i) }; - (QUOT => $e:expr, $l:expr, $c:expr) => { - Token::new($e, TokenType::QUOTED, $l, $c) + (QUOT => $e:expr, $i:expr) => { + Token::new($e, TokenType::QUOTED, &$i) }; - (PUNCT => $e:expr, $l:expr, $c:expr) => { - Token::new($e, TokenType::PUNCT, $l, $c) + (PUNCT => $e:expr, $i:expr) => { + Token::new($e, TokenType::PUNCT, &$i) }; - (DIGIT => $e:expr, $l:expr, $c:expr) => { - Token::new($e, TokenType::DIGIT, $l, $c) + (DIGIT => $e:expr, $i:expr) => { + Token::new($e, TokenType::DIGIT, &$i) }; - ($e:expr, $l:expr, $c:expr) => { - Token::new($e, TokenType::BAREWORD, $l, $c) + ($e:expr, $i:expr) => { + Token::new($e, TokenType::BAREWORD, &$i) }; } /// Helper macro for making expressions. #[allow(unused_macros)] macro_rules! make_expr { - ($e:expr) => { - make_expr!($e, 1, 1) + ($e:expr, $i:expr) => { + Expression::Simple(Value::Symbol(PositionedItem::new_with_pos( + $e.to_string(), + $i, + ))) }; - ($e:expr, $l:expr, $c:expr) => { - Expression::Simple(Value::Symbol(Positioned::new($e.to_string(), $l, $c))) - }; - - ($e:expr => int, $l:expr, $c:expr) => { - Expression::Simple(Value::Int(Positioned::new($e, $l, $c))) + ($e:expr => int, $i:expr) => { + Expression::Simple(Value::Int(PositionedItem::new_with_pos($e, $i))) }; } @@ -180,30 +194,26 @@ macro_rules! make_expr { /// ``` #[allow(unused_macros)] macro_rules! make_selector { - ( $h:expr ) => { - make_selector!($h, 1, 0) - }; - - ( $h:expr, $l:expr, $c:expr ) => { + ( $h:expr, $i:expr) => { SelectorDef::new( SelectorList{head: Box::new($h), tail: None}, - $l, $c) + $i) }; - ( $h: expr, $list:expr, $l:expr, $c:expr) => { + ( $h: expr, $list:expr, $i:expr) => { SelectorDef::new( SelectorList{head: Box::new($h), tail: Some($list)}, - $l, $c) + $i) }; // Tokens - ( $h:expr => [ $( $item:expr ),* ] ) => { + ( $h:expr => [ $( $item:expr ),* ], $i:expr ) => { { - make_selector!($h => [ $( $item, )* ] => 1, 1) + make_selector!($h => [ $( $item, )* ] => $i) } }; - ( $h:expr => [ $( $item:expr ),* ] => $l:expr, $c:expr ) => { + ( $h:expr => [ $( $item:expr ),* ] => $i:expr ) => { { let mut list: Vec = Vec::new(); @@ -211,7 +221,7 @@ macro_rules! make_selector { list.push($item); )* - make_selector!($h, list, $l, $c) + make_selector!($h, list, $i) } }; @@ -223,14 +233,14 @@ macro_rules! make_selector { let mut list: Vec = Vec::new(); $( - list.push(make_tok!($item, 1, col)); + list.push(make_tok!($item, Position::new(1, col, col))); col += $item.len() + 1; )* // Shut up the lint about unused code; assert!(col != 0); - make_selector!($h, list, 1, 1) + make_selector!($h, list, Position::new(1, 1, 1)) } }; @@ -241,14 +251,14 @@ macro_rules! make_selector { let mut list: Vec = Vec::new(); $( - list.push(make_tok!($item, $l, col)); + list.push(make_tok!($item, Position::new($l, col, col))); col += $item.len() + 1; )* // Shut up the linter about unused code; assert!(col != 0); - make_selector!($h, list, $l, $c) + make_selector!($h, list, Position::new($l, $c, $c)) } }; } @@ -314,9 +324,9 @@ pub struct SelectorDef { impl SelectorDef { /// Constructs a new SelectorDef. - pub fn new(sel: SelectorList, line: usize, col: usize) -> Self { + pub fn new>(sel: SelectorList, p: P) -> Self { SelectorDef { - pos: Position::new(line, col), + pos: p.into(), sel: sel, } } @@ -327,13 +337,13 @@ impl SelectorDef { pub enum Value { // Constant Values Empty(Position), - Boolean(Positioned), - Int(Positioned), - Float(Positioned), - Str(Positioned), - Symbol(Positioned), + Boolean(PositionedItem), + Int(PositionedItem), + Float(PositionedItem), + Str(PositionedItem), + Symbol(PositionedItem), // Complex Values - Tuple(Positioned), + Tuple(PositionedItem), List(ListDef), Selector(SelectorDef), } @@ -438,67 +448,67 @@ pub struct SelectDef { /// Adds position information to any type `T`. #[derive(Debug, Clone)] -pub struct Positioned { +pub struct PositionedItem { pub pos: Position, pub val: T, } -impl std::fmt::Display for Positioned { +impl std::fmt::Display for PositionedItem { fn fmt(&self, f: &mut std::fmt::Formatter) -> Result<(), std::fmt::Error> { write!(f, "{}", self.val) } } -impl Positioned { +impl PositionedItem { /// Constructs a new Positioned with a value, line, and column information. - pub fn new(v: T, l: usize, c: usize) -> Self { - Self::new_with_pos(v, Position::new(l, c)) + pub fn new>(v: T, p: P) -> Self { + Self::new_with_pos(v, p.into()) } /// Constructs a new Positioned with a value and a Position. pub fn new_with_pos(v: T, pos: Position) -> Self { - Positioned { pos: pos, val: v } + PositionedItem { pos: pos, val: v } } } -impl PartialEq for Positioned { +impl PartialEq for PositionedItem { fn eq(&self, other: &Self) -> bool { self.val == other.val } } -impl Eq for Positioned {} +impl Eq for PositionedItem {} -impl Ord for Positioned { +impl Ord for PositionedItem { fn cmp(&self, other: &Self) -> Ordering { self.val.cmp(&other.val) } } -impl PartialOrd for Positioned { +impl PartialOrd for PositionedItem { fn partial_cmp(&self, other: &Self) -> Option { self.val.partial_cmp(&other.val) } } -impl Hash for Positioned { +impl Hash for PositionedItem { fn hash(&self, state: &mut H) { self.val.hash(state); } } -impl<'a> From<&'a Token> for Positioned { - fn from(t: &'a Token) -> Positioned { - Positioned { +impl<'a> From<&'a Token> for PositionedItem { + fn from(t: &'a Token) -> PositionedItem { + PositionedItem { pos: t.pos.clone(), val: t.fragment.to_string(), } } } -impl<'a> From<&'a Positioned> for Positioned { - fn from(t: &Positioned) -> Positioned { - Positioned { +impl<'a> From<&'a PositionedItem> for PositionedItem { + fn from(t: &PositionedItem) -> PositionedItem { + PositionedItem { pos: t.pos.clone(), val: t.val.clone(), } @@ -512,7 +522,7 @@ impl<'a> From<&'a Positioned> for Positioned { /// any values except what is defined in their arguments. #[derive(PartialEq, Debug, Clone)] pub struct MacroDef { - pub argdefs: Vec>, + pub argdefs: Vec>, pub fields: FieldList, pub pos: Position, } diff --git a/src/ast/test.rs b/src/ast/test.rs index d8ecc5a..d97060b 100644 --- a/src/ast/test.rs +++ b/src/ast/test.rs @@ -17,21 +17,23 @@ use super::*; #[test] pub fn test_macro_validation_happy_path() { let def = MacroDef { - argdefs: vec![value_node!("foo".to_string(), 1, 0)], + argdefs: vec![value_node!("foo".to_string(), Position::new(1, 0, 0))], fields: vec![( - make_tok!("f1", 1, 1), + make_tok!("f1", Position::new(1, 1, 0)), Expression::Binary(BinaryOpDef { kind: BinaryExprType::Add, left: Box::new(Expression::Simple(Value::Symbol(value_node!( "foo".to_string(), - 1, - 1 + Position::new(1, 1, 0) )))), - right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))), - pos: Position::new(1, 0), + right: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 1, 0) + )))), + pos: Position::new(1, 0, 0), }), )], - pos: Position::new(1, 0), + pos: Position::new(1, 0, 0), }; assert!(def.validate_symbols().unwrap() == ()); } @@ -39,21 +41,23 @@ pub fn test_macro_validation_happy_path() { #[test] pub fn test_macro_validation_fail() { let def = MacroDef { - argdefs: vec![value_node!("foo".to_string(), 1, 0)], + argdefs: vec![value_node!("foo".to_string(), Position::new(1, 0, 0))], fields: vec![( - make_tok!("f1", 1, 1), + make_tok!("f1", Position::new(1, 1, 0)), Expression::Binary(BinaryOpDef { kind: BinaryExprType::Add, left: Box::new(Expression::Simple(Value::Symbol(value_node!( "bar".to_string(), - 1, - 1 + Position::new(1, 1, 0) )))), - right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))), - pos: Position::new(1, 0), + right: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 1, 0) + )))), + pos: Position::new(1, 0, 0), }), )], - pos: Position::new(1, 0), + pos: Position::new(1, 0, 0), }; let mut expected = HashSet::new(); expected.insert("bar".to_string()); @@ -63,20 +67,23 @@ pub fn test_macro_validation_fail() { #[test] pub fn test_macro_validation_selector_happy_path() { let def = MacroDef { - argdefs: vec![value_node!("foo".to_string(), 1, 0)], + argdefs: vec![value_node!("foo".to_string(), Position::new(1, 0, 0))], fields: vec![( - make_tok!("f1", 1, 1), + make_tok!("f1", Position::new(1, 1, 0)), Expression::Binary(BinaryOpDef { kind: BinaryExprType::Add, - left: Box::new(Expression::Simple(Value::Selector( - make_selector!(make_expr!("foo", 1, 1) => [ - make_tok!("quux", 1, 1) ] => 1, 1), - ))), - right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))), - pos: Position::new(1, 0), + left: Box::new(Expression::Simple(Value::Selector(make_selector!( + make_expr!("foo", Position::new(1, 1, 0)) => [ + make_tok!("quux", Position::new(1, 1, 0)) ] + => Position::new(1, 1, 0))))), + right: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 1, 0) + )))), + pos: Position::new(1, 0, 0), }), )], - pos: Position::new(1, 0), + pos: Position::new(1, 0, 0), }; assert!(def.validate_symbols().unwrap() == ()); } @@ -84,20 +91,23 @@ pub fn test_macro_validation_selector_happy_path() { #[test] pub fn test_macro_validation_selector_fail() { let def = MacroDef { - argdefs: vec![value_node!("foo".to_string(), 1, 0)], + argdefs: vec![value_node!("foo".to_string(), Position::new(1, 0, 0))], fields: vec![( - make_tok!("f1", 1, 1), + make_tok!("f1", Position::new(1, 1, 0)), Expression::Binary(BinaryOpDef { kind: BinaryExprType::Add, left: Box::new(Expression::Simple(Value::Selector( - make_selector!(make_expr!("bar", 1, 1) => [ - make_tok!("quux", 1, 1) ] => 1, 1), + make_selector!(make_expr!("bar", Position::new(1, 1, 0)) => [ + make_tok!("quux", Position::new(1, 1, 0)) ] => Position::new(1, 1, 0)), ))), - right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))), - pos: Position::new(1, 0), + right: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 1, 0) + )))), + pos: Position::new(1, 0, 0), }), )], - pos: Position::new(1, 0), + pos: Position::new(1, 0, 0), }; let mut expected = HashSet::new(); expected.insert("bar".to_string()); diff --git a/src/benches/parse.rs b/src/benches/parse.rs index 0cc1c25..30232ea 100644 --- a/src/benches/parse.rs +++ b/src/benches/parse.rs @@ -16,17 +16,20 @@ #[macro_use] extern crate bencher; +extern crate abortable_parser; extern crate cpuprofiler; -extern crate nom_locate; extern crate ucglib; use bencher::Bencher; + +use ucglib::iter::OffsetStrIter; + //use cpuprofiler::PROFILER; use ucglib::parse::*; fn do_parse(i: &str) { - parse(nom_locate::LocatedSpan::new(i)); + parse(OffsetStrIter::new(i)); } fn parse_int(b: &mut Bencher) { diff --git a/src/build/ir.rs b/src/build/ir.rs index 736424d..9d34748 100644 --- a/src/build/ir.rs +++ b/src/build/ir.rs @@ -18,7 +18,7 @@ pub enum Val { Float(f64), Str(String), List(Vec>), - Tuple(Vec<(Positioned, Rc)>), + Tuple(Vec<(PositionedItem, Rc)>), Macro(MacroDef), } @@ -114,7 +114,7 @@ impl Val { } /// Returns the fields if this Val is a tuple. None otherwise. - pub fn get_fields(&self) -> Option<&Vec<(Positioned, Rc)>> { + pub fn get_fields(&self) -> Option<&Vec<(PositionedItem, Rc)>> { if let &Val::Tuple(ref fs) = self { Some(fs) } else { diff --git a/src/build/mod.rs b/src/build/mod.rs index c0fa3e5..19d54eb 100644 --- a/src/build/mod.rs +++ b/src/build/mod.rs @@ -28,8 +28,8 @@ use std::string::ToString; use ast::*; use error; use format; +use iter::OffsetStrIter; use parse::parse; -use tokenizer::Span; pub mod assets; pub mod ir; @@ -44,7 +44,7 @@ impl MacroDef { cache: Rc>, env: Rc, mut args: Vec>, - ) -> Result, Rc)>, Box> { + ) -> Result, Rc)>, Box> { // Error conditions. If the args don't match the length and types of the argdefs then this is // macro call error. if args.len() > self.argdefs.len() { @@ -61,12 +61,12 @@ impl MacroDef { // If the expressions reference Symbols not defined in the MacroDef that is also an error. // TODO(jwall): We should probably enforce that the Expression Symbols must be in argdefs rules // at Macro definition time not evaluation time. - let mut scope = HashMap::, Rc>::new(); + let mut scope = HashMap::, Rc>::new(); for (i, arg) in args.drain(0..).enumerate() { scope.entry(self.argdefs[i].clone()).or_insert(arg.clone()); } let b = Builder::new_with_env_and_scope(root, cache, scope, env); - let mut result: Vec<(Positioned, Rc)> = Vec::new(); + let mut result: Vec<(PositionedItem, Rc)> = Vec::new(); for &(ref key, ref expr) in self.fields.iter() { // We clone the expressions here because this macro may be consumed // multiple times in the future. @@ -81,7 +81,7 @@ impl MacroDef { type BuildResult = Result<(), Box>; /// Defines a set of values in a parsed file. -type ValueMap = HashMap, Rc>; +type ValueMap = HashMap, Rc>; /// AssertCollector collects the results of assertions in the UCG AST. pub struct AssertCollector { @@ -136,7 +136,7 @@ macro_rules! eval_binary_expr { impl<'a> Builder<'a> { // TOOD(jwall): This needs some unit tests. fn tuple_to_val(&self, fields: &Vec<(Token, Expression)>) -> Result, Box> { - let mut new_fields = Vec::<(Positioned, Rc)>::new(); + let mut new_fields = Vec::<(PositionedItem, Rc)>::new(); for &(ref name, ref expr) in fields.iter() { let val = try!(self.eval_expr(expr)); new_fields.push((name.into(), val)); @@ -159,17 +159,18 @@ impl<'a> Builder<'a> { &Value::Int(ref i) => Ok(Rc::new(Val::Int(i.val))), &Value::Float(ref f) => Ok(Rc::new(Val::Float(f.val))), &Value::Str(ref s) => Ok(Rc::new(Val::Str(s.val.to_string()))), - &Value::Symbol(ref s) => self.lookup_sym(&(s.into())).ok_or(Box::new( - error::Error::new( - format!( - "Unable to find {} in file: {}", - s.val, - self.root.to_string_lossy() - ), - error::ErrorType::NoSuchSymbol, - v.pos().clone(), - ), - )), + &Value::Symbol(ref s) => { + self.lookup_sym(&(s.into())) + .ok_or(Box::new(error::Error::new( + format!( + "Unable to find {} in file: {}", + s.val, + self.root.to_string_lossy() + ), + error::ErrorType::NoSuchSymbol, + v.pos().clone(), + ))) + } &Value::List(ref def) => self.list_to_val(def), &Value::Tuple(ref tuple) => self.tuple_to_val(&tuple.val), &Value::Selector(ref selector_list_node) => { @@ -189,9 +190,13 @@ impl<'a> Builder<'a> { cache: Rc>, scope: ValueMap, ) -> Self { - let env_vars: Vec<(Positioned, Rc)> = env::vars() - .map(|t| (Positioned::new(t.0, 0, 0), Rc::new(t.1.into()))) - .collect(); + let env_vars: Vec<(PositionedItem, Rc)> = env::vars() + .map(|t| { + ( + PositionedItem::new(t.0, Position::new(0, 0, 0)), + Rc::new(t.1.into()), + ) + }).collect(); Self::new_with_env_and_scope(root, cache, scope, Rc::new(Val::Tuple(env_vars))) } @@ -220,8 +225,8 @@ impl<'a> Builder<'a> { /// Returns a Val by name from previously built UCG. pub fn get_out_by_name(&self, name: &str) -> Option> { - let key = Positioned { - pos: Position::new(0, 0), + let key = PositionedItem { + pos: Position::new(0, 0, 0), val: name.to_string(), }; self.lookup_sym(&key) @@ -243,8 +248,8 @@ impl<'a> Builder<'a> { Ok(()) } - fn eval_span(&mut self, input: Span) -> Result, Box> { - match parse(input) { + fn eval_span(&mut self, input: OffsetStrIter) -> Result, Box> { + match parse(input.clone()) { Ok(stmts) => { //panic!("Successfully parsed {}", input); let mut out: Option> = None; @@ -256,20 +261,17 @@ impl<'a> Builder<'a> { Some(val) => Ok(val), } } - Err(err) => Err(Box::new(error::Error::new_with_cause( - format!( - "Error while parsing file: {}", - self.curr_file.unwrap_or("") - ), + Err(err) => Err(Box::new(error::Error::new( + format!("{}", err,), error::ErrorType::ParseError, - err, + (&input).into(), ))), } } /// Evaluate an input string as UCG. pub fn eval_string(&mut self, input: &str) -> Result, Box> { - self.eval_span(Span::new(input)) + self.eval_span(OffsetStrIter::new(input)) } /// Builds a ucg file at the named path. @@ -295,7 +297,8 @@ impl<'a> Builder<'a> { let mut b = Self::new(normalized.clone(), self.assets.clone()); let filepath = normalized.to_str().unwrap().clone(); try!(b.build_file(filepath)); - let fields: Vec<(Positioned, Rc)> = b.build_output.drain().collect(); + let fields: Vec<(PositionedItem, Rc)> = + b.build_output.drain().collect(); Rc::new(Val::Tuple(fields)) } }; @@ -360,7 +363,7 @@ impl<'a> Builder<'a> { } } - fn lookup_sym(&self, sym: &Positioned) -> Option> { + fn lookup_sym(&self, sym: &PositionedItem) -> Option> { if &sym.val == "env" { return Some(self.env.clone()); } @@ -370,7 +373,10 @@ impl<'a> Builder<'a> { None } - fn find_in_fieldlist(target: &str, fs: &Vec<(Positioned, Rc)>) -> Option> { + fn find_in_fieldlist( + target: &str, + fs: &Vec<(PositionedItem, Rc)>, + ) -> Option> { for (key, val) in fs.iter().cloned() { if target == &key.val { return Some(val.clone()); @@ -384,7 +390,7 @@ impl<'a> Builder<'a> { stack: &mut VecDeque>, sl: &SelectorList, next: (&Position, &str), - fs: &Vec<(Positioned, Rc)>, + fs: &Vec<(PositionedItem, Rc)>, ) -> Result<(), Box> { if let Some(vv) = Self::find_in_fieldlist(next.1, fs) { stack.push_back(vv.clone()); @@ -775,7 +781,7 @@ impl<'a> Builder<'a> { fn eval_copy(&self, def: &CopyDef) -> Result, Box> { let v = try!(self.lookup_selector(&def.selector.sel)); if let Val::Tuple(ref src_fields) = *v { - let mut m = HashMap::, (i32, Rc)>::new(); + let mut m = HashMap::, (i32, Rc)>::new(); // loop through fields and build up a hashmap let mut count = 0; for &(ref key, ref val) in src_fields.iter() { @@ -824,7 +830,7 @@ impl<'a> Builder<'a> { } }; } - let mut new_fields: Vec<(Positioned, (i32, Rc))> = m.drain().collect(); + let mut new_fields: Vec<(PositionedItem, (i32, Rc))> = m.drain().collect(); // We want to maintain our order for the fields to make comparing tuples // easier in later code. So we sort by the field order before constructing a new tuple. new_fields.sort_by(|a, b| { @@ -839,8 +845,7 @@ impl<'a> Builder<'a> { let first = a.0.clone(); let t = a.1.clone(); (first, t.1) - }) - .collect(), + }).collect(), ))); } Err(Box::new(error::Error::new( @@ -992,11 +997,8 @@ impl<'a> Builder<'a> { let expr = &tok.fragment; expr_as_stmt.push_str(expr); expr_as_stmt.push_str(";"); - let assert_input = Span { - fragment: &expr_as_stmt, - line: tok.pos.line as u32, - offset: tok.pos.column, - }; + let assert_input = + OffsetStrIter::new_with_offsets(&expr_as_stmt, tok.pos.line - 1, tok.pos.column - 1); let ok = match self.eval_span(assert_input) { Ok(v) => v, Err(e) => { diff --git a/src/build/test.rs b/src/build/test.rs index e984621..53f4e1d 100644 --- a/src/build/test.rs +++ b/src/build/test.rs @@ -25,35 +25,6 @@ fn test_expr_to_val(mut cases: Vec<(Expression, Val)>, b: Builder) { } } -#[test] -fn test_eval_div_expr() { - let cache = Rc::new(RefCell::new(MemoryCache::new())); - let b = Builder::new(std::env::current_dir().unwrap(), cache); - test_expr_to_val( - vec![ - ( - Expression::Binary(BinaryOpDef { - kind: BinaryExprType::Div, - left: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))), - right: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))), - pos: Position::new(1, 0), - }), - Val::Int(1), - ), - ( - Expression::Binary(BinaryOpDef { - kind: BinaryExprType::Div, - left: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))), - right: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))), - pos: Position::new(1, 0), - }), - Val::Float(1.0), - ), - ], - b, - ); -} - #[test] #[should_panic(expected = "Expected Float")] fn test_eval_div_expr_fail() { @@ -63,9 +34,15 @@ fn test_eval_div_expr_fail() { vec![( Expression::Binary(BinaryOpDef { kind: BinaryExprType::Div, - left: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))), - right: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))), - pos: Position::new(1, 0), + left: Box::new(Expression::Simple(Value::Float(value_node!( + 2.0, + Position::new(1, 1, 1) + )))), + right: Box::new(Expression::Simple(Value::Int(value_node!( + 2, + Position::new(1, 1, 1) + )))), + pos: Position::new(1, 0, 0), }), Val::Float(1.0), )], @@ -73,35 +50,6 @@ fn test_eval_div_expr_fail() { ); } -#[test] -fn test_eval_mul_expr() { - let cache = Rc::new(RefCell::new(MemoryCache::new())); - let b = Builder::new(std::env::current_dir().unwrap(), cache); - test_expr_to_val( - vec![ - ( - Expression::Binary(BinaryOpDef { - kind: BinaryExprType::Mul, - left: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))), - right: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))), - pos: Position::new(1, 0), - }), - Val::Int(4), - ), - ( - Expression::Binary(BinaryOpDef { - kind: BinaryExprType::Mul, - left: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))), - right: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))), - pos: Position::new(1, 0), - }), - Val::Float(4.0), - ), - ], - b, - ); -} - #[test] #[should_panic(expected = "Expected Float")] fn test_eval_mul_expr_fail() { @@ -111,9 +59,15 @@ fn test_eval_mul_expr_fail() { vec![( Expression::Binary(BinaryOpDef { kind: BinaryExprType::Mul, - left: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))), - right: Box::new(Expression::Simple(Value::Int(value_node!(20, 1, 1)))), - pos: Position::new(1, 0), + left: Box::new(Expression::Simple(Value::Float(value_node!( + 2.0, + Position::new(1, 1, 1) + )))), + right: Box::new(Expression::Simple(Value::Int(value_node!( + 20, + Position::new(1, 1, 1) + )))), + pos: Position::new(1, 0, 0), }), Val::Float(1.0), )], @@ -121,35 +75,6 @@ fn test_eval_mul_expr_fail() { ); } -#[test] -fn test_eval_subtract_expr() { - let cache = Rc::new(RefCell::new(MemoryCache::new())); - let b = Builder::new(std::env::current_dir().unwrap(), cache); - test_expr_to_val( - vec![ - ( - Expression::Binary(BinaryOpDef { - kind: BinaryExprType::Sub, - left: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))), - right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))), - pos: Position::new(1, 0), - }), - Val::Int(1), - ), - ( - Expression::Binary(BinaryOpDef { - kind: BinaryExprType::Sub, - left: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))), - right: Box::new(Expression::Simple(Value::Float(value_node!(1.0, 1, 1)))), - pos: Position::new(1, 0), - }), - Val::Float(1.0), - ), - ], - b, - ); -} - #[test] #[should_panic(expected = "Expected Float")] fn test_eval_subtract_expr_fail() { @@ -159,88 +84,21 @@ fn test_eval_subtract_expr_fail() { vec![( Expression::Binary(BinaryOpDef { kind: BinaryExprType::Sub, - left: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))), - right: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))), - pos: Position::new(1, 0), + left: Box::new(Expression::Simple(Value::Float(value_node!( + 2.0, + Position::new(1, 1, 1) + )))), + right: Box::new(Expression::Simple(Value::Int(value_node!( + 2, + Position::new(1, 1, 1) + )))), + pos: Position::new(1, 0, 0), }), Val::Float(1.0), )], b, ); } - -#[test] -fn test_eval_add_expr() { - let cache = Rc::new(RefCell::new(MemoryCache::new())); - let b = Builder::new(std::env::current_dir().unwrap(), cache); - test_expr_to_val( - vec![ - ( - Expression::Binary(BinaryOpDef { - kind: BinaryExprType::Add, - left: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))), - right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))), - pos: Position::new(1, 0), - }), - Val::Int(2), - ), - ( - Expression::Binary(BinaryOpDef { - kind: BinaryExprType::Add, - left: Box::new(Expression::Simple(Value::Float(value_node!(1.0, 1, 1)))), - right: Box::new(Expression::Simple(Value::Float(value_node!(1.0, 1, 1)))), - pos: Position::new(1, 0), - }), - Val::Float(2.0), - ), - ( - Expression::Binary(BinaryOpDef { - kind: BinaryExprType::Add, - left: Box::new(Expression::Simple(Value::Str(value_node!( - "foo".to_string(), - 1, - 1 - )))), - right: Box::new(Expression::Simple(Value::Str(value_node!( - "bar".to_string(), - 1, - 1 - )))), - pos: Position::new(1, 0), - }), - Val::Str("foobar".to_string()), - ), - ( - Expression::Binary(BinaryOpDef { - kind: BinaryExprType::Add, - left: Box::new(Expression::Simple(Value::List(ListDef { - elems: vec![Expression::Simple(Value::Str(value_node!( - "foo".to_string(), - 1, - 1 - )))], - pos: Position::new(1, 1), - }))), - right: Box::new(Expression::Simple(Value::List(ListDef { - elems: vec![Expression::Simple(Value::Str(value_node!( - "bar".to_string(), - 1, - 1 - )))], - pos: Position::new(1, 1), - }))), - pos: Position::new(1, 0), - }), - Val::List(vec![ - Rc::new(Val::Str("foo".to_string())), - Rc::new(Val::Str("bar".to_string())), - ]), - ), - ], - b, - ); -} - #[test] #[should_panic(expected = "Expected Float")] fn test_eval_add_expr_fail() { @@ -250,9 +108,15 @@ fn test_eval_add_expr_fail() { vec![( Expression::Binary(BinaryOpDef { kind: BinaryExprType::Add, - left: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))), - right: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))), - pos: Position::new(1, 0), + left: Box::new(Expression::Simple(Value::Float(value_node!( + 2.0, + Position::new(1, 1, 1) + )))), + right: Box::new(Expression::Simple(Value::Int(value_node!( + 2, + Position::new(1, 1, 1) + )))), + pos: Position::new(1, 0, 0), }), Val::Float(1.0), )], @@ -260,245 +124,20 @@ fn test_eval_add_expr_fail() { ); } -#[test] -fn test_eval_nested_tuple() { - test_expr_to_val( - vec![ - ( - Expression::Simple(Value::Tuple(value_node!( - vec![( - Token::new("foo", TokenType::BAREWORD, 1, 1), - Expression::Simple(Value::Tuple(value_node!(Vec::new(), 1, 1))), - )], - 1, - 1 - ))), - Val::Tuple(vec![( - Positioned::new("foo".to_string(), 1, 1), - Rc::new(Val::Tuple(Vec::new())), - )]), - ), - ( - Expression::Simple(Value::Tuple(value_node!( - vec![( - Token::new("foo", TokenType::BAREWORD, 1, 1), - Expression::Simple(Value::Tuple(value_node!( - vec![( - Token::new("bar".to_string(), TokenType::BAREWORD, 1, 5), - Expression::Simple(Value::Tuple(value_node!(vec![], 1, 10))), - )], - 1, - 1 - ))), - )], - 1, - 1 - ))), - Val::Tuple(vec![( - Positioned::new("foo".to_string(), 1, 1), - Rc::new(Val::Tuple(vec![( - Positioned::new("bar".to_string(), 1, 10), - Rc::new(Val::Tuple(vec![])), - )])), - )]), - ), - ( - Expression::Simple(Value::Tuple(value_node!( - vec![( - Token::new("foo", TokenType::BAREWORD, 1, 1), - Expression::Simple(Value::Tuple(value_node!( - vec![( - Token::new("bar".to_string(), TokenType::BAREWORD, 1, 5), - Expression::Simple(Value::Tuple(value_node!( - vec![( - Token::new("quux".to_string(), TokenType::BAREWORD, 1, 1), - Expression::Simple(Value::Int(value_node!(3, 1, 1))), - )], - 1, - 10 - ))), - )], - 1, - 1 - ))), - )], - 1, - 1 - ))), - Val::Tuple(vec![( - Positioned::new("foo".to_string(), 1, 1), - Rc::new(Val::Tuple(vec![( - Positioned::new("bar".to_string(), 1, 10), - Rc::new(Val::Tuple(vec![( - Positioned::new("quux".to_string(), 1, 1), - Rc::new(Val::Int(3)), - )])), - )])), - )]), - ), - ], - Builder::new( - std::env::current_dir().unwrap(), - Rc::new(RefCell::new(MemoryCache::new())), - ), - ); -} - -#[test] -fn test_eval_simple_expr() { - test_expr_to_val( - vec![ - ( - Expression::Simple(Value::Int(value_node!(1, 1, 1))), - Val::Int(1), - ), - ( - Expression::Simple(Value::Float(value_node!(2.0, 1, 1))), - Val::Float(2.0), - ), - ( - Expression::Simple(Value::Str(value_node!("foo".to_string(), 1, 1))), - Val::Str("foo".to_string()), - ), - ( - Expression::Simple(Value::Tuple(value_node!( - vec![( - make_tok!("bar", 1, 1), - Expression::Simple(Value::Int(value_node!(1, 1, 1))), - )], - 1, - 1 - ))), - Val::Tuple(vec![( - value_node!("bar".to_string(), 1, 1), - Rc::new(Val::Int(1)), - )]), - ), - ], - Builder::new( - std::env::current_dir().unwrap(), - Rc::new(RefCell::new(MemoryCache::new())), - ), - ); -} - -#[test] -fn test_eval_simple_lookup_expr() { - let cache = Rc::new(RefCell::new(MemoryCache::new())); - let mut b = Builder::new(std::env::current_dir().unwrap(), cache); - b.build_output - .entry(value_node!("var1".to_string(), 1, 0)) - .or_insert(Rc::new(Val::Int(1))); - test_expr_to_val( - vec![( - Expression::Simple(Value::Symbol(value_node!("var1".to_string(), 1, 1))), - Val::Int(1), - )], - b, - ); -} - #[test] fn test_eval_simple_lookup_error() { let cache = Rc::new(RefCell::new(MemoryCache::new())); let mut b = Builder::new(std::env::current_dir().unwrap(), cache); b.build_output - .entry(value_node!("var1".to_string(), 1, 0)) + .entry(value_node!("var1".to_string(), Position::new(1, 0, 0))) .or_insert(Rc::new(Val::Int(1))); - let expr = Expression::Simple(Value::Symbol(value_node!("var".to_string(), 1, 1))); + let expr = Expression::Simple(Value::Symbol(value_node!( + "var".to_string(), + Position::new(1, 1, 1) + ))); assert!(b.eval_expr(&expr).is_err()); } -#[test] -fn test_eval_selector_expr() { - let cache = Rc::new(RefCell::new(MemoryCache::new())); - let mut b = Builder::new(std::env::current_dir().unwrap(), cache); - b.build_output - .entry(value_node!("var1".to_string(), 1, 0)) - .or_insert(Rc::new(Val::Tuple(vec![( - value_node!("lvl1".to_string(), 1, 0), - Rc::new(Val::Tuple(vec![( - value_node!("lvl2".to_string(), 1, 0), - Rc::new(Val::Int(3)), - )])), - )]))); - b.build_output - .entry(value_node!("var2".to_string(), 1, 0)) - .or_insert(Rc::new(Val::Int(2))); - b.build_output - .entry(value_node!("var3".to_string(), 1, 0)) - .or_insert(Rc::new(Val::Tuple(vec![( - value_node!("lvl1".to_string(), 1, 0), - Rc::new(Val::Int(4)), - )]))); - - test_expr_to_val( - vec![ - ( - Expression::Simple(Value::Selector(make_selector!(make_expr!("var1")))), - Val::Tuple(vec![( - value_node!("lvl1".to_string(), 1, 0), - Rc::new(Val::Tuple(vec![( - value_node!("lvl2".to_string(), 1, 0), - Rc::new(Val::Int(3)), - )])), - )]), - ), - ( - Expression::Simple(Value::Selector( - make_selector!(make_expr!("var1") => "lvl1"), - )), - Val::Tuple(vec![( - value_node!("lvl2".to_string(), 1, 0), - Rc::new(Val::Int(3)), - )]), - ), - ( - Expression::Simple(Value::Selector( - make_selector!(make_expr!("var1") => "lvl1", "lvl2"), - )), - Val::Int(3), - ), - ( - Expression::Simple(Value::Selector(make_selector!(make_expr!("var2")))), - Val::Int(2), - ), - ( - Expression::Simple(Value::Selector( - make_selector!(make_expr!("var3") => "lvl1"), - )), - Val::Int(4), - ), - ], - b, - ); -} - -#[test] -fn test_eval_selector_list_expr() { - let cache = Rc::new(RefCell::new(MemoryCache::new())); - let mut b = Builder::new(std::env::current_dir().unwrap(), cache); - b.build_output - .entry(value_node!("var1".to_string(), 1, 1)) - .or_insert(Rc::new(Val::List(vec![ - Rc::new(Val::Str("val1".to_string())), - Rc::new(Val::Tuple(vec![( - value_node!("var2".to_string(), 1, 1), - Rc::new(Val::Int(1)), - )])), - ]))); - - test_expr_to_val( - vec![( - Expression::Simple(Value::Selector( - make_selector!(make_expr!("var1") => "0" => 1, 1), - )), - Val::Str("val1".to_string()), - )], - b, - ); -} - // Include nested for each. #[test] #[should_panic(expected = "Unable to find tpl1")] @@ -508,9 +147,12 @@ fn test_expr_copy_no_such_tuple() { test_expr_to_val( vec![( Expression::Copy(CopyDef { - selector: make_selector!(make_expr!("tpl1")), + selector: make_selector!( + make_expr!("tpl1", Position::new(1, 1, 1)), + Position::new(1, 1, 1) + ), fields: Vec::new(), - pos: Position::new(1, 0), + pos: Position::new(1, 0, 0), }), Val::Tuple(Vec::new()), )], @@ -524,14 +166,17 @@ fn test_expr_copy_not_a_tuple() { let cache = Rc::new(RefCell::new(MemoryCache::new())); let mut b = Builder::new(std::env::current_dir().unwrap(), cache); b.build_output - .entry(value_node!("tpl1".to_string(), 1, 0)) + .entry(value_node!("tpl1".to_string(), Position::new(1, 0, 0))) .or_insert(Rc::new(Val::Int(1))); test_expr_to_val( vec![( Expression::Copy(CopyDef { - selector: make_selector!(make_expr!("tpl1")), + selector: make_selector!( + make_expr!("tpl1", Position::new(1, 1, 1)), + Position::new(1, 1, 1) + ), fields: Vec::new(), - pos: Position::new(1, 0), + pos: Position::new(1, 0, 0), }), Val::Tuple(Vec::new()), )], @@ -545,130 +190,30 @@ fn test_expr_copy_field_type_error() { let cache = Rc::new(RefCell::new(MemoryCache::new())); let mut b = Builder::new(std::env::current_dir().unwrap(), cache); b.build_output - .entry(value_node!("tpl1".to_string(), 1, 0)) + .entry(value_node!("tpl1".to_string(), Position::new(1, 0, 0))) .or_insert(Rc::new(Val::Tuple(vec![( - value_node!("fld1".to_string(), 1, 0), + value_node!("fld1".to_string(), Position::new(1, 0, 0)), Rc::new(Val::Int(1)), )]))); test_expr_to_val( vec![( Expression::Copy(CopyDef { - selector: make_selector!(make_expr!("tpl1")), - fields: vec![( - make_tok!("fld1", 1, 1), - Expression::Simple(Value::Str(value_node!("2".to_string(), 1, 1))), - )], - pos: Position::new(1, 0), - }), - Val::Tuple(vec![( - value_node!("fld1".to_string(), 1, 1), - Rc::new(Val::Str("2".to_string())), - )]), - )], - b, - ); -} - -#[test] -fn test_expr_copy() { - let cache = Rc::new(RefCell::new(MemoryCache::new())); - let mut b = Builder::new(std::env::current_dir().unwrap(), cache); - b.build_output - .entry(value_node!("tpl1".to_string(), 1, 0)) - .or_insert(Rc::new(Val::Tuple(vec![( - value_node!("fld1".to_string(), 1, 0), - Rc::new(Val::Int(1)), - )]))); - test_expr_to_val( - vec![ - ( - Expression::Copy(CopyDef { - selector: make_selector!(make_expr!("tpl1")), - fields: vec![( - make_tok!("fld2", 1, 1), - Expression::Simple(Value::Str(value_node!("2".to_string(), 1, 1))), - )], - pos: Position::new(1, 0), - }), - // Add a new field to the copy - Val::Tuple( - // NOTE(jwall): The order of these is important in order to ensure - // that the compare assertion is correct. The ordering has no - // semantics though so at some point we should probably be less restrictive. - vec![ - (value_node!("fld1".to_string(), 1, 0), Rc::new(Val::Int(1))), - ( - value_node!("fld2".to_string(), 1, 1), - Rc::new(Val::Str("2".to_string())), - ), - ], + selector: make_selector!( + make_expr!("tpl1", Position::new(1, 1, 1)), + Position::new(1, 1, 1) ), - ), - // Overwrite a field in the copy - ( - Expression::Copy(CopyDef { - selector: make_selector!(make_expr!("tpl1")), - fields: vec![ - ( - make_tok!("fld1", 1, 1), - Expression::Simple(Value::Int(value_node!(3, 1, 1))), - ), - ( - make_tok!("fld2", 1, 1), - Expression::Simple(Value::Str(value_node!("2".to_string(), 1, 1))), - ), - ], - pos: Position::new(1, 0), - }), - Val::Tuple(vec![ - (value_node!("fld1".to_string(), 1, 0), Rc::new(Val::Int(3))), - ( - value_node!("fld2".to_string(), 1, 0), - Rc::new(Val::Str("2".to_string())), - ), - ]), - ), - // The source tuple is still unmodified. - ( - Expression::Simple(Value::Selector(make_selector!(make_expr!["tpl1"]))), - Val::Tuple(vec![( - value_node!("fld1".to_string(), 1, 0), - Rc::new(Val::Int(1)), - )]), - ), - ], - b, - ); -} - -#[test] -fn test_macro_call() { - let cache = Rc::new(RefCell::new(MemoryCache::new())); - let mut b = Builder::new(std::env::current_dir().unwrap(), cache); - b.build_output - .entry(value_node!("tstmac".to_string(), 1, 0)) - .or_insert(Rc::new(Val::Macro(MacroDef { - argdefs: vec![value_node!("arg1".to_string(), 1, 0)], - fields: vec![( - make_tok!("foo", 1, 1), - Expression::Simple(Value::Symbol(value_node!("arg1".to_string(), 1, 1))), - )], - pos: Position::new(1, 0), - }))); - test_expr_to_val( - vec![( - Expression::Call(CallDef { - macroref: make_selector!(make_expr!("tstmac")), - arglist: vec![Expression::Simple(Value::Str(value_node!( - "bar".to_string(), - 1, - 1 - )))], - pos: Position::new(1, 0), + fields: vec![( + make_tok!("fld1", Position::new(1, 1, 1)), + Expression::Simple(Value::Str(value_node!( + "2".to_string(), + Position::new(1, 1, 1) + ))), + )], + pos: Position::new(1, 0, 0), }), Val::Tuple(vec![( - value_node!("foo".to_string(), 1, 1), - Rc::new(Val::Str("bar".to_string())), + value_node!("fld1".to_string(), Position::new(1, 1, 1)), + Rc::new(Val::Str("2".to_string())), )]), )], b, @@ -681,31 +226,36 @@ fn test_macro_hermetic() { let cache = Rc::new(RefCell::new(MemoryCache::new())); let mut b = Builder::new(std::env::current_dir().unwrap(), cache); b.build_output - .entry(value_node!("arg1".to_string(), 1, 0)) + .entry(value_node!("arg1".to_string(), Position::new(1, 0, 0))) .or_insert(Rc::new(Val::Str("bar".to_string()))); b.build_output - .entry(value_node!("tstmac".to_string(), 1, 0)) + .entry(value_node!("tstmac".to_string(), Position::new(1, 0, 0))) .or_insert(Rc::new(Val::Macro(MacroDef { - argdefs: vec![value_node!("arg2".to_string(), 1, 0)], + argdefs: vec![value_node!("arg2".to_string(), Position::new(1, 0, 0))], fields: vec![( - make_tok!("foo", 1, 1), - Expression::Simple(Value::Symbol(value_node!("arg1".to_string(), 1, 1))), + make_tok!("foo", Position::new(1, 1, 1)), + Expression::Simple(Value::Symbol(value_node!( + "arg1".to_string(), + Position::new(1, 1, 1) + ))), )], - pos: Position::new(1, 0), + pos: Position::new(1, 0, 0), }))); test_expr_to_val( vec![( Expression::Call(CallDef { - macroref: make_selector!(make_expr!("tstmac")), + macroref: make_selector!( + make_expr!("tstmac", Position::new(1, 1, 1)), + Position::new(1, 1, 1) + ), arglist: vec![Expression::Simple(Value::Str(value_node!( "bar".to_string(), - 1, - 1 + Position::new(1, 1, 1) )))], - pos: Position::new(1, 1), + pos: Position::new(1, 1, 1), }), Val::Tuple(vec![( - value_node!("foo".to_string(), 1, 0), + value_node!("foo".to_string(), Position::new(1, 1, 1)), Rc::new(Val::Str("bar".to_string())), )]), )], @@ -713,126 +263,42 @@ fn test_macro_hermetic() { ); } -#[test] -fn test_select_expr() { - let cache = Rc::new(RefCell::new(MemoryCache::new())); - let mut b = Builder::new(std::env::current_dir().unwrap(), cache); - b.build_output - .entry(value_node!("foo".to_string(), 1, 0)) - .or_insert(Rc::new(Val::Str("bar".to_string()))); - b.build_output - .entry(value_node!("baz".to_string(), 1, 0)) - .or_insert(Rc::new(Val::Str("boo".to_string()))); - test_expr_to_val( - vec![ - ( - Expression::Select(SelectDef { - val: Box::new(Expression::Simple(Value::Symbol(value_node!( - "foo".to_string(), - 1, - 1 - )))), - default: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))), - tuple: vec![ - ( - make_tok!("foo", 1, 1), - Expression::Simple(Value::Str(value_node!("2".to_string(), 1, 1))), - ), - ( - make_tok!("bar", 1, 1), - Expression::Simple(Value::Int(value_node!(2, 1, 1))), - ), - ], - pos: Position::new(1, 0), - }), - Val::Int(2), - ), - ( - Expression::Select(SelectDef { - val: Box::new(Expression::Simple(Value::Symbol(value_node!( - "baz".to_string(), - 1, - 1 - )))), - default: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))), - tuple: vec![ - ( - make_tok!("bar", 1, 1), - Expression::Simple(Value::Int(value_node!(2, 1, 1))), - ), - ( - make_tok!("quux", 1, 1), - Expression::Simple(Value::Str(value_node!("2".to_string(), 1, 1))), - ), - ], - pos: Position::new(1, 0), - }), - // If the field doesn't exist then we get the default. - Val::Int(1), - ), - ], - b, - ); -} - #[test] #[should_panic(expected = "Expected String but got Integer in Select expression")] fn test_select_expr_not_a_string() { let cache = Rc::new(RefCell::new(MemoryCache::new())); let mut b = Builder::new(std::env::current_dir().unwrap(), cache); b.build_output - .entry(value_node!("foo".to_string(), 1, 0)) + .entry(value_node!("foo".to_string(), Position::new(1, 0, 0))) .or_insert(Rc::new(Val::Int(4))); test_expr_to_val( vec![( Expression::Select(SelectDef { val: Box::new(Expression::Simple(Value::Symbol(value_node!( "foo".to_string(), - 1, - 1 + Position::new(1, 1, 1) + )))), + default: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 1, 1) )))), - default: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))), tuple: vec![ ( - make_tok!("bar", 1, 1), - Expression::Simple(Value::Int(value_node!(2, 1, 1))), + make_tok!("bar", Position::new(1, 1, 1)), + Expression::Simple(Value::Int(value_node!(2, Position::new(1, 1, 1)))), ), ( - make_tok!("quux", 1, 1), - Expression::Simple(Value::Str(value_node!("2".to_string(), 1, 1))), + make_tok!("quux", Position::new(1, 1, 1)), + Expression::Simple(Value::Str(value_node!( + "2".to_string(), + Position::new(1, 1, 1) + ))), ), ], - pos: Position::new(1, 0), + pos: Position::new(1, 0, 0), }), Val::Int(2), )], b, ); } - -#[test] -fn test_let_statement() { - let cache = MemoryCache::new(); - let mut b = Builder::new("", Rc::new(RefCell::new(cache))); - let stmt = Statement::Let(LetDef { - name: make_tok!("foo", 1, 1), - value: Expression::Simple(Value::Str(value_node!("bar".to_string(), 1, 1))), - }); - b.build_stmt(&stmt).unwrap(); - test_expr_to_val( - vec![( - Expression::Simple(Value::Symbol(value_node!("foo".to_string(), 1, 1))), - Val::Str("bar".to_string()), - )], - b, - ); -} - -#[test] -fn test_build_file_string() { - let cache = Rc::new(RefCell::new(MemoryCache::new())); - let mut b = Builder::new(std::env::current_dir().unwrap(), cache); - b.eval_string("let foo = 1;").unwrap(); - let key = value_node!("foo".to_string(), 1, 0); - assert!(b.build_output.contains_key(&key)); -} diff --git a/src/convert/env.rs b/src/convert/env.rs index b4eda68..bd25792 100644 --- a/src/convert/env.rs +++ b/src/convert/env.rs @@ -16,7 +16,7 @@ use std::io::Write; use std::rc::Rc; -use ast::Positioned; +use ast::PositionedItem; use build::Val; use convert::traits::{Converter, Result}; @@ -29,7 +29,11 @@ impl EnvConverter { EnvConverter {} } - fn convert_tuple(&self, flds: &Vec<(Positioned, Rc)>, w: &mut Write) -> Result { + fn convert_tuple( + &self, + flds: &Vec<(PositionedItem, Rc)>, + w: &mut Write, + ) -> Result { for &(ref name, ref val) in flds.iter() { if val.is_tuple() { eprintln!("Skipping embedded tuple..."); diff --git a/src/convert/exec.rs b/src/convert/exec.rs index 6d23fa2..9c7efbc 100644 --- a/src/convert/exec.rs +++ b/src/convert/exec.rs @@ -17,7 +17,7 @@ use std; use std::io::{Cursor, Write}; use std::rc::Rc; -use ast::{Position, Positioned}; +use ast::{Position, PositionedItem}; use build::Val; use build::Val::Tuple; use convert; @@ -46,10 +46,10 @@ impl ExecConverter { return Err(Box::new(Error::new( "Exec tuples must have no more than 3 fields", ErrorType::TypeFail, - Position::new(0, 0), + Position::new(0, 0, 0), ))); } - let mut env: Option<&Vec<(Positioned, Rc)>> = None; + let mut env: Option<&Vec<(PositionedItem, Rc)>> = None; let mut command: Option<&str> = None; let mut args: Option<&Vec>> = None; for &(ref name, ref val) in fields.iter() { @@ -115,7 +115,7 @@ impl ExecConverter { return Err(Box::new(Error::new( "An exec tuple must have a command field", ErrorType::TypeFail, - Position::new(0, 0), + Position::new(0, 0, 0), ))); } // Okay if we have made it this far then we are ready to start creating our script. @@ -157,7 +157,7 @@ impl ExecConverter { return Err(Box::new(Error::new( "Exec args must be a list of strings or tuples of strings.", ErrorType::TypeFail, - Position::new(0, 0), + Position::new(0, 0, 0), ))) } } @@ -173,7 +173,7 @@ impl ExecConverter { Err(Box::new(Error::new( "Exec outputs must be of type Tuple", ErrorType::TypeFail, - Position::new(0, 0), + Position::new(0, 0, 0), ))) } } diff --git a/src/convert/json.rs b/src/convert/json.rs index efa8dbb..8e50c24 100644 --- a/src/convert/json.rs +++ b/src/convert/json.rs @@ -36,7 +36,7 @@ impl JsonConverter { fn convert_tuple( &self, - items: &Vec<(ast::Positioned, Rc)>, + items: &Vec<(ast::PositionedItem, Rc)>, ) -> std::io::Result { let mut mp = serde_json::Map::new(); for &(ref k, ref v) in items.iter() { diff --git a/src/convert/yaml.rs b/src/convert/yaml.rs index 28ca3bf..a98927b 100644 --- a/src/convert/yaml.rs +++ b/src/convert/yaml.rs @@ -25,7 +25,7 @@ impl YamlConverter { fn convert_tuple( &self, - items: &Vec<(ast::Positioned, Rc)>, + items: &Vec<(ast::PositionedItem, Rc)>, ) -> std::io::Result { let mut mapping = serde_yaml::Mapping::new(); for &(ref k, ref v) in items.iter() { diff --git a/src/error.rs b/src/error.rs index 77ff70a..e50e012 100644 --- a/src/error.rs +++ b/src/error.rs @@ -15,11 +15,12 @@ //! Errors for use by the ucg compiler. use std::error; use std::fmt; +use std::fmt::Debug; + +use abortable_parser::Positioned; use ast::*; -use nom; - /// ErrorType defines the various types of errors that can result from compiling UCG into an /// output format. pub enum ErrorType { @@ -62,7 +63,6 @@ pub struct Error { pub err_type: ErrorType, pub pos: Position, pub msg: String, - pub cause: Option>, _pkgonly: (), } @@ -72,51 +72,21 @@ impl Error { err_type: t, pos: pos, msg: msg.into(), - cause: None, _pkgonly: (), } } - pub fn new_with_boxed_cause>(msg: S, t: ErrorType, cause: Box) -> Self { - let mut e = Self::new(msg, t, cause.pos.clone()); - e.cause = Some(cause); - return e; - } - - pub fn new_with_cause>(msg: S, t: ErrorType, cause: Self) -> Self { - Self::new_with_boxed_cause(msg, t, Box::new(cause)) - } - - pub fn new_with_errorkind>( - msg: S, - t: ErrorType, - pos: Position, - cause: nom::ErrorKind, - ) -> Self { - match cause { - nom::ErrorKind::Custom(e) => Self::new_with_cause(msg, t, e), - e => Self::new_with_cause( - msg, - t, - Error::new(format!("ErrorKind: {}", e), ErrorType::Unsupported, pos), - ), - } - } - fn render(&self, w: &mut fmt::Formatter) -> fmt::Result { try!(write!( w, - "{}: \"{}\" at line: {} column: {}", - self.err_type, self.msg, self.pos.line, self.pos.column + "{} at line: {} column: {}\nCaused By:\n\t{} ", + self.err_type, self.pos.line, self.pos.column, self.msg )); - if let Some(ref cause) = self.cause { - try!(write!(w, "\n\tCaused By: {}", cause)); - } Ok(()) } } -impl fmt::Debug for Error { +impl Debug for Error { fn fmt(&self, w: &mut fmt::Formatter) -> fmt::Result { self.render(w) } @@ -133,3 +103,50 @@ impl error::Error for Error { &self.msg } } + +#[derive(Debug)] +pub struct StackPrinter { + pub err: abortable_parser::Error, +} + +impl StackPrinter +where + C: abortable_parser::Positioned, +{ + pub fn render(&self, w: &mut fmt::Formatter) -> fmt::Result { + let mut curr_err = Some(&self.err); + let mut tabstop = ""; + loop { + match curr_err { + // our exit condition; + None => break, + Some(err) => { + let context = err.get_context(); + try!(write!( + w, + "{}{}: line: {}, column: {}\n", + tabstop, + err.get_msg(), + context.line(), + context.column(), + )); + tabstop = "\t"; + curr_err = err.get_cause(); + if curr_err.is_some() { + try!(write!(w, "Caused by: \n")); + } + } + } + } + Ok(()) + } +} + +impl fmt::Display for StackPrinter +where + C: Positioned, +{ + fn fmt(&self, w: &mut fmt::Formatter) -> fmt::Result { + self.render(w) + } +} diff --git a/src/format.rs b/src/format.rs index 4f8f627..edda478 100644 --- a/src/format.rs +++ b/src/format.rs @@ -82,21 +82,21 @@ mod test { #[test] fn test_format_happy_path() { let formatter = Formatter::new("foo @ @ \\@", vec!["bar", "quux"]); - let pos = Position { line: 0, column: 0 }; + let pos = Position::new(0, 0, 0); assert_eq!(formatter.render(&pos).unwrap(), "foo bar quux @"); } #[test] fn test_format_happy_wrong_too_few_args() { let formatter = Formatter::new("foo @ @ \\@", vec!["bar"]); - let pos = Position { line: 0, column: 0 }; + let pos = Position::new(0, 0, 0); assert!(formatter.render(&pos).is_err()); } #[test] fn test_format_happy_wrong_too_many_args() { let formatter = Formatter::new("foo @ @ \\@", vec!["bar", "quux", "baz"]); - let pos = Position { line: 0, column: 0 }; + let pos = Position::new(0, 0, 0); assert!(formatter.render(&pos).is_err()); } } diff --git a/src/iter.rs b/src/iter.rs new file mode 100644 index 0000000..d37eb5a --- /dev/null +++ b/src/iter.rs @@ -0,0 +1,107 @@ +use std::convert::From; +use std::iter::Iterator; + +use abortable_parser::iter::{SliceIter, StrIter}; +use abortable_parser::{InputIter, Offsetable, Peekable, Positioned, Seekable, Span, SpanRange}; + +use ast::{Position, Token}; + +#[derive(Debug)] +pub struct OffsetStrIter<'a> { + contained: StrIter<'a>, + line_offset: usize, + col_offset: usize, +} + +impl<'a> OffsetStrIter<'a> { + pub fn new(input: &'a str) -> Self { + Self::new_with_offsets(input, 0, 0) + } + + pub fn new_with_offsets(input: &'a str, line_offset: usize, col_offset: usize) -> Self { + OffsetStrIter { + contained: StrIter::new(input), + line_offset: line_offset, + col_offset: col_offset, + } + } +} + +impl<'a> Iterator for OffsetStrIter<'a> { + type Item = &'a u8; + + fn next(&mut self) -> Option { + self.contained.next() + } +} + +impl<'a> Offsetable for OffsetStrIter<'a> { + fn get_offset(&self) -> usize { + self.contained.get_offset() + } +} + +impl<'a> Clone for OffsetStrIter<'a> { + fn clone(&self) -> Self { + OffsetStrIter { + contained: self.contained.clone(), + line_offset: self.line_offset, + col_offset: self.col_offset, + } + } +} + +impl<'a> From<&'a str> for OffsetStrIter<'a> { + fn from(source: &'a str) -> Self { + OffsetStrIter { + contained: StrIter::new(source), + line_offset: 0, + col_offset: 0, + } + } +} + +impl<'a> Seekable for OffsetStrIter<'a> { + fn seek(&mut self, to: usize) -> usize { + self.contained.seek(to) + } +} + +impl<'a> Span<&'a str> for OffsetStrIter<'a> { + fn span(&self, idx: SpanRange) -> &'a str { + self.contained.span(idx) + } +} + +impl<'a> Peekable<&'a u8> for OffsetStrIter<'a> { + fn peek_next(&self) -> Option<&'a u8> { + self.contained.peek_next() + } +} + +impl<'a> Positioned for OffsetStrIter<'a> { + fn line(&self) -> usize { + self.contained.line() + self.line_offset + } + + fn column(&self) -> usize { + self.contained.column() + self.col_offset + } +} + +impl<'a> InputIter for OffsetStrIter<'a> {} + +impl<'a> From<&'a SliceIter<'a, Token>> for Position { + fn from(source: &'a SliceIter<'a, Token>) -> Self { + match source.peek_next() { + Some(t) => t.pos.clone(), + None => Position::new(0, 0, 0), + } + } +} + +impl<'a> From<&'a OffsetStrIter<'a>> for Position { + fn from(s: &'a OffsetStrIter<'a>) -> Position { + Position::new(s.line(), s.column(), s.get_offset()) + } +} diff --git a/src/lib.rs b/src/lib.rs index 43e8ee8..ec9ec10 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -444,9 +444,7 @@ // to succeed. #![recursion_limit = "128"] #[macro_use] -extern crate nom; -#[macro_use] -extern crate nom_locate; +extern crate abortable_parser; extern crate serde_json; extern crate serde_yaml; extern crate simple_error; @@ -458,6 +456,7 @@ pub mod tokenizer; pub mod build; pub mod convert; pub mod error; +pub mod iter; pub mod parse; mod format; diff --git a/src/parse/mod.rs b/src/parse/mod.rs index 1bd0c47..a65bf13 100644 --- a/src/parse/mod.rs +++ b/src/parse/mod.rs @@ -13,46 +13,30 @@ // limitations under the License. //! The Parsing stage of the ucg compiler. +use std; use std::borrow::Borrow; use std::str::FromStr; -use nom; -use nom::IResult; -use nom::InputLength; -use nom_locate::LocatedSpan; +use abortable_parser; +use abortable_parser::combinators::eoi; +use abortable_parser::iter::SliceIter; +use abortable_parser::{Error, Peekable, Result}; use self::precedence::op_expression; use ast::*; -use error; +use error::StackPrinter; +use iter::OffsetStrIter; use tokenizer::*; -type NomResult<'a, O> = nom::IResult, O, error::Error>; +// TODO(jwall): Rename this to something better. +type NomResult<'a, O> = Result, O>; #[cfg(feature = "tracing")] const ENABLE_TRACE: bool = true; #[cfg(not(feature = "tracing"))] const ENABLE_TRACE: bool = false; -type ParseResult = Result; - -macro_rules! wrap_err { - ($i:expr, $submac:ident, $msg:expr) => { - wrap_err!($i, call!($submac), $msg) - }; - - ($i:expr, $submac:ident!( $($args:tt)* ), $msg:expr) => {{ - let _i = $i.clone(); - match $submac!(_i, $($args)*) { - IResult::Done(rest, mac) => IResult::Done(rest, mac), - IResult::Incomplete(i) => IResult::Incomplete(i), - IResult::Error(nom::ErrorKind::Custom(cause)) => { - let wrapper = error::Error::new_with_cause($msg, error::ErrorType::ParseError, cause); - IResult::Error(nom::ErrorKind::Custom(wrapper)) - } - IResult::Error(e) => IResult::Error(e), - } - }}; -} +type ParseResult<'a, O> = std::result::Result>>; macro_rules! trace_nom { ($i:expr, $rule:ident!( $($args:tt)* )) => { @@ -61,7 +45,7 @@ macro_rules! trace_nom { if ENABLE_TRACE { eprintln!("Entering Rule: {:?} {:?}", stringify!($rule), $i); } - let result = $rule($i, $($args)* ); + let result = $rule!($i, $($args)* ); if ENABLE_TRACE { eprintln!("Exiting Rule: {:?} with {:?}", stringify!($rule), result); } @@ -75,7 +59,7 @@ macro_rules! trace_nom { if ENABLE_TRACE { eprintln!("Entering Rule: {:?} {:?}", stringify!($rule), $i); } - let result = call!($i, $rule); + let result = run!($i, $rule); if ENABLE_TRACE { eprintln!("Exiting Rule: {:?} with {:?}", stringify!($rule), result); } @@ -92,7 +76,8 @@ fn symbol_to_value(s: &Token) -> ParseResult { } // symbol is a bare unquoted field. -named!(symbol, +make_fn!( + symbol, Value>, match_type!(BAREWORD => symbol_to_value) ); @@ -104,14 +89,18 @@ fn str_to_value(s: &Token) -> ParseResult { } // quoted_value is a quoted string. -named!(quoted_value, - match_type!(STR => str_to_value) +make_fn!( + quoted_value, Value>, + match_type!(STR => str_to_value) ); // Helper function to make the return types work for down below. -fn triple_to_number(v: (Option, Option, Option)) -> ParseResult { +fn triple_to_number<'a>( + input: SliceIter<'a, Token>, + v: (Option, Option, Option), +) -> ParseResult<'a, Value> { let (pref, mut pref_pos) = match v.0 { - None => ("", Position::new(0, 0)), + None => ("", Position::new(0, 0, 0)), Some(ref bs) => (bs.fragment.borrow(), bs.pos.clone()), }; @@ -121,11 +110,10 @@ fn triple_to_number(v: (Option, Option, Option)) -> ParseRe let i = match FromStr::from_str(pref) { Ok(i) => i, Err(_) => { - return Err(error::Error::new( + return Err(Error::new( format!("Not an integer! {}", pref), - error::ErrorType::UnexpectedToken, - pref_pos, - )) + Box::new(input.clone()), + )); } }; return Ok(Value::Int(value_node!(i, pref_pos))); @@ -135,35 +123,34 @@ fn triple_to_number(v: (Option, Option, Option)) -> ParseRe pref_pos = v.1.unwrap().pos; } - let (maybepos, suf) = match v.2 { - None => (None, "".to_string()), - Some(bs) => (Some(bs.pos), bs.fragment), + let suf = match v.2 { + None => "".to_string(), + Some(bs) => bs.fragment, }; let to_parse = pref.to_string() + "." + &suf; let f = match FromStr::from_str(&to_parse) { Ok(f) => f, Err(_) => { - return Err(error::Error::new( + return Err(Error::new( format!("Not a float! {}", to_parse), - error::ErrorType::UnexpectedToken, - // NOTE(jwall): This is ugly. I should probably see if I can refactor - // it to something less confusing. - maybepos.unwrap(), + Box::new(input.clone()), )); } }; return Ok(Value::Float(value_node!(f, pref_pos))); } +// FIXME(jwall): This should actually be unnecessary now. + /// alt_peek conditionally runs a combinator if a lookahead combinator matches. macro_rules! alt_peek { (__inner $i:expr, $peekrule:ident!( $($peekargs:tt)* ) => $parserule:ident | $($rest:tt)* ) => ( - alt_peek!(__inner $i, $peekrule!($($peekargs)*) => call!($parserule) | $($rest)* ) + alt_peek!(__inner $i, $peekrule!($($peekargs)*) => run!($parserule) | $($rest)* ) ); (__inner $i:expr, $peekrule:ident => $($rest:tt)* ) => ( - alt_peek!(__inner $i, call!($peekrule) => $($rest)* ) + alt_peek!(__inner $i, run!($peekrule) => $($rest)* ) ); (__inner $i:expr, $peekrule:ident!( $($peekargs:tt)* ) => $parserule:ident!( $($parseargs:tt)* ) | $($rest:tt)* ) => ( @@ -172,14 +159,15 @@ macro_rules! alt_peek { let pre_res = peek!(_i, $peekrule!($($peekargs)*)); match pre_res { // if the peek was incomplete then it might still match so return incomplete. - nom::IResult::Incomplete(i) => nom::IResult::Incomplete(i), + Result::Incomplete(i) => Result::Incomplete(i), // If the peek was in error then try the next peek => parse pair. - nom::IResult::Error(_) => { + Result::Fail(_) => { alt_peek!(__inner $i, $($rest)*) }, + Result::Abort(e) => Result::Abort(e), // If the peek was successful then return the result of the parserule // regardless of it's result. - nom::IResult::Done(_i, _) => { + Result::Complete(_i, _) => { $parserule!(_i, $($parseargs)*) }, } @@ -197,14 +185,15 @@ macro_rules! alt_peek { let pre_res = peek!(_i, $peekrule!($($peekargs)*)); match pre_res { // if the peek was incomplete then it might still match so return incomplete. - nom::IResult::Incomplete(i) => nom::IResult::Incomplete(i), + Result::Incomplete(i) => Result::Incomplete(i), // If the peek was in error then try the next peek => parse pair. - nom::IResult::Error(_) => { + Result::Fail(_) => { alt_peek!(__inner $i, __end) }, + Result::Abort(e) => Result::Abort(e), // If the peek was successful then return the result of the parserule // regardless of it's result. - nom::IResult::Done(_i, _) => { + Result::Complete(_i, _) => { $parserule!(_i, $($parseargs)*) }, } @@ -215,7 +204,7 @@ macro_rules! alt_peek { (__inner $i:expr, $fallback:ident, __end) => ( { let _i = $i.clone(); - call!(_i, $fallback) + run!(_i, $fallback) } ); // In the case of a fallback rule with no peek we just return whatever @@ -230,8 +219,7 @@ macro_rules! alt_peek { // This is our default termination case. // If there is no fallback then we return an Error. (__inner $i:expr, __end) => { - // TODO(jwall): We should do a better custom error here. - nom::IResult::Error(error_position!(nom::ErrorKind::Alt,$i)) + compile_error!("alt_peek! requirs a fallback case"); }; // alt_peek entry_point. @@ -246,234 +234,254 @@ macro_rules! alt_peek { // NOTE(jwall): HERE THERE BE DRAGONS. The order for these matters // alot. We need to process alternatives in order of decreasing // specificity. Unfortunately this means we are required to go in a -// decreasing size order which messes with alt!'s completion logic. To +// decreasing size order which messes with either!'s completion logic. To // work around this we have to force Incomplete to be Error so that -// alt! will try the next in the series instead of aborting. +// either! will try the next in the series instead of aborting. // // *IMPORTANT* // It also means this combinator is risky when used with partial // inputs. So handle with care. -named!(number, - map_res!(alt!( - complete!(do_parse!( // 1.0 - prefix: match_type!(DIGIT) >> - has_dot: punct!(".") >> - suffix: match_type!(DIGIT) >> - (Some(prefix.clone()), Some(has_dot.clone()), Some(suffix.clone())) - )) | - complete!(do_parse!( // 1. - prefix: match_type!(DIGIT) >> - has_dot: punct!(".") >> - (Some(prefix.clone()), Some(has_dot.clone()), None) - )) | - complete!(do_parse!( // .1 - has_dot: punct!(".") >> - suffix: match_type!(DIGIT) >> - (None, Some(has_dot.clone()), Some(suffix.clone())) - )) | - do_parse!( // 1 - prefix: match_type!(DIGIT) >> - (Some(prefix.clone()), None, None) - )), - triple_to_number - ) -); +fn number(input: SliceIter) -> Result, Value> { + let parsed = do_each!(input, + num => either!( + complete!( + "Not a float", + do_each!( // 1.0 + prefix => match_type!(DIGIT), + has_dot => punct!("."), + suffix => match_type!(DIGIT), + (Some(prefix.clone()), Some(has_dot.clone()), Some(suffix.clone())) + )), + complete!( + "Not a float", + do_each!( // 1. + prefix => match_type!(DIGIT), + has_dot => punct!("."), + (Some(prefix.clone()), Some(has_dot.clone()), None) + )), + complete!( + "Not a float", + do_each!( // .1 + has_dot => punct!("."), + suffix => match_type!(DIGIT), + (None, Some(has_dot.clone()), Some(suffix.clone())) + )), + do_each!( // 1 + prefix => match_type!(DIGIT), + (Some(prefix.clone()), None, None) + )), + (num) + ); + match parsed { + Result::Abort(e) => Result::Abort(e), + Result::Fail(e) => Result::Fail(e), + Result::Incomplete(offset) => Result::Incomplete(offset), + Result::Complete(rest, triple) => { + let num = triple_to_number(rest.clone(), triple); + match num { + Ok(val) => Result::Complete(rest, val), + Err(e) => Result::Fail(e), + } + } + } +} // trace_macros!(false); -named!(boolean_value, - do_parse!( - b: match_type!(BOOLEAN) >> - (Value::Boolean(Positioned{ +make_fn!( + boolean_value, Value>, + do_each!( + b => match_type!(BOOLEAN), + (Value::Boolean(PositionedItem{ val: b.fragment == "true", pos: b.pos, })) ) ); -named!( - field_value, - do_parse!( - field: wrap_err!(alt!(match_type!(BAREWORD) | match_type!(STR)), - "Field names must be a bareword or a string.") >> - punct!("=") >> - value: expression >> +make_fn!( + field_value, (Token, Expression)>, + do_each!( + field => wrap_err!(either!(match_type!(BAREWORD), match_type!(STR)), + "Field names must be a bareword or a string."), + _ => punct!("="), + value => expression, (field, value) ) ); // Helper function to make the return types work for down below. -fn vec_to_tuple(t: (Position, Option)) -> ParseResult { - Ok(Value::Tuple(value_node!( - t.1.unwrap_or(Vec::new()), - t.0.line as usize, - t.0.column as usize - ))) +fn vec_to_tuple(pos: Position, fields: Option) -> Value { + Value::Tuple(value_node!(fields.unwrap_or(Vec::new()), pos)) } -named!(field_list, - separated_list!(punct!(","), field_value) +make_fn!( + field_list, FieldList>, + separated!(punct!(","), field_value) ); -named!( - tuple, - map_res!( - do_parse!( - pos: pos >> - punct!("{") >> - v: field_list >> - opt_res!(punct!(",")) >> // nom's opt! macro doesn't preserve error types properly but this one does. - punct!("}") >> - (pos, Some(v)) - ), - vec_to_tuple +make_fn!( + tuple, Value>, + do_each!( + pos => pos, + _ => punct!("{"), + v => optional!(field_list), + _ => optional!(punct!(",")), + _ => punct!("}"), + (vec_to_tuple(pos, v)) ) ); -fn tuple_to_list>(t: (Sp, Vec)) -> ParseResult { - return Ok(Value::List(ListDef { - elems: t.1, - pos: t.0.into(), - })); +fn tuple_to_list>(pos: Sp, elems: Vec) -> Value { + Value::List(ListDef { + elems: elems, + pos: pos.into(), + }) } -named!(list_value, - map_res!( - do_parse!( - start: punct!("[") >> - elements: separated_list!(punct!(","), expression) >> - opt_res!(punct!(",")) >> // nom's opt! macro doesn't preserve error types properly but this one does. - punct!("]") >> - (start.pos, elements) - ), - tuple_to_list - ) -); - -named!(empty_value, - do_parse!( - pos: pos >> - match_type!(EMPTY) >> - (Value::Empty(pos)) +make_fn!( + list_value, Value>, + do_each!( + start => punct!("["), + elements => separated!(punct!(","), expression), + _ => optional!(punct!(",")), // nom's opt! macro doesn't preserve error types properly but this one does. + _ => punct!("]"), + (tuple_to_list(start.pos, elements)) ) ); -named!(compound_value, - alt_peek!( - punct!("[") => trace_nom!(list_value) | - punct!("{") => trace_nom!(tuple) +make_fn!( + empty_value, Value>, + do_each!( + pos => pos, + _ => match_type!(EMPTY), + (Value::Empty(pos.into())) ) ); -named!(value, +make_fn!( + compound_value, Value>, + either!(trace_nom!(list_value), trace_nom!(tuple)) +); + +make_fn!( + value, Value>, alt_peek!( symbol_or_expression => trace_nom!(selector_value) - | alt!(punct!("[") | punct!("{")) => trace_nom!(compound_value) + | either!(punct!("["), punct!("{")) => trace_nom!(compound_value) | match_type!(BOOLEAN) => trace_nom!(boolean_value) | match_type!(EMPTY) => trace_nom!(empty_value) - | alt!(match_type!(DIGIT) | punct!(".")) => trace_nom!(number) + | either!(match_type!(DIGIT), punct!(".")) => trace_nom!(number) | trace_nom!(quoted_value) ) - ); - -fn value_to_expression(v: Value) -> ParseResult { - Ok(Expression::Simple(v)) -} - -named!(simple_expression, - map_res!( - trace_nom!(value), - value_to_expression - ) ); -fn expression_to_grouped_expression(e: Expression) -> ParseResult { - Ok(Expression::Grouped(Box::new(e))) +fn value_to_expression(v: Value) -> Expression { + Expression::Simple(v) } -named!(grouped_expression, - map_res!( - preceded!(punct!("("), terminated!(trace_nom!(expression), punct!(")"))), - expression_to_grouped_expression - ) +make_fn!( + simple_expression, Expression>, + do_each!( + val => trace_nom!(value), + (value_to_expression(val)) + ) ); -fn symbol_or_expression(input: TokenIter) -> NomResult { - let scalar_head = do_parse!(input, sym: alt!(symbol | compound_value) >> (sym)); +fn expression_to_grouped_expression(e: Expression) -> Expression { + Expression::Grouped(Box::new(e)) +} + +make_fn!( + grouped_expression, Expression>, + do_each!( + _ => punct!("("), + expr => do_each!( + expr => trace_nom!(expression), + _ => punct!(")"), + (expr) + ), + (expression_to_grouped_expression(expr)) + ) +); + +fn symbol_or_expression(input: SliceIter) -> NomResult { + let _i = input.clone(); + let scalar_head = do_each!(input, + sym => either!(symbol, compound_value), + (sym) + ); match scalar_head { - IResult::Incomplete(i) => IResult::Incomplete(i), - IResult::Error(_) => grouped_expression(input), - IResult::Done(rest, val) => { + Result::Incomplete(offset) => Result::Incomplete(offset), + Result::Fail(_) => grouped_expression(_i), + Result::Abort(e) => Result::Abort(e), + Result::Complete(rest, val) => { let res = peek!(rest.clone(), punct!(".")); match val { Value::Tuple(_) => { - if res.is_done() { - IResult::Done(rest, Expression::Simple(val)) + if res.is_complete() { + Result::Complete(rest, Expression::Simple(val)) } else { - return IResult::Error(nom::ErrorKind::Custom(error::Error::new( + return Result::Fail(Error::new( "Expected (.) but no dot found".to_string(), - error::ErrorType::IncompleteParsing, - val.pos().clone(), - ))); + Box::new(rest.clone()), + )); } } Value::List(_) => { - if res.is_done() { - IResult::Done(rest, Expression::Simple(val)) + if res.is_complete() { + Result::Complete(rest, Expression::Simple(val)) } else { - return IResult::Error(nom::ErrorKind::Custom(error::Error::new( + return Result::Fail(Error::new( "Expected (.) but no dot found".to_string(), - error::ErrorType::IncompleteParsing, - val.pos().clone(), - ))); + Box::new(rest.clone()), + )); } } - _ => IResult::Done(rest, Expression::Simple(val)), + _ => Result::Complete(rest, Expression::Simple(val)), } } } } -fn selector_list(input: TokenIter) -> NomResult { +fn selector_list(input: SliceIter) -> NomResult { let (rest, head) = match symbol_or_expression(input) { - IResult::Done(rest, val) => (rest, val), - IResult::Error(e) => { - return IResult::Error(e); + Result::Complete(rest, val) => (rest, val), + Result::Fail(e) => { + return Result::Fail(e); } - IResult::Incomplete(i) => { - return IResult::Incomplete(i); + Result::Incomplete(i) => { + return Result::Incomplete(i); } + Result::Abort(e) => return Result::Abort(e), }; let (rest, is_dot) = match punct!(rest, ".") { - IResult::Done(rest, tok) => (rest, Some(tok)), - IResult::Incomplete(i) => { - return IResult::Incomplete(i); + Result::Complete(rest, tok) => (rest, Some(tok)), + Result::Incomplete(i) => { + return Result::Incomplete(i); } - IResult::Error(_) => (rest, None), + Result::Fail(_) => (rest, None), + Result::Abort(e) => return Result::Abort(e), }; let (rest, list) = if is_dot.is_some() { - let (rest, list) = match separated_list!( + let (rest, list) = match separated!( rest, punct!("."), - alt!(match_type!(BAREWORD) | match_type!(DIGIT) | match_type!(STR)) + either!(match_type!(BAREWORD), match_type!(DIGIT), match_type!(STR)) ) { - IResult::Done(rest, val) => (rest, val), - IResult::Incomplete(i) => { - return IResult::Incomplete(i); - } - IResult::Error(e) => { - return IResult::Error(e); - } + Result::Complete(rest, val) => (rest, val), + Result::Incomplete(i) => return Result::Incomplete(i), + Result::Fail(e) => return Result::Fail(e), + Result::Abort(e) => return Result::Abort(e), }; if list.is_empty() { - return IResult::Error(nom::ErrorKind::Custom(error::Error::new( + return Result::Fail(Error::new( "(.) with no selector fields after".to_string(), - error::ErrorType::IncompleteParsing, - is_dot.unwrap().pos, - ))); + Box::new(rest.clone()), + )); } else { (rest, Some(list)) } @@ -486,189 +494,230 @@ fn selector_list(input: TokenIter) -> NomResult { tail: list, }; - return IResult::Done(rest, sel_list); + return Result::Complete(rest, sel_list); } -fn tuple_to_copy(t: (SelectorDef, FieldList)) -> ParseResult { - let pos = t.0.pos.clone(); - Ok(Expression::Copy(CopyDef { - selector: t.0, - fields: t.1, +fn tuple_to_copy(def: SelectorDef, fields: Option) -> Expression { + let pos = def.pos.clone(); + let fields = match fields { + Some(fields) => fields, + None => Vec::new(), + }; + Expression::Copy(CopyDef { + selector: def, + fields: fields, pos: pos, - })) + }) } -named!(copy_expression, - map_res!( - do_parse!( - pos: pos >> - selector: trace_nom!(selector_list) >> - punct!("{") >> - fields: trace_nom!(field_list) >> - opt_res!(punct!(",")) >> // noms opt! macro does not preserve error types properly but this one does. - punct!("}") >> - (SelectorDef::new(selector, pos.line, pos.column as usize), fields) - ), - tuple_to_copy +make_fn!( + copy_expression, Expression>, + do_each!( + pos => pos, + selector => trace_nom!(selector_list), + _ => punct!("{"), + fields => optional!(trace_nom!(field_list)), + _ => optional!(punct!(",")), // noms opt! macro does not preserve error types properly but this one does. + _ => punct!("}"), + (tuple_to_copy(SelectorDef::new(selector, pos), fields)) ) ); -fn tuple_to_macro(mut t: (Position, Vec, Value)) -> ParseResult { - match t.2 { +fn tuple_to_macro<'a>( + input: SliceIter<'a, Token>, + pos: Position, + vals: Option>, + val: Value, +) -> ParseResult<'a, Expression> { + let mut default_args = match vals { + None => Vec::new(), + Some(vals) => vals, + }; + let arglist = default_args + .drain(0..) + .map(|s| PositionedItem { + pos: s.pos().clone(), + val: s.to_string(), + }).collect(); + match val { Value::Tuple(v) => Ok(Expression::Macro(MacroDef { - argdefs: t - .1 - .drain(0..) - .map(|s| Positioned { - pos: s.pos().clone(), - val: s.to_string(), - }) - .collect(), + argdefs: arglist, fields: v.val, - pos: t.0, + pos: pos, })), - val => Err(error::Error::new( + val => Err(Error::new( format!("Expected Tuple Got {:?}", val), - error::ErrorType::UnexpectedToken, - t.0, + Box::new(input.clone()), )), } } -named!(arglist, error::Error>, separated_list!(punct!(","), symbol)); - -named!(macro_expression, - map_res!( - do_parse!( - pos: pos >> - word!("macro") >> - punct!("(") >> - arglist: trace_nom!(arglist) >> - punct!(")") >> - punct!("=>") >> - map: trace_nom!(tuple) >> - (pos, arglist, map) - ), - tuple_to_macro - ) +make_fn!( + arglist, Vec>, + separated!(punct!(","), symbol) ); -fn tuple_to_select(t: (Position, Expression, Expression, Value)) -> ParseResult { - match t.3 { +fn macro_expression(input: SliceIter) -> Result, Expression> { + let parsed = do_each!(input, + pos => pos, + _ => word!("macro"), + _ => punct!("("), + arglist => trace_nom!(optional!(arglist)), + _ => punct!(")"), + _ => punct!("=>"), + map => trace_nom!(tuple), + (pos, arglist, map) + ); + match parsed { + Result::Abort(e) => Result::Abort(e), + Result::Fail(e) => Result::Fail(e), + Result::Incomplete(offset) => Result::Incomplete(offset), + Result::Complete(rest, (pos, arglist, map)) => { + match tuple_to_macro(rest.clone(), pos, arglist, map) { + Ok(expr) => Result::Complete(rest, expr), + Err(e) => Result::Fail(Error::caused_by( + "Invalid Macro syntax", + Box::new(e), + Box::new(rest.clone()), + )), + } + } + } +} + +fn tuple_to_select<'a>( + input: SliceIter<'a, Token>, + e1: Expression, + e2: Expression, + val: Value, +) -> ParseResult<'a, Expression> { + match val { Value::Tuple(v) => Ok(Expression::Select(SelectDef { - val: Box::new(t.1), - default: Box::new(t.2), + val: Box::new(e1), + default: Box::new(e2), tuple: v.val, - pos: t.0, + pos: (&input).into(), })), - val => Err(error::Error::new( + val => Err(Error::new( format!("Expected Tuple Got {:?}", val), - error::ErrorType::UnexpectedToken, - t.0, + Box::new(input.clone()), )), } } -named!(select_expression, - map_res!( - do_parse!( - start: word!("select") >> - val: terminated!(trace_nom!(expression), punct!(",")) >> - default: terminated!(trace_nom!(expression), punct!(",")) >> - map: trace_nom!(tuple) >> - (start.pos.clone(), val, default, map) - ), - tuple_to_select - ) -); - -fn tuple_to_format(t: (Token, Vec)) -> ParseResult { - Ok(Expression::Format(FormatDef { - template: t.0.fragment.to_string(), - args: t.1, - pos: t.0.pos, - })) +fn select_expression(input: SliceIter) -> Result, Expression> { + let parsed = do_each!(input, + _ => word!("select"), + val => do_each!( + expr => trace_nom!(expression), + _ => punct!(","), + (expr) + ), + default => do_each!( + expr => trace_nom!(expression), + _ => punct!(","), + (expr) + ), + map => trace_nom!(tuple), + (val, default, map) + ); + match parsed { + Result::Abort(e) => Result::Abort(e), + Result::Fail(e) => Result::Fail(e), + Result::Incomplete(offset) => Result::Incomplete(offset), + Result::Complete(rest, (val, default, map)) => { + match tuple_to_select(input.clone(), val, default, map) { + Ok(expr) => Result::Complete(rest, expr), + Err(e) => Result::Fail(Error::caused_by( + "Invalid Select Expression", + Box::new(e), + Box::new(rest.clone()), + )), + } + } + } } -named!(format_expression, - map_res!( - do_parse!( - tmpl: match_type!(STR) >> - punct!("%") >> - punct!("(") >> - args: separated_list!(punct!(","), trace_nom!(expression)) >> - punct!(")") >> - (tmpl, args) - ), - tuple_to_format - ) +fn tuple_to_format(tok: Token, exprs: Vec) -> Expression { + Expression::Format(FormatDef { + template: tok.fragment.to_string(), + args: exprs, + pos: tok.pos, + }) +} + +make_fn!( + format_expression, Expression>, + do_each!( + tmpl => match_type!(STR), + _ => punct!("%"), + _ => punct!("("), + args => separated!(punct!(","), trace_nom!(expression)), + _ => punct!(")"), + (tuple_to_format(tmpl, args)) + ) ); -fn tuple_to_call(t: (Position, Value, Vec)) -> ParseResult { - if let Value::Selector(def) = t.1 { +fn tuple_to_call<'a>( + input: SliceIter<'a, Token>, + val: Value, + exprs: Vec, +) -> ParseResult<'a, Expression> { + if let Value::Selector(def) = val { Ok(Expression::Call(CallDef { macroref: def, - arglist: t.2, - pos: Position::new(t.0.line as usize, t.0.column as usize), + arglist: exprs, + pos: (&input).into(), })) } else { - Err(error::Error::new( - format!("Expected Selector Got {:?}", t.0), - error::ErrorType::UnexpectedToken, - Position::new(t.0.line as usize, t.0.column as usize), + Err(Error::new( + format!("Expected Selector Got {:?}", val), + Box::new(input.clone()), )) } } -fn vec_to_selector_value(t: (Position, SelectorList)) -> ParseResult { - Ok(Value::Selector(SelectorDef::new( - t.1, - t.0.line as usize, - t.0.column as usize, - ))) +fn vec_to_selector_value(pos: Position, list: SelectorList) -> Value { + Value::Selector(SelectorDef::new(list, pos)) } -named!(selector_value, - map_res!( - do_parse!( - sl: trace_nom!(selector_list) >> - (sl.head.pos().clone(), sl) - ), - vec_to_selector_value - ) +make_fn!( + selector_value, Value>, + do_each!( + sl => trace_nom!(selector_list), + (vec_to_selector_value(sl.head.pos().clone(), sl)) + ) ); -named!(call_expression, - map_res!( - do_parse!( - macroname: trace_nom!(selector_value) >> - punct!("(") >> - args: separated_list!(punct!(","), trace_nom!(expression)) >> - punct!(")") >> - (macroname.pos().clone(), macroname, args) - ), - tuple_to_call - ) -); +fn call_expression(input: SliceIter) -> Result, Expression> { + let parsed = do_each!(input.clone(), + macroname => trace_nom!(selector_value), + _ => punct!("("), + args => separated!(punct!(","), trace_nom!(expression)), + _ => punct!(")"), + (macroname, args) + ); + match parsed { + Result::Abort(e) => Result::Abort(e), + Result::Fail(e) => Result::Fail(e), + Result::Incomplete(offset) => Result::Incomplete(offset), + Result::Complete(rest, (name, args)) => match tuple_to_call(input.clone(), name, args) { + Ok(expr) => Result::Complete(rest, expr), + Err(e) => Result::Fail(Error::caused_by( + "Invalid Call Syntax", + Box::new(e), + Box::new(rest), + )), + }, + } +} -fn tuple_to_list_op(tpl: (Position, Token, Value, Expression)) -> ParseResult { - let pos = tpl.0; - let t = if &tpl.1.fragment == "map" { - ListOpType::Map - } else if &tpl.1.fragment == "filter" { - ListOpType::Filter - } else { - return Err(error::Error::new( - format!( - "Expected one of 'map' or 'filter' but got '{}'", - tpl.1.fragment - ), - error::ErrorType::UnexpectedToken, - pos, - )); - }; - let macroname = tpl.2; - let list = tpl.3; +fn tuple_to_list_op<'a>( + input: &'a SliceIter, + kind: ListOpType, + macroname: Value, + list: Expression, +) -> ParseResult<'a, Expression> { if let Value::Selector(mut def) = macroname { // First of all we need to assert that this is a selector of at least // two sections. @@ -680,10 +729,9 @@ fn tuple_to_list_op(tpl: (Position, Token, Value, Expression)) -> ParseResult { @@ -694,10 +742,9 @@ fn tuple_to_list_op(tpl: (Position, Token, Value, Expression)) -> ParseResult ParseResult ParseResult, - map_res!( - do_parse!( - pos: pos >> - optype: alt!(word!("map") | word!("filter")) >> - macroname: trace_nom!(selector_value) >> - list: trace_nom!(non_op_expression) >> - (pos, optype, macroname, list) +make_fn!( + list_op_expression, Expression>, + do_each!( + input => input!(), + optype => either!( + do_each!(_ => word!("map"), (ListOpType::Map)), + do_each!(_ => word!("filter"), (ListOpType::Filter)) ), - tuple_to_list_op + macroname => trace_nom!(selector_value), + list => trace_nom!(non_op_expression), + (tuple_to_list_op(&input, optype, macroname, list).unwrap()) ) ); -fn unprefixed_expression(input: TokenIter) -> NomResult { +fn unprefixed_expression(input: SliceIter) -> NomResult { let _input = input.clone(); - let attempt = alt!(input, - trace_nom!(call_expression) | - trace_nom!(copy_expression) | - trace_nom!(format_expression)); - match attempt { - IResult::Incomplete(i) => IResult::Incomplete(i), - IResult::Done(rest, expr) => IResult::Done(rest, expr), - IResult::Error(_) => trace_nom!(_input, simple_expression), - } + let attempt = either!( + input, + trace_nom!(call_expression), + trace_nom!(copy_expression), + trace_nom!(format_expression) + ); + match attempt { + Result::Incomplete(i) => Result::Incomplete(i), + Result::Complete(rest, expr) => Result::Complete(rest, expr), + Result::Fail(_) => trace_nom!(_input, simple_expression), + Result::Abort(e) => Result::Abort(e), + } } -named!(non_op_expression, +make_fn!( + non_op_expression, Expression>, alt_peek!( - alt!(word!("map") | word!("filter")) => trace_nom!(list_op_expression) | + either!(word!("map"), word!("filter")) => trace_nom!(list_op_expression) | word!("macro") => trace_nom!(macro_expression) | word!("select") => trace_nom!(select_expression) | punct!("(") => trace_nom!(grouped_expression) | trace_nom!(unprefixed_expression)) ); -fn expression(input: TokenIter) -> NomResult { +fn expression(input: SliceIter) -> NomResult { let _input = input.clone(); match trace_nom!(_input, op_expression) { - IResult::Incomplete(i) => IResult::Incomplete(i), - IResult::Error(_) => trace_nom!(input, non_op_expression), - IResult::Done(rest, expr) => IResult::Done(rest, expr), + Result::Incomplete(i) => Result::Incomplete(i), + Result::Fail(_) => trace_nom!(input, non_op_expression), + Result::Abort(e) => Result::Abort(e), + Result::Complete(rest, expr) => Result::Complete(rest, expr), } } -//named!(expression, -// alt_complete!(trace_nom!(op_expression) | trace_nom!(non_op_expression)) -//); - -fn expression_to_statement(v: Expression) -> ParseResult { - Ok(Statement::Expression(v)) -} - -named!(expression_statement, - map_res!( - terminated!(trace_nom!(expression), punct!(";")), - expression_to_statement +make_fn!( + expression_statement, Statement>, + do_each!( + e => do_each!( + expr => trace_nom!(expression), + _ => punct!(";"), + (expr) + ), + (Statement::Expression(e)) ) ); -fn tuple_to_let(t: (Token, Expression)) -> ParseResult { - Ok(Statement::Let(LetDef { - name: t.0, - value: t.1, - })) +fn tuple_to_let(tok: Token, expr: Expression) -> Statement { + Statement::Let(LetDef { + name: tok, + value: expr, + }) } -named!(let_stmt_body, - map_res!( - do_parse!( - name: match_type!(BAREWORD) >> - punct!("=") >> - val: trace_nom!(expression) >> - punct!(";") >> - (name, val)), - tuple_to_let +make_fn!( + let_stmt_body, Statement>, + do_each!( + name => match_type!(BAREWORD), + _ => punct!("="), + val => trace_nom!(expression), + _ => punct!(";"), + (tuple_to_let(name, val)) ) ); -named!(let_statement, - wrap_err!(do_parse!( - word!("let") >> - pos: pos >> - stmt: trace_nom!(let_stmt_body) >> +make_fn!( + let_statement, Statement>, + do_each!( + _ => word!("let"), + stmt => trace_nom!(must!(let_stmt_body)), (stmt) - ), "Invalid let statement") -); - -fn tuple_to_import(t: (Token, Token)) -> ParseResult { - Ok(Statement::Import(ImportDef { - path: t.0, - name: t.1, - })) -} - -named!(import_stmt_body, - map_res!( - do_parse!( - path: match_type!(STR) >> - word!("as") >> - name: match_type!(BAREWORD) >> - punct!(";") >> - (path, name)), - tuple_to_import ) ); -named!(import_statement, - wrap_err!(do_parse!( - word!("import") >> +fn tuple_to_import(tok: Token, tok2: Token) -> Statement { + Statement::Import(ImportDef { + path: tok, + name: tok2, + }) +} + +make_fn!( + import_stmt_body, Statement>, + do_each!( + path => match_type!(STR), + _ => word!("as"), + name => match_type!(BAREWORD), + _ => punct!(";"), + (tuple_to_import(path, name)) + ) +); + +make_fn!( + import_statement, Statement>, + do_each!( + _ => word!("import"), // past this point we know this is supposed to be an import statement. - pos: pos >> - stmt: trace_nom!(import_stmt_body) >> + stmt => trace_nom!(must!(import_stmt_body)), (stmt) - ), "Invalid import statement") + ) ); -named!(assert_statement, - wrap_err!(do_parse!( - word!("assert") >> - pos: pos >> - tok: match_type!(PIPEQUOTE) >> - punct!(";") >> - (Statement::Assert(tok.clone())) - ), "Invalid assert statement") +make_fn!( + assert_statement, Statement>, + do_each!( + _ => word!("assert"), + tok => must!(match_type!(PIPEQUOTE)), + _ => must!(punct!(";")), + (Statement::Assert(tok.clone())) + ) ); -named!(out_statement, - wrap_err!(do_parse!( - word!("out") >> - pos: pos >> - typ: match_type!(BAREWORD) >> - expr: expression >> - punct!(";") >> +make_fn!( + out_statement, Statement>, + do_each!( + _ => word!("out"), + typ => must!(match_type!(BAREWORD)), + expr => must!(expression), + _ => must!(punct!(";")), (Statement::Output(typ.clone(), expr.clone())) - ), "Invalid out statement") + ) ); //trace_macros!(true); -fn statement(i: TokenIter) -> nom::IResult { +fn statement(i: SliceIter) -> Result, Statement> { return alt_peek!(i, word!("assert") => trace_nom!(assert_statement) | word!("import") => trace_nom!(import_statement) | @@ -874,48 +921,50 @@ fn statement(i: TokenIter) -> nom::IResult { } //trace_macros!(false); -/// Parses a LocatedSpan into a list of Statements or an error::Error. -pub fn parse(input: LocatedSpan<&str>) -> Result, error::Error> { - match tokenize(input) { +/// Parses a LocatedSpan into a list of Statements or an `error::Error`. +pub fn parse<'a>(input: OffsetStrIter<'a>) -> std::result::Result, String> { + match tokenize(input.clone()) { Ok(tokenized) => { let mut out = Vec::new(); - let mut i_ = TokenIter { - source: tokenized.as_slice(), - }; + let mut i_ = SliceIter::new(&tokenized); loop { let i = i_.clone(); - if i[0].typ == TokenType::END { - break; + if let Some(tok) = i.peek_next() { + if tok.typ == TokenType::END { + break; + } } - match statement(i) { - IResult::Error(nom::ErrorKind::Custom(e)) => { - return Err(e); + match statement(i.clone()) { + Result::Abort(e) => { + let err = abortable_parser::Error::caused_by( + "Statement Parse Error", + Box::new(e), + Box::new(i.clone()), + ); + let ctx_err = StackPrinter { err: err }; + return Err(format!("{}", ctx_err)); } - IResult::Error(e) => { - return Err(error::Error::new_with_errorkind( - "Statement Parse error", - error::ErrorType::ParseError, - Position { - line: i_[0].pos.line, - column: i_[0].pos.column, - }, - e, - )); + Result::Fail(e) => { + let err = abortable_parser::Error::caused_by( + "Statement Parse Error", + Box::new(e), + Box::new(i.clone()), + ); + let ctx_err = StackPrinter { err: err }; + return Err(format!("{}", ctx_err)); } - IResult::Incomplete(ei) => { - return Err(error::Error::new( - format!("Unexpected end of parsing input: {:?}", ei), - error::ErrorType::IncompleteParsing, - Position { - line: i_[0].pos.line, - column: i_[0].pos.column, - }, - )); + Result::Incomplete(_ei) => { + let err = abortable_parser::Error::new( + "Unexpected end of parse input", + Box::new(i.clone()), + ); + let ctx_err = StackPrinter { err: err }; + return Err(format!("{}", ctx_err)); } - IResult::Done(rest, stmt) => { + Result::Complete(rest, stmt) => { out.push(stmt); i_ = rest; - if i_.input_len() == 0 { + if eoi(i).is_complete() { break; } } @@ -924,11 +973,7 @@ pub fn parse(input: LocatedSpan<&str>) -> Result, error::Error> { return Ok(out); } Err(e) => { - return Err(error::Error::new_with_cause( - format!("Tokenization Error"), - error::ErrorType::ParseError, - e, - )); + return Err(e); } } } diff --git a/src/parse/precedence.rs b/src/parse/precedence.rs index 6d06e40..a7bca95 100644 --- a/src/parse/precedence.rs +++ b/src/parse/precedence.rs @@ -14,14 +14,11 @@ //! Bottom up parser for precedence parsing of expressions separated by binary //! operators. -use std; +use abortable_parser::combinators::eoi; +use abortable_parser::{Error, Result, SliceIter}; -use nom::{ErrorKind, IResult, InputIter, InputLength, Slice}; - -use super::{non_op_expression, NomResult, ParseResult}; +use super::{non_op_expression, NomResult}; use ast::*; -use error; -use tokenizer::TokenIter; /// Defines the intermediate stages of our bottom up parser for precedence parsing. #[derive(Debug, PartialEq, Clone)] @@ -31,120 +28,119 @@ pub enum Element { CompareOp(CompareType), } -named!(math_op_type, - alt!( - do_parse!(punct!("+") >> (Element::MathOp(BinaryExprType::Add))) | - do_parse!(punct!("-") >> (Element::MathOp(BinaryExprType::Sub))) | - do_parse!(punct!("*") >> (Element::MathOp(BinaryExprType::Mul))) | - do_parse!(punct!("/") >> (Element::MathOp(BinaryExprType::Div))) +make_fn!( + math_op_type, Element>, + either!( + do_each!( + _ => punct!("+"), + (Element::MathOp(BinaryExprType::Add))), + do_each!( + _ => punct!("-"), + (Element::MathOp(BinaryExprType::Sub))), + do_each!( + _ => punct!("*"), + (Element::MathOp(BinaryExprType::Mul))), + do_each!( + _ => punct!("/"), + (Element::MathOp(BinaryExprType::Div))) ) ); -fn parse_expression(i: OpListIter) -> IResult { - let i_ = i.clone(); - if i_.input_len() == 0 { - return IResult::Error(ErrorKind::Custom(error::Error::new( - format!("Expected Expression found End Of Input"), - error::ErrorType::IncompleteParsing, - // TODO(jwall): This position information is incorrect. - Position { line: 0, column: 0 }, - ))); +fn parse_expression(i: SliceIter) -> Result, Expression> { + let mut i_ = i.clone(); + if eoi(i_.clone()).is_complete() { + return Result::Abort(Error::new( + "Expected Expression found End Of Input", + Box::new(i_), + )); } - let el = &(i_[0]); - if let &Element::Expr(ref expr) = el { - return IResult::Done(i.slice(1..), expr.clone()); + let el = i_.next(); + if let Some(&Element::Expr(ref expr)) = el { + return Result::Complete(i_.clone(), expr.clone()); } - return IResult::Error(ErrorKind::Custom(error::Error::new( + return Result::Fail(Error::new( format!( - "Error while parsing Binary Expression Unexpected Operator {:?}", + "Error while parsing Binary Expression Expected Expression got {:?}", el ), - error::ErrorType::ParseError, - // TODO(jwall): This position information is incorrect. - Position { line: 0, column: 0 }, - ))); + Box::new(i_), + )); } -fn parse_sum_operator(i: OpListIter) -> IResult { - let i_ = i.clone(); - if i_.input_len() == 0 { - return IResult::Error(ErrorKind::Custom(error::Error::new( +fn parse_sum_operator(i: SliceIter) -> Result, BinaryExprType> { + let mut i_ = i.clone(); + if eoi(i_.clone()).is_complete() { + return Result::Fail(Error::new( format!("Expected Expression found End Of Input"), - error::ErrorType::IncompleteParsing, - // TODO(jwall): This position information is incorrect. - Position { line: 0, column: 0 }, - ))); + Box::new(i_), + )); } - let el = &(i_[0]); - if let &Element::MathOp(ref op) = el { + let el = i_.next(); + if let Some(&Element::MathOp(ref op)) = el { match op { &BinaryExprType::Add => { - return IResult::Done(i.slice(1..), op.clone()); + return Result::Complete(i_.clone(), op.clone()); } &BinaryExprType::Sub => { - return IResult::Done(i.slice(1..), op.clone()); + return Result::Complete(i_.clone(), op.clone()); } _other => { // noop } }; } - return IResult::Error(ErrorKind::Custom(error::Error::new( + return Result::Fail(Error::new( format!( "Error while parsing Binary Expression Unexpected Operator {:?}", el ), - error::ErrorType::ParseError, - // TODO(jwall): This position information is incorrect. - Position { line: 0, column: 0 }, - ))); + Box::new(i_), + )); } fn tuple_to_binary_expression( - tpl: (BinaryExprType, Expression, Expression), -) -> ParseResult { - let pos = tpl.1.pos().clone(); - Ok(Expression::Binary(BinaryOpDef { - kind: tpl.0, - left: Box::new(tpl.1), - right: Box::new(tpl.2), + kind: BinaryExprType, + left: Expression, + right: Expression, +) -> Expression { + let pos = left.pos().clone(); + Expression::Binary(BinaryOpDef { + kind: kind, + left: Box::new(left), + right: Box::new(right), pos: pos, - })) + }) } -fn parse_product_operator(i: OpListIter) -> IResult { - let i_ = i.clone(); - if i_.input_len() == 0 { - return IResult::Error(ErrorKind::Custom(error::Error::new( +fn parse_product_operator(i: SliceIter) -> Result, BinaryExprType> { + let mut i_ = i.clone(); + if eoi(i_.clone()).is_complete() { + return Result::Fail(Error::new( format!("Expected Expression found End Of Input"), - error::ErrorType::IncompleteParsing, - // TODO(jwall): This position information is incorrect. - Position { line: 0, column: 0 }, - ))); + Box::new(i_), + )); } - let el = &(i_[0]); - if let &Element::MathOp(ref op) = el { + let el = i_.next(); + if let Some(&Element::MathOp(ref op)) = el { match op { &BinaryExprType::Mul => { - return IResult::Done(i.slice(1..), op.clone()); + return Result::Complete(i_.clone(), op.clone()); } &BinaryExprType::Div => { - return IResult::Done(i.slice(1..), op.clone()); + return Result::Complete(i_.clone(), op.clone()); } _other => { // noop } }; } - return IResult::Error(ErrorKind::Custom(error::Error::new( + return Result::Fail(Error::new( format!( "Error while parsing Binary Expression Unexpected Operator {:?}", el ), - error::ErrorType::ParseError, - // TODO(jwall): This position information is incorrect. - Position { line: 0, column: 0 }, - ))); + Box::new(i_), + )); } /// do_binary_expr implements precedence based parsing where the more tightly bound @@ -152,15 +148,15 @@ fn parse_product_operator(i: OpListIter) -> IResult { - do_binary_expr!($i, call!($oprule), $lowerrule) + do_binary_expr!($i, run!($oprule), $lowerrule) }; ($i:expr, $oprule:ident, $lowerrule:ident!( $($lowerargs:tt)* )) => { - do_binary_expr!($i, call!($oprule), $lowerrule!($($lowerargs)*)) + do_binary_expr!($i, run!($oprule), $lowerrule!($($lowerargs)*)) }; ($i:expr, $oprule:ident) => { - do_binary_expr!($i, call!($oprule)) + do_binary_expr!($i, run!($oprule)) }; ($i:expr, $oprule:ident!( $($args:tt)* )) => { @@ -168,101 +164,97 @@ macro_rules! do_binary_expr { }; ($i:expr, $oprule:ident!( $($args:tt)* ), $lowerrule:ident) => { - do_binary_expr!($i, $oprule!($($args)*), call!($lowerrule)) + do_binary_expr!($i, $oprule!($($args)*), run!($lowerrule)) }; ($i:expr, $oprule:ident!( $($args:tt)* ), $lowerrule:ident!( $($lowerargs:tt)* )) => { - map_res!($i, - do_parse!( - left: $lowerrule!($($lowerargs)*) >> - typ: $oprule!($($args)*) >> - right: $lowerrule!($($lowerargs)*) >> - (typ, left, right) - ), - tuple_to_binary_expression + do_each!($i, + left => $lowerrule!($($lowerargs)*), + typ => $oprule!($($args)*), + right => $lowerrule!($($lowerargs)*), + (tuple_to_binary_expression(typ, left, right)) ) }; } -named!(sum_expression, +make_fn!( + sum_expression, Expression>, do_binary_expr!( parse_sum_operator, - alt!(trace_nom!(product_expression) | trace_nom!(parse_expression))) + either!(trace_nom!(product_expression), trace_nom!(parse_expression)) + ) ); -named!(product_expression, - do_binary_expr!( - parse_product_operator, - trace_nom!(parse_expression)) +make_fn!( + product_expression, Expression>, + do_binary_expr!(parse_product_operator, trace_nom!(parse_expression)) ); -named!(math_expression, - alt!(trace_nom!(sum_expression) | trace_nom!(product_expression)) +make_fn!( + math_expression, Expression>, + either!(trace_nom!(sum_expression), trace_nom!(product_expression)) ); // TODO(jwall): Change comparison operators to use the do_binary_expr! with precedence? fn tuple_to_compare_expression( - tpl: (CompareType, Expression, Expression), -) -> ParseResult { - let pos = tpl.1.pos().clone(); - Ok(Expression::Compare(ComparisonDef { - kind: tpl.0, - left: Box::new(tpl.1), - right: Box::new(tpl.2), + kind: CompareType, + left: Expression, + right: Expression, +) -> Expression { + let pos = left.pos().clone(); + Expression::Compare(ComparisonDef { + kind: kind, + left: Box::new(left), + right: Box::new(right), pos: pos, - })) + }) } -named!(compare_op_type, - alt!( - do_parse!(punct!("==") >> (Element::CompareOp(CompareType::Equal))) | - do_parse!(punct!("!=") >> (Element::CompareOp(CompareType::NotEqual))) | - do_parse!(punct!("<=") >> (Element::CompareOp(CompareType::LTEqual))) | - do_parse!(punct!(">=") >> (Element::CompareOp(CompareType::GTEqual))) | - do_parse!(punct!("<") >> (Element::CompareOp(CompareType::LT))) | - do_parse!(punct!(">") >> (Element::CompareOp(CompareType::GT))) +make_fn!( + compare_op_type, Element>, + either!( + do_each!(_ => punct!("=="), (Element::CompareOp(CompareType::Equal))), + do_each!(_ => punct!("!="), (Element::CompareOp(CompareType::NotEqual))), + do_each!(_ => punct!("<="), (Element::CompareOp(CompareType::LTEqual))), + do_each!(_ => punct!(">="), (Element::CompareOp(CompareType::GTEqual))), + do_each!(_ => punct!("<"), (Element::CompareOp(CompareType::LT))), + do_each!(_ => punct!(">"), (Element::CompareOp(CompareType::GT))) ) ); -fn parse_compare_operator(i: OpListIter) -> IResult { - let i_ = i.clone(); - if i_.input_len() == 0 { - return IResult::Error(ErrorKind::Custom(error::Error::new( +fn parse_compare_operator(i: SliceIter) -> Result, CompareType> { + let mut i_ = i.clone(); + if eoi(i_.clone()).is_complete() { + return Result::Fail(Error::new( format!("Expected Expression found End Of Input"), - error::ErrorType::IncompleteParsing, - // TODO(jwall): This position information is incorrect. - Position { line: 0, column: 0 }, - ))); + Box::new(i_), + )); } - let el = &(i_[0]); - if let &Element::CompareOp(ref op) = el { - return IResult::Done(i.slice(1..), op.clone()); + let el = i_.next(); + if let Some(&Element::CompareOp(ref op)) = el { + return Result::Complete(i_.clone(), op.clone()); } - return IResult::Error(ErrorKind::Custom(error::Error::new( + return Result::Fail(Error::new( format!( "Error while parsing Binary Expression Unexpected Operator {:?}", el ), - error::ErrorType::ParseError, - // TODO(jwall): This position information is incorrect. - Position { line: 0, column: 0 }, - ))); + Box::new(i), + )); } -named!(compare_expression, - map_res!( - do_parse!( - left: alt!(trace_nom!(math_expression) | trace_nom!(parse_expression)) >> - typ: parse_compare_operator >> - right: alt!(trace_nom!(math_expression) | trace_nom!(parse_expression)) >> - (typ, left, right) - ), - tuple_to_compare_expression +make_fn!( + compare_expression, Expression>, + do_each!( + left => either!(trace_nom!(math_expression), trace_nom!(parse_expression)), + typ => parse_compare_operator, + right => either!(trace_nom!(math_expression), trace_nom!(parse_expression)), + (tuple_to_compare_expression(typ, left, right)) ) ); /// Parse a list of expressions separated by operators into a Vec. -fn parse_operand_list(i: TokenIter) -> NomResult> { +fn parse_operand_list<'a>(i: SliceIter<'a, Token>) -> NomResult<'a, Vec> { // 1. First try to parse a non_op_expression, let mut _i = i.clone(); let mut list = Vec::new(); @@ -271,148 +263,101 @@ fn parse_operand_list(i: TokenIter) -> NomResult> { loop { // 2. Parse a non_op_expression. match non_op_expression(_i.clone()) { - IResult::Error(e) => { + Result::Fail(e) => { // A failure to parse an expression // is always an error. - return IResult::Error(e); + return Result::Fail(e); } - IResult::Incomplete(i) => { - return IResult::Incomplete(i); + Result::Abort(e) => { + // A failure to parse an expression + // is always an error. + return Result::Abort(e); } - IResult::Done(rest, expr) => { + Result::Incomplete(i) => { + return Result::Incomplete(i); + } + Result::Complete(rest, expr) => { list.push(Element::Expr(expr)); _i = rest.clone(); } } // 3. Parse an operator. - match alt!(_i, math_op_type | compare_op_type) { - IResult::Error(e) => { + match either!(_i.clone(), math_op_type, compare_op_type) { + Result::Fail(e) => { if firstrun { // If we don't find an operator in our first // run then this is not an operand list. - return IResult::Error(e); + return Result::Fail(e); } // if we don't find one on subsequent runs then // that's the end of the operand list. break; } - IResult::Incomplete(i) => { - return IResult::Incomplete(i); + Result::Abort(e) => { + // A failure to parse an expression + // is always an error. + return Result::Abort(e); } - IResult::Done(rest, el) => { + Result::Incomplete(i) => { + return Result::Incomplete(i); + } + Result::Complete(rest, el) => { list.push(el); _i = rest.clone(); } } firstrun = false; } - return IResult::Done(_i, list); -} - -#[derive(Clone, Debug, PartialEq)] -pub struct OpListIter<'a> { - pub source: &'a [Element], -} - -impl<'a> OpListIter<'a> { - pub fn len(&self) -> usize { - self.source.len() - } -} - -impl<'a> InputLength for OpListIter<'a> { - fn input_len(&self) -> usize { - self.source.input_len() - } -} - -macro_rules! impl_op_iter_slice { - ($r:ty) => { - impl<'a> Slice<$r> for OpListIter<'a> { - fn slice(&self, range: $r) -> Self { - OpListIter { - source: self.source.slice(range), - } - } - } - }; -} - -impl_op_iter_slice!(std::ops::Range); -impl_op_iter_slice!(std::ops::RangeTo); -impl_op_iter_slice!(std::ops::RangeFrom); -impl_op_iter_slice!(std::ops::RangeFull); - -impl<'a> std::ops::Index for OpListIter<'a> { - type Output = Element; - - fn index(&self, i: usize) -> &Self::Output { - &self.source[i] - } -} - -impl<'a> InputIter for OpListIter<'a> { - type Item = &'a Element; - type RawItem = Element; - - type Iter = std::iter::Enumerate>; - type IterElem = std::slice::Iter<'a, Self::RawItem>; - - fn iter_indices(&self) -> Self::Iter { - self.source.iter().enumerate() - } - - fn iter_elements(&self) -> Self::IterElem { - self.source.iter() - } - - fn position

(&self, predicate: P) -> Option - where - P: Fn(Self::RawItem) -> bool, - { - for (o, v) in self.iter_indices() { - if predicate(v.clone()) { - return Some(o); - } - } - None - } - - fn slice_index(&self, count: usize) -> Option { - let mut cnt = 0; - for (index, _) in self.iter_indices() { - if cnt == count { - return Some(index); - } - cnt += 1; - } - if cnt == count { - return Some(self.len()); - } - None - } + return Result::Complete(_i, list); } /// Parse a binary operator expression. -pub fn op_expression(i: TokenIter) -> NomResult { +pub fn op_expression<'a>(i: SliceIter<'a, Token>) -> Result, Expression> { let preparse = parse_operand_list(i.clone()); match preparse { - IResult::Error(e) => IResult::Error(e), - IResult::Incomplete(i) => IResult::Incomplete(i), - IResult::Done(rest, oplist) => { - let mut i_ = OpListIter { - source: oplist.as_slice(), - }; - - let parse_result = alt!( - i_, - trace_nom!(compare_expression) | trace_nom!(math_expression) + Result::Fail(e) => { + let err = Error::caused_by( + "Failed while parsing operator expression", + Box::new(e), + Box::new(i), + ); + Result::Fail(err) + } + Result::Abort(e) => { + let err = Error::caused_by( + "Failed while parsing operator expression", + Box::new(e), + Box::new(i), + ); + Result::Fail(err) + } + Result::Incomplete(i) => Result::Incomplete(i), + Result::Complete(rest, oplist) => { + let mut i_ = SliceIter::new(&oplist); + let parse_result = either!( + i_.clone(), + trace_nom!(compare_expression), + trace_nom!(math_expression) ); match parse_result { - IResult::Error(e) => IResult::Error(e), - IResult::Incomplete(i) => IResult::Incomplete(i), - IResult::Done(_, expr) => IResult::Done(rest.clone(), expr), + Result::Fail(_e) => { + // TODO(jwall): It would be good to be able to use caused_by here. + let err = Error::new( + "Failed while parsing operator expression", + Box::new(rest.clone()), + ); + Result::Fail(err) + } + Result::Abort(_e) => { + let err = Error::new( + "Failed while parsing operator expression", + Box::new(rest.clone()), + ); + Result::Abort(err) + } + Result::Incomplete(_) => Result::Incomplete(i.clone()), + Result::Complete(_, expr) => Result::Complete(rest.clone(), expr), } } } diff --git a/src/parse/test.rs b/src/parse/test.rs index b1520eb..ba20a30 100644 --- a/src/parse/test.rs +++ b/src/parse/test.rs @@ -12,42 +12,55 @@ // See the License for the specific language governing permissions and // limitations under the License. use super::*; -use tokenizer::{tokenize, TokenIter}; +use tokenizer::tokenize; -use nom::IResult; -use nom_locate::LocatedSpan; +use abortable_parser::{Result, SliceIter}; + +use iter::OffsetStrIter; macro_rules! assert_parse { ($parsemac:ident($i:expr), $out:expr) => { assert_parse!($i, $parsemac, $out) }; ($i:expr, $f:expr, $out:expr) => {{ - let input = LocatedSpan::new($i); - match tokenize(input) { + let input = OffsetStrIter::new($i); + match tokenize(input.clone()) { Err(e) => assert!(false, format!("Tokenizer Error: {:?}", e)), - Ok(val) => match $f(TokenIter { - source: val.as_slice(), - }) { - IResult::Done(_, result) => assert_eq!(result, $out), + Ok(val) => match $f(SliceIter::new(val.as_slice())) { + Result::Complete(_, result) => assert_eq!(result, $out), other => assert!(false, format!("Expected Done got {:?}", other)), }, } };}; } -macro_rules! assert_error { +macro_rules! assert_fail { ($parsemac:ident($i:expr)) => { - assert_error!($i, $parsemac) + assert_fail!($i, $parsemac) }; ($i:expr, $f:expr) => {{ - let input = LocatedSpan::new($i); - match tokenize(input) { + let input = OffsetStrIter::new($i); + match tokenize(input.clone()) { Err(_) => assert!(true), Ok(val) => { - let result = $f(TokenIter { - source: val.as_slice(), - }); - assert!(result.is_err(), format!("Not an error: {:?}", result)) + let result = $f(SliceIter::new(val.as_slice())); + assert!(result.is_fail(), format!("Not an abort: {:?}", result)) + } + } + }}; +} + +macro_rules! assert_abort { + ($parsemac:ident($i:expr)) => { + assert_abort!($i, $parsemac) + }; + ($i:expr, $f:expr) => {{ + let input = OffsetStrIter::new($i); + match tokenize(input.clone()) { + Err(_) => assert!(true), + Ok(val) => { + let result = $f(SliceIter::new(val.as_slice())); + assert!(result.is_abort(), format!("Not a fail: {:?}", result)) } } }}; @@ -55,15 +68,15 @@ macro_rules! assert_error { #[test] fn test_null_parsing() { - assert_parse!(empty_value("NULL "), Value::Empty(Position::new(1, 1))); - assert_parse!(value("NULL "), Value::Empty(Position::new(1, 1))); + assert_parse!(empty_value("NULL "), Value::Empty(Position::new(1, 1, 0))); + assert_parse!(value("NULL "), Value::Empty(Position::new(1, 1, 0))); assert_parse!( simple_expression("NULL "), - Expression::Simple(Value::Empty(Position::new(1, 1))) + Expression::Simple(Value::Empty(Position::new(1, 1, 0))) ); assert_parse!( expression("NULL,"), - Expression::Simple(Value::Empty(Position::new(1, 1))) + Expression::Simple(Value::Empty(Position::new(1, 1, 0))) ); } @@ -71,62 +84,66 @@ fn test_null_parsing() { fn test_boolean_parsing() { assert_parse!( boolean_value("true"), - Value::Boolean(Positioned::new(true, 1, 1)) + Value::Boolean(PositionedItem::new(true, Position::new(1, 1, 0))) ); assert_parse!( boolean_value("false"), - Value::Boolean(Positioned::new(false, 1, 1)) + Value::Boolean(PositionedItem::new(false, Position::new(1, 1, 0))) ); - assert_error!(boolean_value("truth")); + assert_fail!(boolean_value("truth")); } #[test] fn test_symbol_parsing() { assert_parse!( symbol("foo"), - Value::Symbol(value_node!("foo".to_string(), 1, 1)) + Value::Symbol(value_node!("foo".to_string(), Position::new(1, 1, 0))) ); assert_parse!( symbol("foo-bar"), - Value::Symbol(value_node!("foo-bar".to_string(), 1, 1)) + Value::Symbol(value_node!("foo-bar".to_string(), Position::new(1, 1, 0))) ); assert_parse!( symbol("foo_bar"), - Value::Symbol(value_node!("foo_bar".to_string(), 1, 1)) + Value::Symbol(value_node!("foo_bar".to_string(), Position::new(1, 1, 0))) ); } #[test] fn test_selector_parsing() { - assert_error!(selector_value("foo.")); + assert_fail!(selector_value("foo.")); assert_parse!( selector_value("foo.bar "), - Value::Selector(make_selector!(make_expr!("foo".to_string(), 1, 1) => [ - make_tok!("bar", 1, 5)] => - 1, 1)) + Value::Selector( + make_selector!(make_expr!("foo".to_string(), Position::new(1, 1, 0)) => [ + make_tok!("bar", Position::new(1, 5, 4))] => + Position::new(1, 1, 0)) + ) ); assert_parse!( selector_value("foo.0 "), - Value::Selector(make_selector!(make_expr!("foo".to_string(), 1, 1) => [ - make_tok!(DIGIT => "0", 1, 5)] => - 1, 1)) + Value::Selector( + make_selector!(make_expr!("foo".to_string(), Position::new(1, 1, 0)) => [ + make_tok!(DIGIT => "0", Position::new(1, 5, 4))] => + Position::new(1, 1, 0)) + ) ); assert_parse!( selector_value("foo.bar;"), - Value::Selector(make_selector!(make_expr!("foo", 1, 1) => + Value::Selector(make_selector!(make_expr!("foo", Position::new(1, 1, 0)) => [ - make_tok!("bar", 1, 5) + make_tok!("bar", Position::new(1, 5, 4)) ] => - 1, 1)) + Position::new(1, 1, 0))) ); assert_parse!( selector_value("({foo=1}).foo "), Value::Selector( make_selector!(Expression::Grouped(Box::new(Expression::Simple( Value::Tuple(value_node!( - vec![(make_tok!("foo", 1, 3), Expression::Simple(Value::Int(Positioned::new(1, 1, 7))))], - 1, 3)) - ))) => [ make_tok!("foo", 1, 11) ] => 1, 2) + vec![(make_tok!("foo", Position::new(1, 3, 2)), Expression::Simple(Value::Int(PositionedItem::new(1, Position::new(1, 7, 6)))))], + Position::new(1, 3, 3))) + ))) => [ make_tok!("foo", Position::new(1, 11, 10)) ] => Position::new(1, 2, 1)) ) ); } @@ -137,35 +154,44 @@ fn test_statement_parse() { assert_parse!( statement(stmt), Statement::Import(ImportDef { - path: make_tok!(QUOT => "foo", 1,8), - name: make_tok!("foo", 1, 17), + path: make_tok!(QUOT => "foo", Position::new(1, 8, 7)), + name: make_tok!("foo", Position::new(1, 17, 16)), }) ); - assert_error!(import_statement("import \"foo\"")); + assert_abort!(import_statement("import \"foo\"")); assert_parse!( statement("let foo = 1.0 ;"), Statement::Let(LetDef { - name: make_tok!("foo", 1, 5), - value: Expression::Simple(Value::Float(value_node!(1.0, 1, 11))), + name: make_tok!("foo", Position::new(1, 5, 4)), + value: Expression::Simple(Value::Float(value_node!(1.0, Position::new(1, 11, 10)))), }) ); assert_parse!( statement("let foo = 1 + 1 * 2;"), Statement::Let(LetDef { - name: make_tok!("foo", 1, 5), + name: make_tok!("foo", Position::new(1, 5, 4)), value: Expression::Binary(BinaryOpDef { kind: BinaryExprType::Add, - left: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 11)))), + left: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 11, 10) + )))), right: Box::new(Expression::Binary(BinaryOpDef { kind: BinaryExprType::Mul, - left: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 15)))), - right: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 19)))), - pos: Position::new(1, 15), + left: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 15, 14) + )))), + right: Box::new(Expression::Simple(Value::Int(value_node!( + 2, + Position::new(1, 19, 18) + )))), + pos: Position::new(1, 15, 14), })), - pos: Position::new(1, 11), + pos: Position::new(1, 11, 10), }), }) ); @@ -173,19 +199,28 @@ fn test_statement_parse() { assert_parse!( statement("let foo = (1 + 1) * 2;"), Statement::Let(LetDef { - name: make_tok!("foo", 1, 5), + name: make_tok!("foo", Position::new(1, 5, 4)), value: Expression::Binary(BinaryOpDef { kind: BinaryExprType::Mul, left: Box::new(Expression::Grouped(Box::new(Expression::Binary( BinaryOpDef { kind: BinaryExprType::Add, - left: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 12)))), - right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 16)))), - pos: Position::new(1, 12), + left: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 12, 11) + )))), + right: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 16, 15) + )))), + pos: Position::new(1, 12, 11), }, )))), - right: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 21)))), - pos: Position::new(1, 12), + right: Box::new(Expression::Simple(Value::Int(value_node!( + 2, + Position::new(1, 21, 20) + )))), + pos: Position::new(1, 12, 11), }), }) ); @@ -193,17 +228,26 @@ fn test_statement_parse() { assert_parse!( statement("let foo = 1 * 1 + 2;"), Statement::Let(LetDef { - name: make_tok!("foo", 1, 5), + name: make_tok!("foo", Position::new(1, 5, 4)), value: Expression::Binary(BinaryOpDef { kind: BinaryExprType::Add, left: Box::new(Expression::Binary(BinaryOpDef { kind: BinaryExprType::Mul, - left: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 11)))), - right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 15)))), - pos: Position::new(1, 11), + left: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 11, 10) + )))), + right: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 15, 14) + )))), + pos: Position::new(1, 11, 10), })), - right: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 19)))), - pos: Position::new(1, 11), + right: Box::new(Expression::Simple(Value::Int(value_node!( + 2, + Position::new(1, 19, 18) + )))), + pos: Position::new(1, 11, 10), }), }) ); @@ -211,113 +255,124 @@ fn test_statement_parse() { assert_parse!( statement("// comment\nlet foo = 1.0 ;"), Statement::Let(LetDef { - name: make_tok!("foo", 2, 5), - value: Expression::Simple(Value::Float(value_node!(1.0, 2, 11))), + name: make_tok!("foo", Position::new(2, 5, 15)), + value: Expression::Simple(Value::Float(value_node!(1.0, Position::new(2, 11, 21)))), }) ); assert_parse!( statement("1.0;"), - Statement::Expression(Expression::Simple(Value::Float(value_node!(1.0, 1, 1)))) + Statement::Expression(Expression::Simple(Value::Float(value_node!( + 1.0, + Position::new(1, 1, 0) + )))) ); } #[test] fn test_import_statement_parse() { - assert_error!(import_statement("import")); - assert_error!(import_statement("import \"foo\"")); - assert_error!(import_statement("import \"foo\" as")); - assert_error!(import_statement("import \"foo\" as foo")); + assert_abort!(import_statement("import")); + assert_abort!(import_statement("import \"foo\"")); + assert_abort!(import_statement("import \"foo\" as")); + assert_abort!(import_statement("import \"foo\" as foo")); let import_stmt = "import \"foo\" as foo;"; assert_parse!( import_statement(import_stmt), Statement::Import(ImportDef { - path: make_tok!(QUOT => "foo", 1, 8), - name: make_tok!("foo", 1, 17), + path: make_tok!(QUOT => "foo", Position::new(1, 8, 7)), + name: make_tok!("foo", Position::new(1, 17, 16)), }) ); } #[test] fn test_let_statement_parse() { - assert_error!(let_statement("foo")); - assert_error!(let_statement("let \"foo\"")); - assert_error!(let_statement("let 1")); - assert_error!(let_statement("let")); - assert_error!(let_statement("let foo")); - assert_error!(let_statement("let foo =")); - assert_error!(let_statement("let foo = ")); - assert_error!(let_statement("let foo = 1")); + assert_fail!(let_statement("foo")); + assert_abort!(let_statement("let \"foo\"")); + assert_abort!(let_statement("let 1")); + assert_abort!(let_statement("let")); + assert_abort!(let_statement("let foo")); + assert_abort!(let_statement("let foo =")); + assert_abort!(let_statement("let foo = ")); + assert_abort!(let_statement("let foo = 1")); assert_parse!( let_statement("let foo = 1.0 ;"), Statement::Let(LetDef { - name: make_tok!("foo", 1, 5), - value: Expression::Simple(Value::Float(value_node!(1.0, 1, 11))), + name: make_tok!("foo", Position::new(1, 5, 4)), + value: Expression::Simple(Value::Float(value_node!(1.0, Position::new(1, 11, 10)))), }) ); assert_parse!( let_statement("let foo = // comment\n1.0 ;"), Statement::Let(LetDef { - name: make_tok!("foo", 1, 5), - value: Expression::Simple(Value::Float(value_node!(1.0, 2, 1))), + name: make_tok!("foo", Position::new(1, 5, 4)), + value: Expression::Simple(Value::Float(value_node!(1.0, Position::new(2, 1, 22)))), }) ); assert_parse!( let_statement("let foo = 1.0 // comment\n;"), Statement::Let(LetDef { - name: make_tok!("foo", 1, 5), - value: Expression::Simple(Value::Float(value_node!(1.0, 1, 11))), + name: make_tok!("foo", Position::new(1, 5, 4)), + value: Expression::Simple(Value::Float(value_node!(1.0, Position::new(1, 11, 10)))), }) ); assert_parse!( let_statement("let foo= 1.0;"), Statement::Let(LetDef { - name: make_tok!("foo", 1, 5), - value: Expression::Simple(Value::Float(value_node!(1.0, 1, 10))), + name: make_tok!("foo", Position::new(1, 5, 4)), + value: Expression::Simple(Value::Float(value_node!(1.0, Position::new(1, 10, 9)))), }) ); assert_parse!( let_statement("let foo =1.0;"), Statement::Let(LetDef { - name: make_tok!("foo", 1, 5), - value: Expression::Simple(Value::Float(value_node!(1.0, 1, 10))), + name: make_tok!("foo", Position::new(1, 5, 4)), + value: Expression::Simple(Value::Float(value_node!(1.0, Position::new(1, 10, 9)))), }) ); } #[test] fn test_out_statement_parse() { - assert_error!(out_statement("out")); - assert_error!(out_statement("out json")); - assert_error!(out_statement("out json foo")); + assert_abort!(out_statement("out")); + assert_abort!(out_statement("out json")); + assert_abort!(out_statement("out json foo")); assert_parse!( out_statement("out json 1.0;"), Statement::Output( Token { - pos: Position { line: 1, column: 5 }, + pos: Position { + line: 1, + column: 5, + offset: 4 + }, fragment: "json".to_string(), typ: TokenType::BAREWORD }, - Expression::Simple(Value::Float(value_node!(1.0, 1, 10))) + Expression::Simple(Value::Float(value_node!(1.0, Position::new(1, 10, 9)))) ) ); } #[test] fn test_assert_statement_parse() { - assert_error!(out_statement("assert")); - assert_error!(out_statement("assert |")); - assert_error!(out_statement("assert |foo")); + assert_fail!(out_statement("assert")); + assert_fail!(out_statement("assert |")); + assert_fail!(out_statement("assert |foo")); assert_parse!( assert_statement("assert |foo|;"), Statement::Assert(Token { - pos: Position { line: 1, column: 8 }, + pos: Position { + line: 1, + column: 8, + offset: 7 + }, fragment: "foo".to_string(), typ: TokenType::PIPEQUOTE }) @@ -325,65 +380,68 @@ fn test_assert_statement_parse() { } #[test] fn test_expression_statement_parse() { - assert_error!(expression_statement("foo")); + assert_fail!(expression_statement("foo")); assert_parse!( expression_statement("1.0;"), - Statement::Expression(Expression::Simple(Value::Float(value_node!(1.0, 1, 1)))) + Statement::Expression(Expression::Simple(Value::Float(value_node!( + 1.0, + Position::new(1, 1, 0) + )))) ); assert_parse!( expression_statement("1.0 ;"), - Statement::Expression(Expression::Simple(Value::Float(value_node!(1.0, 1, 1)))) + Statement::Expression(Expression::Simple(Value::Float(value_node!( + 1.0, + Position::new(1, 1, 0) + )))) ); assert_parse!( expression_statement(" 1.0;"), - Statement::Expression(Expression::Simple(Value::Float(value_node!(1.0, 1, 2)))) + Statement::Expression(Expression::Simple(Value::Float(value_node!( + 1.0, + Position::new(1, 2, 1) + )))) ); assert_parse!( expression_statement("foo;"), Statement::Expression(Expression::Simple(Value::Selector(make_selector!( - make_expr!("foo", 1, 1), - 1, - 1 + make_expr!("foo", Position::new(1, 1, 0)), + Position::new(1, 1, 0) )))) ); assert_parse!( expression_statement("foo ;"), Statement::Expression(Expression::Simple(Value::Selector(make_selector!( - make_expr!("foo", 1, 2), - 1, - 1 + make_expr!("foo", Position::new(1, 2, 1)), + Position::new(1, 1, 0) )))) ); assert_parse!( expression_statement(" foo;"), Statement::Expression(Expression::Simple(Value::Selector(make_selector!( - make_expr!("foo", 1, 2), - 1, - 2 + make_expr!("foo", Position::new(1, 2, 1)), + Position::new(1, 2, 1) )))) ); assert_parse!( expression_statement("\"foo\";"), Statement::Expression(Expression::Simple(Value::Str(value_node!( "foo".to_string(), - 1, - 1 + Position::new(1, 1, 0) )))) ); assert_parse!( expression_statement("\"foo\" ;"), Statement::Expression(Expression::Simple(Value::Str(value_node!( "foo".to_string(), - 1, - 1 + Position::new(1, 1, 0) )))) ); assert_parse!( expression_statement(" \"foo\";"), Statement::Expression(Expression::Simple(Value::Str(value_node!( "foo".to_string(), - 1, - 2 + Position::new(1, 2, 1) )))) ); } @@ -392,29 +450,33 @@ fn test_expression_statement_parse() { fn test_expression_parse() { assert_parse!( expression("NULL "), - Expression::Simple(Value::Empty(Position::new(1, 1))) + Expression::Simple(Value::Empty(Position::new(1, 1, 0))) ); assert_parse!( expression("\"foo\""), - Expression::Simple(Value::Str(value_node!("foo".to_string(), 1, 1))) + Expression::Simple(Value::Str(value_node!( + "foo".to_string(), + Position::new(1, 1, 0) + ))) ); assert_parse!( expression("1"), - Expression::Simple(Value::Int(value_node!(1, 1, 1))) + Expression::Simple(Value::Int(value_node!(1, Position::new(1, 1, 0)))) ); assert_parse!( expression("foo "), Expression::Simple(Value::Selector(make_selector!( - make_expr!("foo", 1, 1), - 1, - 1 + make_expr!("foo", Position::new(1, 1, 0)), + Position::new(1, 1, 0) ))) ); assert_parse!( expression("foo.bar "), - Expression::Simple(Value::Selector(make_selector!(make_expr!("foo", 1, 1) => - [ make_tok!("bar", 1, 5) ] => - 1, 1))) + Expression::Simple(Value::Selector( + make_selector!(make_expr!("foo", Position::new(1, 1, 0)) => + [ make_tok!("bar", Position::new(1, 5, 4)) ] => + Position::new(1, 1, 0)) + )) ); assert_parse!( expression("{foo=1}.foo "), @@ -422,14 +484,14 @@ fn test_expression_parse() { make_selector!(Expression::Simple(Value::Tuple( value_node!(vec![ ( - make_tok!("foo", 1, 2), - Expression::Simple(Value::Int(value_node!(1, 1, 6))), + make_tok!("foo", Position::new(1, 2, 1)), + Expression::Simple(Value::Int(value_node!(1, Position::new(1, 6, 5)))), ), ], - 1, 1), + Position::new(1, 1, 0)), )) => - [ make_tok!("foo", 1, 9) ] => - 1, 1) + [ make_tok!("foo", Position::new(1, 9, 8)) ] => + Position::new(1, 1, 0)) )) ); assert_parse!( @@ -438,49 +500,73 @@ fn test_expression_parse() { make_selector!(Expression::Simple(Value::List( ListDef{ elems: vec![ - Expression::Simple(Value::Int(value_node!(1, 1, 2))), - Expression::Simple(Value::Int(value_node!(2, 1, 5))), + Expression::Simple(Value::Int(value_node!(1, Position::new(1, 2, 1)))), + Expression::Simple(Value::Int(value_node!(2, Position::new(1, 5, 4)))), ], - pos: Position::new(1, 1), + pos: Position::new(1, 1, 0), })) => - [ make_tok!(DIGIT => "1", 1, 8) ] => - 1, 1) + [ make_tok!(DIGIT => "1", Position::new(1, 8, 7)) ] => + Position::new(1, 1, 0)) )) ); assert_parse!( expression("1 + 1"), Expression::Binary(BinaryOpDef { kind: BinaryExprType::Add, - left: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))), - right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 5)))), - pos: Position::new(1, 1), + left: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 1, 0) + )))), + right: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 5, 4) + )))), + pos: Position::new(1, 1, 0), }) ); assert_parse!( expression("1 - 1"), Expression::Binary(BinaryOpDef { kind: BinaryExprType::Sub, - left: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))), - right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 5)))), - pos: Position::new(1, 1), + left: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 1, 0) + )))), + right: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 5, 4) + )))), + pos: Position::new(1, 1, 0), }) ); assert_parse!( expression("1 / 1"), Expression::Binary(BinaryOpDef { kind: BinaryExprType::Div, - left: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))), - right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 5)))), - pos: Position::new(1, 1), + left: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 1, 0) + )))), + right: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 5, 4) + )))), + pos: Position::new(1, 1, 0), }) ); assert_parse!( expression("(1 / 1)"), Expression::Grouped(Box::new(Expression::Binary(BinaryOpDef { kind: BinaryExprType::Div, - left: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 2)))), - right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 6)))), - pos: Position::new(1, 2), + left: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 2, 1) + )))), + right: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 6, 5) + )))), + pos: Position::new(1, 2, 1), }))) ); assert_parse!( @@ -489,12 +575,21 @@ fn test_expression_parse() { kind: BinaryExprType::Add, left: Box::new(Expression::Binary(BinaryOpDef { kind: BinaryExprType::Div, - left: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))), - right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 5)))), - pos: Position::new(1, 1), + left: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 1, 0) + )))), + right: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 5, 4) + )))), + pos: Position::new(1, 1, 0), })), - right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 9)))), - pos: Position::new(1, 1), + right: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 9, 8) + )))), + pos: Position::new(1, 1, 0), }) ); assert_parse!( @@ -504,171 +599,238 @@ fn test_expression_parse() { left: Box::new(Expression::Grouped(Box::new(Expression::Binary( BinaryOpDef { kind: BinaryExprType::Add, - left: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 2)))), - right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 6)))), - pos: Position::new(1, 2), + left: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 2, 1) + )))), + right: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 6, 5) + )))), + pos: Position::new(1, 2, 1), } )))), - right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 11)))), - pos: Position::new(1, 2), + right: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 11, 10) + )))), + pos: Position::new(1, 2, 1), }) ); assert_parse!( expression("1 > 1"), Expression::Compare(ComparisonDef { kind: CompareType::GT, - left: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))), - right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 5)))), - pos: Position::new(1, 1), + left: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 1, 0) + )))), + right: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 5, 4) + )))), + pos: Position::new(1, 1, 0), }) ); assert_parse!( expression("1 < 1"), Expression::Compare(ComparisonDef { kind: CompareType::LT, - left: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))), - right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 5)))), - pos: Position::new(1, 1), + left: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 1, 0) + )))), + right: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 5, 4) + )))), + pos: Position::new(1, 1, 0), }) ); assert_parse!( expression("1 <= 1"), Expression::Compare(ComparisonDef { kind: CompareType::LTEqual, - left: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))), - right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 5)))), - pos: Position::new(1, 1), + left: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 1, 0) + )))), + right: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 5, 4) + )))), + pos: Position::new(1, 1, 0), }) ); assert_parse!( expression("1 >= 1"), Expression::Compare(ComparisonDef { kind: CompareType::GTEqual, - left: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))), - right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 5)))), - pos: Position::new(1, 1), + left: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 1, 0) + )))), + right: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 5, 4) + )))), + pos: Position::new(1, 1, 0), }) ); assert_parse!( expression("1+1"), Expression::Binary(BinaryOpDef { kind: BinaryExprType::Add, - left: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))), - right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 3)))), - pos: Position::new(1, 1), + left: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 1, 0) + )))), + right: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 3, 2) + )))), + pos: Position::new(1, 1, 0), }) ); assert_parse!( expression("1-1"), Expression::Binary(BinaryOpDef { kind: BinaryExprType::Sub, - left: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))), - right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 3)))), - pos: Position::new(1, 1), + left: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 1, 0) + )))), + right: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 3, 2) + )))), + pos: Position::new(1, 1, 0), }) ); assert_parse!( expression("1*1"), Expression::Binary(BinaryOpDef { kind: BinaryExprType::Mul, - left: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))), - right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 3)))), - pos: Position::new(1, 1), + left: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 1, 0) + )))), + right: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 3, 2) + )))), + pos: Position::new(1, 1, 0), }) ); assert_parse!( expression("1/1"), Expression::Binary(BinaryOpDef { kind: BinaryExprType::Div, - left: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))), - right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 3)))), - pos: Position::new(1, 1), + left: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 1, 0) + )))), + right: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 3, 2) + )))), + pos: Position::new(1, 1, 0), }) ); assert_parse!( expression("macro (arg1, arg2) => { foo = arg1 }"), Expression::Macro(MacroDef { argdefs: vec![ - value_node!("arg1".to_string(), 1, 8), - value_node!("arg2".to_string(), 1, 14), + value_node!("arg1".to_string(), Position::new(1, 8, 7)), + value_node!("arg2".to_string(), Position::new(1, 14, 13)), ], fields: vec![( - make_tok!("foo", 1, 25), + make_tok!("foo", Position::new(1, 25, 24)), Expression::Simple(Value::Selector(make_selector!( - make_expr!("arg1", 1, 31), - 1, - 31 + make_expr!("arg1", Position::new(1, 31, 30)), + Position::new(1, 31, 30) ))), )], - pos: Position::new(1, 1), + pos: Position::new(1, 1, 0), }) ); assert_parse!( expression("select foo, 1, { foo = 2 }"), Expression::Select(SelectDef { val: Box::new(Expression::Simple(Value::Selector(make_selector!( - make_expr!("foo", 1, 8), - 1, - 8 + make_expr!("foo", Position::new(1, 8, 7)), + Position::new(1, 8, 7) + )))), + default: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 13, 12) )))), - default: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 13)))), tuple: vec![( - make_tok!("foo", 1, 18), - Expression::Simple(Value::Int(value_node!(2, 1, 24))), + make_tok!("foo", Position::new(1, 18, 17)), + Expression::Simple(Value::Int(value_node!(2, Position::new(1, 24, 23)))), )], - pos: Position::new(1, 1), + pos: Position::new(1, 1, 0), }) ); assert_parse!( expression("foo.bar (1, \"foo\")"), Expression::Call(CallDef { - macroref: make_selector!(make_expr!("foo", 1, 1) => - [ make_tok!("bar", 1, 5) ] => - 1, 1), + macroref: make_selector!(make_expr!("foo", Position::new(1, 1, 0)) => + [ make_tok!("bar", Position::new(1, 5, 4)) ] => + Position::new(1, 1, 0)), arglist: vec![ - Expression::Simple(Value::Int(value_node!(1, 1, 10))), - Expression::Simple(Value::Str(value_node!("foo".to_string(), 1, 13))), + Expression::Simple(Value::Int(value_node!(1, Position::new(1, 10, 9)))), + Expression::Simple(Value::Str(value_node!( + "foo".to_string(), + Position::new(1, 13, 12) + ))), ], - pos: Position::new(1, 1), + pos: Position::new(1, 1, 0), }) ); assert_parse!( expression("(1 + 1)"), Expression::Grouped(Box::new(Expression::Binary(BinaryOpDef { kind: BinaryExprType::Add, - left: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 2)))), - right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 6)))), - pos: Position::new(1, 2), + left: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 2, 1) + )))), + right: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 6, 5) + )))), + pos: Position::new(1, 2, 1), }))) ); assert_parse!( expression("[1, 1]"), Expression::Simple(Value::List(ListDef { elems: vec![ - Expression::Simple(Value::Int(value_node!(1, 1, 2))), - Expression::Simple(Value::Int(value_node!(1, 1, 5))), + Expression::Simple(Value::Int(value_node!(1, Position::new(1, 2, 1)))), + Expression::Simple(Value::Int(value_node!(1, Position::new(1, 5, 4)))), ], - pos: Position::new(1, 1), + pos: Position::new(1, 1, 0), })) ); } #[test] fn test_format_parse() { - assert_error!(format_expression("\"foo")); - assert_error!(format_expression("\"foo\"")); - assert_error!(format_expression("\"foo\" %")); - assert_error!(format_expression("\"foo\" % (, 2")); + assert_fail!(format_expression("\"foo")); + assert_fail!(format_expression("\"foo\"")); + assert_fail!(format_expression("\"foo\" %")); + assert_fail!(format_expression("\"foo\" % (, 2")); assert_parse!( format_expression("\"foo @ @\" % (1, 2)"), Expression::Format(FormatDef { template: "foo @ @".to_string(), args: vec![ - Expression::Simple(Value::Int(value_node!(1, 1, 14))), - Expression::Simple(Value::Int(value_node!(2, 1, 17))), + Expression::Simple(Value::Int(value_node!(1, Position::new(1, 14, 13)))), + Expression::Simple(Value::Int(value_node!(2, Position::new(1, 17, 16)))), ], - pos: Position::new(1, 1), + pos: Position::new(1, 1, 0), }) ); @@ -677,183 +839,229 @@ fn test_format_parse() { Expression::Format(FormatDef { template: "foo @ @".to_string(), args: vec![ - Expression::Simple(Value::Int(value_node!(1, 1, 12))), - Expression::Simple(Value::Int(value_node!(2, 1, 15))), + Expression::Simple(Value::Int(value_node!(1, Position::new(1, 12, 11)))), + Expression::Simple(Value::Int(value_node!(2, Position::new(1, 15, 14)))), ], - pos: Position::new(1, 1), + pos: Position::new(1, 1, 0), }) ); } #[test] fn test_call_parse() { - assert_error!(call_expression("foo")); - assert_error!(call_expression("foo (")); - assert_error!(call_expression("foo (1")); - assert_error!(call_expression("foo (1,")); - assert_error!(call_expression("foo (1,2")); + assert_fail!(call_expression("foo")); + assert_fail!(call_expression("foo (")); + assert_fail!(call_expression("foo (1")); + assert_fail!(call_expression("foo (1,")); + assert_fail!(call_expression("foo (1,2")); assert_parse!( call_expression("foo (1, \"foo\")"), Expression::Call(CallDef { - macroref: make_selector!(make_expr!("foo", 1, 1), 1, 1), + macroref: make_selector!( + make_expr!("foo", Position::new(1, 1, 0)), + Position::new(1, 1, 0) + ), arglist: vec![ - Expression::Simple(Value::Int(value_node!(1, 1, 6))), - Expression::Simple(Value::Str(value_node!("foo".to_string(), 1, 9))), + Expression::Simple(Value::Int(value_node!(1, Position::new(1, 6, 5)))), + Expression::Simple(Value::Str(value_node!( + "foo".to_string(), + Position::new(1, 9, 8) + ))), ], - pos: Position::new(1, 1), + pos: Position::new(1, 1, 0), }) ); assert_parse!( call_expression("foo.bar (1, \"foo\")"), Expression::Call(CallDef { - macroref: make_selector!(make_expr!("foo") => [ make_tok!("bar", 1, 5) ] => 1, 1), + macroref: make_selector!(make_expr!("foo", Position::new(1, 1, 0)) => [ make_tok!("bar", Position::new(1, 5, 4)) ] => Position::new(1, 1, 0)), arglist: vec![ - Expression::Simple(Value::Int(value_node!(1, 1, 10))), - Expression::Simple(Value::Str(value_node!("foo".to_string(), 1, 13))), + Expression::Simple(Value::Int(value_node!(1, Position::new(1, 10, 9)))), + Expression::Simple(Value::Str(value_node!( + "foo".to_string(), + Position::new(1, 13, 12) + ))), ], - pos: Position::new(1, 1), + pos: Position::new(1, 1, 0), }) ); } #[test] fn test_select_parse() { - assert_error!(select_expression("select")); - assert_error!(select_expression("select foo")); - assert_error!(select_expression("select foo, 1")); - assert_error!(select_expression("select foo, 1, {")); + assert_fail!(select_expression("select")); + assert_fail!(select_expression("select foo")); + assert_fail!(select_expression("select foo, 1")); + assert_fail!(select_expression("select foo, 1, {")); assert_parse!( select_expression("select foo, 1, { foo = 2 }"), Expression::Select(SelectDef { val: Box::new(Expression::Simple(Value::Selector(make_selector!( - make_expr!("foo", 1, 8), - 1, - 8 + make_expr!("foo", Position::new(1, 8, 7)), + Position::new(1, 8, 7) + )))), + default: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 13, 12) )))), - default: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 13)))), tuple: vec![( - make_tok!("foo", 1, 18), - Expression::Simple(Value::Int(value_node!(2, 1, 24))), + make_tok!("foo", Position::new(1, 18, 17)), + Expression::Simple(Value::Int(value_node!(2, Position::new(1, 24, 23)))), )], - pos: Position::new(1, 1), + pos: Position::new(1, 1, 0), }) ); } #[test] fn test_macro_expression_parsing() { - assert_error!(macro_expression("foo")); - assert_error!(macro_expression("macro \"foo\"")); - assert_error!(macro_expression("macro 1")); - assert_error!(macro_expression("macro")); - assert_error!(macro_expression("macro (")); - assert_error!(macro_expression("macro (arg")); - assert_error!(macro_expression("macro (arg, arg2")); - assert_error!(macro_expression("macro (arg1, arg2) =>")); - assert_error!(macro_expression("macro (arg1, arg2) => {")); - assert_error!(macro_expression("macro (arg1, arg2) => { foo")); - assert_error!(macro_expression("macro (arg1, arg2) => { foo =")); + assert_fail!(macro_expression("foo")); + assert_fail!(macro_expression("macro \"foo\"")); + assert_fail!(macro_expression("macro 1")); + assert_fail!(macro_expression("macro")); + assert_fail!(macro_expression("macro (")); + assert_fail!(macro_expression("macro (arg")); + assert_fail!(macro_expression("macro (arg, arg2")); + assert_fail!(macro_expression("macro (arg1, arg2) =>")); + assert_fail!(macro_expression("macro (arg1, arg2) => {")); + assert_fail!(macro_expression("macro (arg1, arg2) => { foo")); + assert_fail!(macro_expression("macro (arg1, arg2) => { foo =")); + + assert_parse!( + macro_expression("macro () => {foo=1,bar=2}"), + Expression::Macro(MacroDef { + argdefs: Vec::new(), + fields: vec![ + ( + make_tok!("foo", Position::new(1, 14, 13)), + Expression::Simple(Value::Int(value_node!(1, Position::new(1, 18, 17)))), + ), + ( + make_tok!("bar", Position::new(1, 20, 19)), + Expression::Simple(Value::Int(value_node!(2, Position::new(1, 24, 23)))), + ), + ], + pos: Position::new(1, 1, 0), + }) + ); assert_parse!( macro_expression("macro (arg1, arg2) => {foo=1,bar=2}"), Expression::Macro(MacroDef { argdefs: vec![ - value_node!("arg1".to_string(), 1, 8), - value_node!("arg2".to_string(), 1, 14), + value_node!("arg1".to_string(), Position::new(1, 8, 7)), + value_node!("arg2".to_string(), Position::new(1, 14, 13)), ], fields: vec![ ( - make_tok!("foo", 1, 24), - Expression::Simple(Value::Int(value_node!(1, 1, 28))), + make_tok!("foo", Position::new(1, 24, 23)), + Expression::Simple(Value::Int(value_node!(1, Position::new(1, 28, 27)))), ), ( - make_tok!("bar", 1, 30), - Expression::Simple(Value::Int(value_node!(2, 1, 34))), + make_tok!("bar", Position::new(1, 30, 29)), + Expression::Simple(Value::Int(value_node!(2, Position::new(1, 34, 33)))), ), ], - pos: Position::new(1, 1), + pos: Position::new(1, 1, 0), }) ); } #[test] fn test_copy_parse() { - assert_error!(copy_expression("{}")); - assert_error!(copy_expression("foo")); - assert_error!(copy_expression("foo{")); + assert_fail!(copy_expression("{}")); + assert_fail!(copy_expression("foo")); + assert_fail!(copy_expression("foo{")); assert_parse!( copy_expression("foo{}"), Expression::Copy(CopyDef { - selector: make_selector!(make_expr!("foo", 1, 1), 1, 1), + selector: make_selector!( + make_expr!("foo", Position::new(1, 1, 0)), + Position::new(1, 1, 0) + ), fields: Vec::new(), - pos: Position::new(1, 1), + pos: Position::new(1, 1, 0), }) ); assert_parse!( copy_expression("foo{bar=1}"), Expression::Copy(CopyDef { - selector: make_selector!(make_expr!("foo", 1, 1), 1, 1), + selector: make_selector!( + make_expr!("foo", Position::new(1, 1, 0)), + Position::new(1, 1, 0) + ), fields: vec![( - make_tok!("bar", 1, 5), - Expression::Simple(Value::Int(value_node!(1, 1, 9))), + make_tok!("bar", Position::new(1, 5, 4)), + Expression::Simple(Value::Int(value_node!(1, Position::new(1, 9, 8)))), )], - pos: Position::new(1, 1), + pos: Position::new(1, 1, 0), }) ); assert_parse!( copy_expression("foo{bar=1,}"), Expression::Copy(CopyDef { - selector: make_selector!(make_expr!("foo", 1, 1), 1, 1), + selector: make_selector!( + make_expr!("foo", Position::new(1, 1, 0)), + Position::new(1, 1, 0) + ), fields: vec![( - make_tok!("bar", 1, 5), - Expression::Simple(Value::Int(value_node!(1, 1, 9))), + make_tok!("bar", Position::new(1, 5, 4)), + Expression::Simple(Value::Int(value_node!(1, Position::new(1, 9, 8)))), )], - pos: Position::new(1, 1), + pos: Position::new(1, 1, 0), }) ); } #[test] fn test_grouped_expression_parse() { - assert_error!(grouped_expression("foo")); - assert_error!(grouped_expression("(foo")); + assert_fail!(grouped_expression("foo")); + assert_fail!(grouped_expression("(foo")); assert_parse!( grouped_expression("(foo)"), Expression::Grouped(Box::new(Expression::Simple(Value::Selector( - make_selector!(make_expr!("foo", 1, 2), 1, 2) + make_selector!( + make_expr!("foo", Position::new(1, 2, 1)), + Position::new(1, 2, 1) + ) )))) ); assert_parse!( grouped_expression("(1 + 1)"), Expression::Grouped(Box::new(Expression::Binary(BinaryOpDef { kind: BinaryExprType::Add, - left: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 2)))), - right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 6)))), - pos: Position::new(1, 2), + left: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 2, 1) + )))), + right: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 6, 5) + )))), + pos: Position::new(1, 2, 1), }))) ); } #[test] fn test_list_value_parse() { - assert_error!(list_value("foo")); - assert_error!(list_value("[foo")); - assert_error!(list_value("// commen\n[foo")); + assert_fail!(list_value("foo")); + assert_fail!(list_value("[foo")); + assert_fail!(list_value("// commen\n[foo")); assert_parse!( list_value("[foo]"), Value::List(ListDef { elems: vec![Expression::Simple(Value::Selector(make_selector!( - make_expr!("foo", 1, 2), - 1, - 2 + make_expr!("foo", Position::new(1, 2, 1)), + Position::new(1, 2, 1) )))], - pos: Position::new(1, 1), + pos: Position::new(1, 1, 0), }) ); @@ -861,10 +1069,10 @@ fn test_list_value_parse() { list_value("[1, 1]"), Value::List(ListDef { elems: vec![ - Expression::Simple(Value::Int(value_node!(1, 1, 2))), - Expression::Simple(Value::Int(value_node!(1, 1, 5))), + Expression::Simple(Value::Int(value_node!(1, Position::new(1, 2, 1)))), + Expression::Simple(Value::Int(value_node!(1, Position::new(1, 5, 4)))), ], - pos: Position::new(1, 1), + pos: Position::new(1, 1, 0), }) ); @@ -872,10 +1080,10 @@ fn test_list_value_parse() { list_value("[1, 1,]"), Value::List(ListDef { elems: vec![ - Expression::Simple(Value::Int(value_node!(1, 1, 2))), - Expression::Simple(Value::Int(value_node!(1, 1, 5))), + Expression::Simple(Value::Int(value_node!(1, Position::new(1, 2, 1)))), + Expression::Simple(Value::Int(value_node!(1, Position::new(1, 5, 4)))), ], - pos: Position::new(1, 1), + pos: Position::new(1, 1, 0), }) ); @@ -883,10 +1091,10 @@ fn test_list_value_parse() { list_value("// comment\n[1, 1]"), Value::List(ListDef { elems: vec![ - Expression::Simple(Value::Int(value_node!(1, 2, 2))), - Expression::Simple(Value::Int(value_node!(1, 2, 5))), + Expression::Simple(Value::Int(value_node!(1, Position::new(2, 2, 12)))), + Expression::Simple(Value::Int(value_node!(1, Position::new(2, 5, 14)))), ], - pos: Position::new(2, 1), + pos: Position::new(2, 1, 11), }) ); @@ -894,10 +1102,10 @@ fn test_list_value_parse() { list_value("[// comment\n1, 1]"), Value::List(ListDef { elems: vec![ - Expression::Simple(Value::Int(value_node!(1, 2, 2))), - Expression::Simple(Value::Int(value_node!(1, 2, 5))), + Expression::Simple(Value::Int(value_node!(1, Position::new(2, 2, 12)))), + Expression::Simple(Value::Int(value_node!(1, Position::new(2, 5, 15)))), ], - pos: Position::new(1, 1), + pos: Position::new(1, 1, 0), }) ); @@ -905,10 +1113,10 @@ fn test_list_value_parse() { list_value("[1, // comment\n1]"), Value::List(ListDef { elems: vec![ - Expression::Simple(Value::Int(value_node!(1, 1, 2))), - Expression::Simple(Value::Int(value_node!(1, 2, 1))), + Expression::Simple(Value::Int(value_node!(1, Position::new(1, 2, 1)))), + Expression::Simple(Value::Int(value_node!(1, Position::new(2, 1, 14)))), ], - pos: Position::new(1, 1), + pos: Position::new(1, 1, 0), }) ); @@ -916,35 +1124,37 @@ fn test_list_value_parse() { list_value("[1, 1 // comment\n]"), Value::List(ListDef { elems: vec![ - Expression::Simple(Value::Int(value_node!(1, 1, 2))), - Expression::Simple(Value::Int(value_node!(1, 1, 5))), + Expression::Simple(Value::Int(value_node!(1, Position::new(1, 2, 1)))), + Expression::Simple(Value::Int(value_node!(1, Position::new(1, 5, 4)))), ], - pos: Position::new(1, 1), + pos: Position::new(1, 1, 0), }) ); } #[test] fn test_tuple_parse() { - assert_error!(tuple("{")); - assert_error!(tuple("{ foo")); - assert_error!(tuple("{ foo =")); - assert_error!(tuple("{ foo = 1")); - assert_error!(tuple("{ foo = 1,")); - assert_error!(tuple("{ foo = 1, bar =")); - assert_error!(tuple("// comment\n{ foo = 1, bar =")); + assert_fail!(tuple("{")); + assert_fail!(tuple("{ foo")); + assert_fail!(tuple("{ foo =")); + assert_fail!(tuple("{ foo = 1")); + assert_fail!(tuple("{ foo = 1,")); + assert_fail!(tuple("{ foo = 1, bar =")); + assert_fail!(tuple("// comment\n{ foo = 1, bar =")); - assert_parse!(tuple("{ }"), Value::Tuple(value_node!(vec![], 1, 1))); + assert_parse!( + tuple("{ }"), + Value::Tuple(value_node!(vec![], Position::new(1, 1, 0))) + ); assert_parse!( tuple("{ foo = 1 }"), Value::Tuple(value_node!( vec![( - make_tok!("foo", 1, 3), - Expression::Simple(Value::Int(value_node!(1, 1, 9))), + make_tok!("foo", Position::new(1, 3, 2)), + Expression::Simple(Value::Int(value_node!(1, Position::new(1, 9, 8)))), )], - 1, - 1 + Position::new(1, 1, 0) )) ); @@ -952,11 +1162,10 @@ fn test_tuple_parse() { tuple("// comment\n{ foo = 1 }"), Value::Tuple(value_node!( vec![( - make_tok!("foo", 2, 3), - Expression::Simple(Value::Int(value_node!(1, 2, 9))), + make_tok!("foo", Position::new(2, 3, 13)), + Expression::Simple(Value::Int(value_node!(1, Position::new(2, 9, 19)))), )], - 1, - 1 + Position::new(1, 1, 0) )) ); @@ -964,11 +1173,10 @@ fn test_tuple_parse() { tuple("{// comment\n foo = 1 }"), Value::Tuple(value_node!( vec![( - make_tok!("foo", 2, 2), - Expression::Simple(Value::Int(value_node!(1, 2, 8))), + make_tok!("foo", Position::new(2, 2, 13)), + Expression::Simple(Value::Int(value_node!(1, Position::new(2, 8, 19)))), )], - 1, - 1 + Position::new(1, 1, 0) )) ); @@ -976,11 +1184,10 @@ fn test_tuple_parse() { tuple("{ foo = 1// comment\n }"), Value::Tuple(value_node!( vec![( - make_tok!("foo", 1, 3), - Expression::Simple(Value::Int(value_node!(1, 1, 9))), + make_tok!("foo", Position::new(1, 3, 2)), + Expression::Simple(Value::Int(value_node!(1, Position::new(1, 9, 8)))), )], - 1, - 1 + Position::new(1, 1, 0) )) ); @@ -989,19 +1196,18 @@ fn test_tuple_parse() { Value::Tuple(value_node!( vec![ ( - make_tok!("foo", 1, 3), - Expression::Simple(Value::Int(value_node!(1, 1, 9))), + make_tok!("foo", Position::new(1, 3, 2)), + Expression::Simple(Value::Int(value_node!(1, Position::new(1, 9, 8)))), ), ( - make_tok!("bar", 1, 12), + make_tok!("bar", Position::new(1, 12, 11)), Expression::Simple(Value::Str(value_node!( "1".to_string(), - Position::new(1, 18) + Position::new(1, 18, 17) ))), ), ], - 1, - 1 + Position::new(1, 1, 0) )) ); assert_parse!( @@ -1009,19 +1215,18 @@ fn test_tuple_parse() { Value::Tuple(value_node!( vec![ ( - make_tok!("foo", 1, 3), - Expression::Simple(Value::Int(value_node!(1, 1, 9))), + make_tok!("foo", Position::new(1, 3, 2)), + Expression::Simple(Value::Int(value_node!(1, Position::new(1, 9, 8)))), ), ( - make_tok!("bar", 2, 1), + make_tok!("bar", Position::new(2, 1, 22)), Expression::Simple(Value::Str(value_node!( "1".to_string(), - Position::new(2, 7) + Position::new(2, 7, 28) ))), ), ], - 1, - 1 + Position::new(1, 1, 0) )) ); assert_parse!( @@ -1029,16 +1234,18 @@ fn test_tuple_parse() { Value::Tuple(value_node!( vec![ ( - make_tok!("foo", 1, 3), - Expression::Simple(Value::Int(value_node!(1, Position::new(1, 9)))), + make_tok!("foo", Position::new(1, 3, 2)), + Expression::Simple(Value::Int(value_node!(1, Position::new(1, 9, 8)))), ), ( - make_tok!("bar", 1, 12), - Expression::Simple(Value::Tuple(value_node!(Vec::new(), Position::new(1, 17)))), + make_tok!("bar", Position::new(1, 12, 11)), + Expression::Simple(Value::Tuple(value_node!( + Vec::new(), + Position::new(1, 17, 16) + ))), ), ], - 1, - 1 + Position::new(1, 1, 0) )) ); assert_parse!( @@ -1046,16 +1253,18 @@ fn test_tuple_parse() { Value::Tuple(value_node!( vec![ ( - make_tok!("foo", 1, 3), - Expression::Simple(Value::Int(value_node!(1, Position::new(1, 9)))), + make_tok!("foo", Position::new(1, 3, 2)), + Expression::Simple(Value::Int(value_node!(1, Position::new(1, 9, 8)))), ), ( - make_tok!("bar", 1, 12), - Expression::Simple(Value::Tuple(value_node!(Vec::new(), Position::new(1, 17)))), + make_tok!("bar", Position::new(1, 12, 11)), + Expression::Simple(Value::Tuple(value_node!( + Vec::new(), + Position::new(1, 17, 16) + ))), ), ], - 1, - 1 + Position::new(1, 1, 0) )) ); @@ -1064,26 +1273,35 @@ fn test_tuple_parse() { Expression::Simple(Value::Tuple(value_node!( vec![ ( - make_tok!("foo", 1, 3), - Expression::Simple(Value::Int(value_node!(1, Position::new(1, 9)))), + make_tok!("foo", Position::new(1, 3, 2)), + Expression::Simple(Value::Int(value_node!(1, Position::new(1, 9, 8)))), ), ( - make_tok!("lst", 1, 12), + make_tok!("lst", Position::new(1, 12, 11)), Expression::Simple(Value::List(ListDef { elems: vec![ - Expression::Simple(Value::Int(value_node!(1, Position::new(1, 19)))), - Expression::Simple(Value::Int(value_node!(2, Position::new(1, 22)))), - Expression::Simple(Value::Int(value_node!(3, Position::new(1, 25)))), + Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 19, 18) + ))), + Expression::Simple(Value::Int(value_node!( + 2, + Position::new(1, 22, 21) + ))), + Expression::Simple(Value::Int(value_node!( + 3, + Position::new(1, 25, 24) + ))), ], pos: Position { line: 1, column: 18, + offset: 17, }, })), ), ], - 1, - 1 + Position::new(1, 1, 0) ))) ); } @@ -1094,8 +1312,14 @@ fn test_field_list_parse() { assert_parse!( field_list(f_list), vec![ - (make_tok!("foo", 1, 1), make_expr!(1 => int, 1, 7)), - (make_tok!("quux", 1, 10), make_expr!(2 => int, 1, 17)), + ( + make_tok!("foo", Position::new(1, 1, 0)), + make_expr!(1 => int, Position::new(1, 7, 6)) + ), + ( + make_tok!("quux", Position::new(1, 10, 9)), + make_expr!(2 => int, Position::new(1, 17, 16)) + ), ] ); @@ -1103,8 +1327,14 @@ fn test_field_list_parse() { assert_parse!( field_list(f_list), vec![ - (make_tok!("foo", 1, 1), make_expr!(1 => int, 1, 7)), - (make_tok!("quux", 2, 1), make_expr!(2 => int, 2, 8)), + ( + make_tok!("foo", Position::new(1, 1, 0)), + make_expr!(1 => int, Position::new(1, 7, 6)) + ), + ( + make_tok!("quux", Position::new(2, 1, 20)), + make_expr!(2 => int, Position::new(2, 8, 27)) + ), ] ); @@ -1112,23 +1342,32 @@ fn test_field_list_parse() { assert_parse!( field_list(f_list), vec![ - (make_tok!("foo", 1, 1), make_expr!(1 => int, 1, 7)), - (make_tok!("quux", 3, 1), make_expr!(2 => int, 3, 8)), + ( + make_tok!("foo", Position::new(1, 1, 0)), + make_expr!(1 => int, Position::new(1, 7, 6)) + ), + ( + make_tok!("quux", Position::new(3, 1, 20)), + make_expr!(2 => int, Position::new(3, 8, 28)) + ), ] ); f_list = "foo = 1,\nquux = [1, 2],"; assert_parse!( field_list(f_list), vec![ - (make_tok!("foo", 1, 1), make_expr!(1 => int, 1, 7)), ( - make_tok!("quux", 2, 1), + make_tok!("foo", Position::new(1, 1, 0)), + make_expr!(1 => int, Position::new(1, 7, 6)) + ), + ( + make_tok!("quux", Position::new(2, 1, 9)), Expression::Simple(Value::List(ListDef { elems: vec![ - Expression::Simple(Value::Int(value_node!(1, Position::new(2, 9)))), - Expression::Simple(Value::Int(value_node!(2, Position::new(2, 12)))), + Expression::Simple(Value::Int(value_node!(1, Position::new(2, 9, 17)))), + Expression::Simple(Value::Int(value_node!(2, Position::new(2, 12, 20)))), ], - pos: Position::new(2, 8), + pos: Position::new(2, 8, 16), })), ), ] @@ -1137,75 +1376,77 @@ fn test_field_list_parse() { #[test] fn test_field_value_parse() { - assert_error!(field_value("foo")); - assert_error!(field_value("// comment\nfoo")); - assert_error!(field_value("foo =")); + assert_fail!(field_value("foo")); + assert_fail!(field_value("// comment\nfoo")); + assert_fail!(field_value("foo =")); assert_parse!( field_value("foo = 1"), ( - make_tok!("foo", 1, 1), - Expression::Simple(Value::Int(value_node!(1, 1, 7))) + make_tok!("foo", Position::new(1, 1, 0)), + Expression::Simple(Value::Int(value_node!(1, Position::new(1, 7, 6)))) ) ); assert_parse!( field_value("foo = 1 // foo comment\n"), ( - make_tok!("foo", 1, 1), - Expression::Simple(Value::Int(value_node!(1, 1, 7))) + make_tok!("foo", Position::new(1, 1, 0)), + Expression::Simple(Value::Int(value_node!(1, Position::new(1, 7, 6)))) ) ); assert_parse!( field_value("foo // foo comment\n = 1"), ( - make_tok!("foo", 1, 1), - Expression::Simple(Value::Int(value_node!(1, 2, 4))) + make_tok!("foo", Position::new(1, 1, 0)), + Expression::Simple(Value::Int(value_node!(1, Position::new(2, 4, 4)))) ) ); assert_parse!( field_value("// foo comment\nfoo = 1"), ( - make_tok!("foo", 2, 1), - Expression::Simple(Value::Int(value_node!(1, 2, 7))) + make_tok!("foo", Position::new(2, 1, 15)), + Expression::Simple(Value::Int(value_node!(1, Position::new(2, 7, 21)))) ) ); assert_parse!( field_value("foo = \"1\""), ( - make_tok!("foo", 1, 1), - Expression::Simple(Value::Str(value_node!("1".to_string(), 1, 7))) + make_tok!("foo", Position::new(1, 1, 0)), + Expression::Simple(Value::Str(value_node!( + "1".to_string(), + Position::new(1, 7, 6) + ))) ) ); assert_parse!( field_value("foo = bar "), ( - make_tok!("foo", 1, 1), + make_tok!("foo", Position::new(1, 1, 0)), Expression::Simple(Value::Selector(make_selector!( - make_expr!("bar", 1, 7), - 1, - 7 + make_expr!("bar", Position::new(1, 7, 6)), + Position::new(1, 7, 6) ))) ) ); assert_parse!( field_value("foo = bar.baz "), ( - make_tok!("foo", 1, 1), + make_tok!("foo", Position::new(1, 1, 0)), Expression::Simple(Value::Selector( - make_selector!(make_expr!("bar", 1, 7) => [ make_tok!("baz", 1, 11) ] => 1, 7), + make_selector!(make_expr!("bar", Position::new(1, 7, 6)) => [ make_tok!("baz", Position::new(1, 11, 10)) ] => Position::new(1, 7, 6)), )) ) ); assert_parse!( field_value("foo = [1,2], "), ( - make_tok!("foo", 1, 1), + make_tok!("foo", Position::new(1, 1, 0)), Expression::Simple(Value::List(ListDef { elems: vec![ - Expression::Simple(Value::Int(value_node!(1, Position::new(1, 8)))), - Expression::Simple(Value::Int(value_node!(2, Position::new(1, 10)))), + Expression::Simple(Value::Int(value_node!(1, Position::new(1, 8, 7)))), + Expression::Simple(Value::Int(value_node!(2, Position::new(1, 10, 9)))), ], - pos: Position::new(1, 7), + pos: Position::new(1, 7, 6), })) ) ); @@ -1213,22 +1454,34 @@ fn test_field_value_parse() { #[test] fn test_number_parsing() { - assert_error!(number(".")); - assert_error!(number(". ")); - assert_parse!(number("1.0"), Value::Float(value_node!(1.0, 1, 1))); - assert_parse!(number("1."), Value::Float(value_node!(1.0, 1, 1))); - assert_parse!(number("1"), Value::Int(value_node!(1, 1, 1))); - assert_parse!(number(".1"), Value::Float(value_node!(0.1, 1, 1))); + assert_fail!(number(".")); + assert_fail!(number(". ")); + assert_parse!( + number("1.0"), + Value::Float(value_node!(1.0, Position::new(1, 1, 0))) + ); + assert_parse!( + number("1."), + Value::Float(value_node!(1.0, Position::new(1, 1, 0))) + ); + assert_parse!( + number("1"), + Value::Int(value_node!(1, Position::new(1, 1, 0))) + ); + assert_parse!( + number(".1"), + Value::Float(value_node!(0.1, Position::new(1, 1, 0))) + ); } #[test] fn test_parse() { - let bad_input = LocatedSpan::new("import mylib as lib;"); + let bad_input = OffsetStrIter::new("import mylib as lib;"); let bad_result = parse(bad_input); assert!(bad_result.is_err()); // Valid parsing tree - let input = LocatedSpan::new("import \"mylib\" as lib;let foo = 1;1+1;"); + let input = OffsetStrIter::new("import \"mylib\" as lib;let foo = 1;1+1;"); let result = parse(input); assert!(result.is_ok(), format!("Expected Ok, Got {:?}", result)); let tpl = result.unwrap(); @@ -1236,18 +1489,24 @@ fn test_parse() { tpl, vec![ Statement::Import(ImportDef { - path: make_tok!(QUOT => "mylib", 1, 8), - name: make_tok!("lib", 1, 19), + path: make_tok!(QUOT => "mylib", Position::new(1, 8, 7)), + name: make_tok!("lib", Position::new(1, 19, 18)), }), Statement::Let(LetDef { - name: make_tok!("foo", 1, 27), - value: Expression::Simple(Value::Int(value_node!(1, 1, 33))), + name: make_tok!("foo", Position::new(1, 27, 26)), + value: Expression::Simple(Value::Int(value_node!(1, Position::new(1, 33, 32)))), }), Statement::Expression(Expression::Binary(BinaryOpDef { kind: BinaryExprType::Add, - left: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 35)))), - right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 37)))), - pos: Position::new(1, 35), + left: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 35, 34) + )))), + right: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 37, 36) + )))), + pos: Position::new(1, 35, 34), })), ] ); diff --git a/src/tokenizer/mod.rs b/src/tokenizer/mod.rs index 9ee360a..ec884ff 100644 --- a/src/tokenizer/mod.rs +++ b/src/tokenizer/mod.rs @@ -12,415 +12,424 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! The tokenization stage of the ucg compiler. -use ast::*; -use error; -use nom; -use nom::{alpha, digit, is_alphanumeric, multispace}; -use nom::{InputIter, InputLength, Slice}; -use nom_locate::LocatedSpan; +//! The tokenization stext_tokene of the ucg compiler. use std; -use std::result::Result; -pub type Span<'a> = LocatedSpan<&'a str>; +use abortable_parser::combinators::*; +use abortable_parser::iter::SliceIter; +use abortable_parser::{Error, Offsetable, Result}; -impl<'a> From> for Position { - fn from(s: Span) -> Position { - Position { - line: s.line as usize, - column: s.get_column() as usize, +use ast::*; +use error::StackPrinter; +use iter::OffsetStrIter; + +fn is_symbol_char<'a>(i: OffsetStrIter<'a>) -> Result, u8> { + let mut _i = i.clone(); + let c = match _i.next() { + Some(c) => *c, + None => { + return Result::Fail(Error::new( + "Unexpected End of Input".to_string(), + Box::new(_i.clone()), + )) } + }; + if (c as char).is_ascii_alphanumeric() || c == b'-' || c == b'_' { + Result::Complete(_i, c) + } else { + Result::Fail(Error::new( + "Not a symbol character".to_string(), + Box::new(_i.clone()), + )) } } -fn is_symbol_char(c: char) -> bool { - is_alphanumeric(c as u8) || c == '-' as char || c == '_' as char -} - -fn escapequoted(input: Span) -> nom::IResult { +fn escapequoted<'a>(input: OffsetStrIter<'a>) -> Result, String> { // loop until we find a " that is not preceded by \. // Collapse all \ to just char for escaping. let mut frag = String::new(); let mut escape = false; - for (i, c) in input.iter_indices() { - if c == '\\' && !escape { + let mut _input = input.clone(); + loop { + let c = match _input.next() { + Some(c) => *c, + None => break, + }; + if c == '\\' as u8 && !escape { // eat this slash and set our escaping sentinel escape = true; - } else if c == '"' && !escape { + } else if c == '"' as u8 && !escape { // Bail if this is an unescaped " // we exit here. - return nom::IResult::Done(input.slice(i..), frag); + return Result::Complete(_input, frag); } else { // we accumulate this character. - frag.push(c); + frag.push(c as char); escape = false; // reset our escaping sentinel } } - return nom::IResult::Incomplete(nom::Needed::Unknown); + return Result::Incomplete(_input.clone()); } -named!(strtok( Span ) -> Token, - do_parse!( - span: position!() >> - tag!("\"") >> - frag: escapequoted >> - tag!("\"") >> +make_fn!(strtok, + do_each!( + span => input!(), + _ => text_token!("\""), + frag => escapequoted, (Token{ typ: TokenType::QUOTED, - pos: Position::from(span), - fragment: frag, + pos: Position::from(&span), + fragment: frag.to_string(), }) ) ); -named!(pipequotetok( Span ) -> Token, - do_parse!( - span: position!() >> - tag!("|") >> - frag: take_until!("|") >> - tag!("|") >> +make_fn!(pipequotetok, + do_each!( + p => input!(), + _ => text_token!("|"), + frag => until!(text_token!("|")), + _ => text_token!("|"), (Token{ typ: TokenType::PIPEQUOTE, - pos: Position::from(span), - fragment: frag.fragment.to_string(), + pos: Position::from(&p), + fragment: frag.to_string(), }) ) ); -named!(barewordtok( Span ) -> Token, - do_parse!( - span: position!() >> - frag: preceded!(peek!(alpha), take_while!(is_symbol_char)) >> +make_fn!(barewordtok, + do_each!( + span => input!(), + _ => peek!(ascii_alpha), + frag => consume_all!(is_symbol_char), (Token{ typ: TokenType::BAREWORD, - pos: Position::from(span), - fragment: frag.fragment.to_string(), + pos: Position::from(&span), + fragment: frag.to_string(), }) ) ); -named!(digittok( Span ) -> Token, - do_parse!( - span: position!() >> - digits: digit >> - (Token{ - typ: TokenType::DIGIT, - pos: Position::from(span), - fragment: digits.fragment.to_string(), - }) +make_fn!(digittok, + do_each!( + span => input!(), + _ => peek!(ascii_digit), + digits => consume_all!(ascii_digit), + (Token{ + typ: TokenType::DIGIT, + pos: Position::from(&span), + fragment: digits.to_string(), + }) ) ); -named!(booleantok( Span ) -> Token, - do_parse!( - span: position!() >> - b: alt!( - tag!("true") | - tag!("false") - ) >> +make_fn!(booleantok, + do_each!( + span => input!(), + token => either!( + text_token!("true"), + text_token!("false") + ), (Token{ typ: TokenType::BOOLEAN, - pos: Position::from(span), - fragment: b.fragment.to_string(), + pos: Position::from(&span), + fragment: token.to_string(), }) ) ); -/// do_tag_tok! is a helper macro to make building a simple tag token +/// do_text_token_tok! is a helper macro to make building a simple text_token token /// less code. -macro_rules! do_tag_tok { - // NOTE(jwall): Nom macros do magic with their inputs. They in fact - // rewrite your macro argumets for you by adding an initial argument - // for all their sub-macros. Which means we require this $i paramater - // on the first macro invocation but not the rest. - ($i:expr, $type:expr, $tag:expr,WS) => { - do_parse!( - $i, - span: position!() >> frag: tag!($tag) >> alt!(whitespace | comment) >> (Token { - typ: $type, - pos: Position::from(span), - fragment: frag.fragment.to_string(), - }) - ) +macro_rules! do_text_token_tok { + ($i:expr, $type:expr, $text_token:expr, WS) => { + do_each!($i, + span => input!(), + frag => text_token!($text_token), + _ => either!(whitespace, comment), + (Token { + typ: $type, + pos: Position::from(&span), + fragment: frag.to_string(), + }) + ) }; - ($i:expr, $type:expr, $tag:expr) => { - do_parse!( - $i, - span: position!() >> frag: tag!($tag) >> (Token { - typ: $type, - pos: Position::from(span), - fragment: frag.fragment.to_string(), - }) - ) + + ($i:expr, $type:expr, $text_token:expr) => { + do_each!($i, + span => input!(), + frag => text_token!($text_token), + (Token { + typ: $type, + pos: Position::from(&span), + fragment: frag.to_string(), + }) + ) }; } -named!(emptytok( Span ) -> Token, - do_tag_tok!(TokenType::EMPTY, "NULL") +make_fn!(emptytok, + do_text_token_tok!(TokenType::EMPTY, "NULL") ); -named!(commatok( Span ) -> Token, - do_tag_tok!(TokenType::PUNCT, ",") +make_fn!(commatok, + do_text_token_tok!(TokenType::PUNCT, ",") ); -named!(lbracetok( Span ) -> Token, - do_tag_tok!(TokenType::PUNCT, "{") +make_fn!(lbracetok, + do_text_token_tok!(TokenType::PUNCT, "{") ); -named!(rbracetok( Span ) -> Token, - do_tag_tok!(TokenType::PUNCT, "}") +make_fn!(rbracetok, + do_text_token_tok!(TokenType::PUNCT, "}") ); -named!(lparentok( Span ) -> Token, - do_tag_tok!(TokenType::PUNCT, "(") +make_fn!(lparentok, + do_text_token_tok!(TokenType::PUNCT, "(") ); -named!(rparentok( Span ) -> Token, - do_tag_tok!(TokenType::PUNCT, ")") +make_fn!(rparentok, + do_text_token_tok!(TokenType::PUNCT, ")") ); -named!(dottok( Span ) -> Token, - do_tag_tok!(TokenType::PUNCT, ".") +make_fn!(dottok, + do_text_token_tok!(TokenType::PUNCT, ".") ); -named!(plustok( Span ) -> Token, - do_tag_tok!(TokenType::PUNCT, "+") +make_fn!(plustok, + do_text_token_tok!(TokenType::PUNCT, "+") ); -named!(dashtok( Span ) -> Token, - do_tag_tok!(TokenType::PUNCT, "-") +make_fn!(dashtok, + do_text_token_tok!(TokenType::PUNCT, "-") ); -named!(startok( Span ) -> Token, - do_tag_tok!(TokenType::PUNCT, "*") +make_fn!(startok, + do_text_token_tok!(TokenType::PUNCT, "*") ); -named!(slashtok( Span ) -> Token, - do_tag_tok!(TokenType::PUNCT, "/") +make_fn!(slashtok, + do_text_token_tok!(TokenType::PUNCT, "/") ); -named!(pcttok( Span ) -> Token, - do_tag_tok!(TokenType::PUNCT, "%") +make_fn!(pcttok, + do_text_token_tok!(TokenType::PUNCT, "%") ); -named!(eqeqtok( Span ) -> Token, - do_tag_tok!(TokenType::PUNCT, "==") +make_fn!(eqeqtok, + do_text_token_tok!(TokenType::PUNCT, "==") ); -named!(notequaltok( Span ) -> Token, - do_tag_tok!(TokenType::PUNCT, "!=") +make_fn!(notequaltok, + do_text_token_tok!(TokenType::PUNCT, "!=") ); -named!(gttok( Span ) -> Token, - do_tag_tok!(TokenType::PUNCT, ">") +make_fn!(gttok, + do_text_token_tok!(TokenType::PUNCT, ">") ); -named!(gtequaltok( Span ) -> Token, - do_tag_tok!(TokenType::PUNCT, ">=") +make_fn!(gtequaltok, + do_text_token_tok!(TokenType::PUNCT, ">=") ); -named!(ltequaltok( Span ) -> Token, - do_tag_tok!(TokenType::PUNCT, "<=") +make_fn!(ltequaltok, + do_text_token_tok!(TokenType::PUNCT, "<=") ); -named!(lttok( Span ) -> Token, - do_tag_tok!(TokenType::PUNCT, "<") +make_fn!(lttok, + do_text_token_tok!(TokenType::PUNCT, "<") ); -named!(equaltok( Span ) -> Token, - do_tag_tok!(TokenType::PUNCT, "=") +make_fn!(equaltok, + do_text_token_tok!(TokenType::PUNCT, "=") ); -named!(semicolontok( Span ) -> Token, - do_tag_tok!(TokenType::PUNCT, ";") +make_fn!(semicolontok, + do_text_token_tok!(TokenType::PUNCT, ";") ); -named!(leftsquarebracket( Span ) -> Token, - do_tag_tok!(TokenType::PUNCT, "[") +make_fn!(leftsquarebracket, + do_text_token_tok!(TokenType::PUNCT, "[") ); -named!(rightsquarebracket( Span ) -> Token, - do_tag_tok!(TokenType::PUNCT, "]") +make_fn!(rightsquarebracket, + do_text_token_tok!(TokenType::PUNCT, "]") ); -named!(fatcommatok( Span ) -> Token, - do_tag_tok!(TokenType::PUNCT, "=>") +make_fn!(fatcommatok, + do_text_token_tok!(TokenType::PUNCT, "=>") ); -named!(selecttok( Span ) -> Token, - do_tag_tok!(TokenType::BAREWORD, "select", WS) +make_fn!(selecttok, + do_text_token_tok!(TokenType::BAREWORD, "select", WS) ); -named!(macrotok( Span ) -> Token, - do_tag_tok!(TokenType::BAREWORD, "macro", WS) +make_fn!(macrotok, + do_text_token_tok!(TokenType::BAREWORD, "macro", WS) ); -named!(lettok( Span ) -> Token, - do_tag_tok!(TokenType::BAREWORD, "let", WS) +make_fn!(lettok, + do_text_token_tok!(TokenType::BAREWORD, "let", WS) ); -named!(importtok( Span ) -> Token, - do_tag_tok!(TokenType::BAREWORD, "import", WS) +make_fn!(importtok, + do_text_token_tok!(TokenType::BAREWORD, "import", WS) ); -named!(asserttok( Span ) -> Token, - do_tag_tok!(TokenType::BAREWORD, "assert", WS) +make_fn!(asserttok, + do_text_token_tok!(TokenType::BAREWORD, "assert", WS) ); -named!(outtok( Span ) -> Token, - do_tag_tok!(TokenType::BAREWORD, "out", WS) +make_fn!(outtok, + do_text_token_tok!(TokenType::BAREWORD, "out", WS) ); -named!(astok( Span ) -> Token, - do_tag_tok!(TokenType::BAREWORD, "as", WS) +make_fn!(astok, + do_text_token_tok!(TokenType::BAREWORD, "as", WS) ); -named!(maptok( Span ) -> Token, - do_tag_tok!(TokenType::BAREWORD, "map", WS) +make_fn!(maptok, + do_text_token_tok!(TokenType::BAREWORD, "map", WS) ); -named!(filtertok( Span ) -> Token, - do_tag_tok!(TokenType::BAREWORD, "filter", WS) +make_fn!(filtertok, + do_text_token_tok!(TokenType::BAREWORD, "filter", WS) ); -fn end_of_input(input: Span) -> nom::IResult { - match eof!(input,) { - nom::IResult::Done(_, _) => { - return nom::IResult::Done( - input, - make_tok!(EOF => input.line as usize, - input.get_column() as usize), - ); - } - nom::IResult::Incomplete(_) => { - return nom::IResult::Incomplete(nom::Needed::Unknown); - } - nom::IResult::Error(e) => { - return nom::IResult::Error(e); - } - } -} - -fn comment(input: Span) -> nom::IResult { - match tag!(input, "//") { - nom::IResult::Done(rest, _) => { - match alt!( +fn comment(input: OffsetStrIter) -> Result { + match text_token!(input, "//") { + Result::Complete(rest, _) => { + match until!( rest, - take_until_and_consume!("\r\n") | take_until_and_consume!("\n") + either!( + eoi, + discard!(text_token!("\r\n")), + discard!(text_token!("\n")) + ) ) { - nom::IResult::Done(rest, cmt) => { - return nom::IResult::Done( - rest, - make_tok!(CMT => cmt.fragment.to_string(), - input.line as usize, - input.get_column() as usize), - ); + Result::Complete(rest, cmt) => { + return Result::Complete(rest, make_tok!(CMT => cmt.to_string(), input)); } // If we didn't find a new line then we just grab everything. _ => { - let blen = rest.input_len(); - let next = rest.slice(blen..); - let tok = rest.slice(..blen); - return nom::IResult::Done( - next, - make_tok!(CMT => tok.fragment.to_string(), - input.line as usize, input.get_column() as usize - ), - ); + return Result::Abort(Error::new( + "Unparsable comment".to_string(), + Box::new(rest.clone()), + )); } } } - nom::IResult::Incomplete(i) => return nom::IResult::Incomplete(i), - nom::IResult::Error(e) => return nom::IResult::Error(e), + Result::Incomplete(ctx) => return Result::Incomplete(ctx), + Result::Fail(e) => return Result::Fail(e), + Result::Abort(e) => return Result::Abort(e), } } -named!(whitespace( Span ) -> Token, - do_parse!( - span: position!() >> - many1!(multispace) >> +make_fn!(whitespace, + do_each!( + span => input!(), + _ => peek!(ascii_ws), + _ => repeat!(ascii_ws), (Token{ typ: TokenType::WS, - pos: Position::from(span), + pos: Position::from(&span), fragment: String::new(), }) ) ); -named!(token( Span ) -> Token, - alt!( - strtok | - pipequotetok | - emptytok | // This must come before the barewordtok - digittok | - commatok | - rbracetok | - lbracetok | - lparentok | - rparentok | - dottok | - plustok | - dashtok | - startok | - comment | // Note comment must come before slashtok - slashtok | - pcttok | - eqeqtok | - notequaltok | - complete!(gtequaltok) | - complete!(ltequaltok) | - gttok | - lttok | - fatcommatok | // Note fatcommatok must come before equaltok - equaltok | - semicolontok | - leftsquarebracket | - rightsquarebracket | - booleantok | - lettok | - outtok | - selecttok | - asserttok | - macrotok | - importtok | - astok | - maptok | - filtertok | - barewordtok | - whitespace | - end_of_input) +make_fn!(end_of_input, + do_each!( + span => input!(), + _ => eoi, + (Token{ + typ: TokenType::END, + pos: Position::from(&span), + fragment: String::new(), + }) + ) ); -/// Consumes an input Span and returns either a Vec or a nom::ErrorKind. -pub fn tokenize(input: Span) -> Result, error::Error> { +fn token<'a>(input: OffsetStrIter<'a>) -> Result, Token> { + either!( + input, + strtok, + pipequotetok, + emptytok, // This must come before the barewordtok + digittok, + commatok, + rbracetok, + lbracetok, + lparentok, + rparentok, + dottok, + plustok, + dashtok, + startok, + comment, // Note comment must come before slashtok + slashtok, + pcttok, + eqeqtok, + notequaltok, + complete!("Not >=".to_string(), gtequaltok), + complete!("Not <=".to_string(), ltequaltok), + gttok, + lttok, + fatcommatok, // Note fatcommatok must come before equaltok + equaltok, + semicolontok, + leftsquarebracket, + rightsquarebracket, + booleantok, + lettok, + outtok, + selecttok, + asserttok, + macrotok, + importtok, + astok, + maptok, + filtertok, + barewordtok, + whitespace, + end_of_input + ) +} + +/// Consumes an input OffsetStrIter and returns either a Vec or a error::Error. +pub fn tokenize<'a>(input: OffsetStrIter<'a>) -> std::result::Result, String> { let mut out = Vec::new(); - let mut i = input; + let mut i = input.clone(); loop { - if i.input_len() == 0 { + if let Result::Complete(_, _) = eoi(i.clone()) { break; } - match token(i) { - nom::IResult::Error(_e) => { - return Err(error::Error::new( + match token(i.clone()) { + Result::Abort(e) => { + let err = abortable_parser::Error::caused_by( "Invalid Token encountered", - error::ErrorType::UnexpectedToken, - Position { - line: i.line as usize, - column: i.get_column() as usize, - }, - )); + Box::new(e), + Box::new(i.clone()), + ); + let ctx_err = StackPrinter { err: err }; + return Err(format!("{}", ctx_err)); } - nom::IResult::Incomplete(_) => { - return Err(error::Error::new( - "Unexepcted end of Input", - error::ErrorType::UnexpectedToken, - Position { - line: i.line as usize, - column: i.get_column() as usize, - }, - )); + Result::Fail(e) => { + let err = abortable_parser::Error::caused_by( + "Invalid Token encountered", + Box::new(e), + Box::new(i.clone()), + ); + let ctx_err = StackPrinter { err: err }; + return Err(format!("{}", ctx_err)); } - nom::IResult::Done(rest, tok) => { + Result::Incomplete(_offset) => { + let err = + abortable_parser::Error::new("Invalid Token encountered", Box::new(i.clone())); + let ctx_err = StackPrinter { err: err }; + return Err(format!("{}", ctx_err)); + } + Result::Complete(rest, tok) => { i = rest; if tok.typ == TokenType::COMMENT || tok.typ == TokenType::WS { // we skip comments and whitespace @@ -434,10 +443,7 @@ pub fn tokenize(input: Span) -> Result, error::Error> { out.push(Token { fragment: String::new(), typ: TokenType::END, - pos: Position { - line: i.line as usize, - column: i.get_column() as usize, - }, + pos: Position::from(&i), }); Ok(out) } @@ -445,7 +451,7 @@ pub fn tokenize(input: Span) -> Result, error::Error> { /// Clones a token. /// /// This is necessary to allow the match_type and match_token macros to work. -pub fn token_clone(t: &Token) -> Result { +pub fn token_clone(t: &Token) -> std::result::Result>> { Ok(t.clone()) } @@ -517,28 +523,28 @@ macro_rules! match_type { }; ($i:expr, $t:expr, $msg:expr, $h:expr) => {{ - let i_ = $i.clone(); - use nom::Slice; - use std::convert::Into; - if i_.input_len() == 0 { - nom::IResult::Error(nom::ErrorKind::Custom(error::Error::new( - format!("End of Input! {}", $msg), - error::ErrorType::IncompleteParsing, - Position { line: 0, column: 0 }, - ))) + use abortable_parser::combinators::eoi; + use abortable_parser::{Error, Result}; + use std; + + let mut _i = $i.clone(); + if eoi(_i.clone()).is_complete() { + Result::Fail(Error::new(format!("End of Input! {}", $msg), Box::new(_i))) } else { - let tok = &(i_[0]); - if tok.typ == $t { - match $h(tok) { - Result::Ok(v) => nom::IResult::Done($i.slice(1..), v), - Result::Err(e) => nom::IResult::Error(nom::ErrorKind::Custom(e.into())), + match _i.next() { + Some(tok) => { + if tok.typ == $t { + match $h(tok) { + std::result::Result::Ok(v) => Result::Complete(_i.clone(), v), + std::result::Result::Err(e) => { + Result::Fail(Error::caused_by($msg, Box::new(e), Box::new(_i))) + } + } + } else { + Result::Fail(Error::new($msg.to_string(), Box::new($i))) + } } - } else { - nom::IResult::Error(nom::ErrorKind::Custom(error::Error::new( - $msg.to_string(), - error::ErrorType::UnexpectedToken, - tok.pos.clone(), - ))) + None => Result::Fail(Error::new($msg.to_string(), Box::new($i))), } } }}; @@ -553,7 +559,7 @@ macro_rules! match_token { }}; ($i:expr,PUNCT => $f:expr, $h:expr) => { - match_token!($i, TokenType::PUNCT, $f, format!("Not PUNCT ({})", $f), $h) + match_token!($i, TokenType::PUNCT, $f, format!("({})", $f), $h) }; ($i:expr,BAREWORD => $f:expr) => {{ @@ -572,22 +578,26 @@ macro_rules! match_token { }; ($i:expr, $t:expr, $f:expr, $msg:expr, $h:expr) => {{ - let i_ = $i.clone(); - use nom; - use nom::Slice; - use std::convert::Into; - let tok = &(i_[0]); - if tok.typ == $t && &tok.fragment == $f { - match $h(tok) { - Result::Ok(v) => nom::IResult::Done($i.slice(1..), v), - Result::Err(e) => nom::IResult::Error(nom::ErrorKind::Custom(e.into())), + use abortable_parser::Result; + use std; + let mut i_ = $i.clone(); + let tok = i_.next(); + if let Some(tok) = tok { + if tok.typ == $t && &tok.fragment == $f { + match $h(tok) { + std::result::Result::Ok(v) => Result::Complete(i_.clone(), v), + std::result::Result::Err(e) => { + Result::Fail(Error::caused_by($msg, Box::new(e), Box::new(i_))) + } + } + } else { + Result::Fail(Error::new( + format!("Expected {} Instead is ({})", $msg, tok.fragment), + Box::new(i_), + )) } } else { - nom::IResult::Error(nom::ErrorKind::Custom(error::Error::new( - format!("{} Instead is ({})", $msg, tok.fragment), - error::ErrorType::UnexpectedToken, - tok.pos.clone(), - ))) + Result::Fail(Error::new("Unexpected End Of Input", Box::new(i_))) } }}; } @@ -607,103 +617,12 @@ macro_rules! word { } /// pos gets the current position from a TokenIter input without consuming it. -pub fn pos(i: TokenIter) -> nom::IResult { - let tok = &i[0]; +pub fn pos<'a>(i: SliceIter<'a, Token>) -> Result, Position> { + let mut _i = i.clone(); + let tok = _i.next().unwrap(); let line = tok.pos.line; let column = tok.pos.column; - nom::IResult::Done( - i.clone(), - Position { - line: line, - column: column, - }, - ) -} - -/// TokenIter wraps a slice of Tokens and implements the various necessary -/// nom traits to use it as an input to nom parsers. -#[derive(Clone, Debug, PartialEq)] -pub struct TokenIter<'a> { - pub source: &'a [Token], -} - -impl<'a> TokenIter<'a> { - pub fn len(&self) -> usize { - self.source.len() - } -} - -impl<'a> nom::InputLength for TokenIter<'a> { - fn input_len(&self) -> usize { - self.source.input_len() - } -} - -macro_rules! impl_token_iter_slice { - ($r:ty) => { - impl<'a> nom::Slice<$r> for TokenIter<'a> { - fn slice(&self, range: $r) -> Self { - TokenIter { - source: self.source.slice(range), - } - } - } - }; -} - -impl_token_iter_slice!(std::ops::Range); -impl_token_iter_slice!(std::ops::RangeTo); -impl_token_iter_slice!(std::ops::RangeFrom); -impl_token_iter_slice!(std::ops::RangeFull); - -impl<'a> std::ops::Index for TokenIter<'a> { - type Output = Token; - - fn index(&self, i: usize) -> &Self::Output { - &self.source[i] - } -} - -impl<'a> InputIter for TokenIter<'a> { - type Item = &'a Token; - type RawItem = Token; - - type Iter = std::iter::Enumerate>; - type IterElem = std::slice::Iter<'a, Self::RawItem>; - - fn iter_indices(&self) -> Self::Iter { - self.source.iter().enumerate() - } - - fn iter_elements(&self) -> Self::IterElem { - self.source.iter() - } - - fn position

(&self, predicate: P) -> Option - where - P: Fn(Self::RawItem) -> bool, - { - for (o, v) in self.iter_indices() { - if predicate(v.clone()) { - return Some(o); - } - } - None - } - - fn slice_index(&self, count: usize) -> Option { - let mut cnt = 0; - for (index, _) in self.iter_indices() { - if cnt == count { - return Some(index); - } - cnt += 1; - } - if cnt == count { - return Some(self.len()); - } - None - } + Result::Complete(i.clone(), Position::new(line, column, i.get_offset())) } #[cfg(test)] diff --git a/src/tokenizer/test.rs b/src/tokenizer/test.rs index 5f1dec2..1bc9ce0 100644 --- a/src/tokenizer/test.rs +++ b/src/tokenizer/test.rs @@ -1,12 +1,17 @@ use super::*; -use nom; -use nom_locate::LocatedSpan; + +use abortable_parser::{Result, SliceIter}; + +use iter::OffsetStrIter; #[test] fn test_empty_token() { - let result = emptytok(LocatedSpan::new("NULL ")); - assert!(result.is_done(), format!("result {:?} is not done", result)); - if let nom::IResult::Done(_, tok) = result { + let result = emptytok(OffsetStrIter::new("NULL ")); + assert!( + result.is_complete(), + format!("result {:?} is not done", result) + ); + if let Result::Complete(_, tok) = result { assert_eq!(tok.fragment, "NULL"); assert_eq!(tok.typ, TokenType::EMPTY); } @@ -14,9 +19,12 @@ fn test_empty_token() { #[test] fn test_assert_token() { - let result = asserttok(LocatedSpan::new("assert ")); - assert!(result.is_done(), format!("result {:?} is not done", result)); - if let nom::IResult::Done(_, tok) = result { + let result = asserttok(OffsetStrIter::new("assert ")); + assert!( + result.is_complete(), + format!("result {:?} is not done", result) + ); + if let Result::Complete(_, tok) = result { assert_eq!(tok.fragment, "assert"); assert_eq!(tok.typ, TokenType::BAREWORD); } @@ -24,29 +32,56 @@ fn test_assert_token() { #[test] fn test_out_token() { - let result = outtok(LocatedSpan::new("out ")); - assert!(result.is_done(), format!("result {:?} is not done", result)); - if let nom::IResult::Done(_, tok) = result { + let result = outtok(OffsetStrIter::new("out ")); + assert!( + result.is_complete(), + format!("result {:?} is not done", result) + ); + if let Result::Complete(_, tok) = result { assert_eq!(tok.fragment, "out"); assert_eq!(tok.typ, TokenType::BAREWORD); } } +#[test] +fn test_out_token_with_comment() { + let result = outtok(OffsetStrIter::new("out//comment")); + assert!( + result.is_complete(), + format!("result {:?} is not done", result) + ); + if let Result::Complete(_, tok) = result { + assert_eq!(tok.fragment, "out"); + assert_eq!(tok.typ, TokenType::BAREWORD); + } +} + +#[test] +fn test_not_out_token() { + let result = outtok(OffsetStrIter::new("output")); + assert!(result.is_fail(), format!("result {:?} is not fail", result)); +} + #[test] fn test_escape_quoted() { - let result = escapequoted(LocatedSpan::new("foo \\\"bar\"")); - assert!(result.is_done(), format!("result {:?} is not ok", result)); - if let nom::IResult::Done(rest, frag) = result { + let result = escapequoted(OffsetStrIter::new("foo \\\"bar\"")); + assert!( + result.is_complete(), + format!("result {:?} is not ok", result) + ); + if let Result::Complete(_rest, frag) = result { assert_eq!(frag, "foo \"bar"); - assert_eq!(rest.fragment, "\""); } } #[test] fn test_pipe_quoted() { - let result = pipequotetok(LocatedSpan::new("|foo|")); - assert!(result.is_done(), format!("result {:?} is not ok", result)); - if let nom::IResult::Done(_, tok) = result { + let result = pipequotetok(OffsetStrIter::new("|foo|")); + assert!( + result.is_complete(), + format!("result {:?} is not ok", result) + ); + if let Result::Complete(_, tok) = result { assert_eq!(tok.fragment, "foo".to_string()); assert_eq!(tok.typ, TokenType::PIPEQUOTE); } @@ -54,16 +89,20 @@ fn test_pipe_quoted() { #[test] fn test_string_with_escaping() { - let result = strtok(LocatedSpan::new("\"foo \\\\ \\\"bar\"")); - assert!(result.is_done(), format!("result {:?} is not ok", result)); - if let nom::IResult::Done(_, tok) = result { + let result = strtok(OffsetStrIter::new("\"foo \\\\ \\\"bar\"")); + assert!( + result.is_complete(), + format!("result {:?} is not ok", result) + ); + if let Result::Complete(_, tok) = result { assert_eq!(tok.fragment, "foo \\ \"bar".to_string()); } } #[test] fn test_tokenize_bareword_with_dash() { - let result = tokenize(LocatedSpan::new("foo-bar ")); + let input = OffsetStrIter::new("foo-bar "); + let result = tokenize(input.clone()); assert!(result.is_ok(), format!("result {:?} is not ok", result)); if let Ok(toks) = result { assert_eq!(toks.len(), 2); @@ -73,18 +112,23 @@ fn test_tokenize_bareword_with_dash() { macro_rules! assert_token { ($input:expr, $typ:expr, $msg:expr) => { - let result = token(LocatedSpan::new($input)); + let result = token(OffsetStrIter::new($input)); assert!( - result.is_done(), + result.is_complete(), format!("result {:?} is not a {}", result, $msg) ); - if let nom::IResult::Done(_, tok) = result { - assert_eq!(tok.fragment, $input); + if let Result::Complete(_, tok) = result { assert_eq!(tok.typ, $typ); + assert_eq!(tok.fragment, $input); } }; } +#[test] +fn test_digittok() { + assert_token!("1", TokenType::DIGIT, "1"); +} + #[test] fn test_boolean() { assert_token!("true", TokenType::BOOLEAN, "boolean"); @@ -122,10 +166,11 @@ fn test_lteqtok() { #[test] fn test_tokenize_one_of_each() { - let result = tokenize(LocatedSpan::new( + let input = OffsetStrIter::new( "map out filter assert let import macro select as => [ ] { } ; = % / * \ + - . ( ) , 1 . foo \"bar\" // comment\n ; true false == < > <= >= !=", - )); + ); + let result = tokenize(input.clone()); assert!(result.is_ok(), format!("result {:?} is not ok", result)); let v = result.unwrap(); for (i, t) in v.iter().enumerate() { @@ -137,65 +182,80 @@ fn test_tokenize_one_of_each() { #[test] fn test_parse_has_end() { - let result = tokenize(LocatedSpan::new("foo")); + let input = OffsetStrIter::new("foo"); + let result = tokenize(input.clone()); assert!(result.is_ok()); let v = result.unwrap(); assert_eq!(v.len(), 2); assert_eq!(v[1].typ, TokenType::END); } +#[test] +fn test_whitespace() { + assert!(whitespace(OffsetStrIter::new(" ")).is_complete()); + let result = whitespace(OffsetStrIter::new(" ")); + match result { + Result::Complete(rest, o) => { + assert_eq!(rest.get_offset(), 2); + assert_eq!(o.typ, TokenType::WS); + } + _ => assert!(false, "Not complete"), + } +} + #[test] fn test_parse_comment() { - assert!(comment(LocatedSpan::new("// comment\n")).is_done()); - assert!(comment(LocatedSpan::new("// comment")).is_done()); - assert_eq!( - comment(LocatedSpan::new("// comment\n")), - nom::IResult::Done( - LocatedSpan { - fragment: "", - offset: 11, - line: 2, - }, + assert!(comment(OffsetStrIter::new("// comment\n")).is_complete()); + assert!(comment(OffsetStrIter::new("// comment")).is_complete()); + let mut parsed = comment(OffsetStrIter::new("// comment\n")); + assert!(parsed.is_complete()); + if let Result::Complete(_rest, cmt) = parsed { + assert_eq!( + cmt, Token { typ: TokenType::COMMENT, fragment: " comment".to_string(), - pos: Position { line: 1, column: 1 }, + pos: Position { + line: 1, + column: 1, + offset: 0 + }, } - ) - ); - assert!(comment(LocatedSpan::new("// comment\r\n")).is_done()); - assert_eq!( - comment(LocatedSpan::new("// comment\r\n")), - nom::IResult::Done( - LocatedSpan { - fragment: "", - offset: 12, - line: 2, - }, + ); + } + assert!(comment(OffsetStrIter::new("// comment\r\n")).is_complete()); + parsed = comment(OffsetStrIter::new("// comment\r\n")); + if let Result::Complete(_rest, cmt) = parsed { + assert_eq!( + cmt, Token { typ: TokenType::COMMENT, fragment: " comment".to_string(), - pos: Position { column: 1, line: 1 }, + pos: Position { + column: 1, + line: 1, + offset: 0 + }, } - ) - ); - assert!(comment(LocatedSpan::new("// comment\r\n ")).is_done()); - assert_eq!( - comment(LocatedSpan::new("// comment\r\n ")), - nom::IResult::Done( - LocatedSpan { - fragment: " ", - offset: 12, - line: 2, - }, + ); + } + assert!(comment(OffsetStrIter::new("// comment\r\n ")).is_complete()); + parsed = comment(OffsetStrIter::new("// comment\r\n ")); + if let Result::Complete(_rest, cmt) = parsed { + assert_eq!( + cmt, Token { typ: TokenType::COMMENT, fragment: " comment".to_string(), - pos: Position { column: 1, line: 1 }, + pos: Position { + column: 1, + line: 1, + offset: 0 + }, } - ) - ); - assert!(comment(LocatedSpan::new("// comment")).is_done()); + ); + } + assert!(comment(OffsetStrIter::new("// comment")).is_complete()); } #[test] @@ -203,16 +263,15 @@ fn test_match_word() { let input = vec![Token { fragment: "foo".to_string(), typ: TokenType::BAREWORD, - pos: Position { line: 1, column: 1 }, - }]; - let result = word!( - TokenIter { - source: input.as_slice(), + pos: Position { + line: 1, + column: 1, + offset: 0, }, - "foo" - ); + }]; + let result = word!(SliceIter::new(input.as_slice()), "foo"); match result { - nom::IResult::Done(_, tok) => assert_eq!(tok, input[0]), + Result::Complete(_, tok) => assert_eq!(tok, input[0]), res => assert!(false, format!("Fail: {:?}", res)), } } @@ -222,22 +281,20 @@ fn test_match_word_empty_input() { let input = vec![Token { fragment: "".to_string(), typ: TokenType::END, - pos: Position { line: 1, column: 1 }, - }]; - let result = word!( - TokenIter { - source: input.as_slice(), + pos: Position { + line: 1, + column: 1, + offset: 0, }, - "foo" - ); + }]; + let result = word!(SliceIter::new(input.as_slice()), "foo"); match result { - nom::IResult::Done(_, _) => assert!(false, "Should have been an error but was Done"), - nom::IResult::Incomplete(_) => { - assert!(false, "Should have been an error but was Incomplete") - } - nom::IResult::Error(_) => { + Result::Complete(_, _) => assert!(false, "Should have been an error but was Done"), + Result::Incomplete(_) => assert!(false, "Should have been a Fail but was Incomplete"), + Result::Fail(_) => { // noop } + Result::Abort(_) => assert!(false, "Should have been a Fail but was Abort"), } } @@ -246,16 +303,15 @@ fn test_match_punct() { let input = vec![Token { fragment: "!".to_string(), typ: TokenType::PUNCT, - pos: Position { line: 1, column: 1 }, - }]; - let result = punct!( - TokenIter { - source: input.as_slice(), + pos: Position { + line: 1, + column: 1, + offset: 0, }, - "!" - ); + }]; + let result = punct!(SliceIter::new(input.as_slice()), "!"); match result { - nom::IResult::Done(_, tok) => assert_eq!(tok, input[0]), + Result::Complete(_, tok) => assert_eq!(tok, input[0]), res => assert!(false, format!("Fail: {:?}", res)), } } @@ -265,16 +321,15 @@ fn test_match_type() { let input = vec![Token { fragment: "foo".to_string(), typ: TokenType::BAREWORD, - pos: Position { line: 1, column: 1 }, - }]; - let result = match_type!( - TokenIter { - source: input.as_slice(), + pos: Position { + line: 1, + column: 1, + offset: 0, }, - BAREWORD - ); + }]; + let result = match_type!(SliceIter::new(input.as_slice()), BAREWORD); match result { - nom::IResult::Done(_, tok) => assert_eq!(tok, input[0]), + Result::Complete(_, tok) => assert_eq!(tok, input[0]), res => assert!(false, format!("Fail: {:?}", res)), } }