From 7f47dc3f38bf6a3e8686c1fa3fd50ef97d4bf3af Mon Sep 17 00:00:00 2001 From: Jeremy Wall Date: Sat, 13 Oct 2018 19:42:22 -0500 Subject: [PATCH] REFACTOR: Pass our offsets everywhere. --- examples/test_yaml.yaml | 9 + .../operator_precedence_test.ucg | 4 +- src/ast/mod.rs | 90 +- src/ast/test.rs | 70 +- src/benches/parse.rs | 4 +- src/build/mod.rs | 21 +- src/build/test.rs | 728 ++---------- src/convert/exec.rs | 8 +- src/error.rs | 3 +- src/format.rs | 6 +- src/iter.rs | 103 ++ src/lib.rs | 1 + src/parse/mod.rs | 74 +- src/parse/test.rs | 1054 ++++++++++------- src/tokenizer/mod.rs | 160 ++- src/tokenizer/test.rs | 90 +- 16 files changed, 1128 insertions(+), 1297 deletions(-) create mode 100644 examples/test_yaml.yaml create mode 100644 src/iter.rs diff --git a/examples/test_yaml.yaml b/examples/test_yaml.yaml new file mode 100644 index 0000000..9ba075a --- /dev/null +++ b/examples/test_yaml.yaml @@ -0,0 +1,9 @@ +--- +db_conn1: "db1.prod.net:3306/testdb" +db_conn2: "db2.prod.net:3306/testdb" +tmpldir: "./templates" +prefix: + foo: bar +l: + - foo + - bar \ No newline at end of file diff --git a/integration_tests/operator_precedence_test.ucg b/integration_tests/operator_precedence_test.ucg index f093e0f..d66460b 100644 --- a/integration_tests/operator_precedence_test.ucg +++ b/integration_tests/operator_precedence_test.ucg @@ -4,4 +4,6 @@ assert |2 * (2 + 1) == 6|; assert |2 * 2 + 1 > 4|; assert |2 * 2 + 1 < 6|; assert |2 * 2 + 1 >= 5|; -assert |2 * 2 + 1 <= 5|; \ No newline at end of file +assert |2 * 2 + 1 <= 5|; +assert |2 / 2 == 1|; +assert |2 - 1 == 1|; \ No newline at end of file diff --git a/src/ast/mod.rs b/src/ast/mod.rs index 1a1f0ce..37748ed 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -21,11 +21,10 @@ use std::cmp::PartialEq; use std::cmp::PartialOrd; use std::collections::HashSet; use std::convert::Into; +use std::fmt; use std::hash::Hash; use std::hash::Hasher; -use std::fmt; - macro_rules! enum_type_equality { ( $slf:ident, $r:expr, $( $l:pat ),* ) => { match $slf { @@ -50,14 +49,16 @@ macro_rules! enum_type_equality { pub struct Position { pub line: usize, pub column: usize, + pub offset: usize, } impl Position { /// Construct a new Position. - pub fn new(line: usize, column: usize) -> Self { + pub fn new(line: usize, column: usize, offset: usize) -> Self { Position { line: line, column: column, + offset: offset, } } } @@ -89,8 +90,8 @@ pub struct Token { impl Token { /// Constructs a new Token with a type and line and column information. - pub fn new>(f: S, typ: TokenType, line: usize, col: usize) -> Self { - Self::new_with_pos(f, typ, Position::new(line, col)) + pub fn new, P: Into>(f: S, typ: TokenType, p: P) -> Self { + Self::new_with_pos(f, typ, p.into()) } // Constructs a new Token with a type and a Position. @@ -114,56 +115,49 @@ macro_rules! value_node { ($v:expr, $p:expr) => { Positioned::new_with_pos($v, $p) }; - ($v:expr, $l:expr, $c:expr) => { - Positioned::new($v, $l, $c) - }; } /// Helper macro for making a Token. #[allow(unused_macros)] macro_rules! make_tok { - (EOF => $l:expr, $c:expr) => { - Token::new("", TokenType::END, $l, $c) + (EOF => $i:expr) => { + Token::new("", TokenType::END, $i) }; - (WS => $l:expr, $c:expr) => { - Token::new("", TokenType::WS, $l, $c) + (WS => $i:expr) => { + Token::new("", TokenType::WS, $i) }; - (CMT => $e:expr, $l:expr, $c:expr) => { - Token::new($e, TokenType::COMMENT, $l, $c) + (CMT => $e:expr, $i:expr) => { + Token::new($e, TokenType::COMMENT, $i) }; - (QUOT => $e:expr, $l:expr, $c:expr) => { - Token::new($e, TokenType::QUOTED, $l, $c) + (QUOT => $e:expr, $i:expr) => { + Token::new($e, TokenType::QUOTED, $i) }; - (PUNCT => $e:expr, $l:expr, $c:expr) => { - Token::new($e, TokenType::PUNCT, $l, $c) + (PUNCT => $e:expr, $i:expr) => { + Token::new($e, TokenType::PUNCT, $i) }; - (DIGIT => $e:expr, $l:expr, $c:expr) => { - Token::new($e, TokenType::DIGIT, $l, $c) + (DIGIT => $e:expr, $i:expr) => { + Token::new($e, TokenType::DIGIT, $i) }; - ($e:expr, $l:expr, $c:expr) => { - Token::new($e, TokenType::BAREWORD, $l, $c) + ($e:expr, $i:expr) => { + Token::new($e, TokenType::BAREWORD, $i) }; } /// Helper macro for making expressions. #[allow(unused_macros)] macro_rules! make_expr { - ($e:expr) => { - make_expr!($e, 1, 1) + ($e:expr, $i:expr) => { + Expression::Simple(Value::Symbol(Positioned::new_with_pos($e.to_string(), $i))) }; - ($e:expr, $l:expr, $c:expr) => { - Expression::Simple(Value::Symbol(Positioned::new($e.to_string(), $l, $c))) - }; - - ($e:expr => int, $l:expr, $c:expr) => { - Expression::Simple(Value::Int(Positioned::new($e, $l, $c))) + ($e:expr => int, $i:expr) => { + Expression::Simple(Value::Int(Positioned::new_with_pos($e, $i))) }; } @@ -180,30 +174,26 @@ macro_rules! make_expr { /// ``` #[allow(unused_macros)] macro_rules! make_selector { - ( $h:expr ) => { - make_selector!($h, 1, 0) - }; - - ( $h:expr, $l:expr, $c:expr ) => { + ( $h:expr, $i:expr) => { SelectorDef::new( SelectorList{head: Box::new($h), tail: None}, - $l, $c) + $i) }; - ( $h: expr, $list:expr, $l:expr, $c:expr) => { + ( $h: expr, $list:expr, $i:expr) => { SelectorDef::new( SelectorList{head: Box::new($h), tail: Some($list)}, - $l, $c) + $i) }; // Tokens - ( $h:expr => [ $( $item:expr ),* ] ) => { + ( $h:expr => [ $( $item:expr ),* ], $i:expr ) => { { - make_selector!($h => [ $( $item, )* ] => 1, 1) + make_selector!($h => [ $( $item, )* ] => $i) } }; - ( $h:expr => [ $( $item:expr ),* ] => $l:expr, $c:expr ) => { + ( $h:expr => [ $( $item:expr ),* ] => $i:expr ) => { { let mut list: Vec = Vec::new(); @@ -211,7 +201,7 @@ macro_rules! make_selector { list.push($item); )* - make_selector!($h, list, $l, $c) + make_selector!($h, list, $i) } }; @@ -223,14 +213,14 @@ macro_rules! make_selector { let mut list: Vec = Vec::new(); $( - list.push(make_tok!($item, 1, col)); + list.push(make_tok!($item, Position::new(1, col, col))); col += $item.len() + 1; )* // Shut up the lint about unused code; assert!(col != 0); - make_selector!($h, list, 1, 1) + make_selector!($h, list, Position::new(1, 1, 1)) } }; @@ -241,14 +231,14 @@ macro_rules! make_selector { let mut list: Vec = Vec::new(); $( - list.push(make_tok!($item, $l, col)); + list.push(make_tok!($item, Position::new($l, col, col))); col += $item.len() + 1; )* // Shut up the linter about unused code; assert!(col != 0); - make_selector!($h, list, $l, $c) + make_selector!($h, list, Position::new($l, $c, $c)) } }; } @@ -314,9 +304,9 @@ pub struct SelectorDef { impl SelectorDef { /// Constructs a new SelectorDef. - pub fn new(sel: SelectorList, line: usize, col: usize) -> Self { + pub fn new>(sel: SelectorList, p: P) -> Self { SelectorDef { - pos: Position::new(line, col), + pos: p.into(), sel: sel, } } @@ -451,8 +441,8 @@ impl std::fmt::Display for Positioned { impl Positioned { /// Constructs a new Positioned with a value, line, and column information. - pub fn new(v: T, l: usize, c: usize) -> Self { - Self::new_with_pos(v, Position::new(l, c)) + pub fn new>(v: T, p: P) -> Self { + Self::new_with_pos(v, p.into()) } /// Constructs a new Positioned with a value and a Position. diff --git a/src/ast/test.rs b/src/ast/test.rs index d8ecc5a..d97060b 100644 --- a/src/ast/test.rs +++ b/src/ast/test.rs @@ -17,21 +17,23 @@ use super::*; #[test] pub fn test_macro_validation_happy_path() { let def = MacroDef { - argdefs: vec![value_node!("foo".to_string(), 1, 0)], + argdefs: vec![value_node!("foo".to_string(), Position::new(1, 0, 0))], fields: vec![( - make_tok!("f1", 1, 1), + make_tok!("f1", Position::new(1, 1, 0)), Expression::Binary(BinaryOpDef { kind: BinaryExprType::Add, left: Box::new(Expression::Simple(Value::Symbol(value_node!( "foo".to_string(), - 1, - 1 + Position::new(1, 1, 0) )))), - right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))), - pos: Position::new(1, 0), + right: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 1, 0) + )))), + pos: Position::new(1, 0, 0), }), )], - pos: Position::new(1, 0), + pos: Position::new(1, 0, 0), }; assert!(def.validate_symbols().unwrap() == ()); } @@ -39,21 +41,23 @@ pub fn test_macro_validation_happy_path() { #[test] pub fn test_macro_validation_fail() { let def = MacroDef { - argdefs: vec![value_node!("foo".to_string(), 1, 0)], + argdefs: vec![value_node!("foo".to_string(), Position::new(1, 0, 0))], fields: vec![( - make_tok!("f1", 1, 1), + make_tok!("f1", Position::new(1, 1, 0)), Expression::Binary(BinaryOpDef { kind: BinaryExprType::Add, left: Box::new(Expression::Simple(Value::Symbol(value_node!( "bar".to_string(), - 1, - 1 + Position::new(1, 1, 0) )))), - right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))), - pos: Position::new(1, 0), + right: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 1, 0) + )))), + pos: Position::new(1, 0, 0), }), )], - pos: Position::new(1, 0), + pos: Position::new(1, 0, 0), }; let mut expected = HashSet::new(); expected.insert("bar".to_string()); @@ -63,20 +67,23 @@ pub fn test_macro_validation_fail() { #[test] pub fn test_macro_validation_selector_happy_path() { let def = MacroDef { - argdefs: vec![value_node!("foo".to_string(), 1, 0)], + argdefs: vec![value_node!("foo".to_string(), Position::new(1, 0, 0))], fields: vec![( - make_tok!("f1", 1, 1), + make_tok!("f1", Position::new(1, 1, 0)), Expression::Binary(BinaryOpDef { kind: BinaryExprType::Add, - left: Box::new(Expression::Simple(Value::Selector( - make_selector!(make_expr!("foo", 1, 1) => [ - make_tok!("quux", 1, 1) ] => 1, 1), - ))), - right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))), - pos: Position::new(1, 0), + left: Box::new(Expression::Simple(Value::Selector(make_selector!( + make_expr!("foo", Position::new(1, 1, 0)) => [ + make_tok!("quux", Position::new(1, 1, 0)) ] + => Position::new(1, 1, 0))))), + right: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 1, 0) + )))), + pos: Position::new(1, 0, 0), }), )], - pos: Position::new(1, 0), + pos: Position::new(1, 0, 0), }; assert!(def.validate_symbols().unwrap() == ()); } @@ -84,20 +91,23 @@ pub fn test_macro_validation_selector_happy_path() { #[test] pub fn test_macro_validation_selector_fail() { let def = MacroDef { - argdefs: vec![value_node!("foo".to_string(), 1, 0)], + argdefs: vec![value_node!("foo".to_string(), Position::new(1, 0, 0))], fields: vec![( - make_tok!("f1", 1, 1), + make_tok!("f1", Position::new(1, 1, 0)), Expression::Binary(BinaryOpDef { kind: BinaryExprType::Add, left: Box::new(Expression::Simple(Value::Selector( - make_selector!(make_expr!("bar", 1, 1) => [ - make_tok!("quux", 1, 1) ] => 1, 1), + make_selector!(make_expr!("bar", Position::new(1, 1, 0)) => [ + make_tok!("quux", Position::new(1, 1, 0)) ] => Position::new(1, 1, 0)), ))), - right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))), - pos: Position::new(1, 0), + right: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 1, 0) + )))), + pos: Position::new(1, 0, 0), }), )], - pos: Position::new(1, 0), + pos: Position::new(1, 0, 0), }; let mut expected = HashSet::new(); expected.insert("bar".to_string()); diff --git a/src/benches/parse.rs b/src/benches/parse.rs index bcef0cc..30232ea 100644 --- a/src/benches/parse.rs +++ b/src/benches/parse.rs @@ -22,14 +22,14 @@ extern crate ucglib; use bencher::Bencher; -use abortable_parser::StrIter; +use ucglib::iter::OffsetStrIter; //use cpuprofiler::PROFILER; use ucglib::parse::*; fn do_parse(i: &str) { - parse(StrIter::new(i)); + parse(OffsetStrIter::new(i)); } fn parse_int(b: &mut Bencher) { diff --git a/src/build/mod.rs b/src/build/mod.rs index a26a7fa..74215d2 100644 --- a/src/build/mod.rs +++ b/src/build/mod.rs @@ -25,11 +25,10 @@ use std::path::PathBuf; use std::rc::Rc; use std::string::ToString; -use abortable_parser::StrIter; - use ast::*; use error; use format; +use iter::OffsetStrIter; use parse::parse; pub mod assets; @@ -192,8 +191,12 @@ impl<'a> Builder<'a> { scope: ValueMap, ) -> Self { let env_vars: Vec<(Positioned, Rc)> = env::vars() - .map(|t| (Positioned::new(t.0, 0, 0), Rc::new(t.1.into()))) - .collect(); + .map(|t| { + ( + Positioned::new(t.0, Position::new(0, 0, 0)), + Rc::new(t.1.into()), + ) + }).collect(); Self::new_with_env_and_scope(root, cache, scope, Rc::new(Val::Tuple(env_vars))) } @@ -223,7 +226,7 @@ impl<'a> Builder<'a> { /// Returns a Val by name from previously built UCG. pub fn get_out_by_name(&self, name: &str) -> Option> { let key = Positioned { - pos: Position::new(0, 0), + pos: Position::new(0, 0, 0), val: name.to_string(), }; self.lookup_sym(&key) @@ -245,7 +248,7 @@ impl<'a> Builder<'a> { Ok(()) } - fn eval_span(&mut self, input: StrIter) -> Result, Box> { + fn eval_span(&mut self, input: OffsetStrIter) -> Result, Box> { match parse(input) { Ok(stmts) => { //panic!("Successfully parsed {}", input); @@ -258,6 +261,7 @@ impl<'a> Builder<'a> { Some(val) => Ok(val), } } + // FIXME(jwall): We need to return a error::Error so we have position information. Err(err) => Err(Box::new(error::Error::new_with_boxed_cause( format!( "Error while parsing file: {}", @@ -271,7 +275,7 @@ impl<'a> Builder<'a> { /// Evaluate an input string as UCG. pub fn eval_string(&mut self, input: &str) -> Result, Box> { - self.eval_span(StrIter::new(input)) + self.eval_span(OffsetStrIter::new(input)) } /// Builds a ucg file at the named path. @@ -993,7 +997,8 @@ impl<'a> Builder<'a> { let expr = &tok.fragment; expr_as_stmt.push_str(expr); expr_as_stmt.push_str(";"); - let assert_input = StrIter::new(&expr_as_stmt); + let assert_input = + OffsetStrIter::new_with_offsets(&expr_as_stmt, 0, tok.pos.line - 1, tok.pos.column - 1); let ok = match self.eval_span(assert_input) { Ok(v) => v, Err(e) => { diff --git a/src/build/test.rs b/src/build/test.rs index e984621..53f4e1d 100644 --- a/src/build/test.rs +++ b/src/build/test.rs @@ -25,35 +25,6 @@ fn test_expr_to_val(mut cases: Vec<(Expression, Val)>, b: Builder) { } } -#[test] -fn test_eval_div_expr() { - let cache = Rc::new(RefCell::new(MemoryCache::new())); - let b = Builder::new(std::env::current_dir().unwrap(), cache); - test_expr_to_val( - vec![ - ( - Expression::Binary(BinaryOpDef { - kind: BinaryExprType::Div, - left: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))), - right: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))), - pos: Position::new(1, 0), - }), - Val::Int(1), - ), - ( - Expression::Binary(BinaryOpDef { - kind: BinaryExprType::Div, - left: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))), - right: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))), - pos: Position::new(1, 0), - }), - Val::Float(1.0), - ), - ], - b, - ); -} - #[test] #[should_panic(expected = "Expected Float")] fn test_eval_div_expr_fail() { @@ -63,9 +34,15 @@ fn test_eval_div_expr_fail() { vec![( Expression::Binary(BinaryOpDef { kind: BinaryExprType::Div, - left: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))), - right: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))), - pos: Position::new(1, 0), + left: Box::new(Expression::Simple(Value::Float(value_node!( + 2.0, + Position::new(1, 1, 1) + )))), + right: Box::new(Expression::Simple(Value::Int(value_node!( + 2, + Position::new(1, 1, 1) + )))), + pos: Position::new(1, 0, 0), }), Val::Float(1.0), )], @@ -73,35 +50,6 @@ fn test_eval_div_expr_fail() { ); } -#[test] -fn test_eval_mul_expr() { - let cache = Rc::new(RefCell::new(MemoryCache::new())); - let b = Builder::new(std::env::current_dir().unwrap(), cache); - test_expr_to_val( - vec![ - ( - Expression::Binary(BinaryOpDef { - kind: BinaryExprType::Mul, - left: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))), - right: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))), - pos: Position::new(1, 0), - }), - Val::Int(4), - ), - ( - Expression::Binary(BinaryOpDef { - kind: BinaryExprType::Mul, - left: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))), - right: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))), - pos: Position::new(1, 0), - }), - Val::Float(4.0), - ), - ], - b, - ); -} - #[test] #[should_panic(expected = "Expected Float")] fn test_eval_mul_expr_fail() { @@ -111,9 +59,15 @@ fn test_eval_mul_expr_fail() { vec![( Expression::Binary(BinaryOpDef { kind: BinaryExprType::Mul, - left: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))), - right: Box::new(Expression::Simple(Value::Int(value_node!(20, 1, 1)))), - pos: Position::new(1, 0), + left: Box::new(Expression::Simple(Value::Float(value_node!( + 2.0, + Position::new(1, 1, 1) + )))), + right: Box::new(Expression::Simple(Value::Int(value_node!( + 20, + Position::new(1, 1, 1) + )))), + pos: Position::new(1, 0, 0), }), Val::Float(1.0), )], @@ -121,35 +75,6 @@ fn test_eval_mul_expr_fail() { ); } -#[test] -fn test_eval_subtract_expr() { - let cache = Rc::new(RefCell::new(MemoryCache::new())); - let b = Builder::new(std::env::current_dir().unwrap(), cache); - test_expr_to_val( - vec![ - ( - Expression::Binary(BinaryOpDef { - kind: BinaryExprType::Sub, - left: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))), - right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))), - pos: Position::new(1, 0), - }), - Val::Int(1), - ), - ( - Expression::Binary(BinaryOpDef { - kind: BinaryExprType::Sub, - left: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))), - right: Box::new(Expression::Simple(Value::Float(value_node!(1.0, 1, 1)))), - pos: Position::new(1, 0), - }), - Val::Float(1.0), - ), - ], - b, - ); -} - #[test] #[should_panic(expected = "Expected Float")] fn test_eval_subtract_expr_fail() { @@ -159,88 +84,21 @@ fn test_eval_subtract_expr_fail() { vec![( Expression::Binary(BinaryOpDef { kind: BinaryExprType::Sub, - left: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))), - right: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))), - pos: Position::new(1, 0), + left: Box::new(Expression::Simple(Value::Float(value_node!( + 2.0, + Position::new(1, 1, 1) + )))), + right: Box::new(Expression::Simple(Value::Int(value_node!( + 2, + Position::new(1, 1, 1) + )))), + pos: Position::new(1, 0, 0), }), Val::Float(1.0), )], b, ); } - -#[test] -fn test_eval_add_expr() { - let cache = Rc::new(RefCell::new(MemoryCache::new())); - let b = Builder::new(std::env::current_dir().unwrap(), cache); - test_expr_to_val( - vec![ - ( - Expression::Binary(BinaryOpDef { - kind: BinaryExprType::Add, - left: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))), - right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))), - pos: Position::new(1, 0), - }), - Val::Int(2), - ), - ( - Expression::Binary(BinaryOpDef { - kind: BinaryExprType::Add, - left: Box::new(Expression::Simple(Value::Float(value_node!(1.0, 1, 1)))), - right: Box::new(Expression::Simple(Value::Float(value_node!(1.0, 1, 1)))), - pos: Position::new(1, 0), - }), - Val::Float(2.0), - ), - ( - Expression::Binary(BinaryOpDef { - kind: BinaryExprType::Add, - left: Box::new(Expression::Simple(Value::Str(value_node!( - "foo".to_string(), - 1, - 1 - )))), - right: Box::new(Expression::Simple(Value::Str(value_node!( - "bar".to_string(), - 1, - 1 - )))), - pos: Position::new(1, 0), - }), - Val::Str("foobar".to_string()), - ), - ( - Expression::Binary(BinaryOpDef { - kind: BinaryExprType::Add, - left: Box::new(Expression::Simple(Value::List(ListDef { - elems: vec![Expression::Simple(Value::Str(value_node!( - "foo".to_string(), - 1, - 1 - )))], - pos: Position::new(1, 1), - }))), - right: Box::new(Expression::Simple(Value::List(ListDef { - elems: vec![Expression::Simple(Value::Str(value_node!( - "bar".to_string(), - 1, - 1 - )))], - pos: Position::new(1, 1), - }))), - pos: Position::new(1, 0), - }), - Val::List(vec![ - Rc::new(Val::Str("foo".to_string())), - Rc::new(Val::Str("bar".to_string())), - ]), - ), - ], - b, - ); -} - #[test] #[should_panic(expected = "Expected Float")] fn test_eval_add_expr_fail() { @@ -250,9 +108,15 @@ fn test_eval_add_expr_fail() { vec![( Expression::Binary(BinaryOpDef { kind: BinaryExprType::Add, - left: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))), - right: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))), - pos: Position::new(1, 0), + left: Box::new(Expression::Simple(Value::Float(value_node!( + 2.0, + Position::new(1, 1, 1) + )))), + right: Box::new(Expression::Simple(Value::Int(value_node!( + 2, + Position::new(1, 1, 1) + )))), + pos: Position::new(1, 0, 0), }), Val::Float(1.0), )], @@ -260,245 +124,20 @@ fn test_eval_add_expr_fail() { ); } -#[test] -fn test_eval_nested_tuple() { - test_expr_to_val( - vec![ - ( - Expression::Simple(Value::Tuple(value_node!( - vec![( - Token::new("foo", TokenType::BAREWORD, 1, 1), - Expression::Simple(Value::Tuple(value_node!(Vec::new(), 1, 1))), - )], - 1, - 1 - ))), - Val::Tuple(vec![( - Positioned::new("foo".to_string(), 1, 1), - Rc::new(Val::Tuple(Vec::new())), - )]), - ), - ( - Expression::Simple(Value::Tuple(value_node!( - vec![( - Token::new("foo", TokenType::BAREWORD, 1, 1), - Expression::Simple(Value::Tuple(value_node!( - vec![( - Token::new("bar".to_string(), TokenType::BAREWORD, 1, 5), - Expression::Simple(Value::Tuple(value_node!(vec![], 1, 10))), - )], - 1, - 1 - ))), - )], - 1, - 1 - ))), - Val::Tuple(vec![( - Positioned::new("foo".to_string(), 1, 1), - Rc::new(Val::Tuple(vec![( - Positioned::new("bar".to_string(), 1, 10), - Rc::new(Val::Tuple(vec![])), - )])), - )]), - ), - ( - Expression::Simple(Value::Tuple(value_node!( - vec![( - Token::new("foo", TokenType::BAREWORD, 1, 1), - Expression::Simple(Value::Tuple(value_node!( - vec![( - Token::new("bar".to_string(), TokenType::BAREWORD, 1, 5), - Expression::Simple(Value::Tuple(value_node!( - vec![( - Token::new("quux".to_string(), TokenType::BAREWORD, 1, 1), - Expression::Simple(Value::Int(value_node!(3, 1, 1))), - )], - 1, - 10 - ))), - )], - 1, - 1 - ))), - )], - 1, - 1 - ))), - Val::Tuple(vec![( - Positioned::new("foo".to_string(), 1, 1), - Rc::new(Val::Tuple(vec![( - Positioned::new("bar".to_string(), 1, 10), - Rc::new(Val::Tuple(vec![( - Positioned::new("quux".to_string(), 1, 1), - Rc::new(Val::Int(3)), - )])), - )])), - )]), - ), - ], - Builder::new( - std::env::current_dir().unwrap(), - Rc::new(RefCell::new(MemoryCache::new())), - ), - ); -} - -#[test] -fn test_eval_simple_expr() { - test_expr_to_val( - vec![ - ( - Expression::Simple(Value::Int(value_node!(1, 1, 1))), - Val::Int(1), - ), - ( - Expression::Simple(Value::Float(value_node!(2.0, 1, 1))), - Val::Float(2.0), - ), - ( - Expression::Simple(Value::Str(value_node!("foo".to_string(), 1, 1))), - Val::Str("foo".to_string()), - ), - ( - Expression::Simple(Value::Tuple(value_node!( - vec![( - make_tok!("bar", 1, 1), - Expression::Simple(Value::Int(value_node!(1, 1, 1))), - )], - 1, - 1 - ))), - Val::Tuple(vec![( - value_node!("bar".to_string(), 1, 1), - Rc::new(Val::Int(1)), - )]), - ), - ], - Builder::new( - std::env::current_dir().unwrap(), - Rc::new(RefCell::new(MemoryCache::new())), - ), - ); -} - -#[test] -fn test_eval_simple_lookup_expr() { - let cache = Rc::new(RefCell::new(MemoryCache::new())); - let mut b = Builder::new(std::env::current_dir().unwrap(), cache); - b.build_output - .entry(value_node!("var1".to_string(), 1, 0)) - .or_insert(Rc::new(Val::Int(1))); - test_expr_to_val( - vec![( - Expression::Simple(Value::Symbol(value_node!("var1".to_string(), 1, 1))), - Val::Int(1), - )], - b, - ); -} - #[test] fn test_eval_simple_lookup_error() { let cache = Rc::new(RefCell::new(MemoryCache::new())); let mut b = Builder::new(std::env::current_dir().unwrap(), cache); b.build_output - .entry(value_node!("var1".to_string(), 1, 0)) + .entry(value_node!("var1".to_string(), Position::new(1, 0, 0))) .or_insert(Rc::new(Val::Int(1))); - let expr = Expression::Simple(Value::Symbol(value_node!("var".to_string(), 1, 1))); + let expr = Expression::Simple(Value::Symbol(value_node!( + "var".to_string(), + Position::new(1, 1, 1) + ))); assert!(b.eval_expr(&expr).is_err()); } -#[test] -fn test_eval_selector_expr() { - let cache = Rc::new(RefCell::new(MemoryCache::new())); - let mut b = Builder::new(std::env::current_dir().unwrap(), cache); - b.build_output - .entry(value_node!("var1".to_string(), 1, 0)) - .or_insert(Rc::new(Val::Tuple(vec![( - value_node!("lvl1".to_string(), 1, 0), - Rc::new(Val::Tuple(vec![( - value_node!("lvl2".to_string(), 1, 0), - Rc::new(Val::Int(3)), - )])), - )]))); - b.build_output - .entry(value_node!("var2".to_string(), 1, 0)) - .or_insert(Rc::new(Val::Int(2))); - b.build_output - .entry(value_node!("var3".to_string(), 1, 0)) - .or_insert(Rc::new(Val::Tuple(vec![( - value_node!("lvl1".to_string(), 1, 0), - Rc::new(Val::Int(4)), - )]))); - - test_expr_to_val( - vec![ - ( - Expression::Simple(Value::Selector(make_selector!(make_expr!("var1")))), - Val::Tuple(vec![( - value_node!("lvl1".to_string(), 1, 0), - Rc::new(Val::Tuple(vec![( - value_node!("lvl2".to_string(), 1, 0), - Rc::new(Val::Int(3)), - )])), - )]), - ), - ( - Expression::Simple(Value::Selector( - make_selector!(make_expr!("var1") => "lvl1"), - )), - Val::Tuple(vec![( - value_node!("lvl2".to_string(), 1, 0), - Rc::new(Val::Int(3)), - )]), - ), - ( - Expression::Simple(Value::Selector( - make_selector!(make_expr!("var1") => "lvl1", "lvl2"), - )), - Val::Int(3), - ), - ( - Expression::Simple(Value::Selector(make_selector!(make_expr!("var2")))), - Val::Int(2), - ), - ( - Expression::Simple(Value::Selector( - make_selector!(make_expr!("var3") => "lvl1"), - )), - Val::Int(4), - ), - ], - b, - ); -} - -#[test] -fn test_eval_selector_list_expr() { - let cache = Rc::new(RefCell::new(MemoryCache::new())); - let mut b = Builder::new(std::env::current_dir().unwrap(), cache); - b.build_output - .entry(value_node!("var1".to_string(), 1, 1)) - .or_insert(Rc::new(Val::List(vec![ - Rc::new(Val::Str("val1".to_string())), - Rc::new(Val::Tuple(vec![( - value_node!("var2".to_string(), 1, 1), - Rc::new(Val::Int(1)), - )])), - ]))); - - test_expr_to_val( - vec![( - Expression::Simple(Value::Selector( - make_selector!(make_expr!("var1") => "0" => 1, 1), - )), - Val::Str("val1".to_string()), - )], - b, - ); -} - // Include nested for each. #[test] #[should_panic(expected = "Unable to find tpl1")] @@ -508,9 +147,12 @@ fn test_expr_copy_no_such_tuple() { test_expr_to_val( vec![( Expression::Copy(CopyDef { - selector: make_selector!(make_expr!("tpl1")), + selector: make_selector!( + make_expr!("tpl1", Position::new(1, 1, 1)), + Position::new(1, 1, 1) + ), fields: Vec::new(), - pos: Position::new(1, 0), + pos: Position::new(1, 0, 0), }), Val::Tuple(Vec::new()), )], @@ -524,14 +166,17 @@ fn test_expr_copy_not_a_tuple() { let cache = Rc::new(RefCell::new(MemoryCache::new())); let mut b = Builder::new(std::env::current_dir().unwrap(), cache); b.build_output - .entry(value_node!("tpl1".to_string(), 1, 0)) + .entry(value_node!("tpl1".to_string(), Position::new(1, 0, 0))) .or_insert(Rc::new(Val::Int(1))); test_expr_to_val( vec![( Expression::Copy(CopyDef { - selector: make_selector!(make_expr!("tpl1")), + selector: make_selector!( + make_expr!("tpl1", Position::new(1, 1, 1)), + Position::new(1, 1, 1) + ), fields: Vec::new(), - pos: Position::new(1, 0), + pos: Position::new(1, 0, 0), }), Val::Tuple(Vec::new()), )], @@ -545,130 +190,30 @@ fn test_expr_copy_field_type_error() { let cache = Rc::new(RefCell::new(MemoryCache::new())); let mut b = Builder::new(std::env::current_dir().unwrap(), cache); b.build_output - .entry(value_node!("tpl1".to_string(), 1, 0)) + .entry(value_node!("tpl1".to_string(), Position::new(1, 0, 0))) .or_insert(Rc::new(Val::Tuple(vec![( - value_node!("fld1".to_string(), 1, 0), + value_node!("fld1".to_string(), Position::new(1, 0, 0)), Rc::new(Val::Int(1)), )]))); test_expr_to_val( vec![( Expression::Copy(CopyDef { - selector: make_selector!(make_expr!("tpl1")), - fields: vec![( - make_tok!("fld1", 1, 1), - Expression::Simple(Value::Str(value_node!("2".to_string(), 1, 1))), - )], - pos: Position::new(1, 0), - }), - Val::Tuple(vec![( - value_node!("fld1".to_string(), 1, 1), - Rc::new(Val::Str("2".to_string())), - )]), - )], - b, - ); -} - -#[test] -fn test_expr_copy() { - let cache = Rc::new(RefCell::new(MemoryCache::new())); - let mut b = Builder::new(std::env::current_dir().unwrap(), cache); - b.build_output - .entry(value_node!("tpl1".to_string(), 1, 0)) - .or_insert(Rc::new(Val::Tuple(vec![( - value_node!("fld1".to_string(), 1, 0), - Rc::new(Val::Int(1)), - )]))); - test_expr_to_val( - vec![ - ( - Expression::Copy(CopyDef { - selector: make_selector!(make_expr!("tpl1")), - fields: vec![( - make_tok!("fld2", 1, 1), - Expression::Simple(Value::Str(value_node!("2".to_string(), 1, 1))), - )], - pos: Position::new(1, 0), - }), - // Add a new field to the copy - Val::Tuple( - // NOTE(jwall): The order of these is important in order to ensure - // that the compare assertion is correct. The ordering has no - // semantics though so at some point we should probably be less restrictive. - vec![ - (value_node!("fld1".to_string(), 1, 0), Rc::new(Val::Int(1))), - ( - value_node!("fld2".to_string(), 1, 1), - Rc::new(Val::Str("2".to_string())), - ), - ], + selector: make_selector!( + make_expr!("tpl1", Position::new(1, 1, 1)), + Position::new(1, 1, 1) ), - ), - // Overwrite a field in the copy - ( - Expression::Copy(CopyDef { - selector: make_selector!(make_expr!("tpl1")), - fields: vec![ - ( - make_tok!("fld1", 1, 1), - Expression::Simple(Value::Int(value_node!(3, 1, 1))), - ), - ( - make_tok!("fld2", 1, 1), - Expression::Simple(Value::Str(value_node!("2".to_string(), 1, 1))), - ), - ], - pos: Position::new(1, 0), - }), - Val::Tuple(vec![ - (value_node!("fld1".to_string(), 1, 0), Rc::new(Val::Int(3))), - ( - value_node!("fld2".to_string(), 1, 0), - Rc::new(Val::Str("2".to_string())), - ), - ]), - ), - // The source tuple is still unmodified. - ( - Expression::Simple(Value::Selector(make_selector!(make_expr!["tpl1"]))), - Val::Tuple(vec![( - value_node!("fld1".to_string(), 1, 0), - Rc::new(Val::Int(1)), - )]), - ), - ], - b, - ); -} - -#[test] -fn test_macro_call() { - let cache = Rc::new(RefCell::new(MemoryCache::new())); - let mut b = Builder::new(std::env::current_dir().unwrap(), cache); - b.build_output - .entry(value_node!("tstmac".to_string(), 1, 0)) - .or_insert(Rc::new(Val::Macro(MacroDef { - argdefs: vec![value_node!("arg1".to_string(), 1, 0)], - fields: vec![( - make_tok!("foo", 1, 1), - Expression::Simple(Value::Symbol(value_node!("arg1".to_string(), 1, 1))), - )], - pos: Position::new(1, 0), - }))); - test_expr_to_val( - vec![( - Expression::Call(CallDef { - macroref: make_selector!(make_expr!("tstmac")), - arglist: vec![Expression::Simple(Value::Str(value_node!( - "bar".to_string(), - 1, - 1 - )))], - pos: Position::new(1, 0), + fields: vec![( + make_tok!("fld1", Position::new(1, 1, 1)), + Expression::Simple(Value::Str(value_node!( + "2".to_string(), + Position::new(1, 1, 1) + ))), + )], + pos: Position::new(1, 0, 0), }), Val::Tuple(vec![( - value_node!("foo".to_string(), 1, 1), - Rc::new(Val::Str("bar".to_string())), + value_node!("fld1".to_string(), Position::new(1, 1, 1)), + Rc::new(Val::Str("2".to_string())), )]), )], b, @@ -681,31 +226,36 @@ fn test_macro_hermetic() { let cache = Rc::new(RefCell::new(MemoryCache::new())); let mut b = Builder::new(std::env::current_dir().unwrap(), cache); b.build_output - .entry(value_node!("arg1".to_string(), 1, 0)) + .entry(value_node!("arg1".to_string(), Position::new(1, 0, 0))) .or_insert(Rc::new(Val::Str("bar".to_string()))); b.build_output - .entry(value_node!("tstmac".to_string(), 1, 0)) + .entry(value_node!("tstmac".to_string(), Position::new(1, 0, 0))) .or_insert(Rc::new(Val::Macro(MacroDef { - argdefs: vec![value_node!("arg2".to_string(), 1, 0)], + argdefs: vec![value_node!("arg2".to_string(), Position::new(1, 0, 0))], fields: vec![( - make_tok!("foo", 1, 1), - Expression::Simple(Value::Symbol(value_node!("arg1".to_string(), 1, 1))), + make_tok!("foo", Position::new(1, 1, 1)), + Expression::Simple(Value::Symbol(value_node!( + "arg1".to_string(), + Position::new(1, 1, 1) + ))), )], - pos: Position::new(1, 0), + pos: Position::new(1, 0, 0), }))); test_expr_to_val( vec![( Expression::Call(CallDef { - macroref: make_selector!(make_expr!("tstmac")), + macroref: make_selector!( + make_expr!("tstmac", Position::new(1, 1, 1)), + Position::new(1, 1, 1) + ), arglist: vec![Expression::Simple(Value::Str(value_node!( "bar".to_string(), - 1, - 1 + Position::new(1, 1, 1) )))], - pos: Position::new(1, 1), + pos: Position::new(1, 1, 1), }), Val::Tuple(vec![( - value_node!("foo".to_string(), 1, 0), + value_node!("foo".to_string(), Position::new(1, 1, 1)), Rc::new(Val::Str("bar".to_string())), )]), )], @@ -713,126 +263,42 @@ fn test_macro_hermetic() { ); } -#[test] -fn test_select_expr() { - let cache = Rc::new(RefCell::new(MemoryCache::new())); - let mut b = Builder::new(std::env::current_dir().unwrap(), cache); - b.build_output - .entry(value_node!("foo".to_string(), 1, 0)) - .or_insert(Rc::new(Val::Str("bar".to_string()))); - b.build_output - .entry(value_node!("baz".to_string(), 1, 0)) - .or_insert(Rc::new(Val::Str("boo".to_string()))); - test_expr_to_val( - vec![ - ( - Expression::Select(SelectDef { - val: Box::new(Expression::Simple(Value::Symbol(value_node!( - "foo".to_string(), - 1, - 1 - )))), - default: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))), - tuple: vec![ - ( - make_tok!("foo", 1, 1), - Expression::Simple(Value::Str(value_node!("2".to_string(), 1, 1))), - ), - ( - make_tok!("bar", 1, 1), - Expression::Simple(Value::Int(value_node!(2, 1, 1))), - ), - ], - pos: Position::new(1, 0), - }), - Val::Int(2), - ), - ( - Expression::Select(SelectDef { - val: Box::new(Expression::Simple(Value::Symbol(value_node!( - "baz".to_string(), - 1, - 1 - )))), - default: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))), - tuple: vec![ - ( - make_tok!("bar", 1, 1), - Expression::Simple(Value::Int(value_node!(2, 1, 1))), - ), - ( - make_tok!("quux", 1, 1), - Expression::Simple(Value::Str(value_node!("2".to_string(), 1, 1))), - ), - ], - pos: Position::new(1, 0), - }), - // If the field doesn't exist then we get the default. - Val::Int(1), - ), - ], - b, - ); -} - #[test] #[should_panic(expected = "Expected String but got Integer in Select expression")] fn test_select_expr_not_a_string() { let cache = Rc::new(RefCell::new(MemoryCache::new())); let mut b = Builder::new(std::env::current_dir().unwrap(), cache); b.build_output - .entry(value_node!("foo".to_string(), 1, 0)) + .entry(value_node!("foo".to_string(), Position::new(1, 0, 0))) .or_insert(Rc::new(Val::Int(4))); test_expr_to_val( vec![( Expression::Select(SelectDef { val: Box::new(Expression::Simple(Value::Symbol(value_node!( "foo".to_string(), - 1, - 1 + Position::new(1, 1, 1) + )))), + default: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 1, 1) )))), - default: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))), tuple: vec![ ( - make_tok!("bar", 1, 1), - Expression::Simple(Value::Int(value_node!(2, 1, 1))), + make_tok!("bar", Position::new(1, 1, 1)), + Expression::Simple(Value::Int(value_node!(2, Position::new(1, 1, 1)))), ), ( - make_tok!("quux", 1, 1), - Expression::Simple(Value::Str(value_node!("2".to_string(), 1, 1))), + make_tok!("quux", Position::new(1, 1, 1)), + Expression::Simple(Value::Str(value_node!( + "2".to_string(), + Position::new(1, 1, 1) + ))), ), ], - pos: Position::new(1, 0), + pos: Position::new(1, 0, 0), }), Val::Int(2), )], b, ); } - -#[test] -fn test_let_statement() { - let cache = MemoryCache::new(); - let mut b = Builder::new("", Rc::new(RefCell::new(cache))); - let stmt = Statement::Let(LetDef { - name: make_tok!("foo", 1, 1), - value: Expression::Simple(Value::Str(value_node!("bar".to_string(), 1, 1))), - }); - b.build_stmt(&stmt).unwrap(); - test_expr_to_val( - vec![( - Expression::Simple(Value::Symbol(value_node!("foo".to_string(), 1, 1))), - Val::Str("bar".to_string()), - )], - b, - ); -} - -#[test] -fn test_build_file_string() { - let cache = Rc::new(RefCell::new(MemoryCache::new())); - let mut b = Builder::new(std::env::current_dir().unwrap(), cache); - b.eval_string("let foo = 1;").unwrap(); - let key = value_node!("foo".to_string(), 1, 0); - assert!(b.build_output.contains_key(&key)); -} diff --git a/src/convert/exec.rs b/src/convert/exec.rs index 6d23fa2..cbd9c84 100644 --- a/src/convert/exec.rs +++ b/src/convert/exec.rs @@ -46,7 +46,7 @@ impl ExecConverter { return Err(Box::new(Error::new( "Exec tuples must have no more than 3 fields", ErrorType::TypeFail, - Position::new(0, 0), + Position::new(0, 0, 0), ))); } let mut env: Option<&Vec<(Positioned, Rc)>> = None; @@ -115,7 +115,7 @@ impl ExecConverter { return Err(Box::new(Error::new( "An exec tuple must have a command field", ErrorType::TypeFail, - Position::new(0, 0), + Position::new(0, 0, 0), ))); } // Okay if we have made it this far then we are ready to start creating our script. @@ -157,7 +157,7 @@ impl ExecConverter { return Err(Box::new(Error::new( "Exec args must be a list of strings or tuples of strings.", ErrorType::TypeFail, - Position::new(0, 0), + Position::new(0, 0, 0), ))) } } @@ -173,7 +173,7 @@ impl ExecConverter { Err(Box::new(Error::new( "Exec outputs must be of type Tuple", ErrorType::TypeFail, - Position::new(0, 0), + Position::new(0, 0, 0), ))) } } diff --git a/src/error.rs b/src/error.rs index aa0958e..a963f33 100644 --- a/src/error.rs +++ b/src/error.rs @@ -80,7 +80,8 @@ impl Error { t: ErrorType, cause: Box, ) -> Self { - let mut e = Self::new(msg, t, Position { line: 0, column: 0 }); + // TODO(jwall): This should take a real position instead of this fake one. + let mut e = Self::new(msg, t, Position::new(0, 0, 0)); e.cause = Some(cause); return e; } diff --git a/src/format.rs b/src/format.rs index 4f8f627..edda478 100644 --- a/src/format.rs +++ b/src/format.rs @@ -82,21 +82,21 @@ mod test { #[test] fn test_format_happy_path() { let formatter = Formatter::new("foo @ @ \\@", vec!["bar", "quux"]); - let pos = Position { line: 0, column: 0 }; + let pos = Position::new(0, 0, 0); assert_eq!(formatter.render(&pos).unwrap(), "foo bar quux @"); } #[test] fn test_format_happy_wrong_too_few_args() { let formatter = Formatter::new("foo @ @ \\@", vec!["bar"]); - let pos = Position { line: 0, column: 0 }; + let pos = Position::new(0, 0, 0); assert!(formatter.render(&pos).is_err()); } #[test] fn test_format_happy_wrong_too_many_args() { let formatter = Formatter::new("foo @ @ \\@", vec!["bar", "quux", "baz"]); - let pos = Position { line: 0, column: 0 }; + let pos = Position::new(0, 0, 0); assert!(formatter.render(&pos).is_err()); } } diff --git a/src/iter.rs b/src/iter.rs new file mode 100644 index 0000000..156374d --- /dev/null +++ b/src/iter.rs @@ -0,0 +1,103 @@ +use std::convert::From; +use std::iter::Iterator; + +use abortable_parser::iter::StrIter; +use abortable_parser::{ + InputIter, Offsetable, Peekable, Seekable, Span, SpanRange, TextPositionTracker, +}; + +#[derive(Debug)] +pub struct OffsetStrIter<'a> { + contained: StrIter<'a>, + idx_offset: usize, + line_offset: usize, + col_offset: usize, +} + +impl<'a> OffsetStrIter<'a> { + pub fn new(input: &'a str) -> Self { + Self::new_with_offsets(input, 0, 0, 0) + } + + pub fn new_with_offsets( + input: &'a str, + idx_offset: usize, + line_offset: usize, + col_offset: usize, + ) -> Self { + OffsetStrIter { + contained: StrIter::new(input), + idx_offset: idx_offset, + line_offset: line_offset, + col_offset: col_offset, + } + } +} + +impl<'a> Iterator for OffsetStrIter<'a> { + type Item = &'a u8; + + fn next(&mut self) -> Option { + self.contained.next() + } +} + +impl<'a> Offsetable for OffsetStrIter<'a> { + fn get_offset(&self) -> usize { + self.contained.get_offset() + self.idx_offset + } +} + +impl<'a> Clone for OffsetStrIter<'a> { + fn clone(&self) -> Self { + OffsetStrIter { + contained: self.contained.clone(), + idx_offset: self.idx_offset, + line_offset: self.line_offset, + col_offset: self.col_offset, + } + } +} + +impl<'a> From<&'a str> for OffsetStrIter<'a> { + fn from(source: &'a str) -> Self { + OffsetStrIter { + contained: StrIter::new(source), + idx_offset: 0, + line_offset: 0, + col_offset: 0, + } + } +} + +impl<'a> Seekable for OffsetStrIter<'a> { + fn seek(&mut self, to: usize) -> usize { + let contained_offset = self.contained.seek(to); + self.idx_offset += contained_offset; + contained_offset + } +} + +impl<'a> Span<&'a str> for OffsetStrIter<'a> { + fn span(&self, idx: SpanRange) -> &'a str { + self.contained.span(idx) + } +} + +impl<'a> Peekable<&'a u8> for OffsetStrIter<'a> { + fn peek_next(&self) -> Option<&'a u8> { + self.contained.peek_next() + } +} + +impl<'a> TextPositionTracker for OffsetStrIter<'a> { + fn line(&self) -> usize { + self.contained.line() + self.line_offset + } + + fn column(&self) -> usize { + self.contained.column() + self.col_offset + } +} + +impl<'a> InputIter for OffsetStrIter<'a> {} diff --git a/src/lib.rs b/src/lib.rs index a4f81cc..ec9ec10 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -456,6 +456,7 @@ pub mod tokenizer; pub mod build; pub mod convert; pub mod error; +pub mod iter; pub mod parse; mod format; diff --git a/src/parse/mod.rs b/src/parse/mod.rs index 23c2d9b..d5db62a 100644 --- a/src/parse/mod.rs +++ b/src/parse/mod.rs @@ -19,16 +19,16 @@ use std::str::FromStr; use abortable_parser; use abortable_parser::combinators::eoi; -use abortable_parser::iter::{SliceIter, StrIter}; +use abortable_parser::iter::SliceIter; use abortable_parser::{Error, Peekable, Result}; use self::precedence::op_expression; use ast::*; +use iter::OffsetStrIter; use tokenizer::*; type NomResult<'a, O> = Result, O>; -// FIXME(jwall): All the do_each mappers need to return an actual value. #[cfg(feature = "tracing")] const ENABLE_TRACE: bool = true; #[cfg(not(feature = "tracing"))] @@ -92,11 +92,10 @@ make_fn!( match_type!(STR => str_to_value) ); -// FIXME(jwall): We need to just turn this into a custom parser function. // Helper function to make the return types work for down below. fn triple_to_number(v: (Option, Option, Option)) -> ParseResult { let (pref, mut pref_pos) = match v.0 { - None => ("", Position::new(0, 0)), + None => ("", Position::new(0, 0, 0)), Some(ref bs) => (bs.fragment.borrow(), bs.pos.clone()), }; @@ -106,11 +105,7 @@ fn triple_to_number(v: (Option, Option, Option)) -> ParseRe let i = match FromStr::from_str(pref) { Ok(i) => i, Err(_) => { - return Err(Error::new( - format!("Not an integer! {}", pref), - // FIXME(jwall): This really needs the correct offset. - &0, - )); + return Err(Error::new(format!("Not an integer! {}", pref), &0)); } }; return Ok(Value::Int(value_node!(i, pref_pos))); @@ -120,9 +115,9 @@ fn triple_to_number(v: (Option, Option, Option)) -> ParseRe pref_pos = v.1.unwrap().pos; } - let suf = match v.2 { - None => "".to_string(), - Some(bs) => bs.fragment, + let (suf, pos) = match v.2 { + None => ("".to_string(), Position::new(0, 0, 0)), + Some(bs) => (bs.fragment, bs.pos), }; let to_parse = pref.to_string() + "." + &suf; @@ -131,8 +126,7 @@ fn triple_to_number(v: (Option, Option, Option)) -> ParseRe Err(_) => { return Err(Error::new( format!("Not a float! {}", to_parse), - // FIXME(jwall): This should take the real offset. - &0, + &pos.offset, )); } }; @@ -307,11 +301,7 @@ make_fn!( // Helper function to make the return types work for down below. fn vec_to_tuple(pos: Position, fields: Option) -> Value { - Value::Tuple(value_node!( - fields.unwrap_or(Vec::new()), - pos.line as usize, - pos.column as usize - )) + Value::Tuple(value_node!(fields.unwrap_or(Vec::new()), pos)) } make_fn!( @@ -519,11 +509,10 @@ make_fn!( fields => optional!(trace_nom!(field_list)), _ => optional!(punct!(",")), // noms opt! macro does not preserve error types properly but this one does. _ => punct!("}"), - (tuple_to_copy(SelectorDef::new(selector, pos.line, pos.column), fields)) + (tuple_to_copy(SelectorDef::new(selector, pos), fields)) ) ); -// FIXME(jwall): need to make this into a proper parse function. fn tuple_to_macro(pos: Position, vals: Option>, val: Value) -> ParseResult { let mut default_args = match vals { None => Vec::new(), @@ -543,8 +532,7 @@ fn tuple_to_macro(pos: Position, vals: Option>, val: Value) -> ParseR })), val => Err(Error::new( format!("Expected Tuple Got {:?}", val), - // FIXME(jwall): Should have correct Offset here. - &0, + &val.pos().offset, )), } } @@ -647,7 +635,6 @@ make_fn!( ) ); -// FIXME(jwall): Convert this into a custom parser function. fn tuple_to_call(pos: Position, val: Value, exprs: Vec) -> ParseResult { if let Value::Selector(def) = val { Ok(Expression::Call(CallDef { @@ -656,17 +643,15 @@ fn tuple_to_call(pos: Position, val: Value, exprs: Vec) -> ParseResu pos: pos, })) } else { - // FIXME(jwall): Should get correct offset here. - Err(Error::new(format!("Expected Selector Got {:?}", val), &0)) + Err(Error::new( + format!("Expected Selector Got {:?}", val), + &val.pos().offset, + )) } } fn vec_to_selector_value(pos: Position, list: SelectorList) -> Value { - Value::Selector(SelectorDef::new( - list, - pos.line as usize, - pos.column as usize, - )) + Value::Selector(SelectorDef::new(list, pos)) } make_fn!( @@ -715,8 +700,7 @@ fn tuple_to_list_op( } return Err(Error::new( format!("Missing a result field for the macro"), - // FIXME(jwall): Should have correct offset. - &0, + &def.pos.offset, )); } &mut Some(ref mut tl) => { @@ -729,8 +713,7 @@ fn tuple_to_list_op( } return Err(Error::new( format!("Missing a result field for the macro"), - // FIXME(jwall): Should have correct offset. - &0, + &def.pos.offset, )); } let fname = tl.pop(); @@ -753,12 +736,10 @@ fn tuple_to_list_op( } return Err(Error::new( format!("Expected a selector but got {}", macroname.type_name()), - // FIXME(jwall): Should have correct offset. - &0, + &pos.offset, )); } -// FIXME(jwall): need to make this a custom function to parse it. make_fn!( list_op_expression, Expression>, do_each!( @@ -843,7 +824,7 @@ make_fn!( let_statement, Statement>, do_each!( _ => word!("let"), - stmt => trace_nom!(let_stmt_body), + stmt => trace_nom!(must!(let_stmt_body)), (stmt) ) ); @@ -871,7 +852,7 @@ make_fn!( do_each!( _ => word!("import"), // past this point we know this is supposed to be an import statement. - stmt => trace_nom!(import_stmt_body), + stmt => trace_nom!(must!(import_stmt_body)), (stmt) ) ); @@ -880,8 +861,8 @@ make_fn!( assert_statement, Statement>, do_each!( _ => word!("assert"), - tok => match_type!(PIPEQUOTE), - _ => punct!(";"), + tok => must!(match_type!(PIPEQUOTE)), + _ => must!(punct!(";")), (Statement::Assert(tok.clone())) ) ); @@ -890,9 +871,9 @@ make_fn!( out_statement, Statement>, do_each!( _ => word!("out"), - typ => match_type!(BAREWORD), - expr => expression, - _ => punct!(";"), + typ => must!(match_type!(BAREWORD)), + expr => must!(expression), + _ => must!(punct!(";")), (Statement::Output(typ.clone(), expr.clone())) ) ); @@ -910,7 +891,7 @@ fn statement(i: SliceIter) -> Result, Statement> { //trace_macros!(false); /// Parses a LocatedSpan into a list of Statements or an `abortable_parser::Error`. -pub fn parse(input: StrIter) -> std::result::Result, Error> { +pub fn parse(input: OffsetStrIter) -> std::result::Result, Error> { match tokenize(input.clone()) { Ok(tokenized) => { let mut out = Vec::new(); @@ -922,6 +903,7 @@ pub fn parse(input: StrIter) -> std::result::Result, Error> { break; } } + // FIXME(jwall): We need to return a error::Error so we have position information. match statement(i.clone()) { Result::Abort(e) => { return Err(e); diff --git a/src/parse/test.rs b/src/parse/test.rs index cbf99e9..32097fa 100644 --- a/src/parse/test.rs +++ b/src/parse/test.rs @@ -14,14 +14,16 @@ use super::*; use tokenizer::tokenize; -use abortable_parser::{Result, SliceIter, StrIter}; +use abortable_parser::{Result, SliceIter}; + +use iter::OffsetStrIter; macro_rules! assert_parse { ($parsemac:ident($i:expr), $out:expr) => { assert_parse!($i, $parsemac, $out) }; ($i:expr, $f:expr, $out:expr) => {{ - let input = StrIter::new($i); + let input = OffsetStrIter::new($i); match tokenize(input) { Err(e) => assert!(false, format!("Tokenizer Error: {:?}", e)), Ok(val) => match $f(SliceIter::new(val.as_slice())) { @@ -32,17 +34,33 @@ macro_rules! assert_parse { };}; } -macro_rules! assert_error { +macro_rules! assert_fail { ($parsemac:ident($i:expr)) => { - assert_error!($i, $parsemac) + assert_fail!($i, $parsemac) }; ($i:expr, $f:expr) => {{ - let input = StrIter::new($i); + let input = OffsetStrIter::new($i); match tokenize(input) { Err(_) => assert!(true), Ok(val) => { let result = $f(SliceIter::new(val.as_slice())); - assert!(result.is_fail(), format!("Not a fail: {:?}", result)) + assert!(result.is_fail(), format!("Not an abort: {:?}", result)) + } + } + }}; +} + +macro_rules! assert_abort { + ($parsemac:ident($i:expr)) => { + assert_abort!($i, $parsemac) + }; + ($i:expr, $f:expr) => {{ + let input = OffsetStrIter::new($i); + match tokenize(input) { + Err(_) => assert!(true), + Ok(val) => { + let result = $f(SliceIter::new(val.as_slice())); + assert!(result.is_abort(), format!("Not a fail: {:?}", result)) } } }}; @@ -50,15 +68,15 @@ macro_rules! assert_error { #[test] fn test_null_parsing() { - assert_parse!(empty_value("NULL "), Value::Empty(Position::new(1, 1))); - assert_parse!(value("NULL "), Value::Empty(Position::new(1, 1))); + assert_parse!(empty_value("NULL "), Value::Empty(Position::new(1, 1, 0))); + assert_parse!(value("NULL "), Value::Empty(Position::new(1, 1, 0))); assert_parse!( simple_expression("NULL "), - Expression::Simple(Value::Empty(Position::new(1, 1))) + Expression::Simple(Value::Empty(Position::new(1, 1, 0))) ); assert_parse!( expression("NULL,"), - Expression::Simple(Value::Empty(Position::new(1, 1))) + Expression::Simple(Value::Empty(Position::new(1, 1, 0))) ); } @@ -66,62 +84,66 @@ fn test_null_parsing() { fn test_boolean_parsing() { assert_parse!( boolean_value("true"), - Value::Boolean(Positioned::new(true, 1, 1)) + Value::Boolean(Positioned::new(true, Position::new(1, 1, 0))) ); assert_parse!( boolean_value("false"), - Value::Boolean(Positioned::new(false, 1, 1)) + Value::Boolean(Positioned::new(false, Position::new(1, 1, 0))) ); - assert_error!(boolean_value("truth")); + assert_fail!(boolean_value("truth")); } #[test] fn test_symbol_parsing() { assert_parse!( symbol("foo"), - Value::Symbol(value_node!("foo".to_string(), 1, 1)) + Value::Symbol(value_node!("foo".to_string(), Position::new(1, 1, 0))) ); assert_parse!( symbol("foo-bar"), - Value::Symbol(value_node!("foo-bar".to_string(), 1, 1)) + Value::Symbol(value_node!("foo-bar".to_string(), Position::new(1, 1, 0))) ); assert_parse!( symbol("foo_bar"), - Value::Symbol(value_node!("foo_bar".to_string(), 1, 1)) + Value::Symbol(value_node!("foo_bar".to_string(), Position::new(1, 1, 0))) ); } #[test] fn test_selector_parsing() { - assert_error!(selector_value("foo.")); + assert_fail!(selector_value("foo.")); assert_parse!( selector_value("foo.bar "), - Value::Selector(make_selector!(make_expr!("foo".to_string(), 1, 1) => [ - make_tok!("bar", 1, 5)] => - 1, 1)) + Value::Selector( + make_selector!(make_expr!("foo".to_string(), Position::new(1, 1, 0)) => [ + make_tok!("bar", Position::new(1, 5, 4))] => + Position::new(1, 1, 0)) + ) ); assert_parse!( selector_value("foo.0 "), - Value::Selector(make_selector!(make_expr!("foo".to_string(), 1, 1) => [ - make_tok!(DIGIT => "0", 1, 5)] => - 1, 1)) + Value::Selector( + make_selector!(make_expr!("foo".to_string(), Position::new(1, 1, 0)) => [ + make_tok!(DIGIT => "0", Position::new(1, 5, 4))] => + Position::new(1, 1, 0)) + ) ); assert_parse!( selector_value("foo.bar;"), - Value::Selector(make_selector!(make_expr!("foo", 1, 1) => + Value::Selector(make_selector!(make_expr!("foo", Position::new(1, 1, 0)) => [ - make_tok!("bar", 1, 5) + make_tok!("bar", Position::new(1, 5, 4)) ] => - 1, 1)) + Position::new(1, 1, 0))) ); assert_parse!( selector_value("({foo=1}).foo "), Value::Selector( make_selector!(Expression::Grouped(Box::new(Expression::Simple( Value::Tuple(value_node!( - vec![(make_tok!("foo", 1, 3), Expression::Simple(Value::Int(Positioned::new(1, 1, 7))))], - 1, 3)) - ))) => [ make_tok!("foo", 1, 11) ] => 1, 2) + vec![(make_tok!("foo", Position::new(1, 3, 2)), Expression::Simple(Value::Int(Positioned::new(1, Position::new(1, 7, 6)))))], + Position::new(1, 3, 3))) + ))) => [ make_tok!("foo", Position::new(1, 11, 10)) ] => Position::new(1, 2, 1)) ) ); } @@ -132,35 +154,44 @@ fn test_statement_parse() { assert_parse!( statement(stmt), Statement::Import(ImportDef { - path: make_tok!(QUOT => "foo", 1,8), - name: make_tok!("foo", 1, 17), + path: make_tok!(QUOT => "foo", Position::new(1, 8, 7)), + name: make_tok!("foo", Position::new(1, 17, 16)), }) ); - assert_error!(import_statement("import \"foo\"")); + assert_abort!(import_statement("import \"foo\"")); assert_parse!( statement("let foo = 1.0 ;"), Statement::Let(LetDef { - name: make_tok!("foo", 1, 5), - value: Expression::Simple(Value::Float(value_node!(1.0, 1, 11))), + name: make_tok!("foo", Position::new(1, 5, 4)), + value: Expression::Simple(Value::Float(value_node!(1.0, Position::new(1, 11, 10)))), }) ); assert_parse!( statement("let foo = 1 + 1 * 2;"), Statement::Let(LetDef { - name: make_tok!("foo", 1, 5), + name: make_tok!("foo", Position::new(1, 5, 4)), value: Expression::Binary(BinaryOpDef { kind: BinaryExprType::Add, - left: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 11)))), + left: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 11, 10) + )))), right: Box::new(Expression::Binary(BinaryOpDef { kind: BinaryExprType::Mul, - left: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 15)))), - right: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 19)))), - pos: Position::new(1, 15), + left: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 15, 14) + )))), + right: Box::new(Expression::Simple(Value::Int(value_node!( + 2, + Position::new(1, 19, 18) + )))), + pos: Position::new(1, 15, 14), })), - pos: Position::new(1, 11), + pos: Position::new(1, 11, 10), }), }) ); @@ -168,19 +199,28 @@ fn test_statement_parse() { assert_parse!( statement("let foo = (1 + 1) * 2;"), Statement::Let(LetDef { - name: make_tok!("foo", 1, 5), + name: make_tok!("foo", Position::new(1, 5, 4)), value: Expression::Binary(BinaryOpDef { kind: BinaryExprType::Mul, left: Box::new(Expression::Grouped(Box::new(Expression::Binary( BinaryOpDef { kind: BinaryExprType::Add, - left: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 12)))), - right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 16)))), - pos: Position::new(1, 12), + left: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 12, 11) + )))), + right: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 16, 15) + )))), + pos: Position::new(1, 12, 11), }, )))), - right: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 21)))), - pos: Position::new(1, 12), + right: Box::new(Expression::Simple(Value::Int(value_node!( + 2, + Position::new(1, 21, 20) + )))), + pos: Position::new(1, 12, 11), }), }) ); @@ -188,17 +228,26 @@ fn test_statement_parse() { assert_parse!( statement("let foo = 1 * 1 + 2;"), Statement::Let(LetDef { - name: make_tok!("foo", 1, 5), + name: make_tok!("foo", Position::new(1, 5, 4)), value: Expression::Binary(BinaryOpDef { kind: BinaryExprType::Add, left: Box::new(Expression::Binary(BinaryOpDef { kind: BinaryExprType::Mul, - left: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 11)))), - right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 15)))), - pos: Position::new(1, 11), + left: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 11, 10) + )))), + right: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 15, 14) + )))), + pos: Position::new(1, 11, 10), })), - right: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 19)))), - pos: Position::new(1, 11), + right: Box::new(Expression::Simple(Value::Int(value_node!( + 2, + Position::new(1, 19, 18) + )))), + pos: Position::new(1, 11, 10), }), }) ); @@ -206,113 +255,124 @@ fn test_statement_parse() { assert_parse!( statement("// comment\nlet foo = 1.0 ;"), Statement::Let(LetDef { - name: make_tok!("foo", 2, 5), - value: Expression::Simple(Value::Float(value_node!(1.0, 2, 11))), + name: make_tok!("foo", Position::new(2, 5, 15)), + value: Expression::Simple(Value::Float(value_node!(1.0, Position::new(2, 11, 21)))), }) ); assert_parse!( statement("1.0;"), - Statement::Expression(Expression::Simple(Value::Float(value_node!(1.0, 1, 1)))) + Statement::Expression(Expression::Simple(Value::Float(value_node!( + 1.0, + Position::new(1, 1, 0) + )))) ); } #[test] fn test_import_statement_parse() { - assert_error!(import_statement("import")); - assert_error!(import_statement("import \"foo\"")); - assert_error!(import_statement("import \"foo\" as")); - assert_error!(import_statement("import \"foo\" as foo")); + assert_abort!(import_statement("import")); + assert_abort!(import_statement("import \"foo\"")); + assert_abort!(import_statement("import \"foo\" as")); + assert_abort!(import_statement("import \"foo\" as foo")); let import_stmt = "import \"foo\" as foo;"; assert_parse!( import_statement(import_stmt), Statement::Import(ImportDef { - path: make_tok!(QUOT => "foo", 1, 8), - name: make_tok!("foo", 1, 17), + path: make_tok!(QUOT => "foo", Position::new(1, 8, 7)), + name: make_tok!("foo", Position::new(1, 17, 16)), }) ); } #[test] fn test_let_statement_parse() { - assert_error!(let_statement("foo")); - assert_error!(let_statement("let \"foo\"")); - assert_error!(let_statement("let 1")); - assert_error!(let_statement("let")); - assert_error!(let_statement("let foo")); - assert_error!(let_statement("let foo =")); - assert_error!(let_statement("let foo = ")); - assert_error!(let_statement("let foo = 1")); + assert_fail!(let_statement("foo")); + assert_abort!(let_statement("let \"foo\"")); + assert_abort!(let_statement("let 1")); + assert_abort!(let_statement("let")); + assert_abort!(let_statement("let foo")); + assert_abort!(let_statement("let foo =")); + assert_abort!(let_statement("let foo = ")); + assert_abort!(let_statement("let foo = 1")); assert_parse!( let_statement("let foo = 1.0 ;"), Statement::Let(LetDef { - name: make_tok!("foo", 1, 5), - value: Expression::Simple(Value::Float(value_node!(1.0, 1, 11))), + name: make_tok!("foo", Position::new(1, 5, 4)), + value: Expression::Simple(Value::Float(value_node!(1.0, Position::new(1, 11, 10)))), }) ); assert_parse!( let_statement("let foo = // comment\n1.0 ;"), Statement::Let(LetDef { - name: make_tok!("foo", 1, 5), - value: Expression::Simple(Value::Float(value_node!(1.0, 2, 1))), + name: make_tok!("foo", Position::new(1, 5, 4)), + value: Expression::Simple(Value::Float(value_node!(1.0, Position::new(2, 1, 22)))), }) ); assert_parse!( let_statement("let foo = 1.0 // comment\n;"), Statement::Let(LetDef { - name: make_tok!("foo", 1, 5), - value: Expression::Simple(Value::Float(value_node!(1.0, 1, 11))), + name: make_tok!("foo", Position::new(1, 5, 4)), + value: Expression::Simple(Value::Float(value_node!(1.0, Position::new(1, 11, 10)))), }) ); assert_parse!( let_statement("let foo= 1.0;"), Statement::Let(LetDef { - name: make_tok!("foo", 1, 5), - value: Expression::Simple(Value::Float(value_node!(1.0, 1, 10))), + name: make_tok!("foo", Position::new(1, 5, 4)), + value: Expression::Simple(Value::Float(value_node!(1.0, Position::new(1, 10, 9)))), }) ); assert_parse!( let_statement("let foo =1.0;"), Statement::Let(LetDef { - name: make_tok!("foo", 1, 5), - value: Expression::Simple(Value::Float(value_node!(1.0, 1, 10))), + name: make_tok!("foo", Position::new(1, 5, 4)), + value: Expression::Simple(Value::Float(value_node!(1.0, Position::new(1, 10, 9)))), }) ); } #[test] fn test_out_statement_parse() { - assert_error!(out_statement("out")); - assert_error!(out_statement("out json")); - assert_error!(out_statement("out json foo")); + assert_abort!(out_statement("out")); + assert_abort!(out_statement("out json")); + assert_abort!(out_statement("out json foo")); assert_parse!( out_statement("out json 1.0;"), Statement::Output( Token { - pos: Position { line: 1, column: 5 }, + pos: Position { + line: 1, + column: 5, + offset: 4 + }, fragment: "json".to_string(), typ: TokenType::BAREWORD }, - Expression::Simple(Value::Float(value_node!(1.0, 1, 10))) + Expression::Simple(Value::Float(value_node!(1.0, Position::new(1, 10, 9)))) ) ); } #[test] fn test_assert_statement_parse() { - assert_error!(out_statement("assert")); - assert_error!(out_statement("assert |")); - assert_error!(out_statement("assert |foo")); + assert_fail!(out_statement("assert")); + assert_fail!(out_statement("assert |")); + assert_fail!(out_statement("assert |foo")); assert_parse!( assert_statement("assert |foo|;"), Statement::Assert(Token { - pos: Position { line: 1, column: 8 }, + pos: Position { + line: 1, + column: 8, + offset: 7 + }, fragment: "foo".to_string(), typ: TokenType::PIPEQUOTE }) @@ -320,65 +380,68 @@ fn test_assert_statement_parse() { } #[test] fn test_expression_statement_parse() { - assert_error!(expression_statement("foo")); + assert_fail!(expression_statement("foo")); assert_parse!( expression_statement("1.0;"), - Statement::Expression(Expression::Simple(Value::Float(value_node!(1.0, 1, 1)))) + Statement::Expression(Expression::Simple(Value::Float(value_node!( + 1.0, + Position::new(1, 1, 0) + )))) ); assert_parse!( expression_statement("1.0 ;"), - Statement::Expression(Expression::Simple(Value::Float(value_node!(1.0, 1, 1)))) + Statement::Expression(Expression::Simple(Value::Float(value_node!( + 1.0, + Position::new(1, 1, 0) + )))) ); assert_parse!( expression_statement(" 1.0;"), - Statement::Expression(Expression::Simple(Value::Float(value_node!(1.0, 1, 2)))) + Statement::Expression(Expression::Simple(Value::Float(value_node!( + 1.0, + Position::new(1, 2, 1) + )))) ); assert_parse!( expression_statement("foo;"), Statement::Expression(Expression::Simple(Value::Selector(make_selector!( - make_expr!("foo", 1, 1), - 1, - 1 + make_expr!("foo", Position::new(1, 1, 0)), + Position::new(1, 1, 0) )))) ); assert_parse!( expression_statement("foo ;"), Statement::Expression(Expression::Simple(Value::Selector(make_selector!( - make_expr!("foo", 1, 2), - 1, - 1 + make_expr!("foo", Position::new(1, 2, 1)), + Position::new(1, 1, 0) )))) ); assert_parse!( expression_statement(" foo;"), Statement::Expression(Expression::Simple(Value::Selector(make_selector!( - make_expr!("foo", 1, 2), - 1, - 2 + make_expr!("foo", Position::new(1, 2, 1)), + Position::new(1, 2, 1) )))) ); assert_parse!( expression_statement("\"foo\";"), Statement::Expression(Expression::Simple(Value::Str(value_node!( "foo".to_string(), - 1, - 1 + Position::new(1, 1, 0) )))) ); assert_parse!( expression_statement("\"foo\" ;"), Statement::Expression(Expression::Simple(Value::Str(value_node!( "foo".to_string(), - 1, - 1 + Position::new(1, 1, 0) )))) ); assert_parse!( expression_statement(" \"foo\";"), Statement::Expression(Expression::Simple(Value::Str(value_node!( "foo".to_string(), - 1, - 2 + Position::new(1, 2, 1) )))) ); } @@ -387,29 +450,33 @@ fn test_expression_statement_parse() { fn test_expression_parse() { assert_parse!( expression("NULL "), - Expression::Simple(Value::Empty(Position::new(1, 1))) + Expression::Simple(Value::Empty(Position::new(1, 1, 0))) ); assert_parse!( expression("\"foo\""), - Expression::Simple(Value::Str(value_node!("foo".to_string(), 1, 1))) + Expression::Simple(Value::Str(value_node!( + "foo".to_string(), + Position::new(1, 1, 0) + ))) ); assert_parse!( expression("1"), - Expression::Simple(Value::Int(value_node!(1, 1, 1))) + Expression::Simple(Value::Int(value_node!(1, Position::new(1, 1, 0)))) ); assert_parse!( expression("foo "), Expression::Simple(Value::Selector(make_selector!( - make_expr!("foo", 1, 1), - 1, - 1 + make_expr!("foo", Position::new(1, 1, 0)), + Position::new(1, 1, 0) ))) ); assert_parse!( expression("foo.bar "), - Expression::Simple(Value::Selector(make_selector!(make_expr!("foo", 1, 1) => - [ make_tok!("bar", 1, 5) ] => - 1, 1))) + Expression::Simple(Value::Selector( + make_selector!(make_expr!("foo", Position::new(1, 1, 0)) => + [ make_tok!("bar", Position::new(1, 5, 4)) ] => + Position::new(1, 1, 0)) + )) ); assert_parse!( expression("{foo=1}.foo "), @@ -417,14 +484,14 @@ fn test_expression_parse() { make_selector!(Expression::Simple(Value::Tuple( value_node!(vec![ ( - make_tok!("foo", 1, 2), - Expression::Simple(Value::Int(value_node!(1, 1, 6))), + make_tok!("foo", Position::new(1, 2, 1)), + Expression::Simple(Value::Int(value_node!(1, Position::new(1, 6, 5)))), ), ], - 1, 1), + Position::new(1, 1, 0)), )) => - [ make_tok!("foo", 1, 9) ] => - 1, 1) + [ make_tok!("foo", Position::new(1, 9, 8)) ] => + Position::new(1, 1, 0)) )) ); assert_parse!( @@ -433,49 +500,73 @@ fn test_expression_parse() { make_selector!(Expression::Simple(Value::List( ListDef{ elems: vec![ - Expression::Simple(Value::Int(value_node!(1, 1, 2))), - Expression::Simple(Value::Int(value_node!(2, 1, 5))), + Expression::Simple(Value::Int(value_node!(1, Position::new(1, 2, 1)))), + Expression::Simple(Value::Int(value_node!(2, Position::new(1, 5, 4)))), ], - pos: Position::new(1, 1), + pos: Position::new(1, 1, 0), })) => - [ make_tok!(DIGIT => "1", 1, 8) ] => - 1, 1) + [ make_tok!(DIGIT => "1", Position::new(1, 8, 7)) ] => + Position::new(1, 1, 0)) )) ); assert_parse!( expression("1 + 1"), Expression::Binary(BinaryOpDef { kind: BinaryExprType::Add, - left: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))), - right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 5)))), - pos: Position::new(1, 1), + left: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 1, 0) + )))), + right: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 5, 4) + )))), + pos: Position::new(1, 1, 0), }) ); assert_parse!( expression("1 - 1"), Expression::Binary(BinaryOpDef { kind: BinaryExprType::Sub, - left: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))), - right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 5)))), - pos: Position::new(1, 1), + left: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 1, 0) + )))), + right: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 5, 4) + )))), + pos: Position::new(1, 1, 0), }) ); assert_parse!( expression("1 / 1"), Expression::Binary(BinaryOpDef { kind: BinaryExprType::Div, - left: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))), - right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 5)))), - pos: Position::new(1, 1), + left: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 1, 0) + )))), + right: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 5, 4) + )))), + pos: Position::new(1, 1, 0), }) ); assert_parse!( expression("(1 / 1)"), Expression::Grouped(Box::new(Expression::Binary(BinaryOpDef { kind: BinaryExprType::Div, - left: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 2)))), - right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 6)))), - pos: Position::new(1, 2), + left: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 2, 1) + )))), + right: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 6, 5) + )))), + pos: Position::new(1, 2, 1), }))) ); assert_parse!( @@ -484,12 +575,21 @@ fn test_expression_parse() { kind: BinaryExprType::Add, left: Box::new(Expression::Binary(BinaryOpDef { kind: BinaryExprType::Div, - left: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))), - right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 5)))), - pos: Position::new(1, 1), + left: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 1, 0) + )))), + right: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 5, 4) + )))), + pos: Position::new(1, 1, 0), })), - right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 9)))), - pos: Position::new(1, 1), + right: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 9, 8) + )))), + pos: Position::new(1, 1, 0), }) ); assert_parse!( @@ -499,171 +599,238 @@ fn test_expression_parse() { left: Box::new(Expression::Grouped(Box::new(Expression::Binary( BinaryOpDef { kind: BinaryExprType::Add, - left: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 2)))), - right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 6)))), - pos: Position::new(1, 2), + left: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 2, 1) + )))), + right: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 6, 5) + )))), + pos: Position::new(1, 2, 1), } )))), - right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 11)))), - pos: Position::new(1, 2), + right: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 11, 10) + )))), + pos: Position::new(1, 2, 1), }) ); assert_parse!( expression("1 > 1"), Expression::Compare(ComparisonDef { kind: CompareType::GT, - left: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))), - right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 5)))), - pos: Position::new(1, 1), + left: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 1, 0) + )))), + right: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 5, 4) + )))), + pos: Position::new(1, 1, 0), }) ); assert_parse!( expression("1 < 1"), Expression::Compare(ComparisonDef { kind: CompareType::LT, - left: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))), - right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 5)))), - pos: Position::new(1, 1), + left: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 1, 0) + )))), + right: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 5, 4) + )))), + pos: Position::new(1, 1, 0), }) ); assert_parse!( expression("1 <= 1"), Expression::Compare(ComparisonDef { kind: CompareType::LTEqual, - left: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))), - right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 5)))), - pos: Position::new(1, 1), + left: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 1, 0) + )))), + right: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 5, 4) + )))), + pos: Position::new(1, 1, 0), }) ); assert_parse!( expression("1 >= 1"), Expression::Compare(ComparisonDef { kind: CompareType::GTEqual, - left: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))), - right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 5)))), - pos: Position::new(1, 1), + left: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 1, 0) + )))), + right: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 5, 4) + )))), + pos: Position::new(1, 1, 0), }) ); assert_parse!( expression("1+1"), Expression::Binary(BinaryOpDef { kind: BinaryExprType::Add, - left: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))), - right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 3)))), - pos: Position::new(1, 1), + left: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 1, 0) + )))), + right: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 3, 2) + )))), + pos: Position::new(1, 1, 0), }) ); assert_parse!( expression("1-1"), Expression::Binary(BinaryOpDef { kind: BinaryExprType::Sub, - left: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))), - right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 3)))), - pos: Position::new(1, 1), + left: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 1, 0) + )))), + right: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 3, 2) + )))), + pos: Position::new(1, 1, 0), }) ); assert_parse!( expression("1*1"), Expression::Binary(BinaryOpDef { kind: BinaryExprType::Mul, - left: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))), - right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 3)))), - pos: Position::new(1, 1), + left: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 1, 0) + )))), + right: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 3, 2) + )))), + pos: Position::new(1, 1, 0), }) ); assert_parse!( expression("1/1"), Expression::Binary(BinaryOpDef { kind: BinaryExprType::Div, - left: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))), - right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 3)))), - pos: Position::new(1, 1), + left: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 1, 0) + )))), + right: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 3, 2) + )))), + pos: Position::new(1, 1, 0), }) ); assert_parse!( expression("macro (arg1, arg2) => { foo = arg1 }"), Expression::Macro(MacroDef { argdefs: vec![ - value_node!("arg1".to_string(), 1, 8), - value_node!("arg2".to_string(), 1, 14), + value_node!("arg1".to_string(), Position::new(1, 8, 7)), + value_node!("arg2".to_string(), Position::new(1, 14, 13)), ], fields: vec![( - make_tok!("foo", 1, 25), + make_tok!("foo", Position::new(1, 25, 24)), Expression::Simple(Value::Selector(make_selector!( - make_expr!("arg1", 1, 31), - 1, - 31 + make_expr!("arg1", Position::new(1, 31, 30)), + Position::new(1, 31, 30) ))), )], - pos: Position::new(1, 1), + pos: Position::new(1, 1, 0), }) ); assert_parse!( expression("select foo, 1, { foo = 2 }"), Expression::Select(SelectDef { val: Box::new(Expression::Simple(Value::Selector(make_selector!( - make_expr!("foo", 1, 8), - 1, - 8 + make_expr!("foo", Position::new(1, 8, 7)), + Position::new(1, 8, 7) + )))), + default: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 13, 12) )))), - default: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 13)))), tuple: vec![( - make_tok!("foo", 1, 18), - Expression::Simple(Value::Int(value_node!(2, 1, 24))), + make_tok!("foo", Position::new(1, 18, 17)), + Expression::Simple(Value::Int(value_node!(2, Position::new(1, 24, 23)))), )], - pos: Position::new(1, 1), + pos: Position::new(1, 1, 0), }) ); assert_parse!( expression("foo.bar (1, \"foo\")"), Expression::Call(CallDef { - macroref: make_selector!(make_expr!("foo", 1, 1) => - [ make_tok!("bar", 1, 5) ] => - 1, 1), + macroref: make_selector!(make_expr!("foo", Position::new(1, 1, 0)) => + [ make_tok!("bar", Position::new(1, 5, 4)) ] => + Position::new(1, 1, 0)), arglist: vec![ - Expression::Simple(Value::Int(value_node!(1, 1, 10))), - Expression::Simple(Value::Str(value_node!("foo".to_string(), 1, 13))), + Expression::Simple(Value::Int(value_node!(1, Position::new(1, 10, 9)))), + Expression::Simple(Value::Str(value_node!( + "foo".to_string(), + Position::new(1, 13, 12) + ))), ], - pos: Position::new(1, 1), + pos: Position::new(1, 1, 0), }) ); assert_parse!( expression("(1 + 1)"), Expression::Grouped(Box::new(Expression::Binary(BinaryOpDef { kind: BinaryExprType::Add, - left: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 2)))), - right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 6)))), - pos: Position::new(1, 2), + left: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 2, 1) + )))), + right: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 6, 5) + )))), + pos: Position::new(1, 2, 1), }))) ); assert_parse!( expression("[1, 1]"), Expression::Simple(Value::List(ListDef { elems: vec![ - Expression::Simple(Value::Int(value_node!(1, 1, 2))), - Expression::Simple(Value::Int(value_node!(1, 1, 5))), + Expression::Simple(Value::Int(value_node!(1, Position::new(1, 2, 1)))), + Expression::Simple(Value::Int(value_node!(1, Position::new(1, 5, 4)))), ], - pos: Position::new(1, 1), + pos: Position::new(1, 1, 0), })) ); } #[test] fn test_format_parse() { - assert_error!(format_expression("\"foo")); - assert_error!(format_expression("\"foo\"")); - assert_error!(format_expression("\"foo\" %")); - assert_error!(format_expression("\"foo\" % (, 2")); + assert_fail!(format_expression("\"foo")); + assert_fail!(format_expression("\"foo\"")); + assert_fail!(format_expression("\"foo\" %")); + assert_fail!(format_expression("\"foo\" % (, 2")); assert_parse!( format_expression("\"foo @ @\" % (1, 2)"), Expression::Format(FormatDef { template: "foo @ @".to_string(), args: vec![ - Expression::Simple(Value::Int(value_node!(1, 1, 14))), - Expression::Simple(Value::Int(value_node!(2, 1, 17))), + Expression::Simple(Value::Int(value_node!(1, Position::new(1, 14, 13)))), + Expression::Simple(Value::Int(value_node!(2, Position::new(1, 17, 16)))), ], - pos: Position::new(1, 1), + pos: Position::new(1, 1, 0), }) ); @@ -672,85 +839,96 @@ fn test_format_parse() { Expression::Format(FormatDef { template: "foo @ @".to_string(), args: vec![ - Expression::Simple(Value::Int(value_node!(1, 1, 12))), - Expression::Simple(Value::Int(value_node!(2, 1, 15))), + Expression::Simple(Value::Int(value_node!(1, Position::new(1, 12, 11)))), + Expression::Simple(Value::Int(value_node!(2, Position::new(1, 15, 14)))), ], - pos: Position::new(1, 1), + pos: Position::new(1, 1, 0), }) ); } #[test] fn test_call_parse() { - assert_error!(call_expression("foo")); - assert_error!(call_expression("foo (")); - assert_error!(call_expression("foo (1")); - assert_error!(call_expression("foo (1,")); - assert_error!(call_expression("foo (1,2")); + assert_fail!(call_expression("foo")); + assert_fail!(call_expression("foo (")); + assert_fail!(call_expression("foo (1")); + assert_fail!(call_expression("foo (1,")); + assert_fail!(call_expression("foo (1,2")); assert_parse!( call_expression("foo (1, \"foo\")"), Expression::Call(CallDef { - macroref: make_selector!(make_expr!("foo", 1, 1), 1, 1), + macroref: make_selector!( + make_expr!("foo", Position::new(1, 1, 0)), + Position::new(1, 1, 0) + ), arglist: vec![ - Expression::Simple(Value::Int(value_node!(1, 1, 6))), - Expression::Simple(Value::Str(value_node!("foo".to_string(), 1, 9))), + Expression::Simple(Value::Int(value_node!(1, Position::new(1, 6, 5)))), + Expression::Simple(Value::Str(value_node!( + "foo".to_string(), + Position::new(1, 9, 8) + ))), ], - pos: Position::new(1, 1), + pos: Position::new(1, 1, 0), }) ); assert_parse!( call_expression("foo.bar (1, \"foo\")"), Expression::Call(CallDef { - macroref: make_selector!(make_expr!("foo") => [ make_tok!("bar", 1, 5) ] => 1, 1), + macroref: make_selector!(make_expr!("foo", Position::new(1, 1, 0)) => [ make_tok!("bar", Position::new(1, 5, 4)) ] => Position::new(1, 1, 0)), arglist: vec![ - Expression::Simple(Value::Int(value_node!(1, 1, 10))), - Expression::Simple(Value::Str(value_node!("foo".to_string(), 1, 13))), + Expression::Simple(Value::Int(value_node!(1, Position::new(1, 10, 9)))), + Expression::Simple(Value::Str(value_node!( + "foo".to_string(), + Position::new(1, 13, 12) + ))), ], - pos: Position::new(1, 1), + pos: Position::new(1, 1, 0), }) ); } #[test] fn test_select_parse() { - assert_error!(select_expression("select")); - assert_error!(select_expression("select foo")); - assert_error!(select_expression("select foo, 1")); - assert_error!(select_expression("select foo, 1, {")); + assert_fail!(select_expression("select")); + assert_fail!(select_expression("select foo")); + assert_fail!(select_expression("select foo, 1")); + assert_fail!(select_expression("select foo, 1, {")); assert_parse!( select_expression("select foo, 1, { foo = 2 }"), Expression::Select(SelectDef { val: Box::new(Expression::Simple(Value::Selector(make_selector!( - make_expr!("foo", 1, 8), - 1, - 8 + make_expr!("foo", Position::new(1, 8, 7)), + Position::new(1, 8, 7) + )))), + default: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 13, 12) )))), - default: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 13)))), tuple: vec![( - make_tok!("foo", 1, 18), - Expression::Simple(Value::Int(value_node!(2, 1, 24))), + make_tok!("foo", Position::new(1, 18, 17)), + Expression::Simple(Value::Int(value_node!(2, Position::new(1, 24, 23)))), )], - pos: Position::new(1, 1), + pos: Position::new(1, 1, 0), }) ); } #[test] fn test_macro_expression_parsing() { - assert_error!(macro_expression("foo")); - assert_error!(macro_expression("macro \"foo\"")); - assert_error!(macro_expression("macro 1")); - assert_error!(macro_expression("macro")); - assert_error!(macro_expression("macro (")); - assert_error!(macro_expression("macro (arg")); - assert_error!(macro_expression("macro (arg, arg2")); - assert_error!(macro_expression("macro (arg1, arg2) =>")); - assert_error!(macro_expression("macro (arg1, arg2) => {")); - assert_error!(macro_expression("macro (arg1, arg2) => { foo")); - assert_error!(macro_expression("macro (arg1, arg2) => { foo =")); + assert_fail!(macro_expression("foo")); + assert_fail!(macro_expression("macro \"foo\"")); + assert_fail!(macro_expression("macro 1")); + assert_fail!(macro_expression("macro")); + assert_fail!(macro_expression("macro (")); + assert_fail!(macro_expression("macro (arg")); + assert_fail!(macro_expression("macro (arg, arg2")); + assert_fail!(macro_expression("macro (arg1, arg2) =>")); + assert_fail!(macro_expression("macro (arg1, arg2) => {")); + assert_fail!(macro_expression("macro (arg1, arg2) => { foo")); + assert_fail!(macro_expression("macro (arg1, arg2) => { foo =")); assert_parse!( macro_expression("macro () => {foo=1,bar=2}"), @@ -758,15 +936,15 @@ fn test_macro_expression_parsing() { argdefs: Vec::new(), fields: vec![ ( - make_tok!("foo", 1, 14), - Expression::Simple(Value::Int(value_node!(1, 1, 18))), + make_tok!("foo", Position::new(1, 14, 13)), + Expression::Simple(Value::Int(value_node!(1, Position::new(1, 18, 17)))), ), ( - make_tok!("bar", 1, 20), - Expression::Simple(Value::Int(value_node!(2, 1, 24))), + make_tok!("bar", Position::new(1, 20, 19)), + Expression::Simple(Value::Int(value_node!(2, Position::new(1, 24, 23)))), ), ], - pos: Position::new(1, 1), + pos: Position::new(1, 1, 0), }) ); @@ -774,99 +952,116 @@ fn test_macro_expression_parsing() { macro_expression("macro (arg1, arg2) => {foo=1,bar=2}"), Expression::Macro(MacroDef { argdefs: vec![ - value_node!("arg1".to_string(), 1, 8), - value_node!("arg2".to_string(), 1, 14), + value_node!("arg1".to_string(), Position::new(1, 8, 7)), + value_node!("arg2".to_string(), Position::new(1, 14, 13)), ], fields: vec![ ( - make_tok!("foo", 1, 24), - Expression::Simple(Value::Int(value_node!(1, 1, 28))), + make_tok!("foo", Position::new(1, 24, 23)), + Expression::Simple(Value::Int(value_node!(1, Position::new(1, 28, 27)))), ), ( - make_tok!("bar", 1, 30), - Expression::Simple(Value::Int(value_node!(2, 1, 34))), + make_tok!("bar", Position::new(1, 30, 29)), + Expression::Simple(Value::Int(value_node!(2, Position::new(1, 34, 33)))), ), ], - pos: Position::new(1, 1), + pos: Position::new(1, 1, 0), }) ); } #[test] fn test_copy_parse() { - assert_error!(copy_expression("{}")); - assert_error!(copy_expression("foo")); - assert_error!(copy_expression("foo{")); + assert_fail!(copy_expression("{}")); + assert_fail!(copy_expression("foo")); + assert_fail!(copy_expression("foo{")); assert_parse!( copy_expression("foo{}"), Expression::Copy(CopyDef { - selector: make_selector!(make_expr!("foo", 1, 1), 1, 1), + selector: make_selector!( + make_expr!("foo", Position::new(1, 1, 0)), + Position::new(1, 1, 0) + ), fields: Vec::new(), - pos: Position::new(1, 1), + pos: Position::new(1, 1, 0), }) ); assert_parse!( copy_expression("foo{bar=1}"), Expression::Copy(CopyDef { - selector: make_selector!(make_expr!("foo", 1, 1), 1, 1), + selector: make_selector!( + make_expr!("foo", Position::new(1, 1, 0)), + Position::new(1, 1, 0) + ), fields: vec![( - make_tok!("bar", 1, 5), - Expression::Simple(Value::Int(value_node!(1, 1, 9))), + make_tok!("bar", Position::new(1, 5, 4)), + Expression::Simple(Value::Int(value_node!(1, Position::new(1, 9, 8)))), )], - pos: Position::new(1, 1), + pos: Position::new(1, 1, 0), }) ); assert_parse!( copy_expression("foo{bar=1,}"), Expression::Copy(CopyDef { - selector: make_selector!(make_expr!("foo", 1, 1), 1, 1), + selector: make_selector!( + make_expr!("foo", Position::new(1, 1, 0)), + Position::new(1, 1, 0) + ), fields: vec![( - make_tok!("bar", 1, 5), - Expression::Simple(Value::Int(value_node!(1, 1, 9))), + make_tok!("bar", Position::new(1, 5, 4)), + Expression::Simple(Value::Int(value_node!(1, Position::new(1, 9, 8)))), )], - pos: Position::new(1, 1), + pos: Position::new(1, 1, 0), }) ); } #[test] fn test_grouped_expression_parse() { - assert_error!(grouped_expression("foo")); - assert_error!(grouped_expression("(foo")); + assert_fail!(grouped_expression("foo")); + assert_fail!(grouped_expression("(foo")); assert_parse!( grouped_expression("(foo)"), Expression::Grouped(Box::new(Expression::Simple(Value::Selector( - make_selector!(make_expr!("foo", 1, 2), 1, 2) + make_selector!( + make_expr!("foo", Position::new(1, 2, 1)), + Position::new(1, 2, 1) + ) )))) ); assert_parse!( grouped_expression("(1 + 1)"), Expression::Grouped(Box::new(Expression::Binary(BinaryOpDef { kind: BinaryExprType::Add, - left: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 2)))), - right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 6)))), - pos: Position::new(1, 2), + left: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 2, 1) + )))), + right: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 6, 5) + )))), + pos: Position::new(1, 2, 1), }))) ); } #[test] fn test_list_value_parse() { - assert_error!(list_value("foo")); - assert_error!(list_value("[foo")); - assert_error!(list_value("// commen\n[foo")); + assert_fail!(list_value("foo")); + assert_fail!(list_value("[foo")); + assert_fail!(list_value("// commen\n[foo")); assert_parse!( list_value("[foo]"), Value::List(ListDef { elems: vec![Expression::Simple(Value::Selector(make_selector!( - make_expr!("foo", 1, 2), - 1, - 2 + make_expr!("foo", Position::new(1, 2, 1)), + Position::new(1, 2, 1) )))], - pos: Position::new(1, 1), + pos: Position::new(1, 1, 0), }) ); @@ -874,10 +1069,10 @@ fn test_list_value_parse() { list_value("[1, 1]"), Value::List(ListDef { elems: vec![ - Expression::Simple(Value::Int(value_node!(1, 1, 2))), - Expression::Simple(Value::Int(value_node!(1, 1, 5))), + Expression::Simple(Value::Int(value_node!(1, Position::new(1, 2, 1)))), + Expression::Simple(Value::Int(value_node!(1, Position::new(1, 5, 4)))), ], - pos: Position::new(1, 1), + pos: Position::new(1, 1, 0), }) ); @@ -885,10 +1080,10 @@ fn test_list_value_parse() { list_value("[1, 1,]"), Value::List(ListDef { elems: vec![ - Expression::Simple(Value::Int(value_node!(1, 1, 2))), - Expression::Simple(Value::Int(value_node!(1, 1, 5))), + Expression::Simple(Value::Int(value_node!(1, Position::new(1, 2, 1)))), + Expression::Simple(Value::Int(value_node!(1, Position::new(1, 5, 4)))), ], - pos: Position::new(1, 1), + pos: Position::new(1, 1, 0), }) ); @@ -896,10 +1091,10 @@ fn test_list_value_parse() { list_value("// comment\n[1, 1]"), Value::List(ListDef { elems: vec![ - Expression::Simple(Value::Int(value_node!(1, 2, 2))), - Expression::Simple(Value::Int(value_node!(1, 2, 5))), + Expression::Simple(Value::Int(value_node!(1, Position::new(2, 2, 12)))), + Expression::Simple(Value::Int(value_node!(1, Position::new(2, 5, 14)))), ], - pos: Position::new(2, 1), + pos: Position::new(2, 1, 11), }) ); @@ -907,10 +1102,10 @@ fn test_list_value_parse() { list_value("[// comment\n1, 1]"), Value::List(ListDef { elems: vec![ - Expression::Simple(Value::Int(value_node!(1, 2, 2))), - Expression::Simple(Value::Int(value_node!(1, 2, 5))), + Expression::Simple(Value::Int(value_node!(1, Position::new(2, 2, 12)))), + Expression::Simple(Value::Int(value_node!(1, Position::new(2, 5, 15)))), ], - pos: Position::new(1, 1), + pos: Position::new(1, 1, 0), }) ); @@ -918,10 +1113,10 @@ fn test_list_value_parse() { list_value("[1, // comment\n1]"), Value::List(ListDef { elems: vec![ - Expression::Simple(Value::Int(value_node!(1, 1, 2))), - Expression::Simple(Value::Int(value_node!(1, 2, 1))), + Expression::Simple(Value::Int(value_node!(1, Position::new(1, 2, 1)))), + Expression::Simple(Value::Int(value_node!(1, Position::new(2, 1, 14)))), ], - pos: Position::new(1, 1), + pos: Position::new(1, 1, 0), }) ); @@ -929,35 +1124,37 @@ fn test_list_value_parse() { list_value("[1, 1 // comment\n]"), Value::List(ListDef { elems: vec![ - Expression::Simple(Value::Int(value_node!(1, 1, 2))), - Expression::Simple(Value::Int(value_node!(1, 1, 5))), + Expression::Simple(Value::Int(value_node!(1, Position::new(1, 2, 1)))), + Expression::Simple(Value::Int(value_node!(1, Position::new(1, 5, 4)))), ], - pos: Position::new(1, 1), + pos: Position::new(1, 1, 0), }) ); } #[test] fn test_tuple_parse() { - assert_error!(tuple("{")); - assert_error!(tuple("{ foo")); - assert_error!(tuple("{ foo =")); - assert_error!(tuple("{ foo = 1")); - assert_error!(tuple("{ foo = 1,")); - assert_error!(tuple("{ foo = 1, bar =")); - assert_error!(tuple("// comment\n{ foo = 1, bar =")); + assert_fail!(tuple("{")); + assert_fail!(tuple("{ foo")); + assert_fail!(tuple("{ foo =")); + assert_fail!(tuple("{ foo = 1")); + assert_fail!(tuple("{ foo = 1,")); + assert_fail!(tuple("{ foo = 1, bar =")); + assert_fail!(tuple("// comment\n{ foo = 1, bar =")); - assert_parse!(tuple("{ }"), Value::Tuple(value_node!(vec![], 1, 1))); + assert_parse!( + tuple("{ }"), + Value::Tuple(value_node!(vec![], Position::new(1, 1, 0))) + ); assert_parse!( tuple("{ foo = 1 }"), Value::Tuple(value_node!( vec![( - make_tok!("foo", 1, 3), - Expression::Simple(Value::Int(value_node!(1, 1, 9))), + make_tok!("foo", Position::new(1, 3, 2)), + Expression::Simple(Value::Int(value_node!(1, Position::new(1, 9, 8)))), )], - 1, - 1 + Position::new(1, 1, 0) )) ); @@ -965,11 +1162,10 @@ fn test_tuple_parse() { tuple("// comment\n{ foo = 1 }"), Value::Tuple(value_node!( vec![( - make_tok!("foo", 2, 3), - Expression::Simple(Value::Int(value_node!(1, 2, 9))), + make_tok!("foo", Position::new(2, 3, 13)), + Expression::Simple(Value::Int(value_node!(1, Position::new(2, 9, 19)))), )], - 1, - 1 + Position::new(1, 1, 0) )) ); @@ -977,11 +1173,10 @@ fn test_tuple_parse() { tuple("{// comment\n foo = 1 }"), Value::Tuple(value_node!( vec![( - make_tok!("foo", 2, 2), - Expression::Simple(Value::Int(value_node!(1, 2, 8))), + make_tok!("foo", Position::new(2, 2, 13)), + Expression::Simple(Value::Int(value_node!(1, Position::new(2, 8, 19)))), )], - 1, - 1 + Position::new(1, 1, 0) )) ); @@ -989,11 +1184,10 @@ fn test_tuple_parse() { tuple("{ foo = 1// comment\n }"), Value::Tuple(value_node!( vec![( - make_tok!("foo", 1, 3), - Expression::Simple(Value::Int(value_node!(1, 1, 9))), + make_tok!("foo", Position::new(1, 3, 2)), + Expression::Simple(Value::Int(value_node!(1, Position::new(1, 9, 8)))), )], - 1, - 1 + Position::new(1, 1, 0) )) ); @@ -1002,19 +1196,18 @@ fn test_tuple_parse() { Value::Tuple(value_node!( vec![ ( - make_tok!("foo", 1, 3), - Expression::Simple(Value::Int(value_node!(1, 1, 9))), + make_tok!("foo", Position::new(1, 3, 2)), + Expression::Simple(Value::Int(value_node!(1, Position::new(1, 9, 8)))), ), ( - make_tok!("bar", 1, 12), + make_tok!("bar", Position::new(1, 12, 11)), Expression::Simple(Value::Str(value_node!( "1".to_string(), - Position::new(1, 18) + Position::new(1, 18, 17) ))), ), ], - 1, - 1 + Position::new(1, 1, 0) )) ); assert_parse!( @@ -1022,19 +1215,18 @@ fn test_tuple_parse() { Value::Tuple(value_node!( vec![ ( - make_tok!("foo", 1, 3), - Expression::Simple(Value::Int(value_node!(1, 1, 9))), + make_tok!("foo", Position::new(1, 3, 2)), + Expression::Simple(Value::Int(value_node!(1, Position::new(1, 9, 8)))), ), ( - make_tok!("bar", 2, 1), + make_tok!("bar", Position::new(2, 1, 22)), Expression::Simple(Value::Str(value_node!( "1".to_string(), - Position::new(2, 7) + Position::new(2, 7, 28) ))), ), ], - 1, - 1 + Position::new(1, 1, 0) )) ); assert_parse!( @@ -1042,16 +1234,18 @@ fn test_tuple_parse() { Value::Tuple(value_node!( vec![ ( - make_tok!("foo", 1, 3), - Expression::Simple(Value::Int(value_node!(1, Position::new(1, 9)))), + make_tok!("foo", Position::new(1, 3, 2)), + Expression::Simple(Value::Int(value_node!(1, Position::new(1, 9, 8)))), ), ( - make_tok!("bar", 1, 12), - Expression::Simple(Value::Tuple(value_node!(Vec::new(), Position::new(1, 17)))), + make_tok!("bar", Position::new(1, 12, 11)), + Expression::Simple(Value::Tuple(value_node!( + Vec::new(), + Position::new(1, 17, 16) + ))), ), ], - 1, - 1 + Position::new(1, 1, 0) )) ); assert_parse!( @@ -1059,16 +1253,18 @@ fn test_tuple_parse() { Value::Tuple(value_node!( vec![ ( - make_tok!("foo", 1, 3), - Expression::Simple(Value::Int(value_node!(1, Position::new(1, 9)))), + make_tok!("foo", Position::new(1, 3, 2)), + Expression::Simple(Value::Int(value_node!(1, Position::new(1, 9, 8)))), ), ( - make_tok!("bar", 1, 12), - Expression::Simple(Value::Tuple(value_node!(Vec::new(), Position::new(1, 17)))), + make_tok!("bar", Position::new(1, 12, 11)), + Expression::Simple(Value::Tuple(value_node!( + Vec::new(), + Position::new(1, 17, 16) + ))), ), ], - 1, - 1 + Position::new(1, 1, 0) )) ); @@ -1077,26 +1273,35 @@ fn test_tuple_parse() { Expression::Simple(Value::Tuple(value_node!( vec![ ( - make_tok!("foo", 1, 3), - Expression::Simple(Value::Int(value_node!(1, Position::new(1, 9)))), + make_tok!("foo", Position::new(1, 3, 2)), + Expression::Simple(Value::Int(value_node!(1, Position::new(1, 9, 8)))), ), ( - make_tok!("lst", 1, 12), + make_tok!("lst", Position::new(1, 12, 11)), Expression::Simple(Value::List(ListDef { elems: vec![ - Expression::Simple(Value::Int(value_node!(1, Position::new(1, 19)))), - Expression::Simple(Value::Int(value_node!(2, Position::new(1, 22)))), - Expression::Simple(Value::Int(value_node!(3, Position::new(1, 25)))), + Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 19, 18) + ))), + Expression::Simple(Value::Int(value_node!( + 2, + Position::new(1, 22, 21) + ))), + Expression::Simple(Value::Int(value_node!( + 3, + Position::new(1, 25, 24) + ))), ], pos: Position { line: 1, column: 18, + offset: 17, }, })), ), ], - 1, - 1 + Position::new(1, 1, 0) ))) ); } @@ -1107,8 +1312,14 @@ fn test_field_list_parse() { assert_parse!( field_list(f_list), vec![ - (make_tok!("foo", 1, 1), make_expr!(1 => int, 1, 7)), - (make_tok!("quux", 1, 10), make_expr!(2 => int, 1, 17)), + ( + make_tok!("foo", Position::new(1, 1, 0)), + make_expr!(1 => int, Position::new(1, 7, 6)) + ), + ( + make_tok!("quux", Position::new(1, 10, 9)), + make_expr!(2 => int, Position::new(1, 17, 16)) + ), ] ); @@ -1116,8 +1327,14 @@ fn test_field_list_parse() { assert_parse!( field_list(f_list), vec![ - (make_tok!("foo", 1, 1), make_expr!(1 => int, 1, 7)), - (make_tok!("quux", 2, 1), make_expr!(2 => int, 2, 8)), + ( + make_tok!("foo", Position::new(1, 1, 0)), + make_expr!(1 => int, Position::new(1, 7, 6)) + ), + ( + make_tok!("quux", Position::new(2, 1, 20)), + make_expr!(2 => int, Position::new(2, 8, 27)) + ), ] ); @@ -1125,23 +1342,32 @@ fn test_field_list_parse() { assert_parse!( field_list(f_list), vec![ - (make_tok!("foo", 1, 1), make_expr!(1 => int, 1, 7)), - (make_tok!("quux", 3, 1), make_expr!(2 => int, 3, 8)), + ( + make_tok!("foo", Position::new(1, 1, 0)), + make_expr!(1 => int, Position::new(1, 7, 6)) + ), + ( + make_tok!("quux", Position::new(3, 1, 20)), + make_expr!(2 => int, Position::new(3, 8, 28)) + ), ] ); f_list = "foo = 1,\nquux = [1, 2],"; assert_parse!( field_list(f_list), vec![ - (make_tok!("foo", 1, 1), make_expr!(1 => int, 1, 7)), ( - make_tok!("quux", 2, 1), + make_tok!("foo", Position::new(1, 1, 0)), + make_expr!(1 => int, Position::new(1, 7, 6)) + ), + ( + make_tok!("quux", Position::new(2, 1, 9)), Expression::Simple(Value::List(ListDef { elems: vec![ - Expression::Simple(Value::Int(value_node!(1, Position::new(2, 9)))), - Expression::Simple(Value::Int(value_node!(2, Position::new(2, 12)))), + Expression::Simple(Value::Int(value_node!(1, Position::new(2, 9, 17)))), + Expression::Simple(Value::Int(value_node!(2, Position::new(2, 12, 20)))), ], - pos: Position::new(2, 8), + pos: Position::new(2, 8, 16), })), ), ] @@ -1150,75 +1376,77 @@ fn test_field_list_parse() { #[test] fn test_field_value_parse() { - assert_error!(field_value("foo")); - assert_error!(field_value("// comment\nfoo")); - assert_error!(field_value("foo =")); + assert_fail!(field_value("foo")); + assert_fail!(field_value("// comment\nfoo")); + assert_fail!(field_value("foo =")); assert_parse!( field_value("foo = 1"), ( - make_tok!("foo", 1, 1), - Expression::Simple(Value::Int(value_node!(1, 1, 7))) + make_tok!("foo", Position::new(1, 1, 0)), + Expression::Simple(Value::Int(value_node!(1, Position::new(1, 7, 6)))) ) ); assert_parse!( field_value("foo = 1 // foo comment\n"), ( - make_tok!("foo", 1, 1), - Expression::Simple(Value::Int(value_node!(1, 1, 7))) + make_tok!("foo", Position::new(1, 1, 0)), + Expression::Simple(Value::Int(value_node!(1, Position::new(1, 7, 6)))) ) ); assert_parse!( field_value("foo // foo comment\n = 1"), ( - make_tok!("foo", 1, 1), - Expression::Simple(Value::Int(value_node!(1, 2, 4))) + make_tok!("foo", Position::new(1, 1, 0)), + Expression::Simple(Value::Int(value_node!(1, Position::new(2, 4, 4)))) ) ); assert_parse!( field_value("// foo comment\nfoo = 1"), ( - make_tok!("foo", 2, 1), - Expression::Simple(Value::Int(value_node!(1, 2, 7))) + make_tok!("foo", Position::new(2, 1, 15)), + Expression::Simple(Value::Int(value_node!(1, Position::new(2, 7, 21)))) ) ); assert_parse!( field_value("foo = \"1\""), ( - make_tok!("foo", 1, 1), - Expression::Simple(Value::Str(value_node!("1".to_string(), 1, 7))) + make_tok!("foo", Position::new(1, 1, 0)), + Expression::Simple(Value::Str(value_node!( + "1".to_string(), + Position::new(1, 7, 6) + ))) ) ); assert_parse!( field_value("foo = bar "), ( - make_tok!("foo", 1, 1), + make_tok!("foo", Position::new(1, 1, 0)), Expression::Simple(Value::Selector(make_selector!( - make_expr!("bar", 1, 7), - 1, - 7 + make_expr!("bar", Position::new(1, 7, 6)), + Position::new(1, 7, 6) ))) ) ); assert_parse!( field_value("foo = bar.baz "), ( - make_tok!("foo", 1, 1), + make_tok!("foo", Position::new(1, 1, 0)), Expression::Simple(Value::Selector( - make_selector!(make_expr!("bar", 1, 7) => [ make_tok!("baz", 1, 11) ] => 1, 7), + make_selector!(make_expr!("bar", Position::new(1, 7, 6)) => [ make_tok!("baz", Position::new(1, 11, 10)) ] => Position::new(1, 7, 6)), )) ) ); assert_parse!( field_value("foo = [1,2], "), ( - make_tok!("foo", 1, 1), + make_tok!("foo", Position::new(1, 1, 0)), Expression::Simple(Value::List(ListDef { elems: vec![ - Expression::Simple(Value::Int(value_node!(1, Position::new(1, 8)))), - Expression::Simple(Value::Int(value_node!(2, Position::new(1, 10)))), + Expression::Simple(Value::Int(value_node!(1, Position::new(1, 8, 7)))), + Expression::Simple(Value::Int(value_node!(2, Position::new(1, 10, 9)))), ], - pos: Position::new(1, 7), + pos: Position::new(1, 7, 6), })) ) ); @@ -1226,22 +1454,34 @@ fn test_field_value_parse() { #[test] fn test_number_parsing() { - assert_error!(number(".")); - assert_error!(number(". ")); - assert_parse!(number("1.0"), Value::Float(value_node!(1.0, 1, 1))); - assert_parse!(number("1."), Value::Float(value_node!(1.0, 1, 1))); - assert_parse!(number("1"), Value::Int(value_node!(1, 1, 1))); - assert_parse!(number(".1"), Value::Float(value_node!(0.1, 1, 1))); + assert_fail!(number(".")); + assert_fail!(number(". ")); + assert_parse!( + number("1.0"), + Value::Float(value_node!(1.0, Position::new(1, 1, 0))) + ); + assert_parse!( + number("1."), + Value::Float(value_node!(1.0, Position::new(1, 1, 0))) + ); + assert_parse!( + number("1"), + Value::Int(value_node!(1, Position::new(1, 1, 0))) + ); + assert_parse!( + number(".1"), + Value::Float(value_node!(0.1, Position::new(1, 1, 0))) + ); } #[test] fn test_parse() { - let bad_input = StrIter::new("import mylib as lib;"); + let bad_input = OffsetStrIter::new("import mylib as lib;"); let bad_result = parse(bad_input); assert!(bad_result.is_err()); // Valid parsing tree - let input = StrIter::new("import \"mylib\" as lib;let foo = 1;1+1;"); + let input = OffsetStrIter::new("import \"mylib\" as lib;let foo = 1;1+1;"); let result = parse(input); assert!(result.is_ok(), format!("Expected Ok, Got {:?}", result)); let tpl = result.unwrap(); @@ -1249,18 +1489,24 @@ fn test_parse() { tpl, vec![ Statement::Import(ImportDef { - path: make_tok!(QUOT => "mylib", 1, 8), - name: make_tok!("lib", 1, 19), + path: make_tok!(QUOT => "mylib", Position::new(1, 8, 7)), + name: make_tok!("lib", Position::new(1, 19, 18)), }), Statement::Let(LetDef { - name: make_tok!("foo", 1, 27), - value: Expression::Simple(Value::Int(value_node!(1, 1, 33))), + name: make_tok!("foo", Position::new(1, 27, 26)), + value: Expression::Simple(Value::Int(value_node!(1, Position::new(1, 33, 32)))), }), Statement::Expression(Expression::Binary(BinaryOpDef { kind: BinaryExprType::Add, - left: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 35)))), - right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 37)))), - pos: Position::new(1, 35), + left: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 35, 34) + )))), + right: Box::new(Expression::Simple(Value::Int(value_node!( + 1, + Position::new(1, 37, 36) + )))), + pos: Position::new(1, 35, 34), })), ] ); diff --git a/src/tokenizer/mod.rs b/src/tokenizer/mod.rs index e23f95e..177ca28 100644 --- a/src/tokenizer/mod.rs +++ b/src/tokenizer/mod.rs @@ -16,20 +16,19 @@ use std; use abortable_parser::combinators::*; -use abortable_parser::iter::{SliceIter, StrIter}; +use abortable_parser::iter::SliceIter; use abortable_parser::{Error, Offsetable, Result, TextPositionTracker}; -use ast::*; -impl<'a> From> for Position { - fn from(s: StrIter<'a>) -> Position { - Position { - line: s.line(), - column: s.column(), - } +use ast::*; +use iter::OffsetStrIter; + +impl<'a> From> for Position { + fn from(s: OffsetStrIter<'a>) -> Position { + Position::new(s.line(), s.column(), s.get_offset()) } } -fn is_symbol_char<'a>(i: StrIter<'a>) -> Result, u8> { +fn is_symbol_char<'a>(i: OffsetStrIter<'a>) -> Result, u8> { let mut _i = i.clone(); let c = match _i.next() { Some(c) => *c, @@ -42,7 +41,7 @@ fn is_symbol_char<'a>(i: StrIter<'a>) -> Result, u8> { } } -fn escapequoted<'a>(input: StrIter<'a>) -> Result, String> { +fn escapequoted<'a>(input: OffsetStrIter<'a>) -> Result, String> { // loop until we find a " that is not preceded by \. // Collapse all \ to just char for escaping. let mut frag = String::new(); @@ -69,7 +68,7 @@ fn escapequoted<'a>(input: StrIter<'a>) -> Result, String> { return Result::Incomplete(_input.get_offset()); } -make_fn!(strtok, +make_fn!(strtok, do_each!( span => input!(), _ => text_token!("\""), @@ -82,7 +81,7 @@ make_fn!(strtok, ) ); -make_fn!(pipequotetok, +make_fn!(pipequotetok, do_each!( p => input!(), _ => text_token!("|"), @@ -96,7 +95,7 @@ make_fn!(pipequotetok, ) ); -make_fn!(barewordtok, +make_fn!(barewordtok, do_each!( span => input!(), _ => peek!(ascii_alpha), @@ -109,7 +108,7 @@ make_fn!(barewordtok, ) ); -make_fn!(digittok, +make_fn!(digittok, do_each!( span => input!(), _ => peek!(ascii_digit), @@ -122,7 +121,7 @@ make_fn!(digittok, ) ); -make_fn!(booleantok, +make_fn!(booleantok, do_each!( span => input!(), token => either!( @@ -142,159 +141,159 @@ make_fn!(booleantok, macro_rules! do_text_token_tok { ($i:expr, $type:expr, $text_token:expr, WS) => { do_each!($i, - span => input!(), - frag => text_token!($text_token), - _ => either!(whitespace, comment), - (Token { - typ: $type, - pos: Position::from(span), - fragment: frag.to_string(), - }) - ) + span => input!(), + frag => text_token!($text_token), + _ => either!(whitespace, comment), + (Token { + typ: $type, + pos: Position::from(span), + fragment: frag.to_string(), + }) + ) }; ($i:expr, $type:expr, $text_token:expr) => { do_each!($i, - span => input!(), - frag => text_token!($text_token), - (Token { - typ: $type, - pos: Position::from(span), - fragment: frag.to_string(), - }) - ) + span => input!(), + frag => text_token!($text_token), + (Token { + typ: $type, + pos: Position::from(span), + fragment: frag.to_string(), + }) + ) }; } -make_fn!(emptytok, +make_fn!(emptytok, do_text_token_tok!(TokenType::EMPTY, "NULL") ); -make_fn!(commatok, +make_fn!(commatok, do_text_token_tok!(TokenType::PUNCT, ",") ); -make_fn!(lbracetok, +make_fn!(lbracetok, do_text_token_tok!(TokenType::PUNCT, "{") ); -make_fn!(rbracetok, +make_fn!(rbracetok, do_text_token_tok!(TokenType::PUNCT, "}") ); -make_fn!(lparentok, +make_fn!(lparentok, do_text_token_tok!(TokenType::PUNCT, "(") ); -make_fn!(rparentok, +make_fn!(rparentok, do_text_token_tok!(TokenType::PUNCT, ")") ); -make_fn!(dottok, +make_fn!(dottok, do_text_token_tok!(TokenType::PUNCT, ".") ); -make_fn!(plustok, +make_fn!(plustok, do_text_token_tok!(TokenType::PUNCT, "+") ); -make_fn!(dashtok, +make_fn!(dashtok, do_text_token_tok!(TokenType::PUNCT, "-") ); -make_fn!(startok, +make_fn!(startok, do_text_token_tok!(TokenType::PUNCT, "*") ); -make_fn!(slashtok, +make_fn!(slashtok, do_text_token_tok!(TokenType::PUNCT, "/") ); -make_fn!(pcttok, +make_fn!(pcttok, do_text_token_tok!(TokenType::PUNCT, "%") ); -make_fn!(eqeqtok, +make_fn!(eqeqtok, do_text_token_tok!(TokenType::PUNCT, "==") ); -make_fn!(notequaltok, +make_fn!(notequaltok, do_text_token_tok!(TokenType::PUNCT, "!=") ); -make_fn!(gttok, +make_fn!(gttok, do_text_token_tok!(TokenType::PUNCT, ">") ); -make_fn!(gtequaltok, +make_fn!(gtequaltok, do_text_token_tok!(TokenType::PUNCT, ">=") ); -make_fn!(ltequaltok, +make_fn!(ltequaltok, do_text_token_tok!(TokenType::PUNCT, "<=") ); -make_fn!(lttok, +make_fn!(lttok, do_text_token_tok!(TokenType::PUNCT, "<") ); -make_fn!(equaltok, +make_fn!(equaltok, do_text_token_tok!(TokenType::PUNCT, "=") ); -make_fn!(semicolontok, +make_fn!(semicolontok, do_text_token_tok!(TokenType::PUNCT, ";") ); -make_fn!(leftsquarebracket, +make_fn!(leftsquarebracket, do_text_token_tok!(TokenType::PUNCT, "[") ); -make_fn!(rightsquarebracket, +make_fn!(rightsquarebracket, do_text_token_tok!(TokenType::PUNCT, "]") ); -make_fn!(fatcommatok, +make_fn!(fatcommatok, do_text_token_tok!(TokenType::PUNCT, "=>") ); -make_fn!(selecttok, +make_fn!(selecttok, do_text_token_tok!(TokenType::BAREWORD, "select", WS) ); -make_fn!(macrotok, +make_fn!(macrotok, do_text_token_tok!(TokenType::BAREWORD, "macro", WS) ); -make_fn!(lettok, +make_fn!(lettok, do_text_token_tok!(TokenType::BAREWORD, "let", WS) ); -make_fn!(importtok, +make_fn!(importtok, do_text_token_tok!(TokenType::BAREWORD, "import", WS) ); -make_fn!(asserttok, +make_fn!(asserttok, do_text_token_tok!(TokenType::BAREWORD, "assert", WS) ); -make_fn!(outtok, +make_fn!(outtok, do_text_token_tok!(TokenType::BAREWORD, "out", WS) ); -make_fn!(astok, +make_fn!(astok, do_text_token_tok!(TokenType::BAREWORD, "as", WS) ); -make_fn!(maptok, +make_fn!(maptok, do_text_token_tok!(TokenType::BAREWORD, "map", WS) ); -make_fn!(filtertok, +make_fn!(filtertok, do_text_token_tok!(TokenType::BAREWORD, "filter", WS) ); -fn comment(input: StrIter) -> Result { +fn comment(input: OffsetStrIter) -> Result { match text_token!(input, "//") { Result::Complete(rest, _) => { match until!( @@ -306,12 +305,7 @@ fn comment(input: StrIter) -> Result { ) ) { Result::Complete(rest, cmt) => { - return Result::Complete( - rest, - make_tok!(CMT => cmt.to_string(), - input.line() as usize, - input.column() as usize), - ); + return Result::Complete(rest, make_tok!(CMT => cmt.to_string(), input)); } // If we didn't find a new line then we just grab everything. _ => { @@ -325,7 +319,7 @@ fn comment(input: StrIter) -> Result { } } -make_fn!(whitespace, +make_fn!(whitespace, do_each!( span => input!(), _ => peek!(ascii_ws), @@ -338,7 +332,7 @@ make_fn!(whitespace, ) ); -make_fn!(end_of_input, +make_fn!(end_of_input, do_each!( span => input!(), _ => eoi, @@ -350,7 +344,7 @@ make_fn!(end_of_input, ) ); -make_fn!(token, +make_fn!(token, either!( strtok, pipequotetok, @@ -394,14 +388,15 @@ make_fn!(token, end_of_input) ); -/// Consumes an input StrIter and returns either a Vec or a error::Error. -pub fn tokenize(input: StrIter) -> std::result::Result, Error> { +/// Consumes an input OffsetStrIter and returns either a Vec or a error::Error. +pub fn tokenize(input: OffsetStrIter) -> std::result::Result, Error> { let mut out = Vec::new(); let mut i = input.clone(); loop { if let Result::Complete(_, _) = eoi(i.clone()) { break; } + // FIXME(jwall): We need to return a error::Error so we have position information. match token(i.clone()) { Result::Abort(e) => { return Err(Error::caused_by( @@ -434,10 +429,7 @@ pub fn tokenize(input: StrIter) -> std::result::Result, Error> { out.push(Token { fragment: String::new(), typ: TokenType::END, - pos: Position { - line: i.line(), - column: i.column(), - }, + pos: i.into(), }); Ok(out) } @@ -616,13 +608,7 @@ pub fn pos<'a>(i: SliceIter<'a, Token>) -> Result, Position let tok = _i.next().unwrap(); let line = tok.pos.line; let column = tok.pos.column; - Result::Complete( - i.clone(), - Position { - line: line, - column: column, - }, - ) + Result::Complete(i.clone(), Position::new(line, column, i.get_offset())) } #[cfg(test)] diff --git a/src/tokenizer/test.rs b/src/tokenizer/test.rs index ea3e857..748abaf 100644 --- a/src/tokenizer/test.rs +++ b/src/tokenizer/test.rs @@ -1,10 +1,12 @@ use super::*; -use abortable_parser::{Result, SliceIter, StrIter}; +use abortable_parser::{Result, SliceIter}; + +use iter::OffsetStrIter; #[test] fn test_empty_token() { - let result = emptytok(StrIter::new("NULL ")); + let result = emptytok(OffsetStrIter::new("NULL ")); assert!( result.is_complete(), format!("result {:?} is not done", result) @@ -17,7 +19,7 @@ fn test_empty_token() { #[test] fn test_assert_token() { - let result = asserttok(StrIter::new("assert ")); + let result = asserttok(OffsetStrIter::new("assert ")); assert!( result.is_complete(), format!("result {:?} is not done", result) @@ -30,7 +32,7 @@ fn test_assert_token() { #[test] fn test_out_token() { - let result = outtok(StrIter::new("out ")); + let result = outtok(OffsetStrIter::new("out ")); assert!( result.is_complete(), format!("result {:?} is not done", result) @@ -43,7 +45,7 @@ fn test_out_token() { #[test] fn test_out_token_with_comment() { - let result = outtok(StrIter::new("out//comment")); + let result = outtok(OffsetStrIter::new("out//comment")); assert!( result.is_complete(), format!("result {:?} is not done", result) @@ -56,13 +58,13 @@ fn test_out_token_with_comment() { #[test] fn test_not_out_token() { - let result = outtok(StrIter::new("output")); + let result = outtok(OffsetStrIter::new("output")); assert!(result.is_fail(), format!("result {:?} is not fail", result)); } #[test] fn test_escape_quoted() { - let result = escapequoted(StrIter::new("foo \\\"bar\"")); + let result = escapequoted(OffsetStrIter::new("foo \\\"bar\"")); assert!( result.is_complete(), format!("result {:?} is not ok", result) @@ -74,7 +76,7 @@ fn test_escape_quoted() { #[test] fn test_pipe_quoted() { - let result = pipequotetok(StrIter::new("|foo|")); + let result = pipequotetok(OffsetStrIter::new("|foo|")); assert!( result.is_complete(), format!("result {:?} is not ok", result) @@ -87,7 +89,7 @@ fn test_pipe_quoted() { #[test] fn test_string_with_escaping() { - let result = strtok(StrIter::new("\"foo \\\\ \\\"bar\"")); + let result = strtok(OffsetStrIter::new("\"foo \\\\ \\\"bar\"")); assert!( result.is_complete(), format!("result {:?} is not ok", result) @@ -99,7 +101,7 @@ fn test_string_with_escaping() { #[test] fn test_tokenize_bareword_with_dash() { - let result = tokenize(StrIter::new("foo-bar ")); + let result = tokenize(OffsetStrIter::new("foo-bar ")); assert!(result.is_ok(), format!("result {:?} is not ok", result)); if let Ok(toks) = result { assert_eq!(toks.len(), 2); @@ -109,7 +111,7 @@ fn test_tokenize_bareword_with_dash() { macro_rules! assert_token { ($input:expr, $typ:expr, $msg:expr) => { - let result = token(StrIter::new($input)); + let result = token(OffsetStrIter::new($input)); assert!( result.is_complete(), format!("result {:?} is not a {}", result, $msg) @@ -163,7 +165,7 @@ fn test_lteqtok() { #[test] fn test_tokenize_one_of_each() { - let result = tokenize(StrIter::new( + let result = tokenize(OffsetStrIter::new( "map out filter assert let import macro select as => [ ] { } ; = % / * \ + - . ( ) , 1 . foo \"bar\" // comment\n ; true false == < > <= >= !=", )); @@ -178,7 +180,7 @@ fn test_tokenize_one_of_each() { #[test] fn test_parse_has_end() { - let result = tokenize(StrIter::new("foo")); + let result = tokenize(OffsetStrIter::new("foo")); assert!(result.is_ok()); let v = result.unwrap(); assert_eq!(v.len(), 2); @@ -187,8 +189,8 @@ fn test_parse_has_end() { #[test] fn test_whitespace() { - assert!(whitespace(StrIter::new(" ")).is_complete()); - let result = whitespace(StrIter::new(" ")); + assert!(whitespace(OffsetStrIter::new(" ")).is_complete()); + let result = whitespace(OffsetStrIter::new(" ")); match result { Result::Complete(rest, o) => { assert_eq!(rest.get_offset(), 2); @@ -200,9 +202,9 @@ fn test_whitespace() { #[test] fn test_parse_comment() { - assert!(comment(StrIter::new("// comment\n")).is_complete()); - assert!(comment(StrIter::new("// comment")).is_complete()); - let mut parsed = comment(StrIter::new("// comment\n")); + assert!(comment(OffsetStrIter::new("// comment\n")).is_complete()); + assert!(comment(OffsetStrIter::new("// comment")).is_complete()); + let mut parsed = comment(OffsetStrIter::new("// comment\n")); assert!(parsed.is_complete()); if let Result::Complete(_rest, cmt) = parsed { assert_eq!( @@ -210,35 +212,47 @@ fn test_parse_comment() { Token { typ: TokenType::COMMENT, fragment: " comment".to_string(), - pos: Position { line: 1, column: 1 }, + pos: Position { + line: 1, + column: 1, + offset: 0 + }, } ); } - assert!(comment(StrIter::new("// comment\r\n")).is_complete()); - parsed = comment(StrIter::new("// comment\r\n")); + assert!(comment(OffsetStrIter::new("// comment\r\n")).is_complete()); + parsed = comment(OffsetStrIter::new("// comment\r\n")); if let Result::Complete(_rest, cmt) = parsed { assert_eq!( cmt, Token { typ: TokenType::COMMENT, fragment: " comment".to_string(), - pos: Position { column: 1, line: 1 }, + pos: Position { + column: 1, + line: 1, + offset: 0 + }, } ); } - assert!(comment(StrIter::new("// comment\r\n ")).is_complete()); - parsed = comment(StrIter::new("// comment\r\n ")); + assert!(comment(OffsetStrIter::new("// comment\r\n ")).is_complete()); + parsed = comment(OffsetStrIter::new("// comment\r\n ")); if let Result::Complete(_rest, cmt) = parsed { assert_eq!( cmt, Token { typ: TokenType::COMMENT, fragment: " comment".to_string(), - pos: Position { column: 1, line: 1 }, + pos: Position { + column: 1, + line: 1, + offset: 0 + }, } ); } - assert!(comment(StrIter::new("// comment")).is_complete()); + assert!(comment(OffsetStrIter::new("// comment")).is_complete()); } #[test] @@ -246,7 +260,11 @@ fn test_match_word() { let input = vec![Token { fragment: "foo".to_string(), typ: TokenType::BAREWORD, - pos: Position { line: 1, column: 1 }, + pos: Position { + line: 1, + column: 1, + offset: 0, + }, }]; let result = word!(SliceIter::new(input.as_slice()), "foo"); match result { @@ -260,7 +278,11 @@ fn test_match_word_empty_input() { let input = vec![Token { fragment: "".to_string(), typ: TokenType::END, - pos: Position { line: 1, column: 1 }, + pos: Position { + line: 1, + column: 1, + offset: 0, + }, }]; let result = word!(SliceIter::new(input.as_slice()), "foo"); match result { @@ -278,7 +300,11 @@ fn test_match_punct() { let input = vec![Token { fragment: "!".to_string(), typ: TokenType::PUNCT, - pos: Position { line: 1, column: 1 }, + pos: Position { + line: 1, + column: 1, + offset: 0, + }, }]; let result = punct!(SliceIter::new(input.as_slice()), "!"); match result { @@ -292,7 +318,11 @@ fn test_match_type() { let input = vec![Token { fragment: "foo".to_string(), typ: TokenType::BAREWORD, - pos: Position { line: 1, column: 1 }, + pos: Position { + line: 1, + column: 1, + offset: 0, + }, }]; let result = match_type!(SliceIter::new(input.as_slice()), BAREWORD); match result {