diff --git a/Cargo.toml b/Cargo.toml index 24d71c5..65c9334 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,8 +8,10 @@ readme = "README.md" keywords = ["compiler", "config"] license = "Apache-2.0" +[dependencies.nom] +version = "^3.2" + [dependencies] -nom = "^3.2" nom_locate = "^0.1.1" clap = "~2.26.0" diff --git a/examples/flags.txt b/examples/flags.txt new file mode 100644 index 0000000..1e97b10 --- /dev/null +++ b/examples/flags.txt @@ -0,0 +1 @@ +--db_conn1 'db1.prod.net:3306/testdb' --db_conn2 'db2.prod.net:3306/testdb' --dbconn_list --tmpldir './templates' \ No newline at end of file diff --git a/examples/test.ucg b/examples/test.ucg index e4a728d..f5c1921 100644 --- a/examples/test.ucg +++ b/examples/test.ucg @@ -20,7 +20,7 @@ let db_conns = [db_conn1.conn_string, db_conn2.conn_string]; // Our server configuration. let server_config = { dbconn_list = db_conns, - db_conn1 = db_conns.0, + db_conn1 = db_conns.0, // connection one db_conn2 = db_conns.1, tmpldir = "./templates" }; \ No newline at end of file diff --git a/src/ast.rs b/src/ast.rs index 0e64d3a..0db4cd7 100644 --- a/src/ast.rs +++ b/src/ast.rs @@ -11,6 +11,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. +use std; use std::collections::HashSet; use std::borrow::Borrow; use std::convert::Into; @@ -21,6 +22,29 @@ use std::cmp::PartialEq; use std::hash::Hasher; use std::hash::Hash; +#[derive(Debug,PartialEq)] +pub struct ParseError { + pub pos: Position, + pub description: String, +} + +impl std::fmt::Display for ParseError { + fn fmt(&self, f: &mut std::fmt::Formatter) -> Result<(), std::fmt::Error> { + write!(f, + "Parsing Error {} at line: {} column: {}", + self.description, + self.pos.line, + self.pos.column) + } +} + +impl std::error::Error for ParseError { + fn description(&self) -> &str { + &self.description + } +} + + macro_rules! enum_type_equality { ( $slf:ident, $r:expr, $( $l:pat ),* ) => { match $slf { @@ -52,19 +76,33 @@ impl Position { } } +#[derive(Debug,PartialEq,Eq,Clone,PartialOrd,Ord,Hash)] +pub enum TokenType { + END, + WS, + COMMENT, + QUOTED, + DIGIT, + BAREWORD, + PUNCT, +} + +// FIXME(jwall): We should probably implement copy for this. #[derive(Debug,PartialEq,Eq,Clone,PartialOrd,Ord,Hash)] pub struct Token { + pub typ: TokenType, pub fragment: String, pub pos: Position, } impl Token { - pub fn new>(f: S, line: usize, col: usize) -> Self { - Self::new_with_pos(f, Position::new(line, col)) + pub fn new>(f: S, typ: TokenType, line: usize, col: usize) -> Self { + Self::new_with_pos(f, typ, Position::new(line, col)) } - pub fn new_with_pos>(f: S, pos: Position) -> Self { + pub fn new_with_pos>(f: S, typ: TokenType, pos: Position) -> Self { Token { + typ: typ, fragment: f.into(), pos: pos, } @@ -88,8 +126,32 @@ macro_rules! value_node { #[allow(unused_macros)] macro_rules! make_tok { - ( $e: expr, $l:expr, $c:expr ) => { - Token::new($e, $l, $c) + ( EOF => $l:expr, $c:expr ) => { + Token::new("", TokenType::END, $l, $c) + }; + + ( WS => $l:expr, $c:expr ) => { + Token::new("", TokenType::WS, $l, $c) + }; + + ( CMT => $e:expr, $l:expr, $c:expr ) => { + Token::new($e, TokenType::COMMENT, $l, $c) + }; + + ( QUOT => $e:expr, $l:expr, $c:expr ) => { + Token::new($e, TokenType::QUOTED, $l, $c) + }; + + ( PUNCT => $e:expr, $l:expr, $c:expr ) => { + Token::new($e, TokenType::PUNCT, $l, $c) + }; + + ( DIGIT => $e:expr, $l:expr, $c:expr ) => { + Token::new($e, TokenType::DIGIT, $l, $c) + }; + + ( $e:expr, $l:expr, $c:expr ) => { + Token::new($e, TokenType::BAREWORD, $l, $c) }; } @@ -101,18 +163,22 @@ macro_rules! make_expr { ( $e:expr, $l:expr, $c:expr ) => { Expression::Simple(Value::Symbol(Positioned::new($e.to_string(), $l, $c))) - } + }; + + ( $e:expr => int, $l:expr, $c:expr ) => { + Expression::Simple(Value::Int(Positioned::new($e, $l, $c))) + }; } /// Helper macro for making selectors. -/// +/// /// ``` /// make_selector!(Token::new("tpl", 1, 1), Token::new("fld", 1, 4)); -/// +/// /// make_selector!(Token::new("tpl", 1, 1), vec![Token::new("fld", 1, 4)], => 1, 1); -/// +/// /// make_selector!(foo", ["bar"]); -/// +/// /// make_selector!(foo", ["bar"] => 1, 0); /// ``` #[allow(unused_macros)] @@ -160,7 +226,7 @@ macro_rules! make_selector { let mut list: Vec = Vec::new(); $( - list.push(Token::new($item, 1, col)); + list.push(make_tok!($item, 1, col)); col += $item.len() + 1; )* @@ -178,7 +244,7 @@ macro_rules! make_selector { let mut list: Vec = Vec::new(); $( - list.push(Token::new($item, $l, col)); + list.push(make_tok!($item, $l, col)); col += $item.len() + 1; )* @@ -195,14 +261,14 @@ macro_rules! make_selector { /// /// The expression must evaluate to either a tuple or an array. The token must /// evaluate to either a bareword Symbol or an Int. -/// +/// /// ```ucg /// let foo = { bar = "a thing" }; /// let thing = foo.bar; -/// +/// /// let arr = ["one", "two"]; /// let first = arr.0; -/// +/// /// let berry = {best = "strawberry", unique = "acai"}.best; /// let third = ["uno", "dos", "tres"].1; /// ''' @@ -601,7 +667,7 @@ mod ast_test { let def = MacroDef { argdefs: vec![value_node!("foo".to_string(), 1, 0)], fields: vec![ - (Token::new("f1", 1, 1), Expression::Binary(BinaryOpDef{ + (make_tok!("f1", 1, 1), Expression::Binary(BinaryOpDef{ kind: BinaryExprType::Add, left: Value::Symbol(value_node!("foo".to_string(), 1, 1)), right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))), @@ -618,7 +684,7 @@ mod ast_test { let def = MacroDef { argdefs: vec![value_node!("foo".to_string(), 1, 0)], fields: vec![ - (Token::new("f1", 1, 1), Expression::Binary(BinaryOpDef{ + (make_tok!("f1", 1, 1), Expression::Binary(BinaryOpDef{ kind: BinaryExprType::Add, left: Value::Symbol(value_node!("bar".to_string(), 1, 1)), right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))), @@ -637,10 +703,10 @@ mod ast_test { let def = MacroDef { argdefs: vec![value_node!("foo".to_string(), 1, 0)], fields: vec![ - (Token::new("f1", 1, 1), Expression::Binary(BinaryOpDef{ + (make_tok!("f1", 1, 1), Expression::Binary(BinaryOpDef{ kind: BinaryExprType::Add, left: Value::Selector(make_selector!(make_expr!("foo", 1, 1) => [ - Token::new("quux", 1, 1) ] => 1, 1)), + make_tok!("quux", 1, 1) ] => 1, 1)), right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))), pos: Position::new(1, 0), })), @@ -655,10 +721,10 @@ mod ast_test { let def = MacroDef { argdefs: vec![value_node!("foo".to_string(), 1, 0)], fields: vec![ - (Token::new("f1", 1, 1), Expression::Binary(BinaryOpDef{ + (make_tok!("f1", 1, 1), Expression::Binary(BinaryOpDef{ kind: BinaryExprType::Add, left: Value::Selector(make_selector!(make_expr!("bar", 1, 1) => [ - Token::new("quux", 1, 1) ] => 1, 1)), + make_tok!("quux", 1, 1) ] => 1, 1)), right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))), pos: Position::new(1, 0), })), diff --git a/src/build.rs b/src/build.rs index 1b7af58..71f1378 100644 --- a/src/build.rs +++ b/src/build.rs @@ -22,8 +22,6 @@ use std::ops::Deref; use std::rc::Rc; use std::convert::From; -use nom; - use tokenizer::Span; use ast::*; use format; @@ -153,16 +151,14 @@ impl Display for Val { } write!(f, "]") } - &Val::Macro(_) => { - write!(f, "Macro(..)") - }, + &Val::Macro(_) => write!(f, "Macro(..)"), &Val::Tuple(ref def) => { try!(write!(f, "Tuple(\n")); for v in def.iter() { try!(write!(f, "\t{} = {},\n", v.0.val, v.1)); } write!(f, ")") - }, + } } } } @@ -282,22 +278,15 @@ impl Builder { Ok(()) } - pub fn build_file_string(&mut self, name: &str, input: String) -> BuildResult { + pub fn build_file_string(&mut self, _name: &str, input: String) -> BuildResult { match parse(Span::new(&input)) { - nom::IResult::Done(_span, stmts) => { + Ok(stmts) => { for stmt in stmts.iter() { try!(self.build_stmt(stmt)); } Ok(()) } - nom::IResult::Error(err) => Err(Box::new(err)), - nom::IResult::Incomplete(_) => { - Err(Box::new(error::Error::new( - format!("Could not parse input from file: {}", name), - error::ErrorType::IncompleteParsing, - Position{line: 0, column: 0} - ))) - } + Err(err) => Err(Box::new(err)), } } @@ -433,11 +422,11 @@ impl Builder { &Val::Tuple(_) => { stack.push_back(first.clone()); } - &Val::List(_) =>{ + &Val::List(_) => { stack.push_back(first.clone()); } _ => { - //noop + // noop } } @@ -465,7 +454,8 @@ impl Builder { _ => { return Err(Box::new(error::Error::new(format!("{} is not a Tuple or List", vref), - error::ErrorType::TypeFail, next.pos.clone()))); + error::ErrorType::TypeFail, + next.pos.clone()))); } } } @@ -989,7 +979,7 @@ mod test { (Expression::Simple(Value::String(value_node!("foo".to_string(), 1, 1))), Val::String("foo".to_string())), (Expression::Simple(Value::Tuple(value_node!(vec![ - (Token::new("bar", 1, 1), Expression::Simple(Value::Int(value_node!(1, 1, 1)))) + (make_tok!("bar", 1, 1), Expression::Simple(Value::Int(value_node!(1, 1, 1)))) ], 1, 1))), Val::Tuple(vec![(value_node!("bar".to_string(), 1, 1), Rc::new(Val::Int(1)))])), @@ -1124,7 +1114,7 @@ mod test { (Expression::Copy( CopyDef{ selector: make_selector!(make_expr!("tpl1")), - fields: vec![(Token::new("fld1", 1, 1), + fields: vec![(make_tok!("fld1", 1, 1), Expression::Simple(Value::String(value_node!("2".to_string(), 1, 1))))], pos: Position::new(1, 0)}), Val::Tuple( @@ -1145,7 +1135,7 @@ mod test { (Expression::Copy( CopyDef{ selector: make_selector!(make_expr!("tpl1")), - fields: vec![(Token::new("fld2", 1, 1), + fields: vec![(make_tok!("fld2", 1, 1), Expression::Simple(Value::String(value_node!("2".to_string(), 1, 1))))], pos: Position::new(1, 0), }), @@ -1164,9 +1154,9 @@ mod test { CopyDef{ selector: make_selector!(make_expr!("tpl1")), fields: vec![ - (Token::new("fld1", 1, 1), + (make_tok!("fld1", 1, 1), Expression::Simple(Value::Int(value_node!(3, 1, 1)))), - (Token::new("fld2", 1, 1), + (make_tok!("fld2", 1, 1), Expression::Simple(Value::String(value_node!("2".to_string(), 1, 1)))), ], pos: Position::new(1, 0), @@ -1193,7 +1183,7 @@ mod test { b.out.entry(value_node!("tstmac".to_string(), 1, 0)).or_insert(Rc::new(Val::Macro(MacroDef{ argdefs: vec![value_node!("arg1".to_string(), 1, 0)], fields: vec![ - (Token::new("foo", 1, 1), Expression::Simple(Value::Symbol(value_node!("arg1".to_string(), 1, 1)))), + (make_tok!("foo", 1, 1), Expression::Simple(Value::Symbol(value_node!("arg1".to_string(), 1, 1)))), ], pos: Position::new(1, 0), }))); @@ -1220,7 +1210,7 @@ mod test { b.out.entry(value_node!("tstmac".to_string(), 1, 0)).or_insert(Rc::new(Val::Macro(MacroDef{ argdefs: vec![value_node!("arg2".to_string(), 1, 0)], fields: vec![ - (Token::new("foo", 1, 1), Expression::Simple(Value::Symbol(value_node!("arg1".to_string(), 1, 1)))), + (make_tok!("foo", 1, 1), Expression::Simple(Value::Symbol(value_node!("arg1".to_string(), 1, 1)))), ], pos: Position::new(1, 0), }))); @@ -1250,8 +1240,8 @@ mod test { val: Box::new(Expression::Simple(Value::Symbol(value_node!("foo".to_string(), 1, 1)))), default: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))), tuple: vec![ - (Token::new("foo", 1, 1), Expression::Simple(Value::String(value_node!("2".to_string(), 1, 1)))), - (Token::new("bar", 1, 1), Expression::Simple(Value::Int(value_node!(2, 1, 1)))), + (make_tok!("foo", 1, 1), Expression::Simple(Value::String(value_node!("2".to_string(), 1, 1)))), + (make_tok!("bar", 1, 1), Expression::Simple(Value::Int(value_node!(2, 1, 1)))), ], pos: Position::new(1, 0), }), @@ -1260,8 +1250,8 @@ mod test { val: Box::new(Expression::Simple(Value::Symbol(value_node!("baz".to_string(), 1, 1)))), default: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))), tuple: vec![ - (Token::new("bar", 1, 1), Expression::Simple(Value::Int(value_node!(2, 1, 1)))), - (Token::new("quux", 1, 1), Expression::Simple(Value::String(value_node!("2".to_string(), 1, 1)))), + (make_tok!("bar", 1, 1), Expression::Simple(Value::Int(value_node!(2, 1, 1)))), + (make_tok!("quux", 1, 1), Expression::Simple(Value::String(value_node!("2".to_string(), 1, 1)))), ], pos: Position::new(1, 0), }), @@ -1282,8 +1272,8 @@ mod test { val: Box::new(Expression::Simple(Value::Symbol(value_node!("foo".to_string(), 1, 1)))), default: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))), tuple: vec![ - (Token::new("bar", 1, 1), Expression::Simple(Value::Int(value_node!(2, 1, 1)))), - (Token::new("quux", 1, 1), Expression::Simple(Value::String(value_node!("2".to_string(), 1, 1)))), + (make_tok!("bar", 1, 1), Expression::Simple(Value::Int(value_node!(2, 1, 1)))), + (make_tok!("quux", 1, 1), Expression::Simple(Value::String(value_node!("2".to_string(), 1, 1)))), ], pos: Position::new(1, 0), }), @@ -1295,7 +1285,7 @@ mod test { fn test_let_statement() { let mut b = Builder::new(); let stmt = Statement::Let(LetDef { - name: Token::new("foo", 1, 1), + name: make_tok!("foo", 1, 1), value: Expression::Simple(Value::String(value_node!("bar".to_string(), 1, 1))), }); b.build_stmt(&stmt).unwrap(); diff --git a/src/lib.rs b/src/lib.rs index 97294ec..c7ba447 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -11,7 +11,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -//#![feature(trace_macros,log_syntax)] +// #![feature(trace_macros,log_syntax)] #[macro_use] extern crate nom; @@ -20,6 +20,7 @@ extern crate nom_locate; #[macro_use] pub mod ast; +#[macro_use] pub mod tokenizer; pub mod parse; pub mod build; diff --git a/src/parse.rs b/src/parse.rs index 29b2082..7cecccb 100644 --- a/src/parse.rs +++ b/src/parse.rs @@ -12,41 +12,36 @@ // See the License for the specific language governing permissions and // limitations under the License. use std::str::FromStr; -use std::error::Error; use std::borrow::Borrow; +use nom_locate::LocatedSpan; use nom; -use nom::IResult; use nom::InputLength; +use nom::IResult; use ast::*; use tokenizer::*; -use error as E; -macro_rules! eat_space { - ($i:expr, $($args:tt)*) => ( - { - sep!($i, emptyspace, $($args)*) - } - ) -} +type NomResult<'a, O> = nom::IResult, O, ParseError>; -type ParseResult = Result>; +type ParseResult = Result; -fn symbol_to_value(s: Token) -> ParseResult { - Ok(Value::Symbol(value_node!(s.fragment.to_string(), s.pos))) +fn symbol_to_value(s: &Token) -> ParseResult { + Ok(Value::Symbol(value_node!(s.fragment.to_string(), s.pos.clone()))) } // symbol is a bare unquoted field. -named!(symbol( Span ) -> Value, map_res!(barewordtok, symbol_to_value)); +named!(symbol, + match_type!(BAREWORD => symbol_to_value) +); -fn str_to_value(s: Token) -> ParseResult { - Ok(Value::String(value_node!(s.fragment.to_string(), s.pos))) +fn str_to_value(s: &Token) -> ParseResult { + Ok(Value::String(value_node!(s.fragment.to_string(), s.pos.clone()))) } // quoted_value is a quoted string. -named!(quoted_value( Span ) -> Value, - map_res!(strtok, str_to_value) +named!(quoted_value, + match_type!(STR => str_to_value) ); // Helper function to make the return types work for down below. @@ -59,24 +54,43 @@ fn triple_to_number(v: (Option, Option, Option)) -> ParseRe let has_dot = v.1.is_some(); if v.0.is_some() && !has_dot && v.2.is_none() { - return Ok(Value::Int(value_node!(try!(FromStr::from_str(pref)), pref_pos))); + let i = match FromStr::from_str(pref) { + Ok(i) => i, + Err(_) => { + return Err(ParseError { + description: format!("Not an integer! {}", pref), + pos: pref_pos, + }) + } + }; + return Ok(Value::Int(value_node!(i, pref_pos))); } if v.0.is_none() && has_dot { pref_pos = v.1.unwrap().pos; } - let suf = match v.2 { - None => "", - Some(ref bs) => &bs.fragment, + let (maybepos, suf) = match v.2 { + None => (None, "".to_string()), + Some(bs) => (Some(bs.pos), bs.fragment), }; - let to_parse = pref.to_string() + "." + suf; + let to_parse = pref.to_string() + "." + &suf; // TODO(jwall): if there is an error we should report where that error occured. - let f = try!(FromStr::from_str(&to_parse)); + let f = match FromStr::from_str(&to_parse) { + Ok(f) => f, + Err(_) => { + return Err(ParseError { + description: format!("Not a float! {}", to_parse), + pos: maybepos.unwrap(), + }) + } + }; return Ok(Value::Float(value_node!(f, pref_pos))); } +// trace_macros!(true); + // NOTE(jwall): HERE THERE BE DRAGONS. The order for these matters // alot. We need to process alternatives in order of decreasing // specificity. Unfortunately this means we are required to go in a @@ -87,67 +101,65 @@ fn triple_to_number(v: (Option, Option, Option)) -> ParseRe // *IMPORTANT* // It also means this combinator is risky when used with partial // inputs. So handle with care. -named!(number( Span ) -> Value, +named!(number, map_res!(alt!( complete!(do_parse!( // 1.0 - prefix: digittok >> - has_dot: dottok >> - suffix: digittok >> - peek!(not!(digittok)) >> - (Some(prefix), Some(has_dot), Some(suffix)) + prefix: match_type!(DIGIT) >> + has_dot: punct!(".") >> + suffix: match_type!(DIGIT) >> + (Some(prefix.clone()), Some(has_dot.clone()), Some(suffix.clone())) )) | complete!(do_parse!( // 1. - prefix: digittok >> - has_dot: dottok >> - peek!(not!(digittok)) >> - (Some(prefix), Some(has_dot), None) + prefix: match_type!(DIGIT) >> + has_dot: punct!(".") >> + (Some(prefix.clone()), Some(has_dot.clone()), None) )) | complete!(do_parse!( // .1 - has_dot: dottok >> - suffix: digittok >> - peek!(not!(digittok)) >> - (None, Some(has_dot), Some(suffix)) + has_dot: punct!(".") >> + suffix: match_type!(DIGIT) >> + (None, Some(has_dot.clone()), Some(suffix.clone())) )) | do_parse!( // 1 - prefix: digittok >> + prefix: match_type!(DIGIT) >> // The peek!(not!(..)) make this whole combinator slightly // safer for partial inputs. - peek!(not!(digittok)) >> - (Some(prefix), None, None) + (Some(prefix.clone()), None, None) )), triple_to_number ) ); +// trace_macros!(false); named!( - #[doc="Capture a field and value pair composed of ` = ,`"], - field_value( Span ) -> (Token, Expression), + field_value, do_parse!( - field: barewordtok >> - eat_space!(equaltok) >> + field: match_type!(BAREWORD) >> + punct!("=") >> value: expression >> (field, value) ) ); // Helper function to make the return types work for down below. -fn vec_to_tuple(t: (Span, FieldList)) -> ParseResult { - Ok(Value::Tuple(value_node!(t.1, t.0.line as usize, t.0.offset as usize))) +fn vec_to_tuple(t: (Position, Option)) -> ParseResult { + Ok(Value::Tuple(value_node!(t.1.unwrap_or(Vec::new()), + t.0.line as usize, t.0.column as usize))) } -named!(field_list( Span ) -> FieldList, - separated_list!(commatok, eat_space!(field_value))); +named!(field_list, + separated_list!(punct!(","), field_value) +); named!( #[doc="Capture a tuple of named fields with values. {=,...}"], - tuple( Span ) -> Value, + tuple, map_res!( do_parse!( - pos: position!() >> - v: delimited!(lbracetok, - eat_space!(field_list), - rbracetok) >> - (pos, v) + pos: pos >> + punct!("{") >> + v: field_list >> + punct!("}") >> + (pos, Some(v)) ), vec_to_tuple ) @@ -160,95 +172,97 @@ fn tuple_to_list>(t: (Sp, Vec)) -> ParseResult Value, +named!(list_value, map_res!( do_parse!( - pos: position!() >> - leftsquarebracket >> - elements: eat_space!(separated_list!(eat_space!(commatok), expression)) >> - rightsquarebracket >> - (pos, elements) + start: punct!("[") >> + elements: separated_list!(punct!(","), expression) >> + punct!("]") >> + (start.pos, elements) ), tuple_to_list ) ); -named!(value( Span ) -> Value, +named!(value, alt!( number | quoted_value | list_value | tuple | - selector_value )); + selector_value ) + ); fn value_to_expression(v: Value) -> ParseResult { Ok(Expression::Simple(v)) } -named!(simple_expression( Span ) -> Expression, +named!(simple_expression, map_res!( value, value_to_expression ) ); -fn tuple_to_binary_expression(tpl: (Span, BinaryExprType, Value, Expression)) +fn tuple_to_binary_expression(tpl: (Position, BinaryExprType, Value, Expression)) -> ParseResult { Ok(Expression::Binary(BinaryOpDef { kind: tpl.1, left: tpl.2, right: Box::new(tpl.3), - pos: Position::new(tpl.0.line as usize, tpl.0.offset as usize), + pos: Position::new(tpl.0.line as usize, tpl.0.column as usize), })) } macro_rules! do_binary_expr { - ($i:expr, $fn:expr, $typ:expr) => { + ($i:expr, $subrule:ident!( $($args:tt)* ), $typ:expr) => { // NOTE(jwall): Nom macros do magic with their inputs. They in fact // rewrite your macro argumets for you. Which means we require this $i // paramater even though we don't explicitely pass it below. I don't // particularly like this but I'm living with it for now. - map_res!( - $i, do_parse!( - pos: position!() >> + map_res!($i, + do_parse!( + pos: pos >> left: value >> - eat_space!($fn) >> + $subrule!($($args)*) >> right: expression >> (pos, $typ, left, right) ), tuple_to_binary_expression ) - } + }; } -named!(add_expression( Span ) -> Expression, - do_binary_expr!(plustok, BinaryExprType::Add) +// trace_macros!(true); +named!(add_expression, + do_binary_expr!(punct!("+"), BinaryExprType::Add) +); +// trace_macros!(false); + +named!(sub_expression, + do_binary_expr!(punct!("-"), BinaryExprType::Sub) ); -named!(sub_expression( Span ) -> Expression, - do_binary_expr!(dashtok, BinaryExprType::Sub) +named!(mul_expression, + do_binary_expr!(punct!("*"), BinaryExprType::Mul) ); -named!(mul_expression( Span ) -> Expression, - do_binary_expr!(startok, BinaryExprType::Mul) -); - -named!(div_expression( Span ) -> Expression, - do_binary_expr!(slashtok, BinaryExprType::Div) +named!(div_expression, + do_binary_expr!(punct!("/"), BinaryExprType::Div) ); fn expression_to_grouped_expression(e: Expression) -> ParseResult { Ok(Expression::Grouped(Box::new(e))) } -named!(grouped_expression( Span ) -> Expression, +named!(grouped_expression, map_res!( - preceded!(lparentok, terminated!(expression, rparentok)), + preceded!(punct!("("), terminated!(expression, punct!(")"))), expression_to_grouped_expression ) ); -fn symbol_or_expression(input: Span) -> IResult { +fn symbol_or_expression(input: TokenIter) -> NomResult { let sym = do_parse!(input, sym: symbol >> (sym) @@ -268,11 +282,9 @@ fn symbol_or_expression(input: Span) -> IResult { } } -fn selector_list(input: Span) -> IResult { +fn selector_list(input: TokenIter) -> NomResult { let (rest, head) = match symbol_or_expression(input) { - IResult::Done(rest, val) => { - (rest, val) - } + IResult::Done(rest, val) => (rest, val), IResult::Error(e) => { return IResult::Error(e); } @@ -280,21 +292,17 @@ fn selector_list(input: Span) -> IResult { return IResult::Incomplete(i); } }; - - let (rest, is_dot) = match dottok(rest) { - IResult::Done(rest, _) => { - (rest, true) - } + + let (rest, is_dot) = match punct!(rest, ".") { + IResult::Done(rest, tok) => (rest, Some(tok)), IResult::Incomplete(i) => { return IResult::Incomplete(i); } - IResult::Error(_) => { - (rest, false) - } + IResult::Error(_) => (rest, None), }; - - let (rest, list) = if is_dot { - let (rest, list) = match separated_list!(rest, dottok, alt!(barewordtok | digittok)) { + + let (rest, list) = if is_dot.is_some() { + let (rest, list) = match separated_list!(rest, punct!("."), alt!(match_type!(BAREWORD) | match_type!(DIGIT))) { IResult::Done(rest, val) => { (rest, val) } @@ -307,45 +315,49 @@ fn selector_list(input: Span) -> IResult { }; if list.is_empty() { - return IResult::Error(nom::ErrorKind::Custom(0)); + return IResult::Error(nom::ErrorKind::Custom(ParseError { + description: "(.) with no selector fields after".to_string(), + pos: is_dot.unwrap().pos, + })); } else { (rest, Some(list)) } } else { (rest, None) }; - - let sel_list = SelectorList{ - head: Box::new(head), - tail: list, + + let sel_list = SelectorList { + head: Box::new(head), + tail: list, }; return IResult::Done(rest, sel_list); } -fn tuple_to_copy(t: (Span, SelectorDef, FieldList)) -> ParseResult { +fn tuple_to_copy(t: (SelectorDef, FieldList)) -> ParseResult { + let pos = t.0.pos.clone(); Ok(Expression::Copy(CopyDef { - selector: t.1, - fields: t.2, - pos: Position::new(t.0.line as usize, t.0.offset as usize), + selector: t.0, + fields: t.1, + pos: pos, })) } -named!(copy_expression( Span ) -> Expression, +named!(copy_expression, map_res!( do_parse!( - pos: position!() >> + pos: pos >> selector: selector_list >> - lbracetok >> - fields: eat_space!(field_list) >> - rbracetok >> - (pos, SelectorDef::new(selector, pos.line as usize, pos.offset as usize), fields) + punct!("{") >> + fields: field_list >> + punct!("}") >> + (SelectorDef::new(selector, pos.line, pos.column as usize), fields) ), tuple_to_copy ) ); -fn tuple_to_macro(mut t: (Span, Vec, Value)) -> ParseResult { +fn tuple_to_macro(mut t: (Position, Vec, Value)) -> ParseResult { match t.2 { Value::Tuple(v) => { Ok(Expression::Macro(MacroDef { @@ -359,32 +371,30 @@ fn tuple_to_macro(mut t: (Span, Vec, Value)) -> ParseResult { }) .collect(), fields: v.val, - pos: Position::new(t.0.line as usize, t.0.offset as usize), + pos: t.0, })) } // TODO(jwall): Show a better version of the unexpected parsed value. val => { - Err(Box::new(E::Error::new(format!("Expected Tuple Got {:?}", val), - E::ErrorType::UnexpectedToken, - Position { - line: t.0.line as usize, - column: t.0.offset as usize, - }))) + Err(ParseError { + description: format!("Expected Tuple Got {:?}", val), + pos: t.0, + }) } } } -named!(arglist( Span ) -> Vec, separated_list!(eat_space!(commatok), symbol)); +named!(arglist, ParseError>, separated_list!(punct!(","), symbol)); -named!(macro_expression( Span ) -> Expression, +named!(macro_expression, map_res!( do_parse!( - pos: position!() >> - macrotok >> - eat_space!(lparentok) >> - arglist: eat_space!(arglist) >> - rparentok >> - eat_space!(fatcommatok) >> + pos: pos >> + start: word!("macro") >> + punct!("(") >> + arglist: arglist >> + punct!(")") >> + punct!("=>") >> map: tuple >> (pos, arglist, map) ), @@ -392,36 +402,33 @@ named!(macro_expression( Span ) -> Expression, ) ); -fn tuple_to_select(t: (Span, Expression, Expression, Value)) -> ParseResult { +fn tuple_to_select(t: (Position, Expression, Expression, Value)) -> ParseResult { match t.3 { Value::Tuple(v) => { Ok(Expression::Select(SelectDef { val: Box::new(t.1), default: Box::new(t.2), tuple: v.val, - pos: Position::new(t.0.line as usize, t.0.offset as usize), + pos: t.0, })) } val => { - Err(Box::new(E::Error::new(format!("Expected Tuple Got {:?}", val), - E::ErrorType::UnexpectedToken, - Position { - line: t.0.line as usize, - column: t.0.offset as usize, - }))) + Err(ParseError { + description: format!("Expected Tuple Got {:?}", val), + pos: t.0, + }) } } } -named!(select_expression( Span ) -> Expression, +named!(select_expression, map_res!( do_parse!( - pos: position!() >> - selecttok >> - val: eat_space!(terminated!(expression, commatok)) >> - default: eat_space!(terminated!(expression, commatok)) >> - map: eat_space!(tuple) >> - (pos, val, default, map) + start: word!("select") >> + val: terminated!(expression, punct!(",")) >> + default: terminated!(expression, punct!(",")) >> + map: tuple >> + (start.pos.clone(), val, default, map) ), tuple_to_select ) @@ -435,61 +442,57 @@ fn tuple_to_format(t: (Token, Vec)) -> ParseResult { })) } -named!(format_expression( Span ) -> Expression, +named!(format_expression, map_res!( do_parse!( - tmpl: eat_space!(strtok) >> - eat_space!(pcttok) >> - lparentok >> - args: eat_space!(separated_list!(eat_space!(commatok), expression)) >> - rparentok >> + tmpl: match_type!(STR) >> + punct!("%") >> + punct!("(") >> + args: separated_list!(punct!(","), expression) >> + punct!(")") >> (tmpl, args) ), tuple_to_format ) ); -fn tuple_to_call(t: (Span, Value, Vec)) -> ParseResult { +fn tuple_to_call(t: (Position, Value, Vec)) -> ParseResult { if let Value::Selector(def) = t.1 { Ok(Expression::Call(CallDef { macroref: def, arglist: t.2, - pos: Position::new(t.0.line as usize, t.0.offset as usize), + pos: Position::new(t.0.line as usize, t.0.column as usize), })) } else { - Err(Box::new(E::Error::new(format!("Expected Selector Got {:?}", t.0), - E::ErrorType::UnexpectedToken, - Position { - line: t.0.line as usize, - column: t.0.offset as usize, - }))) + Err(ParseError { + description: format!("Expected Selector Got {:?}", t.0), + pos: Position::new(t.0.line as usize, t.0.column as usize), + }) } } -fn vec_to_selector_value(t: (Span, SelectorList)) -> ParseResult { - Ok(Value::Selector(SelectorDef::new(t.1, t.0.line as usize, t.0.offset as usize))) +fn vec_to_selector_value(t: (Position, SelectorList)) -> ParseResult { + Ok(Value::Selector(SelectorDef::new(t.1, t.0.line as usize, t.0.column as usize))) } -named!(selector_value( Span ) -> Value, +named!(selector_value, map_res!( do_parse!( - pos: position!() >> - sl: eat_space!(selector_list) >> - (pos, sl) + sl: selector_list >> + (sl.head.pos().clone(), sl) ), vec_to_selector_value ) ); -named!(call_expression( Span ) -> Expression, +named!(call_expression, map_res!( do_parse!( - pos: position!() >> macroname: selector_value >> - lparentok >> - args: eat_space!(separated_list!(eat_space!(commatok), expression)) >> - rparentok >> - (pos, macroname, args) + punct!("(") >> + args: separated_list!(punct!(","), expression) >> + punct!(")") >> + (macroname.pos().clone(), macroname, args) ), tuple_to_call ) @@ -505,8 +508,9 @@ named!(call_expression( Span ) -> Expression, // *IMPORTANT* // It also means this combinator is risky when used with partial // inputs. So handle with care. -named!(expression( Span ) -> Expression, - alt!( +named!(expression, + do_parse!( + expr: alt!( complete!(add_expression) | complete!(sub_expression) | complete!(mul_expression) | @@ -517,19 +521,21 @@ named!(expression( Span ) -> Expression, complete!(select_expression) | complete!(call_expression) | complete!(copy_expression) | - eat_space!(simple_expression) - ) + simple_expression + ) >> + (expr) + ) ); fn expression_to_statement(v: Expression) -> ParseResult { Ok(Statement::Expression(v)) } -named!(expression_statement( Span ) -> Statement, - map_res!( - terminated!(eat_space!(expression), semicolontok), - expression_to_statement - ) +named!(expression_statement, + map_res!( + terminated!(expression, punct!(";")), + expression_to_statement + ) ); fn tuple_to_let(t: (Token, Expression)) -> ParseResult { @@ -539,530 +545,429 @@ fn tuple_to_let(t: (Token, Expression)) -> ParseResult { })) } -named!(let_statement( Span ) -> Statement, - map_res!( - terminated!(do_parse!( - lettok >> - name: eat_space!(barewordtok) >> - equaltok >> - val: eat_space!(expression) >> - (name, val) - ), semicolontok), - tuple_to_let - ) +named!(let_statement, + map_res!( + do_parse!( + word!("let") >> + name: match_type!(BAREWORD) >> + punct!("=") >> + val: expression >> + punct!(";") >> + (name, val) + ), + tuple_to_let + ) ); fn tuple_to_import(t: (Token, Token)) -> ParseResult { Ok(Statement::Import(ImportDef { - name: t.0, - path: t.1, + path: t.0, + name: t.1, })) } -named!(import_statement( Span ) -> Statement, - map_res!( - terminated!(do_parse!( - importtok >> - path: eat_space!(strtok) >> - astok >> - name: eat_space!(barewordtok) >> - (name, path) - ), semicolontok), - tuple_to_import - ) +named!(import_statement, + map_res!( + do_parse!( + word!("import") >> + path: match_type!(STR) >> + word!("as") >> + name: match_type!(BAREWORD) >> + punct!(";") >> + (path, name) + ), + tuple_to_import + ) ); -named!(statement( Span ) -> Statement, - alt_complete!( +named!(statement, + do_parse!( + stmt: alt_complete!( import_statement | let_statement | expression_statement - ) + ) >> + (stmt) + ) ); -pub fn parse(input: Span) -> IResult> { - let mut out = Vec::new(); - let mut i = input; - loop { - match eat_space!(i, statement) { - IResult::Error(e) => { - return IResult::Error(e); - } - IResult::Incomplete(i) => { - return IResult::Incomplete(i); - } - IResult::Done(rest, stmt) => { - out.push(stmt); - i = rest; - if i.input_len() == 0 { +pub fn parse(input: LocatedSpan<&str>) -> Result, ParseError> { + match tokenize(input) { + Ok(tokenized) => { + let mut out = Vec::new(); + let mut i_ = TokenIter { source: tokenized.as_slice() }; + loop { + let i = i_.clone(); + if i[0].typ == TokenType::END { break; } + match statement(i) { + IResult::Error(nom::ErrorKind::Custom(e)) => { + return Err(e); + } + IResult::Error(e) => { + return Err(ParseError { + description: + format!("Tokenization error: {:?} current token: {:?}", e, i_[0]), + pos: Position { + line: i_[0].pos.line, + column: i_[0].pos.column, + }, + }); + } + IResult::Incomplete(ei) => { + return Err(ParseError { + description: format!("Unexpected end of parsing input: {:?}", ei), + pos: Position { + line: i_[0].pos.line, + column: i_[0].pos.column, + }, + }); + } + IResult::Done(rest, stmt) => { + out.push(stmt); + i_ = rest; + if i_.input_len() == 0 { + break; + } + } + } + } + return Ok(out); + } + Err(e) => { + // FIXME(jwall): We should really capture the location + // of the tokenization error here. + return Err(ParseError { + description: format!("Tokenize Error: {:?}", e), + pos: Position { + line: 0, + column: 0, + }, + }); + } + } +} + +#[cfg(test)] +mod test { + use super::*; + use tokenizer::{TokenIter, tokenize}; + + use nom_locate::LocatedSpan; + use nom::IResult; + + macro_rules! assert_parse { + ($parsemac:ident( $i:expr ), $out:expr) => { + assert_parse!($i, $parsemac, $out) + }; + ($i:expr, $f:expr, $out:expr) => { + { + let input = LocatedSpan::new($i); + match tokenize(input) { + Err(e) => assert!(false, format!("Tokenizer Error: {:?}", e)), + Ok(val) => { + match $f(TokenIter{source: val.as_slice()}) { + IResult::Done(_, result) => assert_eq!(result, $out), + other => assert!(false, format!("Expected Done got {:?}", other)), + } + }, + } + }; + } + } + + macro_rules! assert_incomplete { + ($parsemac:ident( $i:expr )) => { + assert_incomplete!($i, $parsemac) + }; + ($i:expr, $f:expr) => { + { + let input = LocatedSpan::new($i); + match tokenize(input) { + Err(_) => assert!(false), + Ok(val) => { + let result = $f(TokenIter{source: val.as_slice()}); + assert!(result.is_incomplete(), format!("Not Incomplete: {:?}", result)); + }, + } } } } - return IResult::Done(i, out); -} -// named!(pub parse( Span ) -> Vec, many1!()); - -#[cfg(test)] -mod test { - use super::{Statement, Expression, Value, MacroDef, SelectDef, CallDef}; - use super::{number, symbol, parse, field_value, selector_value, tuple, - grouped_expression, list_value}; - use super::{copy_expression, macro_expression, select_expression}; - use super::{format_expression, call_expression, expression}; - use super::{expression_statement, let_statement, import_statement, statement}; - use ast::*; - use nom_locate::LocatedSpan; - - use nom::{Needed, IResult}; + macro_rules! assert_error { + ($parsemac:ident( $i:expr )) => { + assert_error!($i, $parsemac) + }; + ($i:expr, $f:expr) => { + { + let input = LocatedSpan::new($i); + match tokenize(input) { + Err(_) => assert!(true), + Ok(val) => { + let result = $f(TokenIter{source: val.as_slice()}); + assert!(result.is_err(), format!("Not an error: {:?}", result)) + }, + } + } + } + } #[test] fn test_symbol_parsing() { - assert_eq!(symbol(LocatedSpan::new("foo")), - IResult::Done(LocatedSpan{fragment: "", offset: 3, line: 1}, - Value::Symbol(value_node!("foo".to_string(), 1, 1))) ); - assert_eq!(symbol(LocatedSpan::new("foo-bar")), - IResult::Done(LocatedSpan{fragment: "", offset: 7, line: 1}, - Value::Symbol(value_node!("foo-bar".to_string(), 1, 1))) ); - assert_eq!(symbol(LocatedSpan::new("foo_bar")), - IResult::Done(LocatedSpan{fragment: "", offset: 7, line: 1}, - Value::Symbol(value_node!("foo_bar".to_string(), 1, 1))) ); + assert_parse!(symbol("foo"), + Value::Symbol(value_node!("foo".to_string(), 1, 1)) ); + assert_parse!(symbol("foo-bar"), + Value::Symbol(value_node!("foo-bar".to_string(), 1, 1)) ); + assert_parse!(symbol("foo_bar"), + Value::Symbol(value_node!("foo_bar".to_string(), 1, 1)) ); } #[test] fn test_selector_parsing() { - assert_eq!(selector_value(LocatedSpan::new("foo.")), - IResult::Incomplete(Needed::Unknown) - ); - assert_eq!(selector_value(LocatedSpan::new("foo.bar ")), - IResult::Done(LocatedSpan{fragment: "", offset: 8, line: 1}, + assert_error!(selector_value("foo.")); + assert_parse!(selector_value("foo.bar "), Value::Selector(make_selector!(make_expr!("foo".to_string(), 1, 1) => [ - Token::new("bar", 1, 5)] => - 1, 0))) + make_tok!("bar", 1, 5)] => + 1, 1)) ); - assert_eq!(selector_value(LocatedSpan::new("foo.0 ")), - IResult::Done(LocatedSpan{fragment: "", offset: 6, line: 1}, + assert_parse!(selector_value("foo.0 "), Value::Selector(make_selector!(make_expr!("foo".to_string(), 1, 1) => [ - Token::new("0", 1, 5)] => - 1, 0))) + make_tok!(DIGIT => "0", 1, 5)] => + 1, 1)) ); - assert_eq!(selector_value(LocatedSpan::new("foo.bar;")), - IResult::Done(LocatedSpan{fragment: ";", offset: 7, line: 1}, + assert_parse!(selector_value("foo.bar;"), Value::Selector(make_selector!(make_expr!("foo", 1, 1) => [ - Token{fragment:"bar".to_string(), pos: Position::new(1, 5)} + make_tok!("bar", 1, 5) ] => - 1, 0))) + 1, 1)) ); - - assert_eq!(selector_value(LocatedSpan::new("({foo=1}).foo ")), - IResult::Done(LocatedSpan{fragment: "", offset: 14, line: 1}, + assert_parse!(selector_value("({foo=1}).foo "), Value::Selector(make_selector!(Expression::Grouped(Box::new(Expression::Simple( Value::Tuple(value_node!( vec![(make_tok!("foo", 1, 3), Expression::Simple(Value::Int(Positioned::new(1, 1, 7))))], 1, 3)) - ))) => [ make_tok!("foo", 1, 11) ] => 1, 0) - ))); + ))) => [ make_tok!("foo", 1, 11) ] => 1, 2) + )); } #[test] fn test_statement_parse() { - let mut stmt = "import \"foo\" as foo;"; - let input = LocatedSpan::new(stmt); - assert_eq!(statement(input), - IResult::Done( - LocatedSpan{ - offset: stmt.len(), - line: 1, - fragment: "", - }, + let stmt = "import \"foo\" as foo;"; + assert_parse!(statement(stmt), Statement::Import(ImportDef{ - path: Token{ - fragment: "foo".to_string(), - pos: Position::new(1,8) - }, - name: Token{ - fragment: "foo".to_string(), - pos: Position::new(1,17), - } - }) + path: make_tok!(QUOT => "foo", 1,8), + name: make_tok!("foo", 1,17), + } ) ); - assert!(statement(LocatedSpan::new("import foo")).is_err() ); + assert_error!(import_statement("import \"foo\"") ); - stmt = "let foo = 1.0 ;"; - let input = LocatedSpan::new(stmt); - assert_eq!(statement(input), - IResult::Done( - LocatedSpan{ - offset: stmt.len(), - line: 1, - fragment: "", - }, + assert_parse!(statement("let foo = 1.0 ;"), Statement::Let(LetDef{ - name: Token{ - fragment: "foo".to_string(), - pos: Position::new(1,5), - }, - value: Expression::Simple(Value::Float(value_node!(1.0, 1, 11))) - }))); - stmt = "1.0;"; - let input = LocatedSpan::new(stmt); - assert_eq!(statement(input), - IResult::Done( - LocatedSpan{ - offset: stmt.len(), - line: 1, - fragment: "", - }, - Statement::Expression( - Expression::Simple(Value::Float(value_node!(1.0, 1, 1)))))); + name: make_tok!("foo", 1, 5), + value: Expression::Simple(Value::Float(value_node!(1.0, 1, 11))) + })); + + assert_parse!(statement("// comment\nlet foo = 1.0 ;"), + Statement::Let(LetDef{ + name: make_tok!("foo", 2, 5), + value: Expression::Simple(Value::Float(value_node!(1.0, 2, 11))) + })); + + assert_parse!(statement("1.0;"), + Statement::Expression( + Expression::Simple(Value::Float(value_node!(1.0, 1, 1)))) ); } #[test] - fn test_import_parse() { - assert!(import_statement(LocatedSpan::new("import")).is_incomplete()); - assert!(import_statement(LocatedSpan::new("import \"foo\"")).is_incomplete()); - assert!(import_statement(LocatedSpan::new("import \"foo\" as")).is_incomplete()); - assert!(import_statement(LocatedSpan::new("import \"foo\" as foo")).is_incomplete()); + fn test_import_statement_parse() { + assert_error!(import_statement("import")); + assert_error!(import_statement("import \"foo\"")); + assert_error!(import_statement("import \"foo\" as")); + assert_error!(import_statement("import \"foo\" as foo")); let import_stmt = "import \"foo\" as foo;"; - assert_eq!(import_statement(LocatedSpan::new(import_stmt)), - IResult::Done(LocatedSpan{ - fragment: "", - line: 1, - offset: import_stmt.len(), - }, + assert_parse!(import_statement(import_stmt), Statement::Import(ImportDef{ - path: Token{ - fragment: "foo".to_string(), - pos: Position::new(1, 8), - }, - name: Token{ - fragment: "foo".to_string(), - pos: Position::new(1,17), - } + path: make_tok!(QUOT => "foo", 1, 8), + name: make_tok!("foo", 1,17), }) - ) - ); + ); } #[test] fn test_let_statement_parse() { - assert!(let_statement(LocatedSpan::new("foo")).is_err() ); - assert!(let_statement(LocatedSpan::new("let \"foo\"")).is_err() ); - assert!(let_statement(LocatedSpan::new("let 1")).is_err() ); - assert!(let_statement(LocatedSpan::new("let")).is_incomplete() ); - assert!(let_statement(LocatedSpan::new("let foo")).is_incomplete() ); - assert!(let_statement(LocatedSpan::new("let foo =")).is_incomplete() ); - assert!(let_statement(LocatedSpan::new("let foo = ")).is_incomplete() ); - assert!(let_statement(LocatedSpan::new("let foo = 1")).is_incomplete() ); + assert_error!(let_statement("foo") ); + assert_error!(let_statement("let \"foo\"") ); + assert_error!(let_statement("let 1") ); + assert_error!(let_statement("let") ); + assert_error!(let_statement("let foo") ); + assert_error!(let_statement("let foo =") ); + assert_error!(let_statement("let foo = ") ); + assert_error!(let_statement("let foo = 1") ); - let mut let_stmt = "let foo = 1.0 ;"; - assert_eq!(let_statement(LocatedSpan::new(let_stmt)), - IResult::Done(LocatedSpan{ - fragment: "", - offset: let_stmt.len(), - line: 1, - }, - Statement::Let(LetDef{name: Token{ - fragment: "foo".to_string(), - pos: Position::new(1,5), - }, + assert_parse!(let_statement("let foo = 1.0 ;"), + Statement::Let(LetDef{name: make_tok!("foo", 1,5), value: Expression::Simple(Value::Float(value_node!(1.0, 1, 11))) - }))); + })); - let_stmt = "let foo= 1.0;"; - assert_eq!(let_statement(LocatedSpan::new(let_stmt)), - IResult::Done(LocatedSpan{ - fragment: "", - offset: let_stmt.len(), - line: 1, - }, - Statement::Let(LetDef{name: Token{ - fragment: "foo".to_string(), - pos: Position::new(1,5), - }, - value: Expression::Simple(Value::Float(value_node!(1.0, 1, 10)))}))); - let_stmt = "let foo =1.0;"; - assert_eq!(let_statement(LocatedSpan::new(let_stmt)), - IResult::Done(LocatedSpan{ - fragment: "", - offset: let_stmt.len(), - line: 1, - }, - Statement::Let(LetDef{name: Token{ - fragment: "foo".to_string(), - pos: Position::new(1,5), - }, - value: Expression::Simple(Value::Float(value_node!(1.0, 1, 10)))}))); + assert_parse!(let_statement("let foo = // comment\n1.0 ;"), + Statement::Let(LetDef{name: make_tok!("foo", 1,5), + value: Expression::Simple(Value::Float(value_node!(1.0, 2, 1))) + })); + + assert_parse!(let_statement("let foo = 1.0 // comment\n;"), + Statement::Let(LetDef{name: make_tok!("foo", 1,5), + value: Expression::Simple(Value::Float(value_node!(1.0, 1, 11))) + })); + + assert_parse!(let_statement("let foo= 1.0;"), + Statement::Let(LetDef{name: make_tok!("foo", 1, 5), + value: Expression::Simple(Value::Float(value_node!(1.0, 1, 10)))}) ); + + assert_parse!(let_statement("let foo =1.0;"), + Statement::Let(LetDef{name: make_tok!("foo", 1,5), + value: Expression::Simple(Value::Float(value_node!(1.0, 1, 10)))})); } #[test] fn test_expression_statement_parse() { - assert!(expression_statement(LocatedSpan::new("foo")).is_incomplete() ); - assert_eq!(expression_statement(LocatedSpan::new("1.0;")), - IResult::Done(LocatedSpan{ - fragment: "", - offset: 4, - line: 1, - }, + assert_error!(expression_statement("foo") ); + assert_parse!(expression_statement("1.0;"), Statement::Expression( - Expression::Simple(Value::Float(value_node!(1.0, 1, 1)))))); - assert_eq!(expression_statement(LocatedSpan::new("1.0 ;")), - IResult::Done(LocatedSpan { - fragment: "", - offset: 5, - line: 1, - }, + Expression::Simple(Value::Float(value_node!(1.0, 1, 1))))); + assert_parse!(expression_statement("1.0 ;"), Statement::Expression( - Expression::Simple(Value::Float(value_node!(1.0, 1, 1)))))); - assert_eq!(expression_statement(LocatedSpan::new(" 1.0;")), - IResult::Done(LocatedSpan { - fragment: "", - offset: 5, - line: 1, - }, + Expression::Simple(Value::Float(value_node!(1.0, 1, 1))))); + assert_parse!(expression_statement(" 1.0;"), Statement::Expression( - Expression::Simple(Value::Float(value_node!(1.0, 1, 2)))))); - assert_eq!(expression_statement(LocatedSpan::new("foo;")), - IResult::Done(LocatedSpan { - fragment: "", - offset: 4, - line: 1, - }, + Expression::Simple(Value::Float(value_node!(1.0, 1, 2))))); + assert_parse!(expression_statement("foo;"), Statement::Expression( - Expression::Simple(Value::Selector(make_selector!(make_expr!("foo", 1, 1), 1, 0)))))); - assert_eq!(expression_statement(LocatedSpan::new("foo ;")), - IResult::Done(LocatedSpan { - fragment: "", - offset: 5, - line: 1, - }, + Expression::Simple(Value::Selector(make_selector!(make_expr!("foo", 1, 1), 1, 1))))); + assert_parse!(expression_statement("foo ;"), Statement::Expression( - Expression::Simple(Value::Selector(make_selector!(make_expr!("foo", 1, 2), 1, 0)))))); - assert_eq!(expression_statement(LocatedSpan::new(" foo;")), - IResult::Done(LocatedSpan { - fragment: "", - offset: 5, - line: 1, - }, + Expression::Simple(Value::Selector(make_selector!(make_expr!("foo", 1, 2), 1, 1))))); + assert_parse!(expression_statement(" foo;"), Statement::Expression( - Expression::Simple(Value::Selector(make_selector!(make_expr!("foo", 1, 2), 1, 1)))))); - assert_eq!(expression_statement(LocatedSpan::new("\"foo\";")), - IResult::Done(LocatedSpan { - fragment: "", - offset: 6, - line: 1, - }, + Expression::Simple(Value::Selector(make_selector!(make_expr!("foo", 1, 2), 1, 2))))); + assert_parse!(expression_statement("\"foo\";"), Statement::Expression( - Expression::Simple(Value::String(value_node!("foo".to_string(), 1, 1)))))); - assert_eq!(expression_statement(LocatedSpan::new("\"foo\" ;")), - IResult::Done(LocatedSpan { - fragment: "", - offset: 7, - line: 1, - }, + Expression::Simple(Value::String(value_node!("foo".to_string(), 1, 1))))); + assert_parse!(expression_statement("\"foo\" ;"), Statement::Expression( - Expression::Simple(Value::String(value_node!("foo".to_string(), 1, 1)))))); - assert_eq!(expression_statement(LocatedSpan::new(" \"foo\";")), - IResult::Done(LocatedSpan { - fragment: "", - offset: 7, - line: 1, - }, + Expression::Simple(Value::String(value_node!("foo".to_string(), 1, 1))))); + assert_parse!(expression_statement(" \"foo\";"), Statement::Expression( - Expression::Simple(Value::String(value_node!("foo".to_string(), 1, 2)))))); + Expression::Simple(Value::String(value_node!("foo".to_string(), 1, 2))))); } #[test] fn test_expression_parse() { - assert_eq!(expression(LocatedSpan::new("1")), - IResult::Done(LocatedSpan { - fragment: "", - offset: 1, - line: 1, - }, - Expression::Simple(Value::Int(value_node!(1, 1, 1))))); - assert_eq!(expression(LocatedSpan::new("foo ")), - IResult::Done(LocatedSpan { - fragment: "", - offset: 4, - line: 1, - }, - Expression::Simple(Value::Selector(make_selector!(make_expr!("foo", 1, 1), 1, 0))))); - assert_eq!(expression(LocatedSpan::new("foo.bar ")), - IResult::Done(LocatedSpan { - fragment: "", - offset: 8, - line: 1, - }, + assert_parse!(expression("\"foo\""), + Expression::Simple(Value::String(value_node!("foo".to_string(), 1, 1)))); + assert_parse!(expression("1"), + Expression::Simple(Value::Int(value_node!(1, 1, 1)))); + assert_parse!(expression("foo "), + Expression::Simple(Value::Selector(make_selector!(make_expr!("foo", 1, 1), 1, 1)))); + assert_parse!(expression("foo.bar "), Expression::Simple(Value::Selector(make_selector!(make_expr!("foo", 1, 1) => - [ Token::new("bar", 1, 5) ] => - 1, 0))))); - assert_eq!(expression(LocatedSpan::new("1 + 1")), - IResult::Done(LocatedSpan { - fragment: "", - offset: 5, - line: 1, - }, + [ make_tok!("bar", 1, 5) ] => + 1, 1)))); + assert_parse!(expression("1 + 1"), Expression::Binary(BinaryOpDef{ kind: BinaryExprType::Add, left: Value::Int(value_node!(1, 1, 1)), right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 5)))), - pos: Position::new( 1, 0 ), - }))); - assert_eq!(expression(LocatedSpan::new("1 - 1")), - IResult::Done(LocatedSpan { - fragment: "", - offset: 5, - line: 1, - }, + pos: Position::new( 1, 1 ), + })); + assert_parse!(expression("1 - 1"), Expression::Binary(BinaryOpDef{ kind: BinaryExprType::Sub, left: Value::Int(value_node!(1, 1, 1)), right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 5)))), - pos: Position::new(1, 0), - }))); - assert_eq!(expression(LocatedSpan::new("1 * 1")), - IResult::Done(LocatedSpan { - fragment: "", - offset: 5, - line: 1, - }, + pos: Position::new(1, 1), + })); + assert_parse!(expression("1 * 1"), Expression::Binary(BinaryOpDef{ kind: BinaryExprType::Mul, left: Value::Int(value_node!(1, 1, 1)), right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 5)))), - pos: Position::new(1, 0), - }))); - assert_eq!(expression(LocatedSpan::new("1 / 1")), - IResult::Done(LocatedSpan { - fragment: "", - offset: 5, - line: 1, - }, + pos: Position::new(1, 1), + })); + assert_parse!(expression("1 / 1"), Expression::Binary(BinaryOpDef{ kind: BinaryExprType::Div, left: Value::Int(value_node!(1, 1, 1)), right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 5)))), - pos: Position::new(1, 0), - }))); + pos: Position::new(1, 1), + })); - assert_eq!(expression(LocatedSpan::new("1+1")), - IResult::Done(LocatedSpan { - fragment: "", - offset: 3, - line: 1, - }, + assert_parse!(expression("1+1"), Expression::Binary(BinaryOpDef{ kind: BinaryExprType::Add, left: Value::Int(value_node!(1, 1, 1)), right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 3)))), - pos: Position::new(1, 0), - }))); - assert_eq!(expression(LocatedSpan::new("1-1")), - IResult::Done(LocatedSpan { - fragment: "", - offset: 3, - line: 1, - }, + pos: Position::new(1, 1), + })); + assert_parse!(expression("1-1"), Expression::Binary(BinaryOpDef{ kind: BinaryExprType::Sub, left: Value::Int(value_node!(1, 1, 1)), right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 3)))), - pos: Position::new(1, 0), - }))); - assert_eq!(expression(LocatedSpan::new("1*1")), - IResult::Done(LocatedSpan { - fragment: "", - offset: 3, - line: 1, - }, + pos: Position::new(1, 1), + })); + assert_parse!(expression("1*1"), Expression::Binary(BinaryOpDef{ kind: BinaryExprType::Mul, left: Value::Int(value_node!(1, 1, 1)), right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 3)))), - pos: Position::new(1, 0), - }))); - assert_eq!(expression(LocatedSpan::new("1/1")), - IResult::Done(LocatedSpan { - fragment: "", - offset: 3, - line: 1, - }, + pos: Position::new(1, 1), + })); + assert_parse!(expression("1/1"), Expression::Binary(BinaryOpDef{ kind: BinaryExprType::Div, left: Value::Int(value_node!(1, 1, 1)), right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 3)))), - pos: Position::new(1, 0), - }))); - let macro_expr = "macro (arg1, arg2) => { foo = arg1 }"; - assert_eq!(expression(LocatedSpan::new(macro_expr)), - IResult::Done(LocatedSpan { - fragment: "", - offset: macro_expr.len(), - line: 1, - }, + pos: Position::new(1, 1), + })); + assert_parse!(expression("macro (arg1, arg2) => { foo = arg1 }"), Expression::Macro(MacroDef{ argdefs: vec![ value_node!("arg1".to_string(), 1, 8), value_node!("arg2".to_string(), 1, 14), ], fields: vec![ - (Token::new("foo", 1, 25), - Expression::Simple(Value::Selector(make_selector!(make_expr!("arg1", 1, 31), 1, 30)))), + (make_tok!("foo", 1, 25), + Expression::Simple(Value::Selector(make_selector!(make_expr!("arg1", 1, 31), 1, 31)))), ], - pos: Position::new(1, 0), - }) - ) - ); - let select_expr = "select foo, 1, { foo = 2 }"; - assert_eq!(expression(LocatedSpan::new(select_expr)), - IResult::Done(LocatedSpan { - fragment: "", - offset: select_expr.len(), - line: 1, - }, + pos: Position::new(1, 1), + })); + assert_parse!(expression("select foo, 1, { foo = 2 }"), Expression::Select(SelectDef{ - val: Box::new(Expression::Simple(Value::Selector(make_selector!(make_expr!("foo", 1, 8), 1, 7)))), + val: Box::new(Expression::Simple(Value::Selector(make_selector!(make_expr!("foo", 1, 8), 1, 8)))), default: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 13)))), tuple: vec![ - (Token::new("foo", 1, 18), + (make_tok!("foo", 1, 18), Expression::Simple(Value::Int(value_node!(2, 1, 24)))) ], - pos: Position::new(1, 0), - }) - ) - ); - let call_expr = "foo.bar (1, \"foo\")"; - assert_eq!(expression(LocatedSpan::new(call_expr)), - IResult::Done(LocatedSpan { - fragment: "", - offset: call_expr.len(), - line: 1, - }, + pos: Position::new(1, 1), + })); + assert_parse!(expression("foo.bar (1, \"foo\")"), Expression::Call(CallDef{ macroref: make_selector!(make_expr!("foo", 1, 1) => - [ Token::new("bar", 1, 5) ] => - 1, 0), + [ make_tok!("bar", 1, 5) ] => + 1, 1), arglist: vec![ Expression::Simple(Value::Int(value_node!(1, 1, 10))), Expression::Simple(Value::String(value_node!("foo".to_string(), 1, 13))), ], - pos: Position::new(1, 0), - }) - ) - ); - assert_eq!(expression(LocatedSpan::new("(1 + 1)")), - IResult::Done(LocatedSpan { - fragment: "", - offset: 7, - line: 1, - }, + pos: Position::new(1, 1), + })); + assert_parse!(expression("(1 + 1)"), Expression::Grouped( Box::new( Expression::Binary( @@ -1070,15 +975,9 @@ mod test { kind: BinaryExprType::Add, left: Value::Int(value_node!(1, 1, 2)), right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 6)))), - pos: Position::new(1, 1), - } - ) - ) - ) - ) - ); - assert_eq!(expression(LocatedSpan::new("[1, 1]")), - IResult::Done(LocatedSpan{fragment: "", offset: 6, line: 1}, + pos: Position::new(1, 2), // FIXME(jwall): grouped expressions appear to be getting positioned wrong + })))); + assert_parse!(expression("[1, 1]"), Expression::Simple(Value::List( ListDef{ elems: vec![ @@ -1089,249 +988,221 @@ mod test { } ) ) - )); + ); } #[test] fn test_format_parse() { - assert!(format_expression(LocatedSpan::new("\"foo")).is_err() ); - assert!(format_expression(LocatedSpan::new("\"foo\"")).is_incomplete() ); - assert!(format_expression(LocatedSpan::new("\"foo\" %")).is_incomplete() ); - assert!(format_expression(LocatedSpan::new("\"foo\" % (1, 2")).is_incomplete() ); + assert_error!(format_expression("\"foo") ); + assert_error!(format_expression("\"foo\"") ); + assert_error!(format_expression("\"foo\" %") ); + assert_error!(format_expression("\"foo\" % (, 2") ); - let mut fmt_expr = "\"foo @ @\" % (1, 2)"; - assert_eq!(format_expression(LocatedSpan::new(fmt_expr)), - IResult::Done(LocatedSpan{ - fragment: "", - offset: fmt_expr.len(), - line: 1 - }, + assert_parse!(format_expression("\"foo @ @\" % (1, 2)"), Expression::Format( FormatDef{ template: "foo @ @".to_string(), args: vec![Expression::Simple(Value::Int(value_node!(1, 1, 14))), Expression::Simple(Value::Int(value_node!(2, 1, 17)))], pos: Position::new(1, 1), - } - ) - ) - ); + })); - fmt_expr = "\"foo @ @\"%(1, 2)"; - assert_eq!(format_expression(LocatedSpan::new(fmt_expr)), - IResult::Done(LocatedSpan{ - fragment: "", - offset: fmt_expr.len(), - line: 1, - }, + assert_parse!(format_expression("\"foo @ @\"%(1, 2)"), Expression::Format( FormatDef{ template: "foo @ @".to_string(), args: vec![Expression::Simple(Value::Int(value_node!(1, 1, 12))), Expression::Simple(Value::Int(value_node!(2, 1, 15)))], pos: Position::new(1, 1), - } - ) - ) - ); + })); } #[test] fn test_call_parse() { - assert!(call_expression(LocatedSpan::new("foo")).is_incomplete() ); - assert!(call_expression(LocatedSpan::new("foo (")).is_incomplete() ); - assert!(call_expression(LocatedSpan::new("foo (1")).is_incomplete() ); - assert!(call_expression(LocatedSpan::new("foo (1,")).is_incomplete() ); - assert!(call_expression(LocatedSpan::new("foo (1,2")).is_incomplete() ); + assert_error!(call_expression("foo") ); + assert_error!(call_expression("foo (") ); + assert_error!(call_expression("foo (1") ); + assert_error!(call_expression("foo (1,") ); + assert_error!(call_expression("foo (1,2") ); - let mut copy_expr = "foo (1, \"foo\")"; - assert_eq!(call_expression(LocatedSpan::new(copy_expr)), - IResult::Done( - LocatedSpan{ - fragment: "", - line: 1, - offset: copy_expr.len(), - }, + assert_parse!(call_expression("foo (1, \"foo\")"), Expression::Call(CallDef{ - macroref: make_selector!(make_expr!("foo")), + macroref: make_selector!(make_expr!("foo", 1, 1), 1, 1), arglist: vec![ Expression::Simple(Value::Int(value_node!(1, 1, 6))), Expression::Simple(Value::String(value_node!("foo".to_string(), 1, 9))), ], - pos: Position::new(1, 0), + pos: Position::new(1, 1), }) - ) + ); - copy_expr = "foo.bar (1, \"foo\")"; - assert_eq!(call_expression(LocatedSpan::new(copy_expr)), - IResult::Done( - LocatedSpan{ - fragment: "", - line: 1, - offset: copy_expr.len(), - }, + assert_parse!(call_expression("foo.bar (1, \"foo\")"), Expression::Call(CallDef{ - macroref: make_selector!(make_expr!("foo") => [ make_tok!("bar", 1, 5) ] => 1, 0), + macroref: make_selector!(make_expr!("foo") => [ make_tok!("bar", 1, 5) ] => 1, 1), arglist: vec![ Expression::Simple(Value::Int(value_node!(1, 1, 10))), Expression::Simple(Value::String(value_node!("foo".to_string(), 1, 13))), ], - pos: Position::new(1, 0), + pos: Position::new(1, 1), }) - ) + ); } #[test] fn test_select_parse() { - assert!(select_expression(LocatedSpan::new("select")).is_incomplete()); - assert!(select_expression(LocatedSpan::new("select foo")).is_incomplete()); - assert!(select_expression(LocatedSpan::new("select foo, 1")).is_incomplete()); - assert!(select_expression(LocatedSpan::new("select foo, 1, {")).is_incomplete()); + assert_error!(select_expression("select")); + assert_error!(select_expression("select foo")); + assert_error!(select_expression("select foo, 1")); + assert_error!(select_expression("select foo, 1, {")); - let select_expr = "select foo, 1, { foo = 2 }"; - assert_eq!(select_expression(LocatedSpan::new(select_expr)), - IResult::Done(LocatedSpan { - fragment: "", - offset: select_expr.len(), - line: 1, - }, + assert_parse!(select_expression("select foo, 1, { foo = 2 }"), Expression::Select(SelectDef{ - val: Box::new(Expression::Simple(Value::Selector(make_selector!(make_expr!("foo", 1, 8), 1, 7)))), + val: Box::new(Expression::Simple(Value::Selector(make_selector!(make_expr!("foo", 1, 8), 1, 8)))), default: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 13)))), tuple: vec![ - (Token::new("foo", 1, 18), Expression::Simple(Value::Int(value_node!(2, 1, 24)))) + (make_tok!("foo", 1, 18), Expression::Simple(Value::Int(value_node!(2, 1, 24)))) ], - pos: Position::new(1, 0), - }) - ) - ); + pos: Position::new(1, 1), + })); } #[test] fn test_macro_expression_parsing() { - assert!(macro_expression(LocatedSpan::new("foo")).is_err() ); - assert!(macro_expression(LocatedSpan::new("macro \"foo\"")).is_err() ); - assert!(macro_expression(LocatedSpan::new("macro 1")).is_err() ); - assert!(macro_expression(LocatedSpan::new("macro")).is_incomplete() ); - assert!(macro_expression(LocatedSpan::new("macro (")).is_incomplete() ); - assert!(macro_expression(LocatedSpan::new("macro (arg")).is_incomplete() ); - assert!(macro_expression(LocatedSpan::new("macro (arg, arg2")).is_incomplete() ); - assert!(macro_expression(LocatedSpan::new("macro (arg1, arg2) =>")).is_incomplete() ); - assert!(macro_expression(LocatedSpan::new("macro (arg1, arg2) => {")).is_incomplete() ); - assert!(macro_expression(LocatedSpan::new("macro (arg1, arg2) => { foo")).is_incomplete() ); - assert!(macro_expression(LocatedSpan::new("macro (arg1, arg2) => { foo =")).is_incomplete() ); + assert_error!(macro_expression("foo") ); + assert_error!(macro_expression("macro \"foo\"") ); + assert_error!(macro_expression("macro 1") ); + assert_error!(macro_expression("macro") ); + assert_error!(macro_expression("macro (") ); + assert_error!(macro_expression("macro (arg") ); + assert_error!(macro_expression("macro (arg, arg2") ); + assert_error!(macro_expression("macro (arg1, arg2) =>") ); + assert_error!(macro_expression("macro (arg1, arg2) => {") ); + assert_error!(macro_expression("macro (arg1, arg2) => { foo") ); + assert_error!(macro_expression("macro (arg1, arg2) => { foo =") ); - let macro_expr = "macro (arg1, arg2) => {foo=1,bar=2}"; - assert_eq!(macro_expression(LocatedSpan::new(macro_expr)), - IResult::Done( - LocatedSpan{ - fragment: "", - offset: macro_expr.len(), - line: 1 - }, + assert_parse!(macro_expression("macro (arg1, arg2) => {foo=1,bar=2}"), Expression::Macro(MacroDef{ argdefs: vec![value_node!("arg1".to_string(), 1, 8), value_node!("arg2".to_string(), 1, 14)], - fields: vec![(Token::new("foo", 1, 24), Expression::Simple(Value::Int(value_node!(1, 1, 28)))), - (Token::new("bar", 1, 30), Expression::Simple(Value::Int(value_node!(2, 1, 34)))) + fields: vec![(make_tok!("foo", 1, 24), Expression::Simple(Value::Int(value_node!(1, 1, 28)))), + (make_tok!("bar", 1, 30), Expression::Simple(Value::Int(value_node!(2, 1, 34)))) ], - pos: Position::new(1, 0), - }) - ) - ); + pos: Position::new(1, 1), + })); } #[test] fn test_copy_parse() { - assert!(copy_expression(LocatedSpan::new("{}")).is_err() ); - assert!(copy_expression(LocatedSpan::new("foo")).is_incomplete() ); - assert!(copy_expression(LocatedSpan::new("foo{")).is_incomplete() ); + assert_error!(copy_expression("{}") ); + assert_error!(copy_expression("foo") ); + assert_error!(copy_expression("foo{") ); - let mut copy_expr = "foo{}"; - assert_eq!(copy_expression(LocatedSpan::new(copy_expr)), - IResult::Done( - LocatedSpan{ - fragment: "", - offset: copy_expr.len(), - line: 1 - }, + assert_parse!(copy_expression("foo{}"), Expression::Copy(CopyDef{ - selector: make_selector!(make_expr!("foo")), + selector: make_selector!(make_expr!("foo", 1, 1), 1, 1), fields: Vec::new(), - pos: Position::new(1, 0), - }) - ) - ); + pos: Position::new(1, 1), + })); - copy_expr = "foo{bar=1}"; - assert_eq!(copy_expression(LocatedSpan::new(copy_expr)), - IResult::Done( - LocatedSpan{ - fragment: "", - offset: copy_expr.len(), - line: 1 - }, + assert_parse!(copy_expression("foo{bar=1}"), Expression::Copy(CopyDef{ - selector: make_selector!(make_expr!("foo")), - fields: vec![(Token::new("bar", 1, 5), + selector: make_selector!(make_expr!("foo", 1, 1), 1, 1), + fields: vec![(make_tok!("bar", 1, 5), Expression::Simple(Value::Int(value_node!(1, 1, 9))))], - pos: Position::new(1, 0), - }) - ) - ); + pos: Position::new(1, 1), + })); } #[test] fn test_grouped_expression_parse() { - assert!(grouped_expression(LocatedSpan::new("foo")).is_err() ); - assert!(grouped_expression(LocatedSpan::new("(foo")).is_incomplete() ); - assert_eq!(grouped_expression(LocatedSpan::new("(foo)")), - IResult::Done(LocatedSpan{fragment: "", offset: 5, line: 1}, + assert_error!(grouped_expression("foo") ); + assert_error!(grouped_expression("(foo") ); + assert_parse!(grouped_expression("(foo)"), Expression::Grouped( Box::new( Expression::Simple( - Value::Selector(make_selector!(make_expr!("foo", 1, 2), 1, 1)))))) - ); - assert_eq!(grouped_expression(LocatedSpan::new("(1 + 1)")), - IResult::Done(LocatedSpan{fragment: "", offset: 7, line: 1}, + Value::Selector(make_selector!(make_expr!("foo", 1, 2), 1, 2)))))); + assert_parse!(grouped_expression("(1 + 1)"), Expression::Grouped( Box::new( Expression::Binary( - BinaryOpDef{ - kind: BinaryExprType::Add, - left: Value::Int(value_node!(1, 1, 2)), - right: Box::new(Expression::Simple( - Value::Int(value_node!(1, 1, 6)))), - pos: Position::new(1, 1), - } - ) - ) - ) - ) - ); + BinaryOpDef{ + kind: BinaryExprType::Add, + left: Value::Int(value_node!(1, 1, 2)), + right: Box::new(Expression::Simple( + Value::Int(value_node!(1, 1, 6)))), + pos: Position::new(1, 2), + })))); } #[test] fn test_list_value_parse() { - assert!(list_value(LocatedSpan::new("foo")).is_err() ); - assert!(list_value(LocatedSpan::new("[foo")).is_incomplete() ); - assert_eq!(list_value(LocatedSpan::new("[foo]")), - IResult::Done(LocatedSpan{fragment: "", offset: 5, line: 1}, + assert_error!(list_value("foo") ); + assert_error!(list_value("[foo") ); + assert_error!(list_value("// commen\n[foo") ); + + assert_parse!(list_value("[foo]"), Value::List( ListDef{ elems: vec![ - Expression::Simple(Value::Selector(make_selector!(make_expr!("foo", 1, 2), 1, 1))) + Expression::Simple(Value::Selector(make_selector!(make_expr!("foo", 1, 2), 1, 2))) ], pos: Position::new(1, 1), } ) - ) ); - assert_eq!(list_value(LocatedSpan::new("[1, 1]")), - IResult::Done(LocatedSpan{fragment: "", offset: 6, line: 1}, + assert_parse!(list_value("[1, 1]"), + Value::List( + ListDef{ + elems: vec![ + Expression::Simple(Value::Int(value_node!(1, 1, 2))), + Expression::Simple(Value::Int(value_node!(1, 1, 5))), + ], + pos: Position::new(1, 1), + } + ) + ); + + assert_parse!(list_value("// comment\n[1, 1]"), + Value::List( + ListDef{ + elems: vec![ + Expression::Simple(Value::Int(value_node!(1, 2, 2))), + Expression::Simple(Value::Int(value_node!(1, 2, 5))), + ], + pos: Position::new(2, 1), + } + ) + ); + + assert_parse!(list_value("[// comment\n1, 1]"), + Value::List( + ListDef{ + elems: vec![ + Expression::Simple(Value::Int(value_node!(1, 2, 2))), + Expression::Simple(Value::Int(value_node!(1, 2, 5))), + ], + pos: Position::new(1, 1), + } + ) + ); + + assert_parse!(list_value("[1, // comment\n1]"), + Value::List( + ListDef{ + elems: vec![ + Expression::Simple(Value::Int(value_node!(1, 1, 2))), + Expression::Simple(Value::Int(value_node!(1, 2, 1))), + ], + pos: Position::new(1, 1), + } + ) + ); + + assert_parse!(list_value("[1, 1 // comment\n]"), Value::List( ListDef{ elems: vec![ @@ -1341,111 +1212,140 @@ mod test { pos: Position::new(1, 1), } ) - ) ); } #[test] fn test_tuple_parse() { - assert!(tuple(LocatedSpan::new("{")).is_incomplete() ); - assert!(tuple(LocatedSpan::new("{ foo")).is_incomplete() ); - assert!(tuple(LocatedSpan::new("{ foo =")).is_incomplete() ); - assert!(tuple(LocatedSpan::new("{ foo = 1")).is_incomplete() ); - assert!(tuple(LocatedSpan::new("{ foo = 1,")).is_incomplete() ); - assert!(tuple(LocatedSpan::new("{ foo = 1, bar =")).is_incomplete() ); + assert_error!(tuple("{") ); + assert_error!(tuple("{ foo") ); + assert_error!(tuple("{ foo =") ); + assert_error!(tuple("{ foo = 1") ); + assert_error!(tuple("{ foo = 1,") ); + assert_error!(tuple("{ foo = 1, bar =") ); + assert_error!(tuple("// comment\n{ foo = 1, bar =") ); - let mut tuple_expr = "{ }"; - assert_eq!(tuple(LocatedSpan::new(tuple_expr)), - IResult::Done(LocatedSpan { - fragment: "", - offset: tuple_expr.len(), - line: 1, - }, - Value::Tuple( - value_node!(vec![], 1, 0)))); + assert_parse!(tuple("{ }"), Value::Tuple(value_node!(vec![], 1, 1))); - tuple_expr = "{ foo = 1 }"; - assert_eq!(tuple(LocatedSpan::new(tuple_expr)), - IResult::Done(LocatedSpan { - fragment: "", - offset: tuple_expr.len(), - line: 1, - }, + assert_parse!(tuple("{ foo = 1 }"), + Value::Tuple( + value_node!(vec![ + (make_tok!("foo", 1, 3), + Expression::Simple(Value::Int(value_node!(1, 1, 9)))) + ], 1, 1))); + + assert_parse!(tuple("// comment\n{ foo = 1 }"), + Value::Tuple( + value_node!(vec![ + (make_tok!("foo", 2, 3), + Expression::Simple(Value::Int(value_node!(1, 2, 9)))) + ], 1, 1))); + + assert_parse!(tuple("{// comment\n foo = 1 }"), Value::Tuple( value_node!(vec![ - (Token::new("foo", 1, 3), + (make_tok!("foo", 2, 2), + Expression::Simple(Value::Int(value_node!(1, 2, 8)))) + ], 1, 1))); + + assert_parse!(tuple("{ foo = 1// comment\n }"), + Value::Tuple( + value_node!(vec![ + (make_tok!("foo", 1, 3), Expression::Simple(Value::Int(value_node!(1, 1, 9)))) - ], 1, 0)))); + ], 1, 1))); - tuple_expr = "{ foo = 1, bar = \"1\" }"; - assert_eq!(tuple(LocatedSpan::new(tuple_expr)), - IResult::Done(LocatedSpan { - fragment: "", - offset: tuple_expr.len(), - line: 1, - }, + assert_parse!(tuple("{ foo = 1, bar = \"1\" }"), Value::Tuple( value_node!(vec![ - (Token::new("foo", 1, 3), + (make_tok!("foo", 1, 3), Expression::Simple(Value::Int(value_node!(1, 1, 9)))), - (Token::new("bar", 1, 12), + (make_tok!("bar", 1, 12), Expression::Simple(Value::String(value_node!("1".to_string(), Position::new(1, 18))))) - ], 1, 0)))); - tuple_expr = "{ foo = 1, bar = {} }"; - assert_eq!(tuple(LocatedSpan::new(tuple_expr)), - IResult::Done(LocatedSpan { - fragment: "", - offset: tuple_expr.len(), - line: 1, - }, + ], 1, 1))); + assert_parse!(tuple("{ foo = 1, // comment\nbar = \"1\" }"), Value::Tuple( value_node!(vec![ - (Token::new("foo", 1, 3), + (make_tok!("foo", 1, 3), + Expression::Simple(Value::Int(value_node!(1, 1, 9)))), + (make_tok!("bar", 2, 1), + Expression::Simple(Value::String(value_node!("1".to_string(), Position::new(2, 7))))) + ], 1, 1))); + assert_parse!(tuple("{ foo = 1, bar = {} }"), + Value::Tuple( + value_node!(vec![ + (make_tok!("foo", 1, 3), Expression::Simple(Value::Int(value_node!(1, Position::new(1, 9))))), - (Token::new("bar", 1, 12), + (make_tok!("bar", 1, 12), Expression::Simple(Value::Tuple(value_node!(Vec::new(), Position::new(1, 17))))) - ], 1, 0)))); + ], 1, 1))); + } + + #[test] + fn test_field_list_parse() { + let mut f_list = "foo = 1, quux = 2;"; + assert_parse!(field_list(f_list), + vec![ + (make_tok!("foo", 1, 1), make_expr!(1 => int, 1, 7)), + (make_tok!("quux", 1, 10), make_expr!(2 => int, 1, 17)), + ]); + + f_list = "foo = 1, // comment\nquux = 2;"; + assert_parse!(field_list(f_list), + vec![ + (make_tok!("foo", 1, 1), make_expr!(1 => int, 1, 7)), + (make_tok!("quux", 2, 1), make_expr!(2 => int, 2, 8)), + ]); + + f_list = "foo = 1,\n// comment\nquux = 2;"; + assert_parse!(field_list(f_list), + vec![ + (make_tok!("foo", 1, 1), make_expr!(1 => int, 1, 7)), + (make_tok!("quux", 3, 1), make_expr!(2 => int, 3, 8)), + ]); } #[test] fn test_field_value_parse() { - assert!(field_value(LocatedSpan::new("foo")).is_incomplete() ); - assert!(field_value(LocatedSpan::new("foo =")).is_incomplete() ); + assert_error!(field_value("foo") ); + assert_error!(field_value("// comment\nfoo") ); + assert_error!(field_value("foo =") ); - assert_eq!(field_value(LocatedSpan::new("foo = 1")), - IResult::Done(LocatedSpan { offset: 7, line: 1, fragment: "" }, - (Token::new("foo", 1, 1), - Expression::Simple(Value::Int(value_node!(1, 1, 7))))) ); - assert_eq!(field_value(LocatedSpan::new("foo = \"1\"")), - IResult::Done(LocatedSpan { offset: 9, line: 1, fragment: "" }, - (Token::new("foo", 1, 1), - Expression::Simple(Value::String(value_node!("1".to_string(), 1, 7))))) ); - assert_eq!(field_value(LocatedSpan::new("foo = bar ")), - IResult::Done(LocatedSpan { offset: 10, line: 1, fragment: "" }, - (Token::new("foo", 1, 1), - Expression::Simple(Value::Selector(make_selector!(make_expr!("bar", 1, 7), 1, 6))))) ); - assert_eq!(field_value(LocatedSpan::new("foo = bar.baz ")), - IResult::Done(LocatedSpan { offset: 14, line: 1, fragment: "" }, - (Token::new("foo", 1, 1), - Expression::Simple(Value::Selector(make_selector!(make_expr!("bar", 1, 7) => [ make_tok!("baz", 1, 11) ] => 1, 6)))))); + assert_parse!(field_value("foo = 1"), + (make_tok!("foo", 1, 1), + Expression::Simple(Value::Int(value_node!(1, 1, 7)))) ); + assert_parse!(field_value("foo = 1 // foo comment\n"), + (make_tok!("foo", 1, 1), + Expression::Simple(Value::Int(value_node!(1, 1, 7)))) ); + assert_parse!(field_value("foo // foo comment\n = 1"), + (make_tok!("foo", 1, 1), + Expression::Simple(Value::Int(value_node!(1, 2, 4)))) ); + assert_parse!(field_value("// foo comment\nfoo = 1"), + (make_tok!("foo", 2, 1), + Expression::Simple(Value::Int(value_node!(1, 2, 7)))) ); + assert_parse!(field_value("foo = \"1\""), + (make_tok!("foo", 1, 1), + Expression::Simple(Value::String(value_node!("1".to_string(), 1, 7)))) ); + assert_parse!(field_value("foo = bar "), + (make_tok!("foo", 1, 1), + Expression::Simple(Value::Selector(make_selector!(make_expr!("bar", 1, 7), 1, 7)))) ); + assert_parse!(field_value("foo = bar.baz "), + (make_tok!("foo", 1, 1), + Expression::Simple(Value::Selector(make_selector!(make_expr!("bar", 1, 7) => [ make_tok!("baz", 1, 11) ] => 1, 7)))) ); } #[test] fn test_number_parsing() { - assert!(number(LocatedSpan::new(".")).is_err() ); - assert!(number(LocatedSpan::new(". ")).is_err() ); - assert_eq!(number(LocatedSpan::new("1.0")), - IResult::Done(LocatedSpan{fragment: "", offset: 3, line: 1}, - Value::Float(value_node!(1.0, 1, 1))) ); - assert_eq!(number(LocatedSpan::new("1.")), - IResult::Done(LocatedSpan{fragment: "", offset: 2, line: 1}, - Value::Float(value_node!(1.0, 1, 1))) ); - assert_eq!(number(LocatedSpan::new("1")), - IResult::Done(LocatedSpan{fragment: "", offset: 1, line: 1}, - Value::Int(value_node!(1, 1, 1))) ); - assert_eq!(number(LocatedSpan::new(".1")), - IResult::Done(LocatedSpan{fragment: "", offset: 2, line: 1}, - Value::Float(value_node!(0.1, 1, 1))) ); + assert_error!(number(".") ); + assert_error!(number(". ") ); + assert_parse!(number("1.0"), + Value::Float(value_node!(1.0, 1, 1)) ); + assert_parse!(number("1."), + Value::Float(value_node!(1.0, 1, 1)) ); + assert_parse!(number("1"), + Value::Int(value_node!(1, 1, 1)) ); + assert_parse!(number(".1"), + Value::Float(value_node!(0.1, 1, 1)) ); } #[test] @@ -1457,26 +1357,16 @@ mod test { // Valid parsing tree let input = LocatedSpan::new("import \"mylib\" as lib;let foo = 1;1+1;"); let result = parse(input); - assert!(result.is_done() ); + assert!(result.is_ok(), format!("Expected Ok, Got {:?}", result)); let tpl = result.unwrap(); - assert_eq!(tpl.0.fragment, ""); - assert_eq!(tpl.1, + assert_eq!(tpl, vec![ Statement::Import(ImportDef{ - path: Token{ - fragment: "mylib".to_string(), - pos: Position::new(1, 8), - }, - name: Token{ - fragment: "lib".to_string(), - pos: Position::new(1, 19), - } + path: make_tok!(QUOT => "mylib", 1, 8), + name: make_tok!("lib", 1, 19), }), Statement::Let(LetDef{ - name: Token{ - fragment: "foo".to_string(), - pos: Position::new(1, 27), - }, + name: make_tok!("foo", 1, 27), value: Expression::Simple(Value::Int(value_node!(1, 1, 33))) }), Statement::Expression( @@ -1485,7 +1375,7 @@ mod test { kind: BinaryExprType::Add, left: Value::Int(value_node!(1, 1, 35)), right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 37)))), - pos: Position::new(1, 34), + pos: Position::new(1, 35), }) ) ]); diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 7fe93f3..65b78eb 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -13,9 +13,11 @@ // limitations under the License. use nom_locate::LocatedSpan; use nom; -use nom::{alpha, is_alphanumeric, digit, InputLength, sp}; - +use nom::{alpha, is_alphanumeric, digit, multispace}; +use nom::{InputLength, Slice}; use ast::*; +use std; +use std::result::Result; pub type Span<'a> = LocatedSpan<&'a str>; @@ -32,35 +34,38 @@ fn is_symbol_char(c: char) -> bool { is_alphanumeric(c as u8) || c == '-' as char || c == '_' as char } -named!(pub strtok( Span ) -> Token, +named!(strtok( Span ) -> Token, do_parse!( span: position!() >> tag!("\"") >> frag: take_until!("\"") >> tag!("\"") >> (Token{ + typ: TokenType::QUOTED, pos: Position::from(span), fragment: frag.fragment.to_string(), }) ) ); -named!(pub barewordtok( Span ) -> Token, +named!(barewordtok( Span ) -> Token, do_parse!( span: position!() >> frag: preceded!(peek!(alpha), take_while!(is_symbol_char)) >> (Token{ + typ: TokenType::BAREWORD, pos: Position::from(span), fragment: frag.fragment.to_string(), }) ) ); -named!(pub digittok( Span ) -> Token, +named!(digittok( Span ) -> Token, do_parse!( span: position!() >> digits: digit >> (Token{ + typ: TokenType::DIGIT, pos: Position::from(span), fragment: digits.fragment.to_string(), }) @@ -74,11 +79,12 @@ macro_rules! do_tag_tok { // rewrite your macro argumets for you. Which means we require this $i // paramater even though we don't explicitely pass it below. I don't // particularly like this but I'm living with it for now. - ($i:expr, $tag:expr) => { + ($i:expr, $type:expr, $tag:expr) => { do_parse!($i, span: position!() >> frag: tag!($tag) >> (Token{ + typ: $type, pos: Position::from(span), fragment: frag.fragment.to_string(), }) @@ -86,126 +92,549 @@ macro_rules! do_tag_tok { } } -named!(pub commatok( Span ) -> Token, - do_tag_tok!(",") +named!(commatok( Span ) -> Token, + do_tag_tok!(TokenType::PUNCT, ",") ); -named!(pub lbracetok( Span ) -> Token, - do_tag_tok!("{") +named!(lbracetok( Span ) -> Token, + do_tag_tok!(TokenType::PUNCT, "{") ); -named!(pub rbracetok( Span ) -> Token, - do_tag_tok!("}") +named!(rbracetok( Span ) -> Token, + do_tag_tok!(TokenType::PUNCT, "}") ); -named!(pub lparentok( Span ) -> Token, - do_tag_tok!("(") +named!(lparentok( Span ) -> Token, + do_tag_tok!(TokenType::PUNCT, "(") ); -named!(pub rparentok( Span ) -> Token, - do_tag_tok!(")") +named!(rparentok( Span ) -> Token, + do_tag_tok!(TokenType::PUNCT, ")") ); -named!(pub dottok( Span ) -> Token, - do_tag_tok!(".") +named!(dottok( Span ) -> Token, + do_tag_tok!(TokenType::PUNCT, ".") ); -named!(pub plustok( Span ) -> Token, - do_tag_tok!("+") +named!(plustok( Span ) -> Token, + do_tag_tok!(TokenType::PUNCT, "+") ); -named!(pub dashtok( Span ) -> Token, - do_tag_tok!("-") +named!(dashtok( Span ) -> Token, + do_tag_tok!(TokenType::PUNCT, "-") ); -named!(pub startok( Span ) -> Token, - do_tag_tok!("*") +named!(startok( Span ) -> Token, + do_tag_tok!(TokenType::PUNCT, "*") ); -named!(pub slashtok( Span ) -> Token, - do_tag_tok!("/") +named!(slashtok( Span ) -> Token, + do_tag_tok!(TokenType::PUNCT, "/") ); -named!(pub pcttok( Span ) -> Token, - do_tag_tok!("%") +named!(pcttok( Span ) -> Token, + do_tag_tok!(TokenType::PUNCT, "%") ); -named!(pub equaltok( Span ) -> Token, - do_tag_tok!("=") +named!(equaltok( Span ) -> Token, + do_tag_tok!(TokenType::PUNCT, "=") ); -named!(pub semicolontok( Span ) -> Token, - do_tag_tok!(";") +named!(semicolontok( Span ) -> Token, + do_tag_tok!(TokenType::PUNCT, ";") ); -named!(pub leftsquarebracket( Span ) -> Token, - do_tag_tok!("[") +named!(leftsquarebracket( Span ) -> Token, + do_tag_tok!(TokenType::PUNCT, "[") ); -named!(pub rightsquarebracket( Span ) -> Token, - do_tag_tok!("]") +named!(rightsquarebracket( Span ) -> Token, + do_tag_tok!(TokenType::PUNCT, "]") ); -named!(pub commentprefix( Span ) -> Token, - do_tag_tok!("//") +named!(fatcommatok( Span ) -> Token, + do_tag_tok!(TokenType::PUNCT, "=>") ); -named!(pub fatcommatok( Span ) -> Token, - do_tag_tok!("=>") +named!(lettok( Span ) -> Token, + do_tag_tok!(TokenType::BAREWORD, "let") ); -named!(pub lettok( Span ) -> Token, - do_tag_tok!("let") +named!(selecttok( Span ) -> Token, + do_tag_tok!(TokenType::BAREWORD, "select") ); -named!(pub selecttok( Span ) -> Token, - do_tag_tok!("select") +named!(macrotok( Span ) -> Token, + do_tag_tok!(TokenType::BAREWORD, "macro") ); -named!(pub macrotok( Span ) -> Token, - do_tag_tok!("macro") +named!(importtok( Span ) -> Token, + do_tag_tok!(TokenType::BAREWORD, "import") ); -named!(pub importtok( Span ) -> Token, - do_tag_tok!("import") +named!(astok( Span ) -> Token, + do_tag_tok!(TokenType::BAREWORD, "as") ); -named!(pub astok( Span ) -> Token, - do_tag_tok!("as") -); - -pub fn end_of_input(input: Span) -> nom::IResult { - if input.input_len() == 0 { - return nom::IResult::Done(input, input); - } else { - return nom::IResult::Incomplete(nom::Needed::Unknown); - } -} - -fn comment(input: Span) -> nom::IResult { - match commentprefix(input) { - nom::IResult::Done(rest, _) => { - match alt!(rest, take_until!("\r\n") | take_until!("\n")) { - nom::IResult::Done(rest, cmt) => nom::IResult::Done(rest, cmt), - nom::IResult::Incomplete(i) => nom::IResult::Incomplete(i), - nom::IResult::Error(e) => { - if let nom::ErrorKind::Eof = e { - return nom::IResult::Done(input, input) - } else { - return nom::IResult::Error(e) - } - } - } +fn end_of_input(input: Span) -> nom::IResult { + match eof!(input,) { + nom::IResult::Done(_, _) => { + return nom::IResult::Done(input, + make_tok!(EOF => input.line as usize, + input.get_column() as usize)); } - nom::IResult::Incomplete(i) => { - return nom::IResult::Incomplete(i) + nom::IResult::Incomplete(_) => { + return nom::IResult::Incomplete(nom::Needed::Unknown); } nom::IResult::Error(e) => { - return nom::IResult::Error(e) + return nom::IResult::Error(e); } } } -named!(pub emptyspace( Span ) -> Span, - alt!(sp | comment) -); \ No newline at end of file +fn comment(input: Span) -> nom::IResult { + match tag!(input, "//") { + nom::IResult::Done(rest, _) => { + match alt!(rest, take_until_and_consume!("\r\n") | take_until_and_consume!("\n")) { + nom::IResult::Done(rest, cmt) => { + return nom::IResult::Done(rest, + make_tok!(CMT => cmt.fragment.to_string(), + input.line as usize, + input.get_column() as usize)); + } + // If we didn't find a new line then we just grab everything. + _ => { + let blen = rest.input_len(); + let next = rest.slice(blen..); + let tok = rest.slice(..blen); + return nom::IResult::Done(next, + make_tok!(CMT => tok.fragment.to_string(), + input.line as usize, input.get_column() as usize + )); + } + } + } + nom::IResult::Incomplete(i) => return nom::IResult::Incomplete(i), + nom::IResult::Error(e) => return nom::IResult::Error(e), + } +} + +named!(whitespace( Span ) -> Token, + do_parse!( + span: position!() >> + many1!(multispace) >> + (Token{ + typ: TokenType::WS, + pos: Position::from(span), + fragment: String::new(), + }) + ) +); + +named!(token( Span ) -> Token, + alt!( + strtok | + barewordtok | + digittok | + commatok | + rbracetok | + lbracetok | + lparentok | + rparentok | + dottok | + plustok | + dashtok | + startok | + comment | // Note comment must come before slashtok + slashtok | + pcttok | + fatcommatok | // Note fatcommatok must come before equaltok + equaltok | + semicolontok | + leftsquarebracket | + rightsquarebracket | + lettok | + selecttok | + macrotok | + importtok | + astok | + whitespace | + end_of_input) +); + +// TODO(jwall): This should return a ParseError instead. +pub fn tokenize(input: Span) -> Result, nom::ErrorKind> { + let mut out = Vec::new(); + let mut i = input; + loop { + if i.input_len() == 0 { + break; + } + match token(i) { + nom::IResult::Error(e) => { + return Err(e); + } + nom::IResult::Incomplete(_) => { + return Err(nom::ErrorKind::Complete); + } + nom::IResult::Done(rest, tok) => { + i = rest; + if tok.typ == TokenType::COMMENT || tok.typ == TokenType::WS { + // we skip comments and whitespace + continue; + } + out.push(tok); + } + } + } + // ensure that we always have an END token to go off of. + out.push(Token { + fragment: String::new(), + typ: TokenType::END, + pos: Position { + line: i.line as usize, + column: i.get_column() as usize, + }, + }); + Ok(out) +} + +pub fn token_clone(t: &Token) -> Result { + Ok(t.clone()) +} + +macro_rules! match_type { + ($i:expr, COMMENT => $h:expr) => { + match_type!($i, TokenType::COMMENT, "Not a Comment", $h) + }; + + ($i:expr, COMMENT) => { + match_type!($i, COMMENT => token_clone) + }; + + ($i:expr, BAREWORD => $h:expr) => { + match_type!($i, TokenType::BAREWORD, "Not a Bareword", $h) + }; + + ($i:expr, BAREWORD) => { + match_type!($i, BAREWORD => token_clone) + }; + + ($i:expr, STR => $h:expr) => { + match_type!($i, TokenType::QUOTED, "Not a String", $h) + }; + + ($i:expr, STR) => { + match_type!($i, STR => token_clone) + }; + + ($i:expr, DIGIT => $h:expr) => { + match_type!($i, TokenType::DIGIT, "Not a DIGIT", $h) + }; + + ($i:expr, DIGIT) => { + match_type!($i, DIGIT => token_clone) + }; + + ($i:expr, PUNCT => $h:expr) => { + match_type!($i, TokenType::PUNCT, "Not PUNCTUATION", $h) + }; + + ($i:expr, PUNCT) => { + match_type!($i, PUNCT => token_clone) + }; + + ($i:expr, $t:expr, $msg:expr, $h:expr) => { + { + let i_ = $i.clone(); + use nom::Slice; + use std::convert::Into; + if i_.input_len() == 0 { + nom::IResult::Error( + nom::ErrorKind::Custom(ParseError{ + description: format!("End of Input! {}", $msg), + pos: Position{line: 0, column: 0} + })) + } else { + let tok = &(i_[0]); + if tok.typ == $t { + match $h(tok) { + Result::Ok(v) => nom::IResult::Done($i.slice(1..), v), + Result::Err(e) => nom::IResult::Error( + nom::ErrorKind::Custom(e.into())), + } + } else { + nom::IResult::Error(nom::ErrorKind::Custom(ParseError{ + description: $msg.to_string(), + pos: tok.pos.clone()})) + } + } + } + }; +} + +macro_rules! match_token { + ($i:expr, PUNCT => $f:expr) => { + match_token!($i, PUNCT => $f, token_clone) + }; + + ($i:expr, PUNCT => $f:expr, $h:expr) => { + match_token!($i, TokenType::PUNCT, $f, format!("Not PUNCT ({})", $f), $h) + }; + + ($i:expr, BAREWORD => $f:expr) => { + match_token!($i, BAREWORD => $f, token_clone) + }; + + ($i:expr, BAREWORD => $f:expr, $h:expr) => { + match_token!($i, TokenType::BAREWORD, $f, format!("Not a BAREWORD ({})", $f), $h) + }; + + ($i:expr, $t:expr, $f:expr, $msg:expr, $h:expr) => { + { + let i_ = $i.clone(); + use nom::Slice; + use std::convert::Into; + let tok = &(i_[0]); + if tok.typ == $t && &tok.fragment == $f { + match $h(tok) { + Result::Ok(v) => nom::IResult::Done($i.slice(1..), v), + Result::Err(e) => nom::IResult::Error( + nom::ErrorKind::Custom(e.into())), + } + } else { + nom::IResult::Error(nom::ErrorKind::Custom(ParseError{ + description: format!("{} Instead is ({})", $msg, tok.fragment), + pos: tok.pos.clone()})) + } + } + }; +} + +macro_rules! punct { + ($i:expr, $c:expr) => { + match_token!($i, PUNCT => $c) + }; +} + +macro_rules! word { + ($i:expr, $w:expr) => { + match_token!($i, BAREWORD => $w) + }; +} + +pub fn pos(i: TokenIter) -> nom::IResult { + let tok = &i[0]; + let line = tok.pos.line; + let column = tok.pos.column; + nom::IResult::Done(i.clone(), + Position { + line: line, + column: column, + }) +} + +#[derive(Clone, Debug, PartialEq)] +pub struct TokenIter<'a> { + pub source: &'a [Token], +} + +impl<'a> TokenIter<'a> { + pub fn len(&self) -> usize { + self.source.len() + } +} + +impl<'a> nom::InputLength for TokenIter<'a> { + fn input_len(&self) -> usize { + self.source.input_len() + } +} + +macro_rules! impl_token_iter_slice { + ($r:ty) => { + impl<'a> nom::Slice<$r> for TokenIter<'a> { + fn slice(&self, range: $r) -> Self { + TokenIter { + source: self.source.slice(range), + } + } + } + } +} + +impl_token_iter_slice!(std::ops::Range); +impl_token_iter_slice!(std::ops::RangeTo); +impl_token_iter_slice!(std::ops::RangeFrom); +impl_token_iter_slice!(std::ops::RangeFull); + +impl<'a> std::ops::Index for TokenIter<'a> { + type Output = Token; + + fn index(&self, i: usize) -> &Self::Output { + &self.source[i] + } +} + +impl<'a> nom::InputIter for TokenIter<'a> { + type Item = &'a Token; + type RawItem = Token; + + type Iter = std::iter::Enumerate>; + type IterElem = std::slice::Iter<'a, Self::RawItem>; + + fn iter_indices(&self) -> Self::Iter { + self.source.iter().enumerate() + } + + fn iter_elements(&self) -> Self::IterElem { + self.source.iter() + } + + fn position

(&self, predicate: P) -> Option + where P: Fn(Self::RawItem) -> bool + { + for (o, v) in self.iter_indices() { + if predicate(v.clone()) { + return Some(o); + } + } + None + } + + fn slice_index(&self, count: usize) -> Option { + let mut cnt = 0; + for (index, _) in self.iter_indices() { + if cnt == count { + return Some(index); + } + cnt += 1; + } + if cnt == count { + return Some(self.len()); + } + None + } +} + +#[cfg(test)] +mod tokenizer_test { + use super::*; + use nom; + use nom_locate::LocatedSpan; + + #[test] + fn test_tokenize_one_of_each() { + // 1 1 1 1 1 1 1 1 1 1 2 2 2 2 2 2 2 2 + // 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 + let result = tokenize(LocatedSpan::new("let import macro select as => [ ] { } ; = % / * \ + + - . ( ) , 1 . foo \"bar\" // comment\n ;")); + assert!(result.is_ok(), format!("result {:?} is not ok", result)); + let v = result.unwrap(); + for (i, t) in v.iter().enumerate() { + println!("{}: {:?}", i, t); + } + assert_eq!(v.len(), 27); + assert_eq!(v[26].typ, TokenType::END); + } + + #[test] + fn test_parse_has_end() { + let result = tokenize(LocatedSpan::new("foo")); + assert!(result.is_ok()); + let v = result.unwrap(); + assert_eq!(v.len(), 2); + assert_eq!(v[1].typ, TokenType::END); + } + + #[test] + fn test_parse_comment() { + assert!(comment(LocatedSpan::new("// comment\n")).is_done()); + assert!(comment(LocatedSpan::new("// comment")).is_done()); + assert_eq!(comment(LocatedSpan::new("// comment\n")), + nom::IResult::Done(LocatedSpan{fragment: "", offset: 11, line: 2}, + Token{ + typ: TokenType::COMMENT, + fragment: " comment".to_string(), + pos: Position{line: 1, column: 1}, + })); + assert!(comment(LocatedSpan::new("// comment\r\n")).is_done()); + assert_eq!(comment(LocatedSpan::new("// comment\r\n")), + nom::IResult::Done(LocatedSpan{fragment: "", offset: 12, line: 2}, + Token{ + typ: TokenType::COMMENT, + fragment: " comment".to_string(), + pos: Position{column: 1, line: 1} + })); + assert!(comment(LocatedSpan::new("// comment\r\n ")).is_done()); + assert_eq!(comment(LocatedSpan::new("// comment\r\n ")), + nom::IResult::Done(LocatedSpan{fragment: " ", offset: 12, line: 2}, + Token{ + typ: TokenType::COMMENT, + fragment: " comment".to_string(), + pos: Position{column: 1, line: 1}, + })); + // TODO(jwall): assert!(comment(LocatedSpan::new("// comment")).is_done()); + } + + #[test] + fn test_match_word() { + let input = vec![Token{ + fragment: "foo".to_string(), + typ: TokenType::BAREWORD, + pos: Position{line: 1, column: 1} + }]; + let result = word!(TokenIter{source: input.as_slice()}, "foo"); + match result { + nom::IResult::Done(_, tok) => assert_eq!(tok, input[0]), + res => assert!(false, format!("Fail: {:?}", res)), + } + } + + #[test] + fn test_match_word_empty_input() { + let input = vec![Token{ + fragment: "".to_string(), + typ: TokenType::END, + pos: Position{line: 1, column: 1}, + }]; + let result = word!(TokenIter{source: input.as_slice()}, "foo"); + match result { + nom::IResult::Done(_, _) => assert!(false, "Should have been an error but was Done"), + nom::IResult::Incomplete(_) => { + assert!(false, "Should have been an error but was Incomplete") + } + nom::IResult::Error(_) => { + // noop + } + } + } + + #[test] + fn test_match_punct() { + let input = vec![Token{ + fragment: "!".to_string(), + typ: TokenType::PUNCT, + pos: Position{line: 1, column: 1} + }]; + let result = punct!(TokenIter{source: input.as_slice()}, "!"); + match result { + nom::IResult::Done(_, tok) => assert_eq!(tok, input[0]), + res => assert!(false, format!("Fail: {:?}", res)), + } + } + + #[test] + fn test_match_type() { + let input = vec![Token{ + fragment: "foo".to_string(), + typ: TokenType::BAREWORD, + pos: Position{line: 1, column: 1} + }]; + let result = match_type!(TokenIter{source: input.as_slice()}, BAREWORD); + match result { + nom::IResult::Done(_, tok) => assert_eq!(tok, input[0]), + res => assert!(false, format!("Fail: {:?}", res)), + } + } +}