From 1e3d19755c8508fc1a9fca7bca2d3f2a9ca0fa40 Mon Sep 17 00:00:00 2001 From: Jeremy Wall Date: Sun, 5 Nov 2017 15:26:52 -0600 Subject: [PATCH] Add location information for all tokens. Also add optional position information for some of the AST elements. --- src/ast.rs | 157 +++++-- src/build.rs | 655 ++++++++++++++------------ src/format.rs | 13 +- src/parse.rs | 1147 ++++++++++++++++++++++++++++------------------ src/tokenizer.rs | 20 +- 5 files changed, 1210 insertions(+), 782 deletions(-) diff --git a/src/ast.rs b/src/ast.rs index b07eb19..d23ac6c 100644 --- a/src/ast.rs +++ b/src/ast.rs @@ -15,12 +15,41 @@ use std::collections::HashSet; use std::borrow::Borrow; use std::convert::Into; -#[derive(Debug,PartialEq,Clone)] +#[derive(Debug,PartialEq,Eq,Clone,PartialOrd,Ord,Hash)] pub struct Position { pub line: usize, pub column: usize, } +#[derive(Debug,PartialEq,Eq,Clone,PartialOrd,Ord,Hash)] +pub struct Token { + pub fragment: String, + pub pos: Position, +} + +impl Token { + pub fn new(f: &str) -> Self { + Self::new_with_pos(f, + Position { + line: 0, + column: 0, + }) + } + + pub fn new_with_pos(f: &str, pos: Position) -> Self { + Token { + fragment: f.to_string(), + pos: pos, + } + } +} + +impl Borrow for Token { + fn borrow(&self) -> &str { + &self.fragment + } +} + macro_rules! value_node { ($v:expr) => { LocatedNode::new($v) @@ -30,8 +59,8 @@ macro_rules! value_node { }; } -pub type FieldList = Vec<(String, Expression)>; // str is expected to be a symbol -pub type SelectorList = Vec; // str is expected to always be a symbol. +pub type FieldList = Vec<(Token, Expression)>; // str is expected to be a symbol +pub type SelectorList = Vec; // str is expected to always be a symbol. #[derive(Debug,PartialEq,Clone)] pub struct LocatedNode { @@ -53,6 +82,10 @@ impl LocatedNode { val: v, } } + + pub fn val(&self) -> &T { + return &self.val; + } } @@ -90,7 +123,7 @@ impl Value { buf.push_str("{\n"); for ref t in v.iter() { buf.push_str("\t"); - buf.push_str(&t.0); + buf.push_str(&t.0.fragment); buf.push_str("\n"); } buf.push_str("}"); @@ -104,7 +137,18 @@ impl Value { &Value::String(ref s) => format!("{}", s.val), &Value::Symbol(ref s) => format!("{}", s.val), &Value::Tuple(ref fs) => format!("{}", Self::fields_to_string(&fs.val)), - &Value::Selector(ref v) => v.val.join("."), + &Value::Selector(ref v) => v.val.join("."), + } + } + + pub fn pos(&self) -> &Option { + match self { + &Value::Int(ref i) => &i.pos, + &Value::Float(ref f) => &f.pos, + &Value::String(ref s) => &s.pos, + &Value::Symbol(ref s) => &s.pos, + &Value::Tuple(ref fs) => &fs.pos, + &Value::Selector(ref v) => &v.pos, } } } @@ -128,24 +172,68 @@ pub struct SelectDef { pub pos: Option, } +// TODO(jwall): This should have a way of rendering with position information. +#[derive(PartialEq,Debug,Eq,PartialOrd,Ord,Clone,Hash)] +pub struct Positioned { + pub pos: Option, + pub val: T, +} + +impl Positioned { + pub fn new(v: T) -> Self { + Positioned { + pos: None, + val: v, + } + } + + pub fn new_with_pos(v: T, pos: Position) -> Self { + Positioned { + pos: Some(pos), + val: v, + } + } +} + +impl<'a> From<&'a Token> for Positioned { + fn from(t: &'a Token) -> Positioned { + Positioned { + pos: Some(t.pos.clone()), + val: t.fragment.to_string(), + } + } +} + +impl<'a> From<&'a LocatedNode> for Positioned { + fn from(t: &LocatedNode) -> Positioned { + Positioned { + pos: t.pos.clone(), + val: t.val.clone(), + } + } +} + /// MacroDef is a pure function that always returns a Tuple. /// /// MacroDef's are not closures. They can not reference /// any values except what is defined in their arguments. #[derive(PartialEq,Debug,Clone)] pub struct MacroDef { - pub argdefs: Vec, + pub argdefs: Vec>, pub fields: FieldList, pub pos: Option, } impl MacroDef { - fn validate_value_symbols<'a>(&'a self, stack: &mut Vec<&'a Expression>, val: &'a Value) -> HashSet { + fn validate_value_symbols<'a>(&self, + stack: &mut Vec<&'a Expression>, + val: &'a Value) + -> HashSet { let mut bad_symbols = HashSet::new(); if let &Value::Symbol(ref name) = val { let mut ok = true; for arg in self.argdefs.iter() { - ok &= arg == &name.val + ok &= arg.val == name.val } if !ok { bad_symbols.insert(name.val.clone()); @@ -160,10 +248,10 @@ impl MacroDef { // But we don't know at this time of the value passed into // this macro is a tuple since this isn't a callsite. for arg in self.argdefs.iter() { - ok &= arg == &list[0] + ok &= arg.val == list[0].fragment } if !ok { - bad_symbols.insert(list[0].clone()); + bad_symbols.insert(list[0].fragment.to_string()); } } } else if let &Value::Tuple(ref tuple_node) = val { @@ -186,29 +274,29 @@ impl MacroDef { let mut syms_set = self.validate_value_symbols(&mut stack, &bexpr.left); bad_symbols.extend(syms_set.drain()); stack.push(&bexpr.right); - }, + } &Expression::Grouped(ref expr) => { stack.push(expr); - }, + } &Expression::Format(ref def) => { let exprs = &def.args; for arg_expr in exprs.iter() { stack.push(arg_expr); } - }, + } &Expression::Select(ref def) => { stack.push(def.default.borrow()); stack.push(def.val.borrow()); for &(_, ref expr) in def.tuple.iter() { stack.push(expr); } - }, + } &Expression::Copy(ref def) => { let fields = &def.fields; for &(_, ref expr) in fields.iter() { stack.push(expr); } - }, + } &Expression::Call(ref def) => { for expr in def.arglist.iter() { stack.push(expr); @@ -217,18 +305,18 @@ impl MacroDef { &Expression::Simple(ref val) => { let mut syms_set = self.validate_value_symbols(&mut stack, val); bad_symbols.extend(syms_set.drain()); - }, + } &Expression::Macro(_) => { // noop continue; - }, + } } } } if bad_symbols.len() > 0 { return Err(bad_symbols); } - return Ok(()) + return Ok(()); } } @@ -291,14 +379,14 @@ pub enum Statement { // Named bindings Let { - name: String, + name: Token, value: Expression, }, // Include a file. Import { path: String, - name: String, + name: Token, }, } @@ -308,12 +396,12 @@ mod ast_test { #[test] pub fn test_macro_validation_happy_path() { - let def = MacroDef{ + let def = MacroDef { argdefs: vec![ - "foo".to_string() + Positioned::new("foo".to_string()) ], fields: vec![ - ("f1".to_string(), Expression::Binary(BinaryOpDef{ + (Token::new("f1"), Expression::Binary(BinaryOpDef{ kind: BinaryExprType::Add, left: Value::Symbol(make_value_node("foo".to_string())), right: Box::new(Expression::Simple(Value::Int(make_value_node(1)))), @@ -327,12 +415,12 @@ mod ast_test { #[test] pub fn test_macro_validation_fail() { - let def = MacroDef{ + let def = MacroDef { argdefs: vec![ - "foo".to_string() + Positioned::new("foo".to_string()) ], fields: vec![ - ("f1".to_string(), Expression::Binary(BinaryOpDef{ + (Token::new("f1"), Expression::Binary(BinaryOpDef{ kind: BinaryExprType::Add, left: Value::Symbol(make_value_node("bar".to_string())), right: Box::new(Expression::Simple(Value::Int(make_value_node(1)))), @@ -340,7 +428,6 @@ mod ast_test { })), ], pos: None, - }; let mut expected = HashSet::new(); expected.insert("bar".to_string()); @@ -351,12 +438,12 @@ mod ast_test { pub fn test_macro_validation_selector_happy_path() { let def = MacroDef{ argdefs: vec![ - "foo".to_string() + Positioned::new("foo".to_string()) ], fields: vec![ - ("f1".to_string(), Expression::Binary(BinaryOpDef{ + (Token::new("f1"), Expression::Binary(BinaryOpDef{ kind: BinaryExprType::Add, - left: Value::Selector(make_value_node(vec!["foo".to_string(), "quux".to_string()])), + left: Value::Selector(make_value_node(vec![Token::new("foo"), Token::new("quux")])), right: Box::new(Expression::Simple(Value::Int(make_value_node(1)))), pos: None, })), @@ -368,14 +455,14 @@ mod ast_test { #[test] pub fn test_macro_validation_selector_fail() { - let def = MacroDef{ + let def = MacroDef { argdefs: vec![ - "foo".to_string() + Positioned::new("foo".to_string()), ], fields: vec![ - ("f1".to_string(), Expression::Binary(BinaryOpDef{ + (Token::new("f1"), Expression::Binary(BinaryOpDef{ kind: BinaryExprType::Add, - left: Value::Selector(make_value_node(vec!["bar".to_string(), "quux".to_string()])), + left: Value::Selector(make_value_node(vec![Token::new("bar"), Token::new("quux")])), right: Box::new(Expression::Simple(Value::Int(make_value_node(1)))), pos: None, })), @@ -384,6 +471,6 @@ mod ast_test { }; let mut expected = HashSet::new(); expected.insert("bar".to_string()); - assert_eq!(def.validate_symbols().err().unwrap(), expected); + assert_eq!(def.validate_symbols(), Err(expected)); } } diff --git a/src/build.rs b/src/build.rs index d4b412f..0bb0ec9 100644 --- a/src/build.rs +++ b/src/build.rs @@ -14,43 +14,46 @@ use std::fs::File; use std::io::Read; use std::error::Error; -use std::collections::{HashSet,HashMap,VecDeque}; +use std::collections::{HashSet, HashMap, VecDeque}; use std::collections::hash_map::Entry; use std::fmt; -use std::fmt::{Display,Formatter}; +use std::fmt::{Display, Formatter}; use std::ops::Deref; use std::rc::Rc; +use std::convert::From; use nom; +use tokenizer::Span; use ast::*; use format; use parse::parse; impl MacroDef { - pub fn eval(&self, mut args: Vec>) -> Result)>, Box> { + pub fn eval(&self, + mut args: Vec>) + -> Result, Rc)>, Box> { // Error conditions. If the args don't match the length and types of the argdefs then this is // macro call error. if args.len() > self.argdefs.len() { - return Err(Box::new( - BuildError::BadArgLen( - "Macro called with too many args".to_string()))); + return Err(Box::new(BuildError::BadArgLen("Macro called with too many args" + .to_string()))); } // If the args don't match the types required by the expressions then that is a TypeFail. // If the expressions reference Symbols not defined in the MacroDef that is also an error. // TODO(jwall): We should probably enforce that the Expression Symbols must be in argdefs rules // at Macro definition time not evaluation time. - let mut scope = HashMap::new(); + let mut scope = HashMap::>::new(); for (i, arg) in args.drain(0..).enumerate() { - scope.entry(self.argdefs[i].clone()).or_insert(arg.clone()); + scope.entry(self.argdefs[i].val.clone()).or_insert(arg.clone()); } let b = Builder::new_with_scope(scope); - let mut result: Vec<(String, Rc)> = Vec::new(); + let mut result: Vec<(Positioned, Rc)> = Vec::new(); for &(ref key, ref expr) in self.fields.iter() { // We clone the expressions here because this macro may be consumed // multiple times in the future. - let val = try!(b.eval_expr(expr.clone())); - result.push((key.clone(), val.clone())); + let val = try!(b.eval_expr(expr)); + result.push((key.into(), val.clone())); } Ok(result) } @@ -103,7 +106,7 @@ pub enum Val { Int(i64), Float(f64), String(String), - Tuple(Vec<(String, Rc)>), + Tuple(Vec<(Positioned, Rc)>), Macro(MacroDef), } @@ -120,15 +123,45 @@ impl Val { pub fn type_equal(&self, target: &Self) -> bool { match self { - &Val::Int(_) => if let &Val::Int(_) = target { true } else { false }, - &Val::Float(_) => if let &Val::Float(_) = target { true } else { false }, - &Val::String(_) => if let &Val::String(_) = target { true } else { false }, - &Val::Tuple(_) => if let &Val::Tuple(_) = target { true } else { false }, - &Val::Macro(_) => if let &Val::Macro(_) = target { true } else { false }, + &Val::Int(_) => { + if let &Val::Int(_) = target { + true + } else { + false + } + } + &Val::Float(_) => { + if let &Val::Float(_) = target { + true + } else { + false + } + } + &Val::String(_) => { + if let &Val::String(_) = target { + true + } else { + false + } + } + &Val::Tuple(_) => { + if let &Val::Tuple(_) = target { + true + } else { + false + } + } + &Val::Macro(_) => { + if let &Val::Macro(_) = target { + true + } else { + false + } + } } } - pub fn get_fields(&self) -> Option<&Vec<(String, Rc)>> { + pub fn get_fields(&self) -> Option<&Vec<(Positioned, Rc)>> { if let &Val::Tuple(ref fs) = self { Some(fs) } else { @@ -218,28 +251,28 @@ macro_rules! eval_binary_expr { impl Builder { /// new_builder constructs Builder with initialized fields ready to parse. - fn value_to_val(&self, v: Value) -> Result, Box> { + fn value_to_val(&self, v: &Value) -> Result, Box> { match v { - Value::Int(i) => Ok(Rc::new(Val::Int(i.val))), - Value::Float(f) => Ok(Rc::new(Val::Float(f.val))), - Value::String(s) => Ok(Rc::new(Val::String(s.val.to_string()))), - Value::Symbol(s) => { - self.lookup_sym(&s.val).ok_or(Box::new( - BuildError::NoSuchSymbol(format!("Unable to find {}", s.val)))) - }, - Value::Tuple(mut tuple_node) => { - let fields = &mut tuple_node.val; - let mut new_fields = Vec::new(); - for (name, expr) in fields.drain(0..) { + &Value::Int(ref i) => Ok(Rc::new(Val::Int(i.val))), + &Value::Float(ref f) => Ok(Rc::new(Val::Float(f.val))), + &Value::String(ref s) => Ok(Rc::new(Val::String(s.val.to_string()))), + &Value::Symbol(ref s) => { + self.lookup_sym(&(s.into())) + .ok_or(Box::new(BuildError::NoSuchSymbol(format!("Unable to find {}", s.val)))) + } + &Value::Tuple(ref tuple_node) => { + let fields = tuple_node.val(); + let mut new_fields = Vec::<(Positioned, Rc)>::new(); + for &(ref name, ref expr) in fields.iter() { let val = try!(self.eval_expr(expr)); - new_fields.push((name, val)); + new_fields.push((name.into(), val)); } new_fields.sort_by(|a, b| a.0.cmp(&b.0)); Ok(Rc::new(Val::Tuple(new_fields))) - }, - Value::Selector(selector_list_node) => { - self.lookup_selector(selector_list_node.val) - }, + } + &Value::Selector(ref selector_list_node) => { + self.lookup_selector(&selector_list_node.val) + } } } @@ -261,24 +294,27 @@ impl Builder { } } - pub fn build(&mut self, mut ast: Vec) -> BuildResult { - for stmt in ast.drain(0..) { + pub fn build(&mut self, ast: &Vec) -> BuildResult { + for stmt in ast.iter() { try!(self.build_stmt(stmt)); } Ok(()) } pub fn build_file_string(&mut self, name: &str, input: String) -> BuildResult { - match parse((&input[..]).as_bytes()) { - nom::IResult::Done(_, mut stmts) => { - for stmt in stmts.drain(0..) { + match parse(Span::new(&input)) { + nom::IResult::Done(_, stmts) => { + for stmt in stmts.iter() { try!(self.build_stmt(stmt)); } Ok(()) - }, + } nom::IResult::Error(err) => Err(Box::new(err)), - nom::IResult::Incomplete(_) => Err(Box::new( - BuildError::IncompleteParse(format!("Could not parse input from file: {}", name)))), + nom::IResult::Incomplete(_) => { + Err(Box::new(BuildError::IncompleteParse(format!("Could not parse input from \ + file: {}", + name)))) + } } } @@ -290,64 +326,70 @@ impl Builder { self.build_file_string(name, s) } - fn build_stmt(&mut self, stmt: Statement) -> BuildResult { + fn build_stmt(&mut self, stmt: &Statement) -> BuildResult { match stmt { - Statement::Let { name: sym, value: expr } => { + &Statement::Let { name: ref sym, value: ref expr } => { let val = try!(self.eval_expr(expr)); self.last = Some(val.clone()); - match self.out.entry(sym) { + match self.out.entry(sym.fragment.clone()) { Entry::Occupied(e) => { - return Err(Box::new( - BuildError::DuplicateBinding( - format!("Let binding for {} already exists", e.key())))); - }, + return Err(Box::new(BuildError::DuplicateBinding(format!("Let binding \ + for {:?} already \ + exists", + e.key())))); + } Entry::Vacant(e) => { e.insert(val); - }, + } } } - Statement::Import { path: val, name: sym } => { - if !self.files.contains(&val) { // Only parse the file once on import. - if self.assets.get(&sym).is_none() { + &Statement::Import { path: ref val, name: ref sym } => { + if !self.files.contains(val) { + // Only parse the file once on import. + if self.assets.get(&sym.fragment).is_none() { let mut b = Self::new(); try!(b.build_file(&val)); - let fields: Vec<(String, Rc)> = b.out.drain().collect(); + let fields: Vec<(Positioned, Rc)> = + b.out.drain().map(|t| (Positioned::new(t.0), t.1)) + .collect(); let result = Rc::new(Val::Tuple(fields)); - self.assets.entry(sym).or_insert(result.clone()); - self.files.insert(val); + self.assets.entry(sym.fragment.clone()).or_insert(result.clone()); + self.files.insert(val.clone()); self.last = Some(result); } } } - Statement::Expression(expr) => { + &Statement::Expression(ref expr) => { self.last = Some(try!(self.eval_expr(expr))); } }; Ok(()) } - fn lookup_sym(&self, sym: &str) -> Option> { - if self.out.contains_key(sym) { - return Some(self.out[sym].clone()); - } if self.assets.contains_key(sym) { - return Some(self.assets[sym].clone()); + fn lookup_sym(&self, sym: &Positioned) -> Option> { + if self.out.contains_key(&sym.val) { + return Some(self.out[&sym.val].clone()); + } + if self.assets.contains_key(&sym.val) { + return Some(self.assets[&sym.val].clone()); } None } - fn find_in_fieldlist(target: &str, fs: &Vec<(String, Rc)>) -> Option> { + fn find_in_fieldlist(target: &str, fs: &Vec<(Positioned, Rc)>) -> Option> { for (key, val) in fs.iter().cloned() { - if target == key { - return Some(val.clone()) + if target == &key.val { + return Some(val.clone()); } } - return None + return None; } - fn lookup_selector(&self, sl: SelectorList) -> Result, Box> { + fn lookup_selector(&self, sl: &SelectorList) -> Result, Box> { let len = sl.len(); if len > 0 { - if let Some(v) = self.lookup_sym(&sl[0]) { + let pos_sl = (&sl[0]).into(); + if let Some(v) = self.lookup_sym(&pos_sl) { let mut it = sl.iter().skip(1).peekable(); if it.peek().is_none() { return Ok(v.clone()); @@ -364,159 +406,187 @@ impl Builder { // None above. let k = it.next().unwrap(); if !vref.is_tuple() { - return Err(Box::new(BuildError::NoSuchSymbol( - format!("Attempted to dereference non-tuple {:?} at field {}.", sl, k)))); + // TODO(jeremy) BuildErrors should take a token so they can + // render the location of the error. + return Err(Box::new(BuildError::NoSuchSymbol(format!("Attempted \ + to dereference \ + non-tuple \ + {:?} at field \ + {}.", + sl, + k.fragment)))); } // This unwrap is safe because we already checked for // Tuple above. let fs = vref.get_fields().unwrap(); - if let Some(vv) = Self::find_in_fieldlist(k, fs) { + if let Some(vv) = Self::find_in_fieldlist(&k.fragment, fs) { if vv.is_tuple() { stack.push_back(vv.clone()); continue; } if it.peek().is_some() { - return Err(Box::new(BuildError::NoSuchSymbol( - format!("Unable to match selector path {:?}", sl)))); + return Err(Box::new(BuildError::NoSuchSymbol(format!("Unable to \ + match selector \ + path {:?}", + sl)))); } else { return Ok(vv.clone()); } } else { // TODO(jwall): A better error for this would be nice. - return Err(Box::new(BuildError::NoSuchSymbol( - format!("Unable to match selector path {:?}", sl)))); + return Err(Box::new(BuildError::NoSuchSymbol(format!("Unable to \ + match selector \ + path {:?}", + sl)))); } - }; + } } - return Err(Box::new(BuildError::TypeFail( - format!("{} is not a Tuple", sl[0])))); + return Err(Box::new(BuildError::TypeFail(format!("{} is not a Tuple", + sl[0].fragment)))); } - return Err(Box::new(BuildError::NoSuchSymbol( - format!("Unable to find Symbol {}", sl[0])))); + return Err(Box::new(BuildError::NoSuchSymbol(format!("Unable to find Symbol {}", + sl[0].fragment)))); } - return Err(Box::new(BuildError::NoSuchSymbol( - "Attempted to lookup an empty selector".to_string()))); + return Err(Box::new(BuildError::NoSuchSymbol("Attempted to lookup an empty selector" + .to_string()))); } // eval_expr evals a single Expression in the context of a running Builder. // It does not mutate the builders collected state at all. - pub fn eval_expr(&self, expr: Expression) -> Result, Box> { + pub fn eval_expr(&self, expr: &Expression) -> Result, Box> { // TODO(jwall): We probably don't want to consume these expressions. // Take a reference instead? match expr { - Expression::Simple(val) => { - self.value_to_val(val) - }, - Expression::Binary(def) => { - let kind = def.kind; - let v = def.left; - let expr = def.right; - let expr_result = try!(self.eval_expr(*expr)); + &Expression::Simple(ref val) => self.value_to_val(val), + &Expression::Binary(ref def) => { + let kind = &def.kind; + let v = &def.left; + let expr = &def.right; + let expr_result = try!(self.eval_expr(expr)); let v = try!(self.value_to_val(v)); match kind { - BinaryExprType::Add => { + &BinaryExprType::Add => { match *v { Val::Int(i) => { - eval_binary_expr!(&Val::Int(ii), expr_result, - Val::Int(i + ii), "Integer") - }, + eval_binary_expr!(&Val::Int(ii), + expr_result, + Val::Int(i + ii), + "Integer") + } Val::Float(f) => { - eval_binary_expr!(&Val::Float(ff), expr_result, - Val::Float(f + ff), "Float") - }, + eval_binary_expr!(&Val::Float(ff), + expr_result, + Val::Float(f + ff), + "Float") + } Val::String(ref s) => { match expr_result.as_ref() { &Val::String(ref ss) => { - return Ok(Rc::new( - Val::String([s.to_string(), ss.clone()].concat()))) - }, + return Ok(Rc::new(Val::String([s.to_string(), ss.clone()] + .concat()))) + } val => { - return Err(Box::new( - BuildError::TypeFail( - format!("Expected String but got {:?}", val)))) + return Err(Box::new(BuildError::TypeFail(format!("Expected \ + String \ + but got \ + {:?}", + val)))) } } - }, + } ref expr => { return Err(Box::new( BuildError::Unsupported( format!("{} does not support the '+' operation", expr.type_name())))) } } - }, - BinaryExprType::Sub => { + } + &BinaryExprType::Sub => { match *v { Val::Int(i) => { - eval_binary_expr!(&Val::Int(ii), expr_result, - Val::Int(i - ii), "Integer") - }, + eval_binary_expr!(&Val::Int(ii), + expr_result, + Val::Int(i - ii), + "Integer") + } Val::Float(f) => { - eval_binary_expr!(&Val::Float(ff), expr_result, - Val::Float(f - ff), "Float") - }, + eval_binary_expr!(&Val::Float(ff), + expr_result, + Val::Float(f - ff), + "Float") + } ref expr => { return Err(Box::new( BuildError::Unsupported( format!("{} does not support the '-' operation", expr.type_name())))) } } - }, - BinaryExprType::Mul => { + } + &BinaryExprType::Mul => { match *v { Val::Int(i) => { - eval_binary_expr!(&Val::Int(ii), expr_result, - Val::Int(i * ii), "Integer") - }, + eval_binary_expr!(&Val::Int(ii), + expr_result, + Val::Int(i * ii), + "Integer") + } Val::Float(f) => { - eval_binary_expr!(&Val::Float(ff), expr_result, - Val::Float(f * ff), "Float") - }, + eval_binary_expr!(&Val::Float(ff), + expr_result, + Val::Float(f * ff), + "Float") + } ref expr => { return Err(Box::new( BuildError::Unsupported( format!("{} does not support the '*' operation", expr.type_name())))) } } - }, - BinaryExprType::Div => { - match *v { - Val::Int(i) => { - eval_binary_expr!(&Val::Int(ii), expr_result, - Val::Int(i / ii), "Integer") - }, - Val::Float(f) => { - eval_binary_expr!(&Val::Float(ff), expr_result, - Val::Float(f / ff), "Float") - }, - ref expr => { - return Err(Box::new( - BuildError::Unsupported( - format!("{} does not support the '*' operation", expr.type_name())))) - } - } - }, - } - }, - Expression::Copy(mut def) => { - let v = try!(self.lookup_selector(def.selector)); - if let Val::Tuple(ref src_fields) = *v { - let mut m = HashMap::>::new(); - // loop through fields and build up a hasmap - for &(ref key, ref val) in src_fields.iter() { - if let Entry::Vacant(v) = m.entry(key.to_string()) { - v.insert(val.clone()); - } else { - return Err(Box::new( - BuildError::TypeFail( - format!("Duplicate field: {} in tuple", *key)))); - } } - for (key, val) in def.fields.drain(0..) { + &BinaryExprType::Div => { + match *v { + Val::Int(i) => { + eval_binary_expr!(&Val::Int(ii), + expr_result, + Val::Int(i / ii), + "Integer") + } + Val::Float(f) => { + eval_binary_expr!(&Val::Float(ff), + expr_result, + Val::Float(f / ff), + "Float") + } + ref expr => { + return Err(Box::new( + BuildError::Unsupported( + format!("{} does not support the '*' operation", expr.type_name())))) + } + } + } + } + } + &Expression::Copy(ref def) => { + let v = try!(self.lookup_selector(&def.selector)); + if let Val::Tuple(ref src_fields) = *v { + let mut m = HashMap::<&String, Rc>::new(); + // loop through fields and build up a hasmap + for &(ref key, ref val) in src_fields.iter() { + if let Entry::Vacant(v) = m.entry(&key.val) { + v.insert(val.clone()); + } else { + return Err(Box::new(BuildError::TypeFail(format!("Duplicate \ + field: {} in \ + tuple", + key.val)))); + } + } + for &(ref key, ref val) in def.fields.iter() { let expr_result = try!(self.eval_expr(val)); - match m.entry(key.clone()) { + match m.entry(&key.fragment) { Entry::Vacant(v) => { v.insert(expr_result); - }, + } Entry::Occupied(mut v) => { // Ensure that the new type matches the old type. let src_val = v.get().clone(); @@ -526,90 +596,86 @@ impl Builder { return Err(Box::new( BuildError::TypeFail( format!("Expected type {} for field {} but got {}", - src_val.type_name(), key, expr_result.type_name())))); + src_val.type_name(), key.fragment, expr_result.type_name())))); } - }, + } }; } - let mut new_fields: Vec<(String, Rc)> = m.drain() - .map(|(s, v)| (s.to_string(), v)) + let mut new_fields: Vec<(Positioned, Rc)> = m.drain() + .map(|(s, v)| (Positioned::new(s.to_string()), v)) .collect(); // We want a stable order for the fields to make comparing tuples // easier in later code. So we sort by the field name before constructing a new tuple. new_fields.sort_by(|a, b| a.0.cmp(&b.0)); return Ok(Rc::new(Val::Tuple(new_fields))); } - Err(Box::new( - BuildError::TypeFail( - format!("Expected Tuple got {}", v)))) - }, - Expression::Grouped(expr) => { - return self.eval_expr(*expr); - }, - Expression::Format(def) => { - let tmpl = def.template; - let mut args = def.args; + Err(Box::new(BuildError::TypeFail(format!("Expected Tuple got {}", v)))) + } + &Expression::Grouped(ref expr) => { + return self.eval_expr(expr); + } + &Expression::Format(ref def) => { + let tmpl = &def.template; + let args = &def.args; let mut vals = Vec::new(); - for v in args.drain(0..) { + for v in args.iter() { let rcv = try!(self.eval_expr(v)); vals.push(rcv.deref().clone()); } - let formatter = format::Formatter::new(tmpl, vals); + let formatter = format::Formatter::new(tmpl.clone(), vals); Ok(Rc::new(Val::String(try!(formatter.render())))) - }, - Expression::Call(def) => { - let sel = def.macroref; - let mut args = def.arglist; + } + &Expression::Call(ref def) => { + let sel = &def.macroref; + let args = &def.arglist; let v = try!(self.lookup_selector(sel)); if let &Val::Macro(ref m) = v.deref() { // Congratulations this is actually a macro. let mut argvals: Vec> = Vec::new(); - for arg in args.drain(0..) { + for arg in args.iter() { argvals.push(try!(self.eval_expr(arg))); } let fields = try!(m.eval(argvals)); return Ok(Rc::new(Val::Tuple(fields))); } - Err(Box::new( - BuildError::TypeFail( - // We should pretty print the selectors here. - format!("{} is not a Macro", v)))) - }, - Expression::Macro(def) => { + Err(Box::new(BuildError::TypeFail(// We should pretty print the selectors here. + format!("{} is not a Macro", v)))) + } + &Expression::Macro(ref def) => { match def.validate_symbols() { - Ok(()) => Ok(Rc::new(Val::Macro(def))), - Err(set) => Err(Box::new( - BuildError::NoSuchSymbol( - format!("Macro has the following undefined symbols: {:?}", set)))), + Ok(()) => Ok(Rc::new(Val::Macro(def.clone()))), + Err(set) => { + Err(Box::new(BuildError::NoSuchSymbol(format!("Macro has the following \ + undefined symbols: {:?}", + set)))) + } } - }, - Expression::Select(def) => { - let target = def.val; - let def_expr = def.default; - let mut fields = def.tuple; + } + &Expression::Select(ref def) => { + let target = &def.val; + let def_expr = &def.default; + let fields = &def.tuple; // First resolve the target expression. - let v = try!(self.eval_expr(*target)); + let v = try!(self.eval_expr(target)); // Second ensure that the expression resolves to a string. if let &Val::String(ref name) = v.deref() { // Third find the field with that name in the tuple. - for (fname, val_expr) in fields.drain(0..) { - if &fname == name { + for &(ref fname, ref val_expr) in fields.iter() { + if &fname.fragment == name { // Fourth return the result of evaluating that field. return self.eval_expr(val_expr); } } // Otherwise return the default - return self.eval_expr(*def_expr); + return self.eval_expr(def_expr); } else { - return Err(Box::new( - BuildError::TypeFail( - format!("Expected String but got {} in Select expression", - v.type_name())))); + return Err(Box::new(BuildError::TypeFail(format!("Expected String but got \ + {} in Select expression", + v.type_name())))); } - }, + } } } - } #[cfg(test)] @@ -618,9 +684,9 @@ mod test { use ast::*; use std::rc::Rc; - fn test_expr_to_val(mut cases: Vec<(Expression,Val)>, b: Builder) { + fn test_expr_to_val(mut cases: Vec<(Expression, Val)>, b: Builder) { for tpl in cases.drain(0..) { - assert_eq!(b.eval_expr(tpl.0).unwrap(), Rc::new(tpl.1)); + assert_eq!(b.eval_expr(&tpl.0).unwrap(), Rc::new(tpl.1)); } } @@ -644,7 +710,8 @@ mod test { pos: None, }), Val::Float(1.0)), - ], b); + ], + b); } #[test] @@ -660,7 +727,8 @@ mod test { pos: None, }), Val::Float(1.0)), - ], b); + ], + b); } #[test] @@ -683,7 +751,8 @@ mod test { pos: None, }), Val::Float(4.0)), - ], b); + ], + b); } #[test] @@ -699,7 +768,8 @@ mod test { pos: None, }), Val::Float(1.0)), - ], b); + ], + b); } #[test] @@ -722,7 +792,8 @@ mod test { pos: None, }), Val::Float(1.0)), - ], b); + ], + b); } #[test] @@ -738,7 +809,8 @@ mod test { pos: None, }), Val::Float(1.0)), - ], b); + ], + b); } #[test] @@ -785,7 +857,8 @@ mod test { pos: None, }), Val::Float(1.0)), - ], b); + ], + b); } #[test] @@ -795,10 +868,13 @@ mod test { (Expression::Simple(Value::Float(make_value_node(2.0))), Val::Float(2.0)), (Expression::Simple(Value::String(make_value_node("foo".to_string()))), Val::String("foo".to_string())), - (Expression::Simple(Value::Tuple(make_value_node(vec![("bar".to_string(), - Expression::Simple(Value::Int(make_value_node(1))))]))), - Val::Tuple(vec![("bar".to_string(), Rc::new(Val::Int(1)))])), - ], Builder::new()); + (Expression::Simple(Value::Tuple(make_value_node(vec![ + (Token::new("bar"), Expression::Simple(Value::Int(make_value_node(1)))) + ]))), + Val::Tuple(vec![(Positioned::new_with_pos("bar".to_string(), Position{line: 0, column: 0}), + Rc::new(Val::Int(1)))])), + ], + Builder::new()); } #[test] @@ -807,14 +883,16 @@ mod test { b.out.entry("var1".to_string()).or_insert(Rc::new(Val::Int(1))); test_expr_to_val(vec![ (Expression::Simple(Value::Symbol(make_value_node("var1".to_string()))), Val::Int(1)), - ], b); + ], + b); } #[test] fn test_eval_simple_lookup_error() { let mut b = Builder::new(); b.out.entry("var1".to_string()).or_insert(Rc::new(Val::Int(1))); - assert!(b.eval_expr(Expression::Simple(Value::Symbol(make_value_node("var".to_string())))).is_err()); + let expr = Expression::Simple(Value::Symbol(make_value_node("var".to_string()))); + assert!(b.eval_expr(&expr).is_err()); } #[test] @@ -822,41 +900,42 @@ mod test { // TODO(jwall): Tests for this expression. let mut b = Builder::new(); b.out.entry("var1".to_string()).or_insert(Rc::new(Val::Tuple(vec![ - ("lvl1".to_string(), Rc::new(Val::Tuple( + (Positioned::new("lvl1".to_string()), Rc::new(Val::Tuple( vec![ - ("lvl2".to_string(), Rc::new(Val::Int(3))), + (Positioned::new("lvl2".to_string()), Rc::new(Val::Int(3))), ] ))), ]))); b.out.entry("var2".to_string()).or_insert(Rc::new(Val::Int(2))); - b.out.entry("var3".to_string()).or_insert(Rc::new(Val::Tuple(vec![ - ("lvl1".to_string(), Rc::new(Val::Int(4))) - ]))); + b.out + .entry("var3".to_string()) + .or_insert(Rc::new(Val::Tuple(vec![(Positioned::new("lvl1".to_string()), + Rc::new(Val::Int(4)))]))); test_expr_to_val(vec![ - (Expression::Simple(Value::Selector(make_value_node(vec!["var1".to_string()]))), Val::Tuple( + (Expression::Simple(Value::Selector(make_value_node(vec![Token::new("var1")]))), Val::Tuple( vec![ - ("lvl1".to_string(), Rc::new(Val::Tuple( + (Positioned::new("lvl1".to_string()), Rc::new(Val::Tuple( vec![ - ("lvl2".to_string(), Rc::new(Val::Int(3))), + (Positioned::new("lvl2".to_string()), Rc::new(Val::Int(3))), ] ))), ] )), - (Expression::Simple(Value::Selector(make_value_node(vec!["var1".to_string(), - "lvl1".to_string()]))), + (Expression::Simple(Value::Selector(make_value_node(vec![Token::new("var1"), + Token::new("lvl1")]))), Val::Tuple( vec![ - ("lvl2".to_string(), Rc::new(Val::Int(3))), + (Positioned::new("lvl2".to_string()), Rc::new(Val::Int(3))), ] )), - (Expression::Simple(Value::Selector(make_value_node(vec!["var1".to_string(), - "lvl1".to_string(), - "lvl2".to_string()]))), + (Expression::Simple(Value::Selector(make_value_node(vec![Token::new("var1"), + Token::new("lvl1"), + Token::new("lvl2")]))), Val::Int(3)), - (Expression::Simple(Value::Selector(make_value_node(vec!["var2".to_string()]))), + (Expression::Simple(Value::Selector(make_value_node(vec![Token::new("var2")]))), Val::Int(2)), - (Expression::Simple(Value::Selector(make_value_node(vec!["var3".to_string(), - "lvl1".to_string()]))), + (Expression::Simple(Value::Selector(make_value_node(vec![Token::new("var3"), + Token::new("lvl1")]))), Val::Int(4)), ], b); } @@ -866,7 +945,7 @@ mod test { fn test_expr_copy_no_such_tuple() { let b = Builder::new(); test_expr_to_val(vec![ - (Expression::Copy(CopyDef{selector: vec!["tpl1".to_string()], fields: Vec::new(), pos: None}), + (Expression::Copy(CopyDef{selector: vec![Token::new("tpl1")], fields: Vec::new(), pos: None}), Val::Tuple(Vec::new())), ], b); } @@ -877,7 +956,7 @@ mod test { let mut b = Builder::new(); b.out.entry("tpl1".to_string()).or_insert(Rc::new(Val::Int(1))); test_expr_to_val(vec![ - (Expression::Copy(CopyDef{selector: vec!["tpl1".to_string()], fields: Vec::new(), pos: None}), + (Expression::Copy(CopyDef{selector: vec![Token::new("tpl1")], fields: Vec::new(), pos: None}), Val::Tuple(Vec::new())), ], b); } @@ -887,18 +966,18 @@ mod test { fn test_expr_copy_field_type_error() { let mut b = Builder::new(); b.out.entry("tpl1".to_string()).or_insert(Rc::new(Val::Tuple(vec![ - ("fld1".to_string(), Rc::new(Val::Int(1))), + (Positioned::new("fld1".to_string()), Rc::new(Val::Int(1))), ]))); test_expr_to_val(vec![ (Expression::Copy( CopyDef{ - selector: vec!["tpl1".to_string()], - fields: vec![("fld1".to_string(), + selector: vec![Token::new("tpl1")], + fields: vec![(Token::new("fld1"), Expression::Simple(Value::String(make_value_node("2".to_string()))))], pos: None}), Val::Tuple( vec![ - ("fld1".to_string(), Rc::new(Val::String("2".to_string()))), + (Positioned::new("fld1".to_string()), Rc::new(Val::String("2".to_string()))), ], )), ], b); @@ -911,13 +990,13 @@ mod test { // TODO(jwall): Tests for this expression. let mut b = Builder::new(); b.out.entry("tpl1".to_string()).or_insert(Rc::new(Val::Tuple(vec![ - ("fld1".to_string(), Rc::new(Val::Int(1))), + (Positioned::new("fld1".to_string()), Rc::new(Val::Int(1))), ]))); test_expr_to_val(vec![ (Expression::Copy( CopyDef{ - selector: vec!["tpl1".to_string()], - fields: vec![("fld2".to_string(), + selector: vec![Token::new("tpl1")], + fields: vec![(Token::new("fld2"), Expression::Simple(Value::String(make_value_node("2".to_string()))))], pos: None, }), @@ -927,33 +1006,33 @@ mod test { // that the compare assertion is correct. The ordering has no // semantics though so at some point we should probably be less restrictive. vec![ - ("fld1".to_string(), Rc::new(Val::Int(1))), - ("fld2".to_string(), Rc::new(Val::String("2".to_string()))), + (Positioned::new("fld1".to_string()), Rc::new(Val::Int(1))), + (Positioned::new("fld2".to_string()), Rc::new(Val::String("2".to_string()))), ], )), // Overwrite a field in the copy (Expression::Copy( CopyDef{ - selector: vec!["tpl1".to_string()], + selector: vec![Token::new("tpl1")], fields: vec![ - ("fld1".to_string(), + (Token::new("fld1"), Expression::Simple(Value::Int(make_value_node(3)))), - ("fld2".to_string(), + (Token::new("fld2"), Expression::Simple(Value::String(make_value_node("2".to_string())))), ], pos: None, }), Val::Tuple( vec![ - ("fld1".to_string(), Rc::new(Val::Int(3))), - ("fld2".to_string(), Rc::new(Val::String("2".to_string()))), + (Positioned::new("fld1".to_string()), Rc::new(Val::Int(3))), + (Positioned::new("fld2".to_string()), Rc::new(Val::String("2".to_string()))), ], )), // The source tuple is still unmodified. - (Expression::Simple(Value::Selector(make_value_node(vec!["tpl1".to_string()]))), + (Expression::Simple(Value::Selector(make_value_node(vec![Token::new("tpl1")]))), Val::Tuple( vec![ - ("fld1".to_string(), Rc::new(Val::Int(1))), + (Positioned::new("fld1".to_string()), Rc::new(Val::Int(1))), ], )), ], b); @@ -963,20 +1042,21 @@ mod test { fn test_macro_call() { let mut b = Builder::new(); b.out.entry("tstmac".to_string()).or_insert(Rc::new(Val::Macro(MacroDef{ - argdefs: vec!["arg1".to_string()], + argdefs: vec![Positioned::new("arg1".to_string())], fields: vec![ - ("foo".to_string(), Expression::Simple(Value::Symbol(make_value_node("arg1".to_string())))), + (Token::new("foo"), Expression::Simple(Value::Symbol(make_value_node("arg1".to_string())))), ], pos: None, }))); test_expr_to_val(vec![ (Expression::Call(CallDef{ - macroref: vec!["tstmac".to_string()], + macroref: vec![Token::new("tstmac")], arglist: vec![Expression::Simple(Value::String(make_value_node("bar".to_string())))], pos: None, }), Val::Tuple(vec![ - ("foo".to_string(), Rc::new(Val::String("bar".to_string()))), + (Positioned::new_with_pos("foo".to_string(), Position{line: 0, column: 0}), + Rc::new(Val::String("bar".to_string()))), ])), ], b); } @@ -985,22 +1065,24 @@ mod test { #[should_panic(expected = "Unable to find arg1")] fn test_macro_hermetic() { let mut b = Builder::new(); - b.out.entry("arg1".to_string()).or_insert(Rc::new(Val::String("bar".to_string()))); + b.out + .entry("arg1".to_string()) + .or_insert(Rc::new(Val::String("bar".to_string()))); b.out.entry("tstmac".to_string()).or_insert(Rc::new(Val::Macro(MacroDef{ - argdefs: vec!["arg2".to_string()], + argdefs: vec![Positioned::new("arg2".to_string())], fields: vec![ - ("foo".to_string(), Expression::Simple(Value::Symbol(make_value_node("arg1".to_string())))), + (Token::new("foo"), Expression::Simple(Value::Symbol(make_value_node("arg1".to_string())))), ], pos: None, }))); test_expr_to_val(vec![ (Expression::Call(CallDef{ - macroref: vec!["tstmac".to_string()], + macroref: vec![Token::new("tstmac")], arglist: vec![Expression::Simple(Value::String(make_value_node("bar".to_string())))], pos: None, }), Val::Tuple(vec![ - ("foo".to_string(), Rc::new(Val::String("bar".to_string()))), + (Positioned::new("foo".to_string()), Rc::new(Val::String("bar".to_string()))), ])), ], b); } @@ -1008,15 +1090,19 @@ mod test { #[test] fn test_select_expr() { let mut b = Builder::new(); - b.out.entry("foo".to_string()).or_insert(Rc::new(Val::String("bar".to_string()))); - b.out.entry("baz".to_string()).or_insert(Rc::new(Val::String("boo".to_string()))); + b.out + .entry("foo".to_string()) + .or_insert(Rc::new(Val::String("bar".to_string()))); + b.out + .entry("baz".to_string()) + .or_insert(Rc::new(Val::String("boo".to_string()))); test_expr_to_val(vec![ (Expression::Select(SelectDef{ val: Box::new(Expression::Simple(Value::Symbol(make_value_node("foo".to_string())))), default: Box::new(Expression::Simple(Value::Int(make_value_node(1)))), tuple: vec![ - ("bar".to_string(), Expression::Simple(Value::Int(make_value_node(2)))), - ("quux".to_string(), Expression::Simple(Value::String(make_value_node("2".to_string())))), + (Token::new("foo"), Expression::Simple(Value::String(make_value_node("2".to_string())))), + (Token::new("bar"), Expression::Simple(Value::Int(make_value_node(2)))), ], pos: None, }), @@ -1025,8 +1111,8 @@ mod test { val: Box::new(Expression::Simple(Value::Symbol(make_value_node("baz".to_string())))), default: Box::new(Expression::Simple(Value::Int(make_value_node(1)))), tuple: vec![ - ("bar".to_string(), Expression::Simple(Value::Int(make_value_node(2)))), - ("quux".to_string(), Expression::Simple(Value::String(make_value_node("2".to_string())))), + (Token::new("bar"), Expression::Simple(Value::Int(make_value_node(2)))), + (Token::new("quux"), Expression::Simple(Value::String(make_value_node("2".to_string())))), ], pos: None, }), @@ -1045,8 +1131,8 @@ mod test { val: Box::new(Expression::Simple(Value::Symbol(make_value_node("foo".to_string())))), default: Box::new(Expression::Simple(Value::Int(make_value_node(1)))), tuple: vec![ - ("bar".to_string(), Expression::Simple(Value::Int(make_value_node(2)))), - ("quux".to_string(), Expression::Simple(Value::String(make_value_node("2".to_string())))), + (Token::new("bar"), Expression::Simple(Value::Int(make_value_node(2)))), + (Token::new("quux"), Expression::Simple(Value::String(make_value_node("2".to_string())))), ], pos: None, }), @@ -1057,40 +1143,43 @@ mod test { #[test] fn test_let_statement() { let mut b = Builder::new(); - b.build_stmt(Statement::Let{ - name:"foo".to_string(), + let stmt = Statement::Let { + name: Token::new("foo"), value: Expression::Simple(Value::String(make_value_node("bar".to_string()))), - }).unwrap(); + }; + b.build_stmt(&stmt).unwrap(); test_expr_to_val(vec![ (Expression::Simple(Value::Symbol(make_value_node("foo".to_string()))), Val::String("bar".to_string())), - ], b); + ], + b); } #[test] fn test_build_file_string() { let mut b = Builder::new(); b.build_file_string("foo.ucg", "let foo = 1;".to_string()).unwrap(); - assert!(b.out.contains_key("foo")); + let key = "foo"; + assert!(b.out.contains_key(key)); } #[test] fn test_asset_symbol_lookups() { let mut b = Builder::new(); - b.assets.entry("foo".to_string()).or_insert( - Rc::new(Val::Tuple(vec![ - ("bar".to_string(), Rc::new(Val::Tuple(vec![ - ("quux".to_string(), Rc::new(Val::Int(1))), + b.assets.entry("foo".to_string()).or_insert(Rc::new(Val::Tuple(vec![ + (Positioned::new("bar".to_string()), Rc::new(Val::Tuple(vec![ + (Positioned::new("quux".to_string()), Rc::new(Val::Int(1))), ]))), ]))); test_expr_to_val(vec![ (Expression::Simple(Value::Symbol(make_value_node("foo".to_string()))), Val::Tuple(vec![ - ("bar".to_string(), Rc::new(Val::Tuple(vec![ - ("quux".to_string(), Rc::new(Val::Int(1))), + (Positioned::new("bar".to_string()), Rc::new(Val::Tuple(vec![ + (Positioned::new("quux".to_string()), Rc::new(Val::Int(1))), ]))), ])), - ], b); + ], + b); } // TODO(jwall): Unit test for MacroDef Symbol Validation. } diff --git a/src/format.rs b/src/format.rs index 893b1ff..4684617 100644 --- a/src/format.rs +++ b/src/format.rs @@ -24,7 +24,7 @@ pub struct Formatter + Clone> { impl + Clone> Formatter { pub fn new>(tmpl: S, args: Vec) -> Self { - Formatter{ + Formatter { tmpl: tmpl.into(), args: args, } @@ -37,9 +37,9 @@ impl + Clone> Formatter { for c in self.tmpl.chars() { if c == '@' && !should_escape { if count == self.args.len() { - return Err(Box::new( - BuildError::FormatError( - "Too few arguments to string formatter.".to_string()))) + return Err(Box::new(BuildError::FormatError("Too few arguments to string \ + formatter." + .to_string()))); } let arg = self.args[count].clone(); let strval = arg.into(); @@ -52,9 +52,8 @@ impl + Clone> Formatter { } } if self.args.len() != count { - return Err(Box::new( - BuildError::FormatError( - "Too many arguments to string formatter.".to_string()))) + return Err(Box::new(BuildError::FormatError("Too many arguments to string formatter." + .to_string()))); } return Ok(buf); } diff --git a/src/parse.rs b/src/parse.rs index 93c5ac2..9202721 100644 --- a/src/parse.rs +++ b/src/parse.rs @@ -12,14 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. use std::str::FromStr; -use std::str::from_utf8; use std::error::Error; -use nom_locate::LocatedSpan; -use nom::{alpha, is_alphanumeric, digit}; - use ast::*; - +use tokenizer::*; quick_error! { #[derive(Debug,PartialEq)] @@ -28,72 +24,40 @@ quick_error! { description("Unexpected Token") display("Unexpected Token Expected {} Got {}", expected, actual) } + EmptyExpression(msg: String ) { + description("EmptyExpression") + display("Unexpected EmptyExpression {}", msg) + } } } -type Span<'a> = LocatedSpan<&'a str>; - // TODO(jwall): Convert to tokenizer steps followed by parser steps. // TODO(jwall): Error Reporting with Line and Column information. type ParseResult = Result>; -// sentinels and punctuation -named!(doublequote, tag!("\"")); -named!(comma, tag!(",")); -named!(lbrace, tag!("{")); -named!(rbrace, tag!("}")); -named!(lparen, tag!("(")); -named!(rparen, tag!(")")); -named!(dot, tag!(".")); -named!(plus, tag!("+")); -named!(minus, tag!("-")); -named!(mul, tag!("*")); -named!(div, tag!("/")); -named!(equal, tag!("=")); -named!(semicolon, tag!(";")); -named!(fatcomma, tag!("=>")); - -fn is_symbol_char(c: u8) -> bool { - is_alphanumeric(c) || c == '-' as u8 || c == '_' as u8 -} -// a field is the building block of symbols and tuple field names. -named!(field, - map_res!(preceded!(peek!(alpha), take_while!(is_symbol_char)), - |s| from_utf8(s).map(|s| s.to_string()) - ) -); - -fn symbol_to_value(s: String) -> ParseResult { - Ok(Value::Symbol(value_node!(s))) +fn symbol_to_value(s: Token) -> ParseResult { + Ok(Value::Symbol(value_node!(s.fragment.to_string()))) } // symbol is a bare unquoted field. -named!(symbol, map_res!(field, symbol_to_value)); +named!(symbol( Span ) -> Value, map_res!(barewordtok, symbol_to_value)); -// quoted is a quoted string. -named!(quoted, - map_res!(delimited!(doublequote, take_until!("\""), doublequote), - |s| from_utf8(s).map(|s| s.to_string()) - ) -); - -fn str_to_value(s: String) -> ParseResult { - Ok(Value::String(value_node!(s))) +fn str_to_value(s: Token) -> ParseResult { + Ok(Value::String(value_node!(s.fragment.to_string()))) } // quoted_value is a quoted string. -named!(quoted_value, - map_res!(quoted, str_to_value) +named!(quoted_value( Span ) -> Value, + map_res!(strtok, str_to_value) ); // Helper function to make the return types work for down below. -fn triple_to_number(v: (Option<&[u8]>, Option<&[u8]>, Option<&[u8]>)) - -> ParseResult { +fn triple_to_number(v: (Option, Option, Option)) -> ParseResult { let pref = match v.0 { None => "", - Some(bs) => try!(from_utf8(bs)), + Some(ref bs) => &bs.fragment, }; let has_dot = v.1.is_some(); @@ -104,7 +68,7 @@ fn triple_to_number(v: (Option<&[u8]>, Option<&[u8]>, Option<&[u8]>)) let suf = match v.2 { None => "", - Some(bs) => try!(from_utf8(bs)), + Some(ref bs) => &bs.fragment, }; let to_parse = pref.to_string() + "." + suf; @@ -122,46 +86,44 @@ fn triple_to_number(v: (Option<&[u8]>, Option<&[u8]>, Option<&[u8]>)) // *IMPORTANT* // It also means this combinator is risky when used with partial // inputs. So handle with care. -named!(number, +named!(number( Span ) -> Value, map_res!(alt!( complete!(do_parse!( // 1.0 - prefix: digit >> - has_dot: dot >> - suffix: digit >> - peek!(not!(digit)) >> + prefix: digittok >> + has_dot: dottok >> + suffix: digittok >> + peek!(not!(digittok)) >> (Some(prefix), Some(has_dot), Some(suffix)) )) | complete!(do_parse!( // 1. - prefix: digit >> - has_dot: dot >> - peek!(not!(digit)) >> + prefix: digittok >> + has_dot: dottok >> + peek!(not!(digittok)) >> (Some(prefix), Some(has_dot), None) )) | complete!(do_parse!( // .1 - has_dot: dot >> - suffix: digit >> - peek!(not!(digit)) >> + has_dot: dottok >> + suffix: digittok >> + peek!(not!(digittok)) >> (None, Some(has_dot), Some(suffix)) )) | do_parse!( // 1 - prefix: digit >> + prefix: digittok >> // The peek!(not!(..)) make this whole combinator slightly // safer for partial inputs. - peek!(not!(digit)) >> + peek!(not!(digittok)) >> (Some(prefix), None, None) )), triple_to_number ) ); -named!(value, alt!(number | quoted_value | symbol | tuple)); - named!( #[doc="Capture a field and value pair composed of ` = ,`"], - field_value<(String, Expression) >, + field_value( Span ) -> (Token, Expression), do_parse!( - field: field >> - ws!(equal) >> + field: barewordtok >> + ws!(equaltok) >> value: expression >> (field, value) ) @@ -172,32 +134,27 @@ fn vec_to_tuple(v: FieldList) -> ParseResult { Ok(Value::Tuple(value_node!(v))) } -named!(field_list, - separated_list!(comma, ws!(field_value))); +named!(field_list( Span ) -> FieldList, + separated_list!(commatok, ws!(field_value))); named!( #[doc="Capture a tuple of named fields with values. {=,...}"], - tuple, + tuple( Span ) -> Value, map_res!( - delimited!(lbrace, + delimited!(lbracetok, ws!(field_list), - rbrace), + rbracetok), vec_to_tuple ) ); -// keywords -named!(let_word, tag!("let")); -named!(select_word, tag!("select")); -named!(macro_word, tag!("macro")); -named!(import_word, tag!("import")); -named!(as_word, tag!("as")); +named!(value( Span ) -> Value, alt!(number | quoted_value | symbol | tuple)); fn value_to_expression(v: Value) -> ParseResult { Ok(Expression::Simple(v)) } -named!(simple_expression, +named!(simple_expression( Span ) -> Expression, map_res!( value, value_to_expression @@ -205,7 +162,7 @@ named!(simple_expression, ); fn tuple_to_binary_expression(tpl: (BinaryExprType, Value, Expression)) -> ParseResult { - Ok(Expression::Binary(BinaryOpDef{ + Ok(Expression::Binary(BinaryOpDef { kind: tpl.0, left: tpl.1, right: Box::new(tpl.2), @@ -213,78 +170,81 @@ fn tuple_to_binary_expression(tpl: (BinaryExprType, Value, Expression)) -> Parse })) } -named!(add_expression, - map_res!( - do_parse!( - left: value >> - ws!(plus) >> - right: expression >> - (BinaryExprType::Add, left, right) - ), - tuple_to_binary_expression - ) +macro_rules! do_binary_expr { + ($i:expr, $fn:expr, $typ:expr) => { + // NOTE(jwall): Nom macros do magic with their inputs. They in fact + // rewrite your macro argumets for you. Which means we require this $i + // paramater even though we don't explicitely pass it below. I don't + // particularly like this but I'm living with it for now. + map_res!( + $i, do_parse!( + left: value >> + ws!($fn) >> + right: expression >> + ($typ, left, right) + ), + tuple_to_binary_expression + ) + } +} + +named!(add_expression( Span ) -> Expression, + do_binary_expr!(plustok, BinaryExprType::Add) ); -named!(sub_expression, - map_res!( - do_parse!( - left: value >> - ws!(minus) >> - right: expression >> - (BinaryExprType::Sub, left, right) - ), - tuple_to_binary_expression - ) +named!(sub_expression( Span ) -> Expression, + do_binary_expr!(dashtok, BinaryExprType::Sub) ); -named!(mul_expression, - map_res!( - do_parse!( - left: value >> - ws!(mul) >> - right: expression >> - (BinaryExprType::Mul, left, right) - ), - tuple_to_binary_expression - ) +named!(mul_expression( Span ) -> Expression, + do_binary_expr!(startok, BinaryExprType::Mul) ); -named!(div_expression, - map_res!( - do_parse!( - left: value >> - ws!(div) >> - right: expression >> - (BinaryExprType::Div, left, right) - ), - tuple_to_binary_expression - ) +named!(div_expression( Span ) -> Expression, + do_binary_expr!(slashtok, BinaryExprType::Div) ); fn expression_to_grouped_expression(e: Expression) -> ParseResult { Ok(Expression::Grouped(Box::new(e))) } -named!(grouped_expression, +named!(grouped_expression( Span ) -> Expression, map_res!( - preceded!(lparen, terminated!(expression, rparen)), + preceded!(lparentok, terminated!(expression, rparentok)), expression_to_grouped_expression ) ); -named!(selector_list, separated_nonempty_list!(dot, field)); - -fn tuple_to_copy(t: (SelectorList, FieldList)) -> ParseResult { - Ok(Expression::Copy(CopyDef{selector: t.0, fields: t.1, pos: None})) +fn assert_nonempty_list(v: Vec) -> ParseResult> { + if v.is_empty() { + return Err(Box::new(ParseError::EmptyExpression("Selectors can't be empty.".to_string()))) + } + return Ok(v); } -named!(copy_expression, +// TODO(jwall): We should assert that this is a nonempty list that comes out of here. +named!(selector_list( Span ) -> SelectorList, + map_res!( + separated_list!(dottok, barewordtok), + assert_nonempty_list + ) +); + +fn tuple_to_copy(t: (SelectorList, FieldList)) -> ParseResult { + Ok(Expression::Copy(CopyDef { + selector: t.0, + fields: t.1, + pos: None, + })) +} + +named!(copy_expression( Span ) -> Expression, map_res!( do_parse!( selector: selector_list >> - lbrace >> + lbracetok >> fields: ws!(field_list) >> - rbrace >> + rbracetok >> (selector, fields) ), tuple_to_copy @@ -295,9 +255,18 @@ fn tuple_to_macro(mut t: (Vec, Value)) -> ParseResult { match t.1 { Value::Tuple(v) => { Ok(Expression::Macro(MacroDef { - argdefs: t.0.drain(0..).map(|s| s.to_string()).collect(), + // TODO(jwall): The position information here is not as accurate as we might want. + argdefs: t.0 + .drain(0..) + .map(|s| { + Positioned { + pos: v.pos.clone(), + val: s.to_string(), + } + }) + .collect(), fields: v.val, - pos: None, + pos: v.pos, })) } // TODO(jwall): Show a better version of the unexpected parsed value. @@ -307,16 +276,16 @@ fn tuple_to_macro(mut t: (Vec, Value)) -> ParseResult { } } -named!(arglist >, separated_list!(ws!(comma), symbol)); +named!(arglist( Span ) -> Vec, separated_list!(ws!(commatok), symbol)); -named!(macro_expression, +named!(macro_expression( Span ) -> Expression, map_res!( do_parse!( - macro_word >> - ws!(lparen) >> + macrotok >> + ws!(lparentok) >> arglist: ws!(arglist) >> - rparen >> - ws!(fatcomma) >> + rparentok >> + ws!(fatcommatok) >> map: tuple >> (arglist, map) ), @@ -324,11 +293,10 @@ named!(macro_expression, ) ); -fn tuple_to_select(t: (Expression, Expression, Value)) - -> ParseResult { +fn tuple_to_select(t: (Expression, Expression, Value)) -> ParseResult { match t.2 { Value::Tuple(v) => { - Ok(Expression::Select(SelectDef{ + Ok(Expression::Select(SelectDef { val: Box::new(t.0), default: Box::new(t.1), tuple: v.val, @@ -342,31 +310,35 @@ fn tuple_to_select(t: (Expression, Expression, Value)) } } -named!(select_expression, +named!(select_expression( Span ) -> Expression, map_res!( - terminated!(do_parse!( - select_word >> - val: ws!(terminated!(expression, comma)) >> - default: ws!(terminated!(expression, comma)) >> + do_parse!( + selecttok >> + val: ws!(terminated!(expression, commatok)) >> + default: ws!(terminated!(expression, commatok)) >> map: ws!(tuple) >> (val, default, map) - ), semicolon), + ), tuple_to_select ) ); -fn tuple_to_format(t: (String, Vec)) -> ParseResult { - Ok(Expression::Format(FormatDef{template: t.0, args: t.1, pos: None})) +fn tuple_to_format(t: (Token, Vec)) -> ParseResult { + Ok(Expression::Format(FormatDef { + template: t.0.fragment.to_string(), + args: t.1, + pos: Some(t.0.pos), + })) } -named!(format_expression, +named!(format_expression( Span ) -> Expression, map_res!( do_parse!( - tmpl: ws!(quoted) >> - ws!(tag!("%")) >> - lparen >> - args: ws!(separated_list!(ws!(comma), expression)) >> - rparen >> + tmpl: ws!(strtok) >> + ws!(pcttok) >> + lparentok >> + args: ws!(separated_list!(ws!(commatok), expression)) >> + rparentok >> (tmpl, args) ), tuple_to_format @@ -375,7 +347,7 @@ named!(format_expression, fn tuple_to_call(t: (Value, Vec)) -> ParseResult { if let Value::Selector(sl) = t.0 { - Ok(Expression::Call(CallDef{ + Ok(Expression::Call(CallDef { macroref: sl.val, arglist: t.1, pos: None, @@ -389,20 +361,20 @@ fn vec_to_selector_value(v: SelectorList) -> ParseResult { Ok(Value::Selector(value_node!(v))) } -named!(selector_value, +named!(selector_value( Span ) -> Value, map_res!( ws!(selector_list), vec_to_selector_value ) ); -named!(call_expression, +named!(call_expression( Span ) -> Expression, map_res!( do_parse!( macroname: selector_value >> - lparen >> - args: ws!(separated_list!(ws!(comma), expression)) >> - rparen >> + lparentok >> + args: ws!(separated_list!(ws!(commatok), expression)) >> + rparentok >> (macroname, args) ), tuple_to_call @@ -419,7 +391,7 @@ named!(call_expression, // *IMPORTANT* // It also means this combinator is risky when used with partial // inputs. So handle with care. -named!(expression, +named!(expression( Span ) -> Expression, alt!( complete!(add_expression) | complete!(sub_expression) | @@ -439,54 +411,54 @@ fn expression_to_statement(v: Expression) -> ParseResult { Ok(Statement::Expression(v)) } -named!(expression_statement, +named!(expression_statement( Span ) -> Statement, map_res!( - terminated!(ws!(expression), semicolon), + terminated!(ws!(expression), semicolontok), expression_to_statement ) ); -fn tuple_to_let(t: (String, Expression)) -> ParseResult { +fn tuple_to_let(t: (Token, Expression)) -> ParseResult { Ok(Statement::Let { - name: t.0.to_string(), + name: t.0, value: t.1, }) } -named!(let_statement, +named!(let_statement( Span ) -> Statement, map_res!( terminated!(do_parse!( - let_word >> - name: ws!(field) >> - equal >> + lettok >> + name: ws!(barewordtok) >> + equaltok >> val: ws!(expression) >> (name, val) - ), semicolon), + ), semicolontok), tuple_to_let ) ); -fn tuple_to_import(t: (String, String)) -> ParseResult { +fn tuple_to_import(t: (Token, Token)) -> ParseResult { Ok(Statement::Import { - name: t.0.to_string(), - path: t.1.to_string(), + name: t.0, + path: t.1.fragment.to_string(), }) } -named!(import_statement, +named!(import_statement( Span ) -> Statement, map_res!( terminated!(do_parse!( - import_word >> - path: ws!(quoted) >> - as_word >> - name: ws!(field) >> + importtok >> + path: ws!(strtok) >> + astok >> + name: ws!(barewordtok) >> (name, path) - ), semicolon), + ), semicolontok), tuple_to_import ) ); -named!(statement, +named!(statement( Span ) -> Statement, alt_complete!( import_statement | let_statement | @@ -494,160 +466,303 @@ named!(statement, ) ); -named!(pub parse >, many1!(ws!(statement))); +named!(pub parse( Span ) -> Vec, many1!(ws!(statement))); // TODO(jwall): Full Statement parsing tests. #[cfg(test)] mod test { - use std::str::from_utf8; - use super::{Statement, Expression, Value, MacroDef, SelectDef, CallDef}; use super::{number, symbol, parse, field_value, tuple, grouped_expression}; use super::{copy_expression, macro_expression, select_expression}; use super::{format_expression, call_expression, expression}; use super::{expression_statement, let_statement, import_statement, statement}; use ast::*; + use nom_locate::LocatedSpan; use nom::IResult; #[test] fn test_statement_parse() { - assert_eq!(statement(&b"import \"foo\" as foo;"[..]), - IResult::Done(&b""[..], - Statement::Import{ - path: "foo".to_string(), - name: "foo".to_string() - } + let mut stmt = "import \"foo\" as foo;"; + let input = LocatedSpan::new(stmt); + assert_eq!(statement(input), + IResult::Done( + LocatedSpan{ + offset: stmt.len(), + line: 1, + fragment: "", + }, + Statement::Import{ + path: "foo".to_string(), + name: Token{ + fragment: "foo".to_string(), + pos: Position{ + line: 1, + column: 17, + }, + } + } ) - ); - assert!(statement(&b"import foo"[..]).is_err() ); + ); - assert_eq!(statement(&b"let foo = 1.0 ;"[..]), - IResult::Done(&b""[..], - Statement::Let{name: "foo".to_string(), - value: Expression::Simple(Value::Float(value_node!(1.0)))})); - assert_eq!(statement(&b"1.0;"[..]), - IResult::Done(&b""[..], - Statement::Expression( - Expression::Simple(Value::Float(value_node!(1.0)))))); + assert!(statement(LocatedSpan::new("import foo")).is_err() ); + + stmt = "let foo = 1.0 ;"; + let input = LocatedSpan::new(stmt); + assert_eq!(statement(input), + IResult::Done( + LocatedSpan{ + offset: stmt.len(), + line: 1, + fragment: "", + }, + Statement::Let{ + name: Token{ + fragment: "foo".to_string(), + pos: Position { + line: 1, + column: 5, + }, + }, + value: Expression::Simple(Value::Float(value_node!(1.0))) + })); + stmt = "1.0;"; + let input = LocatedSpan::new(stmt); + assert_eq!(statement(input), + IResult::Done( + LocatedSpan{ + offset: stmt.len(), + line: 1, + fragment: "", + }, + Statement::Expression( + Expression::Simple(Value::Float(value_node!(1.0)))))); } #[test] fn test_import_parse() { - assert!(import_statement(&b"import"[..]).is_incomplete()); - assert!(import_statement(&b"import \"foo\""[..]).is_incomplete()); - assert!(import_statement(&b"import \"foo\" as"[..]).is_incomplete()); - assert!(import_statement(&b"import \"foo\" as foo"[..]).is_incomplete()); + assert!(import_statement(LocatedSpan::new("import")).is_incomplete()); + assert!(import_statement(LocatedSpan::new("import \"foo\"")).is_incomplete()); + assert!(import_statement(LocatedSpan::new("import \"foo\" as")).is_incomplete()); + assert!(import_statement(LocatedSpan::new("import \"foo\" as foo")).is_incomplete()); - assert_eq!(import_statement(&b"import \"foo\" as foo;"[..]), - IResult::Done(&b""[..], - Statement::Import{ - path: "foo".to_string(), - name: "foo".to_string() - } + let import_stmt = "import \"foo\" as foo;"; + assert_eq!(import_statement(LocatedSpan::new(import_stmt)), + IResult::Done(LocatedSpan{ + fragment: "", + line: 1, + offset: import_stmt.len(), + }, + Statement::Import{ + path: "foo".to_string(), + name: Token{ + fragment: "foo".to_string(), + pos: Position{ + line: 1, + column: 17, + }, + } + } ) ); } #[test] fn test_let_statement_parse() { - assert!(let_statement(&b"foo"[..]).is_err() ); - assert!(let_statement(&b"let \"foo\""[..]).is_err() ); - assert!(let_statement(&b"let 1"[..]).is_err() ); - assert!(let_statement(&b"let"[..]).is_incomplete() ); - assert!(let_statement(&b"let foo"[..]).is_incomplete() ); - assert!(let_statement(&b"let foo ="[..]).is_incomplete() ); - assert!(let_statement(&b"let foo = "[..]).is_incomplete() ); - assert!(let_statement(&b"let foo = 1"[..]).is_incomplete() ); + assert!(let_statement(LocatedSpan::new("foo")).is_err() ); + assert!(let_statement(LocatedSpan::new("let \"foo\"")).is_err() ); + assert!(let_statement(LocatedSpan::new("let 1")).is_err() ); + assert!(let_statement(LocatedSpan::new("let")).is_incomplete() ); + assert!(let_statement(LocatedSpan::new("let foo")).is_incomplete() ); + assert!(let_statement(LocatedSpan::new("let foo =")).is_incomplete() ); + assert!(let_statement(LocatedSpan::new("let foo = ")).is_incomplete() ); + assert!(let_statement(LocatedSpan::new("let foo = 1")).is_incomplete() ); - assert_eq!(let_statement(&b"let foo = 1.0 ;"[..]), - IResult::Done(&b""[..], - Statement::Let{name: "foo".to_string(), - value: Expression::Simple(Value::Float(value_node!(1.0)))})); - assert_eq!(let_statement(&b"let foo= 1.0;"[..]), - IResult::Done(&b""[..], - Statement::Let{name: "foo".to_string(), - value: Expression::Simple(Value::Float(value_node!(1.0)))})); - assert_eq!(let_statement(&b"let foo =1.0;"[..]), - IResult::Done(&b""[..], - Statement::Let{name: "foo".to_string(), - value: Expression::Simple(Value::Float(value_node!(1.0)))})); + let mut let_stmt = "let foo = 1.0 ;"; + assert_eq!(let_statement(LocatedSpan::new(let_stmt)), + IResult::Done(LocatedSpan{ + fragment: "", + offset: let_stmt.len(), + line: 1, + }, + Statement::Let{name: Token{ + fragment: "foo".to_string(), + pos: Position{ + line: 1, + column: 5, + }, + }, + value: Expression::Simple(Value::Float(value_node!(1.0))) + })); + + let_stmt = "let foo= 1.0;"; + assert_eq!(let_statement(LocatedSpan::new(let_stmt)), + IResult::Done(LocatedSpan{ + fragment: "", + offset: let_stmt.len(), + line: 1, + }, + Statement::Let{name: Token{ + fragment: "foo".to_string(), + pos: Position{ + line: 1, + column: 5, + } + }, + value: Expression::Simple(Value::Float(value_node!(1.0)))})); + let_stmt = "let foo =1.0;"; + assert_eq!(let_statement(LocatedSpan::new(let_stmt)), + IResult::Done(LocatedSpan{ + fragment: "", + offset: let_stmt.len(), + line: 1, + }, + Statement::Let{name: Token{ + fragment: "foo".to_string(), + pos: Position{ + line: 1, + column: 5, + } + }, + value: Expression::Simple(Value::Float(value_node!(1.0)))})); } #[test] fn test_expression_statement_parse() { - assert!(expression_statement(&b"foo"[..]).is_incomplete() ); - assert_eq!(expression_statement(&b"1.0;"[..]), - IResult::Done(&b""[..], + assert!(expression_statement(LocatedSpan::new("foo")).is_incomplete() ); + assert_eq!(expression_statement(LocatedSpan::new("1.0;")), + IResult::Done(LocatedSpan{ + fragment: "", + offset: 4, + line: 1, + }, Statement::Expression( Expression::Simple(Value::Float(value_node!(1.0)))))); - assert_eq!(expression_statement(&b"1.0 ;"[..]), - IResult::Done(&b""[..], + assert_eq!(expression_statement(LocatedSpan::new("1.0 ;")), + IResult::Done(LocatedSpan { + fragment: "", + offset: 5, + line: 1, + }, Statement::Expression( Expression::Simple(Value::Float(value_node!(1.0)))))); - assert_eq!(expression_statement(&b" 1.0;"[..]), - IResult::Done(&b""[..], + assert_eq!(expression_statement(LocatedSpan::new(" 1.0;")), + IResult::Done(LocatedSpan { + fragment: "", + offset: 5, + line: 1, + }, Statement::Expression( Expression::Simple(Value::Float(value_node!(1.0)))))); - assert_eq!(expression_statement(&b"foo;"[..]), - IResult::Done(&b""[..], + assert_eq!(expression_statement(LocatedSpan::new("foo;")), + IResult::Done(LocatedSpan { + fragment: "", + offset: 4, + line: 1, + }, Statement::Expression( Expression::Simple(Value::Symbol(value_node!("foo".to_string())))))); - assert_eq!(expression_statement(&b"foo ;"[..]), - IResult::Done(&b""[..], + assert_eq!(expression_statement(LocatedSpan::new("foo ;")), + IResult::Done(LocatedSpan { + fragment: "", + offset: 5, + line: 1, + }, Statement::Expression( Expression::Simple(Value::Symbol(value_node!("foo".to_string())))))); - assert_eq!(expression_statement(&b" foo;"[..]), - IResult::Done(&b""[..], + assert_eq!(expression_statement(LocatedSpan::new(" foo;")), + IResult::Done(LocatedSpan { + fragment: "", + offset: 5, + line: 1, + }, Statement::Expression( Expression::Simple(Value::Symbol(value_node!("foo".to_string())))))); - assert_eq!(expression_statement(&b"\"foo\";"[..]), - IResult::Done(&b""[..], + assert_eq!(expression_statement(LocatedSpan::new("\"foo\";")), + IResult::Done(LocatedSpan { + fragment: "", + offset: 6, + line: 1, + }, Statement::Expression( Expression::Simple(Value::String(value_node!("foo".to_string())))))); - assert_eq!(expression_statement(&b"\"foo\" ;"[..]), - IResult::Done(&b""[..], + assert_eq!(expression_statement(LocatedSpan::new("\"foo\" ;")), + IResult::Done(LocatedSpan { + fragment: "", + offset: 7, + line: 1, + }, Statement::Expression( Expression::Simple(Value::String(value_node!("foo".to_string())))))); - assert_eq!(expression_statement(&b" \"foo\";"[..]), - IResult::Done(&b""[..], + assert_eq!(expression_statement(LocatedSpan::new(" \"foo\";")), + IResult::Done(LocatedSpan { + fragment: "", + offset: 7, + line: 1, + }, Statement::Expression( Expression::Simple(Value::String(value_node!("foo".to_string())))))); } #[test] fn test_expression_parse() { - assert_eq!(expression(&b"1"[..]), - IResult::Done(&b""[..], Expression::Simple(Value::Int(value_node!(1))))); - assert_eq!(expression(&b"foo"[..]), - IResult::Done(&b""[..], Expression::Simple(Value::Symbol(value_node!("foo".to_string()))))); - assert_eq!(expression(&b"1 + 1"[..]), - IResult::Done(&b""[..], + assert_eq!(expression(LocatedSpan::new("1")), + IResult::Done(LocatedSpan { + fragment: "", + offset: 1, + line: 1, + }, + Expression::Simple(Value::Int(value_node!(1))))); + assert_eq!(expression(LocatedSpan::new("foo")), + IResult::Done(LocatedSpan { + fragment: "", + offset: 3, + line: 1, + }, + Expression::Simple(Value::Symbol(value_node!("foo".to_string()))))); + assert_eq!(expression(LocatedSpan::new("1 + 1")), + IResult::Done(LocatedSpan { + fragment: "", + offset: 5, + line: 1, + }, Expression::Binary(BinaryOpDef{ kind: BinaryExprType::Add, left: Value::Int(value_node!(1)), right: Box::new(Expression::Simple(Value::Int(value_node!(1)))), pos: None, }))); - assert_eq!(expression(&b"1 - 1"[..]), - IResult::Done(&b""[..], + assert_eq!(expression(LocatedSpan::new("1 - 1")), + IResult::Done(LocatedSpan { + fragment: "", + offset: 5, + line: 1, + }, Expression::Binary(BinaryOpDef{ kind: BinaryExprType::Sub, left: Value::Int(value_node!(1)), right: Box::new(Expression::Simple(Value::Int(value_node!(1)))), pos: None, }))); - assert_eq!(expression(&b"1 * 1"[..]), - IResult::Done(&b""[..], + assert_eq!(expression(LocatedSpan::new("1 * 1")), + IResult::Done(LocatedSpan { + fragment: "", + offset: 5, + line: 1, + }, Expression::Binary(BinaryOpDef{ kind: BinaryExprType::Mul, left: Value::Int(value_node!(1)), right: Box::new(Expression::Simple(Value::Int(value_node!(1)))), pos: None, }))); - assert_eq!(expression(&b"1 / 1"[..]), - IResult::Done(&b""[..], + assert_eq!(expression(LocatedSpan::new("1 / 1")), + IResult::Done(LocatedSpan { + fragment: "", + offset: 5, + line: 1, + }, Expression::Binary(BinaryOpDef{ kind: BinaryExprType::Div, left: Value::Int(value_node!(1)), @@ -655,68 +770,102 @@ mod test { pos: None, }))); - assert_eq!(expression(&b"1+1"[..]), - IResult::Done(&b""[..], + assert_eq!(expression(LocatedSpan::new("1+1")), + IResult::Done(LocatedSpan { + fragment: "", + offset: 3, + line: 1, + }, Expression::Binary(BinaryOpDef{ kind: BinaryExprType::Add, left: Value::Int(value_node!(1)), right: Box::new(Expression::Simple(Value::Int(value_node!(1)))), pos: None, }))); - assert_eq!(expression(&b"1-1"[..]), - IResult::Done(&b""[..], + assert_eq!(expression(LocatedSpan::new("1-1")), + IResult::Done(LocatedSpan { + fragment: "", + offset: 3, + line: 1, + }, Expression::Binary(BinaryOpDef{ kind: BinaryExprType::Sub, left: Value::Int(value_node!(1)), right: Box::new(Expression::Simple(Value::Int(value_node!(1)))), pos: None, }))); - assert_eq!(expression(&b"1*1"[..]), - IResult::Done(&b""[..], + assert_eq!(expression(LocatedSpan::new("1*1")), + IResult::Done(LocatedSpan { + fragment: "", + offset: 3, + line: 1, + }, Expression::Binary(BinaryOpDef{ kind: BinaryExprType::Mul, left: Value::Int(value_node!(1)), right: Box::new(Expression::Simple(Value::Int(value_node!(1)))), pos: None, }))); - assert_eq!(expression(&b"1/1"[..]), - IResult::Done(&b""[..], + assert_eq!(expression(LocatedSpan::new("1/1")), + IResult::Done(LocatedSpan { + fragment: "", + offset: 3, + line: 1, + }, Expression::Binary(BinaryOpDef{ kind: BinaryExprType::Div, left: Value::Int(value_node!(1)), right: Box::new(Expression::Simple(Value::Int(value_node!(1)))), pos: None, }))); - assert_eq!(expression(&b"macro (arg1, arg2) => { foo = arg1 }"[..]), - IResult::Done(&b""[..], + let macro_expr = "macro (arg1, arg2) => { foo = arg1 }"; + assert_eq!(expression(LocatedSpan::new(macro_expr)), + IResult::Done(LocatedSpan { + fragment: "", + offset: macro_expr.len(), + line: 1, + }, Expression::Macro(MacroDef{ argdefs: vec![ - "arg1".to_string(), - "arg2".to_string(), + Positioned::new("arg1".to_string()), + Positioned::new("arg2".to_string()), ], fields: vec![ - ("foo".to_string(), Expression::Simple(Value::Symbol(value_node!("arg1".to_string())))), + (Token::new_with_pos("foo", Position{line: 1, column: 25}), + Expression::Simple(Value::Symbol(value_node!("arg1".to_string())))), ], pos: None, }) ) ); - assert_eq!(expression(&b"select foo, 1, { foo = 2 };"[..]), - IResult::Done(&b""[..], - Expression::Select(SelectDef{ - val: Box::new(Expression::Simple(Value::Symbol(value_node!("foo".to_string())))), - default: Box::new(Expression::Simple(Value::Int(value_node!(1)))), - tuple: vec![ - ("foo".to_string(), Expression::Simple(Value::Int(value_node!(2)))) - ], - pos: None, - }) - ) + let select_expr = "select foo, 1, { foo = 2 }"; + assert_eq!(expression(LocatedSpan::new(select_expr)), + IResult::Done(LocatedSpan { + fragment: "", + offset: select_expr.len(), + line: 1, + }, + Expression::Select(SelectDef{ + val: Box::new(Expression::Simple(Value::Symbol(value_node!("foo".to_string())))), + default: Box::new(Expression::Simple(Value::Int(value_node!(1)))), + tuple: vec![ + (Token::new_with_pos("foo", Position{line: 1, column: 18}), + Expression::Simple(Value::Int(value_node!(2)))) + ], + pos: None, + }) + ) ); - assert_eq!(expression(&b"foo.bar (1, \"foo\")"[..]), - IResult::Done(&b""[..], + let call_expr = "foo.bar (1, \"foo\")"; + assert_eq!(expression(LocatedSpan::new(call_expr)), + IResult::Done(LocatedSpan { + fragment: "", + offset: call_expr.len(), + line: 1, + }, Expression::Call(CallDef{ - macroref: vec!["foo".to_string(),"bar".to_string()], + macroref: vec![Token::new_with_pos("foo", Position{line:1,column: 1}), + Token::new_with_pos("bar", Position{line:1,column: 5})], arglist: vec![ Expression::Simple(Value::Int(value_node!(1))), Expression::Simple(Value::String(value_node!("foo".to_string()))), @@ -725,8 +874,12 @@ mod test { }) ) ); - assert_eq!(expression(&b"(1 + 1)"[..]), - IResult::Done(&b""[..], + assert_eq!(expression(LocatedSpan::new("(1 + 1)")), + IResult::Done(LocatedSpan { + fragment: "", + offset: 7, + line: 1, + }, Expression::Grouped( Box::new( Expression::Binary( @@ -745,84 +898,115 @@ mod test { #[test] fn test_format_parse() { - assert!(format_expression(&b"\"foo"[..]).is_err() ); - assert!(format_expression(&b"\"foo\""[..]).is_incomplete() ); - assert!(format_expression(&b"\"foo\" %"[..]).is_incomplete() ); - assert!(format_expression(&b"\"foo\" % (1, 2"[..]).is_incomplete() ); + assert!(format_expression(LocatedSpan::new("\"foo")).is_err() ); + assert!(format_expression(LocatedSpan::new("\"foo\"")).is_incomplete() ); + assert!(format_expression(LocatedSpan::new("\"foo\" %")).is_incomplete() ); + assert!(format_expression(LocatedSpan::new("\"foo\" % (1, 2")).is_incomplete() ); - assert_eq!(format_expression(&b"\"foo @ @\" % (1, 2)"[..]), - IResult::Done(&b""[..], - Expression::Format( - FormatDef{ - template: "foo @ @".to_string(), - args: vec![Expression::Simple(Value::Int(value_node!(1))), - Expression::Simple(Value::Int(value_node!(2)))], - pos: None, - }) + let mut fmt_expr = "\"foo @ @\" % (1, 2)"; + assert_eq!(format_expression(LocatedSpan::new(fmt_expr)), + IResult::Done(LocatedSpan{ + fragment: "", + offset: fmt_expr.len(), + line: 1 + }, + Expression::Format( + FormatDef{ + template: "foo @ @".to_string(), + args: vec![Expression::Simple(Value::Int(value_node!(1))), + Expression::Simple(Value::Int(value_node!(2)))], + pos: Some(Position{line: 1, column: 1}), + } + ) ) ); - assert_eq!(format_expression(&b"\"foo @ @\"%(1, 2)"[..]), - IResult::Done(&b""[..], - Expression::Format( - FormatDef{ - template: "foo @ @".to_string(), - args: vec![Expression::Simple(Value::Int(value_node!(1))), - Expression::Simple(Value::Int(value_node!(2)))], - pos: None, - }) - ) + + fmt_expr = "\"foo @ @\"%(1, 2)"; + assert_eq!(format_expression(LocatedSpan::new(fmt_expr)), + IResult::Done(LocatedSpan{ + fragment: "", + offset: fmt_expr.len(), + line: 1, + }, + Expression::Format( + FormatDef{ + template: "foo @ @".to_string(), + args: vec![Expression::Simple(Value::Int(value_node!(1))), + Expression::Simple(Value::Int(value_node!(2)))], + pos: Some(Position { line: 1, column: 1 }), + } + ) + ) ); } #[test] fn test_call_parse() { - assert!(call_expression(&b"foo"[..]).is_incomplete() ); - assert!(call_expression(&b"foo ("[..]).is_incomplete() ); - assert!(call_expression(&b"foo (1"[..]).is_incomplete() ); - assert!(call_expression(&b"foo (1,"[..]).is_incomplete() ); - assert!(call_expression(&b"foo (1,2"[..]).is_incomplete() ); + assert!(call_expression(LocatedSpan::new("foo")).is_incomplete() ); + assert!(call_expression(LocatedSpan::new("foo (")).is_incomplete() ); + assert!(call_expression(LocatedSpan::new("foo (1")).is_incomplete() ); + assert!(call_expression(LocatedSpan::new("foo (1,")).is_incomplete() ); + assert!(call_expression(LocatedSpan::new("foo (1,2")).is_incomplete() ); - assert_eq!(call_expression(&b"foo (1, \"foo\")"[..]), - IResult::Done(&b""[..], - Expression::Call(CallDef{ - macroref: vec!["foo".to_string()], - arglist: vec![ - Expression::Simple(Value::Int(value_node!(1))), - Expression::Simple(Value::String(value_node!("foo".to_string()))), - ], - pos: None, - }) + let mut copy_expr = "foo (1, \"foo\")"; + assert_eq!(call_expression(LocatedSpan::new(copy_expr)), + IResult::Done( + LocatedSpan{ + fragment: "", + line: 1, + offset: copy_expr.len(), + }, + Expression::Call(CallDef{ + macroref: vec![Token::new_with_pos("foo", Position{line:1, column: 1})], + arglist: vec![ + Expression::Simple(Value::Int(value_node!(1))), + Expression::Simple(Value::String(value_node!("foo".to_string()))), + ], + pos: None, + }) ) ); - assert_eq!(call_expression(&b"foo.bar (1, \"foo\")"[..]), - IResult::Done(&b""[..], - Expression::Call(CallDef{ - macroref: vec!["foo".to_string(),"bar".to_string()], - arglist: vec![ - Expression::Simple(Value::Int(value_node!(1))), - Expression::Simple(Value::String(value_node!("foo".to_string()))), - ], - pos: None, - }) + copy_expr = "foo.bar (1, \"foo\")"; + assert_eq!(call_expression(LocatedSpan::new(copy_expr)), + IResult::Done( + LocatedSpan{ + fragment: "", + line: 1, + offset: copy_expr.len(), + }, + Expression::Call(CallDef{ + macroref: vec![Token::new_with_pos("foo", Position{line: 1, column: 1}), + Token::new_with_pos("bar", Position{line: 1, column: 5})], + arglist: vec![ + Expression::Simple(Value::Int(value_node!(1))), + Expression::Simple(Value::String(value_node!("foo".to_string()))), + ], + pos: None, + }) ) ); } #[test] fn test_select_parse() { - assert!(select_expression(&b"select"[..]).is_incomplete()); - assert!(select_expression(&b"select foo"[..]).is_incomplete()); - assert!(select_expression(&b"select foo, 1"[..]).is_incomplete()); - assert!(select_expression(&b"select foo, 1, {"[..]).is_incomplete()); + assert!(select_expression(LocatedSpan::new("select")).is_incomplete()); + assert!(select_expression(LocatedSpan::new("select foo")).is_incomplete()); + assert!(select_expression(LocatedSpan::new("select foo, 1")).is_incomplete()); + assert!(select_expression(LocatedSpan::new("select foo, 1, {")).is_incomplete()); - assert_eq!(select_expression(&b"select foo, 1, { foo = 2 };"[..]), - IResult::Done(&b""[..], + let select_expr = "select foo, 1, { foo = 2 }"; + assert_eq!(select_expression(LocatedSpan::new(select_expr)), + IResult::Done(LocatedSpan { + fragment: "", + offset: select_expr.len(), + line: 1, + }, Expression::Select(SelectDef{ val: Box::new(Expression::Simple(Value::Symbol(value_node!("foo".to_string())))), default: Box::new(Expression::Simple(Value::Int(value_node!(1)))), tuple: vec![ - ("foo".to_string(), Expression::Simple(Value::Int(value_node!(2)))) + (Token::new_with_pos("foo", Position{line: 1, column: 18}), Expression::Simple(Value::Int(value_node!(2)))) ], pos: None, }) @@ -832,71 +1016,91 @@ mod test { #[test] fn test_macro_expression_parsing() { - assert!(macro_expression(&b"foo"[..]).is_err() ); - assert!(macro_expression(&b"macro \"foo\""[..]).is_err() ); - assert!(macro_expression(&b"macro 1"[..]).is_err() ); - assert!(macro_expression(&b"macro"[..]).is_incomplete() ); - assert!(macro_expression(&b"macro ("[..]).is_incomplete() ); - assert!(macro_expression(&b"macro (arg"[..]).is_incomplete() ); - assert!(macro_expression(&b"macro (arg, arg2"[..]).is_incomplete() ); - assert!(macro_expression(&b"macro (arg1, arg2) =>"[..]).is_incomplete() ); - assert!(macro_expression(&b"macro (arg1, arg2) => {"[..]).is_incomplete() ); - assert!(macro_expression(&b"macro (arg1, arg2) => { foo"[..]).is_incomplete() ); - assert!(macro_expression(&b"macro (arg1, arg2) => { foo ="[..]).is_incomplete() ); + assert!(macro_expression(LocatedSpan::new("foo")).is_err() ); + assert!(macro_expression(LocatedSpan::new("macro \"foo\"")).is_err() ); + assert!(macro_expression(LocatedSpan::new("macro 1")).is_err() ); + assert!(macro_expression(LocatedSpan::new("macro")).is_incomplete() ); + assert!(macro_expression(LocatedSpan::new("macro (")).is_incomplete() ); + assert!(macro_expression(LocatedSpan::new("macro (arg")).is_incomplete() ); + assert!(macro_expression(LocatedSpan::new("macro (arg, arg2")).is_incomplete() ); + assert!(macro_expression(LocatedSpan::new("macro (arg1, arg2) =>")).is_incomplete() ); + assert!(macro_expression(LocatedSpan::new("macro (arg1, arg2) => {")).is_incomplete() ); + assert!(macro_expression(LocatedSpan::new("macro (arg1, arg2) => { foo")).is_incomplete() ); + assert!(macro_expression(LocatedSpan::new("macro (arg1, arg2) => { foo =")).is_incomplete() ); - assert_eq!(macro_expression(&b"macro (arg1, arg2) => {foo=1,bar=2}"[..]), - IResult::Done(&b""[..], - Expression::Macro(MacroDef{ - argdefs: vec!["arg1".to_string(), - "arg2".to_string()], - fields: vec![("foo".to_string(), Expression::Simple(Value::Int(value_node!(1)))), - ("bar".to_string(), Expression::Simple(Value::Int(value_node!(2)))) - ], - pos: None, - }) + let macro_expr = "macro (arg1, arg2) => {foo=1,bar=2}"; + assert_eq!(macro_expression(LocatedSpan::new(macro_expr)), + IResult::Done( + LocatedSpan{ + fragment: "", + offset: macro_expr.len(), + line: 1 + }, + Expression::Macro(MacroDef{ + argdefs: vec![Positioned::new("arg1".to_string()), + Positioned::new("arg2".to_string())], + fields: vec![(Token::new_with_pos("foo", Position{line: 1, column: 24}), Expression::Simple(Value::Int(value_node!(1)))), + (Token::new_with_pos("bar", Position{line: 1, column: 30}), Expression::Simple(Value::Int(value_node!(2)))) + ], + pos: None, + }) ) ); } #[test] fn test_copy_parse() { - assert!(copy_expression(&b"{}"[..]).is_err() ); - assert!(copy_expression(&b"foo"[..]).is_incomplete() ); - assert!(copy_expression(&b"foo{"[..]).is_incomplete() ); - assert_eq!(copy_expression(&b"foo{}"[..]), - IResult::Done(&b""[..], - Expression::Copy(CopyDef{ - selector: vec!["foo".to_string()], - fields: Vec::new(), - pos: None, - }) + assert!(copy_expression(LocatedSpan::new("{}")).is_err() ); + assert!(copy_expression(LocatedSpan::new("foo")).is_incomplete() ); + assert!(copy_expression(LocatedSpan::new("foo{")).is_incomplete() ); + + let mut copy_expr = "foo{}"; + assert_eq!(copy_expression(LocatedSpan::new(copy_expr)), + IResult::Done( + LocatedSpan{ + fragment: "", + offset: copy_expr.len(), + line: 1 + }, + Expression::Copy(CopyDef{ + selector: vec![Token::new_with_pos("foo", Position{line: 1, column: 1})], + fields: Vec::new(), + pos: None, + }) ) - ); - assert_eq!(copy_expression(&b"foo{bar=1}"[..]), - IResult::Done(&b""[..], - Expression::Copy(CopyDef{ - selector: vec!["foo".to_string()], - fields: vec![("bar".to_string(), - Expression::Simple(Value::Int(value_node!(1))))], - pos: None, - }) - ) - ); + ); + + copy_expr = "foo{bar=1}"; + assert_eq!(copy_expression(LocatedSpan::new(copy_expr)), + IResult::Done( + LocatedSpan{ + fragment: "", + offset: copy_expr.len(), + line: 1 + }, + Expression::Copy(CopyDef{ + selector: vec![Token::new_with_pos("foo", Position{line: 1, column: 1})], + fields: vec![(Token::new_with_pos("bar", Position{line: 1, column: 5}), + Expression::Simple(Value::Int(value_node!(1))))], + pos: None, + }) + ) + ); } #[test] fn test_grouped_expression_parse() { - assert!(grouped_expression(&b"foo"[..]).is_err() ); - assert!(grouped_expression(&b"(foo"[..]).is_incomplete() ); - assert_eq!(grouped_expression(&b"(foo)"[..]), - IResult::Done(&b""[..], + assert!(grouped_expression(LocatedSpan::new("foo")).is_err() ); + assert!(grouped_expression(LocatedSpan::new("(foo")).is_incomplete() ); + assert_eq!(grouped_expression(LocatedSpan::new("(foo)")), + IResult::Done(LocatedSpan{fragment: "", offset: 5, line: 1}, Expression::Grouped( Box::new( Expression::Simple( Value::Symbol(value_node!("foo".to_string())))))) ); - assert_eq!(grouped_expression(&b"(1 + 1)"[..]), - IResult::Done(&b""[..], + assert_eq!(grouped_expression(LocatedSpan::new("(1 + 1)")), + IResult::Done(LocatedSpan{fragment: "", offset: 7, line: 1}, Expression::Grouped( Box::new( Expression::Binary( @@ -916,102 +1120,153 @@ mod test { #[test] fn test_tuple_parse() { - assert!(tuple(&b"{"[..]).is_incomplete() ); - assert!(tuple(&b"{ foo"[..]).is_incomplete() ); - assert!(tuple(&b"{ foo ="[..]).is_incomplete() ); - assert!(tuple(&b"{ foo = 1"[..]).is_incomplete() ); - assert!(tuple(&b"{ foo = 1,"[..]).is_incomplete() ); - assert!(tuple(&b"{ foo = 1, bar ="[..]).is_incomplete() ); + assert!(tuple(LocatedSpan::new("{")).is_incomplete() ); + assert!(tuple(LocatedSpan::new("{ foo")).is_incomplete() ); + assert!(tuple(LocatedSpan::new("{ foo =")).is_incomplete() ); + assert!(tuple(LocatedSpan::new("{ foo = 1")).is_incomplete() ); + assert!(tuple(LocatedSpan::new("{ foo = 1,")).is_incomplete() ); + assert!(tuple(LocatedSpan::new("{ foo = 1, bar =")).is_incomplete() ); - assert_eq!(tuple(&b"{ }"[..]), - IResult::Done(&b""[..], + let mut tuple_expr = "{ }"; + assert_eq!(tuple(LocatedSpan::new(tuple_expr)), + IResult::Done(LocatedSpan { + fragment: "", + offset: tuple_expr.len(), + line: 1, + }, Value::Tuple( value_node!(vec![])))); - assert_eq!(tuple(&b"{ foo = 1 }"[..]), - IResult::Done(&b""[..], + tuple_expr = "{ foo = 1 }"; + assert_eq!(tuple(LocatedSpan::new(tuple_expr)), + IResult::Done(LocatedSpan { + fragment: "", + offset: tuple_expr.len(), + line: 1, + }, Value::Tuple( value_node!(vec![ - ("foo".to_string(), Expression::Simple(Value::Int(value_node!(1)))) + (Token::new_with_pos("foo", Position{line:1, column: 3}), + Expression::Simple(Value::Int(value_node!(1)))) ])))); - assert_eq!(tuple(&b"{ foo = 1, bar = \"1\" }"[..]), - IResult::Done(&b""[..], + tuple_expr = "{ foo = 1, bar = \"1\" }"; + assert_eq!(tuple(LocatedSpan::new(tuple_expr)), + IResult::Done(LocatedSpan { + fragment: "", + offset: tuple_expr.len(), + line: 1, + }, Value::Tuple( value_node!(vec![ - ("foo".to_string(), Expression::Simple(Value::Int(value_node!(1)))), - ("bar".to_string(), Expression::Simple(Value::String(value_node!("1".to_string())))) + (Token::new_with_pos("foo", Position{line: 1, column: 3}), + Expression::Simple(Value::Int(value_node!(1)))), + (Token::new_with_pos("bar", Position{line: 1, column: 12}), + Expression::Simple(Value::String(value_node!("1".to_string())))) ])))); - assert_eq!(tuple(&b"{ foo = 1, bar = {} }"[..]), - IResult::Done(&b""[..], + tuple_expr = "{ foo = 1, bar = {} }"; + assert_eq!(tuple(LocatedSpan::new(tuple_expr)), + IResult::Done(LocatedSpan { + fragment: "", + offset: tuple_expr.len(), + line: 1, + }, Value::Tuple( value_node!(vec![ - ("foo".to_string(), Expression::Simple(Value::Int(value_node!(1)))), - ("bar".to_string(), Expression::Simple(Value::Tuple(value_node!(Vec::new())))) + (Token::new_with_pos("foo", Position{line: 1, column: 3}), + Expression::Simple(Value::Int(value_node!(1)))), + (Token::new_with_pos("bar", Position{line: 1, column: 12}), + Expression::Simple(Value::Tuple(value_node!(Vec::new())))) ])))); } #[test] fn test_field_value_parse() { - assert!(field_value(&b"foo"[..]).is_incomplete() ); - assert!(field_value(&b"foo ="[..]).is_incomplete() ); + assert!(field_value(LocatedSpan::new("foo")).is_incomplete() ); + assert!(field_value(LocatedSpan::new("foo =")).is_incomplete() ); - assert_eq!(field_value(&b"foo = 1"[..]), - IResult::Done(&b""[..], ("foo".to_string(), Expression::Simple(Value::Int(value_node!(1))))) ); - assert_eq!(field_value(&b"foo = \"1\""[..]), - IResult::Done(&b""[..], ("foo".to_string(), Expression::Simple(Value::String(value_node!("1".to_string()))))) ); - assert_eq!(field_value(&b"foo = bar"[..]), - IResult::Done(&b""[..], ("foo".to_string(), Expression::Simple(Value::Symbol(value_node!("bar".to_string()))))) ); - assert_eq!(field_value(&b"foo = bar "[..]), - IResult::Done(&b""[..], ("foo".to_string(), Expression::Simple(Value::Symbol(value_node!("bar".to_string()))))) ); + assert_eq!(field_value(LocatedSpan::new("foo = 1")), + IResult::Done(LocatedSpan { offset: 7, line: 1, fragment: "" }, + (Token::new_with_pos("foo", Position{line: 1, column: 1}), + Expression::Simple(Value::Int(value_node!(1))))) ); + assert_eq!(field_value(LocatedSpan::new("foo = \"1\"")), + IResult::Done(LocatedSpan { offset: 9, line: 1, fragment: "" }, + (Token::new_with_pos("foo", Position{line: 1, column: 1}), + Expression::Simple(Value::String(value_node!("1".to_string()))))) ); + assert_eq!(field_value(LocatedSpan::new("foo = bar")), + IResult::Done(LocatedSpan { offset: 9, line: 1, fragment: "" }, + (Token::new_with_pos("foo", Position{line: 1, column: 1}), + Expression::Simple(Value::Symbol(value_node!("bar".to_string()))))) ); + assert_eq!(field_value(LocatedSpan::new("foo = bar ")), + IResult::Done(LocatedSpan { offset: 10, line: 1, fragment: "" }, + (Token::new_with_pos("foo", Position{line: 1, column: 1}), + Expression::Simple(Value::Symbol(value_node!("bar".to_string()))))) ); } #[test] fn test_number_parsing() { - assert!(number(&b"."[..]).is_err() ); - assert!(number(&b". "[..]).is_err() ); - assert_eq!(number(&b"1.0"[..]), - IResult::Done(&b""[..], Value::Float(value_node!(1.0))) ); - assert_eq!(number(&b"1."[..]), - IResult::Done(&b""[..], Value::Float(value_node!(1.0))) ); - assert_eq!(number(&b"1"[..]), - IResult::Done(&b""[..], Value::Int(value_node!(1))) ); - assert_eq!(number(&b".1"[..]), - IResult::Done(&b""[..], Value::Float(value_node!(0.1))) ); + assert!(number(LocatedSpan::new(".")).is_err() ); + assert!(number(LocatedSpan::new(". ")).is_err() ); + assert_eq!(number(LocatedSpan::new("1.0")), + IResult::Done(LocatedSpan{fragment: "", offset: 3, line: 1}, + Value::Float(value_node!(1.0))) ); + assert_eq!(number(LocatedSpan::new("1.")), + IResult::Done(LocatedSpan{fragment: "", offset: 2, line: 1}, + Value::Float(value_node!(1.0))) ); + assert_eq!(number(LocatedSpan::new("1")), + IResult::Done(LocatedSpan{fragment: "", offset: 1, line: 1}, + Value::Int(value_node!(1))) ); + assert_eq!(number(LocatedSpan::new(".1")), + IResult::Done(LocatedSpan{fragment: "", offset: 2, line: 1}, + Value::Float(value_node!(0.1))) ); } #[test] fn test_symbol_parsing() { - assert_eq!(symbol(&b"foo"[..]), - IResult::Done(&b""[..], Value::Symbol(value_node!("foo".to_string()))) ); - assert_eq!(symbol(&b"foo-bar"[..]), - IResult::Done(&b""[..], Value::Symbol(value_node!("foo-bar".to_string()))) ); - assert_eq!(symbol(&b"foo_bar"[..]), - IResult::Done(&b""[..], Value::Symbol(value_node!("foo_bar".to_string()))) ); + assert_eq!(symbol(LocatedSpan::new("foo")), + IResult::Done(LocatedSpan{fragment: "", offset: 3, line: 1}, + Value::Symbol(value_node!("foo".to_string()))) ); + assert_eq!(symbol(LocatedSpan::new("foo-bar")), + IResult::Done(LocatedSpan{fragment: "", offset: 7, line: 1}, + Value::Symbol(value_node!("foo-bar".to_string()))) ); + assert_eq!(symbol(LocatedSpan::new("foo_bar")), + IResult::Done(LocatedSpan{fragment: "", offset: 7, line: 1}, + Value::Symbol(value_node!("foo_bar".to_string()))) ); } #[test] fn test_parse() { - let bad_input = &b"import mylib as lib;"[..]; + let bad_input = LocatedSpan::new("import mylib as lib;"); let bad_result = parse(bad_input); assert!(bad_result.is_err() ); // Valid parsing tree - let input = &b"import \"mylib\" as lib;\ - let foo = 1;\ - 1+1;"[..]; + let input = + LocatedSpan::new("import \"mylib\" as lib;let foo = 1;1+1;"); let result = parse(input); assert!(result.is_done() ); let tpl = result.unwrap(); - assert_eq!(from_utf8(tpl.0).unwrap(), ""); + assert_eq!(tpl.0.fragment, ""); assert_eq!(tpl.1, vec![ Statement::Import{ path: "mylib".to_string(), - name: "lib".to_string() + name: Token{ + fragment: "lib".to_string(), + pos: Position{ + line: 1, + column: 19, + } + } }, Statement::Let{ - name: "foo".to_string(), + name: Token{ + fragment: "foo".to_string(), + pos: Position{ + line: 1, + column: 27, + } + }, value: Expression::Simple(Value::Int(value_node!(1))) }, Statement::Expression( diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 50dc4cc..e3c0eb2 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -16,7 +16,7 @@ use nom::{alpha, is_alphanumeric, digit}; use ast::*; -type Span<'a> = LocatedSpan<&'a str>; +pub type Span<'a> = LocatedSpan<&'a str>; impl<'a> From> for Position { fn from(s: Span) -> Position { @@ -27,12 +27,6 @@ impl<'a> From> for Position { } } -pub struct Token<'a> { - pub pos: Position, - pub fragment: &'a str, -} - - fn is_symbol_char(c: char) -> bool { is_alphanumeric(c as u8) || c == '-' as char || c == '_' as char } @@ -45,7 +39,7 @@ named!(pub strtok( Span ) -> Token, tag!("\"") >> (Token{ pos: Position::from(span), - fragment: frag.fragment, + fragment: frag.fragment.to_string(), }) ) ); @@ -56,7 +50,7 @@ named!(pub barewordtok( Span ) -> Token, frag: preceded!(peek!(alpha), take_while!(is_symbol_char)) >> (Token{ pos: Position::from(span), - fragment: frag.fragment, + fragment: frag.fragment.to_string(), }) ) ); @@ -67,7 +61,7 @@ named!(pub digittok( Span ) -> Token, digits: digit >> (Token{ pos: Position::from(span), - fragment: digits.fragment, + fragment: digits.fragment.to_string(), }) ) ); @@ -85,7 +79,7 @@ macro_rules! do_tag_tok { frag: tag!($tag) >> (Token{ pos: Position::from(span), - fragment: frag.fragment, + fragment: frag.fragment.to_string(), }) ) } @@ -131,6 +125,10 @@ named!(pub slashtok( Span ) -> Token, do_tag_tok!("/") ); +named!(pub pcttok( Span ) -> Token, + do_tag_tok!("%") +); + named!(pub equaltok( Span ) -> Token, do_tag_tok!("=") );