From e4c80b19f5149fb7dba0a9b785e22ff5323b5470 Mon Sep 17 00:00:00 2001 From: Jeremy Wall Date: Sun, 4 Nov 2018 09:54:16 -0600 Subject: [PATCH] FEATURE: report better stacktraces for parsing. --- Cargo.lock | 6 +-- Cargo.toml | 2 +- example_errors/bad_file.ucg | 1 + examples/test_yaml.yaml | 9 ----- src/ast/mod.rs | 64 ++++++++++++++++++------------ src/build/ir.rs | 4 +- src/build/mod.rs | 44 ++++++++++----------- src/convert/env.rs | 8 +++- src/convert/exec.rs | 4 +- src/convert/json.rs | 2 +- src/convert/yaml.rs | 2 +- src/error.rs | 77 +++++++++++++++++++++++++------------ src/iter.rs | 6 +-- src/parse/mod.rs | 52 +++++++++++-------------- src/parse/test.rs | 12 +++--- src/tokenizer/mod.rs | 72 +++++++++++++++++----------------- src/tokenizer/test.rs | 6 +-- 17 files changed, 202 insertions(+), 169 deletions(-) create mode 100644 example_errors/bad_file.ucg delete mode 100644 examples/test_yaml.yaml diff --git a/Cargo.lock b/Cargo.lock index 00c3f5e..0e8db69 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,6 +1,6 @@ [[package]] name = "abortable_parser" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -228,7 +228,7 @@ dependencies = [ name = "ucg" version = "0.2.0" dependencies = [ - "abortable_parser 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "abortable_parser 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "bencher 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", "clap 2.26.2 (registry+https://github.com/rust-lang/crates.io-index)", "cpuprofiler 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)", @@ -285,7 +285,7 @@ dependencies = [ ] [metadata] -"checksum abortable_parser 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bd280856ce341823c6aa6fddb3bafae236c23223824f47aef3443deb0b8d900c" +"checksum abortable_parser 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "09cdf5378b5e4a079fa886e621519fcb2502d9cb008d3f76b92f61f3890d5906" "checksum ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "23ac7c30002a5accbf7e8987d0632fa6de155b7c3d39d0067317a391e00a2ef6" "checksum atty 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "9a7d5b8723950951411ee34d271d99dddcc2035a16ab25310ea2c8cfd4369652" "checksum backtrace 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "346d7644f0b5f9bc73082d3b2236b69a05fd35cce0cfa3724e184e6a5c9e2a2f" diff --git a/Cargo.toml b/Cargo.toml index 81f5713..bf63b0f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ keywords = ["compiler", "config"] license = "Apache-2.0" [dependencies] -abortable_parser = "~0.2.0" +abortable_parser = "0.2.1" clap = "~2.26.0" serde_json = "~1.0.9" simple-error = "0.1" diff --git a/example_errors/bad_file.ucg b/example_errors/bad_file.ucg new file mode 100644 index 0000000..8329071 --- /dev/null +++ b/example_errors/bad_file.ucg @@ -0,0 +1 @@ +let x = \ No newline at end of file diff --git a/examples/test_yaml.yaml b/examples/test_yaml.yaml deleted file mode 100644 index 9ba075a..0000000 --- a/examples/test_yaml.yaml +++ /dev/null @@ -1,9 +0,0 @@ ---- -db_conn1: "db1.prod.net:3306/testdb" -db_conn2: "db2.prod.net:3306/testdb" -tmpldir: "./templates" -prefix: - foo: bar -l: - - foo - - bar \ No newline at end of file diff --git a/src/ast/mod.rs b/src/ast/mod.rs index 320ea81..3b370d2 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -25,6 +25,8 @@ use std::fmt; use std::hash::Hash; use std::hash::Hasher; +use abortable_parser; + macro_rules! enum_type_equality { ( $slf:ident, $r:expr, $( $l:pat ),* ) => { match $slf { @@ -110,6 +112,15 @@ impl Token { } } +impl abortable_parser::Positioned for Token { + fn line(&self) -> usize { + self.pos.line + } + fn column(&self) -> usize { + self.pos.column + } +} + impl Borrow for Token { fn borrow(&self) -> &str { &self.fragment @@ -119,7 +130,7 @@ impl Borrow for Token { /// Helper macro for making a Positioned Value. macro_rules! value_node { ($v:expr, $p:expr) => { - Positioned::new_with_pos($v, $p) + PositionedItem::new_with_pos($v, $p) }; } @@ -159,11 +170,14 @@ macro_rules! make_tok { #[allow(unused_macros)] macro_rules! make_expr { ($e:expr, $i:expr) => { - Expression::Simple(Value::Symbol(Positioned::new_with_pos($e.to_string(), $i))) + Expression::Simple(Value::Symbol(PositionedItem::new_with_pos( + $e.to_string(), + $i, + ))) }; ($e:expr => int, $i:expr) => { - Expression::Simple(Value::Int(Positioned::new_with_pos($e, $i))) + Expression::Simple(Value::Int(PositionedItem::new_with_pos($e, $i))) }; } @@ -323,13 +337,13 @@ impl SelectorDef { pub enum Value { // Constant Values Empty(Position), - Boolean(Positioned), - Int(Positioned), - Float(Positioned), - Str(Positioned), - Symbol(Positioned), + Boolean(PositionedItem), + Int(PositionedItem), + Float(PositionedItem), + Str(PositionedItem), + Symbol(PositionedItem), // Complex Values - Tuple(Positioned), + Tuple(PositionedItem), List(ListDef), Selector(SelectorDef), } @@ -434,18 +448,18 @@ pub struct SelectDef { /// Adds position information to any type `T`. #[derive(Debug, Clone)] -pub struct Positioned { +pub struct PositionedItem { pub pos: Position, pub val: T, } -impl std::fmt::Display for Positioned { +impl std::fmt::Display for PositionedItem { fn fmt(&self, f: &mut std::fmt::Formatter) -> Result<(), std::fmt::Error> { write!(f, "{}", self.val) } } -impl Positioned { +impl PositionedItem { /// Constructs a new Positioned with a value, line, and column information. pub fn new>(v: T, p: P) -> Self { Self::new_with_pos(v, p.into()) @@ -453,48 +467,48 @@ impl Positioned { /// Constructs a new Positioned with a value and a Position. pub fn new_with_pos(v: T, pos: Position) -> Self { - Positioned { pos: pos, val: v } + PositionedItem { pos: pos, val: v } } } -impl PartialEq for Positioned { +impl PartialEq for PositionedItem { fn eq(&self, other: &Self) -> bool { self.val == other.val } } -impl Eq for Positioned {} +impl Eq for PositionedItem {} -impl Ord for Positioned { +impl Ord for PositionedItem { fn cmp(&self, other: &Self) -> Ordering { self.val.cmp(&other.val) } } -impl PartialOrd for Positioned { +impl PartialOrd for PositionedItem { fn partial_cmp(&self, other: &Self) -> Option { self.val.partial_cmp(&other.val) } } -impl Hash for Positioned { +impl Hash for PositionedItem { fn hash(&self, state: &mut H) { self.val.hash(state); } } -impl<'a> From<&'a Token> for Positioned { - fn from(t: &'a Token) -> Positioned { - Positioned { +impl<'a> From<&'a Token> for PositionedItem { + fn from(t: &'a Token) -> PositionedItem { + PositionedItem { pos: t.pos.clone(), val: t.fragment.to_string(), } } } -impl<'a> From<&'a Positioned> for Positioned { - fn from(t: &Positioned) -> Positioned { - Positioned { +impl<'a> From<&'a PositionedItem> for PositionedItem { + fn from(t: &PositionedItem) -> PositionedItem { + PositionedItem { pos: t.pos.clone(), val: t.val.clone(), } @@ -508,7 +522,7 @@ impl<'a> From<&'a Positioned> for Positioned { /// any values except what is defined in their arguments. #[derive(PartialEq, Debug, Clone)] pub struct MacroDef { - pub argdefs: Vec>, + pub argdefs: Vec>, pub fields: FieldList, pub pos: Position, } diff --git a/src/build/ir.rs b/src/build/ir.rs index 736424d..9d34748 100644 --- a/src/build/ir.rs +++ b/src/build/ir.rs @@ -18,7 +18,7 @@ pub enum Val { Float(f64), Str(String), List(Vec>), - Tuple(Vec<(Positioned, Rc)>), + Tuple(Vec<(PositionedItem, Rc)>), Macro(MacroDef), } @@ -114,7 +114,7 @@ impl Val { } /// Returns the fields if this Val is a tuple. None otherwise. - pub fn get_fields(&self) -> Option<&Vec<(Positioned, Rc)>> { + pub fn get_fields(&self) -> Option<&Vec<(PositionedItem, Rc)>> { if let &Val::Tuple(ref fs) = self { Some(fs) } else { diff --git a/src/build/mod.rs b/src/build/mod.rs index 9cfa178..19d54eb 100644 --- a/src/build/mod.rs +++ b/src/build/mod.rs @@ -44,7 +44,7 @@ impl MacroDef { cache: Rc>, env: Rc, mut args: Vec>, - ) -> Result, Rc)>, Box> { + ) -> Result, Rc)>, Box> { // Error conditions. If the args don't match the length and types of the argdefs then this is // macro call error. if args.len() > self.argdefs.len() { @@ -61,12 +61,12 @@ impl MacroDef { // If the expressions reference Symbols not defined in the MacroDef that is also an error. // TODO(jwall): We should probably enforce that the Expression Symbols must be in argdefs rules // at Macro definition time not evaluation time. - let mut scope = HashMap::, Rc>::new(); + let mut scope = HashMap::, Rc>::new(); for (i, arg) in args.drain(0..).enumerate() { scope.entry(self.argdefs[i].clone()).or_insert(arg.clone()); } let b = Builder::new_with_env_and_scope(root, cache, scope, env); - let mut result: Vec<(Positioned, Rc)> = Vec::new(); + let mut result: Vec<(PositionedItem, Rc)> = Vec::new(); for &(ref key, ref expr) in self.fields.iter() { // We clone the expressions here because this macro may be consumed // multiple times in the future. @@ -81,7 +81,7 @@ impl MacroDef { type BuildResult = Result<(), Box>; /// Defines a set of values in a parsed file. -type ValueMap = HashMap, Rc>; +type ValueMap = HashMap, Rc>; /// AssertCollector collects the results of assertions in the UCG AST. pub struct AssertCollector { @@ -136,7 +136,7 @@ macro_rules! eval_binary_expr { impl<'a> Builder<'a> { // TOOD(jwall): This needs some unit tests. fn tuple_to_val(&self, fields: &Vec<(Token, Expression)>) -> Result, Box> { - let mut new_fields = Vec::<(Positioned, Rc)>::new(); + let mut new_fields = Vec::<(PositionedItem, Rc)>::new(); for &(ref name, ref expr) in fields.iter() { let val = try!(self.eval_expr(expr)); new_fields.push((name.into(), val)); @@ -190,10 +190,10 @@ impl<'a> Builder<'a> { cache: Rc>, scope: ValueMap, ) -> Self { - let env_vars: Vec<(Positioned, Rc)> = env::vars() + let env_vars: Vec<(PositionedItem, Rc)> = env::vars() .map(|t| { ( - Positioned::new(t.0, Position::new(0, 0, 0)), + PositionedItem::new(t.0, Position::new(0, 0, 0)), Rc::new(t.1.into()), ) }).collect(); @@ -225,7 +225,7 @@ impl<'a> Builder<'a> { /// Returns a Val by name from previously built UCG. pub fn get_out_by_name(&self, name: &str) -> Option> { - let key = Positioned { + let key = PositionedItem { pos: Position::new(0, 0, 0), val: name.to_string(), }; @@ -249,8 +249,7 @@ impl<'a> Builder<'a> { } fn eval_span(&mut self, input: OffsetStrIter) -> Result, Box> { - // TODO(jwall): This should really return a better error. - match parse(input) { + match parse(input.clone()) { Ok(stmts) => { //panic!("Successfully parsed {}", input); let mut out: Option> = None; @@ -262,13 +261,10 @@ impl<'a> Builder<'a> { Some(val) => Ok(val), } } - Err(err) => Err(Box::new(error::Error::new_with_cause( - format!( - "Error while parsing file: {}", - self.curr_file.unwrap_or("") - ), + Err(err) => Err(Box::new(error::Error::new( + format!("{}", err,), error::ErrorType::ParseError, - err, + (&input).into(), ))), } } @@ -301,7 +297,8 @@ impl<'a> Builder<'a> { let mut b = Self::new(normalized.clone(), self.assets.clone()); let filepath = normalized.to_str().unwrap().clone(); try!(b.build_file(filepath)); - let fields: Vec<(Positioned, Rc)> = b.build_output.drain().collect(); + let fields: Vec<(PositionedItem, Rc)> = + b.build_output.drain().collect(); Rc::new(Val::Tuple(fields)) } }; @@ -366,7 +363,7 @@ impl<'a> Builder<'a> { } } - fn lookup_sym(&self, sym: &Positioned) -> Option> { + fn lookup_sym(&self, sym: &PositionedItem) -> Option> { if &sym.val == "env" { return Some(self.env.clone()); } @@ -376,7 +373,10 @@ impl<'a> Builder<'a> { None } - fn find_in_fieldlist(target: &str, fs: &Vec<(Positioned, Rc)>) -> Option> { + fn find_in_fieldlist( + target: &str, + fs: &Vec<(PositionedItem, Rc)>, + ) -> Option> { for (key, val) in fs.iter().cloned() { if target == &key.val { return Some(val.clone()); @@ -390,7 +390,7 @@ impl<'a> Builder<'a> { stack: &mut VecDeque>, sl: &SelectorList, next: (&Position, &str), - fs: &Vec<(Positioned, Rc)>, + fs: &Vec<(PositionedItem, Rc)>, ) -> Result<(), Box> { if let Some(vv) = Self::find_in_fieldlist(next.1, fs) { stack.push_back(vv.clone()); @@ -781,7 +781,7 @@ impl<'a> Builder<'a> { fn eval_copy(&self, def: &CopyDef) -> Result, Box> { let v = try!(self.lookup_selector(&def.selector.sel)); if let Val::Tuple(ref src_fields) = *v { - let mut m = HashMap::, (i32, Rc)>::new(); + let mut m = HashMap::, (i32, Rc)>::new(); // loop through fields and build up a hashmap let mut count = 0; for &(ref key, ref val) in src_fields.iter() { @@ -830,7 +830,7 @@ impl<'a> Builder<'a> { } }; } - let mut new_fields: Vec<(Positioned, (i32, Rc))> = m.drain().collect(); + let mut new_fields: Vec<(PositionedItem, (i32, Rc))> = m.drain().collect(); // We want to maintain our order for the fields to make comparing tuples // easier in later code. So we sort by the field order before constructing a new tuple. new_fields.sort_by(|a, b| { diff --git a/src/convert/env.rs b/src/convert/env.rs index b4eda68..bd25792 100644 --- a/src/convert/env.rs +++ b/src/convert/env.rs @@ -16,7 +16,7 @@ use std::io::Write; use std::rc::Rc; -use ast::Positioned; +use ast::PositionedItem; use build::Val; use convert::traits::{Converter, Result}; @@ -29,7 +29,11 @@ impl EnvConverter { EnvConverter {} } - fn convert_tuple(&self, flds: &Vec<(Positioned, Rc)>, w: &mut Write) -> Result { + fn convert_tuple( + &self, + flds: &Vec<(PositionedItem, Rc)>, + w: &mut Write, + ) -> Result { for &(ref name, ref val) in flds.iter() { if val.is_tuple() { eprintln!("Skipping embedded tuple..."); diff --git a/src/convert/exec.rs b/src/convert/exec.rs index cbd9c84..9c7efbc 100644 --- a/src/convert/exec.rs +++ b/src/convert/exec.rs @@ -17,7 +17,7 @@ use std; use std::io::{Cursor, Write}; use std::rc::Rc; -use ast::{Position, Positioned}; +use ast::{Position, PositionedItem}; use build::Val; use build::Val::Tuple; use convert; @@ -49,7 +49,7 @@ impl ExecConverter { Position::new(0, 0, 0), ))); } - let mut env: Option<&Vec<(Positioned, Rc)>> = None; + let mut env: Option<&Vec<(PositionedItem, Rc)>> = None; let mut command: Option<&str> = None; let mut args: Option<&Vec>> = None; for &(ref name, ref val) in fields.iter() { diff --git a/src/convert/json.rs b/src/convert/json.rs index efa8dbb..8e50c24 100644 --- a/src/convert/json.rs +++ b/src/convert/json.rs @@ -36,7 +36,7 @@ impl JsonConverter { fn convert_tuple( &self, - items: &Vec<(ast::Positioned, Rc)>, + items: &Vec<(ast::PositionedItem, Rc)>, ) -> std::io::Result { let mut mp = serde_json::Map::new(); for &(ref k, ref v) in items.iter() { diff --git a/src/convert/yaml.rs b/src/convert/yaml.rs index 28ca3bf..a98927b 100644 --- a/src/convert/yaml.rs +++ b/src/convert/yaml.rs @@ -25,7 +25,7 @@ impl YamlConverter { fn convert_tuple( &self, - items: &Vec<(ast::Positioned, Rc)>, + items: &Vec<(ast::PositionedItem, Rc)>, ) -> std::io::Result { let mut mapping = serde_yaml::Mapping::new(); for &(ref k, ref v) in items.iter() { diff --git a/src/error.rs b/src/error.rs index b607b0b..e50e012 100644 --- a/src/error.rs +++ b/src/error.rs @@ -15,6 +15,9 @@ //! Errors for use by the ucg compiler. use std::error; use std::fmt; +use std::fmt::Debug; + +use abortable_parser::Positioned; use ast::*; @@ -60,7 +63,6 @@ pub struct Error { pub err_type: ErrorType, pub pos: Position, pub msg: String, - pub cause: Option>, _pkgonly: (), } @@ -70,41 +72,21 @@ impl Error { err_type: t, pos: pos, msg: msg.into(), - cause: None, _pkgonly: (), } } - pub fn new_with_boxed_cause>( - msg: S, - t: ErrorType, - cause: Box, - pos: Position, - ) -> Self { - let mut e = Self::new(msg, t, pos); - e.cause = Some(cause); - return e; - } - - pub fn new_with_cause>(msg: S, t: ErrorType, cause: Self) -> Self { - let pos = cause.pos.clone(); - Self::new_with_boxed_cause(msg, t, Box::new(cause), pos) - } - fn render(&self, w: &mut fmt::Formatter) -> fmt::Result { try!(write!( w, - "{}: \"{}\" at line: {} column: {}", - self.err_type, self.msg, self.pos.line, self.pos.column + "{} at line: {} column: {}\nCaused By:\n\t{} ", + self.err_type, self.pos.line, self.pos.column, self.msg )); - if let Some(ref cause) = self.cause { - try!(write!(w, "\n\tCaused By: {}", cause)); - } Ok(()) } } -impl fmt::Debug for Error { +impl Debug for Error { fn fmt(&self, w: &mut fmt::Formatter) -> fmt::Result { self.render(w) } @@ -121,3 +103,50 @@ impl error::Error for Error { &self.msg } } + +#[derive(Debug)] +pub struct StackPrinter { + pub err: abortable_parser::Error, +} + +impl StackPrinter +where + C: abortable_parser::Positioned, +{ + pub fn render(&self, w: &mut fmt::Formatter) -> fmt::Result { + let mut curr_err = Some(&self.err); + let mut tabstop = ""; + loop { + match curr_err { + // our exit condition; + None => break, + Some(err) => { + let context = err.get_context(); + try!(write!( + w, + "{}{}: line: {}, column: {}\n", + tabstop, + err.get_msg(), + context.line(), + context.column(), + )); + tabstop = "\t"; + curr_err = err.get_cause(); + if curr_err.is_some() { + try!(write!(w, "Caused by: \n")); + } + } + } + } + Ok(()) + } +} + +impl fmt::Display for StackPrinter +where + C: Positioned, +{ + fn fmt(&self, w: &mut fmt::Formatter) -> fmt::Result { + self.render(w) + } +} diff --git a/src/iter.rs b/src/iter.rs index 100d168..d37eb5a 100644 --- a/src/iter.rs +++ b/src/iter.rs @@ -2,9 +2,7 @@ use std::convert::From; use std::iter::Iterator; use abortable_parser::iter::{SliceIter, StrIter}; -use abortable_parser::{ - InputIter, Offsetable, Peekable, Seekable, Span, SpanRange, TextPositionTracker, -}; +use abortable_parser::{InputIter, Offsetable, Peekable, Positioned, Seekable, Span, SpanRange}; use ast::{Position, Token}; @@ -81,7 +79,7 @@ impl<'a> Peekable<&'a u8> for OffsetStrIter<'a> { } } -impl<'a> TextPositionTracker for OffsetStrIter<'a> { +impl<'a> Positioned for OffsetStrIter<'a> { fn line(&self) -> usize { self.contained.line() + self.line_offset } diff --git a/src/parse/mod.rs b/src/parse/mod.rs index 893711b..a65bf13 100644 --- a/src/parse/mod.rs +++ b/src/parse/mod.rs @@ -24,7 +24,7 @@ use abortable_parser::{Error, Peekable, Result}; use self::precedence::op_expression; use ast::*; -use error; +use error::StackPrinter; use iter::OffsetStrIter; use tokenizer::*; @@ -291,7 +291,7 @@ make_fn!( boolean_value, Value>, do_each!( b => match_type!(BOOLEAN), - (Value::Boolean(Positioned{ + (Value::Boolean(PositionedItem{ val: b.fragment == "true", pos: b.pos, })) @@ -535,7 +535,7 @@ fn tuple_to_macro<'a>( }; let arglist = default_args .drain(0..) - .map(|s| Positioned { + .map(|s| PositionedItem { pos: s.pos().clone(), val: s.to_string(), }).collect(); @@ -922,11 +922,11 @@ fn statement(i: SliceIter) -> Result, Statement> { //trace_macros!(false); /// Parses a LocatedSpan into a list of Statements or an `error::Error`. -pub fn parse(input: OffsetStrIter) -> std::result::Result, error::Error> { - match tokenize(&input) { +pub fn parse<'a>(input: OffsetStrIter<'a>) -> std::result::Result, String> { + match tokenize(input.clone()) { Ok(tokenized) => { let mut out = Vec::new(); - let mut i_ = SliceIter::from(&tokenized); + let mut i_ = SliceIter::new(&tokenized); loop { let i = i_.clone(); if let Some(tok) = i.peek_next() { @@ -936,31 +936,30 @@ pub fn parse(input: OffsetStrIter) -> std::result::Result, error: } match statement(i.clone()) { Result::Abort(e) => { - let pos: Position = (&i).into(); - let err = error::Error::new( - format!("Statement Parse Error {}", e), - error::ErrorType::ParseError, - pos, + let err = abortable_parser::Error::caused_by( + "Statement Parse Error", + Box::new(e), + Box::new(i.clone()), ); - return Err(err); + let ctx_err = StackPrinter { err: err }; + return Err(format!("{}", ctx_err)); } Result::Fail(e) => { - let pos: Position = (&i).into(); - let err = error::Error::new( - format!("Statement Parse Error {}", e), - error::ErrorType::ParseError, - pos, + let err = abortable_parser::Error::caused_by( + "Statement Parse Error", + Box::new(e), + Box::new(i.clone()), ); - return Err(err); + let ctx_err = StackPrinter { err: err }; + return Err(format!("{}", ctx_err)); } Result::Incomplete(_ei) => { - let pos: Position = (&i).into(); - let err = error::Error::new( + let err = abortable_parser::Error::new( "Unexpected end of parse input", - error::ErrorType::IncompleteParsing, - pos, + Box::new(i.clone()), ); - return Err(err); + let ctx_err = StackPrinter { err: err }; + return Err(format!("{}", ctx_err)); } Result::Complete(rest, stmt) => { out.push(stmt); @@ -974,12 +973,7 @@ pub fn parse(input: OffsetStrIter) -> std::result::Result, error: return Ok(out); } Err(e) => { - let err = error::Error::new_with_cause( - "Tokenization Error", - error::ErrorType::UnexpectedToken, - e, - ); - return Err(err); + return Err(e); } } } diff --git a/src/parse/test.rs b/src/parse/test.rs index 93e2dc5..ba20a30 100644 --- a/src/parse/test.rs +++ b/src/parse/test.rs @@ -24,7 +24,7 @@ macro_rules! assert_parse { }; ($i:expr, $f:expr, $out:expr) => {{ let input = OffsetStrIter::new($i); - match tokenize(&input) { + match tokenize(input.clone()) { Err(e) => assert!(false, format!("Tokenizer Error: {:?}", e)), Ok(val) => match $f(SliceIter::new(val.as_slice())) { Result::Complete(_, result) => assert_eq!(result, $out), @@ -40,7 +40,7 @@ macro_rules! assert_fail { }; ($i:expr, $f:expr) => {{ let input = OffsetStrIter::new($i); - match tokenize(&input) { + match tokenize(input.clone()) { Err(_) => assert!(true), Ok(val) => { let result = $f(SliceIter::new(val.as_slice())); @@ -56,7 +56,7 @@ macro_rules! assert_abort { }; ($i:expr, $f:expr) => {{ let input = OffsetStrIter::new($i); - match tokenize(&input) { + match tokenize(input.clone()) { Err(_) => assert!(true), Ok(val) => { let result = $f(SliceIter::new(val.as_slice())); @@ -84,11 +84,11 @@ fn test_null_parsing() { fn test_boolean_parsing() { assert_parse!( boolean_value("true"), - Value::Boolean(Positioned::new(true, Position::new(1, 1, 0))) + Value::Boolean(PositionedItem::new(true, Position::new(1, 1, 0))) ); assert_parse!( boolean_value("false"), - Value::Boolean(Positioned::new(false, Position::new(1, 1, 0))) + Value::Boolean(PositionedItem::new(false, Position::new(1, 1, 0))) ); assert_fail!(boolean_value("truth")); } @@ -141,7 +141,7 @@ fn test_selector_parsing() { Value::Selector( make_selector!(Expression::Grouped(Box::new(Expression::Simple( Value::Tuple(value_node!( - vec![(make_tok!("foo", Position::new(1, 3, 2)), Expression::Simple(Value::Int(Positioned::new(1, Position::new(1, 7, 6)))))], + vec![(make_tok!("foo", Position::new(1, 3, 2)), Expression::Simple(Value::Int(PositionedItem::new(1, Position::new(1, 7, 6)))))], Position::new(1, 3, 3))) ))) => [ make_tok!("foo", Position::new(1, 11, 10)) ] => Position::new(1, 2, 1)) ) diff --git a/src/tokenizer/mod.rs b/src/tokenizer/mod.rs index 5f47d8d..ec884ff 100644 --- a/src/tokenizer/mod.rs +++ b/src/tokenizer/mod.rs @@ -20,7 +20,7 @@ use abortable_parser::iter::SliceIter; use abortable_parser::{Error, Offsetable, Result}; use ast::*; -use error; +use error::StackPrinter; use iter::OffsetStrIter; fn is_symbol_char<'a>(i: OffsetStrIter<'a>) -> Result, u8> { @@ -144,27 +144,27 @@ make_fn!(booleantok, macro_rules! do_text_token_tok { ($i:expr, $type:expr, $text_token:expr, WS) => { do_each!($i, - span => input!(), - frag => text_token!($text_token), - _ => either!(whitespace, comment), - (Token { - typ: $type, - pos: Position::from(&span), - fragment: frag.to_string(), - }) - ) + span => input!(), + frag => text_token!($text_token), + _ => either!(whitespace, comment), + (Token { + typ: $type, + pos: Position::from(&span), + fragment: frag.to_string(), + }) + ) }; ($i:expr, $type:expr, $text_token:expr) => { do_each!($i, - span => input!(), - frag => text_token!($text_token), - (Token { - typ: $type, - pos: Position::from(&span), - fragment: frag.to_string(), - }) - ) + span => input!(), + frag => text_token!($text_token), + (Token { + typ: $type, + pos: Position::from(&span), + fragment: frag.to_string(), + }) + ) }; } @@ -397,35 +397,37 @@ fn token<'a>(input: OffsetStrIter<'a>) -> Result, Token> { } /// Consumes an input OffsetStrIter and returns either a Vec or a error::Error. -pub fn tokenize(input: &OffsetStrIter) -> std::result::Result, error::Error> { +pub fn tokenize<'a>(input: OffsetStrIter<'a>) -> std::result::Result, String> { let mut out = Vec::new(); let mut i = input.clone(); loop { if let Result::Complete(_, _) = eoi(i.clone()) { break; } - let pos: Position = Position::from(&i); match token(i.clone()) { Result::Abort(e) => { - return Err(error::Error::new( - format!("Invalid Token encountered {}", e), - error::ErrorType::UnexpectedToken, - pos, - )) + let err = abortable_parser::Error::caused_by( + "Invalid Token encountered", + Box::new(e), + Box::new(i.clone()), + ); + let ctx_err = StackPrinter { err: err }; + return Err(format!("{}", ctx_err)); } Result::Fail(e) => { - return Err(error::Error::new( - format!("Invalid Token encountered {}", e), - error::ErrorType::UnexpectedToken, - pos, - )) + let err = abortable_parser::Error::caused_by( + "Invalid Token encountered", + Box::new(e), + Box::new(i.clone()), + ); + let ctx_err = StackPrinter { err: err }; + return Err(format!("{}", ctx_err)); } Result::Incomplete(_offset) => { - return Err(error::Error::new( - "Incomplete Token encountered", - error::ErrorType::IncompleteParsing, - pos, - )) + let err = + abortable_parser::Error::new("Invalid Token encountered", Box::new(i.clone())); + let ctx_err = StackPrinter { err: err }; + return Err(format!("{}", ctx_err)); } Result::Complete(rest, tok) => { i = rest; diff --git a/src/tokenizer/test.rs b/src/tokenizer/test.rs index 277f04a..1bc9ce0 100644 --- a/src/tokenizer/test.rs +++ b/src/tokenizer/test.rs @@ -102,7 +102,7 @@ fn test_string_with_escaping() { #[test] fn test_tokenize_bareword_with_dash() { let input = OffsetStrIter::new("foo-bar "); - let result = tokenize(&input); + let result = tokenize(input.clone()); assert!(result.is_ok(), format!("result {:?} is not ok", result)); if let Ok(toks) = result { assert_eq!(toks.len(), 2); @@ -170,7 +170,7 @@ fn test_tokenize_one_of_each() { "map out filter assert let import macro select as => [ ] { } ; = % / * \ + - . ( ) , 1 . foo \"bar\" // comment\n ; true false == < > <= >= !=", ); - let result = tokenize(&input); + let result = tokenize(input.clone()); assert!(result.is_ok(), format!("result {:?} is not ok", result)); let v = result.unwrap(); for (i, t) in v.iter().enumerate() { @@ -183,7 +183,7 @@ fn test_tokenize_one_of_each() { #[test] fn test_parse_has_end() { let input = OffsetStrIter::new("foo"); - let result = tokenize(&input); + let result = tokenize(input.clone()); assert!(result.is_ok()); let v = result.unwrap(); assert_eq!(v.len(), 2);