From 2821d0953b6f6eb79ecd6e507fa0cca5bce49c64 Mon Sep 17 00:00:00 2001 From: Jeremy Wall Date: Tue, 14 May 2019 20:18:58 -0500 Subject: [PATCH 01/23] FEATURE: First make our AST Walker a little more ergonomic. Use a trait instead of callbacks to make mutable it possible to support Walkers with mutable internal state when necessary. --- src/ast/mod.rs | 5 +- src/ast/printer.rs | 116 +++++++++++++++++++++++++++++++++++++++++++++ src/ast/walk.rs | 83 +++++++++++++++++++------------- 3 files changed, 170 insertions(+), 34 deletions(-) create mode 100644 src/ast/printer.rs diff --git a/src/ast/mod.rs b/src/ast/mod.rs index 041fbba..f3f4b2c 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -31,8 +31,11 @@ use abortable_parser; use crate::build::scope::Scope; use crate::build::Val; +pub mod printer; pub mod walk; +pub use walk::Walker; + macro_rules! enum_type_equality { ( $slf:ident, $r:expr, $( $l:pat ),* ) => { match $slf { @@ -593,7 +596,7 @@ impl ModuleDef { } } }; - let walker = walk::AstWalker::new().with_expr_handler(&rewrite_import); + let mut walker = walk::AstWalker::new().with_expr_handler(&rewrite_import); for stmt in self.statements.iter_mut() { walker.walk_statement(stmt); } diff --git a/src/ast/printer.rs b/src/ast/printer.rs new file mode 100644 index 0000000..6a2ebb2 --- /dev/null +++ b/src/ast/printer.rs @@ -0,0 +1,116 @@ +// Copyright 2019 Jeremy Wall +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +use std::borrow::BorrowMut; +use std::error::Error; +use std::io::Write; + +use crate::ast::walk::Walker; +use crate::ast::*; + +// TODO(jwall): We really need a way to preserve comments for these. +// Perhaps for code formatting we actually want to work on the token stream instead? + +pub struct Printer { + indent: u8, + curr_indent: u8, + w: Box, + pub errs: Vec>, +} + +impl Printer { + pub fn new(indent: u8, w: Box) -> Self { + Printer { + indent: indent, + curr_indent: 0, + w: w, + errs: Vec::new(), + } + } + + pub fn render_list_def(&mut self, def: &ListDef) -> std::io::Result<()> { + panic!("Unimplemented"); + Ok(()) + } + + pub fn render_tuple_def(&mut self, def: &Vec<(Token, Expression)>) -> std::io::Result<()> { + panic!("Unimplemented"); + Ok(()) + } + + pub fn render_value(&mut self, v: &Value) { + // TODO + let w: &mut Write = self.w.borrow_mut(); + let result = match v { + Value::Boolean(b) => write!(w, "{}", b), + Value::Empty(_) => write!(w, "NULL"), + // TODO(jwall): Should we maintain precision for floats? + Value::Float(f) => write!(w, "{}", f), + Value::Int(i) => write!(w, "{}", i), + // TODO(jwall): Make sure that we properly escape quotes here when rendering this? + Value::Str(s) => write!(w, "\"{}\"", s), + Value::Symbol(s) => write!(w, "{}", s), + Value::List(l) => self.render_list_def(l), + Value::Tuple(tpl) => self.render_tuple_def(&tpl.val), + }; + if let Err(e) = result { + self.errs.push(Box::new(e)); + } + } + + fn render_expr(&mut self, expr: &Expression) { + match expr { + Expression::Binary(_def) => {} + Expression::Call(_def) => {} + Expression::Copy(_def) => {} + Expression::Debug(_def) => {} + Expression::Fail(_def) => {} + Expression::Format(_def) => {} + Expression::Func(_def) => {} + Expression::FuncOp(_def) => {} + Expression::Grouped(_expr, _) => {} + Expression::Import(_def) => {} + Expression::Include(_def) => {} + Expression::Module(_def) => {} + Expression::Not(_def) => {} + Expression::Range(_def) => {} + Expression::Select(_def) => {} + Expression::Simple(_def) => {} + } + } + + fn render_stmt(&mut self, stmt: &Statement) { + match stmt { + Statement::Let(_def) => {} + Statement::Expression(_expr) => {} + Statement::Assert(_def) => {} + Statement::Output(_, _tok, _expr) => {} + } + } + + pub fn render(&mut self, stmts: Vec<&mut Statement>) { + self.walk_statement_list(stmts); + } +} + +impl Walker for Printer { + fn visit_value(&mut self, val: &mut Value) { + self.render_value(val); + } + fn visit_expression(&mut self, expr: &mut Expression) { + self.render_expr(expr); + } + fn visit_statement(&mut self, stmt: &mut Statement) { + self.render_stmt(stmt); + } +} diff --git a/src/ast/walk.rs b/src/ast/walk.rs index 023de26..df402df 100644 --- a/src/ast/walk.rs +++ b/src/ast/walk.rs @@ -1,36 +1,13 @@ use crate::ast::*; -pub struct AstWalker<'a> { - handle_value: Option<&'a Fn(&mut Value)>, - handle_expression: Option<&'a Fn(&mut Expression)>, - handle_statment: Option<&'a Fn(&mut Statement)>, -} - -impl<'a> AstWalker<'a> { - pub fn new() -> Self { - AstWalker { - handle_value: None, - handle_expression: None, - handle_statment: None, +pub trait Walker { + fn walk_statement_list(&mut self, stmts: Vec<&mut Statement>) { + for v in stmts { + self.walk_statement(v); } } - pub fn with_value_handler(mut self, h: &'a Fn(&mut Value)) -> Self { - self.handle_value = Some(h); - self - } - - pub fn with_expr_handler(mut self, h: &'a Fn(&mut Expression)) -> Self { - self.handle_expression = Some(h); - self - } - - pub fn with_stmt_handler(mut self, h: &'a Fn(&mut Statement)) -> Self { - self.handle_statment = Some(h); - self - } - - pub fn walk_statement(&self, stmt: &mut Statement) { + fn walk_statement(&mut self, stmt: &mut Statement) { self.visit_statement(stmt); match stmt { Statement::Let(ref mut def) => { @@ -48,13 +25,13 @@ impl<'a> AstWalker<'a> { } } - fn walk_fieldset(&self, fs: &mut FieldList) { + fn walk_fieldset(&mut self, fs: &mut FieldList) { for &mut (_, ref mut expr) in fs.iter_mut() { self.walk_expression(expr); } } - pub fn walk_expression(&self, expr: &mut Expression) { + fn walk_expression(&mut self, expr: &mut Expression) { self.visit_expression(expr); match expr { Expression::Call(ref mut def) => { @@ -135,19 +112,59 @@ impl<'a> AstWalker<'a> { } } - fn visit_value(&self, val: &mut Value) { + fn visit_value(&mut self, val: &mut Value); + + fn visit_expression(&mut self, expr: &mut Expression); + + fn visit_statement(&mut self, stmt: &mut Statement); +} + +// TODO this would be better implemented as a Trait I think. +pub struct AstWalker<'a> { + handle_value: Option<&'a Fn(&mut Value)>, + handle_expression: Option<&'a Fn(&mut Expression)>, + handle_statment: Option<&'a Fn(&mut Statement)>, +} + +impl<'a> AstWalker<'a> { + pub fn new() -> Self { + AstWalker { + handle_value: None, + handle_expression: None, + handle_statment: None, + } + } + + pub fn with_value_handler(mut self, h: &'a Fn(&mut Value)) -> Self { + self.handle_value = Some(h); + self + } + + pub fn with_expr_handler(mut self, h: &'a Fn(&mut Expression)) -> Self { + self.handle_expression = Some(h); + self + } + + pub fn with_stmt_handler(mut self, h: &'a Fn(&mut Statement)) -> Self { + self.handle_statment = Some(h); + self + } +} + +impl<'a> Walker for AstWalker<'a> { + fn visit_value(&mut self, val: &mut Value) { if let Some(h) = self.handle_value { h(val); } } - fn visit_expression(&self, expr: &mut Expression) { + fn visit_expression(&mut self, expr: &mut Expression) { if let Some(h) = self.handle_expression { h(expr); } } - fn visit_statement(&self, stmt: &mut Statement) { + fn visit_statement(&mut self, stmt: &mut Statement) { if let Some(h) = self.handle_statment { h(stmt); } From e86827f613a387b7746657b26bc372df8c93ec19 Mon Sep 17 00:00:00 2001 From: Jeremy Wall Date: Tue, 14 May 2019 21:03:23 -0500 Subject: [PATCH 02/23] FEATURE: Begin the work of pretty printing the AST. --- src/ast/printer.rs | 395 ++++++++++++++++++++++++++++++++++-------- src/parse/mod.rs | 2 +- src/tokenizer/mod.rs | 7 +- src/tokenizer/test.rs | 6 +- 4 files changed, 330 insertions(+), 80 deletions(-) diff --git a/src/ast/printer.rs b/src/ast/printer.rs index 6a2ebb2..b56194e 100644 --- a/src/ast/printer.rs +++ b/src/ast/printer.rs @@ -12,105 +12,352 @@ // See the License for the specific language governing permissions and // limitations under the License. use std::borrow::BorrowMut; -use std::error::Error; use std::io::Write; -use crate::ast::walk::Walker; use crate::ast::*; // TODO(jwall): We really need a way to preserve comments for these. // Perhaps for code formatting we actually want to work on the token stream instead? -pub struct Printer { - indent: u8, - curr_indent: u8, - w: Box, - pub errs: Vec>, +pub struct AstPrinter +where + W: Write, +{ + indent: usize, + curr_indent: usize, + w: W, + pub err: Option, } -impl Printer { - pub fn new(indent: u8, w: Box) -> Self { - Printer { +impl AstPrinter +where + W: Write, +{ + pub fn new(indent: usize, w: W) -> Self { + AstPrinter { indent: indent, curr_indent: 0, w: w, - errs: Vec::new(), + err: None, } } - pub fn render_list_def(&mut self, def: &ListDef) -> std::io::Result<()> { - panic!("Unimplemented"); - Ok(()) - } - - pub fn render_tuple_def(&mut self, def: &Vec<(Token, Expression)>) -> std::io::Result<()> { - panic!("Unimplemented"); - Ok(()) - } - - pub fn render_value(&mut self, v: &Value) { - // TODO + pub fn visit_token(&mut self, t: &Token) -> std::io::Result<()> { let w: &mut Write = self.w.borrow_mut(); - let result = match v { - Value::Boolean(b) => write!(w, "{}", b), - Value::Empty(_) => write!(w, "NULL"), - // TODO(jwall): Should we maintain precision for floats? - Value::Float(f) => write!(w, "{}", f), - Value::Int(i) => write!(w, "{}", i), - // TODO(jwall): Make sure that we properly escape quotes here when rendering this? - Value::Str(s) => write!(w, "\"{}\"", s), - Value::Symbol(s) => write!(w, "{}", s), - Value::List(l) => self.render_list_def(l), - Value::Tuple(tpl) => self.render_tuple_def(&tpl.val), + // Do we care about line length? + match t.typ { + TokenType::BAREWORD | TokenType::BOOLEAN | TokenType::DIGIT => { + write!(w, "{}", t.fragment)?; + } + TokenType::EMPTY => { + write!(w, "NULL")?; + } + TokenType::PUNCT => { + // TODO(jwall): We need to identify the points at which we + // introduce new lines and new indentation scopes. + } + TokenType::COMMENT => { + // We need to track some state here probably. + // Do we leave comments untouched? + } + TokenType::PIPEQUOTE => { + // FIXME I think is supposed to be removed. + } + TokenType::QUOTED => { + w.write(&['"' as u8])?; + write!(w, "{}", Self::escape_quotes(&t.fragment))?; + w.write(&['"' as u8])?; + } + TokenType::WS => { + // TODO(jwall): Track some state around new lines here? + } + TokenType::END => { + // NOOP + } }; - if let Err(e) = result { - self.errs.push(Box::new(e)); - } + Ok(()) } - fn render_expr(&mut self, expr: &Expression) { + fn make_indent(&self) -> String { + // TODO(jwall): This is probably inefficient but we'll improve it after + // we get it correct. + let indent: Vec = std::iter::repeat(' ' as u8) + .take(self.curr_indent) + .collect(); + String::from_utf8_lossy(&indent).to_string() + } + + fn render_list_def(&mut self, def: &ListDef) -> std::io::Result<()> { + write!(self.w, "[\n")?; + self.curr_indent += self.indent; + // If the element list is just 1 we might be able to collapse the tuple. + let indent = self.make_indent(); + for e in def.elems.iter() { + // TODO(jwall): Now print out the elements + write!(self.w, "{}", indent)?; + self.render_expr(e)?; + write!(self.w, "\n")?; + } + self.curr_indent -= self.indent; + self.w.write(&[']' as u8])?; + Ok(()) + } + + fn render_tuple_def(&mut self, def: &Vec<(Token, Expression)>) -> std::io::Result<()> { + self.w.write(&['{' as u8])?; + // If the field list is just 1 we might be able to collapse the tuple. + self.curr_indent += self.indent; + let indent = self.make_indent(); + for &(ref t, ref expr) in def.iter() { + write!(self.w, "{}", indent)?; + // TODO(jwall): Detect if there are strings and render as a quoted string. + write!(&mut self.w, "{} = ", t.fragment)?; + self.render_expr(expr)?; + write!(&mut self.w, ",")?; + write!(self.w, "\n")?; + } + self.w.write(&['}' as u8])?; + Ok(()) + } + + fn escape_quotes(s: &str) -> String { + let mut escaped = String::new(); + for c in s.chars() { + if c == '"' { + escaped.push_str("\\\""); + } else if c == '\\' { + escaped.push_str("\\\\"); + } else { + escaped.push(c); + } + } + escaped + } + + pub fn render_value(&mut self, v: &Value) -> std::io::Result<()> { + match v { + Value::Boolean(b) => write!(self.w, "{}", if b.val { "true" } else { "false" })?, + Value::Empty(_) => write!(self.w, "NULL")?, + // TODO(jwall): We should maintain precision for floats? + Value::Float(f) => write!(self.w, "{}", f.val)?, + Value::Int(i) => write!(self.w, "{}", i.val)?, + Value::Str(s) => write!(self.w, "\"{}\"", Self::escape_quotes(&s.val))?, + Value::Symbol(s) => write!(self.w, "{}", s.val)?, + Value::List(l) => self.render_list_def(l)?, + Value::Tuple(tpl) => self.render_tuple_def(&tpl.val)?, + }; + Ok(()) + } + + fn render_expr(&mut self, expr: &Expression) -> std::io::Result<()> { match expr { - Expression::Binary(_def) => {} - Expression::Call(_def) => {} - Expression::Copy(_def) => {} - Expression::Debug(_def) => {} - Expression::Fail(_def) => {} - Expression::Format(_def) => {} - Expression::Func(_def) => {} - Expression::FuncOp(_def) => {} - Expression::Grouped(_expr, _) => {} - Expression::Import(_def) => {} - Expression::Include(_def) => {} - Expression::Module(_def) => {} - Expression::Not(_def) => {} - Expression::Range(_def) => {} - Expression::Select(_def) => {} - Expression::Simple(_def) => {} - } + Expression::Binary(_def) => { + let op = match _def.kind { + BinaryExprType::AND => " && ", + BinaryExprType::OR => " || ", + BinaryExprType::DOT => ".", + BinaryExprType::Equal => " = ", + BinaryExprType::NotEqual => " != ", + BinaryExprType::GTEqual => " >= ", + BinaryExprType::LTEqual => " <= ", + BinaryExprType::GT => " > ", + BinaryExprType::LT => " < ", + BinaryExprType::Add => " + ", + BinaryExprType::Sub => " - ", + BinaryExprType::Mul => " * ", + BinaryExprType::Div => " / ", + BinaryExprType::Mod => " %% ", + BinaryExprType::IN => " in ", + BinaryExprType::IS => " is ", + BinaryExprType::REMatch => " ~ ", + BinaryExprType::NotREMatch => " !~ ", + }; + self.render_expr(&_def.left)?; + self.w.write(op.as_bytes())?; + self.render_expr(&_def.right)?; + } + Expression::Call(_def) => { + self.render_value(&_def.funcref)?; + self.w.write("(".as_bytes())?; + self.curr_indent += self.indent; + let indent = self.make_indent(); + for e in _def.arglist.iter() { + self.w.write(indent.as_bytes())?; + self.render_expr(e)?; + self.w.write("\n".as_bytes())?; + } + self.curr_indent -= self.indent; + self.w.write("(".as_bytes())?; + } + Expression::Copy(_def) => { + self.render_value(&_def.selector)?; + self.render_tuple_def(&_def.fields)?; + } + Expression::Debug(_def) => { + self.w.write("TRACE ".as_bytes())?; + self.render_expr(&_def.expr)?; + } + Expression::Fail(_def) => { + self.w.write("fail ".as_bytes())?; + self.render_expr(&_def.message)?; + } + Expression::Format(_def) => { + self.w + .write(Self::escape_quotes(&_def.template).as_bytes())?; + write!(self.w, " % ")?; + match _def.args { + FormatArgs::Single(ref e) => { + self.render_expr(e)?; + } + FormatArgs::List(ref es) => { + self.w.write("(".as_bytes())?; + self.curr_indent += self.indent; + let indent = self.make_indent(); + for e in es.iter() { + self.w.write(indent.as_bytes())?; + self.render_expr(e)?; + self.w.write("\n".as_bytes())?; + } + self.curr_indent -= self.indent; + self.w.write(")".as_bytes())?; + } + } + } + Expression::Func(_def) => { + self.w.write("func (".as_bytes())?; + for n in _def.argdefs.iter() { + write!(self.w, "{}, ", n.val)?; + } + self.w.write(") => ".as_bytes())?; + self.render_expr(&_def.fields)?; + } + Expression::FuncOp(_def) => match _def { + FuncOpDef::Filter(_def) => { + write!(self.w, "filter(")?; + self.render_expr(&_def.func)?; + write!(self.w, ", ")?; + self.render_expr(&_def.target)?; + write!(self.w, ")")?; + } + FuncOpDef::Reduce(_def) => { + write!(self.w, "reduce(")?; + self.render_expr(&_def.func)?; + write!(self.w, ", ")?; + self.render_expr(&_def.acc)?; + write!(self.w, ", ")?; + self.render_expr(&_def.target)?; + write!(self.w, ")")?; + } + FuncOpDef::Map(_def) => { + write!(self.w, "map(")?; + self.render_expr(&_def.func)?; + write!(self.w, ", ")?; + self.render_expr(&_def.target)?; + write!(self.w, ")")?; + } + }, + Expression::Grouped(ref expr, _) => { + write!(self.w, "(")?; + self.render_expr(expr)?; + write!(self.w, ")")?; + } + Expression::Import(_def) => { + write!( + self.w, + "import \"{}\"", + Self::escape_quotes(&_def.path.fragment) + )?; + } + Expression::Include(_def) => { + write!( + self.w, + "include {} \"{}\"", + _def.typ.fragment, + Self::escape_quotes(&_def.path.fragment) + )?; + } + Expression::Module(_def) => { + write!(self.w, "module ")?; + self.render_tuple_def(&_def.arg_set)?; + write!(self.w, " => ")?; + if let Some(ref e) = _def.out_expr { + write!(self.w, "(")?; + self.render_expr(e)?; + write!(self.w, ") ")?; + } + write!(self.w, "{{")?; + self.curr_indent += self.indent; + let indent = self.make_indent(); + for stmt in _def.statements.iter() { + write!(self.w, "{}", indent)?; + self.render_stmt(stmt)?; + } + self.curr_indent -= self.indent; + write!(self.w, "}}")?; + } + Expression::Not(_def) => { + write!(self.w, "not ")?; + self.render_expr(&_def.expr)?; + } + Expression::Range(_def) => { + self.render_expr(&_def.start)?; + write!(self.w, ":")?; + if let Some(ref e) = _def.step { + write!(self.w, ":")?; + self.render_expr(e)?; + } + self.render_expr(&_def.end)?; + } + Expression::Select(_def) => { + // + write!(self.w, "select ")?; + self.render_expr(&_def.val)?; + write!(self.w, ", ")?; + if let Some(ref e) = _def.default { + self.render_expr(e)?; + write!(self.w, ", ")?; + } + self.render_tuple_def(&_def.tuple)?; + } + Expression::Simple(ref _def) => { + self.render_value(_def)?; + } + }; + Ok(()) } - fn render_stmt(&mut self, stmt: &Statement) { + fn render_stmt(&mut self, stmt: &Statement) -> std::io::Result<()> { + // All statements start at the beginning of a line. match stmt { - Statement::Let(_def) => {} - Statement::Expression(_expr) => {} - Statement::Assert(_def) => {} - Statement::Output(_, _tok, _expr) => {} - } + Statement::Let(def) => { + write!(&mut self.w, "let {} = ", def.name.fragment)?; + self.render_expr(&def.value)?; + } + Statement::Expression(_expr) => { + self.render_expr(&_expr)?; + // + } + Statement::Assert(def) => { + write!(&mut self.w, "assert ")?; + self.render_expr(&def)?; + // + } + Statement::Output(_, _tok, _expr) => { + write!(&mut self.w, "out {} = ", _tok.fragment)?; + self.render_expr(&_expr)?; + // + } + }; + write!(self.w, ";\n")?; + Ok(()) } pub fn render(&mut self, stmts: Vec<&mut Statement>) { - self.walk_statement_list(stmts); - } -} - -impl Walker for Printer { - fn visit_value(&mut self, val: &mut Value) { - self.render_value(val); - } - fn visit_expression(&mut self, expr: &mut Expression) { - self.render_expr(expr); - } - fn visit_statement(&mut self, stmt: &mut Statement) { - self.render_stmt(stmt); + for v in stmts { + if let Err(e) = self.render_stmt(v) { + self.err = Some(e); + return; + } + } } } diff --git a/src/parse/mod.rs b/src/parse/mod.rs index 4578022..96bec15 100644 --- a/src/parse/mod.rs +++ b/src/parse/mod.rs @@ -854,7 +854,7 @@ fn statement(i: SliceIter) -> Result, Statement> { /// Parses a LocatedSpan into a list of Statements or an `error::Error`. pub fn parse<'a>(input: OffsetStrIter<'a>) -> std::result::Result, String> { - match tokenize(input.clone()) { + match tokenize(input.clone(), true) { Ok(tokenized) => { let mut out = Vec::new(); let mut i_ = SliceIter::new(&tokenized); diff --git a/src/tokenizer/mod.rs b/src/tokenizer/mod.rs index b191fe7..07d5a74 100644 --- a/src/tokenizer/mod.rs +++ b/src/tokenizer/mod.rs @@ -452,7 +452,10 @@ fn token<'a>(input: OffsetStrIter<'a>) -> Result, Token> { } /// Consumes an input OffsetStrIter and returns either a Vec or a error::Error. -pub fn tokenize<'a>(input: OffsetStrIter<'a>) -> std::result::Result, String> { +pub fn tokenize<'a>( + input: OffsetStrIter<'a>, + skip_comments: bool, +) -> std::result::Result, String> { let mut out = Vec::new(); let mut i = input.clone(); loop { @@ -486,7 +489,7 @@ pub fn tokenize<'a>(input: OffsetStrIter<'a>) -> std::result::Result, } Result::Complete(rest, tok) => { i = rest; - if tok.typ == TokenType::COMMENT || tok.typ == TokenType::WS { + if (skip_comments && tok.typ == TokenType::COMMENT) || tok.typ == TokenType::WS { // we skip comments and whitespace continue; } diff --git a/src/tokenizer/test.rs b/src/tokenizer/test.rs index 2458226..0ef757c 100644 --- a/src/tokenizer/test.rs +++ b/src/tokenizer/test.rs @@ -89,7 +89,7 @@ fn test_string_with_escaping() { #[test] fn test_tokenize_bareword_with_dash() { let input = OffsetStrIter::new("foo-bar "); - let result = tokenize(input.clone()); + let result = tokenize(input.clone(), true); assert!(result.is_ok(), format!("result {:?} is not ok", result)); if let Ok(toks) = result { assert_eq!(toks.len(), 2); @@ -157,7 +157,7 @@ fn test_tokenize_one_of_each() { "map out filter assert let import func select as => [ ] { } ; = % / * \ + - . ( ) , 1 . foo \"bar\" // comment\n ; true false == < > <= >= !=", ); - let result = tokenize(input.clone()); + let result = tokenize(input.clone(), true); assert!(result.is_ok(), format!("result {:?} is not ok", result)); let v = result.unwrap(); for (i, t) in v.iter().enumerate() { @@ -170,7 +170,7 @@ fn test_tokenize_one_of_each() { #[test] fn test_parse_has_end() { let input = OffsetStrIter::new("foo"); - let result = tokenize(input.clone()); + let result = tokenize(input.clone(), true); assert!(result.is_ok()); let v = result.unwrap(); assert_eq!(v.len(), 2); From d122fe6e6f468058c2207de9924db5655f88aa22 Mon Sep 17 00:00:00 2001 From: Jeremy Wall Date: Fri, 17 May 2019 18:33:47 -0500 Subject: [PATCH 03/23] FEATURE: Fix a number of bugs found by adding unit tests :-D --- src/ast/{printer.rs => printer/mod.rs} | 48 ++++++- src/ast/printer/test.rs | 167 +++++++++++++++++++++++++ 2 files changed, 210 insertions(+), 5 deletions(-) rename src/ast/{printer.rs => printer/mod.rs} (91%) create mode 100644 src/ast/printer/test.rs diff --git a/src/ast/printer.rs b/src/ast/printer/mod.rs similarity index 91% rename from src/ast/printer.rs rename to src/ast/printer/mod.rs index b56194e..9451da2 100644 --- a/src/ast/printer.rs +++ b/src/ast/printer/mod.rs @@ -87,18 +87,42 @@ where String::from_utf8_lossy(&indent).to_string() } + fn is_bareword(s: &str) -> bool { + match s.chars().nth(0) { + Some(c) => { + if !(c.is_ascii_alphabetic() || c == '_') { + return false; + } + } + None => return false, + }; + for c in s.chars() { + if !c.is_ascii_alphanumeric() { + return false; + } + } + return true; + } + fn render_list_def(&mut self, def: &ListDef) -> std::io::Result<()> { - write!(self.w, "[\n")?; + write!(self.w, "[")?; self.curr_indent += self.indent; // If the element list is just 1 we might be able to collapse the tuple. let indent = self.make_indent(); + let has_fields = def.elems.len() > 0; + if has_fields { + write!(self.w, "\n")?; + } for e in def.elems.iter() { // TODO(jwall): Now print out the elements write!(self.w, "{}", indent)?; self.render_expr(e)?; - write!(self.w, "\n")?; + write!(self.w, ",\n")?; } self.curr_indent -= self.indent; + if has_fields { + write!(self.w, "{}", self.make_indent())?; + } self.w.write(&[']' as u8])?; Ok(()) } @@ -108,14 +132,25 @@ where // If the field list is just 1 we might be able to collapse the tuple. self.curr_indent += self.indent; let indent = self.make_indent(); + let has_fields = def.len() > 0; + if has_fields { + write!(self.w, "\n")?; + } for &(ref t, ref expr) in def.iter() { write!(self.w, "{}", indent)?; - // TODO(jwall): Detect if there are strings and render as a quoted string. - write!(&mut self.w, "{} = ", t.fragment)?; + if Self::is_bareword(&t.fragment) { + write!(&mut self.w, "{} = ", t.fragment)?; + } else { + write!(self.w, "\"{}\" = ", Self::escape_quotes(&t.fragment))?; + } self.render_expr(expr)?; write!(&mut self.w, ",")?; write!(self.w, "\n")?; } + self.curr_indent -= self.indent; + if has_fields { + write!(self.w, "{}", self.make_indent())?; + } self.w.write(&['}' as u8])?; Ok(()) } @@ -352,7 +387,7 @@ where Ok(()) } - pub fn render(&mut self, stmts: Vec<&mut Statement>) { + pub fn render(&mut self, stmts: &Vec) { for v in stmts { if let Err(e) = self.render_stmt(v) { self.err = Some(e); @@ -361,3 +396,6 @@ where } } } + +#[cfg(test)] +mod test; diff --git a/src/ast/printer/test.rs b/src/ast/printer/test.rs new file mode 100644 index 0000000..b9a7896 --- /dev/null +++ b/src/ast/printer/test.rs @@ -0,0 +1,167 @@ +// Copyright 2019 Jeremy Wall +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::ast::printer::*; +use crate::iter::OffsetStrIter; +use crate::parse::*; + +fn assert_parse(input: &str) -> Vec { + parse(OffsetStrIter::new(input)).unwrap() +} + +#[test] +fn test_simple_value_printing() { + let input = "1;"; + let stmts = assert_parse("1;"); + let mut buffer: Vec = Vec::new(); + let mut printer = AstPrinter::new(0, &mut buffer); + printer.render(&stmts); + assert!(printer.err.is_none()); + assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); +} + +#[test] +fn test_simple_quoted_printing() { + let input = "\"foo\";"; + let stmts = assert_parse(input); + let mut buffer: Vec = Vec::new(); + let mut printer = AstPrinter::new(0, &mut buffer); + printer.render(&stmts); + assert!(printer.err.is_none()); + assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); +} + +#[test] +fn test_escaped_quoted_printing() { + let input = "\"f\\\\o\\\"o\";"; + let stmts = assert_parse(input); + let mut buffer: Vec = Vec::new(); + let mut printer = AstPrinter::new(0, &mut buffer); + printer.render(&stmts); + assert!(printer.err.is_none()); + assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); +} + +#[test] +fn test_empty_tuple_printing() { + let input = "{};"; + let stmts = assert_parse(input); + let mut buffer: Vec = Vec::new(); + let mut printer = AstPrinter::new(2, &mut buffer); + printer.render(&stmts); + assert!(printer.err.is_none()); + assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); +} + +#[test] +fn test_empty_list_printing() { + let input = "[];"; + let stmts = assert_parse(input); + let mut buffer: Vec = Vec::new(); + let mut printer = AstPrinter::new(2, &mut buffer); + printer.render(&stmts); + assert!(printer.err.is_none()); + assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); +} + +#[test] +fn test_non_empty_tuple_printing() { + let input = "{\n foo = 1,\n};"; + let stmts = assert_parse(input); + let mut buffer: Vec = Vec::new(); + let mut printer = AstPrinter::new(2, &mut buffer); + printer.render(&stmts); + assert!(printer.err.is_none()); + assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); +} + +#[test] +fn test_nested_empty_tuple_printing() { + let input = "{\n foo = {},\n};"; + let stmts = assert_parse(input); + let mut buffer: Vec = Vec::new(); + let mut printer = AstPrinter::new(2, &mut buffer); + printer.render(&stmts); + assert!(printer.err.is_none()); + assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); +} + +#[test] +fn test_list_nested_empty_tuple_printing() { + let input = "[\n {},\n];"; + let stmts = assert_parse(input); + let mut buffer: Vec = Vec::new(); + let mut printer = AstPrinter::new(2, &mut buffer); + printer.render(&stmts); + assert!(printer.err.is_none()); + assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); +} + +#[test] +fn test_nested_non_empty_tuple_printing() { + let input = "{\n foo = {\n bar = 1,\n },\n};"; + let stmts = assert_parse(input); + let mut buffer: Vec = Vec::new(); + let mut printer = AstPrinter::new(2, &mut buffer); + printer.render(&stmts); + assert!(printer.err.is_none()); + assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); +} + +#[test] +fn test_nested_non_empty_list_printing() { + let input = "[\n [\n 1,\n ],\n];"; + let stmts = assert_parse(input); + let mut buffer: Vec = Vec::new(); + let mut printer = AstPrinter::new(2, &mut buffer); + printer.render(&stmts); + assert!(printer.err.is_none()); + assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); +} + +#[test] +fn test_simple_quoted_field_tuple_printing() { + let input = "{\n \"foo\" = {\n bar = 1,\n },\n};"; + let stmts = assert_parse(input); + let mut buffer: Vec = Vec::new(); + let mut printer = AstPrinter::new(2, &mut buffer); + printer.render(&stmts); + assert!(printer.err.is_none()); + assert_eq!( + String::from_utf8(buffer).unwrap(), + format!("{}\n", "{\n foo = {\n bar = 1,\n },\n};") + ); +} + +#[test] +fn test_special_quoted_field_tuple_printing() { + let input = "{\n \"foo bar\" = {\n bar = 1,\n },\n};"; + let stmts = assert_parse(input); + let mut buffer: Vec = Vec::new(); + let mut printer = AstPrinter::new(2, &mut buffer); + printer.render(&stmts); + assert!(printer.err.is_none()); + assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); +} + +#[test] +fn test_let_statement_printing() { + let input = "let tpl = {\n \"foo bar\" = {\n bar = 1,\n },\n};"; + let stmts = assert_parse(input); + let mut buffer: Vec = Vec::new(); + let mut printer = AstPrinter::new(2, &mut buffer); + printer.render(&stmts); + assert!(printer.err.is_none()); + assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); +} From cd6307824f24b5758b2bf4c153f3e6c3bcee82c4 Mon Sep 17 00:00:00 2001 From: Jeremy Wall Date: Fri, 17 May 2019 19:31:29 -0500 Subject: [PATCH 04/23] FEATURE: more fixes and unit tests. Call and copy have been tweaked properly. --- src/ast/printer/mod.rs | 21 +++++++++++---- src/ast/printer/test.rs | 57 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 72 insertions(+), 6 deletions(-) diff --git a/src/ast/printer/mod.rs b/src/ast/printer/mod.rs index 9451da2..62c4f25 100644 --- a/src/ast/printer/mod.rs +++ b/src/ast/printer/mod.rs @@ -29,6 +29,8 @@ where pub err: Option, } +// TODO(jwall): At some point we probably want to be more aware of line length +// in our formatting. But not at the moment. impl AstPrinter where W: Write, @@ -107,14 +109,12 @@ where fn render_list_def(&mut self, def: &ListDef) -> std::io::Result<()> { write!(self.w, "[")?; self.curr_indent += self.indent; - // If the element list is just 1 we might be able to collapse the tuple. let indent = self.make_indent(); let has_fields = def.elems.len() > 0; if has_fields { write!(self.w, "\n")?; } for e in def.elems.iter() { - // TODO(jwall): Now print out the elements write!(self.w, "{}", indent)?; self.render_expr(e)?; write!(self.w, ",\n")?; @@ -216,13 +216,24 @@ where self.w.write("(".as_bytes())?; self.curr_indent += self.indent; let indent = self.make_indent(); + let has_args = _def.arglist.len() > 1; + if has_args { + write!(self.w, "\n")?; + } for e in _def.arglist.iter() { - self.w.write(indent.as_bytes())?; + if has_args { + write!(self.w, "{}", indent)?; + } self.render_expr(e)?; - self.w.write("\n".as_bytes())?; + if has_args { + self.w.write(",\n".as_bytes())?; + } } self.curr_indent -= self.indent; - self.w.write("(".as_bytes())?; + if has_args { + write!(self.w, "{}", self.make_indent())?; + } + self.w.write(")".as_bytes())?; } Expression::Copy(_def) => { self.render_value(&_def.selector)?; diff --git a/src/ast/printer/test.rs b/src/ast/printer/test.rs index b9a7896..527e0e4 100644 --- a/src/ast/printer/test.rs +++ b/src/ast/printer/test.rs @@ -23,7 +23,18 @@ fn assert_parse(input: &str) -> Vec { #[test] fn test_simple_value_printing() { let input = "1;"; - let stmts = assert_parse("1;"); + let stmts = assert_parse(input); + let mut buffer: Vec = Vec::new(); + let mut printer = AstPrinter::new(0, &mut buffer); + printer.render(&stmts); + assert!(printer.err.is_none()); + assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); +} + +#[test] +fn test_simple_selector_printing() { + let input = "foo.bar.quux;"; + let stmts = assert_parse(input); let mut buffer: Vec = Vec::new(); let mut printer = AstPrinter::new(0, &mut buffer); printer.render(&stmts); @@ -165,3 +176,47 @@ fn test_let_statement_printing() { assert!(printer.err.is_none()); assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); } + +#[test] +fn test_call_expr_printing() { + let input = "call(\n foo,\n bar,\n);"; + let stmts = assert_parse(input); + let mut buffer: Vec = Vec::new(); + let mut printer = AstPrinter::new(2, &mut buffer); + printer.render(&stmts); + assert!(printer.err.is_none()); + assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); +} + +#[test] +fn test_call_expr_one_arg_printing() { + let input = "call(foo);"; + let stmts = assert_parse(input); + let mut buffer: Vec = Vec::new(); + let mut printer = AstPrinter::new(2, &mut buffer); + printer.render(&stmts); + assert!(printer.err.is_none()); + assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); +} + +#[test] +fn test_copy_expr_printing() { + let input = "copy{\n foo = 1,\n bar = 2,\n};"; + let stmts = assert_parse(input); + let mut buffer: Vec = Vec::new(); + let mut printer = AstPrinter::new(2, &mut buffer); + printer.render(&stmts); + assert!(printer.err.is_none()); + assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); +} + +#[test] +fn test_copy_expr_one_arg_printing() { + let input = "copy{\n foo = 1,\n};"; + let stmts = assert_parse(input); + let mut buffer: Vec = Vec::new(); + let mut printer = AstPrinter::new(2, &mut buffer); + printer.render(&stmts); + assert!(printer.err.is_none()); + assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); +} From 525cdd32e6bbb7ac1786b282e976da112ff3ae6e Mon Sep 17 00:00:00 2001 From: Jeremy Wall Date: Fri, 17 May 2019 19:42:50 -0500 Subject: [PATCH 05/23] FEATURE: Use the AST Pretty Printer for TRACE and assert output. --- examples/module_example/modules/host_module.ucg | 2 +- src/ast/printer/mod.rs | 6 +++--- src/build/mod.rs | 17 +++++++++++++++-- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/examples/module_example/modules/host_module.ucg b/examples/module_example/modules/host_module.ucg index 6fb52aa..7f2d2e2 100644 --- a/examples/module_example/modules/host_module.ucg +++ b/examples/module_example/modules/host_module.ucg @@ -1,4 +1,4 @@ -let host_mod = module{ +let host_mod = TRACE module{ hostname="", mem=2048, cpu=2, diff --git a/src/ast/printer/mod.rs b/src/ast/printer/mod.rs index 62c4f25..e0cbfec 100644 --- a/src/ast/printer/mod.rs +++ b/src/ast/printer/mod.rs @@ -184,7 +184,7 @@ where Ok(()) } - fn render_expr(&mut self, expr: &Expression) -> std::io::Result<()> { + pub fn render_expr(&mut self, expr: &Expression) -> std::io::Result<()> { match expr { Expression::Binary(_def) => { let op = match _def.kind { @@ -331,7 +331,7 @@ where self.render_expr(e)?; write!(self.w, ") ")?; } - write!(self.w, "{{")?; + write!(self.w, "{{\n")?; self.curr_indent += self.indent; let indent = self.make_indent(); for stmt in _def.statements.iter() { @@ -372,7 +372,7 @@ where Ok(()) } - fn render_stmt(&mut self, stmt: &Statement) -> std::io::Result<()> { + pub fn render_stmt(&mut self, stmt: &Statement) -> std::io::Result<()> { // All statements start at the beginning of a line. match stmt { Statement::Let(def) => { diff --git a/src/build/mod.rs b/src/build/mod.rs index f889769..4e23874 100644 --- a/src/build/mod.rs +++ b/src/build/mod.rs @@ -1654,11 +1654,17 @@ impl<'a> FileBuilder<'a> { // we are not in validate_mode so build_asserts are noops. return Ok(Rc::new(Val::Empty)); } + let mut buffer: Vec = Vec::new(); + { + let mut printer = crate::ast::printer::AstPrinter::new(2, &mut buffer); + let _ = printer.render_expr(expr); + } + let expr_pretty = String::from_utf8(buffer).unwrap(); let ok = match self.eval_expr(expr, scope) { Ok(v) => v, Err(e) => { // failure! - let msg = format!("CompileError: {}\n", e); + let msg = format!("CompileError: {}\nfor expression:\n{}\n", e, expr_pretty); self.record_assert_result(&msg, false); return Ok(Rc::new(Val::Empty)); } @@ -1965,9 +1971,16 @@ impl<'a> FileBuilder<'a> { }; } &Expression::Debug(ref def) => { + let mut buffer: Vec = Vec::new(); + { + let mut printer = crate::ast::printer::AstPrinter::new(2, &mut buffer); + let _ = printer.render_expr(&def.expr); + } + let expr_pretty = String::from_utf8(buffer).unwrap(); + let val = self.eval_expr(&def.expr, scope); if let Ok(ref val) = val { - eprintln!("TRACE: {} at {}", val, def.pos); + eprintln!("TRACE: {} = {} at {}", expr_pretty, val, def.pos); } val } From 0d78438c2a614dd63c2dc4d30ec62083c48bc3e3 Mon Sep 17 00:00:00 2001 From: Jeremy Wall Date: Fri, 17 May 2019 19:58:48 -0500 Subject: [PATCH 06/23] FEATURE: Fix bug in out statements; Also add tests for modules. --- src/ast/printer/mod.rs | 2 +- src/ast/printer/test.rs | 86 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+), 1 deletion(-) diff --git a/src/ast/printer/mod.rs b/src/ast/printer/mod.rs index e0cbfec..04c2d25 100644 --- a/src/ast/printer/mod.rs +++ b/src/ast/printer/mod.rs @@ -389,7 +389,7 @@ where // } Statement::Output(_, _tok, _expr) => { - write!(&mut self.w, "out {} = ", _tok.fragment)?; + write!(&mut self.w, "out {} ", _tok.fragment)?; self.render_expr(&_expr)?; // } diff --git a/src/ast/printer/test.rs b/src/ast/printer/test.rs index 527e0e4..9c66a20 100644 --- a/src/ast/printer/test.rs +++ b/src/ast/printer/test.rs @@ -220,3 +220,89 @@ fn test_copy_expr_one_arg_printing() { assert!(printer.err.is_none()); assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); } + +#[test] +fn test_out_expr_printing() { + let input = "out json {\n foo = 1,\n};"; + let stmts = assert_parse(input); + let mut buffer: Vec = Vec::new(); + let mut printer = AstPrinter::new(2, &mut buffer); + printer.render(&stmts); + assert!(printer.err.is_none()); + assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); +} + +#[test] +fn test_select_expr_no_default_printing() { + let input = "select true, {\n true = 1,\n false = 2,\n};"; + let stmts = assert_parse(input); + let mut buffer: Vec = Vec::new(); + let mut printer = AstPrinter::new(2, &mut buffer); + printer.render(&stmts); + assert!(printer.err.is_none()); + assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); +} + +#[test] +fn test_select_expr_with_default_printing() { + let input = "select true, 3, {\n true = 1,\n false = 2,\n};"; + let stmts = assert_parse(input); + let mut buffer: Vec = Vec::new(); + let mut printer = AstPrinter::new(2, &mut buffer); + printer.render(&stmts); + assert!(printer.err.is_none()); + assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); +} + +#[test] +fn test_not_expr_printing() { + let input = "not true;"; + let stmts = assert_parse(input); + let mut buffer: Vec = Vec::new(); + let mut printer = AstPrinter::new(2, &mut buffer); + printer.render(&stmts); + assert!(printer.err.is_none()); + assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); +} + +#[test] +fn test_module_no_out_expr_printing() { + let input = "let m = module { + hostname = \"\", + mem = 2048, + cpu = 2, +} => { + let config = { + hostname = mod.hostname, + \"memory_size\" = mod.mem, + \"cpu_count\" = mod.cpu, + }; +};"; + let stmts = assert_parse(input); + let mut buffer: Vec = Vec::new(); + let mut printer = AstPrinter::new(2, &mut buffer); + printer.render(&stmts); + assert!(printer.err.is_none()); + assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); +} + +#[test] +fn test_module_with_out_expr_printing() { + let input = "let m = module { + hostname = \"\", + mem = 2048, + cpu = 2, +} => (config) { + let config = { + hostname = mod.hostname, + \"memory_size\" = mod.mem, + \"cpu_count\" = mod.cpu, + }; +};"; + let stmts = assert_parse(input); + let mut buffer: Vec = Vec::new(); + let mut printer = AstPrinter::new(2, &mut buffer); + printer.render(&stmts); + assert!(printer.err.is_none()); + assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); +} From 9b99bc026cb0bfac76a8d45c71a7ad5d5e1c0793 Mon Sep 17 00:00:00 2001 From: Jeremy Wall Date: Fri, 17 May 2019 20:06:06 -0500 Subject: [PATCH 07/23] FEATURE: Improvements to func expression pretty printing. --- src/ast/printer/mod.rs | 10 ++++++++-- src/ast/printer/test.rs | 27 +++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/src/ast/printer/mod.rs b/src/ast/printer/mod.rs index 04c2d25..1738607 100644 --- a/src/ast/printer/mod.rs +++ b/src/ast/printer/mod.rs @@ -271,8 +271,14 @@ where } Expression::Func(_def) => { self.w.write("func (".as_bytes())?; - for n in _def.argdefs.iter() { - write!(self.w, "{}, ", n.val)?; + if _def.argdefs.len() == 1 { + write!(self.w, "{}", _def.argdefs.first().unwrap())?; + } else { + let mut prefix = ""; + for n in _def.argdefs.iter() { + write!(self.w, "{}{}", prefix, n.val)?; + prefix = ", "; + } } self.w.write(") => ".as_bytes())?; self.render_expr(&_def.fields)?; diff --git a/src/ast/printer/test.rs b/src/ast/printer/test.rs index 9c66a20..0a1aca3 100644 --- a/src/ast/printer/test.rs +++ b/src/ast/printer/test.rs @@ -306,3 +306,30 @@ fn test_module_with_out_expr_printing() { assert!(printer.err.is_none()); assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); } + +#[test] +fn test_func_expr_printing() { + let input = "let f = func (foo, bar) => { + foo = foo, + bar = bar, +};"; + let stmts = assert_parse(input); + let mut buffer: Vec = Vec::new(); + let mut printer = AstPrinter::new(2, &mut buffer); + printer.render(&stmts); + assert!(printer.err.is_none()); + assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); +} + +#[test] +fn test_func_expr_single_arg_printing() { + let input = "let f = func (foo) => { + foo = foo, +};"; + let stmts = assert_parse(input); + let mut buffer: Vec = Vec::new(); + let mut printer = AstPrinter::new(2, &mut buffer); + printer.render(&stmts); + assert!(printer.err.is_none()); + assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); +} From aa9e664c0d05e6a5870a441140599ac883a34f5e Mon Sep 17 00:00:00 2001 From: Jeremy Wall Date: Fri, 17 May 2019 21:19:01 -0500 Subject: [PATCH 08/23] FEATURE: Improvements to format expression printing. --- src/ast/printer/mod.rs | 10 +++++----- src/ast/printer/test.rs | 26 ++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/src/ast/printer/mod.rs b/src/ast/printer/mod.rs index 1738607..b0517c5 100644 --- a/src/ast/printer/mod.rs +++ b/src/ast/printer/mod.rs @@ -248,21 +248,21 @@ where self.render_expr(&_def.message)?; } Expression::Format(_def) => { - self.w - .write(Self::escape_quotes(&_def.template).as_bytes())?; + write!(self.w, "\"{}\"", Self::escape_quotes(&_def.template))?; write!(self.w, " % ")?; match _def.args { FormatArgs::Single(ref e) => { self.render_expr(e)?; } FormatArgs::List(ref es) => { - self.w.write("(".as_bytes())?; + self.w.write("(\n".as_bytes())?; self.curr_indent += self.indent; let indent = self.make_indent(); + let mut prefix = ""; for e in es.iter() { - self.w.write(indent.as_bytes())?; + write!(self.w, "{}{}", prefix, indent)?; self.render_expr(e)?; - self.w.write("\n".as_bytes())?; + prefix = ",\n"; } self.curr_indent -= self.indent; self.w.write(")".as_bytes())?; diff --git a/src/ast/printer/test.rs b/src/ast/printer/test.rs index 0a1aca3..e427921 100644 --- a/src/ast/printer/test.rs +++ b/src/ast/printer/test.rs @@ -333,3 +333,29 @@ fn test_func_expr_single_arg_printing() { assert!(printer.err.is_none()); assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); } + +#[test] +fn test_format_expr_single_arg_printing() { + let input = "\"what? @{item.foo}\" % { + foo = 1, +};"; + let stmts = assert_parse(input); + let mut buffer: Vec = Vec::new(); + let mut printer = AstPrinter::new(2, &mut buffer); + printer.render(&stmts); + assert!(printer.err.is_none()); + assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); +} + +#[test] +fn test_format_expr_list_arg_printing() { + let input = "\"what? @ @\" % ( + 1, + 2);"; + let stmts = assert_parse(input); + let mut buffer: Vec = Vec::new(); + let mut printer = AstPrinter::new(2, &mut buffer); + printer.render(&stmts); + assert!(printer.err.is_none()); + assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); +} From f3a08718dc92cadafaea19afd9c26aaf1ef26652 Mon Sep 17 00:00:00 2001 From: Jeremy Wall Date: Fri, 17 May 2019 21:21:20 -0500 Subject: [PATCH 09/23] MAINT: Tests for fail and trace expressions. --- src/ast/printer/test.rs | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/ast/printer/test.rs b/src/ast/printer/test.rs index e427921..41511e1 100644 --- a/src/ast/printer/test.rs +++ b/src/ast/printer/test.rs @@ -265,6 +265,28 @@ fn test_not_expr_printing() { assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); } +#[test] +fn test_fail_expr_printing() { + let input = "fail \"AHHh\";"; + let stmts = assert_parse(input); + let mut buffer: Vec = Vec::new(); + let mut printer = AstPrinter::new(2, &mut buffer); + printer.render(&stmts); + assert!(printer.err.is_none()); + assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); +} + +#[test] +fn test_trace_expr_printing() { + let input = "TRACE \"AHHh\";"; + let stmts = assert_parse(input); + let mut buffer: Vec = Vec::new(); + let mut printer = AstPrinter::new(2, &mut buffer); + printer.render(&stmts); + assert!(printer.err.is_none()); + assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); +} + #[test] fn test_module_no_out_expr_printing() { let input = "let m = module { From 6661e02a759e9ebca0e25b87065529c428ce4fb5 Mon Sep 17 00:00:00 2001 From: Jeremy Wall Date: Mon, 20 May 2019 21:02:51 -0500 Subject: [PATCH 10/23] DEV: Generate a comment map as part of our tokenization. The comment_map is optional but if passed in it will be populated during tokenization. --- src/ast/printer/mod.rs | 47 +++++++----------------------- src/ast/printer/test.rs | 64 ++++++++++++++++++++--------------------- src/benches/parse.rs | 2 +- src/build/mod.rs | 2 +- src/parse/mod.rs | 9 ++++-- src/tokenizer/mod.rs | 41 ++++++++++++++++++++++---- src/tokenizer/test.rs | 6 ++-- 7 files changed, 90 insertions(+), 81 deletions(-) diff --git a/src/ast/printer/mod.rs b/src/ast/printer/mod.rs index b0517c5..172ddd4 100644 --- a/src/ast/printer/mod.rs +++ b/src/ast/printer/mod.rs @@ -11,27 +11,30 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -use std::borrow::BorrowMut; use std::io::Write; use crate::ast::*; +use crate::parse::CommentMap; // TODO(jwall): We really need a way to preserve comments for these. // Perhaps for code formatting we actually want to work on the token stream instead? -pub struct AstPrinter +pub struct AstPrinter<'a, W> where W: Write, { indent: usize, curr_indent: usize, w: W, + // Indexed by line that the comment was on. + // We use this to determine when to print a comment in our AstPrinter + comment_map: Option<&'a CommentMap>, pub err: Option, } // TODO(jwall): At some point we probably want to be more aware of line length // in our formatting. But not at the moment. -impl AstPrinter +impl<'a, W> AstPrinter<'a, W> where W: Write, { @@ -39,45 +42,15 @@ where AstPrinter { indent: indent, curr_indent: 0, + comment_map: None, w: w, err: None, } } - pub fn visit_token(&mut self, t: &Token) -> std::io::Result<()> { - let w: &mut Write = self.w.borrow_mut(); - // Do we care about line length? - match t.typ { - TokenType::BAREWORD | TokenType::BOOLEAN | TokenType::DIGIT => { - write!(w, "{}", t.fragment)?; - } - TokenType::EMPTY => { - write!(w, "NULL")?; - } - TokenType::PUNCT => { - // TODO(jwall): We need to identify the points at which we - // introduce new lines and new indentation scopes. - } - TokenType::COMMENT => { - // We need to track some state here probably. - // Do we leave comments untouched? - } - TokenType::PIPEQUOTE => { - // FIXME I think is supposed to be removed. - } - TokenType::QUOTED => { - w.write(&['"' as u8])?; - write!(w, "{}", Self::escape_quotes(&t.fragment))?; - w.write(&['"' as u8])?; - } - TokenType::WS => { - // TODO(jwall): Track some state around new lines here? - } - TokenType::END => { - // NOOP - } - }; - Ok(()) + pub fn with_comment_map(mut self, map: &'a CommentMap) -> Self { + self.comment_map = Some(map); + self } fn make_indent(&self) -> String { diff --git a/src/ast/printer/test.rs b/src/ast/printer/test.rs index 41511e1..86db9ba 100644 --- a/src/ast/printer/test.rs +++ b/src/ast/printer/test.rs @@ -16,14 +16,14 @@ use crate::ast::printer::*; use crate::iter::OffsetStrIter; use crate::parse::*; -fn assert_parse(input: &str) -> Vec { - parse(OffsetStrIter::new(input)).unwrap() +fn assert_parse(input: &str, comment_map: Option<&mut CommentMap>) -> Vec { + parse(OffsetStrIter::new(input), comment_map).unwrap() } #[test] fn test_simple_value_printing() { let input = "1;"; - let stmts = assert_parse(input); + let stmts = assert_parse(input, None); let mut buffer: Vec = Vec::new(); let mut printer = AstPrinter::new(0, &mut buffer); printer.render(&stmts); @@ -34,7 +34,7 @@ fn test_simple_value_printing() { #[test] fn test_simple_selector_printing() { let input = "foo.bar.quux;"; - let stmts = assert_parse(input); + let stmts = assert_parse(input, None); let mut buffer: Vec = Vec::new(); let mut printer = AstPrinter::new(0, &mut buffer); printer.render(&stmts); @@ -45,7 +45,7 @@ fn test_simple_selector_printing() { #[test] fn test_simple_quoted_printing() { let input = "\"foo\";"; - let stmts = assert_parse(input); + let stmts = assert_parse(input, None); let mut buffer: Vec = Vec::new(); let mut printer = AstPrinter::new(0, &mut buffer); printer.render(&stmts); @@ -56,7 +56,7 @@ fn test_simple_quoted_printing() { #[test] fn test_escaped_quoted_printing() { let input = "\"f\\\\o\\\"o\";"; - let stmts = assert_parse(input); + let stmts = assert_parse(input, None); let mut buffer: Vec = Vec::new(); let mut printer = AstPrinter::new(0, &mut buffer); printer.render(&stmts); @@ -67,7 +67,7 @@ fn test_escaped_quoted_printing() { #[test] fn test_empty_tuple_printing() { let input = "{};"; - let stmts = assert_parse(input); + let stmts = assert_parse(input, None); let mut buffer: Vec = Vec::new(); let mut printer = AstPrinter::new(2, &mut buffer); printer.render(&stmts); @@ -78,7 +78,7 @@ fn test_empty_tuple_printing() { #[test] fn test_empty_list_printing() { let input = "[];"; - let stmts = assert_parse(input); + let stmts = assert_parse(input, None); let mut buffer: Vec = Vec::new(); let mut printer = AstPrinter::new(2, &mut buffer); printer.render(&stmts); @@ -89,7 +89,7 @@ fn test_empty_list_printing() { #[test] fn test_non_empty_tuple_printing() { let input = "{\n foo = 1,\n};"; - let stmts = assert_parse(input); + let stmts = assert_parse(input, None); let mut buffer: Vec = Vec::new(); let mut printer = AstPrinter::new(2, &mut buffer); printer.render(&stmts); @@ -100,7 +100,7 @@ fn test_non_empty_tuple_printing() { #[test] fn test_nested_empty_tuple_printing() { let input = "{\n foo = {},\n};"; - let stmts = assert_parse(input); + let stmts = assert_parse(input, None); let mut buffer: Vec = Vec::new(); let mut printer = AstPrinter::new(2, &mut buffer); printer.render(&stmts); @@ -111,7 +111,7 @@ fn test_nested_empty_tuple_printing() { #[test] fn test_list_nested_empty_tuple_printing() { let input = "[\n {},\n];"; - let stmts = assert_parse(input); + let stmts = assert_parse(input, None); let mut buffer: Vec = Vec::new(); let mut printer = AstPrinter::new(2, &mut buffer); printer.render(&stmts); @@ -122,7 +122,7 @@ fn test_list_nested_empty_tuple_printing() { #[test] fn test_nested_non_empty_tuple_printing() { let input = "{\n foo = {\n bar = 1,\n },\n};"; - let stmts = assert_parse(input); + let stmts = assert_parse(input, None); let mut buffer: Vec = Vec::new(); let mut printer = AstPrinter::new(2, &mut buffer); printer.render(&stmts); @@ -133,7 +133,7 @@ fn test_nested_non_empty_tuple_printing() { #[test] fn test_nested_non_empty_list_printing() { let input = "[\n [\n 1,\n ],\n];"; - let stmts = assert_parse(input); + let stmts = assert_parse(input, None); let mut buffer: Vec = Vec::new(); let mut printer = AstPrinter::new(2, &mut buffer); printer.render(&stmts); @@ -144,7 +144,7 @@ fn test_nested_non_empty_list_printing() { #[test] fn test_simple_quoted_field_tuple_printing() { let input = "{\n \"foo\" = {\n bar = 1,\n },\n};"; - let stmts = assert_parse(input); + let stmts = assert_parse(input, None); let mut buffer: Vec = Vec::new(); let mut printer = AstPrinter::new(2, &mut buffer); printer.render(&stmts); @@ -158,7 +158,7 @@ fn test_simple_quoted_field_tuple_printing() { #[test] fn test_special_quoted_field_tuple_printing() { let input = "{\n \"foo bar\" = {\n bar = 1,\n },\n};"; - let stmts = assert_parse(input); + let stmts = assert_parse(input, None); let mut buffer: Vec = Vec::new(); let mut printer = AstPrinter::new(2, &mut buffer); printer.render(&stmts); @@ -169,7 +169,7 @@ fn test_special_quoted_field_tuple_printing() { #[test] fn test_let_statement_printing() { let input = "let tpl = {\n \"foo bar\" = {\n bar = 1,\n },\n};"; - let stmts = assert_parse(input); + let stmts = assert_parse(input, None); let mut buffer: Vec = Vec::new(); let mut printer = AstPrinter::new(2, &mut buffer); printer.render(&stmts); @@ -180,7 +180,7 @@ fn test_let_statement_printing() { #[test] fn test_call_expr_printing() { let input = "call(\n foo,\n bar,\n);"; - let stmts = assert_parse(input); + let stmts = assert_parse(input, None); let mut buffer: Vec = Vec::new(); let mut printer = AstPrinter::new(2, &mut buffer); printer.render(&stmts); @@ -191,7 +191,7 @@ fn test_call_expr_printing() { #[test] fn test_call_expr_one_arg_printing() { let input = "call(foo);"; - let stmts = assert_parse(input); + let stmts = assert_parse(input, None); let mut buffer: Vec = Vec::new(); let mut printer = AstPrinter::new(2, &mut buffer); printer.render(&stmts); @@ -202,7 +202,7 @@ fn test_call_expr_one_arg_printing() { #[test] fn test_copy_expr_printing() { let input = "copy{\n foo = 1,\n bar = 2,\n};"; - let stmts = assert_parse(input); + let stmts = assert_parse(input, None); let mut buffer: Vec = Vec::new(); let mut printer = AstPrinter::new(2, &mut buffer); printer.render(&stmts); @@ -213,7 +213,7 @@ fn test_copy_expr_printing() { #[test] fn test_copy_expr_one_arg_printing() { let input = "copy{\n foo = 1,\n};"; - let stmts = assert_parse(input); + let stmts = assert_parse(input, None); let mut buffer: Vec = Vec::new(); let mut printer = AstPrinter::new(2, &mut buffer); printer.render(&stmts); @@ -224,7 +224,7 @@ fn test_copy_expr_one_arg_printing() { #[test] fn test_out_expr_printing() { let input = "out json {\n foo = 1,\n};"; - let stmts = assert_parse(input); + let stmts = assert_parse(input, None); let mut buffer: Vec = Vec::new(); let mut printer = AstPrinter::new(2, &mut buffer); printer.render(&stmts); @@ -235,7 +235,7 @@ fn test_out_expr_printing() { #[test] fn test_select_expr_no_default_printing() { let input = "select true, {\n true = 1,\n false = 2,\n};"; - let stmts = assert_parse(input); + let stmts = assert_parse(input, None); let mut buffer: Vec = Vec::new(); let mut printer = AstPrinter::new(2, &mut buffer); printer.render(&stmts); @@ -246,7 +246,7 @@ fn test_select_expr_no_default_printing() { #[test] fn test_select_expr_with_default_printing() { let input = "select true, 3, {\n true = 1,\n false = 2,\n};"; - let stmts = assert_parse(input); + let stmts = assert_parse(input, None); let mut buffer: Vec = Vec::new(); let mut printer = AstPrinter::new(2, &mut buffer); printer.render(&stmts); @@ -257,7 +257,7 @@ fn test_select_expr_with_default_printing() { #[test] fn test_not_expr_printing() { let input = "not true;"; - let stmts = assert_parse(input); + let stmts = assert_parse(input, None); let mut buffer: Vec = Vec::new(); let mut printer = AstPrinter::new(2, &mut buffer); printer.render(&stmts); @@ -268,7 +268,7 @@ fn test_not_expr_printing() { #[test] fn test_fail_expr_printing() { let input = "fail \"AHHh\";"; - let stmts = assert_parse(input); + let stmts = assert_parse(input, None); let mut buffer: Vec = Vec::new(); let mut printer = AstPrinter::new(2, &mut buffer); printer.render(&stmts); @@ -279,7 +279,7 @@ fn test_fail_expr_printing() { #[test] fn test_trace_expr_printing() { let input = "TRACE \"AHHh\";"; - let stmts = assert_parse(input); + let stmts = assert_parse(input, None); let mut buffer: Vec = Vec::new(); let mut printer = AstPrinter::new(2, &mut buffer); printer.render(&stmts); @@ -300,7 +300,7 @@ fn test_module_no_out_expr_printing() { \"cpu_count\" = mod.cpu, }; };"; - let stmts = assert_parse(input); + let stmts = assert_parse(input, None); let mut buffer: Vec = Vec::new(); let mut printer = AstPrinter::new(2, &mut buffer); printer.render(&stmts); @@ -321,7 +321,7 @@ fn test_module_with_out_expr_printing() { \"cpu_count\" = mod.cpu, }; };"; - let stmts = assert_parse(input); + let stmts = assert_parse(input, None); let mut buffer: Vec = Vec::new(); let mut printer = AstPrinter::new(2, &mut buffer); printer.render(&stmts); @@ -335,7 +335,7 @@ fn test_func_expr_printing() { foo = foo, bar = bar, };"; - let stmts = assert_parse(input); + let stmts = assert_parse(input, None); let mut buffer: Vec = Vec::new(); let mut printer = AstPrinter::new(2, &mut buffer); printer.render(&stmts); @@ -348,7 +348,7 @@ fn test_func_expr_single_arg_printing() { let input = "let f = func (foo) => { foo = foo, };"; - let stmts = assert_parse(input); + let stmts = assert_parse(input, None); let mut buffer: Vec = Vec::new(); let mut printer = AstPrinter::new(2, &mut buffer); printer.render(&stmts); @@ -361,7 +361,7 @@ fn test_format_expr_single_arg_printing() { let input = "\"what? @{item.foo}\" % { foo = 1, };"; - let stmts = assert_parse(input); + let stmts = assert_parse(input, None); let mut buffer: Vec = Vec::new(); let mut printer = AstPrinter::new(2, &mut buffer); printer.render(&stmts); @@ -374,7 +374,7 @@ fn test_format_expr_list_arg_printing() { let input = "\"what? @ @\" % ( 1, 2);"; - let stmts = assert_parse(input); + let stmts = assert_parse(input, None); let mut buffer: Vec = Vec::new(); let mut printer = AstPrinter::new(2, &mut buffer); printer.render(&stmts); diff --git a/src/benches/parse.rs b/src/benches/parse.rs index 61e81ff..782d7c7 100644 --- a/src/benches/parse.rs +++ b/src/benches/parse.rs @@ -26,7 +26,7 @@ use ucglib::iter::OffsetStrIter; use ucglib::parse::*; fn do_parse(i: &str) { - parse(OffsetStrIter::new(i)); + parse(OffsetStrIter::new(i), None); } fn parse_int(b: &mut Bencher) { diff --git a/src/build/mod.rs b/src/build/mod.rs index 4e23874..1864779 100644 --- a/src/build/mod.rs +++ b/src/build/mod.rs @@ -303,7 +303,7 @@ impl<'a> FileBuilder<'a> { } fn eval_input(&mut self, input: OffsetStrIter) -> Result, Box> { - match parse(input.clone()) { + match parse(input.clone(), None) { Ok(stmts) => { //panic!("Successfully parsed {}", input); let mut out: Option> = None; diff --git a/src/parse/mod.rs b/src/parse/mod.rs index 96bec15..a95054c 100644 --- a/src/parse/mod.rs +++ b/src/parse/mod.rs @@ -28,6 +28,8 @@ use crate::error::StackPrinter; use crate::iter::OffsetStrIter; use crate::tokenizer::*; +pub use crate::tokenizer::{CommentGroup, CommentMap}; + type ParseResult<'a, O> = Result, O>; #[cfg(feature = "tracing")] @@ -853,8 +855,11 @@ fn statement(i: SliceIter) -> Result, Statement> { //trace_macros!(false); /// Parses a LocatedSpan into a list of Statements or an `error::Error`. -pub fn parse<'a>(input: OffsetStrIter<'a>) -> std::result::Result, String> { - match tokenize(input.clone(), true) { +pub fn parse<'a>( + input: OffsetStrIter<'a>, + comment_map: Option<&mut CommentMap>, +) -> std::result::Result, String> { + match tokenize(input.clone(), comment_map) { Ok(tokenized) => { let mut out = Vec::new(); let mut i_ = SliceIter::new(&tokenized); diff --git a/src/tokenizer/mod.rs b/src/tokenizer/mod.rs index 07d5a74..e669342 100644 --- a/src/tokenizer/mod.rs +++ b/src/tokenizer/mod.rs @@ -23,6 +23,9 @@ use crate::ast::*; use crate::error::StackPrinter; use crate::iter::OffsetStrIter; +pub type CommentGroup = Vec; +pub type CommentMap = std::collections::HashMap; + fn is_symbol_char<'a>(i: OffsetStrIter<'a>) -> Result, u8> { let mut _i = i.clone(); let c = match _i.next() { @@ -452,12 +455,16 @@ fn token<'a>(input: OffsetStrIter<'a>) -> Result, Token> { } /// Consumes an input OffsetStrIter and returns either a Vec or a error::Error. +/// If a comment_map is passed in then it will store the comments indexed by their +/// line number. pub fn tokenize<'a>( input: OffsetStrIter<'a>, - skip_comments: bool, + mut comment_map: Option<&mut CommentMap>, ) -> std::result::Result, String> { let mut out = Vec::new(); let mut i = input.clone(); + let mut comment_group = Vec::new(); + let mut comment_was_last: Option = None; loop { if let Result::Complete(_, _) = eoi(i.clone()) { break; @@ -489,14 +496,38 @@ pub fn tokenize<'a>( } Result::Complete(rest, tok) => { i = rest; - if (skip_comments && tok.typ == TokenType::COMMENT) || tok.typ == TokenType::WS { - // we skip comments and whitespace - continue; + match (&mut comment_map, &tok.typ) { + // variants with a comment_map + (&mut Some(_), &TokenType::COMMENT) => { + comment_group.push(tok.clone()); + comment_was_last = Some(tok.clone()); + continue; + } + (&mut Some(ref mut map), _) => { + out.push(tok); + if let Some(tok) = comment_was_last { + map.insert(tok.pos.line, comment_group); + comment_group = Vec::new(); + } + } + // variants without a comment_map + (None, TokenType::WS) | (None, TokenType::COMMENT) => continue, + (None, _) => { + out.push(tok); + } } - out.push(tok); + comment_was_last = None; } } } + // if we had a comments at the end then we need to do a final + // insert into our map. + if let Some(ref mut map) = comment_map { + if let Some(ref tok) = comment_group.last() { + let line = tok.pos.line; + map.insert(line, comment_group); + } + } // ensure that we always have an END token to go off of. out.push(Token { fragment: String::new(), diff --git a/src/tokenizer/test.rs b/src/tokenizer/test.rs index 0ef757c..13d58e0 100644 --- a/src/tokenizer/test.rs +++ b/src/tokenizer/test.rs @@ -89,7 +89,7 @@ fn test_string_with_escaping() { #[test] fn test_tokenize_bareword_with_dash() { let input = OffsetStrIter::new("foo-bar "); - let result = tokenize(input.clone(), true); + let result = tokenize(input.clone(), None); assert!(result.is_ok(), format!("result {:?} is not ok", result)); if let Ok(toks) = result { assert_eq!(toks.len(), 2); @@ -157,7 +157,7 @@ fn test_tokenize_one_of_each() { "map out filter assert let import func select as => [ ] { } ; = % / * \ + - . ( ) , 1 . foo \"bar\" // comment\n ; true false == < > <= >= !=", ); - let result = tokenize(input.clone(), true); + let result = tokenize(input.clone(), None); assert!(result.is_ok(), format!("result {:?} is not ok", result)); let v = result.unwrap(); for (i, t) in v.iter().enumerate() { @@ -170,7 +170,7 @@ fn test_tokenize_one_of_each() { #[test] fn test_parse_has_end() { let input = OffsetStrIter::new("foo"); - let result = tokenize(input.clone(), true); + let result = tokenize(input.clone(), None); assert!(result.is_ok()); let v = result.unwrap(); assert_eq!(v.len(), 2); From d884ea9385080fed98c88bf884f2d9ef9a12f9fd Mon Sep 17 00:00:00 2001 From: Jeremy Wall Date: Mon, 20 May 2019 21:12:21 -0500 Subject: [PATCH 11/23] REFACTOR: Remove the err field, rename indent -> indent_size --- src/ast/printer/mod.rs | 37 +++++++---------- src/ast/printer/test.rs | 90 ++++++++++++++--------------------------- 2 files changed, 45 insertions(+), 82 deletions(-) diff --git a/src/ast/printer/mod.rs b/src/ast/printer/mod.rs index 172ddd4..0743c23 100644 --- a/src/ast/printer/mod.rs +++ b/src/ast/printer/mod.rs @@ -23,13 +23,12 @@ pub struct AstPrinter<'a, W> where W: Write, { - indent: usize, + indent_size: usize, curr_indent: usize, w: W, // Indexed by line that the comment was on. // We use this to determine when to print a comment in our AstPrinter comment_map: Option<&'a CommentMap>, - pub err: Option, } // TODO(jwall): At some point we probably want to be more aware of line length @@ -40,11 +39,10 @@ where { pub fn new(indent: usize, w: W) -> Self { AstPrinter { - indent: indent, + indent_size: indent, curr_indent: 0, comment_map: None, w: w, - err: None, } } @@ -81,7 +79,7 @@ where fn render_list_def(&mut self, def: &ListDef) -> std::io::Result<()> { write!(self.w, "[")?; - self.curr_indent += self.indent; + self.curr_indent += self.indent_size; let indent = self.make_indent(); let has_fields = def.elems.len() > 0; if has_fields { @@ -92,7 +90,7 @@ where self.render_expr(e)?; write!(self.w, ",\n")?; } - self.curr_indent -= self.indent; + self.curr_indent -= self.indent_size; if has_fields { write!(self.w, "{}", self.make_indent())?; } @@ -103,7 +101,7 @@ where fn render_tuple_def(&mut self, def: &Vec<(Token, Expression)>) -> std::io::Result<()> { self.w.write(&['{' as u8])?; // If the field list is just 1 we might be able to collapse the tuple. - self.curr_indent += self.indent; + self.curr_indent += self.indent_size; let indent = self.make_indent(); let has_fields = def.len() > 0; if has_fields { @@ -120,7 +118,7 @@ where write!(&mut self.w, ",")?; write!(self.w, "\n")?; } - self.curr_indent -= self.indent; + self.curr_indent -= self.indent_size; if has_fields { write!(self.w, "{}", self.make_indent())?; } @@ -187,7 +185,7 @@ where Expression::Call(_def) => { self.render_value(&_def.funcref)?; self.w.write("(".as_bytes())?; - self.curr_indent += self.indent; + self.curr_indent += self.indent_size; let indent = self.make_indent(); let has_args = _def.arglist.len() > 1; if has_args { @@ -202,7 +200,7 @@ where self.w.write(",\n".as_bytes())?; } } - self.curr_indent -= self.indent; + self.curr_indent -= self.indent_size; if has_args { write!(self.w, "{}", self.make_indent())?; } @@ -229,7 +227,7 @@ where } FormatArgs::List(ref es) => { self.w.write("(\n".as_bytes())?; - self.curr_indent += self.indent; + self.curr_indent += self.indent_size; let indent = self.make_indent(); let mut prefix = ""; for e in es.iter() { @@ -237,7 +235,7 @@ where self.render_expr(e)?; prefix = ",\n"; } - self.curr_indent -= self.indent; + self.curr_indent -= self.indent_size; self.w.write(")".as_bytes())?; } } @@ -311,13 +309,13 @@ where write!(self.w, ") ")?; } write!(self.w, "{{\n")?; - self.curr_indent += self.indent; + self.curr_indent += self.indent_size; let indent = self.make_indent(); for stmt in _def.statements.iter() { write!(self.w, "{}", indent)?; self.render_stmt(stmt)?; } - self.curr_indent -= self.indent; + self.curr_indent -= self.indent_size; write!(self.w, "}}")?; } Expression::Not(_def) => { @@ -360,30 +358,25 @@ where } Statement::Expression(_expr) => { self.render_expr(&_expr)?; - // } Statement::Assert(def) => { write!(&mut self.w, "assert ")?; self.render_expr(&def)?; - // } Statement::Output(_, _tok, _expr) => { write!(&mut self.w, "out {} ", _tok.fragment)?; self.render_expr(&_expr)?; - // } }; write!(self.w, ";\n")?; Ok(()) } - pub fn render(&mut self, stmts: &Vec) { + pub fn render(&mut self, stmts: &Vec) -> std::io::Result<()> { for v in stmts { - if let Err(e) = self.render_stmt(v) { - self.err = Some(e); - return; - } + self.render_stmt(v)?; } + Ok(()) } } diff --git a/src/ast/printer/test.rs b/src/ast/printer/test.rs index 86db9ba..2131a67 100644 --- a/src/ast/printer/test.rs +++ b/src/ast/printer/test.rs @@ -26,8 +26,7 @@ fn test_simple_value_printing() { let stmts = assert_parse(input, None); let mut buffer: Vec = Vec::new(); let mut printer = AstPrinter::new(0, &mut buffer); - printer.render(&stmts); - assert!(printer.err.is_none()); + assert!(printer.render(&stmts).is_ok()); assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); } @@ -37,8 +36,7 @@ fn test_simple_selector_printing() { let stmts = assert_parse(input, None); let mut buffer: Vec = Vec::new(); let mut printer = AstPrinter::new(0, &mut buffer); - printer.render(&stmts); - assert!(printer.err.is_none()); + assert!(printer.render(&stmts).is_ok()); assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); } @@ -48,8 +46,7 @@ fn test_simple_quoted_printing() { let stmts = assert_parse(input, None); let mut buffer: Vec = Vec::new(); let mut printer = AstPrinter::new(0, &mut buffer); - printer.render(&stmts); - assert!(printer.err.is_none()); + assert!(printer.render(&stmts).is_ok()); assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); } @@ -59,8 +56,7 @@ fn test_escaped_quoted_printing() { let stmts = assert_parse(input, None); let mut buffer: Vec = Vec::new(); let mut printer = AstPrinter::new(0, &mut buffer); - printer.render(&stmts); - assert!(printer.err.is_none()); + assert!(printer.render(&stmts).is_ok()); assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); } @@ -70,8 +66,7 @@ fn test_empty_tuple_printing() { let stmts = assert_parse(input, None); let mut buffer: Vec = Vec::new(); let mut printer = AstPrinter::new(2, &mut buffer); - printer.render(&stmts); - assert!(printer.err.is_none()); + assert!(printer.render(&stmts).is_ok()); assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); } @@ -81,8 +76,7 @@ fn test_empty_list_printing() { let stmts = assert_parse(input, None); let mut buffer: Vec = Vec::new(); let mut printer = AstPrinter::new(2, &mut buffer); - printer.render(&stmts); - assert!(printer.err.is_none()); + assert!(printer.render(&stmts).is_ok()); assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); } @@ -92,8 +86,7 @@ fn test_non_empty_tuple_printing() { let stmts = assert_parse(input, None); let mut buffer: Vec = Vec::new(); let mut printer = AstPrinter::new(2, &mut buffer); - printer.render(&stmts); - assert!(printer.err.is_none()); + assert!(printer.render(&stmts).is_ok()); assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); } @@ -103,8 +96,7 @@ fn test_nested_empty_tuple_printing() { let stmts = assert_parse(input, None); let mut buffer: Vec = Vec::new(); let mut printer = AstPrinter::new(2, &mut buffer); - printer.render(&stmts); - assert!(printer.err.is_none()); + assert!(printer.render(&stmts).is_ok()); assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); } @@ -114,8 +106,7 @@ fn test_list_nested_empty_tuple_printing() { let stmts = assert_parse(input, None); let mut buffer: Vec = Vec::new(); let mut printer = AstPrinter::new(2, &mut buffer); - printer.render(&stmts); - assert!(printer.err.is_none()); + assert!(printer.render(&stmts).is_ok()); assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); } @@ -125,8 +116,7 @@ fn test_nested_non_empty_tuple_printing() { let stmts = assert_parse(input, None); let mut buffer: Vec = Vec::new(); let mut printer = AstPrinter::new(2, &mut buffer); - printer.render(&stmts); - assert!(printer.err.is_none()); + assert!(printer.render(&stmts).is_ok()); assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); } @@ -136,8 +126,7 @@ fn test_nested_non_empty_list_printing() { let stmts = assert_parse(input, None); let mut buffer: Vec = Vec::new(); let mut printer = AstPrinter::new(2, &mut buffer); - printer.render(&stmts); - assert!(printer.err.is_none()); + assert!(printer.render(&stmts).is_ok()); assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); } @@ -147,8 +136,7 @@ fn test_simple_quoted_field_tuple_printing() { let stmts = assert_parse(input, None); let mut buffer: Vec = Vec::new(); let mut printer = AstPrinter::new(2, &mut buffer); - printer.render(&stmts); - assert!(printer.err.is_none()); + assert!(printer.render(&stmts).is_ok()); assert_eq!( String::from_utf8(buffer).unwrap(), format!("{}\n", "{\n foo = {\n bar = 1,\n },\n};") @@ -161,8 +149,7 @@ fn test_special_quoted_field_tuple_printing() { let stmts = assert_parse(input, None); let mut buffer: Vec = Vec::new(); let mut printer = AstPrinter::new(2, &mut buffer); - printer.render(&stmts); - assert!(printer.err.is_none()); + assert!(printer.render(&stmts).is_ok()); assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); } @@ -172,8 +159,7 @@ fn test_let_statement_printing() { let stmts = assert_parse(input, None); let mut buffer: Vec = Vec::new(); let mut printer = AstPrinter::new(2, &mut buffer); - printer.render(&stmts); - assert!(printer.err.is_none()); + assert!(printer.render(&stmts).is_ok()); assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); } @@ -183,8 +169,7 @@ fn test_call_expr_printing() { let stmts = assert_parse(input, None); let mut buffer: Vec = Vec::new(); let mut printer = AstPrinter::new(2, &mut buffer); - printer.render(&stmts); - assert!(printer.err.is_none()); + assert!(printer.render(&stmts).is_ok()); assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); } @@ -194,8 +179,7 @@ fn test_call_expr_one_arg_printing() { let stmts = assert_parse(input, None); let mut buffer: Vec = Vec::new(); let mut printer = AstPrinter::new(2, &mut buffer); - printer.render(&stmts); - assert!(printer.err.is_none()); + assert!(printer.render(&stmts).is_ok()); assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); } @@ -205,8 +189,7 @@ fn test_copy_expr_printing() { let stmts = assert_parse(input, None); let mut buffer: Vec = Vec::new(); let mut printer = AstPrinter::new(2, &mut buffer); - printer.render(&stmts); - assert!(printer.err.is_none()); + assert!(printer.render(&stmts).is_ok()); assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); } @@ -216,8 +199,7 @@ fn test_copy_expr_one_arg_printing() { let stmts = assert_parse(input, None); let mut buffer: Vec = Vec::new(); let mut printer = AstPrinter::new(2, &mut buffer); - printer.render(&stmts); - assert!(printer.err.is_none()); + assert!(printer.render(&stmts).is_ok()); assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); } @@ -227,8 +209,7 @@ fn test_out_expr_printing() { let stmts = assert_parse(input, None); let mut buffer: Vec = Vec::new(); let mut printer = AstPrinter::new(2, &mut buffer); - printer.render(&stmts); - assert!(printer.err.is_none()); + assert!(printer.render(&stmts).is_ok()); assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); } @@ -238,8 +219,7 @@ fn test_select_expr_no_default_printing() { let stmts = assert_parse(input, None); let mut buffer: Vec = Vec::new(); let mut printer = AstPrinter::new(2, &mut buffer); - printer.render(&stmts); - assert!(printer.err.is_none()); + assert!(printer.render(&stmts).is_ok()); assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); } @@ -249,8 +229,7 @@ fn test_select_expr_with_default_printing() { let stmts = assert_parse(input, None); let mut buffer: Vec = Vec::new(); let mut printer = AstPrinter::new(2, &mut buffer); - printer.render(&stmts); - assert!(printer.err.is_none()); + assert!(printer.render(&stmts).is_ok()); assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); } @@ -260,8 +239,7 @@ fn test_not_expr_printing() { let stmts = assert_parse(input, None); let mut buffer: Vec = Vec::new(); let mut printer = AstPrinter::new(2, &mut buffer); - printer.render(&stmts); - assert!(printer.err.is_none()); + assert!(printer.render(&stmts).is_ok()); assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); } @@ -271,8 +249,7 @@ fn test_fail_expr_printing() { let stmts = assert_parse(input, None); let mut buffer: Vec = Vec::new(); let mut printer = AstPrinter::new(2, &mut buffer); - printer.render(&stmts); - assert!(printer.err.is_none()); + assert!(printer.render(&stmts).is_ok()); assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); } @@ -282,8 +259,7 @@ fn test_trace_expr_printing() { let stmts = assert_parse(input, None); let mut buffer: Vec = Vec::new(); let mut printer = AstPrinter::new(2, &mut buffer); - printer.render(&stmts); - assert!(printer.err.is_none()); + assert!(printer.render(&stmts).is_ok()); assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); } @@ -303,8 +279,7 @@ fn test_module_no_out_expr_printing() { let stmts = assert_parse(input, None); let mut buffer: Vec = Vec::new(); let mut printer = AstPrinter::new(2, &mut buffer); - printer.render(&stmts); - assert!(printer.err.is_none()); + assert!(printer.render(&stmts).is_ok()); assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); } @@ -324,8 +299,7 @@ fn test_module_with_out_expr_printing() { let stmts = assert_parse(input, None); let mut buffer: Vec = Vec::new(); let mut printer = AstPrinter::new(2, &mut buffer); - printer.render(&stmts); - assert!(printer.err.is_none()); + assert!(printer.render(&stmts).is_ok()); assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); } @@ -338,8 +312,7 @@ fn test_func_expr_printing() { let stmts = assert_parse(input, None); let mut buffer: Vec = Vec::new(); let mut printer = AstPrinter::new(2, &mut buffer); - printer.render(&stmts); - assert!(printer.err.is_none()); + assert!(printer.render(&stmts).is_ok()); assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); } @@ -351,8 +324,7 @@ fn test_func_expr_single_arg_printing() { let stmts = assert_parse(input, None); let mut buffer: Vec = Vec::new(); let mut printer = AstPrinter::new(2, &mut buffer); - printer.render(&stmts); - assert!(printer.err.is_none()); + assert!(printer.render(&stmts).is_ok()); assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); } @@ -364,8 +336,7 @@ fn test_format_expr_single_arg_printing() { let stmts = assert_parse(input, None); let mut buffer: Vec = Vec::new(); let mut printer = AstPrinter::new(2, &mut buffer); - printer.render(&stmts); - assert!(printer.err.is_none()); + assert!(printer.render(&stmts).is_ok()); assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); } @@ -377,7 +348,6 @@ fn test_format_expr_list_arg_printing() { let stmts = assert_parse(input, None); let mut buffer: Vec = Vec::new(); let mut printer = AstPrinter::new(2, &mut buffer); - printer.render(&stmts); - assert!(printer.err.is_none()); + assert!(printer.render(&stmts).is_ok()); assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); } From 957d0c610203cb93f600b036becce39a9d91713f Mon Sep 17 00:00:00 2001 From: Jeremy Wall Date: Tue, 21 May 2019 20:34:42 -0500 Subject: [PATCH 12/23] DEV: Handle comments between statements. --- src/ast/mod.rs | 14 +++++++++- src/ast/printer/mod.rs | 58 ++++++++++++++++++++++++++++++++++++++++- src/ast/printer/test.rs | 51 ++++++++++++++++++++++++++++++++++++ src/ast/walk.rs | 2 +- src/build/mod.rs | 2 +- src/parse/mod.rs | 17 ++++++------ src/tokenizer/mod.rs | 12 +++++++-- src/tokenizer/test.rs | 40 ++++++++++++++++++++++++++++ 8 files changed, 181 insertions(+), 15 deletions(-) diff --git a/src/ast/mod.rs b/src/ast/mod.rs index f3f4b2c..0c7bfe1 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -756,6 +756,7 @@ impl fmt::Display for Expression { /// Encodes a let statement in the UCG AST. #[derive(Debug, PartialEq, Clone)] pub struct LetDef { + pub pos: Position, pub name: Token, pub value: Expression, } @@ -770,8 +771,19 @@ pub enum Statement { Let(LetDef), // Assert statement - Assert(Expression), + Assert(Position, Expression), // Identify an Expression for output. Output(Position, Token, Expression), } + +impl Statement { + fn pos(&self) -> &Position { + match self { + Statement::Expression(ref e) => e.pos(), + Statement::Let(ref def) => &def.pos, + Statement::Assert(ref pos, _) => pos, + Statement::Output(ref pos, _, _) => pos, + } + } +} diff --git a/src/ast/printer/mod.rs b/src/ast/printer/mod.rs index 0743c23..6c19e1d 100644 --- a/src/ast/printer/mod.rs +++ b/src/ast/printer/mod.rs @@ -29,6 +29,8 @@ where // Indexed by line that the comment was on. // We use this to determine when to print a comment in our AstPrinter comment_map: Option<&'a CommentMap>, + last_line: usize, + comment_group_lines: Vec, } // TODO(jwall): At some point we probably want to be more aware of line length @@ -43,10 +45,14 @@ where curr_indent: 0, comment_map: None, w: w, + last_line: 0, + comment_group_lines: Vec::new(), } } pub fn with_comment_map(mut self, map: &'a CommentMap) -> Self { + self.comment_group_lines = map.keys().cloned().collect(); + self.comment_group_lines.reverse(); self.comment_map = Some(map); self } @@ -77,6 +83,47 @@ where return true; } + fn print_comment_group(&mut self, line: usize) -> std::io::Result<()> { + if let Some(ref map) = self.comment_map { + let empty: Vec = Vec::new(); + //eprintln!("comment line candidate: {}", line); + let cg = map.get(&line).unwrap_or(&empty); + //eprintln!("comment_group: {:?}", cg); + for c in cg.iter() { + write!(self.w, "// {}\n", c.fragment.trim())?; + } + self.comment_group_lines.pop(); + } + Ok(()) + } + + fn render_missed_comments(&mut self, line: usize) -> std::io::Result<()> { + loop { + if let Some(next_comment_line) = self.comment_group_lines.last() { + let next_comment_line = *next_comment_line; + if next_comment_line < line { + self.print_comment_group(next_comment_line)?; + } else { + break; + } + if next_comment_line < line - 1 { + write!(self.w, "\n")?; + } + continue; + } + break; + } + Ok(()) + } + + fn render_comment_if_needed(&mut self, line: usize) -> std::io::Result<()> { + if line > self.last_line { + self.render_missed_comments(line)?; + self.last_line = line; + } + Ok(()) + } + fn render_list_def(&mut self, def: &ListDef) -> std::io::Result<()> { write!(self.w, "[")?; self.curr_indent += self.indent_size; @@ -351,6 +398,8 @@ where pub fn render_stmt(&mut self, stmt: &Statement) -> std::io::Result<()> { // All statements start at the beginning of a line. + let line = stmt.pos().line; + self.render_comment_if_needed(line)?; match stmt { Statement::Let(def) => { write!(&mut self.w, "let {} = ", def.name.fragment)?; @@ -359,7 +408,7 @@ where Statement::Expression(_expr) => { self.render_expr(&_expr)?; } - Statement::Assert(def) => { + Statement::Assert(_, def) => { write!(&mut self.w, "assert ")?; self.render_expr(&def)?; } @@ -369,6 +418,7 @@ where } }; write!(self.w, ";\n")?; + self.last_line = line; Ok(()) } @@ -376,6 +426,12 @@ where for v in stmts { self.render_stmt(v)?; } + if let Some(last_comment_line) = self.comment_group_lines.first() { + eprintln!("last_comment_line is: {}", last_comment_line); + eprintln!("comment_map is: {:?}", self.comment_map); + eprintln!("coment_group_lines is: {:?}", self.comment_group_lines); + self.render_missed_comments(*last_comment_line + 1)?; + } Ok(()) } } diff --git a/src/ast/printer/test.rs b/src/ast/printer/test.rs index 2131a67..55a2f53 100644 --- a/src/ast/printer/test.rs +++ b/src/ast/printer/test.rs @@ -11,6 +11,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. +use std::collections::BTreeMap; use crate::ast::printer::*; use crate::iter::OffsetStrIter; @@ -351,3 +352,53 @@ fn test_format_expr_list_arg_printing() { assert!(printer.render(&stmts).is_ok()); assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); } + +#[test] +fn test_statement_with_comment_printing() { + let mut comment_map = BTreeMap::new(); + let input = "// add 1 + 1\n1 + 1;"; + let stmts = assert_parse(input, Some(&mut comment_map)); + let mut buffer: Vec = Vec::new(); + let mut printer = AstPrinter::new(2, &mut buffer).with_comment_map(&comment_map); + assert!(printer.render(&stmts).is_ok()); + assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); +} + +#[test] +fn test_statement_with_comment_printing_groups() { + let mut comment_map = BTreeMap::new(); + let input = "// add 1\n// and 1\n1 + 1;"; + let stmts = assert_parse(input, Some(&mut comment_map)); + let mut buffer: Vec = Vec::new(); + let mut printer = AstPrinter::new(2, &mut buffer).with_comment_map(&comment_map); + assert!(printer.render(&stmts).is_ok()); + assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); +} + +#[test] +fn test_statement_with_comment_printing_multiple_groups() { + let mut comment_map = BTreeMap::new(); + let input = "\n// group 1\n// more group 1\n\n// group 2\n// more group 2\n1 + 1;"; + let stmts = assert_parse(input, Some(&mut comment_map)); + let mut buffer: Vec = Vec::new(); + let mut printer = AstPrinter::new(2, &mut buffer).with_comment_map(&comment_map); + assert!(printer.render(&stmts).is_ok()); + assert_eq!( + String::from_utf8(buffer).unwrap(), + format!("{}\n", input.trim()) + ); +} + +#[test] +fn test_statement_with_comment_printing_comments_at_end() { + let mut comment_map = BTreeMap::new(); + let input = "// group 1\n1 + 1;\n// group 2\n\n"; + let stmts = assert_parse(input, Some(&mut comment_map)); + let mut buffer: Vec = Vec::new(); + let mut printer = AstPrinter::new(2, &mut buffer).with_comment_map(&comment_map); + assert!(printer.render(&stmts).is_ok()); + assert_eq!( + String::from_utf8(buffer).unwrap(), + format!("{}\n", input.trim()) + ); +} diff --git a/src/ast/walk.rs b/src/ast/walk.rs index df402df..349820e 100644 --- a/src/ast/walk.rs +++ b/src/ast/walk.rs @@ -16,7 +16,7 @@ pub trait Walker { Statement::Expression(ref mut expr) => { self.walk_expression(expr); } - Statement::Assert(ref mut expr) => { + Statement::Assert(_, ref mut expr) => { self.walk_expression(expr); } Statement::Output(_, _, ref mut expr) => { diff --git a/src/build/mod.rs b/src/build/mod.rs index 1864779..e68309d 100644 --- a/src/build/mod.rs +++ b/src/build/mod.rs @@ -511,7 +511,7 @@ impl<'a> FileBuilder<'a> { fn eval_stmt(&mut self, stmt: &Statement) -> Result, Box> { let child_scope = self.scope.clone(); match stmt { - &Statement::Assert(ref expr) => self.eval_assert(&expr, &child_scope), + &Statement::Assert(_, ref expr) => self.eval_assert(&expr, &child_scope), &Statement::Let(ref def) => self.eval_let(def), &Statement::Expression(ref expr) => self.eval_expr(expr, &child_scope), // Only one output can be used per file. Right now we enforce this by diff --git a/src/parse/mod.rs b/src/parse/mod.rs index a95054c..e436c19 100644 --- a/src/parse/mod.rs +++ b/src/parse/mod.rs @@ -793,21 +793,19 @@ make_fn!( ) ); -fn tuple_to_let(tok: Token, expr: Expression) -> Statement { - Statement::Let(LetDef { - name: tok, - value: expr, - }) -} - make_fn!( let_stmt_body, Statement>, do_each!( + pos => pos, name => wrap_err!(match_type!(BAREWORD), "Expected name for binding"), _ => punct!("="), val => wrap_err!(trace_parse!(expression), "Expected Expression"), _ => punct!(";"), - (tuple_to_let(name, val)) + (Statement::Let(LetDef { + pos: pos, + name: name, + value: val, + })) ) ); @@ -823,10 +821,11 @@ make_fn!( make_fn!( assert_statement, Statement>, do_each!( + pos => pos, _ => word!("assert"), expr => wrap_err!(must!(expression), "Expected Tuple {ok=, desc=}"), _ => must!(punct!(";")), - (Statement::Assert(expr)) + (Statement::Assert(pos, expr)) ) ); diff --git a/src/tokenizer/mod.rs b/src/tokenizer/mod.rs index e669342..fdb896d 100644 --- a/src/tokenizer/mod.rs +++ b/src/tokenizer/mod.rs @@ -24,7 +24,7 @@ use crate::error::StackPrinter; use crate::iter::OffsetStrIter; pub type CommentGroup = Vec; -pub type CommentMap = std::collections::HashMap; +pub type CommentMap = std::collections::BTreeMap; fn is_symbol_char<'a>(i: OffsetStrIter<'a>) -> Result, u8> { let mut _i = i.clone(); @@ -353,6 +353,12 @@ fn comment(input: OffsetStrIter) -> Result { ) ) { Result::Complete(rest, cmt) => { + // Eat the new lines here before continuing + let rest = + match optional!(rest, either!(text_token!("\r\n"), text_token!("\n"))) { + Result::Complete(next_rest, _) => next_rest, + _ => rest, + }; return Result::Complete(rest, make_tok!(CMT => cmt.to_string(), input)); } // If we didn't find a new line then we just grab everything. @@ -504,7 +510,9 @@ pub fn tokenize<'a>( continue; } (&mut Some(ref mut map), _) => { - out.push(tok); + if tok.typ != TokenType::WS { + out.push(tok); + } if let Some(tok) = comment_was_last { map.insert(tok.pos.line, comment_group); comment_group = Vec::new(); diff --git a/src/tokenizer/test.rs b/src/tokenizer/test.rs index 13d58e0..0d80377 100644 --- a/src/tokenizer/test.rs +++ b/src/tokenizer/test.rs @@ -1,3 +1,5 @@ +use std::collections::BTreeMap; + use super::*; use abortable_parser::{Offsetable, Result, SliceIter}; @@ -167,6 +169,23 @@ fn test_tokenize_one_of_each() { assert_eq!(v[38].typ, TokenType::END); } +#[test] +fn test_tokenize_one_of_each_comment_map_path() { + let input = OffsetStrIter::new( + "map out filter assert let import func select as => [ ] { } ; = % / * \ + + - . ( ) , 1 . foo \"bar\" // comment\n ; true false == < > <= >= !=", + ); + let mut comment_map = BTreeMap::new(); + let result = tokenize(input.clone(), Some(&mut comment_map)); + assert!(result.is_ok(), format!("result {:?} is not ok", result)); + let v = result.unwrap(); + for (i, t) in v.iter().enumerate() { + println!("{}: {:?}", i, t); + } + assert_eq!(v.len(), 39); + assert_eq!(v[38].typ, TokenType::END); +} + #[test] fn test_parse_has_end() { let input = OffsetStrIter::new("foo"); @@ -327,3 +346,24 @@ fn test_match_type() { res => assert!(false, format!("Fail: {:?}", res)), } } + +#[test] +fn test_tokenize_builds_comment_map() { + let input = OffsetStrIter::new("// comment 1\n\n//comment 2"); + let mut comment_map = BTreeMap::new(); + let result = tokenize(input.clone(), Some(&mut comment_map)); + assert!(result.is_ok(), format!("result {:?} is not ok", result)); + + assert_eq!(comment_map.len(), 2); +} + +#[test] +fn test_tokenize_builds_comment_map_groups() { + let input = OffsetStrIter::new("// first part\n// comment 1\n\n//comment 2"); + let mut comment_map = BTreeMap::new(); + let result = tokenize(input.clone(), Some(&mut comment_map)); + assert!(result.is_ok(), format!("result {:?} is not ok", result)); + + assert_eq!(comment_map.len(), 2); + assert_eq!(comment_map[&2].len(), 2); +} From 94ca738ee1cae5c8526bfef43a6dec00d8b344d0 Mon Sep 17 00:00:00 2001 From: Jeremy Wall Date: Wed, 22 May 2019 18:29:25 -0500 Subject: [PATCH 13/23] DEV: Handle comments in tuple field expressions --- src/ast/printer/mod.rs | 15 +++++++++---- src/ast/printer/test.rs | 48 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 4 deletions(-) diff --git a/src/ast/printer/mod.rs b/src/ast/printer/mod.rs index 6c19e1d..54139ea 100644 --- a/src/ast/printer/mod.rs +++ b/src/ast/printer/mod.rs @@ -88,9 +88,10 @@ where let empty: Vec = Vec::new(); //eprintln!("comment line candidate: {}", line); let cg = map.get(&line).unwrap_or(&empty); + let indent = self.make_indent(); //eprintln!("comment_group: {:?}", cg); for c in cg.iter() { - write!(self.w, "// {}\n", c.fragment.trim())?; + write!(self.w, "{}// {}\n", indent, c.fragment.trim())?; } self.comment_group_lines.pop(); } @@ -155,6 +156,12 @@ where write!(self.w, "\n")?; } for &(ref t, ref expr) in def.iter() { + let field_line = t.pos.line; + let expr_line = expr.pos().line; + self.render_comment_if_needed(field_line)?; + if expr_line != field_line { + self.render_comment_if_needed(expr_line)?; + } write!(self.w, "{}", indent)?; if Self::is_bareword(&t.fragment) { write!(&mut self.w, "{} = ", t.fragment)?; @@ -427,9 +434,9 @@ where self.render_stmt(v)?; } if let Some(last_comment_line) = self.comment_group_lines.first() { - eprintln!("last_comment_line is: {}", last_comment_line); - eprintln!("comment_map is: {:?}", self.comment_map); - eprintln!("coment_group_lines is: {:?}", self.comment_group_lines); + //eprintln!("last_comment_line is: {}", last_comment_line); + //eprintln!("comment_map is: {:?}", self.comment_map); + //eprintln!("coment_group_lines is: {:?}", self.comment_group_lines); self.render_missed_comments(*last_comment_line + 1)?; } Ok(()) diff --git a/src/ast/printer/test.rs b/src/ast/printer/test.rs index 55a2f53..d7964d9 100644 --- a/src/ast/printer/test.rs +++ b/src/ast/printer/test.rs @@ -402,3 +402,51 @@ fn test_statement_with_comment_printing_comments_at_end() { format!("{}\n", input.trim()) ); } + +#[test] +fn test_tuple_expression_with_embedded_comment() { + let mut comment_map = BTreeMap::new(); + let input = "{\n foo = bar,\n // a comment\n bar = foo,\n};"; + let stmts = assert_parse(input, Some(&mut comment_map)); + let mut buffer: Vec = Vec::new(); + let mut printer = AstPrinter::new(2, &mut buffer).with_comment_map(&comment_map); + assert!(printer.render(&stmts).is_ok()); + assert_eq!( + String::from_utf8(buffer).unwrap(), + format!("{}\n", input.trim()) + ); +} + +#[test] +fn test_tuple_expression_with_embedded_comment_mid_field_expr() { + let mut comment_map = BTreeMap::new(); + let input = "{\n foo = bar,\n bar =\n// a comment\n foo\n};"; + let stmts = assert_parse(input, Some(&mut comment_map)); + let mut buffer: Vec = Vec::new(); + let mut printer = AstPrinter::new(2, &mut buffer).with_comment_map(&comment_map); + assert!(printer.render(&stmts).is_ok()); + assert_eq!( + String::from_utf8(buffer).unwrap(), + format!( + "{}\n", + "{\n foo = bar,\n // a comment\n bar = foo,\n};".trim() + ) + ); +} + +#[test] +fn test_tuple_expression_with_embedded_comment_and_mid_field_expr() { + let mut comment_map = BTreeMap::new(); + let input = "{\n foo = bar,\n// a comment\n bar =\n// another comment\n foo\n};"; + let stmts = assert_parse(input, Some(&mut comment_map)); + let mut buffer: Vec = Vec::new(); + let mut printer = AstPrinter::new(2, &mut buffer).with_comment_map(&comment_map); + assert!(printer.render(&stmts).is_ok()); + assert_eq!( + String::from_utf8(buffer).unwrap(), + format!( + "{}\n", + "{\n foo = bar,\n // a comment\n // another comment\n bar = foo,\n};".trim() + ) + ); +} From a21d4bd2356edff1b1cb680747d9dabad2626c00 Mon Sep 17 00:00:00 2001 From: Jeremy Wall Date: Wed, 22 May 2019 18:34:42 -0500 Subject: [PATCH 14/23] DEV: Handle comments between list elements. --- src/ast/printer/mod.rs | 6 +----- src/ast/printer/test.rs | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/src/ast/printer/mod.rs b/src/ast/printer/mod.rs index 54139ea..25654df 100644 --- a/src/ast/printer/mod.rs +++ b/src/ast/printer/mod.rs @@ -86,10 +86,8 @@ where fn print_comment_group(&mut self, line: usize) -> std::io::Result<()> { if let Some(ref map) = self.comment_map { let empty: Vec = Vec::new(); - //eprintln!("comment line candidate: {}", line); let cg = map.get(&line).unwrap_or(&empty); let indent = self.make_indent(); - //eprintln!("comment_group: {:?}", cg); for c in cg.iter() { write!(self.w, "{}// {}\n", indent, c.fragment.trim())?; } @@ -134,6 +132,7 @@ where write!(self.w, "\n")?; } for e in def.elems.iter() { + self.render_comment_if_needed(e.pos().line)?; write!(self.w, "{}", indent)?; self.render_expr(e)?; write!(self.w, ",\n")?; @@ -434,9 +433,6 @@ where self.render_stmt(v)?; } if let Some(last_comment_line) = self.comment_group_lines.first() { - //eprintln!("last_comment_line is: {}", last_comment_line); - //eprintln!("comment_map is: {:?}", self.comment_map); - //eprintln!("coment_group_lines is: {:?}", self.comment_group_lines); self.render_missed_comments(*last_comment_line + 1)?; } Ok(()) diff --git a/src/ast/printer/test.rs b/src/ast/printer/test.rs index d7964d9..12c2982 100644 --- a/src/ast/printer/test.rs +++ b/src/ast/printer/test.rs @@ -450,3 +450,17 @@ fn test_tuple_expression_with_embedded_comment_and_mid_field_expr() { ) ); } + +#[test] +fn test_list_expression_with_embedded_comment() { + let mut comment_map = BTreeMap::new(); + let input = "[\n bar,\n // a comment\n foo,\n];"; + let stmts = assert_parse(input, Some(&mut comment_map)); + let mut buffer: Vec = Vec::new(); + let mut printer = AstPrinter::new(2, &mut buffer).with_comment_map(&comment_map); + assert!(printer.render(&stmts).is_ok()); + assert_eq!( + String::from_utf8(buffer).unwrap(), + format!("{}\n", input.trim()) + ); +} From e6f6421ca6a652b977f13b44eef730b74076d35a Mon Sep 17 00:00:00 2001 From: Jeremy Wall Date: Wed, 22 May 2019 18:52:10 -0500 Subject: [PATCH 15/23] DEV: Handle comments embedded in binary expressions and call expressions. --- src/ast/printer/mod.rs | 19 ++++++++++++++++++- src/ast/printer/test.rs | 42 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/src/ast/printer/mod.rs b/src/ast/printer/mod.rs index 25654df..7d0f1c4 100644 --- a/src/ast/printer/mod.rs +++ b/src/ast/printer/mod.rs @@ -123,6 +123,15 @@ where Ok(()) } + fn has_comment(&self, line: usize) -> bool { + if line > self.last_line { + if let Some(next_comment_line) = self.comment_group_lines.last() { + return *next_comment_line < line; + } + } + false + } + fn render_list_def(&mut self, def: &ListDef) -> std::io::Result<()> { write!(self.w, "[")?; self.curr_indent += self.indent_size; @@ -209,13 +218,14 @@ where } pub fn render_expr(&mut self, expr: &Expression) -> std::io::Result<()> { + self.render_comment_if_needed(expr.pos().line)?; match expr { Expression::Binary(_def) => { let op = match _def.kind { BinaryExprType::AND => " && ", BinaryExprType::OR => " || ", BinaryExprType::DOT => ".", - BinaryExprType::Equal => " = ", + BinaryExprType::Equal => " == ", BinaryExprType::NotEqual => " != ", BinaryExprType::GTEqual => " >= ", BinaryExprType::LTEqual => " <= ", @@ -231,8 +241,14 @@ where BinaryExprType::REMatch => " ~ ", BinaryExprType::NotREMatch => " !~ ", }; + let right_line = _def.right.pos().line; self.render_expr(&_def.left)?; self.w.write(op.as_bytes())?; + if self.has_comment(right_line) { + // if we'll be rendering a comment then we should + // add a new line here + self.w.write("\n".as_bytes())?; + } self.render_expr(&_def.right)?; } Expression::Call(_def) => { @@ -245,6 +261,7 @@ where write!(self.w, "\n")?; } for e in _def.arglist.iter() { + self.render_comment_if_needed(e.pos().line)?; if has_args { write!(self.w, "{}", indent)?; } diff --git a/src/ast/printer/test.rs b/src/ast/printer/test.rs index 12c2982..85a8ea6 100644 --- a/src/ast/printer/test.rs +++ b/src/ast/printer/test.rs @@ -464,3 +464,45 @@ fn test_list_expression_with_embedded_comment() { format!("{}\n", input.trim()) ); } + +#[test] +fn test_binary_expression_with_embedded_comment() { + let mut comment_map = BTreeMap::new(); + let input = "true == \n// false is not true\nfalse;"; + let stmts = assert_parse(input, Some(&mut comment_map)); + let mut buffer: Vec = Vec::new(); + let mut printer = AstPrinter::new(2, &mut buffer).with_comment_map(&comment_map); + assert!(printer.render(&stmts).is_ok()); + assert_eq!( + String::from_utf8(buffer).unwrap(), + format!("{}\n", input.trim()) + ); +} + +#[test] +fn test_empty_call_expression_with_comment() { + let mut comment_map = BTreeMap::new(); + let input = "// a comment\nmyfunc();"; + let stmts = assert_parse(input, Some(&mut comment_map)); + let mut buffer: Vec = Vec::new(); + let mut printer = AstPrinter::new(2, &mut buffer).with_comment_map(&comment_map); + assert!(printer.render(&stmts).is_ok()); + assert_eq!( + String::from_utf8(buffer).unwrap(), + format!("{}\n", input.trim()) + ); +} + +#[test] +fn test_call_expression_with_embedded_comment_in_args() { + let mut comment_map = BTreeMap::new(); + let input = "// a comment\nmyfunc(\n arg1,\n // another comment\n arg2,\n);"; + let stmts = assert_parse(input, Some(&mut comment_map)); + let mut buffer: Vec = Vec::new(); + let mut printer = AstPrinter::new(2, &mut buffer).with_comment_map(&comment_map); + assert!(printer.render(&stmts).is_ok()); + assert_eq!( + String::from_utf8(buffer).unwrap(), + format!("{}\n", input.trim()) + ); +} From f88955517ebf55885fe4a1246c1512060ed049da Mon Sep 17 00:00:00 2001 From: Jeremy Wall Date: Wed, 22 May 2019 19:05:48 -0500 Subject: [PATCH 16/23] DEV: Handle TRACE, fail, and format expressions with embedded comments. --- src/ast/printer/mod.rs | 19 +++++++++++++- src/ast/printer/test.rs | 56 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 1 deletion(-) diff --git a/src/ast/printer/mod.rs b/src/ast/printer/mod.rs index 7d0f1c4..956653f 100644 --- a/src/ast/printer/mod.rs +++ b/src/ast/printer/mod.rs @@ -282,10 +282,16 @@ where } Expression::Debug(_def) => { self.w.write("TRACE ".as_bytes())?; + if self.has_comment(_def.expr.pos().line) { + self.w.write("\n".as_bytes())?; + } self.render_expr(&_def.expr)?; } Expression::Fail(_def) => { self.w.write("fail ".as_bytes())?; + if self.has_comment(_def.message.pos().line) { + self.w.write("\n".as_bytes())?; + } self.render_expr(&_def.message)?; } Expression::Format(_def) => { @@ -293,13 +299,24 @@ where write!(self.w, " % ")?; match _def.args { FormatArgs::Single(ref e) => { + if self.has_comment(e.pos().line) { + self.w.write("\n".as_bytes())?; + } self.render_expr(e)?; } FormatArgs::List(ref es) => { self.w.write("(\n".as_bytes())?; self.curr_indent += self.indent_size; let indent = self.make_indent(); - let mut prefix = ""; + let mut prefix = if es + .first() + .and_then(|e| Some(self.has_comment(e.pos().line))) + .unwrap_or(false) + { + "\n" + } else { + "" + }; for e in es.iter() { write!(self.w, "{}{}", prefix, indent)?; self.render_expr(e)?; diff --git a/src/ast/printer/test.rs b/src/ast/printer/test.rs index 85a8ea6..dd619df 100644 --- a/src/ast/printer/test.rs +++ b/src/ast/printer/test.rs @@ -506,3 +506,59 @@ fn test_call_expression_with_embedded_comment_in_args() { format!("{}\n", input.trim()) ); } + +#[test] +fn test_copy_expression_with_embedded_comment_in_args() { + let mut comment_map = BTreeMap::new(); + let input = "// a comment\nmyfunc{\n foo = arg1,\n // another comment\n bar = arg2,\n};"; + let stmts = assert_parse(input, Some(&mut comment_map)); + let mut buffer: Vec = Vec::new(); + let mut printer = AstPrinter::new(2, &mut buffer).with_comment_map(&comment_map); + assert!(printer.render(&stmts).is_ok()); + assert_eq!( + String::from_utf8(buffer).unwrap(), + format!("{}\n", input.trim()) + ); +} + +#[test] +fn test_trace_expression_with_embedded_comment() { + let mut comment_map = BTreeMap::new(); + let input = "// a comment\nTRACE \n// another comment\nfoo;"; + let stmts = assert_parse(input, Some(&mut comment_map)); + let mut buffer: Vec = Vec::new(); + let mut printer = AstPrinter::new(2, &mut buffer).with_comment_map(&comment_map); + assert!(printer.render(&stmts).is_ok()); + assert_eq!( + String::from_utf8(buffer).unwrap(), + format!("{}\n", input.trim()) + ); +} + +#[test] +fn test_fail_expression_with_embedded_comment() { + let mut comment_map = BTreeMap::new(); + let input = "// a comment\nfail \n// another comment\nfoo;"; + let stmts = assert_parse(input, Some(&mut comment_map)); + let mut buffer: Vec = Vec::new(); + let mut printer = AstPrinter::new(2, &mut buffer).with_comment_map(&comment_map); + assert!(printer.render(&stmts).is_ok()); + assert_eq!( + String::from_utf8(buffer).unwrap(), + format!("{}\n", input.trim()) + ); +} + +#[test] +fn test_format_expression_with_embedded_comment() { + let mut comment_map = BTreeMap::new(); + let input = "// a comment\n\"@(item.bar)\" % \n// another comment\nfoo;"; + let stmts = assert_parse(input, Some(&mut comment_map)); + let mut buffer: Vec = Vec::new(); + let mut printer = AstPrinter::new(2, &mut buffer).with_comment_map(&comment_map); + assert!(printer.render(&stmts).is_ok()); + assert_eq!( + String::from_utf8(buffer).unwrap(), + format!("{}\n", input.trim()) + ); +} From 157f123355afcaad394cf020bfc9fcb0c6ee7e99 Mon Sep 17 00:00:00 2001 From: Jeremy Wall Date: Thu, 23 May 2019 19:52:03 -0500 Subject: [PATCH 17/23] DEV: Handle the functional operators with embedded comments. --- src/ast/printer/mod.rs | 68 +++++++- src/ast/printer/test.rs | 360 +++++++++------------------------------- 2 files changed, 143 insertions(+), 285 deletions(-) diff --git a/src/ast/printer/mod.rs b/src/ast/printer/mod.rs index 956653f..2fc7895 100644 --- a/src/ast/printer/mod.rs +++ b/src/ast/printer/mod.rs @@ -218,7 +218,13 @@ where } pub fn render_expr(&mut self, expr: &Expression) -> std::io::Result<()> { + let had_comment = self.has_comment(expr.pos().line); self.render_comment_if_needed(expr.pos().line)?; + let indent = self.make_indent(); + if had_comment { + write!(self.w, "{}", indent)?; + } + let mut did_indent = false; match expr { Expression::Binary(_def) => { let op = match _def.kind { @@ -344,24 +350,75 @@ where Expression::FuncOp(_def) => match _def { FuncOpDef::Filter(_def) => { write!(self.w, "filter(")?; + if self.has_comment(_def.func.pos().line) { + self.curr_indent += self.indent_size; + did_indent = true; + write!(self.w, "\n")?; + } self.render_expr(&_def.func)?; - write!(self.w, ", ")?; + if self.has_comment(_def.target.pos().line) { + write!(self.w, ",")?; + if !did_indent { + self.curr_indent += self.indent_size; + } + did_indent = true; + self.w.write("\n".as_bytes())?; + } else { + write!(self.w, ", ")?; + } self.render_expr(&_def.target)?; write!(self.w, ")")?; } FuncOpDef::Reduce(_def) => { write!(self.w, "reduce(")?; + if self.has_comment(_def.func.pos().line) { + self.curr_indent += self.indent_size; + did_indent = true; + write!(self.w, "\n")?; + } self.render_expr(&_def.func)?; - write!(self.w, ", ")?; + if self.has_comment(_def.acc.pos().line) { + write!(self.w, ",")?; + if !did_indent { + self.curr_indent += self.indent_size; + } + did_indent = true; + self.w.write("\n".as_bytes())?; + } else { + write!(self.w, ", ")?; + } self.render_expr(&_def.acc)?; - write!(self.w, ", ")?; + if self.has_comment(_def.target.pos().line) { + write!(self.w, ",")?; + if !did_indent { + self.curr_indent += self.indent_size; + } + did_indent = true; + self.w.write("\n".as_bytes())?; + } else { + write!(self.w, ", ")?; + } self.render_expr(&_def.target)?; write!(self.w, ")")?; } FuncOpDef::Map(_def) => { write!(self.w, "map(")?; + if self.has_comment(_def.func.pos().line) { + self.curr_indent += self.indent_size; + did_indent = true; + write!(self.w, "\n")?; + } self.render_expr(&_def.func)?; - write!(self.w, ", ")?; + if self.has_comment(_def.target.pos().line) { + write!(self.w, ",")?; + if !did_indent { + self.curr_indent += self.indent_size; + } + did_indent = true; + self.w.write("\n".as_bytes())?; + } else { + write!(self.w, ", ")?; + } self.render_expr(&_def.target)?; write!(self.w, ")")?; } @@ -433,6 +490,9 @@ where self.render_value(_def)?; } }; + if did_indent { + self.curr_indent -= self.indent_size; + } Ok(()) } diff --git a/src/ast/printer/test.rs b/src/ast/printer/test.rs index dd619df..82b26f8 100644 --- a/src/ast/printer/test.rs +++ b/src/ast/printer/test.rs @@ -21,125 +21,86 @@ fn assert_parse(input: &str, comment_map: Option<&mut CommentMap>) -> Vec String { + let mut comment_map = BTreeMap::new(); + let stmts = assert_parse(input, Some(&mut comment_map)); + let mut buffer: Vec = Vec::new(); + let mut printer = AstPrinter::new(2, &mut buffer).with_comment_map(&comment_map); + assert!(printer.render(&stmts).is_ok()); + String::from_utf8(buffer).unwrap() +} + #[test] fn test_simple_value_printing() { let input = "1;"; - let stmts = assert_parse(input, None); - let mut buffer: Vec = Vec::new(); - let mut printer = AstPrinter::new(0, &mut buffer); - assert!(printer.render(&stmts).is_ok()); - assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n", input)); } #[test] fn test_simple_selector_printing() { let input = "foo.bar.quux;"; - let stmts = assert_parse(input, None); - let mut buffer: Vec = Vec::new(); - let mut printer = AstPrinter::new(0, &mut buffer); - assert!(printer.render(&stmts).is_ok()); - assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n", input)); } #[test] fn test_simple_quoted_printing() { let input = "\"foo\";"; - let stmts = assert_parse(input, None); - let mut buffer: Vec = Vec::new(); - let mut printer = AstPrinter::new(0, &mut buffer); - assert!(printer.render(&stmts).is_ok()); - assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n", input)); } #[test] fn test_escaped_quoted_printing() { let input = "\"f\\\\o\\\"o\";"; - let stmts = assert_parse(input, None); - let mut buffer: Vec = Vec::new(); - let mut printer = AstPrinter::new(0, &mut buffer); - assert!(printer.render(&stmts).is_ok()); - assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n", input)); } #[test] fn test_empty_tuple_printing() { let input = "{};"; - let stmts = assert_parse(input, None); - let mut buffer: Vec = Vec::new(); - let mut printer = AstPrinter::new(2, &mut buffer); - assert!(printer.render(&stmts).is_ok()); - assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n", input)); } #[test] fn test_empty_list_printing() { let input = "[];"; - let stmts = assert_parse(input, None); - let mut buffer: Vec = Vec::new(); - let mut printer = AstPrinter::new(2, &mut buffer); - assert!(printer.render(&stmts).is_ok()); - assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n", input)); } #[test] fn test_non_empty_tuple_printing() { let input = "{\n foo = 1,\n};"; - let stmts = assert_parse(input, None); - let mut buffer: Vec = Vec::new(); - let mut printer = AstPrinter::new(2, &mut buffer); - assert!(printer.render(&stmts).is_ok()); - assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n", input)); } #[test] fn test_nested_empty_tuple_printing() { let input = "{\n foo = {},\n};"; - let stmts = assert_parse(input, None); - let mut buffer: Vec = Vec::new(); - let mut printer = AstPrinter::new(2, &mut buffer); - assert!(printer.render(&stmts).is_ok()); - assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n", input)); } #[test] fn test_list_nested_empty_tuple_printing() { let input = "[\n {},\n];"; - let stmts = assert_parse(input, None); - let mut buffer: Vec = Vec::new(); - let mut printer = AstPrinter::new(2, &mut buffer); - assert!(printer.render(&stmts).is_ok()); - assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n", input)); } #[test] fn test_nested_non_empty_tuple_printing() { let input = "{\n foo = {\n bar = 1,\n },\n};"; - let stmts = assert_parse(input, None); - let mut buffer: Vec = Vec::new(); - let mut printer = AstPrinter::new(2, &mut buffer); - assert!(printer.render(&stmts).is_ok()); - assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n", input)); } #[test] fn test_nested_non_empty_list_printing() { let input = "[\n [\n 1,\n ],\n];"; - let stmts = assert_parse(input, None); - let mut buffer: Vec = Vec::new(); - let mut printer = AstPrinter::new(2, &mut buffer); - assert!(printer.render(&stmts).is_ok()); - assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n", input)); } #[test] fn test_simple_quoted_field_tuple_printing() { let input = "{\n \"foo\" = {\n bar = 1,\n },\n};"; - let stmts = assert_parse(input, None); - let mut buffer: Vec = Vec::new(); - let mut printer = AstPrinter::new(2, &mut buffer); - assert!(printer.render(&stmts).is_ok()); assert_eq!( - String::from_utf8(buffer).unwrap(), + print_to_buffer(input), format!("{}\n", "{\n foo = {\n bar = 1,\n },\n};") ); } @@ -147,121 +108,73 @@ fn test_simple_quoted_field_tuple_printing() { #[test] fn test_special_quoted_field_tuple_printing() { let input = "{\n \"foo bar\" = {\n bar = 1,\n },\n};"; - let stmts = assert_parse(input, None); - let mut buffer: Vec = Vec::new(); - let mut printer = AstPrinter::new(2, &mut buffer); - assert!(printer.render(&stmts).is_ok()); - assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n", input)); } #[test] fn test_let_statement_printing() { let input = "let tpl = {\n \"foo bar\" = {\n bar = 1,\n },\n};"; - let stmts = assert_parse(input, None); - let mut buffer: Vec = Vec::new(); - let mut printer = AstPrinter::new(2, &mut buffer); - assert!(printer.render(&stmts).is_ok()); - assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n", input)); } #[test] fn test_call_expr_printing() { let input = "call(\n foo,\n bar,\n);"; - let stmts = assert_parse(input, None); - let mut buffer: Vec = Vec::new(); - let mut printer = AstPrinter::new(2, &mut buffer); - assert!(printer.render(&stmts).is_ok()); - assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n", input)); } #[test] fn test_call_expr_one_arg_printing() { let input = "call(foo);"; - let stmts = assert_parse(input, None); - let mut buffer: Vec = Vec::new(); - let mut printer = AstPrinter::new(2, &mut buffer); - assert!(printer.render(&stmts).is_ok()); - assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n", input)); } #[test] fn test_copy_expr_printing() { let input = "copy{\n foo = 1,\n bar = 2,\n};"; - let stmts = assert_parse(input, None); - let mut buffer: Vec = Vec::new(); - let mut printer = AstPrinter::new(2, &mut buffer); - assert!(printer.render(&stmts).is_ok()); - assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n", input)); } #[test] fn test_copy_expr_one_arg_printing() { let input = "copy{\n foo = 1,\n};"; - let stmts = assert_parse(input, None); - let mut buffer: Vec = Vec::new(); - let mut printer = AstPrinter::new(2, &mut buffer); - assert!(printer.render(&stmts).is_ok()); - assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n", input)); } #[test] fn test_out_expr_printing() { let input = "out json {\n foo = 1,\n};"; - let stmts = assert_parse(input, None); - let mut buffer: Vec = Vec::new(); - let mut printer = AstPrinter::new(2, &mut buffer); - assert!(printer.render(&stmts).is_ok()); - assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n", input)); } #[test] fn test_select_expr_no_default_printing() { let input = "select true, {\n true = 1,\n false = 2,\n};"; - let stmts = assert_parse(input, None); - let mut buffer: Vec = Vec::new(); - let mut printer = AstPrinter::new(2, &mut buffer); - assert!(printer.render(&stmts).is_ok()); - assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n", input)); } #[test] fn test_select_expr_with_default_printing() { let input = "select true, 3, {\n true = 1,\n false = 2,\n};"; - let stmts = assert_parse(input, None); - let mut buffer: Vec = Vec::new(); - let mut printer = AstPrinter::new(2, &mut buffer); - assert!(printer.render(&stmts).is_ok()); - assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n", input)); } #[test] fn test_not_expr_printing() { let input = "not true;"; - let stmts = assert_parse(input, None); - let mut buffer: Vec = Vec::new(); - let mut printer = AstPrinter::new(2, &mut buffer); - assert!(printer.render(&stmts).is_ok()); - assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n", input)); } #[test] fn test_fail_expr_printing() { let input = "fail \"AHHh\";"; - let stmts = assert_parse(input, None); - let mut buffer: Vec = Vec::new(); - let mut printer = AstPrinter::new(2, &mut buffer); - assert!(printer.render(&stmts).is_ok()); - assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n", input)); } #[test] fn test_trace_expr_printing() { let input = "TRACE \"AHHh\";"; - let stmts = assert_parse(input, None); - let mut buffer: Vec = Vec::new(); - let mut printer = AstPrinter::new(2, &mut buffer); - assert!(printer.render(&stmts).is_ok()); - assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n", input)); } #[test] @@ -277,11 +190,7 @@ fn test_module_no_out_expr_printing() { \"cpu_count\" = mod.cpu, }; };"; - let stmts = assert_parse(input, None); - let mut buffer: Vec = Vec::new(); - let mut printer = AstPrinter::new(2, &mut buffer); - assert!(printer.render(&stmts).is_ok()); - assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n", input)); } #[test] @@ -297,11 +206,7 @@ fn test_module_with_out_expr_printing() { \"cpu_count\" = mod.cpu, }; };"; - let stmts = assert_parse(input, None); - let mut buffer: Vec = Vec::new(); - let mut printer = AstPrinter::new(2, &mut buffer); - assert!(printer.render(&stmts).is_ok()); - assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n", input)); } #[test] @@ -310,11 +215,7 @@ fn test_func_expr_printing() { foo = foo, bar = bar, };"; - let stmts = assert_parse(input, None); - let mut buffer: Vec = Vec::new(); - let mut printer = AstPrinter::new(2, &mut buffer); - assert!(printer.render(&stmts).is_ok()); - assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n", input)); } #[test] @@ -322,11 +223,7 @@ fn test_func_expr_single_arg_printing() { let input = "let f = func (foo) => { foo = foo, };"; - let stmts = assert_parse(input, None); - let mut buffer: Vec = Vec::new(); - let mut printer = AstPrinter::new(2, &mut buffer); - assert!(printer.render(&stmts).is_ok()); - assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n", input)); } #[test] @@ -334,11 +231,7 @@ fn test_format_expr_single_arg_printing() { let input = "\"what? @{item.foo}\" % { foo = 1, };"; - let stmts = assert_parse(input, None); - let mut buffer: Vec = Vec::new(); - let mut printer = AstPrinter::new(2, &mut buffer); - assert!(printer.render(&stmts).is_ok()); - assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n", input)); } #[test] @@ -346,219 +239,124 @@ fn test_format_expr_list_arg_printing() { let input = "\"what? @ @\" % ( 1, 2);"; - let stmts = assert_parse(input, None); - let mut buffer: Vec = Vec::new(); - let mut printer = AstPrinter::new(2, &mut buffer); - assert!(printer.render(&stmts).is_ok()); - assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n", input)); } #[test] fn test_statement_with_comment_printing() { - let mut comment_map = BTreeMap::new(); let input = "// add 1 + 1\n1 + 1;"; - let stmts = assert_parse(input, Some(&mut comment_map)); - let mut buffer: Vec = Vec::new(); - let mut printer = AstPrinter::new(2, &mut buffer).with_comment_map(&comment_map); - assert!(printer.render(&stmts).is_ok()); - assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n", input)); } #[test] fn test_statement_with_comment_printing_groups() { - let mut comment_map = BTreeMap::new(); let input = "// add 1\n// and 1\n1 + 1;"; - let stmts = assert_parse(input, Some(&mut comment_map)); - let mut buffer: Vec = Vec::new(); - let mut printer = AstPrinter::new(2, &mut buffer).with_comment_map(&comment_map); - assert!(printer.render(&stmts).is_ok()); - assert_eq!(String::from_utf8(buffer).unwrap(), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n", input)); } #[test] fn test_statement_with_comment_printing_multiple_groups() { - let mut comment_map = BTreeMap::new(); let input = "\n// group 1\n// more group 1\n\n// group 2\n// more group 2\n1 + 1;"; - let stmts = assert_parse(input, Some(&mut comment_map)); - let mut buffer: Vec = Vec::new(); - let mut printer = AstPrinter::new(2, &mut buffer).with_comment_map(&comment_map); - assert!(printer.render(&stmts).is_ok()); - assert_eq!( - String::from_utf8(buffer).unwrap(), - format!("{}\n", input.trim()) - ); + assert_eq!(print_to_buffer(input), format!("{}\n", input.trim())); } #[test] fn test_statement_with_comment_printing_comments_at_end() { - let mut comment_map = BTreeMap::new(); let input = "// group 1\n1 + 1;\n// group 2\n\n"; - let stmts = assert_parse(input, Some(&mut comment_map)); - let mut buffer: Vec = Vec::new(); - let mut printer = AstPrinter::new(2, &mut buffer).with_comment_map(&comment_map); - assert!(printer.render(&stmts).is_ok()); - assert_eq!( - String::from_utf8(buffer).unwrap(), - format!("{}\n", input.trim()) - ); + assert_eq!(print_to_buffer(input), format!("{}\n", input.trim())); } #[test] fn test_tuple_expression_with_embedded_comment() { - let mut comment_map = BTreeMap::new(); let input = "{\n foo = bar,\n // a comment\n bar = foo,\n};"; - let stmts = assert_parse(input, Some(&mut comment_map)); - let mut buffer: Vec = Vec::new(); - let mut printer = AstPrinter::new(2, &mut buffer).with_comment_map(&comment_map); - assert!(printer.render(&stmts).is_ok()); - assert_eq!( - String::from_utf8(buffer).unwrap(), - format!("{}\n", input.trim()) - ); + assert_eq!(print_to_buffer(input), format!("{}\n", input)); } #[test] fn test_tuple_expression_with_embedded_comment_mid_field_expr() { - let mut comment_map = BTreeMap::new(); let input = "{\n foo = bar,\n bar =\n// a comment\n foo\n};"; - let stmts = assert_parse(input, Some(&mut comment_map)); - let mut buffer: Vec = Vec::new(); - let mut printer = AstPrinter::new(2, &mut buffer).with_comment_map(&comment_map); - assert!(printer.render(&stmts).is_ok()); assert_eq!( - String::from_utf8(buffer).unwrap(), - format!( - "{}\n", - "{\n foo = bar,\n // a comment\n bar = foo,\n};".trim() - ) + print_to_buffer(input), + "{\n foo = bar,\n // a comment\n bar = foo,\n};\n" ); } #[test] fn test_tuple_expression_with_embedded_comment_and_mid_field_expr() { - let mut comment_map = BTreeMap::new(); let input = "{\n foo = bar,\n// a comment\n bar =\n// another comment\n foo\n};"; - let stmts = assert_parse(input, Some(&mut comment_map)); - let mut buffer: Vec = Vec::new(); - let mut printer = AstPrinter::new(2, &mut buffer).with_comment_map(&comment_map); - assert!(printer.render(&stmts).is_ok()); assert_eq!( - String::from_utf8(buffer).unwrap(), - format!( - "{}\n", - "{\n foo = bar,\n // a comment\n // another comment\n bar = foo,\n};".trim() - ) + print_to_buffer(input), + "{\n foo = bar,\n // a comment\n // another comment\n bar = foo,\n};\n" ); } #[test] fn test_list_expression_with_embedded_comment() { - let mut comment_map = BTreeMap::new(); let input = "[\n bar,\n // a comment\n foo,\n];"; - let stmts = assert_parse(input, Some(&mut comment_map)); - let mut buffer: Vec = Vec::new(); - let mut printer = AstPrinter::new(2, &mut buffer).with_comment_map(&comment_map); - assert!(printer.render(&stmts).is_ok()); - assert_eq!( - String::from_utf8(buffer).unwrap(), - format!("{}\n", input.trim()) - ); + assert_eq!(print_to_buffer(input), format!("{}\n", input)); } #[test] fn test_binary_expression_with_embedded_comment() { - let mut comment_map = BTreeMap::new(); let input = "true == \n// false is not true\nfalse;"; - let stmts = assert_parse(input, Some(&mut comment_map)); - let mut buffer: Vec = Vec::new(); - let mut printer = AstPrinter::new(2, &mut buffer).with_comment_map(&comment_map); - assert!(printer.render(&stmts).is_ok()); - assert_eq!( - String::from_utf8(buffer).unwrap(), - format!("{}\n", input.trim()) - ); + assert_eq!(print_to_buffer(input), format!("{}\n", input)); } #[test] fn test_empty_call_expression_with_comment() { - let mut comment_map = BTreeMap::new(); let input = "// a comment\nmyfunc();"; - let stmts = assert_parse(input, Some(&mut comment_map)); - let mut buffer: Vec = Vec::new(); - let mut printer = AstPrinter::new(2, &mut buffer).with_comment_map(&comment_map); - assert!(printer.render(&stmts).is_ok()); - assert_eq!( - String::from_utf8(buffer).unwrap(), - format!("{}\n", input.trim()) - ); + assert_eq!(print_to_buffer(input), format!("{}\n", input)); } #[test] fn test_call_expression_with_embedded_comment_in_args() { - let mut comment_map = BTreeMap::new(); let input = "// a comment\nmyfunc(\n arg1,\n // another comment\n arg2,\n);"; - let stmts = assert_parse(input, Some(&mut comment_map)); - let mut buffer: Vec = Vec::new(); - let mut printer = AstPrinter::new(2, &mut buffer).with_comment_map(&comment_map); - assert!(printer.render(&stmts).is_ok()); - assert_eq!( - String::from_utf8(buffer).unwrap(), - format!("{}\n", input.trim()) - ); + assert_eq!(print_to_buffer(input), format!("{}\n", input)); } #[test] fn test_copy_expression_with_embedded_comment_in_args() { - let mut comment_map = BTreeMap::new(); let input = "// a comment\nmyfunc{\n foo = arg1,\n // another comment\n bar = arg2,\n};"; - let stmts = assert_parse(input, Some(&mut comment_map)); - let mut buffer: Vec = Vec::new(); - let mut printer = AstPrinter::new(2, &mut buffer).with_comment_map(&comment_map); - assert!(printer.render(&stmts).is_ok()); - assert_eq!( - String::from_utf8(buffer).unwrap(), - format!("{}\n", input.trim()) - ); + assert_eq!(print_to_buffer(input), format!("{}\n", input)); } #[test] fn test_trace_expression_with_embedded_comment() { - let mut comment_map = BTreeMap::new(); let input = "// a comment\nTRACE \n// another comment\nfoo;"; - let stmts = assert_parse(input, Some(&mut comment_map)); - let mut buffer: Vec = Vec::new(); - let mut printer = AstPrinter::new(2, &mut buffer).with_comment_map(&comment_map); - assert!(printer.render(&stmts).is_ok()); - assert_eq!( - String::from_utf8(buffer).unwrap(), - format!("{}\n", input.trim()) - ); + assert_eq!(print_to_buffer(input), format!("{}\n", input)); } #[test] fn test_fail_expression_with_embedded_comment() { - let mut comment_map = BTreeMap::new(); let input = "// a comment\nfail \n// another comment\nfoo;"; - let stmts = assert_parse(input, Some(&mut comment_map)); - let mut buffer: Vec = Vec::new(); - let mut printer = AstPrinter::new(2, &mut buffer).with_comment_map(&comment_map); - assert!(printer.render(&stmts).is_ok()); - assert_eq!( - String::from_utf8(buffer).unwrap(), - format!("{}\n", input.trim()) - ); + assert_eq!(print_to_buffer(input), format!("{}\n", input)); } #[test] fn test_format_expression_with_embedded_comment() { - let mut comment_map = BTreeMap::new(); let input = "// a comment\n\"@(item.bar)\" % \n// another comment\nfoo;"; - let stmts = assert_parse(input, Some(&mut comment_map)); - let mut buffer: Vec = Vec::new(); - let mut printer = AstPrinter::new(2, &mut buffer).with_comment_map(&comment_map); - assert!(printer.render(&stmts).is_ok()); - assert_eq!( - String::from_utf8(buffer).unwrap(), - format!("{}\n", input.trim()) - ); + let output = print_to_buffer(input); + assert_eq!(output, format!("{}\n", input.trim())); +} + +#[test] +fn test_filter_func_operator_expression_with_embedded_comment() { + //let input = "// a comment\nfilter(foo, bar);"; + let input = "// a comment\nfilter(\n // another comment\n foo,\n // one more\n bar);"; + let output = print_to_buffer(input); + assert_eq!(output, format!("{}\n", input.trim())); +} + +#[test] +fn test_reduce_func_operator_expression_with_embedded_comment() { + //let input = "// a comment\nfilter(foo, bar);"; + let input = "// a comment\nreduce( + // another comment + myfunc, + // one more + acc, + // and the last + target);"; + let output = print_to_buffer(input); + assert_eq!(output, format!("{}\n", input.trim())); } From ff2aafeb98ee917bcd3b9f576ce10be0d3ee6506 Mon Sep 17 00:00:00 2001 From: Jeremy Wall Date: Thu, 23 May 2019 19:57:51 -0500 Subject: [PATCH 18/23] DEV: Handle grouped comments with embedded comments. --- src/ast/printer/mod.rs | 8 ++++++++ src/ast/printer/test.rs | 16 ++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/src/ast/printer/mod.rs b/src/ast/printer/mod.rs index 2fc7895..bb682e6 100644 --- a/src/ast/printer/mod.rs +++ b/src/ast/printer/mod.rs @@ -425,7 +425,15 @@ where }, Expression::Grouped(ref expr, _) => { write!(self.w, "(")?; + if self.has_comment(expr.pos().line) { + self.curr_indent += self.indent_size; + did_indent = true; + write!(self.w, "\n")?; + } self.render_expr(expr)?; + if did_indent { + write!(self.w, "\n")?; + } write!(self.w, ")")?; } Expression::Import(_def) => { diff --git a/src/ast/printer/test.rs b/src/ast/printer/test.rs index 82b26f8..cf07201 100644 --- a/src/ast/printer/test.rs +++ b/src/ast/printer/test.rs @@ -360,3 +360,19 @@ fn test_reduce_func_operator_expression_with_embedded_comment() { let output = print_to_buffer(input); assert_eq!(output, format!("{}\n", input.trim())); } + +#[test] +fn test_map_func_operator_expression_with_embedded_comment() { + //let input = "// a comment\nfilter(foo, bar);"; + let input = "// a comment\nmap(\n // another comment\n foo,\n // one more\n bar);"; + let output = print_to_buffer(input); + assert_eq!(output, format!("{}\n", input.trim())); +} + +#[test] +fn test_grouped_expression_with_embedded_comment() { + //let input = "// a comment\nfilter(foo, bar);"; + let input = "// a comment\n(\n // a comment\n foo\n);"; + let output = print_to_buffer(input); + assert_eq!(output, format!("{}\n", input.trim())); +} From 82d6ca9ecb74ae870f5d4b18137ebc396484eeb5 Mon Sep 17 00:00:00 2001 From: Jeremy Wall Date: Thu, 23 May 2019 20:01:03 -0500 Subject: [PATCH 19/23] DEV: Handle comments embedded in not, import or include expressions --- src/ast/printer/mod.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/ast/printer/mod.rs b/src/ast/printer/mod.rs index bb682e6..6924b5e 100644 --- a/src/ast/printer/mod.rs +++ b/src/ast/printer/mod.rs @@ -437,6 +437,9 @@ where write!(self.w, ")")?; } Expression::Import(_def) => { + if self.has_comment(_def.path.pos.line) { + self.render_missed_comments(_def.path.pos.line)?; + } write!( self.w, "import \"{}\"", @@ -444,6 +447,9 @@ where )?; } Expression::Include(_def) => { + if self.has_comment(_def.path.pos.line) { + self.render_missed_comments(_def.path.pos.line)?; + } write!( self.w, "include {} \"{}\"", From c9b2b9994f884f41ac48776d8b2b94152de505b8 Mon Sep 17 00:00:00 2001 From: Jeremy Wall Date: Thu, 23 May 2019 20:10:18 -0500 Subject: [PATCH 20/23] DEV: Handle some more expressions. --- src/ast/printer/mod.rs | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/ast/printer/mod.rs b/src/ast/printer/mod.rs index 6924b5e..27d91e2 100644 --- a/src/ast/printer/mod.rs +++ b/src/ast/printer/mod.rs @@ -477,10 +477,19 @@ where write!(self.w, "}}")?; } Expression::Not(_def) => { + if self.has_comment(_def.pos.line) { + self.render_missed_comments(_def.pos.line)?; + } write!(self.w, "not ")?; self.render_expr(&_def.expr)?; } Expression::Range(_def) => { + // We print all of the comments we missed before the end of this + // expression before the entire range expression. + let end_line = _def.end.pos().line; + if self.has_comment(end_line) { + self.render_missed_comments(end_line)?; + } self.render_expr(&_def.start)?; write!(self.w, ":")?; if let Some(ref e) = _def.step { @@ -490,7 +499,16 @@ where self.render_expr(&_def.end)?; } Expression::Select(_def) => { - // + let val_line = _def.val.pos().line; + if self.has_comment(val_line) { + self.render_missed_comments(val_line)?; + } + if let Some(ref e) = _def.default { + let default_line = e.pos().line; + if self.has_comment(default_line) { + self.render_missed_comments(default_line)?; + } + } write!(self.w, "select ")?; self.render_expr(&_def.val)?; write!(self.w, ", ")?; From 685917876b7332bbc7166566c9b4b0fdc8ba7e71 Mon Sep 17 00:00:00 2001 From: Jeremy Wall Date: Fri, 24 May 2019 14:45:46 -0500 Subject: [PATCH 21/23] DEV: A bunch of improvments * move comments on the same line up to the previous line. * All statements should have two new lines between them. * Handle comments with indentation whitespace properly. --- src/ast/printer/mod.rs | 28 +++++---- src/ast/printer/test.rs | 123 ++++++++++++++++++++++------------------ 2 files changed, 85 insertions(+), 66 deletions(-) diff --git a/src/ast/printer/mod.rs b/src/ast/printer/mod.rs index 27d91e2..4c0d25b 100644 --- a/src/ast/printer/mod.rs +++ b/src/ast/printer/mod.rs @@ -76,7 +76,7 @@ where None => return false, }; for c in s.chars() { - if !c.is_ascii_alphanumeric() { + if !(c.is_ascii_alphabetic() || c == '_') { return false; } } @@ -89,7 +89,15 @@ where let cg = map.get(&line).unwrap_or(&empty); let indent = self.make_indent(); for c in cg.iter() { - write!(self.w, "{}// {}\n", indent, c.fragment.trim())?; + let first_char = match c.fragment.chars().nth(0) { + Some(c) => c, + None => '\0', + }; + if !first_char.is_whitespace() { + write!(self.w, "{}// {}\n", indent, c.fragment.trim_end())?; + } else { + write!(self.w, "{}//{}\n", indent, c.fragment.trim_end())?; + } } self.comment_group_lines.pop(); } @@ -100,7 +108,7 @@ where loop { if let Some(next_comment_line) = self.comment_group_lines.last() { let next_comment_line = *next_comment_line; - if next_comment_line < line { + if next_comment_line <= line { self.print_comment_group(next_comment_line)?; } else { break; @@ -116,18 +124,14 @@ where } fn render_comment_if_needed(&mut self, line: usize) -> std::io::Result<()> { - if line > self.last_line { - self.render_missed_comments(line)?; - self.last_line = line; - } + self.render_missed_comments(line)?; + self.last_line = line; Ok(()) } fn has_comment(&self, line: usize) -> bool { - if line > self.last_line { - if let Some(next_comment_line) = self.comment_group_lines.last() { - return *next_comment_line < line; - } + if let Some(next_comment_line) = self.comment_group_lines.last() { + return *next_comment_line < line; } false } @@ -549,7 +553,7 @@ where self.render_expr(&_expr)?; } }; - write!(self.w, ";\n")?; + write!(self.w, ";\n\n")?; self.last_line = line; Ok(()) } diff --git a/src/ast/printer/test.rs b/src/ast/printer/test.rs index cf07201..2e6bba9 100644 --- a/src/ast/printer/test.rs +++ b/src/ast/printer/test.rs @@ -33,67 +33,67 @@ fn print_to_buffer(input: &str) -> String { #[test] fn test_simple_value_printing() { let input = "1;"; - assert_eq!(print_to_buffer(input), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n\n", input)); } #[test] fn test_simple_selector_printing() { let input = "foo.bar.quux;"; - assert_eq!(print_to_buffer(input), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n\n", input)); } #[test] fn test_simple_quoted_printing() { let input = "\"foo\";"; - assert_eq!(print_to_buffer(input), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n\n", input)); } #[test] fn test_escaped_quoted_printing() { let input = "\"f\\\\o\\\"o\";"; - assert_eq!(print_to_buffer(input), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n\n", input)); } #[test] fn test_empty_tuple_printing() { let input = "{};"; - assert_eq!(print_to_buffer(input), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n\n", input)); } #[test] fn test_empty_list_printing() { let input = "[];"; - assert_eq!(print_to_buffer(input), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n\n", input)); } #[test] fn test_non_empty_tuple_printing() { let input = "{\n foo = 1,\n};"; - assert_eq!(print_to_buffer(input), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n\n", input)); } #[test] fn test_nested_empty_tuple_printing() { let input = "{\n foo = {},\n};"; - assert_eq!(print_to_buffer(input), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n\n", input)); } #[test] fn test_list_nested_empty_tuple_printing() { let input = "[\n {},\n];"; - assert_eq!(print_to_buffer(input), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n\n", input)); } #[test] fn test_nested_non_empty_tuple_printing() { let input = "{\n foo = {\n bar = 1,\n },\n};"; - assert_eq!(print_to_buffer(input), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n\n", input)); } #[test] fn test_nested_non_empty_list_printing() { let input = "[\n [\n 1,\n ],\n];"; - assert_eq!(print_to_buffer(input), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n\n", input)); } #[test] @@ -101,80 +101,80 @@ fn test_simple_quoted_field_tuple_printing() { let input = "{\n \"foo\" = {\n bar = 1,\n },\n};"; assert_eq!( print_to_buffer(input), - format!("{}\n", "{\n foo = {\n bar = 1,\n },\n};") + format!("{}\n\n", "{\n foo = {\n bar = 1,\n },\n};") ); } #[test] fn test_special_quoted_field_tuple_printing() { let input = "{\n \"foo bar\" = {\n bar = 1,\n },\n};"; - assert_eq!(print_to_buffer(input), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n\n", input)); } #[test] fn test_let_statement_printing() { let input = "let tpl = {\n \"foo bar\" = {\n bar = 1,\n },\n};"; - assert_eq!(print_to_buffer(input), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n\n", input)); } #[test] fn test_call_expr_printing() { let input = "call(\n foo,\n bar,\n);"; - assert_eq!(print_to_buffer(input), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n\n", input)); } #[test] fn test_call_expr_one_arg_printing() { let input = "call(foo);"; - assert_eq!(print_to_buffer(input), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n\n", input)); } #[test] fn test_copy_expr_printing() { let input = "copy{\n foo = 1,\n bar = 2,\n};"; - assert_eq!(print_to_buffer(input), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n\n", input)); } #[test] fn test_copy_expr_one_arg_printing() { let input = "copy{\n foo = 1,\n};"; - assert_eq!(print_to_buffer(input), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n\n", input)); } #[test] fn test_out_expr_printing() { let input = "out json {\n foo = 1,\n};"; - assert_eq!(print_to_buffer(input), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n\n", input)); } #[test] fn test_select_expr_no_default_printing() { let input = "select true, {\n true = 1,\n false = 2,\n};"; - assert_eq!(print_to_buffer(input), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n\n", input)); } #[test] fn test_select_expr_with_default_printing() { let input = "select true, 3, {\n true = 1,\n false = 2,\n};"; - assert_eq!(print_to_buffer(input), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n\n", input)); } #[test] fn test_not_expr_printing() { let input = "not true;"; - assert_eq!(print_to_buffer(input), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n\n", input)); } #[test] fn test_fail_expr_printing() { let input = "fail \"AHHh\";"; - assert_eq!(print_to_buffer(input), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n\n", input)); } #[test] fn test_trace_expr_printing() { let input = "TRACE \"AHHh\";"; - assert_eq!(print_to_buffer(input), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n\n", input)); } #[test] @@ -186,11 +186,12 @@ fn test_module_no_out_expr_printing() { } => { let config = { hostname = mod.hostname, - \"memory_size\" = mod.mem, - \"cpu_count\" = mod.cpu, + memory_size = mod.mem, + cpu_count = mod.cpu, }; + };"; - assert_eq!(print_to_buffer(input), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n\n", input)); } #[test] @@ -202,11 +203,12 @@ fn test_module_with_out_expr_printing() { } => (config) { let config = { hostname = mod.hostname, - \"memory_size\" = mod.mem, - \"cpu_count\" = mod.cpu, + memory_size = mod.mem, + cpu_count = mod.cpu, }; + };"; - assert_eq!(print_to_buffer(input), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n\n", input)); } #[test] @@ -215,7 +217,7 @@ fn test_func_expr_printing() { foo = foo, bar = bar, };"; - assert_eq!(print_to_buffer(input), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n\n", input)); } #[test] @@ -223,7 +225,7 @@ fn test_func_expr_single_arg_printing() { let input = "let f = func (foo) => { foo = foo, };"; - assert_eq!(print_to_buffer(input), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n\n", input)); } #[test] @@ -231,7 +233,7 @@ fn test_format_expr_single_arg_printing() { let input = "\"what? @{item.foo}\" % { foo = 1, };"; - assert_eq!(print_to_buffer(input), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n\n", input)); } #[test] @@ -239,37 +241,51 @@ fn test_format_expr_list_arg_printing() { let input = "\"what? @ @\" % ( 1, 2);"; - assert_eq!(print_to_buffer(input), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n\n", input)); } #[test] fn test_statement_with_comment_printing() { let input = "// add 1 + 1\n1 + 1;"; - assert_eq!(print_to_buffer(input), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n\n", input)); } #[test] fn test_statement_with_comment_printing_groups() { let input = "// add 1\n// and 1\n1 + 1;"; - assert_eq!(print_to_buffer(input), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n\n", input)); } #[test] fn test_statement_with_comment_printing_multiple_groups() { let input = "\n// group 1\n// more group 1\n\n// group 2\n// more group 2\n1 + 1;"; - assert_eq!(print_to_buffer(input), format!("{}\n", input.trim())); + assert_eq!(print_to_buffer(input), format!("{}\n\n", input.trim())); } #[test] fn test_statement_with_comment_printing_comments_at_end() { - let input = "// group 1\n1 + 1;\n// group 2\n\n"; + let input = "// group 1\n1 + 1;\n\n// group 2\n\n"; assert_eq!(print_to_buffer(input), format!("{}\n", input.trim())); } #[test] fn test_tuple_expression_with_embedded_comment() { let input = "{\n foo = bar,\n // a comment\n bar = foo,\n};"; - assert_eq!(print_to_buffer(input), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n\n", input)); +} + +#[test] +fn test_tuple_expression_with_embedded_comment_same_line() { + let input = "{ + foo = bar, // a comment + bar = foo, +};"; + let expected = "{ + // a comment + foo = bar, + bar = foo, +};"; + assert_eq!(print_to_buffer(input), format!("{}\n\n", expected)); } #[test] @@ -277,7 +293,7 @@ fn test_tuple_expression_with_embedded_comment_mid_field_expr() { let input = "{\n foo = bar,\n bar =\n// a comment\n foo\n};"; assert_eq!( print_to_buffer(input), - "{\n foo = bar,\n // a comment\n bar = foo,\n};\n" + "{\n foo = bar,\n // a comment\n bar = foo,\n};\n\n" ); } @@ -286,57 +302,57 @@ fn test_tuple_expression_with_embedded_comment_and_mid_field_expr() { let input = "{\n foo = bar,\n// a comment\n bar =\n// another comment\n foo\n};"; assert_eq!( print_to_buffer(input), - "{\n foo = bar,\n // a comment\n // another comment\n bar = foo,\n};\n" + "{\n foo = bar,\n // a comment\n // another comment\n bar = foo,\n};\n\n" ); } #[test] fn test_list_expression_with_embedded_comment() { let input = "[\n bar,\n // a comment\n foo,\n];"; - assert_eq!(print_to_buffer(input), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n\n", input)); } #[test] fn test_binary_expression_with_embedded_comment() { let input = "true == \n// false is not true\nfalse;"; - assert_eq!(print_to_buffer(input), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n\n", input)); } #[test] fn test_empty_call_expression_with_comment() { let input = "// a comment\nmyfunc();"; - assert_eq!(print_to_buffer(input), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n\n", input)); } #[test] fn test_call_expression_with_embedded_comment_in_args() { let input = "// a comment\nmyfunc(\n arg1,\n // another comment\n arg2,\n);"; - assert_eq!(print_to_buffer(input), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n\n", input)); } #[test] fn test_copy_expression_with_embedded_comment_in_args() { let input = "// a comment\nmyfunc{\n foo = arg1,\n // another comment\n bar = arg2,\n};"; - assert_eq!(print_to_buffer(input), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n\n", input)); } #[test] fn test_trace_expression_with_embedded_comment() { let input = "// a comment\nTRACE \n// another comment\nfoo;"; - assert_eq!(print_to_buffer(input), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n\n", input)); } #[test] fn test_fail_expression_with_embedded_comment() { let input = "// a comment\nfail \n// another comment\nfoo;"; - assert_eq!(print_to_buffer(input), format!("{}\n", input)); + assert_eq!(print_to_buffer(input), format!("{}\n\n", input)); } #[test] fn test_format_expression_with_embedded_comment() { let input = "// a comment\n\"@(item.bar)\" % \n// another comment\nfoo;"; let output = print_to_buffer(input); - assert_eq!(output, format!("{}\n", input.trim())); + assert_eq!(output, format!("{}\n\n", input.trim())); } #[test] @@ -344,12 +360,11 @@ fn test_filter_func_operator_expression_with_embedded_comment() { //let input = "// a comment\nfilter(foo, bar);"; let input = "// a comment\nfilter(\n // another comment\n foo,\n // one more\n bar);"; let output = print_to_buffer(input); - assert_eq!(output, format!("{}\n", input.trim())); + assert_eq!(output, format!("{}\n\n", input.trim())); } #[test] fn test_reduce_func_operator_expression_with_embedded_comment() { - //let input = "// a comment\nfilter(foo, bar);"; let input = "// a comment\nreduce( // another comment myfunc, @@ -358,7 +373,7 @@ fn test_reduce_func_operator_expression_with_embedded_comment() { // and the last target);"; let output = print_to_buffer(input); - assert_eq!(output, format!("{}\n", input.trim())); + assert_eq!(output, format!("{}\n\n", input.trim())); } #[test] @@ -366,7 +381,7 @@ fn test_map_func_operator_expression_with_embedded_comment() { //let input = "// a comment\nfilter(foo, bar);"; let input = "// a comment\nmap(\n // another comment\n foo,\n // one more\n bar);"; let output = print_to_buffer(input); - assert_eq!(output, format!("{}\n", input.trim())); + assert_eq!(output, format!("{}\n\n", input.trim())); } #[test] @@ -374,5 +389,5 @@ fn test_grouped_expression_with_embedded_comment() { //let input = "// a comment\nfilter(foo, bar);"; let input = "// a comment\n(\n // a comment\n foo\n);"; let output = print_to_buffer(input); - assert_eq!(output, format!("{}\n", input.trim())); + assert_eq!(output, format!("{}\n\n", input.trim())); } From 3d3a0a397ccfef36e3835ed35c6fd7f86899d70e Mon Sep 17 00:00:00 2001 From: Jeremy Wall Date: Fri, 24 May 2019 15:37:37 -0500 Subject: [PATCH 22/23] FEATURE: Wire in the fmt command. issue #43 --- src/main.rs | 68 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/src/main.rs b/src/main.rs index 0e220d0..27fadd6 100644 --- a/src/main.rs +++ b/src/main.rs @@ -17,9 +17,11 @@ extern crate dirs; extern crate ucglib; use std::cell::RefCell; +use std::collections::BTreeMap; use std::error::Error; use std::fs::File; use std::io; +use std::io::Read; use std::path::{Path, PathBuf}; use std::process; use std::rc::Rc; @@ -29,6 +31,8 @@ use ucglib::build::assets::{Cache, MemoryCache}; use ucglib::build::Val; use ucglib::convert::traits; use ucglib::convert::{ConverterRegistry, ImporterRegistry}; +use ucglib::iter::OffsetStrIter; +use ucglib::parse::parse; fn do_flags<'a, 'b>() -> clap::App<'a, 'b> { clap_app!( @@ -53,6 +57,12 @@ fn do_flags<'a, 'b>() -> clap::App<'a, 'b> { (@arg recurse: -r "Whether we should recurse or not.") (@arg INPUT: ... "Input ucg files or directories to run test assertions for. If not provided it will scan the current directory for files with _test.ucg") ) + (@subcommand fmt => + (about: "Format ucg files automatically.") + (@arg recurse: -r "Whether we should recurse or not.") + (@arg indent: -i --indent "How many spaces to indent by. Defaults to 4") + (@arg INPUT: ... "Input ucg files or directories to format") + ) (@subcommand converters => (about: "list the available converters") (@arg converter: "Converter name to get help for.") @@ -346,6 +356,60 @@ fn build_command( } } +fn fmt_file(p: &Path, indent: usize) -> std::result::Result<(), Box> { + let mut f = File::open(p)?; + let mut contents = String::new(); + f.read_to_string(&mut contents)?; + let mut comment_map = BTreeMap::new(); + let stmts = parse(OffsetStrIter::new(&contents), Some(&mut comment_map))?; + let mut printer = ucglib::ast::printer::AstPrinter::new(indent, std::io::stdout()) + .with_comment_map(&comment_map); + printer.render(&stmts)?; + Ok(()) +} + +fn fmt_dir(p: &Path, recurse: bool, indent: usize) -> std::result::Result<(), Box> { + // TODO(jwall): We should handle this error more gracefully + // for the user here. + let dir_iter = std::fs::read_dir(p)?.peekable(); + for entry in dir_iter { + let next_item = entry.unwrap(); + let path = next_item.path(); + if path.is_dir() && recurse { + fmt_dir(&path, recurse, indent)?; + } else { + fmt_file(&path, indent)?; + } + } + Ok(()) +} + +fn fmt_command(matches: &clap::ArgMatches) -> std::result::Result<(), Box> { + let files = matches.values_of("INPUT"); + let recurse = matches.is_present("recurse"); + let indent = match matches.value_of("indent") { + Some(s) => s.parse::()?, + None => 4, + }; + + let mut paths = Vec::new(); + if files.is_none() { + paths.push(std::env::current_dir()?); + } else { + for f in files.unwrap() { + paths.push(PathBuf::from(f)); + } + } + for p in paths { + if p.is_dir() { + fmt_dir(&p, recurse, indent)?; + } else { + fmt_file(&p, indent)?; + } + } + Ok(()) +} + fn test_command( matches: &clap::ArgMatches, import_paths: &Vec, @@ -482,6 +546,10 @@ fn main() { importers_command(®istry) } else if let Some(_) = app_matches.subcommand_matches("env") { env_help() + } else if let Some(matches) = app_matches.subcommand_matches("fmt") { + if let Err(e) = fmt_command(matches) { + eprintln!("{}", e); + } } else { app.print_help().unwrap(); println!(""); From 69eb7a398e6315752ecf91ed27f2ea8acbcd6c1c Mon Sep 17 00:00:00 2001 From: Jeremy Wall Date: Fri, 24 May 2019 15:46:29 -0500 Subject: [PATCH 23/23] DOC: Add new command to README doc. --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 201674e..1609351 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,7 @@ SUBCOMMANDS: converters list the available converters env Describe the environment variables ucg uses. eval Evaluate an expression with an optional ucg file as context. + fmt Format ucg files automatically. help Prints this message or the help of the given subcommand(s) importers list the available importers for includes test Check a list of ucg files for errors and run test assertions.