From e46496c6664e0309ba7f7d6f64ebb16e032232a7 Mon Sep 17 00:00:00 2001 From: Jeremy Wall Date: Sun, 6 Jan 2019 20:56:08 -0600 Subject: [PATCH] FEATURE: Add a reduce operator for lists and tuples. fixes #19 fixes #21 --- .../functional_processing_test.ucg | 29 +++ src/ast/mod.rs | 49 +++-- src/build/mod.rs | 167 +++++++++++++----- src/build/test.rs | 2 +- src/parse/mod.rs | 99 ++++++----- src/tokenizer/mod.rs | 5 + 6 files changed, 246 insertions(+), 105 deletions(-) diff --git a/integration_tests/functional_processing_test.ucg b/integration_tests/functional_processing_test.ucg index f5a3b0b..a39c7a8 100644 --- a/integration_tests/functional_processing_test.ucg +++ b/integration_tests/functional_processing_test.ucg @@ -1,3 +1,5 @@ +// List processing + let list1 = [1, 2, 3, 4]; let list2 = ["foo", "bar", "foo", "bar"]; @@ -12,6 +14,22 @@ let boolfiltrator = macro(item) => { result = item < 5, }; +let identity_list_reducer = macro(acc, item) => { + result = acc + [item], +}; + +assert | + reduce identity_list_reducer.result [], list1 == list1; +|; + +let list_reducer = macro(acc, item) => { + result = acc + item, +}; + +assert | + reduce list_reducer.result 0, list1 == 0 + 1 + 2 + 3 + 4; +|; + assert | map mapper.result list1 == [2, 3, 4, 5]; |; @@ -75,4 +93,15 @@ assert | assert | filter tpl_filter.result test_tpl == { quux = "baz" }; +|; + +let tpl_reducer = macro(acc, name, val) => { + result = acc{ + keys = self.keys + [name], + vals = self.vals + [val], + }, +}; + +assert | + reduce tpl_reducer.result {keys = [], vals = []}, test_tpl == {keys = ["foo", "quux"], vals = ["bar", "baz"]}; |; \ No newline at end of file diff --git a/src/ast/mod.rs b/src/ast/mod.rs index 4cfbaf7..aa23679 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -397,7 +397,7 @@ impl MacroDef { ) -> HashSet { let mut bad_symbols = HashSet::new(); if let &Value::Symbol(ref name) = val { - if !self.symbol_is_in_args(&name.val) { + if name.val != "self" && !self.symbol_is_in_args(&name.val) { bad_symbols.insert(name.val.clone()); } } else if let &Value::Tuple(ref tuple_node) = val { @@ -442,12 +442,6 @@ impl MacroDef { stack.push(expr); } } - &Expression::Copy(ref def) => { - let fields = &def.fields; - for &(_, ref expr) in fields.iter() { - stack.push(expr); - } - } &Expression::Call(ref def) => { for expr in def.arglist.iter() { stack.push(expr); @@ -458,8 +452,9 @@ impl MacroDef { bad_symbols.extend(syms_set.drain()); } &Expression::Macro(_) + | &Expression::Copy(_) | &Expression::Module(_) - | &Expression::ListOp(_) + | &Expression::FuncOp(_) | &Expression::Include(_) => { // noop continue; @@ -561,23 +556,41 @@ pub struct ListDef { pub pos: Position, } -/// ListOpType represents the type of list operation for a ListOpDef. #[derive(Debug, PartialEq, Clone)] -pub enum ListOpType { - Map, - Filter, +pub enum FuncOpDef { + Reduce(ReduceOpDef), + Map(MapFilterOpDef), + Filter(MapFilterOpDef), } -/// ListOpDef implements the list operations in the UCG AST. #[derive(Debug, PartialEq, Clone)] -pub struct ListOpDef { - pub typ: ListOpType, +pub struct ReduceOpDef { + pub mac: PositionedItem, + pub field: PositionedItem, + pub acc: Box, + pub target: Box, + pub pos: Position, +} + +/// MapFilterOpDef implements the list operations in the UCG AST. +#[derive(Debug, PartialEq, Clone)] +pub struct MapFilterOpDef { pub mac: PositionedItem, pub field: PositionedItem, pub target: Box, pub pos: Position, } +impl FuncOpDef { + pub fn pos(&self) -> &Position { + match self { + FuncOpDef::Map(def) => &def.pos, + FuncOpDef::Filter(def) => &def.pos, + FuncOpDef::Reduce(def) => &def.pos, + } + } +} + // TODO(jwall): this should probably be moved to a Val::Module IR type. #[derive(Debug, PartialEq, Clone)] pub struct ModuleDef { @@ -632,7 +645,7 @@ pub enum Expression { Call(CallDef), Macro(MacroDef), Select(SelectDef), - ListOp(ListOpDef), + FuncOp(FuncOpDef), Module(ModuleDef), } @@ -649,7 +662,7 @@ impl Expression { &Expression::Macro(ref def) => &def.pos, &Expression::Module(ref def) => &def.pos, &Expression::Select(ref def) => &def.pos, - &Expression::ListOp(ref def) => &def.pos, + &Expression::FuncOp(ref def) => def.pos(), &Expression::Include(ref def) => &def.pos, } } @@ -664,7 +677,7 @@ impl fmt::Display for Expression { &Expression::Binary(_) => { write!(w, "")?; } - &Expression::ListOp(_) => { + &Expression::FuncOp(_) => { write!(w, "")?; } &Expression::Copy(_) => { diff --git a/src/build/mod.rs b/src/build/mod.rs index 3b8a3ae..c39d8d9 100644 --- a/src/build/mod.rs +++ b/src/build/mod.rs @@ -41,6 +41,11 @@ pub mod scope; pub use self::ir::Val; +enum ProcessingOpType { + Map, + Filter, +} + impl MacroDef { /// Expands a ucg Macro using the given arguments into a new Tuple. pub fn eval( @@ -206,7 +211,7 @@ impl<'a> FileBuilder<'a> { } fn eval_tuple( - &mut self, + &self, fields: &Vec<(Token, Expression)>, scope: &Scope, ) -> Result, Box> { @@ -218,7 +223,7 @@ impl<'a> FileBuilder<'a> { Ok(Rc::new(Val::Tuple(new_fields))) } - fn eval_list(&mut self, def: &ListDef, scope: &Scope) -> Result, Box> { + fn eval_list(&self, def: &ListDef, scope: &Scope) -> Result, Box> { let mut vals = Vec::new(); for expr in def.elems.iter() { vals.push(self.eval_expr(expr, scope)?); @@ -226,7 +231,7 @@ impl<'a> FileBuilder<'a> { Ok(Rc::new(Val::List(vals))) } - fn eval_value(&mut self, v: &Value, scope: &Scope) -> Result, Box> { + fn eval_value(&self, v: &Value, scope: &Scope) -> Result, Box> { match v { &Value::Empty(_) => Ok(Rc::new(Val::Empty)), &Value::Boolean(ref b) => Ok(Rc::new(Val::Boolean(b.val))), @@ -736,11 +741,7 @@ impl<'a> FileBuilder<'a> { ))) } - fn do_dot_lookup( - &mut self, - right: &Expression, - scope: &Scope, - ) -> Result, Box> { + fn do_dot_lookup(&self, right: &Expression, scope: &Scope) -> Result, Box> { let pos = right.pos().clone(); match right { Expression::Copy(_) => return self.eval_expr(right, scope), @@ -771,7 +772,7 @@ impl<'a> FileBuilder<'a> { } fn do_element_check( - &mut self, + &self, left: &Expression, right: &Expression, scope: &Scope, @@ -817,7 +818,7 @@ impl<'a> FileBuilder<'a> { } } - fn eval_binary(&mut self, def: &BinaryOpDef, scope: &Scope) -> Result, Box> { + fn eval_binary(&self, def: &BinaryOpDef, scope: &Scope) -> Result, Box> { let kind = &def.kind; if let &BinaryExprType::IN = kind { return self.do_element_check(&def.left, &def.right, scope); @@ -858,7 +859,7 @@ impl<'a> FileBuilder<'a> { } fn copy_from_base( - &mut self, + &self, src_fields: &Vec<(PositionedItem, Rc)>, overrides: &Vec<(Token, Expression)>, scope: &Scope, @@ -935,7 +936,7 @@ impl<'a> FileBuilder<'a> { ))); } - fn eval_copy(&mut self, def: &CopyDef, scope: &Scope) -> Result, Box> { + fn eval_copy(&self, def: &CopyDef, scope: &Scope) -> Result, Box> { let v = self.eval_value(&def.selector, scope)?; if let &Val::Tuple(ref src_fields) = v.as_ref() { let mut child_scope = scope.spawn_child(); @@ -998,7 +999,7 @@ impl<'a> FileBuilder<'a> { ))) } - fn eval_format(&mut self, def: &FormatDef, scope: &Scope) -> Result, Box> { + fn eval_format(&self, def: &FormatDef, scope: &Scope) -> Result, Box> { let tmpl = &def.template; let args = &def.args; let mut vals = Vec::new(); @@ -1010,7 +1011,7 @@ impl<'a> FileBuilder<'a> { Ok(Rc::new(Val::Str(formatter.render(&def.pos)?))) } - fn eval_call(&mut self, def: &CallDef, scope: &Scope) -> Result, Box> { + fn eval_call(&self, def: &CallDef, scope: &Scope) -> Result, Box> { let args = &def.arglist; let v = self.eval_value(&def.macroref, scope)?; if let &Val::Macro(ref m) = v.deref() { @@ -1055,11 +1056,7 @@ impl<'a> FileBuilder<'a> { }; } - fn eval_module_def( - &mut self, - def: &ModuleDef, - scope: &Scope, - ) -> Result, Box> { + fn eval_module_def(&self, def: &ModuleDef, scope: &Scope) -> Result, Box> { let root = self.file_dir(); // Always work on a copy. The original should not be modified. let mut def = def.clone(); @@ -1071,7 +1068,7 @@ impl<'a> FileBuilder<'a> { Ok(Rc::new(Val::Module(def))) } - fn eval_select(&mut self, def: &SelectDef, scope: &Scope) -> Result, Box> { + fn eval_select(&self, def: &SelectDef, scope: &Scope) -> Result, Box> { let target = &def.val; let def_expr = &def.default; let fields = &def.tuple; @@ -1117,7 +1114,7 @@ impl<'a> FileBuilder<'a> { elems: &Vec>, def: &MacroDef, outfield: &PositionedItem, - typ: &ListOpType, + typ: ProcessingOpType, ) -> Result, Box> { let mut out = Vec::new(); for item in elems.iter() { @@ -1125,10 +1122,10 @@ impl<'a> FileBuilder<'a> { let fields = def.eval(self.file.clone(), self, argvals)?; if let Some(v) = find_in_fieldlist(&outfield.val, &fields) { match typ { - ListOpType::Map => { + ProcessingOpType::Map => { out.push(v.clone()); } - ListOpType::Filter => { + ProcessingOpType::Filter => { if let &Val::Empty = v.as_ref() { // noop continue; @@ -1149,7 +1146,7 @@ impl<'a> FileBuilder<'a> { fs: &Vec<(PositionedItem, Rc)>, def: &MacroDef, outfield: &PositionedItem, - typ: &ListOpType, + typ: ProcessingOpType, ) -> Result, Box> { let mut out = Vec::new(); for &(ref name, ref val) in fs { @@ -1157,7 +1154,7 @@ impl<'a> FileBuilder<'a> { let fields = def.eval(self.file.clone(), self, argvals)?; if let Some(v) = find_in_fieldlist(&outfield.val, &fields) { match typ { - ListOpType::Map => { + ProcessingOpType::Map => { if let &Val::List(ref fs) = v.as_ref() { if fs.len() == 2 { // index 0 should be a string for the new field name. @@ -1199,7 +1196,7 @@ impl<'a> FileBuilder<'a> { ))); } } - ListOpType::Filter => { + ProcessingOpType::Filter => { if let &Val::Empty = v.as_ref() { // noop continue; @@ -1210,17 +1207,23 @@ impl<'a> FileBuilder<'a> { out.push((name.clone(), val.clone())); } } + } else { + return Err(Box::new(error::BuildError::new( + format!( + "Result {} field does not exist in macro body!", + outfield.val + ), + error::ErrorType::NoSuchSymbol, + def.pos.clone(), + ))); } } Ok(Rc::new(Val::Tuple(out))) } - fn eval_functional_processing( - &mut self, - def: &ListOpDef, - scope: &Scope, - ) -> Result, Box> { - let maybe_list = self.eval_expr(&def.target, scope)?; + fn eval_reduce_op(&self, def: &ReduceOpDef, scope: &Scope) -> Result, Box> { + let maybe_target = self.eval_expr(&def.target, scope)?; + let mut acc = self.eval_expr(&def.acc, scope)?; let maybe_mac = self.eval_value(&Value::Symbol(def.mac.clone()), &self.scope.clone())?; let macdef = match maybe_mac.as_ref() { &Val::Macro(ref macdef) => macdef, @@ -1232,15 +1235,85 @@ impl<'a> FileBuilder<'a> { ))); } }; - return match maybe_list.as_ref() { + match maybe_target.as_ref() { &Val::List(ref elems) => { - self.eval_functional_list_processing(elems, macdef, &def.field, &def.typ) + for item in elems.iter() { + let argvals = vec![acc.clone(), item.clone()]; + let fields = macdef.eval(self.file.clone(), self, argvals)?; + if let Some(v) = find_in_fieldlist(&def.field.val, &fields) { + acc = v.clone(); + } else { + return Err(Box::new(error::BuildError::new( + format!("Result {} field does not exist in macro body!", def.field), + error::ErrorType::NoSuchSymbol, + def.pos.clone(), + ))); + } + } } &Val::Tuple(ref fs) => { - self.eval_functional_tuple_processing(fs, macdef, &def.field, &def.typ) + for &(ref name, ref val) in fs.iter() { + let argvals = vec![ + acc.clone(), + Rc::new(Val::Str(name.val.clone())), + val.clone(), + ]; + let fields = macdef.eval(self.file.clone(), self, argvals)?; + if let Some(v) = find_in_fieldlist(&def.field.val, &fields) { + acc = v.clone(); + } else { + return Err(Box::new(error::BuildError::new( + format!("Result field {}does not exist in macro body!", def.field), + error::ErrorType::NoSuchSymbol, + def.pos.clone(), + ))); + } + } + } + other => { + return Err(Box::new(error::BuildError::new( + format!( + "Expected List or Tuple as target but got {:?}", + other.type_name() + ), + error::ErrorType::TypeFail, + def.target.pos().clone(), + ))); + } + } + Ok(acc) + } + + fn eval_functional_processing( + &self, + def: &MapFilterOpDef, + typ: ProcessingOpType, + scope: &Scope, + ) -> Result, Box> { + let maybe_target = self.eval_expr(&def.target, scope)?; + let maybe_mac = self.eval_value(&Value::Symbol(def.mac.clone()), &self.scope.clone())?; + let macdef = match maybe_mac.as_ref() { + &Val::Macro(ref macdef) => macdef, + _ => { + return Err(Box::new(error::BuildError::new( + format!("Expected macro but got {:?}", def.mac), + error::ErrorType::TypeFail, + def.pos.clone(), + ))); + } + }; + return match maybe_target.as_ref() { + &Val::List(ref elems) => { + self.eval_functional_list_processing(elems, macdef, &def.field, typ) + } + &Val::Tuple(ref fs) => { + self.eval_functional_tuple_processing(fs, macdef, &def.field, typ) } other => Err(Box::new(error::BuildError::new( - format!("Expected List as target but got {:?}", other.type_name()), + format!( + "Expected List or Tuple as target but got {:?}", + other.type_name() + ), error::ErrorType::TypeFail, def.target.pos().clone(), ))), @@ -1328,7 +1401,7 @@ impl<'a> FileBuilder<'a> { Ok(contents) } - pub fn eval_include(&mut self, def: &IncludeDef) -> Result, Box> { + pub fn eval_include(&self, def: &IncludeDef) -> Result, Box> { return if def.typ.fragment == "str" { Ok(Rc::new(Val::Str( self.get_file_as_string(&def.path.pos, &def.path.fragment)?, @@ -1351,13 +1424,21 @@ impl<'a> FileBuilder<'a> { }; } + fn eval_func_op(&self, def: &FuncOpDef, scope: &Scope) -> Result, Box> { + match def { + FuncOpDef::Filter(ref def) => { + self.eval_functional_processing(def, ProcessingOpType::Filter, scope) + } + FuncOpDef::Map(ref def) => { + self.eval_functional_processing(def, ProcessingOpType::Map, scope) + } + FuncOpDef::Reduce(ref def) => self.eval_reduce_op(def, scope), + } + } + // Evals a single Expression in the context of a running Builder. // It does not mutate the builders collected state at all. - pub fn eval_expr( - &mut self, - expr: &Expression, - scope: &Scope, - ) -> Result, Box> { + pub fn eval_expr(&self, expr: &Expression, scope: &Scope) -> Result, Box> { match expr { &Expression::Simple(ref val) => self.eval_value(val, scope), &Expression::Binary(ref def) => self.eval_binary(def, scope), @@ -1368,7 +1449,7 @@ impl<'a> FileBuilder<'a> { &Expression::Macro(ref def) => self.eval_macro_def(def), &Expression::Module(ref def) => self.eval_module_def(def, scope), &Expression::Select(ref def) => self.eval_select(def, scope), - &Expression::ListOp(ref def) => self.eval_functional_processing(def, scope), + &Expression::FuncOp(ref def) => self.eval_func_op(def, scope), &Expression::Include(ref def) => self.eval_include(def), } } diff --git a/src/build/test.rs b/src/build/test.rs index 604e68d..af32496 100644 --- a/src/build/test.rs +++ b/src/build/test.rs @@ -19,7 +19,7 @@ use std; use std::cell::RefCell; use std::rc::Rc; -fn test_expr_to_val(mut cases: Vec<(Expression, Val)>, mut b: FileBuilder) { +fn test_expr_to_val(mut cases: Vec<(Expression, Val)>, b: FileBuilder) { for tpl in cases.drain(0..) { assert_eq!( b.eval_expr(&tpl.0, &b.scope.spawn_child()).unwrap(), diff --git a/src/parse/mod.rs b/src/parse/mod.rs index 05fbf60..84ad425 100644 --- a/src/parse/mod.rs +++ b/src/parse/mod.rs @@ -567,57 +567,70 @@ fn call_expression(input: SliceIter) -> Result, Expressi } } -fn tuple_to_list_op<'a>( - input: &'a SliceIter, - kind: ListOpType, - macroname: Value, - outfield: Value, - list: Expression, -) -> ConvertResult<'a, Expression> { - let macroname = match macroname { - Value::Symbol(sym) => sym, - _ => { - return Err(Error::new( - format!("Expected a macro name but got {}", macroname.type_name()), - Box::new(input.clone()), - )) - } - }; - let outfield = match outfield { - Value::Symbol(sym) => sym, - _ => { - return Err(Error::new( - format!("Expected a field name but got {}", outfield.type_name()), - Box::new(input.clone()), - )) - } - }; - return Ok(Expression::ListOp(ListOpDef { - typ: kind, - mac: macroname, - field: outfield, - target: Box::new(list), - pos: input.into(), - })); -} +make_fn!( + reduce_expression, Expression>, + do_each!( + pos => pos, + _ => word!("reduce"), + macroname => match_type!(BAREWORD), + _ => punct!("."), + outfield => match_type!(BAREWORD), + acc => trace_parse!(non_op_expression), + _ => punct!(","), + tgt => trace_parse!(non_op_expression), + (Expression::FuncOp(FuncOpDef::Reduce(ReduceOpDef{ + mac: (¯oname).into(), + field: (&outfield).into(), + acc: Box::new(acc), + target: Box::new(tgt), + pos: pos, + }))) + ) +); make_fn!( - list_op_expression, Expression>, + map_expression, Expression>, do_each!( - input => input!(), - optype => either!( - do_each!(_ => word!("map"), (ListOpType::Map)), - do_each!(_ => word!("filter"), (ListOpType::Filter)) - ), + pos => pos, + _ => word!("map"), // TODO This should become just a bareword symbol now - macroname => trace_parse!(symbol), + macroname => match_type!(BAREWORD), _ => punct!("."), - outfield => trace_parse!(symbol), + outfield => match_type!(BAREWORD), list => trace_parse!(non_op_expression), - (tuple_to_list_op(&input, optype, macroname, outfield, list).unwrap()) + (Expression::FuncOp(FuncOpDef::Map(MapFilterOpDef{ + mac: (¯oname).into(), + field: (&outfield).into(), + target: Box::new(list), + pos: pos, + }))) ) ); +make_fn!( + filter_expression, Expression>, + do_each!( + pos => pos, + _ => word!("filter"), + // TODO This should become just a bareword symbol now + macroname => match_type!(BAREWORD), + _ => punct!("."), + outfield => match_type!(BAREWORD), + list => trace_parse!(non_op_expression), + (Expression::FuncOp(FuncOpDef::Filter(MapFilterOpDef{ + mac: (¯oname).into(), + field: (&outfield).into(), + target: Box::new(list), + pos: pos, + }))) + ) +); + +make_fn!( + func_op_expression, Expression>, + either!(reduce_expression, map_expression, filter_expression) +); + fn unprefixed_expression(input: SliceIter) -> ParseResult { let _input = input.clone(); either!( @@ -632,7 +645,7 @@ fn unprefixed_expression(input: SliceIter) -> ParseResult { make_fn!( non_op_expression, Expression>, either!( - trace_parse!(list_op_expression), + trace_parse!(func_op_expression), trace_parse!(macro_expression), trace_parse!(module_expression), trace_parse!(select_expression), diff --git a/src/tokenizer/mod.rs b/src/tokenizer/mod.rs index dcbc6df..65aa18c 100644 --- a/src/tokenizer/mod.rs +++ b/src/tokenizer/mod.rs @@ -308,6 +308,10 @@ make_fn!(filtertok, do_text_token_tok!(TokenType::BAREWORD, "filter", WS) ); +make_fn!(reducetok, + do_text_token_tok!(TokenType::BAREWORD, "reduce", WS) +); + fn comment(input: OffsetStrIter) -> Result { match text_token!(input, "//") { Result::Complete(rest, _) => { @@ -405,6 +409,7 @@ fn token<'a>(input: OffsetStrIter<'a>) -> Result, Token> { astok, maptok, filtertok, + reducetok, barewordtok, whitespace, end_of_input