DEV: Complex Format expressions are supported now.

src/ast/mod.rs

@@ -36,6 +36,13 @@ pub mod walk;
 
 pub use walk::Walker;
 
+#[derive(Debug, PartialEq, Clone)]
+pub enum TemplatePart {
+    Str(Vec<char>),
+    PlaceHolder(usize),
+    Expression(Expression),
+}
+
 macro_rules! enum_type_equality {
     ( $slf:ident, $r:expr, $( $l:pat ),* ) => {
         match $slf {

src/build/format.rs

@@ -18,24 +18,25 @@ use std::clone::Clone;
 use std::error::Error;
 use std::str::Chars;
 
+use abortable_parser::iter::SliceIter;
+use abortable_parser::Result as ParseResult;
+
 use crate::ast::*;
 use crate::build::assets;
 use crate::build::{FileBuilder, Val};
 use crate::error;
+use crate::iter;
+use crate::parse;
+use crate::tokenizer;
 
 pub trait FormatRenderer {
     fn render(&self, pos: &Position) -> Result<String, Box<dyn Error>>;
 }
 
-#[derive(Debug)]
+pub type TemplateResult = Result<Vec<TemplatePart>, Box<dyn Error>>;
-pub enum TemplatePart<'a> {
-    Str(Vec<char>),
-    PlaceHolder(usize),
-    Expression(&'a str),
-}
 
 pub trait TemplateParser {
-    fn parse<'a>(&self, input: &'a str) -> Vec<TemplatePart<'a>>;
+    fn parse(&self, input: &str) -> TemplateResult;
 }
 
 pub struct SimpleTemplate();
@@ -63,7 +64,7 @@ impl<V: Into<String> + Clone> SimpleFormatter<V> {
 }
 
 impl TemplateParser for SimpleTemplate {
-    fn parse<'a>(&self, input: &'a str) -> Vec<TemplatePart<'a>> {
+    fn parse(&self, input: &str) -> TemplateResult {
         let mut result = Vec::new();
         let mut count = 0;
         let mut should_escape = false;
@@ -76,7 +77,6 @@ impl TemplateParser for SimpleTemplate {
                 result.push(TemplatePart::PlaceHolder(count));
                 count += 1;
             } else if c == '\\' && !should_escape {
-                eprintln!("escaping next character");
                 should_escape = true;
                 continue;
             } else {
@@ -88,7 +88,7 @@ impl TemplateParser for SimpleTemplate {
         if buf.len() != 0 {
             result.push(TemplatePart::Str(buf));
         }
-        result
+        Ok(result)
     }
 }
 
@@ -101,7 +101,7 @@ impl<V: Into<String> + Clone> FormatRenderer for SimpleFormatter<V> {
         let mut buf = String::new();
         let mut count = 0;
         let parser = SimpleTemplate::new();
-        let parts = parser.parse(&self.tmpl);
+        let parts = parser.parse(&self.tmpl)?;
         for p in parts {
             match p {
                 TemplatePart::PlaceHolder(idx) => {
@@ -145,6 +145,88 @@ impl<V: Into<String> + Clone> FormatRenderer for SimpleFormatter<V> {
     }
 }
 
+pub struct ExpressionTemplate();
+
+impl ExpressionTemplate {
+    pub fn new() -> Self {
+        ExpressionTemplate()
+    }
+
+    fn consume_expr(&self, iter: &mut Chars) -> Result<Expression, Box<dyn Error>> {
+        let mut result = String::new();
+        let mut brace_count = 0;
+        loop {
+            let c = match iter.next() {
+                Some(c) => c,
+                None => break,
+            };
+            if c == '{' {
+                brace_count += 1;
+                // We ignore the starting brace
+                if brace_count == 1 {
+                    continue;
+                }
+            }
+            if c == '}' {
+                brace_count -= 1;
+                // We ignore the closing brace
+                if brace_count == 0 {
+                    continue;
+                }
+            }
+            if brace_count == 0 {
+                break;
+            }
+            result.push(c);
+        }
+        let str_iter = iter::OffsetStrIter::new(&result);
+        let toks = match tokenizer::tokenize(str_iter, None) {
+            Ok(toks) => toks,
+            Err(_e) => panic!("TODO(jwall): make this not a thing"),
+        };
+
+        let i = SliceIter::new(&toks);
+        match parse::expression(i) {
+            ParseResult::Complete(_, expr) => Ok(expr),
+            ParseResult::Abort(e) | ParseResult::Fail(e) => {
+                panic!("TODO(jwall): make this not a thing")
+            }
+            ParseResult::Incomplete(_ei) => panic!("TODO(jwall): make this not a thing"),
+        }
+    }
+}
+
+impl TemplateParser for ExpressionTemplate {
+    fn parse(&self, input: &str) -> TemplateResult {
+        let mut parts = Vec::new();
+        let mut should_escape = false;
+        let mut iter = input.chars();
+        let mut buf: Vec<char> = Vec::new();
+        loop {
+            let c = match iter.next() {
+                Some(c) => c,
+                None => break,
+            };
+            if c == '@' && !should_escape {
+                parts.push(TemplatePart::Str(buf));
+                buf = Vec::new();
+                // consume our expression here
+                parts.push(TemplatePart::Expression(self.consume_expr(&mut iter)?));
+            } else if c == '\\' && !should_escape {
+                should_escape = true;
+                continue;
+            } else {
+                buf.push(c);
+            }
+            should_escape = false;
+        }
+        if buf.len() != 0 {
+            parts.push(TemplatePart::Str(buf));
+        }
+        Ok(parts)
+    }
+}
+
 pub struct ExpressionFormatter<'a, C>
 where
     C: assets::Cache,

src/build/opcode/mod.rs

@@ -150,8 +150,10 @@ pub enum Hook {
 #[derive(Debug, PartialEq, Clone)]
 pub enum Op {
     // Stack and Name manipulation.
-    Bind, // Bind a Val to a name in the heap
+    Bind,     // Bind a Val to a name in the heap
-    Pop,  // Pop a Value off the value stack and discard it.
+    BindOver, // Overwrite a value in the heap
+    Pop,      // Pop a Value off the value stack and discard it.
+    NewScope(i32),
     // Math ops
     Add,
     Sub,

src/build/opcode/test.rs

@@ -16,9 +16,9 @@ use std::rc::Rc;
 use super::scope::Stack;
 use super::Composite::{List, Tuple};
 use super::Op::{
-    Add, Bang, Bind, Cp, DeRef, Div, Element, Equal, FCall, Field, Func, Index, InitList,
+    Add, Bang, Bind, BindOver, Cp, DeRef, Div, Element, Equal, FCall, Field, Func, Index, InitList,
-    InitThunk, InitTuple, Jump, JumpIfFalse, JumpIfTrue, Module, Mul, Noop, Pop, Render, Return,
+    InitThunk, InitTuple, Jump, JumpIfFalse, JumpIfTrue, Module, Mul, NewScope, Noop, Pop, Render,
-    SelectJump, Sub, Sym, Typ, Val,
+    Return, SelectJump, Sub, Sym, Typ, Val,
 };
 use super::Primitive::{Bool, Empty, Float, Int, Str};
 use super::Value::{C, P};
@@ -78,6 +78,34 @@ fn math_ops() {
     );
 }
 
+#[test]
+fn new_scopes() {
+    assert_cases![
+        vec![
+            Sym("foo".to_owned()),
+            Val(Int(1)),
+            Bind,
+            NewScope(5),
+            Sym("foo".to_owned()),
+            Val(Int(2)),
+            BindOver,
+            DeRef("foo".to_owned()),
+            Return,
+        ] => P(Int(2)),
+        vec![
+            Sym("bar".to_owned()),
+            Val(Int(1)),
+            Bind,
+            NewScope(5),
+            Sym("foo".to_owned()),
+            Val(Int(2)),
+            Bind,
+            DeRef("bar".to_owned()),
+            Return,
+        ] => P(Int(1)),
+    ];
+}
+
 #[test]
 fn bind_op() {
     let mut cases = vec![(
@@ -546,6 +574,7 @@ macro_rules! assert_parse_cases {
     (__impl__ $cases:expr) => {
         for case in $cases.drain(0..) {
             let stmts = parse(OffsetStrIter::from(dbg!(case.0)), None).unwrap();
+            // TODO(jwall): preprocessor
             let ops = Rc::new(translate::AST::translate(stmts));
             assert!(ops.len() > 0);
             let mut vm = VM::new("foo.ucg", ops.clone());
@@ -672,5 +701,6 @@ fn simple_format_expressions() {
         "\"@\" % (NULL);" => P(Str("NULL".to_owned())),
         "\"@ @ @\" % (1, 2, 3);" => P(Str("1 2 3".to_owned())),
         "\"@ \\\\@\" % (1);" => P(Str("1 @".to_owned())),
+        "\"@{item.num}\" % {num=1};" => P(Str("1".to_owned())),
     ];
 }

src/build/opcode/translate.rs

@@ -11,11 +11,9 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-use crate::ast::Position;
 use crate::ast::{BinaryExprType, Expression, FormatArgs, Statement, Value};
-use crate::build::format::{
+use crate::ast::{Position, TemplatePart};
-    ExpressionFormatter, FormatRenderer, SimpleTemplate, TemplateParser, TemplatePart,
+use crate::build::format::{ExpressionTemplate, SimpleTemplate, TemplateParser};
-};
 use crate::build::opcode::Primitive;
 use crate::build::opcode::Value::{C, F, M, P, T};
 use crate::build::opcode::{Hook, Op};
@@ -178,7 +176,9 @@ impl AST {
                 match def.args {
                     FormatArgs::List(mut elems) => {
                         let formatter = SimpleTemplate::new();
-                        let mut parts = dbg!(formatter.parse(&def.template));
+                        // TODO(jwall): This really belongs in a preprocess step
+                        // before here.
+                        let mut parts = formatter.parse(&def.template).unwrap();
                         // We need to push process these in reverse order for the
                         // vm to process things correctly;
                         elems.reverse();
@@ -196,8 +196,37 @@ impl AST {
                             ops.push(Op::Add);
                         }
                     }
-                    FormatArgs::Single(e) => {
+                    FormatArgs::Single(expr) => {
-                        // TODO(jwall): Use expression formatter here.
+                        let formatter = ExpressionTemplate::new();
+                        // TODO(jwall): This really belongs in a preprocess step
+                        // before here.
+                        let mut parts = formatter.parse(&def.template).unwrap();
+                        parts.reverse();
+                        let mut parts_iter = parts.drain(0..);
+                        // TODO(jwall): We need to assume there is a new scope introduced now
+                        ops.push(Op::Noop);
+                        let scope_idx = ops.len() - 1;
+
+                        // Add our item binding shadowing any binding that already
+                        // existed.
+                        ops.push(Op::Sym("item".to_owned()));
+                        Self::translate_expr(*expr, &mut ops);
+                        ops.push(Op::BindOver);
+                        let mut elems = Vec::new();
+                        let mut elems_iter = elems.drain(0..);
+                        Self::translate_template_part(
+                            parts_iter.next().unwrap(),
+                            &mut elems_iter,
+                            &mut ops,
+                            false,
+                        );
+                        for p in parts_iter {
+                            Self::translate_template_part(p, &mut elems_iter, &mut ops, false);
+                            ops.push(Op::Add);
+                        }
+                        ops.push(Op::Return);
+                        let jump_idx = (ops.len() - 1 - scope_idx) as i32;
+                        ops[scope_idx] = Op::NewScope(jump_idx);
                     }
                 }
             }
@@ -233,16 +262,16 @@ impl AST {
                     // In theory this should never be reachable
                     unreachable!();
                 } else {
-                    Self::translate_expr(dbg!(elems.next().unwrap()), &mut ops);
+                    Self::translate_expr(elems.next().unwrap(), &mut ops);
                     ops.push(Op::Render);
                 }
             }
-            TemplatePart::Expression(_expr) => {
+            TemplatePart::Expression(expr) => {
-                // TODO(jwall): We need to parse this.
                 if place_holder {
                     unreachable!();
                 } else {
-                    unimplemented!("Expression Formatters are unimmplemented");
+                    Self::translate_expr(expr, &mut ops);
+                    ops.push(Op::Render);
                 }
             }
         }

src/build/opcode/vm.rs

@@ -86,7 +86,8 @@ impl<'a> VM {
                 Op::Sub => self.op_sub()?,
                 Op::Mul => self.op_mul()?,
                 Op::Div => self.op_div()?,
-                Op::Bind => self.op_bind()?,
+                Op::Bind => self.op_bind(true)?,
+                Op::BindOver => self.op_bind(false)?,
                 Op::Equal => self.op_equal()?,
                 Op::Not => self.op_not()?,
                 Op::Gt => self.op_gt()?,
@@ -116,7 +117,11 @@ impl<'a> VM {
                 Op::Module(mptr) => self.op_module(idx, mptr)?,
                 Op::Func(jptr) => self.op_func(idx, jptr)?,
                 Op::FCall => self.op_fcall()?,
-                Op::Return => return Ok(()),
+                Op::NewScope(jp) => self.op_new_scope(jp, self.ops.clone())?,
+                Op::Return => {
+                    dbg!(&self.stack);
+                    return Ok(());
+                }
                 Op::Pop => {
                     self.pop()?;
                 }
@@ -161,6 +166,7 @@ impl<'a> VM {
                 .map(|v| (v as i32 + jp) as usize)
                 .unwrap_or(jp as usize),
         )?;
+        dbg!(&self.stack);
         Ok(())
     }
 
@@ -206,7 +212,7 @@ impl<'a> VM {
         let cond = self.pop()?;
         if let &P(Bool(cond)) = cond.as_ref() {
             if !cond {
-                self.op_jump(dbg!(jp))?;
+                self.op_jump(jp)?;
             }
         }
         Ok(())
@@ -214,15 +220,15 @@ impl<'a> VM {
 
     fn op_select_jump(&'a mut self, jp: i32) -> Result<(), Error> {
         // pop field value off
-        let field_name = dbg!(self.pop())?;
+        let field_name = self.pop()?;
         // pop search value off
-        let search = dbg!(self.pop())?;
+        let search = self.pop()?;
         // compare them.
-        if dbg!(field_name != search) {
+        if field_name != search {
-            self.op_jump(dbg!(jp))?;
+            self.op_jump(jp)?;
-            self.push(dbg!(search))?;
+            self.push(search)?;
         }
-        dbg!(self.ops.ptr.unwrap());
+        self.ops.ptr.unwrap();
         // if they aren't equal then push search value back on and jump
         Ok(())
     }
@@ -246,11 +252,11 @@ impl<'a> VM {
         let mut ops = self.ops.clone();
         ops.jump(idx)?;
         self.push(Rc::new(M(Module {
-            ptr: dbg!(ops),
+            ptr: ops,
             result_ptr: result_ptr,
-            flds: dbg!(flds),
+            flds: flds,
         })))?;
-        self.ops.jump(dbg!(jptr))
+        self.ops.jump(jptr)
     }
 
     fn op_func(&mut self, idx: usize, jptr: usize) -> Result<(), Error> {
@@ -276,11 +282,11 @@ impl<'a> VM {
         eprintln!("Pushing function definition on stack");
         let mut ops = self.ops.clone();
         ops.jump(idx)?;
-        self.push(Rc::new(dbg!(F(Func {
+        self.push(Rc::new(F(Func {
             ptr: ops, // where the function starts.
             bindings: bindings,
             snapshot: scope_snapshot,
-        }))))?;
+        })))?;
         eprintln!("Jumping to {} past the function body", jptr);
         self.ops.jump(jptr)
     }
@@ -302,13 +308,25 @@ impl<'a> VM {
             // TODO(jwall): This should do a better error if there is
             // nothing on the stack.
             let val = stack.pop().unwrap();
-            vm.binding_push(nm.clone(), val)?;
+            vm.binding_push(nm.clone(), val, false)?;
         }
         // proceed to the function body
         vm.run()?;
         return vm.pop();
     }
 
+    fn op_new_scope(&mut self, jp: i32, ptr: OpPointer) -> Result<(), Error> {
+        let scope_snapshot = self.symbols.snapshot();
+        dbg!(&ptr);
+        let mut vm = Self::with_pointer(&self.path, ptr).to_scoped(scope_snapshot);
+        dbg!(&vm.stack);
+        vm.run()?;
+        dbg!(&vm.stack);
+        self.push(vm.pop()?)?;
+        self.op_jump(jp)?;
+        Ok(())
+    }
+
     fn op_fcall(&mut self) -> Result<(), Error> {
         let f = self.pop()?;
         if let &F(ref f) = f.as_ref() {
@@ -319,7 +337,7 @@ impl<'a> VM {
     }
 
     fn op_thunk(&mut self, idx: usize, jp: i32) -> Result<(), Error> {
-        self.push(Rc::new(dbg!(T(idx))))?;
+        self.push(Rc::new(T(idx)))?;
         self.op_jump(jp)
     }
 
@@ -444,13 +462,13 @@ impl<'a> VM {
         Ok(())
     }
 
-    fn op_bind(&mut self) -> Result<(), Error> {
+    fn op_bind(&mut self, strict: bool) -> Result<(), Error> {
         // pop val off stack.
-        let val = dbg!(self.pop())?;
+        let val = self.pop()?;
         // pop name off stack.
-        let name = dbg!(self.pop())?;
+        let name = self.pop()?;
         if let &S(ref name) = name.as_ref() {
-            self.binding_push(name.clone(), val)?;
+            self.binding_push(name.clone(), val, strict)?;
         } else {
             return Err(dbg!(Error {}));
         }
@@ -542,8 +560,8 @@ impl<'a> VM {
 
     fn op_index(&mut self) -> Result<(), Error> {
         // left and then right
-        let right = dbg!(self.pop()?);
+        let right = self.pop()?;
-        let left = dbg!(self.pop()?);
+        let left = self.pop()?;
         match right.as_ref() {
             &P(Int(i)) => {
                 if let &C(List(ref elems)) = left.as_ref() {
@@ -573,7 +591,7 @@ impl<'a> VM {
     fn op_copy(&mut self) -> Result<(), Error> {
         // TODO Use Cow pointers for this?
         // get next value. It should be a Module or Tuple.
-        let tgt = dbg!(self.pop())?;
+        let tgt = self.pop()?;
         // This value should always be a tuple
         let override_val = self.pop()?;
         let overrides = if let &C(Tuple(ref oflds)) = override_val.as_ref() {
@@ -585,7 +603,7 @@ impl<'a> VM {
             &C(Tuple(ref flds)) => {
                 let mut flds = flds.clone();
                 for (name, val) in overrides {
-                    dbg!(self.merge_field_into_tuple(&mut flds, name, val))?;
+                    self.merge_field_into_tuple(&mut flds, name, val)?;
                 }
                 // Put the copy on the Stack
                 self.push(Rc::new(C(Tuple(flds))))?;
@@ -608,14 +626,14 @@ impl<'a> VM {
                 //self.merge_field_into_tuple(&mut flds, "this".to_owned(), this)?;
                 let mut vm = Self::with_pointer(self.path.clone(), ptr.clone());
                 vm.push(Rc::new(S("mod".to_owned())))?;
-                vm.push(Rc::new(C(Tuple(dbg!(flds)))))?;
+                vm.push(Rc::new(C(Tuple(flds))))?;
                 vm.run()?;
-                if let Some(ptr) = dbg!(result_ptr) {
+                if let Some(ptr) = result_ptr {
                     vm.ops.jump(ptr.clone())?;
                     vm.run()?;
                     self.push(vm.pop()?)?;
                 } else {
-                    self.push(dbg!(Rc::new(vm.symbols_to_tuple(false))))?;
+                    self.push(Rc::new(vm.symbols_to_tuple(false)))?;
                 }
             }
             _ => {
@@ -646,8 +664,13 @@ impl<'a> VM {
         Ok(())
     }
 
-    pub fn binding_push(&mut self, name: String, val: Rc<Value>) -> Result<(), Error> {
+    pub fn binding_push(
-        if self.symbols.is_bound(&name) {
+        &mut self,
+        name: String,
+        val: Rc<Value>,
+        strict: bool,
+    ) -> Result<(), Error> {
+        if self.symbols.is_bound(&name) && strict {
             return Err(dbg!(Error {}));
         }
         self.symbols.add(name, val);

src/parse/mod.rs

@@ -771,7 +771,7 @@ make_fn!(
     )
 );
 
-fn expression(input: SliceIter<Token>) -> ParseResult<Expression> {
+pub fn expression(input: SliceIter<Token>) -> ParseResult<Expression> {
     let _input = input.clone();
     match trace_parse!(_input, op_expression) {
         Result::Incomplete(i) => Result::Incomplete(i),