CLEANUP: The great test module refactor.

Move tests into a separate file for more manageable file
organization.

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name        = "ucg"
-version     = "0.1.0"
+version     = "0.1.1"
 authors     = ["Jeremy Wall <jeremy@marzhillstudios.com>"]
 description = "A configuration generation grammar."
 repository  = "https://github.com/zaphar/ucg"

src/ast/mod.rs

@@ -14,5 +14,729 @@
 
 //! The definitions of the ucg AST and Tokens.
 
-#[macro_use]
+use std;
-pub mod tree;
+use std::borrow::Borrow;
+use std::cmp::Eq;
+use std::cmp::Ordering;
+use std::cmp::PartialEq;
+use std::cmp::PartialOrd;
+use std::collections::HashSet;
+use std::convert::Into;
+use std::hash::Hash;
+use std::hash::Hasher;
+
+macro_rules! enum_type_equality {
+    ( $slf:ident, $r:expr, $( $l:pat ),* ) => {
+        match $slf {
+        $(
+            $l => {
+                if let $l = $r {
+                    true
+                } else {
+                    false
+                }
+            }
+        )*
+        }
+    }
+}
+
+/// Represents a line and a column position in UCG code.
+///
+/// It is used for generating error messages mostly. Most all
+/// parts of the UCG AST have a positioned associated with them.
+#[derive(Debug, PartialEq, Eq, Clone, PartialOrd, Ord, Hash)]
+pub struct Position {
+    pub line: usize,
+    pub column: usize,
+}
+
+impl Position {
+    /// Construct a new Position.
+    pub fn new(line: usize, column: usize) -> Self {
+        Position {
+            line: line,
+            column: column,
+        }
+    }
+}
+
+/// Defines the types of tokens in UCG syntax.
+#[derive(Debug, PartialEq, Eq, Clone, PartialOrd, Ord, Hash)]
+pub enum TokenType {
+    EMPTY,
+    BOOLEAN,
+    END,
+    WS,
+    COMMENT,
+    QUOTED,
+    DIGIT,
+    BAREWORD,
+    PUNCT,
+}
+
+/// Defines a Token representing a building block of UCG syntax.
+///
+/// Token's are passed to the parser stage to be parsed into an AST.
+#[derive(Debug, PartialEq, Eq, Clone, PartialOrd, Ord, Hash)]
+pub struct Token {
+    pub typ: TokenType,
+    pub fragment: String,
+    pub pos: Position,
+}
+
+impl Token {
+    /// Constructs a new Token with a type and line and column information.
+    pub fn new<S: Into<String>>(f: S, typ: TokenType, line: usize, col: usize) -> Self {
+        Self::new_with_pos(f, typ, Position::new(line, col))
+    }
+
+    // Constructs a new Token with a type and a Position.
+    pub fn new_with_pos<S: Into<String>>(f: S, typ: TokenType, pos: Position) -> Self {
+        Token {
+            typ: typ,
+            fragment: f.into(),
+            pos: pos,
+        }
+    }
+}
+
+impl Borrow<str> for Token {
+    fn borrow(&self) -> &str {
+        &self.fragment
+    }
+}
+
+/// Helper macro for making a Positioned Value.
+macro_rules! value_node {
+    ($v:expr, $p:expr) => {
+        Positioned::new_with_pos($v, $p)
+    };
+    ($v:expr, $l:expr, $c:expr) => {
+        Positioned::new($v, $l, $c)
+    };
+}
+
+/// Helper macro for making a Token.
+#[allow(unused_macros)]
+macro_rules! make_tok {
+    (EOF => $l:expr, $c:expr) => {
+        Token::new("", TokenType::END, $l, $c)
+    };
+
+    (WS => $l:expr, $c:expr) => {
+        Token::new("", TokenType::WS, $l, $c)
+    };
+
+    (CMT => $e:expr, $l:expr, $c:expr) => {
+        Token::new($e, TokenType::COMMENT, $l, $c)
+    };
+
+    (QUOT => $e:expr, $l:expr, $c:expr) => {
+        Token::new($e, TokenType::QUOTED, $l, $c)
+    };
+
+    (PUNCT => $e:expr, $l:expr, $c:expr) => {
+        Token::new($e, TokenType::PUNCT, $l, $c)
+    };
+
+    (DIGIT => $e:expr, $l:expr, $c:expr) => {
+        Token::new($e, TokenType::DIGIT, $l, $c)
+    };
+
+    ($e:expr, $l:expr, $c:expr) => {
+        Token::new($e, TokenType::BAREWORD, $l, $c)
+    };
+}
+
+/// Helper macro for making expressions.
+#[allow(unused_macros)]
+macro_rules! make_expr {
+    ($e:expr) => {
+        make_expr!($e, 1, 1)
+    };
+
+    ($e:expr, $l:expr, $c:expr) => {
+        Expression::Simple(Value::Symbol(Positioned::new($e.to_string(), $l, $c)))
+    };
+
+    ($e:expr => int, $l:expr, $c:expr) => {
+        Expression::Simple(Value::Int(Positioned::new($e, $l, $c)))
+    };
+}
+
+/// Helper macro for making selectors.
+///
+/// ```
+/// make_selector!(Token::new("tpl", 1, 1), Token::new("fld", 1, 4));
+///
+/// make_selector!(Token::new("tpl", 1, 1), vec![Token::new("fld", 1, 4)], => 1, 1);
+///
+/// make_selector!(foo", ["bar"]);
+///
+/// make_selector!(foo", ["bar"] => 1, 0);
+/// ```
+#[allow(unused_macros)]
+macro_rules! make_selector {
+    ( $h:expr ) => {
+        make_selector!($h, 1, 0)
+    };
+
+    ( $h:expr, $l:expr, $c:expr ) => {
+        SelectorDef::new(
+            SelectorList{head: Box::new($h), tail: None},
+            $l, $c)
+    };
+
+    ( $h: expr, $list:expr, $l:expr, $c:expr) => {
+        SelectorDef::new(
+            SelectorList{head: Box::new($h), tail: Some($list)},
+            $l, $c)
+    };
+
+    // Tokens
+    ( $h:expr => [ $( $item:expr ),* ] ) => {
+        {
+            make_selector!($h => [ $( $item, )* ] => 1, 1)
+        }
+    };
+
+    ( $h:expr => [ $( $item:expr ),* ] => $l:expr, $c:expr ) => {
+        {
+            let mut list: Vec<Token> = Vec::new();
+
+            $(
+                list.push($item);
+            )*
+
+            make_selector!($h, list, $l, $c)
+        }
+    };
+
+    // Strings not tokens
+    ( $h:expr => $( $item:expr ),* ) => {
+        {
+
+            let mut col = 1;
+            let mut list: Vec<Token> = Vec::new();
+
+            $(
+                list.push(make_tok!($item, 1, col));
+                col += $item.len() + 1;
+            )*
+
+            // Shut up the lint about unused code;
+            assert!(col != 0);
+
+            make_selector!($h, list, 1, 1)
+        }
+
+    };
+
+    ( $h:expr => $( $item:expr ),* => $l:expr, $c:expr ) => {
+        {
+            let mut col = $c;
+            let mut list: Vec<Token> = Vec::new();
+
+            $(
+                list.push(make_tok!($item, $l, col));
+                col += $item.len() + 1;
+            )*
+
+            // Shut up the linter about unused code;
+            assert!(col != 0);
+
+            make_selector!($h, list, $l, $c)
+        }
+    };
+}
+
+/// An Expression with a series of symbols specifying the key
+/// with which to descend into the result of the expression.
+///
+/// The expression must evaluate to either a tuple or an array. The token must
+/// evaluate to either a bareword Symbol or an Int.
+///
+/// ```ucg
+/// let foo = { bar = "a thing" };
+/// let thing = foo.bar;
+///
+/// let arr = ["one", "two"];
+/// let first = arr.0;
+///
+/// let berry = {best = "strawberry", unique = "acai"}.best;
+/// let third = ["uno", "dos", "tres"].1;
+/// '''
+#[derive(Debug, PartialEq, Clone)]
+pub struct SelectorList {
+    pub head: Box<Expression>,
+    pub tail: Option<Vec<Token>>,
+}
+
+impl SelectorList {
+    /// Returns a stringified version of a SelectorList.
+    pub fn to_string(&self) -> String {
+        "TODO".to_string()
+    }
+}
+
+/// An ordered list of Name = Value pairs.
+///
+/// This is usually used as the body of a tuple in the UCG AST.
+pub type FieldList = Vec<(Token, Expression)>; // Token is expected to be a symbol
+
+/// Encodes a selector expression in the UCG AST.
+#[derive(Debug, PartialEq, Clone)]
+pub struct SelectorDef {
+    pub pos: Position,
+    pub sel: SelectorList,
+}
+
+impl SelectorDef {
+    /// Constructs a new SelectorDef.
+    pub fn new(sel: SelectorList, line: usize, col: usize) -> Self {
+        SelectorDef {
+            pos: Position::new(line, col),
+            sel: sel,
+        }
+    }
+}
+
+/// Represents a Value in the UCG parsed AST.
+#[derive(Debug, PartialEq, Clone)]
+pub enum Value {
+    // Constant Values
+    Empty(Position),
+    Boolean(Positioned<bool>),
+    Int(Positioned<i64>),
+    Float(Positioned<f64>),
+    String(Positioned<String>),
+    Symbol(Positioned<String>),
+    // Complex Values
+    Tuple(Positioned<FieldList>),
+    List(ListDef),
+    Selector(SelectorDef),
+}
+
+impl Value {
+    /// Returns the type name of the Value it is called on as a string.
+    pub fn type_name(&self) -> String {
+        match self {
+            &Value::Empty(_) => "EmptyValue".to_string(),
+            &Value::Boolean(_) => "Boolean".to_string(),
+            &Value::Int(_) => "Integer".to_string(),
+            &Value::Float(_) => "Float".to_string(),
+            &Value::String(_) => "String".to_string(),
+            &Value::Symbol(_) => "Symbol".to_string(),
+            &Value::Tuple(_) => "Tuple".to_string(),
+            &Value::List(_) => "List".to_string(),
+            &Value::Selector(_) => "Selector".to_string(),
+        }
+    }
+
+    fn fields_to_string(v: &FieldList) -> String {
+        let mut buf = String::new();
+        buf.push_str("{\n");
+        for ref t in v.iter() {
+            buf.push_str("\t");
+            buf.push_str(&t.0.fragment);
+            buf.push_str("\n");
+        }
+        buf.push_str("}");
+        return buf;
+    }
+
+    fn elems_to_string(v: &Vec<Expression>) -> String {
+        return format!("{}", v.len());
+    }
+
+    /// Returns a stringified version of the Value.
+    pub fn to_string(&self) -> String {
+        match self {
+            &Value::Empty(_) => "EmptyValue".to_string(),
+            &Value::Boolean(ref b) => format!("{}", b.val),
+            &Value::Int(ref i) => format!("{}", i.val),
+            &Value::Float(ref f) => format!("{}", f.val),
+            &Value::String(ref s) => format!("{}", s.val),
+            &Value::Symbol(ref s) => format!("{}", s.val),
+            &Value::Tuple(ref fs) => format!("{}", Self::fields_to_string(&fs.val)),
+            &Value::List(ref def) => format!("[{}]", Self::elems_to_string(&def.elems)),
+            &Value::Selector(ref v) => v.sel.to_string(),
+        }
+    }
+
+    /// Returns the position for a Value.
+    pub fn pos(&self) -> &Position {
+        match self {
+            &Value::Empty(ref pos) => pos,
+            &Value::Boolean(ref b) => &b.pos,
+            &Value::Int(ref i) => &i.pos,
+            &Value::Float(ref f) => &f.pos,
+            &Value::String(ref s) => &s.pos,
+            &Value::Symbol(ref s) => &s.pos,
+            &Value::Tuple(ref fs) => &fs.pos,
+            &Value::List(ref def) => &def.pos,
+            &Value::Selector(ref v) => &v.pos,
+        }
+    }
+
+    /// Returns true if called on a Value that is the same type as itself.
+    pub fn type_equal(&self, target: &Self) -> bool {
+        enum_type_equality!(
+            self,
+            target,
+            &Value::Empty(_),
+            &Value::Boolean(_),
+            &Value::Int(_),
+            &Value::Float(_),
+            &Value::String(_),
+            &Value::Symbol(_),
+            &Value::Tuple(_),
+            &Value::List(_),
+            &Value::Selector(_)
+        )
+    }
+}
+
+/// Represents an expansion of a Macro that is expected to already have been
+/// defined.
+#[derive(PartialEq, Debug, Clone)]
+pub struct CallDef {
+    pub macroref: SelectorDef,
+    pub arglist: Vec<Expression>,
+    pub pos: Position,
+}
+
+/// Encodes a select expression in the UCG AST.
+#[derive(PartialEq, Debug, Clone)]
+pub struct SelectDef {
+    pub val: Box<Expression>,
+    pub default: Box<Expression>,
+    pub tuple: FieldList,
+    pub pos: Position,
+}
+
+// TODO(jwall): This should have a way of rendering with position information.
+
+/// Adds position information to any type `T`.
+#[derive(Debug, Clone)]
+pub struct Positioned<T> {
+    pub pos: Position,
+    pub val: T,
+}
+
+impl<T: std::fmt::Display> std::fmt::Display for Positioned<T> {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> Result<(), std::fmt::Error> {
+        write!(f, "{}", self.val)
+    }
+}
+
+impl<T> Positioned<T> {
+    /// Constructs a new Positioned<T> with a value, line, and column information.
+    pub fn new(v: T, l: usize, c: usize) -> Self {
+        Self::new_with_pos(v, Position::new(l, c))
+    }
+
+    /// Constructs a new Positioned<T> with a value and a Position.
+    pub fn new_with_pos(v: T, pos: Position) -> Self {
+        Positioned { pos: pos, val: v }
+    }
+}
+
+impl<T: PartialEq> PartialEq for Positioned<T> {
+    fn eq(&self, other: &Self) -> bool {
+        self.val == other.val
+    }
+}
+
+impl<T: Eq> Eq for Positioned<T> {}
+
+impl<T: Ord> Ord for Positioned<T> {
+    fn cmp(&self, other: &Self) -> Ordering {
+        self.val.cmp(&other.val)
+    }
+}
+
+impl<T: PartialOrd> PartialOrd for Positioned<T> {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        self.val.partial_cmp(&other.val)
+    }
+}
+
+impl<T: Hash> Hash for Positioned<T> {
+    fn hash<H: Hasher>(&self, state: &mut H) {
+        self.val.hash(state);
+    }
+}
+
+impl<'a> From<&'a Token> for Positioned<String> {
+    fn from(t: &'a Token) -> Positioned<String> {
+        Positioned {
+            pos: t.pos.clone(),
+            val: t.fragment.to_string(),
+        }
+    }
+}
+
+impl<'a> From<&'a Positioned<String>> for Positioned<String> {
+    fn from(t: &Positioned<String>) -> Positioned<String> {
+        Positioned {
+            pos: t.pos.clone(),
+            val: t.val.clone(),
+        }
+    }
+}
+
+/// Encodes a macro expression in the UCG AST..
+///
+/// A macro is a pure function over a tuple.
+/// MacroDefs are not closures. They can not reference
+/// any values except what is defined in their arguments.
+#[derive(PartialEq, Debug, Clone)]
+pub struct MacroDef {
+    pub argdefs: Vec<Positioned<String>>,
+    pub fields: FieldList,
+    pub pos: Position,
+}
+
+impl MacroDef {
+    fn symbol_is_in_args(&self, sym: &String) -> bool {
+        for arg in self.argdefs.iter() {
+            if &arg.val == sym {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    fn validate_value_symbols<'a>(
+        &self,
+        stack: &mut Vec<&'a Expression>,
+        val: &'a Value,
+    ) -> HashSet<String> {
+        let mut bad_symbols = HashSet::new();
+        if let &Value::Symbol(ref name) = val {
+            if !self.symbol_is_in_args(&name.val) {
+                bad_symbols.insert(name.val.clone());
+            }
+        } else if let &Value::Selector(ref sel_node) = val {
+            stack.push(&sel_node.sel.head);
+        } else if let &Value::Tuple(ref tuple_node) = val {
+            let fields = &tuple_node.val;
+            for &(_, ref expr) in fields.iter() {
+                stack.push(expr);
+            }
+        } else if let &Value::List(ref def) = val {
+            for elem in def.elems.iter() {
+                stack.push(elem);
+            }
+        }
+        return bad_symbols;
+    }
+
+    /// Performs typechecking of a ucg macro's arguments to ensure
+    /// that they are valid for the expressions in the macro.
+    pub fn validate_symbols(&self) -> Result<(), HashSet<String>> {
+        let mut bad_symbols = HashSet::new();
+        for &(_, ref expr) in self.fields.iter() {
+            let mut stack = Vec::new();
+            stack.push(expr);
+            while stack.len() > 0 {
+                match stack.pop().unwrap() {
+                    &Expression::Binary(ref bexpr) => {
+                        stack.push(&bexpr.left);
+                        stack.push(&bexpr.right);
+                    }
+                    &Expression::Compare(ref cexpr) => {
+                        stack.push(&cexpr.left);
+                        stack.push(&cexpr.right);
+                    }
+                    &Expression::Grouped(ref expr) => {
+                        stack.push(expr);
+                    }
+                    &Expression::Format(ref def) => {
+                        let exprs = &def.args;
+                        for arg_expr in exprs.iter() {
+                            stack.push(arg_expr);
+                        }
+                    }
+                    &Expression::Select(ref def) => {
+                        stack.push(def.default.borrow());
+                        stack.push(def.val.borrow());
+                        for &(_, ref expr) in def.tuple.iter() {
+                            stack.push(expr);
+                        }
+                    }
+                    &Expression::Copy(ref def) => {
+                        let fields = &def.fields;
+                        for &(_, ref expr) in fields.iter() {
+                            stack.push(expr);
+                        }
+                    }
+                    &Expression::Call(ref def) => for expr in def.arglist.iter() {
+                        stack.push(expr);
+                    },
+                    &Expression::Simple(ref val) => {
+                        let mut syms_set = self.validate_value_symbols(&mut stack, val);
+                        bad_symbols.extend(syms_set.drain());
+                    }
+                    &Expression::Macro(_) => {
+                        // noop
+                        continue;
+                    }
+                    &Expression::ListOp(_) => {
+                        // noop
+                        continue;
+                    }
+                }
+            }
+        }
+        if bad_symbols.len() > 0 {
+            return Err(bad_symbols);
+        }
+        return Ok(());
+    }
+}
+
+/// Specifies the types of binary operations supported in
+/// UCG expression.
+#[derive(Debug, PartialEq, Clone)]
+pub enum BinaryExprType {
+    Add,
+    Sub,
+    Mul,
+    Div,
+}
+
+#[derive(Debug, PartialEq, Clone)]
+pub enum CompareType {
+    Equal,
+    GT,
+    LT,
+    NotEqual,
+    GTEqual,
+    LTEqual,
+}
+
+#[derive(Debug, PartialEq, Clone)]
+pub struct ComparisonDef {
+    pub kind: CompareType,
+    pub left: Box<Expression>,
+    pub right: Box<Expression>,
+    pub pos: Position,
+}
+
+/// Represents an expression with a left and a right side.
+#[derive(Debug, PartialEq, Clone)]
+pub struct BinaryOpDef {
+    pub kind: BinaryExprType,
+    pub left: Box<Expression>,
+    pub right: Box<Expression>,
+    pub pos: Position,
+}
+
+/// Encodes a tuple Copy expression in the UCG AST.
+#[derive(Debug, PartialEq, Clone)]
+pub struct CopyDef {
+    pub selector: SelectorDef,
+    pub fields: FieldList,
+    pub pos: Position,
+}
+
+/// Encodes a format expression in the UCG AST.
+#[derive(Debug, PartialEq, Clone)]
+pub struct FormatDef {
+    pub template: String,
+    pub args: Vec<Expression>,
+    pub pos: Position,
+}
+
+/// Encodes a list expression in the UCG AST.
+#[derive(Debug, PartialEq, Clone)]
+pub struct ListDef {
+    pub elems: Vec<Expression>,
+    pub pos: Position,
+}
+
+#[derive(Debug, PartialEq, Clone)]
+pub enum ListOpType {
+    Map,
+    Filter,
+}
+
+#[derive(Debug, PartialEq, Clone)]
+pub struct ListOpDef {
+    pub typ: ListOpType,
+    pub mac: SelectorDef,
+    pub field: String,
+    pub target: ListDef,
+    pub pos: Position,
+}
+
+/// Encodes a ucg expression. Expressions compute a value from.
+#[derive(Debug, PartialEq, Clone)]
+pub enum Expression {
+    // Base Expression
+    Simple(Value),
+
+    // Binary expressions
+    Binary(BinaryOpDef),
+    Compare(ComparisonDef),
+
+    // Complex Expressions
+    Copy(CopyDef),
+    Grouped(Box<Expression>),
+    Format(FormatDef),
+    Call(CallDef),
+    Macro(MacroDef),
+    Select(SelectDef),
+    ListOp(ListOpDef),
+}
+
+impl Expression {
+    /// Returns the position of the Expression.
+    pub fn pos(&self) -> &Position {
+        match self {
+            &Expression::Simple(ref v) => v.pos(),
+            &Expression::Binary(ref def) => &def.pos,
+            &Expression::Compare(ref def) => &def.pos,
+            &Expression::Copy(ref def) => &def.pos,
+            &Expression::Grouped(ref expr) => expr.pos(),
+            &Expression::Format(ref def) => &def.pos,
+            &Expression::Call(ref def) => &def.pos,
+            &Expression::Macro(ref def) => &def.pos,
+            &Expression::Select(ref def) => &def.pos,
+            &Expression::ListOp(ref def) => &def.pos,
+        }
+    }
+}
+
+/// Encodes a let statement in the UCG AST.
+#[derive(Debug, PartialEq)]
+pub struct LetDef {
+    pub name: Token,
+    pub value: Expression,
+}
+
+/// Encodes an import statement in the UCG AST.
+#[derive(Debug, PartialEq)]
+pub struct ImportDef {
+    pub path: Token,
+    pub name: Token,
+}
+
+/// Encodes a parsed statement in the UCG AST.
+#[derive(Debug, PartialEq)]
+pub enum Statement {
+    // simple expression
+    Expression(Expression),
+
+    // Named bindings
+    Let(LetDef),
+
+    // Import a file.
+    Import(ImportDef),
+}
+
+#[cfg(test)]
+pub mod test;

src/ast/test.rs

@@ -0,0 +1,99 @@
+use super::*;
+
+#[test]
+pub fn test_macro_validation_happy_path() {
+    let def = MacroDef {
+        argdefs: vec![value_node!("foo".to_string(), 1, 0)],
+        fields: vec![
+            (
+                make_tok!("f1", 1, 1),
+                Expression::Binary(BinaryOpDef {
+                    kind: BinaryExprType::Add,
+                    left: Box::new(Expression::Simple(Value::Symbol(value_node!(
+                        "foo".to_string(),
+                        1,
+                        1
+                    )))),
+                    right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
+                    pos: Position::new(1, 0),
+                }),
+            ),
+        ],
+        pos: Position::new(1, 0),
+    };
+    assert!(def.validate_symbols().unwrap() == ());
+}
+
+#[test]
+pub fn test_macro_validation_fail() {
+    let def = MacroDef {
+        argdefs: vec![value_node!("foo".to_string(), 1, 0)],
+        fields: vec![
+            (
+                make_tok!("f1", 1, 1),
+                Expression::Binary(BinaryOpDef {
+                    kind: BinaryExprType::Add,
+                    left: Box::new(Expression::Simple(Value::Symbol(value_node!(
+                        "bar".to_string(),
+                        1,
+                        1
+                    )))),
+                    right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
+                    pos: Position::new(1, 0),
+                }),
+            ),
+        ],
+        pos: Position::new(1, 0),
+    };
+    let mut expected = HashSet::new();
+    expected.insert("bar".to_string());
+    assert_eq!(def.validate_symbols().err().unwrap(), expected);
+}
+
+#[test]
+pub fn test_macro_validation_selector_happy_path() {
+    let def = MacroDef {
+        argdefs: vec![value_node!("foo".to_string(), 1, 0)],
+        fields: vec![
+            (
+                make_tok!("f1", 1, 1),
+                Expression::Binary(BinaryOpDef {
+                    kind: BinaryExprType::Add,
+                    left: Box::new(Expression::Simple(Value::Selector(
+                        make_selector!(make_expr!("foo", 1, 1) => [
+                    make_tok!("quux", 1, 1) ] => 1, 1),
+                    ))),
+                    right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
+                    pos: Position::new(1, 0),
+                }),
+            ),
+        ],
+        pos: Position::new(1, 0),
+    };
+    assert!(def.validate_symbols().unwrap() == ());
+}
+
+#[test]
+pub fn test_macro_validation_selector_fail() {
+    let def = MacroDef {
+        argdefs: vec![value_node!("foo".to_string(), 1, 0)],
+        fields: vec![
+            (
+                make_tok!("f1", 1, 1),
+                Expression::Binary(BinaryOpDef {
+                    kind: BinaryExprType::Add,
+                    left: Box::new(Expression::Simple(Value::Selector(
+                        make_selector!(make_expr!("bar", 1, 1) => [
+                    make_tok!("quux", 1, 1) ] => 1, 1),
+                    ))),
+                    right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
+                    pos: Position::new(1, 0),
+                }),
+            ),
+        ],
+        pos: Position::new(1, 0),
+    };
+    let mut expected = HashSet::new();
+    expected.insert("bar".to_string());
+    assert_eq!(def.validate_symbols(), Err(expected));
+}

src/ast/tree.rs

@@ -1,839 +0,0 @@
-// Copyright 2017 Jeremy Wall <jeremy@marzhillstudios.com>
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License.
-use std;
-use std::borrow::Borrow;
-use std::cmp::Eq;
-use std::cmp::Ordering;
-use std::cmp::PartialEq;
-use std::cmp::PartialOrd;
-use std::collections::HashSet;
-use std::convert::Into;
-use std::hash::Hash;
-use std::hash::Hasher;
-
-macro_rules! enum_type_equality {
-    ( $slf:ident, $r:expr, $( $l:pat ),* ) => {
-        match $slf {
-        $(
-            $l => {
-                if let $l = $r {
-                    true
-                } else {
-                    false
-                }
-            }
-        )*
-        }
-    }
-}
-
-/// Represents a line and a column position in UCG code.
-///
-/// It is used for generating error messages mostly. Most all
-/// parts of the UCG AST have a positioned associated with them.
-#[derive(Debug, PartialEq, Eq, Clone, PartialOrd, Ord, Hash)]
-pub struct Position {
-    pub line: usize,
-    pub column: usize,
-}
-
-impl Position {
-    /// Construct a new Position.
-    pub fn new(line: usize, column: usize) -> Self {
-        Position {
-            line: line,
-            column: column,
-        }
-    }
-}
-
-/// Defines the types of tokens in UCG syntax.
-#[derive(Debug, PartialEq, Eq, Clone, PartialOrd, Ord, Hash)]
-pub enum TokenType {
-    EMPTY,
-    BOOLEAN,
-    END,
-    WS,
-    COMMENT,
-    QUOTED,
-    DIGIT,
-    BAREWORD,
-    PUNCT,
-}
-
-/// Defines a Token representing a building block of UCG syntax.
-///
-/// Token's are passed to the parser stage to be parsed into an AST.
-#[derive(Debug, PartialEq, Eq, Clone, PartialOrd, Ord, Hash)]
-pub struct Token {
-    pub typ: TokenType,
-    pub fragment: String,
-    pub pos: Position,
-}
-
-impl Token {
-    /// Constructs a new Token with a type and line and column information.
-    pub fn new<S: Into<String>>(f: S, typ: TokenType, line: usize, col: usize) -> Self {
-        Self::new_with_pos(f, typ, Position::new(line, col))
-    }
-
-    // Constructs a new Token with a type and a Position.
-    pub fn new_with_pos<S: Into<String>>(f: S, typ: TokenType, pos: Position) -> Self {
-        Token {
-            typ: typ,
-            fragment: f.into(),
-            pos: pos,
-        }
-    }
-}
-
-impl Borrow<str> for Token {
-    fn borrow(&self) -> &str {
-        &self.fragment
-    }
-}
-
-/// Helper macro for making a Positioned Value.
-macro_rules! value_node {
-    ($v:expr, $p:expr) => {
-        Positioned::new_with_pos($v, $p)
-    };
-    ($v:expr, $l:expr, $c:expr) => {
-        Positioned::new($v, $l, $c)
-    };
-}
-
-/// Helper macro for making a Token.
-#[allow(unused_macros)]
-macro_rules! make_tok {
-    (EOF => $l:expr, $c:expr) => {
-        Token::new("", TokenType::END, $l, $c)
-    };
-
-    (WS => $l:expr, $c:expr) => {
-        Token::new("", TokenType::WS, $l, $c)
-    };
-
-    (CMT => $e:expr, $l:expr, $c:expr) => {
-        Token::new($e, TokenType::COMMENT, $l, $c)
-    };
-
-    (QUOT => $e:expr, $l:expr, $c:expr) => {
-        Token::new($e, TokenType::QUOTED, $l, $c)
-    };
-
-    (PUNCT => $e:expr, $l:expr, $c:expr) => {
-        Token::new($e, TokenType::PUNCT, $l, $c)
-    };
-
-    (DIGIT => $e:expr, $l:expr, $c:expr) => {
-        Token::new($e, TokenType::DIGIT, $l, $c)
-    };
-
-    ($e:expr, $l:expr, $c:expr) => {
-        Token::new($e, TokenType::BAREWORD, $l, $c)
-    };
-}
-
-/// Helper macro for making expressions.
-#[allow(unused_macros)]
-macro_rules! make_expr {
-    ($e:expr) => {
-        make_expr!($e, 1, 1)
-    };
-
-    ($e:expr, $l:expr, $c:expr) => {
-        Expression::Simple(Value::Symbol(Positioned::new($e.to_string(), $l, $c)))
-    };
-
-    ($e:expr => int, $l:expr, $c:expr) => {
-        Expression::Simple(Value::Int(Positioned::new($e, $l, $c)))
-    };
-}
-
-/// Helper macro for making selectors.
-///
-/// ```
-/// make_selector!(Token::new("tpl", 1, 1), Token::new("fld", 1, 4));
-///
-/// make_selector!(Token::new("tpl", 1, 1), vec![Token::new("fld", 1, 4)], => 1, 1);
-///
-/// make_selector!(foo", ["bar"]);
-///
-/// make_selector!(foo", ["bar"] => 1, 0);
-/// ```
-#[allow(unused_macros)]
-macro_rules! make_selector {
-    ( $h:expr ) => {
-        make_selector!($h, 1, 0)
-    };
-
-    ( $h:expr, $l:expr, $c:expr ) => {
-        SelectorDef::new(
-            SelectorList{head: Box::new($h), tail: None},
-            $l, $c)
-    };
-
-    ( $h: expr, $list:expr, $l:expr, $c:expr) => {
-        SelectorDef::new(
-            SelectorList{head: Box::new($h), tail: Some($list)},
-            $l, $c)
-    };
-
-    // Tokens
-    ( $h:expr => [ $( $item:expr ),* ] ) => {
-        {
-            make_selector!($h => [ $( $item, )* ] => 1, 1)
-        }
-    };
-
-    ( $h:expr => [ $( $item:expr ),* ] => $l:expr, $c:expr ) => {
-        {
-            let mut list: Vec<Token> = Vec::new();
-
-            $(
-                list.push($item);
-            )*
-
-            make_selector!($h, list, $l, $c)
-        }
-    };
-
-    // Strings not tokens
-    ( $h:expr => $( $item:expr ),* ) => {
-        {
-
-            let mut col = 1;
-            let mut list: Vec<Token> = Vec::new();
-
-            $(
-                list.push(make_tok!($item, 1, col));
-                col += $item.len() + 1;
-            )*
-
-            // Shut up the lint about unused code;
-            assert!(col != 0);
-
-            make_selector!($h, list, 1, 1)
-        }
-
-    };
-
-    ( $h:expr => $( $item:expr ),* => $l:expr, $c:expr ) => {
-        {
-            let mut col = $c;
-            let mut list: Vec<Token> = Vec::new();
-
-            $(
-                list.push(make_tok!($item, $l, col));
-                col += $item.len() + 1;
-            )*
-
-            // Shut up the linter about unused code;
-            assert!(col != 0);
-
-            make_selector!($h, list, $l, $c)
-        }
-    };
-}
-
-/// An Expression with a series of symbols specifying the key
-/// with which to descend into the result of the expression.
-///
-/// The expression must evaluate to either a tuple or an array. The token must
-/// evaluate to either a bareword Symbol or an Int.
-///
-/// ```ucg
-/// let foo = { bar = "a thing" };
-/// let thing = foo.bar;
-///
-/// let arr = ["one", "two"];
-/// let first = arr.0;
-///
-/// let berry = {best = "strawberry", unique = "acai"}.best;
-/// let third = ["uno", "dos", "tres"].1;
-/// '''
-#[derive(Debug, PartialEq, Clone)]
-pub struct SelectorList {
-    pub head: Box<Expression>,
-    pub tail: Option<Vec<Token>>,
-}
-
-impl SelectorList {
-    /// Returns a stringified version of a SelectorList.
-    pub fn to_string(&self) -> String {
-        "TODO".to_string()
-    }
-}
-
-/// An ordered list of Name = Value pairs.
-///
-/// This is usually used as the body of a tuple in the UCG AST.
-pub type FieldList = Vec<(Token, Expression)>; // Token is expected to be a symbol
-
-/// Encodes a selector expression in the UCG AST.
-#[derive(Debug, PartialEq, Clone)]
-pub struct SelectorDef {
-    pub pos: Position,
-    pub sel: SelectorList,
-}
-
-impl SelectorDef {
-    /// Constructs a new SelectorDef.
-    pub fn new(sel: SelectorList, line: usize, col: usize) -> Self {
-        SelectorDef {
-            pos: Position::new(line, col),
-            sel: sel,
-        }
-    }
-}
-
-/// Represents a Value in the UCG parsed AST.
-#[derive(Debug, PartialEq, Clone)]
-pub enum Value {
-    // Constant Values
-    Empty(Position),
-    Boolean(Positioned<bool>),
-    Int(Positioned<i64>),
-    Float(Positioned<f64>),
-    String(Positioned<String>),
-    Symbol(Positioned<String>),
-    // Complex Values
-    Tuple(Positioned<FieldList>),
-    List(ListDef),
-    Selector(SelectorDef),
-}
-
-impl Value {
-    /// Returns the type name of the Value it is called on as a string.
-    pub fn type_name(&self) -> String {
-        match self {
-            &Value::Empty(_) => "EmptyValue".to_string(),
-            &Value::Boolean(_) => "Boolean".to_string(),
-            &Value::Int(_) => "Integer".to_string(),
-            &Value::Float(_) => "Float".to_string(),
-            &Value::String(_) => "String".to_string(),
-            &Value::Symbol(_) => "Symbol".to_string(),
-            &Value::Tuple(_) => "Tuple".to_string(),
-            &Value::List(_) => "List".to_string(),
-            &Value::Selector(_) => "Selector".to_string(),
-        }
-    }
-
-    fn fields_to_string(v: &FieldList) -> String {
-        let mut buf = String::new();
-        buf.push_str("{\n");
-        for ref t in v.iter() {
-            buf.push_str("\t");
-            buf.push_str(&t.0.fragment);
-            buf.push_str("\n");
-        }
-        buf.push_str("}");
-        return buf;
-    }
-
-    fn elems_to_string(v: &Vec<Expression>) -> String {
-        return format!("{}", v.len());
-    }
-
-    /// Returns a stringified version of the Value.
-    pub fn to_string(&self) -> String {
-        match self {
-            &Value::Empty(_) => "EmptyValue".to_string(),
-            &Value::Boolean(ref b) => format!("{}", b.val),
-            &Value::Int(ref i) => format!("{}", i.val),
-            &Value::Float(ref f) => format!("{}", f.val),
-            &Value::String(ref s) => format!("{}", s.val),
-            &Value::Symbol(ref s) => format!("{}", s.val),
-            &Value::Tuple(ref fs) => format!("{}", Self::fields_to_string(&fs.val)),
-            &Value::List(ref def) => format!("[{}]", Self::elems_to_string(&def.elems)),
-            &Value::Selector(ref v) => v.sel.to_string(),
-        }
-    }
-
-    /// Returns the position for a Value.
-    pub fn pos(&self) -> &Position {
-        match self {
-            &Value::Empty(ref pos) => pos,
-            &Value::Boolean(ref b) => &b.pos,
-            &Value::Int(ref i) => &i.pos,
-            &Value::Float(ref f) => &f.pos,
-            &Value::String(ref s) => &s.pos,
-            &Value::Symbol(ref s) => &s.pos,
-            &Value::Tuple(ref fs) => &fs.pos,
-            &Value::List(ref def) => &def.pos,
-            &Value::Selector(ref v) => &v.pos,
-        }
-    }
-
-    /// Returns true if called on a Value that is the same type as itself.
-    pub fn type_equal(&self, target: &Self) -> bool {
-        enum_type_equality!(
-            self,
-            target,
-            &Value::Empty(_),
-            &Value::Boolean(_),
-            &Value::Int(_),
-            &Value::Float(_),
-            &Value::String(_),
-            &Value::Symbol(_),
-            &Value::Tuple(_),
-            &Value::List(_),
-            &Value::Selector(_)
-        )
-    }
-}
-
-/// Represents an expansion of a Macro that is expected to already have been
-/// defined.
-#[derive(PartialEq, Debug, Clone)]
-pub struct CallDef {
-    pub macroref: SelectorDef,
-    pub arglist: Vec<Expression>,
-    pub pos: Position,
-}
-
-/// Encodes a select expression in the UCG AST.
-#[derive(PartialEq, Debug, Clone)]
-pub struct SelectDef {
-    pub val: Box<Expression>,
-    pub default: Box<Expression>,
-    pub tuple: FieldList,
-    pub pos: Position,
-}
-
-// TODO(jwall): This should have a way of rendering with position information.
-
-/// Adds position information to any type `T`.
-#[derive(Debug, Clone)]
-pub struct Positioned<T> {
-    pub pos: Position,
-    pub val: T,
-}
-
-impl<T: std::fmt::Display> std::fmt::Display for Positioned<T> {
-    fn fmt(&self, f: &mut std::fmt::Formatter) -> Result<(), std::fmt::Error> {
-        write!(f, "{}", self.val)
-    }
-}
-
-impl<T> Positioned<T> {
-    /// Constructs a new Positioned<T> with a value, line, and column information.
-    pub fn new(v: T, l: usize, c: usize) -> Self {
-        Self::new_with_pos(v, Position::new(l, c))
-    }
-
-    /// Constructs a new Positioned<T> with a value and a Position.
-    pub fn new_with_pos(v: T, pos: Position) -> Self {
-        Positioned { pos: pos, val: v }
-    }
-}
-
-impl<T: PartialEq> PartialEq for Positioned<T> {
-    fn eq(&self, other: &Self) -> bool {
-        self.val == other.val
-    }
-}
-
-impl<T: Eq> Eq for Positioned<T> {}
-
-impl<T: Ord> Ord for Positioned<T> {
-    fn cmp(&self, other: &Self) -> Ordering {
-        self.val.cmp(&other.val)
-    }
-}
-
-impl<T: PartialOrd> PartialOrd for Positioned<T> {
-    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
-        self.val.partial_cmp(&other.val)
-    }
-}
-
-impl<T: Hash> Hash for Positioned<T> {
-    fn hash<H: Hasher>(&self, state: &mut H) {
-        self.val.hash(state);
-    }
-}
-
-impl<'a> From<&'a Token> for Positioned<String> {
-    fn from(t: &'a Token) -> Positioned<String> {
-        Positioned {
-            pos: t.pos.clone(),
-            val: t.fragment.to_string(),
-        }
-    }
-}
-
-impl<'a> From<&'a Positioned<String>> for Positioned<String> {
-    fn from(t: &Positioned<String>) -> Positioned<String> {
-        Positioned {
-            pos: t.pos.clone(),
-            val: t.val.clone(),
-        }
-    }
-}
-
-/// Encodes a macro expression in the UCG AST..
-///
-/// A macro is a pure function over a tuple.
-/// MacroDefs are not closures. They can not reference
-/// any values except what is defined in their arguments.
-#[derive(PartialEq, Debug, Clone)]
-pub struct MacroDef {
-    pub argdefs: Vec<Positioned<String>>,
-    pub fields: FieldList,
-    pub pos: Position,
-}
-
-impl MacroDef {
-    fn symbol_is_in_args(&self, sym: &String) -> bool {
-        for arg in self.argdefs.iter() {
-            if &arg.val == sym {
-                return true;
-            }
-        }
-        return false;
-    }
-
-    fn validate_value_symbols<'a>(
-        &self,
-        stack: &mut Vec<&'a Expression>,
-        val: &'a Value,
-    ) -> HashSet<String> {
-        let mut bad_symbols = HashSet::new();
-        if let &Value::Symbol(ref name) = val {
-            if !self.symbol_is_in_args(&name.val) {
-                bad_symbols.insert(name.val.clone());
-            }
-        } else if let &Value::Selector(ref sel_node) = val {
-            stack.push(&sel_node.sel.head);
-        } else if let &Value::Tuple(ref tuple_node) = val {
-            let fields = &tuple_node.val;
-            for &(_, ref expr) in fields.iter() {
-                stack.push(expr);
-            }
-        } else if let &Value::List(ref def) = val {
-            for elem in def.elems.iter() {
-                stack.push(elem);
-            }
-        }
-        return bad_symbols;
-    }
-
-    /// Performs typechecking of a ucg macro's arguments to ensure
-    /// that they are valid for the expressions in the macro.
-    pub fn validate_symbols(&self) -> Result<(), HashSet<String>> {
-        let mut bad_symbols = HashSet::new();
-        for &(_, ref expr) in self.fields.iter() {
-            let mut stack = Vec::new();
-            stack.push(expr);
-            while stack.len() > 0 {
-                match stack.pop().unwrap() {
-                    &Expression::Binary(ref bexpr) => {
-                        stack.push(&bexpr.left);
-                        stack.push(&bexpr.right);
-                    }
-                    &Expression::Compare(ref cexpr) => {
-                        stack.push(&cexpr.left);
-                        stack.push(&cexpr.right);
-                    }
-                    &Expression::Grouped(ref expr) => {
-                        stack.push(expr);
-                    }
-                    &Expression::Format(ref def) => {
-                        let exprs = &def.args;
-                        for arg_expr in exprs.iter() {
-                            stack.push(arg_expr);
-                        }
-                    }
-                    &Expression::Select(ref def) => {
-                        stack.push(def.default.borrow());
-                        stack.push(def.val.borrow());
-                        for &(_, ref expr) in def.tuple.iter() {
-                            stack.push(expr);
-                        }
-                    }
-                    &Expression::Copy(ref def) => {
-                        let fields = &def.fields;
-                        for &(_, ref expr) in fields.iter() {
-                            stack.push(expr);
-                        }
-                    }
-                    &Expression::Call(ref def) => for expr in def.arglist.iter() {
-                        stack.push(expr);
-                    },
-                    &Expression::Simple(ref val) => {
-                        let mut syms_set = self.validate_value_symbols(&mut stack, val);
-                        bad_symbols.extend(syms_set.drain());
-                    }
-                    &Expression::Macro(_) => {
-                        // noop
-                        continue;
-                    }
-                    &Expression::ListOp(_) => {
-                        // noop
-                        continue;
-                    }
-                }
-            }
-        }
-        if bad_symbols.len() > 0 {
-            return Err(bad_symbols);
-        }
-        return Ok(());
-    }
-}
-
-/// Specifies the types of binary operations supported in
-/// UCG expression.
-#[derive(Debug, PartialEq, Clone)]
-pub enum BinaryExprType {
-    Add,
-    Sub,
-    Mul,
-    Div,
-}
-
-#[derive(Debug, PartialEq, Clone)]
-pub enum CompareType {
-    Equal,
-    GT,
-    LT,
-    NotEqual,
-    GTEqual,
-    LTEqual,
-}
-
-#[derive(Debug, PartialEq, Clone)]
-pub struct ComparisonDef {
-    pub kind: CompareType,
-    pub left: Box<Expression>,
-    pub right: Box<Expression>,
-    pub pos: Position,
-}
-
-/// Represents an expression with a left and a right side.
-#[derive(Debug, PartialEq, Clone)]
-pub struct BinaryOpDef {
-    pub kind: BinaryExprType,
-    pub left: Box<Expression>,
-    pub right: Box<Expression>,
-    pub pos: Position,
-}
-
-/// Encodes a tuple Copy expression in the UCG AST.
-#[derive(Debug, PartialEq, Clone)]
-pub struct CopyDef {
-    pub selector: SelectorDef,
-    pub fields: FieldList,
-    pub pos: Position,
-}
-
-/// Encodes a format expression in the UCG AST.
-#[derive(Debug, PartialEq, Clone)]
-pub struct FormatDef {
-    pub template: String,
-    pub args: Vec<Expression>,
-    pub pos: Position,
-}
-
-/// Encodes a list expression in the UCG AST.
-#[derive(Debug, PartialEq, Clone)]
-pub struct ListDef {
-    pub elems: Vec<Expression>,
-    pub pos: Position,
-}
-
-#[derive(Debug, PartialEq, Clone)]
-pub enum ListOpType {
-    Map,
-    Filter,
-}
-
-#[derive(Debug, PartialEq, Clone)]
-pub struct ListOpDef {
-    pub typ: ListOpType,
-    pub mac: SelectorDef,
-    pub field: String,
-    pub target: ListDef,
-    pub pos: Position,
-}
-
-/// Encodes a ucg expression. Expressions compute a value from.
-#[derive(Debug, PartialEq, Clone)]
-pub enum Expression {
-    // Base Expression
-    Simple(Value),
-
-    // Binary expressions
-    Binary(BinaryOpDef),
-    Compare(ComparisonDef),
-
-    // Complex Expressions
-    Copy(CopyDef),
-    Grouped(Box<Expression>),
-    Format(FormatDef),
-    Call(CallDef),
-    Macro(MacroDef),
-    Select(SelectDef),
-    ListOp(ListOpDef),
-}
-
-impl Expression {
-    /// Returns the position of the Expression.
-    pub fn pos(&self) -> &Position {
-        match self {
-            &Expression::Simple(ref v) => v.pos(),
-            &Expression::Binary(ref def) => &def.pos,
-            &Expression::Compare(ref def) => &def.pos,
-            &Expression::Copy(ref def) => &def.pos,
-            &Expression::Grouped(ref expr) => expr.pos(),
-            &Expression::Format(ref def) => &def.pos,
-            &Expression::Call(ref def) => &def.pos,
-            &Expression::Macro(ref def) => &def.pos,
-            &Expression::Select(ref def) => &def.pos,
-            &Expression::ListOp(ref def) => &def.pos,
-        }
-    }
-}
-
-/// Encodes a let statement in the UCG AST.
-#[derive(Debug, PartialEq)]
-pub struct LetDef {
-    pub name: Token,
-    pub value: Expression,
-}
-
-/// Encodes an import statement in the UCG AST.
-#[derive(Debug, PartialEq)]
-pub struct ImportDef {
-    pub path: Token,
-    pub name: Token,
-}
-
-/// Encodes a parsed statement in the UCG AST.
-#[derive(Debug, PartialEq)]
-pub enum Statement {
-    // simple expression
-    Expression(Expression),
-
-    // Named bindings
-    Let(LetDef),
-
-    // Import a file.
-    Import(ImportDef),
-}
-
-#[cfg(test)]
-mod ast_test {
-    use super::*;
-
-    #[test]
-    pub fn test_macro_validation_happy_path() {
-        let def = MacroDef {
-            argdefs: vec![value_node!("foo".to_string(), 1, 0)],
-            fields: vec![
-                (
-                    make_tok!("f1", 1, 1),
-                    Expression::Binary(BinaryOpDef {
-                        kind: BinaryExprType::Add,
-                        left: Box::new(Expression::Simple(Value::Symbol(value_node!(
-                            "foo".to_string(),
-                            1,
-                            1
-                        )))),
-                        right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
-                        pos: Position::new(1, 0),
-                    }),
-                ),
-            ],
-            pos: Position::new(1, 0),
-        };
-        assert!(def.validate_symbols().unwrap() == ());
-    }
-
-    #[test]
-    pub fn test_macro_validation_fail() {
-        let def = MacroDef {
-            argdefs: vec![value_node!("foo".to_string(), 1, 0)],
-            fields: vec![
-                (
-                    make_tok!("f1", 1, 1),
-                    Expression::Binary(BinaryOpDef {
-                        kind: BinaryExprType::Add,
-                        left: Box::new(Expression::Simple(Value::Symbol(value_node!(
-                            "bar".to_string(),
-                            1,
-                            1
-                        )))),
-                        right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
-                        pos: Position::new(1, 0),
-                    }),
-                ),
-            ],
-            pos: Position::new(1, 0),
-        };
-        let mut expected = HashSet::new();
-        expected.insert("bar".to_string());
-        assert_eq!(def.validate_symbols().err().unwrap(), expected);
-    }
-
-    #[test]
-    pub fn test_macro_validation_selector_happy_path() {
-        let def = MacroDef {
-            argdefs: vec![value_node!("foo".to_string(), 1, 0)],
-            fields: vec![
-                (
-                    make_tok!("f1", 1, 1),
-                    Expression::Binary(BinaryOpDef {
-                        kind: BinaryExprType::Add,
-                        left: Box::new(Expression::Simple(Value::Selector(
-                            make_selector!(make_expr!("foo", 1, 1) => [
-                        make_tok!("quux", 1, 1) ] => 1, 1),
-                        ))),
-                        right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
-                        pos: Position::new(1, 0),
-                    }),
-                ),
-            ],
-            pos: Position::new(1, 0),
-        };
-        assert!(def.validate_symbols().unwrap() == ());
-    }
-
-    #[test]
-    pub fn test_macro_validation_selector_fail() {
-        let def = MacroDef {
-            argdefs: vec![value_node!("foo".to_string(), 1, 0)],
-            fields: vec![
-                (
-                    make_tok!("f1", 1, 1),
-                    Expression::Binary(BinaryOpDef {
-                        kind: BinaryExprType::Add,
-                        left: Box::new(Expression::Simple(Value::Selector(
-                            make_selector!(make_expr!("bar", 1, 1) => [
-                        make_tok!("quux", 1, 1) ] => 1, 1),
-                        ))),
-                        right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
-                        pos: Position::new(1, 0),
-                    }),
-                ),
-            ],
-            pos: Position::new(1, 0),
-        };
-        let mut expected = HashSet::new();
-        expected.insert("bar".to_string());
-        assert_eq!(def.validate_symbols(), Err(expected));
-    }
-}

src/build/compile_test.rs

@@ -0,0 +1,86 @@
+use super::{Builder, Val};
+
+fn assert_build<S: Into<String>>(input: S, assert: &str) {
+    let mut b = Builder::new();
+    b.build_file_string(input.into()).unwrap();
+    let result = b.eval_string(assert).unwrap();
+    if let &Val::Boolean(ok) = result.as_ref() {
+        assert!(ok, format!("'{}' is not true", assert));
+    } else {
+        assert!(
+            false,
+            format!("'{}' does not evaluate to a boolean: {:?}", assert, result)
+        );
+    }
+}
+
+#[test]
+fn test_comparisons() {
+    let input = "
+    let one = 1;
+    let two = 2;
+    let foo = \"foo\";
+    let bar = \"bar\";
+    let tpl1 = {
+        foo = \"bar\",
+        one = 1
+    };
+    let tpl2 = tpl1{};
+    let tpl3 = {
+        bar = \"foo\",
+        two = 1
+    };
+    let list = [1, 2, 3];
+    let list2 = list;
+    let list3 = [1, 2];
+    ";
+    assert_build(input, "one == one;");
+    assert_build(input, "one >= one;");
+    assert_build(input, "two > one;");
+    assert_build(input, "two >= two;");
+    assert_build(input, "tpl1 == tpl2;");
+    assert_build(input, "tpl1 != tpl3;");
+    assert_build(input, "list == list2;");
+    assert_build(input, "list != list3;");
+}
+
+#[test]
+fn test_deep_comparison() {
+    let input = "
+    let tpl1 = {
+        foo = \"bar\",
+        lst = [1, 2, 3],
+        inner = {
+            fld = \"value\"
+        }
+    };
+    let copy = tpl1;
+    let extra = tpl1{one = 1};
+    let less = {
+        foo = \"bar\"
+    };
+    ";
+
+    assert_build(input, "tpl1.inner == copy.inner;");
+    assert_build(input, "tpl1.inner.fld == copy.inner.fld;");
+    assert_build(input, "tpl1.lst == copy.lst;");
+    assert_build(input, "tpl1.foo == copy.foo;");
+    assert_build(input, "tpl1 == copy;");
+    assert_build(input, "tpl1 != extra;");
+    assert_build(input, "tpl1 != less;");
+}
+
+#[test]
+fn test_expression_comparisons() {
+    assert_build("", "2 == 1+1;");
+    assert_build("", "(1+1) == 2;");
+    assert_build("", "(1+1) == (1+1);");
+    assert_build("", "(\"foo\" + \"bar\") == \"foobar\";");
+}
+
+#[test]
+fn test_binary_operator_precedence() {
+    //assert_build("let result = 2 * 2 + 1;", "result == 6;");
+    assert_build("let result = 2 + 2 * 1;", "result == 4;");
+    assert_build("let result = (2 * 2) + 1;", "result == 5;");
+}

src/build/mod.rs

@@ -25,7 +25,7 @@ use std::io::Read;
 use std::ops::Deref;
 use std::rc::Rc;
 
-use ast::tree::*;
+use ast::*;
 use error;
 use format;
 use parse::parse;
@@ -1110,899 +1110,7 @@ impl Builder {
 }
 
 #[cfg(test)]
-mod compile_test {
+mod compile_test;
-    use super::{Builder, Val};
-
-    fn assert_build<S: Into<String>>(input: S, assert: &str) {
-        let mut b = Builder::new();
-        b.build_file_string(input.into()).unwrap();
-        let result = b.eval_string(assert).unwrap();
-        if let &Val::Boolean(ok) = result.as_ref() {
-            assert!(ok, format!("'{}' is not true", assert));
-        } else {
-            assert!(
-                false,
-                format!("'{}' does not evaluate to a boolean: {:?}", assert, result)
-            );
-        }
-    }
-
-    #[test]
-    fn test_comparisons() {
-        let input = "
-        let one = 1;
-        let two = 2;
-        let foo = \"foo\";
-        let bar = \"bar\";
-        let tpl1 = {
-            foo = \"bar\",
-            one = 1
-        };
-        let tpl2 = tpl1{};
-        let tpl3 = {
-            bar = \"foo\",
-            two = 1
-        };
-        let list = [1, 2, 3];
-        let list2 = list;
-        let list3 = [1, 2];
-        ";
-        assert_build(input, "one == one;");
-        assert_build(input, "one >= one;");
-        assert_build(input, "two > one;");
-        assert_build(input, "two >= two;");
-        assert_build(input, "tpl1 == tpl2;");
-        assert_build(input, "tpl1 != tpl3;");
-        assert_build(input, "list == list2;");
-        assert_build(input, "list != list3;");
-    }
-
-    #[test]
-    fn test_deep_comparison() {
-        let input = "
-        let tpl1 = {
-            foo = \"bar\",
-            lst = [1, 2, 3],
-            inner = {
-                fld = \"value\"
-            }
-        };
-        let copy = tpl1;
-        let extra = tpl1{one = 1};
-        let less = {
-            foo = \"bar\"
-        };
-        ";
-
-        assert_build(input, "tpl1.inner == copy.inner;");
-        assert_build(input, "tpl1.inner.fld == copy.inner.fld;");
-        assert_build(input, "tpl1.lst == copy.lst;");
-        assert_build(input, "tpl1.foo == copy.foo;");
-        assert_build(input, "tpl1 == copy;");
-        assert_build(input, "tpl1 != extra;");
-        assert_build(input, "tpl1 != less;");
-    }
-
-    #[test]
-    fn test_expression_comparisons() {
-        assert_build("", "2 == 1+1;");
-        assert_build("", "(1+1) == 2;");
-        assert_build("", "(1+1) == (1+1);");
-        assert_build("", "(\"foo\" + \"bar\") == \"foobar\";");
-    }
-
-    #[test]
-    fn test_binary_operator_precedence() {
-        //assert_build("let result = 2 * 2 + 1;", "result == 6;");
-        assert_build("let result = 2 + 2 * 1;", "result == 4;");
-        assert_build("let result = (2 * 2) + 1;", "result == 5;");
-    }
-}
 
 #[cfg(test)]
-mod test {
+mod test;
-    use super::{Builder, CallDef, MacroDef, SelectDef, Val};
-    use ast::tree::*;
-    use std::rc::Rc;
-
-    fn test_expr_to_val(mut cases: Vec<(Expression, Val)>, b: Builder) {
-        for tpl in cases.drain(0..) {
-            assert_eq!(b.eval_expr(&tpl.0).unwrap(), Rc::new(tpl.1));
-        }
-    }
-
-    #[test]
-    fn test_eval_div_expr() {
-        let b = Builder::new();
-        test_expr_to_val(
-            vec![
-                (
-                    Expression::Binary(BinaryOpDef {
-                        kind: BinaryExprType::Div,
-                        left: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))),
-                        right: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))),
-                        pos: Position::new(1, 0),
-                    }),
-                    Val::Int(1),
-                ),
-                (
-                    Expression::Binary(BinaryOpDef {
-                        kind: BinaryExprType::Div,
-                        left: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))),
-                        right: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))),
-                        pos: Position::new(1, 0),
-                    }),
-                    Val::Float(1.0),
-                ),
-            ],
-            b,
-        );
-    }
-
-    #[test]
-    #[should_panic(expected = "Expected Float")]
-    fn test_eval_div_expr_fail() {
-        let b = Builder::new();
-        test_expr_to_val(
-            vec![
-                (
-                    Expression::Binary(BinaryOpDef {
-                        kind: BinaryExprType::Div,
-                        left: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))),
-                        right: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))),
-                        pos: Position::new(1, 0),
-                    }),
-                    Val::Float(1.0),
-                ),
-            ],
-            b,
-        );
-    }
-
-    #[test]
-    fn test_eval_mul_expr() {
-        let b = Builder::new();
-        test_expr_to_val(
-            vec![
-                (
-                    Expression::Binary(BinaryOpDef {
-                        kind: BinaryExprType::Mul,
-                        left: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))),
-                        right: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))),
-                        pos: Position::new(1, 0),
-                    }),
-                    Val::Int(4),
-                ),
-                (
-                    Expression::Binary(BinaryOpDef {
-                        kind: BinaryExprType::Mul,
-                        left: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))),
-                        right: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))),
-                        pos: Position::new(1, 0),
-                    }),
-                    Val::Float(4.0),
-                ),
-            ],
-            b,
-        );
-    }
-
-    #[test]
-    #[should_panic(expected = "Expected Float")]
-    fn test_eval_mul_expr_fail() {
-        let b = Builder::new();
-        test_expr_to_val(
-            vec![
-                (
-                    Expression::Binary(BinaryOpDef {
-                        kind: BinaryExprType::Mul,
-                        left: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))),
-                        right: Box::new(Expression::Simple(Value::Int(value_node!(20, 1, 1)))),
-                        pos: Position::new(1, 0),
-                    }),
-                    Val::Float(1.0),
-                ),
-            ],
-            b,
-        );
-    }
-
-    #[test]
-    fn test_eval_subtract_expr() {
-        let b = Builder::new();
-        test_expr_to_val(
-            vec![
-                (
-                    Expression::Binary(BinaryOpDef {
-                        kind: BinaryExprType::Sub,
-                        left: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))),
-                        right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
-                        pos: Position::new(1, 0),
-                    }),
-                    Val::Int(1),
-                ),
-                (
-                    Expression::Binary(BinaryOpDef {
-                        kind: BinaryExprType::Sub,
-                        left: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))),
-                        right: Box::new(Expression::Simple(Value::Float(value_node!(1.0, 1, 1)))),
-                        pos: Position::new(1, 0),
-                    }),
-                    Val::Float(1.0),
-                ),
-            ],
-            b,
-        );
-    }
-
-    #[test]
-    #[should_panic(expected = "Expected Float")]
-    fn test_eval_subtract_expr_fail() {
-        let b = Builder::new();
-        test_expr_to_val(
-            vec![
-                (
-                    Expression::Binary(BinaryOpDef {
-                        kind: BinaryExprType::Sub,
-                        left: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))),
-                        right: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))),
-                        pos: Position::new(1, 0),
-                    }),
-                    Val::Float(1.0),
-                ),
-            ],
-            b,
-        );
-    }
-
-    #[test]
-    fn test_eval_add_expr() {
-        let b = Builder::new();
-        test_expr_to_val(
-            vec![
-                (
-                    Expression::Binary(BinaryOpDef {
-                        kind: BinaryExprType::Add,
-                        left: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
-                        right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
-                        pos: Position::new(1, 0),
-                    }),
-                    Val::Int(2),
-                ),
-                (
-                    Expression::Binary(BinaryOpDef {
-                        kind: BinaryExprType::Add,
-                        left: Box::new(Expression::Simple(Value::Float(value_node!(1.0, 1, 1)))),
-                        right: Box::new(Expression::Simple(Value::Float(value_node!(1.0, 1, 1)))),
-                        pos: Position::new(1, 0),
-                    }),
-                    Val::Float(2.0),
-                ),
-                (
-                    Expression::Binary(BinaryOpDef {
-                        kind: BinaryExprType::Add,
-                        left: Box::new(Expression::Simple(Value::String(value_node!(
-                            "foo".to_string(),
-                            1,
-                            1
-                        )))),
-                        right: Box::new(Expression::Simple(Value::String(value_node!(
-                            "bar".to_string(),
-                            1,
-                            1
-                        )))),
-                        pos: Position::new(1, 0),
-                    }),
-                    Val::String("foobar".to_string()),
-                ),
-                (
-                    Expression::Binary(BinaryOpDef {
-                        kind: BinaryExprType::Add,
-                        left: Box::new(Expression::Simple(Value::List(ListDef {
-                            elems: vec![
-                                Expression::Simple(Value::String(value_node!(
-                                    "foo".to_string(),
-                                    1,
-                                    1
-                                ))),
-                            ],
-                            pos: Position::new(1, 1),
-                        }))),
-                        right: Box::new(Expression::Simple(Value::List(ListDef {
-                            elems: vec![
-                                Expression::Simple(Value::String(value_node!(
-                                    "bar".to_string(),
-                                    1,
-                                    1
-                                ))),
-                            ],
-                            pos: Position::new(1, 1),
-                        }))),
-                        pos: Position::new(1, 0),
-                    }),
-                    Val::List(vec![
-                        Rc::new(Val::String("foo".to_string())),
-                        Rc::new(Val::String("bar".to_string())),
-                    ]),
-                ),
-            ],
-            b,
-        );
-    }
-
-    #[test]
-    #[should_panic(expected = "Expected Float")]
-    fn test_eval_add_expr_fail() {
-        let b = Builder::new();
-        test_expr_to_val(
-            vec![
-                (
-                    Expression::Binary(BinaryOpDef {
-                        kind: BinaryExprType::Add,
-                        left: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))),
-                        right: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))),
-                        pos: Position::new(1, 0),
-                    }),
-                    Val::Float(1.0),
-                ),
-            ],
-            b,
-        );
-    }
-
-    #[test]
-    fn test_eval_simple_expr() {
-        test_expr_to_val(
-            vec![
-                (
-                    Expression::Simple(Value::Int(value_node!(1, 1, 1))),
-                    Val::Int(1),
-                ),
-                (
-                    Expression::Simple(Value::Float(value_node!(2.0, 1, 1))),
-                    Val::Float(2.0),
-                ),
-                (
-                    Expression::Simple(Value::String(value_node!("foo".to_string(), 1, 1))),
-                    Val::String("foo".to_string()),
-                ),
-                (
-                    Expression::Simple(Value::Tuple(value_node!(
-                        vec![
-                            (
-                                make_tok!("bar", 1, 1),
-                                Expression::Simple(Value::Int(value_node!(1, 1, 1))),
-                            ),
-                        ],
-                        1,
-                        1
-                    ))),
-                    Val::Tuple(vec![
-                        (value_node!("bar".to_string(), 1, 1), Rc::new(Val::Int(1))),
-                    ]),
-                ),
-            ],
-            Builder::new(),
-        );
-    }
-
-    #[test]
-    fn test_eval_simple_lookup_expr() {
-        let mut b = Builder::new();
-        b.out
-            .entry(value_node!("var1".to_string(), 1, 0))
-            .or_insert(Rc::new(Val::Int(1)));
-        test_expr_to_val(
-            vec![
-                (
-                    Expression::Simple(Value::Symbol(value_node!("var1".to_string(), 1, 1))),
-                    Val::Int(1),
-                ),
-            ],
-            b,
-        );
-    }
-
-    #[test]
-    fn test_eval_simple_lookup_error() {
-        let mut b = Builder::new();
-        b.out
-            .entry(value_node!("var1".to_string(), 1, 0))
-            .or_insert(Rc::new(Val::Int(1)));
-        let expr = Expression::Simple(Value::Symbol(value_node!("var".to_string(), 1, 1)));
-        assert!(b.eval_expr(&expr).is_err());
-    }
-
-    #[test]
-    fn test_eval_selector_expr() {
-        let mut b = Builder::new();
-        b.out
-            .entry(value_node!("var1".to_string(), 1, 0))
-            .or_insert(Rc::new(Val::Tuple(vec![
-                (
-                    value_node!("lvl1".to_string(), 1, 0),
-                    Rc::new(Val::Tuple(vec![
-                        (value_node!("lvl2".to_string(), 1, 0), Rc::new(Val::Int(3))),
-                    ])),
-                ),
-            ])));
-        b.out
-            .entry(value_node!("var2".to_string(), 1, 0))
-            .or_insert(Rc::new(Val::Int(2)));
-        b.out
-            .entry(value_node!("var3".to_string(), 1, 0))
-            .or_insert(Rc::new(Val::Tuple(vec![
-                (value_node!("lvl1".to_string(), 1, 0), Rc::new(Val::Int(4))),
-            ])));
-
-        test_expr_to_val(
-            vec![
-                (
-                    Expression::Simple(Value::Selector(make_selector!(make_expr!("var1")))),
-                    Val::Tuple(vec![
-                        (
-                            value_node!("lvl1".to_string(), 1, 0),
-                            Rc::new(Val::Tuple(vec![
-                                (value_node!("lvl2".to_string(), 1, 0), Rc::new(Val::Int(3))),
-                            ])),
-                        ),
-                    ]),
-                ),
-                (
-                    Expression::Simple(Value::Selector(
-                        make_selector!(make_expr!("var1") => "lvl1"),
-                    )),
-                    Val::Tuple(vec![
-                        (value_node!("lvl2".to_string(), 1, 0), Rc::new(Val::Int(3))),
-                    ]),
-                ),
-                (
-                    Expression::Simple(Value::Selector(
-                        make_selector!(make_expr!("var1") => "lvl1", "lvl2"),
-                    )),
-                    Val::Int(3),
-                ),
-                (
-                    Expression::Simple(Value::Selector(make_selector!(make_expr!("var2")))),
-                    Val::Int(2),
-                ),
-                (
-                    Expression::Simple(Value::Selector(
-                        make_selector!(make_expr!("var3") => "lvl1"),
-                    )),
-                    Val::Int(4),
-                ),
-            ],
-            b,
-        );
-    }
-
-    #[test]
-    fn test_eval_selector_list_expr() {
-        let mut b = Builder::new();
-        b.out
-            .entry(value_node!("var1".to_string(), 1, 1))
-            .or_insert(Rc::new(Val::List(vec![
-                Rc::new(Val::String("val1".to_string())),
-                Rc::new(Val::Tuple(vec![
-                    (value_node!("var2".to_string(), 1, 1), Rc::new(Val::Int(1))),
-                ])),
-            ])));
-        // TODO(jwall): Assert that we can index into lists using dot syntax.
-
-        test_expr_to_val(
-            vec![
-                (
-                    Expression::Simple(Value::Selector(
-                        make_selector!(make_expr!("var1") =>  "0" => 1, 1),
-                    )),
-                    Val::String("val1".to_string()),
-                ),
-            ],
-            b,
-        );
-    }
-
-    #[test]
-    #[should_panic(expected = "Unable to find tpl1")]
-    fn test_expr_copy_no_such_tuple() {
-        let b = Builder::new();
-        test_expr_to_val(
-            vec![
-                (
-                    Expression::Copy(CopyDef {
-                        selector: make_selector!(make_expr!("tpl1")),
-                        fields: Vec::new(),
-                        pos: Position::new(1, 0),
-                    }),
-                    Val::Tuple(Vec::new()),
-                ),
-            ],
-            b,
-        );
-    }
-
-    #[test]
-    #[should_panic(expected = "Expected Tuple got Int(1)")]
-    fn test_expr_copy_not_a_tuple() {
-        let mut b = Builder::new();
-        b.out
-            .entry(value_node!("tpl1".to_string(), 1, 0))
-            .or_insert(Rc::new(Val::Int(1)));
-        test_expr_to_val(
-            vec![
-                (
-                    Expression::Copy(CopyDef {
-                        selector: make_selector!(make_expr!("tpl1")),
-                        fields: Vec::new(),
-                        pos: Position::new(1, 0),
-                    }),
-                    Val::Tuple(Vec::new()),
-                ),
-            ],
-            b,
-        );
-    }
-
-    #[test]
-    #[should_panic(expected = "Expected type Integer for field fld1 but got String")]
-    fn test_expr_copy_field_type_error() {
-        let mut b = Builder::new();
-        b.out
-            .entry(value_node!("tpl1".to_string(), 1, 0))
-            .or_insert(Rc::new(Val::Tuple(vec![
-                (value_node!("fld1".to_string(), 1, 0), Rc::new(Val::Int(1))),
-            ])));
-        test_expr_to_val(
-            vec![
-                (
-                    Expression::Copy(CopyDef {
-                        selector: make_selector!(make_expr!("tpl1")),
-                        fields: vec![
-                            (
-                                make_tok!("fld1", 1, 1),
-                                Expression::Simple(Value::String(value_node!(
-                                    "2".to_string(),
-                                    1,
-                                    1
-                                ))),
-                            ),
-                        ],
-                        pos: Position::new(1, 0),
-                    }),
-                    Val::Tuple(vec![
-                        (
-                            value_node!("fld1".to_string(), 1, 1),
-                            Rc::new(Val::String("2".to_string())),
-                        ),
-                    ]),
-                ),
-            ],
-            b,
-        );
-    }
-
-    #[test]
-    fn test_expr_copy() {
-        let mut b = Builder::new();
-        b.out
-            .entry(value_node!("tpl1".to_string(), 1, 0))
-            .or_insert(Rc::new(Val::Tuple(vec![
-                (value_node!("fld1".to_string(), 1, 0), Rc::new(Val::Int(1))),
-            ])));
-        test_expr_to_val(
-            vec![
-                (
-                    Expression::Copy(CopyDef {
-                        selector: make_selector!(make_expr!("tpl1")),
-                        fields: vec![
-                            (
-                                make_tok!("fld2", 1, 1),
-                                Expression::Simple(Value::String(value_node!(
-                                    "2".to_string(),
-                                    1,
-                                    1
-                                ))),
-                            ),
-                        ],
-                        pos: Position::new(1, 0),
-                    }),
-                    // Add a new field to the copy
-                    Val::Tuple(
-                        // NOTE(jwall): The order of these is important in order to ensure
-                        // that the compare assertion is correct. The ordering has no
-                        // semantics though so at some point we should probably be less restrictive.
-                        vec![
-                            (value_node!("fld1".to_string(), 1, 0), Rc::new(Val::Int(1))),
-                            (
-                                value_node!("fld2".to_string(), 1, 1),
-                                Rc::new(Val::String("2".to_string())),
-                            ),
-                        ],
-                    ),
-                ),
-                // Overwrite a field in the copy
-                (
-                    Expression::Copy(CopyDef {
-                        selector: make_selector!(make_expr!("tpl1")),
-                        fields: vec![
-                            (
-                                make_tok!("fld1", 1, 1),
-                                Expression::Simple(Value::Int(value_node!(3, 1, 1))),
-                            ),
-                            (
-                                make_tok!("fld2", 1, 1),
-                                Expression::Simple(Value::String(value_node!(
-                                    "2".to_string(),
-                                    1,
-                                    1
-                                ))),
-                            ),
-                        ],
-                        pos: Position::new(1, 0),
-                    }),
-                    Val::Tuple(vec![
-                        (value_node!("fld1".to_string(), 1, 0), Rc::new(Val::Int(3))),
-                        (
-                            value_node!("fld2".to_string(), 1, 0),
-                            Rc::new(Val::String("2".to_string())),
-                        ),
-                    ]),
-                ),
-                // The source tuple is still unmodified.
-                (
-                    Expression::Simple(Value::Selector(make_selector!(make_expr!["tpl1"]))),
-                    Val::Tuple(vec![
-                        (value_node!("fld1".to_string(), 1, 0), Rc::new(Val::Int(1))),
-                    ]),
-                ),
-            ],
-            b,
-        );
-    }
-
-    #[test]
-    fn test_macro_call() {
-        let mut b = Builder::new();
-        b.out
-            .entry(value_node!("tstmac".to_string(), 1, 0))
-            .or_insert(Rc::new(Val::Macro(MacroDef {
-                argdefs: vec![value_node!("arg1".to_string(), 1, 0)],
-                fields: vec![
-                    (
-                        make_tok!("foo", 1, 1),
-                        Expression::Simple(Value::Symbol(value_node!("arg1".to_string(), 1, 1))),
-                    ),
-                ],
-                pos: Position::new(1, 0),
-            })));
-        test_expr_to_val(
-            vec![
-                (
-                    Expression::Call(CallDef {
-                        macroref: make_selector!(make_expr!("tstmac")),
-                        arglist: vec![
-                            Expression::Simple(Value::String(value_node!("bar".to_string(), 1, 1))),
-                        ],
-                        pos: Position::new(1, 0),
-                    }),
-                    Val::Tuple(vec![
-                        (
-                            value_node!("foo".to_string(), 1, 1),
-                            Rc::new(Val::String("bar".to_string())),
-                        ),
-                    ]),
-                ),
-            ],
-            b,
-        );
-    }
-
-    #[test]
-    #[should_panic(expected = "Unable to find arg1")]
-    fn test_macro_hermetic() {
-        let mut b = Builder::new();
-        b.out
-            .entry(value_node!("arg1".to_string(), 1, 0))
-            .or_insert(Rc::new(Val::String("bar".to_string())));
-        b.out
-            .entry(value_node!("tstmac".to_string(), 1, 0))
-            .or_insert(Rc::new(Val::Macro(MacroDef {
-                argdefs: vec![value_node!("arg2".to_string(), 1, 0)],
-                fields: vec![
-                    (
-                        make_tok!("foo", 1, 1),
-                        Expression::Simple(Value::Symbol(value_node!("arg1".to_string(), 1, 1))),
-                    ),
-                ],
-                pos: Position::new(1, 0),
-            })));
-        test_expr_to_val(
-            vec![
-                (
-                    Expression::Call(CallDef {
-                        macroref: make_selector!(make_expr!("tstmac")),
-                        arglist: vec![
-                            Expression::Simple(Value::String(value_node!("bar".to_string(), 1, 1))),
-                        ],
-                        pos: Position::new(1, 1),
-                    }),
-                    Val::Tuple(vec![
-                        (
-                            value_node!("foo".to_string(), 1, 0),
-                            Rc::new(Val::String("bar".to_string())),
-                        ),
-                    ]),
-                ),
-            ],
-            b,
-        );
-    }
-
-    #[test]
-    fn test_select_expr() {
-        let mut b = Builder::new();
-        b.out
-            .entry(value_node!("foo".to_string(), 1, 0))
-            .or_insert(Rc::new(Val::String("bar".to_string())));
-        b.out
-            .entry(value_node!("baz".to_string(), 1, 0))
-            .or_insert(Rc::new(Val::String("boo".to_string())));
-        test_expr_to_val(
-            vec![
-                (
-                    Expression::Select(SelectDef {
-                        val: Box::new(Expression::Simple(Value::Symbol(value_node!(
-                            "foo".to_string(),
-                            1,
-                            1
-                        )))),
-                        default: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
-                        tuple: vec![
-                            (
-                                make_tok!("foo", 1, 1),
-                                Expression::Simple(Value::String(value_node!(
-                                    "2".to_string(),
-                                    1,
-                                    1
-                                ))),
-                            ),
-                            (
-                                make_tok!("bar", 1, 1),
-                                Expression::Simple(Value::Int(value_node!(2, 1, 1))),
-                            ),
-                        ],
-                        pos: Position::new(1, 0),
-                    }),
-                    Val::Int(2),
-                ),
-                (
-                    Expression::Select(SelectDef {
-                        val: Box::new(Expression::Simple(Value::Symbol(value_node!(
-                            "baz".to_string(),
-                            1,
-                            1
-                        )))),
-                        default: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
-                        tuple: vec![
-                            (
-                                make_tok!("bar", 1, 1),
-                                Expression::Simple(Value::Int(value_node!(2, 1, 1))),
-                            ),
-                            (
-                                make_tok!("quux", 1, 1),
-                                Expression::Simple(Value::String(value_node!(
-                                    "2".to_string(),
-                                    1,
-                                    1
-                                ))),
-                            ),
-                        ],
-                        pos: Position::new(1, 0),
-                    }),
-                    // If the field doesn't exist then we get the default.
-                    Val::Int(1),
-                ),
-            ],
-            b,
-        );
-    }
-
-    #[test]
-    #[should_panic(expected = "Expected String but got Integer in Select expression")]
-    fn test_select_expr_not_a_string() {
-        let mut b = Builder::new();
-        b.out
-            .entry(value_node!("foo".to_string(), 1, 0))
-            .or_insert(Rc::new(Val::Int(4)));
-        test_expr_to_val(
-            vec![
-                (
-                    Expression::Select(SelectDef {
-                        val: Box::new(Expression::Simple(Value::Symbol(value_node!(
-                            "foo".to_string(),
-                            1,
-                            1
-                        )))),
-                        default: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
-                        tuple: vec![
-                            (
-                                make_tok!("bar", 1, 1),
-                                Expression::Simple(Value::Int(value_node!(2, 1, 1))),
-                            ),
-                            (
-                                make_tok!("quux", 1, 1),
-                                Expression::Simple(Value::String(value_node!(
-                                    "2".to_string(),
-                                    1,
-                                    1
-                                ))),
-                            ),
-                        ],
-                        pos: Position::new(1, 0),
-                    }),
-                    Val::Int(2),
-                ),
-            ],
-            b,
-        );
-    }
-
-    #[test]
-    fn test_let_statement() {
-        let mut b = Builder::new();
-        let stmt = Statement::Let(LetDef {
-            name: make_tok!("foo", 1, 1),
-            value: Expression::Simple(Value::String(value_node!("bar".to_string(), 1, 1))),
-        });
-        b.build_stmt(&stmt).unwrap();
-        test_expr_to_val(
-            vec![
-                (
-                    Expression::Simple(Value::Symbol(value_node!("foo".to_string(), 1, 1))),
-                    Val::String("bar".to_string()),
-                ),
-            ],
-            b,
-        );
-    }
-
-    #[test]
-    fn test_build_file_string() {
-        let mut b = Builder::new();
-        b.build_file_string("let foo = 1;".to_string()).unwrap();
-        let key = value_node!("foo".to_string(), 1, 0);
-        assert!(b.out.contains_key(&key));
-    }
-
-    #[test]
-    fn test_asset_symbol_lookups() {
-        let mut b = Builder::new();
-        b.assets
-            .entry(value_node!("foo".to_string(), 1, 0))
-            .or_insert(Rc::new(Val::Tuple(vec![
-                (
-                    value_node!("bar".to_string(), 1, 0),
-                    Rc::new(Val::Tuple(vec![
-                        (value_node!("quux".to_string(), 1, 0), Rc::new(Val::Int(1))),
-                    ])),
-                ),
-            ])));
-        test_expr_to_val(
-            vec![
-                (
-                    Expression::Simple(Value::Symbol(value_node!("foo".to_string(), 1, 1))),
-                    Val::Tuple(vec![
-                        (
-                            value_node!("bar".to_string(), 1, 0),
-                            Rc::new(Val::Tuple(vec![
-                                (value_node!("quux".to_string(), 1, 0), Rc::new(Val::Int(1))),
-                            ])),
-                        ),
-                    ]),
-                ),
-            ],
-            b,
-        );
-    }
-}

src/build/test.rs

@@ -0,0 +1,772 @@
+use super::{Builder, CallDef, MacroDef, SelectDef, Val};
+use ast::*;
+use std::rc::Rc;
+
+fn test_expr_to_val(mut cases: Vec<(Expression, Val)>, b: Builder) {
+    for tpl in cases.drain(0..) {
+        assert_eq!(b.eval_expr(&tpl.0).unwrap(), Rc::new(tpl.1));
+    }
+}
+
+#[test]
+fn test_eval_div_expr() {
+    let b = Builder::new();
+    test_expr_to_val(
+        vec![
+            (
+                Expression::Binary(BinaryOpDef {
+                    kind: BinaryExprType::Div,
+                    left: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))),
+                    right: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))),
+                    pos: Position::new(1, 0),
+                }),
+                Val::Int(1),
+            ),
+            (
+                Expression::Binary(BinaryOpDef {
+                    kind: BinaryExprType::Div,
+                    left: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))),
+                    right: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))),
+                    pos: Position::new(1, 0),
+                }),
+                Val::Float(1.0),
+            ),
+        ],
+        b,
+    );
+}
+
+#[test]
+#[should_panic(expected = "Expected Float")]
+fn test_eval_div_expr_fail() {
+    let b = Builder::new();
+    test_expr_to_val(
+        vec![
+            (
+                Expression::Binary(BinaryOpDef {
+                    kind: BinaryExprType::Div,
+                    left: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))),
+                    right: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))),
+                    pos: Position::new(1, 0),
+                }),
+                Val::Float(1.0),
+            ),
+        ],
+        b,
+    );
+}
+
+#[test]
+fn test_eval_mul_expr() {
+    let b = Builder::new();
+    test_expr_to_val(
+        vec![
+            (
+                Expression::Binary(BinaryOpDef {
+                    kind: BinaryExprType::Mul,
+                    left: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))),
+                    right: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))),
+                    pos: Position::new(1, 0),
+                }),
+                Val::Int(4),
+            ),
+            (
+                Expression::Binary(BinaryOpDef {
+                    kind: BinaryExprType::Mul,
+                    left: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))),
+                    right: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))),
+                    pos: Position::new(1, 0),
+                }),
+                Val::Float(4.0),
+            ),
+        ],
+        b,
+    );
+}
+
+#[test]
+#[should_panic(expected = "Expected Float")]
+fn test_eval_mul_expr_fail() {
+    let b = Builder::new();
+    test_expr_to_val(
+        vec![
+            (
+                Expression::Binary(BinaryOpDef {
+                    kind: BinaryExprType::Mul,
+                    left: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))),
+                    right: Box::new(Expression::Simple(Value::Int(value_node!(20, 1, 1)))),
+                    pos: Position::new(1, 0),
+                }),
+                Val::Float(1.0),
+            ),
+        ],
+        b,
+    );
+}
+
+#[test]
+fn test_eval_subtract_expr() {
+    let b = Builder::new();
+    test_expr_to_val(
+        vec![
+            (
+                Expression::Binary(BinaryOpDef {
+                    kind: BinaryExprType::Sub,
+                    left: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))),
+                    right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
+                    pos: Position::new(1, 0),
+                }),
+                Val::Int(1),
+            ),
+            (
+                Expression::Binary(BinaryOpDef {
+                    kind: BinaryExprType::Sub,
+                    left: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))),
+                    right: Box::new(Expression::Simple(Value::Float(value_node!(1.0, 1, 1)))),
+                    pos: Position::new(1, 0),
+                }),
+                Val::Float(1.0),
+            ),
+        ],
+        b,
+    );
+}
+
+#[test]
+#[should_panic(expected = "Expected Float")]
+fn test_eval_subtract_expr_fail() {
+    let b = Builder::new();
+    test_expr_to_val(
+        vec![
+            (
+                Expression::Binary(BinaryOpDef {
+                    kind: BinaryExprType::Sub,
+                    left: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))),
+                    right: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))),
+                    pos: Position::new(1, 0),
+                }),
+                Val::Float(1.0),
+            ),
+        ],
+        b,
+    );
+}
+
+#[test]
+fn test_eval_add_expr() {
+    let b = Builder::new();
+    test_expr_to_val(
+        vec![
+            (
+                Expression::Binary(BinaryOpDef {
+                    kind: BinaryExprType::Add,
+                    left: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
+                    right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
+                    pos: Position::new(1, 0),
+                }),
+                Val::Int(2),
+            ),
+            (
+                Expression::Binary(BinaryOpDef {
+                    kind: BinaryExprType::Add,
+                    left: Box::new(Expression::Simple(Value::Float(value_node!(1.0, 1, 1)))),
+                    right: Box::new(Expression::Simple(Value::Float(value_node!(1.0, 1, 1)))),
+                    pos: Position::new(1, 0),
+                }),
+                Val::Float(2.0),
+            ),
+            (
+                Expression::Binary(BinaryOpDef {
+                    kind: BinaryExprType::Add,
+                    left: Box::new(Expression::Simple(Value::String(value_node!(
+                        "foo".to_string(),
+                        1,
+                        1
+                    )))),
+                    right: Box::new(Expression::Simple(Value::String(value_node!(
+                        "bar".to_string(),
+                        1,
+                        1
+                    )))),
+                    pos: Position::new(1, 0),
+                }),
+                Val::String("foobar".to_string()),
+            ),
+            (
+                Expression::Binary(BinaryOpDef {
+                    kind: BinaryExprType::Add,
+                    left: Box::new(Expression::Simple(Value::List(ListDef {
+                        elems: vec![
+                            Expression::Simple(Value::String(value_node!("foo".to_string(), 1, 1))),
+                        ],
+                        pos: Position::new(1, 1),
+                    }))),
+                    right: Box::new(Expression::Simple(Value::List(ListDef {
+                        elems: vec![
+                            Expression::Simple(Value::String(value_node!("bar".to_string(), 1, 1))),
+                        ],
+                        pos: Position::new(1, 1),
+                    }))),
+                    pos: Position::new(1, 0),
+                }),
+                Val::List(vec![
+                    Rc::new(Val::String("foo".to_string())),
+                    Rc::new(Val::String("bar".to_string())),
+                ]),
+            ),
+        ],
+        b,
+    );
+}
+
+#[test]
+#[should_panic(expected = "Expected Float")]
+fn test_eval_add_expr_fail() {
+    let b = Builder::new();
+    test_expr_to_val(
+        vec![
+            (
+                Expression::Binary(BinaryOpDef {
+                    kind: BinaryExprType::Add,
+                    left: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))),
+                    right: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))),
+                    pos: Position::new(1, 0),
+                }),
+                Val::Float(1.0),
+            ),
+        ],
+        b,
+    );
+}
+
+#[test]
+fn test_eval_simple_expr() {
+    test_expr_to_val(
+        vec![
+            (
+                Expression::Simple(Value::Int(value_node!(1, 1, 1))),
+                Val::Int(1),
+            ),
+            (
+                Expression::Simple(Value::Float(value_node!(2.0, 1, 1))),
+                Val::Float(2.0),
+            ),
+            (
+                Expression::Simple(Value::String(value_node!("foo".to_string(), 1, 1))),
+                Val::String("foo".to_string()),
+            ),
+            (
+                Expression::Simple(Value::Tuple(value_node!(
+                    vec![
+                        (
+                            make_tok!("bar", 1, 1),
+                            Expression::Simple(Value::Int(value_node!(1, 1, 1))),
+                        ),
+                    ],
+                    1,
+                    1
+                ))),
+                Val::Tuple(vec![
+                    (value_node!("bar".to_string(), 1, 1), Rc::new(Val::Int(1))),
+                ]),
+            ),
+        ],
+        Builder::new(),
+    );
+}
+
+#[test]
+fn test_eval_simple_lookup_expr() {
+    let mut b = Builder::new();
+    b.out
+        .entry(value_node!("var1".to_string(), 1, 0))
+        .or_insert(Rc::new(Val::Int(1)));
+    test_expr_to_val(
+        vec![
+            (
+                Expression::Simple(Value::Symbol(value_node!("var1".to_string(), 1, 1))),
+                Val::Int(1),
+            ),
+        ],
+        b,
+    );
+}
+
+#[test]
+fn test_eval_simple_lookup_error() {
+    let mut b = Builder::new();
+    b.out
+        .entry(value_node!("var1".to_string(), 1, 0))
+        .or_insert(Rc::new(Val::Int(1)));
+    let expr = Expression::Simple(Value::Symbol(value_node!("var".to_string(), 1, 1)));
+    assert!(b.eval_expr(&expr).is_err());
+}
+
+#[test]
+fn test_eval_selector_expr() {
+    let mut b = Builder::new();
+    b.out
+        .entry(value_node!("var1".to_string(), 1, 0))
+        .or_insert(Rc::new(Val::Tuple(vec![
+            (
+                value_node!("lvl1".to_string(), 1, 0),
+                Rc::new(Val::Tuple(vec![
+                    (value_node!("lvl2".to_string(), 1, 0), Rc::new(Val::Int(3))),
+                ])),
+            ),
+        ])));
+    b.out
+        .entry(value_node!("var2".to_string(), 1, 0))
+        .or_insert(Rc::new(Val::Int(2)));
+    b.out
+        .entry(value_node!("var3".to_string(), 1, 0))
+        .or_insert(Rc::new(Val::Tuple(vec![
+            (value_node!("lvl1".to_string(), 1, 0), Rc::new(Val::Int(4))),
+        ])));
+
+    test_expr_to_val(
+        vec![
+            (
+                Expression::Simple(Value::Selector(make_selector!(make_expr!("var1")))),
+                Val::Tuple(vec![
+                    (
+                        value_node!("lvl1".to_string(), 1, 0),
+                        Rc::new(Val::Tuple(vec![
+                            (value_node!("lvl2".to_string(), 1, 0), Rc::new(Val::Int(3))),
+                        ])),
+                    ),
+                ]),
+            ),
+            (
+                Expression::Simple(Value::Selector(
+                    make_selector!(make_expr!("var1") => "lvl1"),
+                )),
+                Val::Tuple(vec![
+                    (value_node!("lvl2".to_string(), 1, 0), Rc::new(Val::Int(3))),
+                ]),
+            ),
+            (
+                Expression::Simple(Value::Selector(
+                    make_selector!(make_expr!("var1") => "lvl1", "lvl2"),
+                )),
+                Val::Int(3),
+            ),
+            (
+                Expression::Simple(Value::Selector(make_selector!(make_expr!("var2")))),
+                Val::Int(2),
+            ),
+            (
+                Expression::Simple(Value::Selector(
+                    make_selector!(make_expr!("var3") => "lvl1"),
+                )),
+                Val::Int(4),
+            ),
+        ],
+        b,
+    );
+}
+
+#[test]
+fn test_eval_selector_list_expr() {
+    let mut b = Builder::new();
+    b.out
+        .entry(value_node!("var1".to_string(), 1, 1))
+        .or_insert(Rc::new(Val::List(vec![
+            Rc::new(Val::String("val1".to_string())),
+            Rc::new(Val::Tuple(vec![
+                (value_node!("var2".to_string(), 1, 1), Rc::new(Val::Int(1))),
+            ])),
+        ])));
+    // TODO(jwall): Assert that we can index into lists using dot syntax.
+
+    test_expr_to_val(
+        vec![
+            (
+                Expression::Simple(Value::Selector(
+                    make_selector!(make_expr!("var1") =>  "0" => 1, 1),
+                )),
+                Val::String("val1".to_string()),
+            ),
+        ],
+        b,
+    );
+}
+
+#[test]
+#[should_panic(expected = "Unable to find tpl1")]
+fn test_expr_copy_no_such_tuple() {
+    let b = Builder::new();
+    test_expr_to_val(
+        vec![
+            (
+                Expression::Copy(CopyDef {
+                    selector: make_selector!(make_expr!("tpl1")),
+                    fields: Vec::new(),
+                    pos: Position::new(1, 0),
+                }),
+                Val::Tuple(Vec::new()),
+            ),
+        ],
+        b,
+    );
+}
+
+#[test]
+#[should_panic(expected = "Expected Tuple got Int(1)")]
+fn test_expr_copy_not_a_tuple() {
+    let mut b = Builder::new();
+    b.out
+        .entry(value_node!("tpl1".to_string(), 1, 0))
+        .or_insert(Rc::new(Val::Int(1)));
+    test_expr_to_val(
+        vec![
+            (
+                Expression::Copy(CopyDef {
+                    selector: make_selector!(make_expr!("tpl1")),
+                    fields: Vec::new(),
+                    pos: Position::new(1, 0),
+                }),
+                Val::Tuple(Vec::new()),
+            ),
+        ],
+        b,
+    );
+}
+
+#[test]
+#[should_panic(expected = "Expected type Integer for field fld1 but got String")]
+fn test_expr_copy_field_type_error() {
+    let mut b = Builder::new();
+    b.out
+        .entry(value_node!("tpl1".to_string(), 1, 0))
+        .or_insert(Rc::new(Val::Tuple(vec![
+            (value_node!("fld1".to_string(), 1, 0), Rc::new(Val::Int(1))),
+        ])));
+    test_expr_to_val(
+        vec![
+            (
+                Expression::Copy(CopyDef {
+                    selector: make_selector!(make_expr!("tpl1")),
+                    fields: vec![
+                        (
+                            make_tok!("fld1", 1, 1),
+                            Expression::Simple(Value::String(value_node!("2".to_string(), 1, 1))),
+                        ),
+                    ],
+                    pos: Position::new(1, 0),
+                }),
+                Val::Tuple(vec![
+                    (
+                        value_node!("fld1".to_string(), 1, 1),
+                        Rc::new(Val::String("2".to_string())),
+                    ),
+                ]),
+            ),
+        ],
+        b,
+    );
+}
+
+#[test]
+fn test_expr_copy() {
+    let mut b = Builder::new();
+    b.out
+        .entry(value_node!("tpl1".to_string(), 1, 0))
+        .or_insert(Rc::new(Val::Tuple(vec![
+            (value_node!("fld1".to_string(), 1, 0), Rc::new(Val::Int(1))),
+        ])));
+    test_expr_to_val(
+        vec![
+            (
+                Expression::Copy(CopyDef {
+                    selector: make_selector!(make_expr!("tpl1")),
+                    fields: vec![
+                        (
+                            make_tok!("fld2", 1, 1),
+                            Expression::Simple(Value::String(value_node!("2".to_string(), 1, 1))),
+                        ),
+                    ],
+                    pos: Position::new(1, 0),
+                }),
+                // Add a new field to the copy
+                Val::Tuple(
+                    // NOTE(jwall): The order of these is important in order to ensure
+                    // that the compare assertion is correct. The ordering has no
+                    // semantics though so at some point we should probably be less restrictive.
+                    vec![
+                        (value_node!("fld1".to_string(), 1, 0), Rc::new(Val::Int(1))),
+                        (
+                            value_node!("fld2".to_string(), 1, 1),
+                            Rc::new(Val::String("2".to_string())),
+                        ),
+                    ],
+                ),
+            ),
+            // Overwrite a field in the copy
+            (
+                Expression::Copy(CopyDef {
+                    selector: make_selector!(make_expr!("tpl1")),
+                    fields: vec![
+                        (
+                            make_tok!("fld1", 1, 1),
+                            Expression::Simple(Value::Int(value_node!(3, 1, 1))),
+                        ),
+                        (
+                            make_tok!("fld2", 1, 1),
+                            Expression::Simple(Value::String(value_node!("2".to_string(), 1, 1))),
+                        ),
+                    ],
+                    pos: Position::new(1, 0),
+                }),
+                Val::Tuple(vec![
+                    (value_node!("fld1".to_string(), 1, 0), Rc::new(Val::Int(3))),
+                    (
+                        value_node!("fld2".to_string(), 1, 0),
+                        Rc::new(Val::String("2".to_string())),
+                    ),
+                ]),
+            ),
+            // The source tuple is still unmodified.
+            (
+                Expression::Simple(Value::Selector(make_selector!(make_expr!["tpl1"]))),
+                Val::Tuple(vec![
+                    (value_node!("fld1".to_string(), 1, 0), Rc::new(Val::Int(1))),
+                ]),
+            ),
+        ],
+        b,
+    );
+}
+
+#[test]
+fn test_macro_call() {
+    let mut b = Builder::new();
+    b.out
+        .entry(value_node!("tstmac".to_string(), 1, 0))
+        .or_insert(Rc::new(Val::Macro(MacroDef {
+            argdefs: vec![value_node!("arg1".to_string(), 1, 0)],
+            fields: vec![
+                (
+                    make_tok!("foo", 1, 1),
+                    Expression::Simple(Value::Symbol(value_node!("arg1".to_string(), 1, 1))),
+                ),
+            ],
+            pos: Position::new(1, 0),
+        })));
+    test_expr_to_val(
+        vec![
+            (
+                Expression::Call(CallDef {
+                    macroref: make_selector!(make_expr!("tstmac")),
+                    arglist: vec![
+                        Expression::Simple(Value::String(value_node!("bar".to_string(), 1, 1))),
+                    ],
+                    pos: Position::new(1, 0),
+                }),
+                Val::Tuple(vec![
+                    (
+                        value_node!("foo".to_string(), 1, 1),
+                        Rc::new(Val::String("bar".to_string())),
+                    ),
+                ]),
+            ),
+        ],
+        b,
+    );
+}
+
+#[test]
+#[should_panic(expected = "Unable to find arg1")]
+fn test_macro_hermetic() {
+    let mut b = Builder::new();
+    b.out
+        .entry(value_node!("arg1".to_string(), 1, 0))
+        .or_insert(Rc::new(Val::String("bar".to_string())));
+    b.out
+        .entry(value_node!("tstmac".to_string(), 1, 0))
+        .or_insert(Rc::new(Val::Macro(MacroDef {
+            argdefs: vec![value_node!("arg2".to_string(), 1, 0)],
+            fields: vec![
+                (
+                    make_tok!("foo", 1, 1),
+                    Expression::Simple(Value::Symbol(value_node!("arg1".to_string(), 1, 1))),
+                ),
+            ],
+            pos: Position::new(1, 0),
+        })));
+    test_expr_to_val(
+        vec![
+            (
+                Expression::Call(CallDef {
+                    macroref: make_selector!(make_expr!("tstmac")),
+                    arglist: vec![
+                        Expression::Simple(Value::String(value_node!("bar".to_string(), 1, 1))),
+                    ],
+                    pos: Position::new(1, 1),
+                }),
+                Val::Tuple(vec![
+                    (
+                        value_node!("foo".to_string(), 1, 0),
+                        Rc::new(Val::String("bar".to_string())),
+                    ),
+                ]),
+            ),
+        ],
+        b,
+    );
+}
+
+#[test]
+fn test_select_expr() {
+    let mut b = Builder::new();
+    b.out
+        .entry(value_node!("foo".to_string(), 1, 0))
+        .or_insert(Rc::new(Val::String("bar".to_string())));
+    b.out
+        .entry(value_node!("baz".to_string(), 1, 0))
+        .or_insert(Rc::new(Val::String("boo".to_string())));
+    test_expr_to_val(
+        vec![
+            (
+                Expression::Select(SelectDef {
+                    val: Box::new(Expression::Simple(Value::Symbol(value_node!(
+                        "foo".to_string(),
+                        1,
+                        1
+                    )))),
+                    default: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
+                    tuple: vec![
+                        (
+                            make_tok!("foo", 1, 1),
+                            Expression::Simple(Value::String(value_node!("2".to_string(), 1, 1))),
+                        ),
+                        (
+                            make_tok!("bar", 1, 1),
+                            Expression::Simple(Value::Int(value_node!(2, 1, 1))),
+                        ),
+                    ],
+                    pos: Position::new(1, 0),
+                }),
+                Val::Int(2),
+            ),
+            (
+                Expression::Select(SelectDef {
+                    val: Box::new(Expression::Simple(Value::Symbol(value_node!(
+                        "baz".to_string(),
+                        1,
+                        1
+                    )))),
+                    default: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
+                    tuple: vec![
+                        (
+                            make_tok!("bar", 1, 1),
+                            Expression::Simple(Value::Int(value_node!(2, 1, 1))),
+                        ),
+                        (
+                            make_tok!("quux", 1, 1),
+                            Expression::Simple(Value::String(value_node!("2".to_string(), 1, 1))),
+                        ),
+                    ],
+                    pos: Position::new(1, 0),
+                }),
+                // If the field doesn't exist then we get the default.
+                Val::Int(1),
+            ),
+        ],
+        b,
+    );
+}
+
+#[test]
+#[should_panic(expected = "Expected String but got Integer in Select expression")]
+fn test_select_expr_not_a_string() {
+    let mut b = Builder::new();
+    b.out
+        .entry(value_node!("foo".to_string(), 1, 0))
+        .or_insert(Rc::new(Val::Int(4)));
+    test_expr_to_val(
+        vec![
+            (
+                Expression::Select(SelectDef {
+                    val: Box::new(Expression::Simple(Value::Symbol(value_node!(
+                        "foo".to_string(),
+                        1,
+                        1
+                    )))),
+                    default: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
+                    tuple: vec![
+                        (
+                            make_tok!("bar", 1, 1),
+                            Expression::Simple(Value::Int(value_node!(2, 1, 1))),
+                        ),
+                        (
+                            make_tok!("quux", 1, 1),
+                            Expression::Simple(Value::String(value_node!("2".to_string(), 1, 1))),
+                        ),
+                    ],
+                    pos: Position::new(1, 0),
+                }),
+                Val::Int(2),
+            ),
+        ],
+        b,
+    );
+}
+
+#[test]
+fn test_let_statement() {
+    let mut b = Builder::new();
+    let stmt = Statement::Let(LetDef {
+        name: make_tok!("foo", 1, 1),
+        value: Expression::Simple(Value::String(value_node!("bar".to_string(), 1, 1))),
+    });
+    b.build_stmt(&stmt).unwrap();
+    test_expr_to_val(
+        vec![
+            (
+                Expression::Simple(Value::Symbol(value_node!("foo".to_string(), 1, 1))),
+                Val::String("bar".to_string()),
+            ),
+        ],
+        b,
+    );
+}
+
+#[test]
+fn test_build_file_string() {
+    let mut b = Builder::new();
+    b.build_file_string("let foo = 1;".to_string()).unwrap();
+    let key = value_node!("foo".to_string(), 1, 0);
+    assert!(b.out.contains_key(&key));
+}
+
+#[test]
+fn test_asset_symbol_lookups() {
+    let mut b = Builder::new();
+    b.assets
+        .entry(value_node!("foo".to_string(), 1, 0))
+        .or_insert(Rc::new(Val::Tuple(vec![
+            (
+                value_node!("bar".to_string(), 1, 0),
+                Rc::new(Val::Tuple(vec![
+                    (value_node!("quux".to_string(), 1, 0), Rc::new(Val::Int(1))),
+                ])),
+            ),
+        ])));
+    test_expr_to_val(
+        vec![
+            (
+                Expression::Simple(Value::Symbol(value_node!("foo".to_string(), 1, 1))),
+                Val::Tuple(vec![
+                    (
+                        value_node!("bar".to_string(), 1, 0),
+                        Rc::new(Val::Tuple(vec![
+                            (value_node!("quux".to_string(), 1, 0), Rc::new(Val::Int(1))),
+                        ])),
+                    ),
+                ]),
+            ),
+        ],
+        b,
+    );
+}

src/convert/env.rs

@@ -17,7 +17,7 @@ use std::io::Result;
 use std::io::Write;
 use std::rc::Rc;
 
-use ast::tree::*;
+use ast::*;
 use build::Val;
 use convert::traits::Converter;
 

src/convert/json.rs

@@ -36,7 +36,7 @@ impl JsonConverter {
 
     fn convert_tuple(
         &self,
-        items: &Vec<(ast::tree::Positioned<String>, Rc<Val>)>,
+        items: &Vec<(ast::Positioned<String>, Rc<Val>)>,
     ) -> Result<serde_json::Value> {
         let mut mp = serde_json::Map::new();
         for &(ref k, ref v) in items.iter() {

src/error.rs

@@ -16,7 +16,7 @@
 use std::error;
 use std::fmt;
 
-use ast::tree::*;
+use ast::*;
 
 use nom;
 

src/format.rs

@@ -16,7 +16,7 @@
 use std::clone::Clone;
 use std::error::Error;
 
-use ast::tree::*;
+use ast::*;
 use error;
 
 /// Implements the logic for format strings in UCG format expressions.
@@ -77,7 +77,7 @@ impl<V: Into<String> + Clone> Formatter<V> {
 #[cfg(test)]
 mod test {
     use super::Formatter;
-    use ast::tree::Position;
+    use ast::Position;
 
     #[test]
     fn test_format_happy_path() {

src/lib.rs

@@ -409,9 +409,9 @@ pub mod parse;
 
 mod format;
 
-pub use ast::tree::Expression;
+pub use ast::Expression;
-pub use ast::tree::Statement;
+pub use ast::Statement;
-pub use ast::tree::Value;
+pub use ast::Value;
 
 pub use build::Builder;
 pub use build::Val;

src/parse/mod.rs

@@ -0,0 +1,864 @@
+// Copyright 2017 Jeremy Wall <jeremy@marzhillstudios.com>
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+//! The Parsing stage of the ucg compiler.
+use std::borrow::Borrow;
+use std::str::FromStr;
+
+use nom;
+use nom::IResult;
+use nom::InputLength;
+use nom_locate::LocatedSpan;
+
+use ast::*;
+use error;
+use tokenizer::*;
+
+type NomResult<'a, O> = nom::IResult<TokenIter<'a>, O, error::Error>;
+
+type ParseResult<O> = Result<O, error::Error>;
+
+fn symbol_to_value(s: &Token) -> ParseResult<Value> {
+    Ok(Value::Symbol(value_node!(
+        s.fragment.to_string(),
+        s.pos.clone()
+    )))
+}
+
+// symbol is a bare unquoted field.
+named!(symbol<TokenIter, Value, error::Error>,
+    match_type!(BAREWORD => symbol_to_value)
+);
+
+fn str_to_value(s: &Token) -> ParseResult<Value> {
+    Ok(Value::String(value_node!(
+        s.fragment.to_string(),
+        s.pos.clone()
+    )))
+}
+
+// quoted_value is a quoted string.
+named!(quoted_value<TokenIter, Value, error::Error>,
+       match_type!(STR => str_to_value)
+);
+
+// Helper function to make the return types work for down below.
+fn triple_to_number(v: (Option<Token>, Option<Token>, Option<Token>)) -> ParseResult<Value> {
+    let (pref, mut pref_pos) = match v.0 {
+        None => ("", Position::new(0, 0)),
+        Some(ref bs) => (bs.fragment.borrow(), bs.pos.clone()),
+    };
+
+    let has_dot = v.1.is_some();
+
+    if v.0.is_some() && !has_dot && v.2.is_none() {
+        let i = match FromStr::from_str(pref) {
+            Ok(i) => i,
+            Err(_) => {
+                return Err(error::Error::new(
+                    format!("Not an integer! {}", pref),
+                    error::ErrorType::UnexpectedToken,
+                    pref_pos,
+                ))
+            }
+        };
+        return Ok(Value::Int(value_node!(i, pref_pos)));
+    }
+
+    if v.0.is_none() && has_dot {
+        pref_pos = v.1.unwrap().pos;
+    }
+
+    let (maybepos, suf) = match v.2 {
+        None => (None, "".to_string()),
+        Some(bs) => (Some(bs.pos), bs.fragment),
+    };
+
+    let to_parse = pref.to_string() + "." + &suf;
+    // TODO(jwall): if there is an error we should report where that error occured.
+    let f = match FromStr::from_str(&to_parse) {
+        Ok(f) => f,
+        Err(_) => {
+            return Err(error::Error::new(
+                format!("Not a float! {}", to_parse),
+                error::ErrorType::UnexpectedToken,
+                maybepos.unwrap(),
+            ))
+        }
+    };
+    return Ok(Value::Float(value_node!(f, pref_pos)));
+}
+
+// trace_macros!(true);
+
+// NOTE(jwall): HERE THERE BE DRAGONS. The order for these matters
+// alot. We need to process alternatives in order of decreasing
+// specificity.  Unfortunately this means we are required to go in a
+// decreasing size order which messes with alt!'s completion logic. To
+// work around this we have to force Incomplete to be Error so that
+// alt! will try the next in the series instead of aborting.
+//
+// *IMPORTANT*
+// It also means this combinator is risky when used with partial
+// inputs. So handle with care.
+named!(number<TokenIter, Value, error::Error>,
+       map_res!(alt!(
+           complete!(do_parse!( // 1.0
+               prefix: match_type!(DIGIT) >>
+               has_dot: punct!(".") >>
+               suffix: match_type!(DIGIT) >>
+               (Some(prefix.clone()), Some(has_dot.clone()), Some(suffix.clone()))
+           )) |
+           complete!(do_parse!( // 1.
+               prefix: match_type!(DIGIT) >>
+               has_dot: punct!(".") >>
+               (Some(prefix.clone()), Some(has_dot.clone()), None)
+           )) |
+           complete!(do_parse!( // .1
+               has_dot: punct!(".") >>
+               suffix: match_type!(DIGIT) >>
+               (None, Some(has_dot.clone()), Some(suffix.clone()))
+           )) |
+           do_parse!( // 1
+               prefix: match_type!(DIGIT) >>
+// The peek!(not!(..)) make this whole combinator slightly
+// safer for partial inputs.
+               (Some(prefix.clone()), None, None)
+           )),
+           triple_to_number
+       )
+);
+// trace_macros!(false);
+
+named!(boolean_value<TokenIter, Value, error::Error>,
+    do_parse!(
+        b: match_type!(BOOLEAN) >>
+        (Value::Boolean(Positioned{
+            val: b.fragment == "true",
+            pos: b.pos,
+        }))
+    )
+);
+
+named!(
+    field_value<TokenIter, (Token, Expression), error::Error>,
+    do_parse!(
+            field: match_type!(BAREWORD) >>
+            punct!("=") >>
+            value: expression >>
+            (field, value)
+    )
+);
+
+// Helper function to make the return types work for down below.
+fn vec_to_tuple(t: (Position, Option<FieldList>)) -> ParseResult<Value> {
+    Ok(Value::Tuple(value_node!(
+        t.1.unwrap_or(Vec::new()),
+        t.0.line as usize,
+        t.0.column as usize
+    )))
+}
+
+named!(field_list<TokenIter, FieldList, error::Error>,
+       separated_list!(punct!(","), field_value)
+);
+
+named!(
+    #[doc="Capture a tuple of named fields with values. {<field>=<value>,...}"],
+    tuple<TokenIter, Value, error::Error>,
+    map_res!(
+        do_parse!(
+            pos: pos >>
+            punct!("{") >>
+            v: field_list >>
+            punct!("}") >>
+            (pos, Some(v))
+        ),
+        vec_to_tuple
+    )
+);
+
+fn tuple_to_list<Sp: Into<Position>>(t: (Sp, Vec<Expression>)) -> ParseResult<Value> {
+    return Ok(Value::List(ListDef {
+        elems: t.1,
+        pos: t.0.into(),
+    }));
+}
+
+named!(list_value<TokenIter, Value, error::Error>,
+       map_res!(
+           do_parse!(
+               start: punct!("[") >>
+               elements: separated_list!(punct!(","), expression) >>
+               punct!("]") >>
+               (start.pos, elements)
+           ),
+           tuple_to_list
+       )
+);
+
+named!(empty_value<TokenIter, Value, error::Error>,
+    do_parse!(
+        pos: pos >>
+        match_type!(EMPTY) >>
+        (Value::Empty(pos))
+    )
+);
+
+named!(value<TokenIter, Value, error::Error>,
+    alt!(
+        boolean_value |
+        empty_value |
+        number |
+        quoted_value |
+        list_value |
+        tuple |
+        selector_value )
+ );
+
+fn value_to_expression(v: Value) -> ParseResult<Expression> {
+    Ok(Expression::Simple(v))
+}
+
+named!(simple_expression<TokenIter, Expression, error::Error>,
+       map_res!(
+           value,
+           value_to_expression
+       )
+);
+
+fn tuple_to_binary_expression(
+    tpl: (Position, BinaryExprType, Expression, Expression),
+) -> ParseResult<Expression> {
+    Ok(Expression::Binary(BinaryOpDef {
+        kind: tpl.1,
+        left: Box::new(tpl.2),
+        right: Box::new(tpl.3),
+        pos: Position::new(tpl.0.line as usize, tpl.0.column as usize),
+    }))
+}
+
+/// do_binary_expr implements precedence based parsing where the more tightly bound parsers
+/// are passed in as lowerrule parsers. We default to grouped_expression and simple_expression as
+/// the most tightly bound expressions.
+macro_rules! do_binary_expr {
+    ($i:expr, $oprule:ident!( $($args:tt)* ), $typ:expr) => {
+        do_binary_expr!($i, $oprule!($($args)*), $typ, alt!(grouped_expression | simple_expression))
+    };
+
+    ($i:expr, $oprule:ident!( $($args:tt)* ), $typ:expr, $lowerrule:ident) => {
+        do_binary_expr!($i, $oprule!($($args)*), $typ, call!($lowerrule))
+    };
+
+    ($i:expr, $oprule:ident!( $($args:tt)* ), $typ:expr, $lowerrule:ident!( $($lowerargs:tt)* )) => {
+        map_res!($i,
+            do_parse!(
+                pos: pos >>
+                left: $lowerrule!($($lowerargs)*) >>
+                    $oprule!($($args)*) >>
+                    right: $lowerrule!($($lowerargs)*) >>
+                    (pos, $typ, left, right)
+            ),
+            tuple_to_binary_expression
+        )
+    };
+}
+
+named!(math_expression<TokenIter, Expression, error::Error>,
+    alt!(sum_expression | product_expression)
+);
+
+named!(sum_expression<TokenIter, Expression, error::Error>,
+    alt!(add_expression | sub_expression)
+);
+
+// trace_macros!(true);
+named!(add_expression<TokenIter, Expression, error::Error>,
+       do_binary_expr!(punct!("+"), BinaryExprType::Add, alt!(product_expression | simple_expression | grouped_expression))
+);
+// trace_macros!(false);
+
+named!(sub_expression<TokenIter, Expression, error::Error>,
+       do_binary_expr!(punct!("-"), BinaryExprType::Sub,  alt!(product_expression | simple_expression | grouped_expression))
+);
+
+named!(product_expression<TokenIter, Expression, error::Error>,
+    alt!(mul_expression | div_expression)
+);
+
+named!(mul_expression<TokenIter, Expression, error::Error>,
+       do_binary_expr!(punct!("*"), BinaryExprType::Mul)
+);
+
+named!(div_expression<TokenIter, Expression, error::Error>,
+       do_binary_expr!(punct!("/"), BinaryExprType::Div)
+);
+
+// TODO(jwall): Change comparison operators to use the do_binary_expr! with precedence?
+fn tuple_to_compare_expression(
+    tpl: (Position, CompareType, Expression, Expression),
+) -> ParseResult<Expression> {
+    Ok(Expression::Compare(ComparisonDef {
+        kind: tpl.1,
+        left: Box::new(tpl.2),
+        right: Box::new(tpl.3),
+        pos: Position::new(tpl.0.line as usize, tpl.0.column as usize),
+    }))
+}
+
+macro_rules! do_compare_expr {
+    ($i:expr, $subrule:ident!( $($args:tt)* ), $typ:expr) => {
+        map_res!($i,
+            do_parse!(
+                pos: pos >>
+                left: alt!(simple_expression | grouped_expression | math_expression) >>
+                    $subrule!($($args)*) >>
+                    right: expression >>
+                    (pos, $typ, left, right)
+            ),
+            tuple_to_compare_expression
+        )
+    };
+}
+
+named!(eqeq_expression<TokenIter, Expression, error::Error>,
+       do_compare_expr!(punct!("=="), CompareType::Equal)
+);
+
+named!(not_eqeq_expression<TokenIter, Expression, error::Error>,
+       do_compare_expr!(punct!("!="), CompareType::NotEqual)
+);
+
+named!(lt_eqeq_expression<TokenIter, Expression, error::Error>,
+       do_compare_expr!(punct!("<="), CompareType::LTEqual)
+);
+
+named!(gt_eqeq_expression<TokenIter, Expression, error::Error>,
+       do_compare_expr!(punct!(">="), CompareType::GTEqual)
+);
+
+named!(gt_expression<TokenIter, Expression, error::Error>,
+       do_compare_expr!(punct!(">"), CompareType::GT)
+);
+
+named!(lt_expression<TokenIter, Expression, error::Error>,
+       do_compare_expr!(punct!("<"), CompareType::LT)
+);
+
+fn expression_to_grouped_expression(e: Expression) -> ParseResult<Expression> {
+    Ok(Expression::Grouped(Box::new(e)))
+}
+
+named!(grouped_expression<TokenIter, Expression, error::Error>,
+       map_res!(
+           preceded!(punct!("("), terminated!(expression, punct!(")"))),
+           expression_to_grouped_expression
+       )
+);
+
+fn symbol_or_expression(input: TokenIter) -> NomResult<Expression> {
+    let sym = do_parse!(input, sym: symbol >> (sym));
+
+    match sym {
+        IResult::Incomplete(i) => {
+            return IResult::Incomplete(i);
+        }
+        IResult::Error(_) => {
+            // TODO(jwall): Still missing some. But we need to avoid recursion
+            return grouped_expression(input);
+        }
+        IResult::Done(rest, val) => {
+            return IResult::Done(rest, Expression::Simple(val));
+        }
+    }
+}
+
+fn selector_list(input: TokenIter) -> NomResult<SelectorList> {
+    let (rest, head) = match symbol_or_expression(input) {
+        IResult::Done(rest, val) => (rest, val),
+        IResult::Error(e) => {
+            return IResult::Error(e);
+        }
+        IResult::Incomplete(i) => {
+            return IResult::Incomplete(i);
+        }
+    };
+
+    let (rest, is_dot) = match punct!(rest, ".") {
+        IResult::Done(rest, tok) => (rest, Some(tok)),
+        IResult::Incomplete(i) => {
+            return IResult::Incomplete(i);
+        }
+        IResult::Error(_) => (rest, None),
+    };
+
+    let (rest, list) = if is_dot.is_some() {
+        let (rest, list) = match separated_list!(
+            rest,
+            punct!("."),
+            alt!(match_type!(BAREWORD) | match_type!(DIGIT))
+        ) {
+            IResult::Done(rest, val) => (rest, val),
+            IResult::Incomplete(i) => {
+                return IResult::Incomplete(i);
+            }
+            IResult::Error(e) => {
+                return IResult::Error(e);
+            }
+        };
+
+        if list.is_empty() {
+            return IResult::Error(nom::ErrorKind::Custom(error::Error::new(
+                "(.) with no selector fields after".to_string(),
+                error::ErrorType::IncompleteParsing,
+                is_dot.unwrap().pos,
+            )));
+        } else {
+            (rest, Some(list))
+        }
+    } else {
+        (rest, None)
+    };
+
+    let sel_list = SelectorList {
+        head: Box::new(head),
+        tail: list,
+    };
+
+    return IResult::Done(rest, sel_list);
+}
+
+fn tuple_to_copy(t: (SelectorDef, FieldList)) -> ParseResult<Expression> {
+    let pos = t.0.pos.clone();
+    Ok(Expression::Copy(CopyDef {
+        selector: t.0,
+        fields: t.1,
+        pos: pos,
+    }))
+}
+
+named!(copy_expression<TokenIter, Expression, error::Error>,
+    map_res!(
+        do_parse!(
+            pos: pos >>
+            selector: selector_list >>
+            punct!("{") >>
+            fields: field_list >>
+            punct!("}") >>
+            (SelectorDef::new(selector, pos.line, pos.column as usize), fields)
+        ),
+        tuple_to_copy
+    )
+);
+
+fn tuple_to_macro(mut t: (Position, Vec<Value>, Value)) -> ParseResult<Expression> {
+    match t.2 {
+        Value::Tuple(v) => Ok(Expression::Macro(MacroDef {
+            argdefs: t.1
+                .drain(0..)
+                .map(|s| Positioned {
+                    pos: s.pos().clone(),
+                    val: s.to_string(),
+                })
+                .collect(),
+            fields: v.val,
+            pos: t.0,
+        })),
+        // TODO(jwall): Show a better version of the unexpected parsed value.
+        val => Err(error::Error::new(
+            format!("Expected Tuple Got {:?}", val),
+            error::ErrorType::UnexpectedToken,
+            t.0,
+        )),
+    }
+}
+
+named!(arglist<TokenIter, Vec<Value>, error::Error>, separated_list!(punct!(","), symbol));
+
+named!(macro_expression<TokenIter, Expression, error::Error>,
+       map_res!(
+           do_parse!(
+                pos: pos >>
+                word!("macro") >>
+                punct!("(") >>
+                arglist: arglist >>
+                punct!(")") >>
+                punct!("=>") >>
+                map: tuple >>
+                (pos, arglist, map)
+           ),
+           tuple_to_macro
+       )
+);
+
+fn tuple_to_select(t: (Position, Expression, Expression, Value)) -> ParseResult<Expression> {
+    match t.3 {
+        Value::Tuple(v) => Ok(Expression::Select(SelectDef {
+            val: Box::new(t.1),
+            default: Box::new(t.2),
+            tuple: v.val,
+            pos: t.0,
+        })),
+        val => Err(error::Error::new(
+            format!("Expected Tuple Got {:?}", val),
+            error::ErrorType::UnexpectedToken,
+            t.0,
+        )),
+    }
+}
+
+named!(select_expression<TokenIter, Expression, error::Error>,
+       map_res!(
+           do_parse!(
+               start: word!("select") >>
+               val: terminated!(expression, punct!(",")) >>
+               default: terminated!(expression, punct!(",")) >>
+               map: tuple >>
+               (start.pos.clone(), val, default, map)
+           ),
+           tuple_to_select
+       )
+);
+
+fn tuple_to_format(t: (Token, Vec<Expression>)) -> ParseResult<Expression> {
+    Ok(Expression::Format(FormatDef {
+        template: t.0.fragment.to_string(),
+        args: t.1,
+        pos: t.0.pos,
+    }))
+}
+
+named!(format_expression<TokenIter, Expression, error::Error>,
+       map_res!(
+           do_parse!(
+               tmpl: match_type!(STR) >>
+                   punct!("%") >>
+                   punct!("(") >>
+                   args: separated_list!(punct!(","), expression) >>
+                   punct!(")") >>
+                   (tmpl, args)
+           ),
+           tuple_to_format
+       )
+);
+
+fn tuple_to_call(t: (Position, Value, Vec<Expression>)) -> ParseResult<Expression> {
+    if let Value::Selector(def) = t.1 {
+        Ok(Expression::Call(CallDef {
+            macroref: def,
+            arglist: t.2,
+            pos: Position::new(t.0.line as usize, t.0.column as usize),
+        }))
+    } else {
+        Err(error::Error::new(
+            format!("Expected Selector Got {:?}", t.0),
+            error::ErrorType::UnexpectedToken,
+            Position::new(t.0.line as usize, t.0.column as usize),
+        ))
+    }
+}
+
+fn vec_to_selector_value(t: (Position, SelectorList)) -> ParseResult<Value> {
+    Ok(Value::Selector(SelectorDef::new(
+        t.1,
+        t.0.line as usize,
+        t.0.column as usize,
+    )))
+}
+
+named!(selector_value<TokenIter, Value, error::Error>,
+       map_res!(
+           do_parse!(
+               sl: selector_list >>
+               (sl.head.pos().clone(), sl)
+           ),
+           vec_to_selector_value
+       )
+);
+
+named!(call_expression<TokenIter, Expression, error::Error>,
+       map_res!(
+           do_parse!(
+               macroname: selector_value >>
+               punct!("(") >>
+               args: separated_list!(punct!(","), expression) >>
+               punct!(")") >>
+               (macroname.pos().clone(), macroname, args)
+           ),
+           tuple_to_call
+       )
+);
+
+fn symbol_or_list(input: TokenIter) -> NomResult<Value> {
+    let sym = do_parse!(input, sym: symbol >> (sym));
+
+    match sym {
+        IResult::Incomplete(i) => {
+            return IResult::Incomplete(i);
+        }
+        IResult::Error(_) => {
+            // TODO(jwall): Still missing some. But we need to avoid recursion
+            match list_value(input) {
+                IResult::Incomplete(i) => {
+                    return IResult::Incomplete(i);
+                }
+                IResult::Error(e) => {
+                    return IResult::Error(e);
+                }
+                IResult::Done(i, val) => {
+                    return IResult::Done(i, val);
+                }
+            }
+        }
+        IResult::Done(rest, val) => {
+            return IResult::Done(rest, val);
+        }
+    }
+}
+
+fn tuple_to_list_op(tpl: (Position, Token, Value, Value)) -> ParseResult<Expression> {
+    let pos = tpl.0;
+    let t = if &tpl.1.fragment == "map" {
+        ListOpType::Map
+    } else if &tpl.1.fragment == "filter" {
+        ListOpType::Filter
+    } else {
+        return Err(error::Error::new(
+            format!(
+                "Expected one of 'map' or 'filter' but got '{}'",
+                tpl.1.fragment
+            ),
+            error::ErrorType::UnexpectedToken,
+            pos,
+        ));
+    };
+    let macroname = tpl.2;
+    let list = tpl.3;
+    if let Value::Selector(mut def) = macroname {
+        // First of all we need to assert that this is a selector of at least
+        // two sections.
+        let fieldname: String = match &mut def.sel.tail {
+            &mut None => {
+                return Err(error::Error::new(
+                    format!("Missing a result field for the macro"),
+                    error::ErrorType::IncompleteParsing,
+                    pos,
+                ));
+            }
+            &mut Some(ref mut tl) => {
+                if tl.len() < 1 {
+                    return Err(error::Error::new(
+                        format!("Missing a result field for the macro"),
+                        error::ErrorType::IncompleteParsing,
+                        def.pos.clone(),
+                    ));
+                }
+                let fname = tl.pop();
+                fname.unwrap().fragment
+            }
+        };
+        if let Value::List(ldef) = list {
+            return Ok(Expression::ListOp(ListOpDef {
+                typ: t,
+                mac: def,
+                field: fieldname,
+                target: ldef,
+                pos: pos,
+            }));
+        }
+        // TODO(jwall): We should print a pretter message than debug formatting here.
+        return Err(error::Error::new(
+            format!("Expected a list but got {:?}", list),
+            error::ErrorType::UnexpectedToken,
+            pos,
+        ));
+    }
+    return Err(error::Error::new(
+        format!("Expected a selector but got {:?}", macroname),
+        error::ErrorType::UnexpectedToken,
+        pos,
+    ));
+}
+
+named!(list_op_expression<TokenIter, Expression, error::Error>,
+    map_res!(
+        do_parse!(
+            pos: pos >>
+            optype: alt!(word!("map") | word!("filter")) >>
+            macroname: selector_value >>
+            list: symbol_or_list >>
+            (pos, optype, macroname, list)
+        ),
+        tuple_to_list_op
+    )
+);
+
+// NOTE(jwall): HERE THERE BE DRAGONS. The order for these matters
+// a lot. We need to process alternatives in order of decreasing
+// specificity.  Unfortunately this means we are required to go in a
+// decreasing size order which messes with alt!'s completion logic. To
+// work around this we have to force Incomplete to be Error so that
+// alt! will try the next in the series instead of aborting.
+//
+// *IMPORTANT*
+// It also means this combinator is risky when used with partial
+// inputs. So handle with care.
+named!(expression<TokenIter, Expression, error::Error>,
+    do_parse!(
+        expr: alt!(
+           complete!(list_op_expression) |
+           complete!(math_expression) |
+           complete!(eqeq_expression) |
+           complete!(not_eqeq_expression) |
+           complete!(lt_eqeq_expression) |
+           complete!(gt_eqeq_expression) |
+           complete!(gt_expression) |
+           complete!(lt_expression) |
+           complete!(macro_expression) |
+           complete!(format_expression) |
+           complete!(select_expression) |
+           complete!(call_expression) |
+           complete!(copy_expression) |
+           complete!(grouped_expression) |
+           simple_expression
+       ) >>
+       (expr)
+    )
+);
+
+fn expression_to_statement(v: Expression) -> ParseResult<Statement> {
+    Ok(Statement::Expression(v))
+}
+
+named!(expression_statement<TokenIter, Statement, error::Error>,
+    map_res!(
+        terminated!(expression, punct!(";")),
+        expression_to_statement
+    )
+);
+
+fn tuple_to_let(t: (Token, Expression)) -> ParseResult<Statement> {
+    Ok(Statement::Let(LetDef {
+        name: t.0,
+        value: t.1,
+    }))
+}
+
+named!(let_statement<TokenIter, Statement, error::Error>,
+    map_res!(
+        do_parse!(
+            word!("let") >>
+            name: match_type!(BAREWORD) >>
+            punct!("=") >>
+            val: expression >>
+            punct!(";") >>
+            (name, val)
+        ),
+        tuple_to_let
+    )
+);
+
+fn tuple_to_import(t: (Token, Token)) -> ParseResult<Statement> {
+    Ok(Statement::Import(ImportDef {
+        path: t.0,
+        name: t.1,
+    }))
+}
+
+named!(import_statement<TokenIter, Statement, error::Error>,
+    map_res!(
+       do_parse!(
+           word!("import") >>
+               path: match_type!(STR) >>
+               word!("as") >>
+               name: match_type!(BAREWORD) >>
+               punct!(";") >>
+               (path, name)
+       ),
+       tuple_to_import
+    )
+);
+
+named!(statement<TokenIter, Statement, error::Error>,
+    do_parse!(
+       stmt: alt_complete!(
+           import_statement |
+           let_statement |
+           expression_statement
+       ) >>
+       (stmt)
+    )
+);
+
+/// Parses a LocatedSpan into a list of Statements or an error::Error.
+pub fn parse(input: LocatedSpan<&str>) -> Result<Vec<Statement>, error::Error> {
+    match tokenize(input) {
+        Ok(tokenized) => {
+            let mut out = Vec::new();
+            let mut i_ = TokenIter {
+                source: tokenized.as_slice(),
+            };
+            loop {
+                let i = i_.clone();
+                if i[0].typ == TokenType::END {
+                    break;
+                }
+                match statement(i) {
+                    IResult::Error(nom::ErrorKind::Custom(e)) => {
+                        return Err(e);
+                    }
+                    IResult::Error(e) => {
+                        return Err(error::Error::new_with_errorkind(
+                            format!("Statement Parse error: {:?} current token: {:?}", e, i_[0]),
+                            error::ErrorType::ParseError,
+                            Position {
+                                line: i_[0].pos.line,
+                                column: i_[0].pos.column,
+                            },
+                            e,
+                        ));
+                    }
+                    IResult::Incomplete(ei) => {
+                        return Err(error::Error::new(
+                            format!("Unexpected end of parsing input: {:?}", ei),
+                            error::ErrorType::IncompleteParsing,
+                            Position {
+                                line: i_[0].pos.line,
+                                column: i_[0].pos.column,
+                            },
+                        ));
+                    }
+                    IResult::Done(rest, stmt) => {
+                        out.push(stmt);
+                        i_ = rest;
+                        if i_.input_len() == 0 {
+                            break;
+                        }
+                    }
+                }
+            }
+            return Ok(out);
+        }
+        Err(e) => {
+            return Err(error::Error::new(
+                format!("Tokenization Error {:?}", e.1),
+                error::ErrorType::ParseError,
+                e.0,
+            ));
+        }
+    }
+}
+
+#[cfg(test)]
+mod test;

src/tokenizer/mod.rs

@@ -13,7 +13,7 @@
 //  limitations under the License.
 
 //! The tokenization stage of the ucg compiler.
-use ast::tree::*;
+use ast::*;
 use error;
 use nom;
 use nom::{InputIter, InputLength, Slice};
@@ -659,263 +659,4 @@ impl<'a> InputIter for TokenIter<'a> {
 }
 
 #[cfg(test)]
-mod tokenizer_test {
+mod test;
-    use super::*;
-    use nom;
-    use nom_locate::LocatedSpan;
-
-    #[test]
-    fn test_empty_token() {
-        let result = emptytok(LocatedSpan::new("NULL"));
-        assert!(result.is_done(), format!("result {:?} is not done", result));
-        if let nom::IResult::Done(_, tok) = result {
-            assert_eq!(tok.fragment, "NULL");
-            assert_eq!(tok.typ, TokenType::EMPTY);
-        }
-    }
-
-    #[test]
-    fn test_escape_quoted() {
-        let result = escapequoted(LocatedSpan::new("foo \\\"bar\""));
-        assert!(result.is_done(), format!("result {:?} is not ok", result));
-        if let nom::IResult::Done(rest, frag) = result {
-            assert_eq!(frag, "foo \"bar");
-            assert_eq!(rest.fragment, "\"");
-        }
-    }
-
-    #[test]
-    fn test_string_with_escaping() {
-        let result = strtok(LocatedSpan::new("\"foo \\\\ \\\"bar\""));
-        assert!(result.is_done(), format!("result {:?} is not ok", result));
-        if let nom::IResult::Done(_, tok) = result {
-            assert_eq!(tok.fragment, "foo \\ \"bar".to_string());
-        }
-    }
-
-    #[test]
-    fn test_tokenize_bareword_with_dash() {
-        let result = tokenize(LocatedSpan::new("foo-bar "));
-        assert!(result.is_ok(), format!("result {:?} is not ok", result));
-        if let Ok(toks) = result {
-            assert_eq!(toks.len(), 2);
-            assert_eq!(toks[0].fragment, "foo-bar");
-        }
-    }
-
-    macro_rules! assert_token {
-        ($input:expr, $typ:expr, $msg:expr) => {
-            let result = token(LocatedSpan::new($input));
-            assert!(
-                result.is_done(),
-                format!("result {:?} is not a {}", result, $msg)
-            );
-            if let nom::IResult::Done(_, tok) = result {
-                assert_eq!(tok.fragment, $input);
-                assert_eq!(tok.typ, $typ);
-            }
-        };
-    }
-
-    #[test]
-    fn test_boolean() {
-        assert_token!("true", TokenType::BOOLEAN, "boolean");
-    }
-
-    #[test]
-    fn test_eqeqtok() {
-        assert_token!("==", TokenType::PUNCT, "==");
-    }
-
-    #[test]
-    fn test_notequaltok() {
-        assert_token!("!=", TokenType::PUNCT, "!=");
-    }
-
-    #[test]
-    fn test_gttok() {
-        assert_token!(">", TokenType::PUNCT, ">");
-    }
-
-    #[test]
-    fn test_lttok() {
-        assert_token!("<", TokenType::PUNCT, "<");
-    }
-
-    #[test]
-    fn test_gteqtok() {
-        assert_token!(">=", TokenType::PUNCT, ">=");
-    }
-
-    #[test]
-    fn test_lteqtok() {
-        assert_token!("<=", TokenType::PUNCT, "<=");
-    }
-
-    #[test]
-    fn test_tokenize_one_of_each() {
-        let result = tokenize(LocatedSpan::new(
-            "let import macro select as => [ ] { } ; = % / * \
-             + - . ( ) , 1 . foo \"bar\" // comment\n ; true false == < > <= >= !=",
-        ));
-        assert!(result.is_ok(), format!("result {:?} is not ok", result));
-        let v = result.unwrap();
-        for (i, t) in v.iter().enumerate() {
-            println!("{}: {:?}", i, t);
-        }
-        assert_eq!(v.len(), 35);
-        assert_eq!(v[34].typ, TokenType::END);
-    }
-
-    #[test]
-    fn test_parse_has_end() {
-        let result = tokenize(LocatedSpan::new("foo"));
-        assert!(result.is_ok());
-        let v = result.unwrap();
-        assert_eq!(v.len(), 2);
-        assert_eq!(v[1].typ, TokenType::END);
-    }
-
-    #[test]
-    fn test_parse_comment() {
-        assert!(comment(LocatedSpan::new("// comment\n")).is_done());
-        assert!(comment(LocatedSpan::new("// comment")).is_done());
-        assert_eq!(
-            comment(LocatedSpan::new("// comment\n")),
-            nom::IResult::Done(
-                LocatedSpan {
-                    fragment: "",
-                    offset: 11,
-                    line: 2,
-                },
-                Token {
-                    typ: TokenType::COMMENT,
-                    fragment: " comment".to_string(),
-                    pos: Position { line: 1, column: 1 },
-                }
-            )
-        );
-        assert!(comment(LocatedSpan::new("// comment\r\n")).is_done());
-        assert_eq!(
-            comment(LocatedSpan::new("// comment\r\n")),
-            nom::IResult::Done(
-                LocatedSpan {
-                    fragment: "",
-                    offset: 12,
-                    line: 2,
-                },
-                Token {
-                    typ: TokenType::COMMENT,
-                    fragment: " comment".to_string(),
-                    pos: Position { column: 1, line: 1 },
-                }
-            )
-        );
-        assert!(comment(LocatedSpan::new("// comment\r\n ")).is_done());
-        assert_eq!(
-            comment(LocatedSpan::new("// comment\r\n ")),
-            nom::IResult::Done(
-                LocatedSpan {
-                    fragment: " ",
-                    offset: 12,
-                    line: 2,
-                },
-                Token {
-                    typ: TokenType::COMMENT,
-                    fragment: " comment".to_string(),
-                    pos: Position { column: 1, line: 1 },
-                }
-            )
-        );
-        assert!(comment(LocatedSpan::new("// comment")).is_done());
-    }
-
-    #[test]
-    fn test_match_word() {
-        let input = vec![
-            Token {
-                fragment: "foo".to_string(),
-                typ: TokenType::BAREWORD,
-                pos: Position { line: 1, column: 1 },
-            },
-        ];
-        let result = word!(
-            TokenIter {
-                source: input.as_slice(),
-            },
-            "foo"
-        );
-        match result {
-            nom::IResult::Done(_, tok) => assert_eq!(tok, input[0]),
-            res => assert!(false, format!("Fail: {:?}", res)),
-        }
-    }
-
-    #[test]
-    fn test_match_word_empty_input() {
-        let input = vec![
-            Token {
-                fragment: "".to_string(),
-                typ: TokenType::END,
-                pos: Position { line: 1, column: 1 },
-            },
-        ];
-        let result = word!(
-            TokenIter {
-                source: input.as_slice(),
-            },
-            "foo"
-        );
-        match result {
-            nom::IResult::Done(_, _) => assert!(false, "Should have been an error but was Done"),
-            nom::IResult::Incomplete(_) => {
-                assert!(false, "Should have been an error but was Incomplete")
-            }
-            nom::IResult::Error(_) => {
-                // noop
-            }
-        }
-    }
-
-    #[test]
-    fn test_match_punct() {
-        let input = vec![
-            Token {
-                fragment: "!".to_string(),
-                typ: TokenType::PUNCT,
-                pos: Position { line: 1, column: 1 },
-            },
-        ];
-        let result = punct!(
-            TokenIter {
-                source: input.as_slice(),
-            },
-            "!"
-        );
-        match result {
-            nom::IResult::Done(_, tok) => assert_eq!(tok, input[0]),
-            res => assert!(false, format!("Fail: {:?}", res)),
-        }
-    }
-
-    #[test]
-    fn test_match_type() {
-        let input = vec![
-            Token {
-                fragment: "foo".to_string(),
-                typ: TokenType::BAREWORD,
-                pos: Position { line: 1, column: 1 },
-            },
-        ];
-        let result = match_type!(
-            TokenIter {
-                source: input.as_slice(),
-            },
-            BAREWORD
-        );
-        match result {
-            nom::IResult::Done(_, tok) => assert_eq!(tok, input[0]),
-            res => assert!(false, format!("Fail: {:?}", res)),
-        }
-    }
-}

src/tokenizer/test.rs

@@ -0,0 +1,258 @@
+use super::*;
+use nom;
+use nom_locate::LocatedSpan;
+
+#[test]
+fn test_empty_token() {
+    let result = emptytok(LocatedSpan::new("NULL"));
+    assert!(result.is_done(), format!("result {:?} is not done", result));
+    if let nom::IResult::Done(_, tok) = result {
+        assert_eq!(tok.fragment, "NULL");
+        assert_eq!(tok.typ, TokenType::EMPTY);
+    }
+}
+
+#[test]
+fn test_escape_quoted() {
+    let result = escapequoted(LocatedSpan::new("foo \\\"bar\""));
+    assert!(result.is_done(), format!("result {:?} is not ok", result));
+    if let nom::IResult::Done(rest, frag) = result {
+        assert_eq!(frag, "foo \"bar");
+        assert_eq!(rest.fragment, "\"");
+    }
+}
+
+#[test]
+fn test_string_with_escaping() {
+    let result = strtok(LocatedSpan::new("\"foo \\\\ \\\"bar\""));
+    assert!(result.is_done(), format!("result {:?} is not ok", result));
+    if let nom::IResult::Done(_, tok) = result {
+        assert_eq!(tok.fragment, "foo \\ \"bar".to_string());
+    }
+}
+
+#[test]
+fn test_tokenize_bareword_with_dash() {
+    let result = tokenize(LocatedSpan::new("foo-bar "));
+    assert!(result.is_ok(), format!("result {:?} is not ok", result));
+    if let Ok(toks) = result {
+        assert_eq!(toks.len(), 2);
+        assert_eq!(toks[0].fragment, "foo-bar");
+    }
+}
+
+macro_rules! assert_token {
+    ($input:expr, $typ:expr, $msg:expr) => {
+        let result = token(LocatedSpan::new($input));
+        assert!(
+            result.is_done(),
+            format!("result {:?} is not a {}", result, $msg)
+        );
+        if let nom::IResult::Done(_, tok) = result {
+            assert_eq!(tok.fragment, $input);
+            assert_eq!(tok.typ, $typ);
+        }
+    };
+}
+
+#[test]
+fn test_boolean() {
+    assert_token!("true", TokenType::BOOLEAN, "boolean");
+}
+
+#[test]
+fn test_eqeqtok() {
+    assert_token!("==", TokenType::PUNCT, "==");
+}
+
+#[test]
+fn test_notequaltok() {
+    assert_token!("!=", TokenType::PUNCT, "!=");
+}
+
+#[test]
+fn test_gttok() {
+    assert_token!(">", TokenType::PUNCT, ">");
+}
+
+#[test]
+fn test_lttok() {
+    assert_token!("<", TokenType::PUNCT, "<");
+}
+
+#[test]
+fn test_gteqtok() {
+    assert_token!(">=", TokenType::PUNCT, ">=");
+}
+
+#[test]
+fn test_lteqtok() {
+    assert_token!("<=", TokenType::PUNCT, "<=");
+}
+
+#[test]
+fn test_tokenize_one_of_each() {
+    let result = tokenize(LocatedSpan::new(
+        "let import macro select as => [ ] { } ; = % / * \
+         + - . ( ) , 1 . foo \"bar\" // comment\n ; true false == < > <= >= !=",
+    ));
+    assert!(result.is_ok(), format!("result {:?} is not ok", result));
+    let v = result.unwrap();
+    for (i, t) in v.iter().enumerate() {
+        println!("{}: {:?}", i, t);
+    }
+    assert_eq!(v.len(), 35);
+    assert_eq!(v[34].typ, TokenType::END);
+}
+
+#[test]
+fn test_parse_has_end() {
+    let result = tokenize(LocatedSpan::new("foo"));
+    assert!(result.is_ok());
+    let v = result.unwrap();
+    assert_eq!(v.len(), 2);
+    assert_eq!(v[1].typ, TokenType::END);
+}
+
+#[test]
+fn test_parse_comment() {
+    assert!(comment(LocatedSpan::new("// comment\n")).is_done());
+    assert!(comment(LocatedSpan::new("// comment")).is_done());
+    assert_eq!(
+        comment(LocatedSpan::new("// comment\n")),
+        nom::IResult::Done(
+            LocatedSpan {
+                fragment: "",
+                offset: 11,
+                line: 2,
+            },
+            Token {
+                typ: TokenType::COMMENT,
+                fragment: " comment".to_string(),
+                pos: Position { line: 1, column: 1 },
+            }
+        )
+    );
+    assert!(comment(LocatedSpan::new("// comment\r\n")).is_done());
+    assert_eq!(
+        comment(LocatedSpan::new("// comment\r\n")),
+        nom::IResult::Done(
+            LocatedSpan {
+                fragment: "",
+                offset: 12,
+                line: 2,
+            },
+            Token {
+                typ: TokenType::COMMENT,
+                fragment: " comment".to_string(),
+                pos: Position { column: 1, line: 1 },
+            }
+        )
+    );
+    assert!(comment(LocatedSpan::new("// comment\r\n ")).is_done());
+    assert_eq!(
+        comment(LocatedSpan::new("// comment\r\n ")),
+        nom::IResult::Done(
+            LocatedSpan {
+                fragment: " ",
+                offset: 12,
+                line: 2,
+            },
+            Token {
+                typ: TokenType::COMMENT,
+                fragment: " comment".to_string(),
+                pos: Position { column: 1, line: 1 },
+            }
+        )
+    );
+    assert!(comment(LocatedSpan::new("// comment")).is_done());
+}
+
+#[test]
+fn test_match_word() {
+    let input = vec![
+        Token {
+            fragment: "foo".to_string(),
+            typ: TokenType::BAREWORD,
+            pos: Position { line: 1, column: 1 },
+        },
+    ];
+    let result = word!(
+        TokenIter {
+            source: input.as_slice(),
+        },
+        "foo"
+    );
+    match result {
+        nom::IResult::Done(_, tok) => assert_eq!(tok, input[0]),
+        res => assert!(false, format!("Fail: {:?}", res)),
+    }
+}
+
+#[test]
+fn test_match_word_empty_input() {
+    let input = vec![
+        Token {
+            fragment: "".to_string(),
+            typ: TokenType::END,
+            pos: Position { line: 1, column: 1 },
+        },
+    ];
+    let result = word!(
+        TokenIter {
+            source: input.as_slice(),
+        },
+        "foo"
+    );
+    match result {
+        nom::IResult::Done(_, _) => assert!(false, "Should have been an error but was Done"),
+        nom::IResult::Incomplete(_) => {
+            assert!(false, "Should have been an error but was Incomplete")
+        }
+        nom::IResult::Error(_) => {
+            // noop
+        }
+    }
+}
+
+#[test]
+fn test_match_punct() {
+    let input = vec![
+        Token {
+            fragment: "!".to_string(),
+            typ: TokenType::PUNCT,
+            pos: Position { line: 1, column: 1 },
+        },
+    ];
+    let result = punct!(
+        TokenIter {
+            source: input.as_slice(),
+        },
+        "!"
+    );
+    match result {
+        nom::IResult::Done(_, tok) => assert_eq!(tok, input[0]),
+        res => assert!(false, format!("Fail: {:?}", res)),
+    }
+}
+
+#[test]
+fn test_match_type() {
+    let input = vec![
+        Token {
+            fragment: "foo".to_string(),
+            typ: TokenType::BAREWORD,
+            pos: Position { line: 1, column: 1 },
+        },
+    ];
+    let result = match_type!(
+        TokenIter {
+            source: input.as_slice(),
+        },
+        BAREWORD
+    );
+    match result {
+        nom::IResult::Done(_, tok) => assert_eq!(tok, input[0]),
+        res => assert!(false, format!("Fail: {:?}", res)),
+    }
+}