REFACTOR: Pass our offsets everywhere.

examples/test_yaml.yaml

@@ -0,0 +1,9 @@
+---
+db_conn1: "db1.prod.net:3306/testdb"
+db_conn2: "db2.prod.net:3306/testdb"
+tmpldir: "./templates"
+prefix:
+  foo: bar
+l:
+  - foo
+  - bar

integration_tests/operator_precedence_test.ucg

@@ -5,3 +5,5 @@ assert |2 * 2 + 1 > 4|;
 assert |2 * 2 + 1 < 6|;
 assert |2 * 2 + 1 >= 5|;
 assert |2 * 2 + 1 <= 5|;
+assert |2 / 2  == 1|;
+assert |2 - 1 == 1|;

src/ast/mod.rs

@@ -21,11 +21,10 @@ use std::cmp::PartialEq;
 use std::cmp::PartialOrd;
 use std::collections::HashSet;
 use std::convert::Into;
+use std::fmt;
 use std::hash::Hash;
 use std::hash::Hasher;
 
-use std::fmt;
-
 macro_rules! enum_type_equality {
     ( $slf:ident, $r:expr, $( $l:pat ),* ) => {
         match $slf {
@@ -50,14 +49,16 @@ macro_rules! enum_type_equality {
 pub struct Position {
     pub line: usize,
     pub column: usize,
+    pub offset: usize,
 }
 
 impl Position {
     /// Construct a new Position.
-    pub fn new(line: usize, column: usize) -> Self {
+    pub fn new(line: usize, column: usize, offset: usize) -> Self {
         Position {
             line: line,
             column: column,
+            offset: offset,
         }
     }
 }
@@ -89,8 +90,8 @@ pub struct Token {
 
 impl Token {
     /// Constructs a new Token with a type and line and column information.
-    pub fn new<S: Into<String>>(f: S, typ: TokenType, line: usize, col: usize) -> Self {
+    pub fn new<S: Into<String>, P: Into<Position>>(f: S, typ: TokenType, p: P) -> Self {
-        Self::new_with_pos(f, typ, Position::new(line, col))
+        Self::new_with_pos(f, typ, p.into())
     }
 
     // Constructs a new Token with a type and a Position.
@@ -114,56 +115,49 @@ macro_rules! value_node {
     ($v:expr, $p:expr) => {
         Positioned::new_with_pos($v, $p)
     };
-    ($v:expr, $l:expr, $c:expr) => {
-        Positioned::new($v, $l, $c)
-    };
 }
 
 /// Helper macro for making a Token.
 #[allow(unused_macros)]
 macro_rules! make_tok {
-    (EOF => $l:expr, $c:expr) => {
+    (EOF => $i:expr) => {
-        Token::new("", TokenType::END, $l, $c)
+        Token::new("", TokenType::END, $i)
     };
 
-    (WS => $l:expr, $c:expr) => {
+    (WS => $i:expr) => {
-        Token::new("", TokenType::WS, $l, $c)
+        Token::new("", TokenType::WS, $i)
     };
 
-    (CMT => $e:expr, $l:expr, $c:expr) => {
+    (CMT => $e:expr, $i:expr) => {
-        Token::new($e, TokenType::COMMENT, $l, $c)
+        Token::new($e, TokenType::COMMENT, $i)
     };
 
-    (QUOT => $e:expr, $l:expr, $c:expr) => {
+    (QUOT => $e:expr, $i:expr) => {
-        Token::new($e, TokenType::QUOTED, $l, $c)
+        Token::new($e, TokenType::QUOTED, $i)
     };
 
-    (PUNCT => $e:expr, $l:expr, $c:expr) => {
+    (PUNCT => $e:expr, $i:expr) => {
-        Token::new($e, TokenType::PUNCT, $l, $c)
+        Token::new($e, TokenType::PUNCT, $i)
     };
 
-    (DIGIT => $e:expr, $l:expr, $c:expr) => {
+    (DIGIT => $e:expr, $i:expr) => {
-        Token::new($e, TokenType::DIGIT, $l, $c)
+        Token::new($e, TokenType::DIGIT, $i)
     };
 
-    ($e:expr, $l:expr, $c:expr) => {
+    ($e:expr, $i:expr) => {
-        Token::new($e, TokenType::BAREWORD, $l, $c)
+        Token::new($e, TokenType::BAREWORD, $i)
     };
 }
 
 /// Helper macro for making expressions.
 #[allow(unused_macros)]
 macro_rules! make_expr {
-    ($e:expr) => {
+    ($e:expr, $i:expr) => {
-        make_expr!($e, 1, 1)
+        Expression::Simple(Value::Symbol(Positioned::new_with_pos($e.to_string(), $i)))
     };
 
-    ($e:expr, $l:expr, $c:expr) => {
+    ($e:expr => int, $i:expr) => {
-        Expression::Simple(Value::Symbol(Positioned::new($e.to_string(), $l, $c)))
+        Expression::Simple(Value::Int(Positioned::new_with_pos($e, $i)))
-    };
-
-    ($e:expr => int, $l:expr, $c:expr) => {
-        Expression::Simple(Value::Int(Positioned::new($e, $l, $c)))
     };
 }
 
@@ -180,30 +174,26 @@ macro_rules! make_expr {
 /// ```
 #[allow(unused_macros)]
 macro_rules! make_selector {
-    ( $h:expr ) => {
+    ( $h:expr, $i:expr) => {
-        make_selector!($h, 1, 0)
-    };
-
-    ( $h:expr, $l:expr, $c:expr ) => {
         SelectorDef::new(
             SelectorList{head: Box::new($h), tail: None},
-            $l, $c)
+            $i)
     };
 
-    ( $h: expr, $list:expr, $l:expr, $c:expr) => {
+    ( $h: expr, $list:expr, $i:expr) => {
         SelectorDef::new(
             SelectorList{head: Box::new($h), tail: Some($list)},
-            $l, $c)
+            $i)
     };
 
     // Tokens
-    ( $h:expr => [ $( $item:expr ),* ] ) => {
+    ( $h:expr => [ $( $item:expr ),* ], $i:expr ) => {
         {
-            make_selector!($h => [ $( $item, )* ] => 1, 1)
+            make_selector!($h => [ $( $item, )* ] => $i)
         }
     };
 
-    ( $h:expr => [ $( $item:expr ),* ] => $l:expr, $c:expr ) => {
+    ( $h:expr => [ $( $item:expr ),* ] => $i:expr ) => {
         {
             let mut list: Vec<Token> = Vec::new();
 
@@ -211,7 +201,7 @@ macro_rules! make_selector {
                 list.push($item);
             )*
 
-            make_selector!($h, list, $l, $c)
+            make_selector!($h, list, $i)
         }
     };
 
@@ -223,14 +213,14 @@ macro_rules! make_selector {
             let mut list: Vec<Token> = Vec::new();
 
             $(
-                list.push(make_tok!($item, 1, col));
+                list.push(make_tok!($item, Position::new(1, col, col)));
                 col += $item.len() + 1;
             )*
 
             // Shut up the lint about unused code;
             assert!(col != 0);
 
-            make_selector!($h, list, 1, 1)
+            make_selector!($h, list, Position::new(1, 1, 1))
         }
 
     };
@@ -241,14 +231,14 @@ macro_rules! make_selector {
             let mut list: Vec<Token> = Vec::new();
 
             $(
-                list.push(make_tok!($item, $l, col));
+                list.push(make_tok!($item, Position::new($l, col, col)));
                 col += $item.len() + 1;
             )*
 
             // Shut up the linter about unused code;
             assert!(col != 0);
 
-            make_selector!($h, list, $l, $c)
+            make_selector!($h, list, Position::new($l, $c, $c))
         }
     };
 }
@@ -314,9 +304,9 @@ pub struct SelectorDef {
 
 impl SelectorDef {
     /// Constructs a new SelectorDef.
-    pub fn new(sel: SelectorList, line: usize, col: usize) -> Self {
+    pub fn new<P: Into<Position>>(sel: SelectorList, p: P) -> Self {
         SelectorDef {
-            pos: Position::new(line, col),
+            pos: p.into(),
             sel: sel,
         }
     }
@@ -451,8 +441,8 @@ impl<T: std::fmt::Display> std::fmt::Display for Positioned<T> {
 
 impl<T> Positioned<T> {
     /// Constructs a new Positioned<T> with a value, line, and column information.
-    pub fn new(v: T, l: usize, c: usize) -> Self {
+    pub fn new<P: Into<Position>>(v: T, p: P) -> Self {
-        Self::new_with_pos(v, Position::new(l, c))
+        Self::new_with_pos(v, p.into())
     }
 
     /// Constructs a new Positioned<T> with a value and a Position.

src/ast/test.rs

@@ -17,21 +17,23 @@ use super::*;
 #[test]
 pub fn test_macro_validation_happy_path() {
     let def = MacroDef {
-        argdefs: vec![value_node!("foo".to_string(), 1, 0)],
+        argdefs: vec![value_node!("foo".to_string(), Position::new(1, 0, 0))],
         fields: vec![(
-            make_tok!("f1", 1, 1),
+            make_tok!("f1", Position::new(1, 1, 0)),
             Expression::Binary(BinaryOpDef {
                 kind: BinaryExprType::Add,
                 left: Box::new(Expression::Simple(Value::Symbol(value_node!(
                     "foo".to_string(),
-                    1,
+                    Position::new(1, 1, 0)
-                    1
                 )))),
-                right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
+                right: Box::new(Expression::Simple(Value::Int(value_node!(
-                pos: Position::new(1, 0),
+                    1,
+                    Position::new(1, 1, 0)
+                )))),
+                pos: Position::new(1, 0, 0),
             }),
         )],
-        pos: Position::new(1, 0),
+        pos: Position::new(1, 0, 0),
     };
     assert!(def.validate_symbols().unwrap() == ());
 }
@@ -39,21 +41,23 @@ pub fn test_macro_validation_happy_path() {
 #[test]
 pub fn test_macro_validation_fail() {
     let def = MacroDef {
-        argdefs: vec![value_node!("foo".to_string(), 1, 0)],
+        argdefs: vec![value_node!("foo".to_string(), Position::new(1, 0, 0))],
         fields: vec![(
-            make_tok!("f1", 1, 1),
+            make_tok!("f1", Position::new(1, 1, 0)),
             Expression::Binary(BinaryOpDef {
                 kind: BinaryExprType::Add,
                 left: Box::new(Expression::Simple(Value::Symbol(value_node!(
                     "bar".to_string(),
-                    1,
+                    Position::new(1, 1, 0)
-                    1
                 )))),
-                right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
+                right: Box::new(Expression::Simple(Value::Int(value_node!(
-                pos: Position::new(1, 0),
+                    1,
+                    Position::new(1, 1, 0)
+                )))),
+                pos: Position::new(1, 0, 0),
             }),
         )],
-        pos: Position::new(1, 0),
+        pos: Position::new(1, 0, 0),
     };
     let mut expected = HashSet::new();
     expected.insert("bar".to_string());
@@ -63,20 +67,23 @@ pub fn test_macro_validation_fail() {
 #[test]
 pub fn test_macro_validation_selector_happy_path() {
     let def = MacroDef {
-        argdefs: vec![value_node!("foo".to_string(), 1, 0)],
+        argdefs: vec![value_node!("foo".to_string(), Position::new(1, 0, 0))],
         fields: vec![(
-            make_tok!("f1", 1, 1),
+            make_tok!("f1", Position::new(1, 1, 0)),
             Expression::Binary(BinaryOpDef {
                 kind: BinaryExprType::Add,
-                left: Box::new(Expression::Simple(Value::Selector(
+                left: Box::new(Expression::Simple(Value::Selector(make_selector!(
-                    make_selector!(make_expr!("foo", 1, 1) => [
+                        make_expr!("foo", Position::new(1, 1, 0)) => [
-                    make_tok!("quux", 1, 1) ] => 1, 1),
+                            make_tok!("quux", Position::new(1, 1, 0)) ]
-                ))),
+                        => Position::new(1, 1, 0))))),
-                right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
+                right: Box::new(Expression::Simple(Value::Int(value_node!(
-                pos: Position::new(1, 0),
+                    1,
+                    Position::new(1, 1, 0)
+                )))),
+                pos: Position::new(1, 0, 0),
             }),
         )],
-        pos: Position::new(1, 0),
+        pos: Position::new(1, 0, 0),
     };
     assert!(def.validate_symbols().unwrap() == ());
 }
@@ -84,20 +91,23 @@ pub fn test_macro_validation_selector_happy_path() {
 #[test]
 pub fn test_macro_validation_selector_fail() {
     let def = MacroDef {
-        argdefs: vec![value_node!("foo".to_string(), 1, 0)],
+        argdefs: vec![value_node!("foo".to_string(), Position::new(1, 0, 0))],
         fields: vec![(
-            make_tok!("f1", 1, 1),
+            make_tok!("f1", Position::new(1, 1, 0)),
             Expression::Binary(BinaryOpDef {
                 kind: BinaryExprType::Add,
                 left: Box::new(Expression::Simple(Value::Selector(
-                    make_selector!(make_expr!("bar", 1, 1) => [
+                    make_selector!(make_expr!("bar", Position::new(1, 1, 0)) => [
-                    make_tok!("quux", 1, 1) ] => 1, 1),
+                    make_tok!("quux", Position::new(1, 1, 0)) ] => Position::new(1, 1, 0)),
                 ))),
-                right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
+                right: Box::new(Expression::Simple(Value::Int(value_node!(
-                pos: Position::new(1, 0),
+                    1,
+                    Position::new(1, 1, 0)
+                )))),
+                pos: Position::new(1, 0, 0),
             }),
         )],
-        pos: Position::new(1, 0),
+        pos: Position::new(1, 0, 0),
     };
     let mut expected = HashSet::new();
     expected.insert("bar".to_string());

src/benches/parse.rs

@@ -22,14 +22,14 @@ extern crate ucglib;
 
 use bencher::Bencher;
 
-use abortable_parser::StrIter;
+use ucglib::iter::OffsetStrIter;
 
 //use cpuprofiler::PROFILER;
 
 use ucglib::parse::*;
 
 fn do_parse(i: &str) {
-    parse(StrIter::new(i));
+    parse(OffsetStrIter::new(i));
 }
 
 fn parse_int(b: &mut Bencher) {

src/build/mod.rs

@@ -25,11 +25,10 @@ use std::path::PathBuf;
 use std::rc::Rc;
 use std::string::ToString;
 
-use abortable_parser::StrIter;
-
 use ast::*;
 use error;
 use format;
+use iter::OffsetStrIter;
 use parse::parse;
 
 pub mod assets;
@@ -192,8 +191,12 @@ impl<'a> Builder<'a> {
         scope: ValueMap,
     ) -> Self {
         let env_vars: Vec<(Positioned<String>, Rc<Val>)> = env::vars()
-            .map(|t| (Positioned::new(t.0, 0, 0), Rc::new(t.1.into())))
+            .map(|t| {
-            .collect();
+                (
+                    Positioned::new(t.0, Position::new(0, 0, 0)),
+                    Rc::new(t.1.into()),
+                )
+            }).collect();
         Self::new_with_env_and_scope(root, cache, scope, Rc::new(Val::Tuple(env_vars)))
     }
 
@@ -223,7 +226,7 @@ impl<'a> Builder<'a> {
     /// Returns a Val by name from previously built UCG.
     pub fn get_out_by_name(&self, name: &str) -> Option<Rc<Val>> {
         let key = Positioned {
-            pos: Position::new(0, 0),
+            pos: Position::new(0, 0, 0),
             val: name.to_string(),
         };
         self.lookup_sym(&key)
@@ -245,7 +248,7 @@ impl<'a> Builder<'a> {
         Ok(())
     }
 
-    fn eval_span(&mut self, input: StrIter) -> Result<Rc<Val>, Box<Error>> {
+    fn eval_span(&mut self, input: OffsetStrIter) -> Result<Rc<Val>, Box<Error>> {
         match parse(input) {
             Ok(stmts) => {
                 //panic!("Successfully parsed {}", input);
@@ -258,6 +261,7 @@ impl<'a> Builder<'a> {
                     Some(val) => Ok(val),
                 }
             }
+            // FIXME(jwall): We need to return a error::Error so we have position information.
             Err(err) => Err(Box::new(error::Error::new_with_boxed_cause(
                 format!(
                     "Error while parsing file: {}",
@@ -271,7 +275,7 @@ impl<'a> Builder<'a> {
 
     /// Evaluate an input string as UCG.
     pub fn eval_string(&mut self, input: &str) -> Result<Rc<Val>, Box<Error>> {
-        self.eval_span(StrIter::new(input))
+        self.eval_span(OffsetStrIter::new(input))
     }
 
     /// Builds a ucg file at the named path.
@@ -993,7 +997,8 @@ impl<'a> Builder<'a> {
         let expr = &tok.fragment;
         expr_as_stmt.push_str(expr);
         expr_as_stmt.push_str(";");
-        let assert_input = StrIter::new(&expr_as_stmt);
+        let assert_input =
+            OffsetStrIter::new_with_offsets(&expr_as_stmt, 0, tok.pos.line - 1, tok.pos.column - 1);
         let ok = match self.eval_span(assert_input) {
             Ok(v) => v,
             Err(e) => {

src/build/test.rs

@@ -25,35 +25,6 @@ fn test_expr_to_val(mut cases: Vec<(Expression, Val)>, b: Builder) {
     }
 }
 
-#[test]
-fn test_eval_div_expr() {
-    let cache = Rc::new(RefCell::new(MemoryCache::new()));
-    let b = Builder::new(std::env::current_dir().unwrap(), cache);
-    test_expr_to_val(
-        vec![
-            (
-                Expression::Binary(BinaryOpDef {
-                    kind: BinaryExprType::Div,
-                    left: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))),
-                    right: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))),
-                    pos: Position::new(1, 0),
-                }),
-                Val::Int(1),
-            ),
-            (
-                Expression::Binary(BinaryOpDef {
-                    kind: BinaryExprType::Div,
-                    left: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))),
-                    right: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))),
-                    pos: Position::new(1, 0),
-                }),
-                Val::Float(1.0),
-            ),
-        ],
-        b,
-    );
-}
-
 #[test]
 #[should_panic(expected = "Expected Float")]
 fn test_eval_div_expr_fail() {
@@ -63,9 +34,15 @@ fn test_eval_div_expr_fail() {
         vec![(
             Expression::Binary(BinaryOpDef {
                 kind: BinaryExprType::Div,
-                left: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))),
+                left: Box::new(Expression::Simple(Value::Float(value_node!(
-                right: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))),
+                    2.0,
-                pos: Position::new(1, 0),
+                    Position::new(1, 1, 1)
+                )))),
+                right: Box::new(Expression::Simple(Value::Int(value_node!(
+                    2,
+                    Position::new(1, 1, 1)
+                )))),
+                pos: Position::new(1, 0, 0),
             }),
             Val::Float(1.0),
         )],
@@ -73,35 +50,6 @@ fn test_eval_div_expr_fail() {
     );
 }
 
-#[test]
-fn test_eval_mul_expr() {
-    let cache = Rc::new(RefCell::new(MemoryCache::new()));
-    let b = Builder::new(std::env::current_dir().unwrap(), cache);
-    test_expr_to_val(
-        vec![
-            (
-                Expression::Binary(BinaryOpDef {
-                    kind: BinaryExprType::Mul,
-                    left: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))),
-                    right: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))),
-                    pos: Position::new(1, 0),
-                }),
-                Val::Int(4),
-            ),
-            (
-                Expression::Binary(BinaryOpDef {
-                    kind: BinaryExprType::Mul,
-                    left: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))),
-                    right: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))),
-                    pos: Position::new(1, 0),
-                }),
-                Val::Float(4.0),
-            ),
-        ],
-        b,
-    );
-}
-
 #[test]
 #[should_panic(expected = "Expected Float")]
 fn test_eval_mul_expr_fail() {
@@ -111,9 +59,15 @@ fn test_eval_mul_expr_fail() {
         vec![(
             Expression::Binary(BinaryOpDef {
                 kind: BinaryExprType::Mul,
-                left: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))),
+                left: Box::new(Expression::Simple(Value::Float(value_node!(
-                right: Box::new(Expression::Simple(Value::Int(value_node!(20, 1, 1)))),
+                    2.0,
-                pos: Position::new(1, 0),
+                    Position::new(1, 1, 1)
+                )))),
+                right: Box::new(Expression::Simple(Value::Int(value_node!(
+                    20,
+                    Position::new(1, 1, 1)
+                )))),
+                pos: Position::new(1, 0, 0),
             }),
             Val::Float(1.0),
         )],
@@ -121,35 +75,6 @@ fn test_eval_mul_expr_fail() {
     );
 }
 
-#[test]
-fn test_eval_subtract_expr() {
-    let cache = Rc::new(RefCell::new(MemoryCache::new()));
-    let b = Builder::new(std::env::current_dir().unwrap(), cache);
-    test_expr_to_val(
-        vec![
-            (
-                Expression::Binary(BinaryOpDef {
-                    kind: BinaryExprType::Sub,
-                    left: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))),
-                    right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
-                    pos: Position::new(1, 0),
-                }),
-                Val::Int(1),
-            ),
-            (
-                Expression::Binary(BinaryOpDef {
-                    kind: BinaryExprType::Sub,
-                    left: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))),
-                    right: Box::new(Expression::Simple(Value::Float(value_node!(1.0, 1, 1)))),
-                    pos: Position::new(1, 0),
-                }),
-                Val::Float(1.0),
-            ),
-        ],
-        b,
-    );
-}
-
 #[test]
 #[should_panic(expected = "Expected Float")]
 fn test_eval_subtract_expr_fail() {
@@ -159,88 +84,21 @@ fn test_eval_subtract_expr_fail() {
         vec![(
             Expression::Binary(BinaryOpDef {
                 kind: BinaryExprType::Sub,
-                left: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))),
+                left: Box::new(Expression::Simple(Value::Float(value_node!(
-                right: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))),
+                    2.0,
-                pos: Position::new(1, 0),
+                    Position::new(1, 1, 1)
+                )))),
+                right: Box::new(Expression::Simple(Value::Int(value_node!(
+                    2,
+                    Position::new(1, 1, 1)
+                )))),
+                pos: Position::new(1, 0, 0),
             }),
             Val::Float(1.0),
         )],
         b,
     );
 }
-
-#[test]
-fn test_eval_add_expr() {
-    let cache = Rc::new(RefCell::new(MemoryCache::new()));
-    let b = Builder::new(std::env::current_dir().unwrap(), cache);
-    test_expr_to_val(
-        vec![
-            (
-                Expression::Binary(BinaryOpDef {
-                    kind: BinaryExprType::Add,
-                    left: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
-                    right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
-                    pos: Position::new(1, 0),
-                }),
-                Val::Int(2),
-            ),
-            (
-                Expression::Binary(BinaryOpDef {
-                    kind: BinaryExprType::Add,
-                    left: Box::new(Expression::Simple(Value::Float(value_node!(1.0, 1, 1)))),
-                    right: Box::new(Expression::Simple(Value::Float(value_node!(1.0, 1, 1)))),
-                    pos: Position::new(1, 0),
-                }),
-                Val::Float(2.0),
-            ),
-            (
-                Expression::Binary(BinaryOpDef {
-                    kind: BinaryExprType::Add,
-                    left: Box::new(Expression::Simple(Value::Str(value_node!(
-                        "foo".to_string(),
-                        1,
-                        1
-                    )))),
-                    right: Box::new(Expression::Simple(Value::Str(value_node!(
-                        "bar".to_string(),
-                        1,
-                        1
-                    )))),
-                    pos: Position::new(1, 0),
-                }),
-                Val::Str("foobar".to_string()),
-            ),
-            (
-                Expression::Binary(BinaryOpDef {
-                    kind: BinaryExprType::Add,
-                    left: Box::new(Expression::Simple(Value::List(ListDef {
-                        elems: vec![Expression::Simple(Value::Str(value_node!(
-                            "foo".to_string(),
-                            1,
-                            1
-                        )))],
-                        pos: Position::new(1, 1),
-                    }))),
-                    right: Box::new(Expression::Simple(Value::List(ListDef {
-                        elems: vec![Expression::Simple(Value::Str(value_node!(
-                            "bar".to_string(),
-                            1,
-                            1
-                        )))],
-                        pos: Position::new(1, 1),
-                    }))),
-                    pos: Position::new(1, 0),
-                }),
-                Val::List(vec![
-                    Rc::new(Val::Str("foo".to_string())),
-                    Rc::new(Val::Str("bar".to_string())),
-                ]),
-            ),
-        ],
-        b,
-    );
-}
-
 #[test]
 #[should_panic(expected = "Expected Float")]
 fn test_eval_add_expr_fail() {
@@ -250,9 +108,15 @@ fn test_eval_add_expr_fail() {
         vec![(
             Expression::Binary(BinaryOpDef {
                 kind: BinaryExprType::Add,
-                left: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))),
+                left: Box::new(Expression::Simple(Value::Float(value_node!(
-                right: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))),
+                    2.0,
-                pos: Position::new(1, 0),
+                    Position::new(1, 1, 1)
+                )))),
+                right: Box::new(Expression::Simple(Value::Int(value_node!(
+                    2,
+                    Position::new(1, 1, 1)
+                )))),
+                pos: Position::new(1, 0, 0),
             }),
             Val::Float(1.0),
         )],
@@ -260,245 +124,20 @@ fn test_eval_add_expr_fail() {
     );
 }
 
-#[test]
-fn test_eval_nested_tuple() {
-    test_expr_to_val(
-        vec![
-            (
-                Expression::Simple(Value::Tuple(value_node!(
-                    vec![(
-                        Token::new("foo", TokenType::BAREWORD, 1, 1),
-                        Expression::Simple(Value::Tuple(value_node!(Vec::new(), 1, 1))),
-                    )],
-                    1,
-                    1
-                ))),
-                Val::Tuple(vec![(
-                    Positioned::new("foo".to_string(), 1, 1),
-                    Rc::new(Val::Tuple(Vec::new())),
-                )]),
-            ),
-            (
-                Expression::Simple(Value::Tuple(value_node!(
-                    vec![(
-                        Token::new("foo", TokenType::BAREWORD, 1, 1),
-                        Expression::Simple(Value::Tuple(value_node!(
-                            vec![(
-                                Token::new("bar".to_string(), TokenType::BAREWORD, 1, 5),
-                                Expression::Simple(Value::Tuple(value_node!(vec![], 1, 10))),
-                            )],
-                            1,
-                            1
-                        ))),
-                    )],
-                    1,
-                    1
-                ))),
-                Val::Tuple(vec![(
-                    Positioned::new("foo".to_string(), 1, 1),
-                    Rc::new(Val::Tuple(vec![(
-                        Positioned::new("bar".to_string(), 1, 10),
-                        Rc::new(Val::Tuple(vec![])),
-                    )])),
-                )]),
-            ),
-            (
-                Expression::Simple(Value::Tuple(value_node!(
-                    vec![(
-                        Token::new("foo", TokenType::BAREWORD, 1, 1),
-                        Expression::Simple(Value::Tuple(value_node!(
-                            vec![(
-                                Token::new("bar".to_string(), TokenType::BAREWORD, 1, 5),
-                                Expression::Simple(Value::Tuple(value_node!(
-                                    vec![(
-                                        Token::new("quux".to_string(), TokenType::BAREWORD, 1, 1),
-                                        Expression::Simple(Value::Int(value_node!(3, 1, 1))),
-                                    )],
-                                    1,
-                                    10
-                                ))),
-                            )],
-                            1,
-                            1
-                        ))),
-                    )],
-                    1,
-                    1
-                ))),
-                Val::Tuple(vec![(
-                    Positioned::new("foo".to_string(), 1, 1),
-                    Rc::new(Val::Tuple(vec![(
-                        Positioned::new("bar".to_string(), 1, 10),
-                        Rc::new(Val::Tuple(vec![(
-                            Positioned::new("quux".to_string(), 1, 1),
-                            Rc::new(Val::Int(3)),
-                        )])),
-                    )])),
-                )]),
-            ),
-        ],
-        Builder::new(
-            std::env::current_dir().unwrap(),
-            Rc::new(RefCell::new(MemoryCache::new())),
-        ),
-    );
-}
-
-#[test]
-fn test_eval_simple_expr() {
-    test_expr_to_val(
-        vec![
-            (
-                Expression::Simple(Value::Int(value_node!(1, 1, 1))),
-                Val::Int(1),
-            ),
-            (
-                Expression::Simple(Value::Float(value_node!(2.0, 1, 1))),
-                Val::Float(2.0),
-            ),
-            (
-                Expression::Simple(Value::Str(value_node!("foo".to_string(), 1, 1))),
-                Val::Str("foo".to_string()),
-            ),
-            (
-                Expression::Simple(Value::Tuple(value_node!(
-                    vec![(
-                        make_tok!("bar", 1, 1),
-                        Expression::Simple(Value::Int(value_node!(1, 1, 1))),
-                    )],
-                    1,
-                    1
-                ))),
-                Val::Tuple(vec![(
-                    value_node!("bar".to_string(), 1, 1),
-                    Rc::new(Val::Int(1)),
-                )]),
-            ),
-        ],
-        Builder::new(
-            std::env::current_dir().unwrap(),
-            Rc::new(RefCell::new(MemoryCache::new())),
-        ),
-    );
-}
-
-#[test]
-fn test_eval_simple_lookup_expr() {
-    let cache = Rc::new(RefCell::new(MemoryCache::new()));
-    let mut b = Builder::new(std::env::current_dir().unwrap(), cache);
-    b.build_output
-        .entry(value_node!("var1".to_string(), 1, 0))
-        .or_insert(Rc::new(Val::Int(1)));
-    test_expr_to_val(
-        vec![(
-            Expression::Simple(Value::Symbol(value_node!("var1".to_string(), 1, 1))),
-            Val::Int(1),
-        )],
-        b,
-    );
-}
-
 #[test]
 fn test_eval_simple_lookup_error() {
     let cache = Rc::new(RefCell::new(MemoryCache::new()));
     let mut b = Builder::new(std::env::current_dir().unwrap(), cache);
     b.build_output
-        .entry(value_node!("var1".to_string(), 1, 0))
+        .entry(value_node!("var1".to_string(), Position::new(1, 0, 0)))
         .or_insert(Rc::new(Val::Int(1)));
-    let expr = Expression::Simple(Value::Symbol(value_node!("var".to_string(), 1, 1)));
+    let expr = Expression::Simple(Value::Symbol(value_node!(
+        "var".to_string(),
+        Position::new(1, 1, 1)
+    )));
     assert!(b.eval_expr(&expr).is_err());
 }
 
-#[test]
-fn test_eval_selector_expr() {
-    let cache = Rc::new(RefCell::new(MemoryCache::new()));
-    let mut b = Builder::new(std::env::current_dir().unwrap(), cache);
-    b.build_output
-        .entry(value_node!("var1".to_string(), 1, 0))
-        .or_insert(Rc::new(Val::Tuple(vec![(
-            value_node!("lvl1".to_string(), 1, 0),
-            Rc::new(Val::Tuple(vec![(
-                value_node!("lvl2".to_string(), 1, 0),
-                Rc::new(Val::Int(3)),
-            )])),
-        )])));
-    b.build_output
-        .entry(value_node!("var2".to_string(), 1, 0))
-        .or_insert(Rc::new(Val::Int(2)));
-    b.build_output
-        .entry(value_node!("var3".to_string(), 1, 0))
-        .or_insert(Rc::new(Val::Tuple(vec![(
-            value_node!("lvl1".to_string(), 1, 0),
-            Rc::new(Val::Int(4)),
-        )])));
-
-    test_expr_to_val(
-        vec![
-            (
-                Expression::Simple(Value::Selector(make_selector!(make_expr!("var1")))),
-                Val::Tuple(vec![(
-                    value_node!("lvl1".to_string(), 1, 0),
-                    Rc::new(Val::Tuple(vec![(
-                        value_node!("lvl2".to_string(), 1, 0),
-                        Rc::new(Val::Int(3)),
-                    )])),
-                )]),
-            ),
-            (
-                Expression::Simple(Value::Selector(
-                    make_selector!(make_expr!("var1") => "lvl1"),
-                )),
-                Val::Tuple(vec![(
-                    value_node!("lvl2".to_string(), 1, 0),
-                    Rc::new(Val::Int(3)),
-                )]),
-            ),
-            (
-                Expression::Simple(Value::Selector(
-                    make_selector!(make_expr!("var1") => "lvl1", "lvl2"),
-                )),
-                Val::Int(3),
-            ),
-            (
-                Expression::Simple(Value::Selector(make_selector!(make_expr!("var2")))),
-                Val::Int(2),
-            ),
-            (
-                Expression::Simple(Value::Selector(
-                    make_selector!(make_expr!("var3") => "lvl1"),
-                )),
-                Val::Int(4),
-            ),
-        ],
-        b,
-    );
-}
-
-#[test]
-fn test_eval_selector_list_expr() {
-    let cache = Rc::new(RefCell::new(MemoryCache::new()));
-    let mut b = Builder::new(std::env::current_dir().unwrap(), cache);
-    b.build_output
-        .entry(value_node!("var1".to_string(), 1, 1))
-        .or_insert(Rc::new(Val::List(vec![
-            Rc::new(Val::Str("val1".to_string())),
-            Rc::new(Val::Tuple(vec![(
-                value_node!("var2".to_string(), 1, 1),
-                Rc::new(Val::Int(1)),
-            )])),
-        ])));
-
-    test_expr_to_val(
-        vec![(
-            Expression::Simple(Value::Selector(
-                make_selector!(make_expr!("var1") =>  "0" => 1, 1),
-            )),
-            Val::Str("val1".to_string()),
-        )],
-        b,
-    );
-}
-
 // Include nested for each.
 #[test]
 #[should_panic(expected = "Unable to find tpl1")]
@@ -508,9 +147,12 @@ fn test_expr_copy_no_such_tuple() {
     test_expr_to_val(
         vec![(
             Expression::Copy(CopyDef {
-                selector: make_selector!(make_expr!("tpl1")),
+                selector: make_selector!(
+                    make_expr!("tpl1", Position::new(1, 1, 1)),
+                    Position::new(1, 1, 1)
+                ),
                 fields: Vec::new(),
-                pos: Position::new(1, 0),
+                pos: Position::new(1, 0, 0),
             }),
             Val::Tuple(Vec::new()),
         )],
@@ -524,14 +166,17 @@ fn test_expr_copy_not_a_tuple() {
     let cache = Rc::new(RefCell::new(MemoryCache::new()));
     let mut b = Builder::new(std::env::current_dir().unwrap(), cache);
     b.build_output
-        .entry(value_node!("tpl1".to_string(), 1, 0))
+        .entry(value_node!("tpl1".to_string(), Position::new(1, 0, 0)))
         .or_insert(Rc::new(Val::Int(1)));
     test_expr_to_val(
         vec![(
             Expression::Copy(CopyDef {
-                selector: make_selector!(make_expr!("tpl1")),
+                selector: make_selector!(
+                    make_expr!("tpl1", Position::new(1, 1, 1)),
+                    Position::new(1, 1, 1)
+                ),
                 fields: Vec::new(),
-                pos: Position::new(1, 0),
+                pos: Position::new(1, 0, 0),
             }),
             Val::Tuple(Vec::new()),
         )],
@@ -545,130 +190,30 @@ fn test_expr_copy_field_type_error() {
     let cache = Rc::new(RefCell::new(MemoryCache::new()));
     let mut b = Builder::new(std::env::current_dir().unwrap(), cache);
     b.build_output
-        .entry(value_node!("tpl1".to_string(), 1, 0))
+        .entry(value_node!("tpl1".to_string(), Position::new(1, 0, 0)))
         .or_insert(Rc::new(Val::Tuple(vec![(
-            value_node!("fld1".to_string(), 1, 0),
+            value_node!("fld1".to_string(), Position::new(1, 0, 0)),
             Rc::new(Val::Int(1)),
         )])));
     test_expr_to_val(
         vec![(
             Expression::Copy(CopyDef {
-                selector: make_selector!(make_expr!("tpl1")),
+                selector: make_selector!(
-                fields: vec![(
+                    make_expr!("tpl1", Position::new(1, 1, 1)),
-                    make_tok!("fld1", 1, 1),
+                    Position::new(1, 1, 1)
-                    Expression::Simple(Value::Str(value_node!("2".to_string(), 1, 1))),
-                )],
-                pos: Position::new(1, 0),
-            }),
-            Val::Tuple(vec![(
-                value_node!("fld1".to_string(), 1, 1),
-                Rc::new(Val::Str("2".to_string())),
-            )]),
-        )],
-        b,
-    );
-}
-
-#[test]
-fn test_expr_copy() {
-    let cache = Rc::new(RefCell::new(MemoryCache::new()));
-    let mut b = Builder::new(std::env::current_dir().unwrap(), cache);
-    b.build_output
-        .entry(value_node!("tpl1".to_string(), 1, 0))
-        .or_insert(Rc::new(Val::Tuple(vec![(
-            value_node!("fld1".to_string(), 1, 0),
-            Rc::new(Val::Int(1)),
-        )])));
-    test_expr_to_val(
-        vec![
-            (
-                Expression::Copy(CopyDef {
-                    selector: make_selector!(make_expr!("tpl1")),
-                    fields: vec![(
-                        make_tok!("fld2", 1, 1),
-                        Expression::Simple(Value::Str(value_node!("2".to_string(), 1, 1))),
-                    )],
-                    pos: Position::new(1, 0),
-                }),
-                // Add a new field to the copy
-                Val::Tuple(
-                    // NOTE(jwall): The order of these is important in order to ensure
-                    // that the compare assertion is correct. The ordering has no
-                    // semantics though so at some point we should probably be less restrictive.
-                    vec![
-                        (value_node!("fld1".to_string(), 1, 0), Rc::new(Val::Int(1))),
-                        (
-                            value_node!("fld2".to_string(), 1, 1),
-                            Rc::new(Val::Str("2".to_string())),
-                        ),
-                    ],
                 ),
-            ),
+                fields: vec![(
-            // Overwrite a field in the copy
+                    make_tok!("fld1", Position::new(1, 1, 1)),
-            (
+                    Expression::Simple(Value::Str(value_node!(
-                Expression::Copy(CopyDef {
+                        "2".to_string(),
-                    selector: make_selector!(make_expr!("tpl1")),
+                        Position::new(1, 1, 1)
-                    fields: vec![
+                    ))),
-                        (
+                )],
-                            make_tok!("fld1", 1, 1),
+                pos: Position::new(1, 0, 0),
-                            Expression::Simple(Value::Int(value_node!(3, 1, 1))),
-                        ),
-                        (
-                            make_tok!("fld2", 1, 1),
-                            Expression::Simple(Value::Str(value_node!("2".to_string(), 1, 1))),
-                        ),
-                    ],
-                    pos: Position::new(1, 0),
-                }),
-                Val::Tuple(vec![
-                    (value_node!("fld1".to_string(), 1, 0), Rc::new(Val::Int(3))),
-                    (
-                        value_node!("fld2".to_string(), 1, 0),
-                        Rc::new(Val::Str("2".to_string())),
-                    ),
-                ]),
-            ),
-            // The source tuple is still unmodified.
-            (
-                Expression::Simple(Value::Selector(make_selector!(make_expr!["tpl1"]))),
-                Val::Tuple(vec![(
-                    value_node!("fld1".to_string(), 1, 0),
-                    Rc::new(Val::Int(1)),
-                )]),
-            ),
-        ],
-        b,
-    );
-}
-
-#[test]
-fn test_macro_call() {
-    let cache = Rc::new(RefCell::new(MemoryCache::new()));
-    let mut b = Builder::new(std::env::current_dir().unwrap(), cache);
-    b.build_output
-        .entry(value_node!("tstmac".to_string(), 1, 0))
-        .or_insert(Rc::new(Val::Macro(MacroDef {
-            argdefs: vec![value_node!("arg1".to_string(), 1, 0)],
-            fields: vec![(
-                make_tok!("foo", 1, 1),
-                Expression::Simple(Value::Symbol(value_node!("arg1".to_string(), 1, 1))),
-            )],
-            pos: Position::new(1, 0),
-        })));
-    test_expr_to_val(
-        vec![(
-            Expression::Call(CallDef {
-                macroref: make_selector!(make_expr!("tstmac")),
-                arglist: vec![Expression::Simple(Value::Str(value_node!(
-                    "bar".to_string(),
-                    1,
-                    1
-                )))],
-                pos: Position::new(1, 0),
             }),
             Val::Tuple(vec![(
-                value_node!("foo".to_string(), 1, 1),
+                value_node!("fld1".to_string(), Position::new(1, 1, 1)),
-                Rc::new(Val::Str("bar".to_string())),
+                Rc::new(Val::Str("2".to_string())),
             )]),
         )],
         b,
@@ -681,31 +226,36 @@ fn test_macro_hermetic() {
     let cache = Rc::new(RefCell::new(MemoryCache::new()));
     let mut b = Builder::new(std::env::current_dir().unwrap(), cache);
     b.build_output
-        .entry(value_node!("arg1".to_string(), 1, 0))
+        .entry(value_node!("arg1".to_string(), Position::new(1, 0, 0)))
         .or_insert(Rc::new(Val::Str("bar".to_string())));
     b.build_output
-        .entry(value_node!("tstmac".to_string(), 1, 0))
+        .entry(value_node!("tstmac".to_string(), Position::new(1, 0, 0)))
         .or_insert(Rc::new(Val::Macro(MacroDef {
-            argdefs: vec![value_node!("arg2".to_string(), 1, 0)],
+            argdefs: vec![value_node!("arg2".to_string(), Position::new(1, 0, 0))],
             fields: vec![(
-                make_tok!("foo", 1, 1),
+                make_tok!("foo", Position::new(1, 1, 1)),
-                Expression::Simple(Value::Symbol(value_node!("arg1".to_string(), 1, 1))),
+                Expression::Simple(Value::Symbol(value_node!(
+                    "arg1".to_string(),
+                    Position::new(1, 1, 1)
+                ))),
             )],
-            pos: Position::new(1, 0),
+            pos: Position::new(1, 0, 0),
         })));
     test_expr_to_val(
         vec![(
             Expression::Call(CallDef {
-                macroref: make_selector!(make_expr!("tstmac")),
+                macroref: make_selector!(
+                    make_expr!("tstmac", Position::new(1, 1, 1)),
+                    Position::new(1, 1, 1)
+                ),
                 arglist: vec![Expression::Simple(Value::Str(value_node!(
                     "bar".to_string(),
-                    1,
+                    Position::new(1, 1, 1)
-                    1
                 )))],
-                pos: Position::new(1, 1),
+                pos: Position::new(1, 1, 1),
             }),
             Val::Tuple(vec![(
-                value_node!("foo".to_string(), 1, 0),
+                value_node!("foo".to_string(), Position::new(1, 1, 1)),
                 Rc::new(Val::Str("bar".to_string())),
             )]),
         )],
@@ -713,126 +263,42 @@ fn test_macro_hermetic() {
     );
 }
 
-#[test]
-fn test_select_expr() {
-    let cache = Rc::new(RefCell::new(MemoryCache::new()));
-    let mut b = Builder::new(std::env::current_dir().unwrap(), cache);
-    b.build_output
-        .entry(value_node!("foo".to_string(), 1, 0))
-        .or_insert(Rc::new(Val::Str("bar".to_string())));
-    b.build_output
-        .entry(value_node!("baz".to_string(), 1, 0))
-        .or_insert(Rc::new(Val::Str("boo".to_string())));
-    test_expr_to_val(
-        vec![
-            (
-                Expression::Select(SelectDef {
-                    val: Box::new(Expression::Simple(Value::Symbol(value_node!(
-                        "foo".to_string(),
-                        1,
-                        1
-                    )))),
-                    default: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
-                    tuple: vec![
-                        (
-                            make_tok!("foo", 1, 1),
-                            Expression::Simple(Value::Str(value_node!("2".to_string(), 1, 1))),
-                        ),
-                        (
-                            make_tok!("bar", 1, 1),
-                            Expression::Simple(Value::Int(value_node!(2, 1, 1))),
-                        ),
-                    ],
-                    pos: Position::new(1, 0),
-                }),
-                Val::Int(2),
-            ),
-            (
-                Expression::Select(SelectDef {
-                    val: Box::new(Expression::Simple(Value::Symbol(value_node!(
-                        "baz".to_string(),
-                        1,
-                        1
-                    )))),
-                    default: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
-                    tuple: vec![
-                        (
-                            make_tok!("bar", 1, 1),
-                            Expression::Simple(Value::Int(value_node!(2, 1, 1))),
-                        ),
-                        (
-                            make_tok!("quux", 1, 1),
-                            Expression::Simple(Value::Str(value_node!("2".to_string(), 1, 1))),
-                        ),
-                    ],
-                    pos: Position::new(1, 0),
-                }),
-                // If the field doesn't exist then we get the default.
-                Val::Int(1),
-            ),
-        ],
-        b,
-    );
-}
-
 #[test]
 #[should_panic(expected = "Expected String but got Integer in Select expression")]
 fn test_select_expr_not_a_string() {
     let cache = Rc::new(RefCell::new(MemoryCache::new()));
     let mut b = Builder::new(std::env::current_dir().unwrap(), cache);
     b.build_output
-        .entry(value_node!("foo".to_string(), 1, 0))
+        .entry(value_node!("foo".to_string(), Position::new(1, 0, 0)))
         .or_insert(Rc::new(Val::Int(4)));
     test_expr_to_val(
         vec![(
             Expression::Select(SelectDef {
                 val: Box::new(Expression::Simple(Value::Symbol(value_node!(
                     "foo".to_string(),
-                    1,
+                    Position::new(1, 1, 1)
-                    1
+                )))),
+                default: Box::new(Expression::Simple(Value::Int(value_node!(
+                    1,
+                    Position::new(1, 1, 1)
                 )))),
-                default: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
                 tuple: vec![
                     (
-                        make_tok!("bar", 1, 1),
+                        make_tok!("bar", Position::new(1, 1, 1)),
-                        Expression::Simple(Value::Int(value_node!(2, 1, 1))),
+                        Expression::Simple(Value::Int(value_node!(2, Position::new(1, 1, 1)))),
                     ),
                     (
-                        make_tok!("quux", 1, 1),
+                        make_tok!("quux", Position::new(1, 1, 1)),
-                        Expression::Simple(Value::Str(value_node!("2".to_string(), 1, 1))),
+                        Expression::Simple(Value::Str(value_node!(
+                            "2".to_string(),
+                            Position::new(1, 1, 1)
+                        ))),
                     ),
                 ],
-                pos: Position::new(1, 0),
+                pos: Position::new(1, 0, 0),
             }),
             Val::Int(2),
         )],
         b,
     );
 }
-
-#[test]
-fn test_let_statement() {
-    let cache = MemoryCache::new();
-    let mut b = Builder::new("<Eval>", Rc::new(RefCell::new(cache)));
-    let stmt = Statement::Let(LetDef {
-        name: make_tok!("foo", 1, 1),
-        value: Expression::Simple(Value::Str(value_node!("bar".to_string(), 1, 1))),
-    });
-    b.build_stmt(&stmt).unwrap();
-    test_expr_to_val(
-        vec![(
-            Expression::Simple(Value::Symbol(value_node!("foo".to_string(), 1, 1))),
-            Val::Str("bar".to_string()),
-        )],
-        b,
-    );
-}
-
-#[test]
-fn test_build_file_string() {
-    let cache = Rc::new(RefCell::new(MemoryCache::new()));
-    let mut b = Builder::new(std::env::current_dir().unwrap(), cache);
-    b.eval_string("let foo = 1;").unwrap();
-    let key = value_node!("foo".to_string(), 1, 0);
-    assert!(b.build_output.contains_key(&key));
-}

src/convert/exec.rs

@@ -46,7 +46,7 @@ impl ExecConverter {
                 return Err(Box::new(Error::new(
                     "Exec tuples must have no more than 3 fields",
                     ErrorType::TypeFail,
-                    Position::new(0, 0),
+                    Position::new(0, 0, 0),
                 )));
             }
             let mut env: Option<&Vec<(Positioned<String>, Rc<Val>)>> = None;
@@ -115,7 +115,7 @@ impl ExecConverter {
                 return Err(Box::new(Error::new(
                     "An exec tuple must have a command field",
                     ErrorType::TypeFail,
-                    Position::new(0, 0),
+                    Position::new(0, 0, 0),
                 )));
             }
             // Okay if we have made it this far then we are ready to start creating our script.
@@ -157,7 +157,7 @@ impl ExecConverter {
                             return Err(Box::new(Error::new(
                                 "Exec args must be a list of strings or tuples of strings.",
                                 ErrorType::TypeFail,
-                                Position::new(0, 0),
+                                Position::new(0, 0, 0),
                             )))
                         }
                     }
@@ -173,7 +173,7 @@ impl ExecConverter {
         Err(Box::new(Error::new(
             "Exec outputs must be of type Tuple",
             ErrorType::TypeFail,
-            Position::new(0, 0),
+            Position::new(0, 0, 0),
         )))
     }
 }

src/error.rs

@@ -80,7 +80,8 @@ impl Error {
         t: ErrorType,
         cause: Box<error::Error>,
     ) -> Self {
-        let mut e = Self::new(msg, t, Position { line: 0, column: 0 });
+        // TODO(jwall): This should take a real position instead of this fake one.
+        let mut e = Self::new(msg, t, Position::new(0, 0, 0));
         e.cause = Some(cause);
         return e;
     }

src/format.rs

@@ -82,21 +82,21 @@ mod test {
     #[test]
     fn test_format_happy_path() {
         let formatter = Formatter::new("foo @ @ \\@", vec!["bar", "quux"]);
-        let pos = Position { line: 0, column: 0 };
+        let pos = Position::new(0, 0, 0);
         assert_eq!(formatter.render(&pos).unwrap(), "foo bar quux @");
     }
 
     #[test]
     fn test_format_happy_wrong_too_few_args() {
         let formatter = Formatter::new("foo @ @ \\@", vec!["bar"]);
-        let pos = Position { line: 0, column: 0 };
+        let pos = Position::new(0, 0, 0);
         assert!(formatter.render(&pos).is_err());
     }
 
     #[test]
     fn test_format_happy_wrong_too_many_args() {
         let formatter = Formatter::new("foo @ @ \\@", vec!["bar", "quux", "baz"]);
-        let pos = Position { line: 0, column: 0 };
+        let pos = Position::new(0, 0, 0);
         assert!(formatter.render(&pos).is_err());
     }
 }

src/iter.rs

@@ -0,0 +1,103 @@
+use std::convert::From;
+use std::iter::Iterator;
+
+use abortable_parser::iter::StrIter;
+use abortable_parser::{
+    InputIter, Offsetable, Peekable, Seekable, Span, SpanRange, TextPositionTracker,
+};
+
+#[derive(Debug)]
+pub struct OffsetStrIter<'a> {
+    contained: StrIter<'a>,
+    idx_offset: usize,
+    line_offset: usize,
+    col_offset: usize,
+}
+
+impl<'a> OffsetStrIter<'a> {
+    pub fn new(input: &'a str) -> Self {
+        Self::new_with_offsets(input, 0, 0, 0)
+    }
+
+    pub fn new_with_offsets(
+        input: &'a str,
+        idx_offset: usize,
+        line_offset: usize,
+        col_offset: usize,
+    ) -> Self {
+        OffsetStrIter {
+            contained: StrIter::new(input),
+            idx_offset: idx_offset,
+            line_offset: line_offset,
+            col_offset: col_offset,
+        }
+    }
+}
+
+impl<'a> Iterator for OffsetStrIter<'a> {
+    type Item = &'a u8;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        self.contained.next()
+    }
+}
+
+impl<'a> Offsetable for OffsetStrIter<'a> {
+    fn get_offset(&self) -> usize {
+        self.contained.get_offset() + self.idx_offset
+    }
+}
+
+impl<'a> Clone for OffsetStrIter<'a> {
+    fn clone(&self) -> Self {
+        OffsetStrIter {
+            contained: self.contained.clone(),
+            idx_offset: self.idx_offset,
+            line_offset: self.line_offset,
+            col_offset: self.col_offset,
+        }
+    }
+}
+
+impl<'a> From<&'a str> for OffsetStrIter<'a> {
+    fn from(source: &'a str) -> Self {
+        OffsetStrIter {
+            contained: StrIter::new(source),
+            idx_offset: 0,
+            line_offset: 0,
+            col_offset: 0,
+        }
+    }
+}
+
+impl<'a> Seekable for OffsetStrIter<'a> {
+    fn seek(&mut self, to: usize) -> usize {
+        let contained_offset = self.contained.seek(to);
+        self.idx_offset += contained_offset;
+        contained_offset
+    }
+}
+
+impl<'a> Span<&'a str> for OffsetStrIter<'a> {
+    fn span(&self, idx: SpanRange) -> &'a str {
+        self.contained.span(idx)
+    }
+}
+
+impl<'a> Peekable<&'a u8> for OffsetStrIter<'a> {
+    fn peek_next(&self) -> Option<&'a u8> {
+        self.contained.peek_next()
+    }
+}
+
+impl<'a> TextPositionTracker for OffsetStrIter<'a> {
+    fn line(&self) -> usize {
+        self.contained.line() + self.line_offset
+    }
+
+    fn column(&self) -> usize {
+        self.contained.column() + self.col_offset
+    }
+}
+
+impl<'a> InputIter for OffsetStrIter<'a> {}

src/lib.rs

@@ -456,6 +456,7 @@ pub mod tokenizer;
 pub mod build;
 pub mod convert;
 pub mod error;
+pub mod iter;
 pub mod parse;
 
 mod format;

src/parse/mod.rs

@@ -19,16 +19,16 @@ use std::str::FromStr;
 
 use abortable_parser;
 use abortable_parser::combinators::eoi;
-use abortable_parser::iter::{SliceIter, StrIter};
+use abortable_parser::iter::SliceIter;
 use abortable_parser::{Error, Peekable, Result};
 
 use self::precedence::op_expression;
 use ast::*;
+use iter::OffsetStrIter;
 use tokenizer::*;
 
 type NomResult<'a, O> = Result<SliceIter<'a, Token>, O>;
 
-// FIXME(jwall): All the do_each mappers need to return an actual value.
 #[cfg(feature = "tracing")]
 const ENABLE_TRACE: bool = true;
 #[cfg(not(feature = "tracing"))]
@@ -92,11 +92,10 @@ make_fn!(
     match_type!(STR => str_to_value)
 );
 
-// FIXME(jwall): We need to just turn this into a custom parser function.
 // Helper function to make the return types work for down below.
 fn triple_to_number(v: (Option<Token>, Option<Token>, Option<Token>)) -> ParseResult<Value> {
     let (pref, mut pref_pos) = match v.0 {
-        None => ("", Position::new(0, 0)),
+        None => ("", Position::new(0, 0, 0)),
         Some(ref bs) => (bs.fragment.borrow(), bs.pos.clone()),
     };
 
@@ -106,11 +105,7 @@ fn triple_to_number(v: (Option<Token>, Option<Token>, Option<Token>)) -> ParseRe
         let i = match FromStr::from_str(pref) {
             Ok(i) => i,
             Err(_) => {
-                return Err(Error::new(
+                return Err(Error::new(format!("Not an integer! {}", pref), &0));
-                    format!("Not an integer! {}", pref),
-                    // FIXME(jwall): This really needs the correct offset.
-                    &0,
-                ));
             }
         };
         return Ok(Value::Int(value_node!(i, pref_pos)));
@@ -120,9 +115,9 @@ fn triple_to_number(v: (Option<Token>, Option<Token>, Option<Token>)) -> ParseRe
         pref_pos = v.1.unwrap().pos;
     }
 
-    let suf = match v.2 {
+    let (suf, pos) = match v.2 {
-        None => "".to_string(),
+        None => ("".to_string(), Position::new(0, 0, 0)),
-        Some(bs) => bs.fragment,
+        Some(bs) => (bs.fragment, bs.pos),
     };
 
     let to_parse = pref.to_string() + "." + &suf;
@@ -131,8 +126,7 @@ fn triple_to_number(v: (Option<Token>, Option<Token>, Option<Token>)) -> ParseRe
         Err(_) => {
             return Err(Error::new(
                 format!("Not a float! {}", to_parse),
-                // FIXME(jwall): This should take the real offset.
+                &pos.offset,
-                &0,
             ));
         }
     };
@@ -307,11 +301,7 @@ make_fn!(
 
 // Helper function to make the return types work for down below.
 fn vec_to_tuple(pos: Position, fields: Option<FieldList>) -> Value {
-    Value::Tuple(value_node!(
+    Value::Tuple(value_node!(fields.unwrap_or(Vec::new()), pos))
-        fields.unwrap_or(Vec::new()),
-        pos.line as usize,
-        pos.column as usize
-    ))
 }
 
 make_fn!(
@@ -519,11 +509,10 @@ make_fn!(
         fields => optional!(trace_nom!(field_list)),
         _ => optional!(punct!(",")), // noms opt! macro does not preserve error types properly but this one does.
         _ => punct!("}"),
-        (tuple_to_copy(SelectorDef::new(selector, pos.line, pos.column), fields))
+        (tuple_to_copy(SelectorDef::new(selector, pos), fields))
     )
 );
 
-// FIXME(jwall): need to make this into a proper parse function.
 fn tuple_to_macro(pos: Position, vals: Option<Vec<Value>>, val: Value) -> ParseResult<Expression> {
     let mut default_args = match vals {
         None => Vec::new(),
@@ -543,8 +532,7 @@ fn tuple_to_macro(pos: Position, vals: Option<Vec<Value>>, val: Value) -> ParseR
         })),
         val => Err(Error::new(
             format!("Expected Tuple Got {:?}", val),
-            // FIXME(jwall): Should have correct Offset here.
+            &val.pos().offset,
-            &0,
         )),
     }
 }
@@ -647,7 +635,6 @@ make_fn!(
     )
 );
 
-// FIXME(jwall): Convert this into a custom parser function.
 fn tuple_to_call(pos: Position, val: Value, exprs: Vec<Expression>) -> ParseResult<Expression> {
     if let Value::Selector(def) = val {
         Ok(Expression::Call(CallDef {
@@ -656,17 +643,15 @@ fn tuple_to_call(pos: Position, val: Value, exprs: Vec<Expression>) -> ParseResu
             pos: pos,
         }))
     } else {
-        // FIXME(jwall): Should get correct offset here.
+        Err(Error::new(
-        Err(Error::new(format!("Expected Selector Got {:?}", val), &0))
+            format!("Expected Selector Got {:?}", val),
+            &val.pos().offset,
+        ))
     }
 }
 
 fn vec_to_selector_value(pos: Position, list: SelectorList) -> Value {
-    Value::Selector(SelectorDef::new(
+    Value::Selector(SelectorDef::new(list, pos))
-        list,
-        pos.line as usize,
-        pos.column as usize,
-    ))
 }
 
 make_fn!(
@@ -715,8 +700,7 @@ fn tuple_to_list_op(
                 }
                 return Err(Error::new(
                     format!("Missing a result field for the macro"),
-                    // FIXME(jwall): Should have correct offset.
+                    &def.pos.offset,
-                    &0,
                 ));
             }
             &mut Some(ref mut tl) => {
@@ -729,8 +713,7 @@ fn tuple_to_list_op(
                     }
                     return Err(Error::new(
                         format!("Missing a result field for the macro"),
-                        // FIXME(jwall): Should have correct offset.
+                        &def.pos.offset,
-                        &0,
                     ));
                 }
                 let fname = tl.pop();
@@ -753,12 +736,10 @@ fn tuple_to_list_op(
     }
     return Err(Error::new(
         format!("Expected a selector but got {}", macroname.type_name()),
-        // FIXME(jwall): Should have correct offset.
+        &pos.offset,
-        &0,
     ));
 }
 
-// FIXME(jwall): need to make this a custom function to parse it.
 make_fn!(
     list_op_expression<SliceIter<Token>, Expression>,
     do_each!(
@@ -843,7 +824,7 @@ make_fn!(
     let_statement<SliceIter<Token>, Statement>,
     do_each!(
         _ => word!("let"),
-        stmt => trace_nom!(let_stmt_body),
+        stmt => trace_nom!(must!(let_stmt_body)),
         (stmt)
     )
 );
@@ -871,7 +852,7 @@ make_fn!(
     do_each!(
         _ => word!("import"),
         // past this point we know this is supposed to be an import statement.
-        stmt => trace_nom!(import_stmt_body),
+        stmt => trace_nom!(must!(import_stmt_body)),
         (stmt)
     )
 );
@@ -880,8 +861,8 @@ make_fn!(
     assert_statement<SliceIter<Token>, Statement>,
     do_each!(
         _ => word!("assert"),
-            tok => match_type!(PIPEQUOTE),
+            tok => must!(match_type!(PIPEQUOTE)),
-            _ => punct!(";"),
+            _ => must!(punct!(";")),
             (Statement::Assert(tok.clone()))
     )
 );
@@ -890,9 +871,9 @@ make_fn!(
     out_statement<SliceIter<Token>, Statement>,
     do_each!(
         _ => word!("out"),
-        typ => match_type!(BAREWORD),
+        typ => must!(match_type!(BAREWORD)),
-        expr => expression,
+        expr => must!(expression),
-        _ => punct!(";"),
+        _ => must!(punct!(";")),
         (Statement::Output(typ.clone(), expr.clone()))
     )
 );
@@ -910,7 +891,7 @@ fn statement(i: SliceIter<Token>) -> Result<SliceIter<Token>, Statement> {
 //trace_macros!(false);
 
 /// Parses a LocatedSpan into a list of Statements or an `abortable_parser::Error`.
-pub fn parse(input: StrIter) -> std::result::Result<Vec<Statement>, Error> {
+pub fn parse(input: OffsetStrIter) -> std::result::Result<Vec<Statement>, Error> {
     match tokenize(input.clone()) {
         Ok(tokenized) => {
             let mut out = Vec::new();
@@ -922,6 +903,7 @@ pub fn parse(input: StrIter) -> std::result::Result<Vec<Statement>, Error> {
                         break;
                     }
                 }
+                // FIXME(jwall): We need to return a error::Error so we have position information.
                 match statement(i.clone()) {
                     Result::Abort(e) => {
                         return Err(e);

src/tokenizer/mod.rs

@@ -16,20 +16,19 @@
 use std;
 
 use abortable_parser::combinators::*;
-use abortable_parser::iter::{SliceIter, StrIter};
+use abortable_parser::iter::SliceIter;
 use abortable_parser::{Error, Offsetable, Result, TextPositionTracker};
-use ast::*;
 
-impl<'a> From<StrIter<'a>> for Position {
+use ast::*;
-    fn from(s: StrIter<'a>) -> Position {
+use iter::OffsetStrIter;
-        Position {
+
-            line: s.line(),
+impl<'a> From<OffsetStrIter<'a>> for Position {
-            column: s.column(),
+    fn from(s: OffsetStrIter<'a>) -> Position {
-        }
+        Position::new(s.line(), s.column(), s.get_offset())
     }
 }
 
-fn is_symbol_char<'a>(i: StrIter<'a>) -> Result<StrIter<'a>, u8> {
+fn is_symbol_char<'a>(i: OffsetStrIter<'a>) -> Result<OffsetStrIter<'a>, u8> {
     let mut _i = i.clone();
     let c = match _i.next() {
         Some(c) => *c,
@@ -42,7 +41,7 @@ fn is_symbol_char<'a>(i: StrIter<'a>) -> Result<StrIter<'a>, u8> {
     }
 }
 
-fn escapequoted<'a>(input: StrIter<'a>) -> Result<StrIter<'a>, String> {
+fn escapequoted<'a>(input: OffsetStrIter<'a>) -> Result<OffsetStrIter<'a>, String> {
     // loop until we find a " that is not preceded by \.
     // Collapse all \<char> to just char  for escaping.
     let mut frag = String::new();
@@ -69,7 +68,7 @@ fn escapequoted<'a>(input: StrIter<'a>) -> Result<StrIter<'a>, String> {
     return Result::Incomplete(_input.get_offset());
 }
 
-make_fn!(strtok<StrIter, Token>,
+make_fn!(strtok<OffsetStrIter, Token>,
        do_each!(
            span => input!(),
            _    => text_token!("\""),
@@ -82,7 +81,7 @@ make_fn!(strtok<StrIter, Token>,
        )
 );
 
-make_fn!(pipequotetok<StrIter, Token>,
+make_fn!(pipequotetok<OffsetStrIter, Token>,
        do_each!(
            p    => input!(),
            _    => text_token!("|"),
@@ -96,7 +95,7 @@ make_fn!(pipequotetok<StrIter, Token>,
        )
 );
 
-make_fn!(barewordtok<StrIter, Token>,
+make_fn!(barewordtok<OffsetStrIter, Token>,
        do_each!(
            span => input!(),
            _    => peek!(ascii_alpha),
@@ -109,7 +108,7 @@ make_fn!(barewordtok<StrIter, Token>,
        )
 );
 
-make_fn!(digittok<StrIter, Token>,
+make_fn!(digittok<OffsetStrIter, Token>,
        do_each!(
            span => input!(),
            _ => peek!(ascii_digit),
@@ -122,7 +121,7 @@ make_fn!(digittok<StrIter, Token>,
        )
 );
 
-make_fn!(booleantok<StrIter, Token>,
+make_fn!(booleantok<OffsetStrIter, Token>,
     do_each!(
         span => input!(),
         token => either!(
@@ -142,159 +141,159 @@ make_fn!(booleantok<StrIter, Token>,
 macro_rules! do_text_token_tok {
     ($i:expr, $type:expr, $text_token:expr, WS) => {
         do_each!($i,
-                            span => input!(),
+                                                            span => input!(),
-                            frag => text_token!($text_token),
+                                                            frag => text_token!($text_token),
-                            _ => either!(whitespace, comment),
+                                                            _ => either!(whitespace, comment),
-                            (Token {
+                                                            (Token {
-                                typ: $type,
+                                                                typ: $type,
-                                pos: Position::from(span),
+                                                                pos: Position::from(span),
-                                fragment: frag.to_string(),
+                                                                fragment: frag.to_string(),
-                            })
+                                                            })
-                        )
+                                                        )
     };
 
     ($i:expr, $type:expr, $text_token:expr) => {
         do_each!($i,
-                            span => input!(),
+                                                            span => input!(),
-                            frag => text_token!($text_token),
+                                                            frag => text_token!($text_token),
-                            (Token {
+                                                            (Token {
-                                typ: $type,
+                                                                typ: $type,
-                                pos: Position::from(span),
+                                                                pos: Position::from(span),
-                                fragment: frag.to_string(),
+                                                                fragment: frag.to_string(),
-                            })
+                                                            })
-                        )
+                                                        )
     };
 }
 
-make_fn!(emptytok<StrIter, Token>,
+make_fn!(emptytok<OffsetStrIter, Token>,
        do_text_token_tok!(TokenType::EMPTY, "NULL")
 );
 
-make_fn!(commatok<StrIter, Token>,
+make_fn!(commatok<OffsetStrIter, Token>,
        do_text_token_tok!(TokenType::PUNCT, ",")
 );
 
-make_fn!(lbracetok<StrIter, Token>,
+make_fn!(lbracetok<OffsetStrIter, Token>,
        do_text_token_tok!(TokenType::PUNCT, "{")
 );
 
-make_fn!(rbracetok<StrIter, Token>,
+make_fn!(rbracetok<OffsetStrIter, Token>,
        do_text_token_tok!(TokenType::PUNCT, "}")
 );
 
-make_fn!(lparentok<StrIter, Token>,
+make_fn!(lparentok<OffsetStrIter, Token>,
        do_text_token_tok!(TokenType::PUNCT, "(")
 );
 
-make_fn!(rparentok<StrIter, Token>,
+make_fn!(rparentok<OffsetStrIter, Token>,
        do_text_token_tok!(TokenType::PUNCT, ")")
 );
 
-make_fn!(dottok<StrIter, Token>,
+make_fn!(dottok<OffsetStrIter, Token>,
        do_text_token_tok!(TokenType::PUNCT, ".")
 );
 
-make_fn!(plustok<StrIter, Token>,
+make_fn!(plustok<OffsetStrIter, Token>,
        do_text_token_tok!(TokenType::PUNCT, "+")
 );
 
-make_fn!(dashtok<StrIter, Token>,
+make_fn!(dashtok<OffsetStrIter, Token>,
        do_text_token_tok!(TokenType::PUNCT, "-")
 );
 
-make_fn!(startok<StrIter, Token>,
+make_fn!(startok<OffsetStrIter, Token>,
        do_text_token_tok!(TokenType::PUNCT, "*")
 );
 
-make_fn!(slashtok<StrIter, Token>,
+make_fn!(slashtok<OffsetStrIter, Token>,
        do_text_token_tok!(TokenType::PUNCT, "/")
 );
 
-make_fn!(pcttok<StrIter, Token>,
+make_fn!(pcttok<OffsetStrIter, Token>,
        do_text_token_tok!(TokenType::PUNCT, "%")
 );
 
-make_fn!(eqeqtok<StrIter, Token>,
+make_fn!(eqeqtok<OffsetStrIter, Token>,
        do_text_token_tok!(TokenType::PUNCT, "==")
 );
 
-make_fn!(notequaltok<StrIter, Token>,
+make_fn!(notequaltok<OffsetStrIter, Token>,
        do_text_token_tok!(TokenType::PUNCT, "!=")
 );
 
-make_fn!(gttok<StrIter, Token>,
+make_fn!(gttok<OffsetStrIter, Token>,
        do_text_token_tok!(TokenType::PUNCT, ">")
 );
 
-make_fn!(gtequaltok<StrIter, Token>,
+make_fn!(gtequaltok<OffsetStrIter, Token>,
        do_text_token_tok!(TokenType::PUNCT, ">=")
 );
 
-make_fn!(ltequaltok<StrIter, Token>,
+make_fn!(ltequaltok<OffsetStrIter, Token>,
        do_text_token_tok!(TokenType::PUNCT, "<=")
 );
 
-make_fn!(lttok<StrIter, Token>,
+make_fn!(lttok<OffsetStrIter, Token>,
        do_text_token_tok!(TokenType::PUNCT, "<")
 );
 
-make_fn!(equaltok<StrIter, Token>,
+make_fn!(equaltok<OffsetStrIter, Token>,
        do_text_token_tok!(TokenType::PUNCT, "=")
 );
 
-make_fn!(semicolontok<StrIter, Token>,
+make_fn!(semicolontok<OffsetStrIter, Token>,
        do_text_token_tok!(TokenType::PUNCT, ";")
 );
 
-make_fn!(leftsquarebracket<StrIter, Token>,
+make_fn!(leftsquarebracket<OffsetStrIter, Token>,
     do_text_token_tok!(TokenType::PUNCT, "[")
 );
 
-make_fn!(rightsquarebracket<StrIter, Token>,
+make_fn!(rightsquarebracket<OffsetStrIter, Token>,
     do_text_token_tok!(TokenType::PUNCT, "]")
 );
 
-make_fn!(fatcommatok<StrIter, Token>,
+make_fn!(fatcommatok<OffsetStrIter, Token>,
        do_text_token_tok!(TokenType::PUNCT, "=>")
 );
 
-make_fn!(selecttok<StrIter, Token>,
+make_fn!(selecttok<OffsetStrIter, Token>,
        do_text_token_tok!(TokenType::BAREWORD, "select", WS)
 );
 
-make_fn!(macrotok<StrIter, Token>,
+make_fn!(macrotok<OffsetStrIter, Token>,
        do_text_token_tok!(TokenType::BAREWORD, "macro", WS)
 );
 
-make_fn!(lettok<StrIter, Token>,
+make_fn!(lettok<OffsetStrIter, Token>,
        do_text_token_tok!(TokenType::BAREWORD, "let", WS)
 );
 
-make_fn!(importtok<StrIter, Token>,
+make_fn!(importtok<OffsetStrIter, Token>,
        do_text_token_tok!(TokenType::BAREWORD, "import", WS)
 );
 
-make_fn!(asserttok<StrIter, Token>,
+make_fn!(asserttok<OffsetStrIter, Token>,
        do_text_token_tok!(TokenType::BAREWORD, "assert", WS)
 );
 
-make_fn!(outtok<StrIter, Token>,
+make_fn!(outtok<OffsetStrIter, Token>,
        do_text_token_tok!(TokenType::BAREWORD, "out", WS)
 );
 
-make_fn!(astok<StrIter, Token>,
+make_fn!(astok<OffsetStrIter, Token>,
        do_text_token_tok!(TokenType::BAREWORD, "as", WS)
 );
 
-make_fn!(maptok<StrIter, Token>,
+make_fn!(maptok<OffsetStrIter, Token>,
        do_text_token_tok!(TokenType::BAREWORD, "map", WS)
 );
 
-make_fn!(filtertok<StrIter, Token>,
+make_fn!(filtertok<OffsetStrIter, Token>,
        do_text_token_tok!(TokenType::BAREWORD, "filter", WS)
 );
 
-fn comment(input: StrIter) -> Result<StrIter, Token> {
+fn comment(input: OffsetStrIter) -> Result<OffsetStrIter, Token> {
     match text_token!(input, "//") {
         Result::Complete(rest, _) => {
             match until!(
@@ -306,12 +305,7 @@ fn comment(input: StrIter) -> Result<StrIter, Token> {
                 )
             ) {
                 Result::Complete(rest, cmt) => {
-                    return Result::Complete(
+                    return Result::Complete(rest, make_tok!(CMT => cmt.to_string(), input));
-                        rest,
-                        make_tok!(CMT => cmt.to_string(),
-                                  input.line() as usize,
-                                  input.column() as usize),
-                    );
                 }
                 // If we didn't find a new line then we just grab everything.
                 _ => {
@@ -325,7 +319,7 @@ fn comment(input: StrIter) -> Result<StrIter, Token> {
     }
 }
 
-make_fn!(whitespace<StrIter, Token>,
+make_fn!(whitespace<OffsetStrIter, Token>,
     do_each!(
         span => input!(),
         _ => peek!(ascii_ws),
@@ -338,7 +332,7 @@ make_fn!(whitespace<StrIter, Token>,
     )
 );
 
-make_fn!(end_of_input<StrIter, Token>,
+make_fn!(end_of_input<OffsetStrIter, Token>,
     do_each!(
         span => input!(),
         _ => eoi,
@@ -350,7 +344,7 @@ make_fn!(end_of_input<StrIter, Token>,
     )
 );
 
-make_fn!(token<StrIter, Token>,
+make_fn!(token<OffsetStrIter, Token>,
     either!(
         strtok,
         pipequotetok,
@@ -394,14 +388,15 @@ make_fn!(token<StrIter, Token>,
         end_of_input)
 );
 
-/// Consumes an input StrIter and returns either a Vec<Token> or a error::Error.
+/// Consumes an input OffsetStrIter and returns either a Vec<Token> or a error::Error.
-pub fn tokenize(input: StrIter) -> std::result::Result<Vec<Token>, Error> {
+pub fn tokenize(input: OffsetStrIter) -> std::result::Result<Vec<Token>, Error> {
     let mut out = Vec::new();
     let mut i = input.clone();
     loop {
         if let Result::Complete(_, _) = eoi(i.clone()) {
             break;
         }
+        // FIXME(jwall): We need to return a error::Error so we have position information.
         match token(i.clone()) {
             Result::Abort(e) => {
                 return Err(Error::caused_by(
@@ -434,10 +429,7 @@ pub fn tokenize(input: StrIter) -> std::result::Result<Vec<Token>, Error> {
     out.push(Token {
         fragment: String::new(),
         typ: TokenType::END,
-        pos: Position {
+        pos: i.into(),
-            line: i.line(),
-            column: i.column(),
-        },
     });
     Ok(out)
 }
@@ -616,13 +608,7 @@ pub fn pos<'a>(i: SliceIter<'a, Token>) -> Result<SliceIter<'a, Token>, Position
     let tok = _i.next().unwrap();
     let line = tok.pos.line;
     let column = tok.pos.column;
-    Result::Complete(
+    Result::Complete(i.clone(), Position::new(line, column, i.get_offset()))
-        i.clone(),
-        Position {
-            line: line,
-            column: column,
-        },
-    )
 }
 
 #[cfg(test)]

src/tokenizer/test.rs

@@ -1,10 +1,12 @@
 use super::*;
 
-use abortable_parser::{Result, SliceIter, StrIter};
+use abortable_parser::{Result, SliceIter};
+
+use iter::OffsetStrIter;
 
 #[test]
 fn test_empty_token() {
-    let result = emptytok(StrIter::new("NULL "));
+    let result = emptytok(OffsetStrIter::new("NULL "));
     assert!(
         result.is_complete(),
         format!("result {:?} is not done", result)
@@ -17,7 +19,7 @@ fn test_empty_token() {
 
 #[test]
 fn test_assert_token() {
-    let result = asserttok(StrIter::new("assert "));
+    let result = asserttok(OffsetStrIter::new("assert "));
     assert!(
         result.is_complete(),
         format!("result {:?} is not done", result)
@@ -30,7 +32,7 @@ fn test_assert_token() {
 
 #[test]
 fn test_out_token() {
-    let result = outtok(StrIter::new("out "));
+    let result = outtok(OffsetStrIter::new("out "));
     assert!(
         result.is_complete(),
         format!("result {:?} is not done", result)
@@ -43,7 +45,7 @@ fn test_out_token() {
 
 #[test]
 fn test_out_token_with_comment() {
-    let result = outtok(StrIter::new("out//comment"));
+    let result = outtok(OffsetStrIter::new("out//comment"));
     assert!(
         result.is_complete(),
         format!("result {:?} is not done", result)
@@ -56,13 +58,13 @@ fn test_out_token_with_comment() {
 
 #[test]
 fn test_not_out_token() {
-    let result = outtok(StrIter::new("output"));
+    let result = outtok(OffsetStrIter::new("output"));
     assert!(result.is_fail(), format!("result {:?} is not fail", result));
 }
 
 #[test]
 fn test_escape_quoted() {
-    let result = escapequoted(StrIter::new("foo \\\"bar\""));
+    let result = escapequoted(OffsetStrIter::new("foo \\\"bar\""));
     assert!(
         result.is_complete(),
         format!("result {:?} is not ok", result)
@@ -74,7 +76,7 @@ fn test_escape_quoted() {
 
 #[test]
 fn test_pipe_quoted() {
-    let result = pipequotetok(StrIter::new("|foo|"));
+    let result = pipequotetok(OffsetStrIter::new("|foo|"));
     assert!(
         result.is_complete(),
         format!("result {:?} is not ok", result)
@@ -87,7 +89,7 @@ fn test_pipe_quoted() {
 
 #[test]
 fn test_string_with_escaping() {
-    let result = strtok(StrIter::new("\"foo \\\\ \\\"bar\""));
+    let result = strtok(OffsetStrIter::new("\"foo \\\\ \\\"bar\""));
     assert!(
         result.is_complete(),
         format!("result {:?} is not ok", result)
@@ -99,7 +101,7 @@ fn test_string_with_escaping() {
 
 #[test]
 fn test_tokenize_bareword_with_dash() {
-    let result = tokenize(StrIter::new("foo-bar "));
+    let result = tokenize(OffsetStrIter::new("foo-bar "));
     assert!(result.is_ok(), format!("result {:?} is not ok", result));
     if let Ok(toks) = result {
         assert_eq!(toks.len(), 2);
@@ -109,7 +111,7 @@ fn test_tokenize_bareword_with_dash() {
 
 macro_rules! assert_token {
     ($input:expr, $typ:expr, $msg:expr) => {
-        let result = token(StrIter::new($input));
+        let result = token(OffsetStrIter::new($input));
         assert!(
             result.is_complete(),
             format!("result {:?} is not a {}", result, $msg)
@@ -163,7 +165,7 @@ fn test_lteqtok() {
 
 #[test]
 fn test_tokenize_one_of_each() {
-    let result = tokenize(StrIter::new(
+    let result = tokenize(OffsetStrIter::new(
         "map out filter assert let import macro select as => [ ] { } ; = % / * \
          + - . ( ) , 1 . foo \"bar\" // comment\n ; true false == < > <= >= !=",
     ));
@@ -178,7 +180,7 @@ fn test_tokenize_one_of_each() {
 
 #[test]
 fn test_parse_has_end() {
-    let result = tokenize(StrIter::new("foo"));
+    let result = tokenize(OffsetStrIter::new("foo"));
     assert!(result.is_ok());
     let v = result.unwrap();
     assert_eq!(v.len(), 2);
@@ -187,8 +189,8 @@ fn test_parse_has_end() {
 
 #[test]
 fn test_whitespace() {
-    assert!(whitespace(StrIter::new("    ")).is_complete());
+    assert!(whitespace(OffsetStrIter::new("    ")).is_complete());
-    let result = whitespace(StrIter::new("  "));
+    let result = whitespace(OffsetStrIter::new("  "));
     match result {
         Result::Complete(rest, o) => {
             assert_eq!(rest.get_offset(), 2);
@@ -200,9 +202,9 @@ fn test_whitespace() {
 
 #[test]
 fn test_parse_comment() {
-    assert!(comment(StrIter::new("// comment\n")).is_complete());
+    assert!(comment(OffsetStrIter::new("// comment\n")).is_complete());
-    assert!(comment(StrIter::new("// comment")).is_complete());
+    assert!(comment(OffsetStrIter::new("// comment")).is_complete());
-    let mut parsed = comment(StrIter::new("// comment\n"));
+    let mut parsed = comment(OffsetStrIter::new("// comment\n"));
     assert!(parsed.is_complete());
     if let Result::Complete(_rest, cmt) = parsed {
         assert_eq!(
@@ -210,35 +212,47 @@ fn test_parse_comment() {
             Token {
                 typ: TokenType::COMMENT,
                 fragment: " comment".to_string(),
-                pos: Position { line: 1, column: 1 },
+                pos: Position {
+                    line: 1,
+                    column: 1,
+                    offset: 0
+                },
             }
         );
     }
-    assert!(comment(StrIter::new("// comment\r\n")).is_complete());
+    assert!(comment(OffsetStrIter::new("// comment\r\n")).is_complete());
-    parsed = comment(StrIter::new("// comment\r\n"));
+    parsed = comment(OffsetStrIter::new("// comment\r\n"));
     if let Result::Complete(_rest, cmt) = parsed {
         assert_eq!(
             cmt,
             Token {
                 typ: TokenType::COMMENT,
                 fragment: " comment".to_string(),
-                pos: Position { column: 1, line: 1 },
+                pos: Position {
+                    column: 1,
+                    line: 1,
+                    offset: 0
+                },
             }
         );
     }
-    assert!(comment(StrIter::new("// comment\r\n ")).is_complete());
+    assert!(comment(OffsetStrIter::new("// comment\r\n ")).is_complete());
-    parsed = comment(StrIter::new("// comment\r\n "));
+    parsed = comment(OffsetStrIter::new("// comment\r\n "));
     if let Result::Complete(_rest, cmt) = parsed {
         assert_eq!(
             cmt,
             Token {
                 typ: TokenType::COMMENT,
                 fragment: " comment".to_string(),
-                pos: Position { column: 1, line: 1 },
+                pos: Position {
+                    column: 1,
+                    line: 1,
+                    offset: 0
+                },
             }
         );
     }
-    assert!(comment(StrIter::new("// comment")).is_complete());
+    assert!(comment(OffsetStrIter::new("// comment")).is_complete());
 }
 
 #[test]
@@ -246,7 +260,11 @@ fn test_match_word() {
     let input = vec![Token {
         fragment: "foo".to_string(),
         typ: TokenType::BAREWORD,
-        pos: Position { line: 1, column: 1 },
+        pos: Position {
+            line: 1,
+            column: 1,
+            offset: 0,
+        },
     }];
     let result = word!(SliceIter::new(input.as_slice()), "foo");
     match result {
@@ -260,7 +278,11 @@ fn test_match_word_empty_input() {
     let input = vec![Token {
         fragment: "".to_string(),
         typ: TokenType::END,
-        pos: Position { line: 1, column: 1 },
+        pos: Position {
+            line: 1,
+            column: 1,
+            offset: 0,
+        },
     }];
     let result = word!(SliceIter::new(input.as_slice()), "foo");
     match result {
@@ -278,7 +300,11 @@ fn test_match_punct() {
     let input = vec![Token {
         fragment: "!".to_string(),
         typ: TokenType::PUNCT,
-        pos: Position { line: 1, column: 1 },
+        pos: Position {
+            line: 1,
+            column: 1,
+            offset: 0,
+        },
     }];
     let result = punct!(SliceIter::new(input.as_slice()), "!");
     match result {
@@ -292,7 +318,11 @@ fn test_match_type() {
     let input = vec![Token {
         fragment: "foo".to_string(),
         typ: TokenType::BAREWORD,
-        pos: Position { line: 1, column: 1 },
+        pos: Position {
+            line: 1,
+            column: 1,
+            offset: 0,
+        },
     }];
     let result = match_type!(SliceIter::new(input.as_slice()), BAREWORD);
     match result {