The great documentation roundup.

Add docstrings for all kinds of types.

src/ast.rs

@@ -11,6 +11,8 @@
 //  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
+
+//! The definitions of the ucg AST and Tokens.
 use std;
 use std::collections::HashSet;
 use std::borrow::Borrow;
@@ -22,6 +24,7 @@ use std::cmp::PartialEq;
 use std::hash::Hasher;
 use std::hash::Hash;
 
+/// Encodes a parsing error with position information and a helpful description.
 #[derive(Debug,PartialEq)]
 pub struct ParseError {
     pub pos: Position,
@@ -61,6 +64,10 @@ macro_rules! enum_type_equality {
     }
 }
 
+/// Represents a line and a column position in UCG code.
+/// 
+/// It is used for generating error messages mostly. Most all
+/// parts of the UCG AST have a positioned associated with them.
 #[derive(Debug,PartialEq,Eq,Clone,PartialOrd,Ord,Hash)]
 pub struct Position {
     pub line: usize,
@@ -68,6 +75,7 @@ pub struct Position {
 }
 
 impl Position {
+    /// Construct a new Position.
     pub fn new(line: usize, column: usize) -> Self {
         Position {
             line: line,
@@ -76,6 +84,7 @@ impl Position {
     }
 }
 
+/// Defines the types of tokens in UCG syntax.
 #[derive(Debug,PartialEq,Eq,Clone,PartialOrd,Ord,Hash)]
 pub enum TokenType {
     END,
@@ -88,6 +97,10 @@ pub enum TokenType {
 }
 
 // FIXME(jwall): We should probably implement copy for this.
+
+/// Defines a Token representing a building block of UCG syntax.
+/// 
+/// Token's are passed to the parser stage to be parsed into an AST.
 #[derive(Debug,PartialEq,Eq,Clone,PartialOrd,Ord,Hash)]
 pub struct Token {
     pub typ: TokenType,
@@ -96,10 +109,12 @@ pub struct Token {
 }
 
 impl Token {
+    /// Constructs a new Token with a type and line and column information.
     pub fn new<S: Into<String>>(f: S, typ: TokenType, line: usize, col: usize) -> Self {
         Self::new_with_pos(f, typ, Position::new(line, col))
     }
 
+    // Constructs a new Token with a type and a Position.
     pub fn new_with_pos<S: Into<String>>(f: S, typ: TokenType, pos: Position) -> Self {
         Token {
             typ: typ,
@@ -115,6 +130,7 @@ impl Borrow<str> for Token {
     }
 }
 
+/// Helper macro for making a Positioned Value.
 macro_rules! value_node {
     ($v:expr, $p:expr) => {
         Positioned::new_with_pos($v, $p)
@@ -124,6 +140,7 @@ macro_rules! value_node {
     };
 }
 
+/// Helper macro for making a Token.
 #[allow(unused_macros)]
 macro_rules! make_tok {
     ( EOF => $l:expr, $c:expr  ) => {
@@ -155,6 +172,7 @@ macro_rules! make_tok {
     };
 }
 
+/// Helper macro for making expressions.
 #[allow(unused_macros)]
 macro_rules! make_expr {
     ( $e:expr ) => {
@@ -248,7 +266,7 @@ macro_rules! make_selector {
                 col += $item.len() + 1;
             )*
 
-            // Shut up the lint about unused code;
+            // Shut up the linter about unused code;
             assert!(col != 0);
 
             make_selector!($h, list, $l, $c) 
@@ -256,7 +274,7 @@ macro_rules! make_selector {
     };
 }
 
-/// Selector is an Expression with a series of symbols specifying the key
+/// An Expression with a series of symbols specifying the key
 /// with which to descend into the result of the expression.
 ///
 /// The expression must evaluate to either a tuple or an array. The token must
@@ -279,13 +297,18 @@ pub struct SelectorList {
 }
 
 impl SelectorList {
+    /// Returns a stringified version of a SelectorList.
     pub fn to_string(&self) -> String {
         "TODO".to_string()
     }
 }
 
+/// An ordered list of Name = Value pairs.
+/// 
+/// This is usually used as the body of a tuple in the UCG AST.
 pub type FieldList = Vec<(Token, Expression)>; // Token is expected to be a symbol
 
+/// Encodes a selector expression in the UCG AST.
 #[derive(Debug,PartialEq,Clone)]
 pub struct SelectorDef {
     pub pos: Position,
@@ -293,6 +316,7 @@ pub struct SelectorDef {
 }
 
 impl SelectorDef {
+    /// Constructs a new SelectorDef.
     pub fn new(sel: SelectorList, line: usize, col: usize) -> Self {
         SelectorDef {
             pos: Position::new(line, col),
@@ -301,7 +325,7 @@ impl SelectorDef {
     }
 }
 
-/// Value represents a Value in the UCG parsed AST.
+/// Represents a Value in the UCG parsed AST.
 #[derive(Debug,PartialEq,Clone)]
 pub enum Value {
     // Constant Values
@@ -316,6 +340,7 @@ pub enum Value {
 }
 
 impl Value {
+    /// Returns the type name of the Value it is called on as a string.
     pub fn type_name(&self) -> String {
         match self {
             &Value::Int(_) => "Integer".to_string(),
@@ -344,6 +369,7 @@ impl Value {
         return format!("{}", v.len());
     }
 
+    /// Returns a stringified version of the Value.
     pub fn to_string(&self) -> String {
         match self {
             &Value::Int(ref i) => format!("{}", i.val),
@@ -356,6 +382,7 @@ impl Value {
         }
     }
 
+    /// Returns the position for a Value.
     pub fn pos(&self) -> &Position {
         match self {
             &Value::Int(ref i) => &i.pos,
@@ -368,6 +395,7 @@ impl Value {
         }
     }
 
+    /// Returns true if called on a Value that is the same type as itself.
     pub fn type_equal(&self, target: &Self) -> bool {
         enum_type_equality!(self, target, &Value::Int(_),
                                           &Value::Float(_),
@@ -379,7 +407,7 @@ impl Value {
     }
 }
 
-/// CallDef represents a call to a Macro that is expected to already have been
+/// Represents an expansion of a Macro that is expected to already have been
 /// defined.
 #[derive(PartialEq,Debug,Clone)]
 pub struct CallDef {
@@ -388,8 +416,7 @@ pub struct CallDef {
     pub pos: Position,
 }
 
-/// SelectDef selects a value from a tuple with a default if the value doesn't
+/// Encodes a select expression in the UCG AST.
-/// exist.
 #[derive(PartialEq,Debug,Clone)]
 pub struct SelectDef {
     pub val: Box<Expression>,
@@ -399,6 +426,8 @@ pub struct SelectDef {
 }
 
 // TODO(jwall): This should have a way of rendering with position information.
+
+/// Adds position information to any type `T`.
 #[derive(Debug,Clone)]
 pub struct Positioned<T> {
     pub pos: Position,
@@ -406,10 +435,12 @@ pub struct Positioned<T> {
 }
 
 impl<T> Positioned<T> {
+    /// Constructs a new Positioned<T> with a value, line, and column information.
     pub fn new(v: T, l: usize, c: usize) -> Self {
         Self::new_with_pos(v, Position::new(l, c))
     }
 
+    /// Constructs a new Positioned<T> with a value and a Position.
     pub fn new_with_pos(v: T, pos: Position) -> Self {
         Positioned { pos: pos, val: v }
     }
@@ -459,9 +490,10 @@ impl<'a> From<&'a Positioned<String>> for Positioned<String> {
     }
 }
 
-/// MacroDef is a pure function that always returns a Tuple.
+/// Encodes a macro expression in the UCG AST..
 ///
-/// MacroDef's are not closures. They can not reference
+/// A macro is a pure function over a tuple.
+/// MacroDefs are not closures. They can not reference
 /// any values except what is defined in their arguments.
 #[derive(PartialEq,Debug,Clone)]
 pub struct MacroDef {
@@ -504,6 +536,8 @@ impl MacroDef {
         return bad_symbols;
     }
 
+    /// Performs typechecking of a ucg macro's arguments to ensure
+    /// that they are valid for the expressions in the macro.
     pub fn validate_symbols(&self) -> Result<(), HashSet<String>> {
         let mut bad_symbols = HashSet::new();
         for &(_, ref expr) in self.fields.iter() {
@@ -561,6 +595,8 @@ impl MacroDef {
     }
 }
 
+/// Specifies the types of binary operations supported in
+/// UCG expression.
 #[derive(Debug,PartialEq,Clone)]
 pub enum BinaryExprType {
     Add,
@@ -569,7 +605,7 @@ pub enum BinaryExprType {
     Div,
 }
 
-/// BinaryOpDef represents an expression with a left and a right side.
+/// Represents an expression with a left and a right side.
 #[derive(Debug,PartialEq,Clone)]
 pub struct BinaryOpDef {
     pub kind: BinaryExprType,
@@ -578,6 +614,7 @@ pub struct BinaryOpDef {
     pub pos: Position,
 }
 
+/// Encodes a tuple Copy expression in the UCG AST.
 #[derive(Debug,PartialEq,Clone)]
 pub struct CopyDef {
     pub selector: SelectorDef,
@@ -585,6 +622,7 @@ pub struct CopyDef {
     pub pos: Position,
 }
 
+/// Encodes a format expression in the UCG AST.
 #[derive(Debug,PartialEq,Clone)]
 pub struct FormatDef {
     pub template: String,
@@ -592,33 +630,33 @@ pub struct FormatDef {
     pub pos: Position,
 }
 
+/// Encodes a list expression in the UCG AST.
 #[derive(Debug,PartialEq,Clone)]
 pub struct ListDef {
     pub elems: Vec<Expression>,
     pub pos: Position,
 }
 
-/// Expression encodes an expression. Expressions compute a value from operands.
+/// Encodes a ucg expression. Expressions compute a value from.
 #[derive(Debug,PartialEq,Clone)]
 pub enum Expression {
     // Base Expression
     Simple(Value),
 
+    // Binary expressions
     Binary(BinaryOpDef),
 
     // Complex Expressions
     Copy(CopyDef),
     Grouped(Box<Expression>),
-
     Format(FormatDef),
-
     Call(CallDef),
-
     Macro(MacroDef),
     Select(SelectDef),
 }
 
 impl Expression {
+    /// Returns the position of the Expression.
     pub fn pos(&self) -> &Position {
         match self {
             &Expression::Simple(ref v) => v.pos(),
@@ -633,19 +671,21 @@ impl Expression {
     }
 }
 
+/// Encodes a let statement in the UCG AST.
 #[derive(Debug,PartialEq)]
 pub struct LetDef {
     pub name: Token,
     pub value: Expression,
 }
 
+/// Encodes an import statement in the UCG AST.
 #[derive(Debug,PartialEq)]
 pub struct ImportDef {
     pub path: Token,
     pub name: Token,
 }
 
-/// Statement encodes a parsed Statement in the UCG AST.
+/// Encodes a parsed statement in the UCG AST.
 #[derive(Debug,PartialEq)]
 pub enum Statement {
     // simple expression
@@ -654,7 +694,7 @@ pub enum Statement {
     // Named bindings
     Let(LetDef),
 
-    // Include a file.
+    // Import a file.
     Import(ImportDef),
 }
 

src/build.rs

@@ -11,6 +11,8 @@
 //  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
+
+//! The build stage of the ucg compiler.
 use std::fs::File;
 use std::io::Read;
 use std::error::Error;
@@ -29,6 +31,7 @@ use parse::parse;
 use error;
 
 impl MacroDef {
+    /// Expands a ucg Macro using the given arguments into a new Tuple.
     pub fn eval(&self,
                 mut args: Vec<Rc<Val>>)
                 -> Result<Vec<(Positioned<String>, Rc<Val>)>, Box<Error>> {
@@ -59,10 +62,10 @@ impl MacroDef {
     }
 }
 
-/// BuildResult is the result of a build.
+/// The result of a build.
 type BuildResult = Result<(), Box<Error>>;
 
-/// Val is the Intermediate representation of a compiled UCG AST.
+/// The Intermediate representation of a compiled UCG AST.
 #[derive(PartialEq,Debug,Clone)]
 pub enum Val {
     Int(i64),
@@ -74,6 +77,7 @@ pub enum Val {
 }
 
 impl Val {
+    /// Returns the Type of a Val as a string.
     pub fn type_name(&self) -> String {
         match self {
             &Val::Int(_) => "Integer".to_string(),
@@ -85,6 +89,7 @@ impl Val {
         }
     }
 
+    /// Returns true if called with a Val of the same type as itself.
     pub fn type_equal(&self, target: &Self) -> bool {
         enum_type_equality!(self, target, &Val::Int(_),
                                      &Val::Float(_),
@@ -94,6 +99,7 @@ impl Val {
                                      &Val::Macro(_))
     }
 
+    /// Returns the fields if this Val is a tuple. None otherwise.
     pub fn get_fields(&self) -> Option<&Vec<(Positioned<String>, Rc<Val>)>> {
         if let &Val::Tuple(ref fs) = self {
             Some(fs)
@@ -174,10 +180,10 @@ impl From<Val> for String {
     }
 }
 
-/// ValueMap defines a set of values in a parsed file.
+/// Defines a set of values in a parsed file.
 type ValueMap = HashMap<Positioned<String>, Rc<Val>>;
 
-/// Builder parses one or more statements into a out Tuple.
+/// Handles building ucg code.
 pub struct Builder {
     /// assets are other parsed files from import statements. They
     /// are keyed by the normalized import path. This acts as a cache
@@ -208,7 +214,6 @@ macro_rules! eval_binary_expr {
 }
 
 impl Builder {
-    // TODO(jwall): Maintain order for tuples.
     fn tuple_to_val(&self, fields: &Vec<(Token, Expression)>) -> Result<Rc<Val>, Box<Error>> {
         let mut new_fields = Vec::<(Positioned<String>, Rc<Val>)>::new();
         for &(ref name, ref expr) in fields.iter() {
@@ -245,6 +250,7 @@ impl Builder {
         }
     }
 
+    /// Constructs a new Builder.
     pub fn new() -> Self {
         Builder {
             assets: HashMap::new(),
@@ -254,6 +260,7 @@ impl Builder {
         }
     }
 
+    /// Constructs a new Builder with a provided scope.
     pub fn new_with_scope(scope: ValueMap) -> Self {
         Builder {
             assets: HashMap::new(),
@@ -263,6 +270,7 @@ impl Builder {
         }
     }
 
+    /// Returns a Val by name from previously built UCG.
     pub fn get_out_by_name(&self, name: &str) -> Option<Rc<Val>> {
         let key = Positioned {
             pos: Position::new(0, 0),
@@ -271,6 +279,7 @@ impl Builder {
         self.lookup_sym(&key)
     }
 
+    /// Builds a list of parsed UCG Statements.
     pub fn build(&mut self, ast: &Vec<Statement>) -> BuildResult {
         for stmt in ast.iter() {
             try!(self.build_stmt(stmt));
@@ -278,7 +287,8 @@ impl Builder {
         Ok(())
     }
 
-    pub fn build_file_string(&mut self, _name: &str, input: String) -> BuildResult {
+    /// Builds a string of ucg syntax.
+    pub fn build_file_string(&mut self, input: String) -> BuildResult {
         match parse(Span::new(&input)) {
             Ok(stmts) => {
                 for stmt in stmts.iter() {
@@ -290,12 +300,13 @@ impl Builder {
         }
     }
 
+    /// Builds a ucg file at the named path.
     pub fn build_file(&mut self, name: &str) -> BuildResult {
         let mut f = try!(File::open(name));
         let mut s = String::new();
         // TODO(jwall): It would be nice to be able to do this while streaming
         try!(f.read_to_string(&mut s));
-        self.build_file_string(name, s)
+        self.build_file_string(s)
     }
 
     fn build_import(&mut self, def: &ImportDef) -> BuildResult {
@@ -769,7 +780,7 @@ impl Builder {
         }
     }
 
-    // eval_expr evals a single Expression in the context of a running Builder.
+    // Evals a single Expression in the context of a running Builder.
     // It does not mutate the builders collected state at all.
     pub fn eval_expr(&self, expr: &Expression) -> Result<Rc<Val>, Box<Error>> {
         // TODO(jwall): We probably don't want to consume these expressions.

src/convert/flags.rs

@@ -12,6 +12,7 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
+//! Flags contains code for converting a UCG Val into the command line flag output target.
 use std::rc::Rc;
 use std::io::Write;
 use std::io::Result;
@@ -19,6 +20,7 @@ use std::io::Result;
 use build::Val;
 use convert::traits::Converter;
 
+/// FlagConverter implements the conversion logic for converting a Val into a set of command line flags.
 pub struct FlagConverter {}
 
 impl FlagConverter {

src/convert/json.rs

@@ -7,7 +7,7 @@
 //  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
-
+//! Flags contains code for converting a UCG Val into the json output target.
 use std::rc::Rc;
 use std::io::Write;
 use std::io::Result;
@@ -18,6 +18,7 @@ use ast;
 use build::Val;
 use convert::traits::Converter;
 
+/// JsonConverter implements the logic for converting a Val into the json output format.
 pub struct JsonConverter {}
 
 impl JsonConverter {

src/convert/mod.rs

@@ -11,6 +11,8 @@
 //  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
+
+//! The conversion stage of the ucg compiler.
 pub mod flags;
 pub mod json;
 pub mod traits;
@@ -21,11 +23,16 @@ use std::rc::Rc;
 
 use build::Val;
 
+/// ConverterRunner knows how to run a given converter on a Val.
 pub struct ConverterRunner {
     converter: Box<traits::Converter>,
 }
 
 impl ConverterRunner {
+    /// new creates a new ConverterRunner with a converter for the provided output target.
+    /// 
+    /// * flags
+    /// * json
     pub fn new(typ: &str) -> Result<Self, String> {
         if typ == "flags" {
             return Ok(ConverterRunner { converter: Box::new(flags::FlagConverter::new()) });
@@ -36,6 +43,7 @@ impl ConverterRunner {
         return Err(format!("Unknown Target output type: {}", typ));
     }
 
+    /// convert runs the Converter on a Val and writes the output to the provided writer.
     pub fn convert(&self, v: Rc<Val>, w: Box<Write>) -> io::Result<()> {
         self.converter.convert(v, w)
     }

src/convert/traits.rs

@@ -12,12 +12,15 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
+//! The traits used by the ucg compiler for converting Val intermediate format into the output formats..
 use std::rc::Rc;
 use std::io::Write;
 use std::io::Result;
 
 use build::Val;
 
+/// The trait that Converters from Val to different output formats for the
+/// final conversion stage of the ucg compiler.
 pub trait Converter {
     fn convert(&self, vs: Rc<Val>, w: Box<Write>) -> Result<()>;
 }

src/error.rs

@@ -1,9 +1,25 @@
+// Copyright 2017 Jeremy Wall <jeremy@marzhillstudios.com>
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
 
+//! Errors for use by the ucg compiler.
 use std::error;
 use std::fmt;
 
 use ast::*;
 
+/// ErrorType defines the various types of errors that can result from compiling UCG into an
+/// output format.
 pub enum ErrorType {
     // Build Errors
     TypeFail,
@@ -35,6 +51,7 @@ impl fmt::Display for ErrorType {
     }
 }
 
+/// Error defines an Error type for parsing and building UCG code.
 pub struct Error {
     pub err_type: ErrorType,
     pub pos: Position,

src/format.rs

@@ -12,18 +12,21 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
+//! The format string logic for ucg format expressions.
 use std::clone::Clone;
 use std::error::Error;
 
 use ast::*;
 use error;
 
+/// Implements the logic for format strings in UCG format expressions.
 pub struct Formatter<V: Into<String> + Clone> {
     tmpl: String,
     args: Vec<V>,
 }
 
 impl<V: Into<String> + Clone> Formatter<V> {
+    /// Constructs a Formatter with a template and args.
     pub fn new<S: Into<String>>(tmpl: S, args: Vec<V>) -> Self {
         Formatter {
             tmpl: tmpl.into(),
@@ -31,6 +34,10 @@ impl<V: Into<String> + Clone> Formatter<V> {
         }
     }
 
+    /// Renders a formatter to a string or returns an error.
+    /// 
+    /// If the formatter has the wrong number of arguments for the number of replacements
+    /// it will return an error. Otherwise it will return the formatted string.
     pub fn render(&self, pos: &Position) -> Result<String, Box<Error>> {
         let mut buf = String::new();
         let mut should_escape = false;

src/parse.rs

@@ -11,6 +11,8 @@
 //  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
+
+//! The Parsing stage of the ucg compiler.
 use std::str::FromStr;
 use std::borrow::Borrow;
 
@@ -591,6 +593,7 @@ named!(statement<TokenIter, Statement, ParseError>,
     )
 );
 
+/// Parses a LocatedSpan into a list of Statements or a ParseError.
 pub fn parse(input: LocatedSpan<&str>) -> Result<Vec<Statement>, ParseError> {
     match tokenize(input) {
         Ok(tokenized) => {

src/tokenizer.rs

@@ -11,6 +11,8 @@
 //  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
+
+//! The tokenization stage of the ucg compiler.
 use nom_locate::LocatedSpan;
 use nom;
 use nom::{alpha, is_alphanumeric, digit, multispace};
@@ -286,6 +288,8 @@ named!(token( Span ) -> Token,
 );
 
 // TODO(jwall): This should return a ParseError instead.
+
+/// Consumes an input Span and returns either a Vec<Token> or a nom::ErrorKind.
 pub fn tokenize(input: Span) -> Result<Vec<Token>, nom::ErrorKind> {
     let mut out = Vec::new();
     let mut i = input;
@@ -326,6 +330,8 @@ pub fn token_clone(t: &Token) -> Result<Token, ParseError> {
     Ok(t.clone())
 }
 
+/// nom macro that matches a Token by type and uses an optional conversion handler
+/// for the matched Token.
 macro_rules! match_type {
     ($i:expr, COMMENT => $h:expr) => {
         match_type!($i, TokenType::COMMENT, "Not a Comment", $h)
@@ -396,6 +402,8 @@ macro_rules! match_type {
     };
 }
 
+/// nom style macro that matches various Tokens by type and value and allows optional
+/// conversion handlers for the matched Token.
 macro_rules! match_token {
     ($i:expr, PUNCT => $f:expr) => {
         match_token!($i, PUNCT => $f, token_clone)
@@ -434,18 +442,21 @@ macro_rules! match_token {
     };
 }
 
+/// nom style macro that matches punctuation Tokens.
 macro_rules! punct {
     ($i:expr, $c:expr) => {
         match_token!($i, PUNCT => $c)
     };
 }
 
+/// nom style macro that matches any bareword Token.
 macro_rules! word {
     ($i:expr, $w:expr) => {
         match_token!($i, BAREWORD => $w)
     };
 }
 
+/// pos gets the current position from a TokenIter input without consuming it.
 pub fn pos(i: TokenIter) -> nom::IResult<TokenIter, Position, ParseError> {
     let tok = &i[0];
     let line = tok.pos.line;
@@ -457,6 +468,8 @@ pub fn pos(i: TokenIter) -> nom::IResult<TokenIter, Position, ParseError> {
                        })
 }
 
+/// TokenIter wraps a slice of Tokens and implements the various necessary
+/// nom traits to use it as an input to nom parsers.
 #[derive(Clone, Debug, PartialEq)]
 pub struct TokenIter<'a> {
     pub source: &'a [Token],