From 51edc6d15cae0b056d13b7cc45e249afcac34e5d Mon Sep 17 00:00:00 2001
From: Jeremy Wall <jeremy@marzhillstudios.com>
Date: Wed, 21 Mar 2018 18:20:26 -0500
Subject: [PATCH] Retain the position of errors in the tokenization phase.

---
 src/parse.rs     |  4 ++--
 src/tokenizer.rs | 18 +++++++++++++++---
 2 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/src/parse.rs b/src/parse.rs
index f040c4c..7f43506 100644
--- a/src/parse.rs
+++ b/src/parse.rs
@@ -658,8 +658,8 @@ pub fn parse(input: LocatedSpan<&str>) -> Result<Vec<Statement>, ParseError> {
             // FIXME(jwall): We should really capture the location
             // of the tokenization error here.
             return Err(ParseError {
-                description: format!("Tokenize Error: {:?}", e),
-                pos: Position { line: 0, column: 0 },
+                description: format!("Tokenize Error: {:?}", e.1),
+                pos: e.0,
             });
         }
     }
diff --git a/src/tokenizer.rs b/src/tokenizer.rs
index ced1129..19f9e63 100644
--- a/src/tokenizer.rs
+++ b/src/tokenizer.rs
@@ -304,7 +304,7 @@ named!(token( Span ) -> Token,
 // TODO(jwall): This should return a ParseError instead.
 
 /// Consumes an input Span and returns either a Vec<Token> or a nom::ErrorKind.
-pub fn tokenize(input: Span) -> Result<Vec<Token>, nom::ErrorKind> {
+pub fn tokenize(input: Span) -> Result<Vec<Token>, (Position, nom::ErrorKind)> {
     let mut out = Vec::new();
     let mut i = input;
     loop {
@@ -313,10 +313,22 @@ pub fn tokenize(input: Span) -> Result<Vec<Token>, nom::ErrorKind> {
         }
         match token(i) {
             nom::IResult::Error(e) => {
-                return Err(e);
+                return Err((
+                    Position {
+                        line: i.line as usize,
+                        column: i.get_column() as usize,
+                    },
+                    e,
+                ));
             }
             nom::IResult::Incomplete(_) => {
-                return Err(nom::ErrorKind::Complete);
+                return Err((
+                    Position {
+                        line: i.line as usize,
+                        column: i.get_column() as usize,
+                    },
+                    nom::ErrorKind::Complete,
+                ));
             }
             nom::IResult::Done(rest, tok) => {
                 i = rest;