zaphar/ucg
1
0
Fork 0
mirror of https://github.com/zaphar/ucg.git synced 2025-07-22 18:19:54 -04:00
Code Issues Packages Projects Releases Wiki Activity

Retain the position of errors in the tokenization phase.

Browse Source
This commit is contained in:
Jeremy Wall 2018-03-21 18:20:26 -05:00
parent 141385920a
commit 51edc6d15c
2 changed files with 17 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/parse.rs
View File

@ -658,8 +658,8 @@ pub fn parse(input: LocatedSpan<&str>) -> Result<Vec<Statement>, ParseError> {
// FIXME(jwall): We should really capture the location
// of the tokenization error here.
return Err(ParseError {
description: format!("Tokenize Error: {:?}", e),
pos: Position { line: 0, column: 0 },
description: format!("Tokenize Error: {:?}", e.1),
pos: e.0,
});
}
}

18
src/tokenizer.rs
View File

@ -304,7 +304,7 @@ named!(token( Span ) -> Token,
// TODO(jwall): This should return a ParseError instead.
/// Consumes an input Span and returns either a Vec<Token> or a nom::ErrorKind.
pub fn tokenize(input: Span) -> Result<Vec<Token>, nom::ErrorKind> {
pub fn tokenize(input: Span) -> Result<Vec<Token>, (Position, nom::ErrorKind)> {
let mut out = Vec::new();
let mut i = input;
loop {
@ -313,10 +313,22 @@ pub fn tokenize(input: Span) -> Result<Vec<Token>, nom::ErrorKind> {
}
match token(i) {
nom::IResult::Error(e) => {
return Err(e);
return Err((
Position {
line: i.line as usize,
column: i.get_column() as usize,
},
e,
));
}
nom::IResult::Incomplete(_) => {
return Err(nom::ErrorKind::Complete);
return Err((
Position {
line: i.line as usize,
column: i.get_column() as usize,
},
nom::ErrorKind::Complete,
));
}
nom::IResult::Done(rest, tok) => {
i = rest;