zaphar/ucg
1
0
Fork 0
mirror of https://github.com/zaphar/ucg.git synced 2025-07-22 18:19:54 -04:00
Code Issues Packages Projects Releases Wiki Activity

REFACTOR: Pass our offsets everywhere.

Browse Source
This commit is contained in:
Jeremy Wall 2018-10-13 19:42:22 -05:00
parent 91d7ed690b
commit 7f47dc3f38
16 changed files with 1128 additions and 1297 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
examples/test_yaml.yaml Normal file
View File

@ -0,0 +1,9 @@
---
db_conn1: "db1.prod.net:3306/testdb"
db_conn2: "db2.prod.net:3306/testdb"
tmpldir: "./templates"
prefix:
foo: bar
l:
- foo
- bar

2
integration_tests/operator_precedence_test.ucg
View File

@ -5,3 +5,5 @@ assert |2 * 2 + 1 > 4|;
assert |2 * 2 + 1 < 6|;
assert |2 * 2 + 1 >= 5|;
assert |2 * 2 + 1 <= 5|;
assert |2 / 2 == 1|;
assert |2 - 1 == 1|;

90
src/ast/mod.rs
View File

@ -21,11 +21,10 @@ use std::cmp::PartialEq;
use std::cmp::PartialOrd;
use std::collections::HashSet;
use std::convert::Into;
use std::fmt;
use std::hash::Hash;
use std::hash::Hasher;
use std::fmt;
macro_rules! enum_type_equality {
( $slf:ident, $r:expr, $( $l:pat ),* ) => {
match $slf {
@ -50,14 +49,16 @@ macro_rules! enum_type_equality {
pub struct Position {
pub line: usize,
pub column: usize,
pub offset: usize,
}
impl Position {
/// Construct a new Position.
pub fn new(line: usize, column: usize) -> Self {
pub fn new(line: usize, column: usize, offset: usize) -> Self {
Position {
line: line,
column: column,
offset: offset,
}
}
}
@ -89,8 +90,8 @@ pub struct Token {
impl Token {
/// Constructs a new Token with a type and line and column information.
pub fn new<S: Into<String>>(f: S, typ: TokenType, line: usize, col: usize) -> Self {
Self::new_with_pos(f, typ, Position::new(line, col))
pub fn new<S: Into<String>, P: Into<Position>>(f: S, typ: TokenType, p: P) -> Self {
Self::new_with_pos(f, typ, p.into())
}
// Constructs a new Token with a type and a Position.
@ -114,56 +115,49 @@ macro_rules! value_node {
($v:expr, $p:expr) => {
Positioned::new_with_pos($v, $p)
};
($v:expr, $l:expr, $c:expr) => {
Positioned::new($v, $l, $c)
};
}
/// Helper macro for making a Token.
#[allow(unused_macros)]
macro_rules! make_tok {
(EOF => $l:expr, $c:expr) => {
Token::new("", TokenType::END, $l, $c)
(EOF => $i:expr) => {
Token::new("", TokenType::END, $i)
};
(WS => $l:expr, $c:expr) => {
Token::new("", TokenType::WS, $l, $c)
(WS => $i:expr) => {
Token::new("", TokenType::WS, $i)
};
(CMT => $e:expr, $l:expr, $c:expr) => {
Token::new($e, TokenType::COMMENT, $l, $c)
(CMT => $e:expr, $i:expr) => {
Token::new($e, TokenType::COMMENT, $i)
};
(QUOT => $e:expr, $l:expr, $c:expr) => {
Token::new($e, TokenType::QUOTED, $l, $c)
(QUOT => $e:expr, $i:expr) => {
Token::new($e, TokenType::QUOTED, $i)
};
(PUNCT => $e:expr, $l:expr, $c:expr) => {
Token::new($e, TokenType::PUNCT, $l, $c)
(PUNCT => $e:expr, $i:expr) => {
Token::new($e, TokenType::PUNCT, $i)
};
(DIGIT => $e:expr, $l:expr, $c:expr) => {
Token::new($e, TokenType::DIGIT, $l, $c)
(DIGIT => $e:expr, $i:expr) => {
Token::new($e, TokenType::DIGIT, $i)
};
($e:expr, $l:expr, $c:expr) => {
Token::new($e, TokenType::BAREWORD, $l, $c)
($e:expr, $i:expr) => {
Token::new($e, TokenType::BAREWORD, $i)
};
}
/// Helper macro for making expressions.
#[allow(unused_macros)]
macro_rules! make_expr {
($e:expr) => {
make_expr!($e, 1, 1)
($e:expr, $i:expr) => {
Expression::Simple(Value::Symbol(Positioned::new_with_pos($e.to_string(), $i)))
};
($e:expr, $l:expr, $c:expr) => {
Expression::Simple(Value::Symbol(Positioned::new($e.to_string(), $l, $c)))
};
($e:expr => int, $l:expr, $c:expr) => {
Expression::Simple(Value::Int(Positioned::new($e, $l, $c)))
($e:expr => int, $i:expr) => {
Expression::Simple(Value::Int(Positioned::new_with_pos($e, $i)))
};
}
@ -180,30 +174,26 @@ macro_rules! make_expr {
/// ```
#[allow(unused_macros)]
macro_rules! make_selector {
( $h:expr ) => {
make_selector!($h, 1, 0)
};
( $h:expr, $l:expr, $c:expr ) => {
( $h:expr, $i:expr) => {
SelectorDef::new(
SelectorList{head: Box::new($h), tail: None},
$l, $c)
$i)
};
( $h: expr, $list:expr, $l:expr, $c:expr) => {
( $h: expr, $list:expr, $i:expr) => {
SelectorDef::new(
SelectorList{head: Box::new($h), tail: Some($list)},
$l, $c)
$i)
};
// Tokens
( $h:expr => [ $( $item:expr ),* ] ) => {
( $h:expr => [ $( $item:expr ),* ], $i:expr ) => {
{
make_selector!($h => [ $( $item, )* ] => 1, 1)
make_selector!($h => [ $( $item, )* ] => $i)
}
};
( $h:expr => [ $( $item:expr ),* ] => $l:expr, $c:expr ) => {
( $h:expr => [ $( $item:expr ),* ] => $i:expr ) => {
{
let mut list: Vec<Token> = Vec::new();
@ -211,7 +201,7 @@ macro_rules! make_selector {
list.push($item);
)*
make_selector!($h, list, $l, $c)
make_selector!($h, list, $i)
}
};
@ -223,14 +213,14 @@ macro_rules! make_selector {
let mut list: Vec<Token> = Vec::new();
$(
list.push(make_tok!($item, 1, col));
list.push(make_tok!($item, Position::new(1, col, col)));
col += $item.len() + 1;
)*
// Shut up the lint about unused code;
assert!(col != 0);
make_selector!($h, list, 1, 1)
make_selector!($h, list, Position::new(1, 1, 1))
}
};
@ -241,14 +231,14 @@ macro_rules! make_selector {
let mut list: Vec<Token> = Vec::new();
$(
list.push(make_tok!($item, $l, col));
list.push(make_tok!($item, Position::new($l, col, col)));
col += $item.len() + 1;
)*
// Shut up the linter about unused code;
assert!(col != 0);
make_selector!($h, list, $l, $c)
make_selector!($h, list, Position::new($l, $c, $c))
}
};
}
@ -314,9 +304,9 @@ pub struct SelectorDef {
impl SelectorDef {
/// Constructs a new SelectorDef.
pub fn new(sel: SelectorList, line: usize, col: usize) -> Self {
pub fn new<P: Into<Position>>(sel: SelectorList, p: P) -> Self {
SelectorDef {
pos: Position::new(line, col),
pos: p.into(),
sel: sel,
}
}
@ -451,8 +441,8 @@ impl<T: std::fmt::Display> std::fmt::Display for Positioned<T> {
impl<T> Positioned<T> {
/// Constructs a new Positioned<T> with a value, line, and column information.
pub fn new(v: T, l: usize, c: usize) -> Self {
Self::new_with_pos(v, Position::new(l, c))
pub fn new<P: Into<Position>>(v: T, p: P) -> Self {
Self::new_with_pos(v, p.into())
}
/// Constructs a new Positioned<T> with a value and a Position.

70
src/ast/test.rs
View File

@ -17,21 +17,23 @@ use super::*;
#[test]
pub fn test_macro_validation_happy_path() {
let def = MacroDef {
argdefs: vec![value_node!("foo".to_string(), 1, 0)],
argdefs: vec![value_node!("foo".to_string(), Position::new(1, 0, 0))],
fields: vec![(
make_tok!("f1", 1, 1),
make_tok!("f1", Position::new(1, 1, 0)),
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Add,
left: Box::new(Expression::Simple(Value::Symbol(value_node!(
"foo".to_string(),
1,
1
Position::new(1, 1, 0)
)))),
right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
pos: Position::new(1, 0),
right: Box::new(Expression::Simple(Value::Int(value_node!(
1,
Position::new(1, 1, 0)
)))),
pos: Position::new(1, 0, 0),
}),
)],
pos: Position::new(1, 0),
pos: Position::new(1, 0, 0),
};
assert!(def.validate_symbols().unwrap() == ());
}
@ -39,21 +41,23 @@ pub fn test_macro_validation_happy_path() {
#[test]
pub fn test_macro_validation_fail() {
let def = MacroDef {
argdefs: vec![value_node!("foo".to_string(), 1, 0)],
argdefs: vec![value_node!("foo".to_string(), Position::new(1, 0, 0))],
fields: vec![(
make_tok!("f1", 1, 1),
make_tok!("f1", Position::new(1, 1, 0)),
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Add,
left: Box::new(Expression::Simple(Value::Symbol(value_node!(
"bar".to_string(),
1,
1
Position::new(1, 1, 0)
)))),
right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
pos: Position::new(1, 0),
right: Box::new(Expression::Simple(Value::Int(value_node!(
1,
Position::new(1, 1, 0)
)))),
pos: Position::new(1, 0, 0),
}),
)],
pos: Position::new(1, 0),
pos: Position::new(1, 0, 0),
};
let mut expected = HashSet::new();
expected.insert("bar".to_string());
@ -63,20 +67,23 @@ pub fn test_macro_validation_fail() {
#[test]
pub fn test_macro_validation_selector_happy_path() {
let def = MacroDef {
argdefs: vec![value_node!("foo".to_string(), 1, 0)],
argdefs: vec![value_node!("foo".to_string(), Position::new(1, 0, 0))],
fields: vec![(
make_tok!("f1", 1, 1),
make_tok!("f1", Position::new(1, 1, 0)),
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Add,
left: Box::new(Expression::Simple(Value::Selector(
make_selector!(make_expr!("foo", 1, 1) => [
make_tok!("quux", 1, 1) ] => 1, 1),
))),
right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
pos: Position::new(1, 0),
left: Box::new(Expression::Simple(Value::Selector(make_selector!(
make_expr!("foo", Position::new(1, 1, 0)) => [
make_tok!("quux", Position::new(1, 1, 0)) ]
=> Position::new(1, 1, 0))))),
right: Box::new(Expression::Simple(Value::Int(value_node!(
1,
Position::new(1, 1, 0)
)))),
pos: Position::new(1, 0, 0),
}),
)],
pos: Position::new(1, 0),
pos: Position::new(1, 0, 0),
};
assert!(def.validate_symbols().unwrap() == ());
}
@ -84,20 +91,23 @@ pub fn test_macro_validation_selector_happy_path() {
#[test]
pub fn test_macro_validation_selector_fail() {
let def = MacroDef {
argdefs: vec![value_node!("foo".to_string(), 1, 0)],
argdefs: vec![value_node!("foo".to_string(), Position::new(1, 0, 0))],
fields: vec![(
make_tok!("f1", 1, 1),
make_tok!("f1", Position::new(1, 1, 0)),
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Add,
left: Box::new(Expression::Simple(Value::Selector(
make_selector!(make_expr!("bar", 1, 1) => [
make_tok!("quux", 1, 1) ] => 1, 1),
make_selector!(make_expr!("bar", Position::new(1, 1, 0)) => [
make_tok!("quux", Position::new(1, 1, 0)) ] => Position::new(1, 1, 0)),
))),
right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
pos: Position::new(1, 0),
right: Box::new(Expression::Simple(Value::Int(value_node!(
1,
Position::new(1, 1, 0)
)))),
pos: Position::new(1, 0, 0),
}),
)],
pos: Position::new(1, 0),
pos: Position::new(1, 0, 0),
};
let mut expected = HashSet::new();
expected.insert("bar".to_string());

4
src/benches/parse.rs
View File

@ -22,14 +22,14 @@ extern crate ucglib;
use bencher::Bencher;
use abortable_parser::StrIter;
use ucglib::iter::OffsetStrIter;
//use cpuprofiler::PROFILER;
use ucglib::parse::*;
fn do_parse(i: &str) {
parse(StrIter::new(i));
parse(OffsetStrIter::new(i));
}
fn parse_int(b: &mut Bencher) {

21
src/build/mod.rs
View File

@ -25,11 +25,10 @@ use std::path::PathBuf;
use std::rc::Rc;
use std::string::ToString;
use abortable_parser::StrIter;
use ast::*;
use error;
use format;
use iter::OffsetStrIter;
use parse::parse;
pub mod assets;
@ -192,8 +191,12 @@ impl<'a> Builder<'a> {
scope: ValueMap,
) -> Self {
let env_vars: Vec<(Positioned<String>, Rc<Val>)> = env::vars()
.map(|t| (Positioned::new(t.0, 0, 0), Rc::new(t.1.into())))
.collect();
.map(|t| {
(
Positioned::new(t.0, Position::new(0, 0, 0)),
Rc::new(t.1.into()),
)
}).collect();
Self::new_with_env_and_scope(root, cache, scope, Rc::new(Val::Tuple(env_vars)))
}
@ -223,7 +226,7 @@ impl<'a> Builder<'a> {
/// Returns a Val by name from previously built UCG.
pub fn get_out_by_name(&self, name: &str) -> Option<Rc<Val>> {
let key = Positioned {
pos: Position::new(0, 0),
pos: Position::new(0, 0, 0),
val: name.to_string(),
};
self.lookup_sym(&key)
@ -245,7 +248,7 @@ impl<'a> Builder<'a> {
Ok(())
}
fn eval_span(&mut self, input: StrIter) -> Result<Rc<Val>, Box<Error>> {
fn eval_span(&mut self, input: OffsetStrIter) -> Result<Rc<Val>, Box<Error>> {
match parse(input) {
Ok(stmts) => {
//panic!("Successfully parsed {}", input);
@ -258,6 +261,7 @@ impl<'a> Builder<'a> {
Some(val) => Ok(val),
}
}
// FIXME(jwall): We need to return a error::Error so we have position information.
Err(err) => Err(Box::new(error::Error::new_with_boxed_cause(
format!(
"Error while parsing file: {}",
@ -271,7 +275,7 @@ impl<'a> Builder<'a> {
/// Evaluate an input string as UCG.
pub fn eval_string(&mut self, input: &str) -> Result<Rc<Val>, Box<Error>> {
self.eval_span(StrIter::new(input))
self.eval_span(OffsetStrIter::new(input))
}
/// Builds a ucg file at the named path.
@ -993,7 +997,8 @@ impl<'a> Builder<'a> {
let expr = &tok.fragment;
expr_as_stmt.push_str(expr);
expr_as_stmt.push_str(";");
let assert_input = StrIter::new(&expr_as_stmt);
let assert_input =
OffsetStrIter::new_with_offsets(&expr_as_stmt, 0, tok.pos.line - 1, tok.pos.column - 1);
let ok = match self.eval_span(assert_input) {
Ok(v) => v,
Err(e) => {

728
src/build/test.rs
View File

@ -25,35 +25,6 @@ fn test_expr_to_val(mut cases: Vec<(Expression, Val)>, b: Builder) {
}
}
#[test]
fn test_eval_div_expr() {
let cache = Rc::new(RefCell::new(MemoryCache::new()));
let b = Builder::new(std::env::current_dir().unwrap(), cache);
test_expr_to_val(
vec![
(
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Div,
left: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))),
right: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))),
pos: Position::new(1, 0),
}),
Val::Int(1),
),
(
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Div,
left: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))),
right: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))),
pos: Position::new(1, 0),
}),
Val::Float(1.0),
),
],
b,
);
}
#[test]
#[should_panic(expected = "Expected Float")]
fn test_eval_div_expr_fail() {
@ -63,9 +34,15 @@ fn test_eval_div_expr_fail() {
vec![(
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Div,
left: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))),
right: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))),
pos: Position::new(1, 0),
left: Box::new(Expression::Simple(Value::Float(value_node!(
2.0,
Position::new(1, 1, 1)
)))),
right: Box::new(Expression::Simple(Value::Int(value_node!(
2,
Position::new(1, 1, 1)
)))),
pos: Position::new(1, 0, 0),
}),
Val::Float(1.0),
)],
@ -73,35 +50,6 @@ fn test_eval_div_expr_fail() {
);
}
#[test]
fn test_eval_mul_expr() {
let cache = Rc::new(RefCell::new(MemoryCache::new()));
let b = Builder::new(std::env::current_dir().unwrap(), cache);
test_expr_to_val(
vec![
(
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Mul,
left: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))),
right: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))),
pos: Position::new(1, 0),
}),
Val::Int(4),
),
(
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Mul,
left: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))),
right: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))),
pos: Position::new(1, 0),
}),
Val::Float(4.0),
),
],
b,
);
}
#[test]
#[should_panic(expected = "Expected Float")]
fn test_eval_mul_expr_fail() {
@ -111,9 +59,15 @@ fn test_eval_mul_expr_fail() {
vec![(
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Mul,
left: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))),
right: Box::new(Expression::Simple(Value::Int(value_node!(20, 1, 1)))),
pos: Position::new(1, 0),
left: Box::new(Expression::Simple(Value::Float(value_node!(
2.0,
Position::new(1, 1, 1)
)))),
right: Box::new(Expression::Simple(Value::Int(value_node!(
20,
Position::new(1, 1, 1)
)))),
pos: Position::new(1, 0, 0),
}),
Val::Float(1.0),
)],
@ -121,35 +75,6 @@ fn test_eval_mul_expr_fail() {
);
}
#[test]
fn test_eval_subtract_expr() {
let cache = Rc::new(RefCell::new(MemoryCache::new()));
let b = Builder::new(std::env::current_dir().unwrap(), cache);
test_expr_to_val(
vec![
(
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Sub,
left: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))),
right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
pos: Position::new(1, 0),
}),
Val::Int(1),
),
(
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Sub,
left: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))),
right: Box::new(Expression::Simple(Value::Float(value_node!(1.0, 1, 1)))),
pos: Position::new(1, 0),
}),
Val::Float(1.0),
),
],
b,
);
}
#[test]
#[should_panic(expected = "Expected Float")]
fn test_eval_subtract_expr_fail() {
@ -159,88 +84,21 @@ fn test_eval_subtract_expr_fail() {
vec![(
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Sub,
left: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))),
right: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))),
pos: Position::new(1, 0),
left: Box::new(Expression::Simple(Value::Float(value_node!(
2.0,
Position::new(1, 1, 1)
)))),
right: Box::new(Expression::Simple(Value::Int(value_node!(
2,
Position::new(1, 1, 1)
)))),
pos: Position::new(1, 0, 0),
}),
Val::Float(1.0),
)],
b,
);
}
#[test]
fn test_eval_add_expr() {
let cache = Rc::new(RefCell::new(MemoryCache::new()));
let b = Builder::new(std::env::current_dir().unwrap(), cache);
test_expr_to_val(
vec![
(
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Add,
left: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
pos: Position::new(1, 0),
}),
Val::Int(2),
),
(
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Add,
left: Box::new(Expression::Simple(Value::Float(value_node!(1.0, 1, 1)))),
right: Box::new(Expression::Simple(Value::Float(value_node!(1.0, 1, 1)))),
pos: Position::new(1, 0),
}),
Val::Float(2.0),
),
(
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Add,
left: Box::new(Expression::Simple(Value::Str(value_node!(
"foo".to_string(),
1,
1
)))),
right: Box::new(Expression::Simple(Value::Str(value_node!(
"bar".to_string(),
1,
1
)))),
pos: Position::new(1, 0),
}),
Val::Str("foobar".to_string()),
),
(
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Add,
left: Box::new(Expression::Simple(Value::List(ListDef {
elems: vec![Expression::Simple(Value::Str(value_node!(
"foo".to_string(),
1,
1
)))],
pos: Position::new(1, 1),
}))),
right: Box::new(Expression::Simple(Value::List(ListDef {
elems: vec![Expression::Simple(Value::Str(value_node!(
"bar".to_string(),
1,
1
)))],
pos: Position::new(1, 1),
}))),
pos: Position::new(1, 0),
}),
Val::List(vec![
Rc::new(Val::Str("foo".to_string())),
Rc::new(Val::Str("bar".to_string())),
]),
),
],
b,
);
}
#[test]
#[should_panic(expected = "Expected Float")]
fn test_eval_add_expr_fail() {
@ -250,9 +108,15 @@ fn test_eval_add_expr_fail() {
vec![(
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Add,
left: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))),
right: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))),
pos: Position::new(1, 0),
left: Box::new(Expression::Simple(Value::Float(value_node!(
2.0,
Position::new(1, 1, 1)
)))),
right: Box::new(Expression::Simple(Value::Int(value_node!(
2,
Position::new(1, 1, 1)
)))),
pos: Position::new(1, 0, 0),
}),
Val::Float(1.0),
)],
@ -260,245 +124,20 @@ fn test_eval_add_expr_fail() {
);
}
#[test]
fn test_eval_nested_tuple() {
test_expr_to_val(
vec![
(
Expression::Simple(Value::Tuple(value_node!(
vec![(
Token::new("foo", TokenType::BAREWORD, 1, 1),
Expression::Simple(Value::Tuple(value_node!(Vec::new(), 1, 1))),
)],
1,
1
))),
Val::Tuple(vec![(
Positioned::new("foo".to_string(), 1, 1),
Rc::new(Val::Tuple(Vec::new())),
)]),
),
(
Expression::Simple(Value::Tuple(value_node!(
vec![(
Token::new("foo", TokenType::BAREWORD, 1, 1),
Expression::Simple(Value::Tuple(value_node!(
vec![(
Token::new("bar".to_string(), TokenType::BAREWORD, 1, 5),
Expression::Simple(Value::Tuple(value_node!(vec![], 1, 10))),
)],
1,
1
))),
)],
1,
1
))),
Val::Tuple(vec![(
Positioned::new("foo".to_string(), 1, 1),
Rc::new(Val::Tuple(vec![(
Positioned::new("bar".to_string(), 1, 10),
Rc::new(Val::Tuple(vec![])),
)])),
)]),
),
(
Expression::Simple(Value::Tuple(value_node!(
vec![(
Token::new("foo", TokenType::BAREWORD, 1, 1),
Expression::Simple(Value::Tuple(value_node!(
vec![(
Token::new("bar".to_string(), TokenType::BAREWORD, 1, 5),
Expression::Simple(Value::Tuple(value_node!(
vec![(
Token::new("quux".to_string(), TokenType::BAREWORD, 1, 1),
Expression::Simple(Value::Int(value_node!(3, 1, 1))),
)],
1,
10
))),
)],
1,
1
))),
)],
1,
1
))),
Val::Tuple(vec![(
Positioned::new("foo".to_string(), 1, 1),
Rc::new(Val::Tuple(vec![(
Positioned::new("bar".to_string(), 1, 10),
Rc::new(Val::Tuple(vec![(
Positioned::new("quux".to_string(), 1, 1),
Rc::new(Val::Int(3)),
)])),
)])),
)]),
),
],
Builder::new(
std::env::current_dir().unwrap(),
Rc::new(RefCell::new(MemoryCache::new())),
),
);
}
#[test]
fn test_eval_simple_expr() {
test_expr_to_val(
vec![
(
Expression::Simple(Value::Int(value_node!(1, 1, 1))),
Val::Int(1),
),
(
Expression::Simple(Value::Float(value_node!(2.0, 1, 1))),
Val::Float(2.0),
),
(
Expression::Simple(Value::Str(value_node!("foo".to_string(), 1, 1))),
Val::Str("foo".to_string()),
),
(
Expression::Simple(Value::Tuple(value_node!(
vec![(
make_tok!("bar", 1, 1),
Expression::Simple(Value::Int(value_node!(1, 1, 1))),
)],
1,
1
))),
Val::Tuple(vec![(
value_node!("bar".to_string(), 1, 1),
Rc::new(Val::Int(1)),
)]),
),
],
Builder::new(
std::env::current_dir().unwrap(),
Rc::new(RefCell::new(MemoryCache::new())),
),
);
}
#[test]
fn test_eval_simple_lookup_expr() {
let cache = Rc::new(RefCell::new(MemoryCache::new()));
let mut b = Builder::new(std::env::current_dir().unwrap(), cache);
b.build_output
.entry(value_node!("var1".to_string(), 1, 0))
.or_insert(Rc::new(Val::Int(1)));
test_expr_to_val(
vec![(
Expression::Simple(Value::Symbol(value_node!("var1".to_string(), 1, 1))),
Val::Int(1),
)],
b,
);
}
#[test]
fn test_eval_simple_lookup_error() {
let cache = Rc::new(RefCell::new(MemoryCache::new()));
let mut b = Builder::new(std::env::current_dir().unwrap(), cache);
b.build_output
.entry(value_node!("var1".to_string(), 1, 0))
.entry(value_node!("var1".to_string(), Position::new(1, 0, 0)))
.or_insert(Rc::new(Val::Int(1)));
let expr = Expression::Simple(Value::Symbol(value_node!("var".to_string(), 1, 1)));
let expr = Expression::Simple(Value::Symbol(value_node!(
"var".to_string(),
Position::new(1, 1, 1)
)));
assert!(b.eval_expr(&expr).is_err());
}
#[test]
fn test_eval_selector_expr() {
let cache = Rc::new(RefCell::new(MemoryCache::new()));
let mut b = Builder::new(std::env::current_dir().unwrap(), cache);
b.build_output
.entry(value_node!("var1".to_string(), 1, 0))
.or_insert(Rc::new(Val::Tuple(vec![(
value_node!("lvl1".to_string(), 1, 0),
Rc::new(Val::Tuple(vec![(
value_node!("lvl2".to_string(), 1, 0),
Rc::new(Val::Int(3)),
)])),
)])));
b.build_output
.entry(value_node!("var2".to_string(), 1, 0))
.or_insert(Rc::new(Val::Int(2)));
b.build_output
.entry(value_node!("var3".to_string(), 1, 0))
.or_insert(Rc::new(Val::Tuple(vec![(
value_node!("lvl1".to_string(), 1, 0),
Rc::new(Val::Int(4)),
)])));
test_expr_to_val(
vec![
(
Expression::Simple(Value::Selector(make_selector!(make_expr!("var1")))),
Val::Tuple(vec![(
value_node!("lvl1".to_string(), 1, 0),
Rc::new(Val::Tuple(vec![(
value_node!("lvl2".to_string(), 1, 0),
Rc::new(Val::Int(3)),
)])),
)]),
),
(
Expression::Simple(Value::Selector(
make_selector!(make_expr!("var1") => "lvl1"),
)),
Val::Tuple(vec![(
value_node!("lvl2".to_string(), 1, 0),
Rc::new(Val::Int(3)),
)]),
),
(
Expression::Simple(Value::Selector(
make_selector!(make_expr!("var1") => "lvl1", "lvl2"),
)),
Val::Int(3),
),
(
Expression::Simple(Value::Selector(make_selector!(make_expr!("var2")))),
Val::Int(2),
),
(
Expression::Simple(Value::Selector(
make_selector!(make_expr!("var3") => "lvl1"),
)),
Val::Int(4),
),
],
b,
);
}
#[test]
fn test_eval_selector_list_expr() {
let cache = Rc::new(RefCell::new(MemoryCache::new()));
let mut b = Builder::new(std::env::current_dir().unwrap(), cache);
b.build_output
.entry(value_node!("var1".to_string(), 1, 1))
.or_insert(Rc::new(Val::List(vec![
Rc::new(Val::Str("val1".to_string())),
Rc::new(Val::Tuple(vec![(
value_node!("var2".to_string(), 1, 1),
Rc::new(Val::Int(1)),
)])),
])));
test_expr_to_val(
vec![(
Expression::Simple(Value::Selector(
make_selector!(make_expr!("var1") => "0" => 1, 1),
)),
Val::Str("val1".to_string()),
)],
b,
);
}
// Include nested for each.
#[test]
#[should_panic(expected = "Unable to find tpl1")]
@ -508,9 +147,12 @@ fn test_expr_copy_no_such_tuple() {
test_expr_to_val(
vec![(
Expression::Copy(CopyDef {
selector: make_selector!(make_expr!("tpl1")),
selector: make_selector!(
make_expr!("tpl1", Position::new(1, 1, 1)),
Position::new(1, 1, 1)
),
fields: Vec::new(),
pos: Position::new(1, 0),
pos: Position::new(1, 0, 0),
}),
Val::Tuple(Vec::new()),
)],
@ -524,14 +166,17 @@ fn test_expr_copy_not_a_tuple() {
let cache = Rc::new(RefCell::new(MemoryCache::new()));
let mut b = Builder::new(std::env::current_dir().unwrap(), cache);
b.build_output
.entry(value_node!("tpl1".to_string(), 1, 0))
.entry(value_node!("tpl1".to_string(), Position::new(1, 0, 0)))
.or_insert(Rc::new(Val::Int(1)));
test_expr_to_val(
vec![(
Expression::Copy(CopyDef {
selector: make_selector!(make_expr!("tpl1")),
selector: make_selector!(
make_expr!("tpl1", Position::new(1, 1, 1)),
Position::new(1, 1, 1)
),
fields: Vec::new(),
pos: Position::new(1, 0),
pos: Position::new(1, 0, 0),
}),
Val::Tuple(Vec::new()),
)],
@ -545,130 +190,30 @@ fn test_expr_copy_field_type_error() {
let cache = Rc::new(RefCell::new(MemoryCache::new()));
let mut b = Builder::new(std::env::current_dir().unwrap(), cache);
b.build_output
.entry(value_node!("tpl1".to_string(), 1, 0))
.entry(value_node!("tpl1".to_string(), Position::new(1, 0, 0)))
.or_insert(Rc::new(Val::Tuple(vec![(
value_node!("fld1".to_string(), 1, 0),
value_node!("fld1".to_string(), Position::new(1, 0, 0)),
Rc::new(Val::Int(1)),
)])));
test_expr_to_val(
vec![(
Expression::Copy(CopyDef {
selector: make_selector!(make_expr!("tpl1")),
fields: vec![(
make_tok!("fld1", 1, 1),
Expression::Simple(Value::Str(value_node!("2".to_string(), 1, 1))),
)],
pos: Position::new(1, 0),
}),
Val::Tuple(vec![(
value_node!("fld1".to_string(), 1, 1),
Rc::new(Val::Str("2".to_string())),
)]),
)],
b,
);
}
#[test]
fn test_expr_copy() {
let cache = Rc::new(RefCell::new(MemoryCache::new()));
let mut b = Builder::new(std::env::current_dir().unwrap(), cache);
b.build_output
.entry(value_node!("tpl1".to_string(), 1, 0))
.or_insert(Rc::new(Val::Tuple(vec![(
value_node!("fld1".to_string(), 1, 0),
Rc::new(Val::Int(1)),
)])));
test_expr_to_val(
vec![
(
Expression::Copy(CopyDef {
selector: make_selector!(make_expr!("tpl1")),
fields: vec![(
make_tok!("fld2", 1, 1),
Expression::Simple(Value::Str(value_node!("2".to_string(), 1, 1))),
)],
pos: Position::new(1, 0),
}),
// Add a new field to the copy
Val::Tuple(
// NOTE(jwall): The order of these is important in order to ensure
// that the compare assertion is correct. The ordering has no
// semantics though so at some point we should probably be less restrictive.
vec![
(value_node!("fld1".to_string(), 1, 0), Rc::new(Val::Int(1))),
(
value_node!("fld2".to_string(), 1, 1),
Rc::new(Val::Str("2".to_string())),
),
],
selector: make_selector!(
make_expr!("tpl1", Position::new(1, 1, 1)),
Position::new(1, 1, 1)
),
),
// Overwrite a field in the copy
(
Expression::Copy(CopyDef {
selector: make_selector!(make_expr!("tpl1")),
fields: vec![
(
make_tok!("fld1", 1, 1),
Expression::Simple(Value::Int(value_node!(3, 1, 1))),
),
(
make_tok!("fld2", 1, 1),
Expression::Simple(Value::Str(value_node!("2".to_string(), 1, 1))),
),
],
pos: Position::new(1, 0),
}),
Val::Tuple(vec![
(value_node!("fld1".to_string(), 1, 0), Rc::new(Val::Int(3))),
(
value_node!("fld2".to_string(), 1, 0),
Rc::new(Val::Str("2".to_string())),
),
]),
),
// The source tuple is still unmodified.
(
Expression::Simple(Value::Selector(make_selector!(make_expr!["tpl1"]))),
Val::Tuple(vec![(
value_node!("fld1".to_string(), 1, 0),
Rc::new(Val::Int(1)),
)]),
),
],
b,
);
}
#[test]
fn test_macro_call() {
let cache = Rc::new(RefCell::new(MemoryCache::new()));
let mut b = Builder::new(std::env::current_dir().unwrap(), cache);
b.build_output
.entry(value_node!("tstmac".to_string(), 1, 0))
.or_insert(Rc::new(Val::Macro(MacroDef {
argdefs: vec![value_node!("arg1".to_string(), 1, 0)],
fields: vec![(
make_tok!("foo", 1, 1),
Expression::Simple(Value::Symbol(value_node!("arg1".to_string(), 1, 1))),
)],
pos: Position::new(1, 0),
})));
test_expr_to_val(
vec![(
Expression::Call(CallDef {
macroref: make_selector!(make_expr!("tstmac")),
arglist: vec![Expression::Simple(Value::Str(value_node!(
"bar".to_string(),
1,
1
)))],
pos: Position::new(1, 0),
fields: vec![(
make_tok!("fld1", Position::new(1, 1, 1)),
Expression::Simple(Value::Str(value_node!(
"2".to_string(),
Position::new(1, 1, 1)
))),
)],
pos: Position::new(1, 0, 0),
}),
Val::Tuple(vec![(
value_node!("foo".to_string(), 1, 1),
Rc::new(Val::Str("bar".to_string())),
value_node!("fld1".to_string(), Position::new(1, 1, 1)),
Rc::new(Val::Str("2".to_string())),
)]),
)],
b,
@ -681,31 +226,36 @@ fn test_macro_hermetic() {
let cache = Rc::new(RefCell::new(MemoryCache::new()));
let mut b = Builder::new(std::env::current_dir().unwrap(), cache);
b.build_output
.entry(value_node!("arg1".to_string(), 1, 0))
.entry(value_node!("arg1".to_string(), Position::new(1, 0, 0)))
.or_insert(Rc::new(Val::Str("bar".to_string())));
b.build_output
.entry(value_node!("tstmac".to_string(), 1, 0))
.entry(value_node!("tstmac".to_string(), Position::new(1, 0, 0)))
.or_insert(Rc::new(Val::Macro(MacroDef {
argdefs: vec![value_node!("arg2".to_string(), 1, 0)],
argdefs: vec![value_node!("arg2".to_string(), Position::new(1, 0, 0))],
fields: vec![(
make_tok!("foo", 1, 1),
Expression::Simple(Value::Symbol(value_node!("arg1".to_string(), 1, 1))),
make_tok!("foo", Position::new(1, 1, 1)),
Expression::Simple(Value::Symbol(value_node!(
"arg1".to_string(),
Position::new(1, 1, 1)
))),
)],
pos: Position::new(1, 0),
pos: Position::new(1, 0, 0),
})));
test_expr_to_val(
vec![(
Expression::Call(CallDef {
macroref: make_selector!(make_expr!("tstmac")),
macroref: make_selector!(
make_expr!("tstmac", Position::new(1, 1, 1)),
Position::new(1, 1, 1)
),
arglist: vec![Expression::Simple(Value::Str(value_node!(
"bar".to_string(),
1,
1
Position::new(1, 1, 1)
)))],
pos: Position::new(1, 1),
pos: Position::new(1, 1, 1),
}),
Val::Tuple(vec![(
value_node!("foo".to_string(), 1, 0),
value_node!("foo".to_string(), Position::new(1, 1, 1)),
Rc::new(Val::Str("bar".to_string())),
)]),
)],
@ -713,126 +263,42 @@ fn test_macro_hermetic() {
);
}
#[test]
fn test_select_expr() {
let cache = Rc::new(RefCell::new(MemoryCache::new()));
let mut b = Builder::new(std::env::current_dir().unwrap(), cache);
b.build_output
.entry(value_node!("foo".to_string(), 1, 0))
.or_insert(Rc::new(Val::Str("bar".to_string())));
b.build_output
.entry(value_node!("baz".to_string(), 1, 0))
.or_insert(Rc::new(Val::Str("boo".to_string())));
test_expr_to_val(
vec![
(
Expression::Select(SelectDef {
val: Box::new(Expression::Simple(Value::Symbol(value_node!(
"foo".to_string(),
1,
1
)))),
default: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
tuple: vec![
(
make_tok!("foo", 1, 1),
Expression::Simple(Value::Str(value_node!("2".to_string(), 1, 1))),
),
(
make_tok!("bar", 1, 1),
Expression::Simple(Value::Int(value_node!(2, 1, 1))),
),
],
pos: Position::new(1, 0),
}),
Val::Int(2),
),
(
Expression::Select(SelectDef {
val: Box::new(Expression::Simple(Value::Symbol(value_node!(
"baz".to_string(),
1,
1
)))),
default: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
tuple: vec![
(
make_tok!("bar", 1, 1),
Expression::Simple(Value::Int(value_node!(2, 1, 1))),
),
(
make_tok!("quux", 1, 1),
Expression::Simple(Value::Str(value_node!("2".to_string(), 1, 1))),
),
],
pos: Position::new(1, 0),
}),
// If the field doesn't exist then we get the default.
Val::Int(1),
),
],
b,
);
}
#[test]
#[should_panic(expected = "Expected String but got Integer in Select expression")]
fn test_select_expr_not_a_string() {
let cache = Rc::new(RefCell::new(MemoryCache::new()));
let mut b = Builder::new(std::env::current_dir().unwrap(), cache);
b.build_output
.entry(value_node!("foo".to_string(), 1, 0))
.entry(value_node!("foo".to_string(), Position::new(1, 0, 0)))
.or_insert(Rc::new(Val::Int(4)));
test_expr_to_val(
vec![(
Expression::Select(SelectDef {
val: Box::new(Expression::Simple(Value::Symbol(value_node!(
"foo".to_string(),
1,
1
Position::new(1, 1, 1)
)))),
default: Box::new(Expression::Simple(Value::Int(value_node!(
1,
Position::new(1, 1, 1)
)))),
default: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
tuple: vec![
(
make_tok!("bar", 1, 1),
Expression::Simple(Value::Int(value_node!(2, 1, 1))),
make_tok!("bar", Position::new(1, 1, 1)),
Expression::Simple(Value::Int(value_node!(2, Position::new(1, 1, 1)))),
),
(
make_tok!("quux", 1, 1),
Expression::Simple(Value::Str(value_node!("2".to_string(), 1, 1))),
make_tok!("quux", Position::new(1, 1, 1)),
Expression::Simple(Value::Str(value_node!(
"2".to_string(),
Position::new(1, 1, 1)
))),
),
],
pos: Position::new(1, 0),
pos: Position::new(1, 0, 0),
}),
Val::Int(2),
)],
b,
);
}
#[test]
fn test_let_statement() {
let cache = MemoryCache::new();
let mut b = Builder::new("<Eval>", Rc::new(RefCell::new(cache)));
let stmt = Statement::Let(LetDef {
name: make_tok!("foo", 1, 1),
value: Expression::Simple(Value::Str(value_node!("bar".to_string(), 1, 1))),
});
b.build_stmt(&stmt).unwrap();
test_expr_to_val(
vec![(
Expression::Simple(Value::Symbol(value_node!("foo".to_string(), 1, 1))),
Val::Str("bar".to_string()),
)],
b,
);
}
#[test]
fn test_build_file_string() {
let cache = Rc::new(RefCell::new(MemoryCache::new()));
let mut b = Builder::new(std::env::current_dir().unwrap(), cache);
b.eval_string("let foo = 1;").unwrap();
let key = value_node!("foo".to_string(), 1, 0);
assert!(b.build_output.contains_key(&key));
}

8
src/convert/exec.rs
View File

@ -46,7 +46,7 @@ impl ExecConverter {
return Err(Box::new(Error::new(
"Exec tuples must have no more than 3 fields",
ErrorType::TypeFail,
Position::new(0, 0),
Position::new(0, 0, 0),
)));
}
let mut env: Option<&Vec<(Positioned<String>, Rc<Val>)>> = None;
@ -115,7 +115,7 @@ impl ExecConverter {
return Err(Box::new(Error::new(
"An exec tuple must have a command field",
ErrorType::TypeFail,
Position::new(0, 0),
Position::new(0, 0, 0),
)));
}
// Okay if we have made it this far then we are ready to start creating our script.
@ -157,7 +157,7 @@ impl ExecConverter {
return Err(Box::new(Error::new(
"Exec args must be a list of strings or tuples of strings.",
ErrorType::TypeFail,
Position::new(0, 0),
Position::new(0, 0, 0),
)))
}
}
@ -173,7 +173,7 @@ impl ExecConverter {
Err(Box::new(Error::new(
"Exec outputs must be of type Tuple",
ErrorType::TypeFail,
Position::new(0, 0),
Position::new(0, 0, 0),
)))
}
}

3
src/error.rs
View File

@ -80,7 +80,8 @@ impl Error {
t: ErrorType,
cause: Box<error::Error>,
) -> Self {
let mut e = Self::new(msg, t, Position { line: 0, column: 0 });
// TODO(jwall): This should take a real position instead of this fake one.
let mut e = Self::new(msg, t, Position::new(0, 0, 0));
e.cause = Some(cause);
return e;
}

6
src/format.rs
View File

@ -82,21 +82,21 @@ mod test {
#[test]
fn test_format_happy_path() {
let formatter = Formatter::new("foo @ @ \\@", vec!["bar", "quux"]);
let pos = Position { line: 0, column: 0 };
let pos = Position::new(0, 0, 0);
assert_eq!(formatter.render(&pos).unwrap(), "foo bar quux @");
}
#[test]
fn test_format_happy_wrong_too_few_args() {
let formatter = Formatter::new("foo @ @ \\@", vec!["bar"]);
let pos = Position { line: 0, column: 0 };
let pos = Position::new(0, 0, 0);
assert!(formatter.render(&pos).is_err());
}
#[test]
fn test_format_happy_wrong_too_many_args() {
let formatter = Formatter::new("foo @ @ \\@", vec!["bar", "quux", "baz"]);
let pos = Position { line: 0, column: 0 };
let pos = Position::new(0, 0, 0);
assert!(formatter.render(&pos).is_err());
}
}

103
src/iter.rs Normal file
View File

@ -0,0 +1,103 @@
use std::convert::From;
use std::iter::Iterator;
use abortable_parser::iter::StrIter;
use abortable_parser::{
InputIter, Offsetable, Peekable, Seekable, Span, SpanRange, TextPositionTracker,
};
#[derive(Debug)]
pub struct OffsetStrIter<'a> {
contained: StrIter<'a>,
idx_offset: usize,
line_offset: usize,
col_offset: usize,
}
impl<'a> OffsetStrIter<'a> {
pub fn new(input: &'a str) -> Self {
Self::new_with_offsets(input, 0, 0, 0)
}
pub fn new_with_offsets(
input: &'a str,
idx_offset: usize,
line_offset: usize,
col_offset: usize,
) -> Self {
OffsetStrIter {
contained: StrIter::new(input),
idx_offset: idx_offset,
line_offset: line_offset,
col_offset: col_offset,
}
}
}
impl<'a> Iterator for OffsetStrIter<'a> {
type Item = &'a u8;
fn next(&mut self) -> Option<Self::Item> {
self.contained.next()
}
}
impl<'a> Offsetable for OffsetStrIter<'a> {
fn get_offset(&self) -> usize {
self.contained.get_offset() + self.idx_offset
}
}
impl<'a> Clone for OffsetStrIter<'a> {
fn clone(&self) -> Self {
OffsetStrIter {
contained: self.contained.clone(),
idx_offset: self.idx_offset,
line_offset: self.line_offset,
col_offset: self.col_offset,
}
}
}
impl<'a> From<&'a str> for OffsetStrIter<'a> {
fn from(source: &'a str) -> Self {
OffsetStrIter {
contained: StrIter::new(source),
idx_offset: 0,
line_offset: 0,
col_offset: 0,
}
}
}
impl<'a> Seekable for OffsetStrIter<'a> {
fn seek(&mut self, to: usize) -> usize {
let contained_offset = self.contained.seek(to);
self.idx_offset += contained_offset;
contained_offset
}
}
impl<'a> Span<&'a str> for OffsetStrIter<'a> {
fn span(&self, idx: SpanRange) -> &'a str {
self.contained.span(idx)
}
}
impl<'a> Peekable<&'a u8> for OffsetStrIter<'a> {
fn peek_next(&self) -> Option<&'a u8> {
self.contained.peek_next()
}
}
impl<'a> TextPositionTracker for OffsetStrIter<'a> {
fn line(&self) -> usize {
self.contained.line() + self.line_offset
}
fn column(&self) -> usize {
self.contained.column() + self.col_offset
}
}
impl<'a> InputIter for OffsetStrIter<'a> {}

1
src/lib.rs
View File

@ -456,6 +456,7 @@ pub mod tokenizer;
pub mod build;
pub mod convert;
pub mod error;
pub mod iter;
pub mod parse;
mod format;

74
src/parse/mod.rs
View File

@ -19,16 +19,16 @@ use std::str::FromStr;
use abortable_parser;
use abortable_parser::combinators::eoi;
use abortable_parser::iter::{SliceIter, StrIter};
use abortable_parser::iter::SliceIter;
use abortable_parser::{Error, Peekable, Result};
use self::precedence::op_expression;
use ast::*;
use iter::OffsetStrIter;
use tokenizer::*;
type NomResult<'a, O> = Result<SliceIter<'a, Token>, O>;
// FIXME(jwall): All the do_each mappers need to return an actual value.
#[cfg(feature = "tracing")]
const ENABLE_TRACE: bool = true;
#[cfg(not(feature = "tracing"))]
@ -92,11 +92,10 @@ make_fn!(
match_type!(STR => str_to_value)
);
// FIXME(jwall): We need to just turn this into a custom parser function.
// Helper function to make the return types work for down below.
fn triple_to_number(v: (Option<Token>, Option<Token>, Option<Token>)) -> ParseResult<Value> {
let (pref, mut pref_pos) = match v.0 {
None => ("", Position::new(0, 0)),
None => ("", Position::new(0, 0, 0)),
Some(ref bs) => (bs.fragment.borrow(), bs.pos.clone()),
};
@ -106,11 +105,7 @@ fn triple_to_number(v: (Option<Token>, Option<Token>, Option<Token>)) -> ParseRe
let i = match FromStr::from_str(pref) {
Ok(i) => i,
Err(_) => {
return Err(Error::new(
format!("Not an integer! {}", pref),
// FIXME(jwall): This really needs the correct offset.
&0,
));
return Err(Error::new(format!("Not an integer! {}", pref), &0));
}
};
return Ok(Value::Int(value_node!(i, pref_pos)));
@ -120,9 +115,9 @@ fn triple_to_number(v: (Option<Token>, Option<Token>, Option<Token>)) -> ParseRe
pref_pos = v.1.unwrap().pos;
}
let suf = match v.2 {
None => "".to_string(),
Some(bs) => bs.fragment,
let (suf, pos) = match v.2 {
None => ("".to_string(), Position::new(0, 0, 0)),
Some(bs) => (bs.fragment, bs.pos),
};
let to_parse = pref.to_string() + "." + &suf;
@ -131,8 +126,7 @@ fn triple_to_number(v: (Option<Token>, Option<Token>, Option<Token>)) -> ParseRe
Err(_) => {
return Err(Error::new(
format!("Not a float! {}", to_parse),
// FIXME(jwall): This should take the real offset.
&0,
&pos.offset,
));
}
};
@ -307,11 +301,7 @@ make_fn!(
// Helper function to make the return types work for down below.
fn vec_to_tuple(pos: Position, fields: Option<FieldList>) -> Value {
Value::Tuple(value_node!(
fields.unwrap_or(Vec::new()),
pos.line as usize,
pos.column as usize
))
Value::Tuple(value_node!(fields.unwrap_or(Vec::new()), pos))
}
make_fn!(
@ -519,11 +509,10 @@ make_fn!(
fields => optional!(trace_nom!(field_list)),
_ => optional!(punct!(",")), // noms opt! macro does not preserve error types properly but this one does.
_ => punct!("}"),
(tuple_to_copy(SelectorDef::new(selector, pos.line, pos.column), fields))
(tuple_to_copy(SelectorDef::new(selector, pos), fields))
)
);
// FIXME(jwall): need to make this into a proper parse function.
fn tuple_to_macro(pos: Position, vals: Option<Vec<Value>>, val: Value) -> ParseResult<Expression> {
let mut default_args = match vals {
None => Vec::new(),
@ -543,8 +532,7 @@ fn tuple_to_macro(pos: Position, vals: Option<Vec<Value>>, val: Value) -> ParseR
})),
val => Err(Error::new(
format!("Expected Tuple Got {:?}", val),
// FIXME(jwall): Should have correct Offset here.
&0,
&val.pos().offset,
)),
}
}
@ -647,7 +635,6 @@ make_fn!(
)
);
// FIXME(jwall): Convert this into a custom parser function.
fn tuple_to_call(pos: Position, val: Value, exprs: Vec<Expression>) -> ParseResult<Expression> {
if let Value::Selector(def) = val {
Ok(Expression::Call(CallDef {
@ -656,17 +643,15 @@ fn tuple_to_call(pos: Position, val: Value, exprs: Vec<Expression>) -> ParseResu
pos: pos,
}))
} else {
// FIXME(jwall): Should get correct offset here.
Err(Error::new(format!("Expected Selector Got {:?}", val), &0))
Err(Error::new(
format!("Expected Selector Got {:?}", val),
&val.pos().offset,
))
}
}
fn vec_to_selector_value(pos: Position, list: SelectorList) -> Value {
Value::Selector(SelectorDef::new(
list,
pos.line as usize,
pos.column as usize,
))
Value::Selector(SelectorDef::new(list, pos))
}
make_fn!(
@ -715,8 +700,7 @@ fn tuple_to_list_op(
}
return Err(Error::new(
format!("Missing a result field for the macro"),
// FIXME(jwall): Should have correct offset.
&0,
&def.pos.offset,
));
}
&mut Some(ref mut tl) => {
@ -729,8 +713,7 @@ fn tuple_to_list_op(
}
return Err(Error::new(
format!("Missing a result field for the macro"),
// FIXME(jwall): Should have correct offset.
&0,
&def.pos.offset,
));
}
let fname = tl.pop();
@ -753,12 +736,10 @@ fn tuple_to_list_op(
}
return Err(Error::new(
format!("Expected a selector but got {}", macroname.type_name()),
// FIXME(jwall): Should have correct offset.
&0,
&pos.offset,
));
}
// FIXME(jwall): need to make this a custom function to parse it.
make_fn!(
list_op_expression<SliceIter<Token>, Expression>,
do_each!(
@ -843,7 +824,7 @@ make_fn!(
let_statement<SliceIter<Token>, Statement>,
do_each!(
_ => word!("let"),
stmt => trace_nom!(let_stmt_body),
stmt => trace_nom!(must!(let_stmt_body)),
(stmt)
)
);
@ -871,7 +852,7 @@ make_fn!(
do_each!(
_ => word!("import"),
// past this point we know this is supposed to be an import statement.
stmt => trace_nom!(import_stmt_body),
stmt => trace_nom!(must!(import_stmt_body)),
(stmt)
)
);
@ -880,8 +861,8 @@ make_fn!(
assert_statement<SliceIter<Token>, Statement>,
do_each!(
_ => word!("assert"),
tok => match_type!(PIPEQUOTE),
_ => punct!(";"),
tok => must!(match_type!(PIPEQUOTE)),
_ => must!(punct!(";")),
(Statement::Assert(tok.clone()))
)
);
@ -890,9 +871,9 @@ make_fn!(
out_statement<SliceIter<Token>, Statement>,
do_each!(
_ => word!("out"),
typ => match_type!(BAREWORD),
expr => expression,
_ => punct!(";"),
typ => must!(match_type!(BAREWORD)),
expr => must!(expression),
_ => must!(punct!(";")),
(Statement::Output(typ.clone(), expr.clone()))
)
);
@ -910,7 +891,7 @@ fn statement(i: SliceIter<Token>) -> Result<SliceIter<Token>, Statement> {
//trace_macros!(false);
/// Parses a LocatedSpan into a list of Statements or an `abortable_parser::Error`.
pub fn parse(input: StrIter) -> std::result::Result<Vec<Statement>, Error> {
pub fn parse(input: OffsetStrIter) -> std::result::Result<Vec<Statement>, Error> {
match tokenize(input.clone()) {
Ok(tokenized) => {
let mut out = Vec::new();
@ -922,6 +903,7 @@ pub fn parse(input: StrIter) -> std::result::Result<Vec<Statement>, Error> {
break;
}
}
// FIXME(jwall): We need to return a error::Error so we have position information.
match statement(i.clone()) {
Result::Abort(e) => {
return Err(e);

1054
src/parse/test.rs
View File

File diff suppressed because it is too large Load Diff

160
src/tokenizer/mod.rs
View File

@ -16,20 +16,19 @@
use std;
use abortable_parser::combinators::*;
use abortable_parser::iter::{SliceIter, StrIter};
use abortable_parser::iter::SliceIter;
use abortable_parser::{Error, Offsetable, Result, TextPositionTracker};
use ast::*;
impl<'a> From<StrIter<'a>> for Position {
fn from(s: StrIter<'a>) -> Position {
Position {
line: s.line(),
column: s.column(),
}
use ast::*;
use iter::OffsetStrIter;
impl<'a> From<OffsetStrIter<'a>> for Position {
fn from(s: OffsetStrIter<'a>) -> Position {
Position::new(s.line(), s.column(), s.get_offset())
}
}
fn is_symbol_char<'a>(i: StrIter<'a>) -> Result<StrIter<'a>, u8> {
fn is_symbol_char<'a>(i: OffsetStrIter<'a>) -> Result<OffsetStrIter<'a>, u8> {
let mut _i = i.clone();
let c = match _i.next() {
Some(c) => *c,
@ -42,7 +41,7 @@ fn is_symbol_char<'a>(i: StrIter<'a>) -> Result<StrIter<'a>, u8> {
}
}
fn escapequoted<'a>(input: StrIter<'a>) -> Result<StrIter<'a>, String> {
fn escapequoted<'a>(input: OffsetStrIter<'a>) -> Result<OffsetStrIter<'a>, String> {
// loop until we find a " that is not preceded by \.
// Collapse all \<char> to just char for escaping.
let mut frag = String::new();
@ -69,7 +68,7 @@ fn escapequoted<'a>(input: StrIter<'a>) -> Result<StrIter<'a>, String> {
return Result::Incomplete(_input.get_offset());
}
make_fn!(strtok<StrIter, Token>,
make_fn!(strtok<OffsetStrIter, Token>,
do_each!(
span => input!(),
_ => text_token!("\""),
@ -82,7 +81,7 @@ make_fn!(strtok<StrIter, Token>,
)
);
make_fn!(pipequotetok<StrIter, Token>,
make_fn!(pipequotetok<OffsetStrIter, Token>,
do_each!(
p => input!(),
_ => text_token!("|"),
@ -96,7 +95,7 @@ make_fn!(pipequotetok<StrIter, Token>,
)
);
make_fn!(barewordtok<StrIter, Token>,
make_fn!(barewordtok<OffsetStrIter, Token>,
do_each!(
span => input!(),
_ => peek!(ascii_alpha),
@ -109,7 +108,7 @@ make_fn!(barewordtok<StrIter, Token>,
)
);
make_fn!(digittok<StrIter, Token>,
make_fn!(digittok<OffsetStrIter, Token>,
do_each!(
span => input!(),
_ => peek!(ascii_digit),
@ -122,7 +121,7 @@ make_fn!(digittok<StrIter, Token>,
)
);
make_fn!(booleantok<StrIter, Token>,
make_fn!(booleantok<OffsetStrIter, Token>,
do_each!(
span => input!(),
token => either!(
@ -142,159 +141,159 @@ make_fn!(booleantok<StrIter, Token>,
macro_rules! do_text_token_tok {
($i:expr, $type:expr, $text_token:expr, WS) => {
do_each!($i,
span => input!(),
frag => text_token!($text_token),
_ => either!(whitespace, comment),
(Token {
typ: $type,
pos: Position::from(span),
fragment: frag.to_string(),
})
)
span => input!(),
frag => text_token!($text_token),
_ => either!(whitespace, comment),
(Token {
typ: $type,
pos: Position::from(span),
fragment: frag.to_string(),
})
)
};
($i:expr, $type:expr, $text_token:expr) => {
do_each!($i,
span => input!(),
frag => text_token!($text_token),
(Token {
typ: $type,
pos: Position::from(span),
fragment: frag.to_string(),
})
)
span => input!(),
frag => text_token!($text_token),
(Token {
typ: $type,
pos: Position::from(span),
fragment: frag.to_string(),
})
)
};
}
make_fn!(emptytok<StrIter, Token>,
make_fn!(emptytok<OffsetStrIter, Token>,
do_text_token_tok!(TokenType::EMPTY, "NULL")
);
make_fn!(commatok<StrIter, Token>,
make_fn!(commatok<OffsetStrIter, Token>,
do_text_token_tok!(TokenType::PUNCT, ",")
);
make_fn!(lbracetok<StrIter, Token>,
make_fn!(lbracetok<OffsetStrIter, Token>,
do_text_token_tok!(TokenType::PUNCT, "{")
);
make_fn!(rbracetok<StrIter, Token>,
make_fn!(rbracetok<OffsetStrIter, Token>,
do_text_token_tok!(TokenType::PUNCT, "}")
);
make_fn!(lparentok<StrIter, Token>,
make_fn!(lparentok<OffsetStrIter, Token>,
do_text_token_tok!(TokenType::PUNCT, "(")
);
make_fn!(rparentok<StrIter, Token>,
make_fn!(rparentok<OffsetStrIter, Token>,
do_text_token_tok!(TokenType::PUNCT, ")")
);
make_fn!(dottok<StrIter, Token>,
make_fn!(dottok<OffsetStrIter, Token>,
do_text_token_tok!(TokenType::PUNCT, ".")
);
make_fn!(plustok<StrIter, Token>,
make_fn!(plustok<OffsetStrIter, Token>,
do_text_token_tok!(TokenType::PUNCT, "+")
);
make_fn!(dashtok<StrIter, Token>,
make_fn!(dashtok<OffsetStrIter, Token>,
do_text_token_tok!(TokenType::PUNCT, "-")
);
make_fn!(startok<StrIter, Token>,
make_fn!(startok<OffsetStrIter, Token>,
do_text_token_tok!(TokenType::PUNCT, "*")
);
make_fn!(slashtok<StrIter, Token>,
make_fn!(slashtok<OffsetStrIter, Token>,
do_text_token_tok!(TokenType::PUNCT, "/")
);
make_fn!(pcttok<StrIter, Token>,
make_fn!(pcttok<OffsetStrIter, Token>,
do_text_token_tok!(TokenType::PUNCT, "%")
);
make_fn!(eqeqtok<StrIter, Token>,
make_fn!(eqeqtok<OffsetStrIter, Token>,
do_text_token_tok!(TokenType::PUNCT, "==")
);
make_fn!(notequaltok<StrIter, Token>,
make_fn!(notequaltok<OffsetStrIter, Token>,
do_text_token_tok!(TokenType::PUNCT, "!=")
);
make_fn!(gttok<StrIter, Token>,
make_fn!(gttok<OffsetStrIter, Token>,
do_text_token_tok!(TokenType::PUNCT, ">")
);
make_fn!(gtequaltok<StrIter, Token>,
make_fn!(gtequaltok<OffsetStrIter, Token>,
do_text_token_tok!(TokenType::PUNCT, ">=")
);
make_fn!(ltequaltok<StrIter, Token>,
make_fn!(ltequaltok<OffsetStrIter, Token>,
do_text_token_tok!(TokenType::PUNCT, "<=")
);
make_fn!(lttok<StrIter, Token>,
make_fn!(lttok<OffsetStrIter, Token>,
do_text_token_tok!(TokenType::PUNCT, "<")
);
make_fn!(equaltok<StrIter, Token>,
make_fn!(equaltok<OffsetStrIter, Token>,
do_text_token_tok!(TokenType::PUNCT, "=")
);
make_fn!(semicolontok<StrIter, Token>,
make_fn!(semicolontok<OffsetStrIter, Token>,
do_text_token_tok!(TokenType::PUNCT, ";")
);
make_fn!(leftsquarebracket<StrIter, Token>,
make_fn!(leftsquarebracket<OffsetStrIter, Token>,
do_text_token_tok!(TokenType::PUNCT, "[")
);
make_fn!(rightsquarebracket<StrIter, Token>,
make_fn!(rightsquarebracket<OffsetStrIter, Token>,
do_text_token_tok!(TokenType::PUNCT, "]")
);
make_fn!(fatcommatok<StrIter, Token>,
make_fn!(fatcommatok<OffsetStrIter, Token>,
do_text_token_tok!(TokenType::PUNCT, "=>")
);
make_fn!(selecttok<StrIter, Token>,
make_fn!(selecttok<OffsetStrIter, Token>,
do_text_token_tok!(TokenType::BAREWORD, "select", WS)
);
make_fn!(macrotok<StrIter, Token>,
make_fn!(macrotok<OffsetStrIter, Token>,
do_text_token_tok!(TokenType::BAREWORD, "macro", WS)
);
make_fn!(lettok<StrIter, Token>,
make_fn!(lettok<OffsetStrIter, Token>,
do_text_token_tok!(TokenType::BAREWORD, "let", WS)
);
make_fn!(importtok<StrIter, Token>,
make_fn!(importtok<OffsetStrIter, Token>,
do_text_token_tok!(TokenType::BAREWORD, "import", WS)
);
make_fn!(asserttok<StrIter, Token>,
make_fn!(asserttok<OffsetStrIter, Token>,
do_text_token_tok!(TokenType::BAREWORD, "assert", WS)
);
make_fn!(outtok<StrIter, Token>,
make_fn!(outtok<OffsetStrIter, Token>,
do_text_token_tok!(TokenType::BAREWORD, "out", WS)
);
make_fn!(astok<StrIter, Token>,
make_fn!(astok<OffsetStrIter, Token>,
do_text_token_tok!(TokenType::BAREWORD, "as", WS)
);
make_fn!(maptok<StrIter, Token>,
make_fn!(maptok<OffsetStrIter, Token>,
do_text_token_tok!(TokenType::BAREWORD, "map", WS)
);
make_fn!(filtertok<StrIter, Token>,
make_fn!(filtertok<OffsetStrIter, Token>,
do_text_token_tok!(TokenType::BAREWORD, "filter", WS)
);
fn comment(input: StrIter) -> Result<StrIter, Token> {
fn comment(input: OffsetStrIter) -> Result<OffsetStrIter, Token> {
match text_token!(input, "//") {
Result::Complete(rest, _) => {
match until!(
@ -306,12 +305,7 @@ fn comment(input: StrIter) -> Result<StrIter, Token> {
)
) {
Result::Complete(rest, cmt) => {
return Result::Complete(
rest,
make_tok!(CMT => cmt.to_string(),
input.line() as usize,
input.column() as usize),
);
return Result::Complete(rest, make_tok!(CMT => cmt.to_string(), input));
}
// If we didn't find a new line then we just grab everything.
_ => {
@ -325,7 +319,7 @@ fn comment(input: StrIter) -> Result<StrIter, Token> {
}
}
make_fn!(whitespace<StrIter, Token>,
make_fn!(whitespace<OffsetStrIter, Token>,
do_each!(
span => input!(),
_ => peek!(ascii_ws),
@ -338,7 +332,7 @@ make_fn!(whitespace<StrIter, Token>,
)
);
make_fn!(end_of_input<StrIter, Token>,
make_fn!(end_of_input<OffsetStrIter, Token>,
do_each!(
span => input!(),
_ => eoi,
@ -350,7 +344,7 @@ make_fn!(end_of_input<StrIter, Token>,
)
);
make_fn!(token<StrIter, Token>,
make_fn!(token<OffsetStrIter, Token>,
either!(
strtok,
pipequotetok,
@ -394,14 +388,15 @@ make_fn!(token<StrIter, Token>,
end_of_input)
);
/// Consumes an input StrIter and returns either a Vec<Token> or a error::Error.
pub fn tokenize(input: StrIter) -> std::result::Result<Vec<Token>, Error> {
/// Consumes an input OffsetStrIter and returns either a Vec<Token> or a error::Error.
pub fn tokenize(input: OffsetStrIter) -> std::result::Result<Vec<Token>, Error> {
let mut out = Vec::new();
let mut i = input.clone();
loop {
if let Result::Complete(_, _) = eoi(i.clone()) {
break;
}
// FIXME(jwall): We need to return a error::Error so we have position information.
match token(i.clone()) {
Result::Abort(e) => {
return Err(Error::caused_by(
@ -434,10 +429,7 @@ pub fn tokenize(input: StrIter) -> std::result::Result<Vec<Token>, Error> {
out.push(Token {
fragment: String::new(),
typ: TokenType::END,
pos: Position {
line: i.line(),
column: i.column(),
},
pos: i.into(),
});
Ok(out)
}
@ -616,13 +608,7 @@ pub fn pos<'a>(i: SliceIter<'a, Token>) -> Result<SliceIter<'a, Token>, Position
let tok = _i.next().unwrap();
let line = tok.pos.line;
let column = tok.pos.column;
Result::Complete(
i.clone(),
Position {
line: line,
column: column,
},
)
Result::Complete(i.clone(), Position::new(line, column, i.get_offset()))
}
#[cfg(test)]

90
src/tokenizer/test.rs
View File

@ -1,10 +1,12 @@
use super::*;
use abortable_parser::{Result, SliceIter, StrIter};
use abortable_parser::{Result, SliceIter};
use iter::OffsetStrIter;
#[test]
fn test_empty_token() {
let result = emptytok(StrIter::new("NULL "));
let result = emptytok(OffsetStrIter::new("NULL "));
assert!(
result.is_complete(),
format!("result {:?} is not done", result)
@ -17,7 +19,7 @@ fn test_empty_token() {
#[test]
fn test_assert_token() {
let result = asserttok(StrIter::new("assert "));
let result = asserttok(OffsetStrIter::new("assert "));
assert!(
result.is_complete(),
format!("result {:?} is not done", result)
@ -30,7 +32,7 @@ fn test_assert_token() {
#[test]
fn test_out_token() {
let result = outtok(StrIter::new("out "));
let result = outtok(OffsetStrIter::new("out "));
assert!(
result.is_complete(),
format!("result {:?} is not done", result)
@ -43,7 +45,7 @@ fn test_out_token() {
#[test]
fn test_out_token_with_comment() {
let result = outtok(StrIter::new("out//comment"));
let result = outtok(OffsetStrIter::new("out//comment"));
assert!(
result.is_complete(),
format!("result {:?} is not done", result)
@ -56,13 +58,13 @@ fn test_out_token_with_comment() {
#[test]
fn test_not_out_token() {
let result = outtok(StrIter::new("output"));
let result = outtok(OffsetStrIter::new("output"));
assert!(result.is_fail(), format!("result {:?} is not fail", result));
}
#[test]
fn test_escape_quoted() {
let result = escapequoted(StrIter::new("foo \\\"bar\""));
let result = escapequoted(OffsetStrIter::new("foo \\\"bar\""));
assert!(
result.is_complete(),
format!("result {:?} is not ok", result)
@ -74,7 +76,7 @@ fn test_escape_quoted() {
#[test]
fn test_pipe_quoted() {
let result = pipequotetok(StrIter::new("|foo|"));
let result = pipequotetok(OffsetStrIter::new("|foo|"));
assert!(
result.is_complete(),
format!("result {:?} is not ok", result)
@ -87,7 +89,7 @@ fn test_pipe_quoted() {
#[test]
fn test_string_with_escaping() {
let result = strtok(StrIter::new("\"foo \\\\ \\\"bar\""));
let result = strtok(OffsetStrIter::new("\"foo \\\\ \\\"bar\""));
assert!(
result.is_complete(),
format!("result {:?} is not ok", result)
@ -99,7 +101,7 @@ fn test_string_with_escaping() {
#[test]
fn test_tokenize_bareword_with_dash() {
let result = tokenize(StrIter::new("foo-bar "));
let result = tokenize(OffsetStrIter::new("foo-bar "));
assert!(result.is_ok(), format!("result {:?} is not ok", result));
if let Ok(toks) = result {
assert_eq!(toks.len(), 2);
@ -109,7 +111,7 @@ fn test_tokenize_bareword_with_dash() {
macro_rules! assert_token {
($input:expr, $typ:expr, $msg:expr) => {
let result = token(StrIter::new($input));
let result = token(OffsetStrIter::new($input));
assert!(
result.is_complete(),
format!("result {:?} is not a {}", result, $msg)
@ -163,7 +165,7 @@ fn test_lteqtok() {
#[test]
fn test_tokenize_one_of_each() {
let result = tokenize(StrIter::new(
let result = tokenize(OffsetStrIter::new(
"map out filter assert let import macro select as => [ ] { } ; = % / * \
+ - . ( ) , 1 . foo \"bar\" // comment\n ; true false == < > <= >= !=",
));
@ -178,7 +180,7 @@ fn test_tokenize_one_of_each() {
#[test]
fn test_parse_has_end() {
let result = tokenize(StrIter::new("foo"));
let result = tokenize(OffsetStrIter::new("foo"));
assert!(result.is_ok());
let v = result.unwrap();
assert_eq!(v.len(), 2);
@ -187,8 +189,8 @@ fn test_parse_has_end() {
#[test]
fn test_whitespace() {
assert!(whitespace(StrIter::new(" ")).is_complete());
let result = whitespace(StrIter::new(" "));
assert!(whitespace(OffsetStrIter::new(" ")).is_complete());
let result = whitespace(OffsetStrIter::new(" "));
match result {
Result::Complete(rest, o) => {
assert_eq!(rest.get_offset(), 2);
@ -200,9 +202,9 @@ fn test_whitespace() {
#[test]
fn test_parse_comment() {
assert!(comment(StrIter::new("// comment\n")).is_complete());
assert!(comment(StrIter::new("// comment")).is_complete());
let mut parsed = comment(StrIter::new("// comment\n"));
assert!(comment(OffsetStrIter::new("// comment\n")).is_complete());
assert!(comment(OffsetStrIter::new("// comment")).is_complete());
let mut parsed = comment(OffsetStrIter::new("// comment\n"));
assert!(parsed.is_complete());
if let Result::Complete(_rest, cmt) = parsed {
assert_eq!(
@ -210,35 +212,47 @@ fn test_parse_comment() {
Token {
typ: TokenType::COMMENT,
fragment: " comment".to_string(),
pos: Position { line: 1, column: 1 },
pos: Position {
line: 1,
column: 1,
offset: 0
},
}
);
}
assert!(comment(StrIter::new("// comment\r\n")).is_complete());
parsed = comment(StrIter::new("// comment\r\n"));
assert!(comment(OffsetStrIter::new("// comment\r\n")).is_complete());
parsed = comment(OffsetStrIter::new("// comment\r\n"));
if let Result::Complete(_rest, cmt) = parsed {
assert_eq!(
cmt,
Token {
typ: TokenType::COMMENT,
fragment: " comment".to_string(),
pos: Position { column: 1, line: 1 },
pos: Position {
column: 1,
line: 1,
offset: 0
},
}
);
}
assert!(comment(StrIter::new("// comment\r\n ")).is_complete());
parsed = comment(StrIter::new("// comment\r\n "));
assert!(comment(OffsetStrIter::new("// comment\r\n ")).is_complete());
parsed = comment(OffsetStrIter::new("// comment\r\n "));
if let Result::Complete(_rest, cmt) = parsed {
assert_eq!(
cmt,
Token {
typ: TokenType::COMMENT,
fragment: " comment".to_string(),
pos: Position { column: 1, line: 1 },
pos: Position {
column: 1,
line: 1,
offset: 0
},
}
);
}
assert!(comment(StrIter::new("// comment")).is_complete());
assert!(comment(OffsetStrIter::new("// comment")).is_complete());
}
#[test]
@ -246,7 +260,11 @@ fn test_match_word() {
let input = vec![Token {
fragment: "foo".to_string(),
typ: TokenType::BAREWORD,
pos: Position { line: 1, column: 1 },
pos: Position {
line: 1,
column: 1,
offset: 0,
},
}];
let result = word!(SliceIter::new(input.as_slice()), "foo");
match result {
@ -260,7 +278,11 @@ fn test_match_word_empty_input() {
let input = vec![Token {
fragment: "".to_string(),
typ: TokenType::END,
pos: Position { line: 1, column: 1 },
pos: Position {
line: 1,
column: 1,
offset: 0,
},
}];
let result = word!(SliceIter::new(input.as_slice()), "foo");
match result {
@ -278,7 +300,11 @@ fn test_match_punct() {
let input = vec![Token {
fragment: "!".to_string(),
typ: TokenType::PUNCT,
pos: Position { line: 1, column: 1 },
pos: Position {
line: 1,
column: 1,
offset: 0,
},
}];
let result = punct!(SliceIter::new(input.as_slice()), "!");
match result {
@ -292,7 +318,11 @@ fn test_match_type() {
let input = vec![Token {
fragment: "foo".to_string(),
typ: TokenType::BAREWORD,
pos: Position { line: 1, column: 1 },
pos: Position {
line: 1,
column: 1,
offset: 0,
},
}];
let result = match_type!(SliceIter::new(input.as_slice()), BAREWORD);
match result {