zaphar/ucg
1
0
Fork 0
mirror of https://github.com/zaphar/ucg.git synced 2025-07-24 18:39:50 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge squash of the no_more_nom branch

Browse Source 
commit e4c80b19f5149fb7dba0a9b785e22ff5323b5470
Author: Jeremy Wall <jeremy@marzhillstudios.com>
Date:   Sun Nov 4 09:54:16 2018 -0600

    FEATURE: report better stacktraces for parsing.

commit 24b97c1037d6ddbe21f32e172b1c14cd2cfb9910
Author: Jeremy Wall <jeremy@marzhillstudios.com>
Date:   Wed Oct 17 19:00:29 2018 -0500

    FEATURE: Use context for our error reporting.

commit c22d397545e7575608e34561a274745b28be8c2c
Author: Jeremy Wall <jeremy@marzhillstudios.com>
Date:   Sun Oct 14 14:18:42 2018 -0500

    FIX: Better error reporting.

commit 7f47dc3f38bf6a3e8686c1fa3fd50ef97d4bf3af
Author: Jeremy Wall <jeremy@marzhillstudios.com>
Date:   Sat Oct 13 19:42:22 2018 -0500

    REFACTOR: Pass our offsets everywhere.

commit 91d7ed690bfd9989270ba9fa4f44c70f513d54b9
Author: Jeremy Wall <jeremy@marzhillstudios.com>
Date:   Sun Sep 23 15:08:45 2018 -0500

    REFACTOR: Use abortable_parser.
This commit is contained in:
Jeremy Wall 2018-11-05 21:34:12 -06:00
parent 29aed2c997
commit 6712fc9bb5
24 changed files with 2333 additions and 2513 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
Cargo.lock generated
View File

@ -1,3 +1,8 @@
[[package]]
name = "abortable_parser"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "ansi_term"
version = "0.9.0"
@ -131,31 +136,6 @@ name = "linked-hash-map"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "memchr"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "nom"
version = "3.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"memchr 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "nom_locate"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"memchr 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
"nom 3.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "redox_syscall"
version = "0.1.40"
@ -248,11 +228,10 @@ dependencies = [
name = "ucg"
version = "0.2.0"
dependencies = [
"abortable_parser 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
"bencher 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
"clap 2.26.2 (registry+https://github.com/rust-lang/crates.io-index)",
"cpuprofiler 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)",
"nom 3.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
"nom_locate 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_json 1.0.26 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_yaml 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
"simple-error 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)",
@ -306,6 +285,7 @@ dependencies = [
]
[metadata]
"checksum abortable_parser 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "09cdf5378b5e4a079fa886e621519fcb2502d9cb008d3f76b92f61f3890d5906"
"checksum ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "23ac7c30002a5accbf7e8987d0632fa6de155b7c3d39d0067317a391e00a2ef6"
"checksum atty 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "9a7d5b8723950951411ee34d271d99dddcc2035a16ab25310ea2c8cfd4369652"
"checksum backtrace 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "346d7644f0b5f9bc73082d3b2236b69a05fd35cce0cfa3724e184e6a5c9e2a2f"
@ -324,9 +304,6 @@ dependencies = [
"checksum lazy_static 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "76f033c7ad61445c5b347c7382dd1237847eb1bce590fe50365dcb33d546be73"
"checksum libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)" = "76e3a3ef172f1a0b9a9ff0dd1491ae5e6c948b94479a3021819ba7d860c8645d"
"checksum linked-hash-map 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "70fb39025bc7cdd76305867c4eccf2f2dcf6e9a57f5b21a93e1c2d86cd03ec9e"
"checksum memchr 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "148fab2e51b4f1cfc66da2a7c32981d1d3c083a803978268bb11fe4b86925e7a"
"checksum nom 3.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "05aec50c70fd288702bcd93284a8444607f3292dbdf2a30de5ea5dcdbe72287b"
"checksum nom_locate 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "49b1c61eff39ab6b91ccedfc62aff196eae066d88355b4fe3e4100c23168f0df"
"checksum redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)" = "c214e91d3ecf43e9a4e41e578973adeb14b474f2bee858742d127af75a0112b1"
"checksum redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7e891cfe48e9100a70a3b6eb652fef28920c117d366339687bd5576160db0f76"
"checksum rustc-demangle 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "bcfe5b13211b4d78e5c2cadfebd7769197d95c639c35a50057eb4c05de811395"

5
Cargo.toml
View File

@ -9,11 +9,8 @@ readme = "README.md"
keywords = ["compiler", "config"]
license = "Apache-2.0"
[dependencies.nom]
version = "^3.2"
[dependencies]
nom_locate = "^0.1.1"
abortable_parser = "0.2.1"
clap = "~2.26.0"
serde_json = "~1.0.9"
simple-error = "0.1"

2
TODO.md
View File

@ -9,6 +9,8 @@
You should be able to ask the compiler to tell you any value or set of values in the
compiled configuration.
Inspect is probably the correct location for this.
## Shape equality as a form of type assertion?
# Minor Fixes and Polish

1
example_errors/bad_file.ucg Normal file
View File

@ -0,0 +1 @@
let x =

4
integration_tests/operator_precedence_test.ucg
View File

@ -4,4 +4,6 @@ assert |2 * (2 + 1) == 6|;
assert |2 * 2 + 1 > 4|;
assert |2 * 2 + 1 < 6|;
assert |2 * 2 + 1 >= 5|;
assert |2 * 2 + 1 <= 5|;
assert |2 * 2 + 1 <= 5|;
assert |2 / 2 == 1|;
assert |2 - 1 == 1|;

154
src/ast/mod.rs
View File

@ -21,10 +21,11 @@ use std::cmp::PartialEq;
use std::cmp::PartialOrd;
use std::collections::HashSet;
use std::convert::Into;
use std::fmt;
use std::hash::Hash;
use std::hash::Hasher;
use std::fmt;
use abortable_parser;
macro_rules! enum_type_equality {
( $slf:ident, $r:expr, $( $l:pat ),* ) => {
@ -50,18 +51,26 @@ macro_rules! enum_type_equality {
pub struct Position {
pub line: usize,
pub column: usize,
pub offset: usize,
}
impl Position {
/// Construct a new Position.
pub fn new(line: usize, column: usize) -> Self {
pub fn new(line: usize, column: usize, offset: usize) -> Self {
Position {
line: line,
column: column,
offset: offset,
}
}
}
impl<'a> From<&'a Position> for Position {
fn from(source: &'a Position) -> Self {
source.clone()
}
}
/// Defines the types of tokens in UCG syntax.
#[derive(Debug, PartialEq, Eq, Clone, PartialOrd, Ord, Hash)]
pub enum TokenType {
@ -89,8 +98,8 @@ pub struct Token {
impl Token {
/// Constructs a new Token with a type and line and column information.
pub fn new<S: Into<String>>(f: S, typ: TokenType, line: usize, col: usize) -> Self {
Self::new_with_pos(f, typ, Position::new(line, col))
pub fn new<S: Into<String>, P: Into<Position>>(f: S, typ: TokenType, p: P) -> Self {
Self::new_with_pos(f, typ, p.into())
}
// Constructs a new Token with a type and a Position.
@ -103,6 +112,15 @@ impl Token {
}
}
impl abortable_parser::Positioned for Token {
fn line(&self) -> usize {
self.pos.line
}
fn column(&self) -> usize {
self.pos.column
}
}
impl Borrow<str> for Token {
fn borrow(&self) -> &str {
&self.fragment
@ -112,58 +130,54 @@ impl Borrow<str> for Token {
/// Helper macro for making a Positioned Value.
macro_rules! value_node {
($v:expr, $p:expr) => {
Positioned::new_with_pos($v, $p)
};
($v:expr, $l:expr, $c:expr) => {
Positioned::new($v, $l, $c)
PositionedItem::new_with_pos($v, $p)
};
}
/// Helper macro for making a Token.
#[allow(unused_macros)]
macro_rules! make_tok {
(EOF => $l:expr, $c:expr) => {
Token::new("", TokenType::END, $l, $c)
(EOF => $i:expr) => {
Token::new("", TokenType::END, &$i)
};
(WS => $l:expr, $c:expr) => {
Token::new("", TokenType::WS, $l, $c)
(WS => $i:expr) => {
Token::new("", TokenType::WS, &$i)
};
(CMT => $e:expr, $l:expr, $c:expr) => {
Token::new($e, TokenType::COMMENT, $l, $c)
(CMT => $e:expr, $i:expr) => {
Token::new($e, TokenType::COMMENT, &$i)
};
(QUOT => $e:expr, $l:expr, $c:expr) => {
Token::new($e, TokenType::QUOTED, $l, $c)
(QUOT => $e:expr, $i:expr) => {
Token::new($e, TokenType::QUOTED, &$i)
};
(PUNCT => $e:expr, $l:expr, $c:expr) => {
Token::new($e, TokenType::PUNCT, $l, $c)
(PUNCT => $e:expr, $i:expr) => {
Token::new($e, TokenType::PUNCT, &$i)
};
(DIGIT => $e:expr, $l:expr, $c:expr) => {
Token::new($e, TokenType::DIGIT, $l, $c)
(DIGIT => $e:expr, $i:expr) => {
Token::new($e, TokenType::DIGIT, &$i)
};
($e:expr, $l:expr, $c:expr) => {
Token::new($e, TokenType::BAREWORD, $l, $c)
($e:expr, $i:expr) => {
Token::new($e, TokenType::BAREWORD, &$i)
};
}
/// Helper macro for making expressions.
#[allow(unused_macros)]
macro_rules! make_expr {
($e:expr) => {
make_expr!($e, 1, 1)
($e:expr, $i:expr) => {
Expression::Simple(Value::Symbol(PositionedItem::new_with_pos(
$e.to_string(),
$i,
)))
};
($e:expr, $l:expr, $c:expr) => {
Expression::Simple(Value::Symbol(Positioned::new($e.to_string(), $l, $c)))
};
($e:expr => int, $l:expr, $c:expr) => {
Expression::Simple(Value::Int(Positioned::new($e, $l, $c)))
($e:expr => int, $i:expr) => {
Expression::Simple(Value::Int(PositionedItem::new_with_pos($e, $i)))
};
}
@ -180,30 +194,26 @@ macro_rules! make_expr {
/// ```
#[allow(unused_macros)]
macro_rules! make_selector {
( $h:expr ) => {
make_selector!($h, 1, 0)
};
( $h:expr, $l:expr, $c:expr ) => {
( $h:expr, $i:expr) => {
SelectorDef::new(
SelectorList{head: Box::new($h), tail: None},
$l, $c)
$i)
};
( $h: expr, $list:expr, $l:expr, $c:expr) => {
( $h: expr, $list:expr, $i:expr) => {
SelectorDef::new(
SelectorList{head: Box::new($h), tail: Some($list)},
$l, $c)
$i)
};
// Tokens
( $h:expr => [ $( $item:expr ),* ] ) => {
( $h:expr => [ $( $item:expr ),* ], $i:expr ) => {
{
make_selector!($h => [ $( $item, )* ] => 1, 1)
make_selector!($h => [ $( $item, )* ] => $i)
}
};
( $h:expr => [ $( $item:expr ),* ] => $l:expr, $c:expr ) => {
( $h:expr => [ $( $item:expr ),* ] => $i:expr ) => {
{
let mut list: Vec<Token> = Vec::new();
@ -211,7 +221,7 @@ macro_rules! make_selector {
list.push($item);
)*
make_selector!($h, list, $l, $c)
make_selector!($h, list, $i)
}
};
@ -223,14 +233,14 @@ macro_rules! make_selector {
let mut list: Vec<Token> = Vec::new();
$(
list.push(make_tok!($item, 1, col));
list.push(make_tok!($item, Position::new(1, col, col)));
col += $item.len() + 1;
)*
// Shut up the lint about unused code;
assert!(col != 0);
make_selector!($h, list, 1, 1)
make_selector!($h, list, Position::new(1, 1, 1))
}
};
@ -241,14 +251,14 @@ macro_rules! make_selector {
let mut list: Vec<Token> = Vec::new();
$(
list.push(make_tok!($item, $l, col));
list.push(make_tok!($item, Position::new($l, col, col)));
col += $item.len() + 1;
)*
// Shut up the linter about unused code;
assert!(col != 0);
make_selector!($h, list, $l, $c)
make_selector!($h, list, Position::new($l, $c, $c))
}
};
}
@ -314,9 +324,9 @@ pub struct SelectorDef {
impl SelectorDef {
/// Constructs a new SelectorDef.
pub fn new(sel: SelectorList, line: usize, col: usize) -> Self {
pub fn new<P: Into<Position>>(sel: SelectorList, p: P) -> Self {
SelectorDef {
pos: Position::new(line, col),
pos: p.into(),
sel: sel,
}
}
@ -327,13 +337,13 @@ impl SelectorDef {
pub enum Value {
// Constant Values
Empty(Position),
Boolean(Positioned<bool>),
Int(Positioned<i64>),
Float(Positioned<f64>),
Str(Positioned<String>),
Symbol(Positioned<String>),
Boolean(PositionedItem<bool>),
Int(PositionedItem<i64>),
Float(PositionedItem<f64>),
Str(PositionedItem<String>),
Symbol(PositionedItem<String>),
// Complex Values
Tuple(Positioned<FieldList>),
Tuple(PositionedItem<FieldList>),
List(ListDef),
Selector(SelectorDef),
}
@ -438,67 +448,67 @@ pub struct SelectDef {
/// Adds position information to any type `T`.
#[derive(Debug, Clone)]
pub struct Positioned<T> {
pub struct PositionedItem<T> {
pub pos: Position,
pub val: T,
}
impl<T: std::fmt::Display> std::fmt::Display for Positioned<T> {
impl<T: std::fmt::Display> std::fmt::Display for PositionedItem<T> {
fn fmt(&self, f: &mut std::fmt::Formatter) -> Result<(), std::fmt::Error> {
write!(f, "{}", self.val)
}
}
impl<T> Positioned<T> {
impl<T> PositionedItem<T> {
/// Constructs a new Positioned<T> with a value, line, and column information.
pub fn new(v: T, l: usize, c: usize) -> Self {
Self::new_with_pos(v, Position::new(l, c))
pub fn new<P: Into<Position>>(v: T, p: P) -> Self {
Self::new_with_pos(v, p.into())
}
/// Constructs a new Positioned<T> with a value and a Position.
pub fn new_with_pos(v: T, pos: Position) -> Self {
Positioned { pos: pos, val: v }
PositionedItem { pos: pos, val: v }
}
}
impl<T: PartialEq> PartialEq for Positioned<T> {
impl<T: PartialEq> PartialEq for PositionedItem<T> {
fn eq(&self, other: &Self) -> bool {
self.val == other.val
}
}
impl<T: Eq> Eq for Positioned<T> {}
impl<T: Eq> Eq for PositionedItem<T> {}
impl<T: Ord> Ord for Positioned<T> {
impl<T: Ord> Ord for PositionedItem<T> {
fn cmp(&self, other: &Self) -> Ordering {
self.val.cmp(&other.val)
}
}
impl<T: PartialOrd> PartialOrd for Positioned<T> {
impl<T: PartialOrd> PartialOrd for PositionedItem<T> {
fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
self.val.partial_cmp(&other.val)
}
}
impl<T: Hash> Hash for Positioned<T> {
impl<T: Hash> Hash for PositionedItem<T> {
fn hash<H: Hasher>(&self, state: &mut H) {
self.val.hash(state);
}
}
impl<'a> From<&'a Token> for Positioned<String> {
fn from(t: &'a Token) -> Positioned<String> {
Positioned {
impl<'a> From<&'a Token> for PositionedItem<String> {
fn from(t: &'a Token) -> PositionedItem<String> {
PositionedItem {
pos: t.pos.clone(),
val: t.fragment.to_string(),
}
}
}
impl<'a> From<&'a Positioned<String>> for Positioned<String> {
fn from(t: &Positioned<String>) -> Positioned<String> {
Positioned {
impl<'a> From<&'a PositionedItem<String>> for PositionedItem<String> {
fn from(t: &PositionedItem<String>) -> PositionedItem<String> {
PositionedItem {
pos: t.pos.clone(),
val: t.val.clone(),
}
@ -512,7 +522,7 @@ impl<'a> From<&'a Positioned<String>> for Positioned<String> {
/// any values except what is defined in their arguments.
#[derive(PartialEq, Debug, Clone)]
pub struct MacroDef {
pub argdefs: Vec<Positioned<String>>,
pub argdefs: Vec<PositionedItem<String>>,
pub fields: FieldList,
pub pos: Position,
}

70
src/ast/test.rs
View File

@ -17,21 +17,23 @@ use super::*;
#[test]
pub fn test_macro_validation_happy_path() {
let def = MacroDef {
argdefs: vec![value_node!("foo".to_string(), 1, 0)],
argdefs: vec![value_node!("foo".to_string(), Position::new(1, 0, 0))],
fields: vec![(
make_tok!("f1", 1, 1),
make_tok!("f1", Position::new(1, 1, 0)),
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Add,
left: Box::new(Expression::Simple(Value::Symbol(value_node!(
"foo".to_string(),
1,
1
Position::new(1, 1, 0)
)))),
right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
pos: Position::new(1, 0),
right: Box::new(Expression::Simple(Value::Int(value_node!(
1,
Position::new(1, 1, 0)
)))),
pos: Position::new(1, 0, 0),
}),
)],
pos: Position::new(1, 0),
pos: Position::new(1, 0, 0),
};
assert!(def.validate_symbols().unwrap() == ());
}
@ -39,21 +41,23 @@ pub fn test_macro_validation_happy_path() {
#[test]
pub fn test_macro_validation_fail() {
let def = MacroDef {
argdefs: vec![value_node!("foo".to_string(), 1, 0)],
argdefs: vec![value_node!("foo".to_string(), Position::new(1, 0, 0))],
fields: vec![(
make_tok!("f1", 1, 1),
make_tok!("f1", Position::new(1, 1, 0)),
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Add,
left: Box::new(Expression::Simple(Value::Symbol(value_node!(
"bar".to_string(),
1,
1
Position::new(1, 1, 0)
)))),
right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
pos: Position::new(1, 0),
right: Box::new(Expression::Simple(Value::Int(value_node!(
1,
Position::new(1, 1, 0)
)))),
pos: Position::new(1, 0, 0),
}),
)],
pos: Position::new(1, 0),
pos: Position::new(1, 0, 0),
};
let mut expected = HashSet::new();
expected.insert("bar".to_string());
@ -63,20 +67,23 @@ pub fn test_macro_validation_fail() {
#[test]
pub fn test_macro_validation_selector_happy_path() {
let def = MacroDef {
argdefs: vec![value_node!("foo".to_string(), 1, 0)],
argdefs: vec![value_node!("foo".to_string(), Position::new(1, 0, 0))],
fields: vec![(
make_tok!("f1", 1, 1),
make_tok!("f1", Position::new(1, 1, 0)),
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Add,
left: Box::new(Expression::Simple(Value::Selector(
make_selector!(make_expr!("foo", 1, 1) => [
make_tok!("quux", 1, 1) ] => 1, 1),
))),
right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
pos: Position::new(1, 0),
left: Box::new(Expression::Simple(Value::Selector(make_selector!(
make_expr!("foo", Position::new(1, 1, 0)) => [
make_tok!("quux", Position::new(1, 1, 0)) ]
=> Position::new(1, 1, 0))))),
right: Box::new(Expression::Simple(Value::Int(value_node!(
1,
Position::new(1, 1, 0)
)))),
pos: Position::new(1, 0, 0),
}),
)],
pos: Position::new(1, 0),
pos: Position::new(1, 0, 0),
};
assert!(def.validate_symbols().unwrap() == ());
}
@ -84,20 +91,23 @@ pub fn test_macro_validation_selector_happy_path() {
#[test]
pub fn test_macro_validation_selector_fail() {
let def = MacroDef {
argdefs: vec![value_node!("foo".to_string(), 1, 0)],
argdefs: vec![value_node!("foo".to_string(), Position::new(1, 0, 0))],
fields: vec![(
make_tok!("f1", 1, 1),
make_tok!("f1", Position::new(1, 1, 0)),
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Add,
left: Box::new(Expression::Simple(Value::Selector(
make_selector!(make_expr!("bar", 1, 1) => [
make_tok!("quux", 1, 1) ] => 1, 1),
make_selector!(make_expr!("bar", Position::new(1, 1, 0)) => [
make_tok!("quux", Position::new(1, 1, 0)) ] => Position::new(1, 1, 0)),
))),
right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
pos: Position::new(1, 0),
right: Box::new(Expression::Simple(Value::Int(value_node!(
1,
Position::new(1, 1, 0)
)))),
pos: Position::new(1, 0, 0),
}),
)],
pos: Position::new(1, 0),
pos: Position::new(1, 0, 0),
};
let mut expected = HashSet::new();
expected.insert("bar".to_string());

7
src/benches/parse.rs
View File

@ -16,17 +16,20 @@
#[macro_use]
extern crate bencher;
extern crate abortable_parser;
extern crate cpuprofiler;
extern crate nom_locate;
extern crate ucglib;
use bencher::Bencher;
use ucglib::iter::OffsetStrIter;
//use cpuprofiler::PROFILER;
use ucglib::parse::*;
fn do_parse(i: &str) {
parse(nom_locate::LocatedSpan::new(i));
parse(OffsetStrIter::new(i));
}
fn parse_int(b: &mut Bencher) {

4
src/build/ir.rs
View File

@ -18,7 +18,7 @@ pub enum Val {
Float(f64),
Str(String),
List(Vec<Rc<Val>>),
Tuple(Vec<(Positioned<String>, Rc<Val>)>),
Tuple(Vec<(PositionedItem<String>, Rc<Val>)>),
Macro(MacroDef),
}
@ -114,7 +114,7 @@ impl Val {
}
/// Returns the fields if this Val is a tuple. None otherwise.
pub fn get_fields(&self) -> Option<&Vec<(Positioned<String>, Rc<Val>)>> {
pub fn get_fields(&self) -> Option<&Vec<(PositionedItem<String>, Rc<Val>)>> {
if let &Val::Tuple(ref fs) = self {
Some(fs)
} else {

90
src/build/mod.rs
View File

@ -28,8 +28,8 @@ use std::string::ToString;
use ast::*;
use error;
use format;
use iter::OffsetStrIter;
use parse::parse;
use tokenizer::Span;
pub mod assets;
pub mod ir;
@ -44,7 +44,7 @@ impl MacroDef {
cache: Rc<RefCell<assets::Cache>>,
env: Rc<Val>,
mut args: Vec<Rc<Val>>,
) -> Result<Vec<(Positioned<String>, Rc<Val>)>, Box<Error>> {
) -> Result<Vec<(PositionedItem<String>, Rc<Val>)>, Box<Error>> {
// Error conditions. If the args don't match the length and types of the argdefs then this is
// macro call error.
if args.len() > self.argdefs.len() {
@ -61,12 +61,12 @@ impl MacroDef {
// If the expressions reference Symbols not defined in the MacroDef that is also an error.
// TODO(jwall): We should probably enforce that the Expression Symbols must be in argdefs rules
// at Macro definition time not evaluation time.
let mut scope = HashMap::<Positioned<String>, Rc<Val>>::new();
let mut scope = HashMap::<PositionedItem<String>, Rc<Val>>::new();
for (i, arg) in args.drain(0..).enumerate() {
scope.entry(self.argdefs[i].clone()).or_insert(arg.clone());
}
let b = Builder::new_with_env_and_scope(root, cache, scope, env);
let mut result: Vec<(Positioned<String>, Rc<Val>)> = Vec::new();
let mut result: Vec<(PositionedItem<String>, Rc<Val>)> = Vec::new();
for &(ref key, ref expr) in self.fields.iter() {
// We clone the expressions here because this macro may be consumed
// multiple times in the future.
@ -81,7 +81,7 @@ impl MacroDef {
type BuildResult = Result<(), Box<Error>>;
/// Defines a set of values in a parsed file.
type ValueMap = HashMap<Positioned<String>, Rc<Val>>;
type ValueMap = HashMap<PositionedItem<String>, Rc<Val>>;
/// AssertCollector collects the results of assertions in the UCG AST.
pub struct AssertCollector {
@ -136,7 +136,7 @@ macro_rules! eval_binary_expr {
impl<'a> Builder<'a> {
// TOOD(jwall): This needs some unit tests.
fn tuple_to_val(&self, fields: &Vec<(Token, Expression)>) -> Result<Rc<Val>, Box<Error>> {
let mut new_fields = Vec::<(Positioned<String>, Rc<Val>)>::new();
let mut new_fields = Vec::<(PositionedItem<String>, Rc<Val>)>::new();
for &(ref name, ref expr) in fields.iter() {
let val = try!(self.eval_expr(expr));
new_fields.push((name.into(), val));
@ -159,17 +159,18 @@ impl<'a> Builder<'a> {
&Value::Int(ref i) => Ok(Rc::new(Val::Int(i.val))),
&Value::Float(ref f) => Ok(Rc::new(Val::Float(f.val))),
&Value::Str(ref s) => Ok(Rc::new(Val::Str(s.val.to_string()))),
&Value::Symbol(ref s) => self.lookup_sym(&(s.into())).ok_or(Box::new(
error::Error::new(
format!(
"Unable to find {} in file: {}",
s.val,
self.root.to_string_lossy()
),
error::ErrorType::NoSuchSymbol,
v.pos().clone(),
),
)),
&Value::Symbol(ref s) => {
self.lookup_sym(&(s.into()))
.ok_or(Box::new(error::Error::new(
format!(
"Unable to find {} in file: {}",
s.val,
self.root.to_string_lossy()
),
error::ErrorType::NoSuchSymbol,
v.pos().clone(),
)))
}
&Value::List(ref def) => self.list_to_val(def),
&Value::Tuple(ref tuple) => self.tuple_to_val(&tuple.val),
&Value::Selector(ref selector_list_node) => {
@ -189,9 +190,13 @@ impl<'a> Builder<'a> {
cache: Rc<RefCell<assets::Cache>>,
scope: ValueMap,
) -> Self {
let env_vars: Vec<(Positioned<String>, Rc<Val>)> = env::vars()
.map(|t| (Positioned::new(t.0, 0, 0), Rc::new(t.1.into())))
.collect();
let env_vars: Vec<(PositionedItem<String>, Rc<Val>)> = env::vars()
.map(|t| {
(
PositionedItem::new(t.0, Position::new(0, 0, 0)),
Rc::new(t.1.into()),
)
}).collect();
Self::new_with_env_and_scope(root, cache, scope, Rc::new(Val::Tuple(env_vars)))
}
@ -220,8 +225,8 @@ impl<'a> Builder<'a> {
/// Returns a Val by name from previously built UCG.
pub fn get_out_by_name(&self, name: &str) -> Option<Rc<Val>> {
let key = Positioned {
pos: Position::new(0, 0),
let key = PositionedItem {
pos: Position::new(0, 0, 0),
val: name.to_string(),
};
self.lookup_sym(&key)
@ -243,8 +248,8 @@ impl<'a> Builder<'a> {
Ok(())
}
fn eval_span(&mut self, input: Span) -> Result<Rc<Val>, Box<Error>> {
match parse(input) {
fn eval_span(&mut self, input: OffsetStrIter) -> Result<Rc<Val>, Box<Error>> {
match parse(input.clone()) {
Ok(stmts) => {
//panic!("Successfully parsed {}", input);
let mut out: Option<Rc<Val>> = None;
@ -256,20 +261,17 @@ impl<'a> Builder<'a> {
Some(val) => Ok(val),
}
}
Err(err) => Err(Box::new(error::Error::new_with_cause(
format!(
"Error while parsing file: {}",
self.curr_file.unwrap_or("<eval>")
),
Err(err) => Err(Box::new(error::Error::new(
format!("{}", err,),
error::ErrorType::ParseError,
err,
(&input).into(),
))),
}
}
/// Evaluate an input string as UCG.
pub fn eval_string(&mut self, input: &str) -> Result<Rc<Val>, Box<Error>> {
self.eval_span(Span::new(input))
self.eval_span(OffsetStrIter::new(input))
}
/// Builds a ucg file at the named path.
@ -295,7 +297,8 @@ impl<'a> Builder<'a> {
let mut b = Self::new(normalized.clone(), self.assets.clone());
let filepath = normalized.to_str().unwrap().clone();
try!(b.build_file(filepath));
let fields: Vec<(Positioned<String>, Rc<Val>)> = b.build_output.drain().collect();
let fields: Vec<(PositionedItem<String>, Rc<Val>)> =
b.build_output.drain().collect();
Rc::new(Val::Tuple(fields))
}
};
@ -360,7 +363,7 @@ impl<'a> Builder<'a> {
}
}
fn lookup_sym(&self, sym: &Positioned<String>) -> Option<Rc<Val>> {
fn lookup_sym(&self, sym: &PositionedItem<String>) -> Option<Rc<Val>> {
if &sym.val == "env" {
return Some(self.env.clone());
}
@ -370,7 +373,10 @@ impl<'a> Builder<'a> {
None
}
fn find_in_fieldlist(target: &str, fs: &Vec<(Positioned<String>, Rc<Val>)>) -> Option<Rc<Val>> {
fn find_in_fieldlist(
target: &str,
fs: &Vec<(PositionedItem<String>, Rc<Val>)>,
) -> Option<Rc<Val>> {
for (key, val) in fs.iter().cloned() {
if target == &key.val {
return Some(val.clone());
@ -384,7 +390,7 @@ impl<'a> Builder<'a> {
stack: &mut VecDeque<Rc<Val>>,
sl: &SelectorList,
next: (&Position, &str),
fs: &Vec<(Positioned<String>, Rc<Val>)>,
fs: &Vec<(PositionedItem<String>, Rc<Val>)>,
) -> Result<(), Box<Error>> {
if let Some(vv) = Self::find_in_fieldlist(next.1, fs) {
stack.push_back(vv.clone());
@ -775,7 +781,7 @@ impl<'a> Builder<'a> {
fn eval_copy(&self, def: &CopyDef) -> Result<Rc<Val>, Box<Error>> {
let v = try!(self.lookup_selector(&def.selector.sel));
if let Val::Tuple(ref src_fields) = *v {
let mut m = HashMap::<Positioned<String>, (i32, Rc<Val>)>::new();
let mut m = HashMap::<PositionedItem<String>, (i32, Rc<Val>)>::new();
// loop through fields and build up a hashmap
let mut count = 0;
for &(ref key, ref val) in src_fields.iter() {
@ -824,7 +830,7 @@ impl<'a> Builder<'a> {
}
};
}
let mut new_fields: Vec<(Positioned<String>, (i32, Rc<Val>))> = m.drain().collect();
let mut new_fields: Vec<(PositionedItem<String>, (i32, Rc<Val>))> = m.drain().collect();
// We want to maintain our order for the fields to make comparing tuples
// easier in later code. So we sort by the field order before constructing a new tuple.
new_fields.sort_by(|a, b| {
@ -839,8 +845,7 @@ impl<'a> Builder<'a> {
let first = a.0.clone();
let t = a.1.clone();
(first, t.1)
})
.collect(),
}).collect(),
)));
}
Err(Box::new(error::Error::new(
@ -992,11 +997,8 @@ impl<'a> Builder<'a> {
let expr = &tok.fragment;
expr_as_stmt.push_str(expr);
expr_as_stmt.push_str(";");
let assert_input = Span {
fragment: &expr_as_stmt,
line: tok.pos.line as u32,
offset: tok.pos.column,
};
let assert_input =
OffsetStrIter::new_with_offsets(&expr_as_stmt, tok.pos.line - 1, tok.pos.column - 1);
let ok = match self.eval_span(assert_input) {
Ok(v) => v,
Err(e) => {

728
src/build/test.rs
View File

@ -25,35 +25,6 @@ fn test_expr_to_val(mut cases: Vec<(Expression, Val)>, b: Builder) {
}
}
#[test]
fn test_eval_div_expr() {
let cache = Rc::new(RefCell::new(MemoryCache::new()));
let b = Builder::new(std::env::current_dir().unwrap(), cache);
test_expr_to_val(
vec![
(
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Div,
left: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))),
right: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))),
pos: Position::new(1, 0),
}),
Val::Int(1),
),
(
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Div,
left: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))),
right: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))),
pos: Position::new(1, 0),
}),
Val::Float(1.0),
),
],
b,
);
}
#[test]
#[should_panic(expected = "Expected Float")]
fn test_eval_div_expr_fail() {
@ -63,9 +34,15 @@ fn test_eval_div_expr_fail() {
vec![(
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Div,
left: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))),
right: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))),
pos: Position::new(1, 0),
left: Box::new(Expression::Simple(Value::Float(value_node!(
2.0,
Position::new(1, 1, 1)
)))),
right: Box::new(Expression::Simple(Value::Int(value_node!(
2,
Position::new(1, 1, 1)
)))),
pos: Position::new(1, 0, 0),
}),
Val::Float(1.0),
)],
@ -73,35 +50,6 @@ fn test_eval_div_expr_fail() {
);
}
#[test]
fn test_eval_mul_expr() {
let cache = Rc::new(RefCell::new(MemoryCache::new()));
let b = Builder::new(std::env::current_dir().unwrap(), cache);
test_expr_to_val(
vec![
(
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Mul,
left: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))),
right: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))),
pos: Position::new(1, 0),
}),
Val::Int(4),
),
(
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Mul,
left: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))),
right: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))),
pos: Position::new(1, 0),
}),
Val::Float(4.0),
),
],
b,
);
}
#[test]
#[should_panic(expected = "Expected Float")]
fn test_eval_mul_expr_fail() {
@ -111,9 +59,15 @@ fn test_eval_mul_expr_fail() {
vec![(
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Mul,
left: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))),
right: Box::new(Expression::Simple(Value::Int(value_node!(20, 1, 1)))),
pos: Position::new(1, 0),
left: Box::new(Expression::Simple(Value::Float(value_node!(
2.0,
Position::new(1, 1, 1)
)))),
right: Box::new(Expression::Simple(Value::Int(value_node!(
20,
Position::new(1, 1, 1)
)))),
pos: Position::new(1, 0, 0),
}),
Val::Float(1.0),
)],
@ -121,35 +75,6 @@ fn test_eval_mul_expr_fail() {
);
}
#[test]
fn test_eval_subtract_expr() {
let cache = Rc::new(RefCell::new(MemoryCache::new()));
let b = Builder::new(std::env::current_dir().unwrap(), cache);
test_expr_to_val(
vec![
(
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Sub,
left: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))),
right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
pos: Position::new(1, 0),
}),
Val::Int(1),
),
(
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Sub,
left: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))),
right: Box::new(Expression::Simple(Value::Float(value_node!(1.0, 1, 1)))),
pos: Position::new(1, 0),
}),
Val::Float(1.0),
),
],
b,
);
}
#[test]
#[should_panic(expected = "Expected Float")]
fn test_eval_subtract_expr_fail() {
@ -159,88 +84,21 @@ fn test_eval_subtract_expr_fail() {
vec![(
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Sub,
left: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))),
right: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))),
pos: Position::new(1, 0),
left: Box::new(Expression::Simple(Value::Float(value_node!(
2.0,
Position::new(1, 1, 1)
)))),
right: Box::new(Expression::Simple(Value::Int(value_node!(
2,
Position::new(1, 1, 1)
)))),
pos: Position::new(1, 0, 0),
}),
Val::Float(1.0),
)],
b,
);
}
#[test]
fn test_eval_add_expr() {
let cache = Rc::new(RefCell::new(MemoryCache::new()));
let b = Builder::new(std::env::current_dir().unwrap(), cache);
test_expr_to_val(
vec![
(
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Add,
left: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
pos: Position::new(1, 0),
}),
Val::Int(2),
),
(
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Add,
left: Box::new(Expression::Simple(Value::Float(value_node!(1.0, 1, 1)))),
right: Box::new(Expression::Simple(Value::Float(value_node!(1.0, 1, 1)))),
pos: Position::new(1, 0),
}),
Val::Float(2.0),
),
(
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Add,
left: Box::new(Expression::Simple(Value::Str(value_node!(
"foo".to_string(),
1,
1
)))),
right: Box::new(Expression::Simple(Value::Str(value_node!(
"bar".to_string(),
1,
1
)))),
pos: Position::new(1, 0),
}),
Val::Str("foobar".to_string()),
),
(
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Add,
left: Box::new(Expression::Simple(Value::List(ListDef {
elems: vec![Expression::Simple(Value::Str(value_node!(
"foo".to_string(),
1,
1
)))],
pos: Position::new(1, 1),
}))),
right: Box::new(Expression::Simple(Value::List(ListDef {
elems: vec![Expression::Simple(Value::Str(value_node!(
"bar".to_string(),
1,
1
)))],
pos: Position::new(1, 1),
}))),
pos: Position::new(1, 0),
}),
Val::List(vec![
Rc::new(Val::Str("foo".to_string())),
Rc::new(Val::Str("bar".to_string())),
]),
),
],
b,
);
}
#[test]
#[should_panic(expected = "Expected Float")]
fn test_eval_add_expr_fail() {
@ -250,9 +108,15 @@ fn test_eval_add_expr_fail() {
vec![(
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Add,
left: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))),
right: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))),
pos: Position::new(1, 0),
left: Box::new(Expression::Simple(Value::Float(value_node!(
2.0,
Position::new(1, 1, 1)
)))),
right: Box::new(Expression::Simple(Value::Int(value_node!(
2,
Position::new(1, 1, 1)
)))),
pos: Position::new(1, 0, 0),
}),
Val::Float(1.0),
)],
@ -260,245 +124,20 @@ fn test_eval_add_expr_fail() {
);
}
#[test]
fn test_eval_nested_tuple() {
test_expr_to_val(
vec![
(
Expression::Simple(Value::Tuple(value_node!(
vec![(
Token::new("foo", TokenType::BAREWORD, 1, 1),
Expression::Simple(Value::Tuple(value_node!(Vec::new(), 1, 1))),
)],
1,
1
))),
Val::Tuple(vec![(
Positioned::new("foo".to_string(), 1, 1),
Rc::new(Val::Tuple(Vec::new())),
)]),
),
(
Expression::Simple(Value::Tuple(value_node!(
vec![(
Token::new("foo", TokenType::BAREWORD, 1, 1),
Expression::Simple(Value::Tuple(value_node!(
vec![(
Token::new("bar".to_string(), TokenType::BAREWORD, 1, 5),
Expression::Simple(Value::Tuple(value_node!(vec![], 1, 10))),
)],
1,
1
))),
)],
1,
1
))),
Val::Tuple(vec![(
Positioned::new("foo".to_string(), 1, 1),
Rc::new(Val::Tuple(vec![(
Positioned::new("bar".to_string(), 1, 10),
Rc::new(Val::Tuple(vec![])),
)])),
)]),
),
(
Expression::Simple(Value::Tuple(value_node!(
vec![(
Token::new("foo", TokenType::BAREWORD, 1, 1),
Expression::Simple(Value::Tuple(value_node!(
vec![(
Token::new("bar".to_string(), TokenType::BAREWORD, 1, 5),
Expression::Simple(Value::Tuple(value_node!(
vec![(
Token::new("quux".to_string(), TokenType::BAREWORD, 1, 1),
Expression::Simple(Value::Int(value_node!(3, 1, 1))),
)],
1,
10
))),
)],
1,
1
))),
)],
1,
1
))),
Val::Tuple(vec![(
Positioned::new("foo".to_string(), 1, 1),
Rc::new(Val::Tuple(vec![(
Positioned::new("bar".to_string(), 1, 10),
Rc::new(Val::Tuple(vec![(
Positioned::new("quux".to_string(), 1, 1),
Rc::new(Val::Int(3)),
)])),
)])),
)]),
),
],
Builder::new(
std::env::current_dir().unwrap(),
Rc::new(RefCell::new(MemoryCache::new())),
),
);
}
#[test]
fn test_eval_simple_expr() {
test_expr_to_val(
vec![
(
Expression::Simple(Value::Int(value_node!(1, 1, 1))),
Val::Int(1),
),
(
Expression::Simple(Value::Float(value_node!(2.0, 1, 1))),
Val::Float(2.0),
),
(
Expression::Simple(Value::Str(value_node!("foo".to_string(), 1, 1))),
Val::Str("foo".to_string()),
),
(
Expression::Simple(Value::Tuple(value_node!(
vec![(
make_tok!("bar", 1, 1),
Expression::Simple(Value::Int(value_node!(1, 1, 1))),
)],
1,
1
))),
Val::Tuple(vec![(
value_node!("bar".to_string(), 1, 1),
Rc::new(Val::Int(1)),
)]),
),
],
Builder::new(
std::env::current_dir().unwrap(),
Rc::new(RefCell::new(MemoryCache::new())),
),
);
}
#[test]
fn test_eval_simple_lookup_expr() {
let cache = Rc::new(RefCell::new(MemoryCache::new()));
let mut b = Builder::new(std::env::current_dir().unwrap(), cache);
b.build_output
.entry(value_node!("var1".to_string(), 1, 0))
.or_insert(Rc::new(Val::Int(1)));
test_expr_to_val(
vec![(
Expression::Simple(Value::Symbol(value_node!("var1".to_string(), 1, 1))),
Val::Int(1),
)],
b,
);
}
#[test]
fn test_eval_simple_lookup_error() {
let cache = Rc::new(RefCell::new(MemoryCache::new()));
let mut b = Builder::new(std::env::current_dir().unwrap(), cache);
b.build_output
.entry(value_node!("var1".to_string(), 1, 0))
.entry(value_node!("var1".to_string(), Position::new(1, 0, 0)))
.or_insert(Rc::new(Val::Int(1)));
let expr = Expression::Simple(Value::Symbol(value_node!("var".to_string(), 1, 1)));
let expr = Expression::Simple(Value::Symbol(value_node!(
"var".to_string(),
Position::new(1, 1, 1)
)));
assert!(b.eval_expr(&expr).is_err());
}
#[test]
fn test_eval_selector_expr() {
let cache = Rc::new(RefCell::new(MemoryCache::new()));
let mut b = Builder::new(std::env::current_dir().unwrap(), cache);
b.build_output
.entry(value_node!("var1".to_string(), 1, 0))
.or_insert(Rc::new(Val::Tuple(vec![(
value_node!("lvl1".to_string(), 1, 0),
Rc::new(Val::Tuple(vec![(
value_node!("lvl2".to_string(), 1, 0),
Rc::new(Val::Int(3)),
)])),
)])));
b.build_output
.entry(value_node!("var2".to_string(), 1, 0))
.or_insert(Rc::new(Val::Int(2)));
b.build_output
.entry(value_node!("var3".to_string(), 1, 0))
.or_insert(Rc::new(Val::Tuple(vec![(
value_node!("lvl1".to_string(), 1, 0),
Rc::new(Val::Int(4)),
)])));
test_expr_to_val(
vec![
(
Expression::Simple(Value::Selector(make_selector!(make_expr!("var1")))),
Val::Tuple(vec![(
value_node!("lvl1".to_string(), 1, 0),
Rc::new(Val::Tuple(vec![(
value_node!("lvl2".to_string(), 1, 0),
Rc::new(Val::Int(3)),
)])),
)]),
),
(
Expression::Simple(Value::Selector(
make_selector!(make_expr!("var1") => "lvl1"),
)),
Val::Tuple(vec![(
value_node!("lvl2".to_string(), 1, 0),
Rc::new(Val::Int(3)),
)]),
),
(
Expression::Simple(Value::Selector(
make_selector!(make_expr!("var1") => "lvl1", "lvl2"),
)),
Val::Int(3),
),
(
Expression::Simple(Value::Selector(make_selector!(make_expr!("var2")))),
Val::Int(2),
),
(
Expression::Simple(Value::Selector(
make_selector!(make_expr!("var3") => "lvl1"),
)),
Val::Int(4),
),
],
b,
);
}
#[test]
fn test_eval_selector_list_expr() {
let cache = Rc::new(RefCell::new(MemoryCache::new()));
let mut b = Builder::new(std::env::current_dir().unwrap(), cache);
b.build_output
.entry(value_node!("var1".to_string(), 1, 1))
.or_insert(Rc::new(Val::List(vec![
Rc::new(Val::Str("val1".to_string())),
Rc::new(Val::Tuple(vec![(
value_node!("var2".to_string(), 1, 1),
Rc::new(Val::Int(1)),
)])),
])));
test_expr_to_val(
vec![(
Expression::Simple(Value::Selector(
make_selector!(make_expr!("var1") => "0" => 1, 1),
)),
Val::Str("val1".to_string()),
)],
b,
);
}
// Include nested for each.
#[test]
#[should_panic(expected = "Unable to find tpl1")]
@ -508,9 +147,12 @@ fn test_expr_copy_no_such_tuple() {
test_expr_to_val(
vec![(
Expression::Copy(CopyDef {
selector: make_selector!(make_expr!("tpl1")),
selector: make_selector!(
make_expr!("tpl1", Position::new(1, 1, 1)),
Position::new(1, 1, 1)
),
fields: Vec::new(),
pos: Position::new(1, 0),
pos: Position::new(1, 0, 0),
}),
Val::Tuple(Vec::new()),
)],
@ -524,14 +166,17 @@ fn test_expr_copy_not_a_tuple() {
let cache = Rc::new(RefCell::new(MemoryCache::new()));
let mut b = Builder::new(std::env::current_dir().unwrap(), cache);
b.build_output
.entry(value_node!("tpl1".to_string(), 1, 0))
.entry(value_node!("tpl1".to_string(), Position::new(1, 0, 0)))
.or_insert(Rc::new(Val::Int(1)));
test_expr_to_val(
vec![(
Expression::Copy(CopyDef {
selector: make_selector!(make_expr!("tpl1")),
selector: make_selector!(
make_expr!("tpl1", Position::new(1, 1, 1)),
Position::new(1, 1, 1)
),
fields: Vec::new(),
pos: Position::new(1, 0),
pos: Position::new(1, 0, 0),
}),
Val::Tuple(Vec::new()),
)],
@ -545,130 +190,30 @@ fn test_expr_copy_field_type_error() {
let cache = Rc::new(RefCell::new(MemoryCache::new()));
let mut b = Builder::new(std::env::current_dir().unwrap(), cache);
b.build_output
.entry(value_node!("tpl1".to_string(), 1, 0))
.entry(value_node!("tpl1".to_string(), Position::new(1, 0, 0)))
.or_insert(Rc::new(Val::Tuple(vec![(
value_node!("fld1".to_string(), 1, 0),
value_node!("fld1".to_string(), Position::new(1, 0, 0)),
Rc::new(Val::Int(1)),
)])));
test_expr_to_val(
vec![(
Expression::Copy(CopyDef {
selector: make_selector!(make_expr!("tpl1")),
fields: vec![(
make_tok!("fld1", 1, 1),
Expression::Simple(Value::Str(value_node!("2".to_string(), 1, 1))),
)],
pos: Position::new(1, 0),
}),
Val::Tuple(vec![(
value_node!("fld1".to_string(), 1, 1),
Rc::new(Val::Str("2".to_string())),
)]),
)],
b,
);
}
#[test]
fn test_expr_copy() {
let cache = Rc::new(RefCell::new(MemoryCache::new()));
let mut b = Builder::new(std::env::current_dir().unwrap(), cache);
b.build_output
.entry(value_node!("tpl1".to_string(), 1, 0))
.or_insert(Rc::new(Val::Tuple(vec![(
value_node!("fld1".to_string(), 1, 0),
Rc::new(Val::Int(1)),
)])));
test_expr_to_val(
vec![
(
Expression::Copy(CopyDef {
selector: make_selector!(make_expr!("tpl1")),
fields: vec![(
make_tok!("fld2", 1, 1),
Expression::Simple(Value::Str(value_node!("2".to_string(), 1, 1))),
)],
pos: Position::new(1, 0),
}),
// Add a new field to the copy
Val::Tuple(
// NOTE(jwall): The order of these is important in order to ensure
// that the compare assertion is correct. The ordering has no
// semantics though so at some point we should probably be less restrictive.
vec![
(value_node!("fld1".to_string(), 1, 0), Rc::new(Val::Int(1))),
(
value_node!("fld2".to_string(), 1, 1),
Rc::new(Val::Str("2".to_string())),
),
],
selector: make_selector!(
make_expr!("tpl1", Position::new(1, 1, 1)),
Position::new(1, 1, 1)
),
),
// Overwrite a field in the copy
(
Expression::Copy(CopyDef {
selector: make_selector!(make_expr!("tpl1")),
fields: vec![
(
make_tok!("fld1", 1, 1),
Expression::Simple(Value::Int(value_node!(3, 1, 1))),
),
(
make_tok!("fld2", 1, 1),
Expression::Simple(Value::Str(value_node!("2".to_string(), 1, 1))),
),
],
pos: Position::new(1, 0),
}),
Val::Tuple(vec![
(value_node!("fld1".to_string(), 1, 0), Rc::new(Val::Int(3))),
(
value_node!("fld2".to_string(), 1, 0),
Rc::new(Val::Str("2".to_string())),
),
]),
),
// The source tuple is still unmodified.
(
Expression::Simple(Value::Selector(make_selector!(make_expr!["tpl1"]))),
Val::Tuple(vec![(
value_node!("fld1".to_string(), 1, 0),
Rc::new(Val::Int(1)),
)]),
),
],
b,
);
}
#[test]
fn test_macro_call() {
let cache = Rc::new(RefCell::new(MemoryCache::new()));
let mut b = Builder::new(std::env::current_dir().unwrap(), cache);
b.build_output
.entry(value_node!("tstmac".to_string(), 1, 0))
.or_insert(Rc::new(Val::Macro(MacroDef {
argdefs: vec![value_node!("arg1".to_string(), 1, 0)],
fields: vec![(
make_tok!("foo", 1, 1),
Expression::Simple(Value::Symbol(value_node!("arg1".to_string(), 1, 1))),
)],
pos: Position::new(1, 0),
})));
test_expr_to_val(
vec![(
Expression::Call(CallDef {
macroref: make_selector!(make_expr!("tstmac")),
arglist: vec![Expression::Simple(Value::Str(value_node!(
"bar".to_string(),
1,
1
)))],
pos: Position::new(1, 0),
fields: vec![(
make_tok!("fld1", Position::new(1, 1, 1)),
Expression::Simple(Value::Str(value_node!(
"2".to_string(),
Position::new(1, 1, 1)
))),
)],
pos: Position::new(1, 0, 0),
}),
Val::Tuple(vec![(
value_node!("foo".to_string(), 1, 1),
Rc::new(Val::Str("bar".to_string())),
value_node!("fld1".to_string(), Position::new(1, 1, 1)),
Rc::new(Val::Str("2".to_string())),
)]),
)],
b,
@ -681,31 +226,36 @@ fn test_macro_hermetic() {
let cache = Rc::new(RefCell::new(MemoryCache::new()));
let mut b = Builder::new(std::env::current_dir().unwrap(), cache);
b.build_output
.entry(value_node!("arg1".to_string(), 1, 0))
.entry(value_node!("arg1".to_string(), Position::new(1, 0, 0)))
.or_insert(Rc::new(Val::Str("bar".to_string())));
b.build_output
.entry(value_node!("tstmac".to_string(), 1, 0))
.entry(value_node!("tstmac".to_string(), Position::new(1, 0, 0)))
.or_insert(Rc::new(Val::Macro(MacroDef {
argdefs: vec![value_node!("arg2".to_string(), 1, 0)],
argdefs: vec![value_node!("arg2".to_string(), Position::new(1, 0, 0))],
fields: vec![(
make_tok!("foo", 1, 1),
Expression::Simple(Value::Symbol(value_node!("arg1".to_string(), 1, 1))),
make_tok!("foo", Position::new(1, 1, 1)),
Expression::Simple(Value::Symbol(value_node!(
"arg1".to_string(),
Position::new(1, 1, 1)
))),
)],
pos: Position::new(1, 0),
pos: Position::new(1, 0, 0),
})));
test_expr_to_val(
vec![(
Expression::Call(CallDef {
macroref: make_selector!(make_expr!("tstmac")),
macroref: make_selector!(
make_expr!("tstmac", Position::new(1, 1, 1)),
Position::new(1, 1, 1)
),
arglist: vec![Expression::Simple(Value::Str(value_node!(
"bar".to_string(),
1,
1
Position::new(1, 1, 1)
)))],
pos: Position::new(1, 1),
pos: Position::new(1, 1, 1),
}),
Val::Tuple(vec![(
value_node!("foo".to_string(), 1, 0),
value_node!("foo".to_string(), Position::new(1, 1, 1)),
Rc::new(Val::Str("bar".to_string())),
)]),
)],
@ -713,126 +263,42 @@ fn test_macro_hermetic() {
);
}
#[test]
fn test_select_expr() {
let cache = Rc::new(RefCell::new(MemoryCache::new()));
let mut b = Builder::new(std::env::current_dir().unwrap(), cache);
b.build_output
.entry(value_node!("foo".to_string(), 1, 0))
.or_insert(Rc::new(Val::Str("bar".to_string())));
b.build_output
.entry(value_node!("baz".to_string(), 1, 0))
.or_insert(Rc::new(Val::Str("boo".to_string())));
test_expr_to_val(
vec![
(
Expression::Select(SelectDef {
val: Box::new(Expression::Simple(Value::Symbol(value_node!(
"foo".to_string(),
1,
1
)))),
default: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
tuple: vec![
(
make_tok!("foo", 1, 1),
Expression::Simple(Value::Str(value_node!("2".to_string(), 1, 1))),
),
(
make_tok!("bar", 1, 1),
Expression::Simple(Value::Int(value_node!(2, 1, 1))),
),
],
pos: Position::new(1, 0),
}),
Val::Int(2),
),
(
Expression::Select(SelectDef {
val: Box::new(Expression::Simple(Value::Symbol(value_node!(
"baz".to_string(),
1,
1
)))),
default: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
tuple: vec![
(
make_tok!("bar", 1, 1),
Expression::Simple(Value::Int(value_node!(2, 1, 1))),
),
(
make_tok!("quux", 1, 1),
Expression::Simple(Value::Str(value_node!("2".to_string(), 1, 1))),
),
],
pos: Position::new(1, 0),
}),
// If the field doesn't exist then we get the default.
Val::Int(1),
),
],
b,
);
}
#[test]
#[should_panic(expected = "Expected String but got Integer in Select expression")]
fn test_select_expr_not_a_string() {
let cache = Rc::new(RefCell::new(MemoryCache::new()));
let mut b = Builder::new(std::env::current_dir().unwrap(), cache);
b.build_output
.entry(value_node!("foo".to_string(), 1, 0))
.entry(value_node!("foo".to_string(), Position::new(1, 0, 0)))
.or_insert(Rc::new(Val::Int(4)));
test_expr_to_val(
vec![(
Expression::Select(SelectDef {
val: Box::new(Expression::Simple(Value::Symbol(value_node!(
"foo".to_string(),
1,
1
Position::new(1, 1, 1)
)))),
default: Box::new(Expression::Simple(Value::Int(value_node!(
1,
Position::new(1, 1, 1)
)))),
default: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
tuple: vec![
(
make_tok!("bar", 1, 1),
Expression::Simple(Value::Int(value_node!(2, 1, 1))),
make_tok!("bar", Position::new(1, 1, 1)),
Expression::Simple(Value::Int(value_node!(2, Position::new(1, 1, 1)))),
),
(
make_tok!("quux", 1, 1),
Expression::Simple(Value::Str(value_node!("2".to_string(), 1, 1))),
make_tok!("quux", Position::new(1, 1, 1)),
Expression::Simple(Value::Str(value_node!(
"2".to_string(),
Position::new(1, 1, 1)
))),
),
],
pos: Position::new(1, 0),
pos: Position::new(1, 0, 0),
}),
Val::Int(2),
)],
b,
);
}
#[test]
fn test_let_statement() {
let cache = MemoryCache::new();
let mut b = Builder::new("<Eval>", Rc::new(RefCell::new(cache)));
let stmt = Statement::Let(LetDef {
name: make_tok!("foo", 1, 1),
value: Expression::Simple(Value::Str(value_node!("bar".to_string(), 1, 1))),
});
b.build_stmt(&stmt).unwrap();
test_expr_to_val(
vec![(
Expression::Simple(Value::Symbol(value_node!("foo".to_string(), 1, 1))),
Val::Str("bar".to_string()),
)],
b,
);
}
#[test]
fn test_build_file_string() {
let cache = Rc::new(RefCell::new(MemoryCache::new()));
let mut b = Builder::new(std::env::current_dir().unwrap(), cache);
b.eval_string("let foo = 1;").unwrap();
let key = value_node!("foo".to_string(), 1, 0);
assert!(b.build_output.contains_key(&key));
}

8
src/convert/env.rs
View File

@ -16,7 +16,7 @@
use std::io::Write;
use std::rc::Rc;
use ast::Positioned;
use ast::PositionedItem;
use build::Val;
use convert::traits::{Converter, Result};
@ -29,7 +29,11 @@ impl EnvConverter {
EnvConverter {}
}
fn convert_tuple(&self, flds: &Vec<(Positioned<String>, Rc<Val>)>, w: &mut Write) -> Result {
fn convert_tuple(
&self,
flds: &Vec<(PositionedItem<String>, Rc<Val>)>,
w: &mut Write,
) -> Result {
for &(ref name, ref val) in flds.iter() {
if val.is_tuple() {
eprintln!("Skipping embedded tuple...");

12
src/convert/exec.rs
View File

@ -17,7 +17,7 @@ use std;
use std::io::{Cursor, Write};
use std::rc::Rc;
use ast::{Position, Positioned};
use ast::{Position, PositionedItem};
use build::Val;
use build::Val::Tuple;
use convert;
@ -46,10 +46,10 @@ impl ExecConverter {
return Err(Box::new(Error::new(
"Exec tuples must have no more than 3 fields",
ErrorType::TypeFail,
Position::new(0, 0),
Position::new(0, 0, 0),
)));
}
let mut env: Option<&Vec<(Positioned<String>, Rc<Val>)>> = None;
let mut env: Option<&Vec<(PositionedItem<String>, Rc<Val>)>> = None;
let mut command: Option<&str> = None;
let mut args: Option<&Vec<Rc<Val>>> = None;
for &(ref name, ref val) in fields.iter() {
@ -115,7 +115,7 @@ impl ExecConverter {
return Err(Box::new(Error::new(
"An exec tuple must have a command field",
ErrorType::TypeFail,
Position::new(0, 0),
Position::new(0, 0, 0),
)));
}
// Okay if we have made it this far then we are ready to start creating our script.
@ -157,7 +157,7 @@ impl ExecConverter {
return Err(Box::new(Error::new(
"Exec args must be a list of strings or tuples of strings.",
ErrorType::TypeFail,
Position::new(0, 0),
Position::new(0, 0, 0),
)))
}
}
@ -173,7 +173,7 @@ impl ExecConverter {
Err(Box::new(Error::new(
"Exec outputs must be of type Tuple",
ErrorType::TypeFail,
Position::new(0, 0),
Position::new(0, 0, 0),
)))
}
}

2
src/convert/json.rs
View File

@ -36,7 +36,7 @@ impl JsonConverter {
fn convert_tuple(
&self,
items: &Vec<(ast::Positioned<String>, Rc<Val>)>,
items: &Vec<(ast::PositionedItem<String>, Rc<Val>)>,
) -> std::io::Result<serde_json::Value> {
let mut mp = serde_json::Map::new();
for &(ref k, ref v) in items.iter() {

2
src/convert/yaml.rs
View File

@ -25,7 +25,7 @@ impl YamlConverter {
fn convert_tuple(
&self,
items: &Vec<(ast::Positioned<String>, Rc<Val>)>,
items: &Vec<(ast::PositionedItem<String>, Rc<Val>)>,
) -> std::io::Result<serde_yaml::Value> {
let mut mapping = serde_yaml::Mapping::new();
for &(ref k, ref v) in items.iter() {

89
src/error.rs
View File

@ -15,11 +15,12 @@
//! Errors for use by the ucg compiler.
use std::error;
use std::fmt;
use std::fmt::Debug;
use abortable_parser::Positioned;
use ast::*;
use nom;
/// ErrorType defines the various types of errors that can result from compiling UCG into an
/// output format.
pub enum ErrorType {
@ -62,7 +63,6 @@ pub struct Error {
pub err_type: ErrorType,
pub pos: Position,
pub msg: String,
pub cause: Option<Box<Error>>,
_pkgonly: (),
}
@ -72,51 +72,21 @@ impl Error {
err_type: t,
pos: pos,
msg: msg.into(),
cause: None,
_pkgonly: (),
}
}
pub fn new_with_boxed_cause<S: Into<String>>(msg: S, t: ErrorType, cause: Box<Self>) -> Self {
let mut e = Self::new(msg, t, cause.pos.clone());
e.cause = Some(cause);
return e;
}
pub fn new_with_cause<S: Into<String>>(msg: S, t: ErrorType, cause: Self) -> Self {
Self::new_with_boxed_cause(msg, t, Box::new(cause))
}
pub fn new_with_errorkind<S: Into<String>>(
msg: S,
t: ErrorType,
pos: Position,
cause: nom::ErrorKind<Error>,
) -> Self {
match cause {
nom::ErrorKind::Custom(e) => Self::new_with_cause(msg, t, e),
e => Self::new_with_cause(
msg,
t,
Error::new(format!("ErrorKind: {}", e), ErrorType::Unsupported, pos),
),
}
}
fn render(&self, w: &mut fmt::Formatter) -> fmt::Result {
try!(write!(
w,
"{}: \"{}\" at line: {} column: {}",
self.err_type, self.msg, self.pos.line, self.pos.column
"{} at line: {} column: {}\nCaused By:\n\t{} ",
self.err_type, self.pos.line, self.pos.column, self.msg
));
if let Some(ref cause) = self.cause {
try!(write!(w, "\n\tCaused By: {}", cause));
}
Ok(())
}
}
impl fmt::Debug for Error {
impl Debug for Error {
fn fmt(&self, w: &mut fmt::Formatter) -> fmt::Result {
self.render(w)
}
@ -133,3 +103,50 @@ impl error::Error for Error {
&self.msg
}
}
#[derive(Debug)]
pub struct StackPrinter<C: abortable_parser::Positioned> {
pub err: abortable_parser::Error<C>,
}
impl<C> StackPrinter<C>
where
C: abortable_parser::Positioned,
{
pub fn render(&self, w: &mut fmt::Formatter) -> fmt::Result {
let mut curr_err = Some(&self.err);
let mut tabstop = "";
loop {
match curr_err {
// our exit condition;
None => break,
Some(err) => {
let context = err.get_context();
try!(write!(
w,
"{}{}: line: {}, column: {}\n",
tabstop,
err.get_msg(),
context.line(),
context.column(),
));
tabstop = "\t";
curr_err = err.get_cause();
if curr_err.is_some() {
try!(write!(w, "Caused by: \n"));
}
}
}
}
Ok(())
}
}
impl<C> fmt::Display for StackPrinter<C>
where
C: Positioned,
{
fn fmt(&self, w: &mut fmt::Formatter) -> fmt::Result {
self.render(w)
}
}

6
src/format.rs
View File

@ -82,21 +82,21 @@ mod test {
#[test]
fn test_format_happy_path() {
let formatter = Formatter::new("foo @ @ \\@", vec!["bar", "quux"]);
let pos = Position { line: 0, column: 0 };
let pos = Position::new(0, 0, 0);
assert_eq!(formatter.render(&pos).unwrap(), "foo bar quux @");
}
#[test]
fn test_format_happy_wrong_too_few_args() {
let formatter = Formatter::new("foo @ @ \\@", vec!["bar"]);
let pos = Position { line: 0, column: 0 };
let pos = Position::new(0, 0, 0);
assert!(formatter.render(&pos).is_err());
}
#[test]
fn test_format_happy_wrong_too_many_args() {
let formatter = Formatter::new("foo @ @ \\@", vec!["bar", "quux", "baz"]);
let pos = Position { line: 0, column: 0 };
let pos = Position::new(0, 0, 0);
assert!(formatter.render(&pos).is_err());
}
}

107
src/iter.rs Normal file
View File

@ -0,0 +1,107 @@
use std::convert::From;
use std::iter::Iterator;
use abortable_parser::iter::{SliceIter, StrIter};
use abortable_parser::{InputIter, Offsetable, Peekable, Positioned, Seekable, Span, SpanRange};
use ast::{Position, Token};
#[derive(Debug)]
pub struct OffsetStrIter<'a> {
contained: StrIter<'a>,
line_offset: usize,
col_offset: usize,
}
impl<'a> OffsetStrIter<'a> {
pub fn new(input: &'a str) -> Self {
Self::new_with_offsets(input, 0, 0)
}
pub fn new_with_offsets(input: &'a str, line_offset: usize, col_offset: usize) -> Self {
OffsetStrIter {
contained: StrIter::new(input),
line_offset: line_offset,
col_offset: col_offset,
}
}
}
impl<'a> Iterator for OffsetStrIter<'a> {
type Item = &'a u8;
fn next(&mut self) -> Option<Self::Item> {
self.contained.next()
}
}
impl<'a> Offsetable for OffsetStrIter<'a> {
fn get_offset(&self) -> usize {
self.contained.get_offset()
}
}
impl<'a> Clone for OffsetStrIter<'a> {
fn clone(&self) -> Self {
OffsetStrIter {
contained: self.contained.clone(),
line_offset: self.line_offset,
col_offset: self.col_offset,
}
}
}
impl<'a> From<&'a str> for OffsetStrIter<'a> {
fn from(source: &'a str) -> Self {
OffsetStrIter {
contained: StrIter::new(source),
line_offset: 0,
col_offset: 0,
}
}
}
impl<'a> Seekable for OffsetStrIter<'a> {
fn seek(&mut self, to: usize) -> usize {
self.contained.seek(to)
}
}
impl<'a> Span<&'a str> for OffsetStrIter<'a> {
fn span(&self, idx: SpanRange) -> &'a str {
self.contained.span(idx)
}
}
impl<'a> Peekable<&'a u8> for OffsetStrIter<'a> {
fn peek_next(&self) -> Option<&'a u8> {
self.contained.peek_next()
}
}
impl<'a> Positioned for OffsetStrIter<'a> {
fn line(&self) -> usize {
self.contained.line() + self.line_offset
}
fn column(&self) -> usize {
self.contained.column() + self.col_offset
}
}
impl<'a> InputIter for OffsetStrIter<'a> {}
impl<'a> From<&'a SliceIter<'a, Token>> for Position {
fn from(source: &'a SliceIter<'a, Token>) -> Self {
match source.peek_next() {
Some(t) => t.pos.clone(),
None => Position::new(0, 0, 0),
}
}
}
impl<'a> From<&'a OffsetStrIter<'a>> for Position {
fn from(s: &'a OffsetStrIter<'a>) -> Position {
Position::new(s.line(), s.column(), s.get_offset())
}
}

5
src/lib.rs
View File

@ -444,9 +444,7 @@
// to succeed.
#![recursion_limit = "128"]
#[macro_use]
extern crate nom;
#[macro_use]
extern crate nom_locate;
extern crate abortable_parser;
extern crate serde_json;
extern crate serde_yaml;
extern crate simple_error;
@ -458,6 +456,7 @@ pub mod tokenizer;
pub mod build;
pub mod convert;
pub mod error;
pub mod iter;
pub mod parse;
mod format;

1023
src/parse/mod.rs
View File

File diff suppressed because it is too large Load Diff

433
src/parse/precedence.rs
View File

@ -14,14 +14,11 @@
//! Bottom up parser for precedence parsing of expressions separated by binary
//! operators.
use std;
use abortable_parser::combinators::eoi;
use abortable_parser::{Error, Result, SliceIter};
use nom::{ErrorKind, IResult, InputIter, InputLength, Slice};
use super::{non_op_expression, NomResult, ParseResult};
use super::{non_op_expression, NomResult};
use ast::*;
use error;
use tokenizer::TokenIter;
/// Defines the intermediate stages of our bottom up parser for precedence parsing.
#[derive(Debug, PartialEq, Clone)]
@ -31,120 +28,119 @@ pub enum Element {
CompareOp(CompareType),
}
named!(math_op_type<TokenIter, Element, error::Error>,
alt!(
do_parse!(punct!("+") >> (Element::MathOp(BinaryExprType::Add))) |
do_parse!(punct!("-") >> (Element::MathOp(BinaryExprType::Sub))) |
do_parse!(punct!("*") >> (Element::MathOp(BinaryExprType::Mul))) |
do_parse!(punct!("/") >> (Element::MathOp(BinaryExprType::Div)))
make_fn!(
math_op_type<SliceIter<Token>, Element>,
either!(
do_each!(
_ => punct!("+"),
(Element::MathOp(BinaryExprType::Add))),
do_each!(
_ => punct!("-"),
(Element::MathOp(BinaryExprType::Sub))),
do_each!(
_ => punct!("*"),
(Element::MathOp(BinaryExprType::Mul))),
do_each!(
_ => punct!("/"),
(Element::MathOp(BinaryExprType::Div)))
)
);
fn parse_expression(i: OpListIter) -> IResult<OpListIter, Expression, error::Error> {
let i_ = i.clone();
if i_.input_len() == 0 {
return IResult::Error(ErrorKind::Custom(error::Error::new(
format!("Expected Expression found End Of Input"),
error::ErrorType::IncompleteParsing,
// TODO(jwall): This position information is incorrect.
Position { line: 0, column: 0 },
)));
fn parse_expression(i: SliceIter<Element>) -> Result<SliceIter<Element>, Expression> {
let mut i_ = i.clone();
if eoi(i_.clone()).is_complete() {
return Result::Abort(Error::new(
"Expected Expression found End Of Input",
Box::new(i_),
));
}
let el = &(i_[0]);
if let &Element::Expr(ref expr) = el {
return IResult::Done(i.slice(1..), expr.clone());
let el = i_.next();
if let Some(&Element::Expr(ref expr)) = el {
return Result::Complete(i_.clone(), expr.clone());
}
return IResult::Error(ErrorKind::Custom(error::Error::new(
return Result::Fail(Error::new(
format!(
"Error while parsing Binary Expression Unexpected Operator {:?}",
"Error while parsing Binary Expression Expected Expression got {:?}",
el
),
error::ErrorType::ParseError,
// TODO(jwall): This position information is incorrect.
Position { line: 0, column: 0 },
)));
Box::new(i_),
));
}
fn parse_sum_operator(i: OpListIter) -> IResult<OpListIter, BinaryExprType, error::Error> {
let i_ = i.clone();
if i_.input_len() == 0 {
return IResult::Error(ErrorKind::Custom(error::Error::new(
fn parse_sum_operator(i: SliceIter<Element>) -> Result<SliceIter<Element>, BinaryExprType> {
let mut i_ = i.clone();
if eoi(i_.clone()).is_complete() {
return Result::Fail(Error::new(
format!("Expected Expression found End Of Input"),
error::ErrorType::IncompleteParsing,
// TODO(jwall): This position information is incorrect.
Position { line: 0, column: 0 },
)));
Box::new(i_),
));
}
let el = &(i_[0]);
if let &Element::MathOp(ref op) = el {
let el = i_.next();
if let Some(&Element::MathOp(ref op)) = el {
match op {
&BinaryExprType::Add => {
return IResult::Done(i.slice(1..), op.clone());
return Result::Complete(i_.clone(), op.clone());
}
&BinaryExprType::Sub => {
return IResult::Done(i.slice(1..), op.clone());
return Result::Complete(i_.clone(), op.clone());
}
_other => {
// noop
}
};
}
return IResult::Error(ErrorKind::Custom(error::Error::new(
return Result::Fail(Error::new(
format!(
"Error while parsing Binary Expression Unexpected Operator {:?}",
el
),
error::ErrorType::ParseError,
// TODO(jwall): This position information is incorrect.
Position { line: 0, column: 0 },
)));
Box::new(i_),
));
}
fn tuple_to_binary_expression(
tpl: (BinaryExprType, Expression, Expression),
) -> ParseResult<Expression> {
let pos = tpl.1.pos().clone();
Ok(Expression::Binary(BinaryOpDef {
kind: tpl.0,
left: Box::new(tpl.1),
right: Box::new(tpl.2),
kind: BinaryExprType,
left: Expression,
right: Expression,
) -> Expression {
let pos = left.pos().clone();
Expression::Binary(BinaryOpDef {
kind: kind,
left: Box::new(left),
right: Box::new(right),
pos: pos,
}))
})
}
fn parse_product_operator(i: OpListIter) -> IResult<OpListIter, BinaryExprType, error::Error> {
let i_ = i.clone();
if i_.input_len() == 0 {
return IResult::Error(ErrorKind::Custom(error::Error::new(
fn parse_product_operator(i: SliceIter<Element>) -> Result<SliceIter<Element>, BinaryExprType> {
let mut i_ = i.clone();
if eoi(i_.clone()).is_complete() {
return Result::Fail(Error::new(
format!("Expected Expression found End Of Input"),
error::ErrorType::IncompleteParsing,
// TODO(jwall): This position information is incorrect.
Position { line: 0, column: 0 },
)));
Box::new(i_),
));
}
let el = &(i_[0]);
if let &Element::MathOp(ref op) = el {
let el = i_.next();
if let Some(&Element::MathOp(ref op)) = el {
match op {
&BinaryExprType::Mul => {
return IResult::Done(i.slice(1..), op.clone());
return Result::Complete(i_.clone(), op.clone());
}
&BinaryExprType::Div => {
return IResult::Done(i.slice(1..), op.clone());
return Result::Complete(i_.clone(), op.clone());
}
_other => {
// noop
}
};
}
return IResult::Error(ErrorKind::Custom(error::Error::new(
return Result::Fail(Error::new(
format!(
"Error while parsing Binary Expression Unexpected Operator {:?}",
el
),
error::ErrorType::ParseError,
// TODO(jwall): This position information is incorrect.
Position { line: 0, column: 0 },
)));
Box::new(i_),
));
}
/// do_binary_expr implements precedence based parsing where the more tightly bound
@ -152,15 +148,15 @@ fn parse_product_operator(i: OpListIter) -> IResult<OpListIter, BinaryExprType,
/// as the most tightly bound expressions.
macro_rules! do_binary_expr {
($i:expr, $oprule:ident, $lowerrule:ident) => {
do_binary_expr!($i, call!($oprule), $lowerrule)
do_binary_expr!($i, run!($oprule), $lowerrule)
};
($i:expr, $oprule:ident, $lowerrule:ident!( $($lowerargs:tt)* )) => {
do_binary_expr!($i, call!($oprule), $lowerrule!($($lowerargs)*))
do_binary_expr!($i, run!($oprule), $lowerrule!($($lowerargs)*))
};
($i:expr, $oprule:ident) => {
do_binary_expr!($i, call!($oprule))
do_binary_expr!($i, run!($oprule))
};
($i:expr, $oprule:ident!( $($args:tt)* )) => {
@ -168,101 +164,97 @@ macro_rules! do_binary_expr {
};
($i:expr, $oprule:ident!( $($args:tt)* ), $lowerrule:ident) => {
do_binary_expr!($i, $oprule!($($args)*), call!($lowerrule))
do_binary_expr!($i, $oprule!($($args)*), run!($lowerrule))
};
($i:expr, $oprule:ident!( $($args:tt)* ), $lowerrule:ident!( $($lowerargs:tt)* )) => {
map_res!($i,
do_parse!(
left: $lowerrule!($($lowerargs)*) >>
typ: $oprule!($($args)*) >>
right: $lowerrule!($($lowerargs)*) >>
(typ, left, right)
),
tuple_to_binary_expression
do_each!($i,
left => $lowerrule!($($lowerargs)*),
typ => $oprule!($($args)*),
right => $lowerrule!($($lowerargs)*),
(tuple_to_binary_expression(typ, left, right))
)
};
}
named!(sum_expression<OpListIter, Expression, error::Error>,
make_fn!(
sum_expression<SliceIter<Element>, Expression>,
do_binary_expr!(
parse_sum_operator,
alt!(trace_nom!(product_expression) | trace_nom!(parse_expression)))
either!(trace_nom!(product_expression), trace_nom!(parse_expression))
)
);
named!(product_expression<OpListIter, Expression, error::Error>,
do_binary_expr!(
parse_product_operator,
trace_nom!(parse_expression))
make_fn!(
product_expression<SliceIter<Element>, Expression>,
do_binary_expr!(parse_product_operator, trace_nom!(parse_expression))
);
named!(math_expression<OpListIter, Expression, error::Error>,
alt!(trace_nom!(sum_expression) | trace_nom!(product_expression))
make_fn!(
math_expression<SliceIter<Element>, Expression>,
either!(trace_nom!(sum_expression), trace_nom!(product_expression))
);
// TODO(jwall): Change comparison operators to use the do_binary_expr! with precedence?
fn tuple_to_compare_expression(
tpl: (CompareType, Expression, Expression),
) -> ParseResult<Expression> {
let pos = tpl.1.pos().clone();
Ok(Expression::Compare(ComparisonDef {
kind: tpl.0,
left: Box::new(tpl.1),
right: Box::new(tpl.2),
kind: CompareType,
left: Expression,
right: Expression,
) -> Expression {
let pos = left.pos().clone();
Expression::Compare(ComparisonDef {
kind: kind,
left: Box::new(left),
right: Box::new(right),
pos: pos,
}))
})
}
named!(compare_op_type<TokenIter, Element, error::Error>,
alt!(
do_parse!(punct!("==") >> (Element::CompareOp(CompareType::Equal))) |
do_parse!(punct!("!=") >> (Element::CompareOp(CompareType::NotEqual))) |
do_parse!(punct!("<=") >> (Element::CompareOp(CompareType::LTEqual))) |
do_parse!(punct!(">=") >> (Element::CompareOp(CompareType::GTEqual))) |
do_parse!(punct!("<") >> (Element::CompareOp(CompareType::LT))) |
do_parse!(punct!(">") >> (Element::CompareOp(CompareType::GT)))
make_fn!(
compare_op_type<SliceIter<Token>, Element>,
either!(
do_each!(_ => punct!("=="), (Element::CompareOp(CompareType::Equal))),
do_each!(_ => punct!("!="), (Element::CompareOp(CompareType::NotEqual))),
do_each!(_ => punct!("<="), (Element::CompareOp(CompareType::LTEqual))),
do_each!(_ => punct!(">="), (Element::CompareOp(CompareType::GTEqual))),
do_each!(_ => punct!("<"), (Element::CompareOp(CompareType::LT))),
do_each!(_ => punct!(">"), (Element::CompareOp(CompareType::GT)))
)
);
fn parse_compare_operator(i: OpListIter) -> IResult<OpListIter, CompareType, error::Error> {
let i_ = i.clone();
if i_.input_len() == 0 {
return IResult::Error(ErrorKind::Custom(error::Error::new(
fn parse_compare_operator(i: SliceIter<Element>) -> Result<SliceIter<Element>, CompareType> {
let mut i_ = i.clone();
if eoi(i_.clone()).is_complete() {
return Result::Fail(Error::new(
format!("Expected Expression found End Of Input"),
error::ErrorType::IncompleteParsing,
// TODO(jwall): This position information is incorrect.
Position { line: 0, column: 0 },
)));
Box::new(i_),
));
}
let el = &(i_[0]);
if let &Element::CompareOp(ref op) = el {
return IResult::Done(i.slice(1..), op.clone());
let el = i_.next();
if let Some(&Element::CompareOp(ref op)) = el {
return Result::Complete(i_.clone(), op.clone());
}
return IResult::Error(ErrorKind::Custom(error::Error::new(
return Result::Fail(Error::new(
format!(
"Error while parsing Binary Expression Unexpected Operator {:?}",
el
),
error::ErrorType::ParseError,
// TODO(jwall): This position information is incorrect.
Position { line: 0, column: 0 },
)));
Box::new(i),
));
}
named!(compare_expression<OpListIter, Expression, error::Error>,
map_res!(
do_parse!(
left: alt!(trace_nom!(math_expression) | trace_nom!(parse_expression)) >>
typ: parse_compare_operator >>
right: alt!(trace_nom!(math_expression) | trace_nom!(parse_expression)) >>
(typ, left, right)
),
tuple_to_compare_expression
make_fn!(
compare_expression<SliceIter<Element>, Expression>,
do_each!(
left => either!(trace_nom!(math_expression), trace_nom!(parse_expression)),
typ => parse_compare_operator,
right => either!(trace_nom!(math_expression), trace_nom!(parse_expression)),
(tuple_to_compare_expression(typ, left, right))
)
);
/// Parse a list of expressions separated by operators into a Vec<Element>.
fn parse_operand_list(i: TokenIter) -> NomResult<Vec<Element>> {
fn parse_operand_list<'a>(i: SliceIter<'a, Token>) -> NomResult<'a, Vec<Element>> {
// 1. First try to parse a non_op_expression,
let mut _i = i.clone();
let mut list = Vec::new();
@ -271,148 +263,101 @@ fn parse_operand_list(i: TokenIter) -> NomResult<Vec<Element>> {
loop {
// 2. Parse a non_op_expression.
match non_op_expression(_i.clone()) {
IResult::Error(e) => {
Result::Fail(e) => {
// A failure to parse an expression
// is always an error.
return IResult::Error(e);
return Result::Fail(e);
}
IResult::Incomplete(i) => {
return IResult::Incomplete(i);
Result::Abort(e) => {
// A failure to parse an expression
// is always an error.
return Result::Abort(e);
}
IResult::Done(rest, expr) => {
Result::Incomplete(i) => {
return Result::Incomplete(i);
}
Result::Complete(rest, expr) => {
list.push(Element::Expr(expr));
_i = rest.clone();
}
}
// 3. Parse an operator.
match alt!(_i, math_op_type | compare_op_type) {
IResult::Error(e) => {
match either!(_i.clone(), math_op_type, compare_op_type) {
Result::Fail(e) => {
if firstrun {
// If we don't find an operator in our first
// run then this is not an operand list.
return IResult::Error(e);
return Result::Fail(e);
}
// if we don't find one on subsequent runs then
// that's the end of the operand list.
break;
}
IResult::Incomplete(i) => {
return IResult::Incomplete(i);
Result::Abort(e) => {
// A failure to parse an expression
// is always an error.
return Result::Abort(e);
}
IResult::Done(rest, el) => {
Result::Incomplete(i) => {
return Result::Incomplete(i);
}
Result::Complete(rest, el) => {
list.push(el);
_i = rest.clone();
}
}
firstrun = false;
}
return IResult::Done(_i, list);
}
#[derive(Clone, Debug, PartialEq)]
pub struct OpListIter<'a> {
pub source: &'a [Element],
}
impl<'a> OpListIter<'a> {
pub fn len(&self) -> usize {
self.source.len()
}
}
impl<'a> InputLength for OpListIter<'a> {
fn input_len(&self) -> usize {
self.source.input_len()
}
}
macro_rules! impl_op_iter_slice {
($r:ty) => {
impl<'a> Slice<$r> for OpListIter<'a> {
fn slice(&self, range: $r) -> Self {
OpListIter {
source: self.source.slice(range),
}
}
}
};
}
impl_op_iter_slice!(std::ops::Range<usize>);
impl_op_iter_slice!(std::ops::RangeTo<usize>);
impl_op_iter_slice!(std::ops::RangeFrom<usize>);
impl_op_iter_slice!(std::ops::RangeFull);
impl<'a> std::ops::Index<usize> for OpListIter<'a> {
type Output = Element;
fn index(&self, i: usize) -> &Self::Output {
&self.source[i]
}
}
impl<'a> InputIter for OpListIter<'a> {
type Item = &'a Element;
type RawItem = Element;
type Iter = std::iter::Enumerate<std::slice::Iter<'a, Self::RawItem>>;
type IterElem = std::slice::Iter<'a, Self::RawItem>;
fn iter_indices(&self) -> Self::Iter {
self.source.iter().enumerate()
}
fn iter_elements(&self) -> Self::IterElem {
self.source.iter()
}
fn position<P>(&self, predicate: P) -> Option<usize>
where
P: Fn(Self::RawItem) -> bool,
{
for (o, v) in self.iter_indices() {
if predicate(v.clone()) {
return Some(o);
}
}
None
}
fn slice_index(&self, count: usize) -> Option<usize> {
let mut cnt = 0;
for (index, _) in self.iter_indices() {
if cnt == count {
return Some(index);
}
cnt += 1;
}
if cnt == count {
return Some(self.len());
}
None
}
return Result::Complete(_i, list);
}
/// Parse a binary operator expression.
pub fn op_expression(i: TokenIter) -> NomResult<Expression> {
pub fn op_expression<'a>(i: SliceIter<'a, Token>) -> Result<SliceIter<Token>, Expression> {
let preparse = parse_operand_list(i.clone());
match preparse {
IResult::Error(e) => IResult::Error(e),
IResult::Incomplete(i) => IResult::Incomplete(i),
IResult::Done(rest, oplist) => {
let mut i_ = OpListIter {
source: oplist.as_slice(),
};
let parse_result = alt!(
i_,
trace_nom!(compare_expression) | trace_nom!(math_expression)
Result::Fail(e) => {
let err = Error::caused_by(
"Failed while parsing operator expression",
Box::new(e),
Box::new(i),
);
Result::Fail(err)
}
Result::Abort(e) => {
let err = Error::caused_by(
"Failed while parsing operator expression",
Box::new(e),
Box::new(i),
);
Result::Fail(err)
}
Result::Incomplete(i) => Result::Incomplete(i),
Result::Complete(rest, oplist) => {
let mut i_ = SliceIter::new(&oplist);
let parse_result = either!(
i_.clone(),
trace_nom!(compare_expression),
trace_nom!(math_expression)
);
match parse_result {
IResult::Error(e) => IResult::Error(e),
IResult::Incomplete(i) => IResult::Incomplete(i),
IResult::Done(_, expr) => IResult::Done(rest.clone(), expr),
Result::Fail(_e) => {
// TODO(jwall): It would be good to be able to use caused_by here.
let err = Error::new(
"Failed while parsing operator expression",
Box::new(rest.clone()),
);
Result::Fail(err)
}
Result::Abort(_e) => {
let err = Error::new(
"Failed while parsing operator expression",
Box::new(rest.clone()),
);
Result::Abort(err)
}
Result::Incomplete(_) => Result::Incomplete(i.clone()),
Result::Complete(_, expr) => Result::Complete(rest.clone(), expr),
}
}
}

1079
src/parse/test.rs
View File

File diff suppressed because it is too large Load Diff

717
src/tokenizer/mod.rs
View File

@ -12,415 +12,424 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//! The tokenization stage of the ucg compiler.
use ast::*;
use error;
use nom;
use nom::{alpha, digit, is_alphanumeric, multispace};
use nom::{InputIter, InputLength, Slice};
use nom_locate::LocatedSpan;
//! The tokenization stext_tokene of the ucg compiler.
use std;
use std::result::Result;
pub type Span<'a> = LocatedSpan<&'a str>;
use abortable_parser::combinators::*;
use abortable_parser::iter::SliceIter;
use abortable_parser::{Error, Offsetable, Result};
impl<'a> From<Span<'a>> for Position {
fn from(s: Span) -> Position {
Position {
line: s.line as usize,
column: s.get_column() as usize,
use ast::*;
use error::StackPrinter;
use iter::OffsetStrIter;
fn is_symbol_char<'a>(i: OffsetStrIter<'a>) -> Result<OffsetStrIter<'a>, u8> {
let mut _i = i.clone();
let c = match _i.next() {
Some(c) => *c,
None => {
return Result::Fail(Error::new(
"Unexpected End of Input".to_string(),
Box::new(_i.clone()),
))
}
};
if (c as char).is_ascii_alphanumeric() || c == b'-' || c == b'_' {
Result::Complete(_i, c)
} else {
Result::Fail(Error::new(
"Not a symbol character".to_string(),
Box::new(_i.clone()),
))
}
}
fn is_symbol_char(c: char) -> bool {
is_alphanumeric(c as u8) || c == '-' as char || c == '_' as char
}
fn escapequoted(input: Span) -> nom::IResult<Span, String> {
fn escapequoted<'a>(input: OffsetStrIter<'a>) -> Result<OffsetStrIter<'a>, String> {
// loop until we find a " that is not preceded by \.
// Collapse all \<char> to just char for escaping.
let mut frag = String::new();
let mut escape = false;
for (i, c) in input.iter_indices() {
if c == '\\' && !escape {
let mut _input = input.clone();
loop {
let c = match _input.next() {
Some(c) => *c,
None => break,
};
if c == '\\' as u8 && !escape {
// eat this slash and set our escaping sentinel
escape = true;
} else if c == '"' && !escape {
} else if c == '"' as u8 && !escape {
// Bail if this is an unescaped "
// we exit here.
return nom::IResult::Done(input.slice(i..), frag);
return Result::Complete(_input, frag);
} else {
// we accumulate this character.
frag.push(c);
frag.push(c as char);
escape = false; // reset our escaping sentinel
}
}
return nom::IResult::Incomplete(nom::Needed::Unknown);
return Result::Incomplete(_input.clone());
}
named!(strtok( Span ) -> Token,
do_parse!(
span: position!() >>
tag!("\"") >>
frag: escapequoted >>
tag!("\"") >>
make_fn!(strtok<OffsetStrIter, Token>,
do_each!(
span => input!(),
_ => text_token!("\""),
frag => escapequoted,
(Token{
typ: TokenType::QUOTED,
pos: Position::from(span),
fragment: frag,
pos: Position::from(&span),
fragment: frag.to_string(),
})
)
);
named!(pipequotetok( Span ) -> Token,
do_parse!(
span: position!() >>
tag!("|") >>
frag: take_until!("|") >>
tag!("|") >>
make_fn!(pipequotetok<OffsetStrIter, Token>,
do_each!(
p => input!(),
_ => text_token!("|"),
frag => until!(text_token!("|")),
_ => text_token!("|"),
(Token{
typ: TokenType::PIPEQUOTE,
pos: Position::from(span),
fragment: frag.fragment.to_string(),
pos: Position::from(&p),
fragment: frag.to_string(),
})
)
);
named!(barewordtok( Span ) -> Token,
do_parse!(
span: position!() >>
frag: preceded!(peek!(alpha), take_while!(is_symbol_char)) >>
make_fn!(barewordtok<OffsetStrIter, Token>,
do_each!(
span => input!(),
_ => peek!(ascii_alpha),
frag => consume_all!(is_symbol_char),
(Token{
typ: TokenType::BAREWORD,
pos: Position::from(span),
fragment: frag.fragment.to_string(),
pos: Position::from(&span),
fragment: frag.to_string(),
})
)
);
named!(digittok( Span ) -> Token,
do_parse!(
span: position!() >>
digits: digit >>
(Token{
typ: TokenType::DIGIT,
pos: Position::from(span),
fragment: digits.fragment.to_string(),
})
make_fn!(digittok<OffsetStrIter, Token>,
do_each!(
span => input!(),
_ => peek!(ascii_digit),
digits => consume_all!(ascii_digit),
(Token{
typ: TokenType::DIGIT,
pos: Position::from(&span),
fragment: digits.to_string(),
})
)
);
named!(booleantok( Span ) -> Token,
do_parse!(
span: position!() >>
b: alt!(
tag!("true") |
tag!("false")
) >>
make_fn!(booleantok<OffsetStrIter, Token>,
do_each!(
span => input!(),
token => either!(
text_token!("true"),
text_token!("false")
),
(Token{
typ: TokenType::BOOLEAN,
pos: Position::from(span),
fragment: b.fragment.to_string(),
pos: Position::from(&span),
fragment: token.to_string(),
})
)
);
/// do_tag_tok! is a helper macro to make building a simple tag token
/// do_text_token_tok! is a helper macro to make building a simple text_token token
/// less code.
macro_rules! do_tag_tok {
// NOTE(jwall): Nom macros do magic with their inputs. They in fact
// rewrite your macro argumets for you by adding an initial argument
// for all their sub-macros. Which means we require this $i paramater
// on the first macro invocation but not the rest.
($i:expr, $type:expr, $tag:expr,WS) => {
do_parse!(
$i,
span: position!() >> frag: tag!($tag) >> alt!(whitespace | comment) >> (Token {
typ: $type,
pos: Position::from(span),
fragment: frag.fragment.to_string(),
})
)
macro_rules! do_text_token_tok {
($i:expr, $type:expr, $text_token:expr, WS) => {
do_each!($i,
span => input!(),
frag => text_token!($text_token),
_ => either!(whitespace, comment),
(Token {
typ: $type,
pos: Position::from(&span),
fragment: frag.to_string(),
})
)
};
($i:expr, $type:expr, $tag:expr) => {
do_parse!(
$i,
span: position!() >> frag: tag!($tag) >> (Token {
typ: $type,
pos: Position::from(span),
fragment: frag.fragment.to_string(),
})
)
($i:expr, $type:expr, $text_token:expr) => {
do_each!($i,
span => input!(),
frag => text_token!($text_token),
(Token {
typ: $type,
pos: Position::from(&span),
fragment: frag.to_string(),
})
)
};
}
named!(emptytok( Span ) -> Token,
do_tag_tok!(TokenType::EMPTY, "NULL")
make_fn!(emptytok<OffsetStrIter, Token>,
do_text_token_tok!(TokenType::EMPTY, "NULL")
);
named!(commatok( Span ) -> Token,
do_tag_tok!(TokenType::PUNCT, ",")
make_fn!(commatok<OffsetStrIter, Token>,
do_text_token_tok!(TokenType::PUNCT, ",")
);
named!(lbracetok( Span ) -> Token,
do_tag_tok!(TokenType::PUNCT, "{")
make_fn!(lbracetok<OffsetStrIter, Token>,
do_text_token_tok!(TokenType::PUNCT, "{")
);
named!(rbracetok( Span ) -> Token,
do_tag_tok!(TokenType::PUNCT, "}")
make_fn!(rbracetok<OffsetStrIter, Token>,
do_text_token_tok!(TokenType::PUNCT, "}")
);
named!(lparentok( Span ) -> Token,
do_tag_tok!(TokenType::PUNCT, "(")
make_fn!(lparentok<OffsetStrIter, Token>,
do_text_token_tok!(TokenType::PUNCT, "(")
);
named!(rparentok( Span ) -> Token,
do_tag_tok!(TokenType::PUNCT, ")")
make_fn!(rparentok<OffsetStrIter, Token>,
do_text_token_tok!(TokenType::PUNCT, ")")
);
named!(dottok( Span ) -> Token,
do_tag_tok!(TokenType::PUNCT, ".")
make_fn!(dottok<OffsetStrIter, Token>,
do_text_token_tok!(TokenType::PUNCT, ".")
);
named!(plustok( Span ) -> Token,
do_tag_tok!(TokenType::PUNCT, "+")
make_fn!(plustok<OffsetStrIter, Token>,
do_text_token_tok!(TokenType::PUNCT, "+")
);
named!(dashtok( Span ) -> Token,
do_tag_tok!(TokenType::PUNCT, "-")
make_fn!(dashtok<OffsetStrIter, Token>,
do_text_token_tok!(TokenType::PUNCT, "-")
);
named!(startok( Span ) -> Token,
do_tag_tok!(TokenType::PUNCT, "*")
make_fn!(startok<OffsetStrIter, Token>,
do_text_token_tok!(TokenType::PUNCT, "*")
);
named!(slashtok( Span ) -> Token,
do_tag_tok!(TokenType::PUNCT, "/")
make_fn!(slashtok<OffsetStrIter, Token>,
do_text_token_tok!(TokenType::PUNCT, "/")
);
named!(pcttok( Span ) -> Token,
do_tag_tok!(TokenType::PUNCT, "%")
make_fn!(pcttok<OffsetStrIter, Token>,
do_text_token_tok!(TokenType::PUNCT, "%")
);
named!(eqeqtok( Span ) -> Token,
do_tag_tok!(TokenType::PUNCT, "==")
make_fn!(eqeqtok<OffsetStrIter, Token>,
do_text_token_tok!(TokenType::PUNCT, "==")
);
named!(notequaltok( Span ) -> Token,
do_tag_tok!(TokenType::PUNCT, "!=")
make_fn!(notequaltok<OffsetStrIter, Token>,
do_text_token_tok!(TokenType::PUNCT, "!=")
);
named!(gttok( Span ) -> Token,
do_tag_tok!(TokenType::PUNCT, ">")
make_fn!(gttok<OffsetStrIter, Token>,
do_text_token_tok!(TokenType::PUNCT, ">")
);
named!(gtequaltok( Span ) -> Token,
do_tag_tok!(TokenType::PUNCT, ">=")
make_fn!(gtequaltok<OffsetStrIter, Token>,
do_text_token_tok!(TokenType::PUNCT, ">=")
);
named!(ltequaltok( Span ) -> Token,
do_tag_tok!(TokenType::PUNCT, "<=")
make_fn!(ltequaltok<OffsetStrIter, Token>,
do_text_token_tok!(TokenType::PUNCT, "<=")
);
named!(lttok( Span ) -> Token,
do_tag_tok!(TokenType::PUNCT, "<")
make_fn!(lttok<OffsetStrIter, Token>,
do_text_token_tok!(TokenType::PUNCT, "<")
);
named!(equaltok( Span ) -> Token,
do_tag_tok!(TokenType::PUNCT, "=")
make_fn!(equaltok<OffsetStrIter, Token>,
do_text_token_tok!(TokenType::PUNCT, "=")
);
named!(semicolontok( Span ) -> Token,
do_tag_tok!(TokenType::PUNCT, ";")
make_fn!(semicolontok<OffsetStrIter, Token>,
do_text_token_tok!(TokenType::PUNCT, ";")
);
named!(leftsquarebracket( Span ) -> Token,
do_tag_tok!(TokenType::PUNCT, "[")
make_fn!(leftsquarebracket<OffsetStrIter, Token>,
do_text_token_tok!(TokenType::PUNCT, "[")
);
named!(rightsquarebracket( Span ) -> Token,
do_tag_tok!(TokenType::PUNCT, "]")
make_fn!(rightsquarebracket<OffsetStrIter, Token>,
do_text_token_tok!(TokenType::PUNCT, "]")
);
named!(fatcommatok( Span ) -> Token,
do_tag_tok!(TokenType::PUNCT, "=>")
make_fn!(fatcommatok<OffsetStrIter, Token>,
do_text_token_tok!(TokenType::PUNCT, "=>")
);
named!(selecttok( Span ) -> Token,
do_tag_tok!(TokenType::BAREWORD, "select", WS)
make_fn!(selecttok<OffsetStrIter, Token>,
do_text_token_tok!(TokenType::BAREWORD, "select", WS)
);
named!(macrotok( Span ) -> Token,
do_tag_tok!(TokenType::BAREWORD, "macro", WS)
make_fn!(macrotok<OffsetStrIter, Token>,
do_text_token_tok!(TokenType::BAREWORD, "macro", WS)
);
named!(lettok( Span ) -> Token,
do_tag_tok!(TokenType::BAREWORD, "let", WS)
make_fn!(lettok<OffsetStrIter, Token>,
do_text_token_tok!(TokenType::BAREWORD, "let", WS)
);
named!(importtok( Span ) -> Token,
do_tag_tok!(TokenType::BAREWORD, "import", WS)
make_fn!(importtok<OffsetStrIter, Token>,
do_text_token_tok!(TokenType::BAREWORD, "import", WS)
);
named!(asserttok( Span ) -> Token,
do_tag_tok!(TokenType::BAREWORD, "assert", WS)
make_fn!(asserttok<OffsetStrIter, Token>,
do_text_token_tok!(TokenType::BAREWORD, "assert", WS)
);
named!(outtok( Span ) -> Token,
do_tag_tok!(TokenType::BAREWORD, "out", WS)
make_fn!(outtok<OffsetStrIter, Token>,
do_text_token_tok!(TokenType::BAREWORD, "out", WS)
);
named!(astok( Span ) -> Token,
do_tag_tok!(TokenType::BAREWORD, "as", WS)
make_fn!(astok<OffsetStrIter, Token>,
do_text_token_tok!(TokenType::BAREWORD, "as", WS)
);
named!(maptok( Span ) -> Token,
do_tag_tok!(TokenType::BAREWORD, "map", WS)
make_fn!(maptok<OffsetStrIter, Token>,
do_text_token_tok!(TokenType::BAREWORD, "map", WS)
);
named!(filtertok( Span ) -> Token,
do_tag_tok!(TokenType::BAREWORD, "filter", WS)
make_fn!(filtertok<OffsetStrIter, Token>,
do_text_token_tok!(TokenType::BAREWORD, "filter", WS)
);
fn end_of_input(input: Span) -> nom::IResult<Span, Token> {
match eof!(input,) {
nom::IResult::Done(_, _) => {
return nom::IResult::Done(
input,
make_tok!(EOF => input.line as usize,
input.get_column() as usize),
);
}
nom::IResult::Incomplete(_) => {
return nom::IResult::Incomplete(nom::Needed::Unknown);
}
nom::IResult::Error(e) => {
return nom::IResult::Error(e);
}
}
}
fn comment(input: Span) -> nom::IResult<Span, Token> {
match tag!(input, "//") {
nom::IResult::Done(rest, _) => {
match alt!(
fn comment(input: OffsetStrIter) -> Result<OffsetStrIter, Token> {
match text_token!(input, "//") {
Result::Complete(rest, _) => {
match until!(
rest,
take_until_and_consume!("\r\n") | take_until_and_consume!("\n")
either!(
eoi,
discard!(text_token!("\r\n")),
discard!(text_token!("\n"))
)
) {
nom::IResult::Done(rest, cmt) => {
return nom::IResult::Done(
rest,
make_tok!(CMT => cmt.fragment.to_string(),
input.line as usize,
input.get_column() as usize),
);
Result::Complete(rest, cmt) => {
return Result::Complete(rest, make_tok!(CMT => cmt.to_string(), input));
}
// If we didn't find a new line then we just grab everything.
_ => {
let blen = rest.input_len();
let next = rest.slice(blen..);
let tok = rest.slice(..blen);
return nom::IResult::Done(
next,
make_tok!(CMT => tok.fragment.to_string(),
input.line as usize, input.get_column() as usize
),
);
return Result::Abort(Error::new(
"Unparsable comment".to_string(),
Box::new(rest.clone()),
));
}
}
}
nom::IResult::Incomplete(i) => return nom::IResult::Incomplete(i),
nom::IResult::Error(e) => return nom::IResult::Error(e),
Result::Incomplete(ctx) => return Result::Incomplete(ctx),
Result::Fail(e) => return Result::Fail(e),
Result::Abort(e) => return Result::Abort(e),
}
}
named!(whitespace( Span ) -> Token,
do_parse!(
span: position!() >>
many1!(multispace) >>
make_fn!(whitespace<OffsetStrIter, Token>,
do_each!(
span => input!(),
_ => peek!(ascii_ws),
_ => repeat!(ascii_ws),
(Token{
typ: TokenType::WS,
pos: Position::from(span),
pos: Position::from(&span),
fragment: String::new(),
})
)
);
named!(token( Span ) -> Token,
alt!(
strtok |
pipequotetok |
emptytok | // This must come before the barewordtok
digittok |
commatok |
rbracetok |
lbracetok |
lparentok |
rparentok |
dottok |
plustok |
dashtok |
startok |
comment | // Note comment must come before slashtok
slashtok |
pcttok |
eqeqtok |
notequaltok |
complete!(gtequaltok) |
complete!(ltequaltok) |
gttok |
lttok |
fatcommatok | // Note fatcommatok must come before equaltok
equaltok |
semicolontok |
leftsquarebracket |
rightsquarebracket |
booleantok |
lettok |
outtok |
selecttok |
asserttok |
macrotok |
importtok |
astok |
maptok |
filtertok |
barewordtok |
whitespace |
end_of_input)
make_fn!(end_of_input<OffsetStrIter, Token>,
do_each!(
span => input!(),
_ => eoi,
(Token{
typ: TokenType::END,
pos: Position::from(&span),
fragment: String::new(),
})
)
);
/// Consumes an input Span and returns either a Vec<Token> or a nom::ErrorKind.
pub fn tokenize(input: Span) -> Result<Vec<Token>, error::Error> {
fn token<'a>(input: OffsetStrIter<'a>) -> Result<OffsetStrIter<'a>, Token> {
either!(
input,
strtok,
pipequotetok,
emptytok, // This must come before the barewordtok
digittok,
commatok,
rbracetok,
lbracetok,
lparentok,
rparentok,
dottok,
plustok,
dashtok,
startok,
comment, // Note comment must come before slashtok
slashtok,
pcttok,
eqeqtok,
notequaltok,
complete!("Not >=".to_string(), gtequaltok),
complete!("Not <=".to_string(), ltequaltok),
gttok,
lttok,
fatcommatok, // Note fatcommatok must come before equaltok
equaltok,
semicolontok,
leftsquarebracket,
rightsquarebracket,
booleantok,
lettok,
outtok,
selecttok,
asserttok,
macrotok,
importtok,
astok,
maptok,
filtertok,
barewordtok,
whitespace,
end_of_input
)
}
/// Consumes an input OffsetStrIter and returns either a Vec<Token> or a error::Error.
pub fn tokenize<'a>(input: OffsetStrIter<'a>) -> std::result::Result<Vec<Token>, String> {
let mut out = Vec::new();
let mut i = input;
let mut i = input.clone();
loop {
if i.input_len() == 0 {
if let Result::Complete(_, _) = eoi(i.clone()) {
break;
}
match token(i) {
nom::IResult::Error(_e) => {
return Err(error::Error::new(
match token(i.clone()) {
Result::Abort(e) => {
let err = abortable_parser::Error::caused_by(
"Invalid Token encountered",
error::ErrorType::UnexpectedToken,
Position {
line: i.line as usize,
column: i.get_column() as usize,
},
));
Box::new(e),
Box::new(i.clone()),
);
let ctx_err = StackPrinter { err: err };
return Err(format!("{}", ctx_err));
}
nom::IResult::Incomplete(_) => {
return Err(error::Error::new(
"Unexepcted end of Input",
error::ErrorType::UnexpectedToken,
Position {
line: i.line as usize,
column: i.get_column() as usize,
},
));
Result::Fail(e) => {
let err = abortable_parser::Error::caused_by(
"Invalid Token encountered",
Box::new(e),
Box::new(i.clone()),
);
let ctx_err = StackPrinter { err: err };
return Err(format!("{}", ctx_err));
}
nom::IResult::Done(rest, tok) => {
Result::Incomplete(_offset) => {
let err =
abortable_parser::Error::new("Invalid Token encountered", Box::new(i.clone()));
let ctx_err = StackPrinter { err: err };
return Err(format!("{}", ctx_err));
}
Result::Complete(rest, tok) => {
i = rest;
if tok.typ == TokenType::COMMENT || tok.typ == TokenType::WS {
// we skip comments and whitespace
@ -434,10 +443,7 @@ pub fn tokenize(input: Span) -> Result<Vec<Token>, error::Error> {
out.push(Token {
fragment: String::new(),
typ: TokenType::END,
pos: Position {
line: i.line as usize,
column: i.get_column() as usize,
},
pos: Position::from(&i),
});
Ok(out)
}
@ -445,7 +451,7 @@ pub fn tokenize(input: Span) -> Result<Vec<Token>, error::Error> {
/// Clones a token.
///
/// This is necessary to allow the match_type and match_token macros to work.
pub fn token_clone(t: &Token) -> Result<Token, error::Error> {
pub fn token_clone(t: &Token) -> std::result::Result<Token, Error<SliceIter<Token>>> {
Ok(t.clone())
}
@ -517,28 +523,28 @@ macro_rules! match_type {
};
($i:expr, $t:expr, $msg:expr, $h:expr) => {{
let i_ = $i.clone();
use nom::Slice;
use std::convert::Into;
if i_.input_len() == 0 {
nom::IResult::Error(nom::ErrorKind::Custom(error::Error::new(
format!("End of Input! {}", $msg),
error::ErrorType::IncompleteParsing,
Position { line: 0, column: 0 },
)))
use abortable_parser::combinators::eoi;
use abortable_parser::{Error, Result};
use std;
let mut _i = $i.clone();
if eoi(_i.clone()).is_complete() {
Result::Fail(Error::new(format!("End of Input! {}", $msg), Box::new(_i)))
} else {
let tok = &(i_[0]);
if tok.typ == $t {
match $h(tok) {
Result::Ok(v) => nom::IResult::Done($i.slice(1..), v),
Result::Err(e) => nom::IResult::Error(nom::ErrorKind::Custom(e.into())),
match _i.next() {
Some(tok) => {
if tok.typ == $t {
match $h(tok) {
std::result::Result::Ok(v) => Result::Complete(_i.clone(), v),
std::result::Result::Err(e) => {
Result::Fail(Error::caused_by($msg, Box::new(e), Box::new(_i)))
}
}
} else {
Result::Fail(Error::new($msg.to_string(), Box::new($i)))
}
}
} else {
nom::IResult::Error(nom::ErrorKind::Custom(error::Error::new(
$msg.to_string(),
error::ErrorType::UnexpectedToken,
tok.pos.clone(),
)))
None => Result::Fail(Error::new($msg.to_string(), Box::new($i))),
}
}
}};
@ -553,7 +559,7 @@ macro_rules! match_token {
}};
($i:expr,PUNCT => $f:expr, $h:expr) => {
match_token!($i, TokenType::PUNCT, $f, format!("Not PUNCT ({})", $f), $h)
match_token!($i, TokenType::PUNCT, $f, format!("({})", $f), $h)
};
($i:expr,BAREWORD => $f:expr) => {{
@ -572,22 +578,26 @@ macro_rules! match_token {
};
($i:expr, $t:expr, $f:expr, $msg:expr, $h:expr) => {{
let i_ = $i.clone();
use nom;
use nom::Slice;
use std::convert::Into;
let tok = &(i_[0]);
if tok.typ == $t && &tok.fragment == $f {
match $h(tok) {
Result::Ok(v) => nom::IResult::Done($i.slice(1..), v),
Result::Err(e) => nom::IResult::Error(nom::ErrorKind::Custom(e.into())),
use abortable_parser::Result;
use std;
let mut i_ = $i.clone();
let tok = i_.next();
if let Some(tok) = tok {
if tok.typ == $t && &tok.fragment == $f {
match $h(tok) {
std::result::Result::Ok(v) => Result::Complete(i_.clone(), v),
std::result::Result::Err(e) => {
Result::Fail(Error::caused_by($msg, Box::new(e), Box::new(i_)))
}
}
} else {
Result::Fail(Error::new(
format!("Expected {} Instead is ({})", $msg, tok.fragment),
Box::new(i_),
))
}
} else {
nom::IResult::Error(nom::ErrorKind::Custom(error::Error::new(
format!("{} Instead is ({})", $msg, tok.fragment),
error::ErrorType::UnexpectedToken,
tok.pos.clone(),
)))
Result::Fail(Error::new("Unexpected End Of Input", Box::new(i_)))
}
}};
}
@ -607,103 +617,12 @@ macro_rules! word {
}
/// pos gets the current position from a TokenIter input without consuming it.
pub fn pos(i: TokenIter) -> nom::IResult<TokenIter, Position, error::Error> {
let tok = &i[0];
pub fn pos<'a>(i: SliceIter<'a, Token>) -> Result<SliceIter<'a, Token>, Position> {
let mut _i = i.clone();
let tok = _i.next().unwrap();
let line = tok.pos.line;
let column = tok.pos.column;
nom::IResult::Done(
i.clone(),
Position {
line: line,
column: column,
},
)
}
/// TokenIter wraps a slice of Tokens and implements the various necessary
/// nom traits to use it as an input to nom parsers.
#[derive(Clone, Debug, PartialEq)]
pub struct TokenIter<'a> {
pub source: &'a [Token],
}
impl<'a> TokenIter<'a> {
pub fn len(&self) -> usize {
self.source.len()
}
}
impl<'a> nom::InputLength for TokenIter<'a> {
fn input_len(&self) -> usize {
self.source.input_len()
}
}
macro_rules! impl_token_iter_slice {
($r:ty) => {
impl<'a> nom::Slice<$r> for TokenIter<'a> {
fn slice(&self, range: $r) -> Self {
TokenIter {
source: self.source.slice(range),
}
}
}
};
}
impl_token_iter_slice!(std::ops::Range<usize>);
impl_token_iter_slice!(std::ops::RangeTo<usize>);
impl_token_iter_slice!(std::ops::RangeFrom<usize>);
impl_token_iter_slice!(std::ops::RangeFull);
impl<'a> std::ops::Index<usize> for TokenIter<'a> {
type Output = Token;
fn index(&self, i: usize) -> &Self::Output {
&self.source[i]
}
}
impl<'a> InputIter for TokenIter<'a> {
type Item = &'a Token;
type RawItem = Token;
type Iter = std::iter::Enumerate<std::slice::Iter<'a, Self::RawItem>>;
type IterElem = std::slice::Iter<'a, Self::RawItem>;
fn iter_indices(&self) -> Self::Iter {
self.source.iter().enumerate()
}
fn iter_elements(&self) -> Self::IterElem {
self.source.iter()
}
fn position<P>(&self, predicate: P) -> Option<usize>
where
P: Fn(Self::RawItem) -> bool,
{
for (o, v) in self.iter_indices() {
if predicate(v.clone()) {
return Some(o);
}
}
None
}
fn slice_index(&self, count: usize) -> Option<usize> {
let mut cnt = 0;
for (index, _) in self.iter_indices() {
if cnt == count {
return Some(index);
}
cnt += 1;
}
if cnt == count {
return Some(self.len());
}
None
}
Result::Complete(i.clone(), Position::new(line, column, i.get_offset()))
}
#[cfg(test)]

261
src/tokenizer/test.rs
View File

@ -1,12 +1,17 @@
use super::*;
use nom;
use nom_locate::LocatedSpan;
use abortable_parser::{Result, SliceIter};
use iter::OffsetStrIter;
#[test]
fn test_empty_token() {
let result = emptytok(LocatedSpan::new("NULL "));
assert!(result.is_done(), format!("result {:?} is not done", result));
if let nom::IResult::Done(_, tok) = result {
let result = emptytok(OffsetStrIter::new("NULL "));
assert!(
result.is_complete(),
format!("result {:?} is not done", result)
);
if let Result::Complete(_, tok) = result {
assert_eq!(tok.fragment, "NULL");
assert_eq!(tok.typ, TokenType::EMPTY);
}
@ -14,9 +19,12 @@ fn test_empty_token() {
#[test]
fn test_assert_token() {
let result = asserttok(LocatedSpan::new("assert "));
assert!(result.is_done(), format!("result {:?} is not done", result));
if let nom::IResult::Done(_, tok) = result {
let result = asserttok(OffsetStrIter::new("assert "));
assert!(
result.is_complete(),
format!("result {:?} is not done", result)
);
if let Result::Complete(_, tok) = result {
assert_eq!(tok.fragment, "assert");
assert_eq!(tok.typ, TokenType::BAREWORD);
}
@ -24,29 +32,56 @@ fn test_assert_token() {
#[test]
fn test_out_token() {
let result = outtok(LocatedSpan::new("out "));
assert!(result.is_done(), format!("result {:?} is not done", result));
if let nom::IResult::Done(_, tok) = result {
let result = outtok(OffsetStrIter::new("out "));
assert!(
result.is_complete(),
format!("result {:?} is not done", result)
);
if let Result::Complete(_, tok) = result {
assert_eq!(tok.fragment, "out");
assert_eq!(tok.typ, TokenType::BAREWORD);
}
}
#[test]
fn test_out_token_with_comment() {
let result = outtok(OffsetStrIter::new("out//comment"));
assert!(
result.is_complete(),
format!("result {:?} is not done", result)
);
if let Result::Complete(_, tok) = result {
assert_eq!(tok.fragment, "out");
assert_eq!(tok.typ, TokenType::BAREWORD);
}
}
#[test]
fn test_not_out_token() {
let result = outtok(OffsetStrIter::new("output"));
assert!(result.is_fail(), format!("result {:?} is not fail", result));
}
#[test]
fn test_escape_quoted() {
let result = escapequoted(LocatedSpan::new("foo \\\"bar\""));
assert!(result.is_done(), format!("result {:?} is not ok", result));
if let nom::IResult::Done(rest, frag) = result {
let result = escapequoted(OffsetStrIter::new("foo \\\"bar\""));
assert!(
result.is_complete(),
format!("result {:?} is not ok", result)
);
if let Result::Complete(_rest, frag) = result {
assert_eq!(frag, "foo \"bar");
assert_eq!(rest.fragment, "\"");
}
}
#[test]
fn test_pipe_quoted() {
let result = pipequotetok(LocatedSpan::new("|foo|"));
assert!(result.is_done(), format!("result {:?} is not ok", result));
if let nom::IResult::Done(_, tok) = result {
let result = pipequotetok(OffsetStrIter::new("|foo|"));
assert!(
result.is_complete(),
format!("result {:?} is not ok", result)
);
if let Result::Complete(_, tok) = result {
assert_eq!(tok.fragment, "foo".to_string());
assert_eq!(tok.typ, TokenType::PIPEQUOTE);
}
@ -54,16 +89,20 @@ fn test_pipe_quoted() {
#[test]
fn test_string_with_escaping() {
let result = strtok(LocatedSpan::new("\"foo \\\\ \\\"bar\""));
assert!(result.is_done(), format!("result {:?} is not ok", result));
if let nom::IResult::Done(_, tok) = result {
let result = strtok(OffsetStrIter::new("\"foo \\\\ \\\"bar\""));
assert!(
result.is_complete(),
format!("result {:?} is not ok", result)
);
if let Result::Complete(_, tok) = result {
assert_eq!(tok.fragment, "foo \\ \"bar".to_string());
}
}
#[test]
fn test_tokenize_bareword_with_dash() {
let result = tokenize(LocatedSpan::new("foo-bar "));
let input = OffsetStrIter::new("foo-bar ");
let result = tokenize(input.clone());
assert!(result.is_ok(), format!("result {:?} is not ok", result));
if let Ok(toks) = result {
assert_eq!(toks.len(), 2);
@ -73,18 +112,23 @@ fn test_tokenize_bareword_with_dash() {
macro_rules! assert_token {
($input:expr, $typ:expr, $msg:expr) => {
let result = token(LocatedSpan::new($input));
let result = token(OffsetStrIter::new($input));
assert!(
result.is_done(),
result.is_complete(),
format!("result {:?} is not a {}", result, $msg)
);
if let nom::IResult::Done(_, tok) = result {
assert_eq!(tok.fragment, $input);
if let Result::Complete(_, tok) = result {
assert_eq!(tok.typ, $typ);
assert_eq!(tok.fragment, $input);
}
};
}
#[test]
fn test_digittok() {
assert_token!("1", TokenType::DIGIT, "1");
}
#[test]
fn test_boolean() {
assert_token!("true", TokenType::BOOLEAN, "boolean");
@ -122,10 +166,11 @@ fn test_lteqtok() {
#[test]
fn test_tokenize_one_of_each() {
let result = tokenize(LocatedSpan::new(
let input = OffsetStrIter::new(
"map out filter assert let import macro select as => [ ] { } ; = % / * \
+ - . ( ) , 1 . foo \"bar\" // comment\n ; true false == < > <= >= !=",
));
);
let result = tokenize(input.clone());
assert!(result.is_ok(), format!("result {:?} is not ok", result));
let v = result.unwrap();
for (i, t) in v.iter().enumerate() {
@ -137,65 +182,80 @@ fn test_tokenize_one_of_each() {
#[test]
fn test_parse_has_end() {
let result = tokenize(LocatedSpan::new("foo"));
let input = OffsetStrIter::new("foo");
let result = tokenize(input.clone());
assert!(result.is_ok());
let v = result.unwrap();
assert_eq!(v.len(), 2);
assert_eq!(v[1].typ, TokenType::END);
}
#[test]
fn test_whitespace() {
assert!(whitespace(OffsetStrIter::new(" ")).is_complete());
let result = whitespace(OffsetStrIter::new(" "));
match result {
Result::Complete(rest, o) => {
assert_eq!(rest.get_offset(), 2);
assert_eq!(o.typ, TokenType::WS);
}
_ => assert!(false, "Not complete"),
}
}
#[test]
fn test_parse_comment() {
assert!(comment(LocatedSpan::new("// comment\n")).is_done());
assert!(comment(LocatedSpan::new("// comment")).is_done());
assert_eq!(
comment(LocatedSpan::new("// comment\n")),
nom::IResult::Done(
LocatedSpan {
fragment: "",
offset: 11,
line: 2,
},
assert!(comment(OffsetStrIter::new("// comment\n")).is_complete());
assert!(comment(OffsetStrIter::new("// comment")).is_complete());
let mut parsed = comment(OffsetStrIter::new("// comment\n"));
assert!(parsed.is_complete());
if let Result::Complete(_rest, cmt) = parsed {
assert_eq!(
cmt,
Token {
typ: TokenType::COMMENT,
fragment: " comment".to_string(),
pos: Position { line: 1, column: 1 },
pos: Position {
line: 1,
column: 1,
offset: 0
},
}
)
);
assert!(comment(LocatedSpan::new("// comment\r\n")).is_done());
assert_eq!(
comment(LocatedSpan::new("// comment\r\n")),
nom::IResult::Done(
LocatedSpan {
fragment: "",
offset: 12,
line: 2,
},
);
}
assert!(comment(OffsetStrIter::new("// comment\r\n")).is_complete());
parsed = comment(OffsetStrIter::new("// comment\r\n"));
if let Result::Complete(_rest, cmt) = parsed {
assert_eq!(
cmt,
Token {
typ: TokenType::COMMENT,
fragment: " comment".to_string(),
pos: Position { column: 1, line: 1 },
pos: Position {
column: 1,
line: 1,
offset: 0
},
}
)
);
assert!(comment(LocatedSpan::new("// comment\r\n ")).is_done());
assert_eq!(
comment(LocatedSpan::new("// comment\r\n ")),
nom::IResult::Done(
LocatedSpan {
fragment: " ",
offset: 12,
line: 2,
},
);
}
assert!(comment(OffsetStrIter::new("// comment\r\n ")).is_complete());
parsed = comment(OffsetStrIter::new("// comment\r\n "));
if let Result::Complete(_rest, cmt) = parsed {
assert_eq!(
cmt,
Token {
typ: TokenType::COMMENT,
fragment: " comment".to_string(),
pos: Position { column: 1, line: 1 },
pos: Position {
column: 1,
line: 1,
offset: 0
},
}
)
);
assert!(comment(LocatedSpan::new("// comment")).is_done());
);
}
assert!(comment(OffsetStrIter::new("// comment")).is_complete());
}
#[test]
@ -203,16 +263,15 @@ fn test_match_word() {
let input = vec![Token {
fragment: "foo".to_string(),
typ: TokenType::BAREWORD,
pos: Position { line: 1, column: 1 },
}];
let result = word!(
TokenIter {
source: input.as_slice(),
pos: Position {
line: 1,
column: 1,
offset: 0,
},
"foo"
);
}];
let result = word!(SliceIter::new(input.as_slice()), "foo");
match result {
nom::IResult::Done(_, tok) => assert_eq!(tok, input[0]),
Result::Complete(_, tok) => assert_eq!(tok, input[0]),
res => assert!(false, format!("Fail: {:?}", res)),
}
}
@ -222,22 +281,20 @@ fn test_match_word_empty_input() {
let input = vec![Token {
fragment: "".to_string(),
typ: TokenType::END,
pos: Position { line: 1, column: 1 },
}];
let result = word!(
TokenIter {
source: input.as_slice(),
pos: Position {
line: 1,
column: 1,
offset: 0,
},
"foo"
);
}];
let result = word!(SliceIter::new(input.as_slice()), "foo");
match result {
nom::IResult::Done(_, _) => assert!(false, "Should have been an error but was Done"),
nom::IResult::Incomplete(_) => {
assert!(false, "Should have been an error but was Incomplete")
}
nom::IResult::Error(_) => {
Result::Complete(_, _) => assert!(false, "Should have been an error but was Done"),
Result::Incomplete(_) => assert!(false, "Should have been a Fail but was Incomplete"),
Result::Fail(_) => {
// noop
}
Result::Abort(_) => assert!(false, "Should have been a Fail but was Abort"),
}
}
@ -246,16 +303,15 @@ fn test_match_punct() {
let input = vec![Token {
fragment: "!".to_string(),
typ: TokenType::PUNCT,
pos: Position { line: 1, column: 1 },
}];
let result = punct!(
TokenIter {
source: input.as_slice(),
pos: Position {
line: 1,
column: 1,
offset: 0,
},
"!"
);
}];
let result = punct!(SliceIter::new(input.as_slice()), "!");
match result {
nom::IResult::Done(_, tok) => assert_eq!(tok, input[0]),
Result::Complete(_, tok) => assert_eq!(tok, input[0]),
res => assert!(false, format!("Fail: {:?}", res)),
}
}
@ -265,16 +321,15 @@ fn test_match_type() {
let input = vec![Token {
fragment: "foo".to_string(),
typ: TokenType::BAREWORD,
pos: Position { line: 1, column: 1 },
}];
let result = match_type!(
TokenIter {
source: input.as_slice(),
pos: Position {
line: 1,
column: 1,
offset: 0,
},
BAREWORD
);
}];
let result = match_type!(SliceIter::new(input.as_slice()), BAREWORD);
match result {
nom::IResult::Done(_, tok) => assert_eq!(tok, input[0]),
Result::Complete(_, tok) => assert_eq!(tok, input[0]),
res => assert!(false, format!("Fail: {:?}", res)),
}
}