zaphar/ucg
1
0
Fork 0
mirror of https://github.com/zaphar/ucg.git synced 2025-07-22 18:19:54 -04:00
Code Issues Packages Projects Releases Wiki Activity

CLEANUP: The great test module refactor.

Browse Source 
Move tests into a separate file for more manageable file
organization.
This commit is contained in:
Jeremy Wall 2018-05-22 18:02:44 -05:00
parent 8164792927
commit fdd8a35086
17 changed files with 3919 additions and 3974 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Cargo.toml
View File

@ -1,6 +1,6 @@
[package]
name = "ucg"
version = "0.1.0"
version = "0.1.1"
authors = ["Jeremy Wall <jeremy@marzhillstudios.com>"]
description = "A configuration generation grammar."
repository = "https://github.com/zaphar/ucg"

728
src/ast/mod.rs
View File

@ -14,5 +14,729 @@
//! The definitions of the ucg AST and Tokens.
#[macro_use]
pub mod tree;
use std;
use std::borrow::Borrow;
use std::cmp::Eq;
use std::cmp::Ordering;
use std::cmp::PartialEq;
use std::cmp::PartialOrd;
use std::collections::HashSet;
use std::convert::Into;
use std::hash::Hash;
use std::hash::Hasher;
macro_rules! enum_type_equality {
( $slf:ident, $r:expr, $( $l:pat ),* ) => {
match $slf {
$(
$l => {
if let $l = $r {
true
} else {
false
}
}
)*
}
}
}
/// Represents a line and a column position in UCG code.
///
/// It is used for generating error messages mostly. Most all
/// parts of the UCG AST have a positioned associated with them.
#[derive(Debug, PartialEq, Eq, Clone, PartialOrd, Ord, Hash)]
pub struct Position {
pub line: usize,
pub column: usize,
}
impl Position {
/// Construct a new Position.
pub fn new(line: usize, column: usize) -> Self {
Position {
line: line,
column: column,
}
}
}
/// Defines the types of tokens in UCG syntax.
#[derive(Debug, PartialEq, Eq, Clone, PartialOrd, Ord, Hash)]
pub enum TokenType {
EMPTY,
BOOLEAN,
END,
WS,
COMMENT,
QUOTED,
DIGIT,
BAREWORD,
PUNCT,
}
/// Defines a Token representing a building block of UCG syntax.
///
/// Token's are passed to the parser stage to be parsed into an AST.
#[derive(Debug, PartialEq, Eq, Clone, PartialOrd, Ord, Hash)]
pub struct Token {
pub typ: TokenType,
pub fragment: String,
pub pos: Position,
}
impl Token {
/// Constructs a new Token with a type and line and column information.
pub fn new<S: Into<String>>(f: S, typ: TokenType, line: usize, col: usize) -> Self {
Self::new_with_pos(f, typ, Position::new(line, col))
}
// Constructs a new Token with a type and a Position.
pub fn new_with_pos<S: Into<String>>(f: S, typ: TokenType, pos: Position) -> Self {
Token {
typ: typ,
fragment: f.into(),
pos: pos,
}
}
}
impl Borrow<str> for Token {
fn borrow(&self) -> &str {
&self.fragment
}
}
/// Helper macro for making a Positioned Value.
macro_rules! value_node {
($v:expr, $p:expr) => {
Positioned::new_with_pos($v, $p)
};
($v:expr, $l:expr, $c:expr) => {
Positioned::new($v, $l, $c)
};
}
/// Helper macro for making a Token.
#[allow(unused_macros)]
macro_rules! make_tok {
(EOF => $l:expr, $c:expr) => {
Token::new("", TokenType::END, $l, $c)
};
(WS => $l:expr, $c:expr) => {
Token::new("", TokenType::WS, $l, $c)
};
(CMT => $e:expr, $l:expr, $c:expr) => {
Token::new($e, TokenType::COMMENT, $l, $c)
};
(QUOT => $e:expr, $l:expr, $c:expr) => {
Token::new($e, TokenType::QUOTED, $l, $c)
};
(PUNCT => $e:expr, $l:expr, $c:expr) => {
Token::new($e, TokenType::PUNCT, $l, $c)
};
(DIGIT => $e:expr, $l:expr, $c:expr) => {
Token::new($e, TokenType::DIGIT, $l, $c)
};
($e:expr, $l:expr, $c:expr) => {
Token::new($e, TokenType::BAREWORD, $l, $c)
};
}
/// Helper macro for making expressions.
#[allow(unused_macros)]
macro_rules! make_expr {
($e:expr) => {
make_expr!($e, 1, 1)
};
($e:expr, $l:expr, $c:expr) => {
Expression::Simple(Value::Symbol(Positioned::new($e.to_string(), $l, $c)))
};
($e:expr => int, $l:expr, $c:expr) => {
Expression::Simple(Value::Int(Positioned::new($e, $l, $c)))
};
}
/// Helper macro for making selectors.
///
/// ```
/// make_selector!(Token::new("tpl", 1, 1), Token::new("fld", 1, 4));
///
/// make_selector!(Token::new("tpl", 1, 1), vec![Token::new("fld", 1, 4)], => 1, 1);
///
/// make_selector!(foo", ["bar"]);
///
/// make_selector!(foo", ["bar"] => 1, 0);
/// ```
#[allow(unused_macros)]
macro_rules! make_selector {
( $h:expr ) => {
make_selector!($h, 1, 0)
};
( $h:expr, $l:expr, $c:expr ) => {
SelectorDef::new(
SelectorList{head: Box::new($h), tail: None},
$l, $c)
};
( $h: expr, $list:expr, $l:expr, $c:expr) => {
SelectorDef::new(
SelectorList{head: Box::new($h), tail: Some($list)},
$l, $c)
};
// Tokens
( $h:expr => [ $( $item:expr ),* ] ) => {
{
make_selector!($h => [ $( $item, )* ] => 1, 1)
}
};
( $h:expr => [ $( $item:expr ),* ] => $l:expr, $c:expr ) => {
{
let mut list: Vec<Token> = Vec::new();
$(
list.push($item);
)*
make_selector!($h, list, $l, $c)
}
};
// Strings not tokens
( $h:expr => $( $item:expr ),* ) => {
{
let mut col = 1;
let mut list: Vec<Token> = Vec::new();
$(
list.push(make_tok!($item, 1, col));
col += $item.len() + 1;
)*
// Shut up the lint about unused code;
assert!(col != 0);
make_selector!($h, list, 1, 1)
}
};
( $h:expr => $( $item:expr ),* => $l:expr, $c:expr ) => {
{
let mut col = $c;
let mut list: Vec<Token> = Vec::new();
$(
list.push(make_tok!($item, $l, col));
col += $item.len() + 1;
)*
// Shut up the linter about unused code;
assert!(col != 0);
make_selector!($h, list, $l, $c)
}
};
}
/// An Expression with a series of symbols specifying the key
/// with which to descend into the result of the expression.
///
/// The expression must evaluate to either a tuple or an array. The token must
/// evaluate to either a bareword Symbol or an Int.
///
/// ```ucg
/// let foo = { bar = "a thing" };
/// let thing = foo.bar;
///
/// let arr = ["one", "two"];
/// let first = arr.0;
///
/// let berry = {best = "strawberry", unique = "acai"}.best;
/// let third = ["uno", "dos", "tres"].1;
/// '''
#[derive(Debug, PartialEq, Clone)]
pub struct SelectorList {
pub head: Box<Expression>,
pub tail: Option<Vec<Token>>,
}
impl SelectorList {
/// Returns a stringified version of a SelectorList.
pub fn to_string(&self) -> String {
"TODO".to_string()
}
}
/// An ordered list of Name = Value pairs.
///
/// This is usually used as the body of a tuple in the UCG AST.
pub type FieldList = Vec<(Token, Expression)>; // Token is expected to be a symbol
/// Encodes a selector expression in the UCG AST.
#[derive(Debug, PartialEq, Clone)]
pub struct SelectorDef {
pub pos: Position,
pub sel: SelectorList,
}
impl SelectorDef {
/// Constructs a new SelectorDef.
pub fn new(sel: SelectorList, line: usize, col: usize) -> Self {
SelectorDef {
pos: Position::new(line, col),
sel: sel,
}
}
}
/// Represents a Value in the UCG parsed AST.
#[derive(Debug, PartialEq, Clone)]
pub enum Value {
// Constant Values
Empty(Position),
Boolean(Positioned<bool>),
Int(Positioned<i64>),
Float(Positioned<f64>),
String(Positioned<String>),
Symbol(Positioned<String>),
// Complex Values
Tuple(Positioned<FieldList>),
List(ListDef),
Selector(SelectorDef),
}
impl Value {
/// Returns the type name of the Value it is called on as a string.
pub fn type_name(&self) -> String {
match self {
&Value::Empty(_) => "EmptyValue".to_string(),
&Value::Boolean(_) => "Boolean".to_string(),
&Value::Int(_) => "Integer".to_string(),
&Value::Float(_) => "Float".to_string(),
&Value::String(_) => "String".to_string(),
&Value::Symbol(_) => "Symbol".to_string(),
&Value::Tuple(_) => "Tuple".to_string(),
&Value::List(_) => "List".to_string(),
&Value::Selector(_) => "Selector".to_string(),
}
}
fn fields_to_string(v: &FieldList) -> String {
let mut buf = String::new();
buf.push_str("{\n");
for ref t in v.iter() {
buf.push_str("\t");
buf.push_str(&t.0.fragment);
buf.push_str("\n");
}
buf.push_str("}");
return buf;
}
fn elems_to_string(v: &Vec<Expression>) -> String {
return format!("{}", v.len());
}
/// Returns a stringified version of the Value.
pub fn to_string(&self) -> String {
match self {
&Value::Empty(_) => "EmptyValue".to_string(),
&Value::Boolean(ref b) => format!("{}", b.val),
&Value::Int(ref i) => format!("{}", i.val),
&Value::Float(ref f) => format!("{}", f.val),
&Value::String(ref s) => format!("{}", s.val),
&Value::Symbol(ref s) => format!("{}", s.val),
&Value::Tuple(ref fs) => format!("{}", Self::fields_to_string(&fs.val)),
&Value::List(ref def) => format!("[{}]", Self::elems_to_string(&def.elems)),
&Value::Selector(ref v) => v.sel.to_string(),
}
}
/// Returns the position for a Value.
pub fn pos(&self) -> &Position {
match self {
&Value::Empty(ref pos) => pos,
&Value::Boolean(ref b) => &b.pos,
&Value::Int(ref i) => &i.pos,
&Value::Float(ref f) => &f.pos,
&Value::String(ref s) => &s.pos,
&Value::Symbol(ref s) => &s.pos,
&Value::Tuple(ref fs) => &fs.pos,
&Value::List(ref def) => &def.pos,
&Value::Selector(ref v) => &v.pos,
}
}
/// Returns true if called on a Value that is the same type as itself.
pub fn type_equal(&self, target: &Self) -> bool {
enum_type_equality!(
self,
target,
&Value::Empty(_),
&Value::Boolean(_),
&Value::Int(_),
&Value::Float(_),
&Value::String(_),
&Value::Symbol(_),
&Value::Tuple(_),
&Value::List(_),
&Value::Selector(_)
)
}
}
/// Represents an expansion of a Macro that is expected to already have been
/// defined.
#[derive(PartialEq, Debug, Clone)]
pub struct CallDef {
pub macroref: SelectorDef,
pub arglist: Vec<Expression>,
pub pos: Position,
}
/// Encodes a select expression in the UCG AST.
#[derive(PartialEq, Debug, Clone)]
pub struct SelectDef {
pub val: Box<Expression>,
pub default: Box<Expression>,
pub tuple: FieldList,
pub pos: Position,
}
// TODO(jwall): This should have a way of rendering with position information.
/// Adds position information to any type `T`.
#[derive(Debug, Clone)]
pub struct Positioned<T> {
pub pos: Position,
pub val: T,
}
impl<T: std::fmt::Display> std::fmt::Display for Positioned<T> {
fn fmt(&self, f: &mut std::fmt::Formatter) -> Result<(), std::fmt::Error> {
write!(f, "{}", self.val)
}
}
impl<T> Positioned<T> {
/// Constructs a new Positioned<T> with a value, line, and column information.
pub fn new(v: T, l: usize, c: usize) -> Self {
Self::new_with_pos(v, Position::new(l, c))
}
/// Constructs a new Positioned<T> with a value and a Position.
pub fn new_with_pos(v: T, pos: Position) -> Self {
Positioned { pos: pos, val: v }
}
}
impl<T: PartialEq> PartialEq for Positioned<T> {
fn eq(&self, other: &Self) -> bool {
self.val == other.val
}
}
impl<T: Eq> Eq for Positioned<T> {}
impl<T: Ord> Ord for Positioned<T> {
fn cmp(&self, other: &Self) -> Ordering {
self.val.cmp(&other.val)
}
}
impl<T: PartialOrd> PartialOrd for Positioned<T> {
fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
self.val.partial_cmp(&other.val)
}
}
impl<T: Hash> Hash for Positioned<T> {
fn hash<H: Hasher>(&self, state: &mut H) {
self.val.hash(state);
}
}
impl<'a> From<&'a Token> for Positioned<String> {
fn from(t: &'a Token) -> Positioned<String> {
Positioned {
pos: t.pos.clone(),
val: t.fragment.to_string(),
}
}
}
impl<'a> From<&'a Positioned<String>> for Positioned<String> {
fn from(t: &Positioned<String>) -> Positioned<String> {
Positioned {
pos: t.pos.clone(),
val: t.val.clone(),
}
}
}
/// Encodes a macro expression in the UCG AST..
///
/// A macro is a pure function over a tuple.
/// MacroDefs are not closures. They can not reference
/// any values except what is defined in their arguments.
#[derive(PartialEq, Debug, Clone)]
pub struct MacroDef {
pub argdefs: Vec<Positioned<String>>,
pub fields: FieldList,
pub pos: Position,
}
impl MacroDef {
fn symbol_is_in_args(&self, sym: &String) -> bool {
for arg in self.argdefs.iter() {
if &arg.val == sym {
return true;
}
}
return false;
}
fn validate_value_symbols<'a>(
&self,
stack: &mut Vec<&'a Expression>,
val: &'a Value,
) -> HashSet<String> {
let mut bad_symbols = HashSet::new();
if let &Value::Symbol(ref name) = val {
if !self.symbol_is_in_args(&name.val) {
bad_symbols.insert(name.val.clone());
}
} else if let &Value::Selector(ref sel_node) = val {
stack.push(&sel_node.sel.head);
} else if let &Value::Tuple(ref tuple_node) = val {
let fields = &tuple_node.val;
for &(_, ref expr) in fields.iter() {
stack.push(expr);
}
} else if let &Value::List(ref def) = val {
for elem in def.elems.iter() {
stack.push(elem);
}
}
return bad_symbols;
}
/// Performs typechecking of a ucg macro's arguments to ensure
/// that they are valid for the expressions in the macro.
pub fn validate_symbols(&self) -> Result<(), HashSet<String>> {
let mut bad_symbols = HashSet::new();
for &(_, ref expr) in self.fields.iter() {
let mut stack = Vec::new();
stack.push(expr);
while stack.len() > 0 {
match stack.pop().unwrap() {
&Expression::Binary(ref bexpr) => {
stack.push(&bexpr.left);
stack.push(&bexpr.right);
}
&Expression::Compare(ref cexpr) => {
stack.push(&cexpr.left);
stack.push(&cexpr.right);
}
&Expression::Grouped(ref expr) => {
stack.push(expr);
}
&Expression::Format(ref def) => {
let exprs = &def.args;
for arg_expr in exprs.iter() {
stack.push(arg_expr);
}
}
&Expression::Select(ref def) => {
stack.push(def.default.borrow());
stack.push(def.val.borrow());
for &(_, ref expr) in def.tuple.iter() {
stack.push(expr);
}
}
&Expression::Copy(ref def) => {
let fields = &def.fields;
for &(_, ref expr) in fields.iter() {
stack.push(expr);
}
}
&Expression::Call(ref def) => for expr in def.arglist.iter() {
stack.push(expr);
},
&Expression::Simple(ref val) => {
let mut syms_set = self.validate_value_symbols(&mut stack, val);
bad_symbols.extend(syms_set.drain());
}
&Expression::Macro(_) => {
// noop
continue;
}
&Expression::ListOp(_) => {
// noop
continue;
}
}
}
}
if bad_symbols.len() > 0 {
return Err(bad_symbols);
}
return Ok(());
}
}
/// Specifies the types of binary operations supported in
/// UCG expression.
#[derive(Debug, PartialEq, Clone)]
pub enum BinaryExprType {
Add,
Sub,
Mul,
Div,
}
#[derive(Debug, PartialEq, Clone)]
pub enum CompareType {
Equal,
GT,
LT,
NotEqual,
GTEqual,
LTEqual,
}
#[derive(Debug, PartialEq, Clone)]
pub struct ComparisonDef {
pub kind: CompareType,
pub left: Box<Expression>,
pub right: Box<Expression>,
pub pos: Position,
}
/// Represents an expression with a left and a right side.
#[derive(Debug, PartialEq, Clone)]
pub struct BinaryOpDef {
pub kind: BinaryExprType,
pub left: Box<Expression>,
pub right: Box<Expression>,
pub pos: Position,
}
/// Encodes a tuple Copy expression in the UCG AST.
#[derive(Debug, PartialEq, Clone)]
pub struct CopyDef {
pub selector: SelectorDef,
pub fields: FieldList,
pub pos: Position,
}
/// Encodes a format expression in the UCG AST.
#[derive(Debug, PartialEq, Clone)]
pub struct FormatDef {
pub template: String,
pub args: Vec<Expression>,
pub pos: Position,
}
/// Encodes a list expression in the UCG AST.
#[derive(Debug, PartialEq, Clone)]
pub struct ListDef {
pub elems: Vec<Expression>,
pub pos: Position,
}
#[derive(Debug, PartialEq, Clone)]
pub enum ListOpType {
Map,
Filter,
}
#[derive(Debug, PartialEq, Clone)]
pub struct ListOpDef {
pub typ: ListOpType,
pub mac: SelectorDef,
pub field: String,
pub target: ListDef,
pub pos: Position,
}
/// Encodes a ucg expression. Expressions compute a value from.
#[derive(Debug, PartialEq, Clone)]
pub enum Expression {
// Base Expression
Simple(Value),
// Binary expressions
Binary(BinaryOpDef),
Compare(ComparisonDef),
// Complex Expressions
Copy(CopyDef),
Grouped(Box<Expression>),
Format(FormatDef),
Call(CallDef),
Macro(MacroDef),
Select(SelectDef),
ListOp(ListOpDef),
}
impl Expression {
/// Returns the position of the Expression.
pub fn pos(&self) -> &Position {
match self {
&Expression::Simple(ref v) => v.pos(),
&Expression::Binary(ref def) => &def.pos,
&Expression::Compare(ref def) => &def.pos,
&Expression::Copy(ref def) => &def.pos,
&Expression::Grouped(ref expr) => expr.pos(),
&Expression::Format(ref def) => &def.pos,
&Expression::Call(ref def) => &def.pos,
&Expression::Macro(ref def) => &def.pos,
&Expression::Select(ref def) => &def.pos,
&Expression::ListOp(ref def) => &def.pos,
}
}
}
/// Encodes a let statement in the UCG AST.
#[derive(Debug, PartialEq)]
pub struct LetDef {
pub name: Token,
pub value: Expression,
}
/// Encodes an import statement in the UCG AST.
#[derive(Debug, PartialEq)]
pub struct ImportDef {
pub path: Token,
pub name: Token,
}
/// Encodes a parsed statement in the UCG AST.
#[derive(Debug, PartialEq)]
pub enum Statement {
// simple expression
Expression(Expression),
// Named bindings
Let(LetDef),
// Import a file.
Import(ImportDef),
}
#[cfg(test)]
pub mod test;

99
src/ast/test.rs Normal file
View File

@ -0,0 +1,99 @@
use super::*;
#[test]
pub fn test_macro_validation_happy_path() {
let def = MacroDef {
argdefs: vec![value_node!("foo".to_string(), 1, 0)],
fields: vec![
(
make_tok!("f1", 1, 1),
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Add,
left: Box::new(Expression::Simple(Value::Symbol(value_node!(
"foo".to_string(),
1,
1
)))),
right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
pos: Position::new(1, 0),
}),
),
],
pos: Position::new(1, 0),
};
assert!(def.validate_symbols().unwrap() == ());
}
#[test]
pub fn test_macro_validation_fail() {
let def = MacroDef {
argdefs: vec![value_node!("foo".to_string(), 1, 0)],
fields: vec![
(
make_tok!("f1", 1, 1),
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Add,
left: Box::new(Expression::Simple(Value::Symbol(value_node!(
"bar".to_string(),
1,
1
)))),
right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
pos: Position::new(1, 0),
}),
),
],
pos: Position::new(1, 0),
};
let mut expected = HashSet::new();
expected.insert("bar".to_string());
assert_eq!(def.validate_symbols().err().unwrap(), expected);
}
#[test]
pub fn test_macro_validation_selector_happy_path() {
let def = MacroDef {
argdefs: vec![value_node!("foo".to_string(), 1, 0)],
fields: vec![
(
make_tok!("f1", 1, 1),
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Add,
left: Box::new(Expression::Simple(Value::Selector(
make_selector!(make_expr!("foo", 1, 1) => [
make_tok!("quux", 1, 1) ] => 1, 1),
))),
right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
pos: Position::new(1, 0),
}),
),
],
pos: Position::new(1, 0),
};
assert!(def.validate_symbols().unwrap() == ());
}
#[test]
pub fn test_macro_validation_selector_fail() {
let def = MacroDef {
argdefs: vec![value_node!("foo".to_string(), 1, 0)],
fields: vec![
(
make_tok!("f1", 1, 1),
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Add,
left: Box::new(Expression::Simple(Value::Selector(
make_selector!(make_expr!("bar", 1, 1) => [
make_tok!("quux", 1, 1) ] => 1, 1),
))),
right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
pos: Position::new(1, 0),
}),
),
],
pos: Position::new(1, 0),
};
let mut expected = HashSet::new();
expected.insert("bar".to_string());
assert_eq!(def.validate_symbols(), Err(expected));
}

839
src/ast/tree.rs
View File

@ -1,839 +0,0 @@
// Copyright 2017 Jeremy Wall <jeremy@marzhillstudios.com>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std;
use std::borrow::Borrow;
use std::cmp::Eq;
use std::cmp::Ordering;
use std::cmp::PartialEq;
use std::cmp::PartialOrd;
use std::collections::HashSet;
use std::convert::Into;
use std::hash::Hash;
use std::hash::Hasher;
macro_rules! enum_type_equality {
( $slf:ident, $r:expr, $( $l:pat ),* ) => {
match $slf {
$(
$l => {
if let $l = $r {
true
} else {
false
}
}
)*
}
}
}
/// Represents a line and a column position in UCG code.
///
/// It is used for generating error messages mostly. Most all
/// parts of the UCG AST have a positioned associated with them.
#[derive(Debug, PartialEq, Eq, Clone, PartialOrd, Ord, Hash)]
pub struct Position {
pub line: usize,
pub column: usize,
}
impl Position {
/// Construct a new Position.
pub fn new(line: usize, column: usize) -> Self {
Position {
line: line,
column: column,
}
}
}
/// Defines the types of tokens in UCG syntax.
#[derive(Debug, PartialEq, Eq, Clone, PartialOrd, Ord, Hash)]
pub enum TokenType {
EMPTY,
BOOLEAN,
END,
WS,
COMMENT,
QUOTED,
DIGIT,
BAREWORD,
PUNCT,
}
/// Defines a Token representing a building block of UCG syntax.
///
/// Token's are passed to the parser stage to be parsed into an AST.
#[derive(Debug, PartialEq, Eq, Clone, PartialOrd, Ord, Hash)]
pub struct Token {
pub typ: TokenType,
pub fragment: String,
pub pos: Position,
}
impl Token {
/// Constructs a new Token with a type and line and column information.
pub fn new<S: Into<String>>(f: S, typ: TokenType, line: usize, col: usize) -> Self {
Self::new_with_pos(f, typ, Position::new(line, col))
}
// Constructs a new Token with a type and a Position.
pub fn new_with_pos<S: Into<String>>(f: S, typ: TokenType, pos: Position) -> Self {
Token {
typ: typ,
fragment: f.into(),
pos: pos,
}
}
}
impl Borrow<str> for Token {
fn borrow(&self) -> &str {
&self.fragment
}
}
/// Helper macro for making a Positioned Value.
macro_rules! value_node {
($v:expr, $p:expr) => {
Positioned::new_with_pos($v, $p)
};
($v:expr, $l:expr, $c:expr) => {
Positioned::new($v, $l, $c)
};
}
/// Helper macro for making a Token.
#[allow(unused_macros)]
macro_rules! make_tok {
(EOF => $l:expr, $c:expr) => {
Token::new("", TokenType::END, $l, $c)
};
(WS => $l:expr, $c:expr) => {
Token::new("", TokenType::WS, $l, $c)
};
(CMT => $e:expr, $l:expr, $c:expr) => {
Token::new($e, TokenType::COMMENT, $l, $c)
};
(QUOT => $e:expr, $l:expr, $c:expr) => {
Token::new($e, TokenType::QUOTED, $l, $c)
};
(PUNCT => $e:expr, $l:expr, $c:expr) => {
Token::new($e, TokenType::PUNCT, $l, $c)
};
(DIGIT => $e:expr, $l:expr, $c:expr) => {
Token::new($e, TokenType::DIGIT, $l, $c)
};
($e:expr, $l:expr, $c:expr) => {
Token::new($e, TokenType::BAREWORD, $l, $c)
};
}
/// Helper macro for making expressions.
#[allow(unused_macros)]
macro_rules! make_expr {
($e:expr) => {
make_expr!($e, 1, 1)
};
($e:expr, $l:expr, $c:expr) => {
Expression::Simple(Value::Symbol(Positioned::new($e.to_string(), $l, $c)))
};
($e:expr => int, $l:expr, $c:expr) => {
Expression::Simple(Value::Int(Positioned::new($e, $l, $c)))
};
}
/// Helper macro for making selectors.
///
/// ```
/// make_selector!(Token::new("tpl", 1, 1), Token::new("fld", 1, 4));
///
/// make_selector!(Token::new("tpl", 1, 1), vec![Token::new("fld", 1, 4)], => 1, 1);
///
/// make_selector!(foo", ["bar"]);
///
/// make_selector!(foo", ["bar"] => 1, 0);
/// ```
#[allow(unused_macros)]
macro_rules! make_selector {
( $h:expr ) => {
make_selector!($h, 1, 0)
};
( $h:expr, $l:expr, $c:expr ) => {
SelectorDef::new(
SelectorList{head: Box::new($h), tail: None},
$l, $c)
};
( $h: expr, $list:expr, $l:expr, $c:expr) => {
SelectorDef::new(
SelectorList{head: Box::new($h), tail: Some($list)},
$l, $c)
};
// Tokens
( $h:expr => [ $( $item:expr ),* ] ) => {
{
make_selector!($h => [ $( $item, )* ] => 1, 1)
}
};
( $h:expr => [ $( $item:expr ),* ] => $l:expr, $c:expr ) => {
{
let mut list: Vec<Token> = Vec::new();
$(
list.push($item);
)*
make_selector!($h, list, $l, $c)
}
};
// Strings not tokens
( $h:expr => $( $item:expr ),* ) => {
{
let mut col = 1;
let mut list: Vec<Token> = Vec::new();
$(
list.push(make_tok!($item, 1, col));
col += $item.len() + 1;
)*
// Shut up the lint about unused code;
assert!(col != 0);
make_selector!($h, list, 1, 1)
}
};
( $h:expr => $( $item:expr ),* => $l:expr, $c:expr ) => {
{
let mut col = $c;
let mut list: Vec<Token> = Vec::new();
$(
list.push(make_tok!($item, $l, col));
col += $item.len() + 1;
)*
// Shut up the linter about unused code;
assert!(col != 0);
make_selector!($h, list, $l, $c)
}
};
}
/// An Expression with a series of symbols specifying the key
/// with which to descend into the result of the expression.
///
/// The expression must evaluate to either a tuple or an array. The token must
/// evaluate to either a bareword Symbol or an Int.
///
/// ```ucg
/// let foo = { bar = "a thing" };
/// let thing = foo.bar;
///
/// let arr = ["one", "two"];
/// let first = arr.0;
///
/// let berry = {best = "strawberry", unique = "acai"}.best;
/// let third = ["uno", "dos", "tres"].1;
/// '''
#[derive(Debug, PartialEq, Clone)]
pub struct SelectorList {
pub head: Box<Expression>,
pub tail: Option<Vec<Token>>,
}
impl SelectorList {
/// Returns a stringified version of a SelectorList.
pub fn to_string(&self) -> String {
"TODO".to_string()
}
}
/// An ordered list of Name = Value pairs.
///
/// This is usually used as the body of a tuple in the UCG AST.
pub type FieldList = Vec<(Token, Expression)>; // Token is expected to be a symbol
/// Encodes a selector expression in the UCG AST.
#[derive(Debug, PartialEq, Clone)]
pub struct SelectorDef {
pub pos: Position,
pub sel: SelectorList,
}
impl SelectorDef {
/// Constructs a new SelectorDef.
pub fn new(sel: SelectorList, line: usize, col: usize) -> Self {
SelectorDef {
pos: Position::new(line, col),
sel: sel,
}
}
}
/// Represents a Value in the UCG parsed AST.
#[derive(Debug, PartialEq, Clone)]
pub enum Value {
// Constant Values
Empty(Position),
Boolean(Positioned<bool>),
Int(Positioned<i64>),
Float(Positioned<f64>),
String(Positioned<String>),
Symbol(Positioned<String>),
// Complex Values
Tuple(Positioned<FieldList>),
List(ListDef),
Selector(SelectorDef),
}
impl Value {
/// Returns the type name of the Value it is called on as a string.
pub fn type_name(&self) -> String {
match self {
&Value::Empty(_) => "EmptyValue".to_string(),
&Value::Boolean(_) => "Boolean".to_string(),
&Value::Int(_) => "Integer".to_string(),
&Value::Float(_) => "Float".to_string(),
&Value::String(_) => "String".to_string(),
&Value::Symbol(_) => "Symbol".to_string(),
&Value::Tuple(_) => "Tuple".to_string(),
&Value::List(_) => "List".to_string(),
&Value::Selector(_) => "Selector".to_string(),
}
}
fn fields_to_string(v: &FieldList) -> String {
let mut buf = String::new();
buf.push_str("{\n");
for ref t in v.iter() {
buf.push_str("\t");
buf.push_str(&t.0.fragment);
buf.push_str("\n");
}
buf.push_str("}");
return buf;
}
fn elems_to_string(v: &Vec<Expression>) -> String {
return format!("{}", v.len());
}
/// Returns a stringified version of the Value.
pub fn to_string(&self) -> String {
match self {
&Value::Empty(_) => "EmptyValue".to_string(),
&Value::Boolean(ref b) => format!("{}", b.val),
&Value::Int(ref i) => format!("{}", i.val),
&Value::Float(ref f) => format!("{}", f.val),
&Value::String(ref s) => format!("{}", s.val),
&Value::Symbol(ref s) => format!("{}", s.val),
&Value::Tuple(ref fs) => format!("{}", Self::fields_to_string(&fs.val)),
&Value::List(ref def) => format!("[{}]", Self::elems_to_string(&def.elems)),
&Value::Selector(ref v) => v.sel.to_string(),
}
}
/// Returns the position for a Value.
pub fn pos(&self) -> &Position {
match self {
&Value::Empty(ref pos) => pos,
&Value::Boolean(ref b) => &b.pos,
&Value::Int(ref i) => &i.pos,
&Value::Float(ref f) => &f.pos,
&Value::String(ref s) => &s.pos,
&Value::Symbol(ref s) => &s.pos,
&Value::Tuple(ref fs) => &fs.pos,
&Value::List(ref def) => &def.pos,
&Value::Selector(ref v) => &v.pos,
}
}
/// Returns true if called on a Value that is the same type as itself.
pub fn type_equal(&self, target: &Self) -> bool {
enum_type_equality!(
self,
target,
&Value::Empty(_),
&Value::Boolean(_),
&Value::Int(_),
&Value::Float(_),
&Value::String(_),
&Value::Symbol(_),
&Value::Tuple(_),
&Value::List(_),
&Value::Selector(_)
)
}
}
/// Represents an expansion of a Macro that is expected to already have been
/// defined.
#[derive(PartialEq, Debug, Clone)]
pub struct CallDef {
pub macroref: SelectorDef,
pub arglist: Vec<Expression>,
pub pos: Position,
}
/// Encodes a select expression in the UCG AST.
#[derive(PartialEq, Debug, Clone)]
pub struct SelectDef {
pub val: Box<Expression>,
pub default: Box<Expression>,
pub tuple: FieldList,
pub pos: Position,
}
// TODO(jwall): This should have a way of rendering with position information.
/// Adds position information to any type `T`.
#[derive(Debug, Clone)]
pub struct Positioned<T> {
pub pos: Position,
pub val: T,
}
impl<T: std::fmt::Display> std::fmt::Display for Positioned<T> {
fn fmt(&self, f: &mut std::fmt::Formatter) -> Result<(), std::fmt::Error> {
write!(f, "{}", self.val)
}
}
impl<T> Positioned<T> {
/// Constructs a new Positioned<T> with a value, line, and column information.
pub fn new(v: T, l: usize, c: usize) -> Self {
Self::new_with_pos(v, Position::new(l, c))
}
/// Constructs a new Positioned<T> with a value and a Position.
pub fn new_with_pos(v: T, pos: Position) -> Self {
Positioned { pos: pos, val: v }
}
}
impl<T: PartialEq> PartialEq for Positioned<T> {
fn eq(&self, other: &Self) -> bool {
self.val == other.val
}
}
impl<T: Eq> Eq for Positioned<T> {}
impl<T: Ord> Ord for Positioned<T> {
fn cmp(&self, other: &Self) -> Ordering {
self.val.cmp(&other.val)
}
}
impl<T: PartialOrd> PartialOrd for Positioned<T> {
fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
self.val.partial_cmp(&other.val)
}
}
impl<T: Hash> Hash for Positioned<T> {
fn hash<H: Hasher>(&self, state: &mut H) {
self.val.hash(state);
}
}
impl<'a> From<&'a Token> for Positioned<String> {
fn from(t: &'a Token) -> Positioned<String> {
Positioned {
pos: t.pos.clone(),
val: t.fragment.to_string(),
}
}
}
impl<'a> From<&'a Positioned<String>> for Positioned<String> {
fn from(t: &Positioned<String>) -> Positioned<String> {
Positioned {
pos: t.pos.clone(),
val: t.val.clone(),
}
}
}
/// Encodes a macro expression in the UCG AST..
///
/// A macro is a pure function over a tuple.
/// MacroDefs are not closures. They can not reference
/// any values except what is defined in their arguments.
#[derive(PartialEq, Debug, Clone)]
pub struct MacroDef {
pub argdefs: Vec<Positioned<String>>,
pub fields: FieldList,
pub pos: Position,
}
impl MacroDef {
fn symbol_is_in_args(&self, sym: &String) -> bool {
for arg in self.argdefs.iter() {
if &arg.val == sym {
return true;
}
}
return false;
}
fn validate_value_symbols<'a>(
&self,
stack: &mut Vec<&'a Expression>,
val: &'a Value,
) -> HashSet<String> {
let mut bad_symbols = HashSet::new();
if let &Value::Symbol(ref name) = val {
if !self.symbol_is_in_args(&name.val) {
bad_symbols.insert(name.val.clone());
}
} else if let &Value::Selector(ref sel_node) = val {
stack.push(&sel_node.sel.head);
} else if let &Value::Tuple(ref tuple_node) = val {
let fields = &tuple_node.val;
for &(_, ref expr) in fields.iter() {
stack.push(expr);
}
} else if let &Value::List(ref def) = val {
for elem in def.elems.iter() {
stack.push(elem);
}
}
return bad_symbols;
}
/// Performs typechecking of a ucg macro's arguments to ensure
/// that they are valid for the expressions in the macro.
pub fn validate_symbols(&self) -> Result<(), HashSet<String>> {
let mut bad_symbols = HashSet::new();
for &(_, ref expr) in self.fields.iter() {
let mut stack = Vec::new();
stack.push(expr);
while stack.len() > 0 {
match stack.pop().unwrap() {
&Expression::Binary(ref bexpr) => {
stack.push(&bexpr.left);
stack.push(&bexpr.right);
}
&Expression::Compare(ref cexpr) => {
stack.push(&cexpr.left);
stack.push(&cexpr.right);
}
&Expression::Grouped(ref expr) => {
stack.push(expr);
}
&Expression::Format(ref def) => {
let exprs = &def.args;
for arg_expr in exprs.iter() {
stack.push(arg_expr);
}
}
&Expression::Select(ref def) => {
stack.push(def.default.borrow());
stack.push(def.val.borrow());
for &(_, ref expr) in def.tuple.iter() {
stack.push(expr);
}
}
&Expression::Copy(ref def) => {
let fields = &def.fields;
for &(_, ref expr) in fields.iter() {
stack.push(expr);
}
}
&Expression::Call(ref def) => for expr in def.arglist.iter() {
stack.push(expr);
},
&Expression::Simple(ref val) => {
let mut syms_set = self.validate_value_symbols(&mut stack, val);
bad_symbols.extend(syms_set.drain());
}
&Expression::Macro(_) => {
// noop
continue;
}
&Expression::ListOp(_) => {
// noop
continue;
}
}
}
}
if bad_symbols.len() > 0 {
return Err(bad_symbols);
}
return Ok(());
}
}
/// Specifies the types of binary operations supported in
/// UCG expression.
#[derive(Debug, PartialEq, Clone)]
pub enum BinaryExprType {
Add,
Sub,
Mul,
Div,
}
#[derive(Debug, PartialEq, Clone)]
pub enum CompareType {
Equal,
GT,
LT,
NotEqual,
GTEqual,
LTEqual,
}
#[derive(Debug, PartialEq, Clone)]
pub struct ComparisonDef {
pub kind: CompareType,
pub left: Box<Expression>,
pub right: Box<Expression>,
pub pos: Position,
}
/// Represents an expression with a left and a right side.
#[derive(Debug, PartialEq, Clone)]
pub struct BinaryOpDef {
pub kind: BinaryExprType,
pub left: Box<Expression>,
pub right: Box<Expression>,
pub pos: Position,
}
/// Encodes a tuple Copy expression in the UCG AST.
#[derive(Debug, PartialEq, Clone)]
pub struct CopyDef {
pub selector: SelectorDef,
pub fields: FieldList,
pub pos: Position,
}
/// Encodes a format expression in the UCG AST.
#[derive(Debug, PartialEq, Clone)]
pub struct FormatDef {
pub template: String,
pub args: Vec<Expression>,
pub pos: Position,
}
/// Encodes a list expression in the UCG AST.
#[derive(Debug, PartialEq, Clone)]
pub struct ListDef {
pub elems: Vec<Expression>,
pub pos: Position,
}
#[derive(Debug, PartialEq, Clone)]
pub enum ListOpType {
Map,
Filter,
}
#[derive(Debug, PartialEq, Clone)]
pub struct ListOpDef {
pub typ: ListOpType,
pub mac: SelectorDef,
pub field: String,
pub target: ListDef,
pub pos: Position,
}
/// Encodes a ucg expression. Expressions compute a value from.
#[derive(Debug, PartialEq, Clone)]
pub enum Expression {
// Base Expression
Simple(Value),
// Binary expressions
Binary(BinaryOpDef),
Compare(ComparisonDef),
// Complex Expressions
Copy(CopyDef),
Grouped(Box<Expression>),
Format(FormatDef),
Call(CallDef),
Macro(MacroDef),
Select(SelectDef),
ListOp(ListOpDef),
}
impl Expression {
/// Returns the position of the Expression.
pub fn pos(&self) -> &Position {
match self {
&Expression::Simple(ref v) => v.pos(),
&Expression::Binary(ref def) => &def.pos,
&Expression::Compare(ref def) => &def.pos,
&Expression::Copy(ref def) => &def.pos,
&Expression::Grouped(ref expr) => expr.pos(),
&Expression::Format(ref def) => &def.pos,
&Expression::Call(ref def) => &def.pos,
&Expression::Macro(ref def) => &def.pos,
&Expression::Select(ref def) => &def.pos,
&Expression::ListOp(ref def) => &def.pos,
}
}
}
/// Encodes a let statement in the UCG AST.
#[derive(Debug, PartialEq)]
pub struct LetDef {
pub name: Token,
pub value: Expression,
}
/// Encodes an import statement in the UCG AST.
#[derive(Debug, PartialEq)]
pub struct ImportDef {
pub path: Token,
pub name: Token,
}
/// Encodes a parsed statement in the UCG AST.
#[derive(Debug, PartialEq)]
pub enum Statement {
// simple expression
Expression(Expression),
// Named bindings
Let(LetDef),
// Import a file.
Import(ImportDef),
}
#[cfg(test)]
mod ast_test {
use super::*;
#[test]
pub fn test_macro_validation_happy_path() {
let def = MacroDef {
argdefs: vec![value_node!("foo".to_string(), 1, 0)],
fields: vec![
(
make_tok!("f1", 1, 1),
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Add,
left: Box::new(Expression::Simple(Value::Symbol(value_node!(
"foo".to_string(),
1,
1
)))),
right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
pos: Position::new(1, 0),
}),
),
],
pos: Position::new(1, 0),
};
assert!(def.validate_symbols().unwrap() == ());
}
#[test]
pub fn test_macro_validation_fail() {
let def = MacroDef {
argdefs: vec![value_node!("foo".to_string(), 1, 0)],
fields: vec![
(
make_tok!("f1", 1, 1),
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Add,
left: Box::new(Expression::Simple(Value::Symbol(value_node!(
"bar".to_string(),
1,
1
)))),
right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
pos: Position::new(1, 0),
}),
),
],
pos: Position::new(1, 0),
};
let mut expected = HashSet::new();
expected.insert("bar".to_string());
assert_eq!(def.validate_symbols().err().unwrap(), expected);
}
#[test]
pub fn test_macro_validation_selector_happy_path() {
let def = MacroDef {
argdefs: vec![value_node!("foo".to_string(), 1, 0)],
fields: vec![
(
make_tok!("f1", 1, 1),
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Add,
left: Box::new(Expression::Simple(Value::Selector(
make_selector!(make_expr!("foo", 1, 1) => [
make_tok!("quux", 1, 1) ] => 1, 1),
))),
right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
pos: Position::new(1, 0),
}),
),
],
pos: Position::new(1, 0),
};
assert!(def.validate_symbols().unwrap() == ());
}
#[test]
pub fn test_macro_validation_selector_fail() {
let def = MacroDef {
argdefs: vec![value_node!("foo".to_string(), 1, 0)],
fields: vec![
(
make_tok!("f1", 1, 1),
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Add,
left: Box::new(Expression::Simple(Value::Selector(
make_selector!(make_expr!("bar", 1, 1) => [
make_tok!("quux", 1, 1) ] => 1, 1),
))),
right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
pos: Position::new(1, 0),
}),
),
],
pos: Position::new(1, 0),
};
let mut expected = HashSet::new();
expected.insert("bar".to_string());
assert_eq!(def.validate_symbols(), Err(expected));
}
}

86
src/build/compile_test.rs Normal file
View File

@ -0,0 +1,86 @@
use super::{Builder, Val};
fn assert_build<S: Into<String>>(input: S, assert: &str) {
let mut b = Builder::new();
b.build_file_string(input.into()).unwrap();
let result = b.eval_string(assert).unwrap();
if let &Val::Boolean(ok) = result.as_ref() {
assert!(ok, format!("'{}' is not true", assert));
} else {
assert!(
false,
format!("'{}' does not evaluate to a boolean: {:?}", assert, result)
);
}
}
#[test]
fn test_comparisons() {
let input = "
let one = 1;
let two = 2;
let foo = \"foo\";
let bar = \"bar\";
let tpl1 = {
foo = \"bar\",
one = 1
};
let tpl2 = tpl1{};
let tpl3 = {
bar = \"foo\",
two = 1
};
let list = [1, 2, 3];
let list2 = list;
let list3 = [1, 2];
";
assert_build(input, "one == one;");
assert_build(input, "one >= one;");
assert_build(input, "two > one;");
assert_build(input, "two >= two;");
assert_build(input, "tpl1 == tpl2;");
assert_build(input, "tpl1 != tpl3;");
assert_build(input, "list == list2;");
assert_build(input, "list != list3;");
}
#[test]
fn test_deep_comparison() {
let input = "
let tpl1 = {
foo = \"bar\",
lst = [1, 2, 3],
inner = {
fld = \"value\"
}
};
let copy = tpl1;
let extra = tpl1{one = 1};
let less = {
foo = \"bar\"
};
";
assert_build(input, "tpl1.inner == copy.inner;");
assert_build(input, "tpl1.inner.fld == copy.inner.fld;");
assert_build(input, "tpl1.lst == copy.lst;");
assert_build(input, "tpl1.foo == copy.foo;");
assert_build(input, "tpl1 == copy;");
assert_build(input, "tpl1 != extra;");
assert_build(input, "tpl1 != less;");
}
#[test]
fn test_expression_comparisons() {
assert_build("", "2 == 1+1;");
assert_build("", "(1+1) == 2;");
assert_build("", "(1+1) == (1+1);");
assert_build("", "(\"foo\" + \"bar\") == \"foobar\";");
}
#[test]
fn test_binary_operator_precedence() {
//assert_build("let result = 2 * 2 + 1;", "result == 6;");
assert_build("let result = 2 + 2 * 1;", "result == 4;");
assert_build("let result = (2 * 2) + 1;", "result == 5;");
}

898
src/build.rs → src/build/mod.rs
View File

@ -25,7 +25,7 @@ use std::io::Read;
use std::ops::Deref;
use std::rc::Rc;
use ast::tree::*;
use ast::*;
use error;
use format;
use parse::parse;
@ -1110,899 +1110,7 @@ impl Builder {
}
#[cfg(test)]
mod compile_test {
use super::{Builder, Val};
fn assert_build<S: Into<String>>(input: S, assert: &str) {
let mut b = Builder::new();
b.build_file_string(input.into()).unwrap();
let result = b.eval_string(assert).unwrap();
if let &Val::Boolean(ok) = result.as_ref() {
assert!(ok, format!("'{}' is not true", assert));
} else {
assert!(
false,
format!("'{}' does not evaluate to a boolean: {:?}", assert, result)
);
}
}
#[test]
fn test_comparisons() {
let input = "
let one = 1;
let two = 2;
let foo = \"foo\";
let bar = \"bar\";
let tpl1 = {
foo = \"bar\",
one = 1
};
let tpl2 = tpl1{};
let tpl3 = {
bar = \"foo\",
two = 1
};
let list = [1, 2, 3];
let list2 = list;
let list3 = [1, 2];
";
assert_build(input, "one == one;");
assert_build(input, "one >= one;");
assert_build(input, "two > one;");
assert_build(input, "two >= two;");
assert_build(input, "tpl1 == tpl2;");
assert_build(input, "tpl1 != tpl3;");
assert_build(input, "list == list2;");
assert_build(input, "list != list3;");
}
#[test]
fn test_deep_comparison() {
let input = "
let tpl1 = {
foo = \"bar\",
lst = [1, 2, 3],
inner = {
fld = \"value\"
}
};
let copy = tpl1;
let extra = tpl1{one = 1};
let less = {
foo = \"bar\"
};
";
assert_build(input, "tpl1.inner == copy.inner;");
assert_build(input, "tpl1.inner.fld == copy.inner.fld;");
assert_build(input, "tpl1.lst == copy.lst;");
assert_build(input, "tpl1.foo == copy.foo;");
assert_build(input, "tpl1 == copy;");
assert_build(input, "tpl1 != extra;");
assert_build(input, "tpl1 != less;");
}
#[test]
fn test_expression_comparisons() {
assert_build("", "2 == 1+1;");
assert_build("", "(1+1) == 2;");
assert_build("", "(1+1) == (1+1);");
assert_build("", "(\"foo\" + \"bar\") == \"foobar\";");
}
#[test]
fn test_binary_operator_precedence() {
//assert_build("let result = 2 * 2 + 1;", "result == 6;");
assert_build("let result = 2 + 2 * 1;", "result == 4;");
assert_build("let result = (2 * 2) + 1;", "result == 5;");
}
}
mod compile_test;
#[cfg(test)]
mod test {
use super::{Builder, CallDef, MacroDef, SelectDef, Val};
use ast::tree::*;
use std::rc::Rc;
fn test_expr_to_val(mut cases: Vec<(Expression, Val)>, b: Builder) {
for tpl in cases.drain(0..) {
assert_eq!(b.eval_expr(&tpl.0).unwrap(), Rc::new(tpl.1));
}
}
#[test]
fn test_eval_div_expr() {
let b = Builder::new();
test_expr_to_val(
vec![
(
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Div,
left: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))),
right: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))),
pos: Position::new(1, 0),
}),
Val::Int(1),
),
(
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Div,
left: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))),
right: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))),
pos: Position::new(1, 0),
}),
Val::Float(1.0),
),
],
b,
);
}
#[test]
#[should_panic(expected = "Expected Float")]
fn test_eval_div_expr_fail() {
let b = Builder::new();
test_expr_to_val(
vec![
(
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Div,
left: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))),
right: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))),
pos: Position::new(1, 0),
}),
Val::Float(1.0),
),
],
b,
);
}
#[test]
fn test_eval_mul_expr() {
let b = Builder::new();
test_expr_to_val(
vec![
(
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Mul,
left: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))),
right: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))),
pos: Position::new(1, 0),
}),
Val::Int(4),
),
(
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Mul,
left: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))),
right: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))),
pos: Position::new(1, 0),
}),
Val::Float(4.0),
),
],
b,
);
}
#[test]
#[should_panic(expected = "Expected Float")]
fn test_eval_mul_expr_fail() {
let b = Builder::new();
test_expr_to_val(
vec![
(
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Mul,
left: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))),
right: Box::new(Expression::Simple(Value::Int(value_node!(20, 1, 1)))),
pos: Position::new(1, 0),
}),
Val::Float(1.0),
),
],
b,
);
}
#[test]
fn test_eval_subtract_expr() {
let b = Builder::new();
test_expr_to_val(
vec![
(
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Sub,
left: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))),
right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
pos: Position::new(1, 0),
}),
Val::Int(1),
),
(
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Sub,
left: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))),
right: Box::new(Expression::Simple(Value::Float(value_node!(1.0, 1, 1)))),
pos: Position::new(1, 0),
}),
Val::Float(1.0),
),
],
b,
);
}
#[test]
#[should_panic(expected = "Expected Float")]
fn test_eval_subtract_expr_fail() {
let b = Builder::new();
test_expr_to_val(
vec![
(
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Sub,
left: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))),
right: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))),
pos: Position::new(1, 0),
}),
Val::Float(1.0),
),
],
b,
);
}
#[test]
fn test_eval_add_expr() {
let b = Builder::new();
test_expr_to_val(
vec![
(
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Add,
left: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
pos: Position::new(1, 0),
}),
Val::Int(2),
),
(
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Add,
left: Box::new(Expression::Simple(Value::Float(value_node!(1.0, 1, 1)))),
right: Box::new(Expression::Simple(Value::Float(value_node!(1.0, 1, 1)))),
pos: Position::new(1, 0),
}),
Val::Float(2.0),
),
(
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Add,
left: Box::new(Expression::Simple(Value::String(value_node!(
"foo".to_string(),
1,
1
)))),
right: Box::new(Expression::Simple(Value::String(value_node!(
"bar".to_string(),
1,
1
)))),
pos: Position::new(1, 0),
}),
Val::String("foobar".to_string()),
),
(
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Add,
left: Box::new(Expression::Simple(Value::List(ListDef {
elems: vec![
Expression::Simple(Value::String(value_node!(
"foo".to_string(),
1,
1
))),
],
pos: Position::new(1, 1),
}))),
right: Box::new(Expression::Simple(Value::List(ListDef {
elems: vec![
Expression::Simple(Value::String(value_node!(
"bar".to_string(),
1,
1
))),
],
pos: Position::new(1, 1),
}))),
pos: Position::new(1, 0),
}),
Val::List(vec![
Rc::new(Val::String("foo".to_string())),
Rc::new(Val::String("bar".to_string())),
]),
),
],
b,
);
}
#[test]
#[should_panic(expected = "Expected Float")]
fn test_eval_add_expr_fail() {
let b = Builder::new();
test_expr_to_val(
vec![
(
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Add,
left: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))),
right: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))),
pos: Position::new(1, 0),
}),
Val::Float(1.0),
),
],
b,
);
}
#[test]
fn test_eval_simple_expr() {
test_expr_to_val(
vec![
(
Expression::Simple(Value::Int(value_node!(1, 1, 1))),
Val::Int(1),
),
(
Expression::Simple(Value::Float(value_node!(2.0, 1, 1))),
Val::Float(2.0),
),
(
Expression::Simple(Value::String(value_node!("foo".to_string(), 1, 1))),
Val::String("foo".to_string()),
),
(
Expression::Simple(Value::Tuple(value_node!(
vec![
(
make_tok!("bar", 1, 1),
Expression::Simple(Value::Int(value_node!(1, 1, 1))),
),
],
1,
1
))),
Val::Tuple(vec![
(value_node!("bar".to_string(), 1, 1), Rc::new(Val::Int(1))),
]),
),
],
Builder::new(),
);
}
#[test]
fn test_eval_simple_lookup_expr() {
let mut b = Builder::new();
b.out
.entry(value_node!("var1".to_string(), 1, 0))
.or_insert(Rc::new(Val::Int(1)));
test_expr_to_val(
vec![
(
Expression::Simple(Value::Symbol(value_node!("var1".to_string(), 1, 1))),
Val::Int(1),
),
],
b,
);
}
#[test]
fn test_eval_simple_lookup_error() {
let mut b = Builder::new();
b.out
.entry(value_node!("var1".to_string(), 1, 0))
.or_insert(Rc::new(Val::Int(1)));
let expr = Expression::Simple(Value::Symbol(value_node!("var".to_string(), 1, 1)));
assert!(b.eval_expr(&expr).is_err());
}
#[test]
fn test_eval_selector_expr() {
let mut b = Builder::new();
b.out
.entry(value_node!("var1".to_string(), 1, 0))
.or_insert(Rc::new(Val::Tuple(vec![
(
value_node!("lvl1".to_string(), 1, 0),
Rc::new(Val::Tuple(vec![
(value_node!("lvl2".to_string(), 1, 0), Rc::new(Val::Int(3))),
])),
),
])));
b.out
.entry(value_node!("var2".to_string(), 1, 0))
.or_insert(Rc::new(Val::Int(2)));
b.out
.entry(value_node!("var3".to_string(), 1, 0))
.or_insert(Rc::new(Val::Tuple(vec![
(value_node!("lvl1".to_string(), 1, 0), Rc::new(Val::Int(4))),
])));
test_expr_to_val(
vec![
(
Expression::Simple(Value::Selector(make_selector!(make_expr!("var1")))),
Val::Tuple(vec![
(
value_node!("lvl1".to_string(), 1, 0),
Rc::new(Val::Tuple(vec![
(value_node!("lvl2".to_string(), 1, 0), Rc::new(Val::Int(3))),
])),
),
]),
),
(
Expression::Simple(Value::Selector(
make_selector!(make_expr!("var1") => "lvl1"),
)),
Val::Tuple(vec![
(value_node!("lvl2".to_string(), 1, 0), Rc::new(Val::Int(3))),
]),
),
(
Expression::Simple(Value::Selector(
make_selector!(make_expr!("var1") => "lvl1", "lvl2"),
)),
Val::Int(3),
),
(
Expression::Simple(Value::Selector(make_selector!(make_expr!("var2")))),
Val::Int(2),
),
(
Expression::Simple(Value::Selector(
make_selector!(make_expr!("var3") => "lvl1"),
)),
Val::Int(4),
),
],
b,
);
}
#[test]
fn test_eval_selector_list_expr() {
let mut b = Builder::new();
b.out
.entry(value_node!("var1".to_string(), 1, 1))
.or_insert(Rc::new(Val::List(vec![
Rc::new(Val::String("val1".to_string())),
Rc::new(Val::Tuple(vec![
(value_node!("var2".to_string(), 1, 1), Rc::new(Val::Int(1))),
])),
])));
// TODO(jwall): Assert that we can index into lists using dot syntax.
test_expr_to_val(
vec![
(
Expression::Simple(Value::Selector(
make_selector!(make_expr!("var1") => "0" => 1, 1),
)),
Val::String("val1".to_string()),
),
],
b,
);
}
#[test]
#[should_panic(expected = "Unable to find tpl1")]
fn test_expr_copy_no_such_tuple() {
let b = Builder::new();
test_expr_to_val(
vec![
(
Expression::Copy(CopyDef {
selector: make_selector!(make_expr!("tpl1")),
fields: Vec::new(),
pos: Position::new(1, 0),
}),
Val::Tuple(Vec::new()),
),
],
b,
);
}
#[test]
#[should_panic(expected = "Expected Tuple got Int(1)")]
fn test_expr_copy_not_a_tuple() {
let mut b = Builder::new();
b.out
.entry(value_node!("tpl1".to_string(), 1, 0))
.or_insert(Rc::new(Val::Int(1)));
test_expr_to_val(
vec![
(
Expression::Copy(CopyDef {
selector: make_selector!(make_expr!("tpl1")),
fields: Vec::new(),
pos: Position::new(1, 0),
}),
Val::Tuple(Vec::new()),
),
],
b,
);
}
#[test]
#[should_panic(expected = "Expected type Integer for field fld1 but got String")]
fn test_expr_copy_field_type_error() {
let mut b = Builder::new();
b.out
.entry(value_node!("tpl1".to_string(), 1, 0))
.or_insert(Rc::new(Val::Tuple(vec![
(value_node!("fld1".to_string(), 1, 0), Rc::new(Val::Int(1))),
])));
test_expr_to_val(
vec![
(
Expression::Copy(CopyDef {
selector: make_selector!(make_expr!("tpl1")),
fields: vec![
(
make_tok!("fld1", 1, 1),
Expression::Simple(Value::String(value_node!(
"2".to_string(),
1,
1
))),
),
],
pos: Position::new(1, 0),
}),
Val::Tuple(vec![
(
value_node!("fld1".to_string(), 1, 1),
Rc::new(Val::String("2".to_string())),
),
]),
),
],
b,
);
}
#[test]
fn test_expr_copy() {
let mut b = Builder::new();
b.out
.entry(value_node!("tpl1".to_string(), 1, 0))
.or_insert(Rc::new(Val::Tuple(vec![
(value_node!("fld1".to_string(), 1, 0), Rc::new(Val::Int(1))),
])));
test_expr_to_val(
vec![
(
Expression::Copy(CopyDef {
selector: make_selector!(make_expr!("tpl1")),
fields: vec![
(
make_tok!("fld2", 1, 1),
Expression::Simple(Value::String(value_node!(
"2".to_string(),
1,
1
))),
),
],
pos: Position::new(1, 0),
}),
// Add a new field to the copy
Val::Tuple(
// NOTE(jwall): The order of these is important in order to ensure
// that the compare assertion is correct. The ordering has no
// semantics though so at some point we should probably be less restrictive.
vec![
(value_node!("fld1".to_string(), 1, 0), Rc::new(Val::Int(1))),
(
value_node!("fld2".to_string(), 1, 1),
Rc::new(Val::String("2".to_string())),
),
],
),
),
// Overwrite a field in the copy
(
Expression::Copy(CopyDef {
selector: make_selector!(make_expr!("tpl1")),
fields: vec![
(
make_tok!("fld1", 1, 1),
Expression::Simple(Value::Int(value_node!(3, 1, 1))),
),
(
make_tok!("fld2", 1, 1),
Expression::Simple(Value::String(value_node!(
"2".to_string(),
1,
1
))),
),
],
pos: Position::new(1, 0),
}),
Val::Tuple(vec![
(value_node!("fld1".to_string(), 1, 0), Rc::new(Val::Int(3))),
(
value_node!("fld2".to_string(), 1, 0),
Rc::new(Val::String("2".to_string())),
),
]),
),
// The source tuple is still unmodified.
(
Expression::Simple(Value::Selector(make_selector!(make_expr!["tpl1"]))),
Val::Tuple(vec![
(value_node!("fld1".to_string(), 1, 0), Rc::new(Val::Int(1))),
]),
),
],
b,
);
}
#[test]
fn test_macro_call() {
let mut b = Builder::new();
b.out
.entry(value_node!("tstmac".to_string(), 1, 0))
.or_insert(Rc::new(Val::Macro(MacroDef {
argdefs: vec![value_node!("arg1".to_string(), 1, 0)],
fields: vec![
(
make_tok!("foo", 1, 1),
Expression::Simple(Value::Symbol(value_node!("arg1".to_string(), 1, 1))),
),
],
pos: Position::new(1, 0),
})));
test_expr_to_val(
vec![
(
Expression::Call(CallDef {
macroref: make_selector!(make_expr!("tstmac")),
arglist: vec![
Expression::Simple(Value::String(value_node!("bar".to_string(), 1, 1))),
],
pos: Position::new(1, 0),
}),
Val::Tuple(vec![
(
value_node!("foo".to_string(), 1, 1),
Rc::new(Val::String("bar".to_string())),
),
]),
),
],
b,
);
}
#[test]
#[should_panic(expected = "Unable to find arg1")]
fn test_macro_hermetic() {
let mut b = Builder::new();
b.out
.entry(value_node!("arg1".to_string(), 1, 0))
.or_insert(Rc::new(Val::String("bar".to_string())));
b.out
.entry(value_node!("tstmac".to_string(), 1, 0))
.or_insert(Rc::new(Val::Macro(MacroDef {
argdefs: vec![value_node!("arg2".to_string(), 1, 0)],
fields: vec![
(
make_tok!("foo", 1, 1),
Expression::Simple(Value::Symbol(value_node!("arg1".to_string(), 1, 1))),
),
],
pos: Position::new(1, 0),
})));
test_expr_to_val(
vec![
(
Expression::Call(CallDef {
macroref: make_selector!(make_expr!("tstmac")),
arglist: vec![
Expression::Simple(Value::String(value_node!("bar".to_string(), 1, 1))),
],
pos: Position::new(1, 1),
}),
Val::Tuple(vec![
(
value_node!("foo".to_string(), 1, 0),
Rc::new(Val::String("bar".to_string())),
),
]),
),
],
b,
);
}
#[test]
fn test_select_expr() {
let mut b = Builder::new();
b.out
.entry(value_node!("foo".to_string(), 1, 0))
.or_insert(Rc::new(Val::String("bar".to_string())));
b.out
.entry(value_node!("baz".to_string(), 1, 0))
.or_insert(Rc::new(Val::String("boo".to_string())));
test_expr_to_val(
vec![
(
Expression::Select(SelectDef {
val: Box::new(Expression::Simple(Value::Symbol(value_node!(
"foo".to_string(),
1,
1
)))),
default: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
tuple: vec![
(
make_tok!("foo", 1, 1),
Expression::Simple(Value::String(value_node!(
"2".to_string(),
1,
1
))),
),
(
make_tok!("bar", 1, 1),
Expression::Simple(Value::Int(value_node!(2, 1, 1))),
),
],
pos: Position::new(1, 0),
}),
Val::Int(2),
),
(
Expression::Select(SelectDef {
val: Box::new(Expression::Simple(Value::Symbol(value_node!(
"baz".to_string(),
1,
1
)))),
default: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
tuple: vec![
(
make_tok!("bar", 1, 1),
Expression::Simple(Value::Int(value_node!(2, 1, 1))),
),
(
make_tok!("quux", 1, 1),
Expression::Simple(Value::String(value_node!(
"2".to_string(),
1,
1
))),
),
],
pos: Position::new(1, 0),
}),
// If the field doesn't exist then we get the default.
Val::Int(1),
),
],
b,
);
}
#[test]
#[should_panic(expected = "Expected String but got Integer in Select expression")]
fn test_select_expr_not_a_string() {
let mut b = Builder::new();
b.out
.entry(value_node!("foo".to_string(), 1, 0))
.or_insert(Rc::new(Val::Int(4)));
test_expr_to_val(
vec![
(
Expression::Select(SelectDef {
val: Box::new(Expression::Simple(Value::Symbol(value_node!(
"foo".to_string(),
1,
1
)))),
default: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
tuple: vec![
(
make_tok!("bar", 1, 1),
Expression::Simple(Value::Int(value_node!(2, 1, 1))),
),
(
make_tok!("quux", 1, 1),
Expression::Simple(Value::String(value_node!(
"2".to_string(),
1,
1
))),
),
],
pos: Position::new(1, 0),
}),
Val::Int(2),
),
],
b,
);
}
#[test]
fn test_let_statement() {
let mut b = Builder::new();
let stmt = Statement::Let(LetDef {
name: make_tok!("foo", 1, 1),
value: Expression::Simple(Value::String(value_node!("bar".to_string(), 1, 1))),
});
b.build_stmt(&stmt).unwrap();
test_expr_to_val(
vec![
(
Expression::Simple(Value::Symbol(value_node!("foo".to_string(), 1, 1))),
Val::String("bar".to_string()),
),
],
b,
);
}
#[test]
fn test_build_file_string() {
let mut b = Builder::new();
b.build_file_string("let foo = 1;".to_string()).unwrap();
let key = value_node!("foo".to_string(), 1, 0);
assert!(b.out.contains_key(&key));
}
#[test]
fn test_asset_symbol_lookups() {
let mut b = Builder::new();
b.assets
.entry(value_node!("foo".to_string(), 1, 0))
.or_insert(Rc::new(Val::Tuple(vec![
(
value_node!("bar".to_string(), 1, 0),
Rc::new(Val::Tuple(vec![
(value_node!("quux".to_string(), 1, 0), Rc::new(Val::Int(1))),
])),
),
])));
test_expr_to_val(
vec![
(
Expression::Simple(Value::Symbol(value_node!("foo".to_string(), 1, 1))),
Val::Tuple(vec![
(
value_node!("bar".to_string(), 1, 0),
Rc::new(Val::Tuple(vec![
(value_node!("quux".to_string(), 1, 0), Rc::new(Val::Int(1))),
])),
),
]),
),
],
b,
);
}
}
mod test;

772
src/build/test.rs Normal file
View File

@ -0,0 +1,772 @@
use super::{Builder, CallDef, MacroDef, SelectDef, Val};
use ast::*;
use std::rc::Rc;
fn test_expr_to_val(mut cases: Vec<(Expression, Val)>, b: Builder) {
for tpl in cases.drain(0..) {
assert_eq!(b.eval_expr(&tpl.0).unwrap(), Rc::new(tpl.1));
}
}
#[test]
fn test_eval_div_expr() {
let b = Builder::new();
test_expr_to_val(
vec![
(
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Div,
left: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))),
right: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))),
pos: Position::new(1, 0),
}),
Val::Int(1),
),
(
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Div,
left: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))),
right: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))),
pos: Position::new(1, 0),
}),
Val::Float(1.0),
),
],
b,
);
}
#[test]
#[should_panic(expected = "Expected Float")]
fn test_eval_div_expr_fail() {
let b = Builder::new();
test_expr_to_val(
vec![
(
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Div,
left: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))),
right: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))),
pos: Position::new(1, 0),
}),
Val::Float(1.0),
),
],
b,
);
}
#[test]
fn test_eval_mul_expr() {
let b = Builder::new();
test_expr_to_val(
vec![
(
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Mul,
left: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))),
right: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))),
pos: Position::new(1, 0),
}),
Val::Int(4),
),
(
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Mul,
left: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))),
right: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))),
pos: Position::new(1, 0),
}),
Val::Float(4.0),
),
],
b,
);
}
#[test]
#[should_panic(expected = "Expected Float")]
fn test_eval_mul_expr_fail() {
let b = Builder::new();
test_expr_to_val(
vec![
(
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Mul,
left: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))),
right: Box::new(Expression::Simple(Value::Int(value_node!(20, 1, 1)))),
pos: Position::new(1, 0),
}),
Val::Float(1.0),
),
],
b,
);
}
#[test]
fn test_eval_subtract_expr() {
let b = Builder::new();
test_expr_to_val(
vec![
(
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Sub,
left: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))),
right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
pos: Position::new(1, 0),
}),
Val::Int(1),
),
(
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Sub,
left: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))),
right: Box::new(Expression::Simple(Value::Float(value_node!(1.0, 1, 1)))),
pos: Position::new(1, 0),
}),
Val::Float(1.0),
),
],
b,
);
}
#[test]
#[should_panic(expected = "Expected Float")]
fn test_eval_subtract_expr_fail() {
let b = Builder::new();
test_expr_to_val(
vec![
(
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Sub,
left: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))),
right: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))),
pos: Position::new(1, 0),
}),
Val::Float(1.0),
),
],
b,
);
}
#[test]
fn test_eval_add_expr() {
let b = Builder::new();
test_expr_to_val(
vec![
(
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Add,
left: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
right: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
pos: Position::new(1, 0),
}),
Val::Int(2),
),
(
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Add,
left: Box::new(Expression::Simple(Value::Float(value_node!(1.0, 1, 1)))),
right: Box::new(Expression::Simple(Value::Float(value_node!(1.0, 1, 1)))),
pos: Position::new(1, 0),
}),
Val::Float(2.0),
),
(
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Add,
left: Box::new(Expression::Simple(Value::String(value_node!(
"foo".to_string(),
1,
1
)))),
right: Box::new(Expression::Simple(Value::String(value_node!(
"bar".to_string(),
1,
1
)))),
pos: Position::new(1, 0),
}),
Val::String("foobar".to_string()),
),
(
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Add,
left: Box::new(Expression::Simple(Value::List(ListDef {
elems: vec![
Expression::Simple(Value::String(value_node!("foo".to_string(), 1, 1))),
],
pos: Position::new(1, 1),
}))),
right: Box::new(Expression::Simple(Value::List(ListDef {
elems: vec![
Expression::Simple(Value::String(value_node!("bar".to_string(), 1, 1))),
],
pos: Position::new(1, 1),
}))),
pos: Position::new(1, 0),
}),
Val::List(vec![
Rc::new(Val::String("foo".to_string())),
Rc::new(Val::String("bar".to_string())),
]),
),
],
b,
);
}
#[test]
#[should_panic(expected = "Expected Float")]
fn test_eval_add_expr_fail() {
let b = Builder::new();
test_expr_to_val(
vec![
(
Expression::Binary(BinaryOpDef {
kind: BinaryExprType::Add,
left: Box::new(Expression::Simple(Value::Float(value_node!(2.0, 1, 1)))),
right: Box::new(Expression::Simple(Value::Int(value_node!(2, 1, 1)))),
pos: Position::new(1, 0),
}),
Val::Float(1.0),
),
],
b,
);
}
#[test]
fn test_eval_simple_expr() {
test_expr_to_val(
vec![
(
Expression::Simple(Value::Int(value_node!(1, 1, 1))),
Val::Int(1),
),
(
Expression::Simple(Value::Float(value_node!(2.0, 1, 1))),
Val::Float(2.0),
),
(
Expression::Simple(Value::String(value_node!("foo".to_string(), 1, 1))),
Val::String("foo".to_string()),
),
(
Expression::Simple(Value::Tuple(value_node!(
vec![
(
make_tok!("bar", 1, 1),
Expression::Simple(Value::Int(value_node!(1, 1, 1))),
),
],
1,
1
))),
Val::Tuple(vec![
(value_node!("bar".to_string(), 1, 1), Rc::new(Val::Int(1))),
]),
),
],
Builder::new(),
);
}
#[test]
fn test_eval_simple_lookup_expr() {
let mut b = Builder::new();
b.out
.entry(value_node!("var1".to_string(), 1, 0))
.or_insert(Rc::new(Val::Int(1)));
test_expr_to_val(
vec![
(
Expression::Simple(Value::Symbol(value_node!("var1".to_string(), 1, 1))),
Val::Int(1),
),
],
b,
);
}
#[test]
fn test_eval_simple_lookup_error() {
let mut b = Builder::new();
b.out
.entry(value_node!("var1".to_string(), 1, 0))
.or_insert(Rc::new(Val::Int(1)));
let expr = Expression::Simple(Value::Symbol(value_node!("var".to_string(), 1, 1)));
assert!(b.eval_expr(&expr).is_err());
}
#[test]
fn test_eval_selector_expr() {
let mut b = Builder::new();
b.out
.entry(value_node!("var1".to_string(), 1, 0))
.or_insert(Rc::new(Val::Tuple(vec![
(
value_node!("lvl1".to_string(), 1, 0),
Rc::new(Val::Tuple(vec![
(value_node!("lvl2".to_string(), 1, 0), Rc::new(Val::Int(3))),
])),
),
])));
b.out
.entry(value_node!("var2".to_string(), 1, 0))
.or_insert(Rc::new(Val::Int(2)));
b.out
.entry(value_node!("var3".to_string(), 1, 0))
.or_insert(Rc::new(Val::Tuple(vec![
(value_node!("lvl1".to_string(), 1, 0), Rc::new(Val::Int(4))),
])));
test_expr_to_val(
vec![
(
Expression::Simple(Value::Selector(make_selector!(make_expr!("var1")))),
Val::Tuple(vec![
(
value_node!("lvl1".to_string(), 1, 0),
Rc::new(Val::Tuple(vec![
(value_node!("lvl2".to_string(), 1, 0), Rc::new(Val::Int(3))),
])),
),
]),
),
(
Expression::Simple(Value::Selector(
make_selector!(make_expr!("var1") => "lvl1"),
)),
Val::Tuple(vec![
(value_node!("lvl2".to_string(), 1, 0), Rc::new(Val::Int(3))),
]),
),
(
Expression::Simple(Value::Selector(
make_selector!(make_expr!("var1") => "lvl1", "lvl2"),
)),
Val::Int(3),
),
(
Expression::Simple(Value::Selector(make_selector!(make_expr!("var2")))),
Val::Int(2),
),
(
Expression::Simple(Value::Selector(
make_selector!(make_expr!("var3") => "lvl1"),
)),
Val::Int(4),
),
],
b,
);
}
#[test]
fn test_eval_selector_list_expr() {
let mut b = Builder::new();
b.out
.entry(value_node!("var1".to_string(), 1, 1))
.or_insert(Rc::new(Val::List(vec![
Rc::new(Val::String("val1".to_string())),
Rc::new(Val::Tuple(vec![
(value_node!("var2".to_string(), 1, 1), Rc::new(Val::Int(1))),
])),
])));
// TODO(jwall): Assert that we can index into lists using dot syntax.
test_expr_to_val(
vec![
(
Expression::Simple(Value::Selector(
make_selector!(make_expr!("var1") => "0" => 1, 1),
)),
Val::String("val1".to_string()),
),
],
b,
);
}
#[test]
#[should_panic(expected = "Unable to find tpl1")]
fn test_expr_copy_no_such_tuple() {
let b = Builder::new();
test_expr_to_val(
vec![
(
Expression::Copy(CopyDef {
selector: make_selector!(make_expr!("tpl1")),
fields: Vec::new(),
pos: Position::new(1, 0),
}),
Val::Tuple(Vec::new()),
),
],
b,
);
}
#[test]
#[should_panic(expected = "Expected Tuple got Int(1)")]
fn test_expr_copy_not_a_tuple() {
let mut b = Builder::new();
b.out
.entry(value_node!("tpl1".to_string(), 1, 0))
.or_insert(Rc::new(Val::Int(1)));
test_expr_to_val(
vec![
(
Expression::Copy(CopyDef {
selector: make_selector!(make_expr!("tpl1")),
fields: Vec::new(),
pos: Position::new(1, 0),
}),
Val::Tuple(Vec::new()),
),
],
b,
);
}
#[test]
#[should_panic(expected = "Expected type Integer for field fld1 but got String")]
fn test_expr_copy_field_type_error() {
let mut b = Builder::new();
b.out
.entry(value_node!("tpl1".to_string(), 1, 0))
.or_insert(Rc::new(Val::Tuple(vec![
(value_node!("fld1".to_string(), 1, 0), Rc::new(Val::Int(1))),
])));
test_expr_to_val(
vec![
(
Expression::Copy(CopyDef {
selector: make_selector!(make_expr!("tpl1")),
fields: vec![
(
make_tok!("fld1", 1, 1),
Expression::Simple(Value::String(value_node!("2".to_string(), 1, 1))),
),
],
pos: Position::new(1, 0),
}),
Val::Tuple(vec![
(
value_node!("fld1".to_string(), 1, 1),
Rc::new(Val::String("2".to_string())),
),
]),
),
],
b,
);
}
#[test]
fn test_expr_copy() {
let mut b = Builder::new();
b.out
.entry(value_node!("tpl1".to_string(), 1, 0))
.or_insert(Rc::new(Val::Tuple(vec![
(value_node!("fld1".to_string(), 1, 0), Rc::new(Val::Int(1))),
])));
test_expr_to_val(
vec![
(
Expression::Copy(CopyDef {
selector: make_selector!(make_expr!("tpl1")),
fields: vec![
(
make_tok!("fld2", 1, 1),
Expression::Simple(Value::String(value_node!("2".to_string(), 1, 1))),
),
],
pos: Position::new(1, 0),
}),
// Add a new field to the copy
Val::Tuple(
// NOTE(jwall): The order of these is important in order to ensure
// that the compare assertion is correct. The ordering has no
// semantics though so at some point we should probably be less restrictive.
vec![
(value_node!("fld1".to_string(), 1, 0), Rc::new(Val::Int(1))),
(
value_node!("fld2".to_string(), 1, 1),
Rc::new(Val::String("2".to_string())),
),
],
),
),
// Overwrite a field in the copy
(
Expression::Copy(CopyDef {
selector: make_selector!(make_expr!("tpl1")),
fields: vec![
(
make_tok!("fld1", 1, 1),
Expression::Simple(Value::Int(value_node!(3, 1, 1))),
),
(
make_tok!("fld2", 1, 1),
Expression::Simple(Value::String(value_node!("2".to_string(), 1, 1))),
),
],
pos: Position::new(1, 0),
}),
Val::Tuple(vec![
(value_node!("fld1".to_string(), 1, 0), Rc::new(Val::Int(3))),
(
value_node!("fld2".to_string(), 1, 0),
Rc::new(Val::String("2".to_string())),
),
]),
),
// The source tuple is still unmodified.
(
Expression::Simple(Value::Selector(make_selector!(make_expr!["tpl1"]))),
Val::Tuple(vec![
(value_node!("fld1".to_string(), 1, 0), Rc::new(Val::Int(1))),
]),
),
],
b,
);
}
#[test]
fn test_macro_call() {
let mut b = Builder::new();
b.out
.entry(value_node!("tstmac".to_string(), 1, 0))
.or_insert(Rc::new(Val::Macro(MacroDef {
argdefs: vec![value_node!("arg1".to_string(), 1, 0)],
fields: vec![
(
make_tok!("foo", 1, 1),
Expression::Simple(Value::Symbol(value_node!("arg1".to_string(), 1, 1))),
),
],
pos: Position::new(1, 0),
})));
test_expr_to_val(
vec![
(
Expression::Call(CallDef {
macroref: make_selector!(make_expr!("tstmac")),
arglist: vec![
Expression::Simple(Value::String(value_node!("bar".to_string(), 1, 1))),
],
pos: Position::new(1, 0),
}),
Val::Tuple(vec![
(
value_node!("foo".to_string(), 1, 1),
Rc::new(Val::String("bar".to_string())),
),
]),
),
],
b,
);
}
#[test]
#[should_panic(expected = "Unable to find arg1")]
fn test_macro_hermetic() {
let mut b = Builder::new();
b.out
.entry(value_node!("arg1".to_string(), 1, 0))
.or_insert(Rc::new(Val::String("bar".to_string())));
b.out
.entry(value_node!("tstmac".to_string(), 1, 0))
.or_insert(Rc::new(Val::Macro(MacroDef {
argdefs: vec![value_node!("arg2".to_string(), 1, 0)],
fields: vec![
(
make_tok!("foo", 1, 1),
Expression::Simple(Value::Symbol(value_node!("arg1".to_string(), 1, 1))),
),
],
pos: Position::new(1, 0),
})));
test_expr_to_val(
vec![
(
Expression::Call(CallDef {
macroref: make_selector!(make_expr!("tstmac")),
arglist: vec![
Expression::Simple(Value::String(value_node!("bar".to_string(), 1, 1))),
],
pos: Position::new(1, 1),
}),
Val::Tuple(vec![
(
value_node!("foo".to_string(), 1, 0),
Rc::new(Val::String("bar".to_string())),
),
]),
),
],
b,
);
}
#[test]
fn test_select_expr() {
let mut b = Builder::new();
b.out
.entry(value_node!("foo".to_string(), 1, 0))
.or_insert(Rc::new(Val::String("bar".to_string())));
b.out
.entry(value_node!("baz".to_string(), 1, 0))
.or_insert(Rc::new(Val::String("boo".to_string())));
test_expr_to_val(
vec![
(
Expression::Select(SelectDef {
val: Box::new(Expression::Simple(Value::Symbol(value_node!(
"foo".to_string(),
1,
1
)))),
default: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
tuple: vec![
(
make_tok!("foo", 1, 1),
Expression::Simple(Value::String(value_node!("2".to_string(), 1, 1))),
),
(
make_tok!("bar", 1, 1),
Expression::Simple(Value::Int(value_node!(2, 1, 1))),
),
],
pos: Position::new(1, 0),
}),
Val::Int(2),
),
(
Expression::Select(SelectDef {
val: Box::new(Expression::Simple(Value::Symbol(value_node!(
"baz".to_string(),
1,
1
)))),
default: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
tuple: vec![
(
make_tok!("bar", 1, 1),
Expression::Simple(Value::Int(value_node!(2, 1, 1))),
),
(
make_tok!("quux", 1, 1),
Expression::Simple(Value::String(value_node!("2".to_string(), 1, 1))),
),
],
pos: Position::new(1, 0),
}),
// If the field doesn't exist then we get the default.
Val::Int(1),
),
],
b,
);
}
#[test]
#[should_panic(expected = "Expected String but got Integer in Select expression")]
fn test_select_expr_not_a_string() {
let mut b = Builder::new();
b.out
.entry(value_node!("foo".to_string(), 1, 0))
.or_insert(Rc::new(Val::Int(4)));
test_expr_to_val(
vec![
(
Expression::Select(SelectDef {
val: Box::new(Expression::Simple(Value::Symbol(value_node!(
"foo".to_string(),
1,
1
)))),
default: Box::new(Expression::Simple(Value::Int(value_node!(1, 1, 1)))),
tuple: vec![
(
make_tok!("bar", 1, 1),
Expression::Simple(Value::Int(value_node!(2, 1, 1))),
),
(
make_tok!("quux", 1, 1),
Expression::Simple(Value::String(value_node!("2".to_string(), 1, 1))),
),
],
pos: Position::new(1, 0),
}),
Val::Int(2),
),
],
b,
);
}
#[test]
fn test_let_statement() {
let mut b = Builder::new();
let stmt = Statement::Let(LetDef {
name: make_tok!("foo", 1, 1),
value: Expression::Simple(Value::String(value_node!("bar".to_string(), 1, 1))),
});
b.build_stmt(&stmt).unwrap();
test_expr_to_val(
vec![
(
Expression::Simple(Value::Symbol(value_node!("foo".to_string(), 1, 1))),
Val::String("bar".to_string()),
),
],
b,
);
}
#[test]
fn test_build_file_string() {
let mut b = Builder::new();
b.build_file_string("let foo = 1;".to_string()).unwrap();
let key = value_node!("foo".to_string(), 1, 0);
assert!(b.out.contains_key(&key));
}
#[test]
fn test_asset_symbol_lookups() {
let mut b = Builder::new();
b.assets
.entry(value_node!("foo".to_string(), 1, 0))
.or_insert(Rc::new(Val::Tuple(vec![
(
value_node!("bar".to_string(), 1, 0),
Rc::new(Val::Tuple(vec![
(value_node!("quux".to_string(), 1, 0), Rc::new(Val::Int(1))),
])),
),
])));
test_expr_to_val(
vec![
(
Expression::Simple(Value::Symbol(value_node!("foo".to_string(), 1, 1))),
Val::Tuple(vec![
(
value_node!("bar".to_string(), 1, 0),
Rc::new(Val::Tuple(vec![
(value_node!("quux".to_string(), 1, 0), Rc::new(Val::Int(1))),
])),
),
]),
),
],
b,
);
}

2
src/convert/env.rs
View File

@ -17,7 +17,7 @@ use std::io::Result;
use std::io::Write;
use std::rc::Rc;
use ast::tree::*;
use ast::*;
use build::Val;
use convert::traits::Converter;

2
src/convert/json.rs
View File

@ -36,7 +36,7 @@ impl JsonConverter {
fn convert_tuple(
&self,
items: &Vec<(ast::tree::Positioned<String>, Rc<Val>)>,
items: &Vec<(ast::Positioned<String>, Rc<Val>)>,
) -> Result<serde_json::Value> {
let mut mp = serde_json::Map::new();
for &(ref k, ref v) in items.iter() {

2
src/error.rs
View File

@ -16,7 +16,7 @@
use std::error;
use std::fmt;
use ast::tree::*;
use ast::*;
use nom;

4
src/format.rs
View File

@ -16,7 +16,7 @@
use std::clone::Clone;
use std::error::Error;
use ast::tree::*;
use ast::*;
use error;
/// Implements the logic for format strings in UCG format expressions.
@ -77,7 +77,7 @@ impl<V: Into<String> + Clone> Formatter<V> {
#[cfg(test)]
mod test {
use super::Formatter;
use ast::tree::Position;
use ast::Position;
#[test]
fn test_format_happy_path() {

6
src/lib.rs
View File

@ -409,9 +409,9 @@ pub mod parse;
mod format;
pub use ast::tree::Expression;
pub use ast::tree::Statement;
pub use ast::tree::Value;
pub use ast::Expression;
pub use ast::Statement;
pub use ast::Value;
pub use build::Builder;
pub use build::Val;

1967
src/parse.rs
View File

File diff suppressed because it is too large Load Diff

864
src/parse/mod.rs Normal file
View File

@ -0,0 +1,864 @@
// Copyright 2017 Jeremy Wall <jeremy@marzhillstudios.com>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! The Parsing stage of the ucg compiler.
use std::borrow::Borrow;
use std::str::FromStr;
use nom;
use nom::IResult;
use nom::InputLength;
use nom_locate::LocatedSpan;
use ast::*;
use error;
use tokenizer::*;
type NomResult<'a, O> = nom::IResult<TokenIter<'a>, O, error::Error>;
type ParseResult<O> = Result<O, error::Error>;
fn symbol_to_value(s: &Token) -> ParseResult<Value> {
Ok(Value::Symbol(value_node!(
s.fragment.to_string(),
s.pos.clone()
)))
}
// symbol is a bare unquoted field.
named!(symbol<TokenIter, Value, error::Error>,
match_type!(BAREWORD => symbol_to_value)
);
fn str_to_value(s: &Token) -> ParseResult<Value> {
Ok(Value::String(value_node!(
s.fragment.to_string(),
s.pos.clone()
)))
}
// quoted_value is a quoted string.
named!(quoted_value<TokenIter, Value, error::Error>,
match_type!(STR => str_to_value)
);
// Helper function to make the return types work for down below.
fn triple_to_number(v: (Option<Token>, Option<Token>, Option<Token>)) -> ParseResult<Value> {
let (pref, mut pref_pos) = match v.0 {
None => ("", Position::new(0, 0)),
Some(ref bs) => (bs.fragment.borrow(), bs.pos.clone()),
};
let has_dot = v.1.is_some();
if v.0.is_some() && !has_dot && v.2.is_none() {
let i = match FromStr::from_str(pref) {
Ok(i) => i,
Err(_) => {
return Err(error::Error::new(
format!("Not an integer! {}", pref),
error::ErrorType::UnexpectedToken,
pref_pos,
))
}
};
return Ok(Value::Int(value_node!(i, pref_pos)));
}
if v.0.is_none() && has_dot {
pref_pos = v.1.unwrap().pos;
}
let (maybepos, suf) = match v.2 {
None => (None, "".to_string()),
Some(bs) => (Some(bs.pos), bs.fragment),
};
let to_parse = pref.to_string() + "." + &suf;
// TODO(jwall): if there is an error we should report where that error occured.
let f = match FromStr::from_str(&to_parse) {
Ok(f) => f,
Err(_) => {
return Err(error::Error::new(
format!("Not a float! {}", to_parse),
error::ErrorType::UnexpectedToken,
maybepos.unwrap(),
))
}
};
return Ok(Value::Float(value_node!(f, pref_pos)));
}
// trace_macros!(true);
// NOTE(jwall): HERE THERE BE DRAGONS. The order for these matters
// alot. We need to process alternatives in order of decreasing
// specificity. Unfortunately this means we are required to go in a
// decreasing size order which messes with alt!'s completion logic. To
// work around this we have to force Incomplete to be Error so that
// alt! will try the next in the series instead of aborting.
//
// *IMPORTANT*
// It also means this combinator is risky when used with partial
// inputs. So handle with care.
named!(number<TokenIter, Value, error::Error>,
map_res!(alt!(
complete!(do_parse!( // 1.0
prefix: match_type!(DIGIT) >>
has_dot: punct!(".") >>
suffix: match_type!(DIGIT) >>
(Some(prefix.clone()), Some(has_dot.clone()), Some(suffix.clone()))
)) |
complete!(do_parse!( // 1.
prefix: match_type!(DIGIT) >>
has_dot: punct!(".") >>
(Some(prefix.clone()), Some(has_dot.clone()), None)
)) |
complete!(do_parse!( // .1
has_dot: punct!(".") >>
suffix: match_type!(DIGIT) >>
(None, Some(has_dot.clone()), Some(suffix.clone()))
)) |
do_parse!( // 1
prefix: match_type!(DIGIT) >>
// The peek!(not!(..)) make this whole combinator slightly
// safer for partial inputs.
(Some(prefix.clone()), None, None)
)),
triple_to_number
)
);
// trace_macros!(false);
named!(boolean_value<TokenIter, Value, error::Error>,
do_parse!(
b: match_type!(BOOLEAN) >>
(Value::Boolean(Positioned{
val: b.fragment == "true",
pos: b.pos,
}))
)
);
named!(
field_value<TokenIter, (Token, Expression), error::Error>,
do_parse!(
field: match_type!(BAREWORD) >>
punct!("=") >>
value: expression >>
(field, value)
)
);
// Helper function to make the return types work for down below.
fn vec_to_tuple(t: (Position, Option<FieldList>)) -> ParseResult<Value> {
Ok(Value::Tuple(value_node!(
t.1.unwrap_or(Vec::new()),
t.0.line as usize,
t.0.column as usize
)))
}
named!(field_list<TokenIter, FieldList, error::Error>,
separated_list!(punct!(","), field_value)
);
named!(
#[doc="Capture a tuple of named fields with values. {<field>=<value>,...}"],
tuple<TokenIter, Value, error::Error>,
map_res!(
do_parse!(
pos: pos >>
punct!("{") >>
v: field_list >>
punct!("}") >>
(pos, Some(v))
),
vec_to_tuple
)
);
fn tuple_to_list<Sp: Into<Position>>(t: (Sp, Vec<Expression>)) -> ParseResult<Value> {
return Ok(Value::List(ListDef {
elems: t.1,
pos: t.0.into(),
}));
}
named!(list_value<TokenIter, Value, error::Error>,
map_res!(
do_parse!(
start: punct!("[") >>
elements: separated_list!(punct!(","), expression) >>
punct!("]") >>
(start.pos, elements)
),
tuple_to_list
)
);
named!(empty_value<TokenIter, Value, error::Error>,
do_parse!(
pos: pos >>
match_type!(EMPTY) >>
(Value::Empty(pos))
)
);
named!(value<TokenIter, Value, error::Error>,
alt!(
boolean_value |
empty_value |
number |
quoted_value |
list_value |
tuple |
selector_value )
);
fn value_to_expression(v: Value) -> ParseResult<Expression> {
Ok(Expression::Simple(v))
}
named!(simple_expression<TokenIter, Expression, error::Error>,
map_res!(
value,
value_to_expression
)
);
fn tuple_to_binary_expression(
tpl: (Position, BinaryExprType, Expression, Expression),
) -> ParseResult<Expression> {
Ok(Expression::Binary(BinaryOpDef {
kind: tpl.1,
left: Box::new(tpl.2),
right: Box::new(tpl.3),
pos: Position::new(tpl.0.line as usize, tpl.0.column as usize),
}))
}
/// do_binary_expr implements precedence based parsing where the more tightly bound parsers
/// are passed in as lowerrule parsers. We default to grouped_expression and simple_expression as
/// the most tightly bound expressions.
macro_rules! do_binary_expr {
($i:expr, $oprule:ident!( $($args:tt)* ), $typ:expr) => {
do_binary_expr!($i, $oprule!($($args)*), $typ, alt!(grouped_expression | simple_expression))
};
($i:expr, $oprule:ident!( $($args:tt)* ), $typ:expr, $lowerrule:ident) => {
do_binary_expr!($i, $oprule!($($args)*), $typ, call!($lowerrule))
};
($i:expr, $oprule:ident!( $($args:tt)* ), $typ:expr, $lowerrule:ident!( $($lowerargs:tt)* )) => {
map_res!($i,
do_parse!(
pos: pos >>
left: $lowerrule!($($lowerargs)*) >>
$oprule!($($args)*) >>
right: $lowerrule!($($lowerargs)*) >>
(pos, $typ, left, right)
),
tuple_to_binary_expression
)
};
}
named!(math_expression<TokenIter, Expression, error::Error>,
alt!(sum_expression | product_expression)
);
named!(sum_expression<TokenIter, Expression, error::Error>,
alt!(add_expression | sub_expression)
);
// trace_macros!(true);
named!(add_expression<TokenIter, Expression, error::Error>,
do_binary_expr!(punct!("+"), BinaryExprType::Add, alt!(product_expression | simple_expression | grouped_expression))
);
// trace_macros!(false);
named!(sub_expression<TokenIter, Expression, error::Error>,
do_binary_expr!(punct!("-"), BinaryExprType::Sub, alt!(product_expression | simple_expression | grouped_expression))
);
named!(product_expression<TokenIter, Expression, error::Error>,
alt!(mul_expression | div_expression)
);
named!(mul_expression<TokenIter, Expression, error::Error>,
do_binary_expr!(punct!("*"), BinaryExprType::Mul)
);
named!(div_expression<TokenIter, Expression, error::Error>,
do_binary_expr!(punct!("/"), BinaryExprType::Div)
);
// TODO(jwall): Change comparison operators to use the do_binary_expr! with precedence?
fn tuple_to_compare_expression(
tpl: (Position, CompareType, Expression, Expression),
) -> ParseResult<Expression> {
Ok(Expression::Compare(ComparisonDef {
kind: tpl.1,
left: Box::new(tpl.2),
right: Box::new(tpl.3),
pos: Position::new(tpl.0.line as usize, tpl.0.column as usize),
}))
}
macro_rules! do_compare_expr {
($i:expr, $subrule:ident!( $($args:tt)* ), $typ:expr) => {
map_res!($i,
do_parse!(
pos: pos >>
left: alt!(simple_expression | grouped_expression | math_expression) >>
$subrule!($($args)*) >>
right: expression >>
(pos, $typ, left, right)
),
tuple_to_compare_expression
)
};
}
named!(eqeq_expression<TokenIter, Expression, error::Error>,
do_compare_expr!(punct!("=="), CompareType::Equal)
);
named!(not_eqeq_expression<TokenIter, Expression, error::Error>,
do_compare_expr!(punct!("!="), CompareType::NotEqual)
);
named!(lt_eqeq_expression<TokenIter, Expression, error::Error>,
do_compare_expr!(punct!("<="), CompareType::LTEqual)
);
named!(gt_eqeq_expression<TokenIter, Expression, error::Error>,
do_compare_expr!(punct!(">="), CompareType::GTEqual)
);
named!(gt_expression<TokenIter, Expression, error::Error>,
do_compare_expr!(punct!(">"), CompareType::GT)
);
named!(lt_expression<TokenIter, Expression, error::Error>,
do_compare_expr!(punct!("<"), CompareType::LT)
);
fn expression_to_grouped_expression(e: Expression) -> ParseResult<Expression> {
Ok(Expression::Grouped(Box::new(e)))
}
named!(grouped_expression<TokenIter, Expression, error::Error>,
map_res!(
preceded!(punct!("("), terminated!(expression, punct!(")"))),
expression_to_grouped_expression
)
);
fn symbol_or_expression(input: TokenIter) -> NomResult<Expression> {
let sym = do_parse!(input, sym: symbol >> (sym));
match sym {
IResult::Incomplete(i) => {
return IResult::Incomplete(i);
}
IResult::Error(_) => {
// TODO(jwall): Still missing some. But we need to avoid recursion
return grouped_expression(input);
}
IResult::Done(rest, val) => {
return IResult::Done(rest, Expression::Simple(val));
}
}
}
fn selector_list(input: TokenIter) -> NomResult<SelectorList> {
let (rest, head) = match symbol_or_expression(input) {
IResult::Done(rest, val) => (rest, val),
IResult::Error(e) => {
return IResult::Error(e);
}
IResult::Incomplete(i) => {
return IResult::Incomplete(i);
}
};
let (rest, is_dot) = match punct!(rest, ".") {
IResult::Done(rest, tok) => (rest, Some(tok)),
IResult::Incomplete(i) => {
return IResult::Incomplete(i);
}
IResult::Error(_) => (rest, None),
};
let (rest, list) = if is_dot.is_some() {
let (rest, list) = match separated_list!(
rest,
punct!("."),
alt!(match_type!(BAREWORD) | match_type!(DIGIT))
) {
IResult::Done(rest, val) => (rest, val),
IResult::Incomplete(i) => {
return IResult::Incomplete(i);
}
IResult::Error(e) => {
return IResult::Error(e);
}
};
if list.is_empty() {
return IResult::Error(nom::ErrorKind::Custom(error::Error::new(
"(.) with no selector fields after".to_string(),
error::ErrorType::IncompleteParsing,
is_dot.unwrap().pos,
)));
} else {
(rest, Some(list))
}
} else {
(rest, None)
};
let sel_list = SelectorList {
head: Box::new(head),
tail: list,
};
return IResult::Done(rest, sel_list);
}
fn tuple_to_copy(t: (SelectorDef, FieldList)) -> ParseResult<Expression> {
let pos = t.0.pos.clone();
Ok(Expression::Copy(CopyDef {
selector: t.0,
fields: t.1,
pos: pos,
}))
}
named!(copy_expression<TokenIter, Expression, error::Error>,
map_res!(
do_parse!(
pos: pos >>
selector: selector_list >>
punct!("{") >>
fields: field_list >>
punct!("}") >>
(SelectorDef::new(selector, pos.line, pos.column as usize), fields)
),
tuple_to_copy
)
);
fn tuple_to_macro(mut t: (Position, Vec<Value>, Value)) -> ParseResult<Expression> {
match t.2 {
Value::Tuple(v) => Ok(Expression::Macro(MacroDef {
argdefs: t.1
.drain(0..)
.map(|s| Positioned {
pos: s.pos().clone(),
val: s.to_string(),
})
.collect(),
fields: v.val,
pos: t.0,
})),
// TODO(jwall): Show a better version of the unexpected parsed value.
val => Err(error::Error::new(
format!("Expected Tuple Got {:?}", val),
error::ErrorType::UnexpectedToken,
t.0,
)),
}
}
named!(arglist<TokenIter, Vec<Value>, error::Error>, separated_list!(punct!(","), symbol));
named!(macro_expression<TokenIter, Expression, error::Error>,
map_res!(
do_parse!(
pos: pos >>
word!("macro") >>
punct!("(") >>
arglist: arglist >>
punct!(")") >>
punct!("=>") >>
map: tuple >>
(pos, arglist, map)
),
tuple_to_macro
)
);
fn tuple_to_select(t: (Position, Expression, Expression, Value)) -> ParseResult<Expression> {
match t.3 {
Value::Tuple(v) => Ok(Expression::Select(SelectDef {
val: Box::new(t.1),
default: Box::new(t.2),
tuple: v.val,
pos: t.0,
})),
val => Err(error::Error::new(
format!("Expected Tuple Got {:?}", val),
error::ErrorType::UnexpectedToken,
t.0,
)),
}
}
named!(select_expression<TokenIter, Expression, error::Error>,
map_res!(
do_parse!(
start: word!("select") >>
val: terminated!(expression, punct!(",")) >>
default: terminated!(expression, punct!(",")) >>
map: tuple >>
(start.pos.clone(), val, default, map)
),
tuple_to_select
)
);
fn tuple_to_format(t: (Token, Vec<Expression>)) -> ParseResult<Expression> {
Ok(Expression::Format(FormatDef {
template: t.0.fragment.to_string(),
args: t.1,
pos: t.0.pos,
}))
}
named!(format_expression<TokenIter, Expression, error::Error>,
map_res!(
do_parse!(
tmpl: match_type!(STR) >>
punct!("%") >>
punct!("(") >>
args: separated_list!(punct!(","), expression) >>
punct!(")") >>
(tmpl, args)
),
tuple_to_format
)
);
fn tuple_to_call(t: (Position, Value, Vec<Expression>)) -> ParseResult<Expression> {
if let Value::Selector(def) = t.1 {
Ok(Expression::Call(CallDef {
macroref: def,
arglist: t.2,
pos: Position::new(t.0.line as usize, t.0.column as usize),
}))
} else {
Err(error::Error::new(
format!("Expected Selector Got {:?}", t.0),
error::ErrorType::UnexpectedToken,
Position::new(t.0.line as usize, t.0.column as usize),
))
}
}
fn vec_to_selector_value(t: (Position, SelectorList)) -> ParseResult<Value> {
Ok(Value::Selector(SelectorDef::new(
t.1,
t.0.line as usize,
t.0.column as usize,
)))
}
named!(selector_value<TokenIter, Value, error::Error>,
map_res!(
do_parse!(
sl: selector_list >>
(sl.head.pos().clone(), sl)
),
vec_to_selector_value
)
);
named!(call_expression<TokenIter, Expression, error::Error>,
map_res!(
do_parse!(
macroname: selector_value >>
punct!("(") >>
args: separated_list!(punct!(","), expression) >>
punct!(")") >>
(macroname.pos().clone(), macroname, args)
),
tuple_to_call
)
);
fn symbol_or_list(input: TokenIter) -> NomResult<Value> {
let sym = do_parse!(input, sym: symbol >> (sym));
match sym {
IResult::Incomplete(i) => {
return IResult::Incomplete(i);
}
IResult::Error(_) => {
// TODO(jwall): Still missing some. But we need to avoid recursion
match list_value(input) {
IResult::Incomplete(i) => {
return IResult::Incomplete(i);
}
IResult::Error(e) => {
return IResult::Error(e);
}
IResult::Done(i, val) => {
return IResult::Done(i, val);
}
}
}
IResult::Done(rest, val) => {
return IResult::Done(rest, val);
}
}
}
fn tuple_to_list_op(tpl: (Position, Token, Value, Value)) -> ParseResult<Expression> {
let pos = tpl.0;
let t = if &tpl.1.fragment == "map" {
ListOpType::Map
} else if &tpl.1.fragment == "filter" {
ListOpType::Filter
} else {
return Err(error::Error::new(
format!(
"Expected one of 'map' or 'filter' but got '{}'",
tpl.1.fragment
),
error::ErrorType::UnexpectedToken,
pos,
));
};
let macroname = tpl.2;
let list = tpl.3;
if let Value::Selector(mut def) = macroname {
// First of all we need to assert that this is a selector of at least
// two sections.
let fieldname: String = match &mut def.sel.tail {
&mut None => {
return Err(error::Error::new(
format!("Missing a result field for the macro"),
error::ErrorType::IncompleteParsing,
pos,
));
}
&mut Some(ref mut tl) => {
if tl.len() < 1 {
return Err(error::Error::new(
format!("Missing a result field for the macro"),
error::ErrorType::IncompleteParsing,
def.pos.clone(),
));
}
let fname = tl.pop();
fname.unwrap().fragment
}
};
if let Value::List(ldef) = list {
return Ok(Expression::ListOp(ListOpDef {
typ: t,
mac: def,
field: fieldname,
target: ldef,
pos: pos,
}));
}
// TODO(jwall): We should print a pretter message than debug formatting here.
return Err(error::Error::new(
format!("Expected a list but got {:?}", list),
error::ErrorType::UnexpectedToken,
pos,
));
}
return Err(error::Error::new(
format!("Expected a selector but got {:?}", macroname),
error::ErrorType::UnexpectedToken,
pos,
));
}
named!(list_op_expression<TokenIter, Expression, error::Error>,
map_res!(
do_parse!(
pos: pos >>
optype: alt!(word!("map") | word!("filter")) >>
macroname: selector_value >>
list: symbol_or_list >>
(pos, optype, macroname, list)
),
tuple_to_list_op
)
);
// NOTE(jwall): HERE THERE BE DRAGONS. The order for these matters
// a lot. We need to process alternatives in order of decreasing
// specificity. Unfortunately this means we are required to go in a
// decreasing size order which messes with alt!'s completion logic. To
// work around this we have to force Incomplete to be Error so that
// alt! will try the next in the series instead of aborting.
//
// *IMPORTANT*
// It also means this combinator is risky when used with partial
// inputs. So handle with care.
named!(expression<TokenIter, Expression, error::Error>,
do_parse!(
expr: alt!(
complete!(list_op_expression) |
complete!(math_expression) |
complete!(eqeq_expression) |
complete!(not_eqeq_expression) |
complete!(lt_eqeq_expression) |
complete!(gt_eqeq_expression) |
complete!(gt_expression) |
complete!(lt_expression) |
complete!(macro_expression) |
complete!(format_expression) |
complete!(select_expression) |
complete!(call_expression) |
complete!(copy_expression) |
complete!(grouped_expression) |
simple_expression
) >>
(expr)
)
);
fn expression_to_statement(v: Expression) -> ParseResult<Statement> {
Ok(Statement::Expression(v))
}
named!(expression_statement<TokenIter, Statement, error::Error>,
map_res!(
terminated!(expression, punct!(";")),
expression_to_statement
)
);
fn tuple_to_let(t: (Token, Expression)) -> ParseResult<Statement> {
Ok(Statement::Let(LetDef {
name: t.0,
value: t.1,
}))
}
named!(let_statement<TokenIter, Statement, error::Error>,
map_res!(
do_parse!(
word!("let") >>
name: match_type!(BAREWORD) >>
punct!("=") >>
val: expression >>
punct!(";") >>
(name, val)
),
tuple_to_let
)
);
fn tuple_to_import(t: (Token, Token)) -> ParseResult<Statement> {
Ok(Statement::Import(ImportDef {
path: t.0,
name: t.1,
}))
}
named!(import_statement<TokenIter, Statement, error::Error>,
map_res!(
do_parse!(
word!("import") >>
path: match_type!(STR) >>
word!("as") >>
name: match_type!(BAREWORD) >>
punct!(";") >>
(path, name)
),
tuple_to_import
)
);
named!(statement<TokenIter, Statement, error::Error>,
do_parse!(
stmt: alt_complete!(
import_statement |
let_statement |
expression_statement
) >>
(stmt)
)
);
/// Parses a LocatedSpan into a list of Statements or an error::Error.
pub fn parse(input: LocatedSpan<&str>) -> Result<Vec<Statement>, error::Error> {
match tokenize(input) {
Ok(tokenized) => {
let mut out = Vec::new();
let mut i_ = TokenIter {
source: tokenized.as_slice(),
};
loop {
let i = i_.clone();
if i[0].typ == TokenType::END {
break;
}
match statement(i) {
IResult::Error(nom::ErrorKind::Custom(e)) => {
return Err(e);
}
IResult::Error(e) => {
return Err(error::Error::new_with_errorkind(
format!("Statement Parse error: {:?} current token: {:?}", e, i_[0]),
error::ErrorType::ParseError,
Position {
line: i_[0].pos.line,
column: i_[0].pos.column,
},
e,
));
}
IResult::Incomplete(ei) => {
return Err(error::Error::new(
format!("Unexpected end of parsing input: {:?}", ei),
error::ErrorType::IncompleteParsing,
Position {
line: i_[0].pos.line,
column: i_[0].pos.column,
},
));
}
IResult::Done(rest, stmt) => {
out.push(stmt);
i_ = rest;
if i_.input_len() == 0 {
break;
}
}
}
}
return Ok(out);
}
Err(e) => {
return Err(error::Error::new(
format!("Tokenization Error {:?}", e.1),
error::ErrorType::ParseError,
e.0,
));
}
}
}
#[cfg(test)]
mod test;

1099
src/parse/test.rs Normal file
View File

File diff suppressed because it is too large Load Diff

263
src/tokenizer.rs → src/tokenizer/mod.rs
View File

@ -13,7 +13,7 @@
// limitations under the License.
//! The tokenization stage of the ucg compiler.
use ast::tree::*;
use ast::*;
use error;
use nom;
use nom::{InputIter, InputLength, Slice};
@ -659,263 +659,4 @@ impl<'a> InputIter for TokenIter<'a> {
}
#[cfg(test)]
mod tokenizer_test {
use super::*;
use nom;
use nom_locate::LocatedSpan;
#[test]
fn test_empty_token() {
let result = emptytok(LocatedSpan::new("NULL"));
assert!(result.is_done(), format!("result {:?} is not done", result));
if let nom::IResult::Done(_, tok) = result {
assert_eq!(tok.fragment, "NULL");
assert_eq!(tok.typ, TokenType::EMPTY);
}
}
#[test]
fn test_escape_quoted() {
let result = escapequoted(LocatedSpan::new("foo \\\"bar\""));
assert!(result.is_done(), format!("result {:?} is not ok", result));
if let nom::IResult::Done(rest, frag) = result {
assert_eq!(frag, "foo \"bar");
assert_eq!(rest.fragment, "\"");
}
}
#[test]
fn test_string_with_escaping() {
let result = strtok(LocatedSpan::new("\"foo \\\\ \\\"bar\""));
assert!(result.is_done(), format!("result {:?} is not ok", result));
if let nom::IResult::Done(_, tok) = result {
assert_eq!(tok.fragment, "foo \\ \"bar".to_string());
}
}
#[test]
fn test_tokenize_bareword_with_dash() {
let result = tokenize(LocatedSpan::new("foo-bar "));
assert!(result.is_ok(), format!("result {:?} is not ok", result));
if let Ok(toks) = result {
assert_eq!(toks.len(), 2);
assert_eq!(toks[0].fragment, "foo-bar");
}
}
macro_rules! assert_token {
($input:expr, $typ:expr, $msg:expr) => {
let result = token(LocatedSpan::new($input));
assert!(
result.is_done(),
format!("result {:?} is not a {}", result, $msg)
);
if let nom::IResult::Done(_, tok) = result {
assert_eq!(tok.fragment, $input);
assert_eq!(tok.typ, $typ);
}
};
}
#[test]
fn test_boolean() {
assert_token!("true", TokenType::BOOLEAN, "boolean");
}
#[test]
fn test_eqeqtok() {
assert_token!("==", TokenType::PUNCT, "==");
}
#[test]
fn test_notequaltok() {
assert_token!("!=", TokenType::PUNCT, "!=");
}
#[test]
fn test_gttok() {
assert_token!(">", TokenType::PUNCT, ">");
}
#[test]
fn test_lttok() {
assert_token!("<", TokenType::PUNCT, "<");
}
#[test]
fn test_gteqtok() {
assert_token!(">=", TokenType::PUNCT, ">=");
}
#[test]
fn test_lteqtok() {
assert_token!("<=", TokenType::PUNCT, "<=");
}
#[test]
fn test_tokenize_one_of_each() {
let result = tokenize(LocatedSpan::new(
"let import macro select as => [ ] { } ; = % / * \
+ - . ( ) , 1 . foo \"bar\" // comment\n ; true false == < > <= >= !=",
));
assert!(result.is_ok(), format!("result {:?} is not ok", result));
let v = result.unwrap();
for (i, t) in v.iter().enumerate() {
println!("{}: {:?}", i, t);
}
assert_eq!(v.len(), 35);
assert_eq!(v[34].typ, TokenType::END);
}
#[test]
fn test_parse_has_end() {
let result = tokenize(LocatedSpan::new("foo"));
assert!(result.is_ok());
let v = result.unwrap();
assert_eq!(v.len(), 2);
assert_eq!(v[1].typ, TokenType::END);
}
#[test]
fn test_parse_comment() {
assert!(comment(LocatedSpan::new("// comment\n")).is_done());
assert!(comment(LocatedSpan::new("// comment")).is_done());
assert_eq!(
comment(LocatedSpan::new("// comment\n")),
nom::IResult::Done(
LocatedSpan {
fragment: "",
offset: 11,
line: 2,
},
Token {
typ: TokenType::COMMENT,
fragment: " comment".to_string(),
pos: Position { line: 1, column: 1 },
}
)
);
assert!(comment(LocatedSpan::new("// comment\r\n")).is_done());
assert_eq!(
comment(LocatedSpan::new("// comment\r\n")),
nom::IResult::Done(
LocatedSpan {
fragment: "",
offset: 12,
line: 2,
},
Token {
typ: TokenType::COMMENT,
fragment: " comment".to_string(),
pos: Position { column: 1, line: 1 },
}
)
);
assert!(comment(LocatedSpan::new("// comment\r\n ")).is_done());
assert_eq!(
comment(LocatedSpan::new("// comment\r\n ")),
nom::IResult::Done(
LocatedSpan {
fragment: " ",
offset: 12,
line: 2,
},
Token {
typ: TokenType::COMMENT,
fragment: " comment".to_string(),
pos: Position { column: 1, line: 1 },
}
)
);
assert!(comment(LocatedSpan::new("// comment")).is_done());
}
#[test]
fn test_match_word() {
let input = vec![
Token {
fragment: "foo".to_string(),
typ: TokenType::BAREWORD,
pos: Position { line: 1, column: 1 },
},
];
let result = word!(
TokenIter {
source: input.as_slice(),
},
"foo"
);
match result {
nom::IResult::Done(_, tok) => assert_eq!(tok, input[0]),
res => assert!(false, format!("Fail: {:?}", res)),
}
}
#[test]
fn test_match_word_empty_input() {
let input = vec![
Token {
fragment: "".to_string(),
typ: TokenType::END,
pos: Position { line: 1, column: 1 },
},
];
let result = word!(
TokenIter {
source: input.as_slice(),
},
"foo"
);
match result {
nom::IResult::Done(_, _) => assert!(false, "Should have been an error but was Done"),
nom::IResult::Incomplete(_) => {
assert!(false, "Should have been an error but was Incomplete")
}
nom::IResult::Error(_) => {
// noop
}
}
}
#[test]
fn test_match_punct() {
let input = vec![
Token {
fragment: "!".to_string(),
typ: TokenType::PUNCT,
pos: Position { line: 1, column: 1 },
},
];
let result = punct!(
TokenIter {
source: input.as_slice(),
},
"!"
);
match result {
nom::IResult::Done(_, tok) => assert_eq!(tok, input[0]),
res => assert!(false, format!("Fail: {:?}", res)),
}
}
#[test]
fn test_match_type() {
let input = vec![
Token {
fragment: "foo".to_string(),
typ: TokenType::BAREWORD,
pos: Position { line: 1, column: 1 },
},
];
let result = match_type!(
TokenIter {
source: input.as_slice(),
},
BAREWORD
);
match result {
nom::IResult::Done(_, tok) => assert_eq!(tok, input[0]),
res => assert!(false, format!("Fail: {:?}", res)),
}
}
}
mod test;

258
src/tokenizer/test.rs Normal file
View File

@ -0,0 +1,258 @@
use super::*;
use nom;
use nom_locate::LocatedSpan;
#[test]
fn test_empty_token() {
let result = emptytok(LocatedSpan::new("NULL"));
assert!(result.is_done(), format!("result {:?} is not done", result));
if let nom::IResult::Done(_, tok) = result {
assert_eq!(tok.fragment, "NULL");
assert_eq!(tok.typ, TokenType::EMPTY);
}
}
#[test]
fn test_escape_quoted() {
let result = escapequoted(LocatedSpan::new("foo \\\"bar\""));
assert!(result.is_done(), format!("result {:?} is not ok", result));
if let nom::IResult::Done(rest, frag) = result {
assert_eq!(frag, "foo \"bar");
assert_eq!(rest.fragment, "\"");
}
}
#[test]
fn test_string_with_escaping() {
let result = strtok(LocatedSpan::new("\"foo \\\\ \\\"bar\""));
assert!(result.is_done(), format!("result {:?} is not ok", result));
if let nom::IResult::Done(_, tok) = result {
assert_eq!(tok.fragment, "foo \\ \"bar".to_string());
}
}
#[test]
fn test_tokenize_bareword_with_dash() {
let result = tokenize(LocatedSpan::new("foo-bar "));
assert!(result.is_ok(), format!("result {:?} is not ok", result));
if let Ok(toks) = result {
assert_eq!(toks.len(), 2);
assert_eq!(toks[0].fragment, "foo-bar");
}
}
macro_rules! assert_token {
($input:expr, $typ:expr, $msg:expr) => {
let result = token(LocatedSpan::new($input));
assert!(
result.is_done(),
format!("result {:?} is not a {}", result, $msg)
);
if let nom::IResult::Done(_, tok) = result {
assert_eq!(tok.fragment, $input);
assert_eq!(tok.typ, $typ);
}
};
}
#[test]
fn test_boolean() {
assert_token!("true", TokenType::BOOLEAN, "boolean");
}
#[test]
fn test_eqeqtok() {
assert_token!("==", TokenType::PUNCT, "==");
}
#[test]
fn test_notequaltok() {
assert_token!("!=", TokenType::PUNCT, "!=");
}
#[test]
fn test_gttok() {
assert_token!(">", TokenType::PUNCT, ">");
}
#[test]
fn test_lttok() {
assert_token!("<", TokenType::PUNCT, "<");
}
#[test]
fn test_gteqtok() {
assert_token!(">=", TokenType::PUNCT, ">=");
}
#[test]
fn test_lteqtok() {
assert_token!("<=", TokenType::PUNCT, "<=");
}
#[test]
fn test_tokenize_one_of_each() {
let result = tokenize(LocatedSpan::new(
"let import macro select as => [ ] { } ; = % / * \
+ - . ( ) , 1 . foo \"bar\" // comment\n ; true false == < > <= >= !=",
));
assert!(result.is_ok(), format!("result {:?} is not ok", result));
let v = result.unwrap();
for (i, t) in v.iter().enumerate() {
println!("{}: {:?}", i, t);
}
assert_eq!(v.len(), 35);
assert_eq!(v[34].typ, TokenType::END);
}
#[test]
fn test_parse_has_end() {
let result = tokenize(LocatedSpan::new("foo"));
assert!(result.is_ok());
let v = result.unwrap();
assert_eq!(v.len(), 2);
assert_eq!(v[1].typ, TokenType::END);
}
#[test]
fn test_parse_comment() {
assert!(comment(LocatedSpan::new("// comment\n")).is_done());
assert!(comment(LocatedSpan::new("// comment")).is_done());
assert_eq!(
comment(LocatedSpan::new("// comment\n")),
nom::IResult::Done(
LocatedSpan {
fragment: "",
offset: 11,
line: 2,
},
Token {
typ: TokenType::COMMENT,
fragment: " comment".to_string(),
pos: Position { line: 1, column: 1 },
}
)
);
assert!(comment(LocatedSpan::new("// comment\r\n")).is_done());
assert_eq!(
comment(LocatedSpan::new("// comment\r\n")),
nom::IResult::Done(
LocatedSpan {
fragment: "",
offset: 12,
line: 2,
},
Token {
typ: TokenType::COMMENT,
fragment: " comment".to_string(),
pos: Position { column: 1, line: 1 },
}
)
);
assert!(comment(LocatedSpan::new("// comment\r\n ")).is_done());
assert_eq!(
comment(LocatedSpan::new("// comment\r\n ")),
nom::IResult::Done(
LocatedSpan {
fragment: " ",
offset: 12,
line: 2,
},
Token {
typ: TokenType::COMMENT,
fragment: " comment".to_string(),
pos: Position { column: 1, line: 1 },
}
)
);
assert!(comment(LocatedSpan::new("// comment")).is_done());
}
#[test]
fn test_match_word() {
let input = vec![
Token {
fragment: "foo".to_string(),
typ: TokenType::BAREWORD,
pos: Position { line: 1, column: 1 },
},
];
let result = word!(
TokenIter {
source: input.as_slice(),
},
"foo"
);
match result {
nom::IResult::Done(_, tok) => assert_eq!(tok, input[0]),
res => assert!(false, format!("Fail: {:?}", res)),
}
}
#[test]
fn test_match_word_empty_input() {
let input = vec![
Token {
fragment: "".to_string(),
typ: TokenType::END,
pos: Position { line: 1, column: 1 },
},
];
let result = word!(
TokenIter {
source: input.as_slice(),
},
"foo"
);
match result {
nom::IResult::Done(_, _) => assert!(false, "Should have been an error but was Done"),
nom::IResult::Incomplete(_) => {
assert!(false, "Should have been an error but was Incomplete")
}
nom::IResult::Error(_) => {
// noop
}
}
}
#[test]
fn test_match_punct() {
let input = vec![
Token {
fragment: "!".to_string(),
typ: TokenType::PUNCT,
pos: Position { line: 1, column: 1 },
},
];
let result = punct!(
TokenIter {
source: input.as_slice(),
},
"!"
);
match result {
nom::IResult::Done(_, tok) => assert_eq!(tok, input[0]),
res => assert!(false, format!("Fail: {:?}", res)),
}
}
#[test]
fn test_match_type() {
let input = vec![
Token {
fragment: "foo".to_string(),
typ: TokenType::BAREWORD,
pos: Position { line: 1, column: 1 },
},
];
let result = match_type!(
TokenIter {
source: input.as_slice(),
},
BAREWORD
);
match result {
nom::IResult::Done(_, tok) => assert_eq!(tok, input[0]),
res => assert!(false, format!("Fail: {:?}", res)),
}
}