From 54faeede5ecb97ecc0f1f8c46e9ce7f727fa2ce4 Mon Sep 17 00:00:00 2001 From: Jeremy Wall Date: Sun, 13 Jan 2019 09:37:44 -0600 Subject: [PATCH] FEATURE: Imports as expressions. This is a breaking change for the language. However it makes a number of things easier. Like importing specific symbols from a file. adds: #28 --- docsite/buckets/ucg-docs.ucg | 4 +- .../module_example/modules/site_module.ucg | 2 +- examples/module_example/modules/unified.ucg | 4 +- examples/module_example/test_mod_host.ucg | 2 +- examples/module_example/test_mod_site.ucg | 2 +- examples/test_flags.txt | 1 + examples/test_json.ucg | 2 +- examples/test_yaml.ucg | 2 +- integration_tests/import_test.ucg | 16 ++ integration_tests/libs/shared.ucg | 4 + integration_tests/libs/test_import.ucg | 1 + src/ast/mod.rs | 44 ++++-- src/ast/walk.rs | 137 ++++++++++++++++++ src/build/mod.rs | 26 +--- src/parse/mod.rs | 43 ++---- std/tests/lists_test.ucg | 4 +- std/tests/testing_test.ucg | 2 +- std/tests/tuples_test.ucg | 10 +- 18 files changed, 225 insertions(+), 81 deletions(-) create mode 100644 examples/test_flags.txt create mode 100644 integration_tests/import_test.ucg create mode 100644 integration_tests/libs/shared.ucg create mode 100644 integration_tests/libs/test_import.ucg create mode 100644 src/ast/walk.rs diff --git a/docsite/buckets/ucg-docs.ucg b/docsite/buckets/ucg-docs.ucg index 9bed465..1ce5eb8 100644 --- a/docsite/buckets/ucg-docs.ucg +++ b/docsite/buckets/ucg-docs.ucg @@ -1,5 +1,5 @@ -import "../ucglib/globals.ucg" as globals; -import "../ucglib/macros.ucg" as mks; +let globals = import "../ucglib/globals.ucg"; +let mks = import "../ucglib/macros.ucg"; let doc_bucket_name = "ucg.marzhillstudios.com"; diff --git a/examples/module_example/modules/site_module.ucg b/examples/module_example/modules/site_module.ucg index e275c23..8fcade3 100644 --- a/examples/module_example/modules/site_module.ucg +++ b/examples/module_example/modules/site_module.ucg @@ -8,7 +8,7 @@ let site_mod = module{ // this binding is not visible outside of the module. should be at the time of definition? // or path should be rewritten to be absolue before instantiation. - import "../../shared.ucg" as shared; + let shared = import "../../shared.ucg"; // processing should be delayed till module instantiation. let base_config = shared.mk_site_config(mod.hostname, mod.port); diff --git a/examples/module_example/modules/unified.ucg b/examples/module_example/modules/unified.ucg index 25b9209..b166dcc 100644 --- a/examples/module_example/modules/unified.ucg +++ b/examples/module_example/modules/unified.ucg @@ -1,6 +1,6 @@ let composed = module{} => { - import "host_module.ucg" as host_mod; - import "site_module.ucg" as site_mod; + let host_mod = import "host_module.ucg"; + let site_mod = import "site_module.ucg"; let site_conf = site_mod.site_mod{hostname="example.com", port=80}; let host_conf = host_mod.host_mod{hostname="example.com"}; diff --git a/examples/module_example/test_mod_host.ucg b/examples/module_example/test_mod_host.ucg index e0dde8e..12b7c0b 100644 --- a/examples/module_example/test_mod_host.ucg +++ b/examples/module_example/test_mod_host.ucg @@ -1,3 +1,3 @@ -import "modules/unified.ucg" as unified; +let unified = import "modules/unified.ucg"; out yaml unified.host_conf; \ No newline at end of file diff --git a/examples/module_example/test_mod_site.ucg b/examples/module_example/test_mod_site.ucg index 3f582de..0f93e69 100644 --- a/examples/module_example/test_mod_site.ucg +++ b/examples/module_example/test_mod_site.ucg @@ -1,3 +1,3 @@ -import "modules/unified.ucg" as unified; +let unified = import "modules/unified.ucg"; out json unified.site_conf; \ No newline at end of file diff --git a/examples/test_flags.txt b/examples/test_flags.txt new file mode 100644 index 0000000..dc41662 --- /dev/null +++ b/examples/test_flags.txt @@ -0,0 +1 @@ +--port 8080 --listen '0.0.0.0' --verbose --dir 'some/dir' --dir 'some/other/dir' --log.debug true --log.format 'json' \ No newline at end of file diff --git a/examples/test_json.ucg b/examples/test_json.ucg index a509c5a..1b0f221 100644 --- a/examples/test_json.ucg +++ b/examples/test_json.ucg @@ -1,4 +1,4 @@ -import "shared.ucg" as shared; +let shared = import "shared.ucg"; // A few constants. let dbhost1 = "db1.prod.net"; diff --git a/examples/test_yaml.ucg b/examples/test_yaml.ucg index b2bed5b..c87bc84 100644 --- a/examples/test_yaml.ucg +++ b/examples/test_yaml.ucg @@ -1,4 +1,4 @@ -import "shared.ucg" as shared; +let shared = import "shared.ucg"; // A few constants. let dbhost1 = "db1.prod.net"; diff --git a/integration_tests/import_test.ucg b/integration_tests/import_test.ucg new file mode 100644 index 0000000..9c11740 --- /dev/null +++ b/integration_tests/import_test.ucg @@ -0,0 +1,16 @@ +let shared = import "./libs/shared.ucg"; +let test_mod = (import "./libs/shared.ucg").test_mod{}; + +let script = include str "./include_example.sh"; + +assert { + ok = script == test_mod.script, + desc = "include path worked from an imported module", +}; + +let imported = (import "./libs/test_import.ucg").val; + +assert { + ok = imported == test_mod.imported, + desc = "include path worked from an imported module", +}; \ No newline at end of file diff --git a/integration_tests/libs/shared.ucg b/integration_tests/libs/shared.ucg new file mode 100644 index 0000000..743dd99 --- /dev/null +++ b/integration_tests/libs/shared.ucg @@ -0,0 +1,4 @@ +let test_mod = module{} => { + let imported = (import "./test_import.ucg").val; + let script = include str "../include_example.sh"; +}; \ No newline at end of file diff --git a/integration_tests/libs/test_import.ucg b/integration_tests/libs/test_import.ucg new file mode 100644 index 0000000..0fa19e2 --- /dev/null +++ b/integration_tests/libs/test_import.ucg @@ -0,0 +1 @@ +let val = "imported value"; \ No newline at end of file diff --git a/src/ast/mod.rs b/src/ast/mod.rs index 67b9e2d..6ccd747 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -31,6 +31,8 @@ use abortable_parser; use crate::build::Val; +pub mod walk; + macro_rules! enum_type_equality { ( $slf:ident, $r:expr, $( $l:pat ),* ) => { match $slf { @@ -458,6 +460,7 @@ impl MacroDef { | &Expression::Module(_) | &Expression::Range(_) | &Expression::FuncOp(_) + | &Expression::Import(_) | &Expression::Include(_) => { // noop continue; @@ -618,8 +621,8 @@ impl ModuleDef { } pub fn imports_to_absolute(&mut self, base: PathBuf) { - for stmt in self.statements.iter_mut() { - if let &mut Statement::Import(ref mut def) = stmt { + let rewrite_import = |e: &mut Expression| { + if let Expression::Include(ref mut def) = e { let path = PathBuf::from(&def.path.fragment); if path.is_relative() { def.path.fragment = base @@ -630,6 +633,21 @@ impl ModuleDef { .to_string(); } } + if let Expression::Import(ref mut def) = e { + let path = PathBuf::from(&def.path.fragment); + if path.is_relative() { + def.path.fragment = base + .join(path) + .canonicalize() + .unwrap() + .to_string_lossy() + .to_string(); + } + } + }; + let walker = walk::AstWalker::new().with_expr_handler(&rewrite_import); + for stmt in self.statements.iter_mut() { + walker.walk_statement(stmt); } } } @@ -643,6 +661,13 @@ pub struct RangeDef { pub end: Box, } +/// Encodes an import expression in the UCG AST. +#[derive(Debug, PartialEq, Clone)] +pub struct ImportDef { + pub pos: Position, + pub path: Token, +} + /// Encodes a ucg expression. Expressions compute a value from. #[derive(Debug, PartialEq, Clone)] pub enum Expression { @@ -659,6 +684,7 @@ pub enum Expression { Grouped(Box), Format(FormatDef), Include(IncludeDef), + Import(ImportDef), Call(CallDef), Macro(MacroDef), Select(SelectDef), @@ -682,6 +708,7 @@ impl Expression { &Expression::Select(ref def) => &def.pos, &Expression::FuncOp(ref def) => def.pos(), &Expression::Include(ref def) => &def.pos, + &Expression::Import(ref def) => &def.pos, } } } @@ -725,6 +752,9 @@ impl fmt::Display for Expression { &Expression::Include(_) => { write!(w, "")?; } + &Expression::Import(_) => { + write!(w, "")?; + } } Ok(()) } @@ -737,13 +767,6 @@ pub struct LetDef { pub value: Expression, } -/// Encodes an import statement in the UCG AST. -#[derive(Debug, PartialEq, Clone)] -pub struct ImportDef { - pub path: Token, - pub name: Token, -} - /// Encodes a parsed statement in the UCG AST. #[derive(Debug, PartialEq, Clone)] pub enum Statement { @@ -753,9 +776,6 @@ pub enum Statement { // Named bindings Let(LetDef), - // Import a file. - Import(ImportDef), - // Assert statement Assert(Expression), diff --git a/src/ast/walk.rs b/src/ast/walk.rs new file mode 100644 index 0000000..193a438 --- /dev/null +++ b/src/ast/walk.rs @@ -0,0 +1,137 @@ +use crate::ast::*; + +pub struct AstWalker<'a> { + handle_value: Option<&'a Fn(&mut Value)>, + handle_expression: Option<&'a Fn(&mut Expression)>, + handle_statment: Option<&'a Fn(&mut Statement)>, +} + +impl<'a> AstWalker<'a> { + pub fn new() -> Self { + AstWalker { + handle_value: None, + handle_expression: None, + handle_statment: None, + } + } + + pub fn with_value_handler(mut self, h: &'a Fn(&mut Value)) -> Self { + self.handle_value = Some(h); + self + } + + pub fn with_expr_handler(mut self, h: &'a Fn(&mut Expression)) -> Self { + self.handle_expression = Some(h); + self + } + + pub fn with_stmt_handler(mut self, h: &'a Fn(&mut Statement)) -> Self { + self.handle_statment = Some(h); + self + } + + pub fn walk_statement(&self, stmt: &mut Statement) { + self.visit_statement(stmt); + match stmt { + Statement::Let(ref mut def) => { + self.walk_expression(&mut def.value); + } + Statement::Expression(ref mut expr) => { + self.walk_expression(expr); + } + Statement::Assert(ref mut expr) => { + self.walk_expression(expr); + } + Statement::Output(_, ref mut expr) => { + self.walk_expression(expr); + } + } + } + + fn walk_fieldset(&self, fs: &mut FieldList) { + for &mut (_, ref mut expr) in fs.iter_mut() { + self.walk_expression(expr); + } + } + + pub fn walk_expression(&self, expr: &mut Expression) { + self.visit_expression(expr); + match expr { + Expression::Call(ref mut def) => { + for expr in def.arglist.iter_mut() { + self.walk_expression(expr); + } + } + Expression::Copy(ref mut def) => { + self.walk_fieldset(&mut def.fields); + } + Expression::Format(ref mut def) => { + for expr in def.args.iter_mut() { + self.walk_expression(expr); + } + } + Expression::FuncOp(ref mut def) => match def { + FuncOpDef::Reduce(ref mut def) => { + self.walk_expression(def.target.as_mut()); + self.walk_expression(def.acc.as_mut()) + } + FuncOpDef::Map(ref mut def) => { + self.walk_expression(def.target.as_mut()); + } + FuncOpDef::Filter(ref mut def) => { + self.walk_expression(def.target.as_mut()); + } + }, + Expression::Binary(ref mut def) => { + self.walk_expression(def.left.as_mut()); + self.walk_expression(def.right.as_mut()); + } + Expression::Grouped(ref mut expr) => { + self.walk_expression(expr); + } + Expression::Macro(ref mut def) => self.walk_fieldset(&mut def.fields), + Expression::Module(ref mut def) => { + self.walk_fieldset(&mut def.arg_set); + for stmt in def.statements.iter_mut() { + self.walk_statement(stmt); + } + } + Expression::Range(ref mut def) => { + self.walk_expression(def.start.as_mut()); + self.walk_expression(def.end.as_mut()); + if let Some(ref mut expr) = def.step { + self.walk_expression(expr.as_mut()); + } + } + Expression::Select(ref mut def) => { + self.walk_expression(def.default.as_mut()); + self.walk_expression(def.val.as_mut()); + self.walk_fieldset(&mut def.tuple); + } + Expression::Simple(ref mut val) => { + self.visit_value(val); + } + Expression::Import(_) | Expression::Include(_) => { + //noop + } + } + } + + fn visit_value(&self, val: &mut Value) { + if let Some(h) = self.handle_value { + h(val); + } + } + + fn visit_expression(&self, expr: &mut Expression) { + if let Some(h) = self.handle_expression { + h(expr); + } + } + + fn visit_statement(&self, stmt: &mut Statement) { + if let Some(h) = self.handle_statment { + h(stmt); + } + } +} diff --git a/src/build/mod.rs b/src/build/mod.rs index 7a4bf88..3a522b9 100644 --- a/src/build/mod.rs +++ b/src/build/mod.rs @@ -372,18 +372,7 @@ impl<'a> FileBuilder<'a> { Ok(normalized.canonicalize()?) } - fn eval_import(&mut self, def: &ImportDef) -> Result, Box> { - let sym = &def.name; - if Self::check_reserved_word(&sym.fragment) { - return Err(Box::new(error::BuildError::new( - format!( - "Import {} binding collides with reserved word", - sym.fragment - ), - error::ErrorType::ReservedWordError, - sym.pos.clone(), - ))); - } + fn eval_import(&self, def: &ImportDef) -> Result, Box> { // Try a relative path first. let normalized = self.find_file(&def.path.fragment, true)?; if self.detect_import_cycle(normalized.to_string_lossy().as_ref()) { @@ -394,7 +383,7 @@ impl<'a> FileBuilder<'a> { self.scope.import_stack, ), error::ErrorType::Unsupported, - sym.pos.clone(), + def.pos.clone(), ))); } // Introduce a scope so the above borrow is dropped before we modify @@ -409,15 +398,6 @@ impl<'a> FileBuilder<'a> { b.get_outputs_as_val() } }; - let key = sym.into(); - if self.scope.build_output.contains_key(&key) { - return Err(Box::new(error::BuildError::new( - format!("Binding for import name {} already exists", sym.fragment), - error::ErrorType::DuplicateBinding, - def.path.pos.clone(), - ))); - } - self.scope.build_output.insert(key, result.clone()); let mut mut_assets_cache = self.assets.borrow_mut(); mut_assets_cache.stash(normalized.clone(), result.clone())?; return Ok(result); @@ -459,7 +439,6 @@ impl<'a> FileBuilder<'a> { match stmt { &Statement::Assert(ref expr) => self.build_assert(&expr, &child_scope), &Statement::Let(ref def) => self.eval_let(def), - &Statement::Import(ref def) => self.eval_import(def), &Statement::Expression(ref expr) => self.eval_expr(expr, &child_scope), // Only one output can be used per file. Right now we enforce this by // having a single builder per file. @@ -1613,6 +1592,7 @@ impl<'a> FileBuilder<'a> { &Expression::Select(ref def) => self.eval_select(def, scope), &Expression::FuncOp(ref def) => self.eval_func_op(def, scope), &Expression::Include(ref def) => self.eval_include(def), + &Expression::Import(ref def) => self.eval_import(def), } } } diff --git a/src/parse/mod.rs b/src/parse/mod.rs index 33822ed..839e6ce 100644 --- a/src/parse/mod.rs +++ b/src/parse/mod.rs @@ -654,6 +654,19 @@ make_fn!( ) ); +make_fn!( + import_expression, Expression>, + do_each!( + pos => pos, + _ => word!("import"), + path => must!(wrap_err!(match_type!(STR), "Expected import path")), + (Expression::Import(ImportDef{ + pos: pos, + path: path, + })) + ) +); + fn unprefixed_expression(input: SliceIter) -> ParseResult { let _input = input.clone(); either!( @@ -671,6 +684,7 @@ make_fn!( either!( trace_parse!(func_op_expression), trace_parse!(macro_expression), + trace_parse!(import_expression), trace_parse!(module_expression), trace_parse!(select_expression), trace_parse!(grouped_expression), @@ -728,34 +742,6 @@ make_fn!( ) ); -fn tuple_to_import(tok: Token, tok2: Token) -> Statement { - Statement::Import(ImportDef { - path: tok, - name: tok2, - }) -} - -make_fn!( - import_stmt_body, Statement>, - do_each!( - path => wrap_err!(match_type!(STR), "Expected import path"), - _ => word!("as"), - name => wrap_err!(match_type!(BAREWORD), "Expected import name"), - _ => punct!(";"), - (tuple_to_import(path, name)) - ) -); - -make_fn!( - import_statement, Statement>, - do_each!( - _ => word!("import"), - // past this point we know this is supposed to be an import statement. - stmt => trace_parse!(must!(import_stmt_body)), - (stmt) - ) -); - make_fn!( assert_statement, Statement>, do_each!( @@ -782,7 +768,6 @@ fn statement(i: SliceIter) -> Result, Statement> { return either!( i, trace_parse!(assert_statement), - trace_parse!(import_statement), trace_parse!(let_statement), trace_parse!(out_statement), trace_parse!(expression_statement) diff --git a/std/tests/lists_test.ucg b/std/tests/lists_test.ucg index b982b3d..382f3cb 100644 --- a/std/tests/lists_test.ucg +++ b/std/tests/lists_test.ucg @@ -1,5 +1,5 @@ -import "../lists.ucg" as list; -import "../testing.ucg" as t; +let list = import "../lists.ucg"; +let t = import "../testing.ucg"; let list_to_join = [1, 2, 3]; diff --git a/std/tests/testing_test.ucg b/std/tests/testing_test.ucg index 5522fc5..73f78ea 100644 --- a/std/tests/testing_test.ucg +++ b/std/tests/testing_test.ucg @@ -1,4 +1,4 @@ -import "../testing.ucg" as t; +let t = import "../testing.ucg"; let asserts = t.asserts{}; diff --git a/std/tests/tuples_test.ucg b/std/tests/tuples_test.ucg index 028ee6e..df45a30 100644 --- a/std/tests/tuples_test.ucg +++ b/std/tests/tuples_test.ucg @@ -1,17 +1,17 @@ -import "../tuples.ucg" as tpl; -import "../testing.ucg" as t; +let tpl = import "../tuples.ucg"; +let t = (import "../testing.ucg").asserts{}; -assert t.asserts{}.equal{ +assert t.equal{ left = tpl.fields{tpl={foo=1, bar=2}}.result, right = ["foo", "bar"], }; -assert t.asserts{}.equal{ +assert t.equal{ left = tpl.values{tpl={foo=1, bar=2}}.result, right = [1, 2], }; -assert t.asserts{}.equal{ +assert t.equal{ left = tpl.enumerate{tpl={foo=1, bar=2}}.result, right = [["foo", 1], ["bar", 2]], }; \ No newline at end of file