FEATURE: Imports as expressions.

This is a breaking change for the language. However it makes a number of things
easier. Like importing specific symbols from a file.

adds: #28

docsite/buckets/ucg-docs.ucg

@@ -1,5 +1,5 @@
-import "../ucglib/globals.ucg" as globals;
+let globals = import "../ucglib/globals.ucg";
-import "../ucglib/macros.ucg" as mks;
+let mks = import "../ucglib/macros.ucg";
 
 let doc_bucket_name = "ucg.marzhillstudios.com";
 

examples/module_example/modules/site_module.ucg

@@ -8,7 +8,7 @@ let site_mod = module{
     
     // this binding is not visible outside of the module. should be at the time of definition?
     // or path should be rewritten to be absolue before instantiation.
-    import "../../shared.ucg" as shared;
+    let shared = import "../../shared.ucg";
 
     // processing should be delayed till module instantiation.
     let base_config = shared.mk_site_config(mod.hostname, mod.port); 

examples/module_example/modules/unified.ucg

@@ -1,6 +1,6 @@
 let composed = module{} => {
-    import "host_module.ucg" as host_mod;
+    let host_mod = import "host_module.ucg";
-    import "site_module.ucg" as site_mod;
+    let  site_mod = import "site_module.ucg";
 
     let site_conf = site_mod.site_mod{hostname="example.com", port=80};
     let host_conf = host_mod.host_mod{hostname="example.com"};

examples/module_example/test_mod_host.ucg

@@ -1,3 +1,3 @@
-import "modules/unified.ucg" as unified;
+let unified = import "modules/unified.ucg";
 
 out yaml unified.host_conf;

examples/module_example/test_mod_site.ucg

@@ -1,3 +1,3 @@
-import "modules/unified.ucg" as unified;
+let unified = import "modules/unified.ucg";
 
 out json unified.site_conf;

examples/test_flags.txt

@@ -0,0 +1 @@
+--port 8080 --listen '0.0.0.0' --verbose --dir 'some/dir' --dir 'some/other/dir' --log.debug true --log.format 'json' 

examples/test_json.ucg

@@ -1,4 +1,4 @@
-import "shared.ucg" as shared;
+let shared = import "shared.ucg";
 
 // A few constants.
 let dbhost1 = "db1.prod.net";

examples/test_yaml.ucg

@@ -1,4 +1,4 @@
-import "shared.ucg" as shared;
+let shared = import "shared.ucg";
 
 // A few constants.
 let dbhost1 = "db1.prod.net";

integration_tests/import_test.ucg

@@ -0,0 +1,16 @@
+let shared = import "./libs/shared.ucg";
+let test_mod = (import "./libs/shared.ucg").test_mod{};
+
+let script = include str "./include_example.sh";
+
+assert {
+    ok = script == test_mod.script,
+    desc = "include path worked from an imported module",
+};
+
+let imported = (import "./libs/test_import.ucg").val;
+
+assert {
+    ok = imported == test_mod.imported,
+    desc = "include path worked from an imported module",
+};

integration_tests/libs/shared.ucg

@@ -0,0 +1,4 @@
+let test_mod = module{} => {
+    let imported = (import "./test_import.ucg").val;
+    let script = include str "../include_example.sh";
+};

integration_tests/libs/test_import.ucg

@@ -0,0 +1 @@
+let val = "imported value";

src/ast/mod.rs

@@ -31,6 +31,8 @@ use abortable_parser;
 
 use crate::build::Val;
 
+pub mod walk;
+
 macro_rules! enum_type_equality {
     ( $slf:ident, $r:expr, $( $l:pat ),* ) => {
         match $slf {
@@ -458,6 +460,7 @@ impl MacroDef {
                     | &Expression::Module(_)
                     | &Expression::Range(_)
                     | &Expression::FuncOp(_)
+                    | &Expression::Import(_)
                     | &Expression::Include(_) => {
                         // noop
                         continue;
@@ -618,8 +621,8 @@ impl ModuleDef {
     }
 
     pub fn imports_to_absolute(&mut self, base: PathBuf) {
-        for stmt in self.statements.iter_mut() {
+        let rewrite_import = |e: &mut Expression| {
-            if let &mut Statement::Import(ref mut def) = stmt {
+            if let Expression::Include(ref mut def) = e {
                 let path = PathBuf::from(&def.path.fragment);
                 if path.is_relative() {
                     def.path.fragment = base
@@ -630,6 +633,21 @@ impl ModuleDef {
                         .to_string();
                 }
             }
+            if let Expression::Import(ref mut def) = e {
+                let path = PathBuf::from(&def.path.fragment);
+                if path.is_relative() {
+                    def.path.fragment = base
+                        .join(path)
+                        .canonicalize()
+                        .unwrap()
+                        .to_string_lossy()
+                        .to_string();
+                }
+            }
+        };
+        let walker = walk::AstWalker::new().with_expr_handler(&rewrite_import);
+        for stmt in self.statements.iter_mut() {
+            walker.walk_statement(stmt);
         }
     }
 }
@@ -643,6 +661,13 @@ pub struct RangeDef {
     pub end: Box<Expression>,
 }
 
+/// Encodes an import expression in the UCG AST.
+#[derive(Debug, PartialEq, Clone)]
+pub struct ImportDef {
+    pub pos: Position,
+    pub path: Token,
+}
+
 /// Encodes a ucg expression. Expressions compute a value from.
 #[derive(Debug, PartialEq, Clone)]
 pub enum Expression {
@@ -659,6 +684,7 @@ pub enum Expression {
     Grouped(Box<Expression>),
     Format(FormatDef),
     Include(IncludeDef),
+    Import(ImportDef),
     Call(CallDef),
     Macro(MacroDef),
     Select(SelectDef),
@@ -682,6 +708,7 @@ impl Expression {
             &Expression::Select(ref def) => &def.pos,
             &Expression::FuncOp(ref def) => def.pos(),
             &Expression::Include(ref def) => &def.pos,
+            &Expression::Import(ref def) => &def.pos,
         }
     }
 }
@@ -725,6 +752,9 @@ impl fmt::Display for Expression {
             &Expression::Include(_) => {
                 write!(w, "<Include>")?;
             }
+            &Expression::Import(_) => {
+                write!(w, "<Include>")?;
+            }
         }
         Ok(())
     }
@@ -737,13 +767,6 @@ pub struct LetDef {
     pub value: Expression,
 }
 
-/// Encodes an import statement in the UCG AST.
-#[derive(Debug, PartialEq, Clone)]
-pub struct ImportDef {
-    pub path: Token,
-    pub name: Token,
-}
-
 /// Encodes a parsed statement in the UCG AST.
 #[derive(Debug, PartialEq, Clone)]
 pub enum Statement {
@@ -753,9 +776,6 @@ pub enum Statement {
     // Named bindings
     Let(LetDef),
 
-    // Import a file.
-    Import(ImportDef),
-
     // Assert statement
     Assert(Expression),
 

src/ast/walk.rs

@@ -0,0 +1,137 @@
+use crate::ast::*;
+
+pub struct AstWalker<'a> {
+    handle_value: Option<&'a Fn(&mut Value)>,
+    handle_expression: Option<&'a Fn(&mut Expression)>,
+    handle_statment: Option<&'a Fn(&mut Statement)>,
+}
+
+impl<'a> AstWalker<'a> {
+    pub fn new() -> Self {
+        AstWalker {
+            handle_value: None,
+            handle_expression: None,
+            handle_statment: None,
+        }
+    }
+
+    pub fn with_value_handler(mut self, h: &'a Fn(&mut Value)) -> Self {
+        self.handle_value = Some(h);
+        self
+    }
+
+    pub fn with_expr_handler(mut self, h: &'a Fn(&mut Expression)) -> Self {
+        self.handle_expression = Some(h);
+        self
+    }
+
+    pub fn with_stmt_handler(mut self, h: &'a Fn(&mut Statement)) -> Self {
+        self.handle_statment = Some(h);
+        self
+    }
+
+    pub fn walk_statement(&self, stmt: &mut Statement) {
+        self.visit_statement(stmt);
+        match stmt {
+            Statement::Let(ref mut def) => {
+                self.walk_expression(&mut def.value);
+            }
+            Statement::Expression(ref mut expr) => {
+                self.walk_expression(expr);
+            }
+            Statement::Assert(ref mut expr) => {
+                self.walk_expression(expr);
+            }
+            Statement::Output(_, ref mut expr) => {
+                self.walk_expression(expr);
+            }
+        }
+    }
+
+    fn walk_fieldset(&self, fs: &mut FieldList) {
+        for &mut (_, ref mut expr) in fs.iter_mut() {
+            self.walk_expression(expr);
+        }
+    }
+
+    pub fn walk_expression(&self, expr: &mut Expression) {
+        self.visit_expression(expr);
+        match expr {
+            Expression::Call(ref mut def) => {
+                for expr in def.arglist.iter_mut() {
+                    self.walk_expression(expr);
+                }
+            }
+            Expression::Copy(ref mut def) => {
+                self.walk_fieldset(&mut def.fields);
+            }
+            Expression::Format(ref mut def) => {
+                for expr in def.args.iter_mut() {
+                    self.walk_expression(expr);
+                }
+            }
+            Expression::FuncOp(ref mut def) => match def {
+                FuncOpDef::Reduce(ref mut def) => {
+                    self.walk_expression(def.target.as_mut());
+                    self.walk_expression(def.acc.as_mut())
+                }
+                FuncOpDef::Map(ref mut def) => {
+                    self.walk_expression(def.target.as_mut());
+                }
+                FuncOpDef::Filter(ref mut def) => {
+                    self.walk_expression(def.target.as_mut());
+                }
+            },
+            Expression::Binary(ref mut def) => {
+                self.walk_expression(def.left.as_mut());
+                self.walk_expression(def.right.as_mut());
+            }
+            Expression::Grouped(ref mut expr) => {
+                self.walk_expression(expr);
+            }
+            Expression::Macro(ref mut def) => self.walk_fieldset(&mut def.fields),
+            Expression::Module(ref mut def) => {
+                self.walk_fieldset(&mut def.arg_set);
+                for stmt in def.statements.iter_mut() {
+                    self.walk_statement(stmt);
+                }
+            }
+            Expression::Range(ref mut def) => {
+                self.walk_expression(def.start.as_mut());
+                self.walk_expression(def.end.as_mut());
+                if let Some(ref mut expr) = def.step {
+                    self.walk_expression(expr.as_mut());
+                }
+            }
+            Expression::Select(ref mut def) => {
+                self.walk_expression(def.default.as_mut());
+                self.walk_expression(def.val.as_mut());
+                self.walk_fieldset(&mut def.tuple);
+            }
+            Expression::Simple(ref mut val) => {
+                self.visit_value(val);
+            }
+            Expression::Import(_) | Expression::Include(_) => {
+                //noop
+            }
+        }
+    }
+
+    fn visit_value(&self, val: &mut Value) {
+        if let Some(h) = self.handle_value {
+            h(val);
+        }
+    }
+
+    fn visit_expression(&self, expr: &mut Expression) {
+        if let Some(h) = self.handle_expression {
+            h(expr);
+        }
+    }
+
+    fn visit_statement(&self, stmt: &mut Statement) {
+        if let Some(h) = self.handle_statment {
+            h(stmt);
+        }
+    }
+}

src/build/mod.rs

@@ -372,18 +372,7 @@ impl<'a> FileBuilder<'a> {
         Ok(normalized.canonicalize()?)
     }
 
-    fn eval_import(&mut self, def: &ImportDef) -> Result<Rc<Val>, Box<dyn Error>> {
+    fn eval_import(&self, def: &ImportDef) -> Result<Rc<Val>, Box<dyn Error>> {
-        let sym = &def.name;
-        if Self::check_reserved_word(&sym.fragment) {
-            return Err(Box::new(error::BuildError::new(
-                format!(
-                    "Import {} binding collides with reserved word",
-                    sym.fragment
-                ),
-                error::ErrorType::ReservedWordError,
-                sym.pos.clone(),
-            )));
-        }
         // Try a relative path first.
         let normalized = self.find_file(&def.path.fragment, true)?;
         if self.detect_import_cycle(normalized.to_string_lossy().as_ref()) {
@@ -394,7 +383,7 @@ impl<'a> FileBuilder<'a> {
                     self.scope.import_stack,
                 ),
                 error::ErrorType::Unsupported,
-                sym.pos.clone(),
+                def.pos.clone(),
             )));
         }
         // Introduce a scope so the above borrow is dropped before we modify
@@ -409,15 +398,6 @@ impl<'a> FileBuilder<'a> {
                 b.get_outputs_as_val()
             }
         };
-        let key = sym.into();
-        if self.scope.build_output.contains_key(&key) {
-            return Err(Box::new(error::BuildError::new(
-                format!("Binding for import name {} already exists", sym.fragment),
-                error::ErrorType::DuplicateBinding,
-                def.path.pos.clone(),
-            )));
-        }
-        self.scope.build_output.insert(key, result.clone());
         let mut mut_assets_cache = self.assets.borrow_mut();
         mut_assets_cache.stash(normalized.clone(), result.clone())?;
         return Ok(result);
@@ -459,7 +439,6 @@ impl<'a> FileBuilder<'a> {
         match stmt {
             &Statement::Assert(ref expr) => self.build_assert(&expr, &child_scope),
             &Statement::Let(ref def) => self.eval_let(def),
-            &Statement::Import(ref def) => self.eval_import(def),
             &Statement::Expression(ref expr) => self.eval_expr(expr, &child_scope),
             // Only one output can be used per file. Right now we enforce this by
             // having a single builder per file.
@@ -1613,6 +1592,7 @@ impl<'a> FileBuilder<'a> {
             &Expression::Select(ref def) => self.eval_select(def, scope),
             &Expression::FuncOp(ref def) => self.eval_func_op(def, scope),
             &Expression::Include(ref def) => self.eval_include(def),
+            &Expression::Import(ref def) => self.eval_import(def),
         }
     }
 }

src/parse/mod.rs

@@ -654,6 +654,19 @@ make_fn!(
     )
 );
 
+make_fn!(
+    import_expression<SliceIter<Token>, Expression>,
+    do_each!(
+        pos => pos,
+        _ => word!("import"),
+        path => must!(wrap_err!(match_type!(STR), "Expected import path")),
+        (Expression::Import(ImportDef{
+            pos: pos,
+            path: path,
+        }))
+    )
+);
+
 fn unprefixed_expression(input: SliceIter<Token>) -> ParseResult<Expression> {
     let _input = input.clone();
     either!(
@@ -671,6 +684,7 @@ make_fn!(
     either!(
         trace_parse!(func_op_expression),
         trace_parse!(macro_expression),
+        trace_parse!(import_expression),
         trace_parse!(module_expression),
         trace_parse!(select_expression),
         trace_parse!(grouped_expression),
@@ -728,34 +742,6 @@ make_fn!(
     )
 );
 
-fn tuple_to_import(tok: Token, tok2: Token) -> Statement {
-    Statement::Import(ImportDef {
-        path: tok,
-        name: tok2,
-    })
-}
-
-make_fn!(
-    import_stmt_body<SliceIter<Token>, Statement>,
-    do_each!(
-        path => wrap_err!(match_type!(STR), "Expected import path"),
-        _ => word!("as"),
-        name => wrap_err!(match_type!(BAREWORD), "Expected import name"),
-        _ => punct!(";"),
-        (tuple_to_import(path, name))
-    )
-);
-
-make_fn!(
-    import_statement<SliceIter<Token>, Statement>,
-    do_each!(
-        _ => word!("import"),
-        // past this point we know this is supposed to be an import statement.
-        stmt => trace_parse!(must!(import_stmt_body)),
-        (stmt)
-    )
-);
-
 make_fn!(
     assert_statement<SliceIter<Token>, Statement>,
     do_each!(
@@ -782,7 +768,6 @@ fn statement(i: SliceIter<Token>) -> Result<SliceIter<Token>, Statement> {
     return either!(
         i,
         trace_parse!(assert_statement),
-        trace_parse!(import_statement),
         trace_parse!(let_statement),
         trace_parse!(out_statement),
         trace_parse!(expression_statement)

std/tests/lists_test.ucg

@@ -1,5 +1,5 @@
-import "../lists.ucg" as list;
+let list = import "../lists.ucg";
-import "../testing.ucg" as t;
+let t = import "../testing.ucg";
 
 let list_to_join = [1, 2, 3];
 

std/tests/testing_test.ucg

@@ -1,4 +1,4 @@
-import "../testing.ucg" as t;
+let t = import "../testing.ucg";
 
 let asserts = t.asserts{};
 

std/tests/tuples_test.ucg

@@ -1,17 +1,17 @@
-import "../tuples.ucg" as tpl;
+let tpl = import "../tuples.ucg";
-import "../testing.ucg" as t;
+let t = (import "../testing.ucg").asserts{};
 
-assert t.asserts{}.equal{
+assert t.equal{
     left = tpl.fields{tpl={foo=1, bar=2}}.result,
     right = ["foo", "bar"],
 };
 
-assert t.asserts{}.equal{
+assert t.equal{
     left = tpl.values{tpl={foo=1, bar=2}}.result,
     right = [1, 2],
 };
 
-assert t.asserts{}.equal{
+assert t.equal{
     left = tpl.enumerate{tpl={foo=1, bar=2}}.result,
     right = [["foo", 1], ["bar", 2]],
 };