FEATURE: Support a convert expression.

docsite/site/content/reference/expressions.md

@@ -681,4 +681,29 @@ This will output a line to stderr something like the below:
 
 This is helpful when developing shared modules or ucg libraries.
 
+Convert Expressions
+-------------------
+
+UCG has convert expressions which will turn any UCG value into a string using the specified conversion format.
+This expression is similar to the out expression except instead of writing to a file it writes to a string.
+
+It's useful for previewing the result of converting a ucg value in the repl or for composing multiple conversion
+formats together into a single composite ucg value.
+
+You can experiment with conversion in the repl:
+
+```
+> convert json {foo="bar"};
+'{
+  "foo": "bar"
+}'
+> 
+```
+
+Or store a converted value into a UCG string:
+
+```
+let converted = convert json {foo="bar"};
+```
+
 Next: <a href="/reference/statements">Statements</a>

docsite/site/content/reference/grammar.md

@@ -55,6 +55,7 @@ filter_keyword: "filter" ;
 module_keyword: "module" ;
 mod_keyword: "mod" ;
 out_keyword: "out" ;
+convert_keyword: "convert" ;
 assert_keyword: "assert" ;
 fail_keyword: "fail" ;
 trace_keyword: "TRACE" ;
@@ -230,11 +231,13 @@ expr: binary_expr | non_operator_expr ;
 ```
 let_statement: let_keyword, bareword, equal, expr ;
 out_statement: out_keyword, bareword, str ;
+convert_statement: convert_keyword, bareword, str ;
 assert_statement: assert_keyword, pipe, { statement }, pipe ;
 simple_statement: expr ;
 
 statement: ( let_statement
              | out_statement
+             | convert_statement
              | assert_statement
              | simple_statement ), semicolon ;
 ```

src/ast/mod.rs

@@ -775,6 +775,9 @@ pub enum Statement {
 
     // Identify an Expression for output.
     Output(Position, Token, Expression),
+
+    // Print the expression to stdout.
+    Print(Position, Token, Expression),
 }
 
 impl Statement {
@@ -784,6 +787,7 @@ impl Statement {
             Statement::Let(ref def) => &def.pos,
             Statement::Assert(ref pos, _) => pos,
             Statement::Output(ref pos, _, _) => pos,
+            Statement::Print(ref pos, _, _) => pos,
         }
     }
 }

src/ast/printer/mod.rs

@@ -552,6 +552,10 @@ where
                 write!(&mut self.w, "out {} ", _tok.fragment)?;
                 self.render_expr(&_expr)?;
             }
+            Statement::Print(_, _tok, _expr) => {
+                write!(&mut self.w, "print {} ", _tok.fragment)?;
+                self.render_expr(&_expr)?;
+            }
         };
         write!(self.w, ";\n\n")?;
         self.last_line = line;

src/ast/walk.rs

@@ -22,6 +22,9 @@ pub trait Walker {
             Statement::Output(_, _, ref mut expr) => {
                 self.walk_expression(expr);
             }
+            Statement::Print(_, _, ref mut expr) => {
+                self.walk_expression(expr);
+            }
         }
     }
 

src/build/compile_test.rs

@@ -19,11 +19,13 @@ use regex::Regex;
 
 use super::assets::MemoryCache;
 use super::FileBuilder;
+use crate::convert::ConverterRegistry;
 
 fn assert_build(input: &str) {
     let i_paths = Vec::new();
     let cache = MemoryCache::new();
-    let mut b = FileBuilder::new("<Eval>", &i_paths, Rc::new(RefCell::new(cache)));
+    let registry = ConverterRegistry::make_registry();
+    let mut b = FileBuilder::new("<Eval>", &i_paths, Rc::new(RefCell::new(cache)), &registry);
     b.enable_validate_mode();
     b.eval_string(input).unwrap();
     if !b.assert_collector.success {
@@ -34,7 +36,8 @@ fn assert_build(input: &str) {
 fn assert_build_failure(input: &str, expect: Vec<Regex>) {
     let i_paths = Vec::new();
     let cache = MemoryCache::new();
-    let mut b = FileBuilder::new("<Eval>", &i_paths, Rc::new(RefCell::new(cache)));
+    let registry = ConverterRegistry::make_registry();
+    let mut b = FileBuilder::new("<Eval>", &i_paths, Rc::new(RefCell::new(cache)), &registry);
     b.enable_validate_mode();
     let err = b.eval_string(input);
     match err {

src/build/mod.rs

@@ -32,6 +32,7 @@ use unicode_segmentation::UnicodeSegmentation;
 use crate::ast::*;
 use crate::build::format::{ExpressionFormatter, FormatRenderer, SimpleFormatter};
 use crate::build::scope::{find_in_fieldlist, Scope, ValueMap};
+use crate::convert::ConverterRegistry;
 use crate::convert::ImporterRegistry;
 use crate::error;
 use crate::iter::OffsetStrIter;
@@ -112,6 +113,7 @@ where
     pub assert_collector: AssertCollector,
     scope: Scope,
     import_registry: ImporterRegistry,
+    converter_registry: &'a ConverterRegistry,
     // NOTE(jwall): We use interior mutability here because we need
     // our asset cache to be shared by multiple different sub-builders.
     // We use Rc to handle the reference counting for us and we use
@@ -156,6 +158,7 @@ where
         working_dir: P,
         import_paths: &'a Vec<PathBuf>,
         cache: Rc<RefCell<C>>,
+        converter_registry: &'a ConverterRegistry,
     ) -> Self {
         let env_vars: Vec<(String, String)> = env::vars().collect();
         let scope = scope::Scope::new(Rc::new(Val::Env(env_vars)));
@@ -173,6 +176,7 @@ where
             },
             scope: scope,
             import_registry: ImporterRegistry::make_registry(),
+            converter_registry: converter_registry,
             assets: cache,
             out_lock: None,
             is_module: false,
@@ -195,6 +199,7 @@ where
             assets: self.assets.clone(),
             // This is admittedly a little wasteful but we can live with it for now.
             import_registry: ImporterRegistry::make_registry(),
+            converter_registry: self.converter_registry,
             scope: self.scope.spawn_clean(),
             out_lock: None,
             is_module: false,
@@ -365,6 +370,8 @@ where
             normalized.push(&path);
             // First see if the normalized file exists or not.
             if !normalized.exists() && use_import_path {
+                // TODO(jwall): Support importing from a zip file in this
+                // import_path?
                 // If it does not then look for it in the list of import_paths
                 for mut p in self.import_path.iter().cloned() {
                     p.push(&path);
@@ -425,7 +432,6 @@ where
         }
         let sep = format!("{}", std::path::MAIN_SEPARATOR);
         let raw_path = def.path.fragment.replace("/", &sep);
-        // Try a relative path first.
         let normalized = match self.find_file(&raw_path, true) {
             Ok(p) => p,
             Err(e) => {
@@ -551,6 +557,31 @@ where
                     .to_boxed())
                 }
             }
+            &Statement::Print(ref pos, ref typ, ref expr) => {
+                if let None = self.out_lock {
+                    let val = self.eval_expr(expr, &child_scope)?;
+                    match self.converter_registry.get_converter(&typ.fragment) {
+                        Some(c) => {
+                            let mut buf = Vec::new();
+                            c.convert(val.clone(), &mut buf)?;
+                            Ok(Rc::new(Val::Str(String::from_utf8(buf)?)))
+                        }
+                        None => Err(error::BuildError::with_pos(
+                            format!("Invalid Converter specified for print {}", typ.fragment),
+                            error::ErrorType::Unsupported,
+                            pos.clone(),
+                        )
+                        .to_boxed()),
+                    }
+                } else {
+                    Err(error::BuildError::with_pos(
+                        format!("You can only have one output per file."),
+                        error::ErrorType::Unsupported,
+                        pos.clone(),
+                    )
+                    .to_boxed())
+                }
+            }
         }
     }
 

src/build/test.rs

@@ -15,6 +15,7 @@ use super::assets;
 use super::assets::MemoryCache;
 use super::{FileBuilder, SelectDef, Val};
 use crate::ast::*;
+use crate::convert::ConverterRegistry;
 
 use std;
 use std::cell::RefCell;
@@ -37,7 +38,8 @@ fn test_expr_to_val<'a, C: assets::Cache>(
 fn test_eval_div_expr_fail() {
     let i_paths = Vec::new();
     let cache = Rc::new(RefCell::new(MemoryCache::new()));
-    let b = FileBuilder::new(std::env::current_dir().unwrap(), &i_paths, cache);
+    let registry = ConverterRegistry::make_registry();
+    let b = FileBuilder::new(std::env::current_dir().unwrap(), &i_paths, cache, &registry);
     test_expr_to_val(
         vec![(
             Expression::Binary(BinaryOpDef {
@@ -63,7 +65,8 @@ fn test_eval_div_expr_fail() {
 fn test_eval_mul_expr_fail() {
     let i_paths = Vec::new();
     let cache = Rc::new(RefCell::new(MemoryCache::new()));
-    let b = FileBuilder::new(std::env::current_dir().unwrap(), &i_paths, cache);
+    let registry = ConverterRegistry::make_registry();
+    let b = FileBuilder::new(std::env::current_dir().unwrap(), &i_paths, cache, &registry);
     test_expr_to_val(
         vec![(
             Expression::Binary(BinaryOpDef {
@@ -89,7 +92,8 @@ fn test_eval_mul_expr_fail() {
 fn test_eval_subtract_expr_fail() {
     let i_paths = Vec::new();
     let cache = Rc::new(RefCell::new(MemoryCache::new()));
-    let b = FileBuilder::new(std::env::current_dir().unwrap(), &i_paths, cache);
+    let registry = ConverterRegistry::make_registry();
+    let b = FileBuilder::new(std::env::current_dir().unwrap(), &i_paths, cache, &registry);
     test_expr_to_val(
         vec![(
             Expression::Binary(BinaryOpDef {
@@ -114,7 +118,8 @@ fn test_eval_subtract_expr_fail() {
 fn test_eval_add_expr_fail() {
     let i_paths = Vec::new();
     let cache = Rc::new(RefCell::new(MemoryCache::new()));
-    let b = FileBuilder::new(std::env::current_dir().unwrap(), &i_paths, cache);
+    let registry = ConverterRegistry::make_registry();
+    let b = FileBuilder::new(std::env::current_dir().unwrap(), &i_paths, cache, &registry);
     test_expr_to_val(
         vec![(
             Expression::Binary(BinaryOpDef {
@@ -139,7 +144,8 @@ fn test_eval_add_expr_fail() {
 fn test_eval_simple_lookup_error() {
     let i_paths = Vec::new();
     let cache = Rc::new(RefCell::new(MemoryCache::new()));
-    let mut b = FileBuilder::new(std::env::current_dir().unwrap(), &i_paths, cache);
+    let registry = ConverterRegistry::make_registry();
+    let mut b = FileBuilder::new(std::env::current_dir().unwrap(), &i_paths, cache, &registry);
     b.scope
         .build_output
         .entry(value_node!("var1".to_string(), Position::new(1, 0, 0)))
@@ -157,7 +163,8 @@ fn test_eval_simple_lookup_error() {
 fn test_expr_copy_no_such_tuple() {
     let i_paths = Vec::new();
     let cache = Rc::new(RefCell::new(MemoryCache::new()));
-    let b = FileBuilder::new(std::env::current_dir().unwrap(), &i_paths, cache);
+    let registry = ConverterRegistry::make_registry();
+    let b = FileBuilder::new(std::env::current_dir().unwrap(), &i_paths, cache, &registry);
     test_expr_to_val(
         vec![(
             Expression::Copy(CopyDef {
@@ -179,7 +186,8 @@ fn test_expr_copy_no_such_tuple() {
 fn test_select_expr_not_a_string() {
     let i_paths = Vec::new();
     let cache = Rc::new(RefCell::new(MemoryCache::new()));
-    let mut b = FileBuilder::new(std::env::current_dir().unwrap(), &i_paths, cache);
+    let registry = ConverterRegistry::make_registry();
+    let mut b = FileBuilder::new(std::env::current_dir().unwrap(), &i_paths, cache, &registry);
     b.scope
         .build_output
         .entry(value_node!("foo".to_string(), Position::new(1, 0, 0)))

src/convert/exec.rs

@@ -197,6 +197,7 @@ mod exec_test {
     use crate::build::assets::MemoryCache;
     use crate::build::FileBuilder;
     use crate::convert::traits::Converter;
+    use crate::convert::ConverterRegistry;
 
     use std;
     use std::cell::RefCell;
@@ -206,7 +207,8 @@ mod exec_test {
     fn convert_just_command_test() {
         let i_paths = Vec::new();
         let cache = Rc::new(RefCell::new(MemoryCache::new()));
-        let mut b = FileBuilder::new(std::env::current_dir().unwrap(), &i_paths, cache);
+        let registry = ConverterRegistry::make_registry();
+        let mut b = FileBuilder::new(std::env::current_dir().unwrap(), &i_paths, cache, &registry);
         let conv = ExecConverter::new();
         b.eval_string(
             "let script = {
@@ -228,7 +230,8 @@ mod exec_test {
     fn convert_command_with_env_test() {
         let i_paths = Vec::new();
         let cache = Rc::new(RefCell::new(MemoryCache::new()));
-        let mut b = FileBuilder::new(std::env::current_dir().unwrap(), &i_paths, cache);
+        let registry = ConverterRegistry::make_registry();
+        let mut b = FileBuilder::new(std::env::current_dir().unwrap(), &i_paths, cache, &registry);
         let conv = ExecConverter::new();
         b.eval_string(
             "let script = {
@@ -257,7 +260,8 @@ mod exec_test {
     fn convert_command_with_arg_test() {
         let i_paths = Vec::new();
         let cache = Rc::new(RefCell::new(MemoryCache::new()));
-        let mut b = FileBuilder::new(std::env::current_dir().unwrap(), &i_paths, cache);
+        let registry = ConverterRegistry::make_registry();
+        let mut b = FileBuilder::new(std::env::current_dir().unwrap(), &i_paths, cache, &registry);
         let conv = ExecConverter::new();
         b.eval_string(
             "let script = {

src/main.rs

@@ -99,12 +99,14 @@ fn build_file<'a, C: Cache>(
     strict: bool,
     import_paths: &'a Vec<PathBuf>,
     cache: Rc<RefCell<C>>,
+    registry: &'a ConverterRegistry,
 ) -> Result<build::FileBuilder<'a, C>, Box<dyn Error>> {
     let mut file_path_buf = PathBuf::from(file);
     if file_path_buf.is_relative() {
         file_path_buf = std::env::current_dir()?.join(file_path_buf);
     }
-    let mut builder = build::FileBuilder::new(std::env::current_dir()?, import_paths, cache);
+    let mut builder =
+        build::FileBuilder::new(std::env::current_dir()?, import_paths, cache, registry);
     builder.set_strict(strict);
     if validate {
         builder.enable_validate_mode();
@@ -121,9 +123,10 @@ fn do_validate<C: Cache>(
     strict: bool,
     import_paths: &Vec<PathBuf>,
     cache: Rc<RefCell<C>>,
+    registry: &ConverterRegistry,
 ) -> bool {
     println!("Validating {}", file);
-    match build_file(file, true, strict, import_paths, cache) {
+    match build_file(file, true, strict, import_paths, cache, registry) {
         Ok(b) => {
             if b.assert_collector.success {
                 println!("File {} Pass\n", file);
@@ -148,7 +151,7 @@ fn do_compile<C: Cache>(
     registry: &ConverterRegistry,
 ) -> bool {
     println!("Building {}", file);
-    let builder = match build_file(file, false, strict, import_paths, cache.clone()) {
+    let builder = match build_file(file, false, strict, import_paths, cache.clone(), registry) {
         Ok(builder) => builder,
         Err(err) => {
             eprintln!("{}", err);
@@ -214,7 +217,13 @@ fn visit_ucg_files<C: Cache>(
                 }
             } else {
                 if validate && path_as_string.ends_with("_test.ucg") {
-                    if !do_validate(&path_as_string, strict, import_paths, cache.clone()) {
+                    if !do_validate(
+                        &path_as_string,
+                        strict,
+                        import_paths,
+                        cache.clone(),
+                        registry,
+                    ) {
                         result = false;
                         summary.push_str(format!("{} - FAIL\n", path_as_string).as_str())
                     } else {
@@ -234,7 +243,7 @@ fn visit_ucg_files<C: Cache>(
             }
         }
     } else if validate && our_path.ends_with("_test.ucg") {
-        if !do_validate(&our_path, strict, import_paths, cache) {
+        if !do_validate(&our_path, strict, import_paths, cache, registry) {
             result = false;
             summary.push_str(format!("{} - FAIL\n", our_path).as_str());
         } else {
@@ -262,8 +271,12 @@ fn inspect_command<C: Cache>(
     let file = matches.value_of("INPUT").unwrap_or("std/functional.ucg");
     let sym = matches.value_of("expr");
     let target = matches.value_of("target").unwrap_or("json");
-    let mut builder =
+    let mut builder = build::FileBuilder::new(
-        build::FileBuilder::new(std::env::current_dir().unwrap(), import_paths, cache);
+        std::env::current_dir().unwrap(),
+        import_paths,
+        cache,
+        registry,
+    );
     builder.set_strict(strict);
     match registry.get_converter(target) {
         Some(converter) => {
@@ -512,6 +525,7 @@ fn print_repl_help() {
 fn do_repl<C: Cache>(
     import_paths: &Vec<PathBuf>,
     cache: Rc<RefCell<C>>,
+    registry: &ConverterRegistry,
 ) -> std::result::Result<(), Box<dyn Error>> {
     let config = rustyline::Config::builder();
     let mut editor = rustyline::Editor::<()>::with_config(
@@ -544,7 +558,8 @@ fn do_repl<C: Cache>(
             }
         }
     }
-    let mut builder = build::FileBuilder::new(std::env::current_dir()?, import_paths, cache);
+    let mut builder =
+        build::FileBuilder::new(std::env::current_dir()?, import_paths, cache, registry);
     // loop
     let mut lines = ucglib::io::StatementAccumulator::new();
     println!("Welcome to the UCG repl. Ctrl-D to exit");
@@ -607,8 +622,12 @@ fn do_repl<C: Cache>(
     }
 }
 
-fn repl<C: Cache>(import_paths: &Vec<PathBuf>, cache: Rc<RefCell<C>>) {
+fn repl<C: Cache>(
-    if let Err(e) = do_repl(import_paths, cache) {
+    import_paths: &Vec<PathBuf>,
+    cache: Rc<RefCell<C>>,
+    registry: &ConverterRegistry,
+) {
+    if let Err(e) = do_repl(import_paths, cache, registry) {
         eprintln!("{}", e);
         process::exit(1);
     }
@@ -652,7 +671,7 @@ fn main() {
     } else if let Some(_) = app_matches.subcommand_matches("env") {
         env_help()
     } else if let Some(_) = app_matches.subcommand_matches("repl") {
-        repl(&import_paths, cache)
+        repl(&import_paths, cache, &registry)
     } else if let Some(matches) = app_matches.subcommand_matches("fmt") {
         if let Err(e) = fmt_command(matches) {
             eprintln!("{}", e);

src/parse/mod.rs

@@ -841,6 +841,18 @@ make_fn!(
     )
 );
 
+make_fn!(
+    print_statement<SliceIter<Token>, Statement>,
+    do_each!(
+        pos => pos,
+        _ => word!("convert"),
+        typ => wrap_err!(must!(match_type!(BAREWORD)), "Expected converter name"),
+        expr => wrap_err!(must!(expression), "Expected Expression to print"),
+        _ => must!(punct!(";")),
+        (Statement::Print(pos, typ.clone(), expr.clone()))
+    )
+);
+
 //trace_macros!(true);
 fn statement(i: SliceIter<Token>) -> Result<SliceIter<Token>, Statement> {
     return either!(
@@ -848,6 +860,7 @@ fn statement(i: SliceIter<Token>) -> Result<SliceIter<Token>, Statement> {
         trace_parse!(assert_statement),
         trace_parse!(let_statement),
         trace_parse!(out_statement),
+        trace_parse!(print_statement),
         trace_parse!(expression_statement)
     );
 }

src/tokenizer/mod.rs

@@ -325,6 +325,10 @@ make_fn!(outtok<OffsetStrIter, Token>,
        do_text_token_tok!(TokenType::BAREWORD, "out", WS)
 );
 
+make_fn!(converttok<OffsetStrIter, Token>,
+       do_text_token_tok!(TokenType::BAREWORD, "convert", WS)
+);
+
 make_fn!(astok<OffsetStrIter, Token>,
        do_text_token_tok!(TokenType::BAREWORD, "as", WS)
 );
@@ -442,6 +446,7 @@ fn token<'a>(input: OffsetStrIter<'a>) -> Result<OffsetStrIter<'a>, Token> {
         nottok,
         lettok,
         outtok,
+        converttok,
         selecttok,
         asserttok,
         failtok,