Add base64 as an importer type aside from string.

Cargo.lock

@@ -41,6 +41,14 @@ dependencies = [
  "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "base64"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "byteorder 1.2.7 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "bencher"
 version = "0.1.5"
@@ -51,6 +59,11 @@ name = "bitflags"
 version = "0.9.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "byteorder"
+version = "1.2.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "cc"
 version = "1.0.22"
@@ -237,6 +250,7 @@ name = "ucg"
 version = "0.2.9"
 dependencies = [
  "abortable_parser 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "base64 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "bencher 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
  "clap 2.26.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "cpuprofiler 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -305,8 +319,10 @@ dependencies = [
 "checksum atty 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "9a7d5b8723950951411ee34d271d99dddcc2035a16ab25310ea2c8cfd4369652"
 "checksum backtrace 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "346d7644f0b5f9bc73082d3b2236b69a05fd35cce0cfa3724e184e6a5c9e2a2f"
 "checksum backtrace-sys 0.1.24 (registry+https://github.com/rust-lang/crates.io-index)" = "c66d56ac8dabd07f6aacdaf633f4b8262f5b3601a810a0dcddffd5c22c69daa0"
+"checksum base64 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "621fc7ecb8008f86d7fb9b95356cd692ce9514b80a86d85b397f32a22da7b9e2"
 "checksum bencher 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "7dfdb4953a096c551ce9ace855a604d702e6e62d77fac690575ae347571717f5"
 "checksum bitflags 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4efd02e230a02e18f92fc2735f44597385ed02ad8f831e7c1c1156ee5e1ab3a5"
+"checksum byteorder 1.2.7 (registry+https://github.com/rust-lang/crates.io-index)" = "94f88df23a25417badc922ab0f5716cc1330e87f71ddd9203b3a3ccd9cedf75d"
 "checksum cc 1.0.22 (registry+https://github.com/rust-lang/crates.io-index)" = "4a6007c146fdd28d4512a794b07ffe9d8e89e6bf86e2e0c4ddff2e1fb54a0007"
 "checksum cfg-if 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "0c4e7bb64a8ebb0d856483e1e682ea3422f883c5f5615a90d51a2c82fe87fdd3"
 "checksum clap 2.26.2 (registry+https://github.com/rust-lang/crates.io-index)" = "3451e409013178663435d6f15fdb212f14ee4424a3d74f979d081d0a66b6f1f2"

Cargo.toml

@@ -18,6 +18,7 @@ simple-error = "0.1"
 serde_yaml = "~0.8.1"
 toml = "~0.4.8"
 xml-rs = "0.8.0"
+base64 = "0.10.0"
 
 [dev-dependencies]
 bencher = "~0.1.5"

integration_tests/include_test.ucg

@@ -3,3 +3,12 @@ assert |
     script == "#!/usr/bin/env bash
 echo \"included\"";
 |;
+let expected = "IyEvdXNyL2Jpbi9lbnYgYmFzaAplY2hvICJpbmNsdWRlZCI=";
+let base64 = include b64 "./include_example.sh";
+assert |
+    base64 == expected;
+|;
+let base64 = include b64urlsafe "./include_example.sh";
+assert |
+    base64 == expected;
+|;

src/build/mod.rs

@@ -29,6 +29,7 @@ use simple_error;
 
 use crate::ast::*;
 use crate::build::scope::{find_in_fieldlist, Scope, ValueMap};
+use crate::convert::ImporterRegistry;
 use crate::error;
 use crate::format;
 use crate::iter::OffsetStrIter;
@@ -100,6 +101,7 @@ pub struct FileBuilder<'a> {
     pub assert_collector: AssertCollector,
     strict: bool,
     scope: Scope,
+    import_registry: ImporterRegistry,
     // NOTE(jwall): We use interior mutability here because we need
     // our asset cache to be shared by multiple different sub-builders.
     // We use Rc to handle the reference counting for us and we use
@@ -166,6 +168,7 @@ impl<'a> FileBuilder<'a> {
             },
             scope: scope,
             strict: true,
+            import_registry: ImporterRegistry::make_registry(),
             assets: cache,
             out_lock: None,
             is_module: false,
@@ -185,6 +188,8 @@ impl<'a> FileBuilder<'a> {
             },
             strict: true,
             assets: self.assets.clone(),
+            // This is admittedly a little wasteful but we can live with it for now.
+            import_registry: ImporterRegistry::make_registry(),
             scope: self.scope.spawn_clean(),
             out_lock: None,
             is_module: false,
@@ -335,7 +340,7 @@ impl<'a> FileBuilder<'a> {
     }
 
     fn find_file<P: Into<PathBuf>>(
-        &mut self,
+        &self,
         path: P,
         use_import_path: bool,
     ) -> Result<PathBuf, Box<dyn Error>> {
@@ -1194,39 +1199,52 @@ impl<'a> FileBuilder<'a> {
         Ok(ok)
     }
 
+    fn get_file_as_string(&self, pos: &Position, path: &str) -> Result<String, Box<dyn Error>> {
+        let normalized = match self.find_file(path, false) {
+            Ok(p) => p,
+            Err(e) => {
+                return Err(Box::new(error::BuildError::new(
+                    format!("Error finding file {} {}", path, e),
+                    error::ErrorType::TypeFail,
+                    pos.clone(),
+                )))
+            }
+        };
+        let mut f = match File::open(&normalized) {
+            Ok(f) => f,
+            Err(e) => {
+                return Err(Box::new(error::BuildError::new(
+                    format!("Error opening file {} {}", normalized.to_string_lossy(), e),
+                    error::ErrorType::TypeFail,
+                    pos.clone(),
+                )))
+            }
+        };
+        let mut contents = String::new();
+        f.read_to_string(&mut contents)?;
+        Ok(contents)
+    }
+
     pub fn eval_include(&mut self, def: &IncludeDef) -> Result<Rc<Val>, Box<dyn Error>> {
         return if def.typ.fragment == "str" {
-            let normalized = match self.find_file(&def.path.fragment, false) {
-                Ok(p) => p,
-                Err(e) => {
-                    return Err(Box::new(error::BuildError::new(
-                        format!("Error finding file {} {}", def.path.fragment, e),
-                        error::ErrorType::TypeFail,
-                        def.typ.pos.clone(),
-                    )))
-                }
-            };
-            let mut f = match File::open(&normalized) {
-                Ok(f) => f,
-                Err(e) => {
-                    return Err(Box::new(error::BuildError::new(
-                        format!("Error opening file {} {}", normalized.to_string_lossy(), e),
-                        error::ErrorType::TypeFail,
-                        def.typ.pos.clone(),
-                    )))
-                }
-            };
-            let mut contents = String::new();
-            f.read_to_string(&mut contents)?;
-            Ok(Rc::new(Val::Str(contents)))
-        } else {
-            // TODO(jwall): Run the conversion on the contents of the file and return it as
-            //               an Rc<Val>.
-            Err(Box::new(error::BuildError::new(
-                format!("Unknown include conversion type {}", def.typ.fragment),
-                error::ErrorType::Unsupported,
-                def.typ.pos.clone(),
+            Ok(Rc::new(Val::Str(
+                self.get_file_as_string(&def.path.pos, &def.path.fragment)?,
             )))
+        } else {
+            let maybe_importer = self.import_registry.get_importer(&def.typ.fragment);
+            match maybe_importer {
+                Some(importer) => {
+                    let file_contents =
+                        self.get_file_as_string(&def.path.pos, &def.path.fragment)?;
+                    let val = importer.import(file_contents.as_bytes())?;
+                    Ok(val)
+                }
+                None => Err(Box::new(error::BuildError::new(
+                    format!("Unknown include conversion type {}", def.typ.fragment),
+                    error::ErrorType::Unsupported,
+                    def.typ.pos.clone(),
+                ))),
+            }
         };
     }
 

src/convert/b64.rs

@@ -0,0 +1,23 @@
+use std::error::Error;
+use std::rc::Rc;
+use std::result::Result;
+
+use base64::{encode, encode_config, URL_SAFE};
+
+use crate::build::Val;
+use crate::convert::traits::Importer;
+
+pub struct Base64Importer {
+    pub url_safe: bool,
+}
+
+impl Importer for Base64Importer {
+    fn import(&self, bytes: &[u8]) -> Result<Rc<Val>, Box<dyn Error>> {
+        let bslice = bytes.into();
+        return if self.url_safe {
+            Ok(Rc::new(Val::Str(encode(bslice))))
+        } else {
+            Ok(Rc::new(Val::Str(encode_config(bslice, URL_SAFE))))
+        };
+    }
+}

src/convert/mod.rs

@@ -13,6 +13,7 @@
 //  limitations under the License.
 
 //! The conversion stage of the ucg compiler.
+pub mod b64;
 pub mod env;
 pub mod exec;
 pub mod flags;
@@ -26,7 +27,7 @@ use std::collections::HashMap;
 
 /// ConverterRunner knows how to run a given converter on a Val.
 pub struct ConverterRegistry {
-    converters: HashMap<String, Box<traits::Converter>>,
+    converters: HashMap<String, Box<dyn traits::Converter>>,
 }
 
 impl ConverterRegistry {
@@ -68,3 +69,45 @@ impl ConverterRegistry {
 
     // TODO(jwall): Support converter help descriptions.
 }
+
+pub struct ImporterRegistry {
+    importers: HashMap<String, Box<dyn traits::Importer>>,
+}
+
+impl ImporterRegistry {
+    /// new creates a new ConverterRunner with a converter for the provided output target.
+    ///
+    /// * flags
+    /// * json
+    /// * env
+    /// * exec
+    fn new() -> Self {
+        ImporterRegistry {
+            importers: HashMap::new(),
+        }
+    }
+
+    pub fn make_registry() -> Self {
+        let mut registry = Self::new();
+        registry.register("b64", Box::new(b64::Base64Importer { url_safe: false }));
+        registry.register(
+            "b64urlsafe",
+            Box::new(b64::Base64Importer { url_safe: true }),
+        );
+        registry
+    }
+
+    pub fn register<S: Into<String>>(&mut self, typ: S, importer: Box<dyn traits::Importer>) {
+        self.importers.insert(typ.into(), importer);
+    }
+
+    pub fn get_importer(&self, typ: &str) -> Option<&dyn traits::Importer> {
+        self.importers.get(typ).map(|c| c.as_ref())
+    }
+
+    pub fn get_importer_list(&self) -> Vec<(&String, &Box<dyn traits::Importer>)> {
+        self.importers.iter().collect()
+    }
+
+    // TODO(jwall): Support converter help descriptions.
+}

src/convert/traits.rs

@@ -29,3 +29,7 @@ pub trait Converter {
     fn file_ext(&self) -> String;
     fn description(&self) -> String;
 }
+
+pub trait Importer {
+    fn import(&self, bytes: &[u8]) -> result::Result<Rc<Val>, Box<dyn Error>>;
+}

src/lib.rs

@@ -22,6 +22,7 @@
 #![recursion_limit = "128"]
 #[macro_use]
 extern crate abortable_parser;
+extern crate base64;
 extern crate serde_json;
 extern crate serde_yaml;
 extern crate simple_error;

src/main.rs

@@ -27,7 +27,7 @@ use ucglib::build;
 use ucglib::build::assets::{Cache, MemoryCache};
 use ucglib::build::Val;
 use ucglib::convert::traits;
-use ucglib::convert::ConverterRegistry;
+use ucglib::convert::{ConverterRegistry, ImporterRegistry};
 
 fn do_flags<'a, 'b>() -> clap::App<'a, 'b> {
     clap_app!(
@@ -55,6 +55,9 @@ fn do_flags<'a, 'b>() -> clap::App<'a, 'b> {
             (@subcommand converters =>
              (about: "list the available converters")
             )
+            (@subcommand importers =>
+             (about: "list the available importers for includes")
+            )
             (@subcommand env =>
              (about: "Describe the environment variables ucg uses.")
             )
@@ -380,6 +383,14 @@ fn converters_command(registry: &ConverterRegistry) {
     }
 }
 
+fn importers_command(registry: &ImporterRegistry) {
+    println!("Available importers");
+    println!("");
+    for (name, _importer) in registry.get_importer_list().iter() {
+        println!("- {}", name);
+    }
+}
+
 fn env_help() {
     println!("Universal Configuration Grammar compiler.");
     println!("");
@@ -419,6 +430,9 @@ fn main() {
         test_command(matches, &import_paths, cache, &registry, strict);
     } else if let Some(_) = app_matches.subcommand_matches("converters") {
         converters_command(&registry)
+    } else if let Some(_) = app_matches.subcommand_matches("importers") {
+        let registry = ImporterRegistry::make_registry();
+        importers_command(&registry)
     } else if let Some(_) = app_matches.subcommand_matches("env") {
         env_help()
     } else {