diff --git a/Cargo.lock b/Cargo.lock index b23746b..ad4eb89 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -41,6 +41,14 @@ dependencies = [ "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "base64" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "byteorder 1.2.7 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "bencher" version = "0.1.5" @@ -51,6 +59,11 @@ name = "bitflags" version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "byteorder" +version = "1.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "cc" version = "1.0.22" @@ -237,6 +250,7 @@ name = "ucg" version = "0.2.9" dependencies = [ "abortable_parser 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "base64 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", "bencher 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", "clap 2.26.2 (registry+https://github.com/rust-lang/crates.io-index)", "cpuprofiler 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)", @@ -305,8 +319,10 @@ dependencies = [ "checksum atty 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "9a7d5b8723950951411ee34d271d99dddcc2035a16ab25310ea2c8cfd4369652" "checksum backtrace 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "346d7644f0b5f9bc73082d3b2236b69a05fd35cce0cfa3724e184e6a5c9e2a2f" "checksum backtrace-sys 0.1.24 (registry+https://github.com/rust-lang/crates.io-index)" = "c66d56ac8dabd07f6aacdaf633f4b8262f5b3601a810a0dcddffd5c22c69daa0" +"checksum base64 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "621fc7ecb8008f86d7fb9b95356cd692ce9514b80a86d85b397f32a22da7b9e2" "checksum bencher 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "7dfdb4953a096c551ce9ace855a604d702e6e62d77fac690575ae347571717f5" "checksum bitflags 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4efd02e230a02e18f92fc2735f44597385ed02ad8f831e7c1c1156ee5e1ab3a5" +"checksum byteorder 1.2.7 (registry+https://github.com/rust-lang/crates.io-index)" = "94f88df23a25417badc922ab0f5716cc1330e87f71ddd9203b3a3ccd9cedf75d" "checksum cc 1.0.22 (registry+https://github.com/rust-lang/crates.io-index)" = "4a6007c146fdd28d4512a794b07ffe9d8e89e6bf86e2e0c4ddff2e1fb54a0007" "checksum cfg-if 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "0c4e7bb64a8ebb0d856483e1e682ea3422f883c5f5615a90d51a2c82fe87fdd3" "checksum clap 2.26.2 (registry+https://github.com/rust-lang/crates.io-index)" = "3451e409013178663435d6f15fdb212f14ee4424a3d74f979d081d0a66b6f1f2" diff --git a/Cargo.toml b/Cargo.toml index 8093215..f60ecc5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,6 +18,7 @@ simple-error = "0.1" serde_yaml = "~0.8.1" toml = "~0.4.8" xml-rs = "0.8.0" +base64 = "0.10.0" [dev-dependencies] bencher = "~0.1.5" diff --git a/integration_tests/include_test.ucg b/integration_tests/include_test.ucg index b5e8879..8066613 100644 --- a/integration_tests/include_test.ucg +++ b/integration_tests/include_test.ucg @@ -2,4 +2,13 @@ let script = include str "./include_example.sh"; assert | script == "#!/usr/bin/env bash echo \"included\""; +|; +let expected = "IyEvdXNyL2Jpbi9lbnYgYmFzaAplY2hvICJpbmNsdWRlZCI="; +let base64 = include b64 "./include_example.sh"; +assert | + base64 == expected; +|; +let base64 = include b64urlsafe "./include_example.sh"; +assert | + base64 == expected; |; \ No newline at end of file diff --git a/src/build/mod.rs b/src/build/mod.rs index 690181b..229410c 100644 --- a/src/build/mod.rs +++ b/src/build/mod.rs @@ -29,6 +29,7 @@ use simple_error; use crate::ast::*; use crate::build::scope::{find_in_fieldlist, Scope, ValueMap}; +use crate::convert::ImporterRegistry; use crate::error; use crate::format; use crate::iter::OffsetStrIter; @@ -100,6 +101,7 @@ pub struct FileBuilder<'a> { pub assert_collector: AssertCollector, strict: bool, scope: Scope, + import_registry: ImporterRegistry, // NOTE(jwall): We use interior mutability here because we need // our asset cache to be shared by multiple different sub-builders. // We use Rc to handle the reference counting for us and we use @@ -166,6 +168,7 @@ impl<'a> FileBuilder<'a> { }, scope: scope, strict: true, + import_registry: ImporterRegistry::make_registry(), assets: cache, out_lock: None, is_module: false, @@ -185,6 +188,8 @@ impl<'a> FileBuilder<'a> { }, strict: true, assets: self.assets.clone(), + // This is admittedly a little wasteful but we can live with it for now. + import_registry: ImporterRegistry::make_registry(), scope: self.scope.spawn_clean(), out_lock: None, is_module: false, @@ -335,7 +340,7 @@ impl<'a> FileBuilder<'a> { } fn find_file>( - &mut self, + &self, path: P, use_import_path: bool, ) -> Result> { @@ -1194,39 +1199,52 @@ impl<'a> FileBuilder<'a> { Ok(ok) } + fn get_file_as_string(&self, pos: &Position, path: &str) -> Result> { + let normalized = match self.find_file(path, false) { + Ok(p) => p, + Err(e) => { + return Err(Box::new(error::BuildError::new( + format!("Error finding file {} {}", path, e), + error::ErrorType::TypeFail, + pos.clone(), + ))) + } + }; + let mut f = match File::open(&normalized) { + Ok(f) => f, + Err(e) => { + return Err(Box::new(error::BuildError::new( + format!("Error opening file {} {}", normalized.to_string_lossy(), e), + error::ErrorType::TypeFail, + pos.clone(), + ))) + } + }; + let mut contents = String::new(); + f.read_to_string(&mut contents)?; + Ok(contents) + } + pub fn eval_include(&mut self, def: &IncludeDef) -> Result, Box> { return if def.typ.fragment == "str" { - let normalized = match self.find_file(&def.path.fragment, false) { - Ok(p) => p, - Err(e) => { - return Err(Box::new(error::BuildError::new( - format!("Error finding file {} {}", def.path.fragment, e), - error::ErrorType::TypeFail, - def.typ.pos.clone(), - ))) - } - }; - let mut f = match File::open(&normalized) { - Ok(f) => f, - Err(e) => { - return Err(Box::new(error::BuildError::new( - format!("Error opening file {} {}", normalized.to_string_lossy(), e), - error::ErrorType::TypeFail, - def.typ.pos.clone(), - ))) - } - }; - let mut contents = String::new(); - f.read_to_string(&mut contents)?; - Ok(Rc::new(Val::Str(contents))) - } else { - // TODO(jwall): Run the conversion on the contents of the file and return it as - // an Rc. - Err(Box::new(error::BuildError::new( - format!("Unknown include conversion type {}", def.typ.fragment), - error::ErrorType::Unsupported, - def.typ.pos.clone(), + Ok(Rc::new(Val::Str( + self.get_file_as_string(&def.path.pos, &def.path.fragment)?, ))) + } else { + let maybe_importer = self.import_registry.get_importer(&def.typ.fragment); + match maybe_importer { + Some(importer) => { + let file_contents = + self.get_file_as_string(&def.path.pos, &def.path.fragment)?; + let val = importer.import(file_contents.as_bytes())?; + Ok(val) + } + None => Err(Box::new(error::BuildError::new( + format!("Unknown include conversion type {}", def.typ.fragment), + error::ErrorType::Unsupported, + def.typ.pos.clone(), + ))), + } }; } diff --git a/src/convert/b64.rs b/src/convert/b64.rs new file mode 100644 index 0000000..3d86a4d --- /dev/null +++ b/src/convert/b64.rs @@ -0,0 +1,23 @@ +use std::error::Error; +use std::rc::Rc; +use std::result::Result; + +use base64::{encode, encode_config, URL_SAFE}; + +use crate::build::Val; +use crate::convert::traits::Importer; + +pub struct Base64Importer { + pub url_safe: bool, +} + +impl Importer for Base64Importer { + fn import(&self, bytes: &[u8]) -> Result, Box> { + let bslice = bytes.into(); + return if self.url_safe { + Ok(Rc::new(Val::Str(encode(bslice)))) + } else { + Ok(Rc::new(Val::Str(encode_config(bslice, URL_SAFE)))) + }; + } +} diff --git a/src/convert/mod.rs b/src/convert/mod.rs index 9b4947f..ff62d63 100644 --- a/src/convert/mod.rs +++ b/src/convert/mod.rs @@ -13,6 +13,7 @@ // limitations under the License. //! The conversion stage of the ucg compiler. +pub mod b64; pub mod env; pub mod exec; pub mod flags; @@ -26,7 +27,7 @@ use std::collections::HashMap; /// ConverterRunner knows how to run a given converter on a Val. pub struct ConverterRegistry { - converters: HashMap>, + converters: HashMap>, } impl ConverterRegistry { @@ -68,3 +69,45 @@ impl ConverterRegistry { // TODO(jwall): Support converter help descriptions. } + +pub struct ImporterRegistry { + importers: HashMap>, +} + +impl ImporterRegistry { + /// new creates a new ConverterRunner with a converter for the provided output target. + /// + /// * flags + /// * json + /// * env + /// * exec + fn new() -> Self { + ImporterRegistry { + importers: HashMap::new(), + } + } + + pub fn make_registry() -> Self { + let mut registry = Self::new(); + registry.register("b64", Box::new(b64::Base64Importer { url_safe: false })); + registry.register( + "b64urlsafe", + Box::new(b64::Base64Importer { url_safe: true }), + ); + registry + } + + pub fn register>(&mut self, typ: S, importer: Box) { + self.importers.insert(typ.into(), importer); + } + + pub fn get_importer(&self, typ: &str) -> Option<&dyn traits::Importer> { + self.importers.get(typ).map(|c| c.as_ref()) + } + + pub fn get_importer_list(&self) -> Vec<(&String, &Box)> { + self.importers.iter().collect() + } + + // TODO(jwall): Support converter help descriptions. +} diff --git a/src/convert/traits.rs b/src/convert/traits.rs index caef4fb..91f930a 100644 --- a/src/convert/traits.rs +++ b/src/convert/traits.rs @@ -29,3 +29,7 @@ pub trait Converter { fn file_ext(&self) -> String; fn description(&self) -> String; } + +pub trait Importer { + fn import(&self, bytes: &[u8]) -> result::Result, Box>; +} diff --git a/src/lib.rs b/src/lib.rs index 6987339..493171e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -22,6 +22,7 @@ #![recursion_limit = "128"] #[macro_use] extern crate abortable_parser; +extern crate base64; extern crate serde_json; extern crate serde_yaml; extern crate simple_error; diff --git a/src/main.rs b/src/main.rs index 5d3f327..10ac7cd 100644 --- a/src/main.rs +++ b/src/main.rs @@ -27,7 +27,7 @@ use ucglib::build; use ucglib::build::assets::{Cache, MemoryCache}; use ucglib::build::Val; use ucglib::convert::traits; -use ucglib::convert::ConverterRegistry; +use ucglib::convert::{ConverterRegistry, ImporterRegistry}; fn do_flags<'a, 'b>() -> clap::App<'a, 'b> { clap_app!( @@ -55,6 +55,9 @@ fn do_flags<'a, 'b>() -> clap::App<'a, 'b> { (@subcommand converters => (about: "list the available converters") ) + (@subcommand importers => + (about: "list the available importers for includes") + ) (@subcommand env => (about: "Describe the environment variables ucg uses.") ) @@ -380,6 +383,14 @@ fn converters_command(registry: &ConverterRegistry) { } } +fn importers_command(registry: &ImporterRegistry) { + println!("Available importers"); + println!(""); + for (name, _importer) in registry.get_importer_list().iter() { + println!("- {}", name); + } +} + fn env_help() { println!("Universal Configuration Grammar compiler."); println!(""); @@ -419,6 +430,9 @@ fn main() { test_command(matches, &import_paths, cache, ®istry, strict); } else if let Some(_) = app_matches.subcommand_matches("converters") { converters_command(®istry) + } else if let Some(_) = app_matches.subcommand_matches("importers") { + let registry = ImporterRegistry::make_registry(); + importers_command(®istry) } else if let Some(_) = app_matches.subcommand_matches("env") { env_help() } else {