diff --git a/Cargo.lock b/Cargo.lock index ae3ef42..82ff06e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -24,6 +24,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de45108900e1f9b9242f7f2e254aa3e2c029c921c258fe9e6b4217eeebd54288" dependencies = [ "axum-core", + "axum-macros", "bytes", "form_urlencoded", "futures-util", @@ -71,6 +72,17 @@ dependencies = [ "tracing", ] +[[package]] +name = "axum-macros" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "604fde5e028fea851ce1d8570bbdc034bec850d157f7569d10f347d06808c05c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "backtrace" version = "0.3.74" @@ -86,6 +98,30 @@ dependencies = [ "windows-targets", ] +[[package]] +name = "bitflags" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd" + +[[package]] +name = "blake2" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" +dependencies = [ + "digest", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "bytes" version = "1.10.1" @@ -98,6 +134,27 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", + "subtle", +] + [[package]] name = "fnv" version = "1.0.7" @@ -146,6 +203,28 @@ dependencies = [ "pin-utils", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73fea8450eea4bac3940448fb7ae50d91f034f941199fcd9d909a5a07aa455f0" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasi 0.14.2+wasi-0.2.4", +] + [[package]] name = "gimli" version = "0.31.1" @@ -285,7 +364,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd" dependencies = [ "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", "windows-sys", ] @@ -303,7 +382,10 @@ name = "offline-web" version = "0.1.0" dependencies = [ "axum", + "blake2", + "rand", "serde", + "tokio", ] [[package]] @@ -330,6 +412,15 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + [[package]] name = "proc-macro2" version = "1.0.94" @@ -348,6 +439,42 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "5.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" + +[[package]] +name = "rand" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3779b94aeb87e8bd4e834cee3650289ee9e0d5677f976ecdb6d219e5f4f6cd94" +dependencies = [ + "rand_chacha", + "rand_core", + "zerocopy", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" +dependencies = [ + "getrandom", +] + [[package]] name = "rustc-demangle" version = "0.1.24" @@ -436,6 +563,12 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + [[package]] name = "syn" version = "2.0.100" @@ -527,18 +660,39 @@ dependencies = [ "once_cell", ] +[[package]] +name = "typenum" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" + [[package]] name = "unicode-ident" version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasi" +version = "0.14.2+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" +dependencies = [ + "wit-bindgen-rt", +] + [[package]] name = "windows-sys" version = "0.52.0" @@ -611,3 +765,32 @@ name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "wit-bindgen-rt" +version = "0.39.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" +dependencies = [ + "bitflags", +] + +[[package]] +name = "zerocopy" +version = "0.8.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2586fea28e186957ef732a5f8b3be2da217d65c5969d4b1e17f973ebbe876879" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a996a8f63c5c4448cd959ac1bab0aaa3306ccfd060472f85943ee0750f0169be" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/Cargo.toml b/Cargo.toml index b2faf43..3dac612 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,16 @@ name = "offline-web" version = "0.1.0" edition = "2021" +[lib] +path = "src/lib.rs" + +[[bin]] +name = "example" +path = "src/main.rs" + [dependencies] -axum = "0.8.3" +axum = { version = "0.8.3", features = ["macros"] } +blake2 = "0.10.6" +rand = "0.9.0" serde = { version = "1.0.219", features = ["derive", "rc"] } +tokio = "1.44.1" diff --git a/DESIGN.md b/DESIGN.md index 6885137..981549f 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -21,11 +21,20 @@ Resource reference paths are rooted at `/api/v1/ref/`. ```json { - "objectId": , + "objectId": , + "content_address": , "path": "/path/name0", "dependents": [ - { "path": "path/name1", "objectId": , content: }, - { "path": "path/name2", "objectId": , content: } + { + "path": "path/name1", + "content_address": , + "objectId": , content: + }, + { + "path": "path/name2", + "content_address": , + "objectId": , content: + } ], } ``` @@ -49,17 +58,25 @@ content key. ```json { - "objectId": , + "objectId": , "path": "/path/name0", "dependents": [ { - "objectId": , + "objectId": , "path": "path/name1", "content": { - "objectId": , + "objectId": , "dependents": [ - { "path": "path/name1", "objectId": , content: }, - { "path": "path/name2", "objectId": , content: } + { + "path": "path/name1", + "content_address": , + "objectId": , content: + }, + { + "path": "path/name2", + "content_address": , + "objectId": , content: + } ], } } @@ -90,7 +107,7 @@ operations. ### Bootstrapping -* Load `/api/v1/resource/all/` and then follow the sub resources to +* Load `/api/v1/ref/all/` and then follow the sub resources to load the entire dataset locally making sure to keep the content-addresses around for comparison. diff --git a/src/datamodel/mod.rs b/src/datamodel/mod.rs index 3b146e9..ab22961 100644 --- a/src/datamodel/mod.rs +++ b/src/datamodel/mod.rs @@ -1,30 +1,36 @@ -use std::rc::Rc; +use std::sync::Arc; use serde::{Serialize, Deserialize}; -#[derive(Serialize, Deserialize)] +#[derive(Serialize, Deserialize, Debug)] pub struct Reference { - object_id: String, - path: String, + pub object_id: String, + pub content_address: String, + pub path: String, #[serde(skip_serializing_if = "Vec::is_empty")] - dependents: Vec>, + pub dependents: Vec>, } impl Reference { - pub fn new(object_id: String, path: String) -> Self { + pub fn new(object_id: String, content_address: String, path: String) -> Self { Self { object_id, + content_address, path, dependents: Vec::new(), } } - pub fn add_dep(mut self, dep: Rc) -> Self { + pub fn add_dep(mut self, dep: Arc) -> Self { self.dependents.push(dep); self } - pub fn to_rc(self) -> Rc { - Rc::new(self) + pub fn to_arc(self) -> Arc { + Arc::new(self) + } + + pub fn is_leaf(&self) -> bool { + return self.dependents.is_empty(); } } diff --git a/src/lib.rs b/src/lib.rs index 9175220..6bf8767 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,10 +1,139 @@ -use axum::{ - routing::get, - Router -}; +use std::{collections::HashMap, sync::Arc}; +use axum::{extract::Path, http, response::{Html, IntoResponse}, routing::get, Json, Router}; +use blake2::{Blake2b512, Digest}; +use rand::Rng; + +use datamodel::Reference; mod datamodel; -pub fn endpoints() -> Router { - Router::new().route("/api/v1/ref/all/username", get(|| async { "hello world" })) +#[derive(Debug)] +pub struct AddressableObject { + pub address: String, + pub content: String, +} + +fn random_object() -> AddressableObject { + let mut rng = rand::rng(); + let random_size = rng.random_range(50..=4096); + let random_string: String = (0..random_size) + .map(|_| rng.sample(rand::distr::Alphanumeric) as char) + .collect(); + + let mut hasher = Blake2b512::new(); + hasher.update(&random_string); + let hash = format!("{:x}", hasher.finalize()); + + AddressableObject { + address: hash, + content: random_string, + } +} + +fn random_references_and_objects() -> (Arc, Arc>>, Arc>) { + let path_root = String::from("ref/0"); + let mut objects = HashMap::new(); + let mut refs = HashMap::new(); + let mut root_ref = Reference::new( + "username:0".to_string(), + String::from("0"), + path_root.clone(), + ); + let mut root_hasher = Blake2b512::new(); + for i in 1..=10 { + let mut item_ref = Reference::new( + format!("item:{}", i), + format!("0:{}", i), + format!("/item/{}", i), + ); + let mut hasher = Blake2b512::new(); + for j in 1..=10 { + let object = random_object(); + hasher.update(&object.content); + let leaf_ref = Reference::new( + format!("item:{}:subitem:{}", i, j), + format!("{}", object.address), + format!("/item/{}/subitem/{}", i, j), + ).to_arc(); + item_ref = item_ref.add_dep(leaf_ref.clone()); + objects.insert(object.address.clone(), object); + hasher.update(&leaf_ref.content_address); + refs.insert(leaf_ref.path.clone(), leaf_ref); + } + let hash = format!("{:x}", hasher.finalize()); + item_ref.content_address = hash; + root_hasher.update(&item_ref.content_address); + let rc_ref = item_ref.to_arc(); + root_ref = root_ref.add_dep(rc_ref.clone()); + refs.insert(rc_ref.path.clone(), rc_ref); + } + root_ref.content_address = format!("{:x}", root_hasher.finalize()); + let rc_root = root_ref.to_arc(); + refs.insert(rc_root.path.clone(), rc_root.clone()); + dbg!(&objects); + (rc_root, Arc::new(refs), Arc::new(objects)) +} + +async fn all_references(root_ref: Arc) -> Json> { + Json(root_ref) +} + +async fn ref_path(refs: Arc>>, Path(path): Path) -> Json> { + let path = format!("/item/{}", path); + match refs.get(&path) { + Some(r) => Json(r.clone()), + None => todo!("Return a 404?"), + } +} + +async fn object_path(objects: Arc>, Path(path): Path) -> String { + dbg!(&path); + match objects.get(&path) { + Some(o) => o.content.clone(), + None => todo!("Return a 404?"), + } +} + +async fn get_client_js() -> impl IntoResponse { + ( + [(http::header::CONTENT_TYPE, "application/javascript")], + include_str!("../static/client.js"), + ) +} + +pub fn endpoints(root_ref: Arc, refs: Arc>>, objects: Arc>) -> Router { + Router::new().nest( + "/api/v1", + Router::new().nest( + "/ref", + Router::new() + .route("/all/username", get({ + let state = root_ref.clone(); + move || all_references(state) + })) + .route("/item/{*path}", get({ + let refs = refs.clone(); + move |path| ref_path(refs, path) + })) + ).nest( + "/object", + Router::new() + .route("/{addr}", get({ + let objects = objects.clone(); + move |path| object_path(objects, path) + })) + ), + ) + .route("/lib/client.js", get(get_client_js)) + .route("/ui/", get(|| async { Html(include_str!("../static/index.html")).into_response() })) +} + +// TODO(jwall): Javascript test script +pub async fn serve() { + // run our app with hyper, listening globally on port 3000 + let (root_ref, refs, objects) = random_references_and_objects(); + let listener = tokio::net::TcpListener::bind("127.0.0.1:3000") + .await + .unwrap(); + axum::serve(listener, endpoints(root_ref, refs, objects)).await.unwrap(); } diff --git a/src/main.rs b/src/main.rs new file mode 100644 index 0000000..7a64137 --- /dev/null +++ b/src/main.rs @@ -0,0 +1,5 @@ + +#[tokio::main(flavor = "current_thread")] +async fn main() { + offline_web::serve().await; +} diff --git a/static/client.js b/static/client.js new file mode 100644 index 0000000..82de876 --- /dev/null +++ b/static/client.js @@ -0,0 +1,155 @@ +export { bootstrap }; + +/** + * @typedef {Object} Reference + * @property {Array} dependents + * @property {string} path + * @property {string} object_id + * @property {string} content_address + */ + +async function load_bootstrap() { + let response = await fetch("/api/v1/ref/all/username"); + if (!response.ok) { + throw new Error("Network response was not ok: " + response.statusText); + } + return await response.json(); +} + +/** + * @param {String} dbName + * @param {Array} storeNames + * @returns {Promise} + */ +async function openDatabase(dbName, storeNames) { + return await new Promise((resolve, reject) => { + const request = indexedDB.open(dbName, 1); + + request.onupgradeneeded = (event) => { + const db = event.target.result; + for (var storeName of storeNames) { + // Create the object store if it doesn't exist + if (!db.objectStoreNames.contains(storeName)) { + db.createObjectStore(storeName); + } + } + }; + + request.onsuccess = (event) => { + const db = event.target.result; + resolve(db); + }; + + request.onerror = (event) => { + reject(event.target.error); + }; + }); +} + +/** + * Stores a reference object in the IndexedDB. + * @param {IDBObjectStore} store + * @param {Object} reference + * @param {string} root_path + * @returns {Promise} + */ +function storeObject(store, reference, root_path) { + return new Promise((resolve, reject) => { + const request = store.put(JSON.stringify(reference), root_path); + request.onerror = (evt) => { + reject(evt.target.error); + console.log("Failed to store object", evt); + }; + request.onsuccess = (evt) => { + resolve(evt.target.result); + }; + }); +} + +/** + * @param {IDBObjectStore} refStore + * @param {Object} reference + * @returns {Promise>} An array of references +*/ +function load_reference_paths(refStore, reference) { + return new Promise(async (resolve, reject) => { + let references = []; + references.push(reference); + if (reference.dependents) { + for (var dep of reference.dependents) { + references = references.concat(await load_reference_paths(refStore, dep)); + } + } + + await storeObject(refStore, reference, reference.path); + resolve(references); + }); +} + +/** + * @param {IDBDatabase} db + * @param {string} storeName + * @param {Array} references +*/ +async function load_objects_and_store(db, references, storeName) { + let objects = [] + for (var ref of references) { + /** @type {Response} */ + if (ref.dependents && ref.dependents.length != 0) { + continue; // not a leaf object + } + let response = await fetch("/api/v1/object/" + ref.content_address); + if (!response.ok) { + throw new Error("Network response was not ok: " + response.statusText); + } + const object = await response.text(); + objects.push({ id: ref.content_address, content: object }); + } + const objectTrxAndStore = await getStoreAndTransaction(db, storeName); + for (var obj of objects) { + await storeObject(objectTrxAndStore.store, obj.content, obj.id); + } + await new Promise((resolve, reject) => { + objectTrxAndStore.trx.oncomplete = () => resolve(); + objectTrxAndStore.trx.onerror = (event) => reject(event.target.error); + }); +} + +/** + * @param {string} storeName + * @param {IDBDatabase} db + * @returns {Promise<{trx: IDBTransaction, store: IDBObjectStore}>} The transaction and object store. + */ +async function getStoreAndTransaction(db, storeName) { + const transaction = db.transaction([storeName], "readwrite"); + return { trx: transaction, store: transaction.objectStore(storeName) }; +} + +/** +* @returns {Number} The number of milliseconds it took to bootstrap. +*/ +async function bootstrap() { + const refStoreName = "references"; + const objectStoreName = "objects"; + const databaseName = "MerkleStore"; + const start = new Date().getTime(); + const root = await load_bootstrap(); + const db = await openDatabase(databaseName, [refStoreName, objectStoreName]); + const refTrxAndStore = await getStoreAndTransaction(db, refStoreName); + + // Use a promise to wait for the transaction to complete + const transactionComplete = new Promise((resolve, reject) => { + refTrxAndStore.trx.oncomplete = () => resolve(); + refTrxAndStore.trx.onerror = (event) => reject(event.target.error); + }); + + const refs = await load_reference_paths(refTrxAndStore.store, root); + + // Wait for the transaction to complete + await transactionComplete; + + await load_objects_and_store(db, refs, objectStoreName); + + const end = new Date().getTime(); + return end - start; +} diff --git a/static/index.html b/static/index.html new file mode 100644 index 0000000..321d1a4 --- /dev/null +++ b/static/index.html @@ -0,0 +1,26 @@ + + + + + + Bootstrap Time Display + + +

Bootstrap Time

+
Loading...
+ + + + +