zaphar/offline-web
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

wip: switch to the superior indexed-db crate

Browse Source
This commit is contained in:
Jeremy Wall 2025-07-24 21:05:04 -04:00
parent b486cb3d53
commit b43ca2e1f0
5 changed files with 224 additions and 353 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
CLAUDE.md
View File

@ -25,10 +25,10 @@ The framework uses a content-addressable system where:
- Build individual crates: `make model-native`, `make storage-native`, `make model-wasm`, `make storage-wasm`
## Testing
- Unit tests: `cargo test` (for native features)
- Model tests: `cargo test -p offline-web-model --features native`
- Storage integration tests: `cargo test -p offline-web-storage --features native`
- WASM builds require `--target=wasm32-unknown-unknown` and `--features wasm`
- Unit tests: `make test` (for native features)
- Model tests: `make test-model`
- Storage integrations tests: `make test-storage-native test-storage-wasm`
- Cargo WASM builds require `--target=wasm32-unknown-unknown` and `--features wasm`
## Feature Flags
Both core crates use conditional compilation:

60
Cargo.lock generated
View File

@ -586,6 +586,17 @@ version = "0.3.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6"
[[package]]
name = "futures-macro"
version = "0.3.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.100",
]
[[package]]
name = "futures-sink"
version = "0.3.31"
@ -606,6 +617,7 @@ checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81"
dependencies = [
"futures-core",
"futures-io",
"futures-macro",
"futures-sink",
"futures-task",
"memchr",
@ -913,20 +925,6 @@ dependencies = [
"zerovec",
]
[[package]]
name = "idb"
version = "0.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3afe8830d5802f769dc0be20a87f9f116798c896650cb6266eb5c19a3c109eed"
dependencies = [
"js-sys",
"num-traits",
"thiserror 1.0.69",
"tokio",
"wasm-bindgen",
"web-sys",
]
[[package]]
name = "idna"
version = "1.0.3"
@ -948,6 +946,20 @@ dependencies = [
"icu_properties",
]
[[package]]
name = "indexed-db"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78f4ecbb6cd50773303683617a93fc2782267d2c94546e9545ec4190eb69aa1a"
dependencies = [
"futures-channel",
"futures-util",
"pin-project-lite",
"scoped-tls",
"thiserror 2.0.12",
"web-sys",
]
[[package]]
name = "indexmap"
version = "2.10.0"
@ -1199,9 +1211,10 @@ dependencies = [
"anyhow",
"blake2",
"chrono",
"idb",
"indexed-db",
"offline-web-model",
"serde",
"serde-wasm-bindgen",
"serde_json",
"sqlx",
"thiserror 2.0.12",
@ -1530,6 +1543,12 @@ dependencies = [
"winapi-util",
]
[[package]]
name = "scoped-tls"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294"
[[package]]
name = "scopeguard"
version = "1.2.0"
@ -1555,6 +1574,17 @@ dependencies = [
"serde_derive",
]
[[package]]
name = "serde-wasm-bindgen"
version = "0.6.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8302e169f0eddcc139c70f139d19d6467353af16f9fce27e8c30158036a1e16b"
dependencies = [
"js-sys",
"serde",
"wasm-bindgen",
]
[[package]]
name = "serde_derive"
version = "1.0.219"

5
offline-web-storage/Cargo.toml
View File

@ -16,7 +16,8 @@ chrono = { version = "0.4", features = ["serde"] }
blake2 = "0.10"
thiserror = "2.0.12"
wasm-bindgen = { version = "0.2.100", optional=true }
idb = { version = "0.6.1", optional=true }
indexed-db = { version = "0.4.2", optional=true }
serde-wasm-bindgen = { version = "0.6", optional=true }
web-sys = { version = "0.3", features = ["console"], optional=true }
[dev-dependencies]
@ -28,4 +29,4 @@ crate-type = ["cdylib", "rlib"]
[features]
native = ["sqlx", "tokio", "tokio-test"]
wasm = ["offline-web-model/wasm", "uuid/js", "idb", "wasm-bindgen", "web-sys"]
wasm = ["offline-web-model/wasm", "uuid/js", "indexed-db", "serde-wasm-bindgen", "wasm-bindgen", "web-sys"]

425
offline-web-storage/src/indexeddb.rs
View File

@ -1,12 +1,9 @@
use std::collections::{HashSet, VecDeque};
use std::sync::Arc;
use idb::{
Database, DatabaseEvent, Factory, KeyPath, ObjectStore, ObjectStoreParams, Query, TransactionMode,
};
use indexed_db::{Database, Factory};
use offline_web_model::Reference;
use serde::{Deserialize, Serialize};
use wasm_bindgen::JsValue;
use crate::ReferenceStore;
use crate::StoreError;
@ -37,146 +34,119 @@ struct ContentEntry {
}
pub struct IndexedDbReferenceStore {
db: Database,
db: Database<StoreError>,
}
impl IndexedDbReferenceStore {
pub async fn new() -> Result<Self, StoreError> {
let factory = Factory::new()?;
let factory = Factory::<StoreError>::get()?;
let mut open_request = factory.open(DB_NAME, Some(DB_VERSION))?;
// Set up database upgrade handler
open_request.on_upgrade_needed(|event| {
let db = event.database().expect("Failed to get indexeddb database");
let db = factory
.open(DB_NAME, DB_VERSION, |event| async move {
let db = event.database();
// Create ref_entries object store
if !db
.store_names()
.iter()
.any(|name| name == REF_ENTRIES_STORE)
{
let ref_store = db.create_object_store(REF_ENTRIES_STORE, ObjectStoreParams::new())
.expect(&format!(
"Failed to create ref_entries store: {:?}",
REF_ENTRIES_STORE
));
let ref_store = db
.build_object_store(REF_ENTRIES_STORE)
.create()?;
// Create name index for get_graph method
ref_store.create_index("name", KeyPath::new_single("name"), None)
.expect("Failed to create name index");
}
ref_store
.build_index("name", "name")
.create()?;
// Create ref_dependencies object store
if !db
.store_names()
.iter()
.any(|name| name == REF_DEPENDENCIES_STORE)
{
db.create_object_store(REF_DEPENDENCIES_STORE, ObjectStoreParams::new())
.expect(&format!(
"Failed to create ref_dependencies store: {:?}",
REF_DEPENDENCIES_STORE
));
}
db.build_object_store(REF_DEPENDENCIES_STORE)
.create()?;
// Create content_store object store
if !db.store_names().iter().any(|name| name == CONTENT_STORE) {
db.create_object_store(CONTENT_STORE, ObjectStoreParams::new())
.expect(&format!(
"Failed to create content_store: {:?}",
CONTENT_STORE
));
}
});
db.build_object_store(CONTENT_STORE)
.create()?;
let db = open_request.await?;
Ok(())
})
.await?;
Ok(Self { db })
}
async fn get_reference_without_dependents(&self, id: &str) -> Result<Reference, StoreError> {
let id = id.to_string();
let reference = self.db
.transaction(&[REF_ENTRIES_STORE])
.run(|transaction| async move {
let store = transaction.object_store(REF_ENTRIES_STORE)?;
async fn clear_dependencies_sync(
&self,
dep_store: &ObjectStore,
parent_id: &str,
) -> Result<(), StoreError> {
let cursor_request = dep_store.open_cursor(None, None)?;
let mut keys_to_delete: Vec<String> = Vec::new();
let id_value = serde_wasm_bindgen::to_value(&id)
.map_err(|e| {
indexed_db::Error::User(StoreError::SerializationError(Box::new(e)))
})?;
let value = store.get(&id_value).await?;
if let Ok(Some(cursor)) = cursor_request.await {
loop {
let should_continue = match cursor.value() {
Ok(value) => {
if let Some(dep_str) = value.as_string() {
if let Ok(dep_entry) = serde_json::from_str::<RefDependency>(&dep_str) {
if let Some(js_value) = value {
let ref_entry: RefEntry = serde_wasm_bindgen::from_value(js_value)
.map_err(|e| {
indexed_db::Error::User(StoreError::SerializationError(Box::new(e)))
})?;
Ok(Reference {
id: ref_entry.id,
content_address: ref_entry.content_address,
name: ref_entry.name,
dependents: Vec::new(),
})
} else {
Err(indexed_db::Error::User(StoreError::NoSuchReference))
}
})
.await?;
Ok(reference)
}
async fn get_direct_children(&self, parent_id: &str) -> Result<Vec<String>, StoreError> {
let parent_id = parent_id.to_string();
self.db
.transaction(&[REF_DEPENDENCIES_STORE])
.run(|transaction| async move {
let store = transaction.object_store(REF_DEPENDENCIES_STORE)?;
let mut children = Vec::new();
let mut cursor = store.cursor().open().await?;
while let Some(value) = cursor.value() {
if let Ok(dep_entry) = serde_wasm_bindgen::from_value::<RefDependency>(value) {
if dep_entry.parent_id == parent_id {
if let Ok(key) = cursor.key() {
if let Some(key_str) = key.as_string() {
keys_to_delete.push(key_str);
children.push(dep_entry.dependent_id);
}
}
}
}
}
true
}
Err(_) => false,
};
if !should_continue {
break;
cursor.advance(1).await?;
}
// Try to continue to next item
match cursor.advance(1) {
Ok(advance_request) => {
if advance_request.await.is_err() {
break;
}
if cursor.value().is_err() {
break;
}
}
Err(_) => break,
}
}
}
// Delete the found keys
for key in keys_to_delete {
let delete_request = dep_store.delete(JsValue::from_str(&key))?;
delete_request.await?;
}
Ok(())
Ok(children)
})
.await.map_err(StoreError::from)
}
async fn get_dependents(&self, parent_id: &str) -> Result<Vec<Arc<Reference>>, StoreError> {
// Use a completely iterative approach to build the dependency tree
let mut all_refs = std::collections::HashMap::new();
let mut dependency_map = std::collections::HashMap::<String, Vec<String>>::new();
let mut to_process = std::collections::VecDeque::new();
let mut processed = std::collections::HashSet::new();
// Start with direct children of the parent
let direct_children = self.get_direct_children(parent_id).await?;
for child_id in direct_children.clone() {
to_process.push_back(child_id);
}
// Process all references iteratively to build both refs map and dependency map
while let Some(ref_id) = to_process.pop_front() {
if processed.contains(&ref_id) {
continue;
}
processed.insert(ref_id.clone());
// Get the reference without dependents first
if let Ok(reference) = self.get_reference_without_dependents(&ref_id).await {
all_refs.insert(ref_id.clone(), reference);
// Get its direct children and add them to the processing queue
if let Ok(children) = self.get_direct_children(&ref_id).await {
dependency_map.insert(ref_id.clone(), children.clone());
for child_id in children {
@ -188,7 +158,6 @@ impl IndexedDbReferenceStore {
}
}
// Now build the dependency tree iteratively using the maps
let mut result = Vec::new();
for child_id in direct_children {
if let Some(reference) = self.build_full_reference(&child_id, &all_refs, &dependency_map) {
@ -209,7 +178,6 @@ impl IndexedDbReferenceStore {
let mut to_build = std::collections::VecDeque::new();
let mut processed = std::collections::HashSet::new();
// Start from leaves and work up
to_build.push_back(target_id.to_string());
while let Some(ref_id) = to_build.pop_back() {
@ -220,11 +188,9 @@ impl IndexedDbReferenceStore {
let base_ref = all_refs.get(&ref_id)?;
let children = dependency_map.get(&ref_id).cloned().unwrap_or_default();
// Check if all children are already built
let all_children_built = children.iter().all(|child_id| built_refs.contains_key(child_id));
if all_children_built {
// Build this reference with its dependents
let mut dependents = Vec::new();
for child_id in &children {
if let Some(child_ref) = built_refs.get(child_id) {
@ -242,7 +208,6 @@ impl IndexedDbReferenceStore {
built_refs.insert(ref_id.clone(), full_ref);
processed.insert(ref_id);
} else {
// Add children to be built first, then re-add this ref
to_build.push_front(ref_id);
for child_id in children {
if !processed.contains(&child_id) && !built_refs.contains_key(&child_id) {
@ -256,127 +221,26 @@ impl IndexedDbReferenceStore {
}
async fn find_reference_by_name(&self, name: &str) -> Result<Option<RefEntry>, StoreError> {
let transaction = self
.db
.transaction(&[REF_ENTRIES_STORE], TransactionMode::ReadOnly)?;
let name = name.to_string();
let ref_store = transaction.object_store(REF_ENTRIES_STORE)?;
self.db
.transaction(&[REF_ENTRIES_STORE])
.run(|transaction| async move {
let store = transaction.object_store(REF_ENTRIES_STORE)?;
let mut cursor = store.cursor().open().await?;
// Use cursor to search through all references
let cursor_request = ref_store.open_cursor(None, None)?;
if let Ok(Some(cursor)) = cursor_request.await {
loop {
let should_continue = match cursor.value() {
Ok(value) => {
if let Some(ref_str) = value.as_string() {
if let Ok(ref_entry) = serde_json::from_str::<RefEntry>(&ref_str) {
while let Some(value) = cursor.value() {
if let Ok(ref_entry) = serde_wasm_bindgen::from_value::<RefEntry>(value) {
if ref_entry.name == name {
return Ok(Some(ref_entry));
}
}
}
true
}
Err(_) => false,
};
if !should_continue {
break;
}
// Try to continue to next item
match cursor.advance(1) {
Ok(advance_request) => {
if advance_request.await.is_err() {
break;
}
if cursor.value().is_err() {
break;
}
}
Err(_) => break,
}
}
cursor.advance(1).await?;
}
Ok(None)
}
async fn get_reference_without_dependents(&self, id: &str) -> Result<Reference, StoreError> {
let transaction = self
.db
.transaction(&[REF_ENTRIES_STORE], TransactionMode::ReadOnly)?;
let ref_store = transaction.object_store(REF_ENTRIES_STORE)?;
let request = ref_store.get(Query::Key(JsValue::from_str(id)))?;
let value = request.await?;
if let Some(value) = value {
if let Some(ref_str) = value.as_string() {
if let Ok(ref_entry) = serde_json::from_str::<RefEntry>(&ref_str) {
return Ok(Reference {
id: ref_entry.id,
content_address: ref_entry.content_address,
name: ref_entry.name,
dependents: Vec::new(), // No dependents in this method
});
}
}
}
Err(StoreError::NoSuchReference)
}
async fn get_direct_children(&self, parent_id: &str) -> Result<Vec<String>, StoreError> {
let transaction = self
.db
.transaction(&[REF_DEPENDENCIES_STORE], TransactionMode::ReadOnly)?;
let dep_store = transaction.object_store(REF_DEPENDENCIES_STORE)?;
let mut children = Vec::new();
let cursor_request = dep_store.open_cursor(None, None)?;
if let Ok(Some(cursor)) = cursor_request.await {
loop {
let should_continue = match cursor.value() {
Ok(value) => {
if let Some(dep_str) = value.as_string() {
if let Ok(dep_entry) = serde_json::from_str::<RefDependency>(&dep_str) {
if dep_entry.parent_id == parent_id {
children.push(dep_entry.dependent_id);
}
}
}
true
}
Err(_) => false,
};
if !should_continue {
break;
}
// Try to continue to next item
match cursor.advance(1) {
Ok(advance_request) => {
// Wait for the advance operation, but if it fails, we're done
if advance_request.await.is_err() {
break;
}
// Check if cursor is still valid
if cursor.value().is_err() {
break;
}
}
Err(_) => break,
}
}
}
Ok(children)
})
.await.map_err(StoreError::from)
}
}
@ -388,50 +252,27 @@ impl ReferenceStore for IndexedDbReferenceStore {
}
async fn get_content_for_reference(&self, reference: Reference) -> Result<String, StoreError> {
if let Some(content_address) = &reference.content_address {
let transaction = self
.db
.transaction(&[CONTENT_STORE], TransactionMode::ReadOnly)
.map_err(|e| {
StoreError::StorageError(Box::new(std::io::Error::new(
std::io::ErrorKind::Other,
format!("Failed to create transaction: {:?}", e),
)))
})?;
if let Some(content_address) = reference.content_address {
self.db
.transaction(&[CONTENT_STORE])
.run(|transaction| async move {
let store = transaction.object_store(CONTENT_STORE)?;
let content_store = transaction.object_store(CONTENT_STORE).map_err(|e| {
StoreError::StorageError(Box::new(std::io::Error::new(
std::io::ErrorKind::Other,
format!("Failed to get content_store: {:?}", e),
)))
})?;
let content_key = serde_wasm_bindgen::to_value(&content_address)
.map_err(|e| indexed_db::Error::User(StoreError::SerializationError(Box::new(e))))?;
let value = store.get(&content_key).await?;
let request = content_store
.get(Query::Key(JsValue::from_str(content_address)))
.map_err(|e| {
StoreError::StorageError(Box::new(std::io::Error::new(
std::io::ErrorKind::Other,
format!("Failed to get content: {:?}", e),
)))
})?;
if let Some(js_value) = value {
let content_entry: ContentEntry = serde_wasm_bindgen::from_value(js_value)
.map_err(|e| indexed_db::Error::User(StoreError::SerializationError(Box::new(e))))?;
let value = request.await.map_err(|e| {
StoreError::StorageError(Box::new(std::io::Error::new(
std::io::ErrorKind::Other,
format!("Failed to await content: {:?}", e),
)))
})?;
if let Some(value) = value {
if let Some(content_str) = value.as_string() {
if let Ok(content_entry) = serde_json::from_str::<ContentEntry>(&content_str) {
return String::from_utf8(content_entry.content)
.map_err(|e| StoreError::StorageError(Box::new(e)));
String::from_utf8(content_entry.content)
.map_err(|e| indexed_db::Error::User(StoreError::StorageError(Box::new(e))))
} else {
Err(indexed_db::Error::User(StoreError::NoSuchContentAddress))
}
}
}
Err(StoreError::NoSuchContentAddress)
})
.await.map_err(StoreError::from)
} else {
Err(StoreError::NoSuchContentAddress)
}
@ -451,7 +292,6 @@ impl ReferenceStore for IndexedDbReferenceStore {
visited.insert(current_name.clone());
// Find reference by name - use a cursor to search through all references
if let Ok(reference_opt) = self.find_reference_by_name(&current_name).await {
if let Some(ref_entry) = reference_opt {
let dependents = self.get_dependents(&ref_entry.id).await?;
@ -465,7 +305,6 @@ impl ReferenceStore for IndexedDbReferenceStore {
result.push(reference);
// Add dependent names to queue
for dependent in dependents {
if !visited.contains(&dependent.name) {
queue.push_back(dependent.name.clone());
@ -479,13 +318,12 @@ impl ReferenceStore for IndexedDbReferenceStore {
}
async fn store_reference(&self, reference: &Reference) -> Result<(), StoreError> {
let transaction = self
.db
.transaction(
&[REF_ENTRIES_STORE, REF_DEPENDENCIES_STORE],
TransactionMode::ReadWrite,
)?;
let reference = reference.clone();
self.db
.transaction(&[REF_ENTRIES_STORE, REF_DEPENDENCIES_STORE])
.rw()
.run(|transaction| async move {
let ref_store = transaction.object_store(REF_ENTRIES_STORE)?;
let dep_store = transaction.object_store(REF_DEPENDENCIES_STORE)?;
@ -496,15 +334,11 @@ impl ReferenceStore for IndexedDbReferenceStore {
name: reference.name.clone(),
};
let ref_value = serde_json::to_string(&ref_entry)?;
let ref_request = ref_store.put(
&JsValue::from_str(&ref_value),
Some(&JsValue::from_str(&reference.id)),
)?;
ref_request.await?;
// Clear existing dependencies for this reference
self.clear_dependencies_sync(&dep_store, &reference.id).await?;
let ref_value = serde_wasm_bindgen::to_value(&ref_entry)
.map_err(|e| indexed_db::Error::User(StoreError::SerializationError(Box::new(e))))?;
let ref_key = serde_wasm_bindgen::to_value(&reference.id)
.map_err(|e| indexed_db::Error::User(StoreError::SerializationError(Box::new(e))))?;
ref_store.add_kv(&ref_key, &ref_value).await?;
// Store new dependencies
for dependent in &reference.dependents {
@ -513,45 +347,42 @@ impl ReferenceStore for IndexedDbReferenceStore {
dependent_id: dependent.id.clone(),
};
let dep_value = serde_json::to_string(&dep_entry)?;
let dep_key = format!("{}:{}", reference.id, dependent.id);
let dep_value = serde_wasm_bindgen::to_value(&dep_entry)
.map_err(|e| indexed_db::Error::User(StoreError::SerializationError(Box::new(e))))?;
let dep_key = serde_wasm_bindgen::to_value(&format!("{}:{}", reference.id, dependent.id))
.map_err(|e| indexed_db::Error::User(StoreError::SerializationError(Box::new(e))))?;
let dep_request = dep_store.put(
&JsValue::from_str(&dep_value),
Some(&JsValue::from_str(&dep_key)),
)?;
dep_request.await?;
dep_store.add_kv(&dep_key, &dep_value).await?;
}
let commit_request = transaction.commit()?;
commit_request.await?;
Ok(())
})
.await.map_err(StoreError::from)
}
async fn store_content(&self, content_address: &str, content: &[u8]) -> Result<(), StoreError> {
let transaction = self
.db
.transaction(&[CONTENT_STORE], TransactionMode::ReadWrite)?;
let content_address = content_address.to_string();
let content = content.to_vec();
let content_store = transaction.object_store(CONTENT_STORE)?;
self.db
.transaction(&[CONTENT_STORE])
.rw()
.run(|transaction| async move {
let store = transaction.object_store(CONTENT_STORE)?;
let content_entry = ContentEntry {
content_address: content_address.to_string(),
content: content.to_vec(),
content_address: content_address.clone(),
content,
};
let content_value = serde_json::to_string(&content_entry)?;
let put_request = content_store.put(
&JsValue::from_str(&content_value),
Some(&JsValue::from_str(content_address)),
)?;
put_request.await?;
let commit_request = transaction.commit()?;
commit_request.await?;
let content_value = serde_wasm_bindgen::to_value(&content_entry)
.map_err(|e| indexed_db::Error::User(StoreError::SerializationError(Box::new(e))))?;
let content_key = serde_wasm_bindgen::to_value(&content_entry.content_address)
.map_err(|e| indexed_db::Error::User(StoreError::SerializationError(Box::new(e))))?;
store.add_kv(&content_key, &content_value).await?;
Ok(())
})
.await.map_err(StoreError::from)
}
}

17
offline-web-storage/src/lib.rs
View File

@ -16,10 +16,19 @@ pub enum StoreError {
}
#[cfg(feature="wasm")]
impl From<idb::Error> for StoreError {
fn from(value: idb::Error) -> Self {
// TODO(jwall): We can probably be more helpful in our error message here.
StoreError::StorageError(Box::new(value))
impl From<indexed_db::Error<StoreError>> for StoreError {
fn from(value: indexed_db::Error<StoreError>) -> Self {
match value {
indexed_db::Error::User(store_error) => store_error,
other => StoreError::StorageError(Box::new(other))
}
}
}
#[cfg(feature="wasm")]
impl From<serde_wasm_bindgen::Error> for StoreError {
fn from(value: serde_wasm_bindgen::Error) -> Self {
StoreError::SerializationError(Box::new(value))
}
}