diff --git a/src/dag.rs b/src/dag.rs index 81c0a95..7804369 100644 --- a/src/dag.rs +++ b/src/dag.rs @@ -12,10 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::{BTreeMap, BTreeSet}; +use std::{collections::BTreeSet, marker::PhantomData}; -use crate::hash::{ByteEncoder, HashWriter}; -use crate::node::Node; +use crate::{ + hash::{ByteEncoder, HashWriter}, + node::Node, + store::{Store, StoreError}, +}; /// Node comparison values. In a given Merkle DAG a Node can come `After`, `Before`, be `Equivalent`, or `Uncomparable`. /// If the two nodes have the same id they are eqivalent. If two nodes are not part of the same sub graph within the DAG @@ -29,11 +32,6 @@ pub enum NodeCompare { Uncomparable, } -#[derive(Debug)] -pub enum EdgeError { - NoSuchDependents, -} - /// A Merkle-DAG implementation. This is a modification on the standard Merkle Tree data structure /// but instead of a tree it is a DAG and as a result can have multiple roots. A merkle-dag specifies /// a partial ordering on all the nodes and utilizes the api to ensure that this ordering is @@ -46,19 +44,22 @@ pub enum EdgeError { /// A merkle DAG instance is tied to a specific implementation of the HashWriter interface to ensure /// that all hash identifiers are of the same hash algorithm. #[derive(Clone, Debug)] -pub struct Merkle +pub struct Merkle where N: ByteEncoder, HW: HashWriter, + S: Store, { roots: BTreeSet<[u8; HASH_LEN]>, - nodes: BTreeMap<[u8; HASH_LEN], Node>, + nodes: S, + _phantom_node: PhantomData>, } -impl Merkle +impl Merkle where N: ByteEncoder, HW: HashWriter, + S: Store, { /// Construct a new empty DAG. The empty DAG is also the default for a DAG. pub fn new() -> Self { @@ -75,31 +76,35 @@ where &'a mut self, item: N, dependency_ids: BTreeSet<[u8; HASH_LEN]>, - ) -> Result<[u8; HASH_LEN], EdgeError> { + ) -> Result<[u8; HASH_LEN], StoreError> { let node = Node::::new(item, dependency_ids.clone()); let id = node.id().clone(); - if self.nodes.contains_key(&id) { + if self.nodes.contains(&id) { // We've already added this node so there is nothing left to do. return Ok(id); } + let mut root_removals = Vec::new(); for dep_id in dependency_ids.iter() { - if !self.nodes.contains_key(dep_id) { - return Err(EdgeError::NoSuchDependents); + if !self.nodes.contains(dep_id) { + return Err(StoreError::NoSuchDependents); } // If any of our dependencies is in the roots pointer list then - // it is time to remove it from there. + // we need to remove it below. if self.roots.contains(dep_id) { - self.roots.remove(dep_id); + root_removals.push(dep_id); } } - self.roots.insert(id.clone()); - self.nodes.insert(id.clone(), node); + self.nodes.store(node)?; + for removal in root_removals { + self.roots.remove(removal); + } + self.roots.insert(id); Ok(id) } /// Check if we already have a copy of a node. pub fn check_for_node(&self, id: &[u8; HASH_LEN]) -> bool { - return self.nodes.contains_key(id); + return self.nodes.contains(id); } /// Get a node from the DAG by it's hash identifier if it exists. @@ -113,7 +118,7 @@ where } /// Get the map of all nodes in the DAG. - pub fn get_nodes(&self) -> &BTreeMap<[u8; HASH_LEN], Node> { + pub fn get_nodes(&self) -> &S { &self.nodes } @@ -166,15 +171,17 @@ where } } -impl Default for Merkle +impl Default for Merkle where N: ByteEncoder, HW: HashWriter, + S: Store, { fn default() -> Self { Self { roots: BTreeSet::new(), - nodes: BTreeMap::new(), + nodes: S::default(), + _phantom_node: Default::default(), } } } diff --git a/src/lib.rs b/src/lib.rs index b21026f..dfdbbc8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -15,6 +15,7 @@ pub mod dag; pub mod hash; pub mod node; pub mod prelude; +pub mod store; #[cfg(test)] mod test; diff --git a/src/proptest.rs b/src/proptest.rs new file mode 100644 index 0000000..4598dca --- /dev/null +++ b/src/proptest.rs @@ -0,0 +1,133 @@ +// Copyright 2022 Jeremy Wall (Jeremy@marzhilsltudios.com) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +use std::collections::{hash_map::DefaultHasher, BTreeMap, BTreeSet}; + +use proptest::prelude::*; + +use crate::prelude::*; + +type TestDag = Merkle>, String, DefaultHasher, 8>; + +fn simple_edge_strategy( + nodes_count: usize, +) -> impl Strategy, BTreeSet)> { + prop::collection::vec(".*", 4..nodes_count).prop_flat_map(|payloads| { + let nodes_len = payloads.len(); + ( + // our total list of nodes. + Just(payloads), + // our list of roots. + prop::collection::btree_set(1..nodes_len, 1..(nodes_len / 2)), + ) + }) +} + +fn complex_dag_strategy( + nodes_count: usize, + depth: usize, + branch: usize, +) -> impl Strategy { + prop::collection::vec(".*", depth..nodes_count).prop_flat_map(move |payloads| { + let nodes_len = payloads.len(); + let mut dag = TestDag::new(); + // partition the payloads into depth pieces + let mut id_stack: Vec<[u8; 8]> = Vec::new(); + for chunk in payloads.chunks(nodes_len / depth) { + // loop through the partions adding each partions nodes to the dag. + let dep_sets: Vec> = if id_stack.is_empty() { + vec![BTreeSet::new()] + } else { + let mut dep_sets = Vec::new(); + for id_chunk in id_stack.chunks(branch) { + let id_set = id_chunk.iter().fold(BTreeSet::new(), |mut acc, item| { + acc.insert(item.clone()); + acc + }); + dep_sets.push(id_set); + } + dep_sets + }; + let dep_set_len = dep_sets.len(); + for (idx, p) in chunk.iter().enumerate() { + let dep_idx = idx % dep_set_len; + let dep_set = dep_sets[dep_idx].clone(); + id_stack.push(dag.add_node(p.clone(), dep_set).unwrap().clone()); + } + } + Just(dag) + }) +} + +proptest! { + #[test] + fn test_dag_add_node_properties((nodes, parent_idxs) in simple_edge_strategy(100)) { + // TODO implement the tests now + let mut dag = TestDag::new(); + let parent_count = parent_idxs.len(); + let mut dependents = BTreeMap::new(); + let mut node_set = BTreeSet::new(); + for (idx, n) in nodes.iter().cloned().enumerate() { + if !parent_idxs.contains(&idx) { + let node_id = dag.add_node(n, BTreeSet::new()).unwrap(); + node_set.insert(node_id.clone()); + let parent = idx % parent_count; + if dependents.contains_key(&parent) { + dependents.get_mut(&parent).map(|v: &mut BTreeSet<[u8; 8]>| v.insert(node_id)); + } else { + dependents.insert(parent, BTreeSet::from([node_id])); + } + } + } + for (pidx, dep_ids) in dependents { + let node_id = dag.add_node(nodes[pidx].clone(), dep_ids).unwrap(); + node_set.insert(node_id.clone()); + } + assert!(dag.get_roots().len() <= parent_count); + assert!(dag.get_nodes().len() == node_set.len()); + } +} + +proptest! { + #[test] + fn test_complex_dag_node_properties(dag in complex_dag_strategy(100, 10, 3)) { + // TODO(jwall): We can assert much more about the Merkle if we get more clever in what we return. + let nodes = dag.get_nodes(); + assert!(nodes.len() <= 100); + + let roots = dag.get_roots(); + assert!(roots.len() < dag.get_nodes().len()); + + for node_id in nodes.keys() { + let mut is_descendant = false; + if roots.contains(node_id) { + continue; + } + for root in roots.iter() { + if let NodeCompare::After = dag.compare(root, node_id) { + // success + is_descendant = true; + } + } + assert!(is_descendant); + } + // Check that every root node is uncomparable. + for left_root in roots.iter() { + for right_root in roots.iter() { + if left_root != right_root { + assert_eq!(dag.compare(left_root, right_root), NodeCompare::Uncomparable); + } + } + } + } +} diff --git a/src/store.rs b/src/store.rs index 84d3f80..58c87a0 100644 --- a/src/store.rs +++ b/src/store.rs @@ -11,17 +11,45 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. +use std::collections::BTreeMap; -use crate::hash::{ByteEncoder, HashWriter}; -use crate::node::Node; +use crate::{ + hash::{ByteEncoder, HashWriter}, + node::Node, +}; -pub enum StoreError {} +#[derive(Debug, Clone)] +pub enum StoreError { + StoreFailure, + NoSuchDependents, +} -pub trait Store +pub trait Store: Default where N: ByteEncoder, HW: HashWriter, { - fn get(&self, id: &[u8; HASH_LEN]) -> &Node; + fn contains(&self, id: &[u8; HASH_LEN]) -> bool; + fn get(&self, id: &[u8; HASH_LEN]) -> Option<&Node>; fn store(&mut self, node: Node) -> Result<(), StoreError>; } + +impl Store + for BTreeMap<[u8; HASH_LEN], Node> +where + N: ByteEncoder, + HW: HashWriter, +{ + fn contains(&self, id: &[u8; HASH_LEN]) -> bool { + self.contains_key(id) + } + + fn get(&self, id: &[u8; HASH_LEN]) -> Option<&Node> { + self.get(id) + } + + fn store(&mut self, node: Node) -> Result<(), StoreError> { + self.insert(node.id().clone(), node); + Ok(()) + } +} diff --git a/src/test.rs b/src/test.rs index 17636b2..fc853be 100644 --- a/src/test.rs +++ b/src/test.rs @@ -12,13 +12,20 @@ // See the License for the specific language governing permissions and // limitations under the License. use std::collections::hash_map::DefaultHasher; -use std::collections::BTreeSet; +use std::collections::{BTreeMap, BTreeSet}; use crate::prelude::*; +type TestDag<'a> = Merkle< + BTreeMap<[u8; 8], Node<&'a str, std::collections::hash_map::DefaultHasher, 8>>, + &'a str, + std::collections::hash_map::DefaultHasher, + 8, +>; + #[test] fn test_root_pointer_hygiene() { - let mut dag = Merkle::<&str, DefaultHasher, 8>::new(); + let mut dag = TestDag::new(); let quax_node_id = dag.add_node("quax", BTreeSet::new()).unwrap(); assert_eq!( quax_node_id, @@ -39,7 +46,7 @@ fn test_root_pointer_hygiene() { #[test] fn test_insert_no_such_dependents_error() { let missing_dependent = Node::<&str, DefaultHasher, 8>::new("missing", BTreeSet::new()); - let mut dag = Merkle::<&str, DefaultHasher, 8>::new(); + let mut dag = TestDag::new(); let mut dep_set = BTreeSet::new(); dep_set.insert(*missing_dependent.id()); assert!(dag.add_node("foo", dep_set).is_err()); @@ -49,7 +56,7 @@ fn test_insert_no_such_dependents_error() { #[test] fn test_adding_nodes_is_idempotent() { - let mut dag = Merkle::<&str, DefaultHasher, 8>::new(); + let mut dag = TestDag::new(); let quax_node_id = dag.add_node("quax", BTreeSet::new()).unwrap(); assert_eq!( quax_node_id, @@ -65,7 +72,7 @@ fn test_adding_nodes_is_idempotent() { #[test] fn test_adding_nodes_is_idempotent_regardless_of_dep_order() { - let mut dag = Merkle::<&str, DefaultHasher, 8>::new(); + let mut dag = TestDag::new(); let quake_node_id = dag.add_node("quake", BTreeSet::new()).unwrap(); let qualm_node_id = dag.add_node("qualm", BTreeSet::new()).unwrap(); let quell_node_id = dag.add_node("quell", BTreeSet::new()).unwrap(); @@ -87,7 +94,7 @@ fn test_adding_nodes_is_idempotent_regardless_of_dep_order() { #[test] fn test_node_comparison_equivalent() { - let mut dag = Merkle::<&str, DefaultHasher, 8>::new(); + let mut dag = TestDag::new(); let quake_node_id = dag.add_node("quake", BTreeSet::new()).unwrap(); assert_eq!( dag.compare(&quake_node_id, &quake_node_id), @@ -97,7 +104,7 @@ fn test_node_comparison_equivalent() { #[test] fn test_node_comparison_before() { - let mut dag = Merkle::<&str, DefaultHasher, 8>::new(); + let mut dag = TestDag::new(); let quake_node_id = dag.add_node("quake", BTreeSet::new()).unwrap(); let qualm_node_id = dag .add_node("qualm", BTreeSet::from([quake_node_id.clone()])) @@ -117,7 +124,7 @@ fn test_node_comparison_before() { #[test] fn test_node_comparison_after() { - let mut dag = Merkle::<&str, DefaultHasher, 8>::new(); + let mut dag = TestDag::new(); let quake_node_id = dag.add_node("quake", BTreeSet::new()).unwrap(); let qualm_node_id = dag .add_node("qualm", BTreeSet::from([quake_node_id.clone()])) @@ -137,7 +144,7 @@ fn test_node_comparison_after() { #[test] fn test_node_comparison_no_shared_graph() { - let mut dag = Merkle::<&str, DefaultHasher, 8>::new(); + let mut dag = TestDag::new(); let quake_node_id = dag.add_node("quake", BTreeSet::new()).unwrap(); let qualm_node_id = dag.add_node("qualm", BTreeSet::new()).unwrap(); let quell_node_id = dag.add_node("quell", BTreeSet::new()).unwrap();