Introduce a Node storage trait

src/dag.rs

@@ -12,10 +12,13 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::{BTreeMap, BTreeSet};
+use std::{collections::BTreeSet, marker::PhantomData};
 
-use crate::hash::{ByteEncoder, HashWriter};
+use crate::{
-use crate::node::Node;
+    hash::{ByteEncoder, HashWriter},
+    node::Node,
+    store::{Store, StoreError},
+};
 
 /// Node comparison values. In a given Merkle DAG a Node can come `After`, `Before`, be `Equivalent`, or `Uncomparable`.
 /// If the two nodes have the same id they are eqivalent. If two nodes are not part of the same sub graph within the DAG
@@ -29,11 +32,6 @@ pub enum NodeCompare {
     Uncomparable,
 }
 
-#[derive(Debug)]
-pub enum EdgeError {
-    NoSuchDependents,
-}
-
 /// A Merkle-DAG implementation. This is a modification on the standard Merkle Tree data structure
 /// but instead of a tree it is a DAG and as a result can have multiple roots. A merkle-dag specifies
 /// a partial ordering on all the nodes and utilizes the api to ensure that this ordering is
@@ -46,19 +44,22 @@ pub enum EdgeError {
 /// A merkle DAG instance is tied to a specific implementation of the HashWriter interface to ensure
 /// that all hash identifiers are of the same hash algorithm.
 #[derive(Clone, Debug)]
-pub struct Merkle<N, HW, const HASH_LEN: usize>
+pub struct Merkle<S, N, HW, const HASH_LEN: usize>
 where
     N: ByteEncoder,
     HW: HashWriter<HASH_LEN>,
+    S: Store<N, HW, HASH_LEN>,
 {
     roots: BTreeSet<[u8; HASH_LEN]>,
-    nodes: BTreeMap<[u8; HASH_LEN], Node<N, HW, HASH_LEN>>,
+    nodes: S,
+    _phantom_node: PhantomData<Node<N, HW, HASH_LEN>>,
 }
 
-impl<N, HW, const HASH_LEN: usize> Merkle<N, HW, HASH_LEN>
+impl<S, N, HW, const HASH_LEN: usize> Merkle<S, N, HW, HASH_LEN>
 where
     N: ByteEncoder,
     HW: HashWriter<HASH_LEN>,
+    S: Store<N, HW, HASH_LEN>,
 {
     /// Construct a new empty DAG. The empty DAG is also the default for a DAG.
     pub fn new() -> Self {
@@ -75,31 +76,35 @@ where
         &'a mut self,
         item: N,
         dependency_ids: BTreeSet<[u8; HASH_LEN]>,
-    ) -> Result<[u8; HASH_LEN], EdgeError> {
+    ) -> Result<[u8; HASH_LEN], StoreError> {
         let node = Node::<N, HW, HASH_LEN>::new(item, dependency_ids.clone());
         let id = node.id().clone();
-        if self.nodes.contains_key(&id) {
+        if self.nodes.contains(&id) {
             // We've already added this node so there is nothing left to do.
             return Ok(id);
         }
+        let mut root_removals = Vec::new();
         for dep_id in dependency_ids.iter() {
-            if !self.nodes.contains_key(dep_id) {
+            if !self.nodes.contains(dep_id) {
-                return Err(EdgeError::NoSuchDependents);
+                return Err(StoreError::NoSuchDependents);
             }
             // If any of our dependencies is in the roots pointer list then
-            // it is time to remove it from there.
+            // we need to remove it below.
             if self.roots.contains(dep_id) {
-                self.roots.remove(dep_id);
+                root_removals.push(dep_id);
             }
         }
-        self.roots.insert(id.clone());
+        self.nodes.store(node)?;
-        self.nodes.insert(id.clone(), node);
+        for removal in root_removals {
+            self.roots.remove(removal);
+        }
+        self.roots.insert(id);
         Ok(id)
     }
 
     /// Check if we already have a copy of a node.
     pub fn check_for_node(&self, id: &[u8; HASH_LEN]) -> bool {
-        return self.nodes.contains_key(id);
+        return self.nodes.contains(id);
     }
 
     /// Get a node from the DAG by it's hash identifier if it exists.
@@ -113,7 +118,7 @@ where
     }
 
     /// Get the map of all nodes in the DAG.
-    pub fn get_nodes(&self) -> &BTreeMap<[u8; HASH_LEN], Node<N, HW, HASH_LEN>> {
+    pub fn get_nodes(&self) -> &S {
         &self.nodes
     }
 
@@ -166,15 +171,17 @@ where
     }
 }
 
-impl<N, HW, const HASH_LEN: usize> Default for Merkle<N, HW, HASH_LEN>
+impl<S, N, HW, const HASH_LEN: usize> Default for Merkle<S, N, HW, HASH_LEN>
 where
     N: ByteEncoder,
     HW: HashWriter<HASH_LEN>,
+    S: Store<N, HW, HASH_LEN>,
 {
     fn default() -> Self {
         Self {
             roots: BTreeSet::new(),
-            nodes: BTreeMap::new(),
+            nodes: S::default(),
+            _phantom_node: Default::default(),
         }
     }
 }

src/lib.rs

@@ -15,6 +15,7 @@ pub mod dag;
 pub mod hash;
 pub mod node;
 pub mod prelude;
+pub mod store;
 
 #[cfg(test)]
 mod test;

src/proptest.rs

@@ -0,0 +1,133 @@
+// Copyright 2022 Jeremy Wall (Jeremy@marzhilsltudios.com)
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+use std::collections::{hash_map::DefaultHasher, BTreeMap, BTreeSet};
+
+use proptest::prelude::*;
+
+use crate::prelude::*;
+
+type TestDag = Merkle<BTreeMap<[u8; 8], Node<String, DefaultHasher, 8>>, String, DefaultHasher, 8>;
+
+fn simple_edge_strategy(
+    nodes_count: usize,
+) -> impl Strategy<Value = (Vec<String>, BTreeSet<usize>)> {
+    prop::collection::vec(".*", 4..nodes_count).prop_flat_map(|payloads| {
+        let nodes_len = payloads.len();
+        (
+            // our total list of nodes.
+            Just(payloads),
+            // our list of roots.
+            prop::collection::btree_set(1..nodes_len, 1..(nodes_len / 2)),
+        )
+    })
+}
+
+fn complex_dag_strategy(
+    nodes_count: usize,
+    depth: usize,
+    branch: usize,
+) -> impl Strategy<Value = TestDag> {
+    prop::collection::vec(".*", depth..nodes_count).prop_flat_map(move |payloads| {
+        let nodes_len = payloads.len();
+        let mut dag = TestDag::new();
+        // partition the payloads into depth pieces
+        let mut id_stack: Vec<[u8; 8]> = Vec::new();
+        for chunk in payloads.chunks(nodes_len / depth) {
+            // loop through the partions adding each partions nodes to the dag.
+            let dep_sets: Vec<BTreeSet<[u8; 8]>> = if id_stack.is_empty() {
+                vec![BTreeSet::new()]
+            } else {
+                let mut dep_sets = Vec::new();
+                for id_chunk in id_stack.chunks(branch) {
+                    let id_set = id_chunk.iter().fold(BTreeSet::new(), |mut acc, item| {
+                        acc.insert(item.clone());
+                        acc
+                    });
+                    dep_sets.push(id_set);
+                }
+                dep_sets
+            };
+            let dep_set_len = dep_sets.len();
+            for (idx, p) in chunk.iter().enumerate() {
+                let dep_idx = idx % dep_set_len;
+                let dep_set = dep_sets[dep_idx].clone();
+                id_stack.push(dag.add_node(p.clone(), dep_set).unwrap().clone());
+            }
+        }
+        Just(dag)
+    })
+}
+
+proptest! {
+    #[test]
+    fn test_dag_add_node_properties((nodes, parent_idxs) in simple_edge_strategy(100)) {
+        // TODO implement the tests now
+        let mut dag = TestDag::new();
+        let parent_count = parent_idxs.len();
+        let mut dependents = BTreeMap::new();
+        let mut node_set = BTreeSet::new();
+        for (idx, n) in nodes.iter().cloned().enumerate() {
+            if !parent_idxs.contains(&idx) {
+                let node_id = dag.add_node(n, BTreeSet::new()).unwrap();
+                node_set.insert(node_id.clone());
+                let parent = idx % parent_count;
+                if dependents.contains_key(&parent) {
+                    dependents.get_mut(&parent).map(|v: &mut BTreeSet<[u8; 8]>| v.insert(node_id));
+                } else {
+                    dependents.insert(parent, BTreeSet::from([node_id]));
+                }
+            }
+        }
+        for (pidx, dep_ids) in dependents {
+            let node_id = dag.add_node(nodes[pidx].clone(), dep_ids).unwrap();
+            node_set.insert(node_id.clone());
+        }
+        assert!(dag.get_roots().len() <= parent_count);
+        assert!(dag.get_nodes().len() == node_set.len());
+    }
+}
+
+proptest! {
+    #[test]
+    fn test_complex_dag_node_properties(dag in complex_dag_strategy(100, 10, 3)) {
+        // TODO(jwall): We can assert much more about the Merkle if we get more clever in what we return.
+        let nodes = dag.get_nodes();
+        assert!(nodes.len() <= 100);
+
+        let roots = dag.get_roots();
+        assert!(roots.len() < dag.get_nodes().len());
+
+        for node_id in nodes.keys() {
+            let mut is_descendant = false;
+            if roots.contains(node_id) {
+                continue;
+            }
+            for root in roots.iter() {
+                if let NodeCompare::After = dag.compare(root, node_id) {
+                    // success
+                    is_descendant = true;
+                }
+            }
+            assert!(is_descendant);
+        }
+        // Check that every root node is uncomparable.
+        for left_root in roots.iter() {
+            for right_root in roots.iter() {
+                if left_root != right_root {
+                    assert_eq!(dag.compare(left_root, right_root), NodeCompare::Uncomparable);
+                }
+            }
+        }
+    }
+}

src/store.rs

@@ -11,17 +11,45 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+use std::collections::BTreeMap;
 
-use crate::hash::{ByteEncoder, HashWriter};
+use crate::{
-use crate::node::Node;
+    hash::{ByteEncoder, HashWriter},
+    node::Node,
+};
 
-pub enum StoreError {}
+#[derive(Debug, Clone)]
+pub enum StoreError {
+    StoreFailure,
+    NoSuchDependents,
+}
 
-pub trait Store<N, HW, const HASH_LEN: usize>
+pub trait Store<N, HW, const HASH_LEN: usize>: Default
 where
     N: ByteEncoder,
     HW: HashWriter<HASH_LEN>,
 {
-    fn get(&self, id: &[u8; HASH_LEN]) -> &Node<N, HW, HASH_LEN>;
+    fn contains(&self, id: &[u8; HASH_LEN]) -> bool;
+    fn get(&self, id: &[u8; HASH_LEN]) -> Option<&Node<N, HW, HASH_LEN>>;
     fn store(&mut self, node: Node<N, HW, HASH_LEN>) -> Result<(), StoreError>;
 }
+
+impl<N, HW, const HASH_LEN: usize> Store<N, HW, HASH_LEN>
+    for BTreeMap<[u8; HASH_LEN], Node<N, HW, HASH_LEN>>
+where
+    N: ByteEncoder,
+    HW: HashWriter<HASH_LEN>,
+{
+    fn contains(&self, id: &[u8; HASH_LEN]) -> bool {
+        self.contains_key(id)
+    }
+
+    fn get(&self, id: &[u8; HASH_LEN]) -> Option<&Node<N, HW, HASH_LEN>> {
+        self.get(id)
+    }
+
+    fn store(&mut self, node: Node<N, HW, HASH_LEN>) -> Result<(), StoreError> {
+        self.insert(node.id().clone(), node);
+        Ok(())
+    }
+}

src/test.rs

@@ -12,13 +12,20 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 use std::collections::hash_map::DefaultHasher;
-use std::collections::BTreeSet;
+use std::collections::{BTreeMap, BTreeSet};
 
 use crate::prelude::*;
 
+type TestDag<'a> = Merkle<
+    BTreeMap<[u8; 8], Node<&'a str, std::collections::hash_map::DefaultHasher, 8>>,
+    &'a str,
+    std::collections::hash_map::DefaultHasher,
+    8,
+>;
+
 #[test]
 fn test_root_pointer_hygiene() {
-    let mut dag = Merkle::<&str, DefaultHasher, 8>::new();
+    let mut dag = TestDag::new();
     let quax_node_id = dag.add_node("quax", BTreeSet::new()).unwrap();
     assert_eq!(
         quax_node_id,
@@ -39,7 +46,7 @@ fn test_root_pointer_hygiene() {
 #[test]
 fn test_insert_no_such_dependents_error() {
     let missing_dependent = Node::<&str, DefaultHasher, 8>::new("missing", BTreeSet::new());
-    let mut dag = Merkle::<&str, DefaultHasher, 8>::new();
+    let mut dag = TestDag::new();
     let mut dep_set = BTreeSet::new();
     dep_set.insert(*missing_dependent.id());
     assert!(dag.add_node("foo", dep_set).is_err());
@@ -49,7 +56,7 @@ fn test_insert_no_such_dependents_error() {
 
 #[test]
 fn test_adding_nodes_is_idempotent() {
-    let mut dag = Merkle::<&str, DefaultHasher, 8>::new();
+    let mut dag = TestDag::new();
     let quax_node_id = dag.add_node("quax", BTreeSet::new()).unwrap();
     assert_eq!(
         quax_node_id,
@@ -65,7 +72,7 @@ fn test_adding_nodes_is_idempotent() {
 
 #[test]
 fn test_adding_nodes_is_idempotent_regardless_of_dep_order() {
-    let mut dag = Merkle::<&str, DefaultHasher, 8>::new();
+    let mut dag = TestDag::new();
     let quake_node_id = dag.add_node("quake", BTreeSet::new()).unwrap();
     let qualm_node_id = dag.add_node("qualm", BTreeSet::new()).unwrap();
     let quell_node_id = dag.add_node("quell", BTreeSet::new()).unwrap();
@@ -87,7 +94,7 @@ fn test_adding_nodes_is_idempotent_regardless_of_dep_order() {
 
 #[test]
 fn test_node_comparison_equivalent() {
-    let mut dag = Merkle::<&str, DefaultHasher, 8>::new();
+    let mut dag = TestDag::new();
     let quake_node_id = dag.add_node("quake", BTreeSet::new()).unwrap();
     assert_eq!(
         dag.compare(&quake_node_id, &quake_node_id),
@@ -97,7 +104,7 @@ fn test_node_comparison_equivalent() {
 
 #[test]
 fn test_node_comparison_before() {
-    let mut dag = Merkle::<&str, DefaultHasher, 8>::new();
+    let mut dag = TestDag::new();
     let quake_node_id = dag.add_node("quake", BTreeSet::new()).unwrap();
     let qualm_node_id = dag
         .add_node("qualm", BTreeSet::from([quake_node_id.clone()]))
@@ -117,7 +124,7 @@ fn test_node_comparison_before() {
 
 #[test]
 fn test_node_comparison_after() {
-    let mut dag = Merkle::<&str, DefaultHasher, 8>::new();
+    let mut dag = TestDag::new();
     let quake_node_id = dag.add_node("quake", BTreeSet::new()).unwrap();
     let qualm_node_id = dag
         .add_node("qualm", BTreeSet::from([quake_node_id.clone()]))
@@ -137,7 +144,7 @@ fn test_node_comparison_after() {
 
 #[test]
 fn test_node_comparison_no_shared_graph() {
-    let mut dag = Merkle::<&str, DefaultHasher, 8>::new();
+    let mut dag = TestDag::new();
     let quake_node_id = dag.add_node("quake", BTreeSet::new()).unwrap();
     let qualm_node_id = dag.add_node("qualm", BTreeSet::new()).unwrap();
     let quell_node_id = dag.add_node("quell", BTreeSet::new()).unwrap();