zaphar/merkle-dag
1
0
Fork 0
mirror of https://github.com/zaphar/merkle-dag.git synced 2025-07-24 03:29:48 -04:00
Code Issues Packages Projects Releases Wiki Activity

Introduce a Node storage trait

Browse Source
This commit is contained in:
Jeremy Wall 2022-08-05 14:29:07 -04:00
parent 016f6d10af
commit 06e0dc270c
5 changed files with 213 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
src/dag.rs
View File

@ -12,10 +12,13 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use std::collections::{BTreeMap, BTreeSet};
use std::{collections::BTreeSet, marker::PhantomData};
use crate::hash::{ByteEncoder, HashWriter};
use crate::node::Node;
use crate::{
hash::{ByteEncoder, HashWriter},
node::Node,
store::{Store, StoreError},
};
/// Node comparison values. In a given Merkle DAG a Node can come `After`, `Before`, be `Equivalent`, or `Uncomparable`.
/// If the two nodes have the same id they are eqivalent. If two nodes are not part of the same sub graph within the DAG
@ -29,11 +32,6 @@ pub enum NodeCompare {
Uncomparable,
}
#[derive(Debug)]
pub enum EdgeError {
NoSuchDependents,
}
/// A Merkle-DAG implementation. This is a modification on the standard Merkle Tree data structure
/// but instead of a tree it is a DAG and as a result can have multiple roots. A merkle-dag specifies
/// a partial ordering on all the nodes and utilizes the api to ensure that this ordering is
@ -46,19 +44,22 @@ pub enum EdgeError {
/// A merkle DAG instance is tied to a specific implementation of the HashWriter interface to ensure
/// that all hash identifiers are of the same hash algorithm.
#[derive(Clone, Debug)]
pub struct Merkle<N, HW, const HASH_LEN: usize>
pub struct Merkle<S, N, HW, const HASH_LEN: usize>
where
N: ByteEncoder,
HW: HashWriter<HASH_LEN>,
S: Store<N, HW, HASH_LEN>,
{
roots: BTreeSet<[u8; HASH_LEN]>,
nodes: BTreeMap<[u8; HASH_LEN], Node<N, HW, HASH_LEN>>,
nodes: S,
_phantom_node: PhantomData<Node<N, HW, HASH_LEN>>,
}
impl<N, HW, const HASH_LEN: usize> Merkle<N, HW, HASH_LEN>
impl<S, N, HW, const HASH_LEN: usize> Merkle<S, N, HW, HASH_LEN>
where
N: ByteEncoder,
HW: HashWriter<HASH_LEN>,
S: Store<N, HW, HASH_LEN>,
{
/// Construct a new empty DAG. The empty DAG is also the default for a DAG.
pub fn new() -> Self {
@ -75,31 +76,35 @@ where
&'a mut self,
item: N,
dependency_ids: BTreeSet<[u8; HASH_LEN]>,
) -> Result<[u8; HASH_LEN], EdgeError> {
) -> Result<[u8; HASH_LEN], StoreError> {
let node = Node::<N, HW, HASH_LEN>::new(item, dependency_ids.clone());
let id = node.id().clone();
if self.nodes.contains_key(&id) {
if self.nodes.contains(&id) {
// We've already added this node so there is nothing left to do.
return Ok(id);
}
let mut root_removals = Vec::new();
for dep_id in dependency_ids.iter() {
if !self.nodes.contains_key(dep_id) {
return Err(EdgeError::NoSuchDependents);
if !self.nodes.contains(dep_id) {
return Err(StoreError::NoSuchDependents);
}
// If any of our dependencies is in the roots pointer list then
// it is time to remove it from there.
// we need to remove it below.
if self.roots.contains(dep_id) {
self.roots.remove(dep_id);
root_removals.push(dep_id);
}
}
self.roots.insert(id.clone());
self.nodes.insert(id.clone(), node);
self.nodes.store(node)?;
for removal in root_removals {
self.roots.remove(removal);
}
self.roots.insert(id);
Ok(id)
}
/// Check if we already have a copy of a node.
pub fn check_for_node(&self, id: &[u8; HASH_LEN]) -> bool {
return self.nodes.contains_key(id);
return self.nodes.contains(id);
}
/// Get a node from the DAG by it's hash identifier if it exists.
@ -113,7 +118,7 @@ where
}
/// Get the map of all nodes in the DAG.
pub fn get_nodes(&self) -> &BTreeMap<[u8; HASH_LEN], Node<N, HW, HASH_LEN>> {
pub fn get_nodes(&self) -> &S {
&self.nodes
}
@ -166,15 +171,17 @@ where
}
}
impl<N, HW, const HASH_LEN: usize> Default for Merkle<N, HW, HASH_LEN>
impl<S, N, HW, const HASH_LEN: usize> Default for Merkle<S, N, HW, HASH_LEN>
where
N: ByteEncoder,
HW: HashWriter<HASH_LEN>,
S: Store<N, HW, HASH_LEN>,
{
fn default() -> Self {
Self {
roots: BTreeSet::new(),
nodes: BTreeMap::new(),
nodes: S::default(),
_phantom_node: Default::default(),
}
}
}

1
src/lib.rs
View File

@ -15,6 +15,7 @@ pub mod dag;
pub mod hash;
pub mod node;
pub mod prelude;
pub mod store;
#[cfg(test)]
mod test;

133
src/proptest.rs Normal file
View File

@ -0,0 +1,133 @@
// Copyright 2022 Jeremy Wall (Jeremy@marzhilsltudios.com)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::collections::{hash_map::DefaultHasher, BTreeMap, BTreeSet};
use proptest::prelude::*;
use crate::prelude::*;
type TestDag = Merkle<BTreeMap<[u8; 8], Node<String, DefaultHasher, 8>>, String, DefaultHasher, 8>;
fn simple_edge_strategy(
nodes_count: usize,
) -> impl Strategy<Value = (Vec<String>, BTreeSet<usize>)> {
prop::collection::vec(".*", 4..nodes_count).prop_flat_map(|payloads| {
let nodes_len = payloads.len();
(
// our total list of nodes.
Just(payloads),
// our list of roots.
prop::collection::btree_set(1..nodes_len, 1..(nodes_len / 2)),
)
})
}
fn complex_dag_strategy(
nodes_count: usize,
depth: usize,
branch: usize,
) -> impl Strategy<Value = TestDag> {
prop::collection::vec(".*", depth..nodes_count).prop_flat_map(move |payloads| {
let nodes_len = payloads.len();
let mut dag = TestDag::new();
// partition the payloads into depth pieces
let mut id_stack: Vec<[u8; 8]> = Vec::new();
for chunk in payloads.chunks(nodes_len / depth) {
// loop through the partions adding each partions nodes to the dag.
let dep_sets: Vec<BTreeSet<[u8; 8]>> = if id_stack.is_empty() {
vec![BTreeSet::new()]
} else {
let mut dep_sets = Vec::new();
for id_chunk in id_stack.chunks(branch) {
let id_set = id_chunk.iter().fold(BTreeSet::new(), |mut acc, item| {
acc.insert(item.clone());
acc
});
dep_sets.push(id_set);
}
dep_sets
};
let dep_set_len = dep_sets.len();
for (idx, p) in chunk.iter().enumerate() {
let dep_idx = idx % dep_set_len;
let dep_set = dep_sets[dep_idx].clone();
id_stack.push(dag.add_node(p.clone(), dep_set).unwrap().clone());
}
}
Just(dag)
})
}
proptest! {
#[test]
fn test_dag_add_node_properties((nodes, parent_idxs) in simple_edge_strategy(100)) {
// TODO implement the tests now
let mut dag = TestDag::new();
let parent_count = parent_idxs.len();
let mut dependents = BTreeMap::new();
let mut node_set = BTreeSet::new();
for (idx, n) in nodes.iter().cloned().enumerate() {
if !parent_idxs.contains(&idx) {
let node_id = dag.add_node(n, BTreeSet::new()).unwrap();
node_set.insert(node_id.clone());
let parent = idx % parent_count;
if dependents.contains_key(&parent) {
dependents.get_mut(&parent).map(|v: &mut BTreeSet<[u8; 8]>| v.insert(node_id));
} else {
dependents.insert(parent, BTreeSet::from([node_id]));
}
}
}
for (pidx, dep_ids) in dependents {
let node_id = dag.add_node(nodes[pidx].clone(), dep_ids).unwrap();
node_set.insert(node_id.clone());
}
assert!(dag.get_roots().len() <= parent_count);
assert!(dag.get_nodes().len() == node_set.len());
}
}
proptest! {
#[test]
fn test_complex_dag_node_properties(dag in complex_dag_strategy(100, 10, 3)) {
// TODO(jwall): We can assert much more about the Merkle if we get more clever in what we return.
let nodes = dag.get_nodes();
assert!(nodes.len() <= 100);
let roots = dag.get_roots();
assert!(roots.len() < dag.get_nodes().len());
for node_id in nodes.keys() {
let mut is_descendant = false;
if roots.contains(node_id) {
continue;
}
for root in roots.iter() {
if let NodeCompare::After = dag.compare(root, node_id) {
// success
is_descendant = true;
}
}
assert!(is_descendant);
}
// Check that every root node is uncomparable.
for left_root in roots.iter() {
for right_root in roots.iter() {
if left_root != right_root {
assert_eq!(dag.compare(left_root, right_root), NodeCompare::Uncomparable);
}
}
}
}
}

38
src/store.rs
View File

@ -11,17 +11,45 @@
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::collections::BTreeMap;
use crate::hash::{ByteEncoder, HashWriter};
use crate::node::Node;
use crate::{
hash::{ByteEncoder, HashWriter},
node::Node,
};
pub enum StoreError {}
#[derive(Debug, Clone)]
pub enum StoreError {
StoreFailure,
NoSuchDependents,
}
pub trait Store<N, HW, const HASH_LEN: usize>
pub trait Store<N, HW, const HASH_LEN: usize>: Default
where
N: ByteEncoder,
HW: HashWriter<HASH_LEN>,
{
fn get(&self, id: &[u8; HASH_LEN]) -> &Node<N, HW, HASH_LEN>;
fn contains(&self, id: &[u8; HASH_LEN]) -> bool;
fn get(&self, id: &[u8; HASH_LEN]) -> Option<&Node<N, HW, HASH_LEN>>;
fn store(&mut self, node: Node<N, HW, HASH_LEN>) -> Result<(), StoreError>;
}
impl<N, HW, const HASH_LEN: usize> Store<N, HW, HASH_LEN>
for BTreeMap<[u8; HASH_LEN], Node<N, HW, HASH_LEN>>
where
N: ByteEncoder,
HW: HashWriter<HASH_LEN>,
{
fn contains(&self, id: &[u8; HASH_LEN]) -> bool {
self.contains_key(id)
}
fn get(&self, id: &[u8; HASH_LEN]) -> Option<&Node<N, HW, HASH_LEN>> {
self.get(id)
}
fn store(&mut self, node: Node<N, HW, HASH_LEN>) -> Result<(), StoreError> {
self.insert(node.id().clone(), node);
Ok(())
}
}

25
src/test.rs
View File

@ -12,13 +12,20 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use std::collections::hash_map::DefaultHasher;
use std::collections::BTreeSet;
use std::collections::{BTreeMap, BTreeSet};
use crate::prelude::*;
type TestDag<'a> = Merkle<
BTreeMap<[u8; 8], Node<&'a str, std::collections::hash_map::DefaultHasher, 8>>,
&'a str,
std::collections::hash_map::DefaultHasher,
8,
>;
#[test]
fn test_root_pointer_hygiene() {
let mut dag = Merkle::<&str, DefaultHasher, 8>::new();
let mut dag = TestDag::new();
let quax_node_id = dag.add_node("quax", BTreeSet::new()).unwrap();
assert_eq!(
quax_node_id,
@ -39,7 +46,7 @@ fn test_root_pointer_hygiene() {
#[test]
fn test_insert_no_such_dependents_error() {
let missing_dependent = Node::<&str, DefaultHasher, 8>::new("missing", BTreeSet::new());
let mut dag = Merkle::<&str, DefaultHasher, 8>::new();
let mut dag = TestDag::new();
let mut dep_set = BTreeSet::new();
dep_set.insert(*missing_dependent.id());
assert!(dag.add_node("foo", dep_set).is_err());
@ -49,7 +56,7 @@ fn test_insert_no_such_dependents_error() {
#[test]
fn test_adding_nodes_is_idempotent() {
let mut dag = Merkle::<&str, DefaultHasher, 8>::new();
let mut dag = TestDag::new();
let quax_node_id = dag.add_node("quax", BTreeSet::new()).unwrap();
assert_eq!(
quax_node_id,
@ -65,7 +72,7 @@ fn test_adding_nodes_is_idempotent() {
#[test]
fn test_adding_nodes_is_idempotent_regardless_of_dep_order() {
let mut dag = Merkle::<&str, DefaultHasher, 8>::new();
let mut dag = TestDag::new();
let quake_node_id = dag.add_node("quake", BTreeSet::new()).unwrap();
let qualm_node_id = dag.add_node("qualm", BTreeSet::new()).unwrap();
let quell_node_id = dag.add_node("quell", BTreeSet::new()).unwrap();
@ -87,7 +94,7 @@ fn test_adding_nodes_is_idempotent_regardless_of_dep_order() {
#[test]
fn test_node_comparison_equivalent() {
let mut dag = Merkle::<&str, DefaultHasher, 8>::new();
let mut dag = TestDag::new();
let quake_node_id = dag.add_node("quake", BTreeSet::new()).unwrap();
assert_eq!(
dag.compare(&quake_node_id, &quake_node_id),
@ -97,7 +104,7 @@ fn test_node_comparison_equivalent() {
#[test]
fn test_node_comparison_before() {
let mut dag = Merkle::<&str, DefaultHasher, 8>::new();
let mut dag = TestDag::new();
let quake_node_id = dag.add_node("quake", BTreeSet::new()).unwrap();
let qualm_node_id = dag
.add_node("qualm", BTreeSet::from([quake_node_id.clone()]))
@ -117,7 +124,7 @@ fn test_node_comparison_before() {
#[test]
fn test_node_comparison_after() {
let mut dag = Merkle::<&str, DefaultHasher, 8>::new();
let mut dag = TestDag::new();
let quake_node_id = dag.add_node("quake", BTreeSet::new()).unwrap();
let qualm_node_id = dag
.add_node("qualm", BTreeSet::from([quake_node_id.clone()]))
@ -137,7 +144,7 @@ fn test_node_comparison_after() {
#[test]
fn test_node_comparison_no_shared_graph() {
let mut dag = Merkle::<&str, DefaultHasher, 8>::new();
let mut dag = TestDag::new();
let quake_node_id = dag.add_node("quake", BTreeSet::new()).unwrap();
let qualm_node_id = dag.add_node("qualm", BTreeSet::new()).unwrap();
let quell_node_id = dag.add_node("quell", BTreeSet::new()).unwrap();