vinteumorg · Davidson-Souza · Jul 25, 2024 · Jul 2, 2024 · Jul 2, 2024 · Jul 9, 2024
diff --git a/crates/floresta-cli/src/lib.rs b/crates/floresta-cli/src/lib.rs
@@ -95,14 +95,6 @@ mod tests {
         (Florestad { proc: fld }, client)
     }
 
-    #[test]
-    fn test_rescan() {
-        let (_proc, client) = start_florestad();
-
-        let rescan = client.rescan(0).expect("rpc not working");
-        assert!(rescan);
-    }
-
     #[test]
     fn test_stop() {
         let (mut _proc, client) = start_florestad();
@@ -183,18 +175,6 @@ mod tests {
         assert!(block_filter.is_err());
     }
 
-    #[test]
-    fn test_load_descriptor() {
-        let (_proc, client) = start_florestad();
-
-        let desc = "
-            wsh(sortedmulti(1,[54ff5a12/48h/1h/0h/2h]tpubDDw6pwZA3hYxcSN32q7a5ynsKmWr4BbkBNHydHPKkM4BZwUfiK7tQ26h7USm8kA1E2FvCy7f7Er7QXKF8RNptATywydARtzgrxuPDwyYv4x/<0;1>/*,[bcf969c0/48h/1h/0h/2h]tpubDEFdgZdCPgQBTNtGj4h6AehK79Jm4LH54JrYBJjAtHMLEAth7LuY87awx9ZMiCURFzFWhxToRJK6xp39aqeJWrG5nuW3eBnXeMJcvDeDxfp/<0;1>/*))#fuw35j0q";
-
-        let res = client.load_descriptor(desc.to_string(), Some(0)).unwrap();
-
-        assert!(res)
-    }
-
     #[test]
     fn test_get_height() {
         let (_proc, client) = start_florestad();

diff --git a/crates/floresta-cli/src/main.rs b/crates/floresta-cli/src/main.rs
@@ -58,8 +58,8 @@ fn do_request(cmd: &Cli, client: ReqwestClient) -> anyhow::Result<String> {
         Methods::GetBlockHeader { hash } => {
             serde_json::to_string_pretty(&client.get_block_header(hash)?)?
         }
-        Methods::LoadDescriptor { desc, rescan } => {
-            serde_json::to_string_pretty(&client.load_descriptor(desc, rescan)?)?
+        Methods::LoadDescriptor { desc } => {
+            serde_json::to_string_pretty(&client.load_descriptor(desc)?)?
         }
         Methods::GetRoots => serde_json::to_string_pretty(&client.get_roots()?)?,
         Methods::GetBlock { hash, .. } => serde_json::to_string_pretty(&client.get_block(hash)?)?,
@@ -128,7 +128,7 @@ pub enum Methods {
     GetBlockHeader { hash: BlockHash },
     /// Loads a new descriptor to the watch only wallet
     #[command(name = "loaddescriptor")]
-    LoadDescriptor { desc: String, rescan: Option<u32> },
+    LoadDescriptor { desc: String },
     /// Returns the roots of the current utreexo forest
     #[command(name = "getroots")]
     GetRoots,

diff --git a/crates/floresta-cli/src/rpc.rs b/crates/floresta-cli/src/rpc.rs
@@ -56,7 +56,7 @@ pub trait FlorestaRPC {
     /// compact block filters enabled, this process will be much faster and use less bandwidth.
     /// The rescan parameter is the height at which to start the rescan, and should be at least
     /// as old as the oldest transaction this descriptor could have been used in.
-    fn load_descriptor(&self, descriptor: String, rescan: Option<u32>) -> Result<bool>;
+    fn load_descriptor(&self, descriptor: String) -> Result<bool>;
     /// Trigger a rescan of the wallet
     ///
     /// This method triggers a rescan of the wallet. If you have compact block filters enabled,
@@ -181,15 +181,8 @@ impl<T: JsonRPCClient> FlorestaRPC for T {
         )
     }
 
-    fn load_descriptor(&self, descriptor: String, rescan: Option<u32>) -> Result<bool> {
-        let rescan = rescan.unwrap_or(0);
-        self.call(
-            "loaddescriptor",
-            &[
-                Value::String(descriptor),
-                Value::Number(Number::from(rescan)),
-            ],
-        )
+    fn load_descriptor(&self, descriptor: String) -> Result<bool> {
+        self.call("loaddescriptor", &[Value::String(descriptor)])
     }
 
     fn get_block_filter(&self, heigth: u32) -> Result<String> {

diff --git a/crates/floresta-compact-filters/src/flat_filters_store.rs b/crates/floresta-compact-filters/src/flat_filters_store.rs
@@ -0,0 +1,178 @@
+use std::convert::TryFrom;
+use std::fs::File;
+use std::io::BufReader;
+use std::io::Read;
+use std::io::Seek;
+use std::io::SeekFrom;
+use std::io::Write;
+use std::path::PathBuf;
+use std::sync::Mutex;
+use std::sync::MutexGuard;
+use std::sync::PoisonError;
+
+use crate::IteratableFilterStore;
+use crate::IteratableFilterStoreError;
+
+pub struct FiltersIterator {
+    reader: BufReader<File>,
+    current: u32,
+}
+
+impl Iterator for FiltersIterator {
+    type Item = (u32, crate::bip158::BlockFilter);
+
+    fn next(&mut self) -> Option<Self::Item> {
+        let mut buf = [0; 4];
+        self.reader.read_exact(&mut buf).ok()?;
+        let length = u32::from_le_bytes(buf);
+
+        debug_assert!(
+            length < 1_000_000,
+            "filter for block {} has length {}",
+            self.current,
+            length
+        );
+
+        let mut buf = vec![0_u8; length as usize];
+        self.reader.read_exact(&mut buf).ok()?;
+        let filter = crate::bip158::BlockFilter::new(&buf);
+        self.current += 1;
+
+        Some((self.current - 1, filter))
+    }
+}
+
+struct FlatFiltersStoreInner {
+    file: std::fs::File,
+    path: PathBuf,
+}
+
+impl From<PoisonError<MutexGuard<'_, FlatFiltersStoreInner>>> for IteratableFilterStoreError {
+    fn from(_: PoisonError<MutexGuard<'_, FlatFiltersStoreInner>>) -> Self {
+        IteratableFilterStoreError::Poisoned
+    }
+}
+
+pub struct FlatFiltersStore(Mutex<FlatFiltersStoreInner>);
+
+impl FlatFiltersStore {
+    pub fn new(path: PathBuf) -> Self {
+        let file = std::fs::OpenOptions::new()
+            .read(true)
+            .write(true)
+            .create(true)
+            .truncate(false)
+            .open(&path)
+            .unwrap();
+
+        Self(Mutex::new(FlatFiltersStoreInner { file, path }))
+    }
+}
+
+impl TryFrom<&PathBuf> for FlatFiltersStore {
+    type Error = std::io::Error;
+
+    fn try_from(path: &PathBuf) -> Result<Self, Self::Error> {
+        let file = std::fs::OpenOptions::new()
+            .read(true)
+            .write(true)
+            .create(true)
+            .truncate(false)
+            .open(path)?;
+
+        Ok(Self(Mutex::new(FlatFiltersStoreInner {
+            file,
+            path: path.clone(),
+        })))
+    }
+}
+
+impl IntoIterator for FlatFiltersStore {
+    type Item = (u32, crate::bip158::BlockFilter);
+    type IntoIter = FiltersIterator;
+
+    fn into_iter(self) -> Self::IntoIter {
+        let mut inner = self.0.lock().unwrap();
+        inner.file.seek(SeekFrom::Start(4)).unwrap();
+        let reader = BufReader::new(inner.file.try_clone().unwrap());
+        FiltersIterator { reader, current: 0 }
+    }
+}
+
+impl IteratableFilterStore for FlatFiltersStore {
+    type I = FiltersIterator;
+    fn set_height(&self, height: u32) -> Result<(), IteratableFilterStoreError> {
+        let mut inner = self.0.lock()?;
+        inner.file.seek(SeekFrom::Start(0))?;
+        inner.file.write_all(&height.to_le_bytes())?;
+
+        Ok(())
+    }
+
+    fn get_height(&self) -> Result<u32, IteratableFilterStoreError> {
+        let mut inner = self.0.lock()?;
+
+        let mut buf = [0; 4];
+        inner.file.seek(SeekFrom::Start(0))?;
+        inner.file.read_exact(&mut buf)?;
+
+        Ok(u32::from_le_bytes(buf))
+    }
+
+    fn iter(&self) -> Result<Self::I, IteratableFilterStoreError> {
+        let inner = self.0.lock()?;
+        let new_file = File::open(inner.path.clone())?;
+        let mut reader = BufReader::new(new_file);
+        reader.seek(SeekFrom::Start(4))?;
+        Ok(FiltersIterator { reader, current: 1 })
+    }
+
+    fn put_filter(
+        &self,
+        block_filter: crate::bip158::BlockFilter,
+    ) -> Result<(), IteratableFilterStoreError> {
+        let length = block_filter.content.len() as u32;
+
+        if length > 1_000_000 {
+            return Err(IteratableFilterStoreError::FilterTooLarge);
+        }
+        let mut inner = self.0.lock()?;
+
+        inner.file.seek(SeekFrom::End(0))?;
+        inner.file.write_all(&length.to_le_bytes())?;
+        inner.file.write_all(&block_filter.content)?;
+
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::fs::remove_file;
+
+    use super::FlatFiltersStore;
+    use crate::bip158::BlockFilter;
+    use crate::IteratableFilterStore;
+
+    #[test]
+    fn test_filter_store() {
+        let path = "test_filter_store";
+        let store = FlatFiltersStore::new(path.into());
+
+        let res = store.get_height().unwrap_err();
+        assert!(matches!(res, crate::IteratableFilterStoreError::Io(_)));
+        store.set_height(1).expect("could not set height");
+        assert_eq!(store.get_height().unwrap(), 1);
+
+        let filter = BlockFilter::new(&[10, 11, 12, 13]);
+        store
+            .put_filter(filter.clone())
+            .expect("could not put filter");
+
+        let mut iter = store.iter().expect("could not get iterator");
+        assert_eq!((1, filter), iter.next().unwrap());
+
+        assert_eq!(iter.next(), None);
+        remove_file(path).expect("could not remove file after test");
+    }
+}
diff --git a/crates/floresta-compact-filters/src/lib.rs b/crates/floresta-compact-filters/src/lib.rs
@@ -11,18 +11,23 @@
 //! since we wouldn't have the filter for all blocks yet.
 use core::fmt::Debug;
 use core::ops::BitAnd;
+use std::fmt::Display;
 use std::io::Write;
+use std::sync::PoisonError;
+use std::sync::RwLockWriteGuard;
 
 use bitcoin::hashes::Hash;
 use bitcoin::Block;
 use bitcoin::BlockHash;
 use bitcoin::OutPoint;
 use bitcoin::Transaction;
 use bitcoin::Txid;
+use flat_filters_store::FlatFiltersStore;
 use floresta_chain::BlockConsumer;
 use log::error;
 
 mod bip158;
+pub mod flat_filters_store;
 pub mod kv_filter_database;
 pub mod network_filters;
 
@@ -41,6 +46,64 @@ pub trait BlockFilterStore: Send + Sync {
     fn get_height(&self) -> Option<u32>;
 }
 
+pub enum IteratableFilterStoreError {
+    /// I/O error
+    Io(std::io::Error),
+    /// End of the file
+    Eof,
+    /// Lock error
+    Poisoned,
+    /// Filter too large, probably a bug
+    FilterTooLarge,
+}
+
+impl Debug for IteratableFilterStoreError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            IteratableFilterStoreError::Io(e) => write!(f, "I/O error: {e}"),
+            IteratableFilterStoreError::Eof => write!(f, "End of file"),
+            IteratableFilterStoreError::Poisoned => write!(f, "Lock poisoned"),
+            IteratableFilterStoreError::FilterTooLarge => write!(f, "Filter too large"),
+        }
+    }
+}
+
+impl From<std::io::Error> for IteratableFilterStoreError {
+    fn from(e: std::io::Error) -> Self {
+        IteratableFilterStoreError::Io(e)
+    }
+}
+
+impl Display for IteratableFilterStoreError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        Debug::fmt(self, f)
+    }
+}
+
+impl From<PoisonError<RwLockWriteGuard<'_, FlatFiltersStore>>> for IteratableFilterStoreError {
+    fn from(_: PoisonError<RwLockWriteGuard<'_, FlatFiltersStore>>) -> Self {
+        IteratableFilterStoreError::Poisoned
+    }
+}
+
+pub trait IteratableFilterStore:
+    Send + Sync + IntoIterator<Item = (u32, bip158::BlockFilter)>
+{
+    type I: Iterator<Item = (u32, bip158::BlockFilter)>;
+    /// Fetches the first filter and sets our internal cursor to the first filter,
+    /// succeeding calls to [next] will return the next filter until we reach the end
+    fn iter(&self) -> Result<Self::I, IteratableFilterStoreError>;
+    /// Writes a new filter to the store
+    fn put_filter(
+        &self,
+        block_filter: bip158::BlockFilter,
+    ) -> Result<(), IteratableFilterStoreError>;
+    /// Persists the height of the last filter we have
+    fn set_height(&self, height: u32) -> Result<(), IteratableFilterStoreError>;
+    /// Fetches the height of the last filter we have
+    fn get_height(&self) -> Result<u32, IteratableFilterStoreError>;
+}
+
 /// All standard outputs type define in the Bitcoin network
 #[derive(Debug, Hash)]
 pub enum OutputTypes {