Integrate with db

Cargo.toml

@@ -10,7 +10,10 @@ fuser = "0.17.0"
 libc = "0.2.186"
 notify = "8.2.0"
 time = "0.3.49"
-
 symphonia = { version = "0.5", default-features = false, features = [
     "aac", "alac", "flac", "mp3", "ogg", "vorbis", "wav"
 ] }
+twox-hash = "2.1.2"
+
+sea-orm = { version = "2.0.0-rc", features = [ "sqlx-postgres", "runtime-tokio", "macros" ] }
+tokio = { version = "1", features = ["macros", "rt-multi-thread"] }

db/schema.sql

@@ -0,0 +1,20 @@
+CREATE TABLE items (
+    inode           BIGINT PRIMARY KEY,
+    name            TEXT NOT NULL,
+    original_path   TEXT NOT NULL,
+    local_path      TEXT NOT NULL,
+    file_type       TEXT NOT NULL CHECK (file_type IN ('directory', 'file')),
+    hash            BIGINT NOT NULL
+);
+
+CREATE TABLE music_metadata (
+    inode           BIGINT PRIMARY KEY REFERENCES items(inode) ON DELETE CASCADE,
+    track_title     TEXT NOT NULL,
+    album           TEXT NOT NULL,
+    track_number    INTEGER NOT NULL
+);
+
+CREATE TABLE music_metadata_artists (
+    inode           BIGINT NOT NULL REFERENCES music_metadata(inode) ON DELETE CASCADE,
+    artist          TEXT NOT NULL
+);

devenv.lock

@@ -55,11 +55,11 @@
     "devenv": {
       "locked": {
         "dir": "src/modules",
-        "lastModified": 1781800860,
-        "narHash": "sha256-LrEo0eC5ckMvjpBRCuk5q5/vjItKlxnb4n/clHNRZlk=",
+        "lastModified": 1782331842,
+        "narHash": "sha256-7CJ2EqNVPMq0ly39aaP6dGgdO627MqUtM/+Dm+QwNdU=",
         "owner": "cachix",
         "repo": "devenv",
-        "rev": "d59d872d80876d9eeb3e214d3b088bc4a14a9c4f",
+        "rev": "885e1c9d62cfa12232802de77b36aaded1ca609b",
         "type": "github"
       },
       "original": {
@@ -292,11 +292,11 @@
     "nixpkgs-src": {
       "flake": false,
       "locked": {
-        "lastModified": 1781454065,
-        "narHash": "sha256-d2xfDjnfRuf/xYGdu9VVRHiav/2w5hDL/5cw2TuVAXw=",
+        "lastModified": 1781607440,
+        "narHash": "sha256-rxO+uc/KFbSJp+pgyXRuAX6QlG9hJdnt0BXpEQRXY+U=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "9eac87a12312b8f60dd52e1c6e1a265f6fc7f5fc",
+        "rev": "3e41b24abd260e8f71dbe2f5737d24122f972158",
         "type": "github"
       },
       "original": {
@@ -311,11 +311,11 @@
         "nixpkgs-src": "nixpkgs-src"
       },
       "locked": {
-        "lastModified": 1781620901,
-        "narHash": "sha256-UF6scQlG+6lRkZBUpn/3KNavhOo5G8kDWhjVHcno8uc=",
+        "lastModified": 1782132010,
+        "narHash": "sha256-ZnAVHdVrotp80iIMm5CSR1fdxPlw7Uwmwxb+O/wsgZ8=",
         "owner": "cachix",
         "repo": "devenv-nixpkgs",
-        "rev": "2df109b343d3c68efd752e32a444a1d9b9f89afa",
+        "rev": "12866ae2dddbc0ab8b329915f8072bb9c75bde89",
         "type": "github"
       },
       "original": {

devenv.nix

@@ -28,11 +28,19 @@
     git
     just
 
-    ffprobe
-
     opencode
   ];
 
+  services.postgres = {
+    enable = true;
+    initialDatabases = [
+      {
+        name = "musicfs";
+        schema = ./db/schema.sql;
+      }
+    ];
+  };
+
   outputs = {
     rust-app = config.languages.rust.import ./. { };
   };

justfile

@@ -1,2 +1,4 @@
+alias b := build
+
 build:
     cargo build

src/local.rs

@@ -1,16 +1,22 @@
 use std::{
     collections::BTreeMap,
-    fs, io,
-    os::unix::fs::{MetadataExt, PermissionsExt},
+    fs,
+    hash::{Hash, Hasher},
+    io,
+    os::unix::{
+        ffi::OsStrExt,
+        fs::{MetadataExt, PermissionsExt},
+    },
     path::{Path, PathBuf},
     sync::{Arc, Mutex, mpsc},
     thread,
     time::{Duration, SystemTime},
 };
 
-use clap::builder::OsStr;
 use fuser::{Errno, FileAttr, Filesystem, Generation, INodeNo, Request};
 use notify::{Event, EventKind, RecursiveMode, Watcher};
+use sea_orm::entity::prelude::*;
+use twox_hash::XxHash64;
 
 use crate::file_watcher::FileWatcher;
 use crate::music_metadata::MusicMetadata;
@@ -30,45 +36,115 @@ struct LocalItem {
     file_type: FileType,
     metadata: fs::Metadata,
     music_metadata: Option<MusicMetadata>,
+    hash: u64,
+}
+
+impl LocalItem {
+    fn compute_hash(&self) -> u64 {
+        let seed = 1234;
+        let mut hasher = XxHash64::with_seed(seed);
+
+        let metadata = &self.metadata;
+        let mtime: u64 = metadata
+            .modified()
+            .unwrap_or(SystemTime::UNIX_EPOCH)
+            .duration_since(SystemTime::UNIX_EPOCH)
+            .map(|d| d.as_secs())
+            .unwrap_or(0);
+        let crtime: u64 = metadata
+            .created()
+            .unwrap_or(SystemTime::UNIX_EPOCH)
+            .duration_since(SystemTime::UNIX_EPOCH)
+            .map(|d| d.as_secs())
+            .unwrap_or(0);
+
+        hasher.write_u64(self.inode.0);
+        hasher.write(self.name.as_bytes());
+        hasher.write(self.original_path.as_os_str().as_bytes());
+        hasher.write(self.local_path.as_os_str().as_bytes());
+        hasher.write_u64(metadata.ctime() as u64);
+        hasher.write_u64(mtime);
+        hasher.write_u64(crtime);
+
+        return hasher.finish();
+    }
 }
 
-#[derive(Debug)]
 pub struct LocalOrigin {
     source: PathBuf,
     destination: PathBuf,
     files: Arc<Mutex<BTreeMap<INodeNo, LocalItem>>>,
+
+    client: sea_orm::DatabaseConnection,
+}
+
+impl std::fmt::Debug for LocalOrigin {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("LocalOrigin")
+            .field("source", &self.source)
+            .field("destination", &self.destination)
+            .field("files", &self.files)
+            .finish()
+    }
 }
 
 impl LocalOrigin {
-    pub fn new(source: String, destination: String) -> Result<LocalOrigin, io::Error> {
+    pub async fn new(
+        source: String,
+        destination: String,
+        client: sea_orm::DatabaseConnection,
+    ) -> Result<LocalOrigin, io::Error> {
         println!("Initializing the LocalOrigin");
 
-        let map: Arc<Mutex<BTreeMap<INodeNo, LocalItem>>> = Arc::new(Mutex::new(BTreeMap::new()));
+        let snapshot = LocalOrigin::build_snapshot(Path::new(&source), Path::new(&destination))?;
 
-        let local_root = LocalItem {
-            inode: INodeNo::ROOT,
-            name: "/".to_string(),
-            original_path: PathBuf::from(&source),
-            local_path: PathBuf::from(&destination),
-            file_type: FileType::Directory,
-            metadata: fs::metadata(&source).unwrap(),
-            music_metadata: Option::None,
-        };
-        map.lock().unwrap().insert(INodeNo::ROOT, local_root);
+        let db_map: std::collections::HashMap<i64, i64> = Entity::find()
+            .all(&client)
+            .await
+            .unwrap()
+            .into_iter()
+            .map(|e| (e.inode, e.hash))
+            .collect();
 
-        match LocalOrigin::read_source(Path::new(&source), Path::new(&destination), &map) {
-            Ok(_) => {
-                let local_origin = LocalOrigin {
-                    source: source.into(),
-                    destination: destination.into(),
-                    files: map,
-                };
-                dbg!(&local_origin);
+        let mut to_insert: Vec<ActiveModel> = vec![];
+        let mut to_update: Vec<ActiveModel> = vec![];
+        let mut to_delete: Vec<i64> = vec![];
 
-                return Ok(local_origin);
+        for (ino, item) in &snapshot {
+            let ino_i64 = ino.0 as i64;
+            match db_map.get(&ino_i64) {
+                None => to_insert.push(ActiveModel::from(item)),
+                Some(&db_hash) if db_hash != item.hash as i64 => {
+                    to_update.push(ActiveModel::from(item))
+                }
+                _ => {}
             }
-            Err(err) => return Err(err),
         }
+
+        let fresh_inodes: std::collections::HashSet<i64> =
+            snapshot.keys().map(|i| i.0 as i64).collect();
+        for ino in db_map.keys().filter(|i| !fresh_inodes.contains(i)) {
+            to_delete.push(*ino);
+        }
+
+        if !to_insert.is_empty() {
+            Entity::insert_many(to_insert).exec(&client).await.unwrap();
+        }
+        for model in to_update {
+            model.update(&client).await.unwrap();
+        }
+        for ino in to_delete {
+            Entity::delete_by_id(ino).exec(&client).await.unwrap();
+        }
+
+        let local_origin = LocalOrigin {
+            source: source.into(),
+            destination: destination.into(),
+            files: Arc::new(Mutex::new(snapshot)),
+            client,
+        };
+        //dbg!(&local_origin);
+        Ok(local_origin)
     }
 
     fn fill_fileset(
@@ -76,84 +152,100 @@ impl LocalOrigin {
         source: &Path,
         destination: &Path,
     ) {
+        match LocalOrigin::build_snapshot(source, destination) {
+            Ok(new_snapshot) => {
+                let mut files = map.lock().unwrap();
+
+                files.retain(|ino, _| new_snapshot.contains_key(ino));
+
+                for (ino, new_item) in new_snapshot {
+                    match files.get(&ino) {
+                        Some(existing) if existing.hash == new_item.hash => {}
+                        _ => {
+                            files.insert(ino, new_item);
+                        }
+                    }
+                }
+            }
+            Err(e) => eprintln!("Error while reading source: {e}"),
+        }
+    }
+
+    fn build_snapshot(
+        source: &Path,
+        destination: &Path,
+    ) -> Result<BTreeMap<INodeNo, LocalItem>, io::Error> {
+        let mut map = BTreeMap::new();
+
         let local_root = LocalItem {
             inode: INodeNo::ROOT,
             name: "/".to_string(),
             original_path: source.to_path_buf(),
             local_path: destination.to_path_buf(),
             file_type: FileType::Directory,
-            metadata: fs::metadata(&source).unwrap(),
-            music_metadata: Option::None,
+            metadata: fs::metadata(source)?,
+            music_metadata: None,
+            hash: 0,
         };
-        map.lock().unwrap().insert(INodeNo::ROOT, local_root);
+        let local_root = LocalItem {
+            hash: local_root.compute_hash(),
+            ..local_root
+        };
+        map.insert(INodeNo::ROOT, local_root);
 
-        let _ = LocalOrigin::read_source(&source, &destination, &map)
-            .inspect_err(|e| eprintln!("Error while reading source: {e}"));
+        LocalOrigin::read_into_map(source, destination, &mut map)?;
+        Ok(map)
     }
 
-    fn read_source(
+    fn read_into_map(
         source: &Path,
         destination: &Path,
-        map: &Arc<Mutex<BTreeMap<INodeNo, LocalItem>>>,
+        map: &mut BTreeMap<INodeNo, LocalItem>,
     ) -> Result<(), io::Error> {
-        match fs::read_dir(source) {
-            Ok(dir) => {
-                for item in dir {
-                    match item {
-                        Ok(entry) => {
-                            let name = entry.file_name().to_string_lossy().into_owned();
-                            let item_path: PathBuf = entry.path();
-                            let metadata: fs::Metadata = entry.metadata().unwrap();
-                            let file_type = if entry.file_type().unwrap().is_dir() {
-                                FileType::Directory
-                            } else {
-                                FileType::File
-                            };
+        for item in fs::read_dir(source)? {
+            let entry = item?;
+            let name = entry.file_name().to_string_lossy().into_owned();
+            let item_path = entry.path();
+            let metadata = entry.metadata()?;
+            let file_type = if entry.file_type()?.is_dir() {
+                FileType::Directory
+            } else {
+                FileType::File
+            };
 
-                            let mut local_path: PathBuf = PathBuf::new();
-                            // local_path.push(destination);
-                            local_path.push(name.clone());
+            let mut local_path = PathBuf::new();
+            local_path.push(name.clone());
 
-                            let music_metadata: Option<MusicMetadata> = if item_path
-                                .extension()
-                                .and_then(std::ffi::OsStr::to_str)
-                                .unwrap()
-                                == "flac"
-                            {
-                                MusicMetadata::parse_music_metadata(&item_path)
-                            } else {
-                                Option::None
-                            };
+            let music_metadata =
+                if item_path.extension().and_then(std::ffi::OsStr::to_str) == Some("flac") {
+                    MusicMetadata::parse_music_metadata(&item_path)
+                } else {
+                    None
+                };
 
-                            let local_item = LocalItem {
-                                inode: INodeNo(metadata.ino()),
-                                name: name,
-                                original_path: item_path.clone(),
-                                local_path: local_path,
-                                file_type: file_type,
-                                metadata: metadata,
-                                music_metadata: music_metadata,
-                            };
-                            let inode = local_item.inode;
+            let inode = INodeNo(metadata.ino());
+            let local_item = LocalItem {
+                inode,
+                name,
+                original_path: item_path.clone(),
+                local_path,
+                file_type,
+                metadata,
+                music_metadata,
+                hash: 0,
+            };
+            let local_item = LocalItem {
+                hash: local_item.compute_hash(),
+                ..local_item
+            };
 
-                            map.lock().unwrap().insert(inode, local_item);
+            map.insert(inode, local_item);
 
-                            if file_type == FileType::Directory {
-                                match LocalOrigin::read_source(&item_path, destination, map) {
-                                    Ok(_) => {}
-                                    Err(err) => return Err(err),
-                                };
-                            }
-                        }
-
-                        Err(err) => return Err(err),
-                    }
-                }
+            if file_type == FileType::Directory {
+                LocalOrigin::read_into_map(&item_path, destination, map)?;
             }
-            Err(err) => return Err(err),
-        };
-
-        return Ok(());
+        }
+        Ok(())
     }
 
     fn local_file_to_file_attr(local_file: &LocalItem) -> FileAttr {
@@ -243,12 +335,13 @@ impl Filesystem for LocalOrigin {
                 .lock()
                 .unwrap()
                 .iter()
+                .filter(|(key, _)| **key != INodeNo::ROOT)
                 .skip(offset as usize)
                 .enumerate()
             {
                 let offset: u64 = 2 + i as u64;
                 let file_type: fuser::FileType = LocalOrigin::file_type(value.file_type);
-                let path = &value.local_path;
+                let path = &value.name;
 
                 let buffer_full: bool = reply.add(*key, offset, file_type, path);
 
@@ -339,3 +432,61 @@ impl FileWatcher for LocalOrigin {
         });
     }
 }
+
+#[derive(Clone, Debug, PartialEq, Eq, sea_orm::DeriveEntityModel)]
+#[sea_orm(table_name = "items")]
+pub struct Model {
+    #[sea_orm(primary_key, auto_increment = false)]
+    pub inode: i64,
+    pub name: String,
+    pub original_path: String,
+    pub local_path: String,
+    pub file_type: String,
+    pub hash: i64,
+}
+
+pub type LocalItemEntity = Model;
+
+#[derive(Copy, Clone, Debug, sea_orm::EnumIter, sea_orm::DeriveRelation)]
+pub enum Relation {}
+
+impl sea_orm::ActiveModelBehavior for ActiveModel {}
+
+impl From<&LocalItem> for ActiveModel {
+    fn from(item: &LocalItem) -> Self {
+        use sea_orm::ActiveValue::Set;
+        ActiveModel {
+            inode: Set(item.inode.0 as i64),
+            name: Set(item.name.clone()),
+            original_path: Set(item.original_path.to_string_lossy().into_owned()),
+            local_path: Set(item.local_path.to_string_lossy().into_owned()),
+            file_type: Set(match item.file_type {
+                FileType::Directory => "directory".to_string(),
+                FileType::File => "file".to_string(),
+            }),
+            hash: Set(item.hash as i64),
+        }
+    }
+}
+
+impl From<(LocalItemEntity, fs::Metadata)> for LocalItem {
+    fn from((entity, metadata): (LocalItemEntity, fs::Metadata)) -> Self {
+        let local_item = LocalItem {
+            inode: INodeNo(entity.inode as u64),
+            name: entity.name,
+            original_path: PathBuf::from(entity.original_path),
+            local_path: PathBuf::from(entity.local_path),
+            file_type: match entity.file_type.as_str() {
+                "directory" => FileType::Directory,
+                _ => FileType::File,
+            },
+            metadata,
+            music_metadata: None,
+            hash: 0,
+        };
+        LocalItem {
+            hash: local_item.compute_hash(),
+            ..local_item
+        }
+    }
+}

src/main.rs

@@ -17,9 +17,13 @@ struct Args {
 
     #[arg(short, long, required = true)]
     source: String,
+
+    #[arg(short, long, required = true)]
+    database: String,
 }
 
-fn main() {
+#[tokio::main]
+async fn main() {
     let args = Args::parse();
     let mountpoint = args.mountpoint;
 
@@ -32,9 +36,13 @@ fn main() {
     })
     .expect("Error setting Ctrl+C handler");
 
+    let db = sea_orm::Database::connect(&args.database).await.unwrap();
+
     // TODO don't think clone is necessary
     // TODO unwrap might not be safe and better would be to handle it properly
-    let fs = local::LocalOrigin::new(args.source.clone(), mountpoint.clone()).unwrap();
+    let fs = local::LocalOrigin::new(args.source.clone(), mountpoint.clone(), db)
+        .await
+        .unwrap();
 
     // start watching for changes in source files
     fs.watch();