From fd236a1b57b610047deafa16a3738f32ca591f54 Mon Sep 17 00:00:00 2001 From: Brendan Allan Date: Tue, 6 Jun 2023 16:42:52 +0200 Subject: [PATCH] [ENG-679] Reserve ids for built in indexer rules (#909) * indexer rules pub ids * should work? * better migrator * errors * debugging * maybe? * double migrate * please * maybe fix? * update lockfile * SD_ACCEPT_DATA_LOSS message * put tracing back * dumb * fix system indexer rule ui --- Cargo.lock | 8 +- Cargo.toml | 6 +- apps/desktop/src-tauri/Cargo.toml | 2 +- apps/server/src/main.rs | 7 +- core/prisma/schema.prisma | 1 + core/src/lib.rs | 1 - core/src/library/config.rs | 66 +++-- core/src/library/manager.rs | 65 +++-- core/src/location/indexer/rules.rs | 245 +++++++++++++++--- core/src/migrations.rs | 25 -- core/src/node/config.rs | 35 ++- core/src/util/db.rs | 20 +- core/src/util/migrator.rs | 200 ++++++-------- core/src/util/mod.rs | 1 - core/src/util/seeder.rs | 144 ---------- .../IndexerRuleEditor/RuleButton.tsx | 9 +- packages/client/src/core.ts | 2 +- 17 files changed, 432 insertions(+), 405 deletions(-) delete mode 100644 core/src/migrations.rs delete mode 100644 core/src/util/seeder.rs diff --git a/Cargo.lock b/Cargo.lock index c0e0508ce..de7157dc3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5661,7 +5661,7 @@ dependencies = [ [[package]] name = "prisma-client-rust" version = "0.6.8" -source = "git+https://github.com/Brendonovich/prisma-client-rust?rev=3b805c459ec1d52f163ecdc527b0d82e6556a022#3b805c459ec1d52f163ecdc527b0d82e6556a022" +source = "git+https://github.com/Brendonovich/prisma-client-rust?rev=7e67b4550dd5323d479c96008678032f396b9060#7e67b4550dd5323d479c96008678032f396b9060" dependencies = [ "base64 0.13.1", "bigdecimal", @@ -5694,7 +5694,7 @@ dependencies = [ [[package]] name = "prisma-client-rust-cli" version = "0.6.8" -source = "git+https://github.com/Brendonovich/prisma-client-rust?rev=3b805c459ec1d52f163ecdc527b0d82e6556a022#3b805c459ec1d52f163ecdc527b0d82e6556a022" +source = "git+https://github.com/Brendonovich/prisma-client-rust?rev=7e67b4550dd5323d479c96008678032f396b9060#7e67b4550dd5323d479c96008678032f396b9060" dependencies = [ "directories", "flate2", @@ -5714,7 +5714,7 @@ dependencies = [ [[package]] name = "prisma-client-rust-macros" version = "0.6.8" -source = "git+https://github.com/Brendonovich/prisma-client-rust?rev=3b805c459ec1d52f163ecdc527b0d82e6556a022#3b805c459ec1d52f163ecdc527b0d82e6556a022" +source = "git+https://github.com/Brendonovich/prisma-client-rust?rev=7e67b4550dd5323d479c96008678032f396b9060#7e67b4550dd5323d479c96008678032f396b9060" dependencies = [ "convert_case 0.6.0", "proc-macro2", @@ -5726,7 +5726,7 @@ dependencies = [ [[package]] name = "prisma-client-rust-sdk" version = "0.6.8" -source = "git+https://github.com/Brendonovich/prisma-client-rust?rev=3b805c459ec1d52f163ecdc527b0d82e6556a022#3b805c459ec1d52f163ecdc527b0d82e6556a022" +source = "git+https://github.com/Brendonovich/prisma-client-rust?rev=7e67b4550dd5323d479c96008678032f396b9060#7e67b4550dd5323d479c96008678032f396b9060" dependencies = [ "convert_case 0.5.0", "dmmf", diff --git a/Cargo.toml b/Cargo.toml index 39c14cfd1..5baea3b60 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,19 +18,19 @@ edition = "2021" repository = "https://github.com/spacedriveapp/spacedrive" [workspace.dependencies] -prisma-client-rust = { git = "https://github.com/Brendonovich/prisma-client-rust", rev = "3b805c459ec1d52f163ecdc527b0d82e6556a022", features = [ +prisma-client-rust = { git = "https://github.com/Brendonovich/prisma-client-rust", rev = "7e67b4550dd5323d479c96008678032f396b9060", features = [ "rspc", "sqlite-create-many", "migrations", "sqlite", ] } -prisma-client-rust-cli = { git = "https://github.com/Brendonovich/prisma-client-rust", rev = "3b805c459ec1d52f163ecdc527b0d82e6556a022", features = [ +prisma-client-rust-cli = { git = "https://github.com/Brendonovich/prisma-client-rust", rev = "7e67b4550dd5323d479c96008678032f396b9060", features = [ "rspc", "sqlite-create-many", "migrations", "sqlite", ] } -prisma-client-rust-sdk = { git = "https://github.com/Brendonovich/prisma-client-rust", rev = "3b805c459ec1d52f163ecdc527b0d82e6556a022", features = [ +prisma-client-rust-sdk = { git = "https://github.com/Brendonovich/prisma-client-rust", rev = "7e67b4550dd5323d479c96008678032f396b9060", features = [ "sqlite", ] } diff --git a/apps/desktop/src-tauri/Cargo.toml b/apps/desktop/src-tauri/Cargo.toml index 763cad131..9be31d6e3 100644 --- a/apps/desktop/src-tauri/Cargo.toml +++ b/apps/desktop/src-tauri/Cargo.toml @@ -18,7 +18,7 @@ httpz = { workspace = true, features = [ sd-core = { path = "../../../core", features = [ "ffmpeg", "location-watcher", - "heif", + # "heif", ] } tokio = { workspace = true, features = ["sync"] } window-shadows = "0.2.1" diff --git a/apps/server/src/main.rs b/apps/server/src/main.rs index 17218f856..ced5e3579 100644 --- a/apps/server/src/main.rs +++ b/apps/server/src/main.rs @@ -34,7 +34,12 @@ async fn main() { let _guard = Node::init_logger(&data_dir); - let (node, router) = Node::new(data_dir).await.expect("Unable to create node"); + let (node, router) = match Node::new(data_dir).await { + Ok(d) => d, + Err(e) => { + panic!("{}", e.to_string()) + } + }; let signal = utils::axum_shutdown_signal(node.clone()); let app = axum::Router::new() diff --git a/core/prisma/schema.prisma b/core/prisma/schema.prisma index c5bd439e1..8a1412640 100644 --- a/core/prisma/schema.prisma +++ b/core/prisma/schema.prisma @@ -446,6 +446,7 @@ model Comment { model IndexerRule { id Int @id @default(autoincrement()) + pub_id Bytes? @unique name String default Boolean @default(false) rules_per_kind Bytes diff --git a/core/src/lib.rs b/core/src/lib.rs index 3d1da1153..59f4e6baf 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -29,7 +29,6 @@ pub mod custom_uri; pub(crate) mod job; pub mod library; pub(crate) mod location; -pub(crate) mod migrations; pub(crate) mod node; pub(crate) mod object; pub(crate) mod p2p; diff --git a/core/src/library/config.rs b/core/src/library/config.rs index e33ce6769..41d2bb09c 100644 --- a/core/src/library/config.rs +++ b/core/src/library/config.rs @@ -1,17 +1,13 @@ -use std::{marker::PhantomData, path::PathBuf}; - use serde::{Deserialize, Serialize}; use specta::Type; use uuid::Uuid; -use crate::{migrations, util::migrator::FileMigrator}; - -use super::LibraryManagerError; - -const MIGRATOR: FileMigrator = FileMigrator { - current_version: migrations::LIBRARY_VERSION, - migration_fn: migrations::migration_library, - phantom: PhantomData, +use crate::{ + prisma::{indexer_rule, PrismaClient}, + util::{ + db::uuid_to_bytes, + migrator::{Migrate, MigratorError}, + }, }; /// LibraryConfig holds the configuration for a specific library. This is stored as a '{uuid}.sdlibrary' file. @@ -26,18 +22,46 @@ pub struct LibraryConfig { // pub is_encrypted: bool, } -impl LibraryConfig { - /// read will read the configuration from disk and return it. - pub(super) fn read(file_dir: PathBuf) -> Result { - MIGRATOR.load(&file_dir).map_err(Into::into) - } +#[async_trait::async_trait] +impl Migrate for LibraryConfig { + const CURRENT_VERSION: u32 = 1; + + type Ctx = PrismaClient; + + async fn migrate( + to_version: u32, + _config: &mut serde_json::Map, + db: &Self::Ctx, + ) -> Result<(), MigratorError> { + match to_version { + 0 => {} + 1 => { + let rules = vec![ + format!("No OS protected"), + format!("No Hidden"), + format!("Only Git Repositories"), + format!("Only Images"), + ]; + + db._batch( + rules + .into_iter() + .enumerate() + .map(|(i, name)| { + db.indexer_rule().update_many( + vec![indexer_rule::name::equals(name)], + vec![indexer_rule::pub_id::set(Some(uuid_to_bytes( + Uuid::from_u128(i as u128), + )))], + ) + }) + .collect::>(), + ) + .await?; + } + v => unreachable!("Missing migration for library version {}", v), + } - /// save will write the configuration back to disk - pub(super) fn save( - file_dir: PathBuf, - config: &LibraryConfig, - ) -> Result<(), LibraryManagerError> { - MIGRATOR.save(&file_dir, config.clone())?; Ok(()) } } diff --git a/core/src/library/manager.rs b/core/src/library/manager.rs index c19c3fd33..3584eb5f1 100644 --- a/core/src/library/manager.rs +++ b/core/src/library/manager.rs @@ -1,15 +1,14 @@ use crate::{ invalidate_query, - location::LocationManagerError, + location::{indexer::rules, LocationManagerError}, node::Platform, object::orphan_remover::OrphanRemoverActor, prisma::{location, node, PrismaClient}, sync::{SyncManager, SyncMessage}, util::{ - db::{load_and_migrate, MigrationError}, + db, error::{FileIOError, NonUtf8PathError}, - migrator::MigratorError, - seeder::{indexer_rules_seeder, SeederError}, + migrator::{Migrate, MigratorError}, }, NodeContext, }; @@ -57,14 +56,14 @@ pub enum LibraryManagerError { Migration(String), #[error("failed to parse uuid")] Uuid(#[from] uuid::Error), - #[error("failed to run seeder")] - Seeder(#[from] SeederError), + #[error("failed to run indexer rules seeder")] + IndexerRulesSeeder(#[from] rules::SeederError), #[error("failed to initialise the key manager")] KeyManager(#[from] sd_crypto::Error), - #[error("failed to run library migrations")] + #[error("failed to run library migrations: {0}")] MigratorError(#[from] MigratorError), #[error("error migrating the library: {0}")] - MigrationError(#[from] MigrationError), + MigrationError(#[from] db::MigrationError), #[error("invalid library configuration: {0}")] InvalidConfig(String), #[error(transparent)] @@ -154,41 +153,41 @@ impl LibraryManager { .await .map_err(|e| FileIOError::from((&libraries_dir, e)))? { - let entry_path = entry.path(); + let config_path = entry.path(); let metadata = entry .metadata() .await - .map_err(|e| FileIOError::from((&entry_path, e)))?; + .map_err(|e| FileIOError::from((&config_path, e)))?; if metadata.is_file() - && entry_path + && config_path .extension() .map(|ext| ext == "sdlibrary") .unwrap_or(false) { - let Some(Ok(library_id)) = entry_path + let Some(Ok(library_id)) = config_path .file_stem() .and_then(|v| v.to_str().map(Uuid::from_str)) else { - warn!("Attempted to load library from path '{}' but it has an invalid filename. Skipping...", entry_path.display()); + warn!("Attempted to load library from path '{}' but it has an invalid filename. Skipping...", config_path.display()); continue; }; - let db_path = entry_path.with_extension("db"); + let db_path = config_path.with_extension("db"); match fs::metadata(&db_path).await { Ok(_) => {} Err(e) if e.kind() == io::ErrorKind::NotFound => { warn!( "Found library '{}' but no matching database file was found. Skipping...", - entry_path.display() + config_path.display() ); continue; } Err(e) => return Err(FileIOError::from((db_path, e)).into()), } - let config = LibraryConfig::read(entry_path)?; - libraries - .push(Self::load(library_id, &db_path, config, node_context.clone()).await?); + libraries.push( + Self::load(library_id, &db_path, config_path, node_context.clone()).await?, + ); } } @@ -222,21 +221,19 @@ impl LibraryManager { )); } - LibraryConfig::save( - Path::new(&self.libraries_dir).join(format!("{id}.sdlibrary")), - &config, - )?; + let config_path = self.libraries_dir.join(format!("{id}.sdlibrary")); + config.save(&config_path)?; let library = Self::load( id, self.libraries_dir.join(format!("{id}.db")), - config.clone(), + config_path, self.node_context.clone(), ) .await?; // Run seeders - indexer_rules_seeder(&library.db).await?; + rules::seeder(&library.db).await?; invalidate_query!(library, "library.list"); @@ -282,8 +279,8 @@ impl LibraryManager { } LibraryConfig::save( - Path::new(&self.libraries_dir).join(format!("{id}.sdlibrary")), &library.config, + &self.libraries_dir.join(format!("{id}.sdlibrary")), )?; invalidate_query!(library, "library.list"); @@ -361,19 +358,19 @@ impl LibraryManager { pub(crate) async fn load( id: Uuid, db_path: impl AsRef, - config: LibraryConfig, + config_path: PathBuf, node_context: NodeContext, ) -> Result { let db_path = db_path.as_ref(); - let db = Arc::new( - load_and_migrate(&format!( - "file:{}", - db_path.as_os_str().to_str().ok_or_else(|| { - LibraryManagerError::NonUtf8Path(NonUtf8PathError(db_path.into())) - })? - )) - .await?, + let db_url = format!( + "file:{}", + db_path.as_os_str().to_str().ok_or_else(|| { + LibraryManagerError::NonUtf8Path(NonUtf8PathError(db_path.into())) + })? ); + let db = Arc::new(db::load_and_migrate(&db_url).await?); + + let config = LibraryConfig::load_and_migrate(&config_path, &db).await?; let node_config = node_context.config.get().await; diff --git a/core/src/location/indexer/rules.rs b/core/src/location/indexer/rules.rs index a3bc6551a..8e15ba328 100644 --- a/core/src/location/indexer/rules.rs +++ b/core/src/location/indexer/rules.rs @@ -1,7 +1,10 @@ use crate::{ library::Library, - prisma::{indexer_rule, PrismaClient}, - util::error::{FileIOError, NonUtf8PathError}, + prisma::indexer_rule, + util::{ + db::uuid_to_bytes, + error::{FileIOError, NonUtf8PathError}, + }, }; use chrono::{DateTime, Utc}; @@ -19,6 +22,7 @@ use std::{ use thiserror::Error; use tokio::fs; use tracing::debug; +use uuid::Uuid; #[derive(Error, Debug)] pub enum IndexerRuleError { @@ -121,7 +125,13 @@ impl IndexerRuleCreateArgs { library .db .indexer_rule() - .create(self.name, rules_data, vec![]) + .create( + self.name, + rules_data, + vec![indexer_rule::pub_id::set(Some(uuid_to_bytes( + generate_pub_id(), + )))], + ) .exec() .await?, )) @@ -461,36 +471,6 @@ impl IndexerRule { try_join_all(self.rules.iter().map(|rule| rule.apply(source.as_ref()))).await } - pub async fn save(self, client: &PrismaClient) -> Result<(), IndexerRuleError> { - if let Some(id) = self.id { - client - .indexer_rule() - .upsert( - indexer_rule::id::equals(id), - indexer_rule::create( - self.name, - rmp_serde::to_vec_named(&self.rules)?, - vec![indexer_rule::default::set(self.default)], - ), - vec![indexer_rule::date_modified::set(Utc::now().into())], - ) - .exec() - .await?; - } else { - client - .indexer_rule() - .create( - self.name, - rmp_serde::to_vec_named(&self.rules)?, - vec![indexer_rule::default::set(self.default)], - ) - .exec() - .await?; - } - - Ok(()) - } - pub async fn apply_all( rules: &[IndexerRule], source: impl AsRef, @@ -626,6 +606,205 @@ async fn reject_dir_for_its_children( Ok(true) } +pub fn generate_pub_id() -> Uuid { + loop { + let pub_id = Uuid::new_v4(); + if pub_id.as_u128() >= 0xFFF { + return pub_id; + } + } +} + +mod seeder { + use crate::{ + location::indexer::rules::{IndexerRuleError, RulePerKind}, + prisma::PrismaClient, + util::db::uuid_to_bytes, + }; + use sd_prisma::prisma::indexer_rule; + use thiserror::Error; + use uuid::Uuid; + + #[derive(Error, Debug)] + pub enum SeederError { + #[error("Failed to run indexer rules seeder: {0}")] + IndexerRules(#[from] IndexerRuleError), + #[error("An error occurred with the database while applying migrations: {0}")] + DatabaseError(#[from] prisma_client_rust::QueryError), + } + + struct SystemIndexerRule { + name: &'static str, + rules: Vec, + } + + pub async fn seeder(client: &PrismaClient) -> Result<(), SeederError> { + // DO NOT REORDER THIS ARRAY! + for (i, rule) in [ + no_os_protected(), + no_hidden(), + only_git_repos(), + only_images(), + ] + .into_iter() + .enumerate() + { + let pub_id = uuid_to_bytes(Uuid::from_u128(i as u128)); + let rules = rmp_serde::to_vec_named(&rule.rules).map_err(IndexerRuleError::from)?; + + client + .indexer_rule() + .upsert( + indexer_rule::pub_id::equals(pub_id.clone()), + indexer_rule::create( + rule.name.to_string(), + rules.clone(), + vec![indexer_rule::pub_id::set(Some(pub_id.clone()))], + ), + vec![ + indexer_rule::name::set(rule.name.to_string()), + indexer_rule::rules_per_kind::set(rules), + indexer_rule::pub_id::set(Some(pub_id.clone())), + ], + ) + .exec() + .await?; + } + + Ok(()) + } + + fn no_os_protected() -> SystemIndexerRule { + SystemIndexerRule { + // TODO: On windows, beside the listed files, any file with the FILE_ATTRIBUTE_SYSTEM should be considered a system file + // https://learn.microsoft.com/en-us/windows/win32/fileio/file-attribute-constants#FILE_ATTRIBUTE_SYSTEM + name: "No OS protected", + rules: vec![ + RulePerKind::new_reject_files_by_globs_str( + [ + vec![ + "**/.spacedrive", + ], + // Globset, even on Windows, requires the use of / as a separator + // https://github.com/github/gitignore/blob/main/Global/Windows.gitignore + // https://learn.microsoft.com/en-us/windows/win32/fileio/naming-a-file + #[cfg(target_os = "windows")] + vec![ + // Windows thumbnail cache files + "**/{Thumbs.db,Thumbs.db:encryptable,ehthumbs.db,ehthumbs_vista.db}", + // Dump file + "**/*.stackdump", + // Folder config file + "**/[Dd]esktop.ini", + // Recycle Bin used on file shares + "**/$RECYCLE.BIN", + // Chkdsk recovery directory + "**/FOUND.[0-9][0-9][0-9]", + // Reserved names + "**/{CON,PRN,AUX,NUL,COM0,COM1,COM2,COM3,COM4,COM5,COM6,COM7,COM8,COM9,LPT0,LPT1,LPT2,LPT3,LPT4,LPT5,LPT6,LPT7,LPT8,LPT9}", + "**/{CON,PRN,AUX,NUL,COM0,COM1,COM2,COM3,COM4,COM5,COM6,COM7,COM8,COM9,LPT0,LPT1,LPT2,LPT3,LPT4,LPT5,LPT6,LPT7,LPT8,LPT9}.*", + // User special files + "C:/Users/*/NTUSER.DAT*", + "C:/Users/*/ntuser.dat*", + "C:/Users/*/{ntuser.ini,ntuser.dat,NTUSER.DAT}", + // User special folders (most of these the user dont even have permission to access) + "C:/Users/*/{Cookies,AppData,NetHood,Recent,PrintHood,SendTo,Templates,Start Menu,Application Data,Local Settings}", + // System special folders + "C:/{$Recycle.Bin,$WinREAgent,Documents and Settings,Program Files,Program Files (x86),ProgramData,Recovery,PerfLogs,Windows,Windows.old}", + // NTFS internal dir, can exists on any drive + "[A-Z]:/System Volume Information", + // System special files + "C:/{config,pagefile,hiberfil}.sys", + // Windows can create a swapfile on any drive + "[A-Z]:/swapfile.sys", + "C:/DumpStack.log.tmp", + ], + // https://github.com/github/gitignore/blob/main/Global/macOS.gitignore + // https://developer.apple.com/library/archive/documentation/FileManagement/Conceptual/FileSystemProgrammingGuide/FileSystemOverview/FileSystemOverview.html#//apple_ref/doc/uid/TP40010672-CH2-SW14 + #[cfg(any(target_os = "ios", target_os = "macos"))] + vec![ + "**/.{DS_Store,AppleDouble,LSOverride}", + // Icon must end with two \r + "**/Icon\r\r", + // Thumbnails + "**/._*", + ], + #[cfg(target_os = "macos")] + vec![ + "/{System,Network,Library,Applications}", + "/Users/*/{Library,Applications}", + // Files that might appear in the root of a volume + "**/.{DocumentRevisions-V100,fseventsd,Spotlight-V100,TemporaryItems,Trashes,VolumeIcon.icns,com.apple.timemachine.donotpresent}", + // Directories potentially created on remote AFP share + "**/.{AppleDB,AppleDesktop,apdisk}", + "**/{Network Trash Folder,Temporary Items}", + ], + // https://github.com/github/gitignore/blob/main/Global/Linux.gitignore + #[cfg(target_os = "linux")] + vec![ + "**/*~", + // temporary files which can be created if a process still has a handle open of a deleted file + "**/.fuse_hidden*", + // KDE directory preferences + "**/.directory", + // Linux trash folder which might appear on any partition or disk + "**/.Trash-*", + // .nfs files are created when an open file is removed but is still being accessed + "**/.nfs*", + ], + #[cfg(target_os = "android")] + vec![ + "**/.nomedia", + "**/.thumbnails", + ], + // https://en.wikipedia.org/wiki/Unix_filesystem#Conventional_directory_layout + // https://en.wikipedia.org/wiki/Filesystem_Hierarchy_Standard + #[cfg(target_family = "unix")] + vec![ + // Directories containing unix memory/device mapped files/dirs + "/{dev,sys,proc}", + // Directories containing special files for current running programs + "/{run,var,boot}", + // ext2-4 recovery directory + "**/lost+found", + ], + ] + .into_iter() + .flatten() + ).unwrap(), + ], + } + } + + fn no_hidden() -> SystemIndexerRule { + SystemIndexerRule { + name: "No Hidden", + rules: vec![RulePerKind::new_reject_files_by_globs_str(["**/.*"]).unwrap()], + } + } + + fn only_git_repos() -> SystemIndexerRule { + SystemIndexerRule { + name: "Only Git Repositories", + rules: vec![RulePerKind::AcceptIfChildrenDirectoriesArePresent( + [".git".to_string()].into_iter().collect(), + )], + } + } + + fn only_images() -> SystemIndexerRule { + SystemIndexerRule { + name: "Only Images", + rules: vec![RulePerKind::new_accept_files_by_globs_str([ + "*.{avif,bmp,gif,ico,jpeg,jpg,png,svg,tif,tiff,webp}", + ]) + .unwrap()], + } + } +} + +pub use seeder::*; + #[cfg(test)] mod tests { use super::*; diff --git a/core/src/migrations.rs b/core/src/migrations.rs deleted file mode 100644 index fc8aff5aa..000000000 --- a/core/src/migrations.rs +++ /dev/null @@ -1,25 +0,0 @@ -use serde_json::{Map, Value}; - -use crate::util::migrator::MigratorError; - -pub(crate) const NODE_VERSION: u32 = 0; -pub(crate) const LIBRARY_VERSION: u32 = 0; - -/// Used to run migrations at a node level. This is useful for breaking changes to the `NodeConfig` file. -pub fn migration_node(version: u32, _config: &mut Map) -> Result<(), MigratorError> { - match version { - 0 => Ok(()), - v => unreachable!("Missing migration for library version {}", v), - } -} - -/// Used to run migrations at a library level. This will be run for every library as necessary. -pub fn migration_library( - version: u32, - _config: &mut Map, -) -> Result<(), MigratorError> { - match version { - 0 => Ok(()), - v => unreachable!("Missing migration for library version {}", v), - } -} diff --git a/core/src/node/config.rs b/core/src/node/config.rs index afdb11137..a369e1fe1 100644 --- a/core/src/node/config.rs +++ b/core/src/node/config.rs @@ -1,28 +1,19 @@ use sd_p2p::Keypair; use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; use specta::Type; use std::{ - marker::PhantomData, path::{Path, PathBuf}, sync::Arc, }; use tokio::sync::{RwLock, RwLockWriteGuard}; use uuid::Uuid; -use crate::{ - migrations, - util::migrator::{FileMigrator, MigratorError}, -}; +use crate::util::migrator::{Migrate, MigratorError}; /// NODE_STATE_CONFIG_NAME is the name of the file which stores the NodeState pub const NODE_STATE_CONFIG_NAME: &str = "node_state.sdconfig"; -const MIGRATOR: FileMigrator = FileMigrator { - current_version: migrations::NODE_VERSION, - migration_fn: migrations::migration_node, - phantom: PhantomData, -}; - /// NodeConfig is the configuration for a node. This is shared between all libraries and is stored in a JSON file on disk. #[derive(Debug, Serialize, Deserialize, Clone, Type)] pub struct NodeConfig { @@ -40,6 +31,24 @@ pub struct NodeConfig { pub p2p_img_url: Option, } +#[async_trait::async_trait] +impl Migrate for NodeConfig { + const CURRENT_VERSION: u32 = 0; + + type Ctx = (); + + async fn migrate( + from_version: u32, + config: &mut Map, + ctx: &Self::Ctx, + ) -> Result<(), MigratorError> { + match from_version { + 0 => Ok(()), + v => unreachable!("Missing migration for library version {}", v), + } + } +} + impl Default for NodeConfig { fn default() -> Self { NodeConfig { @@ -66,7 +75,7 @@ impl NodeConfigManager { /// new will create a new NodeConfigManager with the given path to the config file. pub(crate) async fn new(data_path: PathBuf) -> Result, MigratorError> { Ok(Arc::new(Self( - RwLock::new(MIGRATOR.load(&Self::path(&data_path))?), + RwLock::new(NodeConfig::load_and_migrate(&Self::path(&data_path), &()).await?), data_path, ))) } @@ -99,7 +108,7 @@ impl NodeConfigManager { /// save will write the configuration back to disk fn save(base_path: &Path, config: &NodeConfig) -> Result<(), MigratorError> { - MIGRATOR.save(&Self::path(base_path), config.clone())?; + NodeConfig::save(config, &Self::path(base_path))?; Ok(()) } } diff --git a/core/src/util/db.rs b/core/src/util/db.rs index 1f9c8ca43..101855e3a 100644 --- a/core/src/util/db.rs +++ b/core/src/util/db.rs @@ -28,14 +28,30 @@ pub async fn load_and_migrate(db_url: &str) -> Result {} + Err(e @ DbPushError::PossibleDataLoss(_)) => { + eprintln!("Pushing Prisma schema may result in data loss. Use `SD_ACCEPT_DATA_LOSS=true` to force it."); + Err(e)?; + } + Err(e) => Err(e)?, + } } #[cfg(not(debug_assertions))] diff --git a/core/src/util/migrator.rs b/core/src/util/migrator.rs index 9f8b95cbd..6b01b4a81 100644 --- a/core/src/util/migrator.rs +++ b/core/src/util/migrator.rs @@ -2,7 +2,6 @@ use std::{ any::type_name, fs::File, io::{self, BufReader, Seek, Write}, - marker::PhantomData, path::Path, }; @@ -23,32 +22,19 @@ pub struct BaseConfig { } /// System for managing app level migrations on a config file so we can introduce breaking changes to the app without the user needing to reset their whole system. -pub struct FileMigrator -where - T: Serialize + DeserializeOwned + Default, -{ - pub current_version: u32, - pub migration_fn: fn(u32, &mut Map) -> Result<(), MigratorError>, - pub phantom: PhantomData, -} +#[async_trait::async_trait] +pub trait Migrate: Sized + DeserializeOwned + Serialize + Default { + const CURRENT_VERSION: u32; -impl FileMigrator -where - T: Serialize + DeserializeOwned + Default, -{ - // TODO: This is blocked on Rust. Make sure to make all fields private when this is introduced! Tracking issue: https://github.com/rust-lang/rust/issues/57349 - // pub const fn new( - // current_version: u32, - // migration_fn: fn(u32, &mut Map) -> Result<(), MigratorError>, - // ) -> Self { - // Self { - // current_version, - // migration_fn, - // phantom: PhantomData, - // } - // } + type Ctx: Sync; - pub fn load(&self, path: &Path) -> Result { + async fn migrate( + from_version: u32, + config: &mut Map, + ctx: &Self::Ctx, + ) -> Result<(), MigratorError>; + + async fn load_and_migrate(path: &Path, ctx: &Self::Ctx) -> Result { match path.try_exists()? { true => { let mut file = File::options().read(true).write(true).open(path)?; @@ -81,14 +67,14 @@ where }; file.rewind()?; // Fail early so we don't end up invalid state - if cfg.version > self.current_version { + if cfg.version > Self::CURRENT_VERSION { return Err(MigratorError::YourAppIsOutdated); } - let is_latest = cfg.version == self.current_version; - for v in (cfg.version + 1)..=self.current_version { + let is_latest = cfg.version == Self::CURRENT_VERSION; + for v in (cfg.version + 1)..=Self::CURRENT_VERSION { cfg.version = v; - match (self.migration_fn)(v, &mut cfg.other) { + match Self::migrate(v, &mut cfg.other, &ctx).await { Ok(()) => (), Err(err) => { file.write_all(serde_json::to_string(&cfg)?.as_bytes())?; // Writes updated version @@ -104,18 +90,18 @@ where Ok(serde_json::from_value(Value::Object(cfg.other))?) } false => Ok(serde_json::from_value(Value::Object( - self.save(path, T::default())?.other, + Self::default().save(path)?.other, ))?), } } - pub fn save(&self, path: &Path, content: T) -> Result { + fn save(&self, path: &Path) -> Result { let config = BaseConfig { - version: self.current_version, - other: match serde_json::to_value(content)? { + version: Self::CURRENT_VERSION, + other: match serde_json::to_value(self)? { Value::Object(map) => map, _ => { - return Err(MigratorError::InvalidType(type_name::())); + return Err(MigratorError::InvalidType(type_name::())); } }, }; @@ -138,6 +124,8 @@ pub enum MigratorError { YourAppIsOutdated, #[error("Type '{0}' as generic `Migrator::T` must be serialiable to a Serde object!")] InvalidType(&'static str), + #[error("{0}")] + Database(#[from] prisma_client_rust::QueryError), #[error("We detected a Spacedrive config from a super early version of the app!")] HasSuperLegacyConfig, #[error("custom migration error: {0}")] @@ -148,40 +136,54 @@ pub enum MigratorError { mod test { use std::{fs, io::Read, path::PathBuf}; - use futures::executor::block_on; + use serde_json::json; use super::*; #[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq, Eq)] - pub struct MyConfigType { - a: u8, + struct MyConfigType { // For testing add new fields without breaking the passing. #[serde(flatten)] other: Map, } - pub fn migration_node( - version: u32, - config: &mut Map, - ) -> Result<(), MigratorError> { - match version { - 0 => Ok(()), - // Add field to config - 1 => { - config.insert("b".into(), 2.into()); - Ok(()) + #[async_trait::async_trait] + impl Migrate for MyConfigType { + const CURRENT_VERSION: u32 = 3; + + type Ctx = (); + + async fn migrate( + to_version: u32, + config: &mut Map, + ctx: &Self::Ctx, + ) -> Result<(), MigratorError> { + match to_version { + 0 => Ok(()), + 1 => { + config.insert("a".into(), json!({})); + Ok(()) + } + 2 => { + config + .get_mut("a") + .and_then(|v| v.as_object_mut()) + .map(|v| v.insert("b".into(), json!({}))); + + Ok(()) + } + 3 => { + config + .get_mut("a") + .and_then(|v| v.as_object_mut()) + .and_then(|v| v.get_mut("b")) + .and_then(|v| v.as_object_mut()) + .map(|v| v.insert("c".into(), json!("it works"))); + + Ok(()) + } + v => unreachable!("Missing migration for library version {}", v), } - // Async migration - 2 => { - let mut a = false; - block_on(async { - a = true; - config.insert("c".into(), 3.into()); - }); - assert!(a, "Async block was not blocked on correctly!"); - Ok(()) - } - v => unreachable!("Missing migration for library version {}", v), } } @@ -203,83 +205,41 @@ mod test { file.write_all(contents.as_bytes()).unwrap(); } - #[test] - fn test_migrator_happy_path() { - let migrator = FileMigrator:: { - current_version: 0, - migration_fn: migration_node, - phantom: PhantomData, - }; - + #[tokio::test] + async fn test_migrator_happy_path() { let p = path("test_migrator_happy_path.config"); // Check config is created when it's missing assert!(!p.exists(), "config file should start out deleted"); - let default_cfg = migrator.load(&p).unwrap(); - assert!(p.exists(), "config file was not initialised"); - assert_eq!(file_as_str(&p), r#"{"version":0,"a":0}"#); - - // Check config can be loaded back into the system correctly - let config = migrator.load(&p).unwrap(); - assert_eq!(default_cfg, config, "Config has got mangled somewhere"); - - // Update the config and check it saved correctly - let mut new_config = config; - new_config.a = 1; - migrator.save(&p, new_config.clone()).unwrap(); - assert_eq!(file_as_str(&p), r#"{"version":0,"a":1}"#); - - // Try loading in the new config and check it's correct - let config = migrator.load(&p).unwrap(); - assert_eq!( - new_config, config, - "Config has got mangled during the saving process" + std::fs::write( + &p, + serde_json::to_string(&json!({ + "version": 0 + })) + .unwrap(), ); + assert!(p.exists(), "config file was not initialised"); + assert_eq!(file_as_str(&p), r#"{"version":0}"#); - // Test upgrading to a new version which adds a field - let migrator = FileMigrator:: { - current_version: 1, - migration_fn: migration_node, - phantom: PhantomData, - }; + // Load + migrate config + let config = MyConfigType::load_and_migrate(&p, &()).await.unwrap(); - // Try loading in the new config and check it was updated - let config = migrator.load(&p).unwrap(); - assert_eq!(file_as_str(&p), r#"{"version":1,"a":1,"b":2}"#); - - // Check editing works - let mut new_config = config; - new_config.a = 2; - migrator.save(&p, new_config).unwrap(); - assert_eq!(file_as_str(&p), r#"{"version":1,"a":2,"b":2}"#); - - // Test upgrading to a new version which adds a field asynchronously - let migrator = FileMigrator:: { - current_version: 2, - migration_fn: migration_node, - phantom: PhantomData, - }; - - // Try loading in the new config and check it was updated - migrator.load(&p).unwrap(); - assert_eq!(file_as_str(&p), r#"{"version":2,"a":2,"b":2,"c":3}"#); + assert_eq!( + file_as_str(&p), + r#"{"version":3,"a":{"b":{"c":"it works"}}}"# + ); // Cleanup fs::remove_file(&p).unwrap(); } - #[test] - pub fn test_time_traveling_backwards() { + #[tokio::test] + pub async fn test_time_traveling_backwards() { let p = path("test_time_traveling_backwards.config"); // You opened a new database in an older version of the app - write_to_file(&p, r#"{"version":5,"a":1}"#); - let migrator = FileMigrator:: { - current_version: 2, - migration_fn: migration_node, - phantom: PhantomData, - }; - match migrator.load(&p) { + write_to_file(&p, r#"{"version":5}"#); + match MyConfigType::load_and_migrate(&p, &()).await { Err(MigratorError::YourAppIsOutdated) => (), _ => panic!("Should have failed to load config from a super newer version of the app"), } diff --git a/core/src/util/mod.rs b/core/src/util/mod.rs index d772b5099..849552798 100644 --- a/core/src/util/mod.rs +++ b/core/src/util/mod.rs @@ -4,6 +4,5 @@ pub mod db; pub mod debug_initializer; pub mod error; pub mod migrator; -pub mod seeder; pub use abort_on_drop::*; diff --git a/core/src/util/seeder.rs b/core/src/util/seeder.rs deleted file mode 100644 index e20c0c237..000000000 --- a/core/src/util/seeder.rs +++ /dev/null @@ -1,144 +0,0 @@ -use crate::{ - location::indexer::rules::{IndexerRule, IndexerRuleError, RulePerKind}, - prisma::PrismaClient, -}; -use thiserror::Error; - -#[derive(Error, Debug)] -pub enum SeederError { - #[error("Failed to run indexer rules seeder: {0}")] - IndexerRules(#[from] IndexerRuleError), - #[error("An error occurred with the database while applying migrations: {0}")] - DatabaseError(#[from] prisma_client_rust::QueryError), -} - -pub async fn indexer_rules_seeder(client: &PrismaClient) -> Result<(), SeederError> { - if client.indexer_rule().count(vec![]).exec().await? == 0 { - for rule in [ - IndexerRule::new( - // TODO: On windows, beside the listed files, any file with the FILE_ATTRIBUTE_SYSTEM should be considered a system file - // https://learn.microsoft.com/en-us/windows/win32/fileio/file-attribute-constants#FILE_ATTRIBUTE_SYSTEM - "No OS protected".to_string(), - true, - vec![ - RulePerKind::new_reject_files_by_globs_str( - [ - vec![ - "**/.spacedrive", - ], - // Globset, even on Windows, requires the use of / as a separator - // https://github.com/github/gitignore/blob/main/Global/Windows.gitignore - // https://learn.microsoft.com/en-us/windows/win32/fileio/naming-a-file - #[cfg(target_os = "windows")] - vec![ - // Windows thumbnail cache files - "**/{Thumbs.db,Thumbs.db:encryptable,ehthumbs.db,ehthumbs_vista.db}", - // Dump file - "**/*.stackdump", - // Folder config file - "**/[Dd]esktop.ini", - // Recycle Bin used on file shares - "**/$RECYCLE.BIN", - // Chkdsk recovery directory - "**/FOUND.[0-9][0-9][0-9]", - // Reserved names - "**/{CON,PRN,AUX,NUL,COM0,COM1,COM2,COM3,COM4,COM5,COM6,COM7,COM8,COM9,LPT0,LPT1,LPT2,LPT3,LPT4,LPT5,LPT6,LPT7,LPT8,LPT9}", - "**/{CON,PRN,AUX,NUL,COM0,COM1,COM2,COM3,COM4,COM5,COM6,COM7,COM8,COM9,LPT0,LPT1,LPT2,LPT3,LPT4,LPT5,LPT6,LPT7,LPT8,LPT9}.*", - // User special files - "C:/Users/*/NTUSER.DAT*", - "C:/Users/*/ntuser.dat*", - "C:/Users/*/{ntuser.ini,ntuser.dat,NTUSER.DAT}", - // User special folders (most of these the user dont even have permission to access) - "C:/Users/*/{Cookies,AppData,NetHood,Recent,PrintHood,SendTo,Templates,Start Menu,Application Data,Local Settings}", - // System special folders - "C:/{$Recycle.Bin,$WinREAgent,Documents and Settings,Program Files,Program Files (x86),ProgramData,Recovery,PerfLogs,Windows,Windows.old}", - // NTFS internal dir, can exists on any drive - "[A-Z]:/System Volume Information", - // System special files - "C:/{config,pagefile,hiberfil}.sys", - // Windows can create a swapfile on any drive - "[A-Z]:/swapfile.sys", - "C:/DumpStack.log.tmp", - ], - // https://github.com/github/gitignore/blob/main/Global/macOS.gitignore - // https://developer.apple.com/library/archive/documentation/FileManagement/Conceptual/FileSystemProgrammingGuide/FileSystemOverview/FileSystemOverview.html#//apple_ref/doc/uid/TP40010672-CH2-SW14 - #[cfg(any(target_os = "ios", target_os = "macos"))] - vec![ - "**/.{DS_Store,AppleDouble,LSOverride}", - // Icon must end with two \r - "**/Icon\r\r", - // Thumbnails - "**/._*", - ], - #[cfg(target_os = "macos")] - vec![ - "/{System,Network,Library,Applications}", - "/Users/*/{Library,Applications}", - // Files that might appear in the root of a volume - "**/.{DocumentRevisions-V100,fseventsd,Spotlight-V100,TemporaryItems,Trashes,VolumeIcon.icns,com.apple.timemachine.donotpresent}", - // Directories potentially created on remote AFP share - "**/.{AppleDB,AppleDesktop,apdisk}", - "**/{Network Trash Folder,Temporary Items}", - ], - // https://github.com/github/gitignore/blob/main/Global/Linux.gitignore - #[cfg(target_os = "linux")] - vec![ - "**/*~", - // temporary files which can be created if a process still has a handle open of a deleted file - "**/.fuse_hidden*", - // KDE directory preferences - "**/.directory", - // Linux trash folder which might appear on any partition or disk - "**/.Trash-*", - // .nfs files are created when an open file is removed but is still being accessed - "**/.nfs*", - ], - #[cfg(target_os = "android")] - vec![ - "**/.nomedia", - "**/.thumbnails", - ], - // https://en.wikipedia.org/wiki/Unix_filesystem#Conventional_directory_layout - // https://en.wikipedia.org/wiki/Filesystem_Hierarchy_Standard - #[cfg(target_family = "unix")] - vec![ - // Directories containing unix memory/device mapped files/dirs - "/{dev,sys,proc}", - // Directories containing special files for current running programs - "/{run,var,boot}", - // ext2-4 recovery directory - "**/lost+found", - ], - ] - .into_iter() - .flatten() - )?, - ], - ), - IndexerRule::new( - "No Hidden".to_string(), - true, - vec![RulePerKind::new_reject_files_by_globs_str( - ["**/.*"], - )?], - ), - IndexerRule::new( - "Only Git Repositories".into(), - false, - vec![RulePerKind::AcceptIfChildrenDirectoriesArePresent( - [".git".to_string()].into_iter().collect(), - )], - ), - IndexerRule::new( - "Only Images".to_string(), - false, - vec![RulePerKind::new_accept_files_by_globs_str(["*.{avif,bmp,gif,ico,jpeg,jpg,png,svg,tif,tiff,webp}"])?], - ) - - ] { - rule.save(client).await?; - } - } - - Ok(()) -} diff --git a/interface/app/$libraryId/settings/library/locations/IndexerRuleEditor/RuleButton.tsx b/interface/app/$libraryId/settings/library/locations/IndexerRuleEditor/RuleButton.tsx index 253853435..cec7b4a2a 100644 --- a/interface/app/$libraryId/settings/library/locations/IndexerRuleEditor/RuleButton.tsx +++ b/interface/app/$libraryId/settings/library/locations/IndexerRuleEditor/RuleButton.tsx @@ -4,6 +4,11 @@ import { IndexerRule } from '@sd/client'; import { InfoPill } from '~/app/$libraryId/Explorer/Inspector'; import { IndexerRuleIdFieldType } from '.'; +function ruleIsSystem(rule: IndexerRule) { + const num = rule.pub_id?.[15 - 3]; + return num !== undefined ? num === 0 : false; +} + interface RuleButtonProps { rule: IndexerRule; field?: T; @@ -57,7 +62,9 @@ function RuleButton({ > {ruleEnabled ? 'Enabled' : 'Disabled'} - {rule.default && System} + {ruleIsSystem(rule) && ( + System + )} diff --git a/packages/client/src/core.ts b/packages/client/src/core.ts index 0709c7e75..b6978f18b 100644 --- a/packages/client/src/core.ts +++ b/packages/client/src/core.ts @@ -162,7 +162,7 @@ export type HashingAlgorithm = { name: "Argon2id"; params: Params } | { name: "B export type IdentifyUniqueFilesArgs = { id: number; path: string } -export type IndexerRule = { id: number; name: string; default: boolean; rules_per_kind: number[]; date_created: string; date_modified: string } +export type IndexerRule = { id: number; pub_id: number[] | null; name: string; default: boolean; rules_per_kind: number[]; date_created: string; date_modified: string } /** * `IndexerRuleCreateArgs` is the argument received from the client using rspc to create a new indexer rule.