Software-Mac/spacedrive
Software-Mac/spacedrive
1
0
Fork 0
mirror of https://github.com/spacedriveapp/spacedrive synced 2024-07-02 08:53:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

[ENG-679] Reserve ids for built in indexer rules (#909)

Browse source 
* indexer rules pub ids

* should work?

* better migrator

* errors

* debugging

* maybe?

* double migrate

* please

* maybe fix?

* update lockfile

* SD_ACCEPT_DATA_LOSS message

* put tracing back

* dumb

* fix system indexer rule ui
This commit is contained in:
Brendan Allan 2023-06-06 16:42:52 +02:00 committed by GitHub
parent f3a35a9c13
commit fd236a1b57
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
17 changed files with 432 additions and 405 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
Cargo.lock generated
View file

@ -5661,7 +5661,7 @@ dependencies = [
[[package]]
name = "prisma-client-rust"
version = "0.6.8"
source = "git+https://github.com/Brendonovich/prisma-client-rust?rev=3b805c459ec1d52f163ecdc527b0d82e6556a022#3b805c459ec1d52f163ecdc527b0d82e6556a022"
source = "git+https://github.com/Brendonovich/prisma-client-rust?rev=7e67b4550dd5323d479c96008678032f396b9060#7e67b4550dd5323d479c96008678032f396b9060"
dependencies = [
"base64 0.13.1",
"bigdecimal",
@ -5694,7 +5694,7 @@ dependencies = [
[[package]]
name = "prisma-client-rust-cli"
version = "0.6.8"
source = "git+https://github.com/Brendonovich/prisma-client-rust?rev=3b805c459ec1d52f163ecdc527b0d82e6556a022#3b805c459ec1d52f163ecdc527b0d82e6556a022"
source = "git+https://github.com/Brendonovich/prisma-client-rust?rev=7e67b4550dd5323d479c96008678032f396b9060#7e67b4550dd5323d479c96008678032f396b9060"
dependencies = [
"directories",
"flate2",
@ -5714,7 +5714,7 @@ dependencies = [
[[package]]
name = "prisma-client-rust-macros"
version = "0.6.8"
source = "git+https://github.com/Brendonovich/prisma-client-rust?rev=3b805c459ec1d52f163ecdc527b0d82e6556a022#3b805c459ec1d52f163ecdc527b0d82e6556a022"
source = "git+https://github.com/Brendonovich/prisma-client-rust?rev=7e67b4550dd5323d479c96008678032f396b9060#7e67b4550dd5323d479c96008678032f396b9060"
dependencies = [
"convert_case 0.6.0",
"proc-macro2",
@ -5726,7 +5726,7 @@ dependencies = [
[[package]]
name = "prisma-client-rust-sdk"
version = "0.6.8"
source = "git+https://github.com/Brendonovich/prisma-client-rust?rev=3b805c459ec1d52f163ecdc527b0d82e6556a022#3b805c459ec1d52f163ecdc527b0d82e6556a022"
source = "git+https://github.com/Brendonovich/prisma-client-rust?rev=7e67b4550dd5323d479c96008678032f396b9060#7e67b4550dd5323d479c96008678032f396b9060"
dependencies = [
"convert_case 0.5.0",
"dmmf",

6
Cargo.toml
View file

@ -18,19 +18,19 @@ edition = "2021"
repository = "https://github.com/spacedriveapp/spacedrive"
[workspace.dependencies]
prisma-client-rust = { git = "https://github.com/Brendonovich/prisma-client-rust", rev = "3b805c459ec1d52f163ecdc527b0d82e6556a022", features = [
prisma-client-rust = { git = "https://github.com/Brendonovich/prisma-client-rust", rev = "7e67b4550dd5323d479c96008678032f396b9060", features = [
"rspc",
"sqlite-create-many",
"migrations",
"sqlite",
] }
prisma-client-rust-cli = { git = "https://github.com/Brendonovich/prisma-client-rust", rev = "3b805c459ec1d52f163ecdc527b0d82e6556a022", features = [
prisma-client-rust-cli = { git = "https://github.com/Brendonovich/prisma-client-rust", rev = "7e67b4550dd5323d479c96008678032f396b9060", features = [
"rspc",
"sqlite-create-many",
"migrations",
"sqlite",
] }
prisma-client-rust-sdk = { git = "https://github.com/Brendonovich/prisma-client-rust", rev = "3b805c459ec1d52f163ecdc527b0d82e6556a022", features = [
prisma-client-rust-sdk = { git = "https://github.com/Brendonovich/prisma-client-rust", rev = "7e67b4550dd5323d479c96008678032f396b9060", features = [
"sqlite",
] }

2
apps/desktop/src-tauri/Cargo.toml
View file

@ -18,7 +18,7 @@ httpz = { workspace = true, features = [
sd-core = { path = "../../../core", features = [
"ffmpeg",
"location-watcher",
"heif",
# "heif",
] }
tokio = { workspace = true, features = ["sync"] }
window-shadows = "0.2.1"

7
apps/server/src/main.rs
View file

@ -34,7 +34,12 @@ async fn main() {
let _guard = Node::init_logger(&data_dir);
let (node, router) = Node::new(data_dir).await.expect("Unable to create node");
let (node, router) = match Node::new(data_dir).await {
Ok(d) => d,
Err(e) => {
panic!("{}", e.to_string())
}
};
let signal = utils::axum_shutdown_signal(node.clone());
let app = axum::Router::new()

1
core/prisma/schema.prisma
View file

@ -446,6 +446,7 @@ model Comment {
model IndexerRule {
id Int @id @default(autoincrement())
pub_id Bytes? @unique
name String
default Boolean @default(false)
rules_per_kind Bytes

1
core/src/lib.rs
View file

@ -29,7 +29,6 @@ pub mod custom_uri;
pub(crate) mod job;
pub mod library;
pub(crate) mod location;
pub(crate) mod migrations;
pub(crate) mod node;
pub(crate) mod object;
pub(crate) mod p2p;

66
core/src/library/config.rs
View file

@ -1,17 +1,13 @@
use std::{marker::PhantomData, path::PathBuf};
use serde::{Deserialize, Serialize};
use specta::Type;
use uuid::Uuid;
use crate::{migrations, util::migrator::FileMigrator};
use super::LibraryManagerError;
const MIGRATOR: FileMigrator<LibraryConfig> = FileMigrator {
current_version: migrations::LIBRARY_VERSION,
migration_fn: migrations::migration_library,
phantom: PhantomData,
use crate::{
prisma::{indexer_rule, PrismaClient},
util::{
db::uuid_to_bytes,
migrator::{Migrate, MigratorError},
},
};
/// LibraryConfig holds the configuration for a specific library. This is stored as a '{uuid}.sdlibrary' file.
@ -26,18 +22,46 @@ pub struct LibraryConfig {
// pub is_encrypted: bool,
}
impl LibraryConfig {
/// read will read the configuration from disk and return it.
pub(super) fn read(file_dir: PathBuf) -> Result<LibraryConfig, LibraryManagerError> {
MIGRATOR.load(&file_dir).map_err(Into::into)
}
#[async_trait::async_trait]
impl Migrate for LibraryConfig {
const CURRENT_VERSION: u32 = 1;
type Ctx = PrismaClient;
async fn migrate(
to_version: u32,
_config: &mut serde_json::Map<String, serde_json::Value>,
db: &Self::Ctx,
) -> Result<(), MigratorError> {
match to_version {
0 => {}
1 => {
let rules = vec![
format!("No OS protected"),
format!("No Hidden"),
format!("Only Git Repositories"),
format!("Only Images"),
];
db._batch(
rules
.into_iter()
.enumerate()
.map(|(i, name)| {
db.indexer_rule().update_many(
vec![indexer_rule::name::equals(name)],
vec![indexer_rule::pub_id::set(Some(uuid_to_bytes(
Uuid::from_u128(i as u128),
)))],
)
})
.collect::<Vec<_>>(),
)
.await?;
}
v => unreachable!("Missing migration for library version {}", v),
}
/// save will write the configuration back to disk
pub(super) fn save(
file_dir: PathBuf,
config: &LibraryConfig,
) -> Result<(), LibraryManagerError> {
MIGRATOR.save(&file_dir, config.clone())?;
Ok(())
}
}

65
core/src/library/manager.rs
View file

@ -1,15 +1,14 @@
use crate::{
invalidate_query,
location::LocationManagerError,
location::{indexer::rules, LocationManagerError},
node::Platform,
object::orphan_remover::OrphanRemoverActor,
prisma::{location, node, PrismaClient},
sync::{SyncManager, SyncMessage},
util::{
db::{load_and_migrate, MigrationError},
db,
error::{FileIOError, NonUtf8PathError},
migrator::MigratorError,
seeder::{indexer_rules_seeder, SeederError},
migrator::{Migrate, MigratorError},
},
NodeContext,
};
@ -57,14 +56,14 @@ pub enum LibraryManagerError {
Migration(String),
#[error("failed to parse uuid")]
Uuid(#[from] uuid::Error),
#[error("failed to run seeder")]
Seeder(#[from] SeederError),
#[error("failed to run indexer rules seeder")]
IndexerRulesSeeder(#[from] rules::SeederError),
#[error("failed to initialise the key manager")]
KeyManager(#[from] sd_crypto::Error),
#[error("failed to run library migrations")]
#[error("failed to run library migrations: {0}")]
MigratorError(#[from] MigratorError),
#[error("error migrating the library: {0}")]
MigrationError(#[from] MigrationError),
MigrationError(#[from] db::MigrationError),
#[error("invalid library configuration: {0}")]
InvalidConfig(String),
#[error(transparent)]
@ -154,41 +153,41 @@ impl LibraryManager {
.await
.map_err(|e| FileIOError::from((&libraries_dir, e)))?
{
let entry_path = entry.path();
let config_path = entry.path();
let metadata = entry
.metadata()
.await
.map_err(|e| FileIOError::from((&entry_path, e)))?;
.map_err(|e| FileIOError::from((&config_path, e)))?;
if metadata.is_file()
&& entry_path
&& config_path
.extension()
.map(|ext| ext == "sdlibrary")
.unwrap_or(false)
{
let Some(Ok(library_id)) = entry_path
let Some(Ok(library_id)) = config_path
.file_stem()
.and_then(|v| v.to_str().map(Uuid::from_str))
else {
warn!("Attempted to load library from path '{}' but it has an invalid filename. Skipping...", entry_path.display());
warn!("Attempted to load library from path '{}' but it has an invalid filename. Skipping...", config_path.display());
continue;
};
let db_path = entry_path.with_extension("db");
let db_path = config_path.with_extension("db");
match fs::metadata(&db_path).await {
Ok(_) => {}
Err(e) if e.kind() == io::ErrorKind::NotFound => {
warn!(
"Found library '{}' but no matching database file was found. Skipping...",
entry_path.display()
config_path.display()
);
continue;
}
Err(e) => return Err(FileIOError::from((db_path, e)).into()),
}
let config = LibraryConfig::read(entry_path)?;
libraries
.push(Self::load(library_id, &db_path, config, node_context.clone()).await?);
libraries.push(
Self::load(library_id, &db_path, config_path, node_context.clone()).await?,
);
}
}
@ -222,21 +221,19 @@ impl LibraryManager {
));
}
LibraryConfig::save(
Path::new(&self.libraries_dir).join(format!("{id}.sdlibrary")),
&config,
)?;
let config_path = self.libraries_dir.join(format!("{id}.sdlibrary"));
config.save(&config_path)?;
let library = Self::load(
id,
self.libraries_dir.join(format!("{id}.db")),
config.clone(),
config_path,
self.node_context.clone(),
)
.await?;
// Run seeders
indexer_rules_seeder(&library.db).await?;
rules::seeder(&library.db).await?;
invalidate_query!(library, "library.list");
@ -282,8 +279,8 @@ impl LibraryManager {
}
LibraryConfig::save(
Path::new(&self.libraries_dir).join(format!("{id}.sdlibrary")),
&library.config,
&self.libraries_dir.join(format!("{id}.sdlibrary")),
)?;
invalidate_query!(library, "library.list");
@ -361,19 +358,19 @@ impl LibraryManager {
pub(crate) async fn load(
id: Uuid,
db_path: impl AsRef<Path>,
config: LibraryConfig,
config_path: PathBuf,
node_context: NodeContext,
) -> Result<Library, LibraryManagerError> {
let db_path = db_path.as_ref();
let db = Arc::new(
load_and_migrate(&format!(
"file:{}",
db_path.as_os_str().to_str().ok_or_else(|| {
LibraryManagerError::NonUtf8Path(NonUtf8PathError(db_path.into()))
})?
))
.await?,
let db_url = format!(
"file:{}",
db_path.as_os_str().to_str().ok_or_else(|| {
LibraryManagerError::NonUtf8Path(NonUtf8PathError(db_path.into()))
})?
);
let db = Arc::new(db::load_and_migrate(&db_url).await?);
let config = LibraryConfig::load_and_migrate(&config_path, &db).await?;
let node_config = node_context.config.get().await;

245
core/src/location/indexer/rules.rs
View file

@ -1,7 +1,10 @@
use crate::{
library::Library,
prisma::{indexer_rule, PrismaClient},
util::error::{FileIOError, NonUtf8PathError},
prisma::indexer_rule,
util::{
db::uuid_to_bytes,
error::{FileIOError, NonUtf8PathError},
},
};
use chrono::{DateTime, Utc};
@ -19,6 +22,7 @@ use std::{
use thiserror::Error;
use tokio::fs;
use tracing::debug;
use uuid::Uuid;
#[derive(Error, Debug)]
pub enum IndexerRuleError {
@ -121,7 +125,13 @@ impl IndexerRuleCreateArgs {
library
.db
.indexer_rule()
.create(self.name, rules_data, vec![])
.create(
self.name,
rules_data,
vec![indexer_rule::pub_id::set(Some(uuid_to_bytes(
generate_pub_id(),
)))],
)
.exec()
.await?,
))
@ -461,36 +471,6 @@ impl IndexerRule {
try_join_all(self.rules.iter().map(|rule| rule.apply(source.as_ref()))).await
}
pub async fn save(self, client: &PrismaClient) -> Result<(), IndexerRuleError> {
if let Some(id) = self.id {
client
.indexer_rule()
.upsert(
indexer_rule::id::equals(id),
indexer_rule::create(
self.name,
rmp_serde::to_vec_named(&self.rules)?,
vec![indexer_rule::default::set(self.default)],
),
vec![indexer_rule::date_modified::set(Utc::now().into())],
)
.exec()
.await?;
} else {
client
.indexer_rule()
.create(
self.name,
rmp_serde::to_vec_named(&self.rules)?,
vec![indexer_rule::default::set(self.default)],
)
.exec()
.await?;
}
Ok(())
}
pub async fn apply_all(
rules: &[IndexerRule],
source: impl AsRef<Path>,
@ -626,6 +606,205 @@ async fn reject_dir_for_its_children(
Ok(true)
}
pub fn generate_pub_id() -> Uuid {
loop {
let pub_id = Uuid::new_v4();
if pub_id.as_u128() >= 0xFFF {
return pub_id;
}
}
}
mod seeder {
use crate::{
location::indexer::rules::{IndexerRuleError, RulePerKind},
prisma::PrismaClient,
util::db::uuid_to_bytes,
};
use sd_prisma::prisma::indexer_rule;
use thiserror::Error;
use uuid::Uuid;
#[derive(Error, Debug)]
pub enum SeederError {
#[error("Failed to run indexer rules seeder: {0}")]
IndexerRules(#[from] IndexerRuleError),
#[error("An error occurred with the database while applying migrations: {0}")]
DatabaseError(#[from] prisma_client_rust::QueryError),
}
struct SystemIndexerRule {
name: &'static str,
rules: Vec<RulePerKind>,
}
pub async fn seeder(client: &PrismaClient) -> Result<(), SeederError> {
// DO NOT REORDER THIS ARRAY!
for (i, rule) in [
no_os_protected(),
no_hidden(),
only_git_repos(),
only_images(),
]
.into_iter()
.enumerate()
{
let pub_id = uuid_to_bytes(Uuid::from_u128(i as u128));
let rules = rmp_serde::to_vec_named(&rule.rules).map_err(IndexerRuleError::from)?;
client
.indexer_rule()
.upsert(
indexer_rule::pub_id::equals(pub_id.clone()),
indexer_rule::create(
rule.name.to_string(),
rules.clone(),
vec![indexer_rule::pub_id::set(Some(pub_id.clone()))],
),
vec![
indexer_rule::name::set(rule.name.to_string()),
indexer_rule::rules_per_kind::set(rules),
indexer_rule::pub_id::set(Some(pub_id.clone())),
],
)
.exec()
.await?;
}
Ok(())
}
fn no_os_protected() -> SystemIndexerRule {
SystemIndexerRule {
// TODO: On windows, beside the listed files, any file with the FILE_ATTRIBUTE_SYSTEM should be considered a system file
// https://learn.microsoft.com/en-us/windows/win32/fileio/file-attribute-constants#FILE_ATTRIBUTE_SYSTEM
name: "No OS protected",
rules: vec![
RulePerKind::new_reject_files_by_globs_str(
[
vec![
"**/.spacedrive",
],
// Globset, even on Windows, requires the use of / as a separator
// https://github.com/github/gitignore/blob/main/Global/Windows.gitignore
// https://learn.microsoft.com/en-us/windows/win32/fileio/naming-a-file
#[cfg(target_os = "windows")]
vec![
// Windows thumbnail cache files
"**/{Thumbs.db,Thumbs.db:encryptable,ehthumbs.db,ehthumbs_vista.db}",
// Dump file
"**/*.stackdump",
// Folder config file
"**/[Dd]esktop.ini",
// Recycle Bin used on file shares
"**/$RECYCLE.BIN",
// Chkdsk recovery directory
"**/FOUND.[0-9][0-9][0-9]",
// Reserved names
"**/{CON,PRN,AUX,NUL,COM0,COM1,COM2,COM3,COM4,COM5,COM6,COM7,COM8,COM9,LPT0,LPT1,LPT2,LPT3,LPT4,LPT5,LPT6,LPT7,LPT8,LPT9}",
"**/{CON,PRN,AUX,NUL,COM0,COM1,COM2,COM3,COM4,COM5,COM6,COM7,COM8,COM9,LPT0,LPT1,LPT2,LPT3,LPT4,LPT5,LPT6,LPT7,LPT8,LPT9}.*",
// User special files
"C:/Users/*/NTUSER.DAT*",
"C:/Users/*/ntuser.dat*",
"C:/Users/*/{ntuser.ini,ntuser.dat,NTUSER.DAT}",
// User special folders (most of these the user dont even have permission to access)
"C:/Users/*/{Cookies,AppData,NetHood,Recent,PrintHood,SendTo,Templates,Start Menu,Application Data,Local Settings}",
// System special folders
"C:/{$Recycle.Bin,$WinREAgent,Documents and Settings,Program Files,Program Files (x86),ProgramData,Recovery,PerfLogs,Windows,Windows.old}",
// NTFS internal dir, can exists on any drive
"[A-Z]:/System Volume Information",
// System special files
"C:/{config,pagefile,hiberfil}.sys",
// Windows can create a swapfile on any drive
"[A-Z]:/swapfile.sys",
"C:/DumpStack.log.tmp",
],
// https://github.com/github/gitignore/blob/main/Global/macOS.gitignore
// https://developer.apple.com/library/archive/documentation/FileManagement/Conceptual/FileSystemProgrammingGuide/FileSystemOverview/FileSystemOverview.html#//apple_ref/doc/uid/TP40010672-CH2-SW14
#[cfg(any(target_os = "ios", target_os = "macos"))]
vec![
"**/.{DS_Store,AppleDouble,LSOverride}",
// Icon must end with two \r
"**/Icon\r\r",
// Thumbnails
"**/._*",
],
#[cfg(target_os = "macos")]
vec![
"/{System,Network,Library,Applications}",
"/Users/*/{Library,Applications}",
// Files that might appear in the root of a volume
"**/.{DocumentRevisions-V100,fseventsd,Spotlight-V100,TemporaryItems,Trashes,VolumeIcon.icns,com.apple.timemachine.donotpresent}",
// Directories potentially created on remote AFP share
"**/.{AppleDB,AppleDesktop,apdisk}",
"**/{Network Trash Folder,Temporary Items}",
],
// https://github.com/github/gitignore/blob/main/Global/Linux.gitignore
#[cfg(target_os = "linux")]
vec![
"**/*~",
// temporary files which can be created if a process still has a handle open of a deleted file
"**/.fuse_hidden*",
// KDE directory preferences
"**/.directory",
// Linux trash folder which might appear on any partition or disk
"**/.Trash-*",
// .nfs files are created when an open file is removed but is still being accessed
"**/.nfs*",
],
#[cfg(target_os = "android")]
vec![
"**/.nomedia",
"**/.thumbnails",
],
// https://en.wikipedia.org/wiki/Unix_filesystem#Conventional_directory_layout
// https://en.wikipedia.org/wiki/Filesystem_Hierarchy_Standard
#[cfg(target_family = "unix")]
vec![
// Directories containing unix memory/device mapped files/dirs
"/{dev,sys,proc}",
// Directories containing special files for current running programs
"/{run,var,boot}",
// ext2-4 recovery directory
"**/lost+found",
],
]
.into_iter()
.flatten()
).unwrap(),
],
}
}
fn no_hidden() -> SystemIndexerRule {
SystemIndexerRule {
name: "No Hidden",
rules: vec![RulePerKind::new_reject_files_by_globs_str(["**/.*"]).unwrap()],
}
}
fn only_git_repos() -> SystemIndexerRule {
SystemIndexerRule {
name: "Only Git Repositories",
rules: vec![RulePerKind::AcceptIfChildrenDirectoriesArePresent(
[".git".to_string()].into_iter().collect(),
)],
}
}
fn only_images() -> SystemIndexerRule {
SystemIndexerRule {
name: "Only Images",
rules: vec![RulePerKind::new_accept_files_by_globs_str([
"*.{avif,bmp,gif,ico,jpeg,jpg,png,svg,tif,tiff,webp}",
])
.unwrap()],
}
}
}
pub use seeder::*;
#[cfg(test)]
mod tests {
use super::*;

25
core/src/migrations.rs
View file

@ -1,25 +0,0 @@
use serde_json::{Map, Value};
use crate::util::migrator::MigratorError;
pub(crate) const NODE_VERSION: u32 = 0;
pub(crate) const LIBRARY_VERSION: u32 = 0;
/// Used to run migrations at a node level. This is useful for breaking changes to the `NodeConfig` file.
pub fn migration_node(version: u32, _config: &mut Map<String, Value>) -> Result<(), MigratorError> {
match version {
0 => Ok(()),
v => unreachable!("Missing migration for library version {}", v),
}
}
/// Used to run migrations at a library level. This will be run for every library as necessary.
pub fn migration_library(
version: u32,
_config: &mut Map<String, Value>,
) -> Result<(), MigratorError> {
match version {
0 => Ok(()),
v => unreachable!("Missing migration for library version {}", v),
}
}

35
core/src/node/config.rs
View file

@ -1,28 +1,19 @@
use sd_p2p::Keypair;
use serde::{Deserialize, Serialize};
use serde_json::{Map, Value};
use specta::Type;
use std::{
marker::PhantomData,
path::{Path, PathBuf},
sync::Arc,
};
use tokio::sync::{RwLock, RwLockWriteGuard};
use uuid::Uuid;
use crate::{
migrations,
util::migrator::{FileMigrator, MigratorError},
};
use crate::util::migrator::{Migrate, MigratorError};
/// NODE_STATE_CONFIG_NAME is the name of the file which stores the NodeState
pub const NODE_STATE_CONFIG_NAME: &str = "node_state.sdconfig";
const MIGRATOR: FileMigrator<NodeConfig> = FileMigrator {
current_version: migrations::NODE_VERSION,
migration_fn: migrations::migration_node,
phantom: PhantomData,
};
/// NodeConfig is the configuration for a node. This is shared between all libraries and is stored in a JSON file on disk.
#[derive(Debug, Serialize, Deserialize, Clone, Type)]
pub struct NodeConfig {
@ -40,6 +31,24 @@ pub struct NodeConfig {
pub p2p_img_url: Option<String>,
}
#[async_trait::async_trait]
impl Migrate for NodeConfig {
const CURRENT_VERSION: u32 = 0;
type Ctx = ();
async fn migrate(
from_version: u32,
config: &mut Map<String, Value>,
ctx: &Self::Ctx,
) -> Result<(), MigratorError> {
match from_version {
0 => Ok(()),
v => unreachable!("Missing migration for library version {}", v),
}
}
}
impl Default for NodeConfig {
fn default() -> Self {
NodeConfig {
@ -66,7 +75,7 @@ impl NodeConfigManager {
/// new will create a new NodeConfigManager with the given path to the config file.
pub(crate) async fn new(data_path: PathBuf) -> Result<Arc<Self>, MigratorError> {
Ok(Arc::new(Self(
RwLock::new(MIGRATOR.load(&Self::path(&data_path))?),
RwLock::new(NodeConfig::load_and_migrate(&Self::path(&data_path), &()).await?),
data_path,
)))
}
@ -99,7 +108,7 @@ impl NodeConfigManager {
/// save will write the configuration back to disk
fn save(base_path: &Path, config: &NodeConfig) -> Result<(), MigratorError> {
MIGRATOR.save(&Self::path(base_path), config.clone())?;
NodeConfig::save(config, &Self::path(base_path))?;
Ok(())
}
}

20
core/src/util/db.rs
View file

@ -28,14 +28,30 @@ pub async fn load_and_migrate(db_url: &str) -> Result<PrismaClient, MigrationErr
{
let mut builder = client._db_push();
if std::env::var("SD_ACCEPT_DATA_LOSS")
.map(|v| v == "true")
.unwrap_or(false)
{
builder = builder.accept_data_loss();
}
if std::env::var("SD_FORCE_RESET_DB")
.map(|v| v == "true")
.unwrap_or(false)
{
builder = builder.accept_data_loss().force_reset();
builder = builder.force_reset();
}
builder.await?;
let res = builder.await;
match res {
Ok(_) => {}
Err(e @ DbPushError::PossibleDataLoss(_)) => {
eprintln!("Pushing Prisma schema may result in data loss. Use `SD_ACCEPT_DATA_LOSS=true` to force it.");
Err(e)?;
}
Err(e) => Err(e)?,
}
}
#[cfg(not(debug_assertions))]

200
core/src/util/migrator.rs
View file

@ -2,7 +2,6 @@ use std::{
any::type_name,
fs::File,
io::{self, BufReader, Seek, Write},
marker::PhantomData,
path::Path,
};
@ -23,32 +22,19 @@ pub struct BaseConfig {
}
/// System for managing app level migrations on a config file so we can introduce breaking changes to the app without the user needing to reset their whole system.
pub struct FileMigrator<T>
where
T: Serialize + DeserializeOwned + Default,
{
pub current_version: u32,
pub migration_fn: fn(u32, &mut Map<String, Value>) -> Result<(), MigratorError>,
pub phantom: PhantomData<T>,
}
#[async_trait::async_trait]
pub trait Migrate: Sized + DeserializeOwned + Serialize + Default {
const CURRENT_VERSION: u32;
impl<T> FileMigrator<T>
where
T: Serialize + DeserializeOwned + Default,
{
// TODO: This is blocked on Rust. Make sure to make all fields private when this is introduced! Tracking issue: https://github.com/rust-lang/rust/issues/57349
// pub const fn new(
// current_version: u32,
// migration_fn: fn(u32, &mut Map<String, Value>) -> Result<(), MigratorError>,
// ) -> Self {
// Self {
// current_version,
// migration_fn,
// phantom: PhantomData,
// }
// }
type Ctx: Sync;
pub fn load(&self, path: &Path) -> Result<T, MigratorError> {
async fn migrate(
from_version: u32,
config: &mut Map<String, Value>,
ctx: &Self::Ctx,
) -> Result<(), MigratorError>;
async fn load_and_migrate(path: &Path, ctx: &Self::Ctx) -> Result<Self, MigratorError> {
match path.try_exists()? {
true => {
let mut file = File::options().read(true).write(true).open(path)?;
@ -81,14 +67,14 @@ where
};
file.rewind()?; // Fail early so we don't end up invalid state
if cfg.version > self.current_version {
if cfg.version > Self::CURRENT_VERSION {
return Err(MigratorError::YourAppIsOutdated);
}
let is_latest = cfg.version == self.current_version;
for v in (cfg.version + 1)..=self.current_version {
let is_latest = cfg.version == Self::CURRENT_VERSION;
for v in (cfg.version + 1)..=Self::CURRENT_VERSION {
cfg.version = v;
match (self.migration_fn)(v, &mut cfg.other) {
match Self::migrate(v, &mut cfg.other, &ctx).await {
Ok(()) => (),
Err(err) => {
file.write_all(serde_json::to_string(&cfg)?.as_bytes())?; // Writes updated version
@ -104,18 +90,18 @@ where
Ok(serde_json::from_value(Value::Object(cfg.other))?)
}
false => Ok(serde_json::from_value(Value::Object(
self.save(path, T::default())?.other,
Self::default().save(path)?.other,
))?),
}
}
pub fn save(&self, path: &Path, content: T) -> Result<BaseConfig, MigratorError> {
fn save(&self, path: &Path) -> Result<BaseConfig, MigratorError> {
let config = BaseConfig {
version: self.current_version,
other: match serde_json::to_value(content)? {
version: Self::CURRENT_VERSION,
other: match serde_json::to_value(self)? {
Value::Object(map) => map,
_ => {
return Err(MigratorError::InvalidType(type_name::<T>()));
return Err(MigratorError::InvalidType(type_name::<Self>()));
}
},
};
@ -138,6 +124,8 @@ pub enum MigratorError {
YourAppIsOutdated,
#[error("Type '{0}' as generic `Migrator::T` must be serialiable to a Serde object!")]
InvalidType(&'static str),
#[error("{0}")]
Database(#[from] prisma_client_rust::QueryError),
#[error("We detected a Spacedrive config from a super early version of the app!")]
HasSuperLegacyConfig,
#[error("custom migration error: {0}")]
@ -148,40 +136,54 @@ pub enum MigratorError {
mod test {
use std::{fs, io::Read, path::PathBuf};
use futures::executor::block_on;
use serde_json::json;
use super::*;
#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq, Eq)]
pub struct MyConfigType {
a: u8,
struct MyConfigType {
// For testing add new fields without breaking the passing.
#[serde(flatten)]
other: Map<String, Value>,
}
pub fn migration_node(
version: u32,
config: &mut Map<String, Value>,
) -> Result<(), MigratorError> {
match version {
0 => Ok(()),
// Add field to config
1 => {
config.insert("b".into(), 2.into());
Ok(())
#[async_trait::async_trait]
impl Migrate for MyConfigType {
const CURRENT_VERSION: u32 = 3;
type Ctx = ();
async fn migrate(
to_version: u32,
config: &mut Map<String, Value>,
ctx: &Self::Ctx,
) -> Result<(), MigratorError> {
match to_version {
0 => Ok(()),
1 => {
config.insert("a".into(), json!({}));
Ok(())
}
2 => {
config
.get_mut("a")
.and_then(|v| v.as_object_mut())
.map(|v| v.insert("b".into(), json!({})));
Ok(())
}
3 => {
config
.get_mut("a")
.and_then(|v| v.as_object_mut())
.and_then(|v| v.get_mut("b"))
.and_then(|v| v.as_object_mut())
.map(|v| v.insert("c".into(), json!("it works")));
Ok(())
}
v => unreachable!("Missing migration for library version {}", v),
}
// Async migration
2 => {
let mut a = false;
block_on(async {
a = true;
config.insert("c".into(), 3.into());
});
assert!(a, "Async block was not blocked on correctly!");
Ok(())
}
v => unreachable!("Missing migration for library version {}", v),
}
}
@ -203,83 +205,41 @@ mod test {
file.write_all(contents.as_bytes()).unwrap();
}
#[test]
fn test_migrator_happy_path() {
let migrator = FileMigrator::<MyConfigType> {
current_version: 0,
migration_fn: migration_node,
phantom: PhantomData,
};
#[tokio::test]
async fn test_migrator_happy_path() {
let p = path("test_migrator_happy_path.config");
// Check config is created when it's missing
assert!(!p.exists(), "config file should start out deleted");
let default_cfg = migrator.load(&p).unwrap();
assert!(p.exists(), "config file was not initialised");
assert_eq!(file_as_str(&p), r#"{"version":0,"a":0}"#);
// Check config can be loaded back into the system correctly
let config = migrator.load(&p).unwrap();
assert_eq!(default_cfg, config, "Config has got mangled somewhere");
// Update the config and check it saved correctly
let mut new_config = config;
new_config.a = 1;
migrator.save(&p, new_config.clone()).unwrap();
assert_eq!(file_as_str(&p), r#"{"version":0,"a":1}"#);
// Try loading in the new config and check it's correct
let config = migrator.load(&p).unwrap();
assert_eq!(
new_config, config,
"Config has got mangled during the saving process"
std::fs::write(
&p,
serde_json::to_string(&json!({
"version": 0
}))
.unwrap(),
);
assert!(p.exists(), "config file was not initialised");
assert_eq!(file_as_str(&p), r#"{"version":0}"#);
// Test upgrading to a new version which adds a field
let migrator = FileMigrator::<MyConfigType> {
current_version: 1,
migration_fn: migration_node,
phantom: PhantomData,
};
// Load + migrate config
let config = MyConfigType::load_and_migrate(&p, &()).await.unwrap();
// Try loading in the new config and check it was updated
let config = migrator.load(&p).unwrap();
assert_eq!(file_as_str(&p), r#"{"version":1,"a":1,"b":2}"#);
// Check editing works
let mut new_config = config;
new_config.a = 2;
migrator.save(&p, new_config).unwrap();
assert_eq!(file_as_str(&p), r#"{"version":1,"a":2,"b":2}"#);
// Test upgrading to a new version which adds a field asynchronously
let migrator = FileMigrator::<MyConfigType> {
current_version: 2,
migration_fn: migration_node,
phantom: PhantomData,
};
// Try loading in the new config and check it was updated
migrator.load(&p).unwrap();
assert_eq!(file_as_str(&p), r#"{"version":2,"a":2,"b":2,"c":3}"#);
assert_eq!(
file_as_str(&p),
r#"{"version":3,"a":{"b":{"c":"it works"}}}"#
);
// Cleanup
fs::remove_file(&p).unwrap();
}
#[test]
pub fn test_time_traveling_backwards() {
#[tokio::test]
pub async fn test_time_traveling_backwards() {
let p = path("test_time_traveling_backwards.config");
// You opened a new database in an older version of the app
write_to_file(&p, r#"{"version":5,"a":1}"#);
let migrator = FileMigrator::<MyConfigType> {
current_version: 2,
migration_fn: migration_node,
phantom: PhantomData,
};
match migrator.load(&p) {
write_to_file(&p, r#"{"version":5}"#);
match MyConfigType::load_and_migrate(&p, &()).await {
Err(MigratorError::YourAppIsOutdated) => (),
_ => panic!("Should have failed to load config from a super newer version of the app"),
}

1
core/src/util/mod.rs
View file

@ -4,6 +4,5 @@ pub mod db;
pub mod debug_initializer;
pub mod error;
pub mod migrator;
pub mod seeder;
pub use abort_on_drop::*;

144
core/src/util/seeder.rs
View file

@ -1,144 +0,0 @@
use crate::{
location::indexer::rules::{IndexerRule, IndexerRuleError, RulePerKind},
prisma::PrismaClient,
};
use thiserror::Error;
#[derive(Error, Debug)]
pub enum SeederError {
#[error("Failed to run indexer rules seeder: {0}")]
IndexerRules(#[from] IndexerRuleError),
#[error("An error occurred with the database while applying migrations: {0}")]
DatabaseError(#[from] prisma_client_rust::QueryError),
}
pub async fn indexer_rules_seeder(client: &PrismaClient) -> Result<(), SeederError> {
if client.indexer_rule().count(vec![]).exec().await? == 0 {
for rule in [
IndexerRule::new(
// TODO: On windows, beside the listed files, any file with the FILE_ATTRIBUTE_SYSTEM should be considered a system file
// https://learn.microsoft.com/en-us/windows/win32/fileio/file-attribute-constants#FILE_ATTRIBUTE_SYSTEM
"No OS protected".to_string(),
true,
vec![
RulePerKind::new_reject_files_by_globs_str(
[
vec![
"**/.spacedrive",
],
// Globset, even on Windows, requires the use of / as a separator
// https://github.com/github/gitignore/blob/main/Global/Windows.gitignore
// https://learn.microsoft.com/en-us/windows/win32/fileio/naming-a-file
#[cfg(target_os = "windows")]
vec![
// Windows thumbnail cache files
"**/{Thumbs.db,Thumbs.db:encryptable,ehthumbs.db,ehthumbs_vista.db}",
// Dump file
"**/*.stackdump",
// Folder config file
"**/[Dd]esktop.ini",
// Recycle Bin used on file shares
"**/$RECYCLE.BIN",
// Chkdsk recovery directory
"**/FOUND.[0-9][0-9][0-9]",
// Reserved names
"**/{CON,PRN,AUX,NUL,COM0,COM1,COM2,COM3,COM4,COM5,COM6,COM7,COM8,COM9,LPT0,LPT1,LPT2,LPT3,LPT4,LPT5,LPT6,LPT7,LPT8,LPT9}",
"**/{CON,PRN,AUX,NUL,COM0,COM1,COM2,COM3,COM4,COM5,COM6,COM7,COM8,COM9,LPT0,LPT1,LPT2,LPT3,LPT4,LPT5,LPT6,LPT7,LPT8,LPT9}.*",
// User special files
"C:/Users/*/NTUSER.DAT*",
"C:/Users/*/ntuser.dat*",
"C:/Users/*/{ntuser.ini,ntuser.dat,NTUSER.DAT}",
// User special folders (most of these the user dont even have permission to access)
"C:/Users/*/{Cookies,AppData,NetHood,Recent,PrintHood,SendTo,Templates,Start Menu,Application Data,Local Settings}",
// System special folders
"C:/{$Recycle.Bin,$WinREAgent,Documents and Settings,Program Files,Program Files (x86),ProgramData,Recovery,PerfLogs,Windows,Windows.old}",
// NTFS internal dir, can exists on any drive
"[A-Z]:/System Volume Information",
// System special files
"C:/{config,pagefile,hiberfil}.sys",
// Windows can create a swapfile on any drive
"[A-Z]:/swapfile.sys",
"C:/DumpStack.log.tmp",
],
// https://github.com/github/gitignore/blob/main/Global/macOS.gitignore
// https://developer.apple.com/library/archive/documentation/FileManagement/Conceptual/FileSystemProgrammingGuide/FileSystemOverview/FileSystemOverview.html#//apple_ref/doc/uid/TP40010672-CH2-SW14
#[cfg(any(target_os = "ios", target_os = "macos"))]
vec![
"**/.{DS_Store,AppleDouble,LSOverride}",
// Icon must end with two \r
"**/Icon\r\r",
// Thumbnails
"**/._*",
],
#[cfg(target_os = "macos")]
vec![
"/{System,Network,Library,Applications}",
"/Users/*/{Library,Applications}",
// Files that might appear in the root of a volume
"**/.{DocumentRevisions-V100,fseventsd,Spotlight-V100,TemporaryItems,Trashes,VolumeIcon.icns,com.apple.timemachine.donotpresent}",
// Directories potentially created on remote AFP share
"**/.{AppleDB,AppleDesktop,apdisk}",
"**/{Network Trash Folder,Temporary Items}",
],
// https://github.com/github/gitignore/blob/main/Global/Linux.gitignore
#[cfg(target_os = "linux")]
vec![
"**/*~",
// temporary files which can be created if a process still has a handle open of a deleted file
"**/.fuse_hidden*",
// KDE directory preferences
"**/.directory",
// Linux trash folder which might appear on any partition or disk
"**/.Trash-*",
// .nfs files are created when an open file is removed but is still being accessed
"**/.nfs*",
],
#[cfg(target_os = "android")]
vec![
"**/.nomedia",
"**/.thumbnails",
],
// https://en.wikipedia.org/wiki/Unix_filesystem#Conventional_directory_layout
// https://en.wikipedia.org/wiki/Filesystem_Hierarchy_Standard
#[cfg(target_family = "unix")]
vec![
// Directories containing unix memory/device mapped files/dirs
"/{dev,sys,proc}",
// Directories containing special files for current running programs
"/{run,var,boot}",
// ext2-4 recovery directory
"**/lost+found",
],
]
.into_iter()
.flatten()
)?,
],
),
IndexerRule::new(
"No Hidden".to_string(),
true,
vec![RulePerKind::new_reject_files_by_globs_str(
["**/.*"],
)?],
),
IndexerRule::new(
"Only Git Repositories".into(),
false,
vec![RulePerKind::AcceptIfChildrenDirectoriesArePresent(
[".git".to_string()].into_iter().collect(),
)],
),
IndexerRule::new(
"Only Images".to_string(),
false,
vec![RulePerKind::new_accept_files_by_globs_str(["*.{avif,bmp,gif,ico,jpeg,jpg,png,svg,tif,tiff,webp}"])?],
)
] {
rule.save(client).await?;
}
}
Ok(())
}

9
interface/app/$libraryId/settings/library/locations/IndexerRuleEditor/RuleButton.tsx
View file

@ -4,6 +4,11 @@ import { IndexerRule } from '@sd/client';
import { InfoPill } from '~/app/$libraryId/Explorer/Inspector';
import { IndexerRuleIdFieldType } from '.';
function ruleIsSystem(rule: IndexerRule) {
const num = rule.pub_id?.[15 - 3];
return num !== undefined ? num === 0 : false;
}
interface RuleButtonProps<T extends IndexerRuleIdFieldType> {
rule: IndexerRule;
field?: T;
@ -57,7 +62,9 @@ function RuleButton<T extends IndexerRuleIdFieldType>({
>
{ruleEnabled ? 'Enabled' : 'Disabled'}
</InfoPill>
{rule.default && <InfoPill className="px-2 text-ink-faint">System</InfoPill>}
{ruleIsSystem(rule) && (
<InfoPill className="px-2 text-ink-faint">System</InfoPill>
)}
</div>
</div>
</div>

2
packages/client/src/core.ts
View file

@ -162,7 +162,7 @@ export type HashingAlgorithm = { name: "Argon2id"; params: Params } | { name: "B
export type IdentifyUniqueFilesArgs = { id: number; path: string }
export type IndexerRule = { id: number; name: string; default: boolean; rules_per_kind: number[]; date_created: string; date_modified: string }
export type IndexerRule = { id: number; pub_id: number[] | null; name: string; default: boolean; rules_per_kind: number[]; date_created: string; date_modified: string }
/**
* `IndexerRuleCreateArgs` is the argument received from the client using rspc to create a new indexer rule.