Software-Mac/spacedrive
Software-Mac/spacedrive
1
0
Fork 0
mirror of https://github.com/spacedriveapp/spacedrive synced 2024-07-07 06:43:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

[ENG-782] Seeded tags (#994)

Browse source 
* seeded tags infra

* dumb

* here's my seed

---------

Co-authored-by: James Pine <ijamespine@me.com>
Co-authored-by: Jamie Pine <32987599+jamiepine@users.noreply.github.com>
This commit is contained in:
Brendan Allan 2023-06-22 14:08:38 +02:00 committed by GitHub
parent 7acd584d6f
commit 32824a88be
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 285 additions and 274 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

34
core/src/api/tags.rs
View file

@ -3,11 +3,11 @@ use serde::Deserialize;
use specta::Type;
use serde_json::json;
use uuid::Uuid;
use crate::{
invalidate_query,
library::Library,
object::tag::TagCreateArgs,
prisma::{tag, tag_on_object},
sync,
};
@ -46,39 +46,9 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
})
})
.procedure("create", {
#[derive(Type, Deserialize)]
pub struct TagCreateArgs {
pub name: String,
pub color: String,
}
R.with2(library())
.mutation(|(_, library), args: TagCreateArgs| async move {
let Library { db, sync, .. } = &library;
let pub_id = Uuid::new_v4().as_bytes().to_vec();
let created_tag = sync
.write_op(
db,
sync.unique_shared_create(
sync::tag::SyncId {
pub_id: pub_id.clone(),
},
[
(tag::name::NAME, json!(args.name)),
(tag::color::NAME, json!(args.color)),
],
),
db.tag().create(
pub_id,
vec![
tag::name::set(Some(args.name)),
tag::color::set(Some(args.color)),
],
),
)
.await?;
let created_tag = args.exec(&library).await?;
invalidate_query!(library, "tags.list");

13
core/src/library/manager.rs
View file

@ -1,8 +1,8 @@
use crate::{
invalidate_query,
location::{indexer::rules, LocationManagerError},
location::{indexer, LocationManagerError},
node::{NodeConfig, Platform},
object::orphan_remover::OrphanRemoverActor,
object::{orphan_remover::OrphanRemoverActor, tag},
prisma::{location, node},
sync::{SyncManager, SyncMessage},
util::{
@ -77,7 +77,7 @@ pub enum LibraryManagerError {
#[error("failed to parse uuid: {0}")]
Uuid(#[from] uuid::Error),
#[error("failed to run indexer rules seeder: {0}")]
IndexerRulesSeeder(#[from] rules::SeederError),
IndexerRulesSeeder(#[from] indexer::rules::seed::SeederError),
// #[error("failed to initialise the key manager: {0}")]
// KeyManager(#[from] sd_crypto::Error),
#[error("failed to run library migrations: {0}")]
@ -251,7 +251,8 @@ impl LibraryManager {
debug!("Loaded library '{id:?}'");
// Run seeders
rules::seeder(&library.db).await?;
tag::seed::new_library(&library).await?;
indexer::rules::seed::new_or_existing_library(&library).await?;
debug!("Seeded library '{id:?}'");
@ -458,8 +459,6 @@ impl LibraryManager {
// let key_manager = Arc::new(KeyManager::new(vec![]).await?);
// seed_keymanager(&db, &key_manager).await?;
rules::seeder(&db).await?;
let (sync_manager, sync_rx) = SyncManager::new(&db, id);
Self::emit(
@ -481,6 +480,8 @@ impl LibraryManager {
identity,
};
indexer::rules::seed::new_or_existing_library(&library).await?;
for location in library
.db
.location()

202
core/src/location/indexer/rules.rs → core/src/location/indexer/rules/mod.rs
View file

@ -1,3 +1,5 @@
pub mod seed;
use crate::{
library::Library,
prisma::indexer_rule,
@ -616,206 +618,6 @@ pub fn generate_pub_id() -> Uuid {
}
}
mod seeder {
use crate::{
location::indexer::rules::{IndexerRuleError, RulePerKind},
prisma::PrismaClient,
util::db::uuid_to_bytes,
};
use chrono::Utc;
use sd_prisma::prisma::indexer_rule;
use thiserror::Error;
use uuid::Uuid;
#[derive(Error, Debug)]
pub enum SeederError {
#[error("Failed to run indexer rules seeder: {0}")]
IndexerRules(#[from] IndexerRuleError),
#[error("An error occurred with the database while applying migrations: {0}")]
DatabaseError(#[from] prisma_client_rust::QueryError),
}
struct SystemIndexerRule {
name: &'static str,
rules: Vec<RulePerKind>,
default: bool,
}
pub async fn seeder(client: &PrismaClient) -> Result<(), SeederError> {
// DO NOT REORDER THIS ARRAY!
for (i, rule) in [
no_os_protected(),
no_hidden(),
only_git_repos(),
only_images(),
]
.into_iter()
.enumerate()
{
let pub_id = uuid_to_bytes(Uuid::from_u128(i as u128));
let rules = rmp_serde::to_vec_named(&rule.rules).map_err(IndexerRuleError::from)?;
use indexer_rule::*;
let data = vec![
name::set(Some(rule.name.to_string())),
rules_per_kind::set(Some(rules.clone())),
default::set(Some(rule.default)),
date_created::set(Some(Utc::now().into())),
date_modified::set(Some(Utc::now().into())),
];
client
.indexer_rule()
.upsert(
indexer_rule::pub_id::equals(pub_id.clone()),
indexer_rule::create(pub_id.clone(), data.clone()),
data,
)
.exec()
.await?;
}
Ok(())
}
fn no_os_protected() -> SystemIndexerRule {
SystemIndexerRule {
// TODO: On windows, beside the listed files, any file with the FILE_ATTRIBUTE_SYSTEM should be considered a system file
// https://learn.microsoft.com/en-us/windows/win32/fileio/file-attribute-constants#FILE_ATTRIBUTE_SYSTEM
name: "No OS protected",
default: true,
rules: vec![
RulePerKind::new_reject_files_by_globs_str(
[
vec![
"**/.spacedrive",
],
// Globset, even on Windows, requires the use of / as a separator
// https://github.com/github/gitignore/blob/main/Global/Windows.gitignore
// https://learn.microsoft.com/en-us/windows/win32/fileio/naming-a-file
#[cfg(target_os = "windows")]
vec![
// Windows thumbnail cache files
"**/{Thumbs.db,Thumbs.db:encryptable,ehthumbs.db,ehthumbs_vista.db}",
// Dump file
"**/*.stackdump",
// Folder config file
"**/[Dd]esktop.ini",
// Recycle Bin used on file shares
"**/$RECYCLE.BIN",
// Chkdsk recovery directory
"**/FOUND.[0-9][0-9][0-9]",
// Reserved names
"**/{CON,PRN,AUX,NUL,COM0,COM1,COM2,COM3,COM4,COM5,COM6,COM7,COM8,COM9,LPT0,LPT1,LPT2,LPT3,LPT4,LPT5,LPT6,LPT7,LPT8,LPT9}",
"**/{CON,PRN,AUX,NUL,COM0,COM1,COM2,COM3,COM4,COM5,COM6,COM7,COM8,COM9,LPT0,LPT1,LPT2,LPT3,LPT4,LPT5,LPT6,LPT7,LPT8,LPT9}.*",
// User special files
"C:/Users/*/NTUSER.DAT*",
"C:/Users/*/ntuser.dat*",
"C:/Users/*/{ntuser.ini,ntuser.dat,NTUSER.DAT}",
// User special folders (most of these the user dont even have permission to access)
"C:/Users/*/{Cookies,AppData,NetHood,Recent,PrintHood,SendTo,Templates,Start Menu,Application Data,Local Settings}",
// System special folders
"C:/{$Recycle.Bin,$WinREAgent,Documents and Settings,Program Files,Program Files (x86),ProgramData,Recovery,PerfLogs,Windows,Windows.old}",
// NTFS internal dir, can exists on any drive
"[A-Z]:/System Volume Information",
// System special files
"C:/{config,pagefile,hiberfil}.sys",
// Windows can create a swapfile on any drive
"[A-Z]:/swapfile.sys",
"C:/DumpStack.log.tmp",
],
// https://github.com/github/gitignore/blob/main/Global/macOS.gitignore
// https://developer.apple.com/library/archive/documentation/FileManagement/Conceptual/FileSystemProgrammingGuide/FileSystemOverview/FileSystemOverview.html#//apple_ref/doc/uid/TP40010672-CH2-SW14
#[cfg(any(target_os = "ios", target_os = "macos"))]
vec![
"**/.{DS_Store,AppleDouble,LSOverride}",
// Icon must end with two \r
"**/Icon\r\r",
// Thumbnails
"**/._*",
],
#[cfg(target_os = "macos")]
vec![
"/{System,Network,Library,Applications}",
"/Users/*/{Library,Applications}",
"**/*.photoslibrary/{database,external,private,resources,scope}",
// Files that might appear in the root of a volume
"**/.{DocumentRevisions-V100,fseventsd,Spotlight-V100,TemporaryItems,Trashes,VolumeIcon.icns,com.apple.timemachine.donotpresent}",
// Directories potentially created on remote AFP share
"**/.{AppleDB,AppleDesktop,apdisk}",
"**/{Network Trash Folder,Temporary Items}",
],
// https://github.com/github/gitignore/blob/main/Global/Linux.gitignore
#[cfg(target_os = "linux")]
vec![
"**/*~",
// temporary files which can be created if a process still has a handle open of a deleted file
"**/.fuse_hidden*",
// KDE directory preferences
"**/.directory",
// Linux trash folder which might appear on any partition or disk
"**/.Trash-*",
// .nfs files are created when an open file is removed but is still being accessed
"**/.nfs*",
],
#[cfg(target_os = "android")]
vec![
"**/.nomedia",
"**/.thumbnails",
],
// https://en.wikipedia.org/wiki/Unix_filesystem#Conventional_directory_layout
// https://en.wikipedia.org/wiki/Filesystem_Hierarchy_Standard
#[cfg(target_family = "unix")]
vec![
// Directories containing unix memory/device mapped files/dirs
"/{dev,sys,proc}",
// Directories containing special files for current running programs
"/{run,var,boot}",
// ext2-4 recovery directory
"**/lost+found",
],
]
.into_iter()
.flatten()
).expect("this is hardcoded and should always work"),
],
}
}
fn no_hidden() -> SystemIndexerRule {
SystemIndexerRule {
name: "No Hidden",
default: true,
rules: vec![RulePerKind::new_reject_files_by_globs_str(["**/.*"])
.expect("this is hardcoded and should always work")],
}
}
fn only_git_repos() -> SystemIndexerRule {
SystemIndexerRule {
name: "Only Git Repositories",
default: false,
rules: vec![RulePerKind::AcceptIfChildrenDirectoriesArePresent(
[".git".to_string()].into_iter().collect(),
)],
}
}
fn only_images() -> SystemIndexerRule {
SystemIndexerRule {
name: "Only Images",
default: false,
rules: vec![RulePerKind::new_accept_files_by_globs_str([
"*.{avif,bmp,gif,ico,jpeg,jpg,png,svg,tif,tiff,webp}",
])
.expect("this is hardcoded and should always work")],
}
}
}
pub use seeder::*;
#[cfg(test)]
#[allow(clippy::unwrap_used)]
mod tests {

197
core/src/location/indexer/rules/seed.rs Normal file
View file

@ -0,0 +1,197 @@
use crate::{
library::Library,
location::indexer::rules::{IndexerRuleError, RulePerKind},
util::db::uuid_to_bytes,
};
use chrono::Utc;
use sd_prisma::prisma::indexer_rule;
use thiserror::Error;
use uuid::Uuid;
#[derive(Error, Debug)]
pub enum SeederError {
#[error("Failed to run indexer rules seeder: {0}")]
IndexerRules(#[from] IndexerRuleError),
#[error("An error occurred with the database while applying migrations: {0}")]
DatabaseError(#[from] prisma_client_rust::QueryError),
}
struct SystemIndexerRule {
name: &'static str,
rules: Vec<RulePerKind>,
default: bool,
}
/// Seeds system indexer rules into a new or existing library,
pub async fn new_or_existing_library(library: &Library) -> Result<(), SeederError> {
// DO NOT REORDER THIS ARRAY!
for (i, rule) in [
no_os_protected(),
no_hidden(),
only_git_repos(),
only_images(),
]
.into_iter()
.enumerate()
{
let pub_id = uuid_to_bytes(Uuid::from_u128(i as u128));
let rules = rmp_serde::to_vec_named(&rule.rules).map_err(IndexerRuleError::from)?;
use indexer_rule::*;
let data = vec![
name::set(Some(rule.name.to_string())),
rules_per_kind::set(Some(rules.clone())),
default::set(Some(rule.default)),
date_created::set(Some(Utc::now().into())),
date_modified::set(Some(Utc::now().into())),
];
library
.db
.indexer_rule()
.upsert(
indexer_rule::pub_id::equals(pub_id.clone()),
indexer_rule::create(pub_id.clone(), data.clone()),
data,
)
.exec()
.await?;
}
Ok(())
}
fn no_os_protected() -> SystemIndexerRule {
SystemIndexerRule {
// TODO: On windows, beside the listed files, any file with the FILE_ATTRIBUTE_SYSTEM should be considered a system file
// https://learn.microsoft.com/en-us/windows/win32/fileio/file-attribute-constants#FILE_ATTRIBUTE_SYSTEM
name: "No OS protected",
default: true,
rules: vec![
RulePerKind::new_reject_files_by_globs_str(
[
vec![
"**/.spacedrive",
],
// Globset, even on Windows, requires the use of / as a separator
// https://github.com/github/gitignore/blob/main/Global/Windows.gitignore
// https://learn.microsoft.com/en-us/windows/win32/fileio/naming-a-file
#[cfg(target_os = "windows")]
vec![
// Windows thumbnail cache files
"**/{Thumbs.db,Thumbs.db:encryptable,ehthumbs.db,ehthumbs_vista.db}",
// Dump file
"**/*.stackdump",
// Folder config file
"**/[Dd]esktop.ini",
// Recycle Bin used on file shares
"**/$RECYCLE.BIN",
// Chkdsk recovery directory
"**/FOUND.[0-9][0-9][0-9]",
// Reserved names
"**/{CON,PRN,AUX,NUL,COM0,COM1,COM2,COM3,COM4,COM5,COM6,COM7,COM8,COM9,LPT0,LPT1,LPT2,LPT3,LPT4,LPT5,LPT6,LPT7,LPT8,LPT9}",
"**/{CON,PRN,AUX,NUL,COM0,COM1,COM2,COM3,COM4,COM5,COM6,COM7,COM8,COM9,LPT0,LPT1,LPT2,LPT3,LPT4,LPT5,LPT6,LPT7,LPT8,LPT9}.*",
// User special files
"C:/Users/*/NTUSER.DAT*",
"C:/Users/*/ntuser.dat*",
"C:/Users/*/{ntuser.ini,ntuser.dat,NTUSER.DAT}",
// User special folders (most of these the user dont even have permission to access)
"C:/Users/*/{Cookies,AppData,NetHood,Recent,PrintHood,SendTo,Templates,Start Menu,Application Data,Local Settings}",
// System special folders
"C:/{$Recycle.Bin,$WinREAgent,Documents and Settings,Program Files,Program Files (x86),ProgramData,Recovery,PerfLogs,Windows,Windows.old}",
// NTFS internal dir, can exists on any drive
"[A-Z]:/System Volume Information",
// System special files
"C:/{config,pagefile,hiberfil}.sys",
// Windows can create a swapfile on any drive
"[A-Z]:/swapfile.sys",
"C:/DumpStack.log.tmp",
],
// https://github.com/github/gitignore/blob/main/Global/macOS.gitignore
// https://developer.apple.com/library/archive/documentation/FileManagement/Conceptual/FileSystemProgrammingGuide/FileSystemOverview/FileSystemOverview.html#//apple_ref/doc/uid/TP40010672-CH2-SW14
#[cfg(any(target_os = "ios", target_os = "macos"))]
vec![
"**/.{DS_Store,AppleDouble,LSOverride}",
// Icon must end with two \r
"**/Icon\r\r",
// Thumbnails
"**/._*",
],
#[cfg(target_os = "macos")]
vec![
"/{System,Network,Library,Applications}",
"/Users/*/{Library,Applications}",
"**/*.photoslibrary/{database,external,private,resources,scope}",
// Files that might appear in the root of a volume
"**/.{DocumentRevisions-V100,fseventsd,Spotlight-V100,TemporaryItems,Trashes,VolumeIcon.icns,com.apple.timemachine.donotpresent}",
// Directories potentially created on remote AFP share
"**/.{AppleDB,AppleDesktop,apdisk}",
"**/{Network Trash Folder,Temporary Items}",
],
// https://github.com/github/gitignore/blob/main/Global/Linux.gitignore
#[cfg(target_os = "linux")]
vec![
"**/*~",
// temporary files which can be created if a process still has a handle open of a deleted file
"**/.fuse_hidden*",
// KDE directory preferences
"**/.directory",
// Linux trash folder which might appear on any partition or disk
"**/.Trash-*",
// .nfs files are created when an open file is removed but is still being accessed
"**/.nfs*",
],
#[cfg(target_os = "android")]
vec![
"**/.nomedia",
"**/.thumbnails",
],
// https://en.wikipedia.org/wiki/Unix_filesystem#Conventional_directory_layout
// https://en.wikipedia.org/wiki/Filesystem_Hierarchy_Standard
#[cfg(target_family = "unix")]
vec![
// Directories containing unix memory/device mapped files/dirs
"/{dev,sys,proc}",
// Directories containing special files for current running programs
"/{run,var,boot}",
// ext2-4 recovery directory
"**/lost+found",
],
]
.into_iter()
.flatten()
).expect("this is hardcoded and should always work"),
],
}
}
fn no_hidden() -> SystemIndexerRule {
SystemIndexerRule {
name: "No Hidden",
default: true,
rules: vec![RulePerKind::new_reject_files_by_globs_str(["**/.*"])
.expect("this is hardcoded and should always work")],
}
}
fn only_git_repos() -> SystemIndexerRule {
SystemIndexerRule {
name: "Only Git Repositories",
default: false,
rules: vec![RulePerKind::AcceptIfChildrenDirectoriesArePresent(
[".git".to_string()].into_iter().collect(),
)],
}
}
fn only_images() -> SystemIndexerRule {
SystemIndexerRule {
name: "Only Images",
default: false,
rules: vec![RulePerKind::new_accept_files_by_globs_str([
"*.{avif,bmp,gif,ico,jpeg,jpg,png,svg,tif,tiff,webp}",
])
.expect("this is hardcoded and should always work")],
}
}

36
core/src/object/tag.rs
View file

@ -1,36 +0,0 @@
use prisma_client_rust::QueryError;
use serde::Deserialize;
use specta::Type;
use uuid::Uuid;
use crate::prisma::{tag, PrismaClient};
#[derive(Type, Deserialize)]
pub struct Tag {
pub name: String,
pub color: String,
}
impl Tag {
#[allow(dead_code)]
pub fn new(name: String, color: String) -> Self {
Self { name, color }
}
#[allow(dead_code)]
pub async fn save(self, db: &PrismaClient) -> Result<(), QueryError> {
db.tag()
.create(
Uuid::new_v4().as_bytes().to_vec(),
vec![
tag::name::set(Some(self.name)),
tag::color::set(Some(self.color)),
],
)
.exec()
.await?;
Ok(())
}
}

45
core/src/object/tag/mod.rs Normal file
View file

@ -0,0 +1,45 @@
pub mod seed;
use serde::Deserialize;
use serde_json::json;
use specta::Type;
use uuid::Uuid;
use crate::{library::Library, prisma::tag, sync};
#[derive(Type, Deserialize)]
pub struct TagCreateArgs {
pub name: String,
pub color: String,
}
impl TagCreateArgs {
pub async fn exec(
self,
Library { db, sync, .. }: &Library,
) -> prisma_client_rust::Result<tag::Data> {
let pub_id = Uuid::new_v4().as_bytes().to_vec();
sync.write_op(
db,
sync.unique_shared_create(
sync::tag::SyncId {
pub_id: pub_id.clone(),
},
[
(tag::name::NAME, json!(&self.name)),
(tag::color::NAME, json!(&self.color)),
],
),
db.tag().create(
pub_id,
vec![
tag::name::set(Some(self.name)),
tag::color::set(Some(self.color)),
],
),
)
.await
}
}

32
core/src/object/tag/seed.rs Normal file
View file

@ -0,0 +1,32 @@
use super::TagCreateArgs;
use crate::library::Library;
/// Seeds tags in a new library.
/// Shouldn't be called more than once!
pub async fn new_library(library: &Library) -> prisma_client_rust::Result<()> {
// remove type after tags are added
let tags = [
TagCreateArgs {
name: "Keepsafe".to_string(),
color: "#D9188E".to_string(),
},
TagCreateArgs {
name: "Hidden".to_string(),
color: "#646278".to_string(),
},
TagCreateArgs {
name: "Projects".to_string(),
color: "#42D097".to_string(),
},
TagCreateArgs {
name: "Memes".to_string(),
color: "#A718D9".to_string(),
},
];
for tag in tags {
tag.exec(&library).await?;
}
Ok(())
}