Added crates folder (#389)

* restructure core for crates folder

- improve naming consistency
- added placeholder files for near future code
- added docs

* - removed all typescript from core
- moved rspc bindings to client

* remove useless git ignore & rustfmt

.cspell/project_words.txt

@@ -1,42 +1,46 @@
-spacedrive
-spacedriveapp
-vdfs
-haoyuan
-brendonovich
-codegen
-elon
-deel
-haden
 akar
-benja
-haris
-mehrzad
-OSSC
-josephjacks
-rauch
-ravikant
-neha
-narkhede
 allred
-lütke
-tobiaslutke
-justinhoffman
-rywalker
-zacharysmith
-sanjay
-poonen
-mytton
-davidmytton
-richelsen
-lesterlee
 alluxio
 augusto
+benja
+brendonovich
+chacha
+codegen
+davidmytton
+deel
+elon
+encryptor
+haden
+haoyuan
+haris
+josephjacks
+justinhoffman
+Keyslot
+keyslots
+lesterlee
+lütke
 marietti
-vijay
+mehrzad
-sharma
+mytton
+narkhede
 naveen
+neha
 noco
+OSSC
+poonen
+rauch
+ravikant
+richelsen
 rspc
 rspcws
-stringly
+rywalker
+sanjay
+sharma
+spacedrive
+spacedriveapp
 specta
+stringly
+tobiaslutke
+vdfs
+vijay
+zacharysmith

.gitignore

@@ -27,6 +27,7 @@ apps/*/stats.html
 docs/public/*.st
 docs/public/*.toml
 dev.db
+stats.html
 
 !cli/cmd/turbo
 cli/npm/turbo-android-arm64/bin
@@ -61,6 +62,7 @@ yalc.lock
 todos.md
 examples/*/*.lock
 /target
+/core/src/prisma.rs
 
 /sdserver_data
 .spacedrive

.prettierrc.json

@@ -7,13 +7,7 @@
 	"bracketSameLine": false,
 	"semi": true,
 	"quoteProps": "consistent",
-	"importOrder": [
+	"importOrder": ["^@sd/interface/(.*)$", "^@sd/client/(.*)$", "^@sd/ui/(.*)$", "^[./]"],
-		"^@sd/core/(.*)$",
-		"^@sd/interface/(.*)$",
-		"^@sd/client/(.*)$",
-		"^@sd/ui/(.*)$",
-		"^[./]"
-	],
 	"importOrderSeparation": true,
 	"importOrderSortSpecifiers": true
 }

.rustfmt.toml

@@ -1,6 +1,12 @@
+max_width = 100
 hard_tabs = true
-match_block_trailing_comma = true
-max_width = 90
 newline_style = "Unix"
-use_field_init_shorthand = true
+use_small_heuristics = "Default"
-use_try_shorthand = true
+reorder_imports = true
+reorder_modules = true
+remove_nested_parens = true
+edition = "2021"
+merge_derives = true
+use_try_shorthand = false
+use_field_init_shorthand = false
+force_explicit_abi = true

.vscode/.todo

@@ -1,50 +0,0 @@
-####
-# Completed
-####
-✔ Configured a Tauri window 
-✔ Primative UI components (Button, Input, Shortcut etc) 
-✔ Connect & create sqlite database in application data folder
-✔ Auto migrate database on app launch 
-✔ Define data schema in Rust 
-✔ Scan given directory recursively and write to database 
-✔ App frame and router set up 
-✔ Render basic file list with database data 
-✔ Create a global store 
-✔ Virtualized list for explorer @done(21-10-24 04:03)
-    Also cache folder contents and serve lazy, updating in background
-✔ Native file type icons @done(21-10-24 04:04)
-✔ Cache native icons in app data folder @done(21-10-24 11:30)
-✔ File inspector @done(21-10-24 23:30)
-✔ Upgrade to mono-repo @done(21-12-27 18:30)
-✔ Create project readme @done(21-12-28 18:30)
-✔ Refactor indexer to scan efficiently @done(21-12-29 18:30)
-✔ File previews
-✔ Generate CAS ids
-✔ Settings screen
-✔ Job queue system
-✔ Landing page
-✔ Volume identification
-✔ Statistic calucations
-    Folder size, Volume size, Total capacity, Total unique etc..
-    
-####
-# Next up 
-####
-☐ Tag creation and assignment
-☐ Right click menu
-☐ Set up Tauri updater
-☐ Onboarding flow
-☐ Explorer grid view
-☐ Changelog
-
-####
-# Planned features
-####
-☐ Action system & command pallete
-☐ Explorer path viewer / editor
-☐ File encryptor
-☐ File viewer / player
-☐ Secret keystore
-☐ Search 
-    Efficient way to search sqlite: make file table WITHOUT ROWID
-☐ Open with

Cargo.lock

@@ -29,6 +29,41 @@ version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234"
 
+[[package]]
+name = "aead"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c192eb8f11fc081b0fe4259ba5af04217d4e0faddd02417310a927911abd7c8"
+dependencies = [
+ "crypto-common",
+ "generic-array",
+]
+
+[[package]]
+name = "aes"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bfe0133578c0986e1fe3dfcd4af1cc5b2dd6c3dbf534d69916ce16a2701d40ba"
+dependencies = [
+ "cfg-if 1.0.0",
+ "cipher",
+ "cpufeatures",
+]
+
+[[package]]
+name = "aes-gcm"
+version = "0.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "82e1366e0c69c9f927b1fa5ce2c7bf9eafc8f9268c0b9800729e8b267612447c"
+dependencies = [
+ "aead",
+ "aes",
+ "cipher",
+ "ctr",
+ "ghash",
+ "subtle",
+]
+
 [[package]]
 name = "ahash"
 version = "0.7.6"
@@ -88,6 +123,29 @@ version = "1.0.65"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "98161a4e3e2184da77bb14f02184cdd111e83bbbcc9979dfee3c44b9a85f5602"
 
+[[package]]
+name = "argon2"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "db4ce4441f99dbd377ca8a8f57b698c44d0d6e712d8329b5040da5a64aa1ce73"
+dependencies = [
+ "base64ct",
+ "blake2",
+ "password-hash",
+]
+
+[[package]]
+name = "arrayref"
+version = "0.3.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544"
+
+[[package]]
+name = "arrayvec"
+version = "0.7.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6"
+
 [[package]]
 name = "ascii"
 version = "0.9.3"
@@ -257,6 +315,17 @@ dependencies = [
  "rustc-demangle",
 ]
 
+[[package]]
+name = "balloon-hash"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1fdbafd8c776e0a7c250b5dfaf4723477e0d92b42c5a91ce4e50bd9bb2dcc9a1"
+dependencies = [
+ "crypto-bigint",
+ "digest 0.10.5",
+ "password-hash",
+]
+
 [[package]]
 name = "base64"
 version = "0.12.3"
@@ -269,6 +338,12 @@ version = "0.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd"
 
+[[package]]
+name = "base64ct"
+version = "1.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ea2b2456fd614d856680dcd9fcc660a51a820fa09daef2e49772b56a193c8474"
+
 [[package]]
 name = "bigdecimal"
 version = "0.2.2"
@@ -321,6 +396,29 @@ dependencies = [
  "typenum",
 ]
 
+[[package]]
+name = "blake2"
+version = "0.10.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b9cf849ee05b2ee5fba5e36f97ff8ec2533916700fc0758d40d92136a42f3388"
+dependencies = [
+ "digest 0.10.5",
+]
+
+[[package]]
+name = "blake3"
+version = "1.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a08e53fc5a564bb15bfe6fae56bd71522205f1f91893f9c0116edad6496c183f"
+dependencies = [
+ "arrayref",
+ "arrayvec",
+ "cc",
+ "cfg-if 1.0.0",
+ "constant_time_eq",
+ "digest 0.10.5",
+]
+
 [[package]]
 name = "block"
 version = "0.1.6"
@@ -498,6 +596,30 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
+[[package]]
+name = "chacha20"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c7fc89c7c5b9e7a02dfe45cd2367bae382f9ed31c61ca8debe5f827c420a2f08"
+dependencies = [
+ "cfg-if 1.0.0",
+ "cipher",
+ "cpufeatures",
+]
+
+[[package]]
+name = "chacha20poly1305"
+version = "0.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "10cd79432192d1c0f4e1a0fef9527696cc039165d729fb41b3f4f4f354c2dc35"
+dependencies = [
+ "aead",
+ "chacha20",
+ "cipher",
+ "poly1305",
+ "zeroize",
+]
+
 [[package]]
 name = "chrono"
 version = "0.4.22"
@@ -514,6 +636,17 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "cipher"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d1873270f8f7942c191139cb8a40fd228da6c3fd2fc376d7e92d47aa14aeb59e"
+dependencies = [
+ "crypto-common",
+ "inout",
+ "zeroize",
+]
+
 [[package]]
 name = "clang-sys"
 version = "1.4.0"
@@ -615,6 +748,12 @@ dependencies = [
  "wasm-bindgen",
 ]
 
+[[package]]
+name = "constant_time_eq"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc"
+
 [[package]]
 name = "convert_case"
 version = "0.4.0"
@@ -741,6 +880,24 @@ dependencies = [
  "once_cell",
 ]
 
+[[package]]
+name = "crypto"
+version = "0.1.0"
+dependencies = [
+ "chacha20poly1305",
+ "rand 0.8.5",
+]
+
+[[package]]
+name = "crypto-bigint"
+version = "0.4.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9f2b443d17d49dad5ef0ede301c3179cc923b8822f3393b4d2c28c269dd4a122"
+dependencies = [
+ "generic-array",
+ "subtle",
+]
+
 [[package]]
 name = "crypto-common"
 version = "0.1.6"
@@ -748,6 +905,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
 dependencies = [
  "generic-array",
+ "rand_core 0.6.4",
  "typenum",
 ]
 
@@ -788,6 +946,15 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "ctr"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0d14f329cfbaf5d0e06b5e87fff7e265d2673c5ea7d2c27691a2c107db1442a0"
+dependencies = [
+ "cipher",
+]
+
 [[package]]
 name = "ctrlc"
 version = "3.2.3"
@@ -913,6 +1080,18 @@ dependencies = [
  "byteorder",
 ]
 
+[[package]]
+name = "deoxys"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "00603a49e114bd99d87a4ab8d480f36ecd1451a9d6474f66973d1a829ff77789"
+dependencies = [
+ "aead",
+ "aes",
+ "subtle",
+ "zeroize",
+]
+
 [[package]]
 name = "derive_more"
 version = "0.99.17"
@@ -952,6 +1131,7 @@ checksum = "adfbc57365a37acbd2ebf2b64d7e69bb766e2fea813521ed536f5d0520dcf86c"
 dependencies = [
  "block-buffer 0.10.3",
  "crypto-common",
+ "subtle",
 ]
 
 [[package]]
@@ -1481,6 +1661,16 @@ dependencies = [
  "wasm-bindgen",
 ]
 
+[[package]]
+name = "ghash"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d930750de5717d2dd0b8c0d42c076c0e884c81a73e6cab859bbd2339c71e3e40"
+dependencies = [
+ "opaque-debug",
+ "polyval",
+]
+
 [[package]]
 name = "gif"
 version = "0.11.4"
@@ -1603,6 +1793,22 @@ dependencies = [
  "system-deps 6.0.2",
 ]
 
+[[package]]
+name = "goop"
+version = "0.1.0"
+dependencies = [
+ "aead",
+ "aes-gcm",
+ "anyhow",
+ "argon2",
+ "balloon-hash",
+ "blake3",
+ "chacha20poly1305",
+ "deoxys",
+ "rand 0.8.5",
+ "zeroize",
+]
+
 [[package]]
 name = "graphql-parser"
 version = "0.3.0"
@@ -2013,6 +2219,15 @@ dependencies = [
  "adler32",
 ]
 
+[[package]]
+name = "inout"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5"
+dependencies = [
+ "generic-array",
+]
+
 [[package]]
 name = "instant"
 version = "0.1.12"
@@ -3004,6 +3219,10 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "p2p"
+version = "0.1.0"
+
 [[package]]
 name = "pango"
 version = "0.15.10"
@@ -3113,6 +3332,17 @@ dependencies = [
  "schema-ast",
 ]
 
+[[package]]
+name = "password-hash"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7676374caaee8a325c9e7a2ae557f216c5563a171d6997b0ef8a65af35147700"
+dependencies = [
+ "base64ct",
+ "rand_core 0.6.4",
+ "subtle",
+]
+
 [[package]]
 name = "paste"
 version = "1.0.9"
@@ -3365,6 +3595,29 @@ dependencies = [
  "miniz_oxide",
 ]
 
+[[package]]
+name = "poly1305"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8159bd90725d2df49889a078b54f4f79e87f1f8a8444194cdca81d38f5393abf"
+dependencies = [
+ "cpufeatures",
+ "opaque-debug",
+ "universal-hash",
+]
+
+[[package]]
+name = "polyval"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7ef234e08c11dfcb2e56f79fd70f6f2eb7f025c0ce2333e82f4f0518ecad30c6"
+dependencies = [
+ "cfg-if 1.0.0",
+ "cpufeatures",
+ "opaque-debug",
+ "universal-hash",
+]
+
 [[package]]
 name = "ppv-lite86"
 version = "0.2.16"
@@ -4201,6 +4454,17 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
 
+[[package]]
+name = "sd_ffmpeg"
+version = "0.1.0"
+dependencies = [
+ "ffmpeg-sys-next",
+ "tempfile",
+ "thiserror",
+ "tokio",
+ "webp",
+]
+
 [[package]]
 name = "sdcore"
 version = "0.1.0"
@@ -4209,6 +4473,7 @@ dependencies = [
  "async-trait",
  "base64 0.13.0",
  "chrono",
+ "crypto",
  "ctor",
  "data-encoding",
  "enumflags2",
@@ -4227,12 +4492,12 @@ dependencies = [
  "rmp",
  "rmp-serde",
  "rspc",
+ "sd_ffmpeg",
  "serde",
  "serde_json",
  "sysinfo",
  "tempfile",
  "thiserror",
- "thumbnailer",
  "tokio",
  "tracing",
  "tracing-subscriber",
@@ -4862,6 +5127,12 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "subtle"
+version = "2.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601"
+
 [[package]]
 name = "swift-rs"
 version = "0.3.0"
@@ -4894,6 +5165,10 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "sync"
+version = "0.1.0"
+
 [[package]]
 name = "sync_wrapper"
 version = "0.1.1"
@@ -5274,17 +5549,6 @@ dependencies = [
  "num_cpus",
 ]
 
-[[package]]
-name = "thumbnailer"
-version = "0.1.0"
-dependencies = [
- "ffmpeg-sys-next",
- "tempfile",
- "thiserror",
- "tokio",
- "webp",
-]
-
 [[package]]
 name = "tiff"
 version = "0.7.3"
@@ -5654,6 +5918,16 @@ version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e"
 
+[[package]]
+name = "universal-hash"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7d3160b73c9a19f7e2939a2fdad446c57c1bbbbf4d919d3213ff1267a580d8b5"
+dependencies = [
+ "crypto-common",
+ "subtle",
+]
+
 [[package]]
 name = "unreachable"
 version = "1.0.0"
@@ -6329,3 +6603,9 @@ name = "xml-rs"
 version = "0.8.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d2d7d3948613f75c98fd9328cfdcc45acc4d360655289d0a7d4ec931392200a3"
+
+[[package]]
+name = "zeroize"
+version = "1.5.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c394b5bd0c6f669e7275d9c20aa90ae064cb22e75a1cad54e1b34088034b149f"

Cargo.toml

@@ -1,9 +1,9 @@
 [workspace]
 members = [
+  "core",
+  "crates/*",
   "apps/desktop/src-tauri",
   "apps/mobile/rust",
-  "core",
-  "core/prisma",
   "apps/server",
 ]
 resolver = "2"

apps/desktop/package.json

@@ -14,7 +14,6 @@
 		"@rspc/client": "^0.0.6",
 		"@rspc/tauri": "^0.0.6",
 		"@sd/client": "workspace:*",
-		"@sd/core": "workspace:*",
 		"@sd/interface": "workspace:*",
 		"@sd/ui": "workspace:*",
 		"@tanstack/react-query": "^4.0.10",

apps/desktop/tsconfig.json

@@ -1,5 +1,7 @@
 {
 	"extends": "../../packages/config/interface.tsconfig.json",
-	"compilerOptions": {},
+	"compilerOptions": {
+		"target": "es2020"
+	},
 	"include": ["src"]
 }

apps/web/package.json

@@ -11,7 +11,6 @@
 		"@fontsource/inter": "^4.5.11",
 		"@rspc/client": "^0.0.6",
 		"@sd/client": "workspace:*",
-		"@sd/core": "workspace:*",
 		"@sd/interface": "workspace:*",
 		"@sd/ui": "workspace:*",
 		"@tanstack/react-query": "^4.0.10",

core/.gitignore

@@ -1,4 +0,0 @@
-/target
-/types
-*.db*
-/src/prisma.rs

core/.rustfmt.toml

@@ -1,13 +0,0 @@
-max_width = 100
-hard_tabs = true
-newline_style = "Unix"
-use_small_heuristics = "Default"
-reorder_imports = true
-reorder_modules = true
-remove_nested_parens = true
-edition = "2021"
-merge_derives = true
-use_try_shorthand = false
-use_field_init_shorthand = false
-force_explicit_abi = true
-# normalize_comments = true

core/Cargo.toml

@@ -16,7 +16,7 @@ mobile = [
 ] # This feature allows features to be disabled when the Core is running on mobile.
 ffmpeg = [
   "dep:ffmpeg-next",
-  "dep:thumbnailer",
+  "dep:sd_ffmpeg",
 ] # This feature controls whether the Spacedrive Core contains functionality which requires FFmpeg.
 
 [dependencies]
@@ -56,7 +56,8 @@ async-trait = "^0.1.52"
 image = "0.24.1"
 webp = "0.2.2"
 ffmpeg-next = { version = "5.0.3", optional = true, features = [] }
-thumbnailer = { path = "./thumbnailer", optional = true }
+sd_ffmpeg = { path = "../crates/ffmpeg", optional = true }
+crypto = { path = "../crates/crypto" }
 fs_extra = "1.2.0"
 tracing = "0.1.35"
 tracing-subscriber = { version = "0.3.14", features = ["env-filter"] }

core/package.json

@@ -1,17 +0,0 @@
-{
-	"name": "@sd/core",
-	"version": "0.0.0",
-	"main": "index.js",
-	"license": "GPL-3.0-only",
-	"scripts": {
-		"codegen": "cargo test",
-		"build": "cargo build",
-		"test": "cargo test",
-		"test:log": "cargo test -- --nocapture",
-		"prisma": "cargo prisma"
-	},
-	"devDependencies": {
-		"@types/node": "^18.6.1",
-		"typescript": "^4.7.4"
-	}
-}

core/scripts/bindingsIndex.ts

@@ -1,30 +0,0 @@
-import * as fs from 'fs/promises';
-import * as path from 'path';
-
-(async function main() {
-	async function exists(path: string) {
-		try {
-			await fs.access(path);
-			return true;
-		} catch {
-			return false;
-		}
-	}
-
-	const files = await fs.readdir(path.join(__dirname, '../bindings'));
-	const bindings = files.filter((f) => f.endsWith('.ts'));
-	let str = '';
-	// str += `export * from './types';\n`;
-
-	for (let binding of bindings) {
-		str += `export * from './bindings/${binding.split('.')[0]}';\n`;
-	}
-
-	let indexExists = await exists(path.join(__dirname, '../index.ts'));
-
-	if (indexExists) {
-		await fs.rm(path.join(__dirname, '../index.ts'));
-	}
-
-	await fs.writeFile(path.join(__dirname, '../index.ts'), str);
-})();

core/src/api/jobs.rs

@@ -1,8 +1,10 @@
 use crate::{
-	encode::{ThumbnailJob, ThumbnailJobInit},
-	file::cas::{FileIdentifierJob, FileIdentifierJobInit},
 	job::{Job, JobManager},
 	location::{fetch_location, LocationError},
+	object::{
+		identifier_job::{FileIdentifierJob, FileIdentifierJobInit},
+		preview::{ThumbnailJob, ThumbnailJobInit},
+	},
 	prisma::location,
 };
 

core/src/api/locations.rs

@@ -1,11 +1,11 @@
 use crate::{
-	encode::THUMBNAIL_CACHE_DIR_NAME,
 	invalidate_query,
 	location::{
 		fetch_location,
-		indexer::{indexer_job::indexer_job_location, indexer_rules::IndexerRuleCreateArgs},
+		indexer::{indexer_job::indexer_job_location, rules::IndexerRuleCreateArgs},
 		scan_location, LocationCreateArgs, LocationError, LocationUpdateArgs,
 	},
+	object::preview::THUMBNAIL_CACHE_DIR_NAME,
 	prisma::{file, file_path, indexer_rule, indexer_rules_in_location, location, tag},
 };
 

core/src/api/mod.rs

@@ -110,8 +110,10 @@ mod tests {
 	#[test]
 	fn test_and_export_rspc_bindings() {
 		let r = super::mount();
-		r.export_ts(PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("./index.ts"))
+		r.export_ts(
-			.expect("Error exporting rspc Typescript bindings!");
+			PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../packages/client/src/core.ts"),
+		)
+		.expect("Error exporting rspc Typescript bindings!");
 		r.export_ts(
 			PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../apps/mobile/src/types/bindings.ts"),
 		)

core/src/api/tags.rs

@@ -5,8 +5,8 @@ use uuid::Uuid;
 
 use crate::{
 	api::locations::{file_with_paths, ExplorerContext, ExplorerData, ExplorerItem},
-	encode::THUMBNAIL_CACHE_DIR_NAME,
 	invalidate_query,
+	object::preview::THUMBNAIL_CACHE_DIR_NAME,
 	prisma::{file, tag, tag_on_file},
 };
 

core/src/file/cas/mod.rs

@@ -1,5 +0,0 @@
-mod checksum;
-mod identifier;
-
-pub use checksum::*;
-pub use identifier::*;

core/src/file/mod.rs

@@ -1 +0,0 @@
-pub mod cas;

core/src/job/job_manager.rs

@@ -1,9 +1,11 @@
 use crate::{
-	encode::{ThumbnailJob, THUMBNAIL_JOB_NAME},
-	file::cas::{FileIdentifierJob, IDENTIFIER_JOB_NAME},
 	job::{worker::Worker, DynJob, Job, JobError},
 	library::LibraryContext,
 	location::indexer::indexer_job::{IndexerJob, INDEXER_JOB_NAME},
+	object::{
+		identifier_job::{FileIdentifierJob, IDENTIFIER_JOB_NAME},
+		preview::{ThumbnailJob, THUMBNAIL_JOB_NAME},
+	},
 	prisma::{job, node},
 };
 

core/src/lib.rs

@@ -13,17 +13,16 @@ use tracing::{error, info};
 use tracing_subscriber::{filter::LevelFilter, fmt, prelude::*, EnvFilter};
 
 pub mod api;
-pub(crate) mod encode;
-pub(crate) mod file;
 pub(crate) mod job;
 pub(crate) mod library;
 pub(crate) mod location;
 pub(crate) mod node;
 pub(crate) mod object;
-pub(crate) mod prisma;
 pub(crate) mod util;
 pub(crate) mod volume;
 
+pub(crate) mod prisma;
+
 #[derive(Clone)]
 pub struct NodeContext {
 	pub config: Arc<NodeConfigManager>,

core/src/location/indexer/indexer_job.rs

@@ -12,7 +12,7 @@ use tokio::time::Instant;
 use tracing::info;
 
 use super::{
-	indexer_rules::IndexerRule,
+	rules::IndexerRule,
 	walk::{walk, WalkEntry},
 };
 

core/src/location/indexer/mod.rs

@@ -1,12 +1,12 @@
 pub mod indexer_job;
-pub mod indexer_rules;
+pub mod rules;
 mod walk;
 
 use globset::Error;
-use indexer_rules::RuleKind;
 use int_enum::IntEnumError;
 use rmp_serde::{decode::Error as RMPDecodeError, encode::Error as RMPEncodeError};
 use rspc::ErrorCode;
+use rules::RuleKind;
 use serde_json::Error as SerdeJsonError;
 use std::io;
 use thiserror::Error;

core/src/location/indexer/walk.rs

@@ -9,7 +9,7 @@ use tokio::fs;
 use tracing::{debug, error};
 
 use super::{
-	indexer_rules::{IndexerRule, RuleKind},
+	rules::{IndexerRule, RuleKind},
 	IndexerError,
 };
 
@@ -273,7 +273,7 @@ pub(super) async fn walk(
 
 #[cfg(test)]
 mod tests {
-	use super::super::indexer_rules::ParametersPerKind;
+	use super::super::rules::ParametersPerKind;
 	use super::*;
 	use chrono::Utc;
 	use globset::Glob;

core/src/location/mod.rs

@@ -1,9 +1,11 @@
 use crate::{
-	encode::{ThumbnailJob, ThumbnailJobInit},
-	file::cas::{FileIdentifierJob, FileIdentifierJobInit},
 	invalidate_query,
 	job::Job,
 	library::LibraryContext,
+	object::{
+		identifier_job::{FileIdentifierJob, FileIdentifierJobInit},
+		preview::{ThumbnailJob, ThumbnailJobInit},
+	},
 	prisma::{indexer_rules_in_location, location, node},
 };
 

core/src/object/identifier_job.rs

@@ -14,7 +14,7 @@ use std::{
 use tokio::{fs, io};
 use tracing::{error, info};
 
-use super::checksum::generate_cas_id;
+use super::cas::generate_cas_id;
 
 // we break this job into chunks of 100 to improve performance
 static CHUNK_SIZE: usize = 100;

core/src/object/mod.rs

@@ -1,3 +1,7 @@
+pub mod cas;
+pub mod identifier_job;
+pub mod preview;
+
 // Objects are primarily created by the identifier from Paths
 // Some Objects are purely virtual, unless they have one or more associated Paths, which refer to a file found in a Location
 // Objects are what can be added to Spaces

core/src/object/preview/thumb.rs

@@ -298,7 +298,7 @@ async fn generate_video_thumbnail<P: AsRef<Path>>(
 	file_path: P,
 	output_path: P,
 ) -> Result<(), Box<dyn Error>> {
-	use thumbnailer::to_thumbnail;
+	use sd_ffmpeg::to_thumbnail;
 
 	to_thumbnail(file_path, output_path, 256, THUMBNAIL_QUALITY).await?;
 

core/src/util/seeder.rs

@@ -1,6 +1,6 @@
 use crate::{
 	location::indexer::{
-		indexer_rules::{IndexerRule, ParametersPerKind, RuleKind},
+		rules::{IndexerRule, ParametersPerKind, RuleKind},
 		IndexerError,
 	},
 	prisma::PrismaClient,

crates/ffmpeg/Cargo.toml

@@ -1,5 +1,5 @@
 [package]
-name = "thumbnailer"
+name = "sd_ffmpeg"
 version = "0.1.0"
 authors = ["Ericson Soares <ericson.ds999@gmail.com>"]
 edition = "2021"

crates/p2p/Cargo.toml

@@ -0,0 +1,8 @@
+[package]
+name = "p2p"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]

crates/p2p/src/main.rs

@@ -0,0 +1,3 @@
+fn main() {
+	println!("Hello, world!");
+}

crates/prisma/schema.prisma

@@ -5,7 +5,7 @@ datasource db {
 
 generator client {
   provider = "cargo prisma"
-  output   = "../src/prisma.rs"
+  output   = "../../core/src/prisma.rs"
 }
 
 model Migration {

crates/sync/Cargo.toml

@@ -0,0 +1,8 @@
+[package]
+name = "sync"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]

crates/sync/src/main.rs

@@ -0,0 +1,3 @@
+fn main() {
+	println!("Hello, world!");
+}

docs/product/guides/adding-locations.md

@@ -4,4 +4,39 @@ index: 5
 
 # Adding Locations
 
-![image](/locations.webp)
+Locations are places Spacedrive looks for files. You can add Locations from any device to your Library, they will be scanned and monitored for filesystem changes.
+
+![Spacedrive Locations from the Settings view](/locations.webp)
+
+Locations can be managed from the settings. The "online" indicator shows if that Location is currently accessible from one of your devices.
+
+## Scanning
+When a Location is added it is immediately scanned. The scan happens via several phases.
+
+1. The [Indexer](): directory structure is indexed and saved to your library's database.
+2. The [Identifier](): metadata is extracted and unique identifiers generated.
+3. The [Thumbnailer](): compressed previews are generated for all media and text files.
+
+If any one of these jobs fail or the application is closed, the state is saved and it will resume when the app is relaunched. 
+
+::: slot info 
+If you wish to pause or cancel these jobs you can do so from the [Job Manager]()
+:::
+
+There is a button to trigger a full re-scan of this Location. Otherwise the location will keep watch for filesystem changes automatically.
+
+Deleting a location will remove the data from the database permanently.
+
+## Archiving
+::: slot warning 
+This feature is WIP
+:::
+Locations can be archived, meaning the directory structure is extracted from the Spacedrive database and preserved as a standalone database file. This file can be opened by Spacedrive in the [Database]() view.
+
+Archives are useful when files have been moved or deleted from a Location, but you want to keep a record of the directory structure of that Location without cluttering the Spacedrive interface. File metadata can optionally be included/excluded from the archive file.
+
+## Dynamic Space
+A slider in the Location settings allows you to allocate an amount of GB to use as redundancy for other Locations. The storage space will be filled and emptied dynamically, recent and important files will be copied as priority.
+
+## Encrypted Space
+Allocate a portion of this location to be encrypted 

docs/product/guides/connecting-nodes.md

@@ -2,4 +2,13 @@
 index: 6
 ---
 
-# Connecting Nodes
+# Connecting Nodes
+
+::: slot note WIP 
+It is not possible to connect Nodes yet, please check our [roadmap](/roadmap).
+:::
+
+Nodes are instances of Spacedrive, commonly your devices. Currently Spacedrive supports Windows, Linux, Mac, iOS and Android. 
+
+These devices can be connected via P2P to directly and securely synchronize and move data. 
+

docs/product/guides/creating-spaces.md

@@ -2,4 +2,6 @@
 index: 7
 ---
 
-# Creating Spaces
+# Creating Spaces
+
+Spaces are virtual folders, they can contain files from various locations.

docs/product/guides/database-backup.md

@@ -1,5 +0,0 @@
----
-index: 6
----
-
-# Database Backup

docs/product/guides/find-my-file.md

@@ -0,0 +1,7 @@
+---
+index: 10
+---
+
+# Find My File
+
+Mark a file as lost, if Spacedrive ever discovers that file when indexing, you will be notified and Spacedrive will make a backup.

docs/product/guides/library-setup.md

@@ -0,0 +1,14 @@
+---
+index: 0
+---
+
+# Library Setup
+
+A Library is where all your Spacedrive data lives, but not the files themselves.
+
+For the convenience of finding, organizing and sharing files even when you don't have access; directory structures, metadata and preview media are extracted and stored within the Spacedrive Library.
+
+This data is heavily compressed and very portable, you can have multiple libraries and even lock and encrypt them.
+
+## Deleting a Library
+A Library can be deleted, 

package.json

@@ -3,18 +3,17 @@
 	"version": "0.0.0",
 	"private": true,
 	"scripts": {
-		"prep": "pnpm db:gen && pnpm core codegen",
+		"prep": "pnpm db:gen && cargo test",
 		"build": "turbo run build",
 		"landing-web": "turbo run dev --parallel --filter=@sd/landing --filter=@sd/web",
-		"db:migrate": "pnpm core prisma migrate dev",
+		"db:migrate": "cd crates && cargo prisma migrate dev",
-		"db:gen": "pnpm core prisma generate",
+		"db:gen": "cd crates && cargo prisma generate",
 		"format": "prettier --config .prettierrc.cli.js --write \"**/*.{ts,tsx,html,scss,json,yml,md}\"",
 		"desktop": "pnpm --filter @sd/desktop --",
 		"web": "pnpm --filter @sd/web -- ",
 		"landing": "pnpm --filter @sd/landing -- ",
 		"ui": "pnpm --filter @sd/ui -- ",
 		"interface": "pnpm --filter @sd/interface -- ",
-		"core": "pnpm --filter @sd/core -- ",
 		"docs": "pnpm --filter @sd/docs -- ",
 		"client": "pnpm --filter @sd/client -- ",
 		"server": "pnpm --filter @sd/server -- ",

packages/client/package.json

@@ -20,7 +20,6 @@
 		"@rspc/client": "^0.0.6",
 		"@rspc/react": "^0.0.6",
 		"@sd/config": "workspace:*",
-		"@sd/core": "workspace:*",
 		"@sd/interface": "workspace:*",
 		"@tanstack/react-query": "^4.0.10",
 		"eventemitter3": "^4.0.7",

packages/client/src/index.ts

@@ -1,5 +1,5 @@
 export * from './stores';
 export * from './context';
 export * from './rspc';
+export * from './core';
 export * from './hooks';
-export type { Operations } from '@sd/core';

packages/client/src/rspc.ts

@@ -1,6 +1,5 @@
 import { RSPCError } from '@rspc/client';
 import { createReactQueryHooks } from '@rspc/react';
-import { LibraryArgs, Operations } from '@sd/core';
 import {
 	QueryClient,
 	UseInfiniteQueryOptions,
@@ -12,6 +11,7 @@ import {
 	useMutation as _useMutation
 } from '@tanstack/react-query';
 
+import { LibraryArgs, Operations } from './core';
 import { useCurrentLibrary } from './index';
 
 export const queryClient = new QueryClient();

packages/client/src/types/file.ts

@@ -1,16 +0,0 @@
-import { FilePath } from '@sd/core';
-
-export interface ExplorerItem {
-	id: number;
-	name: string;
-	is_dir: boolean;
-	// kind: ObjectKind;
-	extension: string;
-	size_in_bytes: number;
-	created_at: string;
-	updated_at: string;
-	favorite?: boolean;
-
-	// computed
-	paths?: FilePath[];
-}

packages/client/src/types/index.ts

@@ -1 +0,0 @@
-export * from './file';

packages/interface/package.json

@@ -27,7 +27,6 @@
 		"@radix-ui/react-tooltip": "^1.0.0",
 		"@sd/assets": "workspace:*",
 		"@sd/client": "workspace:*",
-		"@sd/core": "workspace:*",
 		"@sd/ui": "workspace:*",
 		"@tailwindcss/forms": "^0.5.2",
 		"@tanstack/react-query": "^4.2.3",

packages/interface/src/components/explorer/Explorer.tsx

@@ -1,5 +1,4 @@
-import { getExplorerStore, rspc, useCurrentLibrary, useExplorerStore } from '@sd/client';
+import { ExplorerData, rspc, useCurrentLibrary, useExplorerStore } from '@sd/client';
-import { ExplorerData } from '@sd/core';
 
 import { Inspector } from '../explorer/Inspector';
 import { TopBar } from '../layout/TopBar';

packages/interface/src/components/explorer/FileItem.tsx

@@ -1,5 +1,4 @@
-import { getExplorerStore, useExplorerStore } from '@sd/client';
+import { ExplorerItem, getExplorerStore } from '@sd/client';
-import { ExplorerItem } from '@sd/core';
 import clsx from 'clsx';
 import { HTMLAttributes } from 'react';
 

packages/interface/src/components/explorer/FileRow.tsx

@@ -1,4 +1,4 @@
-import { ExplorerItem } from '@sd/core';
+import { ExplorerItem } from '@sd/client';
 import clsx from 'clsx';
 import { HTMLAttributes } from 'react';
 

packages/interface/src/components/explorer/FileThumb.tsx

@@ -1,5 +1,5 @@
 import { getExplorerStore, useExplorerStore, usePlatform } from '@sd/client';
-import { ExplorerItem } from '@sd/core';
+import { ExplorerItem } from '@sd/client';
 import clsx from 'clsx';
 import { useState } from 'react';
 import { useSnapshot } from 'valtio';

packages/interface/src/components/explorer/Inspector.tsx

@@ -1,6 +1,6 @@
 import { ShareIcon } from '@heroicons/react/24/solid';
 import { useLibraryQuery } from '@sd/client';
-import { ExplorerContext, ExplorerItem, File, FilePath, Location } from '@sd/core';
+import { ExplorerContext, ExplorerItem, File, FilePath, Location } from '@sd/client';
 import { Button, TextArea } from '@sd/ui';
 import clsx from 'clsx';
 import moment from 'moment';

packages/interface/src/components/explorer/VirtualizedList.tsx

@@ -1,5 +1,5 @@
 import { ExplorerLayoutMode, getExplorerStore, useExplorerStore } from '@sd/client';
-import { ExplorerContext, ExplorerItem, FilePath } from '@sd/core';
+import { ExplorerContext, ExplorerItem, FilePath } from '@sd/client';
 import { useVirtualizer } from '@tanstack/react-virtual';
 import { memo, useCallback, useLayoutEffect, useMemo, useRef, useState } from 'react';
 import { useSearchParams } from 'react-router-dom';

packages/interface/src/components/explorer/inspector/FavoriteButton.tsx

@@ -1,5 +1,5 @@
 import { useLibraryMutation } from '@sd/client';
-import { File } from '@sd/core';
+import { File } from '@sd/client';
 import { Button } from '@sd/ui';
 import { Heart } from 'phosphor-react';
 import { useEffect, useState } from 'react';

packages/interface/src/components/explorer/inspector/Note.tsx

@@ -1,6 +1,6 @@
 /* eslint-disable react-hooks/exhaustive-deps */
 import { useLibraryMutation } from '@sd/client';
-import { File } from '@sd/core';
+import { File } from '@sd/client';
 import { TextArea } from '@sd/ui';
 import { debounce } from 'lodash';
 import { useCallback, useState } from 'react';

packages/interface/src/components/explorer/utils.ts

@@ -1,4 +1,4 @@
-import { ExplorerItem, File, FilePath } from '@sd/core';
+import { ExplorerItem, File, FilePath } from '@sd/client';
 
 export function isPath(item: ExplorerItem): item is Extract<ExplorerItem, { type: 'Path' }> {
 	return item.type === 'Path';

packages/interface/src/components/jobs/JobManager.tsx

@@ -1,6 +1,6 @@
 import { EyeIcon, FolderIcon, PhotoIcon, XMarkIcon } from '@heroicons/react/24/outline';
 import { useLibraryQuery } from '@sd/client';
-import { JobReport } from '@sd/core';
+import { JobReport } from '@sd/client';
 import { Button } from '@sd/ui';
 import clsx from 'clsx';
 import { formatDistanceToNow, formatDuration } from 'date-fns';

packages/interface/src/components/layout/Sidebar.tsx

@@ -1,7 +1,7 @@
 import { CogIcon, LockClosedIcon, PhotoIcon } from '@heroicons/react/24/outline';
 import { PlusIcon } from '@heroicons/react/24/solid';
 import { useCurrentLibrary, useLibraryMutation, useLibraryQuery, usePlatform } from '@sd/client';
-import { LocationCreateArgs } from '@sd/core';
+import { LocationCreateArgs } from '@sd/client';
 import { Button, Dropdown, OverlayPanel } from '@sd/ui';
 import clsx from 'clsx';
 import { CheckCircle, CirclesFour, Planet, WaveTriangle } from 'phosphor-react';

packages/interface/src/components/location/LocationListItem.tsx

@@ -1,13 +1,14 @@
-import { Folder } from '../icons/Folder';
-import Dialog from '../layout/Dialog';
 import { TrashIcon } from '@heroicons/react/24/solid';
 import { useLibraryMutation } from '@sd/client';
-import { Location, Node } from '@sd/core';
+import { Location, Node } from '@sd/client';
 import { Button } from '@sd/ui';
 import clsx from 'clsx';
 import { Repeat } from 'phosphor-react';
 import React, { useState } from 'react';
 
+import { Folder } from '../icons/Folder';
+import Dialog from '../layout/Dialog';
+
 interface LocationListItemProps {
 	location: Location & { node: Node };
 }

packages/interface/src/screens/Overview.tsx

@@ -1,6 +1,6 @@
 import { ExclamationCircleIcon, PlusIcon } from '@heroicons/react/24/solid';
 import { useBridgeQuery, useLibraryQuery, usePlatform } from '@sd/client';
-import { Statistics } from '@sd/core';
+import { Statistics } from '@sd/client';
 import { Button, Input } from '@sd/ui';
 import byteSize from 'byte-size';
 import clsx from 'clsx';
@@ -163,7 +163,7 @@ export const OverviewScreen = () => {
 					</div>
 
 					<div className="flex-grow" />
-					<div className="space-x-2 h-full flex items-center">
+					<div className="flex items-center h-full space-x-2">
 						<div>
 							<Dialog
 								title="Add Device"

packages/interface/src/screens/settings/library/LocationSettings.tsx

@@ -1,5 +1,5 @@
 import { useLibraryMutation, useLibraryQuery, usePlatform } from '@sd/client';
-import { LocationCreateArgs } from '@sd/core';
+import { LocationCreateArgs } from '@sd/client';
 import { Button } from '@sd/ui';
 
 import LocationListItem from '../../../components/location/LocationListItem';

packages/interface/src/screens/settings/library/TagsSettings.tsx

@@ -1,3 +1,12 @@
+import { TrashIcon } from '@heroicons/react/24/outline';
+import { useLibraryMutation, useLibraryQuery } from '@sd/client';
+import { TagUpdateArgs } from '@sd/client';
+import { Button, Input } from '@sd/ui';
+import clsx from 'clsx';
+import { useCallback, useEffect, useMemo, useState } from 'react';
+import { Controller, useForm } from 'react-hook-form';
+import { useDebounce } from 'rooks';
+
 import Card from '../../../components/layout/Card';
 import Dialog from '../../../components/layout/Dialog';
 import { Toggle } from '../../../components/primitive';
@@ -5,14 +14,6 @@ import { InputContainer } from '../../../components/primitive/InputContainer';
 import { PopoverPicker } from '../../../components/primitive/PopoverPicker';
 import { SettingsContainer } from '../../../components/settings/SettingsContainer';
 import { SettingsHeader } from '../../../components/settings/SettingsHeader';
-import { TrashIcon } from '@heroicons/react/24/outline';
-import { useLibraryMutation, useLibraryQuery } from '@sd/client';
-import { TagUpdateArgs } from '@sd/core';
-import { Button, Input } from '@sd/ui';
-import clsx from 'clsx';
-import { useCallback, useEffect, useMemo, useState } from 'react';
-import { Controller, useForm } from 'react-hook-form';
-import { useDebounce } from 'rooks';
 
 export default function TagsSettings() {
 	const [openCreateModal, setOpenCreateModal] = useState(false);

packages/interface/src/screens/settings/node/LibrariesSettings.tsx

@@ -1,6 +1,6 @@
 import { PencilIcon, TrashIcon } from '@heroicons/react/24/outline';
 import { useBridgeMutation, useBridgeQuery } from '@sd/client';
-import { LibraryConfigWrapped } from '@sd/core';
+import { LibraryConfigWrapped } from '@sd/client';
 import { Button } from '@sd/ui';
 import { DotsSixVertical } from 'phosphor-react';
 import { useState } from 'react';