From 3e8c33d1c0290d06eabd5544b4bdd82822884b77 Mon Sep 17 00:00:00 2001 From: Jamie Pine <32987599+jamiepine@users.noreply.github.com> Date: Thu, 29 Sep 2022 21:02:29 -0700 Subject: [PATCH] Added crates folder (#389) * restructure core for crates folder - improve naming consistency - added placeholder files for near future code - added docs * - removed all typescript from core - moved rspc bindings to client * remove useless git ignore & rustfmt --- .cspell/project_words.txt | 68 ++-- .gitignore | 2 + .prettierrc.json | 8 +- .rustfmt.toml | 14 +- .vscode/.todo | 50 --- Cargo.lock | 304 +++++++++++++++++- Cargo.toml | 4 +- apps/desktop/package.json | 1 - apps/desktop/tsconfig.json | 4 +- apps/web/package.json | 1 - core/.gitignore | 4 - core/.rustfmt.toml | 13 - core/Cargo.toml | 5 +- core/package.json | 17 - core/scripts/bindingsIndex.ts | 30 -- core/src/api/jobs.rs | 6 +- core/src/api/locations.rs | 4 +- core/src/api/mod.rs | 6 +- core/src/api/tags.rs | 2 +- core/src/file/cas/mod.rs | 5 - core/src/file/mod.rs | 1 - core/src/job/job_manager.rs | 6 +- core/src/lib.rs | 5 +- .../location/archive/archive_job.rs} | 0 .../src/location/archive/mod.rs | 0 core/src/location/indexer/indexer_job.rs | 2 +- core/src/location/indexer/mod.rs | 4 +- .../indexer/{indexer_rules.rs => rules.rs} | 0 core/src/location/indexer/walk.rs | 4 +- core/src/location/mod.rs | 6 +- .../{file/cas/checksum.rs => object/cas.rs} | 0 core/src/object/fs/archive.rs | 0 core/src/object/fs/convert.rs | 0 core/src/object/fs/copy.rs | 0 core/src/object/fs/decrypt.rs | 0 core/src/object/fs/encrypt.rs | 0 core/src/object/fs/erase.rs | 0 core/src/object/fs/mod.rs | 0 core/src/object/fs/move.rs | 0 core/src/object/fs/sync.rs | 0 .../identifier_job.rs} | 2 +- core/src/object/mod.rs | 4 + .../{encode => object/preview}/metadata.rs | 0 core/src/{encode => object/preview}/mod.rs | 0 core/src/{encode => object/preview}/thumb.rs | 2 +- core/src/object/reader.rs | 0 core/src/object/validation/hash.rs | 0 core/src/object/validation/mod.rs | 0 core/src/object/validation/validator_job.rs | 0 core/src/util/seeder.rs | 2 +- core/src/{volume/mod.rs => volume.rs} | 0 .../thumbnailer => crates/ffmpeg}/.gitignore | 0 .../thumbnailer => crates/ffmpeg}/Cargo.toml | 2 +- {core/thumbnailer => crates/ffmpeg}/README.md | 0 .../ffmpeg}/src/error.rs | 0 .../ffmpeg}/src/film_strip.rs | 0 .../thumbnailer => crates/ffmpeg}/src/lib.rs | 0 .../ffmpeg}/src/movie_decoder.rs | 0 .../ffmpeg}/src/thumbnailer.rs | 0 .../ffmpeg}/src/utils.rs | 0 .../ffmpeg}/src/video_frame.rs | 0 crates/p2p/Cargo.toml | 8 + crates/p2p/src/main.rs | 3 + {core => crates}/prisma/Cargo.toml | 0 .../20220909073230_init/migration.sql | 0 .../prisma/migrations/migration_lock.toml | 0 {core => crates}/prisma/schema.prisma | 2 +- {core => crates}/prisma/src/main.rs | 0 crates/sync/Cargo.toml | 8 + crates/sync/src/main.rs | 3 + docs/product/guides/adding-locations.md | 37 ++- docs/product/guides/connecting-nodes.md | 11 +- docs/product/guides/creating-spaces.md | 4 +- docs/product/guides/database-backup.md | 5 - docs/product/guides/find-my-file.md | 7 + docs/product/guides/library-setup.md | 14 + package.json | 7 +- packages/client/package.json | 1 - core/index.ts => packages/client/src/core.ts | 0 packages/client/src/index.ts | 2 +- packages/client/src/rspc.ts | 2 +- packages/client/src/types/file.ts | 16 - packages/client/src/types/index.ts | 1 - packages/interface/package.json | 1 - .../src/components/explorer/Explorer.tsx | 3 +- .../src/components/explorer/FileItem.tsx | 3 +- .../src/components/explorer/FileRow.tsx | 2 +- .../src/components/explorer/FileThumb.tsx | 2 +- .../src/components/explorer/Inspector.tsx | 2 +- .../components/explorer/VirtualizedList.tsx | 2 +- .../explorer/inspector/FavoriteButton.tsx | 2 +- .../components/explorer/inspector/Note.tsx | 2 +- .../src/components/explorer/utils.ts | 2 +- .../src/components/jobs/JobManager.tsx | 2 +- .../src/components/layout/Sidebar.tsx | 2 +- .../components/location/LocationListItem.tsx | 7 +- packages/interface/src/screens/Overview.tsx | 4 +- .../settings/library/LocationSettings.tsx | 2 +- .../screens/settings/library/TagsSettings.tsx | 17 +- .../settings/node/LibrariesSettings.tsx | 2 +- packages/interface/tsconfig.json | 1 + 101 files changed, 510 insertions(+), 267 deletions(-) delete mode 100644 .vscode/.todo delete mode 100644 core/.gitignore delete mode 100644 core/.rustfmt.toml delete mode 100644 core/package.json delete mode 100644 core/scripts/bindingsIndex.ts delete mode 100644 core/src/file/cas/mod.rs delete mode 100644 core/src/file/mod.rs rename core/{scripts/tsconfig.json => src/location/archive/archive_job.rs} (100%) rename packages/client/.gitignore => core/src/location/archive/mod.rs (100%) rename core/src/location/indexer/{indexer_rules.rs => rules.rs} (100%) rename core/src/{file/cas/checksum.rs => object/cas.rs} (100%) create mode 100644 core/src/object/fs/archive.rs create mode 100644 core/src/object/fs/convert.rs create mode 100644 core/src/object/fs/copy.rs create mode 100644 core/src/object/fs/decrypt.rs create mode 100644 core/src/object/fs/encrypt.rs create mode 100644 core/src/object/fs/erase.rs create mode 100644 core/src/object/fs/mod.rs create mode 100644 core/src/object/fs/move.rs create mode 100644 core/src/object/fs/sync.rs rename core/src/{file/cas/identifier.rs => object/identifier_job.rs} (99%) rename core/src/{encode => object/preview}/metadata.rs (100%) rename core/src/{encode => object/preview}/mod.rs (100%) rename core/src/{encode => object/preview}/thumb.rs (99%) create mode 100644 core/src/object/reader.rs create mode 100644 core/src/object/validation/hash.rs create mode 100644 core/src/object/validation/mod.rs create mode 100644 core/src/object/validation/validator_job.rs rename core/src/{volume/mod.rs => volume.rs} (100%) rename {core/thumbnailer => crates/ffmpeg}/.gitignore (100%) rename {core/thumbnailer => crates/ffmpeg}/Cargo.toml (96%) rename {core/thumbnailer => crates/ffmpeg}/README.md (100%) rename {core/thumbnailer => crates/ffmpeg}/src/error.rs (100%) rename {core/thumbnailer => crates/ffmpeg}/src/film_strip.rs (100%) rename {core/thumbnailer => crates/ffmpeg}/src/lib.rs (100%) rename {core/thumbnailer => crates/ffmpeg}/src/movie_decoder.rs (100%) rename {core/thumbnailer => crates/ffmpeg}/src/thumbnailer.rs (100%) rename {core/thumbnailer => crates/ffmpeg}/src/utils.rs (100%) rename {core/thumbnailer => crates/ffmpeg}/src/video_frame.rs (100%) create mode 100644 crates/p2p/Cargo.toml create mode 100644 crates/p2p/src/main.rs rename {core => crates}/prisma/Cargo.toml (100%) rename {core => crates}/prisma/migrations/20220909073230_init/migration.sql (100%) rename {core => crates}/prisma/migrations/migration_lock.toml (100%) rename {core => crates}/prisma/schema.prisma (99%) rename {core => crates}/prisma/src/main.rs (100%) create mode 100644 crates/sync/Cargo.toml create mode 100644 crates/sync/src/main.rs delete mode 100644 docs/product/guides/database-backup.md create mode 100644 docs/product/guides/find-my-file.md create mode 100644 docs/product/guides/library-setup.md rename core/index.ts => packages/client/src/core.ts (100%) delete mode 100644 packages/client/src/types/file.ts delete mode 100644 packages/client/src/types/index.ts diff --git a/.cspell/project_words.txt b/.cspell/project_words.txt index 882cf4dda..1125e97df 100644 --- a/.cspell/project_words.txt +++ b/.cspell/project_words.txt @@ -1,42 +1,46 @@ -spacedrive -spacedriveapp -vdfs -haoyuan -brendonovich -codegen -elon -deel -haden akar -benja -haris -mehrzad -OSSC -josephjacks -rauch -ravikant -neha -narkhede allred -lütke -tobiaslutke -justinhoffman -rywalker -zacharysmith -sanjay -poonen -mytton -davidmytton -richelsen -lesterlee alluxio augusto +benja +brendonovich +chacha +codegen +davidmytton +deel +elon +encryptor +haden +haoyuan +haris +josephjacks +justinhoffman +Keyslot +keyslots +lesterlee +lütke marietti -vijay -sharma +mehrzad +mytton +narkhede naveen +neha noco +OSSC +poonen +rauch +ravikant +richelsen rspc rspcws -stringly +rywalker +sanjay +sharma +spacedrive +spacedriveapp specta +stringly +tobiaslutke +vdfs +vijay +zacharysmith diff --git a/.gitignore b/.gitignore index 958c7791a..73aded57f 100644 --- a/.gitignore +++ b/.gitignore @@ -27,6 +27,7 @@ apps/*/stats.html docs/public/*.st docs/public/*.toml dev.db +stats.html !cli/cmd/turbo cli/npm/turbo-android-arm64/bin @@ -61,6 +62,7 @@ yalc.lock todos.md examples/*/*.lock /target +/core/src/prisma.rs /sdserver_data .spacedrive diff --git a/.prettierrc.json b/.prettierrc.json index 921c7343c..5347ec663 100644 --- a/.prettierrc.json +++ b/.prettierrc.json @@ -7,13 +7,7 @@ "bracketSameLine": false, "semi": true, "quoteProps": "consistent", - "importOrder": [ - "^@sd/core/(.*)$", - "^@sd/interface/(.*)$", - "^@sd/client/(.*)$", - "^@sd/ui/(.*)$", - "^[./]" - ], + "importOrder": ["^@sd/interface/(.*)$", "^@sd/client/(.*)$", "^@sd/ui/(.*)$", "^[./]"], "importOrderSeparation": true, "importOrderSortSpecifiers": true } diff --git a/.rustfmt.toml b/.rustfmt.toml index b6bfaa825..9c4c7300b 100644 --- a/.rustfmt.toml +++ b/.rustfmt.toml @@ -1,6 +1,12 @@ +max_width = 100 hard_tabs = true -match_block_trailing_comma = true -max_width = 90 newline_style = "Unix" -use_field_init_shorthand = true -use_try_shorthand = true \ No newline at end of file +use_small_heuristics = "Default" +reorder_imports = true +reorder_modules = true +remove_nested_parens = true +edition = "2021" +merge_derives = true +use_try_shorthand = false +use_field_init_shorthand = false +force_explicit_abi = true \ No newline at end of file diff --git a/.vscode/.todo b/.vscode/.todo deleted file mode 100644 index f71fc3809..000000000 --- a/.vscode/.todo +++ /dev/null @@ -1,50 +0,0 @@ -#### -# Completed -#### -✔ Configured a Tauri window -✔ Primative UI components (Button, Input, Shortcut etc) -✔ Connect & create sqlite database in application data folder -✔ Auto migrate database on app launch -✔ Define data schema in Rust -✔ Scan given directory recursively and write to database -✔ App frame and router set up -✔ Render basic file list with database data -✔ Create a global store -✔ Virtualized list for explorer @done(21-10-24 04:03) - Also cache folder contents and serve lazy, updating in background -✔ Native file type icons @done(21-10-24 04:04) -✔ Cache native icons in app data folder @done(21-10-24 11:30) -✔ File inspector @done(21-10-24 23:30) -✔ Upgrade to mono-repo @done(21-12-27 18:30) -✔ Create project readme @done(21-12-28 18:30) -✔ Refactor indexer to scan efficiently @done(21-12-29 18:30) -✔ File previews -✔ Generate CAS ids -✔ Settings screen -✔ Job queue system -✔ Landing page -✔ Volume identification -✔ Statistic calucations - Folder size, Volume size, Total capacity, Total unique etc.. - -#### -# Next up -#### -☐ Tag creation and assignment -☐ Right click menu -☐ Set up Tauri updater -☐ Onboarding flow -☐ Explorer grid view -☐ Changelog - -#### -# Planned features -#### -☐ Action system & command pallete -☐ Explorer path viewer / editor -☐ File encryptor -☐ File viewer / player -☐ Secret keystore -☐ Search - Efficient way to search sqlite: make file table WITHOUT ROWID -☐ Open with diff --git a/Cargo.lock b/Cargo.lock index f1e9de695..ca0f61538 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -29,6 +29,41 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" +[[package]] +name = "aead" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c192eb8f11fc081b0fe4259ba5af04217d4e0faddd02417310a927911abd7c8" +dependencies = [ + "crypto-common", + "generic-array", +] + +[[package]] +name = "aes" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfe0133578c0986e1fe3dfcd4af1cc5b2dd6c3dbf534d69916ce16a2701d40ba" +dependencies = [ + "cfg-if 1.0.0", + "cipher", + "cpufeatures", +] + +[[package]] +name = "aes-gcm" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82e1366e0c69c9f927b1fa5ce2c7bf9eafc8f9268c0b9800729e8b267612447c" +dependencies = [ + "aead", + "aes", + "cipher", + "ctr", + "ghash", + "subtle", +] + [[package]] name = "ahash" version = "0.7.6" @@ -88,6 +123,29 @@ version = "1.0.65" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "98161a4e3e2184da77bb14f02184cdd111e83bbbcc9979dfee3c44b9a85f5602" +[[package]] +name = "argon2" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db4ce4441f99dbd377ca8a8f57b698c44d0d6e712d8329b5040da5a64aa1ce73" +dependencies = [ + "base64ct", + "blake2", + "password-hash", +] + +[[package]] +name = "arrayref" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544" + +[[package]] +name = "arrayvec" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" + [[package]] name = "ascii" version = "0.9.3" @@ -257,6 +315,17 @@ dependencies = [ "rustc-demangle", ] +[[package]] +name = "balloon-hash" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fdbafd8c776e0a7c250b5dfaf4723477e0d92b42c5a91ce4e50bd9bb2dcc9a1" +dependencies = [ + "crypto-bigint", + "digest 0.10.5", + "password-hash", +] + [[package]] name = "base64" version = "0.12.3" @@ -269,6 +338,12 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" +[[package]] +name = "base64ct" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea2b2456fd614d856680dcd9fcc660a51a820fa09daef2e49772b56a193c8474" + [[package]] name = "bigdecimal" version = "0.2.2" @@ -321,6 +396,29 @@ dependencies = [ "typenum", ] +[[package]] +name = "blake2" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9cf849ee05b2ee5fba5e36f97ff8ec2533916700fc0758d40d92136a42f3388" +dependencies = [ + "digest 0.10.5", +] + +[[package]] +name = "blake3" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a08e53fc5a564bb15bfe6fae56bd71522205f1f91893f9c0116edad6496c183f" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if 1.0.0", + "constant_time_eq", + "digest 0.10.5", +] + [[package]] name = "block" version = "0.1.6" @@ -498,6 +596,30 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "chacha20" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7fc89c7c5b9e7a02dfe45cd2367bae382f9ed31c61ca8debe5f827c420a2f08" +dependencies = [ + "cfg-if 1.0.0", + "cipher", + "cpufeatures", +] + +[[package]] +name = "chacha20poly1305" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10cd79432192d1c0f4e1a0fef9527696cc039165d729fb41b3f4f4f354c2dc35" +dependencies = [ + "aead", + "chacha20", + "cipher", + "poly1305", + "zeroize", +] + [[package]] name = "chrono" version = "0.4.22" @@ -514,6 +636,17 @@ dependencies = [ "winapi", ] +[[package]] +name = "cipher" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1873270f8f7942c191139cb8a40fd228da6c3fd2fc376d7e92d47aa14aeb59e" +dependencies = [ + "crypto-common", + "inout", + "zeroize", +] + [[package]] name = "clang-sys" version = "1.4.0" @@ -615,6 +748,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "constant_time_eq" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" + [[package]] name = "convert_case" version = "0.4.0" @@ -741,6 +880,24 @@ dependencies = [ "once_cell", ] +[[package]] +name = "crypto" +version = "0.1.0" +dependencies = [ + "chacha20poly1305", + "rand 0.8.5", +] + +[[package]] +name = "crypto-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f2b443d17d49dad5ef0ede301c3179cc923b8822f3393b4d2c28c269dd4a122" +dependencies = [ + "generic-array", + "subtle", +] + [[package]] name = "crypto-common" version = "0.1.6" @@ -748,6 +905,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" dependencies = [ "generic-array", + "rand_core 0.6.4", "typenum", ] @@ -788,6 +946,15 @@ dependencies = [ "syn", ] +[[package]] +name = "ctr" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d14f329cfbaf5d0e06b5e87fff7e265d2673c5ea7d2c27691a2c107db1442a0" +dependencies = [ + "cipher", +] + [[package]] name = "ctrlc" version = "3.2.3" @@ -913,6 +1080,18 @@ dependencies = [ "byteorder", ] +[[package]] +name = "deoxys" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00603a49e114bd99d87a4ab8d480f36ecd1451a9d6474f66973d1a829ff77789" +dependencies = [ + "aead", + "aes", + "subtle", + "zeroize", +] + [[package]] name = "derive_more" version = "0.99.17" @@ -952,6 +1131,7 @@ checksum = "adfbc57365a37acbd2ebf2b64d7e69bb766e2fea813521ed536f5d0520dcf86c" dependencies = [ "block-buffer 0.10.3", "crypto-common", + "subtle", ] [[package]] @@ -1481,6 +1661,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "ghash" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d930750de5717d2dd0b8c0d42c076c0e884c81a73e6cab859bbd2339c71e3e40" +dependencies = [ + "opaque-debug", + "polyval", +] + [[package]] name = "gif" version = "0.11.4" @@ -1603,6 +1793,22 @@ dependencies = [ "system-deps 6.0.2", ] +[[package]] +name = "goop" +version = "0.1.0" +dependencies = [ + "aead", + "aes-gcm", + "anyhow", + "argon2", + "balloon-hash", + "blake3", + "chacha20poly1305", + "deoxys", + "rand 0.8.5", + "zeroize", +] + [[package]] name = "graphql-parser" version = "0.3.0" @@ -2013,6 +2219,15 @@ dependencies = [ "adler32", ] +[[package]] +name = "inout" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5" +dependencies = [ + "generic-array", +] + [[package]] name = "instant" version = "0.1.12" @@ -3004,6 +3219,10 @@ dependencies = [ "winapi", ] +[[package]] +name = "p2p" +version = "0.1.0" + [[package]] name = "pango" version = "0.15.10" @@ -3113,6 +3332,17 @@ dependencies = [ "schema-ast", ] +[[package]] +name = "password-hash" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7676374caaee8a325c9e7a2ae557f216c5563a171d6997b0ef8a65af35147700" +dependencies = [ + "base64ct", + "rand_core 0.6.4", + "subtle", +] + [[package]] name = "paste" version = "1.0.9" @@ -3365,6 +3595,29 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "poly1305" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8159bd90725d2df49889a078b54f4f79e87f1f8a8444194cdca81d38f5393abf" +dependencies = [ + "cpufeatures", + "opaque-debug", + "universal-hash", +] + +[[package]] +name = "polyval" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef234e08c11dfcb2e56f79fd70f6f2eb7f025c0ce2333e82f4f0518ecad30c6" +dependencies = [ + "cfg-if 1.0.0", + "cpufeatures", + "opaque-debug", + "universal-hash", +] + [[package]] name = "ppv-lite86" version = "0.2.16" @@ -4201,6 +4454,17 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "sd_ffmpeg" +version = "0.1.0" +dependencies = [ + "ffmpeg-sys-next", + "tempfile", + "thiserror", + "tokio", + "webp", +] + [[package]] name = "sdcore" version = "0.1.0" @@ -4209,6 +4473,7 @@ dependencies = [ "async-trait", "base64 0.13.0", "chrono", + "crypto", "ctor", "data-encoding", "enumflags2", @@ -4227,12 +4492,12 @@ dependencies = [ "rmp", "rmp-serde", "rspc", + "sd_ffmpeg", "serde", "serde_json", "sysinfo", "tempfile", "thiserror", - "thumbnailer", "tokio", "tracing", "tracing-subscriber", @@ -4862,6 +5127,12 @@ dependencies = [ "syn", ] +[[package]] +name = "subtle" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" + [[package]] name = "swift-rs" version = "0.3.0" @@ -4894,6 +5165,10 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync" +version = "0.1.0" + [[package]] name = "sync_wrapper" version = "0.1.1" @@ -5274,17 +5549,6 @@ dependencies = [ "num_cpus", ] -[[package]] -name = "thumbnailer" -version = "0.1.0" -dependencies = [ - "ffmpeg-sys-next", - "tempfile", - "thiserror", - "tokio", - "webp", -] - [[package]] name = "tiff" version = "0.7.3" @@ -5654,6 +5918,16 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" +[[package]] +name = "universal-hash" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d3160b73c9a19f7e2939a2fdad446c57c1bbbbf4d919d3213ff1267a580d8b5" +dependencies = [ + "crypto-common", + "subtle", +] + [[package]] name = "unreachable" version = "1.0.0" @@ -6329,3 +6603,9 @@ name = "xml-rs" version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2d7d3948613f75c98fd9328cfdcc45acc4d360655289d0a7d4ec931392200a3" + +[[package]] +name = "zeroize" +version = "1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c394b5bd0c6f669e7275d9c20aa90ae064cb22e75a1cad54e1b34088034b149f" diff --git a/Cargo.toml b/Cargo.toml index 063d0a62a..96f10f8f7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,9 +1,9 @@ [workspace] members = [ + "core", + "crates/*", "apps/desktop/src-tauri", "apps/mobile/rust", - "core", - "core/prisma", "apps/server", ] resolver = "2" diff --git a/apps/desktop/package.json b/apps/desktop/package.json index 3e300ad93..818905ec2 100644 --- a/apps/desktop/package.json +++ b/apps/desktop/package.json @@ -14,7 +14,6 @@ "@rspc/client": "^0.0.6", "@rspc/tauri": "^0.0.6", "@sd/client": "workspace:*", - "@sd/core": "workspace:*", "@sd/interface": "workspace:*", "@sd/ui": "workspace:*", "@tanstack/react-query": "^4.0.10", diff --git a/apps/desktop/tsconfig.json b/apps/desktop/tsconfig.json index 168f12434..435bfd6d3 100644 --- a/apps/desktop/tsconfig.json +++ b/apps/desktop/tsconfig.json @@ -1,5 +1,7 @@ { "extends": "../../packages/config/interface.tsconfig.json", - "compilerOptions": {}, + "compilerOptions": { + "target": "es2020" + }, "include": ["src"] } diff --git a/apps/web/package.json b/apps/web/package.json index ebf8ff194..67d29d392 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -11,7 +11,6 @@ "@fontsource/inter": "^4.5.11", "@rspc/client": "^0.0.6", "@sd/client": "workspace:*", - "@sd/core": "workspace:*", "@sd/interface": "workspace:*", "@sd/ui": "workspace:*", "@tanstack/react-query": "^4.0.10", diff --git a/core/.gitignore b/core/.gitignore deleted file mode 100644 index 7a75bd931..000000000 --- a/core/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -/target -/types -*.db* -/src/prisma.rs \ No newline at end of file diff --git a/core/.rustfmt.toml b/core/.rustfmt.toml deleted file mode 100644 index 89f5d09e3..000000000 --- a/core/.rustfmt.toml +++ /dev/null @@ -1,13 +0,0 @@ -max_width = 100 -hard_tabs = true -newline_style = "Unix" -use_small_heuristics = "Default" -reorder_imports = true -reorder_modules = true -remove_nested_parens = true -edition = "2021" -merge_derives = true -use_try_shorthand = false -use_field_init_shorthand = false -force_explicit_abi = true -# normalize_comments = true \ No newline at end of file diff --git a/core/Cargo.toml b/core/Cargo.toml index ddfff8133..8283d61e7 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -16,7 +16,7 @@ mobile = [ ] # This feature allows features to be disabled when the Core is running on mobile. ffmpeg = [ "dep:ffmpeg-next", - "dep:thumbnailer", + "dep:sd_ffmpeg", ] # This feature controls whether the Spacedrive Core contains functionality which requires FFmpeg. [dependencies] @@ -56,7 +56,8 @@ async-trait = "^0.1.52" image = "0.24.1" webp = "0.2.2" ffmpeg-next = { version = "5.0.3", optional = true, features = [] } -thumbnailer = { path = "./thumbnailer", optional = true } +sd_ffmpeg = { path = "../crates/ffmpeg", optional = true } +crypto = { path = "../crates/crypto" } fs_extra = "1.2.0" tracing = "0.1.35" tracing-subscriber = { version = "0.3.14", features = ["env-filter"] } diff --git a/core/package.json b/core/package.json deleted file mode 100644 index c05d412fe..000000000 --- a/core/package.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "name": "@sd/core", - "version": "0.0.0", - "main": "index.js", - "license": "GPL-3.0-only", - "scripts": { - "codegen": "cargo test", - "build": "cargo build", - "test": "cargo test", - "test:log": "cargo test -- --nocapture", - "prisma": "cargo prisma" - }, - "devDependencies": { - "@types/node": "^18.6.1", - "typescript": "^4.7.4" - } -} diff --git a/core/scripts/bindingsIndex.ts b/core/scripts/bindingsIndex.ts deleted file mode 100644 index addea2bf6..000000000 --- a/core/scripts/bindingsIndex.ts +++ /dev/null @@ -1,30 +0,0 @@ -import * as fs from 'fs/promises'; -import * as path from 'path'; - -(async function main() { - async function exists(path: string) { - try { - await fs.access(path); - return true; - } catch { - return false; - } - } - - const files = await fs.readdir(path.join(__dirname, '../bindings')); - const bindings = files.filter((f) => f.endsWith('.ts')); - let str = ''; - // str += `export * from './types';\n`; - - for (let binding of bindings) { - str += `export * from './bindings/${binding.split('.')[0]}';\n`; - } - - let indexExists = await exists(path.join(__dirname, '../index.ts')); - - if (indexExists) { - await fs.rm(path.join(__dirname, '../index.ts')); - } - - await fs.writeFile(path.join(__dirname, '../index.ts'), str); -})(); diff --git a/core/src/api/jobs.rs b/core/src/api/jobs.rs index 2c267a5ff..7668d1ee8 100644 --- a/core/src/api/jobs.rs +++ b/core/src/api/jobs.rs @@ -1,8 +1,10 @@ use crate::{ - encode::{ThumbnailJob, ThumbnailJobInit}, - file::cas::{FileIdentifierJob, FileIdentifierJobInit}, job::{Job, JobManager}, location::{fetch_location, LocationError}, + object::{ + identifier_job::{FileIdentifierJob, FileIdentifierJobInit}, + preview::{ThumbnailJob, ThumbnailJobInit}, + }, prisma::location, }; diff --git a/core/src/api/locations.rs b/core/src/api/locations.rs index cd276de40..aed2faa5f 100644 --- a/core/src/api/locations.rs +++ b/core/src/api/locations.rs @@ -1,11 +1,11 @@ use crate::{ - encode::THUMBNAIL_CACHE_DIR_NAME, invalidate_query, location::{ fetch_location, - indexer::{indexer_job::indexer_job_location, indexer_rules::IndexerRuleCreateArgs}, + indexer::{indexer_job::indexer_job_location, rules::IndexerRuleCreateArgs}, scan_location, LocationCreateArgs, LocationError, LocationUpdateArgs, }, + object::preview::THUMBNAIL_CACHE_DIR_NAME, prisma::{file, file_path, indexer_rule, indexer_rules_in_location, location, tag}, }; diff --git a/core/src/api/mod.rs b/core/src/api/mod.rs index 053600121..2266bf330 100644 --- a/core/src/api/mod.rs +++ b/core/src/api/mod.rs @@ -110,8 +110,10 @@ mod tests { #[test] fn test_and_export_rspc_bindings() { let r = super::mount(); - r.export_ts(PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("./index.ts")) - .expect("Error exporting rspc Typescript bindings!"); + r.export_ts( + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../packages/client/src/core.ts"), + ) + .expect("Error exporting rspc Typescript bindings!"); r.export_ts( PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../apps/mobile/src/types/bindings.ts"), ) diff --git a/core/src/api/tags.rs b/core/src/api/tags.rs index 7a21ae8e2..111d5e242 100644 --- a/core/src/api/tags.rs +++ b/core/src/api/tags.rs @@ -5,8 +5,8 @@ use uuid::Uuid; use crate::{ api::locations::{file_with_paths, ExplorerContext, ExplorerData, ExplorerItem}, - encode::THUMBNAIL_CACHE_DIR_NAME, invalidate_query, + object::preview::THUMBNAIL_CACHE_DIR_NAME, prisma::{file, tag, tag_on_file}, }; diff --git a/core/src/file/cas/mod.rs b/core/src/file/cas/mod.rs deleted file mode 100644 index 697b7ae31..000000000 --- a/core/src/file/cas/mod.rs +++ /dev/null @@ -1,5 +0,0 @@ -mod checksum; -mod identifier; - -pub use checksum::*; -pub use identifier::*; diff --git a/core/src/file/mod.rs b/core/src/file/mod.rs deleted file mode 100644 index 584df6c94..000000000 --- a/core/src/file/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod cas; diff --git a/core/src/job/job_manager.rs b/core/src/job/job_manager.rs index c4f40a1de..f2d7be4fb 100644 --- a/core/src/job/job_manager.rs +++ b/core/src/job/job_manager.rs @@ -1,9 +1,11 @@ use crate::{ - encode::{ThumbnailJob, THUMBNAIL_JOB_NAME}, - file::cas::{FileIdentifierJob, IDENTIFIER_JOB_NAME}, job::{worker::Worker, DynJob, Job, JobError}, library::LibraryContext, location::indexer::indexer_job::{IndexerJob, INDEXER_JOB_NAME}, + object::{ + identifier_job::{FileIdentifierJob, IDENTIFIER_JOB_NAME}, + preview::{ThumbnailJob, THUMBNAIL_JOB_NAME}, + }, prisma::{job, node}, }; diff --git a/core/src/lib.rs b/core/src/lib.rs index fd11734e6..3ab679658 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -13,17 +13,16 @@ use tracing::{error, info}; use tracing_subscriber::{filter::LevelFilter, fmt, prelude::*, EnvFilter}; pub mod api; -pub(crate) mod encode; -pub(crate) mod file; pub(crate) mod job; pub(crate) mod library; pub(crate) mod location; pub(crate) mod node; pub(crate) mod object; -pub(crate) mod prisma; pub(crate) mod util; pub(crate) mod volume; +pub(crate) mod prisma; + #[derive(Clone)] pub struct NodeContext { pub config: Arc, diff --git a/core/scripts/tsconfig.json b/core/src/location/archive/archive_job.rs similarity index 100% rename from core/scripts/tsconfig.json rename to core/src/location/archive/archive_job.rs diff --git a/packages/client/.gitignore b/core/src/location/archive/mod.rs similarity index 100% rename from packages/client/.gitignore rename to core/src/location/archive/mod.rs diff --git a/core/src/location/indexer/indexer_job.rs b/core/src/location/indexer/indexer_job.rs index dc804838f..f958961b5 100644 --- a/core/src/location/indexer/indexer_job.rs +++ b/core/src/location/indexer/indexer_job.rs @@ -12,7 +12,7 @@ use tokio::time::Instant; use tracing::info; use super::{ - indexer_rules::IndexerRule, + rules::IndexerRule, walk::{walk, WalkEntry}, }; diff --git a/core/src/location/indexer/mod.rs b/core/src/location/indexer/mod.rs index 52f10a8f3..c6fa351fb 100644 --- a/core/src/location/indexer/mod.rs +++ b/core/src/location/indexer/mod.rs @@ -1,12 +1,12 @@ pub mod indexer_job; -pub mod indexer_rules; +pub mod rules; mod walk; use globset::Error; -use indexer_rules::RuleKind; use int_enum::IntEnumError; use rmp_serde::{decode::Error as RMPDecodeError, encode::Error as RMPEncodeError}; use rspc::ErrorCode; +use rules::RuleKind; use serde_json::Error as SerdeJsonError; use std::io; use thiserror::Error; diff --git a/core/src/location/indexer/indexer_rules.rs b/core/src/location/indexer/rules.rs similarity index 100% rename from core/src/location/indexer/indexer_rules.rs rename to core/src/location/indexer/rules.rs diff --git a/core/src/location/indexer/walk.rs b/core/src/location/indexer/walk.rs index 0d65f99f6..0056f3341 100644 --- a/core/src/location/indexer/walk.rs +++ b/core/src/location/indexer/walk.rs @@ -9,7 +9,7 @@ use tokio::fs; use tracing::{debug, error}; use super::{ - indexer_rules::{IndexerRule, RuleKind}, + rules::{IndexerRule, RuleKind}, IndexerError, }; @@ -273,7 +273,7 @@ pub(super) async fn walk( #[cfg(test)] mod tests { - use super::super::indexer_rules::ParametersPerKind; + use super::super::rules::ParametersPerKind; use super::*; use chrono::Utc; use globset::Glob; diff --git a/core/src/location/mod.rs b/core/src/location/mod.rs index b51143b58..bc13355d5 100644 --- a/core/src/location/mod.rs +++ b/core/src/location/mod.rs @@ -1,9 +1,11 @@ use crate::{ - encode::{ThumbnailJob, ThumbnailJobInit}, - file::cas::{FileIdentifierJob, FileIdentifierJobInit}, invalidate_query, job::Job, library::LibraryContext, + object::{ + identifier_job::{FileIdentifierJob, FileIdentifierJobInit}, + preview::{ThumbnailJob, ThumbnailJobInit}, + }, prisma::{indexer_rules_in_location, location, node}, }; diff --git a/core/src/file/cas/checksum.rs b/core/src/object/cas.rs similarity index 100% rename from core/src/file/cas/checksum.rs rename to core/src/object/cas.rs diff --git a/core/src/object/fs/archive.rs b/core/src/object/fs/archive.rs new file mode 100644 index 000000000..e69de29bb diff --git a/core/src/object/fs/convert.rs b/core/src/object/fs/convert.rs new file mode 100644 index 000000000..e69de29bb diff --git a/core/src/object/fs/copy.rs b/core/src/object/fs/copy.rs new file mode 100644 index 000000000..e69de29bb diff --git a/core/src/object/fs/decrypt.rs b/core/src/object/fs/decrypt.rs new file mode 100644 index 000000000..e69de29bb diff --git a/core/src/object/fs/encrypt.rs b/core/src/object/fs/encrypt.rs new file mode 100644 index 000000000..e69de29bb diff --git a/core/src/object/fs/erase.rs b/core/src/object/fs/erase.rs new file mode 100644 index 000000000..e69de29bb diff --git a/core/src/object/fs/mod.rs b/core/src/object/fs/mod.rs new file mode 100644 index 000000000..e69de29bb diff --git a/core/src/object/fs/move.rs b/core/src/object/fs/move.rs new file mode 100644 index 000000000..e69de29bb diff --git a/core/src/object/fs/sync.rs b/core/src/object/fs/sync.rs new file mode 100644 index 000000000..e69de29bb diff --git a/core/src/file/cas/identifier.rs b/core/src/object/identifier_job.rs similarity index 99% rename from core/src/file/cas/identifier.rs rename to core/src/object/identifier_job.rs index 77d92922d..39e50fc54 100644 --- a/core/src/file/cas/identifier.rs +++ b/core/src/object/identifier_job.rs @@ -14,7 +14,7 @@ use std::{ use tokio::{fs, io}; use tracing::{error, info}; -use super::checksum::generate_cas_id; +use super::cas::generate_cas_id; // we break this job into chunks of 100 to improve performance static CHUNK_SIZE: usize = 100; diff --git a/core/src/object/mod.rs b/core/src/object/mod.rs index fd268ba87..d5dd6cd56 100644 --- a/core/src/object/mod.rs +++ b/core/src/object/mod.rs @@ -1,3 +1,7 @@ +pub mod cas; +pub mod identifier_job; +pub mod preview; + // Objects are primarily created by the identifier from Paths // Some Objects are purely virtual, unless they have one or more associated Paths, which refer to a file found in a Location // Objects are what can be added to Spaces diff --git a/core/src/encode/metadata.rs b/core/src/object/preview/metadata.rs similarity index 100% rename from core/src/encode/metadata.rs rename to core/src/object/preview/metadata.rs diff --git a/core/src/encode/mod.rs b/core/src/object/preview/mod.rs similarity index 100% rename from core/src/encode/mod.rs rename to core/src/object/preview/mod.rs diff --git a/core/src/encode/thumb.rs b/core/src/object/preview/thumb.rs similarity index 99% rename from core/src/encode/thumb.rs rename to core/src/object/preview/thumb.rs index 551cfbf51..e2e94d868 100644 --- a/core/src/encode/thumb.rs +++ b/core/src/object/preview/thumb.rs @@ -298,7 +298,7 @@ async fn generate_video_thumbnail>( file_path: P, output_path: P, ) -> Result<(), Box> { - use thumbnailer::to_thumbnail; + use sd_ffmpeg::to_thumbnail; to_thumbnail(file_path, output_path, 256, THUMBNAIL_QUALITY).await?; diff --git a/core/src/object/reader.rs b/core/src/object/reader.rs new file mode 100644 index 000000000..e69de29bb diff --git a/core/src/object/validation/hash.rs b/core/src/object/validation/hash.rs new file mode 100644 index 000000000..e69de29bb diff --git a/core/src/object/validation/mod.rs b/core/src/object/validation/mod.rs new file mode 100644 index 000000000..e69de29bb diff --git a/core/src/object/validation/validator_job.rs b/core/src/object/validation/validator_job.rs new file mode 100644 index 000000000..e69de29bb diff --git a/core/src/util/seeder.rs b/core/src/util/seeder.rs index 655ea1270..81bd6c92c 100644 --- a/core/src/util/seeder.rs +++ b/core/src/util/seeder.rs @@ -1,6 +1,6 @@ use crate::{ location::indexer::{ - indexer_rules::{IndexerRule, ParametersPerKind, RuleKind}, + rules::{IndexerRule, ParametersPerKind, RuleKind}, IndexerError, }, prisma::PrismaClient, diff --git a/core/src/volume/mod.rs b/core/src/volume.rs similarity index 100% rename from core/src/volume/mod.rs rename to core/src/volume.rs diff --git a/core/thumbnailer/.gitignore b/crates/ffmpeg/.gitignore similarity index 100% rename from core/thumbnailer/.gitignore rename to crates/ffmpeg/.gitignore diff --git a/core/thumbnailer/Cargo.toml b/crates/ffmpeg/Cargo.toml similarity index 96% rename from core/thumbnailer/Cargo.toml rename to crates/ffmpeg/Cargo.toml index 83508df6b..216a7103b 100644 --- a/core/thumbnailer/Cargo.toml +++ b/crates/ffmpeg/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "thumbnailer" +name = "sd_ffmpeg" version = "0.1.0" authors = ["Ericson Soares "] edition = "2021" diff --git a/core/thumbnailer/README.md b/crates/ffmpeg/README.md similarity index 100% rename from core/thumbnailer/README.md rename to crates/ffmpeg/README.md diff --git a/core/thumbnailer/src/error.rs b/crates/ffmpeg/src/error.rs similarity index 100% rename from core/thumbnailer/src/error.rs rename to crates/ffmpeg/src/error.rs diff --git a/core/thumbnailer/src/film_strip.rs b/crates/ffmpeg/src/film_strip.rs similarity index 100% rename from core/thumbnailer/src/film_strip.rs rename to crates/ffmpeg/src/film_strip.rs diff --git a/core/thumbnailer/src/lib.rs b/crates/ffmpeg/src/lib.rs similarity index 100% rename from core/thumbnailer/src/lib.rs rename to crates/ffmpeg/src/lib.rs diff --git a/core/thumbnailer/src/movie_decoder.rs b/crates/ffmpeg/src/movie_decoder.rs similarity index 100% rename from core/thumbnailer/src/movie_decoder.rs rename to crates/ffmpeg/src/movie_decoder.rs diff --git a/core/thumbnailer/src/thumbnailer.rs b/crates/ffmpeg/src/thumbnailer.rs similarity index 100% rename from core/thumbnailer/src/thumbnailer.rs rename to crates/ffmpeg/src/thumbnailer.rs diff --git a/core/thumbnailer/src/utils.rs b/crates/ffmpeg/src/utils.rs similarity index 100% rename from core/thumbnailer/src/utils.rs rename to crates/ffmpeg/src/utils.rs diff --git a/core/thumbnailer/src/video_frame.rs b/crates/ffmpeg/src/video_frame.rs similarity index 100% rename from core/thumbnailer/src/video_frame.rs rename to crates/ffmpeg/src/video_frame.rs diff --git a/crates/p2p/Cargo.toml b/crates/p2p/Cargo.toml new file mode 100644 index 000000000..a564b62cf --- /dev/null +++ b/crates/p2p/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "p2p" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] diff --git a/crates/p2p/src/main.rs b/crates/p2p/src/main.rs new file mode 100644 index 000000000..a30eb952c --- /dev/null +++ b/crates/p2p/src/main.rs @@ -0,0 +1,3 @@ +fn main() { + println!("Hello, world!"); +} diff --git a/core/prisma/Cargo.toml b/crates/prisma/Cargo.toml similarity index 100% rename from core/prisma/Cargo.toml rename to crates/prisma/Cargo.toml diff --git a/core/prisma/migrations/20220909073230_init/migration.sql b/crates/prisma/migrations/20220909073230_init/migration.sql similarity index 100% rename from core/prisma/migrations/20220909073230_init/migration.sql rename to crates/prisma/migrations/20220909073230_init/migration.sql diff --git a/core/prisma/migrations/migration_lock.toml b/crates/prisma/migrations/migration_lock.toml similarity index 100% rename from core/prisma/migrations/migration_lock.toml rename to crates/prisma/migrations/migration_lock.toml diff --git a/core/prisma/schema.prisma b/crates/prisma/schema.prisma similarity index 99% rename from core/prisma/schema.prisma rename to crates/prisma/schema.prisma index 0b8c45a6c..9e4a3ce83 100644 --- a/core/prisma/schema.prisma +++ b/crates/prisma/schema.prisma @@ -5,7 +5,7 @@ datasource db { generator client { provider = "cargo prisma" - output = "../src/prisma.rs" + output = "../../core/src/prisma.rs" } model Migration { diff --git a/core/prisma/src/main.rs b/crates/prisma/src/main.rs similarity index 100% rename from core/prisma/src/main.rs rename to crates/prisma/src/main.rs diff --git a/crates/sync/Cargo.toml b/crates/sync/Cargo.toml new file mode 100644 index 000000000..ac99e6ce2 --- /dev/null +++ b/crates/sync/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "sync" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] diff --git a/crates/sync/src/main.rs b/crates/sync/src/main.rs new file mode 100644 index 000000000..a30eb952c --- /dev/null +++ b/crates/sync/src/main.rs @@ -0,0 +1,3 @@ +fn main() { + println!("Hello, world!"); +} diff --git a/docs/product/guides/adding-locations.md b/docs/product/guides/adding-locations.md index 9734fa753..d0df3f043 100644 --- a/docs/product/guides/adding-locations.md +++ b/docs/product/guides/adding-locations.md @@ -4,4 +4,39 @@ index: 5 # Adding Locations -![image](/locations.webp) \ No newline at end of file +Locations are places Spacedrive looks for files. You can add Locations from any device to your Library, they will be scanned and monitored for filesystem changes. + +![Spacedrive Locations from the Settings view](/locations.webp) + +Locations can be managed from the settings. The "online" indicator shows if that Location is currently accessible from one of your devices. + +## Scanning +When a Location is added it is immediately scanned. The scan happens via several phases. + +1. The [Indexer](): directory structure is indexed and saved to your library's database. +2. The [Identifier](): metadata is extracted and unique identifiers generated. +3. The [Thumbnailer](): compressed previews are generated for all media and text files. + +If any one of these jobs fail or the application is closed, the state is saved and it will resume when the app is relaunched. + +::: slot info +If you wish to pause or cancel these jobs you can do so from the [Job Manager]() +::: + +There is a button to trigger a full re-scan of this Location. Otherwise the location will keep watch for filesystem changes automatically. + +Deleting a location will remove the data from the database permanently. + +## Archiving +::: slot warning +This feature is WIP +::: +Locations can be archived, meaning the directory structure is extracted from the Spacedrive database and preserved as a standalone database file. This file can be opened by Spacedrive in the [Database]() view. + +Archives are useful when files have been moved or deleted from a Location, but you want to keep a record of the directory structure of that Location without cluttering the Spacedrive interface. File metadata can optionally be included/excluded from the archive file. + +## Dynamic Space +A slider in the Location settings allows you to allocate an amount of GB to use as redundancy for other Locations. The storage space will be filled and emptied dynamically, recent and important files will be copied as priority. + +## Encrypted Space +Allocate a portion of this location to be encrypted \ No newline at end of file diff --git a/docs/product/guides/connecting-nodes.md b/docs/product/guides/connecting-nodes.md index 78a306a8d..009639b06 100644 --- a/docs/product/guides/connecting-nodes.md +++ b/docs/product/guides/connecting-nodes.md @@ -2,4 +2,13 @@ index: 6 --- -# Connecting Nodes \ No newline at end of file +# Connecting Nodes + +::: slot note WIP +It is not possible to connect Nodes yet, please check our [roadmap](/roadmap). +::: + +Nodes are instances of Spacedrive, commonly your devices. Currently Spacedrive supports Windows, Linux, Mac, iOS and Android. + +These devices can be connected via P2P to directly and securely synchronize and move data. + diff --git a/docs/product/guides/creating-spaces.md b/docs/product/guides/creating-spaces.md index 94770fb86..8ff1acdf5 100644 --- a/docs/product/guides/creating-spaces.md +++ b/docs/product/guides/creating-spaces.md @@ -2,4 +2,6 @@ index: 7 --- -# Creating Spaces \ No newline at end of file +# Creating Spaces + +Spaces are virtual folders, they can contain files from various locations. \ No newline at end of file diff --git a/docs/product/guides/database-backup.md b/docs/product/guides/database-backup.md deleted file mode 100644 index b86376ec6..000000000 --- a/docs/product/guides/database-backup.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -index: 6 ---- - -# Database Backup \ No newline at end of file diff --git a/docs/product/guides/find-my-file.md b/docs/product/guides/find-my-file.md new file mode 100644 index 000000000..06b00c652 --- /dev/null +++ b/docs/product/guides/find-my-file.md @@ -0,0 +1,7 @@ +--- +index: 10 +--- + +# Find My File + +Mark a file as lost, if Spacedrive ever discovers that file when indexing, you will be notified and Spacedrive will make a backup. diff --git a/docs/product/guides/library-setup.md b/docs/product/guides/library-setup.md new file mode 100644 index 000000000..217dc60d5 --- /dev/null +++ b/docs/product/guides/library-setup.md @@ -0,0 +1,14 @@ +--- +index: 0 +--- + +# Library Setup + +A Library is where all your Spacedrive data lives, but not the files themselves. + +For the convenience of finding, organizing and sharing files even when you don't have access; directory structures, metadata and preview media are extracted and stored within the Spacedrive Library. + +This data is heavily compressed and very portable, you can have multiple libraries and even lock and encrypt them. + +## Deleting a Library +A Library can be deleted, \ No newline at end of file diff --git a/package.json b/package.json index e3406c04f..ab0932e3d 100644 --- a/package.json +++ b/package.json @@ -3,18 +3,17 @@ "version": "0.0.0", "private": true, "scripts": { - "prep": "pnpm db:gen && pnpm core codegen", + "prep": "pnpm db:gen && cargo test", "build": "turbo run build", "landing-web": "turbo run dev --parallel --filter=@sd/landing --filter=@sd/web", - "db:migrate": "pnpm core prisma migrate dev", - "db:gen": "pnpm core prisma generate", + "db:migrate": "cd crates && cargo prisma migrate dev", + "db:gen": "cd crates && cargo prisma generate", "format": "prettier --config .prettierrc.cli.js --write \"**/*.{ts,tsx,html,scss,json,yml,md}\"", "desktop": "pnpm --filter @sd/desktop --", "web": "pnpm --filter @sd/web -- ", "landing": "pnpm --filter @sd/landing -- ", "ui": "pnpm --filter @sd/ui -- ", "interface": "pnpm --filter @sd/interface -- ", - "core": "pnpm --filter @sd/core -- ", "docs": "pnpm --filter @sd/docs -- ", "client": "pnpm --filter @sd/client -- ", "server": "pnpm --filter @sd/server -- ", diff --git a/packages/client/package.json b/packages/client/package.json index 506be5674..4ed47610f 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -20,7 +20,6 @@ "@rspc/client": "^0.0.6", "@rspc/react": "^0.0.6", "@sd/config": "workspace:*", - "@sd/core": "workspace:*", "@sd/interface": "workspace:*", "@tanstack/react-query": "^4.0.10", "eventemitter3": "^4.0.7", diff --git a/core/index.ts b/packages/client/src/core.ts similarity index 100% rename from core/index.ts rename to packages/client/src/core.ts diff --git a/packages/client/src/index.ts b/packages/client/src/index.ts index 8cc82b66b..bc08fb913 100644 --- a/packages/client/src/index.ts +++ b/packages/client/src/index.ts @@ -1,5 +1,5 @@ export * from './stores'; export * from './context'; export * from './rspc'; +export * from './core'; export * from './hooks'; -export type { Operations } from '@sd/core'; diff --git a/packages/client/src/rspc.ts b/packages/client/src/rspc.ts index 56e763ef5..fa2aff113 100644 --- a/packages/client/src/rspc.ts +++ b/packages/client/src/rspc.ts @@ -1,6 +1,5 @@ import { RSPCError } from '@rspc/client'; import { createReactQueryHooks } from '@rspc/react'; -import { LibraryArgs, Operations } from '@sd/core'; import { QueryClient, UseInfiniteQueryOptions, @@ -12,6 +11,7 @@ import { useMutation as _useMutation } from '@tanstack/react-query'; +import { LibraryArgs, Operations } from './core'; import { useCurrentLibrary } from './index'; export const queryClient = new QueryClient(); diff --git a/packages/client/src/types/file.ts b/packages/client/src/types/file.ts deleted file mode 100644 index e48d01533..000000000 --- a/packages/client/src/types/file.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { FilePath } from '@sd/core'; - -export interface ExplorerItem { - id: number; - name: string; - is_dir: boolean; - // kind: ObjectKind; - extension: string; - size_in_bytes: number; - created_at: string; - updated_at: string; - favorite?: boolean; - - // computed - paths?: FilePath[]; -} diff --git a/packages/client/src/types/index.ts b/packages/client/src/types/index.ts deleted file mode 100644 index 706b0d228..000000000 --- a/packages/client/src/types/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './file'; diff --git a/packages/interface/package.json b/packages/interface/package.json index abc8cc1f1..d875a80df 100644 --- a/packages/interface/package.json +++ b/packages/interface/package.json @@ -27,7 +27,6 @@ "@radix-ui/react-tooltip": "^1.0.0", "@sd/assets": "workspace:*", "@sd/client": "workspace:*", - "@sd/core": "workspace:*", "@sd/ui": "workspace:*", "@tailwindcss/forms": "^0.5.2", "@tanstack/react-query": "^4.2.3", diff --git a/packages/interface/src/components/explorer/Explorer.tsx b/packages/interface/src/components/explorer/Explorer.tsx index 09fb14160..7dcf6911f 100644 --- a/packages/interface/src/components/explorer/Explorer.tsx +++ b/packages/interface/src/components/explorer/Explorer.tsx @@ -1,5 +1,4 @@ -import { getExplorerStore, rspc, useCurrentLibrary, useExplorerStore } from '@sd/client'; -import { ExplorerData } from '@sd/core'; +import { ExplorerData, rspc, useCurrentLibrary, useExplorerStore } from '@sd/client'; import { Inspector } from '../explorer/Inspector'; import { TopBar } from '../layout/TopBar'; diff --git a/packages/interface/src/components/explorer/FileItem.tsx b/packages/interface/src/components/explorer/FileItem.tsx index 2e7ee6e26..fbf98e4da 100644 --- a/packages/interface/src/components/explorer/FileItem.tsx +++ b/packages/interface/src/components/explorer/FileItem.tsx @@ -1,5 +1,4 @@ -import { getExplorerStore, useExplorerStore } from '@sd/client'; -import { ExplorerItem } from '@sd/core'; +import { ExplorerItem, getExplorerStore } from '@sd/client'; import clsx from 'clsx'; import { HTMLAttributes } from 'react'; diff --git a/packages/interface/src/components/explorer/FileRow.tsx b/packages/interface/src/components/explorer/FileRow.tsx index 9384b8d91..16abf00d8 100644 --- a/packages/interface/src/components/explorer/FileRow.tsx +++ b/packages/interface/src/components/explorer/FileRow.tsx @@ -1,4 +1,4 @@ -import { ExplorerItem } from '@sd/core'; +import { ExplorerItem } from '@sd/client'; import clsx from 'clsx'; import { HTMLAttributes } from 'react'; diff --git a/packages/interface/src/components/explorer/FileThumb.tsx b/packages/interface/src/components/explorer/FileThumb.tsx index 9f282b039..1acb66384 100644 --- a/packages/interface/src/components/explorer/FileThumb.tsx +++ b/packages/interface/src/components/explorer/FileThumb.tsx @@ -1,5 +1,5 @@ import { getExplorerStore, useExplorerStore, usePlatform } from '@sd/client'; -import { ExplorerItem } from '@sd/core'; +import { ExplorerItem } from '@sd/client'; import clsx from 'clsx'; import { useState } from 'react'; import { useSnapshot } from 'valtio'; diff --git a/packages/interface/src/components/explorer/Inspector.tsx b/packages/interface/src/components/explorer/Inspector.tsx index 61a81912d..fec89f030 100644 --- a/packages/interface/src/components/explorer/Inspector.tsx +++ b/packages/interface/src/components/explorer/Inspector.tsx @@ -1,6 +1,6 @@ import { ShareIcon } from '@heroicons/react/24/solid'; import { useLibraryQuery } from '@sd/client'; -import { ExplorerContext, ExplorerItem, File, FilePath, Location } from '@sd/core'; +import { ExplorerContext, ExplorerItem, File, FilePath, Location } from '@sd/client'; import { Button, TextArea } from '@sd/ui'; import clsx from 'clsx'; import moment from 'moment'; diff --git a/packages/interface/src/components/explorer/VirtualizedList.tsx b/packages/interface/src/components/explorer/VirtualizedList.tsx index 50fadd719..d668bffb2 100644 --- a/packages/interface/src/components/explorer/VirtualizedList.tsx +++ b/packages/interface/src/components/explorer/VirtualizedList.tsx @@ -1,5 +1,5 @@ import { ExplorerLayoutMode, getExplorerStore, useExplorerStore } from '@sd/client'; -import { ExplorerContext, ExplorerItem, FilePath } from '@sd/core'; +import { ExplorerContext, ExplorerItem, FilePath } from '@sd/client'; import { useVirtualizer } from '@tanstack/react-virtual'; import { memo, useCallback, useLayoutEffect, useMemo, useRef, useState } from 'react'; import { useSearchParams } from 'react-router-dom'; diff --git a/packages/interface/src/components/explorer/inspector/FavoriteButton.tsx b/packages/interface/src/components/explorer/inspector/FavoriteButton.tsx index d73b8f966..c95ef374f 100644 --- a/packages/interface/src/components/explorer/inspector/FavoriteButton.tsx +++ b/packages/interface/src/components/explorer/inspector/FavoriteButton.tsx @@ -1,5 +1,5 @@ import { useLibraryMutation } from '@sd/client'; -import { File } from '@sd/core'; +import { File } from '@sd/client'; import { Button } from '@sd/ui'; import { Heart } from 'phosphor-react'; import { useEffect, useState } from 'react'; diff --git a/packages/interface/src/components/explorer/inspector/Note.tsx b/packages/interface/src/components/explorer/inspector/Note.tsx index 2c4e0268c..500cafe79 100644 --- a/packages/interface/src/components/explorer/inspector/Note.tsx +++ b/packages/interface/src/components/explorer/inspector/Note.tsx @@ -1,6 +1,6 @@ /* eslint-disable react-hooks/exhaustive-deps */ import { useLibraryMutation } from '@sd/client'; -import { File } from '@sd/core'; +import { File } from '@sd/client'; import { TextArea } from '@sd/ui'; import { debounce } from 'lodash'; import { useCallback, useState } from 'react'; diff --git a/packages/interface/src/components/explorer/utils.ts b/packages/interface/src/components/explorer/utils.ts index cc2a7d2da..0cdf23b85 100644 --- a/packages/interface/src/components/explorer/utils.ts +++ b/packages/interface/src/components/explorer/utils.ts @@ -1,4 +1,4 @@ -import { ExplorerItem, File, FilePath } from '@sd/core'; +import { ExplorerItem, File, FilePath } from '@sd/client'; export function isPath(item: ExplorerItem): item is Extract { return item.type === 'Path'; diff --git a/packages/interface/src/components/jobs/JobManager.tsx b/packages/interface/src/components/jobs/JobManager.tsx index 5da33ac40..70d4a8349 100644 --- a/packages/interface/src/components/jobs/JobManager.tsx +++ b/packages/interface/src/components/jobs/JobManager.tsx @@ -1,6 +1,6 @@ import { EyeIcon, FolderIcon, PhotoIcon, XMarkIcon } from '@heroicons/react/24/outline'; import { useLibraryQuery } from '@sd/client'; -import { JobReport } from '@sd/core'; +import { JobReport } from '@sd/client'; import { Button } from '@sd/ui'; import clsx from 'clsx'; import { formatDistanceToNow, formatDuration } from 'date-fns'; diff --git a/packages/interface/src/components/layout/Sidebar.tsx b/packages/interface/src/components/layout/Sidebar.tsx index 3146b00e1..9c06f598d 100644 --- a/packages/interface/src/components/layout/Sidebar.tsx +++ b/packages/interface/src/components/layout/Sidebar.tsx @@ -1,7 +1,7 @@ import { CogIcon, LockClosedIcon, PhotoIcon } from '@heroicons/react/24/outline'; import { PlusIcon } from '@heroicons/react/24/solid'; import { useCurrentLibrary, useLibraryMutation, useLibraryQuery, usePlatform } from '@sd/client'; -import { LocationCreateArgs } from '@sd/core'; +import { LocationCreateArgs } from '@sd/client'; import { Button, Dropdown, OverlayPanel } from '@sd/ui'; import clsx from 'clsx'; import { CheckCircle, CirclesFour, Planet, WaveTriangle } from 'phosphor-react'; diff --git a/packages/interface/src/components/location/LocationListItem.tsx b/packages/interface/src/components/location/LocationListItem.tsx index 6fb2081ff..cab808c03 100644 --- a/packages/interface/src/components/location/LocationListItem.tsx +++ b/packages/interface/src/components/location/LocationListItem.tsx @@ -1,13 +1,14 @@ -import { Folder } from '../icons/Folder'; -import Dialog from '../layout/Dialog'; import { TrashIcon } from '@heroicons/react/24/solid'; import { useLibraryMutation } from '@sd/client'; -import { Location, Node } from '@sd/core'; +import { Location, Node } from '@sd/client'; import { Button } from '@sd/ui'; import clsx from 'clsx'; import { Repeat } from 'phosphor-react'; import React, { useState } from 'react'; +import { Folder } from '../icons/Folder'; +import Dialog from '../layout/Dialog'; + interface LocationListItemProps { location: Location & { node: Node }; } diff --git a/packages/interface/src/screens/Overview.tsx b/packages/interface/src/screens/Overview.tsx index c2ce5bfbd..45d53ce23 100644 --- a/packages/interface/src/screens/Overview.tsx +++ b/packages/interface/src/screens/Overview.tsx @@ -1,6 +1,6 @@ import { ExclamationCircleIcon, PlusIcon } from '@heroicons/react/24/solid'; import { useBridgeQuery, useLibraryQuery, usePlatform } from '@sd/client'; -import { Statistics } from '@sd/core'; +import { Statistics } from '@sd/client'; import { Button, Input } from '@sd/ui'; import byteSize from 'byte-size'; import clsx from 'clsx'; @@ -163,7 +163,7 @@ export const OverviewScreen = () => {
-
+