mirror of
https://github.com/spacedriveapp/spacedrive
synced 2024-06-28 01:03:27 +00:00
[ENG-1479] AI Prototype (#1845)
* First draft on image labeling * Fixing execution providers for other OSs * Better error handling and shutdown * Working with shallow media processor * bruh * Fix warnings * Now hooked to media processor job * Link desktop app with libonnxruntime to avoid TLS error during startup * Be able to change models on runtime Revert to use labels table instead of tags * A bug on a model-less inference * Show AI labels on Inspector - Change yolo inference to use half precision - Add labels api to core * Remove LD_PRELOAD * Fix race condition on model executor shutdown * Don't load all images in memory moron * Embeed yolo model in prod build - Change yolo model path to new one relative to executable * Disable volume watcher on linux, it was crashing the app - Invalidate labels when they are updated * Rust fmt * Minor changes * Gate onnxruntime linking to the ai-models feature * Add build script to sd-server to handle onnxruntime linking workaround * Move AI stuff to its own crate and normalize deps * Rust fmt * Don't regenerate labels unless asked to * Now blazingly fast * Bad merge * Fix * Fix * Add backend logic to download extra yolo models * Add models api route - Add api call to get available model version - Add api call to change the model version * Improve new model download logic - Add frontend to change image labeler model * Fix new model downloader * Fix model select width * invalidate labels count after media_processor generates a new output * Rename AI crate and first draft on download notifications * fix types --------- Co-authored-by: Vítor Vasconcellos <vasconcellos.dev@gmail.com> Co-authored-by: Brendan Allan <brendonovich@outlook.com>
This commit is contained in:
Ericson "Fogo" Soares
GitHub
parent
7aa0452ba3
commit
7c90bcb95b
|
|
@ -64,7 +64,7 @@ indent_style = space
|
|||
# Prisma
|
||||
# https://www.prisma.io/docs/reference/tools-and-interfaces/prisma-schema/data-model#formatting
|
||||
[*.prisma]
|
||||
indent_size = 4
|
||||
indent_size = 2
|
||||
indent_style = space
|
||||
|
||||
# YAML
|
||||
|
|
|
|||
3
.vscode/launch.json
vendored
3
.vscode/launch.json
vendored
|
|
@ -12,7 +12,8 @@
|
|||
"args": [
|
||||
"build",
|
||||
"--manifest-path=./apps/desktop/src-tauri/Cargo.toml",
|
||||
"--no-default-features"
|
||||
"--no-default-features",
|
||||
"--features=ai-models"
|
||||
],
|
||||
"problemMatcher": "$rustc"
|
||||
},
|
||||
|
|
|
|||
3
.vscode/tasks.json
vendored
3
.vscode/tasks.json
vendored
|
|
@ -56,7 +56,8 @@
|
|||
"command": "run",
|
||||
"args": [
|
||||
"--manifest-path=./apps/desktop/src-tauri/Cargo.toml",
|
||||
"--no-default-features"
|
||||
"--no-default-features",
|
||||
"--features=ai-models"
|
||||
],
|
||||
"env": {
|
||||
"RUST_BACKTRACE": "short"
|
||||
|
|
|
|||
246
Cargo.lock
generated
246
Cargo.lock
generated
|
|
@ -151,30 +151,30 @@ checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87"
|
|||
|
||||
[[package]]
|
||||
name = "anstyle-parse"
|
||||
version = "0.2.2"
|
||||
version = "0.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "317b9a89c1868f5ea6ff1d9539a69f45dffc21ce321ac1fd1160dfa48c8e2140"
|
||||
checksum = "c75ac65da39e5fe5ab759307499ddad880d724eed2f6ce5b5e8a26f4f387928c"
|
||||
dependencies = [
|
||||
"utf8parse",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstyle-query"
|
||||
version = "1.0.0"
|
||||
version = "1.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b"
|
||||
checksum = "a3a318f1f38d2418400f8209655bfd825785afd25aa30bb7ba6cc792e4596748"
|
||||
dependencies = [
|
||||
"windows-sys 0.48.0",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstyle-wincon"
|
||||
version = "3.0.1"
|
||||
version = "3.0.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f0699d10d2f4d628a98ee7b57b289abbc98ff3bad977cb3152709d4bf2330628"
|
||||
checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7"
|
||||
dependencies = [
|
||||
"anstyle",
|
||||
"windows-sys 0.48.0",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1031,9 +1031,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "clap"
|
||||
version = "4.4.10"
|
||||
version = "4.4.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "41fffed7514f420abec6d183b1d3acfd9099c79c3a10a06ade4f8203f1411272"
|
||||
checksum = "bfaff671f6b22ca62406885ece523383b9b64022e341e53e009a62ebc47a45f2"
|
||||
dependencies = [
|
||||
"clap_builder",
|
||||
"clap_derive",
|
||||
|
|
@ -1041,9 +1041,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "clap_builder"
|
||||
version = "4.4.9"
|
||||
version = "4.4.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "63361bae7eef3771745f02d8d892bec2fee5f6e34af316ba556e7f97a7069ff1"
|
||||
checksum = "a216b506622bb1d316cd51328dce24e07bdff4a6128a47c7e7fad11878d5adbb"
|
||||
dependencies = [
|
||||
"anstream",
|
||||
"anstyle",
|
||||
|
|
@ -1996,12 +1996,12 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "exr"
|
||||
version = "1.71.0"
|
||||
version = "1.6.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "832a761f35ab3e6664babfbdc6cef35a4860e816ec3916dcfd0882954e98a8a8"
|
||||
checksum = "279d3efcc55e19917fff7ab3ddd6c14afb6a90881a0078465196fe2f99d08c56"
|
||||
dependencies = [
|
||||
"bit_field",
|
||||
"flume 0.11.0",
|
||||
"flume",
|
||||
"half",
|
||||
"lebe",
|
||||
"miniz_oxide",
|
||||
|
|
@ -2100,14 +2100,14 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "filetime"
|
||||
version = "0.2.22"
|
||||
version = "0.2.23"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d4029edd3e734da6fe05b6cd7bd2960760a616bd2ddd0d59a0124746d6272af0"
|
||||
checksum = "1ee447700ac8aa0b2f2bd7bc4462ad686ba06baa6727ac149a2d6277f0d240fd"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"redox_syscall 0.3.5",
|
||||
"windows-sys 0.48.0",
|
||||
"redox_syscall 0.4.1",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -2145,15 +2145,6 @@ dependencies = [
|
|||
"spin 0.9.8",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "flume"
|
||||
version = "0.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "55ac459de2512911e4b674ce33cf20befaba382d05b62b008afc1c8b57cbf181"
|
||||
dependencies = [
|
||||
"spin 0.9.8",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fnv"
|
||||
version = "1.0.7"
|
||||
|
|
@ -2760,11 +2751,13 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "half"
|
||||
version = "2.2.1"
|
||||
version = "2.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "02b4af3693f1b705df946e9fe5631932443781d0aabb423b62fcd4d73f6d2fd0"
|
||||
checksum = "bc52e53916c08643f1b56ec082790d1e86a32e58dc5268f897f313fbae7b4872"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"crunchy",
|
||||
"num-traits",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -3387,9 +3380,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "itertools"
|
||||
version = "0.11.0"
|
||||
version = "0.12.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57"
|
||||
checksum = "25db6b064527c5d482d0423354fcd07a89a2dfe07b67892e62411946db7f07b0"
|
||||
dependencies = [
|
||||
"either",
|
||||
]
|
||||
|
|
@ -4202,6 +4195,16 @@ version = "0.7.3"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94"
|
||||
|
||||
[[package]]
|
||||
name = "matrixmultiply"
|
||||
version = "0.3.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7574c1cf36da4798ab73da5b215bbf444f50718207754cb522201d78d1cd0ff2"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"rawpointer",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "maybe-owned"
|
||||
version = "0.3.4"
|
||||
|
|
@ -4220,7 +4223,7 @@ version = "0.9.3"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4c97183f9949c1f97921e4dda6bc059dfc30a6da5e4460a4f5218847dbe2ae1f"
|
||||
dependencies = [
|
||||
"flume 0.10.14",
|
||||
"flume",
|
||||
"if-addrs 0.10.2",
|
||||
"log",
|
||||
"polling",
|
||||
|
|
@ -4402,9 +4405,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "mio"
|
||||
version = "0.8.9"
|
||||
version = "0.8.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3dce281c5e46beae905d4de1870d8b1509a9142b62eedf18b443b011ca8343d0"
|
||||
checksum = "8f3d0b296e374a4e6f3c7b0a1f5a51d748a0d34c85e7dc48fc3fa9a87657fe09"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"log",
|
||||
|
|
@ -4538,6 +4541,19 @@ dependencies = [
|
|||
"socket2 0.4.10",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ndarray"
|
||||
version = "0.15.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "adb12d4e967ec485a5f71c6311fe28158e9d6f4bc4a447b474184d0f91a8fa32"
|
||||
dependencies = [
|
||||
"matrixmultiply",
|
||||
"num-complex",
|
||||
"num-integer",
|
||||
"num-traits",
|
||||
"rawpointer",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ndk"
|
||||
version = "0.6.0"
|
||||
|
|
@ -4827,6 +4843,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"libm",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -4919,9 +4936,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "once_cell"
|
||||
version = "1.18.0"
|
||||
version = "1.19.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d"
|
||||
checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
|
||||
|
||||
[[package]]
|
||||
name = "opaque-debug"
|
||||
|
|
@ -4954,9 +4971,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "openssl"
|
||||
version = "0.10.57"
|
||||
version = "0.10.61"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bac25ee399abb46215765b1cb35bc0212377e58a061560d8b29b024fd0430e7c"
|
||||
checksum = "6b8419dc8cc6d866deb801274bba2e6f8f6108c1bb7fcc10ee5ab864931dbb45"
|
||||
dependencies = [
|
||||
"bitflags 2.4.1",
|
||||
"cfg-if",
|
||||
|
|
@ -4995,9 +5012,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "openssl-sys"
|
||||
version = "0.9.93"
|
||||
version = "0.9.97"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "db4d56a4c0478783083cfafcc42493dd4a981d41669da64b4572a2a089b51b1d"
|
||||
checksum = "c3eaad34cdd97d81de97964fc7f29e2d104f483840d906ef56daa1912338460b"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"libc",
|
||||
|
|
@ -5049,6 +5066,33 @@ version = "0.3.5"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a86ed3f5f244b372d6b1a00b72ef7f8876d0bc6a78a4c9985c53614041512063"
|
||||
|
||||
[[package]]
|
||||
name = "ort"
|
||||
version = "2.0.0-alpha.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1a094a17bfe4f9eb561bfdf4454b8d0f6f89deaf9a5a572a1ef29c29ce708627"
|
||||
dependencies = [
|
||||
"half",
|
||||
"libloading 0.8.1",
|
||||
"ndarray",
|
||||
"once_cell",
|
||||
"ort-sys",
|
||||
"thiserror",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ort-sys"
|
||||
version = "2.0.0-alpha.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dee05e7997a1cd9c0dc63fb9ddf79543efe646d9398089bf5b0b95e7e9441984"
|
||||
dependencies = [
|
||||
"flate2",
|
||||
"sha2 0.10.8",
|
||||
"tar",
|
||||
"ureq",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "os_info"
|
||||
version = "3.7.0"
|
||||
|
|
@ -6245,6 +6289,12 @@ version = "0.5.2"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f2ff9a1f06a88b01621b7ae906ef0211290d1c8a168a15542486a8f61c0833b9"
|
||||
|
||||
[[package]]
|
||||
name = "rawpointer"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3"
|
||||
|
||||
[[package]]
|
||||
name = "rayon"
|
||||
version = "1.8.0"
|
||||
|
|
@ -6301,15 +6351,6 @@ dependencies = [
|
|||
"bitflags 1.3.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "redox_syscall"
|
||||
version = "0.3.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29"
|
||||
dependencies = [
|
||||
"bitflags 1.3.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "redox_syscall"
|
||||
version = "0.4.1"
|
||||
|
|
@ -6524,9 +6565,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "ring"
|
||||
version = "0.17.6"
|
||||
version = "0.17.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "684d5e6e18f669ccebf64a92236bb7db9a34f07be010e3627368182027180866"
|
||||
checksum = "688c63d65483050968b2a8937f7995f443e27041a0f7700aa59b0822aedebb74"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"getrandom 0.2.11",
|
||||
|
|
@ -6692,7 +6733,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "629648aced5775d558af50b2b4c7b02983a04b312126d45eeead26e7caa498b9"
|
||||
dependencies = [
|
||||
"log",
|
||||
"ring 0.17.6",
|
||||
"ring 0.17.7",
|
||||
"rustls-webpki",
|
||||
"sct",
|
||||
]
|
||||
|
|
@ -6703,7 +6744,7 @@ version = "0.101.7"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765"
|
||||
dependencies = [
|
||||
"ring 0.17.6",
|
||||
"ring 0.17.7",
|
||||
"untrusted 0.9.0",
|
||||
]
|
||||
|
||||
|
|
@ -6849,10 +6890,38 @@ version = "0.7.1"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414"
|
||||
dependencies = [
|
||||
"ring 0.17.6",
|
||||
"ring 0.17.7",
|
||||
"untrusted 0.9.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sd-ai"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"async-channel",
|
||||
"chrono",
|
||||
"futures",
|
||||
"futures-concurrency",
|
||||
"half",
|
||||
"image",
|
||||
"ndarray",
|
||||
"once_cell",
|
||||
"ort",
|
||||
"prisma-client-rust",
|
||||
"reqwest",
|
||||
"rmp-serde",
|
||||
"sd-file-path-helper",
|
||||
"sd-prisma",
|
||||
"sd-utils",
|
||||
"serde",
|
||||
"thiserror",
|
||||
"tokio",
|
||||
"tokio-stream",
|
||||
"tracing",
|
||||
"url",
|
||||
"uuid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sd-cache"
|
||||
version = "0.0.0"
|
||||
|
|
@ -6912,7 +6981,7 @@ dependencies = [
|
|||
"http-range",
|
||||
"image",
|
||||
"int-enum",
|
||||
"itertools 0.11.0",
|
||||
"itertools 0.12.0",
|
||||
"mini-moka",
|
||||
"normpath",
|
||||
"notify",
|
||||
|
|
@ -6927,12 +6996,14 @@ dependencies = [
|
|||
"rmp-serde",
|
||||
"rmpv",
|
||||
"rspc",
|
||||
"sd-ai",
|
||||
"sd-cache",
|
||||
"sd-cloud-api",
|
||||
"sd-core-sync",
|
||||
"sd-crypto",
|
||||
"sd-ffmpeg",
|
||||
"sd-file-ext",
|
||||
"sd-file-path-helper",
|
||||
"sd-images",
|
||||
"sd-media-metadata",
|
||||
"sd-p2p",
|
||||
|
|
@ -6962,7 +7033,6 @@ dependencies = [
|
|||
"tracing-test",
|
||||
"uuid",
|
||||
"webp",
|
||||
"winapi-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -7091,9 +7161,26 @@ dependencies = [
|
|||
"serde_json",
|
||||
"specta",
|
||||
"strum",
|
||||
"strum_macros",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sd-file-path-helper"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"prisma-client-rust",
|
||||
"regex",
|
||||
"sd-prisma",
|
||||
"sd-utils",
|
||||
"serde",
|
||||
"thiserror",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"winapi-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sd-images"
|
||||
version = "0.0.0"
|
||||
|
|
@ -7164,7 +7251,7 @@ version = "0.1.0"
|
|||
dependencies = [
|
||||
"base64 0.21.5",
|
||||
"ed25519-dalek",
|
||||
"flume 0.10.14",
|
||||
"flume",
|
||||
"futures-core",
|
||||
"if-watch",
|
||||
"libp2p",
|
||||
|
|
@ -7235,6 +7322,10 @@ dependencies = [
|
|||
name = "sd-utils"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"prisma-client-rust",
|
||||
"rspc",
|
||||
"sd-prisma",
|
||||
"thiserror",
|
||||
"uuid",
|
||||
]
|
||||
|
||||
|
|
@ -7839,11 +7930,11 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "sqlformat"
|
||||
version = "0.2.2"
|
||||
version = "0.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6b7b278788e7be4d0d29c0f39497a0eef3fba6bbc8e70d8bf7fde46edeaa9e85"
|
||||
checksum = "ce81b7bd7c4493975347ef60d8c7e8b742d4694f4c49f93e0a12ea263938176c"
|
||||
dependencies = [
|
||||
"itertools 0.11.0",
|
||||
"itertools 0.12.0",
|
||||
"nom",
|
||||
"unicode_categories",
|
||||
]
|
||||
|
|
@ -8967,9 +9058,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "try-lock"
|
||||
version = "0.2.4"
|
||||
version = "0.2.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed"
|
||||
checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
|
||||
|
||||
[[package]]
|
||||
name = "ttf-parser"
|
||||
|
|
@ -9057,9 +9148,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "unicode-bidi"
|
||||
version = "0.3.13"
|
||||
version = "0.3.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460"
|
||||
checksum = "6f2528f27a9eb2b21e69c95319b30bd0efd85d09c379741b0f78ea1d86be2416"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-bidi-mirroring"
|
||||
|
|
@ -9165,6 +9256,21 @@ version = "0.9.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
|
||||
|
||||
[[package]]
|
||||
name = "ureq"
|
||||
version = "2.9.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8cdd25c339e200129fe4de81451814e5228c9b771d57378817d6117cc2b3f97"
|
||||
dependencies = [
|
||||
"base64 0.21.5",
|
||||
"log",
|
||||
"once_cell",
|
||||
"rustls",
|
||||
"rustls-webpki",
|
||||
"url",
|
||||
"webpki-roots",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "url"
|
||||
version = "2.5.0"
|
||||
|
|
@ -9554,6 +9660,12 @@ dependencies = [
|
|||
"libwebp-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "webpki-roots"
|
||||
version = "0.25.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1778a42e8b3b90bff8d0f5032bf22250792889a5cdc752aa0020c84abe3aaf10"
|
||||
|
||||
[[package]]
|
||||
name = "webview2-com"
|
||||
version = "0.19.1"
|
||||
|
|
@ -10018,9 +10130,9 @@ checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04"
|
|||
|
||||
[[package]]
|
||||
name = "winnow"
|
||||
version = "0.5.19"
|
||||
version = "0.5.26"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "829846f3e3db426d4cee4510841b71a8e58aa2a76b1132579487ae430ccd9c7b"
|
||||
checksum = "b67b5f0a4e7a27a64c651977932b9dc5667ca7fc31ac44b03ed37a0cf42fdfff"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
|
|
|||
39
Cargo.toml
39
Cargo.toml
|
|
@ -21,6 +21,7 @@ edition = "2021"
|
|||
repository = "https://github.com/spacedriveapp/spacedrive"
|
||||
|
||||
[workspace.dependencies]
|
||||
# First party dependencies
|
||||
prisma-client-rust = { git = "https://github.com/spacedriveapp/prisma-client-rust", rev = "9f8ac122e8f2b2e4957b71f48a37e06565adba40", features = [
|
||||
"rspc",
|
||||
"sqlite-create-many",
|
||||
|
|
@ -47,10 +48,40 @@ tauri-specta = { version = "=2.0.0-rc.4" }
|
|||
|
||||
swift-rs = { version = "1.0.6" }
|
||||
|
||||
tokio = { version = "1.34.0" }
|
||||
uuid = { version = "1.5.0", features = ["v4", "serde"] }
|
||||
serde = { version = "1.0" }
|
||||
serde_json = { version = "1.0" }
|
||||
|
||||
# Third party dependencies used by one or more of our crates
|
||||
anyhow = "1.0.75"
|
||||
async-channel = "2.0.0"
|
||||
axum = "0.6.20"
|
||||
base64 = "0.21.5"
|
||||
blake3 = "1.5.0"
|
||||
chrono = "0.4.31"
|
||||
clap = "4.4.7"
|
||||
futures = "0.3.29"
|
||||
futures-concurrency = "7.4.3"
|
||||
hex = "0.4.3"
|
||||
http = "0.2.9"
|
||||
image = "0.24.7"
|
||||
normpath = "1.1.1"
|
||||
once_cell = "1.18.0"
|
||||
pin-project-lite = "0.2.13"
|
||||
rand = "0.8.5"
|
||||
rand_chacha = "0.3.1"
|
||||
regex = "1.10.2"
|
||||
reqwest = "0.11.22"
|
||||
rmp-serde = "1.1.2"
|
||||
serde = "1.0"
|
||||
serde_json = "1.0"
|
||||
strum = "0.25"
|
||||
strum_macros = "0.25"
|
||||
tempfile = "3.8.1"
|
||||
thiserror = "1.0.50"
|
||||
tokio = "1.34.0"
|
||||
tokio-stream = "0.1.14"
|
||||
tokio-util = "0.7.10"
|
||||
uhlc = "=0.5.2"
|
||||
uuid = "1.5.0"
|
||||
webp = "0.2.6"
|
||||
|
||||
[patch.crates-io]
|
||||
# Proper IOS Support
|
||||
|
|
|
|||
|
|
@ -6,9 +6,11 @@ repository = { workspace = true }
|
|||
edition = { workspace = true }
|
||||
|
||||
[dependencies]
|
||||
indoc = "2.0.4"
|
||||
clap = { version = "4.4.7", features = ["derive"] }
|
||||
anyhow = "1.0.75"
|
||||
hex = "0.4.3"
|
||||
sd-crypto = { path = "../../crates/crypto" }
|
||||
|
||||
anyhow = { workspace = true }
|
||||
clap = { workspace = true, features = ["derive"] }
|
||||
hex = { workspace = true }
|
||||
tokio = { workspace = true, features = ["io-util", "rt-multi-thread"] }
|
||||
|
||||
indoc = "2.0.4"
|
||||
|
|
|
|||
|
|
@ -6,8 +6,8 @@ repository = { workspace = true }
|
|||
edition = { workspace = true }
|
||||
|
||||
[dependencies]
|
||||
libc = "0.2"
|
||||
tokio = { workspace = true, features = ["fs"] }
|
||||
libc = "0.2"
|
||||
|
||||
[target.'cfg(target_os = "linux")'.dependencies]
|
||||
# WARNING: gtk should follow the same version used by tauri
|
||||
|
|
|
|||
|
|
@ -6,8 +6,9 @@ repository = { workspace = true }
|
|||
edition = { workspace = true }
|
||||
|
||||
[dependencies]
|
||||
thiserror = "1.0.50"
|
||||
normpath = "1.1.1"
|
||||
normpath = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
|
||||
libc = "0.2"
|
||||
|
||||
[target.'cfg(target_os = "windows")'.dependencies.windows]
|
||||
|
|
|
|||
|
|
@ -9,6 +9,28 @@ repository = { workspace = true }
|
|||
edition = { workspace = true }
|
||||
|
||||
[dependencies]
|
||||
sd-core = { path = "../../../core", features = [
|
||||
"ffmpeg",
|
||||
"location-watcher",
|
||||
"heif",
|
||||
] }
|
||||
sd-fda = { path = "../../../crates/fda" }
|
||||
sd-prisma = { path = "../../../crates/prisma" }
|
||||
|
||||
axum = { workspace = true, features = ["headers", "query"] }
|
||||
futures = { workspace = true }
|
||||
http = { workspace = true }
|
||||
prisma-client-rust = { workspace = true }
|
||||
rand = { workspace = true }
|
||||
rspc = { workspace = true, features = ["tauri"] }
|
||||
serde = { workspace = true }
|
||||
specta = { workspace = true }
|
||||
tokio = { workspace = true, features = ["sync"] }
|
||||
tracing = { workspace = true }
|
||||
tauri-specta = { workspace = true, features = ["typescript"] }
|
||||
uuid = { workspace = true, features = ["serde"] }
|
||||
|
||||
opener = { version = "0.6.1", features = ["reveal"] }
|
||||
tauri = { version = "1.5.3", features = [
|
||||
"macos-private-api",
|
||||
"path-all",
|
||||
|
|
@ -22,30 +44,9 @@ tauri = { version = "1.5.3", features = [
|
|||
"native-tls-vendored",
|
||||
"tracing",
|
||||
] }
|
||||
|
||||
rspc = { workspace = true, features = ["tauri"] }
|
||||
sd-core = { path = "../../../core", features = [
|
||||
"ffmpeg",
|
||||
"location-watcher",
|
||||
"heif",
|
||||
] }
|
||||
sd-fda = { path = "../../../crates/fda" }
|
||||
tokio = { workspace = true, features = ["sync"] }
|
||||
tracing = { workspace = true }
|
||||
serde = "1.0.190"
|
||||
http = "0.2.9"
|
||||
opener = { version = "0.6.1", features = ["reveal"] }
|
||||
specta = { workspace = true }
|
||||
tauri-specta = { workspace = true, features = ["typescript"] }
|
||||
uuid = { version = "1.5.0", features = ["serde"] }
|
||||
futures = "0.3"
|
||||
axum = { version = "0.6.20", features = ["headers", "query"] }
|
||||
rand = "0.8.5"
|
||||
|
||||
prisma-client-rust = { workspace = true }
|
||||
sd-prisma = { path = "../../../crates/prisma" }
|
||||
tauri-plugin-window-state = "0.1.0"
|
||||
|
||||
|
||||
[target.'cfg(target_os = "linux")'.dependencies]
|
||||
sd-desktop-linux = { path = "../crates/linux" }
|
||||
webkit2gtk = { version = "0.18.2", features = ["v2_2"] }
|
||||
|
|
@ -61,5 +62,6 @@ webview2-com = "0.19.1"
|
|||
tauri-build = "1.5.0"
|
||||
|
||||
[features]
|
||||
default = ["custom-protocol"]
|
||||
default = ["ai-models", "custom-protocol"]
|
||||
ai-models = ["sd-core/ai"]
|
||||
custom-protocol = ["tauri/custom-protocol"]
|
||||
|
|
|
|||
|
|
@ -1,3 +1,12 @@
|
|||
fn main() {
|
||||
#[cfg(all(not(target_os = "windows"), feature = "ai-models"))]
|
||||
// This is required because libonnxruntime.so is incorrectly built with the Initial Executable (IE) thread-Local storage access model by zig
|
||||
// https://docs.oracle.com/cd/E23824_01/html/819-0690/chapter8-20.html
|
||||
// https://github.com/ziglang/zig/issues/16152
|
||||
// https://github.com/ziglang/zig/pull/17702
|
||||
// Due to this, the linker will fail to dlopen libonnxruntime.so because it runs out of the static TLS space reserved after initial load
|
||||
// To workaround this problem libonnxruntime.so is added as a dependency to the binaries, which makes the linker allocate its TLS space during initial load
|
||||
println!("cargo:rustc-link-lib=onnxruntime");
|
||||
|
||||
tauri_build::build();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,3 +1,6 @@
|
|||
use sd_core::Node;
|
||||
use sd_prisma::prisma::{file_path, location};
|
||||
|
||||
use std::{
|
||||
collections::{BTreeSet, HashMap, HashSet},
|
||||
hash::{Hash, Hasher},
|
||||
|
|
@ -6,10 +9,6 @@ use std::{
|
|||
};
|
||||
|
||||
use futures::future::join_all;
|
||||
use sd_core::{
|
||||
prisma::{file_path, location},
|
||||
Node,
|
||||
};
|
||||
use serde::Serialize;
|
||||
use specta::Type;
|
||||
use tauri::async_runtime::spawn_blocking;
|
||||
|
|
|
|||
|
|
@ -11,9 +11,6 @@
|
|||
"tauri": {
|
||||
"macOSPrivateApi": true,
|
||||
"bundle": {
|
||||
"appimage": {
|
||||
"bundleMediaFramework": true
|
||||
},
|
||||
"active": true,
|
||||
"targets": ["deb", "msi", "dmg", "updater"],
|
||||
"identifier": "com.spacedrive.desktop",
|
||||
|
|
@ -24,27 +21,32 @@
|
|||
"icons/icon.icns",
|
||||
"icons/icon.ico"
|
||||
],
|
||||
"resources": [],
|
||||
"resources": {},
|
||||
"externalBin": [],
|
||||
"copyright": "Spacedrive Technology Inc.",
|
||||
"shortDescription": "File explorer from the future.",
|
||||
"longDescription": "Cross-platform universal file explorer, powered by an open-source virtual distributed filesystem.",
|
||||
"deb": {
|
||||
"files": {
|
||||
"/usr/share/models/yolov8s.onnx": "../../.deps/models/yolov8s.onnx"
|
||||
},
|
||||
"depends": ["libc6"]
|
||||
},
|
||||
"macOS": {
|
||||
"frameworks": ["../../.deps/Spacedrive.framework"],
|
||||
"minimumSystemVersion": "10.15",
|
||||
"exceptionDomain": "",
|
||||
"entitlements": null
|
||||
"exceptionDomain": null,
|
||||
"entitlements": null,
|
||||
"frameworks": ["../../.deps/Spacedrive.framework"]
|
||||
},
|
||||
"windows": {
|
||||
"certificateThumbprint": null,
|
||||
"webviewInstallMode": { "type": "embedBootstrapper", "silent": true },
|
||||
"digestAlgorithm": "sha256",
|
||||
"timestampUrl": "",
|
||||
"wix": {
|
||||
"bannerPath": "icons/WindowsBanner.bmp",
|
||||
"dialogImagePath": "icons/WindowsDialogImage.bmp"
|
||||
"enableElevatedUpdateTask": true,
|
||||
"dialogImagePath": "icons/WindowsDialogImage.bmp",
|
||||
"bannerPath": "icons/WindowsBanner.bmp"
|
||||
}
|
||||
}
|
||||
},
|
||||
|
|
|
|||
|
|
@ -7,14 +7,16 @@ repository = { workspace = true }
|
|||
edition = { workspace = true }
|
||||
|
||||
[dependencies]
|
||||
once_cell = "1.18.0"
|
||||
sd-core = { path = "../../../../../core", features = [
|
||||
"mobile",
|
||||
], default-features = false }
|
||||
|
||||
futures = { workspace = true }
|
||||
once_cell = { workspace = true }
|
||||
rspc = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
tokio = { workspace = true }
|
||||
futures = "0.3.29"
|
||||
tracing = { workspace = true }
|
||||
|
||||
futures-channel = "0.3.29"
|
||||
futures-locks = "0.7.1"
|
||||
|
|
|
|||
|
|
@ -23,7 +23,6 @@ import {
|
|||
createCache,
|
||||
initPlausible,
|
||||
LibraryContextProvider,
|
||||
NotificationContextProvider,
|
||||
P2PContextProvider,
|
||||
RspcProvider,
|
||||
useBridgeQuery,
|
||||
|
|
@ -138,9 +137,7 @@ function AppContainer() {
|
|||
<ClientContextProvider currentLibraryId={id}>
|
||||
<P2PContextProvider>
|
||||
<P2P />
|
||||
<NotificationContextProvider>
|
||||
<AppNavigation />
|
||||
</NotificationContextProvider>
|
||||
<AppNavigation />
|
||||
</P2PContextProvider>
|
||||
</ClientContextProvider>
|
||||
</BottomSheetModalProvider>
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import { useQueryClient } from '@tanstack/react-query';
|
|||
import { forwardRef, useEffect, useState } from 'react';
|
||||
import { Pressable, Text, View } from 'react-native';
|
||||
import ColorPicker from 'react-native-wheel-color-picker';
|
||||
import { useLibraryMutation, usePlausibleEvent } from '@sd/client';
|
||||
import { ToastDefautlColor, useLibraryMutation, usePlausibleEvent } from '@sd/client';
|
||||
import { FadeInAnimation } from '~/components/animation/layout';
|
||||
import { ModalInput } from '~/components/form/Input';
|
||||
import { Modal, ModalRef } from '~/components/layout/Modal';
|
||||
|
|
@ -16,7 +16,7 @@ const CreateTagModal = forwardRef<ModalRef, unknown>((_, ref) => {
|
|||
const modalRef = useForwardedRef(ref);
|
||||
|
||||
const [tagName, setTagName] = useState('');
|
||||
const [tagColor, setTagColor] = useState('#A717D9');
|
||||
const [tagColor, setTagColor] = useState(ToastDefautlColor);
|
||||
const [showPicker, setShowPicker] = useState(false);
|
||||
|
||||
// TODO: Use react-hook-form?
|
||||
|
|
@ -30,7 +30,7 @@ const CreateTagModal = forwardRef<ModalRef, unknown>((_, ref) => {
|
|||
onSuccess: () => {
|
||||
// Reset form
|
||||
setTagName('');
|
||||
setTagColor('#A717D9');
|
||||
setTagColor(ToastDefautlColor);
|
||||
setShowPicker(false);
|
||||
|
||||
queryClient.invalidateQueries(['tags.list']);
|
||||
|
|
@ -61,7 +61,7 @@ const CreateTagModal = forwardRef<ModalRef, unknown>((_, ref) => {
|
|||
onDismiss={() => {
|
||||
// Resets form onDismiss
|
||||
setTagName('');
|
||||
setTagColor('#A717D9');
|
||||
setTagColor(ToastDefautlColor);
|
||||
setShowPicker(false);
|
||||
}}
|
||||
showCloseButton
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ edition = { workspace = true }
|
|||
|
||||
[features]
|
||||
assets = []
|
||||
ai-models = ["sd-core/ai"]
|
||||
|
||||
[dependencies]
|
||||
sd-core = { path = "../../core", features = [
|
||||
|
|
@ -14,10 +15,12 @@ sd-core = { path = "../../core", features = [
|
|||
"location-watcher",
|
||||
"heif",
|
||||
] }
|
||||
|
||||
axum = { workspace = true }
|
||||
http = { workspace = true }
|
||||
rspc = { workspace = true, features = ["axum"] }
|
||||
axum = "0.6.20"
|
||||
tokio = { workspace = true, features = ["sync", "rt-multi-thread", "signal"] }
|
||||
tracing = { workspace = true }
|
||||
http = "0.2.9"
|
||||
|
||||
include_dir = "0.7.3"
|
||||
mime_guess = "2.0.4"
|
||||
|
|
|
|||
10
apps/server/build.rs
Normal file
10
apps/server/build.rs
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
fn main() {
|
||||
#[cfg(all(not(target_os = "windows"), feature = "ai-models"))]
|
||||
// This is required because libonnxruntime.so is incorrectly built with the Initial Executable (IE) thread-Local storage access model by zig
|
||||
// https://docs.oracle.com/cd/E23824_01/html/819-0690/chapter8-20.html
|
||||
// https://github.com/ziglang/zig/issues/16152
|
||||
// https://github.com/ziglang/zig/pull/17702
|
||||
// Due to this, the linker will fail to dlopen libonnxruntime.so because it runs out of the static TLS space reserved after initial load
|
||||
// To workaround this problem libonnxruntime.so is added as a dependency to the binaries, which makes the linker allocate its TLS space during initial load
|
||||
println!("cargo:rustc-link-lib=onnxruntime");
|
||||
}
|
||||
114
core/Cargo.toml
114
core/Cargo.toml
|
|
@ -16,31 +16,52 @@ mobile = []
|
|||
ffmpeg = ["dep:sd-ffmpeg"]
|
||||
location-watcher = ["dep:notify"]
|
||||
heif = ["sd-images/heif"]
|
||||
ai = ["dep:sd-ai"]
|
||||
|
||||
[dependencies]
|
||||
sd-media-metadata = { path = "../crates/media-metadata" }
|
||||
sd-prisma = { path = "../crates/prisma" }
|
||||
sd-ffmpeg = { path = "../crates/ffmpeg", optional = true }
|
||||
# Sub-crates
|
||||
sd-cache = { path = "../crates/cache" }
|
||||
sd-core-sync = { path = "./crates/sync" }
|
||||
# sd-cloud-api = { path = "../crates/cloud-api" }
|
||||
sd-crypto = { path = "../crates/crypto", features = [
|
||||
"rspc",
|
||||
"specta",
|
||||
"serde",
|
||||
"keymanager",
|
||||
] }
|
||||
sd-cache = { path = "../crates/cache" }
|
||||
sd-file-path-helper = { path = "../crates/file-path-helper" }
|
||||
sd-ffmpeg = { path = "../crates/ffmpeg", optional = true }
|
||||
sd-file-ext = { path = "../crates/file-ext" }
|
||||
sd-images = { path = "../crates/images", features = [
|
||||
"rspc",
|
||||
"serde",
|
||||
"specta",
|
||||
] }
|
||||
sd-file-ext = { path = "../crates/file-ext" }
|
||||
sd-sync = { path = "../crates/sync" }
|
||||
sd-media-metadata = { path = "../crates/media-metadata" }
|
||||
sd-p2p = { path = "../crates/p2p", features = ["specta", "serde"] }
|
||||
sd-prisma = { path = "../crates/prisma" }
|
||||
sd-ai = { path = "../crates/ai", optional = true }
|
||||
sd-sync = { path = "../crates/sync" }
|
||||
sd-utils = { path = "../crates/utils" }
|
||||
# sd-cloud-api = { path = "../crates/cloud-api" }
|
||||
sd-cloud-api = { version = "0.1.0", path = "../crates/cloud-api" }
|
||||
|
||||
sd-core-sync = { path = "./crates/sync" }
|
||||
|
||||
# Workspace dependencies
|
||||
async-channel = { workspace = true }
|
||||
axum = { workspace = true }
|
||||
base64 = { workspace = true }
|
||||
blake3 = { workspace = true }
|
||||
chrono = { workspace = true, features = ["serde"] }
|
||||
futures = { workspace = true }
|
||||
futures-concurrency = { workspace = true }
|
||||
image = { workspace = true }
|
||||
normpath = { workspace = true, features = ["localization"] }
|
||||
once_cell = { workspace = true }
|
||||
pin-project-lite = { workspace = true }
|
||||
prisma-client-rust = { workspace = true }
|
||||
regex = { workspace = true }
|
||||
reqwest = { workspace = true, features = ["json", "native-tls-vendored"] }
|
||||
rmp-serde = { workspace = true }
|
||||
rspc = { workspace = true, features = [
|
||||
"uuid",
|
||||
"chrono",
|
||||
|
|
@ -48,8 +69,13 @@ rspc = { workspace = true, features = [
|
|||
"alpha",
|
||||
"unstable",
|
||||
] }
|
||||
prisma-client-rust = { workspace = true }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
serde_json = { workspace = true }
|
||||
specta = { workspace = true }
|
||||
strum = { workspace = true, features = ["derive"] }
|
||||
strum_macros = { workspace = true }
|
||||
tempfile = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
tokio = { workspace = true, features = [
|
||||
"sync",
|
||||
"rt-multi-thread",
|
||||
|
|
@ -58,74 +84,54 @@ tokio = { workspace = true, features = [
|
|||
"time",
|
||||
"process",
|
||||
] }
|
||||
serde = { version = "1.0", features = ["derive"] }
|
||||
chrono = { version = "0.4.31", features = ["serde"] }
|
||||
serde_json = { workspace = true }
|
||||
serde_repr = "0.1"
|
||||
futures = "0.3"
|
||||
rmp-serde = "^1.1.2"
|
||||
rmpv = "^1.0.1"
|
||||
blake3 = "1.5.0"
|
||||
hostname = "0.3.1"
|
||||
uuid = { workspace = true }
|
||||
sysinfo = "0.29.10"
|
||||
thiserror = "1.0.50"
|
||||
async-trait = "^0.1.74"
|
||||
image = "0.24.7"
|
||||
webp = "0.2.6"
|
||||
tokio-stream = { workspace = true, features = ["fs"] }
|
||||
tokio-util = { workspace = true, features = ["io"] }
|
||||
tracing = { workspace = true }
|
||||
tracing-subscriber = { workspace = true, features = ["env-filter"] }
|
||||
tracing-appender = { workspace = true }
|
||||
tracing-subscriber = { workspace = true, features = ["env-filter"] }
|
||||
uuid = { workspace = true, features = ["v4", "serde"] }
|
||||
webp = { workspace = true }
|
||||
|
||||
|
||||
# Specific Core dependencies
|
||||
async-recursion = "1.0.5"
|
||||
async-stream = "0.3.5"
|
||||
once_cell = "1.18.0"
|
||||
async-trait = "^0.1.74"
|
||||
bytes = "1.5.0"
|
||||
ctor = "0.2.5"
|
||||
directories = "5.0.1"
|
||||
flate2 = "1.0.28"
|
||||
globset = { version = "^0.4.13", features = ["serde1"] }
|
||||
itertools = "^0.11.0"
|
||||
hostname = "0.3.1"
|
||||
http-body = "0.4.5"
|
||||
http-range = "0.1.5"
|
||||
int-enum = "0.5.0"
|
||||
itertools = "0.12.0"
|
||||
mini-moka = "0.10.2"
|
||||
serde_with = "3.4.0"
|
||||
notify = { version = "=5.2.0", default-features = false, features = [
|
||||
"macos_fsevent",
|
||||
], optional = true }
|
||||
static_assertions = "1.1.0"
|
||||
rmpv = "^1.0.1"
|
||||
serde-hashkey = "0.4.5"
|
||||
normpath = { version = "1.1.1", features = ["localization"] }
|
||||
strum = { version = "0.25", features = ["derive"] }
|
||||
strum_macros = "0.25"
|
||||
regex = "1.10.2"
|
||||
int-enum = "0.5.0"
|
||||
tokio-stream = { version = "0.1.14", features = ["fs"] }
|
||||
futures-concurrency = "7.4.3"
|
||||
async-channel = "2.0.0"
|
||||
tokio-util = { version = "0.7.10", features = ["io"] }
|
||||
serde_repr = "0.1"
|
||||
serde_with = "3.4.0"
|
||||
slotmap = "1.0.6"
|
||||
flate2 = "1.0.28"
|
||||
static_assertions = "1.1.0"
|
||||
sysinfo = "0.29.10"
|
||||
tar = "0.4.40"
|
||||
tempfile = "^3.8.1"
|
||||
axum = "0.6.20"
|
||||
http-body = "0.4.5"
|
||||
pin-project-lite = "0.2.13"
|
||||
bytes = "1.5.0"
|
||||
reqwest = { version = "0.11.22", features = ["json", "native-tls-vendored"] }
|
||||
directories = "5.0.1"
|
||||
async-recursion = "1.0.5"
|
||||
base64 = "0.21.5"
|
||||
sd-cloud-api = { version = "0.1.0", path = "../crates/cloud-api" }
|
||||
|
||||
# Override features of transitive dependencies
|
||||
[dependencies.openssl]
|
||||
version = "=0.10.57"
|
||||
version = "=0.10.61"
|
||||
features = ["vendored"]
|
||||
[dependencies.openssl-sys]
|
||||
version = "=0.9.93"
|
||||
version = "=0.9.97"
|
||||
features = ["vendored"]
|
||||
|
||||
# Platform-specific dependencies
|
||||
[target.'cfg(target_os = "macos")'.dependencies]
|
||||
plist = "1"
|
||||
|
||||
[target.'cfg(windows)'.dependencies.winapi-util]
|
||||
version = "0.1.6"
|
||||
|
||||
[dev-dependencies]
|
||||
tracing-test = "^0.2.4"
|
||||
aovec = "1.1.0"
|
||||
|
|
|
|||
|
|
@ -17,4 +17,4 @@ serde_json = { workspace = true }
|
|||
tokio = { workspace = true }
|
||||
uuid = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
uhlc = "=0.5.2"
|
||||
uhlc = { workspace = true }
|
||||
|
|
|
|||
|
|
@ -1,12 +1,14 @@
|
|||
use crate::{
|
||||
db_operation::*, ingest, relation_op_db, shared_op_db, SharedState, SyncMessage, NTP64,
|
||||
};
|
||||
|
||||
use sd_prisma::prisma::{
|
||||
cloud_relation_operation, cloud_shared_operation, instance, relation_operation,
|
||||
shared_operation, PrismaClient, SortOrder,
|
||||
};
|
||||
use sd_sync::{CRDTOperation, CRDTOperationType, OperationFactory};
|
||||
use sd_utils::uuid_to_bytes;
|
||||
|
||||
use std::{
|
||||
cmp::Ordering,
|
||||
collections::HashMap,
|
||||
|
|
@ -16,6 +18,7 @@ use std::{
|
|||
Arc,
|
||||
},
|
||||
};
|
||||
|
||||
use tokio::sync::{broadcast, RwLock};
|
||||
use uhlc::{HLCBuilder, HLC};
|
||||
use uuid::Uuid;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,39 @@
|
|||
/*
|
||||
Warnings:
|
||||
|
||||
- You are about to drop the column `redundancy_goal` on the `tag` table. All the data in the column will be lost.
|
||||
- Made the column `name` on table `label` required. This step will fail if there are existing NULL values in that column.
|
||||
|
||||
*/
|
||||
-- AlterTable
|
||||
ALTER TABLE "saved_search" ADD COLUMN "search" TEXT;
|
||||
|
||||
-- RedefineTables
|
||||
PRAGMA foreign_keys=OFF;
|
||||
CREATE TABLE "new_label" (
|
||||
"id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
|
||||
"pub_id" BLOB NOT NULL,
|
||||
"name" TEXT NOT NULL,
|
||||
"date_created" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
"date_modified" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
INSERT INTO "new_label" ("date_created", "date_modified", "id", "name", "pub_id") SELECT "date_created", "date_modified", "id", "name", "pub_id" FROM "label";
|
||||
DROP TABLE "label";
|
||||
ALTER TABLE "new_label" RENAME TO "label";
|
||||
CREATE UNIQUE INDEX "label_pub_id_key" ON "label"("pub_id");
|
||||
CREATE UNIQUE INDEX "label_name_key" ON "label"("name");
|
||||
CREATE TABLE "new_tag" (
|
||||
"id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
|
||||
"pub_id" BLOB NOT NULL,
|
||||
"name" TEXT,
|
||||
"color" TEXT,
|
||||
"is_hidden" BOOLEAN,
|
||||
"date_created" DATETIME,
|
||||
"date_modified" DATETIME
|
||||
);
|
||||
INSERT INTO "new_tag" ("color", "date_created", "date_modified", "id", "name", "pub_id") SELECT "color", "date_created", "date_modified", "id", "name", "pub_id" FROM "tag";
|
||||
DROP TABLE "tag";
|
||||
ALTER TABLE "new_tag" RENAME TO "tag";
|
||||
CREATE UNIQUE INDEX "tag_pub_id_key" ON "tag"("pub_id");
|
||||
PRAGMA foreign_key_check;
|
||||
PRAGMA foreign_keys=ON;
|
||||
|
|
@ -337,8 +337,7 @@ model Tag {
|
|||
name String?
|
||||
color String?
|
||||
|
||||
// Enum: ??
|
||||
redundancy_goal Int?
|
||||
is_hidden Boolean? // user hidden entire tag
|
||||
|
||||
date_created DateTime?
|
||||
date_modified DateTime?
|
||||
|
|
@ -367,7 +366,7 @@ model TagOnObject {
|
|||
model Label {
|
||||
id Int @id @default(autoincrement())
|
||||
pub_id Bytes @unique
|
||||
name String?
|
||||
name String @unique
|
||||
date_created DateTime @default(now())
|
||||
date_modified DateTime @default(now())
|
||||
|
||||
|
|
@ -379,11 +378,11 @@ model Label {
|
|||
model LabelOnObject {
|
||||
date_created DateTime @default(now())
|
||||
|
||||
label_id Int
|
||||
label Label @relation(fields: [label_id], references: [id], onDelete: Restrict)
|
||||
label_id Int
|
||||
label Label @relation(fields: [label_id], references: [id], onDelete: Restrict)
|
||||
|
||||
object_id Int
|
||||
object Object @relation(fields: [object_id], references: [id], onDelete: Restrict)
|
||||
object_id Int
|
||||
object Object @relation(fields: [object_id], references: [id], onDelete: Restrict)
|
||||
|
||||
@@id([label_id, object_id])
|
||||
@@map("label_on_object")
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
use crate::util::http::ensure_response;
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use reqwest::{Response, StatusCode};
|
||||
|
|
@ -5,8 +7,6 @@ use rspc::alpha::AlphaRouter;
|
|||
use serde::{de::DeserializeOwned, Deserialize, Serialize};
|
||||
use specta::Type;
|
||||
|
||||
use crate::util::http::ensure_response;
|
||||
|
||||
use super::{Ctx, R};
|
||||
|
||||
async fn parse_json_body<T: DeserializeOwned>(response: Response) -> Result<T, rspc::Error> {
|
||||
|
|
|
|||
|
|
@ -1,3 +1,11 @@
|
|||
use crate::{
|
||||
invalidate_query,
|
||||
library::{Library, LibraryManagerError},
|
||||
Node,
|
||||
};
|
||||
|
||||
use sd_utils::error::FileIOError;
|
||||
|
||||
use std::{
|
||||
cmp,
|
||||
path::{Path, PathBuf},
|
||||
|
|
@ -25,13 +33,6 @@ use tokio::{
|
|||
use tracing::{error, info};
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::{
|
||||
invalidate_query,
|
||||
library::{Library, LibraryManagerError},
|
||||
util::error::FileIOError,
|
||||
Node,
|
||||
};
|
||||
|
||||
use super::{utils::library, Ctx, R};
|
||||
|
||||
pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
||||
|
|
|
|||
|
|
@ -1,13 +1,29 @@
|
|||
use super::{utils::library, Ctx, R};
|
||||
use crate::{api::libraries::LibraryConfigWrapped, invalidate_query, library::LibraryName};
|
||||
|
||||
use reqwest::Response;
|
||||
use rspc::alpha::AlphaRouter;
|
||||
use serde::de::DeserializeOwned;
|
||||
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::{utils::library, Ctx, R};
|
||||
|
||||
#[allow(unused)]
|
||||
async fn parse_json_body<T: DeserializeOwned>(response: Response) -> Result<T, rspc::Error> {
|
||||
response.json().await.map_err(|_| {
|
||||
rspc::Error::new(
|
||||
rspc::ErrorCode::InternalServerError,
|
||||
"JSON conversion failed".to_string(),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
||||
R.router().merge("library.", library::mount())
|
||||
}
|
||||
|
||||
mod library {
|
||||
|
||||
use super::*;
|
||||
|
||||
pub fn mount() -> AlphaRouter<Ctx> {
|
||||
|
|
|
|||
|
|
@ -2,18 +2,18 @@ use crate::{
|
|||
api::utils::library,
|
||||
invalidate_query,
|
||||
library::Library,
|
||||
location::file_path_helper::IsolatedFilePathData,
|
||||
object::{
|
||||
fs::{error::FileSystemJobsError, find_available_filename_for_duplicate},
|
||||
media::media_data_extractor::{
|
||||
can_extract_media_data_for_image, extract_media_data, MediaDataError,
|
||||
},
|
||||
},
|
||||
util::error::FileIOError,
|
||||
};
|
||||
|
||||
use sd_file_ext::extensions::ImageExtension;
|
||||
use sd_file_path_helper::IsolatedFilePathData;
|
||||
use sd_media_metadata::MediaMetadata;
|
||||
use sd_utils::error::FileIOError;
|
||||
|
||||
use std::{ffi::OsStr, path::PathBuf, str::FromStr};
|
||||
|
||||
|
|
|
|||
|
|
@ -3,12 +3,7 @@ use crate::{
|
|||
invalidate_query,
|
||||
job::Job,
|
||||
library::Library,
|
||||
location::{
|
||||
file_path_helper::{
|
||||
file_path_to_isolate, file_path_to_isolate_with_id, FilePathError, IsolatedFilePathData,
|
||||
},
|
||||
get_location_path_from_location_id, LocationError,
|
||||
},
|
||||
location::{get_location_path_from_location_id, LocationError},
|
||||
object::{
|
||||
fs::{
|
||||
copy::FileCopierJobInit, cut::FileCutterJobInit, delete::FileDeleterJobInit,
|
||||
|
|
@ -17,14 +12,17 @@ use crate::{
|
|||
},
|
||||
media::media_data_image_from_prisma_data,
|
||||
},
|
||||
prisma::{file_path, location, object},
|
||||
util::{db::maybe_missing, error::FileIOError},
|
||||
};
|
||||
|
||||
use sd_cache::{CacheNode, Model, NormalisedResult, Reference};
|
||||
use sd_file_ext::kind::ObjectKind;
|
||||
use sd_file_path_helper::{
|
||||
file_path_to_isolate, file_path_to_isolate_with_id, FilePathError, IsolatedFilePathData,
|
||||
};
|
||||
use sd_images::ConvertableExtension;
|
||||
use sd_media_metadata::MediaMetadata;
|
||||
use sd_prisma::prisma::{file_path, location, object};
|
||||
use sd_utils::{db::maybe_missing, error::FileIOError};
|
||||
|
||||
use std::{
|
||||
ffi::OsString,
|
||||
|
|
|
|||
|
|
@ -6,9 +6,10 @@ use crate::{
|
|||
file_identifier::file_identifier_job::FileIdentifierJobInit, media::MediaProcessorJobInit,
|
||||
validation::validator_job::ObjectValidatorJobInit,
|
||||
},
|
||||
prisma::{job, location, SortOrder},
|
||||
};
|
||||
|
||||
use sd_prisma::prisma::{job, location, SortOrder};
|
||||
|
||||
use std::{
|
||||
collections::{hash_map::Entry, BTreeMap, HashMap, VecDeque},
|
||||
path::PathBuf,
|
||||
|
|
@ -246,6 +247,39 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
|||
location,
|
||||
sub_path: Some(path),
|
||||
regenerate_thumbnails: regenerate,
|
||||
regenerate_labels: false,
|
||||
})
|
||||
.spawn(&node, &library)
|
||||
.await
|
||||
.map_err(Into::into)
|
||||
},
|
||||
)
|
||||
})
|
||||
.procedure("generateLabelsForLocation", {
|
||||
#[derive(Type, Deserialize)]
|
||||
pub struct GenerateLabelsForLocationArgs {
|
||||
pub id: location::id::Type,
|
||||
pub path: PathBuf,
|
||||
#[serde(default)]
|
||||
pub regenerate: bool,
|
||||
}
|
||||
|
||||
R.with2(library()).mutation(
|
||||
|(node, library),
|
||||
GenerateLabelsForLocationArgs {
|
||||
id,
|
||||
path,
|
||||
regenerate,
|
||||
}: GenerateLabelsForLocationArgs| async move {
|
||||
let Some(location) = find_location(&library, id).exec().await? else {
|
||||
return Err(LocationError::IdNotFound(id).into());
|
||||
};
|
||||
|
||||
Job::new(MediaProcessorJobInit {
|
||||
location,
|
||||
sub_path: Some(path),
|
||||
regenerate_thumbnails: false,
|
||||
regenerate_labels: regenerate,
|
||||
})
|
||||
.spawn(&node, &library)
|
||||
.await
|
||||
|
|
|
|||
85
core/src/api/labels.rs
Normal file
85
core/src/api/labels.rs
Normal file
|
|
@ -0,0 +1,85 @@
|
|||
use crate::{invalidate_query, library::Library};
|
||||
|
||||
use sd_prisma::prisma::{label, label_on_object, object};
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use rspc::alpha::AlphaRouter;
|
||||
|
||||
use super::{utils::library, Ctx, R};
|
||||
|
||||
pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
||||
R.router()
|
||||
.procedure("list", {
|
||||
R.with2(library()).query(|(_, library), _: ()| async move {
|
||||
Ok(library.db.label().find_many(vec![]).exec().await?)
|
||||
})
|
||||
})
|
||||
.procedure("getForObject", {
|
||||
R.with2(library())
|
||||
.query(|(_, library), object_id: i32| async move {
|
||||
Ok(library
|
||||
.db
|
||||
.label()
|
||||
.find_many(vec![label::label_objects::some(vec![
|
||||
label_on_object::object_id::equals(object_id),
|
||||
])])
|
||||
.exec()
|
||||
.await?)
|
||||
})
|
||||
})
|
||||
.procedure("getWithObjects", {
|
||||
R.with2(library()).query(
|
||||
|(_, library), object_ids: Vec<object::id::Type>| async move {
|
||||
let Library { db, .. } = library.as_ref();
|
||||
let labels_with_objects = db
|
||||
.label()
|
||||
.find_many(vec![label::label_objects::some(vec![
|
||||
label_on_object::object_id::in_vec(object_ids.clone()),
|
||||
])])
|
||||
.select(label::select!({
|
||||
id
|
||||
label_objects(vec![label_on_object::object_id::in_vec(object_ids.clone())]): select {
|
||||
date_created
|
||||
object: select {
|
||||
id
|
||||
}
|
||||
}
|
||||
}))
|
||||
.exec()
|
||||
.await?;
|
||||
Ok(labels_with_objects
|
||||
.into_iter()
|
||||
.map(|label| (label.id, label.label_objects))
|
||||
.collect::<BTreeMap<_, _>>())
|
||||
},
|
||||
)
|
||||
})
|
||||
.procedure("get", {
|
||||
R.with2(library())
|
||||
.query(|(_, library), label_id: i32| async move {
|
||||
Ok(library
|
||||
.db
|
||||
.label()
|
||||
.find_unique(label::id::equals(label_id))
|
||||
.exec()
|
||||
.await?)
|
||||
})
|
||||
})
|
||||
.procedure(
|
||||
"delete",
|
||||
R.with2(library())
|
||||
.mutation(|(_, library), label_id: i32| async move {
|
||||
library
|
||||
.db
|
||||
.label()
|
||||
.delete(label::id::equals(label_id))
|
||||
.exec()
|
||||
.await?;
|
||||
|
||||
invalidate_query!(library, "labels.list");
|
||||
|
||||
Ok(())
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
|
@ -11,19 +11,22 @@ use crate::{
|
|||
},
|
||||
object::file_identifier::file_identifier_job::FileIdentifierJobInit,
|
||||
p2p::PeerMetadata,
|
||||
prisma::{file_path, indexer_rule, indexer_rules_in_location, location, object, SortOrder},
|
||||
util::AbortOnDrop,
|
||||
};
|
||||
|
||||
use sd_cache::{CacheNode, Model, Normalise, NormalisedResult, NormalisedResults, Reference};
|
||||
use sd_prisma::prisma::{
|
||||
file_path, indexer_rule, indexer_rules_in_location, location, object, SortOrder,
|
||||
};
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use chrono::{DateTime, FixedOffset, Utc};
|
||||
use directories::UserDirs;
|
||||
use rspc::{self, alpha::AlphaRouter, ErrorCode};
|
||||
use sd_cache::{CacheNode, Model, Normalise, NormalisedResult, NormalisedResults, Reference};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use specta::Type;
|
||||
use tracing::error;
|
||||
use tracing::{debug, error};
|
||||
|
||||
use super::{utils::library, Ctx, R};
|
||||
|
||||
|
|
@ -371,7 +374,7 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
|||
reidentify_objects,
|
||||
}| async move {
|
||||
if reidentify_objects {
|
||||
library
|
||||
let count = library
|
||||
.db
|
||||
.file_path()
|
||||
.update_many(
|
||||
|
|
@ -388,6 +391,8 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
|||
.exec()
|
||||
.await?;
|
||||
|
||||
debug!("Disconnected {count} file paths from objects");
|
||||
|
||||
library.orphan_remover.invoke().await;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,14 +1,15 @@
|
|||
use std::sync::{atomic::Ordering, Arc};
|
||||
|
||||
use crate::{
|
||||
invalidate_query,
|
||||
job::JobProgressEvent,
|
||||
node::config::{NodeConfig, NodePreferences},
|
||||
Node,
|
||||
};
|
||||
|
||||
use sd_cache::patch_typedef;
|
||||
use sd_p2p::P2PStatus;
|
||||
|
||||
use std::sync::{atomic::Ordering, Arc};
|
||||
|
||||
use itertools::Itertools;
|
||||
use rspc::{alpha::Rspc, Config, ErrorCode};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
|
@ -23,8 +24,10 @@ mod ephemeral_files;
|
|||
mod files;
|
||||
mod jobs;
|
||||
mod keys;
|
||||
mod labels;
|
||||
mod libraries;
|
||||
pub mod locations;
|
||||
mod models;
|
||||
mod nodes;
|
||||
pub mod notifications;
|
||||
mod p2p;
|
||||
|
|
@ -92,6 +95,7 @@ pub struct SanitisedNodeConfig {
|
|||
pub p2p_port: Option<u16>,
|
||||
pub features: Vec<BackendFeature>,
|
||||
pub preferences: NodePreferences,
|
||||
pub image_labeler_version: Option<String>,
|
||||
}
|
||||
|
||||
impl From<NodeConfig> for SanitisedNodeConfig {
|
||||
|
|
@ -103,6 +107,7 @@ impl From<NodeConfig> for SanitisedNodeConfig {
|
|||
p2p_port: value.p2p.port,
|
||||
features: value.features,
|
||||
preferences: value.preferences,
|
||||
image_labeler_version: value.image_labeler_version,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -194,6 +199,7 @@ pub(crate) fn mount() -> Arc<Router> {
|
|||
.merge("library.", libraries::mount())
|
||||
.merge("volumes.", volumes::mount())
|
||||
.merge("tags.", tags::mount())
|
||||
.merge("labels.", labels::mount())
|
||||
// .merge("categories.", categories::mount())
|
||||
// .merge("keys.", keys::mount())
|
||||
.merge("locations.", locations::mount())
|
||||
|
|
@ -201,6 +207,7 @@ pub(crate) fn mount() -> Arc<Router> {
|
|||
.merge("files.", files::mount())
|
||||
.merge("jobs.", jobs::mount())
|
||||
.merge("p2p.", p2p::mount())
|
||||
.merge("models.", models::mount())
|
||||
.merge("nodes.", nodes::mount())
|
||||
.merge("sync.", sync::mount())
|
||||
.merge("preferences.", preferences::mount())
|
||||
|
|
|
|||
23
core/src/api/models.rs
Normal file
23
core/src/api/models.rs
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
use rspc::alpha::AlphaRouter;
|
||||
|
||||
use super::{Ctx, R};
|
||||
|
||||
pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
||||
R.router().procedure("image_detection.list", {
|
||||
R.query(
|
||||
|_, _: ()| -> std::result::Result<Vec<&'static str>, rspc::Error> {
|
||||
#[cfg(not(feature = "ai"))]
|
||||
return Err(rspc::Error::new(
|
||||
rspc::ErrorCode::MethodNotSupported,
|
||||
"AI feature is not aviailable".to_string(),
|
||||
));
|
||||
|
||||
#[cfg(feature = "ai")]
|
||||
{
|
||||
use sd_ai::image_labeler::{Model, YoloV8};
|
||||
Ok(YoloV8::versions())
|
||||
}
|
||||
},
|
||||
)
|
||||
})
|
||||
}
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
use crate::{invalidate_query, prisma::location, util::MaybeUndefined};
|
||||
use crate::{invalidate_query, util::MaybeUndefined};
|
||||
|
||||
use sd_prisma::prisma::instance;
|
||||
use sd_prisma::prisma::{instance, location};
|
||||
|
||||
use rspc::{alpha::AlphaRouter, ErrorCode};
|
||||
use serde::Deserialize;
|
||||
|
|
@ -16,8 +16,9 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
|||
#[derive(Deserialize, Type)]
|
||||
pub struct ChangeNodeNameArgs {
|
||||
pub name: Option<String>,
|
||||
pub p2p_enabled: Option<bool>,
|
||||
pub p2p_port: MaybeUndefined<u16>,
|
||||
pub p2p_enabled: Option<bool>,
|
||||
pub image_labeler_version: Option<String>,
|
||||
}
|
||||
R.mutation(|node, args: ChangeNodeNameArgs| async move {
|
||||
if let Some(name) = &args.name {
|
||||
|
|
@ -32,6 +33,9 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
|||
let does_p2p_need_refresh =
|
||||
args.p2p_enabled.is_some() || args.p2p_port.is_defined();
|
||||
|
||||
#[cfg(feature = "ai")]
|
||||
let mut new_model = None;
|
||||
|
||||
node.config
|
||||
.write(|config| {
|
||||
if let Some(name) = args.name {
|
||||
|
|
@ -43,6 +47,28 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
|||
if let Some(v) = args.p2p_port.into() {
|
||||
config.p2p.port = v;
|
||||
}
|
||||
|
||||
#[cfg(feature = "ai")]
|
||||
if let Some(version) = args.image_labeler_version {
|
||||
if config
|
||||
.image_labeler_version
|
||||
.as_ref()
|
||||
.map(|node_version| version != *node_version)
|
||||
.unwrap_or(true)
|
||||
{
|
||||
new_model = sd_ai::image_labeler::YoloV8::model(Some(&version))
|
||||
.map_err(|e| {
|
||||
error!(
|
||||
"Failed to crate image_detection model: '{}'; Error: {e:#?}",
|
||||
&version,
|
||||
);
|
||||
})
|
||||
.ok();
|
||||
if new_model.is_some() {
|
||||
config.image_labeler_version = Some(version);
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(|err| {
|
||||
|
|
@ -63,6 +89,34 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
|||
|
||||
invalidate_query!(node; node, "nodeState");
|
||||
|
||||
#[cfg(feature = "ai")]
|
||||
{
|
||||
use super::notifications::{NotificationData, NotificationKind};
|
||||
|
||||
if let Some(model) = new_model {
|
||||
let version = model.version().to_string();
|
||||
tokio::spawn(async move {
|
||||
let notification = if let Err(e) =
|
||||
node.image_labeller.change_model(model).await
|
||||
{
|
||||
NotificationData {
|
||||
title: String::from("Failed to change image detection model"),
|
||||
content: format!("Error: {e}"),
|
||||
kind: NotificationKind::Error,
|
||||
}
|
||||
} else {
|
||||
NotificationData {
|
||||
title: String::from("Model download completed"),
|
||||
content: format!("Sucessfuly loaded model: {version}"),
|
||||
kind: NotificationKind::Success,
|
||||
}
|
||||
};
|
||||
|
||||
node.emit_notification(notification, None).await;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
})
|
||||
})
|
||||
|
|
|
|||
|
|
@ -1,14 +1,14 @@
|
|||
use sd_prisma::prisma::notification;
|
||||
|
||||
use crate::api::{Ctx, R};
|
||||
use async_stream::stream;
|
||||
use chrono::{DateTime, Utc};
|
||||
use futures::future::join_all;
|
||||
use rspc::{alpha::AlphaRouter, ErrorCode};
|
||||
use sd_prisma::prisma::notification;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use specta::Type;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::api::{Ctx, R};
|
||||
|
||||
use super::utils::library;
|
||||
|
||||
/// Represents a single notification.
|
||||
|
|
@ -27,13 +27,22 @@ pub enum NotificationId {
|
|||
Library(Uuid, u32),
|
||||
Node(u32),
|
||||
}
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, Type)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub enum NotificationKind {
|
||||
Info,
|
||||
Success,
|
||||
Error,
|
||||
Warning,
|
||||
}
|
||||
|
||||
/// Represents the data of a single notification.
|
||||
/// This data is used by the frontend to properly display the notification.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, Type)]
|
||||
pub enum NotificationData {
|
||||
PairingRequest { id: Uuid, pairing_id: u16 },
|
||||
Test,
|
||||
pub struct NotificationData {
|
||||
pub title: String,
|
||||
pub content: String,
|
||||
pub kind: NotificationKind,
|
||||
}
|
||||
|
||||
pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
||||
|
|
@ -159,21 +168,4 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
|||
}
|
||||
})
|
||||
})
|
||||
.procedure("test", {
|
||||
R.mutation(|node, _: ()| async move {
|
||||
node.emit_notification(NotificationData::Test, None).await;
|
||||
|
||||
Ok(())
|
||||
})
|
||||
})
|
||||
.procedure("testLibrary", {
|
||||
R.with2(library())
|
||||
.mutation(|(_, library), _: ()| async move {
|
||||
library
|
||||
.emit_notification(NotificationData::Test, None)
|
||||
.await;
|
||||
|
||||
Ok(())
|
||||
})
|
||||
})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,12 +1,13 @@
|
|||
use rspc::{alpha::AlphaRouter, ErrorCode};
|
||||
use crate::p2p::{operations, P2PEvent, PairingDecision};
|
||||
|
||||
use sd_p2p::spacetunnel::RemoteIdentity;
|
||||
|
||||
use rspc::{alpha::AlphaRouter, ErrorCode};
|
||||
use serde::Deserialize;
|
||||
use specta::Type;
|
||||
use std::path::PathBuf;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::p2p::{operations, P2PEvent, PairingDecision};
|
||||
|
||||
use super::{Ctx, R};
|
||||
|
||||
pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
use crate::preferences::LibraryPreferences;
|
||||
|
||||
use rspc::alpha::AlphaRouter;
|
||||
|
||||
use super::{utils::library, Ctx, R};
|
||||
use crate::preferences::LibraryPreferences;
|
||||
|
||||
pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
||||
R.router()
|
||||
|
|
|
|||
|
|
@ -1,18 +1,19 @@
|
|||
use crate::location::LocationError;
|
||||
|
||||
use sd_file_path_helper::{check_file_path_exists, IsolatedFilePathData};
|
||||
use sd_prisma::prisma::{self, file_path};
|
||||
|
||||
use chrono::{DateTime, FixedOffset, Utc};
|
||||
use prisma_client_rust::{OrderByQuery, PaginatedQuery, WhereQuery};
|
||||
use rspc::ErrorCode;
|
||||
use sd_prisma::prisma::{self, file_path};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use specta::Type;
|
||||
|
||||
use crate::location::{
|
||||
file_path_helper::{check_file_path_exists, IsolatedFilePathData},
|
||||
LocationError,
|
||||
use super::{
|
||||
object::*,
|
||||
utils::{self, *},
|
||||
};
|
||||
|
||||
use super::object::*;
|
||||
use super::utils::{self, *};
|
||||
|
||||
#[derive(Serialize, Deserialize, Type, Debug, Clone)]
|
||||
#[serde(rename_all = "camelCase", tag = "field", content = "value")]
|
||||
pub enum FilePathOrder {
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
use sd_prisma::prisma::{self, media_data};
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use specta::Type;
|
||||
|
||||
|
|
|
|||
|
|
@ -1,11 +1,3 @@
|
|||
pub mod file_path;
|
||||
pub mod media_data;
|
||||
pub mod object;
|
||||
pub mod saved;
|
||||
mod utils;
|
||||
|
||||
pub use self::{file_path::*, object::*, utils::*};
|
||||
|
||||
use crate::{
|
||||
api::{
|
||||
locations::{file_path_with_object, object_with_file_paths, ExplorerItem},
|
||||
|
|
@ -16,14 +8,23 @@ use crate::{
|
|||
object::media::thumbnail::get_indexed_thumb_key,
|
||||
};
|
||||
|
||||
use sd_cache::{CacheNode, Model, Normalise, Reference};
|
||||
use sd_prisma::prisma::{self, PrismaClient};
|
||||
|
||||
use std::path::PathBuf;
|
||||
|
||||
use rspc::{alpha::AlphaRouter, ErrorCode};
|
||||
use sd_cache::{CacheNode, Model, Normalise, Reference};
|
||||
use sd_prisma::prisma::{self, PrismaClient};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use specta::Type;
|
||||
|
||||
pub mod file_path;
|
||||
pub mod media_data;
|
||||
pub mod object;
|
||||
pub mod saved;
|
||||
mod utils;
|
||||
|
||||
pub use self::{file_path::*, object::*, utils::*};
|
||||
|
||||
use super::{Ctx, R};
|
||||
|
||||
const MAX_TAKE: u8 = 100;
|
||||
|
|
|
|||
|
|
@ -1,14 +1,16 @@
|
|||
use chrono::{DateTime, FixedOffset};
|
||||
use prisma_client_rust::not;
|
||||
use prisma_client_rust::{or, OrderByQuery, PaginatedQuery, WhereQuery};
|
||||
// use crate::library::Category;
|
||||
|
||||
use sd_prisma::prisma::{self, object, tag_on_object};
|
||||
|
||||
use chrono::{DateTime, FixedOffset};
|
||||
use prisma_client_rust::{not, or, OrderByQuery, PaginatedQuery, WhereQuery};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use specta::Type;
|
||||
|
||||
// use crate::library::Category;
|
||||
|
||||
use super::media_data::*;
|
||||
use super::utils::{self, *};
|
||||
use super::{
|
||||
media_data::*,
|
||||
utils::{self, *},
|
||||
};
|
||||
|
||||
#[derive(Deserialize, Type, Debug)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
|
|
@ -28,25 +30,25 @@ impl ObjectCursor {
|
|||
|
||||
query.add_where(or![
|
||||
match item.order {
|
||||
SortOrder::Asc => prisma::object::$field::gt(data),
|
||||
SortOrder::Desc => prisma::object::$field::lt(data),
|
||||
SortOrder::Asc => object::$field::gt(data),
|
||||
SortOrder::Desc => object::$field::lt(data),
|
||||
},
|
||||
prisma_client_rust::and![
|
||||
prisma::object::$field::equals(Some(item.data)),
|
||||
object::$field::equals(Some(item.data)),
|
||||
match item.order {
|
||||
SortOrder::Asc => prisma::object::id::gt(id),
|
||||
SortOrder::Desc => prisma::object::id::lt(id),
|
||||
SortOrder::Asc => object::id::gt(id),
|
||||
SortOrder::Desc => object::id::lt(id),
|
||||
}
|
||||
]
|
||||
]);
|
||||
|
||||
query.add_order_by(prisma::object::$field::order(item.order.into()));
|
||||
query.add_order_by(object::$field::order(item.order.into()));
|
||||
}};
|
||||
}
|
||||
|
||||
match self {
|
||||
Self::None => {
|
||||
query.add_where(prisma::object::id::gt(id));
|
||||
query.add_where(object::id::gt(id));
|
||||
}
|
||||
Self::Kind(item) => arm!(kind, item),
|
||||
Self::DateAccessed(item) => arm!(date_accessed, item),
|
||||
|
|
@ -146,7 +148,7 @@ impl ObjectFilterArgs {
|
|||
}
|
||||
|
||||
pub type OrderAndPagination =
|
||||
utils::OrderAndPagination<prisma::object::id::Type, ObjectOrder, ObjectCursor>;
|
||||
utils::OrderAndPagination<object::id::Type, ObjectOrder, ObjectCursor>;
|
||||
|
||||
impl OrderAndPagination {
|
||||
pub fn apply(self, query: &mut object::FindManyQuery) {
|
||||
|
|
@ -164,7 +166,7 @@ impl OrderAndPagination {
|
|||
Self::Cursor { id, cursor } => {
|
||||
cursor.apply(query, id);
|
||||
|
||||
query.add_order_by(prisma::object::pub_id::order(prisma::SortOrder::Asc))
|
||||
query.add_order_by(object::pub_id::order(prisma::SortOrder::Asc))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
use crate::{api::utils::library, invalidate_query, prisma::saved_search};
|
||||
use crate::{api::utils::library, invalidate_query};
|
||||
|
||||
use sd_prisma::prisma::saved_search;
|
||||
use sd_utils::chain_optional_iter;
|
||||
|
||||
use chrono::{DateTime, FixedOffset, Utc};
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
use sd_prisma::prisma;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use specta::Type;
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
use rspc::alpha::AlphaRouter;
|
||||
use sd_core_sync::GetOpsArgs;
|
||||
|
||||
use rspc::alpha::AlphaRouter;
|
||||
|
||||
use super::{utils::library, Ctx, R};
|
||||
|
||||
pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
||||
|
|
|
|||
|
|
@ -1,26 +1,24 @@
|
|||
use crate::{invalidate_query, library::Library, object::tag::TagCreateArgs};
|
||||
|
||||
use sd_cache::{CacheNode, Normalise, NormalisedResult, NormalisedResults, Reference};
|
||||
use sd_file_ext::kind::ObjectKind;
|
||||
use sd_prisma::{
|
||||
prisma::{file_path, object, tag, tag_on_object},
|
||||
prisma_sync,
|
||||
};
|
||||
use sd_sync::OperationFactory;
|
||||
use sd_utils::uuid_to_bytes;
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use itertools::{Either, Itertools};
|
||||
use rspc::{alpha::AlphaRouter, ErrorCode};
|
||||
use sd_cache::{CacheNode, Normalise, NormalisedResult, NormalisedResults, Reference};
|
||||
use sd_file_ext::kind::ObjectKind;
|
||||
use sd_prisma::{prisma, prisma_sync};
|
||||
use sd_sync::OperationFactory;
|
||||
use sd_utils::uuid_to_bytes;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use specta::Type;
|
||||
|
||||
use serde_json::json;
|
||||
use specta::Type;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::{
|
||||
invalidate_query,
|
||||
library::Library,
|
||||
object::tag::TagCreateArgs,
|
||||
prisma::{file_path, object, tag, tag_on_object},
|
||||
};
|
||||
|
||||
use super::{utils::library, Ctx, R};
|
||||
|
||||
pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
||||
|
|
@ -119,8 +117,8 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
|||
#[derive(Debug, Type, Deserialize)]
|
||||
#[specta(inline)]
|
||||
enum Target {
|
||||
Object(prisma::object::id::Type),
|
||||
FilePath(prisma::file_path::id::Type),
|
||||
Object(object::id::Type),
|
||||
FilePath(file_path::id::Type),
|
||||
}
|
||||
|
||||
#[derive(Debug, Type, Deserialize)]
|
||||
|
|
@ -276,8 +274,14 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
|||
},
|
||||
);
|
||||
|
||||
sync.write_ops(db, (sync_ops, db.tag_on_object().create_many(db_creates)))
|
||||
.await?;
|
||||
sync.write_ops(
|
||||
db,
|
||||
(
|
||||
sync_ops,
|
||||
db.tag_on_object().create_many(db_creates).skip_duplicates(),
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
|
||||
invalidate_query!(library, "tags.getForObject");
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
use rspc::alpha::AlphaRouter;
|
||||
use crate::volume::get_volumes;
|
||||
|
||||
use sd_cache::{Normalise, NormalisedResults};
|
||||
|
||||
use crate::volume::get_volumes;
|
||||
use rspc::alpha::AlphaRouter;
|
||||
|
||||
use super::{Ctx, R};
|
||||
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
use crate::util::http::ensure_response;
|
||||
|
||||
use rspc::alpha::AlphaRouter;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use specta::Type;
|
||||
|
||||
use crate::util::http::ensure_response;
|
||||
|
||||
use super::{Ctx, R};
|
||||
|
||||
pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
||||
|
|
|
|||
|
|
@ -1,9 +1,11 @@
|
|||
use crate::cloud::sync::err_return;
|
||||
|
||||
use super::Library;
|
||||
use std::sync::Arc;
|
||||
|
||||
use tokio::sync::Notify;
|
||||
|
||||
use super::Library;
|
||||
|
||||
pub async fn run_actor((library, notify): (Arc<Library>, Arc<Notify>)) {
|
||||
let Library { sync, .. } = library.as_ref();
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
use std::sync::{atomic, Arc};
|
||||
|
||||
use crate::{library::Library, Node};
|
||||
|
||||
use std::sync::{atomic, Arc};
|
||||
|
||||
mod ingest;
|
||||
mod receive;
|
||||
mod send;
|
||||
|
|
|
|||
|
|
@ -3,22 +3,25 @@ use crate::{
|
|||
library::Library,
|
||||
Node,
|
||||
};
|
||||
use base64::prelude::*;
|
||||
use chrono::Utc;
|
||||
use itertools::{Either, Itertools};
|
||||
|
||||
use sd_core_sync::NTP64;
|
||||
use sd_prisma::prisma::{
|
||||
cloud_relation_operation, cloud_shared_operation, instance, PrismaClient, SortOrder,
|
||||
};
|
||||
use sd_sync::*;
|
||||
use sd_utils::{from_bytes_to_uuid, uuid_to_bytes};
|
||||
use serde::Deserialize;
|
||||
use serde_json::{json, to_vec};
|
||||
|
||||
use std::{
|
||||
collections::{hash_map::Entry, HashMap},
|
||||
sync::Arc,
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use base64::prelude::*;
|
||||
use chrono::Utc;
|
||||
use itertools::{Either, Itertools};
|
||||
use serde::Deserialize;
|
||||
use serde_json::{json, to_vec};
|
||||
use tokio::{sync::Notify, time::sleep};
|
||||
use uuid::Uuid;
|
||||
|
||||
|
|
|
|||
|
|
@ -1,14 +1,18 @@
|
|||
use super::Library;
|
||||
use crate::{cloud::sync::err_break, Node};
|
||||
|
||||
use sd_core_sync::{GetOpsArgs, SyncMessage, NTP64};
|
||||
use sd_prisma::prisma::instance;
|
||||
use sd_utils::from_bytes_to_uuid;
|
||||
|
||||
use std::{sync::Arc, time::Duration};
|
||||
|
||||
use serde::Deserialize;
|
||||
use serde_json::json;
|
||||
use std::{sync::Arc, time::Duration};
|
||||
use tokio::time::sleep;
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::Library;
|
||||
|
||||
pub async fn run_actor((library, node): (Arc<Library>, Arc<Node>)) {
|
||||
let db = &library.db;
|
||||
let api_url = &library.env.api_url;
|
||||
|
|
|
|||
|
|
@ -1,20 +1,23 @@
|
|||
use crate::{
|
||||
api::{utils::InvalidateOperationEvent, CoreEvent},
|
||||
library::Library,
|
||||
location::file_path_helper::{file_path_to_handle_custom_uri, IsolatedFilePathData},
|
||||
object::media::thumbnail::WEBP_EXTENSION,
|
||||
p2p::{operations, IdentityOrRemoteIdentity},
|
||||
prisma::{file_path, location},
|
||||
util::{db::*, InfallibleResponse},
|
||||
util::InfallibleResponse,
|
||||
Node,
|
||||
};
|
||||
|
||||
use sd_file_ext::text::is_text;
|
||||
use sd_file_path_helper::{file_path_to_handle_custom_uri, IsolatedFilePathData};
|
||||
use sd_p2p::{spaceblock::Range, spacetunnel::RemoteIdentity};
|
||||
use sd_prisma::prisma::{file_path, location};
|
||||
use sd_utils::db::maybe_missing;
|
||||
|
||||
use std::{
|
||||
cmp::min,
|
||||
ffi::OsStr,
|
||||
fmt::Debug,
|
||||
fs::Metadata,
|
||||
io::{self, SeekFrom},
|
||||
path::{Path, PathBuf},
|
||||
str::FromStr,
|
||||
sync::{atomic::Ordering, Arc},
|
||||
|
|
@ -30,13 +33,10 @@ use axum::{
|
|||
Router,
|
||||
};
|
||||
use bytes::Bytes;
|
||||
|
||||
use mini_moka::sync::Cache;
|
||||
use sd_file_ext::text::is_text;
|
||||
use sd_p2p::{spaceblock::Range, spacetunnel::RemoteIdentity};
|
||||
use tokio::{
|
||||
fs::{self, File},
|
||||
io::{AsyncReadExt, AsyncSeekExt},
|
||||
io::{self, AsyncReadExt, AsyncSeekExt, SeekFrom},
|
||||
};
|
||||
use tokio_util::sync::PollSender;
|
||||
use tracing::error;
|
||||
|
|
|
|||
|
|
@ -1,11 +1,10 @@
|
|||
use std::{
|
||||
io,
|
||||
pin::Pin,
|
||||
task::{Context, Poll},
|
||||
};
|
||||
|
||||
use bytes::Bytes;
|
||||
use tokio::io::AsyncWrite;
|
||||
use tokio::io::{self, AsyncWrite};
|
||||
use tokio_util::sync::PollSender;
|
||||
|
||||
/// Allowing wrapping an `mpsc::Sender` into an `AsyncWrite`
|
||||
|
|
|
|||
|
|
@ -1,17 +1,16 @@
|
|||
use crate::util::InfallibleResponse;
|
||||
|
||||
use std::{
|
||||
fs::Metadata,
|
||||
io::{self, SeekFrom},
|
||||
time::UNIX_EPOCH,
|
||||
};
|
||||
use std::{fs::Metadata, time::UNIX_EPOCH};
|
||||
|
||||
use axum::{
|
||||
body::{self, BoxBody, Full, StreamBody},
|
||||
http::{header, request, HeaderValue, Method, Response, StatusCode},
|
||||
};
|
||||
use http_range::HttpRange;
|
||||
use tokio::{fs::File, io::AsyncSeekExt};
|
||||
use tokio::{
|
||||
fs::File,
|
||||
io::{self, AsyncSeekExt, SeekFrom},
|
||||
};
|
||||
use tokio_util::io::ReaderStream;
|
||||
use tracing::error;
|
||||
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
use crate::util::InfallibleResponse;
|
||||
|
||||
use std::{fmt::Debug, panic::Location};
|
||||
|
||||
use axum::{
|
||||
|
|
@ -8,8 +10,6 @@ use axum::{
|
|||
use http_body::Full;
|
||||
use tracing::debug;
|
||||
|
||||
use crate::util::InfallibleResponse;
|
||||
|
||||
#[track_caller]
|
||||
pub(crate) fn bad_request(err: impl Debug) -> http::Response<BoxBody> {
|
||||
debug!("400: Bad Request at {}: {err:?}", Location::caller());
|
||||
|
|
|
|||
|
|
@ -4,10 +4,10 @@ use crate::{
|
|||
file_identifier::FileIdentifierJobError, fs::error::FileSystemJobsError,
|
||||
media::media_processor::MediaProcessorError, validation::ValidatorError,
|
||||
},
|
||||
util::{db::MissingFieldError, error::FileIOError},
|
||||
};
|
||||
|
||||
use sd_crypto::Error as CryptoError;
|
||||
use sd_utils::{db::MissingFieldError, error::FileIOError};
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
|
|
|
|||
|
|
@ -11,10 +11,11 @@ use crate::{
|
|||
media::media_processor::MediaProcessorJobInit,
|
||||
validation::validator_job::ObjectValidatorJobInit,
|
||||
},
|
||||
prisma::job,
|
||||
Node,
|
||||
};
|
||||
|
||||
use sd_prisma::prisma::job;
|
||||
|
||||
use std::{
|
||||
collections::{HashMap, HashSet, VecDeque},
|
||||
sync::Arc,
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
use crate::{library::Library, Node};
|
||||
|
||||
use sd_prisma::prisma::location;
|
||||
|
||||
use std::{
|
||||
collections::{hash_map::DefaultHasher, VecDeque},
|
||||
fmt,
|
||||
|
|
@ -10,8 +12,6 @@ use std::{
|
|||
time::Instant,
|
||||
};
|
||||
|
||||
use sd_prisma::prisma::location;
|
||||
|
||||
use async_channel as chan;
|
||||
use futures::stream::{self, StreamExt};
|
||||
use futures_concurrency::stream::Merge;
|
||||
|
|
|
|||
|
|
@ -1,8 +1,7 @@
|
|||
use crate::{
|
||||
library::Library,
|
||||
prisma::job,
|
||||
util::db::{maybe_missing, MissingFieldError},
|
||||
};
|
||||
use crate::library::Library;
|
||||
|
||||
use sd_prisma::prisma::job;
|
||||
use sd_utils::db::{maybe_missing, MissingFieldError};
|
||||
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
|
|
|
|||
|
|
@ -6,12 +6,14 @@ use crate::{
|
|||
object::media::thumbnail::actor::Thumbnailer,
|
||||
};
|
||||
|
||||
#[cfg(feature = "ai")]
|
||||
use sd_ai::image_labeler::{DownloadModelError, ImageLabeler, YoloV8};
|
||||
|
||||
use api::notifications::{Notification, NotificationData, NotificationId};
|
||||
use chrono::{DateTime, Utc};
|
||||
use node::config;
|
||||
use notifications::Notifications;
|
||||
use reqwest::{RequestBuilder, Response};
|
||||
pub use sd_prisma::*;
|
||||
|
||||
use std::{
|
||||
fmt,
|
||||
|
|
@ -64,6 +66,8 @@ pub struct Node {
|
|||
pub cloud_sync_flag: Arc<AtomicBool>,
|
||||
pub env: Arc<env::Env>,
|
||||
pub http: reqwest::Client,
|
||||
#[cfg(feature = "ai")]
|
||||
pub image_labeller: ImageLabeler,
|
||||
}
|
||||
|
||||
impl fmt::Debug for Node {
|
||||
|
|
@ -96,6 +100,11 @@ impl Node {
|
|||
.await
|
||||
.map_err(NodeError::FailedToInitializeConfig)?;
|
||||
|
||||
#[cfg(feature = "ai")]
|
||||
sd_ai::init()?;
|
||||
#[cfg(feature = "ai")]
|
||||
let image_labeler_version = config.get().await.image_labeler_version;
|
||||
|
||||
let (locations, locations_actor) = location::Locations::new();
|
||||
let (jobs, jobs_actor) = job::Jobs::new();
|
||||
let libraries = library::Libraries::new(data_dir.join("libraries")).await?;
|
||||
|
|
@ -107,7 +116,7 @@ impl Node {
|
|||
notifications: notifications::Notifications::new(),
|
||||
p2p,
|
||||
thumbnailer: Thumbnailer::new(
|
||||
data_dir.to_path_buf(),
|
||||
data_dir,
|
||||
libraries.clone(),
|
||||
event_bus.0.clone(),
|
||||
config.preferences_watcher(),
|
||||
|
|
@ -120,6 +129,10 @@ impl Node {
|
|||
cloud_sync_flag: Arc::new(AtomicBool::new(false)),
|
||||
http: reqwest::Client::new(),
|
||||
env,
|
||||
#[cfg(feature = "ai")]
|
||||
image_labeller: ImageLabeler::new(YoloV8::model(image_labeler_version)?, data_dir)
|
||||
.await
|
||||
.map_err(sd_ai::Error::from)?,
|
||||
});
|
||||
|
||||
// Restore backend feature flags
|
||||
|
|
@ -165,7 +178,7 @@ impl Node {
|
|||
|
||||
std::env::set_var(
|
||||
"RUST_LOG",
|
||||
format!("info,sd_core={level},sd_core::location::manager=info"),
|
||||
format!("info,sd_core={level},sd_core::location::manager=info,sd_ai={level}"),
|
||||
);
|
||||
}
|
||||
|
||||
|
|
@ -207,6 +220,8 @@ impl Node {
|
|||
self.thumbnailer.shutdown().await;
|
||||
self.jobs.shutdown().await;
|
||||
self.p2p.shutdown().await;
|
||||
#[cfg(feature = "ai")]
|
||||
self.image_labeller.shutdown().await;
|
||||
info!("Spacedrive Core shutdown successful!");
|
||||
}
|
||||
|
||||
|
|
@ -300,4 +315,10 @@ pub enum NodeError {
|
|||
InitConfig(#[from] util::debug_initializer::InitConfigError),
|
||||
#[error("logger error: {0}")]
|
||||
Logger(#[from] FromEnvError),
|
||||
#[cfg(feature = "ai")]
|
||||
#[error("ai error: {0}")]
|
||||
AI(#[from] sd_ai::Error),
|
||||
#[cfg(feature = "ai")]
|
||||
#[error("Failed to download model: {0}")]
|
||||
DownloadModel(#[from] DownloadModelError),
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
use crate::prisma::object;
|
||||
use prisma_client_rust::not;
|
||||
use sd_file_ext::kind::ObjectKind;
|
||||
use sd_prisma::prisma::object;
|
||||
|
||||
use prisma_client_rust::not;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use specta::Type;
|
||||
use std::vec;
|
||||
|
|
|
|||
|
|
@ -1,16 +1,12 @@
|
|||
use crate::{
|
||||
node::{config::NodeConfig, Platform},
|
||||
p2p::IdentityOrRemoteIdentity,
|
||||
prisma::{file_path, indexer_rule, PrismaClient},
|
||||
util::{
|
||||
db::maybe_missing,
|
||||
error::FileIOError,
|
||||
version_manager::{Kind, ManagedVersion, VersionManager, VersionManagerError},
|
||||
},
|
||||
util::version_manager::{Kind, ManagedVersion, VersionManager, VersionManagerError},
|
||||
};
|
||||
|
||||
use sd_p2p::spacetunnel::Identity;
|
||||
use sd_prisma::prisma::{instance, location, node};
|
||||
use sd_prisma::prisma::{file_path, indexer_rule, instance, location, node, PrismaClient};
|
||||
use sd_utils::{db::maybe_missing, error::FileIOError};
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
|
|
|
|||
|
|
@ -1,19 +1,13 @@
|
|||
use crate::{
|
||||
api::{
|
||||
notifications::{Notification, NotificationData, NotificationId},
|
||||
CoreEvent,
|
||||
},
|
||||
location::file_path_helper::{file_path_to_full_path, IsolatedFilePathData},
|
||||
notifications,
|
||||
api::CoreEvent,
|
||||
object::{media::thumbnail::get_indexed_thumbnail_path, orphan_remover::OrphanRemoverActor},
|
||||
prisma::{file_path, location, PrismaClient},
|
||||
sync,
|
||||
util::{db::maybe_missing, error::FileIOError},
|
||||
Node,
|
||||
sync, Node,
|
||||
};
|
||||
|
||||
use sd_file_path_helper::{file_path_to_full_path, IsolatedFilePathData};
|
||||
use sd_p2p::spacetunnel::Identity;
|
||||
use sd_prisma::prisma::notification;
|
||||
use sd_prisma::prisma::{file_path, location, PrismaClient};
|
||||
use sd_utils::{db::maybe_missing, error::FileIOError};
|
||||
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
|
|
@ -22,7 +16,6 @@ use std::{
|
|||
sync::Arc,
|
||||
};
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use tokio::{fs, io, sync::broadcast, sync::RwLock};
|
||||
use tracing::warn;
|
||||
use uuid::Uuid;
|
||||
|
|
@ -54,7 +47,6 @@ pub struct Library {
|
|||
// The UUID which matches `config.instance_id`'s primary key.
|
||||
pub instance_uuid: Uuid,
|
||||
|
||||
notifications: notifications::Notifications,
|
||||
pub env: Arc<crate::env::Env>,
|
||||
|
||||
// Look, I think this shouldn't be here but our current invalidation system needs it.
|
||||
|
|
@ -95,7 +87,6 @@ impl Library {
|
|||
// key_manager,
|
||||
identity,
|
||||
orphan_remover: OrphanRemoverActor::spawn(db),
|
||||
notifications: node.notifications.clone(),
|
||||
instance_uuid,
|
||||
env: node.env.clone(),
|
||||
event_bus_tx: node.event_bus.0.clone(),
|
||||
|
|
@ -180,45 +171,4 @@ impl Library {
|
|||
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Create a new notification which will be stored into the DB and emitted to the UI.
|
||||
pub async fn emit_notification(&self, data: NotificationData, expires: Option<DateTime<Utc>>) {
|
||||
let result = match self
|
||||
.db
|
||||
.notification()
|
||||
.create(
|
||||
match rmp_serde::to_vec(&data).map_err(|err| err.to_string()) {
|
||||
Ok(data) => data,
|
||||
Err(err) => {
|
||||
warn!(
|
||||
"Failed to serialize notification data for library '{}': {}",
|
||||
self.id, err
|
||||
);
|
||||
return;
|
||||
}
|
||||
},
|
||||
expires
|
||||
.map(|e| vec![notification::expires_at::set(Some(e.fixed_offset()))])
|
||||
.unwrap_or_default(),
|
||||
)
|
||||
.exec()
|
||||
.await
|
||||
{
|
||||
Ok(result) => result,
|
||||
Err(err) => {
|
||||
warn!(
|
||||
"Failed to create notification in library '{}': {}",
|
||||
self.id, err
|
||||
);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
self.notifications._internal_send(Notification {
|
||||
id: NotificationId::Library(self.id, result.id as u32),
|
||||
data,
|
||||
read: false,
|
||||
expires,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,10 +2,11 @@ use crate::{
|
|||
library::LibraryConfigError,
|
||||
location::{indexer, LocationManagerError},
|
||||
p2p::IdentityOrRemoteIdentityErr,
|
||||
util::{
|
||||
db::{self, MissingFieldError},
|
||||
error::{FileIOError, NonUtf8PathError},
|
||||
},
|
||||
};
|
||||
|
||||
use sd_utils::{
|
||||
db::{self, MissingFieldError},
|
||||
error::{FileIOError, NonUtf8PathError},
|
||||
};
|
||||
|
||||
use thiserror::Error;
|
||||
|
|
|
|||
|
|
@ -8,21 +8,19 @@ use crate::{
|
|||
node::Platform,
|
||||
object::tag,
|
||||
p2p::{self, IdentityOrRemoteIdentity},
|
||||
prisma::location,
|
||||
sync,
|
||||
util::{
|
||||
db,
|
||||
error::{FileIOError, NonUtf8PathError},
|
||||
mpscrr, MaybeUndefined,
|
||||
},
|
||||
volume::watcher::spawn_volume_watcher,
|
||||
util::{mpscrr, MaybeUndefined},
|
||||
Node,
|
||||
};
|
||||
|
||||
use sd_core_sync::SyncMessage;
|
||||
use sd_p2p::spacetunnel::Identity;
|
||||
use sd_prisma::prisma::{instance, shared_operation};
|
||||
use sd_utils::from_bytes_to_uuid;
|
||||
use sd_prisma::prisma::{instance, location, shared_operation};
|
||||
use sd_utils::{
|
||||
db,
|
||||
error::{FileIOError, NonUtf8PathError},
|
||||
from_bytes_to_uuid,
|
||||
};
|
||||
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
|
|
@ -131,11 +129,17 @@ impl Libraries {
|
|||
Err(e) => return Err(FileIOError::from((db_path, e)).into()),
|
||||
}
|
||||
|
||||
let library_arc = self
|
||||
let _library_arc = self
|
||||
.load(library_id, &db_path, config_path, None, true, node)
|
||||
.await?;
|
||||
|
||||
spawn_volume_watcher(library_arc.clone());
|
||||
// This is compleaty breaking on linux now, no ideia why, but it will be irrelevant in a short while
|
||||
// So let's leave it disable for now
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
{
|
||||
use crate::volume::watcher::spawn_volume_watcher;
|
||||
spawn_volume_watcher(_library_arc.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,10 +1,9 @@
|
|||
use std::ops::Deref;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use specta::Type;
|
||||
use thiserror::Error;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Debug, Serialize, Clone, Type)]
|
||||
pub struct LibraryName(String);
|
||||
|
||||
|
|
|
|||
|
|
@ -1,9 +1,8 @@
|
|||
use crate::{
|
||||
prisma::location,
|
||||
util::{
|
||||
db::MissingFieldError,
|
||||
error::{FileIOError, NonUtf8PathError},
|
||||
},
|
||||
use sd_file_path_helper::FilePathError;
|
||||
use sd_prisma::prisma::location;
|
||||
use sd_utils::{
|
||||
db::MissingFieldError,
|
||||
error::{FileIOError, NonUtf8PathError},
|
||||
};
|
||||
|
||||
use std::path::Path;
|
||||
|
|
@ -12,9 +11,7 @@ use rspc::{self, ErrorCode};
|
|||
use thiserror::Error;
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::{
|
||||
file_path_helper::FilePathError, manager::LocationManagerError, metadata::LocationMetadataError,
|
||||
};
|
||||
use super::{manager::LocationManagerError, metadata::LocationMetadataError};
|
||||
|
||||
/// Error type for location related errors
|
||||
#[derive(Error, Debug)]
|
||||
|
|
|
|||
|
|
@ -5,21 +5,20 @@ use crate::{
|
|||
JobStepOutput, StatefulJob, WorkerContext,
|
||||
},
|
||||
library::Library,
|
||||
location::{
|
||||
file_path_helper::{
|
||||
ensure_file_path_exists, ensure_sub_path_is_directory, ensure_sub_path_is_in_location,
|
||||
IsolatedFilePathData,
|
||||
},
|
||||
location_with_indexer_rules, update_location_size,
|
||||
},
|
||||
prisma::{file_path, location},
|
||||
location::{location_with_indexer_rules, update_location_size},
|
||||
to_remove_db_fetcher_fn,
|
||||
util::db::maybe_missing,
|
||||
};
|
||||
|
||||
use sd_prisma::prisma_sync;
|
||||
use sd_file_path_helper::{
|
||||
ensure_file_path_exists, ensure_sub_path_is_directory, ensure_sub_path_is_in_location,
|
||||
IsolatedFilePathData,
|
||||
};
|
||||
use sd_prisma::{
|
||||
prisma::{file_path, location},
|
||||
prisma_sync,
|
||||
};
|
||||
use sd_sync::*;
|
||||
use sd_utils::from_bytes_to_uuid;
|
||||
use sd_utils::{db::maybe_missing, from_bytes_to_uuid};
|
||||
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
|
|
|
|||
|
|
@ -1,14 +1,14 @@
|
|||
use crate::{
|
||||
library::Library,
|
||||
util::{db::inode_to_db, error::FileIOError},
|
||||
};
|
||||
use crate::library::Library;
|
||||
|
||||
use sd_file_path_helper::{
|
||||
file_path_pub_and_cas_ids, FilePathError, IsolatedFilePathData, IsolatedFilePathDataParts,
|
||||
};
|
||||
use sd_prisma::{
|
||||
prisma::{file_path, location, object as prisma_object, PrismaClient},
|
||||
prisma_sync,
|
||||
};
|
||||
use sd_sync::*;
|
||||
use sd_utils::from_bytes_to_uuid;
|
||||
use sd_utils::{db::inode_to_db, error::FileIOError, from_bytes_to_uuid};
|
||||
|
||||
use std::{collections::HashMap, path::Path};
|
||||
|
||||
|
|
@ -22,10 +22,7 @@ use serde_json::json;
|
|||
use thiserror::Error;
|
||||
use tracing::{trace, warn};
|
||||
|
||||
use super::{
|
||||
file_path_helper::{file_path_pub_and_cas_ids, FilePathError, IsolatedFilePathData},
|
||||
location_with_indexer_rules,
|
||||
};
|
||||
use super::location_with_indexer_rules;
|
||||
|
||||
pub mod indexer_job;
|
||||
pub mod rules;
|
||||
|
|
@ -97,13 +94,13 @@ async fn execute_indexer_save_step(
|
|||
.walked
|
||||
.iter()
|
||||
.map(|entry| {
|
||||
let IsolatedFilePathData {
|
||||
let IsolatedFilePathDataParts {
|
||||
materialized_path,
|
||||
is_dir,
|
||||
name,
|
||||
extension,
|
||||
..
|
||||
} = &entry.iso_file_path;
|
||||
} = &entry.iso_file_path.to_parts();
|
||||
|
||||
use file_path::*;
|
||||
|
||||
|
|
@ -197,7 +194,7 @@ async fn execute_indexer_update_step(
|
|||
.to_update
|
||||
.iter()
|
||||
.map(|entry| async move {
|
||||
let IsolatedFilePathData { is_dir, .. } = &entry.iso_file_path;
|
||||
let IsolatedFilePathDataParts { is_dir, .. } = &entry.iso_file_path.to_parts();
|
||||
|
||||
let pub_id = sd_utils::uuid_to_bytes(entry.pub_id);
|
||||
|
||||
|
|
@ -338,7 +335,7 @@ macro_rules! file_paths_db_fetcher_fn {
|
|||
.find_many(vec![::prisma_client_rust::operator::or(
|
||||
founds.collect::<Vec<_>>(),
|
||||
)])
|
||||
.select($crate::location::file_path_helper::file_path_walker::select())
|
||||
.select(::sd_file_path_helper::file_path_walker::select())
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
|
|
@ -358,13 +355,13 @@ macro_rules! file_paths_db_fetcher_fn {
|
|||
macro_rules! to_remove_db_fetcher_fn {
|
||||
($location_id:expr, $db:expr) => {{
|
||||
|parent_iso_file_path, unique_location_id_materialized_path_name_extension_params| async {
|
||||
let location_id: $crate::prisma::location::id::Type = $location_id;
|
||||
let db: &$crate::prisma::PrismaClient = $db;
|
||||
let parent_iso_file_path: $crate::location::file_path_helper::IsolatedFilePathData<
|
||||
let location_id: ::sd_prisma::prisma::location::id::Type = $location_id;
|
||||
let db: &::sd_prisma::prisma::PrismaClient = $db;
|
||||
let parent_iso_file_path: ::sd_file_path_helper::IsolatedFilePathData<
|
||||
'static,
|
||||
> = parent_iso_file_path;
|
||||
let unique_location_id_materialized_path_name_extension_params: ::std::vec::Vec<
|
||||
$crate::prisma::file_path::WhereParam,
|
||||
::sd_prisma::prisma::file_path::WhereParam,
|
||||
> = unique_location_id_materialized_path_name_extension_params;
|
||||
|
||||
// FIXME: Can't pass this chunks variable direct to _batch because of lifetime issues
|
||||
|
|
@ -377,7 +374,7 @@ macro_rules! to_remove_db_fetcher_fn {
|
|||
.find_many(vec![::prisma_client_rust::operator::or(
|
||||
unique_params.collect(),
|
||||
)])
|
||||
.select($crate::prisma::file_path::select!({ id }))
|
||||
.select(::sd_prisma::prisma::file_path::select!({ id }))
|
||||
})
|
||||
.collect::<::std::vec::Vec<_>>();
|
||||
|
||||
|
|
@ -398,17 +395,17 @@ macro_rules! to_remove_db_fetcher_fn {
|
|||
loop {
|
||||
let found = $db.file_path()
|
||||
.find_many(vec![
|
||||
$crate::prisma::file_path::location_id::equals(Some(location_id)),
|
||||
$crate::prisma::file_path::materialized_path::equals(Some(
|
||||
::sd_prisma::prisma::file_path::location_id::equals(Some(location_id)),
|
||||
::sd_prisma::prisma::file_path::materialized_path::equals(Some(
|
||||
parent_iso_file_path
|
||||
.materialized_path_for_children()
|
||||
.expect("the received isolated file path must be from a directory"),
|
||||
)),
|
||||
])
|
||||
.order_by($crate::prisma::file_path::id::order($crate::prisma::SortOrder::Asc))
|
||||
.order_by(::sd_prisma::prisma::file_path::id::order(::sd_prisma::prisma::SortOrder::Asc))
|
||||
.take(BATCH_SIZE)
|
||||
.cursor($crate::prisma::file_path::id::equals(cursor))
|
||||
.select($crate::prisma::file_path::select!({ id pub_id cas_id }))
|
||||
.cursor(::sd_prisma::prisma::file_path::id::equals(cursor))
|
||||
.select(::sd_prisma::prisma::file_path::select!({ id pub_id cas_id }))
|
||||
.exec()
|
||||
.await?;
|
||||
|
||||
|
|
@ -424,7 +421,7 @@ macro_rules! to_remove_db_fetcher_fn {
|
|||
found
|
||||
.into_iter()
|
||||
.filter(|file_path| !founds_ids.contains(&file_path.id))
|
||||
.map(|file_path| $crate::location::file_path_helper::file_path_pub_and_cas_ids::Data {
|
||||
.map(|file_path| ::sd_file_path_helper::file_path_pub_and_cas_ids::Data {
|
||||
pub_id: file_path.pub_id,
|
||||
cas_id: file_path.cas_id,
|
||||
}),
|
||||
|
|
|
|||
|
|
@ -1,12 +1,9 @@
|
|||
pub mod seed;
|
||||
use crate::library::Library;
|
||||
|
||||
use crate::{
|
||||
library::Library,
|
||||
prisma::indexer_rule,
|
||||
util::{
|
||||
db::{maybe_missing, MissingFieldError},
|
||||
error::{FileIOError, NonUtf8PathError},
|
||||
},
|
||||
use sd_prisma::prisma::indexer_rule;
|
||||
use sd_utils::{
|
||||
db::{maybe_missing, MissingFieldError},
|
||||
error::{FileIOError, NonUtf8PathError},
|
||||
};
|
||||
|
||||
use std::{
|
||||
|
|
@ -27,6 +24,8 @@ use tokio::fs;
|
|||
use tracing::debug;
|
||||
use uuid::Uuid;
|
||||
|
||||
pub mod seed;
|
||||
|
||||
#[derive(Error, Debug)]
|
||||
pub enum IndexerRuleError {
|
||||
// User errors
|
||||
|
|
|
|||
|
|
@ -2,8 +2,10 @@ use crate::{
|
|||
library::Library,
|
||||
location::indexer::rules::{IndexerRule, IndexerRuleError, RulePerKind},
|
||||
};
|
||||
use chrono::Utc;
|
||||
|
||||
use sd_prisma::prisma::indexer_rule;
|
||||
|
||||
use chrono::Utc;
|
||||
use thiserror::Error;
|
||||
use uuid::Uuid;
|
||||
|
||||
|
|
|
|||
|
|
@ -3,20 +3,20 @@ use crate::{
|
|||
job::JobError,
|
||||
library::Library,
|
||||
location::{
|
||||
file_path_helper::{
|
||||
check_file_path_exists, ensure_sub_path_is_directory, ensure_sub_path_is_in_location,
|
||||
IsolatedFilePathData,
|
||||
},
|
||||
indexer::{
|
||||
execute_indexer_update_step, reverse_update_directories_sizes, IndexerJobUpdateStep,
|
||||
},
|
||||
scan_location_sub_path, update_location_size,
|
||||
},
|
||||
to_remove_db_fetcher_fn,
|
||||
util::db::maybe_missing,
|
||||
Node,
|
||||
to_remove_db_fetcher_fn, Node,
|
||||
};
|
||||
|
||||
use sd_file_path_helper::{
|
||||
check_file_path_exists, ensure_sub_path_is_directory, ensure_sub_path_is_in_location,
|
||||
IsolatedFilePathData,
|
||||
};
|
||||
use sd_utils::db::maybe_missing;
|
||||
|
||||
use std::{
|
||||
collections::HashSet,
|
||||
path::{Path, PathBuf},
|
||||
|
|
|
|||
|
|
@ -1,10 +1,8 @@
|
|||
use crate::{
|
||||
location::file_path_helper::{
|
||||
file_path_pub_and_cas_ids, file_path_walker, FilePathMetadata, IsolatedFilePathData,
|
||||
},
|
||||
prisma::file_path,
|
||||
util::{db::inode_from_db, error::FileIOError},
|
||||
use sd_file_path_helper::{
|
||||
file_path_pub_and_cas_ids, file_path_walker, FilePathMetadata, IsolatedFilePathData,
|
||||
};
|
||||
use sd_prisma::prisma::file_path;
|
||||
use sd_utils::{db::inode_from_db, error::FileIOError};
|
||||
|
||||
use std::{
|
||||
collections::{HashMap, HashSet, VecDeque},
|
||||
|
|
@ -388,7 +386,7 @@ where
|
|||
// We ignore the size of directories because it is not reliable, we need to
|
||||
// calculate it ourselves later
|
||||
&& !(
|
||||
entry.iso_file_path.is_dir
|
||||
entry.iso_file_path.to_parts().is_dir
|
||||
&& metadata.size_in_bytes
|
||||
!= file_path
|
||||
.size_in_bytes_bytes
|
||||
|
|
|
|||
|
|
@ -1,10 +1,11 @@
|
|||
use crate::{
|
||||
library::{Library, LibraryId},
|
||||
prisma::location,
|
||||
util::db::maybe_missing,
|
||||
Node,
|
||||
};
|
||||
|
||||
use sd_prisma::prisma::location;
|
||||
use sd_utils::db::maybe_missing;
|
||||
|
||||
use std::{
|
||||
collections::{HashMap, HashSet},
|
||||
path::{Path, PathBuf},
|
||||
|
|
|
|||
|
|
@ -1,11 +1,13 @@
|
|||
use crate::{
|
||||
job::JobManagerError,
|
||||
library::{Library, LibraryManagerEvent},
|
||||
prisma::location,
|
||||
util::{db::MissingFieldError, error::FileIOError},
|
||||
Node,
|
||||
};
|
||||
|
||||
use sd_file_path_helper::FilePathError;
|
||||
use sd_prisma::prisma::location;
|
||||
use sd_utils::{db::MissingFieldError, error::FileIOError};
|
||||
|
||||
use std::{
|
||||
collections::BTreeSet,
|
||||
path::{Path, PathBuf},
|
||||
|
|
@ -24,8 +26,6 @@ use tracing::error;
|
|||
use tokio::sync::mpsc;
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::file_path_helper::FilePathError;
|
||||
|
||||
#[cfg(feature = "location-watcher")]
|
||||
mod watcher;
|
||||
|
||||
|
|
|
|||
|
|
@ -6,10 +6,10 @@
|
|||
//! Aside from that, when a directory is moved to our watched location from the outside, we receive
|
||||
//! a Create Dir event, this one is actually ok at least.
|
||||
|
||||
use crate::{
|
||||
invalidate_query, library::Library, location::manager::LocationManagerError, prisma::location,
|
||||
util::error::FileIOError, Node,
|
||||
};
|
||||
use crate::{invalidate_query, library::Library, location::manager::LocationManagerError, Node};
|
||||
|
||||
use sd_prisma::prisma::location;
|
||||
use sd_utils::error::FileIOError;
|
||||
|
||||
use std::{
|
||||
collections::{BTreeMap, HashMap},
|
||||
|
|
|
|||
|
|
@ -9,19 +9,11 @@
|
|||
//! current location from anywhere else, we just receive the new path rename event, which means a
|
||||
//! creation.
|
||||
|
||||
use crate::{
|
||||
invalidate_query,
|
||||
library::Library,
|
||||
location::{
|
||||
file_path_helper::{
|
||||
check_file_path_exists, get_inode, FilePathError, IsolatedFilePathData,
|
||||
},
|
||||
manager::LocationManagerError,
|
||||
},
|
||||
prisma::location,
|
||||
util::error::FileIOError,
|
||||
Node,
|
||||
};
|
||||
use crate::{invalidate_query, library::Library, location::manager::LocationManagerError, Node};
|
||||
|
||||
use sd_file_path_helper::{check_file_path_exists, get_inode, FilePathError, IsolatedFilePathData};
|
||||
use sd_prisma::prisma::location;
|
||||
use sd_utils::error::FileIOError;
|
||||
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
|
|
|
|||
|
|
@ -1,4 +1,7 @@
|
|||
use crate::{library::Library, prisma::location, util::db::maybe_missing, Node};
|
||||
use crate::{library::Library, Node};
|
||||
|
||||
use sd_prisma::prisma::location;
|
||||
use sd_utils::db::maybe_missing;
|
||||
|
||||
use std::{
|
||||
collections::HashSet,
|
||||
|
|
|
|||
|
|
@ -2,19 +2,9 @@ use crate::{
|
|||
invalidate_query,
|
||||
library::Library,
|
||||
location::{
|
||||
delete_directory,
|
||||
file_path_helper::{
|
||||
check_file_path_exists, create_file_path, file_path_with_object,
|
||||
filter_existing_file_path_params,
|
||||
isolated_file_path_data::extract_normalized_materialized_path_str,
|
||||
loose_find_existing_file_path_params, path_is_hidden, FilePathError, FilePathMetadata,
|
||||
IsolatedFilePathData, MetadataExt,
|
||||
},
|
||||
find_location,
|
||||
indexer::reverse_update_directories_sizes,
|
||||
location_with_indexer_rules,
|
||||
manager::LocationManagerError,
|
||||
scan_location_sub_path, update_location_size,
|
||||
create_file_path, delete_directory, find_location,
|
||||
indexer::reverse_update_directories_sizes, location_with_indexer_rules,
|
||||
manager::LocationManagerError, scan_location_sub_path, update_location_size,
|
||||
},
|
||||
object::{
|
||||
file_identifier::FileMetadata,
|
||||
|
|
@ -25,19 +15,32 @@ use crate::{
|
|||
},
|
||||
validation::hash::file_checksum,
|
||||
},
|
||||
prisma::{file_path, location, object},
|
||||
util::{
|
||||
db::{inode_from_db, inode_to_db, maybe_missing},
|
||||
error::FileIOError,
|
||||
},
|
||||
Node,
|
||||
};
|
||||
|
||||
use sd_file_ext::{extensions::ImageExtension, kind::ObjectKind};
|
||||
use sd_file_path_helper::{
|
||||
check_file_path_exists, file_path_with_object, filter_existing_file_path_params,
|
||||
isolated_file_path_data::extract_normalized_materialized_path_str,
|
||||
loose_find_existing_file_path_params, path_is_hidden, FilePathError, FilePathMetadata,
|
||||
IsolatedFilePathData, MetadataExt,
|
||||
};
|
||||
use sd_prisma::{
|
||||
prisma::{file_path, location, media_data, object},
|
||||
prisma_sync,
|
||||
};
|
||||
use sd_sync::OperationFactory;
|
||||
use sd_utils::{
|
||||
db::{inode_from_db, inode_to_db, maybe_missing},
|
||||
error::FileIOError,
|
||||
uuid_to_bytes,
|
||||
};
|
||||
|
||||
#[cfg(target_family = "unix")]
|
||||
use crate::location::file_path_helper::get_inode;
|
||||
use sd_file_path_helper::get_inode;
|
||||
|
||||
#[cfg(target_family = "windows")]
|
||||
use crate::location::file_path_helper::get_inode_from_path;
|
||||
use sd_file_path_helper::get_inode_from_path;
|
||||
|
||||
use std::{
|
||||
collections::{HashMap, HashSet},
|
||||
|
|
@ -48,14 +51,9 @@ use std::{
|
|||
sync::Arc,
|
||||
};
|
||||
|
||||
use sd_file_ext::{extensions::ImageExtension, kind::ObjectKind};
|
||||
|
||||
use chrono::{DateTime, FixedOffset, Local, Utc};
|
||||
use notify::Event;
|
||||
use prisma_client_rust::{raw, PrismaValue};
|
||||
use sd_prisma::{prisma::media_data, prisma_sync};
|
||||
use sd_sync::OperationFactory;
|
||||
use sd_utils::uuid_to_bytes;
|
||||
use serde_json::json;
|
||||
use tokio::{
|
||||
fs,
|
||||
|
|
@ -122,7 +120,7 @@ pub(super) async fn create_dir(
|
|||
|
||||
create_file_path(
|
||||
library,
|
||||
iso_file_path,
|
||||
iso_file_path.to_parts(),
|
||||
None,
|
||||
FilePathMetadata::from_path(&path, metadata).await?,
|
||||
)
|
||||
|
|
@ -178,7 +176,8 @@ async fn inner_create_file(
|
|||
);
|
||||
|
||||
let iso_file_path = IsolatedFilePathData::new(location_id, location_path, path, false)?;
|
||||
let extension = iso_file_path.extension.to_string();
|
||||
let iso_file_path_parts = iso_file_path.to_parts();
|
||||
let extension = iso_file_path_parts.extension.to_string();
|
||||
|
||||
let metadata = FilePathMetadata::from_path(&path, metadata).await?;
|
||||
|
||||
|
|
@ -202,9 +201,9 @@ async fn inner_create_file(
|
|||
.file_path()
|
||||
.find_unique(file_path::location_id_materialized_path_name_extension(
|
||||
location_id,
|
||||
iso_file_path.materialized_path.to_string(),
|
||||
iso_file_path.name.to_string(),
|
||||
iso_file_path.extension.to_string(),
|
||||
iso_file_path_parts.materialized_path.to_string(),
|
||||
iso_file_path_parts.name.to_string(),
|
||||
iso_file_path_parts.extension.to_string(),
|
||||
))
|
||||
.include(file_path_with_object::include())
|
||||
.exec()
|
||||
|
|
@ -242,7 +241,8 @@ async fn inner_create_file(
|
|||
|
||||
debug!("Creating path: {}", iso_file_path);
|
||||
|
||||
let created_file = create_file_path(library, iso_file_path, cas_id.clone(), metadata).await?;
|
||||
let created_file =
|
||||
create_file_path(library, iso_file_path_parts, cas_id.clone(), metadata).await?;
|
||||
|
||||
object::select!(object_ids { id pub_id });
|
||||
|
||||
|
|
@ -776,10 +776,12 @@ pub(super) async fn rename(
|
|||
let is_dir = maybe_missing(file_path.is_dir, "file_path.is_dir")?;
|
||||
|
||||
let new = IsolatedFilePathData::new(location_id, &location_path, new_path, is_dir)?;
|
||||
let new_parts = new.to_parts();
|
||||
|
||||
// If the renamed path is a directory, we have to update every successor
|
||||
if is_dir {
|
||||
let old = IsolatedFilePathData::new(location_id, &location_path, old_path, is_dir)?;
|
||||
let old_parts = old.to_parts();
|
||||
// TODO: Fetch all file_paths that will be updated and dispatch sync events
|
||||
|
||||
let updated = library
|
||||
|
|
@ -788,8 +790,14 @@ pub(super) async fn rename(
|
|||
"UPDATE file_path \
|
||||
SET materialized_path = REPLACE(materialized_path, {}, {}) \
|
||||
WHERE location_id = {}",
|
||||
PrismaValue::String(format!("{}/{}/", old.materialized_path, old.name)),
|
||||
PrismaValue::String(format!("{}/{}/", new.materialized_path, new.name)),
|
||||
PrismaValue::String(format!(
|
||||
"{}/{}/",
|
||||
old_parts.materialized_path, old_parts.name
|
||||
)),
|
||||
PrismaValue::String(format!(
|
||||
"{}/{}/",
|
||||
new_parts.materialized_path, new_parts.name
|
||||
)),
|
||||
PrismaValue::Int(location_id as i64)
|
||||
))
|
||||
.exec()
|
||||
|
|
@ -806,8 +814,8 @@ pub(super) async fn rename(
|
|||
file_path::pub_id::equals(file_path.pub_id),
|
||||
vec![
|
||||
file_path::materialized_path::set(Some(new_path_materialized_str)),
|
||||
file_path::name::set(Some(new.name.to_string())),
|
||||
file_path::extension::set(Some(new.extension.to_string())),
|
||||
file_path::name::set(Some(new_parts.name.to_string())),
|
||||
file_path::extension::set(Some(new_parts.extension.to_string())),
|
||||
file_path::date_modified::set(Some(
|
||||
DateTime::<Utc>::from(new_path_metadata.modified_or_now()).into(),
|
||||
)),
|
||||
|
|
|
|||
|
|
@ -7,17 +7,11 @@
|
|||
//! a remove event to see if a create event is emitted. If it is, we just update the `file_path`
|
||||
//! in the database. If not, we remove the file from the database.
|
||||
|
||||
use crate::{
|
||||
invalidate_query,
|
||||
library::Library,
|
||||
location::{
|
||||
file_path_helper::{get_inode_from_path, FilePathError},
|
||||
manager::LocationManagerError,
|
||||
},
|
||||
prisma::location,
|
||||
util::error::FileIOError,
|
||||
Node,
|
||||
};
|
||||
use crate::{invalidate_query, library::Library, location::manager::LocationManagerError, Node};
|
||||
|
||||
use sd_file_path_helper::{get_inode_from_path, FilePathError};
|
||||
use sd_prisma::prisma::location;
|
||||
use sd_utils::error::FileIOError;
|
||||
|
||||
use std::{
|
||||
collections::{BTreeMap, HashMap},
|
||||
|
|
|
|||
|
|
@ -2,19 +2,28 @@ use crate::{
|
|||
invalidate_query,
|
||||
job::{JobBuilder, JobError, JobManagerError},
|
||||
library::Library,
|
||||
location::file_path_helper::filter_existing_file_path_params,
|
||||
object::{
|
||||
file_identifier::{self, file_identifier_job::FileIdentifierJobInit},
|
||||
media::{media_processor, MediaProcessorJobInit},
|
||||
},
|
||||
prisma::{file_path, indexer_rules_in_location, location, PrismaClient},
|
||||
util::{
|
||||
db::{maybe_missing, MissingFieldError},
|
||||
error::{FileIOError, NonUtf8PathError},
|
||||
},
|
||||
Node,
|
||||
};
|
||||
|
||||
use sd_file_path_helper::{filter_existing_file_path_params, IsolatedFilePathData};
|
||||
use sd_prisma::{
|
||||
prisma::{file_path, indexer_rules_in_location, location, PrismaClient},
|
||||
prisma_sync,
|
||||
};
|
||||
use sd_sync::*;
|
||||
use sd_utils::{
|
||||
db::{maybe_missing, MissingFieldError},
|
||||
error::{FileIOError, NonUtf8PathError},
|
||||
uuid_to_bytes,
|
||||
};
|
||||
|
||||
#[cfg(feature = "location-watcher")]
|
||||
use sd_file_path_helper::IsolatedFilePathDataParts;
|
||||
|
||||
use std::{
|
||||
collections::HashSet,
|
||||
path::{Component, Path, PathBuf},
|
||||
|
|
@ -25,9 +34,6 @@ use chrono::Utc;
|
|||
use futures::future::TryFutureExt;
|
||||
use normpath::PathExt;
|
||||
use prisma_client_rust::{operator::and, or, QueryError};
|
||||
use sd_prisma::prisma_sync;
|
||||
use sd_sync::*;
|
||||
use sd_utils::uuid_to_bytes;
|
||||
use serde::Deserialize;
|
||||
use serde_json::json;
|
||||
use specta::Type;
|
||||
|
|
@ -36,7 +42,6 @@ use tracing::{debug, info, warn};
|
|||
use uuid::Uuid;
|
||||
|
||||
mod error;
|
||||
pub mod file_path_helper;
|
||||
pub mod indexer;
|
||||
mod manager;
|
||||
pub mod metadata;
|
||||
|
|
@ -47,8 +52,6 @@ use indexer::IndexerJobInit;
|
|||
pub use manager::{LocationManagerError, Locations};
|
||||
use metadata::SpacedriveLocationMetadataFile;
|
||||
|
||||
use file_path_helper::IsolatedFilePathData;
|
||||
|
||||
pub type LocationPubId = Uuid;
|
||||
|
||||
// Location includes!
|
||||
|
|
@ -464,6 +467,7 @@ pub async fn scan_location(
|
|||
location: location_base_data,
|
||||
sub_path: None,
|
||||
regenerate_thumbnails: false,
|
||||
regenerate_labels: false,
|
||||
})
|
||||
.spawn(node, library)
|
||||
.await
|
||||
|
|
@ -503,6 +507,7 @@ pub async fn scan_location_sub_path(
|
|||
location: location_base_data,
|
||||
sub_path: Some(sub_path),
|
||||
regenerate_thumbnails: false,
|
||||
regenerate_labels: false,
|
||||
})
|
||||
.spawn(node, library)
|
||||
.await
|
||||
|
|
@ -526,7 +531,15 @@ pub async fn light_scan_location(
|
|||
|
||||
indexer::shallow(&location, &sub_path, &node, &library).await?;
|
||||
file_identifier::shallow(&location_base_data, &sub_path, &library).await?;
|
||||
media_processor::shallow(&location_base_data, &sub_path, &library, &node).await?;
|
||||
media_processor::shallow(
|
||||
&location_base_data,
|
||||
&sub_path,
|
||||
&library,
|
||||
#[cfg(feature = "ai")]
|
||||
false,
|
||||
&node,
|
||||
)
|
||||
.await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -1017,3 +1030,97 @@ pub async fn get_location_path_from_location_id(
|
|||
})
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(feature = "location-watcher")]
|
||||
pub async fn create_file_path(
|
||||
crate::location::Library { db, sync, .. }: &crate::location::Library,
|
||||
IsolatedFilePathDataParts {
|
||||
materialized_path,
|
||||
is_dir,
|
||||
location_id,
|
||||
name,
|
||||
extension,
|
||||
..
|
||||
}: IsolatedFilePathDataParts<'_>,
|
||||
cas_id: Option<String>,
|
||||
metadata: sd_file_path_helper::FilePathMetadata,
|
||||
) -> Result<file_path::Data, sd_file_path_helper::FilePathError> {
|
||||
use sd_utils::db::inode_to_db;
|
||||
|
||||
use sd_prisma::prisma;
|
||||
|
||||
let indexed_at = Utc::now();
|
||||
|
||||
let location = db
|
||||
.location()
|
||||
.find_unique(location::id::equals(location_id))
|
||||
.select(location::select!({ id pub_id }))
|
||||
.exec()
|
||||
.await?
|
||||
.ok_or(sd_file_path_helper::FilePathError::LocationNotFound(
|
||||
location_id,
|
||||
))?;
|
||||
|
||||
let params = {
|
||||
use file_path::*;
|
||||
|
||||
vec![
|
||||
(
|
||||
location::NAME,
|
||||
json!(prisma_sync::location::SyncId {
|
||||
pub_id: location.pub_id
|
||||
}),
|
||||
),
|
||||
(cas_id::NAME, json!(cas_id)),
|
||||
(materialized_path::NAME, json!(materialized_path)),
|
||||
(name::NAME, json!(name)),
|
||||
(extension::NAME, json!(extension)),
|
||||
(
|
||||
size_in_bytes_bytes::NAME,
|
||||
json!(metadata.size_in_bytes.to_be_bytes().to_vec()),
|
||||
),
|
||||
(inode::NAME, json!(metadata.inode.to_le_bytes())),
|
||||
(is_dir::NAME, json!(is_dir)),
|
||||
(date_created::NAME, json!(metadata.created_at)),
|
||||
(date_modified::NAME, json!(metadata.modified_at)),
|
||||
(date_indexed::NAME, json!(indexed_at)),
|
||||
]
|
||||
};
|
||||
|
||||
let pub_id = sd_utils::uuid_to_bytes(Uuid::new_v4());
|
||||
|
||||
let created_path = sync
|
||||
.write_ops(
|
||||
db,
|
||||
(
|
||||
sync.shared_create(
|
||||
prisma_sync::file_path::SyncId {
|
||||
pub_id: pub_id.clone(),
|
||||
},
|
||||
params,
|
||||
),
|
||||
db.file_path().create(pub_id, {
|
||||
use file_path::*;
|
||||
vec![
|
||||
location::connect(prisma::location::id::equals(location.id)),
|
||||
materialized_path::set(Some(materialized_path.into())),
|
||||
name::set(Some(name.into())),
|
||||
extension::set(Some(extension.into())),
|
||||
inode::set(Some(inode_to_db(metadata.inode))),
|
||||
cas_id::set(cas_id),
|
||||
is_dir::set(Some(is_dir)),
|
||||
size_in_bytes_bytes::set(Some(
|
||||
metadata.size_in_bytes.to_be_bytes().to_vec(),
|
||||
)),
|
||||
date_created::set(Some(metadata.created_at.into())),
|
||||
date_modified::set(Some(metadata.modified_at.into())),
|
||||
date_indexed::set(Some(indexed_at.into())),
|
||||
hidden::set(Some(metadata.hidden)),
|
||||
]
|
||||
}),
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
|
||||
Ok(created_path)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,22 +5,22 @@ use crate::{
|
|||
cas::generate_cas_id,
|
||||
media::thumbnail::{get_ephemeral_thumb_key, BatchToProcess, GenerateThumbnailArgs},
|
||||
},
|
||||
prisma::location,
|
||||
util::error::FileIOError,
|
||||
Node,
|
||||
};
|
||||
|
||||
use sd_file_ext::{extensions::Extension, kind::ObjectKind};
|
||||
use sd_file_path_helper::{path_is_hidden, MetadataExt};
|
||||
use sd_prisma::prisma::location;
|
||||
use sd_utils::{chain_optional_iter, error::FileIOError};
|
||||
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
path::{Path, PathBuf},
|
||||
sync::Arc,
|
||||
};
|
||||
|
||||
use sd_file_ext::{extensions::Extension, kind::ObjectKind};
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use rspc::ErrorCode;
|
||||
use sd_utils::chain_optional_iter;
|
||||
use serde::Serialize;
|
||||
use specta::Type;
|
||||
use thiserror::Error;
|
||||
|
|
@ -28,7 +28,6 @@ use tokio::{fs, io};
|
|||
use tracing::{error, warn};
|
||||
|
||||
use super::{
|
||||
file_path_helper::{path_is_hidden, MetadataExt},
|
||||
indexer::rules::{
|
||||
seed::{no_hidden, no_os_protected},
|
||||
IndexerRule, RuleKind,
|
||||
|
|
|
|||
|
|
@ -1,13 +1,11 @@
|
|||
use crate::{
|
||||
api::{notifications::Notification, BackendFeature},
|
||||
object::media::thumbnail::preferences::ThumbnailerPreferences,
|
||||
util::{
|
||||
error::FileIOError,
|
||||
version_manager::{Kind, ManagedVersion, VersionManager, VersionManagerError},
|
||||
},
|
||||
util::version_manager::{Kind, ManagedVersion, VersionManager, VersionManagerError},
|
||||
};
|
||||
|
||||
use sd_p2p::{Keypair, ManagerConfig};
|
||||
use sd_utils::error::FileIOError;
|
||||
|
||||
use std::{
|
||||
path::{Path, PathBuf},
|
||||
|
|
@ -51,9 +49,10 @@ pub struct NodeConfig {
|
|||
pub features: Vec<BackendFeature>,
|
||||
/// Authentication for Spacedrive Accounts
|
||||
pub auth_token: Option<sd_cloud_api::auth::OAuthToken>,
|
||||
|
||||
/// The aggreagation of many different preferences for the node
|
||||
pub preferences: NodePreferences,
|
||||
// Model version for the image labeler
|
||||
pub image_labeler_version: Option<String>,
|
||||
|
||||
version: NodeConfigVersion,
|
||||
}
|
||||
|
|
@ -89,6 +88,11 @@ impl ManagedVersion<NodeConfigVersion> for NodeConfig {
|
|||
};
|
||||
name.truncate(250);
|
||||
|
||||
#[cfg(feature = "ai")]
|
||||
let image_labeler_version = Some(sd_ai::image_labeler::DEFAULT_MODEL_VERSION.to_string());
|
||||
#[cfg(not(feature = "ai"))]
|
||||
let image_labeler_version = None;
|
||||
|
||||
Some(Self {
|
||||
id: Uuid::new_v4(),
|
||||
name,
|
||||
|
|
@ -99,6 +103,7 @@ impl ManagedVersion<NodeConfigVersion> for NodeConfig {
|
|||
notifications: vec![],
|
||||
auth_token: None,
|
||||
preferences: NodePreferences::default(),
|
||||
image_labeler_version,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -205,7 +210,18 @@ impl Manager {
|
|||
let data_directory_path = data_directory_path.as_ref().to_path_buf();
|
||||
let config_file_path = data_directory_path.join(NODE_STATE_CONFIG_NAME);
|
||||
|
||||
let config = NodeConfig::load(&config_file_path).await?;
|
||||
let mut config = NodeConfig::load(&config_file_path).await?;
|
||||
|
||||
#[cfg(feature = "ai")]
|
||||
if config.image_labeler_version.is_none() {
|
||||
config.image_labeler_version =
|
||||
Some(sd_ai::image_labeler::DEFAULT_MODEL_VERSION.to_string());
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "ai"))]
|
||||
{
|
||||
config.image_labeler_version = None;
|
||||
}
|
||||
|
||||
let (preferences_watcher_tx, _preferences_watcher_rx) =
|
||||
watch::channel(config.preferences.clone());
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
use crate::NodeError;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use specta::Type;
|
||||
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
use crate::api::notifications::Notification;
|
||||
|
||||
use std::sync::{atomic::AtomicU32, Arc};
|
||||
|
||||
use tokio::sync::broadcast;
|
||||
|
||||
use crate::api::notifications::Notification;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct Notifications(
|
||||
// Keep this private and use `Node::emit_notification` or `Library::emit_notification` instead.
|
||||
|
|
|
|||
|
|
@ -4,14 +4,15 @@ use crate::{
|
|||
JobStepOutput, StatefulJob, WorkerContext,
|
||||
},
|
||||
library::Library,
|
||||
location::file_path_helper::{
|
||||
ensure_file_path_exists, ensure_sub_path_is_directory, ensure_sub_path_is_in_location,
|
||||
file_path_for_file_identifier, IsolatedFilePathData,
|
||||
},
|
||||
prisma::{file_path, location, PrismaClient, SortOrder},
|
||||
util::db::maybe_missing,
|
||||
};
|
||||
|
||||
use sd_file_path_helper::{
|
||||
ensure_file_path_exists, ensure_sub_path_is_directory, ensure_sub_path_is_in_location,
|
||||
file_path_for_file_identifier, IsolatedFilePathData,
|
||||
};
|
||||
use sd_prisma::prisma::{file_path, location, PrismaClient, SortOrder};
|
||||
use sd_utils::db::maybe_missing;
|
||||
|
||||
use std::{
|
||||
hash::{Hash, Hasher},
|
||||
path::{Path, PathBuf},
|
||||
|
|
|
|||
|
|
@ -1,19 +1,17 @@
|
|||
use crate::{
|
||||
job::JobError,
|
||||
library::Library,
|
||||
location::file_path_helper::{
|
||||
file_path_for_file_identifier, FilePathError, IsolatedFilePathData,
|
||||
},
|
||||
object::{cas::generate_cas_id, object_for_file_identifier},
|
||||
prisma::{file_path, location, object, PrismaClient},
|
||||
util::{db::maybe_missing, error::FileIOError},
|
||||
};
|
||||
|
||||
use sd_file_ext::{extensions::Extension, kind::ObjectKind};
|
||||
|
||||
use sd_prisma::prisma_sync;
|
||||
use sd_file_path_helper::{file_path_for_file_identifier, FilePathError, IsolatedFilePathData};
|
||||
use sd_prisma::{
|
||||
prisma::{file_path, location, object, PrismaClient},
|
||||
prisma_sync,
|
||||
};
|
||||
use sd_sync::{CRDTOperation, OperationFactory};
|
||||
use sd_utils::uuid_to_bytes;
|
||||
use sd_utils::{db::maybe_missing, error::FileIOError, uuid_to_bytes};
|
||||
|
||||
use std::{
|
||||
collections::{HashMap, HashSet},
|
||||
|
|
|
|||
|
|
@ -1,14 +1,11 @@
|
|||
use crate::{
|
||||
invalidate_query,
|
||||
job::JobError,
|
||||
library::Library,
|
||||
location::file_path_helper::{
|
||||
ensure_file_path_exists, ensure_sub_path_is_directory, ensure_sub_path_is_in_location,
|
||||
file_path_for_file_identifier, IsolatedFilePathData,
|
||||
},
|
||||
prisma::{file_path, location, PrismaClient, SortOrder},
|
||||
util::db::maybe_missing,
|
||||
use crate::{invalidate_query, job::JobError, library::Library};
|
||||
|
||||
use sd_file_path_helper::{
|
||||
ensure_file_path_exists, ensure_sub_path_is_directory, ensure_sub_path_is_in_location,
|
||||
file_path_for_file_identifier, IsolatedFilePathData,
|
||||
};
|
||||
use sd_prisma::prisma::{file_path, location, PrismaClient, SortOrder};
|
||||
use sd_utils::db::maybe_missing;
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
|
|
|
|||
|
|
@ -5,11 +5,12 @@ use crate::{
|
|||
WorkerContext,
|
||||
},
|
||||
library::Library,
|
||||
location::file_path_helper::{join_location_relative_path, IsolatedFilePathData},
|
||||
prisma::{file_path, location},
|
||||
util::{db::maybe_missing, error::FileIOError},
|
||||
};
|
||||
|
||||
use sd_file_path_helper::{join_location_relative_path, IsolatedFilePathData};
|
||||
use sd_prisma::prisma::{file_path, location};
|
||||
use sd_utils::{db::maybe_missing, error::FileIOError};
|
||||
|
||||
use std::{hash::Hash, path::PathBuf};
|
||||
|
||||
use futures_concurrency::future::TryJoin;
|
||||
|
|
|
|||
|
|
@ -5,12 +5,13 @@ use crate::{
|
|||
WorkerContext,
|
||||
},
|
||||
library::Library,
|
||||
location::file_path_helper::push_location_relative_path,
|
||||
object::fs::{construct_target_filename, error::FileSystemJobsError},
|
||||
prisma::{file_path, location},
|
||||
util::error::FileIOError,
|
||||
};
|
||||
|
||||
use sd_file_path_helper::push_location_relative_path;
|
||||
use sd_prisma::prisma::{file_path, location};
|
||||
use sd_utils::error::FileIOError;
|
||||
|
||||
use std::{hash::Hash, path::PathBuf};
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
|
|
|||
|
|
@ -5,10 +5,11 @@ use crate::{
|
|||
},
|
||||
library::Library,
|
||||
location::get_location_path_from_location_id,
|
||||
prisma::{file_path, location},
|
||||
util::{db::maybe_missing, error::FileIOError},
|
||||
};
|
||||
|
||||
use sd_prisma::prisma::{file_path, location};
|
||||
use sd_utils::{db::maybe_missing, error::FileIOError};
|
||||
|
||||
use std::hash::Hash;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
|
|
|||
|
|
@ -5,11 +5,13 @@ use crate::{
|
|||
StatefulJob, WorkerContext,
|
||||
},
|
||||
library::Library,
|
||||
location::{file_path_helper::IsolatedFilePathData, get_location_path_from_location_id},
|
||||
prisma::{file_path, location},
|
||||
util::{db::maybe_missing, error::FileIOError},
|
||||
location::get_location_path_from_location_id,
|
||||
};
|
||||
|
||||
use sd_file_path_helper::IsolatedFilePathData;
|
||||
use sd_prisma::prisma::{file_path, location};
|
||||
use sd_utils::{db::maybe_missing, error::FileIOError};
|
||||
|
||||
use std::{hash::Hash, path::PathBuf};
|
||||
|
||||
use futures::future::try_join_all;
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
use crate::{
|
||||
location::{file_path_helper::FilePathError, LocationError},
|
||||
prisma::file_path,
|
||||
util::{
|
||||
db::MissingFieldError,
|
||||
error::{FileIOError, NonUtf8PathError},
|
||||
},
|
||||
use crate::location::LocationError;
|
||||
|
||||
use sd_file_path_helper::FilePathError;
|
||||
use sd_prisma::prisma::file_path;
|
||||
use sd_utils::{
|
||||
db::MissingFieldError,
|
||||
error::{FileIOError, NonUtf8PathError},
|
||||
};
|
||||
|
||||
use std::path::Path;
|
||||
|
|
|
|||
|
|
@ -1,13 +1,10 @@
|
|||
use crate::{
|
||||
location::{
|
||||
file_path_helper::{file_path_with_object, IsolatedFilePathData},
|
||||
LocationError,
|
||||
},
|
||||
prisma::{file_path, location, PrismaClient},
|
||||
util::{
|
||||
db::maybe_missing,
|
||||
error::{FileIOError, NonUtf8PathError},
|
||||
},
|
||||
use crate::location::LocationError;
|
||||
|
||||
use sd_file_path_helper::{file_path_with_object, IsolatedFilePathData};
|
||||
use sd_prisma::prisma::{file_path, location, PrismaClient};
|
||||
use sd_utils::{
|
||||
db::maybe_missing,
|
||||
error::{FileIOError, NonUtf8PathError},
|
||||
};
|
||||
|
||||
use std::{
|
||||
|
|
|
|||
|
|
@ -1,12 +1,10 @@
|
|||
use crate::{
|
||||
job::JobRunErrors,
|
||||
location::file_path_helper::{file_path_for_media_processor, IsolatedFilePathData},
|
||||
prisma::{location, media_data, PrismaClient},
|
||||
util::error::FileIOError,
|
||||
};
|
||||
use crate::job::JobRunErrors;
|
||||
|
||||
use sd_file_ext::extensions::{Extension, ImageExtension, ALL_IMAGE_EXTENSIONS};
|
||||
use sd_file_path_helper::{file_path_for_media_processor, IsolatedFilePathData};
|
||||
use sd_media_metadata::ImageMetadata;
|
||||
use sd_prisma::prisma::{location, media_data, PrismaClient};
|
||||
use sd_utils::error::FileIOError;
|
||||
|
||||
use std::{collections::HashSet, path::Path};
|
||||
|
||||
|
|
|
|||
|
|
@ -5,16 +5,25 @@ use crate::{
|
|||
StatefulJob, WorkerContext,
|
||||
},
|
||||
library::Library,
|
||||
location::file_path_helper::{
|
||||
ensure_file_path_exists, ensure_sub_path_is_directory, ensure_sub_path_is_in_location,
|
||||
file_path_for_media_processor, IsolatedFilePathData,
|
||||
},
|
||||
prisma::{location, PrismaClient},
|
||||
util::db::maybe_missing,
|
||||
Node,
|
||||
};
|
||||
|
||||
#[cfg(feature = "ai")]
|
||||
use crate::job::JobRunErrors;
|
||||
|
||||
use sd_file_ext::extensions::Extension;
|
||||
use sd_file_path_helper::{
|
||||
ensure_file_path_exists, ensure_sub_path_is_directory, ensure_sub_path_is_in_location,
|
||||
file_path_for_media_processor, IsolatedFilePathData,
|
||||
};
|
||||
use sd_prisma::prisma::{location, PrismaClient};
|
||||
use sd_utils::db::maybe_missing;
|
||||
|
||||
#[cfg(feature = "ai")]
|
||||
use sd_ai::image_labeler::{BatchToken as ImageLabelerBatchToken, LabelerOutput};
|
||||
|
||||
#[cfg(feature = "ai")]
|
||||
use std::sync::Arc;
|
||||
|
||||
use std::{
|
||||
hash::Hash,
|
||||
|
|
@ -45,6 +54,7 @@ pub struct MediaProcessorJobInit {
|
|||
pub location: location::Data,
|
||||
pub sub_path: Option<PathBuf>,
|
||||
pub regenerate_thumbnails: bool,
|
||||
pub regenerate_labels: bool,
|
||||
}
|
||||
|
||||
impl Hash for MediaProcessorJobInit {
|
||||
|
|
@ -62,12 +72,19 @@ pub struct MediaProcessorJobData {
|
|||
to_process_path: PathBuf,
|
||||
#[serde(skip, default)]
|
||||
maybe_thumbnailer_progress_rx: Option<chan::Receiver<(u32, u32)>>,
|
||||
#[cfg(feature = "ai")]
|
||||
labeler_batch_token: ImageLabelerBatchToken,
|
||||
#[cfg(feature = "ai")]
|
||||
#[serde(skip, default)]
|
||||
maybe_labels_rx: Option<chan::Receiver<LabelerOutput>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
pub enum MediaProcessorJobStep {
|
||||
ExtractMediaData(Vec<file_path_for_media_processor::Data>),
|
||||
WaitThumbnails(usize),
|
||||
#[cfg(feature = "ai")]
|
||||
WaitLabels(usize),
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
|
|
@ -134,7 +151,7 @@ impl StatefulJob for MediaProcessorJobInit {
|
|||
&iso_file_path,
|
||||
&ctx.library,
|
||||
&ctx.node,
|
||||
false,
|
||||
self.regenerate_thumbnails,
|
||||
)
|
||||
.await?;
|
||||
|
||||
|
|
@ -153,23 +170,58 @@ impl StatefulJob for MediaProcessorJobInit {
|
|||
|
||||
let file_paths = get_files_for_media_data_extraction(db, &iso_file_path).await?;
|
||||
|
||||
#[cfg(feature = "ai")]
|
||||
let file_paths_for_labeling =
|
||||
get_files_for_labeling(db, &iso_file_path, self.regenerate_labels).await?;
|
||||
|
||||
#[cfg(feature = "ai")]
|
||||
let total_files_for_labeling = file_paths_for_labeling.len();
|
||||
|
||||
#[cfg(feature = "ai")]
|
||||
let (labeler_batch_token, labels_rx) = ctx
|
||||
.node
|
||||
.image_labeller
|
||||
.new_resumable_batch(
|
||||
location_id,
|
||||
location_path.clone(),
|
||||
file_paths_for_labeling,
|
||||
Arc::clone(db),
|
||||
)
|
||||
.await;
|
||||
|
||||
let total_files = file_paths.len();
|
||||
|
||||
let chunked_files =
|
||||
file_paths
|
||||
let chunked_files = file_paths
|
||||
.into_iter()
|
||||
.chunks(BATCH_SIZE)
|
||||
.into_iter()
|
||||
.map(|chunk| chunk.collect::<Vec<_>>())
|
||||
.map(MediaProcessorJobStep::ExtractMediaData)
|
||||
.chain(
|
||||
[
|
||||
(thumbs_to_process_count > 0).then_some(MediaProcessorJobStep::WaitThumbnails(
|
||||
thumbs_to_process_count as usize,
|
||||
)),
|
||||
]
|
||||
.into_iter()
|
||||
.chunks(BATCH_SIZE)
|
||||
.flatten(),
|
||||
)
|
||||
.chain(
|
||||
[
|
||||
#[cfg(feature = "ai")]
|
||||
{
|
||||
(total_files_for_labeling > 0)
|
||||
.then_some(MediaProcessorJobStep::WaitLabels(total_files_for_labeling))
|
||||
},
|
||||
#[cfg(not(feature = "ai"))]
|
||||
{
|
||||
None
|
||||
},
|
||||
]
|
||||
.into_iter()
|
||||
.map(|chunk| chunk.collect::<Vec<_>>())
|
||||
.map(MediaProcessorJobStep::ExtractMediaData)
|
||||
.chain(
|
||||
[(thumbs_to_process_count > 0).then_some(
|
||||
MediaProcessorJobStep::WaitThumbnails(thumbs_to_process_count as usize),
|
||||
)]
|
||||
.into_iter()
|
||||
.flatten(),
|
||||
)
|
||||
.collect::<Vec<_>>();
|
||||
.flatten(),
|
||||
)
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
ctx.progress(vec![
|
||||
JobReportUpdate::TaskCount(total_files),
|
||||
|
|
@ -184,6 +236,10 @@ impl StatefulJob for MediaProcessorJobInit {
|
|||
location_path,
|
||||
to_process_path,
|
||||
maybe_thumbnailer_progress_rx,
|
||||
#[cfg(feature = "ai")]
|
||||
labeler_batch_token,
|
||||
#[cfg(feature = "ai")]
|
||||
maybe_labels_rx: Some(labels_rx),
|
||||
});
|
||||
|
||||
Ok((
|
||||
|
|
@ -218,6 +274,7 @@ impl StatefulJob for MediaProcessorJobInit {
|
|||
.await
|
||||
.map(Into::into)
|
||||
.map_err(Into::into),
|
||||
|
||||
MediaProcessorJobStep::WaitThumbnails(total_thumbs) => {
|
||||
ctx.progress(vec![
|
||||
JobReportUpdate::TaskCount(*total_thumbs),
|
||||
|
|
@ -262,6 +319,66 @@ impl StatefulJob for MediaProcessorJobInit {
|
|||
|
||||
Ok(None.into())
|
||||
}
|
||||
|
||||
#[cfg(feature = "ai")]
|
||||
MediaProcessorJobStep::WaitLabels(total_labels) => {
|
||||
ctx.progress(vec![
|
||||
JobReportUpdate::TaskCount(*total_labels),
|
||||
JobReportUpdate::Phase("labels".to_string()),
|
||||
JobReportUpdate::Message(
|
||||
format!("Extracting labels for {total_labels} files",),
|
||||
),
|
||||
]);
|
||||
|
||||
let mut labels_rx = pin!(if let Some(labels_rx) = data.maybe_labels_rx.clone() {
|
||||
labels_rx
|
||||
} else {
|
||||
match ctx
|
||||
.node
|
||||
.image_labeller
|
||||
.resume_batch(data.labeler_batch_token, Arc::clone(&ctx.library.db))
|
||||
.await
|
||||
{
|
||||
Ok(labels_rx) => labels_rx,
|
||||
Err(e) => return Ok(JobRunErrors(vec![e.to_string()]).into()),
|
||||
}
|
||||
});
|
||||
|
||||
let mut total_labeled = 0;
|
||||
|
||||
let mut errors = Vec::new();
|
||||
|
||||
while let Some(LabelerOutput {
|
||||
file_path_id,
|
||||
has_new_labels,
|
||||
result,
|
||||
}) = labels_rx.next().await
|
||||
{
|
||||
total_labeled += 1;
|
||||
ctx.progress(vec![JobReportUpdate::CompletedTaskCount(total_labeled)]);
|
||||
|
||||
if let Err(e) = result {
|
||||
error!(
|
||||
"Failed to generate labels <file_path_id='{}'>: {e:#?}",
|
||||
file_path_id
|
||||
);
|
||||
|
||||
errors.push(e.to_string());
|
||||
} else if has_new_labels {
|
||||
invalidate_query!(&ctx.library, "labels.count");
|
||||
}
|
||||
}
|
||||
|
||||
invalidate_query!(&ctx.library, "labels.list");
|
||||
invalidate_query!(&ctx.library, "labels.getForObject");
|
||||
invalidate_query!(&ctx.library, "labels.getWithObjects");
|
||||
|
||||
if !errors.is_empty() {
|
||||
Ok(JobRunErrors(errors).into())
|
||||
} else {
|
||||
Ok(None.into())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -377,6 +494,51 @@ async fn get_files_for_media_data_extraction(
|
|||
.map_err(Into::into)
|
||||
}
|
||||
|
||||
#[cfg(feature = "ai")]
|
||||
async fn get_files_for_labeling(
|
||||
db: &PrismaClient,
|
||||
parent_iso_file_path: &IsolatedFilePathData<'_>,
|
||||
regenerate_labels: bool,
|
||||
) -> Result<Vec<file_path_for_media_processor::Data>, MediaProcessorError> {
|
||||
// FIXME: Had to use format! macro because PCR doesn't support IN with Vec for SQLite
|
||||
// We have no data coming from the user, so this is sql injection safe
|
||||
db._query_raw(raw!(
|
||||
&format!(
|
||||
"SELECT id, materialized_path, is_dir, name, extension, cas_id, object_id
|
||||
FROM file_path f
|
||||
WHERE
|
||||
location_id={{}}
|
||||
AND cas_id IS NOT NULL
|
||||
AND LOWER(extension) IN ({})
|
||||
AND materialized_path LIKE {{}}
|
||||
{}
|
||||
ORDER BY materialized_path ASC",
|
||||
// Orderind by materialized_path so we can prioritize processing the first files
|
||||
// in the above part of the directories tree
|
||||
&media_data_extractor::FILTERED_IMAGE_EXTENSIONS
|
||||
.iter()
|
||||
.map(|ext| format!("LOWER('{ext}')"))
|
||||
.collect::<Vec<_>>()
|
||||
.join(","),
|
||||
if !regenerate_labels {
|
||||
"AND NOT EXISTS (SELECT 1 FROM label_on_object WHERE object_id = f.object_id)"
|
||||
} else {
|
||||
""
|
||||
}
|
||||
),
|
||||
PrismaValue::Int(parent_iso_file_path.location_id() as i64),
|
||||
PrismaValue::String(format!(
|
||||
"{}%",
|
||||
parent_iso_file_path
|
||||
.materialized_path_for_children()
|
||||
.expect("sub path iso_file_path must be a directory")
|
||||
))
|
||||
))
|
||||
.exec()
|
||||
.await
|
||||
.map_err(Into::into)
|
||||
}
|
||||
|
||||
async fn get_all_children_files_by_extensions(
|
||||
db: &PrismaClient,
|
||||
parent_iso_file_path: &IsolatedFilePathData<'_>,
|
||||
|
|
|
|||
|
|
@ -1,8 +1,6 @@
|
|||
use crate::{
|
||||
job::{JobRunErrors, JobRunMetadata},
|
||||
location::file_path_helper::{file_path_for_media_processor, FilePathError},
|
||||
};
|
||||
use crate::job::{JobRunErrors, JobRunMetadata};
|
||||
|
||||
use sd_file_path_helper::{file_path_for_media_processor, FilePathError};
|
||||
use sd_prisma::prisma::{location, PrismaClient};
|
||||
|
||||
use std::path::Path;
|
||||
|
|
@ -42,6 +40,7 @@ pub enum MediaProcessorError {
|
|||
pub struct MediaProcessorMetadata {
|
||||
media_data: MediaDataExtractorMetadata,
|
||||
thumbs_processed: u32,
|
||||
labels_extracted: u32,
|
||||
}
|
||||
|
||||
impl From<MediaDataExtractorMetadata> for MediaProcessorMetadata {
|
||||
|
|
@ -49,6 +48,7 @@ impl From<MediaDataExtractorMetadata> for MediaProcessorMetadata {
|
|||
Self {
|
||||
media_data,
|
||||
thumbs_processed: 0,
|
||||
labels_extracted: 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -58,6 +58,7 @@ impl JobRunMetadata for MediaProcessorMetadata {
|
|||
self.media_data.extracted += new_data.media_data.extracted;
|
||||
self.media_data.skipped += new_data.media_data.skipped;
|
||||
self.thumbs_processed += new_data.thumbs_processed;
|
||||
self.labels_extracted += new_data.labels_extracted;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -2,23 +2,33 @@ use crate::{
|
|||
invalidate_query,
|
||||
job::{JobError, JobRunMetadata},
|
||||
library::Library,
|
||||
location::file_path_helper::{
|
||||
ensure_file_path_exists, ensure_sub_path_is_directory, ensure_sub_path_is_in_location,
|
||||
file_path_for_media_processor, IsolatedFilePathData,
|
||||
},
|
||||
object::media::thumbnail::GenerateThumbnailArgs,
|
||||
prisma::{location, PrismaClient},
|
||||
util::db::maybe_missing,
|
||||
Node,
|
||||
};
|
||||
|
||||
use sd_file_ext::extensions::Extension;
|
||||
use sd_file_path_helper::{
|
||||
ensure_file_path_exists, ensure_sub_path_is_directory, ensure_sub_path_is_in_location,
|
||||
file_path_for_media_processor, IsolatedFilePathData,
|
||||
};
|
||||
use sd_prisma::prisma::{location, PrismaClient};
|
||||
use sd_utils::db::maybe_missing;
|
||||
|
||||
#[cfg(feature = "ai")]
|
||||
use sd_ai::image_labeler::LabelerOutput;
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
#[cfg(feature = "ai")]
|
||||
use std::sync::Arc;
|
||||
|
||||
use itertools::Itertools;
|
||||
use prisma_client_rust::{raw, PrismaValue};
|
||||
use sd_file_ext::extensions::Extension;
|
||||
use tracing::{debug, error};
|
||||
|
||||
#[cfg(feature = "ai")]
|
||||
use futures::StreamExt;
|
||||
|
||||
use super::{
|
||||
media_data_extractor::{self, process},
|
||||
thumbnail::{self, BatchToProcess},
|
||||
|
|
@ -30,11 +40,10 @@ const BATCH_SIZE: usize = 10;
|
|||
pub async fn shallow(
|
||||
location: &location::Data,
|
||||
sub_path: &PathBuf,
|
||||
library: &Library,
|
||||
library @ Library { db, .. }: &Library,
|
||||
#[cfg(feature = "ai")] regenerate_labels: bool,
|
||||
node: &Node,
|
||||
) -> Result<(), JobError> {
|
||||
let Library { db, .. } = library;
|
||||
|
||||
let location_id = location.id;
|
||||
let location_path = maybe_missing(&location.path, "location.path").map(PathBuf::from)?;
|
||||
|
||||
|
|
@ -64,7 +73,7 @@ pub async fn shallow(
|
|||
.map_err(MediaProcessorError::from)?
|
||||
};
|
||||
|
||||
debug!("Searching for images in location {location_id} at path {iso_file_path}");
|
||||
debug!("Searching for media in location {location_id} at path {iso_file_path}");
|
||||
|
||||
dispatch_thumbnails_for_processing(
|
||||
location.id,
|
||||
|
|
@ -78,6 +87,13 @@ pub async fn shallow(
|
|||
|
||||
let file_paths = get_files_for_media_data_extraction(db, &iso_file_path).await?;
|
||||
|
||||
#[cfg(feature = "ai")]
|
||||
let file_paths_for_labelling =
|
||||
get_files_for_labeling(db, &iso_file_path, regenerate_labels).await?;
|
||||
|
||||
#[cfg(feature = "ai")]
|
||||
let has_labels = !file_paths_for_labelling.is_empty();
|
||||
|
||||
let total_files = file_paths.len();
|
||||
|
||||
let chunked_files = file_paths
|
||||
|
|
@ -92,6 +108,17 @@ pub async fn shallow(
|
|||
chunked_files.len()
|
||||
);
|
||||
|
||||
#[cfg(feature = "ai")]
|
||||
let labels_rx = node
|
||||
.image_labeller
|
||||
.new_batch(
|
||||
location_id,
|
||||
location_path.clone(),
|
||||
file_paths_for_labelling,
|
||||
Arc::clone(db),
|
||||
)
|
||||
.await;
|
||||
|
||||
let mut run_metadata = MediaProcessorMetadata::default();
|
||||
|
||||
for files in chunked_files {
|
||||
|
|
@ -113,6 +140,33 @@ pub async fn shallow(
|
|||
invalidate_query!(library, "search.objects");
|
||||
}
|
||||
|
||||
#[cfg(feature = "ai")]
|
||||
{
|
||||
if has_labels {
|
||||
labels_rx
|
||||
.for_each(
|
||||
|LabelerOutput {
|
||||
file_path_id,
|
||||
has_new_labels,
|
||||
result,
|
||||
}| async move {
|
||||
if let Err(e) = result {
|
||||
error!(
|
||||
"Failed to generate labels <file_path_id='{file_path_id}'>: {e:#?}"
|
||||
);
|
||||
} else if has_new_labels {
|
||||
invalidate_query!(library, "labels.count");
|
||||
}
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
invalidate_query!(library, "labels.list");
|
||||
invalidate_query!(library, "labels.getForObject");
|
||||
invalidate_query!(library, "labels.getWithObjects");
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
|
@ -129,6 +183,47 @@ async fn get_files_for_media_data_extraction(
|
|||
.map_err(Into::into)
|
||||
}
|
||||
|
||||
#[cfg(feature = "ai")]
|
||||
async fn get_files_for_labeling(
|
||||
db: &PrismaClient,
|
||||
parent_iso_file_path: &IsolatedFilePathData<'_>,
|
||||
regenerate_labels: bool,
|
||||
) -> Result<Vec<file_path_for_media_processor::Data>, MediaProcessorError> {
|
||||
// FIXME: Had to use format! macro because PCR doesn't support IN with Vec for SQLite
|
||||
// We have no data coming from the user, so this is sql injection safe
|
||||
db._query_raw(raw!(
|
||||
&format!(
|
||||
"SELECT id, materialized_path, is_dir, name, extension, cas_id, object_id
|
||||
FROM file_path f
|
||||
WHERE
|
||||
location_id={{}}
|
||||
AND cas_id IS NOT NULL
|
||||
AND LOWER(extension) IN ({})
|
||||
AND materialized_path = {{}}
|
||||
{}",
|
||||
&media_data_extractor::FILTERED_IMAGE_EXTENSIONS
|
||||
.iter()
|
||||
.map(|ext| format!("LOWER('{ext}')"))
|
||||
.collect::<Vec<_>>()
|
||||
.join(","),
|
||||
if !regenerate_labels {
|
||||
"AND NOT EXISTS (SELECT 1 FROM label_on_object WHERE object_id = f.object_id)"
|
||||
} else {
|
||||
""
|
||||
}
|
||||
),
|
||||
PrismaValue::Int(parent_iso_file_path.location_id() as i64),
|
||||
PrismaValue::String(
|
||||
parent_iso_file_path
|
||||
.materialized_path_for_children()
|
||||
.expect("sub path iso_file_path must be a directory")
|
||||
)
|
||||
))
|
||||
.exec()
|
||||
.await
|
||||
.map_err(Into::into)
|
||||
}
|
||||
|
||||
async fn dispatch_thumbnails_for_processing(
|
||||
location_id: location::id::Type,
|
||||
location_path: impl AsRef<Path>,
|
||||
|
|
|
|||
|
|
@ -2,10 +2,10 @@ use crate::{
|
|||
api::CoreEvent,
|
||||
library::{Libraries, LibraryId, LibraryManagerEvent},
|
||||
node::config::NodePreferences,
|
||||
util::error::{FileIOError, NonUtf8PathError},
|
||||
};
|
||||
|
||||
use sd_prisma::prisma::{location, PrismaClient};
|
||||
use sd_utils::error::{FileIOError, NonUtf8PathError};
|
||||
|
||||
use std::{
|
||||
path::{Path, PathBuf},
|
||||
|
|
@ -72,19 +72,18 @@ pub struct Thumbnailer {
|
|||
|
||||
impl Thumbnailer {
|
||||
pub async fn new(
|
||||
data_dir: PathBuf,
|
||||
data_dir: impl AsRef<Path>,
|
||||
libraries_manager: Arc<Libraries>,
|
||||
reporter: broadcast::Sender<CoreEvent>,
|
||||
node_preferences_rx: watch::Receiver<NodePreferences>,
|
||||
) -> Self {
|
||||
let data_dir = data_dir.as_ref();
|
||||
let thumbnails_directory = Arc::new(
|
||||
init_thumbnail_dir(&data_dir, Arc::clone(&libraries_manager))
|
||||
init_thumbnail_dir(data_dir, Arc::clone(&libraries_manager))
|
||||
.await
|
||||
.unwrap_or_else(|e| {
|
||||
error!("Failed to initialize thumbnail directory: {e:#?}");
|
||||
let mut thumbnails_directory = data_dir;
|
||||
thumbnails_directory.push(THUMBNAIL_CACHE_DIR_NAME);
|
||||
thumbnails_directory
|
||||
data_dir.join(THUMBNAIL_CACHE_DIR_NAME)
|
||||
}),
|
||||
);
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
use crate::{library::LibraryId, util::error::FileIOError};
|
||||
use crate::library::LibraryId;
|
||||
|
||||
use sd_prisma::prisma::{file_path, PrismaClient};
|
||||
use sd_utils::error::FileIOError;
|
||||
|
||||
use std::{collections::HashSet, ffi::OsString, path::PathBuf, sync::Arc};
|
||||
|
||||
|
|
|
|||
|
|
@ -1,13 +1,11 @@
|
|||
use crate::{
|
||||
library::{Libraries, LibraryId},
|
||||
object::media::thumbnail::ONE_SEC,
|
||||
util::{
|
||||
error::FileIOError,
|
||||
version_manager::{Kind, ManagedVersion, VersionManager, VersionManagerError},
|
||||
},
|
||||
util::version_manager::{Kind, ManagedVersion, VersionManager, VersionManagerError},
|
||||
};
|
||||
|
||||
use sd_prisma::prisma::{file_path, PrismaClient};
|
||||
use sd_utils::error::FileIOError;
|
||||
use serde_repr::{Deserialize_repr, Serialize_repr};
|
||||
|
||||
use std::{
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue