mirror of
https://github.com/spacedriveapp/spacedrive
synced 2024-06-30 12:33:31 +00:00
[ENG-1775] Scan location using new jobs (#2476)
* First draft on task system usage, still missing job system * Scan location roughly working, a ton of stuff to fix yet * Updating some deps due to crashes and bugs * Exposing non critical errors to frontend * Getting active job reports from job system * Using boxed opaque type to avoid a downcast issue with generics * Task system issues discovered on race conditions * Enable debug * Fix job report in the job manager * Fix race condition on steal tasks * Fixed race condition on task suspend * Some fixes on job progress reporting and save * Fixed many race conditions and a hard deadlock Also some progress report polishing * Ignore .ts and .mts video files for now * Some better logs * bruh * Internal deadlocks and excess of communication in the task system - Also better logs * Bunch of fixes and optimizations * WIP at fixing file identifier * Fixed file identifier job - still need to work on its progress report frontend * A bunch of polishing * Fixed some bugs and did more polishing * Cleanup * Bridging old and new job systems * A ton of fixes * A bunch of bugs related to shutdown and resume * Indexer and watcher bugs * Log normalizing * Fixing CI * Change error! to warn! on non critical errors log * Fix redirect to new location * Type annotation * Bogus merge resolution on cargo lock
This commit is contained in:
parent
f8ed254a22
commit
bdc242a852
1
.vscode/launch.json
vendored
1
.vscode/launch.json
vendored
|
@ -11,6 +11,7 @@
|
||||||
"cargo": {
|
"cargo": {
|
||||||
"args": [
|
"args": [
|
||||||
"build",
|
"build",
|
||||||
|
"--profile=dev-debug",
|
||||||
"--manifest-path=./apps/desktop/src-tauri/Cargo.toml",
|
"--manifest-path=./apps/desktop/src-tauri/Cargo.toml",
|
||||||
"--no-default-features"
|
"--no-default-features"
|
||||||
],
|
],
|
||||||
|
|
346
Cargo.lock
generated
346
Cargo.lock
generated
|
@ -282,7 +282,7 @@ dependencies = [
|
||||||
"proc-macro2",
|
"proc-macro2",
|
||||||
"quote",
|
"quote",
|
||||||
"syn 1.0.109",
|
"syn 1.0.109",
|
||||||
"synstructure",
|
"synstructure 0.12.6",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
|
@ -383,9 +383,9 @@ dependencies = [
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "async-signal"
|
name = "async-signal"
|
||||||
version = "0.2.7"
|
version = "0.2.8"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "329972aa325176e89114919f2a80fdae4f4c040f66a370b1a1159c6c0f94e7aa"
|
checksum = "794f185324c2f00e771cd9f1ae8b5ac68be2ca7abb129a87afd6e86d228bc54d"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"async-io",
|
"async-io",
|
||||||
"async-lock",
|
"async-lock",
|
||||||
|
@ -826,9 +826,9 @@ dependencies = [
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "aws-smithy-runtime"
|
name = "aws-smithy-runtime"
|
||||||
version = "1.5.5"
|
version = "1.5.7"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "d0d3965f6417a92a6d1009c5958a67042f57e46342afb37ca58f9ad26744ec73"
|
checksum = "8508de54f34b8feca6638466c2bd2de9d1df5bf79c578de9a649b72d644006b3"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"aws-smithy-async",
|
"aws-smithy-async",
|
||||||
"aws-smithy-http",
|
"aws-smithy-http",
|
||||||
|
@ -840,6 +840,7 @@ dependencies = [
|
||||||
"http 0.2.12",
|
"http 0.2.12",
|
||||||
"http-body 0.4.6",
|
"http-body 0.4.6",
|
||||||
"http-body 1.0.0",
|
"http-body 1.0.0",
|
||||||
|
"httparse",
|
||||||
"hyper 0.14.29",
|
"hyper 0.14.29",
|
||||||
"hyper-rustls 0.24.2",
|
"hyper-rustls 0.24.2",
|
||||||
"once_cell",
|
"once_cell",
|
||||||
|
@ -852,9 +853,9 @@ dependencies = [
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "aws-smithy-runtime-api"
|
name = "aws-smithy-runtime-api"
|
||||||
version = "1.6.2"
|
version = "1.6.3"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "4179bd8a1c943e1aceb46c5b9fc014a561bd6c35a2153e816ba29076ee49d245"
|
checksum = "aa6dbabc7629fab4e4467f95f119c2e1a9b00b44c893affa98e23b040a0e2567"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"aws-smithy-async",
|
"aws-smithy-async",
|
||||||
"aws-smithy-types",
|
"aws-smithy-types",
|
||||||
|
@ -869,9 +870,9 @@ dependencies = [
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "aws-smithy-types"
|
name = "aws-smithy-types"
|
||||||
version = "1.1.10"
|
version = "1.2.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "5b6764ba7e1c5ede1c9f9e4046645534f06c2581402461c559b481a420330a83"
|
checksum = "cfe321a6b21f5d8eabd0ade9c55d3d0335f3c3157fc2b3e87f05f34b539e4df5"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"base64-simd",
|
"base64-simd",
|
||||||
"bytes",
|
"bytes",
|
||||||
|
@ -1306,7 +1307,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "05efc5cfd9110c8416e471df0e96702d58690178e206e61b7173706673c93706"
|
checksum = "05efc5cfd9110c8416e471df0e96702d58690178e206e61b7173706673c93706"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"memchr",
|
"memchr",
|
||||||
"regex-automata 0.4.6",
|
"regex-automata 0.4.7",
|
||||||
"serde",
|
"serde",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
@ -1618,9 +1619,9 @@ dependencies = [
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "clap"
|
name = "clap"
|
||||||
version = "4.5.6"
|
version = "4.5.7"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "a9689a29b593160de5bc4aacab7b5d54fb52231de70122626c178e6a368994c7"
|
checksum = "5db83dced34638ad474f39f250d7fea9598bdd239eaced1bdf45d597da0f433f"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"clap_builder",
|
"clap_builder",
|
||||||
"clap_derive",
|
"clap_derive",
|
||||||
|
@ -1628,9 +1629,9 @@ dependencies = [
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "clap_builder"
|
name = "clap_builder"
|
||||||
version = "4.5.6"
|
version = "4.5.7"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "2e5387378c84f6faa26890ebf9f0a92989f8873d4d380467bcd0d8d8620424df"
|
checksum = "f7e204572485eb3fbf28f871612191521df159bc3e15a9f5064c66dba3a8c05f"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"anstream",
|
"anstream",
|
||||||
"anstyle",
|
"anstyle",
|
||||||
|
@ -2749,7 +2750,7 @@ dependencies = [
|
||||||
"proc-macro2",
|
"proc-macro2",
|
||||||
"quote",
|
"quote",
|
||||||
"syn 1.0.109",
|
"syn 1.0.109",
|
||||||
"synstructure",
|
"synstructure 0.12.6",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
|
@ -3015,6 +3016,17 @@ dependencies = [
|
||||||
"futures-util",
|
"futures-util",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "futures-buffered"
|
||||||
|
version = "0.2.6"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "02dcae03ee5afa5ea17b1aebc793806b8ddfc6dc500e0b8e8e1eb30b9dad22c0"
|
||||||
|
dependencies = [
|
||||||
|
"futures-core",
|
||||||
|
"futures-util",
|
||||||
|
"pin-project-lite",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "futures-channel"
|
name = "futures-channel"
|
||||||
version = "0.3.30"
|
version = "0.3.30"
|
||||||
|
@ -3027,11 +3039,12 @@ dependencies = [
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "futures-concurrency"
|
name = "futures-concurrency"
|
||||||
version = "7.6.0"
|
version = "7.6.1"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "51ee14e256b9143bfafbf2fddeede6f396650bacf95d06fc1b3f2b503df129a0"
|
checksum = "4b14ac911e85d57c5ea6eef76d7b4d4a3177ecd15f4bea2e61927e9e3823e19f"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"bitvec",
|
"bitvec",
|
||||||
|
"futures-buffered",
|
||||||
"futures-core",
|
"futures-core",
|
||||||
"futures-lite 1.13.0",
|
"futures-lite 1.13.0",
|
||||||
"pin-project",
|
"pin-project",
|
||||||
|
@ -3542,8 +3555,8 @@ dependencies = [
|
||||||
"aho-corasick 1.1.3",
|
"aho-corasick 1.1.3",
|
||||||
"bstr",
|
"bstr",
|
||||||
"log",
|
"log",
|
||||||
"regex-automata 0.4.6",
|
"regex-automata 0.4.7",
|
||||||
"regex-syntax 0.8.3",
|
"regex-syntax 0.8.4",
|
||||||
"serde",
|
"serde",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
@ -4009,12 +4022,12 @@ dependencies = [
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "http-body-util"
|
name = "http-body-util"
|
||||||
version = "0.1.1"
|
version = "0.1.2"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "0475f8b2ac86659c21b64320d5d653f9efe42acd2a4e560073ec61a155a34f1d"
|
checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"bytes",
|
"bytes",
|
||||||
"futures-core",
|
"futures-util",
|
||||||
"http 1.1.0",
|
"http 1.1.0",
|
||||||
"http-body 1.0.0",
|
"http-body 1.0.0",
|
||||||
"pin-project-lite",
|
"pin-project-lite",
|
||||||
|
@ -4223,6 +4236,124 @@ dependencies = [
|
||||||
"objc2",
|
"objc2",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "icu_collections"
|
||||||
|
version = "1.5.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526"
|
||||||
|
dependencies = [
|
||||||
|
"displaydoc",
|
||||||
|
"yoke",
|
||||||
|
"zerofrom",
|
||||||
|
"zerovec",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "icu_locid"
|
||||||
|
version = "1.5.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637"
|
||||||
|
dependencies = [
|
||||||
|
"displaydoc",
|
||||||
|
"litemap",
|
||||||
|
"tinystr",
|
||||||
|
"writeable",
|
||||||
|
"zerovec",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "icu_locid_transform"
|
||||||
|
version = "1.5.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e"
|
||||||
|
dependencies = [
|
||||||
|
"displaydoc",
|
||||||
|
"icu_locid",
|
||||||
|
"icu_locid_transform_data",
|
||||||
|
"icu_provider",
|
||||||
|
"tinystr",
|
||||||
|
"zerovec",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "icu_locid_transform_data"
|
||||||
|
version = "1.5.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "icu_normalizer"
|
||||||
|
version = "1.5.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f"
|
||||||
|
dependencies = [
|
||||||
|
"displaydoc",
|
||||||
|
"icu_collections",
|
||||||
|
"icu_normalizer_data",
|
||||||
|
"icu_properties",
|
||||||
|
"icu_provider",
|
||||||
|
"smallvec",
|
||||||
|
"utf16_iter",
|
||||||
|
"utf8_iter",
|
||||||
|
"write16",
|
||||||
|
"zerovec",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "icu_normalizer_data"
|
||||||
|
version = "1.5.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "icu_properties"
|
||||||
|
version = "1.5.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "1f8ac670d7422d7f76b32e17a5db556510825b29ec9154f235977c9caba61036"
|
||||||
|
dependencies = [
|
||||||
|
"displaydoc",
|
||||||
|
"icu_collections",
|
||||||
|
"icu_locid_transform",
|
||||||
|
"icu_properties_data",
|
||||||
|
"icu_provider",
|
||||||
|
"tinystr",
|
||||||
|
"zerovec",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "icu_properties_data"
|
||||||
|
version = "1.5.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "icu_provider"
|
||||||
|
version = "1.5.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9"
|
||||||
|
dependencies = [
|
||||||
|
"displaydoc",
|
||||||
|
"icu_locid",
|
||||||
|
"icu_provider_macros",
|
||||||
|
"stable_deref_trait",
|
||||||
|
"tinystr",
|
||||||
|
"writeable",
|
||||||
|
"yoke",
|
||||||
|
"zerofrom",
|
||||||
|
"zerovec",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "icu_provider_macros"
|
||||||
|
version = "1.5.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn 2.0.66",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "ident_case"
|
name = "ident_case"
|
||||||
version = "1.0.1"
|
version = "1.0.1"
|
||||||
|
@ -4241,12 +4372,14 @@ dependencies = [
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "idna"
|
name = "idna"
|
||||||
version = "0.5.0"
|
version = "1.0.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6"
|
checksum = "4716a3a0933a1d01c2f72450e89596eb51dd34ef3c211ccd875acdf1f8fe47ed"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"unicode-bidi",
|
"icu_normalizer",
|
||||||
"unicode-normalization",
|
"icu_properties",
|
||||||
|
"smallvec",
|
||||||
|
"utf8_iter",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
|
@ -4512,9 +4645,9 @@ checksum = "f8478577c03552c21db0e2724ffb8986a5ce7af88107e6be5d2ee6e158c12800"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "iter_tools"
|
name = "iter_tools"
|
||||||
version = "0.17.0"
|
version = "0.18.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "55f9f40b3308a2367d5201430790786748b3e038982317dd880677c0f7b3f3f0"
|
checksum = "f85582248e8796b1d7146eabe9f70c5b9de4db16bf934ca893581d33c66403b6"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"itertools 0.11.0",
|
"itertools 0.11.0",
|
||||||
]
|
]
|
||||||
|
@ -5417,6 +5550,12 @@ version = "0.4.14"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89"
|
checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "litemap"
|
||||||
|
version = "0.7.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "643cb0b8d4fcc284004d5fd0d67ccf61dfffadb7f75e1e71bc420f4688a3a704"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "litrs"
|
name = "litrs"
|
||||||
version = "0.4.1"
|
version = "0.4.1"
|
||||||
|
@ -8087,14 +8226,14 @@ dependencies = [
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "regex"
|
name = "regex"
|
||||||
version = "1.10.4"
|
version = "1.10.5"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c"
|
checksum = "b91213439dad192326a0d7c6ee3955910425f441d7038e0d6933b0aec5c4517f"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"aho-corasick 1.1.3",
|
"aho-corasick 1.1.3",
|
||||||
"memchr",
|
"memchr",
|
||||||
"regex-automata 0.4.6",
|
"regex-automata 0.4.7",
|
||||||
"regex-syntax 0.8.3",
|
"regex-syntax 0.8.4",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
|
@ -8108,20 +8247,20 @@ dependencies = [
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "regex-automata"
|
name = "regex-automata"
|
||||||
version = "0.4.6"
|
version = "0.4.7"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea"
|
checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"aho-corasick 1.1.3",
|
"aho-corasick 1.1.3",
|
||||||
"memchr",
|
"memchr",
|
||||||
"regex-syntax 0.8.3",
|
"regex-syntax 0.8.4",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "regex-lite"
|
name = "regex-lite"
|
||||||
version = "0.1.5"
|
version = "0.1.6"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "30b661b2f27137bdbc16f00eda72866a92bb28af1753ffbd56744fb6e2e9cd8e"
|
checksum = "53a49587ad06b26609c52e423de037e7f57f20d53535d66e08c695f347df952a"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "regex-syntax"
|
name = "regex-syntax"
|
||||||
|
@ -8131,9 +8270,9 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "regex-syntax"
|
name = "regex-syntax"
|
||||||
version = "0.8.3"
|
version = "0.8.4"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56"
|
checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "renderdoc-sys"
|
name = "renderdoc-sys"
|
||||||
|
@ -8864,6 +9003,7 @@ dependencies = [
|
||||||
"sd-p2p-tunnel",
|
"sd-p2p-tunnel",
|
||||||
"sd-prisma",
|
"sd-prisma",
|
||||||
"sd-sync",
|
"sd-sync",
|
||||||
|
"sd-task-system",
|
||||||
"sd-utils",
|
"sd-utils",
|
||||||
"serde",
|
"serde",
|
||||||
"serde-hashkey",
|
"serde-hashkey",
|
||||||
|
@ -8984,7 +9124,10 @@ version = "0.1.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"prisma-client-rust",
|
"prisma-client-rust",
|
||||||
"sd-prisma",
|
"sd-prisma",
|
||||||
|
"sd-utils",
|
||||||
"serde",
|
"serde",
|
||||||
|
"specta",
|
||||||
|
"uuid",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
|
@ -9329,6 +9472,7 @@ dependencies = [
|
||||||
"tokio",
|
"tokio",
|
||||||
"tokio-stream",
|
"tokio-stream",
|
||||||
"tracing",
|
"tracing",
|
||||||
|
"tracing-subscriber",
|
||||||
"tracing-test",
|
"tracing-test",
|
||||||
"uuid",
|
"uuid",
|
||||||
]
|
]
|
||||||
|
@ -10227,6 +10371,17 @@ dependencies = [
|
||||||
"unicode-xid",
|
"unicode-xid",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "synstructure"
|
||||||
|
version = "0.13.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn 2.0.66",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "sys-locale"
|
name = "sys-locale"
|
||||||
version = "0.3.1"
|
version = "0.3.1"
|
||||||
|
@ -10851,6 +11006,16 @@ dependencies = [
|
||||||
"strict-num",
|
"strict-num",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "tinystr"
|
||||||
|
version = "0.7.6"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f"
|
||||||
|
dependencies = [
|
||||||
|
"displaydoc",
|
||||||
|
"zerovec",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "tinyvec"
|
name = "tinyvec"
|
||||||
version = "1.6.0"
|
version = "1.6.0"
|
||||||
|
@ -11516,12 +11681,12 @@ dependencies = [
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "url"
|
name = "url"
|
||||||
version = "2.5.0"
|
version = "2.5.1"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633"
|
checksum = "f7c25da092f0a868cdf09e8674cd3b7ef3a7d92a24253e663a2fb85e2496de56"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"form_urlencoded",
|
"form_urlencoded",
|
||||||
"idna 0.5.0",
|
"idna 1.0.0",
|
||||||
"percent-encoding",
|
"percent-encoding",
|
||||||
"serde",
|
"serde",
|
||||||
]
|
]
|
||||||
|
@ -11603,6 +11768,12 @@ version = "0.7.6"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9"
|
checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "utf16_iter"
|
||||||
|
version = "1.0.5"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "utf16string"
|
name = "utf16string"
|
||||||
version = "0.2.0"
|
version = "0.2.0"
|
||||||
|
@ -11612,6 +11783,12 @@ dependencies = [
|
||||||
"byteorder",
|
"byteorder",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "utf8_iter"
|
||||||
|
version = "1.0.4"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "utf8parse"
|
name = "utf8parse"
|
||||||
version = "0.2.2"
|
version = "0.2.2"
|
||||||
|
@ -12521,6 +12698,18 @@ dependencies = [
|
||||||
"windows-sys 0.48.0",
|
"windows-sys 0.48.0",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "write16"
|
||||||
|
version = "1.0.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "writeable"
|
||||||
|
version = "0.5.5"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "wry"
|
name = "wry"
|
||||||
version = "0.39.5"
|
version = "0.39.5"
|
||||||
|
@ -12635,12 +12824,12 @@ dependencies = [
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "xdg-home"
|
name = "xdg-home"
|
||||||
version = "1.1.0"
|
version = "1.2.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "21e5a325c3cb8398ad6cf859c1135b25dd29e186679cf2da7581d9679f63b38e"
|
checksum = "ca91dcf8f93db085f3a0a29358cd0b9d670915468f4290e8b85d118a34211ab8"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"libc",
|
"libc",
|
||||||
"winapi",
|
"windows-sys 0.52.0",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
|
@ -12710,6 +12899,30 @@ dependencies = [
|
||||||
"time",
|
"time",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "yoke"
|
||||||
|
version = "0.7.4"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "6c5b1314b079b0930c31e3af543d8ee1757b1951ae1e1565ec704403a7240ca5"
|
||||||
|
dependencies = [
|
||||||
|
"serde",
|
||||||
|
"stable_deref_trait",
|
||||||
|
"yoke-derive",
|
||||||
|
"zerofrom",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "yoke-derive"
|
||||||
|
version = "0.7.4"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn 2.0.66",
|
||||||
|
"synstructure 0.13.1",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "zbus"
|
name = "zbus"
|
||||||
version = "4.0.1"
|
version = "4.0.1"
|
||||||
|
@ -12789,6 +13002,27 @@ dependencies = [
|
||||||
"syn 2.0.66",
|
"syn 2.0.66",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "zerofrom"
|
||||||
|
version = "0.1.4"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "91ec111ce797d0e0784a1116d0ddcdbea84322cd79e5d5ad173daeba4f93ab55"
|
||||||
|
dependencies = [
|
||||||
|
"zerofrom-derive",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "zerofrom-derive"
|
||||||
|
version = "0.1.4"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn 2.0.66",
|
||||||
|
"synstructure 0.13.1",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "zeroize"
|
name = "zeroize"
|
||||||
version = "1.8.1"
|
version = "1.8.1"
|
||||||
|
@ -12809,6 +13043,28 @@ dependencies = [
|
||||||
"syn 2.0.66",
|
"syn 2.0.66",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "zerovec"
|
||||||
|
version = "0.10.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "bb2cc8827d6c0994478a15c53f374f46fbd41bea663d809b14744bc42e6b109c"
|
||||||
|
dependencies = [
|
||||||
|
"yoke",
|
||||||
|
"zerofrom",
|
||||||
|
"zerovec-derive",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "zerovec-derive"
|
||||||
|
version = "0.10.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "97cf56601ee5052b4417d90c8755c6683473c926039908196cf35d99f893ebe7"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn 2.0.66",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "zip"
|
name = "zip"
|
||||||
version = "0.6.6"
|
version = "0.6.6"
|
||||||
|
|
18
Cargo.toml
18
Cargo.toml
|
@ -114,6 +114,17 @@ lto = false
|
||||||
codegen-units = 256
|
codegen-units = 256
|
||||||
incremental = true
|
incremental = true
|
||||||
|
|
||||||
|
[profile.dev-debug]
|
||||||
|
inherits = "dev"
|
||||||
|
# Enables debugger
|
||||||
|
split-debuginfo = "none"
|
||||||
|
opt-level = 0
|
||||||
|
debug = "full"
|
||||||
|
strip = "none"
|
||||||
|
lto = "off"
|
||||||
|
codegen-units = 256
|
||||||
|
incremental = true
|
||||||
|
|
||||||
# Set the settings for build scripts and proc-macros.
|
# Set the settings for build scripts and proc-macros.
|
||||||
[profile.dev.build-override]
|
[profile.dev.build-override]
|
||||||
opt-level = 3
|
opt-level = 3
|
||||||
|
@ -123,6 +134,13 @@ opt-level = 3
|
||||||
opt-level = 3
|
opt-level = 3
|
||||||
incremental = false
|
incremental = false
|
||||||
|
|
||||||
|
# Set the default for dependencies, except workspace members.
|
||||||
|
[profile.dev-debug.package."*"]
|
||||||
|
inherits = "dev"
|
||||||
|
opt-level = 3
|
||||||
|
debug = "full"
|
||||||
|
incremental = false
|
||||||
|
|
||||||
# Optimize release builds
|
# Optimize release builds
|
||||||
[profile.release]
|
[profile.release]
|
||||||
panic = "abort" # Strip expensive panic clean-up logic
|
panic = "abort" # Strip expensive panic clean-up logic
|
||||||
|
|
|
@ -32,7 +32,7 @@ async fn app_ready(app_handle: AppHandle) {
|
||||||
|
|
||||||
#[tauri::command(async)]
|
#[tauri::command(async)]
|
||||||
#[specta::specta]
|
#[specta::specta]
|
||||||
// If this erorrs, we don't have FDA and we need to re-prompt for it
|
// If this errors, we don't have FDA and we need to re-prompt for it
|
||||||
async fn request_fda_macos() {
|
async fn request_fda_macos() {
|
||||||
DiskAccess::request_fda().expect("Unable to request full disk access");
|
DiskAccess::request_fda().expect("Unable to request full disk access");
|
||||||
}
|
}
|
||||||
|
|
|
@ -45,9 +45,11 @@ function constructServerUrl(urlSuffix: string) {
|
||||||
|
|
||||||
export const platform = {
|
export const platform = {
|
||||||
platform: 'tauri',
|
platform: 'tauri',
|
||||||
getThumbnailUrlByThumbKey: (keyParts) =>
|
getThumbnailUrlByThumbKey: (thumbKey) =>
|
||||||
constructServerUrl(
|
constructServerUrl(
|
||||||
`/thumbnail/${keyParts.map((i) => encodeURIComponent(i)).join('/')}.webp`
|
`/thumbnail/${encodeURIComponent(
|
||||||
|
thumbKey.base_directory_str
|
||||||
|
)}/${encodeURIComponent(thumbKey.shard_hex)}/${encodeURIComponent(thumbKey.cas_id)}.webp`
|
||||||
),
|
),
|
||||||
getFileUrl: (libraryId, locationLocalId, filePathId) =>
|
getFileUrl: (libraryId, locationLocalId, filePathId) =>
|
||||||
constructServerUrl(`/file/${libraryId}/${locationLocalId}/${filePathId}`),
|
constructServerUrl(`/file/${libraryId}/${locationLocalId}/${filePathId}`),
|
||||||
|
|
|
@ -76,8 +76,8 @@ pub fn handle_core_msg(
|
||||||
|
|
||||||
let new_node = match Node::new(data_dir, sd_core::Env::new(CLIENT_ID)).await {
|
let new_node = match Node::new(data_dir, sd_core::Env::new(CLIENT_ID)).await {
|
||||||
Ok(node) => node,
|
Ok(node) => node,
|
||||||
Err(err) => {
|
Err(e) => {
|
||||||
error!("failed to initialise node: {}", err);
|
error!(?e, "Failed to initialize node;");
|
||||||
callback(Err(query));
|
callback(Err(query));
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
@ -94,8 +94,8 @@ pub fn handle_core_msg(
|
||||||
false => from_value::<Request>(v).map(|v| vec![v]),
|
false => from_value::<Request>(v).map(|v| vec![v]),
|
||||||
}) {
|
}) {
|
||||||
Ok(v) => v,
|
Ok(v) => v,
|
||||||
Err(err) => {
|
Err(e) => {
|
||||||
error!("failed to decode JSON-RPC request: {}", err); // Don't use tracing here because it's before the `Node` is initialised which sets that config!
|
error!(?e, "Failed to decode JSON-RPC request;");
|
||||||
callback(Err(query));
|
callback(Err(query));
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
@ -133,8 +133,8 @@ pub fn spawn_core_event_listener(callback: impl Fn(String) + Send + 'static) {
|
||||||
while let Some(event) = rx.next().await {
|
while let Some(event) = rx.next().await {
|
||||||
let data = match to_string(&event) {
|
let data = match to_string(&event) {
|
||||||
Ok(json) => json,
|
Ok(json) => json,
|
||||||
Err(err) => {
|
Err(e) => {
|
||||||
error!("Failed to serialize event: {err}");
|
error!(?e, "Failed to serialize event;");
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
|
@ -1,24 +1,25 @@
|
||||||
import { DocumentDirectoryPath } from '@dr.pogodin/react-native-fs';
|
import { DocumentDirectoryPath } from '@dr.pogodin/react-native-fs';
|
||||||
import { getIcon } from '@sd/assets/util';
|
import { getIcon } from '@sd/assets/util';
|
||||||
|
import { Image } from 'expo-image';
|
||||||
|
import { useEffect, useLayoutEffect, useMemo, useState, type PropsWithChildren } from 'react';
|
||||||
|
import { View } from 'react-native';
|
||||||
import {
|
import {
|
||||||
getExplorerItemData,
|
getExplorerItemData,
|
||||||
getItemFilePath,
|
getItemFilePath,
|
||||||
getItemLocation,
|
getItemLocation,
|
||||||
isDarkTheme,
|
isDarkTheme,
|
||||||
|
ThumbKey,
|
||||||
type ExplorerItem
|
type ExplorerItem
|
||||||
} from '@sd/client';
|
} from '@sd/client';
|
||||||
import { Image } from 'expo-image';
|
|
||||||
import { useEffect, useLayoutEffect, useMemo, useState, type PropsWithChildren } from 'react';
|
|
||||||
import { View } from 'react-native';
|
|
||||||
import { flattenThumbnailKey, useExplorerStore } from '~/stores/explorerStore';
|
import { flattenThumbnailKey, useExplorerStore } from '~/stores/explorerStore';
|
||||||
|
|
||||||
import { tw } from '../../lib/tailwind';
|
import { tw } from '../../lib/tailwind';
|
||||||
|
|
||||||
// NOTE: `file://` is required for Android to load local files!
|
// NOTE: `file://` is required for Android to load local files!
|
||||||
export const getThumbnailUrlByThumbKey = (thumbKey: string[]) => {
|
export const getThumbnailUrlByThumbKey = (thumbKey: ThumbKey) => {
|
||||||
return `file://${DocumentDirectoryPath}/thumbnails/${thumbKey
|
return `file://${DocumentDirectoryPath}/thumbnails/${encodeURIComponent(
|
||||||
.map((i) => encodeURIComponent(i))
|
thumbKey.base_directory_str
|
||||||
.join('/')}.webp`;
|
)}/${encodeURIComponent(thumbKey.shard_hex)}/${encodeURIComponent(thumbKey.cas_id)}.webp`;
|
||||||
};
|
};
|
||||||
|
|
||||||
const FileThumbWrapper = ({ children, size = 1 }: PropsWithChildren<{ size: number }>) => (
|
const FileThumbWrapper = ({ children, size = 1 }: PropsWithChildren<{ size: number }>) => (
|
||||||
|
|
|
@ -1,4 +1,3 @@
|
||||||
import { JobProgressEvent, JobReport, useJobInfo } from '@sd/client';
|
|
||||||
import {
|
import {
|
||||||
Copy,
|
Copy,
|
||||||
Fingerprint,
|
Fingerprint,
|
||||||
|
@ -11,13 +10,14 @@ import {
|
||||||
} from 'phosphor-react-native';
|
} from 'phosphor-react-native';
|
||||||
import { memo } from 'react';
|
import { memo } from 'react';
|
||||||
import { View, ViewStyle } from 'react-native';
|
import { View, ViewStyle } from 'react-native';
|
||||||
|
import { JobProgressEvent, Report, useJobInfo } from '@sd/client';
|
||||||
import { tw, twStyle } from '~/lib/tailwind';
|
import { tw, twStyle } from '~/lib/tailwind';
|
||||||
|
|
||||||
import { ProgressBar } from '../animation/ProgressBar';
|
import { ProgressBar } from '../animation/ProgressBar';
|
||||||
import JobContainer from './JobContainer';
|
import JobContainer from './JobContainer';
|
||||||
|
|
||||||
type JobProps = {
|
type JobProps = {
|
||||||
job: JobReport;
|
job: Report;
|
||||||
isChild?: boolean;
|
isChild?: boolean;
|
||||||
containerStyle?: ViewStyle;
|
containerStyle?: ViewStyle;
|
||||||
progress: JobProgressEvent | null;
|
progress: JobProgressEvent | null;
|
||||||
|
|
|
@ -1,19 +1,19 @@
|
||||||
import { Folder } from '@sd/assets/icons';
|
import { Folder } from '@sd/assets/icons';
|
||||||
import {
|
|
||||||
getJobNiceActionName,
|
|
||||||
getTotalTasks,
|
|
||||||
JobGroup,
|
|
||||||
JobProgressEvent,
|
|
||||||
JobReport,
|
|
||||||
useLibraryMutation,
|
|
||||||
useRspcLibraryContext,
|
|
||||||
useTotalElapsedTimeText
|
|
||||||
} from '@sd/client';
|
|
||||||
import dayjs from 'dayjs';
|
import dayjs from 'dayjs';
|
||||||
import { DotsThreeVertical, Eye, Pause, Play, Stop, Trash } from 'phosphor-react-native';
|
import { DotsThreeVertical, Eye, Pause, Play, Stop, Trash } from 'phosphor-react-native';
|
||||||
import { SetStateAction, useMemo, useState } from 'react';
|
import { SetStateAction, useMemo, useState } from 'react';
|
||||||
import { Animated, Pressable, View } from 'react-native';
|
import { Animated, Pressable, View } from 'react-native';
|
||||||
import { Swipeable } from 'react-native-gesture-handler';
|
import { Swipeable } from 'react-native-gesture-handler';
|
||||||
|
import {
|
||||||
|
getJobNiceActionName,
|
||||||
|
getTotalTasks,
|
||||||
|
JobGroup,
|
||||||
|
JobProgressEvent,
|
||||||
|
Report,
|
||||||
|
useLibraryMutation,
|
||||||
|
useRspcLibraryContext,
|
||||||
|
useTotalElapsedTimeText
|
||||||
|
} from '@sd/client';
|
||||||
import { tw, twStyle } from '~/lib/tailwind';
|
import { tw, twStyle } from '~/lib/tailwind';
|
||||||
|
|
||||||
import { AnimatedHeight } from '../animation/layout';
|
import { AnimatedHeight } from '../animation/layout';
|
||||||
|
@ -64,7 +64,12 @@ export default function ({ group, progress }: JobGroupProps) {
|
||||||
{ transform: [{ translateX: translate }] }
|
{ transform: [{ translateX: translate }] }
|
||||||
]}
|
]}
|
||||||
>
|
>
|
||||||
<Options showChildJobs={showChildJobs} setShowChildJobs={setShowChildJobs} activeJob={runningJob} group={group} />
|
<Options
|
||||||
|
showChildJobs={showChildJobs}
|
||||||
|
setShowChildJobs={setShowChildJobs}
|
||||||
|
activeJob={runningJob}
|
||||||
|
group={group}
|
||||||
|
/>
|
||||||
</Animated.View>
|
</Animated.View>
|
||||||
);
|
);
|
||||||
};
|
};
|
||||||
|
@ -169,22 +174,20 @@ const toastErrorSuccess = (
|
||||||
};
|
};
|
||||||
|
|
||||||
interface OptionsProps {
|
interface OptionsProps {
|
||||||
activeJob?: JobReport;
|
activeJob?: Report;
|
||||||
group: JobGroup;
|
group: JobGroup;
|
||||||
showChildJobs: boolean;
|
showChildJobs: boolean;
|
||||||
setShowChildJobs: React.Dispatch<SetStateAction<boolean>>
|
setShowChildJobs: React.Dispatch<SetStateAction<boolean>>;
|
||||||
}
|
}
|
||||||
|
|
||||||
function Options({ activeJob, group, setShowChildJobs, showChildJobs }: OptionsProps) {
|
function Options({ activeJob, group, setShowChildJobs, showChildJobs }: OptionsProps) {
|
||||||
|
|
||||||
const rspc = useRspcLibraryContext();
|
const rspc = useRspcLibraryContext();
|
||||||
|
|
||||||
const clearJob = useLibraryMutation(
|
const clearJob = useLibraryMutation(['jobs.clear'], {
|
||||||
['jobs.clear'], {
|
onSuccess: () => {
|
||||||
onSuccess: () => {
|
rspc.queryClient.invalidateQueries(['jobs.reports']);
|
||||||
rspc.queryClient.invalidateQueries(['jobs.reports']);
|
}
|
||||||
}
|
});
|
||||||
})
|
|
||||||
|
|
||||||
const resumeJob = useLibraryMutation(
|
const resumeJob = useLibraryMutation(
|
||||||
['jobs.resume'],
|
['jobs.resume'],
|
||||||
|
@ -208,8 +211,7 @@ function Options({ activeJob, group, setShowChildJobs, showChildJobs }: OptionsP
|
||||||
group.jobs.forEach((job) => {
|
group.jobs.forEach((job) => {
|
||||||
clearJob.mutate(job.id);
|
clearJob.mutate(job.id);
|
||||||
//only one toast for all jobs
|
//only one toast for all jobs
|
||||||
if (job.id === group.id)
|
if (job.id === group.id) toast.success('Job has been removed');
|
||||||
toast.success('Job has been removed');
|
|
||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -217,35 +219,68 @@ function Options({ activeJob, group, setShowChildJobs, showChildJobs }: OptionsP
|
||||||
<>
|
<>
|
||||||
{/* Resume */}
|
{/* Resume */}
|
||||||
{(group.status === 'Queued' || group.status === 'Paused' || isJobPaused) && (
|
{(group.status === 'Queued' || group.status === 'Paused' || isJobPaused) && (
|
||||||
<Button style={tw`h-7 w-7`} variant="outline" size="sm" onPress={() => resumeJob.mutate(group.id)}>
|
<Button
|
||||||
|
style={tw`h-7 w-7`}
|
||||||
|
variant="outline"
|
||||||
|
size="sm"
|
||||||
|
onPress={() =>
|
||||||
|
resumeJob.mutate(
|
||||||
|
group.running_job_id != null ? group.running_job_id : group.id
|
||||||
|
)
|
||||||
|
}
|
||||||
|
>
|
||||||
<Play size={16} color="white" />
|
<Play size={16} color="white" />
|
||||||
</Button>
|
</Button>
|
||||||
)}
|
)}
|
||||||
{/* TODO: This should remove the job from panel */}
|
{/* TODO: This should remove the job from panel */}
|
||||||
{!activeJob !== undefined ? (
|
{activeJob !== undefined ? (
|
||||||
<Menu
|
|
||||||
containerStyle={tw`max-w-25`}
|
|
||||||
trigger={
|
|
||||||
<View style={tw`flex h-7 w-7 flex-row items-center justify-center rounded-md border border-app-inputborder`}>
|
|
||||||
<DotsThreeVertical size={16} color="white" />
|
|
||||||
</View>
|
|
||||||
}
|
|
||||||
>
|
|
||||||
<MenuItem
|
|
||||||
style={twStyle(showChildJobs ? 'rounded bg-app-screen/50' : 'bg-transparent')}
|
|
||||||
onSelect={() => setShowChildJobs(!showChildJobs)}
|
|
||||||
text="Expand" icon={Eye}/>
|
|
||||||
<MenuItem onSelect={clearJobHandler} text='Remove' icon={Trash}/>
|
|
||||||
</Menu>
|
|
||||||
) : (
|
|
||||||
<View style={tw`flex flex-row gap-2`}>
|
<View style={tw`flex flex-row gap-2`}>
|
||||||
<Button style={tw`h-7 w-7`} variant="outline" size="sm" onPress={() => pauseJob.mutate(group.id)}>
|
<Button
|
||||||
|
style={tw`h-7 w-7`}
|
||||||
|
variant="outline"
|
||||||
|
size="sm"
|
||||||
|
onPress={() =>
|
||||||
|
pauseJob.mutate(
|
||||||
|
group.running_job_id != null ? group.running_job_id : group.id
|
||||||
|
)
|
||||||
|
}
|
||||||
|
>
|
||||||
<Pause size={16} color="white" />
|
<Pause size={16} color="white" />
|
||||||
</Button>
|
</Button>
|
||||||
<Button style={tw`h-7 w-7`} variant="outline" size="sm" onPress={() => cancelJob.mutate(group.id)}>
|
<Button
|
||||||
|
style={tw`h-7 w-7`}
|
||||||
|
variant="outline"
|
||||||
|
size="sm"
|
||||||
|
onPress={() =>
|
||||||
|
cancelJob.mutate(
|
||||||
|
group.running_job_id != null ? group.running_job_id : group.id
|
||||||
|
)
|
||||||
|
}
|
||||||
|
>
|
||||||
<Stop size={16} color="white" />
|
<Stop size={16} color="white" />
|
||||||
</Button>
|
</Button>
|
||||||
</View>
|
</View>
|
||||||
|
) : (
|
||||||
|
<Menu
|
||||||
|
containerStyle={tw`max-w-25`}
|
||||||
|
trigger={
|
||||||
|
<View
|
||||||
|
style={tw`flex h-7 w-7 flex-row items-center justify-center rounded-md border border-app-inputborder`}
|
||||||
|
>
|
||||||
|
<DotsThreeVertical size={16} color="white" />
|
||||||
|
</View>
|
||||||
|
}
|
||||||
|
>
|
||||||
|
<MenuItem
|
||||||
|
style={twStyle(
|
||||||
|
showChildJobs ? 'rounded bg-app-screen/50' : 'bg-transparent'
|
||||||
|
)}
|
||||||
|
onSelect={() => setShowChildJobs(!showChildJobs)}
|
||||||
|
text="Expand"
|
||||||
|
icon={Eye}
|
||||||
|
/>
|
||||||
|
<MenuItem onSelect={clearJobHandler} text="Remove" icon={Trash} />
|
||||||
|
</Menu>
|
||||||
)}
|
)}
|
||||||
</>
|
</>
|
||||||
);
|
);
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
import { resetStore } from '@sd/client';
|
import { ThumbKey, resetStore } from '@sd/client';
|
||||||
import { proxy, useSnapshot } from 'valtio';
|
import { proxy, useSnapshot } from 'valtio';
|
||||||
import { proxySet } from 'valtio/utils';
|
import { proxySet } from 'valtio/utils';
|
||||||
|
|
||||||
|
@ -26,14 +26,14 @@ const state = {
|
||||||
orderDirection: 'Asc' as 'Asc' | 'Desc'
|
orderDirection: 'Asc' as 'Asc' | 'Desc'
|
||||||
};
|
};
|
||||||
|
|
||||||
export function flattenThumbnailKey(thumbKey: string[]) {
|
export function flattenThumbnailKey(thumbKey: ThumbKey) {
|
||||||
return thumbKey.join('/');
|
return `${thumbKey.base_directory_str}/${thumbKey.shard_hex}/${thumbKey.cas_id}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
const store = proxy({
|
const store = proxy({
|
||||||
...state,
|
...state,
|
||||||
reset: () => resetStore(store, state),
|
reset: () => resetStore(store, state),
|
||||||
addNewThumbnail: (thumbKey: string[]) => {
|
addNewThumbnail: (thumbKey: ThumbKey) => {
|
||||||
store.newThumbnails.add(flattenThumbnailKey(thumbKey));
|
store.newThumbnails.add(flattenThumbnailKey(thumbKey));
|
||||||
},
|
},
|
||||||
// this should be done when the explorer query is refreshed
|
// this should be done when the explorer query is refreshed
|
||||||
|
|
|
@ -42,8 +42,10 @@ const spacedriveURL = (() => {
|
||||||
|
|
||||||
const platform: Platform = {
|
const platform: Platform = {
|
||||||
platform: 'web',
|
platform: 'web',
|
||||||
getThumbnailUrlByThumbKey: (keyParts) =>
|
getThumbnailUrlByThumbKey: (thumbKey) =>
|
||||||
`${spacedriveURL}/thumbnail/${keyParts.map((i) => encodeURIComponent(i)).join('/')}.webp`,
|
`${spacedriveURL}/thumbnail/${encodeURIComponent(
|
||||||
|
thumbKey.base_directory_str
|
||||||
|
)}/${encodeURIComponent(thumbKey.shard_hex)}/${encodeURIComponent(thumbKey.cas_id)}.webp`,
|
||||||
getFileUrl: (libraryId, locationLocalId, filePathId) =>
|
getFileUrl: (libraryId, locationLocalId, filePathId) =>
|
||||||
`${spacedriveURL}/file/${encodeURIComponent(libraryId)}/${encodeURIComponent(
|
`${spacedriveURL}/file/${encodeURIComponent(libraryId)}/${encodeURIComponent(
|
||||||
locationLocalId
|
locationLocalId
|
||||||
|
|
|
@ -47,6 +47,7 @@ sd-p2p-proto = { path = "../crates/p2p/crates/proto" }
|
||||||
sd-p2p-tunnel = { path = "../crates/p2p/crates/tunnel" }
|
sd-p2p-tunnel = { path = "../crates/p2p/crates/tunnel" }
|
||||||
sd-prisma = { path = "../crates/prisma" }
|
sd-prisma = { path = "../crates/prisma" }
|
||||||
sd-sync = { path = "../crates/sync" }
|
sd-sync = { path = "../crates/sync" }
|
||||||
|
sd-task-system = { path = "../crates/task-system" }
|
||||||
sd-utils = { path = "../crates/utils" }
|
sd-utils = { path = "../crates/utils" }
|
||||||
|
|
||||||
# Workspace dependencies
|
# Workspace dependencies
|
||||||
|
|
|
@ -1,3 +1,5 @@
|
||||||
|
use sd_core_prisma_helpers::CasId;
|
||||||
|
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
|
|
||||||
use blake3::Hasher;
|
use blake3::Hasher;
|
||||||
|
@ -6,6 +8,7 @@ use tokio::{
|
||||||
fs::{self, File},
|
fs::{self, File},
|
||||||
io::{self, AsyncReadExt, AsyncSeekExt, SeekFrom},
|
io::{self, AsyncReadExt, AsyncSeekExt, SeekFrom},
|
||||||
};
|
};
|
||||||
|
use tracing::{instrument, trace, Level};
|
||||||
|
|
||||||
const SAMPLE_COUNT: u64 = 4;
|
const SAMPLE_COUNT: u64 = 4;
|
||||||
const SAMPLE_SIZE: u64 = 1024 * 10;
|
const SAMPLE_SIZE: u64 = 1024 * 10;
|
||||||
|
@ -20,20 +23,29 @@ const_assert!((HEADER_OR_FOOTER_SIZE * 2 + SAMPLE_COUNT * SAMPLE_SIZE) < MINIMUM
|
||||||
// Asserting that the sample size is larger than header/footer size, as the same buffer is used for both
|
// Asserting that the sample size is larger than header/footer size, as the same buffer is used for both
|
||||||
const_assert!(SAMPLE_SIZE > HEADER_OR_FOOTER_SIZE);
|
const_assert!(SAMPLE_SIZE > HEADER_OR_FOOTER_SIZE);
|
||||||
|
|
||||||
|
#[instrument(
|
||||||
|
skip(path),
|
||||||
|
ret(level = Level::TRACE),
|
||||||
|
err,
|
||||||
|
fields(path = %path.as_ref().display()
|
||||||
|
))]
|
||||||
// SAFETY: Casts here are safe, they're hardcoded values we have some const assertions above to make sure they're correct
|
// SAFETY: Casts here are safe, they're hardcoded values we have some const assertions above to make sure they're correct
|
||||||
#[allow(clippy::cast_possible_truncation)]
|
#[allow(clippy::cast_possible_truncation)]
|
||||||
#[allow(clippy::cast_possible_wrap)]
|
#[allow(clippy::cast_possible_wrap)]
|
||||||
pub async fn generate_cas_id(
|
pub async fn generate_cas_id(
|
||||||
path: impl AsRef<Path> + Send,
|
path: impl AsRef<Path> + Send,
|
||||||
size: u64,
|
size: u64,
|
||||||
) -> Result<String, io::Error> {
|
) -> Result<CasId<'static>, io::Error> {
|
||||||
let mut hasher = Hasher::new();
|
let mut hasher = Hasher::new();
|
||||||
hasher.update(&size.to_le_bytes());
|
hasher.update(&size.to_le_bytes());
|
||||||
|
|
||||||
if size <= MINIMUM_FILE_SIZE {
|
if size <= MINIMUM_FILE_SIZE {
|
||||||
|
trace!("File is small, hashing the whole file");
|
||||||
// For small files, we hash the whole file
|
// For small files, we hash the whole file
|
||||||
hasher.update(&fs::read(path).await?);
|
hasher.update(&fs::read(path).await?);
|
||||||
} else {
|
} else {
|
||||||
|
trace!("File bigger than threshold, hashing samples");
|
||||||
|
|
||||||
let mut file = File::open(path).await?;
|
let mut file = File::open(path).await?;
|
||||||
let mut buf = vec![0; SAMPLE_SIZE as usize].into_boxed_slice();
|
let mut buf = vec![0; SAMPLE_SIZE as usize].into_boxed_slice();
|
||||||
|
|
||||||
|
@ -64,5 +76,5 @@ pub async fn generate_cas_id(
|
||||||
hasher.update(&buf[..HEADER_OR_FOOTER_SIZE as usize]);
|
hasher.update(&buf[..HEADER_OR_FOOTER_SIZE as usize]);
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(hasher.finalize().to_hex()[..16].to_string())
|
Ok(hasher.finalize().to_hex()[..16].to_string().into())
|
||||||
}
|
}
|
||||||
|
|
File diff suppressed because it is too large
Load diff
|
@ -1,12 +1,20 @@
|
||||||
use crate::utils::sub_path;
|
use crate::{utils::sub_path, OuterContext};
|
||||||
|
|
||||||
use sd_core_file_path_helper::{FilePathError, IsolatedFilePathData};
|
use sd_core_file_path_helper::{FilePathError, IsolatedFilePathData};
|
||||||
|
use sd_core_prisma_helpers::CasId;
|
||||||
|
|
||||||
use sd_file_ext::{extensions::Extension, kind::ObjectKind};
|
use sd_file_ext::{extensions::Extension, kind::ObjectKind};
|
||||||
use sd_prisma::prisma::{file_path, location};
|
use sd_prisma::prisma::{file_path, location};
|
||||||
|
use sd_task_system::{TaskDispatcher, TaskHandle};
|
||||||
use sd_utils::{db::MissingFieldError, error::FileIOError};
|
use sd_utils::{db::MissingFieldError, error::FileIOError};
|
||||||
|
|
||||||
use std::{fs::Metadata, path::Path};
|
use std::{
|
||||||
|
collections::{hash_map::Entry, HashMap},
|
||||||
|
fs::Metadata,
|
||||||
|
mem,
|
||||||
|
path::Path,
|
||||||
|
sync::Arc,
|
||||||
|
};
|
||||||
|
|
||||||
use prisma_client_rust::{or, QueryError};
|
use prisma_client_rust::{or, QueryError};
|
||||||
use rspc::ErrorCode;
|
use rspc::ErrorCode;
|
||||||
|
@ -20,11 +28,13 @@ pub mod job;
|
||||||
mod shallow;
|
mod shallow;
|
||||||
mod tasks;
|
mod tasks;
|
||||||
|
|
||||||
use cas_id::generate_cas_id;
|
pub use cas_id::generate_cas_id;
|
||||||
|
|
||||||
pub use job::FileIdentifier;
|
pub use job::FileIdentifier;
|
||||||
pub use shallow::shallow;
|
pub use shallow::shallow;
|
||||||
|
|
||||||
|
use tasks::FilePathToCreateOrLinkObject;
|
||||||
|
|
||||||
// we break these tasks into chunks of 100 to improve performance
|
// we break these tasks into chunks of 100 to improve performance
|
||||||
const CHUNK_SIZE: usize = 100;
|
const CHUNK_SIZE: usize = 100;
|
||||||
|
|
||||||
|
@ -44,17 +54,18 @@ pub enum Error {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl From<Error> for rspc::Error {
|
impl From<Error> for rspc::Error {
|
||||||
fn from(err: Error) -> Self {
|
fn from(e: Error) -> Self {
|
||||||
match err {
|
match e {
|
||||||
Error::SubPath(sub_path_err) => sub_path_err.into(),
|
Error::SubPath(sub_path_err) => sub_path_err.into(),
|
||||||
|
|
||||||
_ => Self::with_cause(ErrorCode::InternalServerError, err.to_string(), err),
|
_ => Self::with_cause(ErrorCode::InternalServerError, e.to_string(), e),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(thiserror::Error, Debug, Serialize, Deserialize, Type)]
|
#[derive(thiserror::Error, Debug, Serialize, Deserialize, Type, Clone)]
|
||||||
pub enum NonCriticalError {
|
#[serde(rename_all = "snake_case")]
|
||||||
|
pub enum NonCriticalFileIdentifierError {
|
||||||
#[error("failed to extract file metadata: {0}")]
|
#[error("failed to extract file metadata: {0}")]
|
||||||
FailedToExtractFileMetadata(String),
|
FailedToExtractFileMetadata(String),
|
||||||
#[cfg(target_os = "windows")]
|
#[cfg(target_os = "windows")]
|
||||||
|
@ -66,7 +77,7 @@ pub enum NonCriticalError {
|
||||||
|
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
pub struct FileMetadata {
|
pub struct FileMetadata {
|
||||||
pub cas_id: Option<String>,
|
pub cas_id: Option<CasId<'static>>,
|
||||||
pub kind: ObjectKind,
|
pub kind: ObjectKind,
|
||||||
pub fs_metadata: Metadata,
|
pub fs_metadata: Metadata,
|
||||||
}
|
}
|
||||||
|
@ -87,10 +98,14 @@ impl FileMetadata {
|
||||||
.await
|
.await
|
||||||
.map_err(|e| FileIOError::from((&path, e)))?;
|
.map_err(|e| FileIOError::from((&path, e)))?;
|
||||||
|
|
||||||
assert!(
|
if fs_metadata.is_dir() {
|
||||||
!fs_metadata.is_dir(),
|
trace!(path = %path.display(), "Skipping directory;");
|
||||||
"We can't generate cas_id for directories"
|
return Ok(Self {
|
||||||
);
|
cas_id: None,
|
||||||
|
kind: ObjectKind::Folder,
|
||||||
|
fs_metadata,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
// derive Object kind
|
// derive Object kind
|
||||||
let kind = Extension::resolve_conflicting(&path, false)
|
let kind = Extension::resolve_conflicting(&path, false)
|
||||||
|
@ -108,8 +123,10 @@ impl FileMetadata {
|
||||||
};
|
};
|
||||||
|
|
||||||
trace!(
|
trace!(
|
||||||
"Analyzed file: <path='{}', cas_id={cas_id:?}, object_kind={kind}>",
|
path = %path.display(),
|
||||||
path.display()
|
?cas_id,
|
||||||
|
%kind,
|
||||||
|
"Analyzed file;",
|
||||||
);
|
);
|
||||||
|
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
|
@ -140,7 +157,7 @@ fn orphan_path_filters_shallow(
|
||||||
)),
|
)),
|
||||||
file_path::size_in_bytes_bytes::not(Some(0u64.to_be_bytes().to_vec())),
|
file_path::size_in_bytes_bytes::not(Some(0u64.to_be_bytes().to_vec())),
|
||||||
],
|
],
|
||||||
[file_path_id.map(file_path::id::gte)],
|
[file_path_id.map(file_path::id::gt)],
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -161,7 +178,7 @@ fn orphan_path_filters_deep(
|
||||||
],
|
],
|
||||||
[
|
[
|
||||||
// this is a workaround for the cursor not working properly
|
// this is a workaround for the cursor not working properly
|
||||||
file_path_id.map(file_path::id::gte),
|
file_path_id.map(file_path::id::gt),
|
||||||
maybe_sub_iso_file_path.as_ref().map(|sub_iso_file_path| {
|
maybe_sub_iso_file_path.as_ref().map(|sub_iso_file_path| {
|
||||||
file_path::materialized_path::starts_with(
|
file_path::materialized_path::starts_with(
|
||||||
sub_iso_file_path
|
sub_iso_file_path
|
||||||
|
@ -172,3 +189,91 @@ fn orphan_path_filters_deep(
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn dispatch_object_processor_tasks<Iter, Dispatcher>(
|
||||||
|
file_paths_by_cas_id: Iter,
|
||||||
|
ctx: &impl OuterContext,
|
||||||
|
dispatcher: &Dispatcher,
|
||||||
|
with_priority: bool,
|
||||||
|
) -> Result<Vec<TaskHandle<crate::Error>>, Dispatcher::DispatchError>
|
||||||
|
where
|
||||||
|
Iter: IntoIterator<Item = (CasId<'static>, Vec<FilePathToCreateOrLinkObject>)> + Send,
|
||||||
|
Iter::IntoIter: Send,
|
||||||
|
Dispatcher: TaskDispatcher<crate::Error>,
|
||||||
|
{
|
||||||
|
let mut current_batch = HashMap::<_, Vec<_>>::new();
|
||||||
|
let mut tasks = vec![];
|
||||||
|
|
||||||
|
let mut current_batch_size = 0;
|
||||||
|
|
||||||
|
for (cas_id, objects_to_create_or_link) in file_paths_by_cas_id {
|
||||||
|
if objects_to_create_or_link.len() >= CHUNK_SIZE {
|
||||||
|
tasks.push(
|
||||||
|
dispatcher
|
||||||
|
.dispatch(tasks::ObjectProcessor::new(
|
||||||
|
HashMap::from([(cas_id, objects_to_create_or_link)]),
|
||||||
|
Arc::clone(ctx.db()),
|
||||||
|
Arc::clone(ctx.sync()),
|
||||||
|
with_priority,
|
||||||
|
))
|
||||||
|
.await?,
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
current_batch_size += objects_to_create_or_link.len();
|
||||||
|
match current_batch.entry(cas_id) {
|
||||||
|
Entry::Occupied(entry) => {
|
||||||
|
entry.into_mut().extend(objects_to_create_or_link);
|
||||||
|
}
|
||||||
|
Entry::Vacant(entry) => {
|
||||||
|
entry.insert(objects_to_create_or_link);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if current_batch_size >= CHUNK_SIZE {
|
||||||
|
tasks.push(
|
||||||
|
dispatcher
|
||||||
|
.dispatch(tasks::ObjectProcessor::new(
|
||||||
|
mem::take(&mut current_batch),
|
||||||
|
Arc::clone(ctx.db()),
|
||||||
|
Arc::clone(ctx.sync()),
|
||||||
|
with_priority,
|
||||||
|
))
|
||||||
|
.await?,
|
||||||
|
);
|
||||||
|
|
||||||
|
current_batch_size = 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if !current_batch.is_empty() {
|
||||||
|
tasks.push(
|
||||||
|
dispatcher
|
||||||
|
.dispatch(tasks::ObjectProcessor::new(
|
||||||
|
current_batch,
|
||||||
|
Arc::clone(ctx.db()),
|
||||||
|
Arc::clone(ctx.sync()),
|
||||||
|
with_priority,
|
||||||
|
))
|
||||||
|
.await?,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(tasks)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn accumulate_file_paths_by_cas_id(
|
||||||
|
input: HashMap<CasId<'static>, Vec<FilePathToCreateOrLinkObject>>,
|
||||||
|
accumulator: &mut HashMap<CasId<'static>, Vec<FilePathToCreateOrLinkObject>>,
|
||||||
|
) {
|
||||||
|
for (cas_id, file_paths) in input {
|
||||||
|
match accumulator.entry(cas_id) {
|
||||||
|
Entry::<_, Vec<_>>::Occupied(entry) => {
|
||||||
|
entry.into_mut().extend(file_paths);
|
||||||
|
}
|
||||||
|
Entry::Vacant(entry) => {
|
||||||
|
entry.insert(file_paths);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
use crate::{
|
use crate::{
|
||||||
file_identifier, utils::sub_path::maybe_get_iso_file_path_from_sub_path, Error,
|
file_identifier, utils::sub_path::maybe_get_iso_file_path_from_sub_path, Error,
|
||||||
NonCriticalError, OuterContext,
|
NonCriticalError, OuterContext, UpdateEvent,
|
||||||
};
|
};
|
||||||
|
|
||||||
use sd_core_file_path_helper::IsolatedFilePathData;
|
use sd_core_file_path_helper::IsolatedFilePathData;
|
||||||
|
@ -8,34 +8,40 @@ use sd_core_prisma_helpers::file_path_for_file_identifier;
|
||||||
|
|
||||||
use sd_prisma::prisma::{file_path, location, SortOrder};
|
use sd_prisma::prisma::{file_path, location, SortOrder};
|
||||||
use sd_task_system::{
|
use sd_task_system::{
|
||||||
BaseTaskDispatcher, CancelTaskOnDrop, TaskDispatcher, TaskOutput, TaskStatus,
|
BaseTaskDispatcher, CancelTaskOnDrop, TaskDispatcher, TaskHandle, TaskOutput, TaskStatus,
|
||||||
};
|
};
|
||||||
use sd_utils::db::maybe_missing;
|
use sd_utils::db::maybe_missing;
|
||||||
|
|
||||||
use std::{
|
use std::{
|
||||||
|
collections::HashMap,
|
||||||
path::{Path, PathBuf},
|
path::{Path, PathBuf},
|
||||||
sync::Arc,
|
sync::Arc,
|
||||||
};
|
};
|
||||||
|
|
||||||
use futures_concurrency::future::FutureGroup;
|
use futures::{stream::FuturesUnordered, StreamExt};
|
||||||
use lending_stream::{LendingStream, StreamExt};
|
use tracing::{debug, instrument, trace, warn};
|
||||||
use tracing::{debug, warn};
|
|
||||||
|
|
||||||
use super::{
|
use super::{
|
||||||
orphan_path_filters_shallow,
|
accumulate_file_paths_by_cas_id, dispatch_object_processor_tasks, orphan_path_filters_shallow,
|
||||||
tasks::{
|
tasks::{self, identifier, object_processor},
|
||||||
extract_file_metadata, object_processor, ExtractFileMetadataTask, ObjectProcessorTask,
|
|
||||||
},
|
|
||||||
CHUNK_SIZE,
|
CHUNK_SIZE,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
#[instrument(
|
||||||
|
skip_all,
|
||||||
|
fields(
|
||||||
|
location_id = location.id,
|
||||||
|
location_path = ?location.path,
|
||||||
|
sub_path = %sub_path.as_ref().display()
|
||||||
|
)
|
||||||
|
err,
|
||||||
|
)]
|
||||||
pub async fn shallow(
|
pub async fn shallow(
|
||||||
location: location::Data,
|
location: location::Data,
|
||||||
sub_path: impl AsRef<Path> + Send,
|
sub_path: impl AsRef<Path> + Send,
|
||||||
dispatcher: BaseTaskDispatcher<Error>,
|
dispatcher: &BaseTaskDispatcher<Error>,
|
||||||
ctx: impl OuterContext,
|
ctx: &impl OuterContext,
|
||||||
) -> Result<Vec<NonCriticalError>, Error> {
|
) -> Result<Vec<NonCriticalError>, Error> {
|
||||||
let sub_path = sub_path.as_ref();
|
|
||||||
let db = ctx.db();
|
let db = ctx.db();
|
||||||
|
|
||||||
let location_path = maybe_missing(&location.path, "location.path")
|
let location_path = maybe_missing(&location.path, "location.path")
|
||||||
|
@ -45,22 +51,25 @@ pub async fn shallow(
|
||||||
|
|
||||||
let location = Arc::new(location);
|
let location = Arc::new(location);
|
||||||
|
|
||||||
let sub_iso_file_path =
|
let sub_iso_file_path = maybe_get_iso_file_path_from_sub_path::<file_identifier::Error>(
|
||||||
maybe_get_iso_file_path_from_sub_path(location.id, &Some(sub_path), &*location_path, db)
|
location.id,
|
||||||
.await
|
Some(sub_path.as_ref()),
|
||||||
.map_err(file_identifier::Error::from)?
|
&*location_path,
|
||||||
.map_or_else(
|
db,
|
||||||
|| {
|
)
|
||||||
IsolatedFilePathData::new(location.id, &*location_path, &*location_path, true)
|
.await?
|
||||||
.map_err(file_identifier::Error::from)
|
.map_or_else(
|
||||||
},
|
|| {
|
||||||
Ok,
|
IsolatedFilePathData::new(location.id, &*location_path, &*location_path, true)
|
||||||
)?;
|
.map_err(file_identifier::Error::from)
|
||||||
|
},
|
||||||
|
Ok,
|
||||||
|
)?;
|
||||||
|
|
||||||
let mut orphans_count = 0;
|
let mut orphans_count = 0;
|
||||||
let mut last_orphan_file_path_id = None;
|
let mut last_orphan_file_path_id = None;
|
||||||
|
|
||||||
let mut pending_running_tasks = FutureGroup::new();
|
let mut identifier_tasks = vec![];
|
||||||
|
|
||||||
loop {
|
loop {
|
||||||
#[allow(clippy::cast_possible_wrap)]
|
#[allow(clippy::cast_possible_wrap)]
|
||||||
|
@ -87,70 +96,89 @@ pub async fn shallow(
|
||||||
orphans_count += orphan_paths.len() as u64;
|
orphans_count += orphan_paths.len() as u64;
|
||||||
last_orphan_file_path_id = Some(last_orphan.id);
|
last_orphan_file_path_id = Some(last_orphan.id);
|
||||||
|
|
||||||
pending_running_tasks.insert(CancelTaskOnDrop(
|
let Ok(tasks) = dispatcher
|
||||||
dispatcher
|
.dispatch(tasks::Identifier::new(
|
||||||
.dispatch(ExtractFileMetadataTask::new(
|
Arc::clone(&location),
|
||||||
Arc::clone(&location),
|
Arc::clone(&location_path),
|
||||||
Arc::clone(&location_path),
|
orphan_paths,
|
||||||
orphan_paths,
|
true,
|
||||||
true,
|
Arc::clone(ctx.db()),
|
||||||
))
|
Arc::clone(ctx.sync()),
|
||||||
.await,
|
))
|
||||||
));
|
.await
|
||||||
|
else {
|
||||||
|
debug!("Task system is shutting down while a shallow file identifier was in progress");
|
||||||
|
return Ok(vec![]);
|
||||||
|
};
|
||||||
|
|
||||||
|
identifier_tasks.push(tasks);
|
||||||
}
|
}
|
||||||
|
|
||||||
if orphans_count == 0 {
|
if orphans_count == 0 {
|
||||||
debug!(
|
trace!("No orphans found");
|
||||||
"No orphans found on <location_id={}, sub_path='{}'>",
|
|
||||||
location.id,
|
|
||||||
sub_path.display()
|
|
||||||
);
|
|
||||||
return Ok(vec![]);
|
return Ok(vec![]);
|
||||||
}
|
}
|
||||||
|
|
||||||
let errors = process_tasks(pending_running_tasks, dispatcher, ctx).await?;
|
process_tasks(identifier_tasks, dispatcher, ctx).await
|
||||||
|
|
||||||
Ok(errors)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn process_tasks(
|
async fn process_tasks(
|
||||||
pending_running_tasks: FutureGroup<CancelTaskOnDrop<Error>>,
|
identifier_tasks: Vec<TaskHandle<Error>>,
|
||||||
dispatcher: BaseTaskDispatcher<Error>,
|
dispatcher: &BaseTaskDispatcher<Error>,
|
||||||
ctx: impl OuterContext,
|
ctx: &impl OuterContext,
|
||||||
) -> Result<Vec<NonCriticalError>, Error> {
|
) -> Result<Vec<NonCriticalError>, Error> {
|
||||||
let mut pending_running_tasks = pending_running_tasks.lend_mut();
|
let total_identifier_tasks = identifier_tasks.len();
|
||||||
|
|
||||||
let db = ctx.db();
|
let mut pending_running_tasks = identifier_tasks
|
||||||
let sync = ctx.sync();
|
.into_iter()
|
||||||
|
.map(CancelTaskOnDrop::new)
|
||||||
|
.collect::<FuturesUnordered<_>>();
|
||||||
|
|
||||||
let mut errors = vec![];
|
let mut errors = vec![];
|
||||||
|
let mut completed_identifier_tasks = 0;
|
||||||
|
let mut file_paths_accumulator = HashMap::new();
|
||||||
|
|
||||||
while let Some((pending_running_tasks, task_result)) = pending_running_tasks.next().await {
|
while let Some(task_result) = pending_running_tasks.next().await {
|
||||||
match task_result {
|
match task_result {
|
||||||
Ok(TaskStatus::Done((_, TaskOutput::Out(any_task_output)))) => {
|
Ok(TaskStatus::Done((_, TaskOutput::Out(any_task_output)))) => {
|
||||||
// We only care about ExtractFileMetadataTaskOutput because we need to dispatch further tasks
|
// We only care about ExtractFileMetadataTaskOutput because we need to dispatch further tasks
|
||||||
// and the ObjectProcessorTask only gives back some metrics not much important for
|
// and the ObjectProcessorTask only gives back some metrics not much important for
|
||||||
// shallow file identifier
|
// shallow file identifier
|
||||||
if any_task_output.is::<extract_file_metadata::Output>() {
|
if any_task_output.is::<identifier::Output>() {
|
||||||
let extract_file_metadata::Output {
|
let identifier::Output {
|
||||||
identified_files,
|
file_path_ids_with_new_object,
|
||||||
|
file_paths_by_cas_id,
|
||||||
errors: more_errors,
|
errors: more_errors,
|
||||||
..
|
..
|
||||||
} = *any_task_output.downcast().expect("just checked");
|
} = *any_task_output.downcast().expect("just checked");
|
||||||
|
|
||||||
|
completed_identifier_tasks += 1;
|
||||||
|
|
||||||
|
ctx.report_update(UpdateEvent::NewIdentifiedObjects {
|
||||||
|
file_path_ids: file_path_ids_with_new_object,
|
||||||
|
});
|
||||||
|
|
||||||
|
accumulate_file_paths_by_cas_id(
|
||||||
|
file_paths_by_cas_id,
|
||||||
|
&mut file_paths_accumulator,
|
||||||
|
);
|
||||||
|
|
||||||
errors.extend(more_errors);
|
errors.extend(more_errors);
|
||||||
|
|
||||||
if !identified_files.is_empty() {
|
if total_identifier_tasks == completed_identifier_tasks {
|
||||||
pending_running_tasks.insert(CancelTaskOnDrop(
|
let Ok(tasks) = dispatch_object_processor_tasks(
|
||||||
dispatcher
|
file_paths_accumulator.drain(),
|
||||||
.dispatch(ObjectProcessorTask::new(
|
ctx,
|
||||||
identified_files,
|
dispatcher,
|
||||||
Arc::clone(db),
|
true,
|
||||||
Arc::clone(sync),
|
)
|
||||||
true,
|
.await
|
||||||
))
|
else {
|
||||||
.await,
|
debug!("Task system is shutting down while a shallow file identifier was in progress");
|
||||||
));
|
continue;
|
||||||
|
};
|
||||||
|
|
||||||
|
pending_running_tasks.extend(tasks.into_iter().map(CancelTaskOnDrop::new));
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
let object_processor::Output {
|
let object_processor::Output {
|
||||||
|
@ -158,21 +186,21 @@ async fn process_tasks(
|
||||||
..
|
..
|
||||||
} = *any_task_output.downcast().expect("just checked");
|
} = *any_task_output.downcast().expect("just checked");
|
||||||
|
|
||||||
ctx.report_update(crate::UpdateEvent::NewIdentifiedObjects {
|
ctx.report_update(UpdateEvent::NewIdentifiedObjects {
|
||||||
file_path_ids: file_path_ids_with_new_object,
|
file_path_ids: file_path_ids_with_new_object,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(TaskStatus::Done((task_id, TaskOutput::Empty))) => {
|
Ok(TaskStatus::Done((task_id, TaskOutput::Empty))) => {
|
||||||
warn!("Task <id='{task_id}'> returned an empty output");
|
warn!(%task_id, "Task returned an empty output");
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(TaskStatus::Shutdown(_)) => {
|
Ok(TaskStatus::Shutdown(_)) => {
|
||||||
debug!(
|
debug!(
|
||||||
"Spacedrive is shutting down while a shallow file identifier was in progress"
|
"Spacedrive is shutting down while a shallow file identifier was in progress"
|
||||||
);
|
);
|
||||||
return Ok(vec![]);
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(TaskStatus::Error(e)) => {
|
Ok(TaskStatus::Error(e)) => {
|
||||||
|
@ -181,7 +209,7 @@ async fn process_tasks(
|
||||||
|
|
||||||
Ok(TaskStatus::Canceled | TaskStatus::ForcedAbortion) => {
|
Ok(TaskStatus::Canceled | TaskStatus::ForcedAbortion) => {
|
||||||
warn!("Task was cancelled or aborted on shallow file identifier");
|
warn!("Task was cancelled or aborted on shallow file identifier");
|
||||||
return Ok(vec![]);
|
return Ok(errors);
|
||||||
}
|
}
|
||||||
|
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
|
|
|
@ -1,267 +0,0 @@
|
||||||
use crate::{
|
|
||||||
file_identifier::{self, FileMetadata},
|
|
||||||
Error, NonCriticalError,
|
|
||||||
};
|
|
||||||
|
|
||||||
use sd_core_file_path_helper::IsolatedFilePathData;
|
|
||||||
use sd_core_prisma_helpers::file_path_for_file_identifier;
|
|
||||||
|
|
||||||
use sd_prisma::prisma::location;
|
|
||||||
use sd_task_system::{
|
|
||||||
ExecStatus, Interrupter, InterruptionKind, IntoAnyTaskOutput, SerializableTask, Task, TaskId,
|
|
||||||
};
|
|
||||||
use sd_utils::error::FileIOError;
|
|
||||||
|
|
||||||
use std::{
|
|
||||||
collections::HashMap, future::IntoFuture, mem, path::PathBuf, pin::pin, sync::Arc,
|
|
||||||
time::Duration,
|
|
||||||
};
|
|
||||||
|
|
||||||
use futures::stream::{self, FuturesUnordered, StreamExt};
|
|
||||||
use futures_concurrency::stream::Merge;
|
|
||||||
use serde::{Deserialize, Serialize};
|
|
||||||
use tokio::time::Instant;
|
|
||||||
use tracing::error;
|
|
||||||
use uuid::Uuid;
|
|
||||||
|
|
||||||
use super::IdentifiedFile;
|
|
||||||
|
|
||||||
#[derive(Debug, Serialize, Deserialize)]
|
|
||||||
pub struct ExtractFileMetadataTask {
|
|
||||||
id: TaskId,
|
|
||||||
location: Arc<location::Data>,
|
|
||||||
location_path: Arc<PathBuf>,
|
|
||||||
file_paths_by_id: HashMap<Uuid, file_path_for_file_identifier::Data>,
|
|
||||||
identified_files: HashMap<Uuid, IdentifiedFile>,
|
|
||||||
extract_metadata_time: Duration,
|
|
||||||
errors: Vec<NonCriticalError>,
|
|
||||||
with_priority: bool,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug)]
|
|
||||||
pub struct Output {
|
|
||||||
pub identified_files: HashMap<Uuid, IdentifiedFile>,
|
|
||||||
pub extract_metadata_time: Duration,
|
|
||||||
pub errors: Vec<NonCriticalError>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl ExtractFileMetadataTask {
|
|
||||||
#[must_use]
|
|
||||||
pub fn new(
|
|
||||||
location: Arc<location::Data>,
|
|
||||||
location_path: Arc<PathBuf>,
|
|
||||||
file_paths: Vec<file_path_for_file_identifier::Data>,
|
|
||||||
with_priority: bool,
|
|
||||||
) -> Self {
|
|
||||||
Self {
|
|
||||||
id: TaskId::new_v4(),
|
|
||||||
location,
|
|
||||||
location_path,
|
|
||||||
identified_files: HashMap::with_capacity(file_paths.len()),
|
|
||||||
file_paths_by_id: file_paths
|
|
||||||
.into_iter()
|
|
||||||
.map(|file_path| {
|
|
||||||
// SAFETY: This should never happen
|
|
||||||
(
|
|
||||||
Uuid::from_slice(&file_path.pub_id).expect("file_path.pub_id is invalid!"),
|
|
||||||
file_path,
|
|
||||||
)
|
|
||||||
})
|
|
||||||
.collect(),
|
|
||||||
extract_metadata_time: Duration::ZERO,
|
|
||||||
errors: Vec::new(),
|
|
||||||
with_priority,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[async_trait::async_trait]
|
|
||||||
impl Task<Error> for ExtractFileMetadataTask {
|
|
||||||
fn id(&self) -> TaskId {
|
|
||||||
self.id
|
|
||||||
}
|
|
||||||
|
|
||||||
fn with_priority(&self) -> bool {
|
|
||||||
self.with_priority
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn run(&mut self, interrupter: &Interrupter) -> Result<ExecStatus, Error> {
|
|
||||||
// `Processed` is larger than `Interrupt`, but it's much more common
|
|
||||||
// so we ignore the size difference to optimize for usage
|
|
||||||
#[allow(clippy::large_enum_variant)]
|
|
||||||
enum StreamMessage {
|
|
||||||
Processed(Uuid, Result<FileMetadata, FileIOError>),
|
|
||||||
Interrupt(InterruptionKind),
|
|
||||||
}
|
|
||||||
|
|
||||||
let Self {
|
|
||||||
location,
|
|
||||||
location_path,
|
|
||||||
file_paths_by_id,
|
|
||||||
identified_files,
|
|
||||||
extract_metadata_time,
|
|
||||||
errors,
|
|
||||||
..
|
|
||||||
} = self;
|
|
||||||
|
|
||||||
let start_time = Instant::now();
|
|
||||||
|
|
||||||
if !file_paths_by_id.is_empty() {
|
|
||||||
let extraction_futures = file_paths_by_id
|
|
||||||
.iter()
|
|
||||||
.filter_map(|(file_path_id, file_path)| {
|
|
||||||
try_iso_file_path_extraction(
|
|
||||||
location.id,
|
|
||||||
*file_path_id,
|
|
||||||
file_path,
|
|
||||||
Arc::clone(location_path),
|
|
||||||
errors,
|
|
||||||
)
|
|
||||||
})
|
|
||||||
.map(|(file_path_id, iso_file_path, location_path)| async move {
|
|
||||||
StreamMessage::Processed(
|
|
||||||
file_path_id,
|
|
||||||
FileMetadata::new(&*location_path, &iso_file_path).await,
|
|
||||||
)
|
|
||||||
})
|
|
||||||
.collect::<FuturesUnordered<_>>();
|
|
||||||
|
|
||||||
let mut msg_stream = pin!((
|
|
||||||
extraction_futures,
|
|
||||||
stream::once(interrupter.into_future()).map(StreamMessage::Interrupt)
|
|
||||||
)
|
|
||||||
.merge());
|
|
||||||
|
|
||||||
while let Some(msg) = msg_stream.next().await {
|
|
||||||
match msg {
|
|
||||||
StreamMessage::Processed(file_path_pub_id, res) => {
|
|
||||||
let file_path = file_paths_by_id
|
|
||||||
.remove(&file_path_pub_id)
|
|
||||||
.expect("file_path must be here");
|
|
||||||
|
|
||||||
match res {
|
|
||||||
Ok(FileMetadata { cas_id, kind, .. }) => {
|
|
||||||
identified_files.insert(
|
|
||||||
file_path_pub_id,
|
|
||||||
IdentifiedFile {
|
|
||||||
file_path,
|
|
||||||
cas_id,
|
|
||||||
kind,
|
|
||||||
},
|
|
||||||
);
|
|
||||||
}
|
|
||||||
Err(e) => {
|
|
||||||
handle_non_critical_errors(
|
|
||||||
location.id,
|
|
||||||
file_path_pub_id,
|
|
||||||
&e,
|
|
||||||
errors,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if file_paths_by_id.is_empty() {
|
|
||||||
// All files have been processed so we can end this merged stream and don't keep waiting an
|
|
||||||
// interrupt signal
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
StreamMessage::Interrupt(kind) => {
|
|
||||||
*extract_metadata_time += start_time.elapsed();
|
|
||||||
return Ok(match kind {
|
|
||||||
InterruptionKind::Pause => ExecStatus::Paused,
|
|
||||||
InterruptionKind::Cancel => ExecStatus::Canceled,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(ExecStatus::Done(
|
|
||||||
Output {
|
|
||||||
identified_files: mem::take(identified_files),
|
|
||||||
extract_metadata_time: *extract_metadata_time + start_time.elapsed(),
|
|
||||||
errors: mem::take(errors),
|
|
||||||
}
|
|
||||||
.into_output(),
|
|
||||||
))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn handle_non_critical_errors(
|
|
||||||
location_id: location::id::Type,
|
|
||||||
file_path_pub_id: Uuid,
|
|
||||||
e: &FileIOError,
|
|
||||||
errors: &mut Vec<NonCriticalError>,
|
|
||||||
) {
|
|
||||||
error!("Failed to extract file metadata <location_id={location_id}, file_path_pub_id='{file_path_pub_id}'>: {e:#?}");
|
|
||||||
|
|
||||||
let formatted_error = format!("<file_path_pub_id='{file_path_pub_id}', error={e}>");
|
|
||||||
|
|
||||||
#[cfg(target_os = "windows")]
|
|
||||||
{
|
|
||||||
// Handle case where file is on-demand (NTFS only)
|
|
||||||
if e.source.raw_os_error().map_or(false, |code| code == 362) {
|
|
||||||
errors.push(
|
|
||||||
file_identifier::NonCriticalError::FailedToExtractMetadataFromOnDemandFile(
|
|
||||||
formatted_error,
|
|
||||||
)
|
|
||||||
.into(),
|
|
||||||
);
|
|
||||||
} else {
|
|
||||||
errors.push(
|
|
||||||
file_identifier::NonCriticalError::FailedToExtractFileMetadata(formatted_error)
|
|
||||||
.into(),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(not(target_os = "windows"))]
|
|
||||||
{
|
|
||||||
errors.push(
|
|
||||||
file_identifier::NonCriticalError::FailedToExtractFileMetadata(formatted_error).into(),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn try_iso_file_path_extraction(
|
|
||||||
location_id: location::id::Type,
|
|
||||||
file_path_pub_id: Uuid,
|
|
||||||
file_path: &file_path_for_file_identifier::Data,
|
|
||||||
location_path: Arc<PathBuf>,
|
|
||||||
errors: &mut Vec<NonCriticalError>,
|
|
||||||
) -> Option<(Uuid, IsolatedFilePathData<'static>, Arc<PathBuf>)> {
|
|
||||||
IsolatedFilePathData::try_from((location_id, file_path))
|
|
||||||
.map(IsolatedFilePathData::to_owned)
|
|
||||||
.map(|iso_file_path| (file_path_pub_id, iso_file_path, location_path))
|
|
||||||
.map_err(|e| {
|
|
||||||
error!("Failed to extract isolated file path data: {e:#?}");
|
|
||||||
errors.push(
|
|
||||||
file_identifier::NonCriticalError::FailedToExtractIsolatedFilePathData(format!(
|
|
||||||
"<file_path_pub_id='{file_path_pub_id}', error={e}>"
|
|
||||||
))
|
|
||||||
.into(),
|
|
||||||
);
|
|
||||||
})
|
|
||||||
.ok()
|
|
||||||
}
|
|
||||||
|
|
||||||
impl SerializableTask<Error> for ExtractFileMetadataTask {
|
|
||||||
type SerializeError = rmp_serde::encode::Error;
|
|
||||||
|
|
||||||
type DeserializeError = rmp_serde::decode::Error;
|
|
||||||
|
|
||||||
type DeserializeCtx = ();
|
|
||||||
|
|
||||||
async fn serialize(self) -> Result<Vec<u8>, Self::SerializeError> {
|
|
||||||
rmp_serde::to_vec_named(&self)
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn deserialize(
|
|
||||||
data: &[u8],
|
|
||||||
(): Self::DeserializeCtx,
|
|
||||||
) -> Result<Self, Self::DeserializeError> {
|
|
||||||
rmp_serde::from_slice(data)
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -0,0 +1,508 @@
|
||||||
|
use crate::{
|
||||||
|
file_identifier::{self, FileMetadata},
|
||||||
|
Error, NonCriticalError,
|
||||||
|
};
|
||||||
|
|
||||||
|
use sd_core_file_path_helper::IsolatedFilePathData;
|
||||||
|
use sd_core_prisma_helpers::{file_path_for_file_identifier, CasId, FilePathPubId};
|
||||||
|
use sd_core_sync::Manager as SyncManager;
|
||||||
|
|
||||||
|
use sd_file_ext::kind::ObjectKind;
|
||||||
|
use sd_prisma::{
|
||||||
|
prisma::{file_path, location, PrismaClient},
|
||||||
|
prisma_sync,
|
||||||
|
};
|
||||||
|
use sd_sync::OperationFactory;
|
||||||
|
use sd_task_system::{
|
||||||
|
ExecStatus, Interrupter, InterruptionKind, IntoAnyTaskOutput, SerializableTask, Task, TaskId,
|
||||||
|
};
|
||||||
|
use sd_utils::{error::FileIOError, msgpack};
|
||||||
|
|
||||||
|
use std::{
|
||||||
|
collections::HashMap, future::IntoFuture, mem, path::PathBuf, pin::pin, sync::Arc,
|
||||||
|
time::Duration,
|
||||||
|
};
|
||||||
|
|
||||||
|
use futures::stream::{self, FuturesUnordered, StreamExt};
|
||||||
|
use futures_concurrency::{future::TryJoin, stream::Merge};
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
use tokio::time::Instant;
|
||||||
|
use tracing::{error, instrument, trace, Level};
|
||||||
|
|
||||||
|
use super::{create_objects_and_update_file_paths, FilePathToCreateOrLinkObject};
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize, Deserialize)]
|
||||||
|
struct IdentifiedFile {
|
||||||
|
file_path: file_path_for_file_identifier::Data,
|
||||||
|
cas_id: CasId<'static>,
|
||||||
|
kind: ObjectKind,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl IdentifiedFile {
|
||||||
|
pub fn new(
|
||||||
|
file_path: file_path_for_file_identifier::Data,
|
||||||
|
cas_id: impl Into<CasId<'static>>,
|
||||||
|
kind: ObjectKind,
|
||||||
|
) -> Self {
|
||||||
|
Self {
|
||||||
|
file_path,
|
||||||
|
cas_id: cas_id.into(),
|
||||||
|
kind,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub struct Identifier {
|
||||||
|
// Task control
|
||||||
|
id: TaskId,
|
||||||
|
with_priority: bool,
|
||||||
|
|
||||||
|
// Received input args
|
||||||
|
location: Arc<location::Data>,
|
||||||
|
location_path: Arc<PathBuf>,
|
||||||
|
file_paths_by_id: HashMap<FilePathPubId, file_path_for_file_identifier::Data>,
|
||||||
|
|
||||||
|
// Inner state
|
||||||
|
identified_files: HashMap<FilePathPubId, IdentifiedFile>,
|
||||||
|
file_paths_without_cas_id: Vec<FilePathToCreateOrLinkObject>,
|
||||||
|
|
||||||
|
// Out collector
|
||||||
|
output: Output,
|
||||||
|
|
||||||
|
// Dependencies
|
||||||
|
db: Arc<PrismaClient>,
|
||||||
|
sync: Arc<SyncManager>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Output from the `[Identifier]` task
|
||||||
|
#[derive(Debug, Default, Serialize, Deserialize)]
|
||||||
|
pub struct Output {
|
||||||
|
/// To send to frontend for priority reporting of new objects
|
||||||
|
pub file_path_ids_with_new_object: Vec<file_path::id::Type>,
|
||||||
|
|
||||||
|
/// Files that need to be aggregate between many identifier tasks to be processed by the
|
||||||
|
/// object processor tasks
|
||||||
|
pub file_paths_by_cas_id: HashMap<CasId<'static>, Vec<FilePathToCreateOrLinkObject>>,
|
||||||
|
|
||||||
|
/// Collected metric about time elapsed extracting metadata from file system
|
||||||
|
pub extract_metadata_time: Duration,
|
||||||
|
|
||||||
|
/// Collected metric about time spent saving objects on disk
|
||||||
|
pub save_db_time: Duration,
|
||||||
|
|
||||||
|
/// Total number of objects already created as they didn't have `cas_id`, like directories or empty files
|
||||||
|
pub created_objects_count: u64,
|
||||||
|
|
||||||
|
/// Total number of files that we were able to identify
|
||||||
|
pub total_identified_files: u64,
|
||||||
|
|
||||||
|
/// Non critical errors that happened during the task execution
|
||||||
|
pub errors: Vec<NonCriticalError>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait::async_trait]
|
||||||
|
impl Task<Error> for Identifier {
|
||||||
|
fn id(&self) -> TaskId {
|
||||||
|
self.id
|
||||||
|
}
|
||||||
|
|
||||||
|
fn with_priority(&self) -> bool {
|
||||||
|
self.with_priority
|
||||||
|
}
|
||||||
|
|
||||||
|
#[instrument(
|
||||||
|
skip(self, interrupter),
|
||||||
|
fields(
|
||||||
|
task_id = %self.id,
|
||||||
|
location_id = %self.location.id,
|
||||||
|
location_path = %self.location_path.display(),
|
||||||
|
files_count = %self.file_paths_by_id.len(),
|
||||||
|
),
|
||||||
|
ret(level = Level::TRACE),
|
||||||
|
err,
|
||||||
|
)]
|
||||||
|
#[allow(clippy::blocks_in_conditions)] // Due to `err` on `instrument` macro above
|
||||||
|
async fn run(&mut self, interrupter: &Interrupter) -> Result<ExecStatus, Error> {
|
||||||
|
// `Processed` is larger than `Interrupt`, but it's much more common
|
||||||
|
// so we ignore the size difference to optimize for usage
|
||||||
|
#[allow(clippy::large_enum_variant)]
|
||||||
|
enum StreamMessage {
|
||||||
|
Processed(FilePathPubId, Result<FileMetadata, FileIOError>),
|
||||||
|
Interrupt(InterruptionKind),
|
||||||
|
}
|
||||||
|
|
||||||
|
let Self {
|
||||||
|
location,
|
||||||
|
location_path,
|
||||||
|
file_paths_by_id,
|
||||||
|
file_paths_without_cas_id,
|
||||||
|
identified_files,
|
||||||
|
output,
|
||||||
|
..
|
||||||
|
} = self;
|
||||||
|
|
||||||
|
if !file_paths_by_id.is_empty() {
|
||||||
|
let start_time = Instant::now();
|
||||||
|
|
||||||
|
let extraction_futures = file_paths_by_id
|
||||||
|
.iter()
|
||||||
|
.filter_map(|(file_path_id, file_path)| {
|
||||||
|
try_iso_file_path_extraction(
|
||||||
|
location.id,
|
||||||
|
file_path_id.clone(),
|
||||||
|
file_path,
|
||||||
|
Arc::clone(location_path),
|
||||||
|
&mut output.errors,
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.map(|(file_path_id, iso_file_path, location_path)| async move {
|
||||||
|
StreamMessage::Processed(
|
||||||
|
file_path_id,
|
||||||
|
FileMetadata::new(&*location_path, &iso_file_path).await,
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.collect::<FuturesUnordered<_>>();
|
||||||
|
|
||||||
|
let mut msg_stream = pin!((
|
||||||
|
extraction_futures,
|
||||||
|
stream::once(interrupter.into_future()).map(StreamMessage::Interrupt)
|
||||||
|
)
|
||||||
|
.merge());
|
||||||
|
|
||||||
|
while let Some(msg) = msg_stream.next().await {
|
||||||
|
match msg {
|
||||||
|
StreamMessage::Processed(file_path_pub_id, res) => {
|
||||||
|
let file_path = file_paths_by_id
|
||||||
|
.remove(&file_path_pub_id)
|
||||||
|
.expect("file_path must be here");
|
||||||
|
|
||||||
|
trace!(
|
||||||
|
files_remaining = file_paths_by_id.len(),
|
||||||
|
%file_path_pub_id,
|
||||||
|
"Processed file;",
|
||||||
|
);
|
||||||
|
|
||||||
|
match res {
|
||||||
|
Ok(FileMetadata {
|
||||||
|
cas_id: Some(cas_id),
|
||||||
|
kind,
|
||||||
|
..
|
||||||
|
}) => {
|
||||||
|
identified_files.insert(
|
||||||
|
file_path_pub_id,
|
||||||
|
IdentifiedFile::new(file_path, cas_id, kind),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
Ok(FileMetadata {
|
||||||
|
cas_id: None, kind, ..
|
||||||
|
}) => {
|
||||||
|
let file_path_for_file_identifier::Data {
|
||||||
|
id,
|
||||||
|
pub_id,
|
||||||
|
date_created,
|
||||||
|
..
|
||||||
|
} = file_path;
|
||||||
|
file_paths_without_cas_id.push(FilePathToCreateOrLinkObject {
|
||||||
|
id,
|
||||||
|
file_path_pub_id: pub_id.into(),
|
||||||
|
kind,
|
||||||
|
created_at: date_created,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
handle_non_critical_errors(
|
||||||
|
file_path_pub_id,
|
||||||
|
&e,
|
||||||
|
&mut output.errors,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if file_paths_by_id.is_empty() {
|
||||||
|
trace!("All files have been processed");
|
||||||
|
// All files have been processed so we can end this merged stream
|
||||||
|
// and don't keep waiting an interrupt signal
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
StreamMessage::Interrupt(kind) => {
|
||||||
|
trace!(?kind, "Interrupted;");
|
||||||
|
output.extract_metadata_time += start_time.elapsed();
|
||||||
|
return Ok(match kind {
|
||||||
|
InterruptionKind::Pause => ExecStatus::Paused,
|
||||||
|
InterruptionKind::Cancel => ExecStatus::Canceled,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
output.extract_metadata_time = start_time.elapsed();
|
||||||
|
|
||||||
|
output.total_identified_files =
|
||||||
|
identified_files.len() as u64 + file_paths_without_cas_id.len() as u64;
|
||||||
|
|
||||||
|
trace!(
|
||||||
|
identified_files_count = identified_files.len(),
|
||||||
|
"All files have been processed, saving cas_ids to db...;"
|
||||||
|
);
|
||||||
|
let start_time = Instant::now();
|
||||||
|
// Assign cas_id to each file path
|
||||||
|
let ((), file_path_ids_with_new_object) = (
|
||||||
|
assign_cas_id_to_file_paths(identified_files, &self.db, &self.sync),
|
||||||
|
create_objects_and_update_file_paths(
|
||||||
|
file_paths_without_cas_id.drain(..),
|
||||||
|
&self.db,
|
||||||
|
&self.sync,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.try_join()
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
output.save_db_time = start_time.elapsed();
|
||||||
|
output.created_objects_count = file_path_ids_with_new_object.len() as u64;
|
||||||
|
output.file_path_ids_with_new_object =
|
||||||
|
file_path_ids_with_new_object.into_keys().collect();
|
||||||
|
|
||||||
|
output.file_paths_by_cas_id = identified_files.drain().fold(
|
||||||
|
HashMap::new(),
|
||||||
|
|mut map,
|
||||||
|
(
|
||||||
|
file_path_pub_id,
|
||||||
|
IdentifiedFile {
|
||||||
|
cas_id,
|
||||||
|
kind,
|
||||||
|
file_path:
|
||||||
|
file_path_for_file_identifier::Data {
|
||||||
|
id, date_created, ..
|
||||||
|
},
|
||||||
|
},
|
||||||
|
)| {
|
||||||
|
map.entry(cas_id)
|
||||||
|
.or_insert_with(|| Vec::with_capacity(1))
|
||||||
|
.push(FilePathToCreateOrLinkObject {
|
||||||
|
id,
|
||||||
|
file_path_pub_id,
|
||||||
|
kind,
|
||||||
|
created_at: date_created,
|
||||||
|
});
|
||||||
|
|
||||||
|
map
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
trace!(save_db_time = ?output.save_db_time, "Cas_ids saved to db;");
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(ExecStatus::Done(mem::take(output).into_output()))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Identifier {
|
||||||
|
#[must_use]
|
||||||
|
pub fn new(
|
||||||
|
location: Arc<location::Data>,
|
||||||
|
location_path: Arc<PathBuf>,
|
||||||
|
file_paths: Vec<file_path_for_file_identifier::Data>,
|
||||||
|
with_priority: bool,
|
||||||
|
db: Arc<PrismaClient>,
|
||||||
|
sync: Arc<SyncManager>,
|
||||||
|
) -> Self {
|
||||||
|
Self {
|
||||||
|
id: TaskId::new_v4(),
|
||||||
|
location,
|
||||||
|
location_path,
|
||||||
|
identified_files: HashMap::with_capacity(file_paths.len()),
|
||||||
|
file_paths_without_cas_id: Vec::with_capacity(file_paths.len()),
|
||||||
|
file_paths_by_id: file_paths
|
||||||
|
.into_iter()
|
||||||
|
.map(|file_path| (file_path.pub_id.as_slice().into(), file_path))
|
||||||
|
.collect(),
|
||||||
|
output: Output::default(),
|
||||||
|
with_priority,
|
||||||
|
db,
|
||||||
|
sync,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[instrument(skip_all, err, fields(identified_files_count = identified_files.len()))]
|
||||||
|
async fn assign_cas_id_to_file_paths(
|
||||||
|
identified_files: &HashMap<FilePathPubId, IdentifiedFile>,
|
||||||
|
db: &PrismaClient,
|
||||||
|
sync: &SyncManager,
|
||||||
|
) -> Result<(), file_identifier::Error> {
|
||||||
|
// Assign cas_id to each file path
|
||||||
|
sync.write_ops(
|
||||||
|
db,
|
||||||
|
identified_files
|
||||||
|
.iter()
|
||||||
|
.map(|(pub_id, IdentifiedFile { cas_id, .. })| {
|
||||||
|
(
|
||||||
|
sync.shared_update(
|
||||||
|
prisma_sync::file_path::SyncId {
|
||||||
|
pub_id: pub_id.to_db(),
|
||||||
|
},
|
||||||
|
file_path::cas_id::NAME,
|
||||||
|
msgpack!(cas_id),
|
||||||
|
),
|
||||||
|
db.file_path()
|
||||||
|
.update(
|
||||||
|
file_path::pub_id::equals(pub_id.to_db()),
|
||||||
|
vec![file_path::cas_id::set(cas_id.into())],
|
||||||
|
)
|
||||||
|
// We don't need any data here, just the id avoids receiving the entire object
|
||||||
|
// as we can't pass an empty select macro call
|
||||||
|
.select(file_path::select!({ id })),
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.unzip::<_, _, _, Vec<_>>(),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[instrument(skip(errors))]
|
||||||
|
fn handle_non_critical_errors(
|
||||||
|
file_path_pub_id: FilePathPubId,
|
||||||
|
e: &FileIOError,
|
||||||
|
errors: &mut Vec<NonCriticalError>,
|
||||||
|
) {
|
||||||
|
let formatted_error = format!("<file_path_pub_id='{file_path_pub_id}', error={e}>");
|
||||||
|
|
||||||
|
#[cfg(target_os = "windows")]
|
||||||
|
{
|
||||||
|
// Handle case where file is on-demand (NTFS only)
|
||||||
|
if e.source.raw_os_error().map_or(false, |code| code == 362) {
|
||||||
|
errors.push(
|
||||||
|
file_identifier::NonCriticalFileIdentifierError::FailedToExtractMetadataFromOnDemandFile(
|
||||||
|
formatted_error,
|
||||||
|
)
|
||||||
|
.into(),
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
errors.push(
|
||||||
|
file_identifier::NonCriticalFileIdentifierError::FailedToExtractFileMetadata(
|
||||||
|
formatted_error,
|
||||||
|
)
|
||||||
|
.into(),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(not(target_os = "windows"))]
|
||||||
|
{
|
||||||
|
errors.push(
|
||||||
|
file_identifier::NonCriticalFileIdentifierError::FailedToExtractFileMetadata(
|
||||||
|
formatted_error,
|
||||||
|
)
|
||||||
|
.into(),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[instrument(
|
||||||
|
skip(location_id, file_path, location_path, errors),
|
||||||
|
fields(
|
||||||
|
file_path_id = file_path.id,
|
||||||
|
materialized_path = ?file_path.materialized_path,
|
||||||
|
name = ?file_path.name,
|
||||||
|
extension = ?file_path.extension,
|
||||||
|
)
|
||||||
|
)]
|
||||||
|
fn try_iso_file_path_extraction(
|
||||||
|
location_id: location::id::Type,
|
||||||
|
file_path_pub_id: FilePathPubId,
|
||||||
|
file_path: &file_path_for_file_identifier::Data,
|
||||||
|
location_path: Arc<PathBuf>,
|
||||||
|
errors: &mut Vec<NonCriticalError>,
|
||||||
|
) -> Option<(FilePathPubId, IsolatedFilePathData<'static>, Arc<PathBuf>)> {
|
||||||
|
IsolatedFilePathData::try_from((location_id, file_path))
|
||||||
|
.map(IsolatedFilePathData::to_owned)
|
||||||
|
.map_err(|e| {
|
||||||
|
error!(?e, "Failed to extract isolated file path data;");
|
||||||
|
errors.push(
|
||||||
|
file_identifier::NonCriticalFileIdentifierError::FailedToExtractIsolatedFilePathData(format!(
|
||||||
|
"<file_path_pub_id='{file_path_pub_id}', error={e}>"
|
||||||
|
))
|
||||||
|
.into(),
|
||||||
|
);
|
||||||
|
})
|
||||||
|
.map(|iso_file_path| (file_path_pub_id, iso_file_path, location_path))
|
||||||
|
.ok()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize, Deserialize)]
|
||||||
|
struct SaveState {
|
||||||
|
id: TaskId,
|
||||||
|
location: Arc<location::Data>,
|
||||||
|
location_path: Arc<PathBuf>,
|
||||||
|
file_paths_by_id: HashMap<FilePathPubId, file_path_for_file_identifier::Data>,
|
||||||
|
identified_files: HashMap<FilePathPubId, IdentifiedFile>,
|
||||||
|
file_paths_without_cas_id: Vec<FilePathToCreateOrLinkObject>,
|
||||||
|
output: Output,
|
||||||
|
with_priority: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl SerializableTask<Error> for Identifier {
|
||||||
|
type SerializeError = rmp_serde::encode::Error;
|
||||||
|
|
||||||
|
type DeserializeError = rmp_serde::decode::Error;
|
||||||
|
|
||||||
|
type DeserializeCtx = (Arc<PrismaClient>, Arc<SyncManager>);
|
||||||
|
|
||||||
|
async fn serialize(self) -> Result<Vec<u8>, Self::SerializeError> {
|
||||||
|
let Self {
|
||||||
|
id,
|
||||||
|
location,
|
||||||
|
location_path,
|
||||||
|
file_paths_by_id,
|
||||||
|
identified_files,
|
||||||
|
file_paths_without_cas_id,
|
||||||
|
output,
|
||||||
|
with_priority,
|
||||||
|
..
|
||||||
|
} = self;
|
||||||
|
rmp_serde::to_vec_named(&SaveState {
|
||||||
|
id,
|
||||||
|
location,
|
||||||
|
location_path,
|
||||||
|
file_paths_by_id,
|
||||||
|
identified_files,
|
||||||
|
file_paths_without_cas_id,
|
||||||
|
output,
|
||||||
|
with_priority,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn deserialize(
|
||||||
|
data: &[u8],
|
||||||
|
(db, sync): Self::DeserializeCtx,
|
||||||
|
) -> Result<Self, Self::DeserializeError> {
|
||||||
|
rmp_serde::from_slice::<SaveState>(data).map(
|
||||||
|
|SaveState {
|
||||||
|
id,
|
||||||
|
location,
|
||||||
|
location_path,
|
||||||
|
file_paths_by_id,
|
||||||
|
identified_files,
|
||||||
|
file_paths_without_cas_id,
|
||||||
|
output,
|
||||||
|
with_priority,
|
||||||
|
}| Self {
|
||||||
|
id,
|
||||||
|
with_priority,
|
||||||
|
location,
|
||||||
|
location_path,
|
||||||
|
file_paths_by_id,
|
||||||
|
identified_files,
|
||||||
|
file_paths_without_cas_id,
|
||||||
|
output,
|
||||||
|
db,
|
||||||
|
sync,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
|
@ -1,18 +1,171 @@
|
||||||
use sd_core_prisma_helpers::file_path_for_file_identifier;
|
use crate::file_identifier;
|
||||||
|
|
||||||
|
use sd_core_prisma_helpers::{file_path_id, FilePathPubId, ObjectPubId};
|
||||||
|
use sd_core_sync::Manager as SyncManager;
|
||||||
|
|
||||||
use sd_file_ext::kind::ObjectKind;
|
use sd_file_ext::kind::ObjectKind;
|
||||||
|
use sd_prisma::{
|
||||||
|
prisma::{file_path, object, PrismaClient},
|
||||||
|
prisma_sync,
|
||||||
|
};
|
||||||
|
use sd_sync::{CRDTOperation, OperationFactory};
|
||||||
|
use sd_utils::msgpack;
|
||||||
|
|
||||||
|
use std::collections::{HashMap, HashSet};
|
||||||
|
|
||||||
|
use chrono::{DateTime, FixedOffset};
|
||||||
|
use prisma_client_rust::Select;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
use tracing::{instrument, trace, Level};
|
||||||
|
|
||||||
pub mod extract_file_metadata;
|
pub mod identifier;
|
||||||
pub mod object_processor;
|
pub mod object_processor;
|
||||||
|
|
||||||
pub use extract_file_metadata::ExtractFileMetadataTask;
|
pub use identifier::Identifier;
|
||||||
pub use object_processor::ObjectProcessorTask;
|
pub use object_processor::ObjectProcessor;
|
||||||
|
|
||||||
|
/// This object has all needed data to create a new `object` for a `file_path` or link an existing one.
|
||||||
#[derive(Debug, Serialize, Deserialize)]
|
#[derive(Debug, Serialize, Deserialize)]
|
||||||
pub(super) struct IdentifiedFile {
|
pub(super) struct FilePathToCreateOrLinkObject {
|
||||||
pub(super) file_path: file_path_for_file_identifier::Data,
|
id: file_path::id::Type,
|
||||||
pub(super) cas_id: Option<String>,
|
file_path_pub_id: FilePathPubId,
|
||||||
pub(super) kind: ObjectKind,
|
kind: ObjectKind,
|
||||||
|
created_at: Option<DateTime<FixedOffset>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[instrument(skip(sync, db))]
|
||||||
|
fn connect_file_path_to_object<'db>(
|
||||||
|
file_path_pub_id: &FilePathPubId,
|
||||||
|
object_pub_id: &ObjectPubId,
|
||||||
|
db: &'db PrismaClient,
|
||||||
|
sync: &SyncManager,
|
||||||
|
) -> (CRDTOperation, Select<'db, file_path_id::Data>) {
|
||||||
|
trace!("Connecting");
|
||||||
|
|
||||||
|
(
|
||||||
|
sync.shared_update(
|
||||||
|
prisma_sync::file_path::SyncId {
|
||||||
|
pub_id: file_path_pub_id.to_db(),
|
||||||
|
},
|
||||||
|
file_path::object::NAME,
|
||||||
|
msgpack!(prisma_sync::object::SyncId {
|
||||||
|
pub_id: object_pub_id.to_db(),
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
db.file_path()
|
||||||
|
.update(
|
||||||
|
file_path::pub_id::equals(file_path_pub_id.to_db()),
|
||||||
|
vec![file_path::object::connect(object::pub_id::equals(
|
||||||
|
object_pub_id.to_db(),
|
||||||
|
))],
|
||||||
|
)
|
||||||
|
// selecting just id to avoid fetching the whole object
|
||||||
|
.select(file_path_id::select()),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[instrument(skip_all, ret(level = Level::TRACE), err)]
|
||||||
|
async fn create_objects_and_update_file_paths(
|
||||||
|
files_and_kinds: impl IntoIterator<Item = FilePathToCreateOrLinkObject> + Send,
|
||||||
|
db: &PrismaClient,
|
||||||
|
sync: &SyncManager,
|
||||||
|
) -> Result<HashMap<file_path::id::Type, ObjectPubId>, file_identifier::Error> {
|
||||||
|
trace!("Preparing objects");
|
||||||
|
let (object_create_args, file_path_args) = files_and_kinds
|
||||||
|
.into_iter()
|
||||||
|
.map(
|
||||||
|
|FilePathToCreateOrLinkObject {
|
||||||
|
id,
|
||||||
|
file_path_pub_id,
|
||||||
|
kind,
|
||||||
|
created_at,
|
||||||
|
}| {
|
||||||
|
let object_pub_id = ObjectPubId::new();
|
||||||
|
|
||||||
|
let kind = kind as i32;
|
||||||
|
|
||||||
|
let (sync_params, db_params) = [
|
||||||
|
(
|
||||||
|
(object::date_created::NAME, msgpack!(created_at)),
|
||||||
|
object::date_created::set(created_at),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
(object::kind::NAME, msgpack!(kind)),
|
||||||
|
object::kind::set(Some(kind)),
|
||||||
|
),
|
||||||
|
]
|
||||||
|
.into_iter()
|
||||||
|
.unzip::<_, _, Vec<_>, Vec<_>>();
|
||||||
|
|
||||||
|
(
|
||||||
|
(
|
||||||
|
sync.shared_create(
|
||||||
|
prisma_sync::object::SyncId {
|
||||||
|
pub_id: object_pub_id.to_db(),
|
||||||
|
},
|
||||||
|
sync_params,
|
||||||
|
),
|
||||||
|
object::create_unchecked(object_pub_id.to_db(), db_params),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
(id, object_pub_id.clone()),
|
||||||
|
connect_file_path_to_object(&file_path_pub_id, &object_pub_id, db, sync),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.unzip::<_, _, Vec<_>, Vec<_>>();
|
||||||
|
|
||||||
|
let (mut object_pub_id_by_file_path_id, file_path_update_args) = file_path_args
|
||||||
|
.into_iter()
|
||||||
|
.unzip::<_, _, HashMap<_, _>, Vec<_>>(
|
||||||
|
);
|
||||||
|
|
||||||
|
trace!(
|
||||||
|
new_objects_count = object_create_args.len(),
|
||||||
|
"Creating new Objects!;",
|
||||||
|
);
|
||||||
|
|
||||||
|
// create new object records with assembled values
|
||||||
|
let created_objects_count = sync
|
||||||
|
.write_ops(db, {
|
||||||
|
let (sync, db_params) = object_create_args
|
||||||
|
.into_iter()
|
||||||
|
.unzip::<_, _, Vec<_>, Vec<_>>();
|
||||||
|
|
||||||
|
(
|
||||||
|
sync.into_iter().flatten().collect(),
|
||||||
|
db.object().create_many(db_params),
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
trace!(%created_objects_count, "Created new Objects;");
|
||||||
|
|
||||||
|
if created_objects_count > 0 {
|
||||||
|
trace!("Updating file paths with created objects");
|
||||||
|
|
||||||
|
let updated_file_path_ids = sync
|
||||||
|
.write_ops(
|
||||||
|
db,
|
||||||
|
file_path_update_args
|
||||||
|
.into_iter()
|
||||||
|
.unzip::<_, _, Vec<_>, Vec<_>>(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map(|file_paths| {
|
||||||
|
file_paths
|
||||||
|
.into_iter()
|
||||||
|
.map(|file_path_id::Data { id }| id)
|
||||||
|
.collect::<HashSet<_>>()
|
||||||
|
})?;
|
||||||
|
|
||||||
|
object_pub_id_by_file_path_id
|
||||||
|
.retain(|file_path_id, _| updated_file_path_ids.contains(file_path_id));
|
||||||
|
|
||||||
|
Ok(object_pub_id_by_file_path_id)
|
||||||
|
} else {
|
||||||
|
trace!("No objects created, skipping file path updates");
|
||||||
|
Ok(HashMap::new())
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,98 +1,76 @@
|
||||||
use crate::{file_identifier, Error};
|
use crate::{file_identifier, Error};
|
||||||
|
|
||||||
use sd_core_prisma_helpers::{
|
use sd_core_prisma_helpers::{file_path_id, object_for_file_identifier, CasId, ObjectPubId};
|
||||||
file_path_for_file_identifier, file_path_pub_id, object_for_file_identifier,
|
|
||||||
};
|
|
||||||
use sd_core_sync::Manager as SyncManager;
|
use sd_core_sync::Manager as SyncManager;
|
||||||
|
|
||||||
use sd_prisma::{
|
use sd_prisma::prisma::{file_path, object, PrismaClient};
|
||||||
prisma::{file_path, object, PrismaClient},
|
|
||||||
prisma_sync,
|
|
||||||
};
|
|
||||||
use sd_sync::{CRDTOperation, OperationFactory};
|
|
||||||
use sd_task_system::{
|
use sd_task_system::{
|
||||||
check_interruption, ExecStatus, Interrupter, IntoAnyTaskOutput, SerializableTask, Task, TaskId,
|
check_interruption, ExecStatus, Interrupter, IntoAnyTaskOutput, SerializableTask, Task, TaskId,
|
||||||
};
|
};
|
||||||
use sd_utils::{msgpack, uuid_to_bytes};
|
|
||||||
|
|
||||||
use std::{
|
use std::{collections::HashMap, mem, sync::Arc, time::Duration};
|
||||||
collections::{HashMap, HashSet},
|
|
||||||
mem,
|
|
||||||
sync::Arc,
|
|
||||||
time::Duration,
|
|
||||||
};
|
|
||||||
|
|
||||||
use prisma_client_rust::Select;
|
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use tokio::time::Instant;
|
use tokio::time::Instant;
|
||||||
use tracing::{debug, trace};
|
use tracing::{instrument, trace, Level};
|
||||||
use uuid::Uuid;
|
|
||||||
|
|
||||||
use super::IdentifiedFile;
|
use super::{
|
||||||
|
connect_file_path_to_object, create_objects_and_update_file_paths, FilePathToCreateOrLinkObject,
|
||||||
|
};
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
pub struct ObjectProcessorTask {
|
pub struct ObjectProcessor {
|
||||||
|
// Task control
|
||||||
id: TaskId,
|
id: TaskId,
|
||||||
|
with_priority: bool,
|
||||||
|
|
||||||
|
// Received input args
|
||||||
|
file_paths_by_cas_id: HashMap<CasId<'static>, Vec<FilePathToCreateOrLinkObject>>,
|
||||||
|
|
||||||
|
// Inner state
|
||||||
|
stage: Stage,
|
||||||
|
|
||||||
|
// Out collector
|
||||||
|
output: Output,
|
||||||
|
|
||||||
|
// Dependencies
|
||||||
db: Arc<PrismaClient>,
|
db: Arc<PrismaClient>,
|
||||||
sync: Arc<SyncManager>,
|
sync: Arc<SyncManager>,
|
||||||
identified_files: HashMap<Uuid, IdentifiedFile>,
|
|
||||||
output: Output,
|
|
||||||
stage: Stage,
|
|
||||||
with_priority: bool,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Serialize, Deserialize)]
|
|
||||||
pub struct SaveState {
|
|
||||||
id: TaskId,
|
|
||||||
identified_files: HashMap<Uuid, IdentifiedFile>,
|
|
||||||
output: Output,
|
|
||||||
stage: Stage,
|
|
||||||
with_priority: bool,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Serialize, Deserialize, Default)]
|
|
||||||
pub struct Output {
|
|
||||||
pub file_path_ids_with_new_object: Vec<file_path::id::Type>,
|
|
||||||
pub assign_cas_ids_time: Duration,
|
|
||||||
pub fetch_existing_objects_time: Duration,
|
|
||||||
pub assign_to_existing_object_time: Duration,
|
|
||||||
pub create_object_time: Duration,
|
|
||||||
pub created_objects_count: u64,
|
|
||||||
pub linked_objects_count: u64,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Serialize, Deserialize)]
|
#[derive(Debug, Serialize, Deserialize)]
|
||||||
enum Stage {
|
enum Stage {
|
||||||
Starting,
|
Starting,
|
||||||
FetchExistingObjects,
|
|
||||||
AssignFilePathsToExistingObjects {
|
AssignFilePathsToExistingObjects {
|
||||||
existing_objects_by_cas_id: HashMap<String, object_for_file_identifier::Data>,
|
existing_objects_by_cas_id: HashMap<CasId<'static>, ObjectPubId>,
|
||||||
},
|
},
|
||||||
CreateObjects,
|
CreateObjects,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ObjectProcessorTask {
|
/// Output from the `[ObjectProcessor]` task
|
||||||
#[must_use]
|
#[derive(Debug, Serialize, Deserialize, Default)]
|
||||||
pub fn new(
|
pub struct Output {
|
||||||
identified_files: HashMap<Uuid, IdentifiedFile>,
|
/// To send to frontend for priority reporting of new objects
|
||||||
db: Arc<PrismaClient>,
|
pub file_path_ids_with_new_object: Vec<file_path::id::Type>,
|
||||||
sync: Arc<SyncManager>,
|
|
||||||
with_priority: bool,
|
/// Time elapsed fetching existing `objects` from db to be linked to `file_paths`
|
||||||
) -> Self {
|
pub fetch_existing_objects_time: Duration,
|
||||||
Self {
|
|
||||||
id: TaskId::new_v4(),
|
/// Time spent linking `file_paths` to already existing `objects`
|
||||||
db,
|
pub assign_to_existing_object_time: Duration,
|
||||||
sync,
|
|
||||||
identified_files,
|
/// Time spent creating new `objects`
|
||||||
stage: Stage::Starting,
|
pub create_object_time: Duration,
|
||||||
output: Output::default(),
|
|
||||||
with_priority,
|
/// Number of new `objects` created
|
||||||
}
|
pub created_objects_count: u64,
|
||||||
}
|
|
||||||
|
/// Number of `objects` that were linked to `file_paths`
|
||||||
|
pub linked_objects_count: u64,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[async_trait::async_trait]
|
#[async_trait::async_trait]
|
||||||
impl Task<Error> for ObjectProcessorTask {
|
impl Task<Error> for ObjectProcessor {
|
||||||
fn id(&self) -> TaskId {
|
fn id(&self) -> TaskId {
|
||||||
self.id
|
self.id
|
||||||
}
|
}
|
||||||
|
@ -101,16 +79,25 @@ impl Task<Error> for ObjectProcessorTask {
|
||||||
self.with_priority
|
self.with_priority
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[instrument(
|
||||||
|
skip(self, interrupter),
|
||||||
|
fields(
|
||||||
|
task_id = %self.id,
|
||||||
|
cas_ids_count = %self.file_paths_by_cas_id.len(),
|
||||||
|
),
|
||||||
|
ret(level = Level::TRACE),
|
||||||
|
err,
|
||||||
|
)]
|
||||||
|
#[allow(clippy::blocks_in_conditions)] // Due to `err` on `instrument` macro above
|
||||||
async fn run(&mut self, interrupter: &Interrupter) -> Result<ExecStatus, Error> {
|
async fn run(&mut self, interrupter: &Interrupter) -> Result<ExecStatus, Error> {
|
||||||
let Self {
|
let Self {
|
||||||
db,
|
db,
|
||||||
sync,
|
sync,
|
||||||
identified_files,
|
file_paths_by_cas_id,
|
||||||
stage,
|
stage,
|
||||||
output:
|
output:
|
||||||
Output {
|
Output {
|
||||||
file_path_ids_with_new_object,
|
file_path_ids_with_new_object,
|
||||||
assign_cas_ids_time,
|
|
||||||
fetch_existing_objects_time,
|
fetch_existing_objects_time,
|
||||||
assign_to_existing_object_time,
|
assign_to_existing_object_time,
|
||||||
create_object_time,
|
create_object_time,
|
||||||
|
@ -123,17 +110,17 @@ impl Task<Error> for ObjectProcessorTask {
|
||||||
loop {
|
loop {
|
||||||
match stage {
|
match stage {
|
||||||
Stage::Starting => {
|
Stage::Starting => {
|
||||||
let start = Instant::now();
|
trace!("Starting object processor task");
|
||||||
assign_cas_id_to_file_paths(identified_files, db, sync).await?;
|
|
||||||
*assign_cas_ids_time = start.elapsed();
|
|
||||||
*stage = Stage::FetchExistingObjects;
|
|
||||||
}
|
|
||||||
|
|
||||||
Stage::FetchExistingObjects => {
|
|
||||||
let start = Instant::now();
|
let start = Instant::now();
|
||||||
let existing_objects_by_cas_id =
|
let existing_objects_by_cas_id =
|
||||||
fetch_existing_objects_by_cas_id(identified_files, db).await?;
|
fetch_existing_objects_by_cas_id(file_paths_by_cas_id.keys(), db).await?;
|
||||||
*fetch_existing_objects_time = start.elapsed();
|
*fetch_existing_objects_time = start.elapsed();
|
||||||
|
|
||||||
|
trace!(
|
||||||
|
elapsed_time = ?fetch_existing_objects_time,
|
||||||
|
existing_objects_count = existing_objects_by_cas_id.len(),
|
||||||
|
"Fetched existing Objects;",
|
||||||
|
);
|
||||||
*stage = Stage::AssignFilePathsToExistingObjects {
|
*stage = Stage::AssignFilePathsToExistingObjects {
|
||||||
existing_objects_by_cas_id,
|
existing_objects_by_cas_id,
|
||||||
};
|
};
|
||||||
|
@ -142,48 +129,53 @@ impl Task<Error> for ObjectProcessorTask {
|
||||||
Stage::AssignFilePathsToExistingObjects {
|
Stage::AssignFilePathsToExistingObjects {
|
||||||
existing_objects_by_cas_id,
|
existing_objects_by_cas_id,
|
||||||
} => {
|
} => {
|
||||||
|
trace!(
|
||||||
|
existing_objects_to_link = existing_objects_by_cas_id.len(),
|
||||||
|
"Assigning file paths to existing Objects;",
|
||||||
|
);
|
||||||
let start = Instant::now();
|
let start = Instant::now();
|
||||||
let assigned_file_path_pub_ids = assign_existing_objects_to_file_paths(
|
let more_file_path_ids_with_new_object = assign_existing_objects_to_file_paths(
|
||||||
identified_files,
|
file_paths_by_cas_id,
|
||||||
existing_objects_by_cas_id,
|
existing_objects_by_cas_id,
|
||||||
db,
|
db,
|
||||||
sync,
|
sync,
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
*assign_to_existing_object_time = start.elapsed();
|
*assign_to_existing_object_time = start.elapsed();
|
||||||
*linked_objects_count = assigned_file_path_pub_ids.len() as u64;
|
file_path_ids_with_new_object.extend(more_file_path_ids_with_new_object);
|
||||||
|
*linked_objects_count += file_path_ids_with_new_object.len() as u64;
|
||||||
|
|
||||||
debug!(
|
trace!(
|
||||||
"Found {} existing Objects, linked file paths to them",
|
existing_objects_to_link = existing_objects_by_cas_id.len(),
|
||||||
existing_objects_by_cas_id.len()
|
%linked_objects_count,
|
||||||
|
"Found existing Objects, linked file paths to them;",
|
||||||
);
|
);
|
||||||
|
|
||||||
for file_path_pub_id::Data { pub_id } in assigned_file_path_pub_ids {
|
|
||||||
let pub_id = Uuid::from_slice(&pub_id).expect("uuid bytes are invalid");
|
|
||||||
trace!("Assigned file path <file_path_pub_id={pub_id}> to existing object");
|
|
||||||
|
|
||||||
identified_files
|
|
||||||
.remove(&pub_id)
|
|
||||||
.expect("file_path must be here");
|
|
||||||
}
|
|
||||||
|
|
||||||
*stage = Stage::CreateObjects;
|
*stage = Stage::CreateObjects;
|
||||||
|
|
||||||
if identified_files.is_empty() {
|
if file_paths_by_cas_id.is_empty() {
|
||||||
|
trace!("No more objects to be created, finishing task");
|
||||||
// No objects to be created, we're good to finish already
|
// No objects to be created, we're good to finish already
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Stage::CreateObjects => {
|
Stage::CreateObjects => {
|
||||||
|
trace!(
|
||||||
|
creating_count = file_paths_by_cas_id.len(),
|
||||||
|
"Creating new Objects;"
|
||||||
|
);
|
||||||
let start = Instant::now();
|
let start = Instant::now();
|
||||||
*created_objects_count = create_objects(identified_files, db, sync).await?;
|
let (more_file_paths_with_new_object, more_linked_objects_count) =
|
||||||
|
assign_objects_to_duplicated_orphans(file_paths_by_cas_id, db, sync)
|
||||||
|
.await?;
|
||||||
*create_object_time = start.elapsed();
|
*create_object_time = start.elapsed();
|
||||||
|
file_path_ids_with_new_object.extend(more_file_paths_with_new_object);
|
||||||
|
*linked_objects_count += more_linked_objects_count;
|
||||||
|
|
||||||
*file_path_ids_with_new_object = identified_files
|
*created_objects_count = file_path_ids_with_new_object.len() as u64;
|
||||||
.values()
|
|
||||||
.map(|IdentifiedFile { file_path, .. }| file_path.id)
|
trace!(%created_objects_count, ?create_object_time, "Created new Objects;");
|
||||||
.collect();
|
|
||||||
|
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
@ -196,225 +188,188 @@ impl Task<Error> for ObjectProcessorTask {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn assign_cas_id_to_file_paths(
|
impl ObjectProcessor {
|
||||||
identified_files: &HashMap<Uuid, IdentifiedFile>,
|
#[must_use]
|
||||||
db: &PrismaClient,
|
pub fn new(
|
||||||
sync: &SyncManager,
|
file_paths_by_cas_id: HashMap<CasId<'static>, Vec<FilePathToCreateOrLinkObject>>,
|
||||||
) -> Result<(), file_identifier::Error> {
|
db: Arc<PrismaClient>,
|
||||||
// Assign cas_id to each file path
|
sync: Arc<SyncManager>,
|
||||||
sync.write_ops(
|
with_priority: bool,
|
||||||
db,
|
) -> Self {
|
||||||
identified_files
|
Self {
|
||||||
.iter()
|
id: TaskId::new_v4(),
|
||||||
.map(|(pub_id, IdentifiedFile { cas_id, .. })| {
|
db,
|
||||||
(
|
sync,
|
||||||
sync.shared_update(
|
file_paths_by_cas_id,
|
||||||
prisma_sync::file_path::SyncId {
|
stage: Stage::Starting,
|
||||||
pub_id: uuid_to_bytes(*pub_id),
|
output: Output::default(),
|
||||||
},
|
with_priority,
|
||||||
file_path::cas_id::NAME,
|
}
|
||||||
msgpack!(cas_id),
|
}
|
||||||
),
|
|
||||||
db.file_path()
|
|
||||||
.update(
|
|
||||||
file_path::pub_id::equals(uuid_to_bytes(*pub_id)),
|
|
||||||
vec![file_path::cas_id::set(cas_id.clone())],
|
|
||||||
)
|
|
||||||
// We don't need any data here, just the id avoids receiving the entire object
|
|
||||||
// as we can't pass an empty select macro call
|
|
||||||
.select(file_path::select!({ id })),
|
|
||||||
)
|
|
||||||
})
|
|
||||||
.unzip::<_, _, _, Vec<_>>(),
|
|
||||||
)
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
Ok(())
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn fetch_existing_objects_by_cas_id(
|
/// Retrieves objects that are already connected to file paths with the same cas_id
|
||||||
identified_files: &HashMap<Uuid, IdentifiedFile>,
|
#[instrument(skip_all, err)]
|
||||||
|
async fn fetch_existing_objects_by_cas_id<'cas_id, Iter>(
|
||||||
|
cas_ids: Iter,
|
||||||
db: &PrismaClient,
|
db: &PrismaClient,
|
||||||
) -> Result<HashMap<String, object_for_file_identifier::Data>, file_identifier::Error> {
|
) -> Result<HashMap<CasId<'static>, ObjectPubId>, file_identifier::Error>
|
||||||
// Retrieves objects that are already connected to file paths with the same id
|
where
|
||||||
db.object()
|
Iter: IntoIterator<Item = &'cas_id CasId<'cas_id>> + Send,
|
||||||
.find_many(vec![object::file_paths::some(vec![
|
Iter::IntoIter: Send,
|
||||||
file_path::cas_id::in_vec(
|
{
|
||||||
identified_files
|
async fn inner(
|
||||||
.values()
|
stringed_cas_ids: Vec<String>,
|
||||||
.filter_map(|IdentifiedFile { cas_id, .. }| cas_id.as_ref())
|
db: &PrismaClient,
|
||||||
.cloned()
|
) -> Result<HashMap<CasId<'static>, ObjectPubId>, file_identifier::Error> {
|
||||||
.collect::<HashSet<_>>()
|
db.object()
|
||||||
|
.find_many(vec![object::file_paths::some(vec![
|
||||||
|
file_path::cas_id::in_vec(stringed_cas_ids),
|
||||||
|
file_path::object_id::not(None),
|
||||||
|
])])
|
||||||
|
.select(object_for_file_identifier::select())
|
||||||
|
.exec()
|
||||||
|
.await
|
||||||
|
.map_err(Into::into)
|
||||||
|
.map(|objects| {
|
||||||
|
objects
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.collect(),
|
.filter_map(|object_for_file_identifier::Data { pub_id, file_paths }| {
|
||||||
),
|
file_paths
|
||||||
])])
|
.first()
|
||||||
.select(object_for_file_identifier::select())
|
.and_then(|file_path| {
|
||||||
.exec()
|
file_path
|
||||||
.await
|
.cas_id
|
||||||
.map_err(Into::into)
|
.as_ref()
|
||||||
.map(|objects| {
|
.map(CasId::from)
|
||||||
objects
|
.map(CasId::into_owned)
|
||||||
.into_iter()
|
})
|
||||||
.filter_map(|object| {
|
.map(|cas_id| (cas_id, pub_id.into()))
|
||||||
object
|
})
|
||||||
.file_paths
|
.collect()
|
||||||
.first()
|
})
|
||||||
.and_then(|file_path| file_path.cas_id.clone())
|
}
|
||||||
.map(|cas_id| (cas_id, object))
|
|
||||||
})
|
let stringed_cas_ids = cas_ids.into_iter().map(Into::into).collect::<Vec<_>>();
|
||||||
.collect()
|
|
||||||
})
|
trace!(
|
||||||
|
cas_ids_count = stringed_cas_ids.len(),
|
||||||
|
"Fetching existing objects by cas_ids;",
|
||||||
|
);
|
||||||
|
|
||||||
|
inner(stringed_cas_ids, db).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Attempt to associate each file path with an object that has been
|
||||||
|
/// connected to file paths with the same cas_id
|
||||||
|
#[instrument(skip_all, err, fields(identified_files_count = file_paths_by_cas_id.len()))]
|
||||||
async fn assign_existing_objects_to_file_paths(
|
async fn assign_existing_objects_to_file_paths(
|
||||||
identified_files: &HashMap<Uuid, IdentifiedFile>,
|
file_paths_by_cas_id: &mut HashMap<CasId<'static>, Vec<FilePathToCreateOrLinkObject>>,
|
||||||
objects_by_cas_id: &HashMap<String, object_for_file_identifier::Data>,
|
objects_by_cas_id: &HashMap<CasId<'static>, ObjectPubId>,
|
||||||
db: &PrismaClient,
|
db: &PrismaClient,
|
||||||
sync: &SyncManager,
|
sync: &SyncManager,
|
||||||
) -> Result<Vec<file_path_pub_id::Data>, file_identifier::Error> {
|
) -> Result<Vec<file_path::id::Type>, file_identifier::Error> {
|
||||||
// Attempt to associate each file path with an object that has been
|
|
||||||
// connected to file paths with the same cas_id
|
|
||||||
sync.write_ops(
|
sync.write_ops(
|
||||||
db,
|
db,
|
||||||
identified_files
|
objects_by_cas_id
|
||||||
.iter()
|
.iter()
|
||||||
.filter_map(|(pub_id, IdentifiedFile { cas_id, .. })| {
|
.flat_map(|(cas_id, object_pub_id)| {
|
||||||
objects_by_cas_id
|
file_paths_by_cas_id
|
||||||
// Filtering out files without cas_id due to being empty
|
.remove(cas_id)
|
||||||
.get(cas_id.as_ref()?)
|
.map(|file_paths| {
|
||||||
.map(|object| (*pub_id, object))
|
file_paths.into_iter().map(
|
||||||
})
|
|FilePathToCreateOrLinkObject {
|
||||||
.map(|(pub_id, object)| {
|
file_path_pub_id, ..
|
||||||
connect_file_path_to_object(
|
}| {
|
||||||
pub_id,
|
connect_file_path_to_object(
|
||||||
// SAFETY: This pub_id is generated by the uuid lib, but we have to store bytes in sqlite
|
&file_path_pub_id,
|
||||||
Uuid::from_slice(&object.pub_id).expect("uuid bytes are invalid"),
|
object_pub_id,
|
||||||
sync,
|
db,
|
||||||
db,
|
sync,
|
||||||
)
|
)
|
||||||
|
},
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.expect("must be here")
|
||||||
})
|
})
|
||||||
.unzip::<_, _, Vec<_>, Vec<_>>(),
|
.unzip::<_, _, Vec<_>, Vec<_>>(),
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
|
.map(|file_paths| {
|
||||||
|
file_paths
|
||||||
|
.into_iter()
|
||||||
|
.map(|file_path_id::Data { id }| id)
|
||||||
|
.collect()
|
||||||
|
})
|
||||||
.map_err(Into::into)
|
.map_err(Into::into)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn connect_file_path_to_object<'db>(
|
async fn assign_objects_to_duplicated_orphans(
|
||||||
file_path_pub_id: Uuid,
|
file_paths_by_cas_id: &mut HashMap<CasId<'static>, Vec<FilePathToCreateOrLinkObject>>,
|
||||||
object_pub_id: Uuid,
|
|
||||||
sync: &SyncManager,
|
|
||||||
db: &'db PrismaClient,
|
|
||||||
) -> (CRDTOperation, Select<'db, file_path_pub_id::Data>) {
|
|
||||||
trace!("Connecting <file_path_pub_id={file_path_pub_id}> to <object_pub_id={object_pub_id}'>");
|
|
||||||
|
|
||||||
let vec_id = object_pub_id.as_bytes().to_vec();
|
|
||||||
|
|
||||||
(
|
|
||||||
sync.shared_update(
|
|
||||||
prisma_sync::file_path::SyncId {
|
|
||||||
pub_id: uuid_to_bytes(file_path_pub_id),
|
|
||||||
},
|
|
||||||
file_path::object::NAME,
|
|
||||||
msgpack!(prisma_sync::object::SyncId {
|
|
||||||
pub_id: vec_id.clone()
|
|
||||||
}),
|
|
||||||
),
|
|
||||||
db.file_path()
|
|
||||||
.update(
|
|
||||||
file_path::pub_id::equals(uuid_to_bytes(file_path_pub_id)),
|
|
||||||
vec![file_path::object::connect(object::pub_id::equals(vec_id))],
|
|
||||||
)
|
|
||||||
.select(file_path_pub_id::select()),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn create_objects(
|
|
||||||
identified_files: &HashMap<Uuid, IdentifiedFile>,
|
|
||||||
db: &PrismaClient,
|
db: &PrismaClient,
|
||||||
sync: &SyncManager,
|
sync: &SyncManager,
|
||||||
) -> Result<u64, file_identifier::Error> {
|
) -> Result<(Vec<file_path::id::Type>, u64), file_identifier::Error> {
|
||||||
trace!("Creating {} new Objects", identified_files.len(),);
|
// at least 1 file path per cas_id
|
||||||
|
let mut selected_file_paths = Vec::with_capacity(file_paths_by_cas_id.len());
|
||||||
|
let mut cas_ids_by_file_path_id = HashMap::with_capacity(file_paths_by_cas_id.len());
|
||||||
|
|
||||||
let (object_create_args, file_path_update_args) = identified_files
|
file_paths_by_cas_id.retain(|cas_id, file_paths| {
|
||||||
.iter()
|
let file_path = file_paths.pop().expect("file_paths can't be empty");
|
||||||
.map(
|
let has_more_file_paths = !file_paths.is_empty();
|
||||||
|(
|
|
||||||
file_path_pub_id,
|
|
||||||
IdentifiedFile {
|
|
||||||
file_path: file_path_for_file_identifier::Data { date_created, .. },
|
|
||||||
kind,
|
|
||||||
..
|
|
||||||
},
|
|
||||||
)| {
|
|
||||||
let object_pub_id = Uuid::new_v4();
|
|
||||||
|
|
||||||
let kind = *kind as i32;
|
if has_more_file_paths {
|
||||||
|
cas_ids_by_file_path_id.insert(file_path.id, cas_id.clone());
|
||||||
|
}
|
||||||
|
selected_file_paths.push(file_path);
|
||||||
|
|
||||||
let (sync_params, db_params) = [
|
has_more_file_paths
|
||||||
(
|
});
|
||||||
(object::date_created::NAME, msgpack!(date_created)),
|
|
||||||
object::date_created::set(*date_created),
|
|
||||||
),
|
|
||||||
(
|
|
||||||
(object::kind::NAME, msgpack!(kind)),
|
|
||||||
object::kind::set(Some(kind)),
|
|
||||||
),
|
|
||||||
]
|
|
||||||
.into_iter()
|
|
||||||
.unzip::<_, _, Vec<_>, Vec<_>>();
|
|
||||||
|
|
||||||
|
let (mut file_paths_with_new_object, objects_by_cas_id) =
|
||||||
|
create_objects_and_update_file_paths(selected_file_paths, db, sync)
|
||||||
|
.await?
|
||||||
|
.into_iter()
|
||||||
|
.map(|(file_path_id, object_pub_id)| {
|
||||||
(
|
(
|
||||||
(
|
file_path_id,
|
||||||
sync.shared_create(
|
cas_ids_by_file_path_id
|
||||||
prisma_sync::object::SyncId {
|
.remove(&file_path_id)
|
||||||
pub_id: uuid_to_bytes(object_pub_id),
|
.map(|cas_id| (cas_id, object_pub_id)),
|
||||||
},
|
|
||||||
sync_params,
|
|
||||||
),
|
|
||||||
object::create_unchecked(uuid_to_bytes(object_pub_id), db_params),
|
|
||||||
),
|
|
||||||
connect_file_path_to_object(*file_path_pub_id, object_pub_id, sync, db),
|
|
||||||
)
|
)
|
||||||
},
|
})
|
||||||
)
|
.unzip::<_, _, Vec<_>, Vec<_>>();
|
||||||
.unzip::<_, _, Vec<_>, Vec<_>>();
|
|
||||||
|
|
||||||
// create new object records with assembled values
|
let more_file_paths_ids_with_new_object = assign_existing_objects_to_file_paths(
|
||||||
let total_created_files = sync
|
file_paths_by_cas_id,
|
||||||
.write_ops(db, {
|
&objects_by_cas_id.into_iter().flatten().collect(),
|
||||||
let (sync, db_params) = object_create_args
|
db,
|
||||||
.into_iter()
|
sync,
|
||||||
.unzip::<_, _, Vec<_>, Vec<_>>();
|
)
|
||||||
|
.await?;
|
||||||
|
|
||||||
(
|
// Sanity check
|
||||||
sync.into_iter().flatten().collect(),
|
assert!(
|
||||||
db.object().create_many(db_params),
|
file_paths_by_cas_id.is_empty(),
|
||||||
)
|
"We MUST have processed all pending `file_paths` by now"
|
||||||
})
|
);
|
||||||
.await?;
|
|
||||||
|
|
||||||
trace!("Created {total_created_files} new Objects");
|
let linked_objects_count = more_file_paths_ids_with_new_object.len() as u64;
|
||||||
|
|
||||||
if total_created_files > 0 {
|
file_paths_with_new_object.extend(more_file_paths_ids_with_new_object);
|
||||||
trace!("Updating file paths with created objects");
|
|
||||||
|
|
||||||
sync.write_ops(
|
Ok((file_paths_with_new_object, linked_objects_count))
|
||||||
db,
|
|
||||||
file_path_update_args
|
|
||||||
.into_iter()
|
|
||||||
.unzip::<_, _, Vec<_>, Vec<_>>(),
|
|
||||||
)
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
trace!("Updated file paths with created objects");
|
|
||||||
}
|
|
||||||
|
|
||||||
#[allow(clippy::cast_sign_loss)] // SAFETY: We're sure the value is positive
|
|
||||||
Ok(total_created_files as u64)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl SerializableTask<Error> for ObjectProcessorTask {
|
#[derive(Debug, Serialize, Deserialize)]
|
||||||
|
pub struct SaveState {
|
||||||
|
id: TaskId,
|
||||||
|
file_paths_by_cas_id: HashMap<CasId<'static>, Vec<FilePathToCreateOrLinkObject>>,
|
||||||
|
stage: Stage,
|
||||||
|
output: Output,
|
||||||
|
with_priority: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl SerializableTask<Error> for ObjectProcessor {
|
||||||
type SerializeError = rmp_serde::encode::Error;
|
type SerializeError = rmp_serde::encode::Error;
|
||||||
|
|
||||||
type DeserializeError = rmp_serde::decode::Error;
|
type DeserializeError = rmp_serde::decode::Error;
|
||||||
|
@ -424,18 +379,18 @@ impl SerializableTask<Error> for ObjectProcessorTask {
|
||||||
async fn serialize(self) -> Result<Vec<u8>, Self::SerializeError> {
|
async fn serialize(self) -> Result<Vec<u8>, Self::SerializeError> {
|
||||||
let Self {
|
let Self {
|
||||||
id,
|
id,
|
||||||
identified_files,
|
file_paths_by_cas_id,
|
||||||
output,
|
|
||||||
stage,
|
stage,
|
||||||
|
output,
|
||||||
with_priority,
|
with_priority,
|
||||||
..
|
..
|
||||||
} = self;
|
} = self;
|
||||||
|
|
||||||
rmp_serde::to_vec_named(&SaveState {
|
rmp_serde::to_vec_named(&SaveState {
|
||||||
id,
|
id,
|
||||||
identified_files,
|
file_paths_by_cas_id,
|
||||||
output,
|
|
||||||
stage,
|
stage,
|
||||||
|
output,
|
||||||
with_priority,
|
with_priority,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
@ -447,18 +402,18 @@ impl SerializableTask<Error> for ObjectProcessorTask {
|
||||||
rmp_serde::from_slice(data).map(
|
rmp_serde::from_slice(data).map(
|
||||||
|SaveState {
|
|SaveState {
|
||||||
id,
|
id,
|
||||||
identified_files,
|
file_paths_by_cas_id,
|
||||||
output,
|
|
||||||
stage,
|
stage,
|
||||||
|
output,
|
||||||
with_priority,
|
with_priority,
|
||||||
}| Self {
|
}| Self {
|
||||||
id,
|
id,
|
||||||
|
with_priority,
|
||||||
|
file_paths_by_cas_id,
|
||||||
|
stage,
|
||||||
|
output,
|
||||||
db,
|
db,
|
||||||
sync,
|
sync,
|
||||||
identified_files,
|
|
||||||
output,
|
|
||||||
stage,
|
|
||||||
with_priority,
|
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
File diff suppressed because it is too large
Load diff
|
@ -1,7 +1,6 @@
|
||||||
use crate::{utils::sub_path, OuterContext};
|
use crate::{utils::sub_path, OuterContext};
|
||||||
|
|
||||||
use sd_core_file_path_helper::{FilePathError, IsolatedFilePathData};
|
use sd_core_file_path_helper::{FilePathError, IsolatedFilePathData};
|
||||||
use sd_core_indexer_rules::IndexerRuleError;
|
|
||||||
use sd_core_prisma_helpers::{
|
use sd_core_prisma_helpers::{
|
||||||
file_path_pub_and_cas_ids, file_path_to_isolate_with_pub_id, file_path_walker,
|
file_path_pub_and_cas_ids, file_path_to_isolate_with_pub_id, file_path_walker,
|
||||||
};
|
};
|
||||||
|
@ -27,11 +26,11 @@ use std::{
|
||||||
};
|
};
|
||||||
|
|
||||||
use itertools::Itertools;
|
use itertools::Itertools;
|
||||||
use prisma_client_rust::{operator::or, Select};
|
use prisma_client_rust::{operator::or, QueryError, Select};
|
||||||
use rspc::ErrorCode;
|
use rspc::ErrorCode;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use specta::Type;
|
use specta::Type;
|
||||||
use tracing::warn;
|
use tracing::{instrument, warn};
|
||||||
|
|
||||||
pub mod job;
|
pub mod job;
|
||||||
mod shallow;
|
mod shallow;
|
||||||
|
@ -53,8 +52,8 @@ pub enum Error {
|
||||||
SubPath(#[from] sub_path::Error),
|
SubPath(#[from] sub_path::Error),
|
||||||
|
|
||||||
// Internal Errors
|
// Internal Errors
|
||||||
#[error("database Error: {0}")]
|
#[error("database error: {0}")]
|
||||||
Database(#[from] prisma_client_rust::QueryError),
|
Database(#[from] QueryError),
|
||||||
#[error(transparent)]
|
#[error(transparent)]
|
||||||
FileIO(#[from] FileIOError),
|
FileIO(#[from] FileIOError),
|
||||||
#[error(transparent)]
|
#[error(transparent)]
|
||||||
|
@ -68,27 +67,28 @@ pub enum Error {
|
||||||
|
|
||||||
// Mixed errors
|
// Mixed errors
|
||||||
#[error(transparent)]
|
#[error(transparent)]
|
||||||
Rules(#[from] IndexerRuleError),
|
Rules(#[from] sd_core_indexer_rules::Error),
|
||||||
}
|
}
|
||||||
|
|
||||||
impl From<Error> for rspc::Error {
|
impl From<Error> for rspc::Error {
|
||||||
fn from(err: Error) -> Self {
|
fn from(e: Error) -> Self {
|
||||||
match err {
|
match e {
|
||||||
Error::IndexerRuleNotFound(_) => {
|
Error::IndexerRuleNotFound(_) => {
|
||||||
Self::with_cause(ErrorCode::NotFound, err.to_string(), err)
|
Self::with_cause(ErrorCode::NotFound, e.to_string(), e)
|
||||||
}
|
}
|
||||||
|
|
||||||
Error::SubPath(sub_path_err) => sub_path_err.into(),
|
Error::SubPath(sub_path_err) => sub_path_err.into(),
|
||||||
|
|
||||||
Error::Rules(rule_err) => rule_err.into(),
|
Error::Rules(rule_err) => rule_err.into(),
|
||||||
|
|
||||||
_ => Self::with_cause(ErrorCode::InternalServerError, err.to_string(), err),
|
_ => Self::with_cause(ErrorCode::InternalServerError, e.to_string(), e),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(thiserror::Error, Debug, Serialize, Deserialize, Type)]
|
#[derive(thiserror::Error, Debug, Serialize, Deserialize, Type, Clone)]
|
||||||
pub enum NonCriticalError {
|
#[serde(rename_all = "snake_case")]
|
||||||
|
pub enum NonCriticalIndexerError {
|
||||||
#[error("failed to read directory entry: {0}")]
|
#[error("failed to read directory entry: {0}")]
|
||||||
FailedDirectoryEntry(String),
|
FailedDirectoryEntry(String),
|
||||||
#[error("failed to fetch metadata: {0}")]
|
#[error("failed to fetch metadata: {0}")]
|
||||||
|
@ -153,10 +153,12 @@ async fn update_directory_sizes(
|
||||||
file_path::size_in_bytes_bytes::NAME,
|
file_path::size_in_bytes_bytes::NAME,
|
||||||
msgpack!(size_bytes),
|
msgpack!(size_bytes),
|
||||||
),
|
),
|
||||||
db.file_path().update(
|
db.file_path()
|
||||||
file_path::pub_id::equals(file_path.pub_id),
|
.update(
|
||||||
vec![file_path::size_in_bytes_bytes::set(Some(size_bytes))],
|
file_path::pub_id::equals(file_path.pub_id),
|
||||||
),
|
vec![file_path::size_in_bytes_bytes::set(Some(size_bytes))],
|
||||||
|
)
|
||||||
|
.select(file_path::select!({ id })),
|
||||||
))
|
))
|
||||||
})
|
})
|
||||||
.collect::<Result<Vec<_>, Error>>()?
|
.collect::<Result<Vec<_>, Error>>()?
|
||||||
|
@ -240,8 +242,16 @@ async fn remove_non_existing_file_paths(
|
||||||
.map_err(Into::into)
|
.map_err(Into::into)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[instrument(
|
||||||
|
skip(base_path, location_path, db, sync, errors),
|
||||||
|
fields(
|
||||||
|
base_path = %base_path.as_ref().display(),
|
||||||
|
location_path = %location_path.as_ref().display(),
|
||||||
|
),
|
||||||
|
err,
|
||||||
|
)]
|
||||||
#[allow(clippy::missing_panics_doc)] // Can't actually panic as we only deal with directories
|
#[allow(clippy::missing_panics_doc)] // Can't actually panic as we only deal with directories
|
||||||
async fn reverse_update_directories_sizes(
|
pub async fn reverse_update_directories_sizes(
|
||||||
base_path: impl AsRef<Path> + Send,
|
base_path: impl AsRef<Path> + Send,
|
||||||
location_id: location::id::Type,
|
location_id: location::id::Type,
|
||||||
location_path: impl AsRef<Path> + Send,
|
location_path: impl AsRef<Path> + Send,
|
||||||
|
@ -278,7 +288,7 @@ async fn reverse_update_directories_sizes(
|
||||||
IsolatedFilePathData::try_from(file_path)
|
IsolatedFilePathData::try_from(file_path)
|
||||||
.map_err(|e| {
|
.map_err(|e| {
|
||||||
errors.push(
|
errors.push(
|
||||||
NonCriticalError::MissingFilePathData(format!(
|
NonCriticalIndexerError::MissingFilePathData(format!(
|
||||||
"Found a file_path missing data: <pub_id='{:#?}'>, error: {e:#?}",
|
"Found a file_path missing data: <pub_id='{:#?}'>, error: {e:#?}",
|
||||||
from_bytes_to_uuid(&pub_id)
|
from_bytes_to_uuid(&pub_id)
|
||||||
))
|
))
|
||||||
|
@ -328,7 +338,7 @@ async fn reverse_update_directories_sizes(
|
||||||
),
|
),
|
||||||
))
|
))
|
||||||
} else {
|
} else {
|
||||||
warn!("Got a missing ancestor for a file_path in the database, maybe we have a corruption");
|
warn!("Got a missing ancestor for a file_path in the database, ignoring...");
|
||||||
None
|
None
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
@ -345,8 +355,9 @@ async fn compute_sizes(
|
||||||
pub_id_by_ancestor_materialized_path: &mut HashMap<String, (file_path::pub_id::Type, u64)>,
|
pub_id_by_ancestor_materialized_path: &mut HashMap<String, (file_path::pub_id::Type, u64)>,
|
||||||
db: &PrismaClient,
|
db: &PrismaClient,
|
||||||
errors: &mut Vec<crate::NonCriticalError>,
|
errors: &mut Vec<crate::NonCriticalError>,
|
||||||
) -> Result<(), Error> {
|
) -> Result<(), QueryError> {
|
||||||
db.file_path()
|
for file_path in db
|
||||||
|
.file_path()
|
||||||
.find_many(vec![
|
.find_many(vec![
|
||||||
file_path::location_id::equals(Some(location_id)),
|
file_path::location_id::equals(Some(location_id)),
|
||||||
file_path::materialized_path::in_vec(materialized_paths),
|
file_path::materialized_path::in_vec(materialized_paths),
|
||||||
|
@ -354,30 +365,29 @@ async fn compute_sizes(
|
||||||
.select(file_path::select!({ pub_id materialized_path size_in_bytes_bytes }))
|
.select(file_path::select!({ pub_id materialized_path size_in_bytes_bytes }))
|
||||||
.exec()
|
.exec()
|
||||||
.await?
|
.await?
|
||||||
.into_iter()
|
{
|
||||||
.for_each(|file_path| {
|
if let Some(materialized_path) = file_path.materialized_path {
|
||||||
if let Some(materialized_path) = file_path.materialized_path {
|
if let Some((_, size)) =
|
||||||
if let Some((_, size)) =
|
pub_id_by_ancestor_materialized_path.get_mut(&materialized_path)
|
||||||
pub_id_by_ancestor_materialized_path.get_mut(&materialized_path)
|
{
|
||||||
{
|
*size += file_path.size_in_bytes_bytes.map_or_else(
|
||||||
*size += file_path.size_in_bytes_bytes.map_or_else(
|
|| {
|
||||||
|| {
|
warn!("Got a directory missing its size in bytes");
|
||||||
warn!("Got a directory missing its size in bytes");
|
0
|
||||||
0
|
},
|
||||||
},
|
|size_in_bytes_bytes| size_in_bytes_from_db(&size_in_bytes_bytes),
|
||||||
|size_in_bytes_bytes| size_in_bytes_from_db(&size_in_bytes_bytes),
|
);
|
||||||
);
|
}
|
||||||
}
|
} else {
|
||||||
} else {
|
errors.push(
|
||||||
errors.push(
|
NonCriticalIndexerError::MissingFilePathData(format!(
|
||||||
NonCriticalError::MissingFilePathData(format!(
|
|
||||||
"Corrupt database possessing a file_path entry without materialized_path: <pub_id='{:#?}'>",
|
"Corrupt database possessing a file_path entry without materialized_path: <pub_id='{:#?}'>",
|
||||||
from_bytes_to_uuid(&file_path.pub_id)
|
from_bytes_to_uuid(&file_path.pub_id)
|
||||||
))
|
))
|
||||||
.into(),
|
.into(),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
});
|
}
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
@ -433,57 +443,76 @@ impl walker::WalkerDBProxy for WalkerDBProxy {
|
||||||
async fn fetch_file_paths_to_remove(
|
async fn fetch_file_paths_to_remove(
|
||||||
&self,
|
&self,
|
||||||
parent_iso_file_path: &IsolatedFilePathData<'_>,
|
parent_iso_file_path: &IsolatedFilePathData<'_>,
|
||||||
|
mut existing_inodes: HashSet<Vec<u8>>,
|
||||||
unique_location_id_materialized_path_name_extension_params: Vec<file_path::WhereParam>,
|
unique_location_id_materialized_path_name_extension_params: Vec<file_path::WhereParam>,
|
||||||
) -> Result<Vec<file_path_pub_and_cas_ids::Data>, NonCriticalError> {
|
) -> Result<Vec<file_path_pub_and_cas_ids::Data>, NonCriticalIndexerError> {
|
||||||
// NOTE: This batch size can be increased if we wish to trade memory for more performance
|
// NOTE: This batch size can be increased if we wish to trade memory for more performance
|
||||||
const BATCH_SIZE: i64 = 1000;
|
const BATCH_SIZE: i64 = 1000;
|
||||||
|
|
||||||
let founds_ids = self
|
let founds_ids = {
|
||||||
.db
|
let found_chunks = self
|
||||||
._batch(
|
.db
|
||||||
unique_location_id_materialized_path_name_extension_params
|
._batch(
|
||||||
.into_iter()
|
unique_location_id_materialized_path_name_extension_params
|
||||||
.chunks(200)
|
.into_iter()
|
||||||
.into_iter()
|
.chunks(200)
|
||||||
.map(|unique_params| {
|
.into_iter()
|
||||||
self.db
|
.map(|unique_params| {
|
||||||
.file_path()
|
self.db
|
||||||
.find_many(vec![or(unique_params.collect())])
|
.file_path()
|
||||||
.select(file_path::select!({ id }))
|
.find_many(vec![or(unique_params.collect())])
|
||||||
})
|
.select(file_path::select!({ id inode }))
|
||||||
.collect::<Vec<_>>(),
|
})
|
||||||
)
|
.collect::<Vec<_>>(),
|
||||||
.await
|
)
|
||||||
.map(|founds_chunk| {
|
.await
|
||||||
founds_chunk
|
.map_err(|e| {
|
||||||
.into_iter()
|
NonCriticalIndexerError::FetchAlreadyExistingFilePathIds(e.to_string())
|
||||||
.flat_map(|file_paths| file_paths.into_iter().map(|file_path| file_path.id))
|
})?;
|
||||||
.collect::<HashSet<_>>()
|
|
||||||
})
|
found_chunks
|
||||||
.map_err(|e| NonCriticalError::FetchAlreadyExistingFilePathIds(e.to_string()))?;
|
.into_iter()
|
||||||
|
.flatten()
|
||||||
|
.map(|file_path| {
|
||||||
|
if let Some(inode) = file_path.inode {
|
||||||
|
existing_inodes.remove(&inode);
|
||||||
|
}
|
||||||
|
file_path.id
|
||||||
|
})
|
||||||
|
.collect::<HashSet<_>>()
|
||||||
|
};
|
||||||
|
|
||||||
let mut to_remove = vec![];
|
let mut to_remove = vec![];
|
||||||
let mut cursor = 1;
|
let mut cursor = 1;
|
||||||
|
|
||||||
loop {
|
loop {
|
||||||
|
let materialized_path_param = file_path::materialized_path::equals(Some(
|
||||||
|
parent_iso_file_path
|
||||||
|
.materialized_path_for_children()
|
||||||
|
.expect("the received isolated file path must be from a directory"),
|
||||||
|
));
|
||||||
|
|
||||||
let found = self
|
let found = self
|
||||||
.db
|
.db
|
||||||
.file_path()
|
.file_path()
|
||||||
.find_many(vec![
|
.find_many(vec![
|
||||||
file_path::location_id::equals(Some(self.location_id)),
|
file_path::location_id::equals(Some(self.location_id)),
|
||||||
file_path::materialized_path::equals(Some(
|
if existing_inodes.is_empty() {
|
||||||
parent_iso_file_path
|
materialized_path_param
|
||||||
.materialized_path_for_children()
|
} else {
|
||||||
.expect("the received isolated file path must be from a directory"),
|
or(vec![
|
||||||
)),
|
materialized_path_param,
|
||||||
|
file_path::inode::in_vec(existing_inodes.iter().cloned().collect()),
|
||||||
|
])
|
||||||
|
},
|
||||||
])
|
])
|
||||||
.order_by(file_path::id::order(SortOrder::Asc))
|
.order_by(file_path::id::order(SortOrder::Asc))
|
||||||
.take(BATCH_SIZE)
|
.take(BATCH_SIZE)
|
||||||
.cursor(file_path::id::equals(cursor))
|
.cursor(file_path::id::equals(cursor))
|
||||||
.select(file_path_pub_and_cas_ids::select())
|
.select(file_path::select!({ id pub_id cas_id inode }))
|
||||||
.exec()
|
.exec()
|
||||||
.await
|
.await
|
||||||
.map_err(|e| NonCriticalError::FetchFilePathsToRemove(e.to_string()))?;
|
.map_err(|e| NonCriticalIndexerError::FetchFilePathsToRemove(e.to_string()))?;
|
||||||
|
|
||||||
#[allow(clippy::cast_possible_truncation)] // Safe because we are using a constant
|
#[allow(clippy::cast_possible_truncation)] // Safe because we are using a constant
|
||||||
let should_stop = found.len() < BATCH_SIZE as usize;
|
let should_stop = found.len() < BATCH_SIZE as usize;
|
||||||
|
@ -494,11 +523,17 @@ impl walker::WalkerDBProxy for WalkerDBProxy {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
to_remove.extend(
|
to_remove.extend(found.into_iter().filter_map(|file_path| {
|
||||||
found
|
if let Some(inode) = file_path.inode {
|
||||||
.into_iter()
|
existing_inodes.remove(&inode);
|
||||||
.filter(|file_path| !founds_ids.contains(&file_path.id)),
|
}
|
||||||
);
|
|
||||||
|
(!founds_ids.contains(&file_path.id)).then_some(file_path_pub_and_cas_ids::Data {
|
||||||
|
id: file_path.id,
|
||||||
|
pub_id: file_path.pub_id,
|
||||||
|
cas_id: file_path.cas_id,
|
||||||
|
})
|
||||||
|
}));
|
||||||
|
|
||||||
if should_stop {
|
if should_stop {
|
||||||
break;
|
break;
|
||||||
|
|
|
@ -18,25 +18,32 @@ use std::{
|
||||||
|
|
||||||
use futures_concurrency::future::TryJoin;
|
use futures_concurrency::future::TryJoin;
|
||||||
use itertools::Itertools;
|
use itertools::Itertools;
|
||||||
use tracing::{debug, warn};
|
use tracing::{debug, instrument, warn};
|
||||||
|
|
||||||
use super::{
|
use super::{
|
||||||
remove_non_existing_file_paths, reverse_update_directories_sizes,
|
remove_non_existing_file_paths, reverse_update_directories_sizes,
|
||||||
tasks::{
|
tasks::{
|
||||||
saver::{SaveTask, SaveTaskOutput},
|
self, saver, updater,
|
||||||
updater::{UpdateTask, UpdateTaskOutput},
|
walker::{self, ToWalkEntry, WalkedEntry},
|
||||||
walker::{ToWalkEntry, WalkDirTask, WalkTaskOutput, WalkedEntry},
|
|
||||||
},
|
},
|
||||||
update_directory_sizes, update_location_size, IsoFilePathFactory, WalkerDBProxy, BATCH_SIZE,
|
update_directory_sizes, update_location_size, IsoFilePathFactory, WalkerDBProxy, BATCH_SIZE,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
#[instrument(
|
||||||
|
skip_all,
|
||||||
|
fields(
|
||||||
|
location_id = location.id,
|
||||||
|
location_path = ?location.path,
|
||||||
|
sub_path = %sub_path.as_ref().display()
|
||||||
|
)
|
||||||
|
err,
|
||||||
|
)]
|
||||||
pub async fn shallow(
|
pub async fn shallow(
|
||||||
location: location_with_indexer_rules::Data,
|
location: location_with_indexer_rules::Data,
|
||||||
sub_path: impl AsRef<Path> + Send,
|
sub_path: impl AsRef<Path> + Send,
|
||||||
dispatcher: BaseTaskDispatcher<Error>,
|
dispatcher: &BaseTaskDispatcher<Error>,
|
||||||
ctx: impl OuterContext,
|
ctx: &impl OuterContext,
|
||||||
) -> Result<Vec<NonCriticalError>, Error> {
|
) -> Result<Vec<NonCriticalError>, Error> {
|
||||||
let sub_path = sub_path.as_ref();
|
|
||||||
let db = ctx.db();
|
let db = ctx.db();
|
||||||
let sync = ctx.sync();
|
let sync = ctx.sync();
|
||||||
|
|
||||||
|
@ -46,15 +53,20 @@ pub async fn shallow(
|
||||||
.map_err(indexer::Error::from)?;
|
.map_err(indexer::Error::from)?;
|
||||||
|
|
||||||
let to_walk_path = Arc::new(
|
let to_walk_path = Arc::new(
|
||||||
get_full_path_from_sub_path(location.id, &Some(sub_path), &*location_path, db)
|
get_full_path_from_sub_path::<indexer::Error>(
|
||||||
.await
|
location.id,
|
||||||
.map_err(indexer::Error::from)?,
|
Some(sub_path.as_ref()),
|
||||||
|
&*location_path,
|
||||||
|
db,
|
||||||
|
)
|
||||||
|
.await?,
|
||||||
);
|
);
|
||||||
|
|
||||||
let Some(WalkTaskOutput {
|
let Some(walker::Output {
|
||||||
to_create,
|
to_create,
|
||||||
to_update,
|
to_update,
|
||||||
to_remove,
|
to_remove,
|
||||||
|
non_indexed_paths,
|
||||||
mut errors,
|
mut errors,
|
||||||
directory_iso_file_path,
|
directory_iso_file_path,
|
||||||
total_size,
|
total_size,
|
||||||
|
@ -64,13 +76,16 @@ pub async fn shallow(
|
||||||
Arc::clone(&location_path),
|
Arc::clone(&location_path),
|
||||||
Arc::clone(&to_walk_path),
|
Arc::clone(&to_walk_path),
|
||||||
Arc::clone(db),
|
Arc::clone(db),
|
||||||
&dispatcher,
|
dispatcher,
|
||||||
)
|
)
|
||||||
.await?
|
.await?
|
||||||
else {
|
else {
|
||||||
return Ok(vec![]);
|
return Ok(vec![]);
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// TODO use non_indexed_paths here in the future, sending it to frontend, showing then alongside the indexed files from db
|
||||||
|
debug!(non_indexed_paths_count = non_indexed_paths.len());
|
||||||
|
|
||||||
let removed_count = remove_non_existing_file_paths(to_remove, db, sync).await?;
|
let removed_count = remove_non_existing_file_paths(to_remove, db, sync).await?;
|
||||||
|
|
||||||
let Some(Metadata {
|
let Some(Metadata {
|
||||||
|
@ -82,7 +97,7 @@ pub async fn shallow(
|
||||||
to_update,
|
to_update,
|
||||||
Arc::clone(db),
|
Arc::clone(db),
|
||||||
Arc::clone(sync),
|
Arc::clone(sync),
|
||||||
&dispatcher,
|
dispatcher,
|
||||||
)
|
)
|
||||||
.await?
|
.await?
|
||||||
else {
|
else {
|
||||||
|
@ -109,7 +124,7 @@ pub async fn shallow(
|
||||||
.await?;
|
.await?;
|
||||||
}
|
}
|
||||||
|
|
||||||
update_location_size(location.id, db, &ctx).await?;
|
update_location_size(location.id, db, ctx).await?;
|
||||||
}
|
}
|
||||||
|
|
||||||
if indexed_count > 0 || removed_count > 0 {
|
if indexed_count > 0 || removed_count > 0 {
|
||||||
|
@ -119,15 +134,19 @@ pub async fn shallow(
|
||||||
Ok(errors)
|
Ok(errors)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[instrument(
|
||||||
|
skip_all,
|
||||||
|
fields(to_walk_path = %to_walk_path.display())
|
||||||
|
)]
|
||||||
async fn walk(
|
async fn walk(
|
||||||
location: &location_with_indexer_rules::Data,
|
location: &location_with_indexer_rules::Data,
|
||||||
location_path: Arc<PathBuf>,
|
location_path: Arc<PathBuf>,
|
||||||
to_walk_path: Arc<PathBuf>,
|
to_walk_path: Arc<PathBuf>,
|
||||||
db: Arc<PrismaClient>,
|
db: Arc<PrismaClient>,
|
||||||
dispatcher: &BaseTaskDispatcher<Error>,
|
dispatcher: &BaseTaskDispatcher<Error>,
|
||||||
) -> Result<Option<WalkTaskOutput>, Error> {
|
) -> Result<Option<walker::Output<WalkerDBProxy, IsoFilePathFactory>>, Error> {
|
||||||
match dispatcher
|
let Ok(task_handle) = dispatcher
|
||||||
.dispatch(WalkDirTask::new_shallow(
|
.dispatch(tasks::Walker::new_shallow(
|
||||||
ToWalkEntry::from(&*to_walk_path),
|
ToWalkEntry::from(&*to_walk_path),
|
||||||
to_walk_path,
|
to_walk_path,
|
||||||
location
|
location
|
||||||
|
@ -147,11 +166,15 @@ async fn walk(
|
||||||
},
|
},
|
||||||
)?)
|
)?)
|
||||||
.await
|
.await
|
||||||
.await?
|
else {
|
||||||
{
|
debug!("Task system is shutting down while a shallow indexer was in progress");
|
||||||
|
return Ok(None);
|
||||||
|
};
|
||||||
|
|
||||||
|
match task_handle.await? {
|
||||||
sd_task_system::TaskStatus::Done((_, TaskOutput::Out(data))) => Ok(Some(
|
sd_task_system::TaskStatus::Done((_, TaskOutput::Out(data))) => Ok(Some(
|
||||||
*data
|
*data
|
||||||
.downcast::<WalkTaskOutput>()
|
.downcast::<walker::Output<WalkerDBProxy, IsoFilePathFactory>>()
|
||||||
.expect("we just dispatched this task"),
|
.expect("we just dispatched this task"),
|
||||||
)),
|
)),
|
||||||
sd_task_system::TaskStatus::Done((_, TaskOutput::Empty)) => {
|
sd_task_system::TaskStatus::Done((_, TaskOutput::Empty)) => {
|
||||||
|
@ -188,7 +211,7 @@ async fn save_and_update(
|
||||||
.chunks(BATCH_SIZE)
|
.chunks(BATCH_SIZE)
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.map(|chunk| {
|
.map(|chunk| {
|
||||||
SaveTask::new_shallow(
|
tasks::Saver::new_shallow(
|
||||||
location.id,
|
location.id,
|
||||||
location.pub_id.clone(),
|
location.pub_id.clone(),
|
||||||
chunk.collect::<Vec<_>>(),
|
chunk.collect::<Vec<_>>(),
|
||||||
|
@ -203,7 +226,7 @@ async fn save_and_update(
|
||||||
.chunks(BATCH_SIZE)
|
.chunks(BATCH_SIZE)
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.map(|chunk| {
|
.map(|chunk| {
|
||||||
UpdateTask::new_shallow(
|
tasks::Updater::new_shallow(
|
||||||
chunk.collect::<Vec<_>>(),
|
chunk.collect::<Vec<_>>(),
|
||||||
Arc::clone(&db),
|
Arc::clone(&db),
|
||||||
Arc::clone(&sync),
|
Arc::clone(&sync),
|
||||||
|
@ -218,25 +241,28 @@ async fn save_and_update(
|
||||||
updated_count: 0,
|
updated_count: 0,
|
||||||
};
|
};
|
||||||
|
|
||||||
for task_status in dispatcher
|
let Ok(tasks_handles) = dispatcher.dispatch_many_boxed(save_and_update_tasks).await else {
|
||||||
.dispatch_many_boxed(save_and_update_tasks)
|
debug!("Task system is shutting down while a shallow indexer was in progress");
|
||||||
.await
|
return Ok(None);
|
||||||
|
};
|
||||||
|
|
||||||
|
for task_status in tasks_handles
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.map(CancelTaskOnDrop)
|
.map(CancelTaskOnDrop::new)
|
||||||
.collect::<Vec<_>>()
|
.collect::<Vec<_>>()
|
||||||
.try_join()
|
.try_join()
|
||||||
.await?
|
.await?
|
||||||
{
|
{
|
||||||
match task_status {
|
match task_status {
|
||||||
sd_task_system::TaskStatus::Done((_, TaskOutput::Out(data))) => {
|
sd_task_system::TaskStatus::Done((_, TaskOutput::Out(data))) => {
|
||||||
if data.is::<SaveTaskOutput>() {
|
if data.is::<saver::Output>() {
|
||||||
metadata.indexed_count += data
|
metadata.indexed_count += data
|
||||||
.downcast::<SaveTaskOutput>()
|
.downcast::<saver::Output>()
|
||||||
.expect("just checked")
|
.expect("just checked")
|
||||||
.saved_count;
|
.saved_count;
|
||||||
} else {
|
} else {
|
||||||
metadata.updated_count += data
|
metadata.updated_count += data
|
||||||
.downcast::<UpdateTaskOutput>()
|
.downcast::<updater::Output>()
|
||||||
.expect("just checked")
|
.expect("just checked")
|
||||||
.updated_count;
|
.updated_count;
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,3 +1,7 @@
|
||||||
pub mod saver;
|
pub mod saver;
|
||||||
pub mod updater;
|
pub mod updater;
|
||||||
pub mod walker;
|
pub mod walker;
|
||||||
|
|
||||||
|
pub use saver::Saver;
|
||||||
|
pub use updater::Updater;
|
||||||
|
pub use walker::Walker;
|
||||||
|
|
|
@ -16,22 +16,165 @@ use std::{sync::Arc, time::Duration};
|
||||||
use chrono::Utc;
|
use chrono::Utc;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use tokio::time::Instant;
|
use tokio::time::Instant;
|
||||||
use tracing::trace;
|
use tracing::{instrument, trace, Level};
|
||||||
|
|
||||||
use super::walker::WalkedEntry;
|
use super::walker::WalkedEntry;
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
pub struct SaveTask {
|
pub struct Saver {
|
||||||
|
// Task control
|
||||||
id: TaskId,
|
id: TaskId,
|
||||||
|
is_shallow: bool,
|
||||||
|
|
||||||
|
// Received input args
|
||||||
location_id: location::id::Type,
|
location_id: location::id::Type,
|
||||||
location_pub_id: location::pub_id::Type,
|
location_pub_id: location::pub_id::Type,
|
||||||
walked_entries: Vec<WalkedEntry>,
|
walked_entries: Vec<WalkedEntry>,
|
||||||
|
|
||||||
|
// Dependencies
|
||||||
db: Arc<PrismaClient>,
|
db: Arc<PrismaClient>,
|
||||||
sync: Arc<SyncManager>,
|
sync: Arc<SyncManager>,
|
||||||
is_shallow: bool,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl SaveTask {
|
/// [`Save`] Task output
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub struct Output {
|
||||||
|
/// Number of records inserted on database
|
||||||
|
pub saved_count: u64,
|
||||||
|
/// Time spent saving records
|
||||||
|
pub save_duration: Duration,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait::async_trait]
|
||||||
|
impl Task<Error> for Saver {
|
||||||
|
fn id(&self) -> TaskId {
|
||||||
|
self.id
|
||||||
|
}
|
||||||
|
|
||||||
|
fn with_priority(&self) -> bool {
|
||||||
|
// If we're running in shallow mode, then we want priority
|
||||||
|
self.is_shallow
|
||||||
|
}
|
||||||
|
|
||||||
|
#[instrument(
|
||||||
|
skip_all,
|
||||||
|
fields(
|
||||||
|
task_id = %self.id,
|
||||||
|
location_id = %self.location_id,
|
||||||
|
to_save_count = %self.walked_entries.len(),
|
||||||
|
is_shallow = self.is_shallow,
|
||||||
|
),
|
||||||
|
ret(level = Level::TRACE),
|
||||||
|
err,
|
||||||
|
)]
|
||||||
|
#[allow(clippy::blocks_in_conditions)] // Due to `err` on `instrument` macro above
|
||||||
|
async fn run(&mut self, _: &Interrupter) -> Result<ExecStatus, Error> {
|
||||||
|
use file_path::{
|
||||||
|
create_unchecked, date_created, date_indexed, date_modified, extension, hidden, inode,
|
||||||
|
is_dir, location, location_id, materialized_path, name, size_in_bytes_bytes,
|
||||||
|
};
|
||||||
|
|
||||||
|
let start_time = Instant::now();
|
||||||
|
|
||||||
|
let Self {
|
||||||
|
location_id,
|
||||||
|
location_pub_id,
|
||||||
|
walked_entries,
|
||||||
|
db,
|
||||||
|
sync,
|
||||||
|
..
|
||||||
|
} = self;
|
||||||
|
|
||||||
|
let (sync_stuff, paths): (Vec<_>, Vec<_>) = walked_entries
|
||||||
|
.drain(..)
|
||||||
|
.map(
|
||||||
|
|WalkedEntry {
|
||||||
|
pub_id,
|
||||||
|
maybe_object_id,
|
||||||
|
iso_file_path,
|
||||||
|
metadata,
|
||||||
|
}| {
|
||||||
|
let IsolatedFilePathDataParts {
|
||||||
|
materialized_path,
|
||||||
|
is_dir,
|
||||||
|
name,
|
||||||
|
extension,
|
||||||
|
..
|
||||||
|
} = iso_file_path.to_parts();
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
maybe_object_id.is_none(),
|
||||||
|
"Object ID must be None as this tasks only created \
|
||||||
|
new file_paths and they were not identified yet"
|
||||||
|
);
|
||||||
|
|
||||||
|
let (sync_params, db_params): (Vec<_>, Vec<_>) = [
|
||||||
|
(
|
||||||
|
(
|
||||||
|
location::NAME,
|
||||||
|
msgpack!(prisma_sync::location::SyncId {
|
||||||
|
pub_id: location_pub_id.clone()
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
location_id::set(Some(*location_id)),
|
||||||
|
),
|
||||||
|
sync_db_entry!(materialized_path.to_string(), materialized_path),
|
||||||
|
sync_db_entry!(name.to_string(), name),
|
||||||
|
sync_db_entry!(is_dir, is_dir),
|
||||||
|
sync_db_entry!(extension.to_string(), extension),
|
||||||
|
sync_db_entry!(
|
||||||
|
metadata.size_in_bytes.to_be_bytes().to_vec(),
|
||||||
|
size_in_bytes_bytes
|
||||||
|
),
|
||||||
|
sync_db_entry!(inode_to_db(metadata.inode), inode),
|
||||||
|
sync_db_entry!(metadata.created_at.into(), date_created),
|
||||||
|
sync_db_entry!(metadata.modified_at.into(), date_modified),
|
||||||
|
sync_db_entry!(Utc::now().into(), date_indexed),
|
||||||
|
sync_db_entry!(metadata.hidden, hidden),
|
||||||
|
]
|
||||||
|
.into_iter()
|
||||||
|
.unzip();
|
||||||
|
|
||||||
|
(
|
||||||
|
sync.shared_create(
|
||||||
|
prisma_sync::file_path::SyncId {
|
||||||
|
pub_id: pub_id.to_db(),
|
||||||
|
},
|
||||||
|
sync_params,
|
||||||
|
),
|
||||||
|
create_unchecked(pub_id.into(), db_params),
|
||||||
|
)
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.unzip();
|
||||||
|
|
||||||
|
#[allow(clippy::cast_sign_loss)]
|
||||||
|
let saved_count = sync
|
||||||
|
.write_ops(
|
||||||
|
db,
|
||||||
|
(
|
||||||
|
sync_stuff.into_iter().flatten().collect(),
|
||||||
|
db.file_path().create_many(paths).skip_duplicates(),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(indexer::Error::from)? as u64;
|
||||||
|
|
||||||
|
let save_duration = start_time.elapsed();
|
||||||
|
|
||||||
|
trace!(saved_count, "Inserted records;");
|
||||||
|
|
||||||
|
Ok(ExecStatus::Done(
|
||||||
|
Output {
|
||||||
|
saved_count,
|
||||||
|
save_duration,
|
||||||
|
}
|
||||||
|
.into_output(),
|
||||||
|
))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Saver {
|
||||||
#[must_use]
|
#[must_use]
|
||||||
pub fn new_deep(
|
pub fn new_deep(
|
||||||
location_id: location::id::Type,
|
location_id: location::id::Type,
|
||||||
|
@ -72,15 +215,16 @@ impl SaveTask {
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Serialize, Deserialize)]
|
#[derive(Debug, Serialize, Deserialize)]
|
||||||
struct SaveTaskSaveState {
|
struct SaveState {
|
||||||
id: TaskId,
|
id: TaskId,
|
||||||
|
is_shallow: bool,
|
||||||
|
|
||||||
location_id: location::id::Type,
|
location_id: location::id::Type,
|
||||||
location_pub_id: location::pub_id::Type,
|
location_pub_id: location::pub_id::Type,
|
||||||
walked_entries: Vec<WalkedEntry>,
|
walked_entries: Vec<WalkedEntry>,
|
||||||
is_shallow: bool,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl SerializableTask<Error> for SaveTask {
|
impl SerializableTask<Error> for Saver {
|
||||||
type SerializeError = rmp_serde::encode::Error;
|
type SerializeError = rmp_serde::encode::Error;
|
||||||
|
|
||||||
type DeserializeError = rmp_serde::decode::Error;
|
type DeserializeError = rmp_serde::decode::Error;
|
||||||
|
@ -90,18 +234,18 @@ impl SerializableTask<Error> for SaveTask {
|
||||||
async fn serialize(self) -> Result<Vec<u8>, Self::SerializeError> {
|
async fn serialize(self) -> Result<Vec<u8>, Self::SerializeError> {
|
||||||
let Self {
|
let Self {
|
||||||
id,
|
id,
|
||||||
|
is_shallow,
|
||||||
location_id,
|
location_id,
|
||||||
location_pub_id,
|
location_pub_id,
|
||||||
walked_entries,
|
walked_entries,
|
||||||
is_shallow,
|
|
||||||
..
|
..
|
||||||
} = self;
|
} = self;
|
||||||
rmp_serde::to_vec_named(&SaveTaskSaveState {
|
rmp_serde::to_vec_named(&SaveState {
|
||||||
id,
|
id,
|
||||||
|
is_shallow,
|
||||||
location_id,
|
location_id,
|
||||||
location_pub_id,
|
location_pub_id,
|
||||||
walked_entries,
|
walked_entries,
|
||||||
is_shallow,
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -110,131 +254,21 @@ impl SerializableTask<Error> for SaveTask {
|
||||||
(db, sync): Self::DeserializeCtx,
|
(db, sync): Self::DeserializeCtx,
|
||||||
) -> Result<Self, Self::DeserializeError> {
|
) -> Result<Self, Self::DeserializeError> {
|
||||||
rmp_serde::from_slice(data).map(
|
rmp_serde::from_slice(data).map(
|
||||||
|SaveTaskSaveState {
|
|SaveState {
|
||||||
id,
|
id,
|
||||||
|
is_shallow,
|
||||||
location_id,
|
location_id,
|
||||||
location_pub_id,
|
location_pub_id,
|
||||||
walked_entries,
|
walked_entries,
|
||||||
is_shallow,
|
|
||||||
}| Self {
|
}| Self {
|
||||||
id,
|
id,
|
||||||
|
is_shallow,
|
||||||
location_id,
|
location_id,
|
||||||
location_pub_id,
|
location_pub_id,
|
||||||
walked_entries,
|
walked_entries,
|
||||||
db,
|
db,
|
||||||
sync,
|
sync,
|
||||||
is_shallow,
|
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug)]
|
|
||||||
pub struct SaveTaskOutput {
|
|
||||||
pub saved_count: u64,
|
|
||||||
pub save_duration: Duration,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[async_trait::async_trait]
|
|
||||||
impl Task<Error> for SaveTask {
|
|
||||||
fn id(&self) -> TaskId {
|
|
||||||
self.id
|
|
||||||
}
|
|
||||||
|
|
||||||
fn with_priority(&self) -> bool {
|
|
||||||
// If we're running in shallow mode, then we want priority
|
|
||||||
self.is_shallow
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn run(&mut self, _: &Interrupter) -> Result<ExecStatus, Error> {
|
|
||||||
use file_path::{
|
|
||||||
create_unchecked, date_created, date_indexed, date_modified, extension, hidden, inode,
|
|
||||||
is_dir, location, location_id, materialized_path, name, size_in_bytes_bytes,
|
|
||||||
};
|
|
||||||
|
|
||||||
let start_time = Instant::now();
|
|
||||||
|
|
||||||
let Self {
|
|
||||||
location_id,
|
|
||||||
location_pub_id,
|
|
||||||
walked_entries,
|
|
||||||
db,
|
|
||||||
sync,
|
|
||||||
..
|
|
||||||
} = self;
|
|
||||||
|
|
||||||
let (sync_stuff, paths): (Vec<_>, Vec<_>) = walked_entries
|
|
||||||
.drain(..)
|
|
||||||
.map(|entry| {
|
|
||||||
let IsolatedFilePathDataParts {
|
|
||||||
materialized_path,
|
|
||||||
is_dir,
|
|
||||||
name,
|
|
||||||
extension,
|
|
||||||
..
|
|
||||||
} = entry.iso_file_path.to_parts();
|
|
||||||
|
|
||||||
let pub_id = sd_utils::uuid_to_bytes(entry.pub_id);
|
|
||||||
|
|
||||||
let (sync_params, db_params): (Vec<_>, Vec<_>) = [
|
|
||||||
(
|
|
||||||
(
|
|
||||||
location::NAME,
|
|
||||||
msgpack!(prisma_sync::location::SyncId {
|
|
||||||
pub_id: location_pub_id.clone()
|
|
||||||
}),
|
|
||||||
),
|
|
||||||
location_id::set(Some(*location_id)),
|
|
||||||
),
|
|
||||||
sync_db_entry!(materialized_path.to_string(), materialized_path),
|
|
||||||
sync_db_entry!(name.to_string(), name),
|
|
||||||
sync_db_entry!(is_dir, is_dir),
|
|
||||||
sync_db_entry!(extension.to_string(), extension),
|
|
||||||
sync_db_entry!(
|
|
||||||
entry.metadata.size_in_bytes.to_be_bytes().to_vec(),
|
|
||||||
size_in_bytes_bytes
|
|
||||||
),
|
|
||||||
sync_db_entry!(inode_to_db(entry.metadata.inode), inode),
|
|
||||||
sync_db_entry!(entry.metadata.created_at.into(), date_created),
|
|
||||||
sync_db_entry!(entry.metadata.modified_at.into(), date_modified),
|
|
||||||
sync_db_entry!(Utc::now().into(), date_indexed),
|
|
||||||
sync_db_entry!(entry.metadata.hidden, hidden),
|
|
||||||
]
|
|
||||||
.into_iter()
|
|
||||||
.unzip();
|
|
||||||
|
|
||||||
(
|
|
||||||
sync.shared_create(
|
|
||||||
prisma_sync::file_path::SyncId {
|
|
||||||
pub_id: sd_utils::uuid_to_bytes(entry.pub_id),
|
|
||||||
},
|
|
||||||
sync_params,
|
|
||||||
),
|
|
||||||
create_unchecked(pub_id, db_params),
|
|
||||||
)
|
|
||||||
})
|
|
||||||
.unzip();
|
|
||||||
|
|
||||||
#[allow(clippy::cast_sign_loss)]
|
|
||||||
let saved_count = sync
|
|
||||||
.write_ops(
|
|
||||||
db,
|
|
||||||
(
|
|
||||||
sync_stuff.into_iter().flatten().collect(),
|
|
||||||
db.file_path().create_many(paths).skip_duplicates(),
|
|
||||||
),
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.map_err(indexer::Error::from)? as u64;
|
|
||||||
|
|
||||||
trace!("Inserted {saved_count} records");
|
|
||||||
|
|
||||||
Ok(ExecStatus::Done(
|
|
||||||
SaveTaskOutput {
|
|
||||||
saved_count,
|
|
||||||
save_duration: start_time.elapsed(),
|
|
||||||
}
|
|
||||||
.into_output(),
|
|
||||||
))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
|
@ -17,21 +17,169 @@ use std::{collections::HashSet, sync::Arc, time::Duration};
|
||||||
|
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use tokio::time::Instant;
|
use tokio::time::Instant;
|
||||||
use tracing::trace;
|
use tracing::{instrument, trace, Level};
|
||||||
|
|
||||||
use super::walker::WalkedEntry;
|
use super::walker::WalkedEntry;
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
pub struct UpdateTask {
|
pub struct Updater {
|
||||||
|
// Task control
|
||||||
id: TaskId,
|
id: TaskId,
|
||||||
|
is_shallow: bool,
|
||||||
|
|
||||||
|
// Received input args
|
||||||
walked_entries: Vec<WalkedEntry>,
|
walked_entries: Vec<WalkedEntry>,
|
||||||
|
|
||||||
|
// Inner state
|
||||||
object_ids_that_should_be_unlinked: HashSet<object::id::Type>,
|
object_ids_that_should_be_unlinked: HashSet<object::id::Type>,
|
||||||
|
|
||||||
|
// Dependencies
|
||||||
db: Arc<PrismaClient>,
|
db: Arc<PrismaClient>,
|
||||||
sync: Arc<SyncManager>,
|
sync: Arc<SyncManager>,
|
||||||
is_shallow: bool,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl UpdateTask {
|
/// [`Update`] Task output
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub struct Output {
|
||||||
|
/// Number of records updated on database
|
||||||
|
pub updated_count: u64,
|
||||||
|
/// Time spent updating records
|
||||||
|
pub update_duration: Duration,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait::async_trait]
|
||||||
|
impl Task<Error> for Updater {
|
||||||
|
fn id(&self) -> TaskId {
|
||||||
|
self.id
|
||||||
|
}
|
||||||
|
|
||||||
|
fn with_priority(&self) -> bool {
|
||||||
|
// If we're running in shallow mode, then we want priority
|
||||||
|
self.is_shallow
|
||||||
|
}
|
||||||
|
|
||||||
|
#[instrument(
|
||||||
|
skip_all,
|
||||||
|
fields(
|
||||||
|
task_id = %self.id,
|
||||||
|
to_update_count = %self.walked_entries.len(),
|
||||||
|
is_shallow = self.is_shallow,
|
||||||
|
),
|
||||||
|
ret(level = Level::TRACE),
|
||||||
|
err,
|
||||||
|
)]
|
||||||
|
#[allow(clippy::blocks_in_conditions)] // Due to `err` on `instrument` macro above
|
||||||
|
async fn run(&mut self, interrupter: &Interrupter) -> Result<ExecStatus, Error> {
|
||||||
|
use file_path::{
|
||||||
|
cas_id, date_created, date_modified, hidden, inode, is_dir, object, object_id,
|
||||||
|
size_in_bytes_bytes,
|
||||||
|
};
|
||||||
|
|
||||||
|
let start_time = Instant::now();
|
||||||
|
|
||||||
|
let Self {
|
||||||
|
walked_entries,
|
||||||
|
db,
|
||||||
|
sync,
|
||||||
|
object_ids_that_should_be_unlinked,
|
||||||
|
..
|
||||||
|
} = self;
|
||||||
|
|
||||||
|
fetch_objects_ids_to_unlink(walked_entries, object_ids_that_should_be_unlinked, db).await?;
|
||||||
|
|
||||||
|
check_interruption!(interrupter);
|
||||||
|
|
||||||
|
let (sync_stuff, paths_to_update) = walked_entries
|
||||||
|
.drain(..)
|
||||||
|
.map(
|
||||||
|
|WalkedEntry {
|
||||||
|
pub_id,
|
||||||
|
maybe_object_id,
|
||||||
|
iso_file_path,
|
||||||
|
metadata,
|
||||||
|
}| {
|
||||||
|
let IsolatedFilePathDataParts { is_dir, .. } = &iso_file_path.to_parts();
|
||||||
|
|
||||||
|
let should_unlink_object = maybe_object_id.map_or(false, |object_id| {
|
||||||
|
object_ids_that_should_be_unlinked.contains(&object_id)
|
||||||
|
});
|
||||||
|
|
||||||
|
let (sync_params, db_params) = chain_optional_iter(
|
||||||
|
[
|
||||||
|
((cas_id::NAME, msgpack!(nil)), cas_id::set(None)),
|
||||||
|
sync_db_entry!(*is_dir, is_dir),
|
||||||
|
sync_db_entry!(
|
||||||
|
metadata.size_in_bytes.to_be_bytes().to_vec(),
|
||||||
|
size_in_bytes_bytes
|
||||||
|
),
|
||||||
|
sync_db_entry!(inode_to_db(metadata.inode), inode),
|
||||||
|
{
|
||||||
|
let v = metadata.created_at.into();
|
||||||
|
sync_db_entry!(v, date_created)
|
||||||
|
},
|
||||||
|
{
|
||||||
|
let v = metadata.modified_at.into();
|
||||||
|
sync_db_entry!(v, date_modified)
|
||||||
|
},
|
||||||
|
sync_db_entry!(metadata.hidden, hidden),
|
||||||
|
],
|
||||||
|
[
|
||||||
|
// As this file was updated while Spacedrive was offline, we mark the object_id and cas_id as null
|
||||||
|
// So this file_path will be updated at file identifier job
|
||||||
|
should_unlink_object.then_some((
|
||||||
|
(object_id::NAME, msgpack!(nil)),
|
||||||
|
object::disconnect(),
|
||||||
|
)),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
.into_iter()
|
||||||
|
.unzip::<_, _, Vec<_>, Vec<_>>();
|
||||||
|
|
||||||
|
(
|
||||||
|
sync_params
|
||||||
|
.into_iter()
|
||||||
|
.map(|(field, value)| {
|
||||||
|
sync.shared_update(
|
||||||
|
prisma_sync::file_path::SyncId {
|
||||||
|
pub_id: pub_id.to_db(),
|
||||||
|
},
|
||||||
|
field,
|
||||||
|
value,
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>(),
|
||||||
|
db.file_path()
|
||||||
|
.update(file_path::pub_id::equals(pub_id.into()), db_params)
|
||||||
|
// selecting id to avoid fetching whole object from database
|
||||||
|
.select(file_path::select!({ id })),
|
||||||
|
)
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.unzip::<_, _, Vec<_>, Vec<_>>();
|
||||||
|
|
||||||
|
let updated = sync
|
||||||
|
.write_ops(
|
||||||
|
db,
|
||||||
|
(sync_stuff.into_iter().flatten().collect(), paths_to_update),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(indexer::Error::from)?;
|
||||||
|
|
||||||
|
let update_duration = start_time.elapsed();
|
||||||
|
|
||||||
|
trace!(?updated, "Updated records;");
|
||||||
|
|
||||||
|
Ok(ExecStatus::Done(
|
||||||
|
Output {
|
||||||
|
updated_count: updated.len() as u64,
|
||||||
|
update_duration,
|
||||||
|
}
|
||||||
|
.into_output(),
|
||||||
|
))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Updater {
|
||||||
#[must_use]
|
#[must_use]
|
||||||
pub fn new_deep(
|
pub fn new_deep(
|
||||||
walked_entries: Vec<WalkedEntry>,
|
walked_entries: Vec<WalkedEntry>,
|
||||||
|
@ -65,177 +213,6 @@ impl UpdateTask {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Serialize, Deserialize)]
|
|
||||||
struct UpdateTaskSaveState {
|
|
||||||
id: TaskId,
|
|
||||||
walked_entries: Vec<WalkedEntry>,
|
|
||||||
object_ids_that_should_be_unlinked: HashSet<object::id::Type>,
|
|
||||||
is_shallow: bool,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl SerializableTask<Error> for UpdateTask {
|
|
||||||
type SerializeError = rmp_serde::encode::Error;
|
|
||||||
|
|
||||||
type DeserializeError = rmp_serde::decode::Error;
|
|
||||||
|
|
||||||
type DeserializeCtx = (Arc<PrismaClient>, Arc<SyncManager>);
|
|
||||||
|
|
||||||
async fn serialize(self) -> Result<Vec<u8>, Self::SerializeError> {
|
|
||||||
let Self {
|
|
||||||
id,
|
|
||||||
walked_entries,
|
|
||||||
object_ids_that_should_be_unlinked,
|
|
||||||
is_shallow,
|
|
||||||
..
|
|
||||||
} = self;
|
|
||||||
|
|
||||||
rmp_serde::to_vec_named(&UpdateTaskSaveState {
|
|
||||||
id,
|
|
||||||
walked_entries,
|
|
||||||
object_ids_that_should_be_unlinked,
|
|
||||||
is_shallow,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn deserialize(
|
|
||||||
data: &[u8],
|
|
||||||
(db, sync): Self::DeserializeCtx,
|
|
||||||
) -> Result<Self, Self::DeserializeError> {
|
|
||||||
rmp_serde::from_slice(data).map(
|
|
||||||
|UpdateTaskSaveState {
|
|
||||||
id,
|
|
||||||
walked_entries,
|
|
||||||
object_ids_that_should_be_unlinked,
|
|
||||||
is_shallow,
|
|
||||||
}| Self {
|
|
||||||
id,
|
|
||||||
walked_entries,
|
|
||||||
object_ids_that_should_be_unlinked,
|
|
||||||
db,
|
|
||||||
sync,
|
|
||||||
is_shallow,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug)]
|
|
||||||
pub struct UpdateTaskOutput {
|
|
||||||
pub updated_count: u64,
|
|
||||||
pub update_duration: Duration,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[async_trait::async_trait]
|
|
||||||
impl Task<Error> for UpdateTask {
|
|
||||||
fn id(&self) -> TaskId {
|
|
||||||
self.id
|
|
||||||
}
|
|
||||||
|
|
||||||
fn with_priority(&self) -> bool {
|
|
||||||
// If we're running in shallow mode, then we want priority
|
|
||||||
self.is_shallow
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn run(&mut self, interrupter: &Interrupter) -> Result<ExecStatus, Error> {
|
|
||||||
use file_path::{
|
|
||||||
cas_id, date_created, date_modified, hidden, inode, is_dir, object, object_id,
|
|
||||||
size_in_bytes_bytes,
|
|
||||||
};
|
|
||||||
|
|
||||||
let start_time = Instant::now();
|
|
||||||
|
|
||||||
let Self {
|
|
||||||
walked_entries,
|
|
||||||
db,
|
|
||||||
sync,
|
|
||||||
object_ids_that_should_be_unlinked,
|
|
||||||
..
|
|
||||||
} = self;
|
|
||||||
|
|
||||||
fetch_objects_ids_to_unlink(walked_entries, object_ids_that_should_be_unlinked, db).await?;
|
|
||||||
|
|
||||||
check_interruption!(interrupter);
|
|
||||||
|
|
||||||
let (sync_stuff, paths_to_update) = walked_entries
|
|
||||||
.drain(..)
|
|
||||||
.map(|entry| {
|
|
||||||
let IsolatedFilePathDataParts { is_dir, .. } = &entry.iso_file_path.to_parts();
|
|
||||||
|
|
||||||
let pub_id = sd_utils::uuid_to_bytes(entry.pub_id);
|
|
||||||
|
|
||||||
let should_unlink_object = entry.maybe_object_id.map_or(false, |object_id| {
|
|
||||||
object_ids_that_should_be_unlinked.contains(&object_id)
|
|
||||||
});
|
|
||||||
|
|
||||||
let (sync_params, db_params) = chain_optional_iter(
|
|
||||||
[
|
|
||||||
((cas_id::NAME, msgpack!(nil)), cas_id::set(None)),
|
|
||||||
sync_db_entry!(*is_dir, is_dir),
|
|
||||||
sync_db_entry!(
|
|
||||||
entry.metadata.size_in_bytes.to_be_bytes().to_vec(),
|
|
||||||
size_in_bytes_bytes
|
|
||||||
),
|
|
||||||
sync_db_entry!(inode_to_db(entry.metadata.inode), inode),
|
|
||||||
{
|
|
||||||
let v = entry.metadata.created_at.into();
|
|
||||||
sync_db_entry!(v, date_created)
|
|
||||||
},
|
|
||||||
{
|
|
||||||
let v = entry.metadata.modified_at.into();
|
|
||||||
sync_db_entry!(v, date_modified)
|
|
||||||
},
|
|
||||||
sync_db_entry!(entry.metadata.hidden, hidden),
|
|
||||||
],
|
|
||||||
[
|
|
||||||
// As this file was updated while Spacedrive was offline, we mark the object_id and cas_id as null
|
|
||||||
// So this file_path will be updated at file identifier job
|
|
||||||
should_unlink_object
|
|
||||||
.then_some(((object_id::NAME, msgpack!(nil)), object::disconnect())),
|
|
||||||
],
|
|
||||||
)
|
|
||||||
.into_iter()
|
|
||||||
.unzip::<_, _, Vec<_>, Vec<_>>();
|
|
||||||
|
|
||||||
(
|
|
||||||
sync_params
|
|
||||||
.into_iter()
|
|
||||||
.map(|(field, value)| {
|
|
||||||
sync.shared_update(
|
|
||||||
prisma_sync::file_path::SyncId {
|
|
||||||
pub_id: pub_id.clone(),
|
|
||||||
},
|
|
||||||
field,
|
|
||||||
value,
|
|
||||||
)
|
|
||||||
})
|
|
||||||
.collect::<Vec<_>>(),
|
|
||||||
db.file_path()
|
|
||||||
.update(file_path::pub_id::equals(pub_id), db_params)
|
|
||||||
.select(file_path::select!({ id })),
|
|
||||||
)
|
|
||||||
})
|
|
||||||
.unzip::<_, _, Vec<_>, Vec<_>>();
|
|
||||||
|
|
||||||
let updated = sync
|
|
||||||
.write_ops(
|
|
||||||
db,
|
|
||||||
(sync_stuff.into_iter().flatten().collect(), paths_to_update),
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.map_err(indexer::Error::from)?;
|
|
||||||
|
|
||||||
trace!("Updated {updated:?} records");
|
|
||||||
|
|
||||||
Ok(ExecStatus::Done(
|
|
||||||
UpdateTaskOutput {
|
|
||||||
updated_count: updated.len() as u64,
|
|
||||||
update_duration: start_time.elapsed(),
|
|
||||||
}
|
|
||||||
.into_output(),
|
|
||||||
))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn fetch_objects_ids_to_unlink(
|
async fn fetch_objects_ids_to_unlink(
|
||||||
walked_entries: &[WalkedEntry],
|
walked_entries: &[WalkedEntry],
|
||||||
object_ids_that_should_be_unlinked: &mut HashSet<object::id::Type>,
|
object_ids_that_should_be_unlinked: &mut HashSet<object::id::Type>,
|
||||||
|
@ -269,3 +246,59 @@ async fn fetch_objects_ids_to_unlink(
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize, Deserialize)]
|
||||||
|
struct SaveState {
|
||||||
|
id: TaskId,
|
||||||
|
is_shallow: bool,
|
||||||
|
|
||||||
|
walked_entries: Vec<WalkedEntry>,
|
||||||
|
|
||||||
|
object_ids_that_should_be_unlinked: HashSet<object::id::Type>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl SerializableTask<Error> for Updater {
|
||||||
|
type SerializeError = rmp_serde::encode::Error;
|
||||||
|
|
||||||
|
type DeserializeError = rmp_serde::decode::Error;
|
||||||
|
|
||||||
|
type DeserializeCtx = (Arc<PrismaClient>, Arc<SyncManager>);
|
||||||
|
|
||||||
|
async fn serialize(self) -> Result<Vec<u8>, Self::SerializeError> {
|
||||||
|
let Self {
|
||||||
|
id,
|
||||||
|
walked_entries,
|
||||||
|
object_ids_that_should_be_unlinked,
|
||||||
|
is_shallow,
|
||||||
|
..
|
||||||
|
} = self;
|
||||||
|
|
||||||
|
rmp_serde::to_vec_named(&SaveState {
|
||||||
|
id,
|
||||||
|
is_shallow,
|
||||||
|
walked_entries,
|
||||||
|
object_ids_that_should_be_unlinked,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn deserialize(
|
||||||
|
data: &[u8],
|
||||||
|
(db, sync): Self::DeserializeCtx,
|
||||||
|
) -> Result<Self, Self::DeserializeError> {
|
||||||
|
rmp_serde::from_slice(data).map(
|
||||||
|
|SaveState {
|
||||||
|
id,
|
||||||
|
is_shallow,
|
||||||
|
walked_entries,
|
||||||
|
object_ids_that_should_be_unlinked,
|
||||||
|
}| Self {
|
||||||
|
id,
|
||||||
|
is_shallow,
|
||||||
|
walked_entries,
|
||||||
|
object_ids_that_should_be_unlinked,
|
||||||
|
db,
|
||||||
|
sync,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
File diff suppressed because it is too large
Load diff
93
core/crates/heavy-lifting/src/indexer/tasks/walker/entry.rs
Normal file
93
core/crates/heavy-lifting/src/indexer/tasks/walker/entry.rs
Normal file
|
@ -0,0 +1,93 @@
|
||||||
|
use sd_core_file_path_helper::{FilePathMetadata, IsolatedFilePathData};
|
||||||
|
|
||||||
|
use sd_core_prisma_helpers::FilePathPubId;
|
||||||
|
use sd_prisma::prisma::file_path;
|
||||||
|
|
||||||
|
use std::{
|
||||||
|
hash::{Hash, Hasher},
|
||||||
|
path::{Path, PathBuf},
|
||||||
|
};
|
||||||
|
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
|
/// `WalkedEntry` represents a single path in the filesystem
|
||||||
|
#[derive(Debug, Serialize, Deserialize)]
|
||||||
|
pub struct WalkedEntry {
|
||||||
|
pub pub_id: FilePathPubId,
|
||||||
|
pub maybe_object_id: file_path::object_id::Type,
|
||||||
|
pub iso_file_path: IsolatedFilePathData<'static>,
|
||||||
|
pub metadata: FilePathMetadata,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl PartialEq for WalkedEntry {
|
||||||
|
fn eq(&self, other: &Self) -> bool {
|
||||||
|
self.iso_file_path == other.iso_file_path
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Eq for WalkedEntry {}
|
||||||
|
|
||||||
|
impl Hash for WalkedEntry {
|
||||||
|
fn hash<H: Hasher>(&self, state: &mut H) {
|
||||||
|
self.iso_file_path.hash(state);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize, Deserialize)]
|
||||||
|
pub(super) struct WalkingEntry {
|
||||||
|
pub(super) iso_file_path: IsolatedFilePathData<'static>,
|
||||||
|
pub(super) metadata: FilePathMetadata,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<WalkingEntry> for WalkedEntry {
|
||||||
|
fn from(
|
||||||
|
WalkingEntry {
|
||||||
|
iso_file_path,
|
||||||
|
metadata,
|
||||||
|
}: WalkingEntry,
|
||||||
|
) -> Self {
|
||||||
|
Self {
|
||||||
|
pub_id: FilePathPubId::new(),
|
||||||
|
maybe_object_id: None,
|
||||||
|
iso_file_path,
|
||||||
|
metadata,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<PubId: Into<FilePathPubId>> From<(PubId, file_path::object_id::Type, WalkingEntry)>
|
||||||
|
for WalkedEntry
|
||||||
|
{
|
||||||
|
fn from(
|
||||||
|
(
|
||||||
|
pub_id,
|
||||||
|
maybe_object_id,
|
||||||
|
WalkingEntry {
|
||||||
|
iso_file_path,
|
||||||
|
metadata,
|
||||||
|
},
|
||||||
|
): (PubId, file_path::object_id::Type, WalkingEntry),
|
||||||
|
) -> Self {
|
||||||
|
Self {
|
||||||
|
pub_id: pub_id.into(),
|
||||||
|
maybe_object_id,
|
||||||
|
iso_file_path,
|
||||||
|
metadata,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize, Deserialize)]
|
||||||
|
pub struct ToWalkEntry {
|
||||||
|
pub(super) path: PathBuf,
|
||||||
|
pub(super) parent_dir_accepted_by_its_children: Option<bool>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<P: AsRef<Path>> From<P> for ToWalkEntry {
|
||||||
|
fn from(path: P) -> Self {
|
||||||
|
Self {
|
||||||
|
path: path.as_ref().into(),
|
||||||
|
parent_dir_accepted_by_its_children: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,64 @@
|
||||||
|
use crate::indexer;
|
||||||
|
|
||||||
|
use sd_core_file_path_helper::FilePathMetadata;
|
||||||
|
use sd_core_indexer_rules::MetadataForIndexerRules;
|
||||||
|
|
||||||
|
use std::{fs::Metadata, path::Path};
|
||||||
|
|
||||||
|
use chrono::{DateTime, Utc};
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize, Deserialize)]
|
||||||
|
pub(super) struct InnerMetadata {
|
||||||
|
pub is_dir: bool,
|
||||||
|
pub is_symlink: bool,
|
||||||
|
pub inode: u64,
|
||||||
|
pub size_in_bytes: u64,
|
||||||
|
pub hidden: bool,
|
||||||
|
pub created_at: DateTime<Utc>,
|
||||||
|
pub modified_at: DateTime<Utc>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl InnerMetadata {
|
||||||
|
pub fn new(
|
||||||
|
path: impl AsRef<Path>,
|
||||||
|
metadata: &Metadata,
|
||||||
|
) -> Result<Self, indexer::NonCriticalIndexerError> {
|
||||||
|
let FilePathMetadata {
|
||||||
|
inode,
|
||||||
|
size_in_bytes,
|
||||||
|
created_at,
|
||||||
|
modified_at,
|
||||||
|
hidden,
|
||||||
|
} = FilePathMetadata::from_path(path, metadata)
|
||||||
|
.map_err(|e| indexer::NonCriticalIndexerError::FilePathMetadata(e.to_string()))?;
|
||||||
|
|
||||||
|
Ok(Self {
|
||||||
|
is_dir: metadata.is_dir(),
|
||||||
|
is_symlink: metadata.is_symlink(),
|
||||||
|
inode,
|
||||||
|
size_in_bytes,
|
||||||
|
hidden,
|
||||||
|
created_at,
|
||||||
|
modified_at,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl MetadataForIndexerRules for InnerMetadata {
|
||||||
|
fn is_dir(&self) -> bool {
|
||||||
|
self.is_dir
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<InnerMetadata> for FilePathMetadata {
|
||||||
|
fn from(metadata: InnerMetadata) -> Self {
|
||||||
|
Self {
|
||||||
|
inode: metadata.inode,
|
||||||
|
size_in_bytes: metadata.size_in_bytes,
|
||||||
|
hidden: metadata.hidden,
|
||||||
|
created_at: metadata.created_at,
|
||||||
|
modified_at: metadata.modified_at,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
1176
core/crates/heavy-lifting/src/indexer/tasks/walker/mod.rs
Normal file
1176
core/crates/heavy-lifting/src/indexer/tasks/walker/mod.rs
Normal file
File diff suppressed because it is too large
Load diff
261
core/crates/heavy-lifting/src/indexer/tasks/walker/rules.rs
Normal file
261
core/crates/heavy-lifting/src/indexer/tasks/walker/rules.rs
Normal file
|
@ -0,0 +1,261 @@
|
||||||
|
use crate::{indexer, NonCriticalError};
|
||||||
|
|
||||||
|
use sd_core_file_path_helper::{FilePathMetadata, IsolatedFilePathData};
|
||||||
|
use sd_core_indexer_rules::{IndexerRuler, MetadataForIndexerRules, RuleKind};
|
||||||
|
|
||||||
|
use sd_utils::error::FileIOError;
|
||||||
|
|
||||||
|
use std::{
|
||||||
|
collections::{hash_map::Entry, HashMap, HashSet},
|
||||||
|
path::{Path, PathBuf},
|
||||||
|
sync::Arc,
|
||||||
|
};
|
||||||
|
|
||||||
|
use futures_concurrency::future::Join;
|
||||||
|
use tokio::fs;
|
||||||
|
use tracing::{instrument, trace};
|
||||||
|
|
||||||
|
use super::{
|
||||||
|
entry::{ToWalkEntry, WalkingEntry},
|
||||||
|
InnerMetadata, IsoFilePathFactory, WalkedEntry,
|
||||||
|
};
|
||||||
|
|
||||||
|
pub(super) async fn apply_indexer_rules(
|
||||||
|
paths_and_metadatas: &mut HashMap<PathBuf, InnerMetadata>,
|
||||||
|
indexer_ruler: &IndexerRuler,
|
||||||
|
errors: &mut Vec<NonCriticalError>,
|
||||||
|
) -> HashMap<PathBuf, (InnerMetadata, HashMap<RuleKind, Vec<bool>>)> {
|
||||||
|
paths_and_metadatas
|
||||||
|
.drain()
|
||||||
|
// TODO: Hard ignoring symlinks for now, but this should be configurable
|
||||||
|
.filter(|(_, metadata)| !metadata.is_symlink)
|
||||||
|
.map(|(current_path, metadata)| async {
|
||||||
|
indexer_ruler
|
||||||
|
.apply_all(¤t_path, &metadata)
|
||||||
|
.await
|
||||||
|
.map(|acceptance_per_rule_kind| {
|
||||||
|
(current_path, (metadata, acceptance_per_rule_kind))
|
||||||
|
})
|
||||||
|
.map_err(|e| indexer::NonCriticalIndexerError::IndexerRule(e.to_string()))
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join()
|
||||||
|
.await
|
||||||
|
.into_iter()
|
||||||
|
.filter_map(|res| res.map_err(|e| errors.push(e.into())).ok())
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) async fn process_rules_results(
|
||||||
|
root: &Arc<PathBuf>,
|
||||||
|
iso_file_path_factory: &impl IsoFilePathFactory,
|
||||||
|
parent_dir_accepted_by_its_children: Option<bool>,
|
||||||
|
paths_metadatas_and_acceptance: &mut HashMap<
|
||||||
|
PathBuf,
|
||||||
|
(InnerMetadata, HashMap<RuleKind, Vec<bool>>),
|
||||||
|
>,
|
||||||
|
maybe_to_keep_walking: &mut Option<Vec<ToWalkEntry>>,
|
||||||
|
collect_rejected_paths: bool,
|
||||||
|
errors: &mut Vec<NonCriticalError>,
|
||||||
|
) -> (
|
||||||
|
HashMap<PathBuf, InnerMetadata>,
|
||||||
|
HashSet<WalkedEntry>,
|
||||||
|
Vec<PathBuf>,
|
||||||
|
) {
|
||||||
|
let (accepted, accepted_ancestors, rejected) = segregate_paths(
|
||||||
|
root,
|
||||||
|
iso_file_path_factory,
|
||||||
|
paths_metadatas_and_acceptance.drain(),
|
||||||
|
parent_dir_accepted_by_its_children,
|
||||||
|
maybe_to_keep_walking,
|
||||||
|
collect_rejected_paths,
|
||||||
|
errors,
|
||||||
|
);
|
||||||
|
|
||||||
|
(
|
||||||
|
accepted,
|
||||||
|
accepted_ancestors
|
||||||
|
.into_iter()
|
||||||
|
.map(|(ancestor_iso_file_path, ancestor_path)| async move {
|
||||||
|
fs::metadata(&ancestor_path)
|
||||||
|
.await
|
||||||
|
.map_err(|e| {
|
||||||
|
indexer::NonCriticalIndexerError::Metadata(
|
||||||
|
FileIOError::from((&ancestor_path, e)).to_string(),
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.and_then(|metadata| {
|
||||||
|
FilePathMetadata::from_path(&ancestor_path, &metadata)
|
||||||
|
.map(|metadata| {
|
||||||
|
WalkingEntry {
|
||||||
|
iso_file_path: ancestor_iso_file_path,
|
||||||
|
metadata,
|
||||||
|
}
|
||||||
|
.into()
|
||||||
|
})
|
||||||
|
.map_err(|e| {
|
||||||
|
indexer::NonCriticalIndexerError::FilePathMetadata(e.to_string())
|
||||||
|
})
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join()
|
||||||
|
.await
|
||||||
|
.into_iter()
|
||||||
|
.filter_map(|res| res.map_err(|e| errors.push(e.into())).ok())
|
||||||
|
.collect(),
|
||||||
|
rejected,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn segregate_paths(
|
||||||
|
root: &Arc<PathBuf>,
|
||||||
|
iso_file_path_factory: &impl IsoFilePathFactory,
|
||||||
|
paths_metadatas_and_acceptance: impl IntoIterator<
|
||||||
|
Item = (PathBuf, (InnerMetadata, HashMap<RuleKind, Vec<bool>>)),
|
||||||
|
>,
|
||||||
|
parent_dir_accepted_by_its_children: Option<bool>,
|
||||||
|
maybe_to_keep_walking: &mut Option<Vec<ToWalkEntry>>,
|
||||||
|
collect_rejected_paths: bool,
|
||||||
|
errors: &mut Vec<NonCriticalError>,
|
||||||
|
) -> (
|
||||||
|
HashMap<PathBuf, InnerMetadata>,
|
||||||
|
HashMap<IsolatedFilePathData<'static>, PathBuf>,
|
||||||
|
Vec<PathBuf>,
|
||||||
|
) {
|
||||||
|
let root = root.as_ref();
|
||||||
|
|
||||||
|
let mut accepted = HashMap::new();
|
||||||
|
let mut accepted_ancestors = HashMap::new();
|
||||||
|
let mut rejected = Vec::new();
|
||||||
|
|
||||||
|
for (current_path, (metadata, acceptance_per_rule_kind)) in paths_metadatas_and_acceptance {
|
||||||
|
// Accept by children has three states,
|
||||||
|
// None if we don't now yet or if this check doesn't apply
|
||||||
|
// Some(true) if this check applies and it passes
|
||||||
|
// Some(false) if this check applies and it was rejected
|
||||||
|
// and we pass the current parent state to its children
|
||||||
|
let mut accept_by_children_dir = parent_dir_accepted_by_its_children;
|
||||||
|
|
||||||
|
if !reject_path(
|
||||||
|
¤t_path,
|
||||||
|
&metadata,
|
||||||
|
&acceptance_per_rule_kind,
|
||||||
|
&mut accept_by_children_dir,
|
||||||
|
maybe_to_keep_walking,
|
||||||
|
) && accept_by_children_dir.unwrap_or(true)
|
||||||
|
{
|
||||||
|
accept_path_and_ancestors(
|
||||||
|
current_path,
|
||||||
|
metadata,
|
||||||
|
root,
|
||||||
|
&mut accepted,
|
||||||
|
iso_file_path_factory,
|
||||||
|
&mut accepted_ancestors,
|
||||||
|
errors,
|
||||||
|
);
|
||||||
|
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if collect_rejected_paths {
|
||||||
|
rejected.push(current_path);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
(accepted, accepted_ancestors, rejected)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[instrument(skip_all, fields(current_path = %current_path.display()))]
|
||||||
|
fn reject_path(
|
||||||
|
current_path: &Path,
|
||||||
|
metadata: &InnerMetadata,
|
||||||
|
acceptance_per_rule_kind: &HashMap<RuleKind, Vec<bool>>,
|
||||||
|
accept_by_children_dir: &mut Option<bool>,
|
||||||
|
maybe_to_keep_walking: &mut Option<Vec<ToWalkEntry>>,
|
||||||
|
) -> bool {
|
||||||
|
IndexerRuler::rejected_by_reject_glob(acceptance_per_rule_kind)
|
||||||
|
|| IndexerRuler::rejected_by_git_ignore(acceptance_per_rule_kind)
|
||||||
|
|| (metadata.is_dir()
|
||||||
|
&& process_and_maybe_reject_by_directory_rules(
|
||||||
|
current_path,
|
||||||
|
acceptance_per_rule_kind,
|
||||||
|
accept_by_children_dir,
|
||||||
|
maybe_to_keep_walking,
|
||||||
|
)) || IndexerRuler::rejected_by_accept_glob(acceptance_per_rule_kind)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn process_and_maybe_reject_by_directory_rules(
|
||||||
|
current_path: &Path,
|
||||||
|
acceptance_per_rule_kind: &HashMap<RuleKind, Vec<bool>>,
|
||||||
|
accept_by_children_dir: &mut Option<bool>,
|
||||||
|
maybe_to_keep_walking: &mut Option<Vec<ToWalkEntry>>,
|
||||||
|
) -> bool {
|
||||||
|
// If it is a directory, first we check if we must reject it and its children entirely
|
||||||
|
if IndexerRuler::rejected_by_children_directories(acceptance_per_rule_kind) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Then we check if we must accept it and its children
|
||||||
|
if let Some(accepted_by_children_rules) =
|
||||||
|
acceptance_per_rule_kind.get(&RuleKind::AcceptIfChildrenDirectoriesArePresent)
|
||||||
|
{
|
||||||
|
if accepted_by_children_rules.iter().any(|accept| *accept) {
|
||||||
|
*accept_by_children_dir = Some(true);
|
||||||
|
}
|
||||||
|
|
||||||
|
// If it wasn't accepted then we mark as rejected
|
||||||
|
if accept_by_children_dir.is_none() {
|
||||||
|
trace!(
|
||||||
|
"Rejected because it didn't passed in any \
|
||||||
|
`RuleKind::AcceptIfChildrenDirectoriesArePresent` rule",
|
||||||
|
);
|
||||||
|
*accept_by_children_dir = Some(false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Then we mark this directory to maybe be walked in too
|
||||||
|
if let Some(ref mut to_keep_walking) = maybe_to_keep_walking {
|
||||||
|
to_keep_walking.push(ToWalkEntry {
|
||||||
|
path: current_path.to_path_buf(),
|
||||||
|
parent_dir_accepted_by_its_children: *accept_by_children_dir,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
false
|
||||||
|
}
|
||||||
|
|
||||||
|
fn accept_path_and_ancestors(
|
||||||
|
current_path: PathBuf,
|
||||||
|
metadata: InnerMetadata,
|
||||||
|
root: &Path,
|
||||||
|
accepted: &mut HashMap<PathBuf, InnerMetadata>,
|
||||||
|
iso_file_path_factory: &impl IsoFilePathFactory,
|
||||||
|
accepted_ancestors: &mut HashMap<IsolatedFilePathData<'static>, PathBuf>,
|
||||||
|
errors: &mut Vec<NonCriticalError>,
|
||||||
|
) {
|
||||||
|
// If the ancestors directories wasn't indexed before, now we do
|
||||||
|
for ancestor in current_path
|
||||||
|
.ancestors()
|
||||||
|
.skip(1) // Skip the current directory as it was already indexed
|
||||||
|
.take_while(|&ancestor| ancestor != root)
|
||||||
|
{
|
||||||
|
if let Ok(iso_file_path) = iso_file_path_factory.build(ancestor, true).map_err(|e| {
|
||||||
|
errors.push(indexer::NonCriticalIndexerError::IsoFilePath(e.to_string()).into());
|
||||||
|
}) {
|
||||||
|
match accepted_ancestors.entry(iso_file_path) {
|
||||||
|
Entry::Occupied(_) => {
|
||||||
|
// If we already accepted this ancestor, then it will contain
|
||||||
|
// also all if its ancestors too, so we can stop here
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
Entry::Vacant(entry) => {
|
||||||
|
trace!(ancestor = %ancestor.display(), "Accepted ancestor");
|
||||||
|
entry.insert(ancestor.to_path_buf());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
accepted.insert(current_path, metadata);
|
||||||
|
}
|
219
core/crates/heavy-lifting/src/indexer/tasks/walker/save_state.rs
Normal file
219
core/crates/heavy-lifting/src/indexer/tasks/walker/save_state.rs
Normal file
|
@ -0,0 +1,219 @@
|
||||||
|
use crate::{Error, NonCriticalError};
|
||||||
|
|
||||||
|
use sd_core_file_path_helper::IsolatedFilePathData;
|
||||||
|
use sd_core_indexer_rules::{IndexerRuler, RuleKind};
|
||||||
|
use sd_core_prisma_helpers::file_path_pub_and_cas_ids;
|
||||||
|
|
||||||
|
use std::{
|
||||||
|
collections::{HashMap, HashSet},
|
||||||
|
path::PathBuf,
|
||||||
|
sync::Arc,
|
||||||
|
time::Duration,
|
||||||
|
};
|
||||||
|
|
||||||
|
use sd_task_system::{SerializableTask, TaskId};
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
|
use super::{
|
||||||
|
entry::{ToWalkEntry, WalkingEntry},
|
||||||
|
metadata::InnerMetadata,
|
||||||
|
IsoFilePathFactory, WalkedEntry, Walker, WalkerDBProxy, WalkerStage,
|
||||||
|
};
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize, Deserialize)]
|
||||||
|
pub(super) struct WalkDirSaveState {
|
||||||
|
id: TaskId,
|
||||||
|
is_shallow: bool,
|
||||||
|
|
||||||
|
entry: ToWalkEntry,
|
||||||
|
root: Arc<PathBuf>,
|
||||||
|
entry_iso_file_path: IsolatedFilePathData<'static>,
|
||||||
|
|
||||||
|
stage: WalkerStageSaveState,
|
||||||
|
|
||||||
|
errors: Vec<NonCriticalError>,
|
||||||
|
scan_time: Duration,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize, Deserialize)]
|
||||||
|
pub(super) enum WalkerStageSaveState {
|
||||||
|
Start,
|
||||||
|
CollectingMetadata {
|
||||||
|
found_paths: Vec<PathBuf>,
|
||||||
|
},
|
||||||
|
CheckingIndexerRules {
|
||||||
|
paths_and_metadatas: HashMap<PathBuf, InnerMetadata>,
|
||||||
|
},
|
||||||
|
ProcessingRulesResults {
|
||||||
|
paths_metadatas_and_acceptance:
|
||||||
|
HashMap<PathBuf, (InnerMetadata, HashMap<RuleKind, Vec<bool>>)>,
|
||||||
|
},
|
||||||
|
GatheringFilePathsToRemove {
|
||||||
|
accepted_paths: HashMap<PathBuf, InnerMetadata>,
|
||||||
|
maybe_to_keep_walking: Option<Vec<ToWalkEntry>>,
|
||||||
|
accepted_ancestors: HashSet<WalkedEntry>,
|
||||||
|
non_indexed_paths: Vec<PathBuf>,
|
||||||
|
},
|
||||||
|
Finalize {
|
||||||
|
walking_entries: Vec<WalkingEntry>,
|
||||||
|
accepted_ancestors: HashSet<WalkedEntry>,
|
||||||
|
to_remove_entries: Vec<file_path_pub_and_cas_ids::Data>,
|
||||||
|
maybe_to_keep_walking: Option<Vec<ToWalkEntry>>,
|
||||||
|
non_indexed_paths: Vec<PathBuf>,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<WalkerStage> for WalkerStageSaveState {
|
||||||
|
fn from(stage: WalkerStage) -> Self {
|
||||||
|
match stage {
|
||||||
|
// We can't store the current state of `ReadDirStream` so we start again from the beginning
|
||||||
|
WalkerStage::Start | WalkerStage::Walking { .. } => Self::Start,
|
||||||
|
WalkerStage::CollectingMetadata { found_paths } => {
|
||||||
|
Self::CollectingMetadata { found_paths }
|
||||||
|
}
|
||||||
|
WalkerStage::CheckingIndexerRules {
|
||||||
|
paths_and_metadatas,
|
||||||
|
} => Self::CheckingIndexerRules {
|
||||||
|
paths_and_metadatas,
|
||||||
|
},
|
||||||
|
WalkerStage::ProcessingRulesResults {
|
||||||
|
paths_metadatas_and_acceptance,
|
||||||
|
} => Self::ProcessingRulesResults {
|
||||||
|
paths_metadatas_and_acceptance,
|
||||||
|
},
|
||||||
|
WalkerStage::GatheringFilePathsToRemove {
|
||||||
|
accepted_paths,
|
||||||
|
maybe_to_keep_walking,
|
||||||
|
accepted_ancestors,
|
||||||
|
non_indexed_paths,
|
||||||
|
} => Self::GatheringFilePathsToRemove {
|
||||||
|
accepted_paths,
|
||||||
|
maybe_to_keep_walking,
|
||||||
|
accepted_ancestors,
|
||||||
|
non_indexed_paths,
|
||||||
|
},
|
||||||
|
WalkerStage::Finalize {
|
||||||
|
walking_entries,
|
||||||
|
accepted_ancestors,
|
||||||
|
to_remove_entries,
|
||||||
|
maybe_to_keep_walking,
|
||||||
|
non_indexed_paths,
|
||||||
|
} => Self::Finalize {
|
||||||
|
walking_entries,
|
||||||
|
accepted_ancestors,
|
||||||
|
to_remove_entries,
|
||||||
|
maybe_to_keep_walking,
|
||||||
|
non_indexed_paths,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<WalkerStageSaveState> for WalkerStage {
|
||||||
|
fn from(value: WalkerStageSaveState) -> Self {
|
||||||
|
match value {
|
||||||
|
WalkerStageSaveState::Start => Self::Start,
|
||||||
|
WalkerStageSaveState::CollectingMetadata { found_paths } => {
|
||||||
|
Self::CollectingMetadata { found_paths }
|
||||||
|
}
|
||||||
|
WalkerStageSaveState::CheckingIndexerRules {
|
||||||
|
paths_and_metadatas,
|
||||||
|
} => Self::CheckingIndexerRules {
|
||||||
|
paths_and_metadatas,
|
||||||
|
},
|
||||||
|
WalkerStageSaveState::ProcessingRulesResults {
|
||||||
|
paths_metadatas_and_acceptance,
|
||||||
|
} => Self::ProcessingRulesResults {
|
||||||
|
paths_metadatas_and_acceptance,
|
||||||
|
},
|
||||||
|
WalkerStageSaveState::GatheringFilePathsToRemove {
|
||||||
|
accepted_paths,
|
||||||
|
maybe_to_keep_walking,
|
||||||
|
accepted_ancestors,
|
||||||
|
non_indexed_paths,
|
||||||
|
} => Self::GatheringFilePathsToRemove {
|
||||||
|
accepted_paths,
|
||||||
|
maybe_to_keep_walking,
|
||||||
|
accepted_ancestors,
|
||||||
|
non_indexed_paths,
|
||||||
|
},
|
||||||
|
WalkerStageSaveState::Finalize {
|
||||||
|
walking_entries,
|
||||||
|
accepted_ancestors,
|
||||||
|
to_remove_entries,
|
||||||
|
maybe_to_keep_walking,
|
||||||
|
non_indexed_paths,
|
||||||
|
} => Self::Finalize {
|
||||||
|
walking_entries,
|
||||||
|
accepted_ancestors,
|
||||||
|
to_remove_entries,
|
||||||
|
maybe_to_keep_walking,
|
||||||
|
non_indexed_paths,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<DBProxy, IsoPathFactory> SerializableTask<Error> for Walker<DBProxy, IsoPathFactory>
|
||||||
|
where
|
||||||
|
DBProxy: WalkerDBProxy,
|
||||||
|
IsoPathFactory: IsoFilePathFactory,
|
||||||
|
{
|
||||||
|
type SerializeError = rmp_serde::encode::Error;
|
||||||
|
type DeserializeError = rmp_serde::decode::Error;
|
||||||
|
type DeserializeCtx = (IndexerRuler, DBProxy, IsoPathFactory);
|
||||||
|
|
||||||
|
async fn serialize(self) -> Result<Vec<u8>, Self::SerializeError> {
|
||||||
|
let Self {
|
||||||
|
id,
|
||||||
|
entry,
|
||||||
|
root,
|
||||||
|
entry_iso_file_path,
|
||||||
|
stage,
|
||||||
|
errors,
|
||||||
|
scan_time,
|
||||||
|
is_shallow,
|
||||||
|
..
|
||||||
|
} = self;
|
||||||
|
rmp_serde::to_vec_named(&WalkDirSaveState {
|
||||||
|
id,
|
||||||
|
is_shallow,
|
||||||
|
entry,
|
||||||
|
root,
|
||||||
|
entry_iso_file_path,
|
||||||
|
stage: stage.into(),
|
||||||
|
errors,
|
||||||
|
scan_time,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn deserialize(
|
||||||
|
data: &[u8],
|
||||||
|
(indexer_ruler, db_proxy, iso_file_path_factory): Self::DeserializeCtx,
|
||||||
|
) -> Result<Self, Self::DeserializeError> {
|
||||||
|
rmp_serde::from_slice(data).map(
|
||||||
|
|WalkDirSaveState {
|
||||||
|
id,
|
||||||
|
entry,
|
||||||
|
root,
|
||||||
|
entry_iso_file_path,
|
||||||
|
stage,
|
||||||
|
errors,
|
||||||
|
scan_time,
|
||||||
|
is_shallow,
|
||||||
|
}| Self {
|
||||||
|
id,
|
||||||
|
entry,
|
||||||
|
root,
|
||||||
|
entry_iso_file_path,
|
||||||
|
indexer_ruler,
|
||||||
|
iso_file_path_factory,
|
||||||
|
db_proxy,
|
||||||
|
stage: stage.into(),
|
||||||
|
errors,
|
||||||
|
scan_time,
|
||||||
|
is_shallow,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
|
@ -1,5 +1,4 @@
|
||||||
use crate::Error;
|
use sd_task_system::{DispatcherShutdownError, Task};
|
||||||
|
|
||||||
use sd_utils::error::FileIOError;
|
use sd_utils::error::FileIOError;
|
||||||
|
|
||||||
use prisma_client_rust::QueryError;
|
use prisma_client_rust::QueryError;
|
||||||
|
@ -17,9 +16,6 @@ pub enum JobSystemError {
|
||||||
already_running_id: JobId,
|
already_running_id: JobId,
|
||||||
},
|
},
|
||||||
|
|
||||||
#[error("job canceled: <id='{0}'>")]
|
|
||||||
Canceled(JobId),
|
|
||||||
|
|
||||||
#[error("failed to load job reports from database to resume jobs: {0}")]
|
#[error("failed to load job reports from database to resume jobs: {0}")]
|
||||||
LoadReportsForResume(#[from] QueryError),
|
LoadReportsForResume(#[from] QueryError),
|
||||||
|
|
||||||
|
@ -34,9 +30,6 @@ pub enum JobSystemError {
|
||||||
|
|
||||||
#[error(transparent)]
|
#[error(transparent)]
|
||||||
Report(#[from] ReportError),
|
Report(#[from] ReportError),
|
||||||
|
|
||||||
#[error(transparent)]
|
|
||||||
Processing(#[from] Error),
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl From<JobSystemError> for rspc::Error {
|
impl From<JobSystemError> for rspc::Error {
|
||||||
|
@ -45,17 +38,36 @@ impl From<JobSystemError> for rspc::Error {
|
||||||
JobSystemError::NotFound(_) => {
|
JobSystemError::NotFound(_) => {
|
||||||
Self::with_cause(rspc::ErrorCode::NotFound, e.to_string(), e)
|
Self::with_cause(rspc::ErrorCode::NotFound, e.to_string(), e)
|
||||||
}
|
}
|
||||||
|
|
||||||
JobSystemError::AlreadyRunning { .. } => {
|
JobSystemError::AlreadyRunning { .. } => {
|
||||||
Self::with_cause(rspc::ErrorCode::Conflict, e.to_string(), e)
|
Self::with_cause(rspc::ErrorCode::Conflict, e.to_string(), e)
|
||||||
}
|
}
|
||||||
|
|
||||||
JobSystemError::Canceled(_) => {
|
|
||||||
Self::with_cause(rspc::ErrorCode::ClientClosedRequest, e.to_string(), e)
|
|
||||||
}
|
|
||||||
JobSystemError::Processing(e) => e.into(),
|
|
||||||
JobSystemError::Report(e) => e.into(),
|
JobSystemError::Report(e) => e.into(),
|
||||||
|
|
||||||
_ => Self::with_cause(rspc::ErrorCode::InternalServerError, e.to_string(), e),
|
_ => Self::with_cause(rspc::ErrorCode::InternalServerError, e.to_string(), e),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(thiserror::Error, Debug)]
|
||||||
|
pub enum DispatcherError {
|
||||||
|
#[error("job canceled: <id='{0}'>")]
|
||||||
|
JobCanceled(JobId),
|
||||||
|
#[error("system entered on shutdown mode <task_count={}>", .0.len())]
|
||||||
|
Shutdown(Vec<Box<dyn Task<crate::Error>>>),
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, thiserror::Error)]
|
||||||
|
pub enum JobErrorOrDispatcherError<JobError: Into<crate::Error>> {
|
||||||
|
#[error(transparent)]
|
||||||
|
JobError(#[from] JobError),
|
||||||
|
#[error(transparent)]
|
||||||
|
Dispatcher(#[from] DispatcherError),
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<DispatcherShutdownError<crate::Error>> for DispatcherError {
|
||||||
|
fn from(DispatcherShutdownError(tasks): DispatcherShutdownError<crate::Error>) -> Self {
|
||||||
|
Self::Shutdown(tasks)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
File diff suppressed because it is too large
Load diff
|
@ -1,16 +1,22 @@
|
||||||
use crate::Error;
|
use crate::{Error, JobContext};
|
||||||
|
|
||||||
use sd_prisma::prisma::location;
|
use sd_prisma::prisma::location;
|
||||||
use sd_task_system::BaseTaskDispatcher;
|
use sd_task_system::BaseTaskDispatcher;
|
||||||
use sd_utils::error::FileIOError;
|
use sd_utils::error::FileIOError;
|
||||||
|
|
||||||
use std::{cell::RefCell, collections::hash_map::HashMap, path::Path, sync::Arc};
|
use std::{
|
||||||
|
cell::RefCell,
|
||||||
|
collections::hash_map::HashMap,
|
||||||
|
panic,
|
||||||
|
path::{Path, PathBuf},
|
||||||
|
sync::Arc,
|
||||||
|
};
|
||||||
|
|
||||||
use async_channel as chan;
|
use async_channel as chan;
|
||||||
use futures::Stream;
|
use futures::Stream;
|
||||||
use futures_concurrency::future::{Join, TryJoin};
|
use futures_concurrency::future::{Join, TryJoin};
|
||||||
use tokio::{fs, spawn, sync::oneshot, task::JoinHandle};
|
use tokio::{fs, spawn, sync::oneshot, task::JoinHandle};
|
||||||
use tracing::{error, info, trace, warn};
|
use tracing::{debug, error, info, instrument, trace, warn};
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
mod error;
|
mod error;
|
||||||
|
@ -20,8 +26,9 @@ mod runner;
|
||||||
mod store;
|
mod store;
|
||||||
pub mod utils;
|
pub mod utils;
|
||||||
|
|
||||||
use error::JobSystemError;
|
pub use error::{DispatcherError, JobErrorOrDispatcherError, JobSystemError};
|
||||||
use job::{IntoJob, Job, JobName, JobOutput, OuterContext};
|
use job::{IntoJob, Job, JobName, JobOutput, OuterContext};
|
||||||
|
use report::Report;
|
||||||
use runner::{run, JobSystemRunner, RunnerMessage};
|
use runner::{run, JobSystemRunner, RunnerMessage};
|
||||||
use store::{load_jobs, StoredJobEntry};
|
use store::{load_jobs, StoredJobEntry};
|
||||||
|
|
||||||
|
@ -36,22 +43,23 @@ pub enum Command {
|
||||||
Pause,
|
Pause,
|
||||||
Resume,
|
Resume,
|
||||||
Cancel,
|
Cancel,
|
||||||
|
Shutdown,
|
||||||
}
|
}
|
||||||
|
|
||||||
pub struct JobSystem<Ctx: OuterContext> {
|
pub struct JobSystem<OuterCtx: OuterContext, JobCtx: JobContext<OuterCtx>> {
|
||||||
msgs_tx: chan::Sender<RunnerMessage<Ctx>>,
|
msgs_tx: chan::Sender<RunnerMessage<OuterCtx, JobCtx>>,
|
||||||
job_outputs_rx: chan::Receiver<(JobId, Result<JobOutput, JobSystemError>)>,
|
job_outputs_rx: chan::Receiver<(JobId, Result<JobOutput, Error>)>,
|
||||||
|
store_jobs_file: Arc<PathBuf>,
|
||||||
runner_handle: RefCell<Option<JoinHandle<()>>>,
|
runner_handle: RefCell<Option<JoinHandle<()>>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl<Ctx: OuterContext> JobSystem<Ctx> {
|
impl<OuterCtx: OuterContext, JobCtx: JobContext<OuterCtx>> JobSystem<OuterCtx, JobCtx> {
|
||||||
pub async fn new(
|
pub fn new(
|
||||||
base_dispatcher: BaseTaskDispatcher<Error>,
|
base_dispatcher: BaseTaskDispatcher<Error>,
|
||||||
data_directory: impl AsRef<Path> + Send,
|
data_directory: impl AsRef<Path>,
|
||||||
previously_existing_contexts: &HashMap<Uuid, Ctx>,
|
) -> Self {
|
||||||
) -> Result<Self, JobSystemError> {
|
|
||||||
let (job_outputs_tx, job_outputs_rx) = chan::unbounded();
|
let (job_outputs_tx, job_outputs_rx) = chan::unbounded();
|
||||||
let (job_return_status_tx, job_return_status_rx) = chan::bounded(16);
|
let (job_done_tx, job_done_rx) = chan::bounded(16);
|
||||||
let (msgs_tx, msgs_rx) = chan::bounded(8);
|
let (msgs_tx, msgs_rx) = chan::bounded(8);
|
||||||
|
|
||||||
let store_jobs_file = Arc::new(data_directory.as_ref().join(PENDING_JOBS_FILE));
|
let store_jobs_file = Arc::new(data_directory.as_ref().join(PENDING_JOBS_FILE));
|
||||||
|
@ -63,8 +71,8 @@ impl<Ctx: OuterContext> JobSystem<Ctx> {
|
||||||
while let Err(e) = spawn({
|
while let Err(e) = spawn({
|
||||||
let store_jobs_file = Arc::clone(&store_jobs_file);
|
let store_jobs_file = Arc::clone(&store_jobs_file);
|
||||||
let base_dispatcher = base_dispatcher.clone();
|
let base_dispatcher = base_dispatcher.clone();
|
||||||
let job_return_status_tx = job_return_status_tx.clone();
|
let job_return_status_tx = job_done_tx.clone();
|
||||||
let job_return_status_rx = job_return_status_rx.clone();
|
let job_done_rx = job_done_rx.clone();
|
||||||
let job_outputs_tx = job_outputs_tx.clone();
|
let job_outputs_tx = job_outputs_tx.clone();
|
||||||
let msgs_rx = msgs_rx.clone();
|
let msgs_rx = msgs_rx.clone();
|
||||||
|
|
||||||
|
@ -77,7 +85,7 @@ impl<Ctx: OuterContext> JobSystem<Ctx> {
|
||||||
),
|
),
|
||||||
store_jobs_file.as_ref(),
|
store_jobs_file.as_ref(),
|
||||||
msgs_rx,
|
msgs_rx,
|
||||||
job_return_status_rx,
|
job_done_rx,
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
}
|
}
|
||||||
|
@ -85,7 +93,7 @@ impl<Ctx: OuterContext> JobSystem<Ctx> {
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
if e.is_panic() {
|
if e.is_panic() {
|
||||||
error!("Job system panicked: {e:#?}");
|
error!(?e, "Job system panicked;");
|
||||||
} else {
|
} else {
|
||||||
trace!("JobSystemRunner received shutdown signal and will exit...");
|
trace!("JobSystemRunner received shutdown signal and will exit...");
|
||||||
break;
|
break;
|
||||||
|
@ -97,22 +105,47 @@ impl<Ctx: OuterContext> JobSystem<Ctx> {
|
||||||
}
|
}
|
||||||
})));
|
})));
|
||||||
|
|
||||||
load_stored_job_entries(
|
Self {
|
||||||
store_jobs_file.as_ref(),
|
|
||||||
previously_existing_contexts,
|
|
||||||
&msgs_tx,
|
|
||||||
)
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
Ok(Self {
|
|
||||||
msgs_tx,
|
msgs_tx,
|
||||||
job_outputs_rx,
|
job_outputs_rx,
|
||||||
|
store_jobs_file,
|
||||||
runner_handle,
|
runner_handle,
|
||||||
})
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn init(
|
||||||
|
&self,
|
||||||
|
previously_existing_contexts: &HashMap<Uuid, OuterCtx>,
|
||||||
|
) -> Result<(), JobSystemError> {
|
||||||
|
load_stored_job_entries(
|
||||||
|
&*self.store_jobs_file,
|
||||||
|
previously_existing_contexts,
|
||||||
|
&self.msgs_tx,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Get a map of all active reports with their respective job ids
|
||||||
|
///
|
||||||
|
/// # Panics
|
||||||
|
///
|
||||||
|
/// Panics only happen if internal channels are unexpectedly closed
|
||||||
|
pub async fn get_active_reports(&self) -> HashMap<JobId, Report> {
|
||||||
|
let (ack_tx, ack_rx) = oneshot::channel();
|
||||||
|
self.msgs_tx
|
||||||
|
.send(RunnerMessage::GetActiveReports { ack_tx })
|
||||||
|
.await
|
||||||
|
.expect("runner msgs channel unexpectedly closed on get active reports request");
|
||||||
|
|
||||||
|
ack_rx
|
||||||
|
.await
|
||||||
|
.expect("ack channel closed before receiving get active reports response")
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Checks if *any* of the desired jobs is running for the desired location
|
/// Checks if *any* of the desired jobs is running for the desired location
|
||||||
|
///
|
||||||
/// # Panics
|
/// # Panics
|
||||||
|
///
|
||||||
/// Panics only happen if internal channels are unexpectedly closed
|
/// Panics only happen if internal channels are unexpectedly closed
|
||||||
pub async fn check_running_jobs(
|
pub async fn check_running_jobs(
|
||||||
&self,
|
&self,
|
||||||
|
@ -122,7 +155,7 @@ impl<Ctx: OuterContext> JobSystem<Ctx> {
|
||||||
let (ack_tx, ack_rx) = oneshot::channel();
|
let (ack_tx, ack_rx) = oneshot::channel();
|
||||||
|
|
||||||
self.msgs_tx
|
self.msgs_tx
|
||||||
.send(RunnerMessage::CheckIfJobAreRunning {
|
.send(RunnerMessage::CheckIfJobsAreRunning {
|
||||||
job_names,
|
job_names,
|
||||||
location_id,
|
location_id,
|
||||||
ack_tx,
|
ack_tx,
|
||||||
|
@ -136,7 +169,9 @@ impl<Ctx: OuterContext> JobSystem<Ctx> {
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Shutdown the job system
|
/// Shutdown the job system
|
||||||
|
///
|
||||||
/// # Panics
|
/// # Panics
|
||||||
|
///
|
||||||
/// Panics only happen if internal channels are unexpectedly closed
|
/// Panics only happen if internal channels are unexpectedly closed
|
||||||
pub async fn shutdown(&self) {
|
pub async fn shutdown(&self) {
|
||||||
if let Some(handle) = self
|
if let Some(handle) = self
|
||||||
|
@ -152,7 +187,7 @@ impl<Ctx: OuterContext> JobSystem<Ctx> {
|
||||||
|
|
||||||
if let Err(e) = handle.await {
|
if let Err(e) = handle.await {
|
||||||
if e.is_panic() {
|
if e.is_panic() {
|
||||||
error!("JobSystem panicked: {e:#?}");
|
error!(?e, "JobSystem panicked;");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
info!("JobSystem gracefully shutdown");
|
info!("JobSystem gracefully shutdown");
|
||||||
|
@ -162,13 +197,15 @@ impl<Ctx: OuterContext> JobSystem<Ctx> {
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Dispatch a new job to the system
|
/// Dispatch a new job to the system
|
||||||
|
///
|
||||||
/// # Panics
|
/// # Panics
|
||||||
|
///
|
||||||
/// Panics only happen if internal channels are unexpectedly closed
|
/// Panics only happen if internal channels are unexpectedly closed
|
||||||
pub async fn dispatch<J: Job + SerializableJob<Ctx>>(
|
pub async fn dispatch<J: Job + SerializableJob<OuterCtx>>(
|
||||||
&mut self,
|
&self,
|
||||||
job: impl IntoJob<J, Ctx> + Send,
|
job: impl IntoJob<J, OuterCtx, JobCtx> + Send,
|
||||||
location_id: location::id::Type,
|
location_id: location::id::Type,
|
||||||
ctx: Ctx,
|
ctx: OuterCtx,
|
||||||
) -> Result<JobId, JobSystemError> {
|
) -> Result<JobId, JobSystemError> {
|
||||||
let dyn_job = job.into_job();
|
let dyn_job = job.into_job();
|
||||||
let id = dyn_job.id();
|
let id = dyn_job.id();
|
||||||
|
@ -176,7 +213,7 @@ impl<Ctx: OuterContext> JobSystem<Ctx> {
|
||||||
let (ack_tx, ack_rx) = oneshot::channel();
|
let (ack_tx, ack_rx) = oneshot::channel();
|
||||||
self.msgs_tx
|
self.msgs_tx
|
||||||
.send(RunnerMessage::NewJob {
|
.send(RunnerMessage::NewJob {
|
||||||
id,
|
job_id: id,
|
||||||
location_id,
|
location_id,
|
||||||
dyn_job,
|
dyn_job,
|
||||||
ctx,
|
ctx,
|
||||||
|
@ -191,17 +228,35 @@ impl<Ctx: OuterContext> JobSystem<Ctx> {
|
||||||
.map(|()| id)
|
.map(|()| id)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn receive_job_outputs(
|
/// Check if there are any active jobs for the desired [`OuterContext`]
|
||||||
&self,
|
///
|
||||||
) -> impl Stream<Item = (JobId, Result<JobOutput, JobSystemError>)> {
|
/// # Panics
|
||||||
|
///
|
||||||
|
/// Panics only happen if internal channels are unexpectedly closed
|
||||||
|
pub async fn has_active_jobs(&self, ctx: OuterCtx) -> bool {
|
||||||
|
let ctx_id = ctx.id();
|
||||||
|
|
||||||
|
let (ack_tx, ack_rx) = oneshot::channel();
|
||||||
|
self.msgs_tx
|
||||||
|
.send(RunnerMessage::HasActiveJobs { ctx_id, ack_tx })
|
||||||
|
.await
|
||||||
|
.expect("runner msgs channel unexpectedly closed on has active jobs request");
|
||||||
|
|
||||||
|
ack_rx
|
||||||
|
.await
|
||||||
|
.expect("ack channel closed before receiving has active jobs response")
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn receive_job_outputs(&self) -> impl Stream<Item = (JobId, Result<JobOutput, Error>)> {
|
||||||
self.job_outputs_rx.clone()
|
self.job_outputs_rx.clone()
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn send_command(&self, id: JobId, command: Command) -> Result<(), JobSystemError> {
|
#[instrument(skip(self), err)]
|
||||||
|
async fn send_command(&self, job_id: JobId, command: Command) -> Result<(), JobSystemError> {
|
||||||
let (ack_tx, ack_rx) = oneshot::channel();
|
let (ack_tx, ack_rx) = oneshot::channel();
|
||||||
self.msgs_tx
|
self.msgs_tx
|
||||||
.send(RunnerMessage::Command {
|
.send(RunnerMessage::Command {
|
||||||
id,
|
job_id,
|
||||||
command,
|
command,
|
||||||
ack_tx,
|
ack_tx,
|
||||||
})
|
})
|
||||||
|
@ -215,38 +270,48 @@ impl<Ctx: OuterContext> JobSystem<Ctx> {
|
||||||
.unwrap_or_else(|_| panic!("ack channel closed before receiving {command:?} response"))
|
.unwrap_or_else(|_| panic!("ack channel closed before receiving {command:?} response"))
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn pause(&self, id: JobId) -> Result<(), JobSystemError> {
|
pub async fn pause(&self, job_id: JobId) -> Result<(), JobSystemError> {
|
||||||
self.send_command(id, Command::Pause).await
|
self.send_command(job_id, Command::Pause).await
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn resume(&self, id: JobId) -> Result<(), JobSystemError> {
|
pub async fn resume(&self, job_id: JobId) -> Result<(), JobSystemError> {
|
||||||
self.send_command(id, Command::Resume).await
|
self.send_command(job_id, Command::Resume).await
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn cancel(&self, id: JobId) -> Result<(), JobSystemError> {
|
pub async fn cancel(&self, job_id: JobId) -> Result<(), JobSystemError> {
|
||||||
self.send_command(id, Command::Cancel).await
|
self.send_command(job_id, Command::Cancel).await
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// SAFETY: Due to usage of refcell we lost `Sync` impl, but we only use it to have a shutdown method
|
/// SAFETY: Due to usage of refcell we lost `Sync` impl, but we only use it to have a shutdown method
|
||||||
/// receiving `&self` which is called once, and we also use `try_borrow_mut` so we never panic
|
/// receiving `&self` which is called once, and we also use `try_borrow_mut` so we never panic
|
||||||
unsafe impl<Ctx: OuterContext> Sync for JobSystem<Ctx> {}
|
unsafe impl<OuterCtx: OuterContext, JobCtx: JobContext<OuterCtx>> Sync
|
||||||
|
for JobSystem<OuterCtx, JobCtx>
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
async fn load_stored_job_entries<Ctx: OuterContext>(
|
async fn load_stored_job_entries<OuterCtx: OuterContext, JobCtx: JobContext<OuterCtx>>(
|
||||||
store_jobs_file: impl AsRef<Path> + Send,
|
store_jobs_file: impl AsRef<Path> + Send,
|
||||||
previously_existing_job_contexts: &HashMap<Uuid, Ctx>,
|
previously_existing_job_contexts: &HashMap<Uuid, OuterCtx>,
|
||||||
msgs_tx: &chan::Sender<RunnerMessage<Ctx>>,
|
msgs_tx: &chan::Sender<RunnerMessage<OuterCtx, JobCtx>>,
|
||||||
) -> Result<(), JobSystemError> {
|
) -> Result<(), JobSystemError> {
|
||||||
let store_jobs_file = store_jobs_file.as_ref();
|
let store_jobs_file = store_jobs_file.as_ref();
|
||||||
|
|
||||||
let stores_jobs_by_db = rmp_serde::from_slice::<HashMap<Uuid, Vec<StoredJobEntry>>>(
|
let stores_jobs_by_db = rmp_serde::from_slice::<HashMap<Uuid, Vec<StoredJobEntry>>>(
|
||||||
&fs::read(store_jobs_file).await.map_err(|e| {
|
&match fs::read(store_jobs_file).await {
|
||||||
JobSystemError::StoredJobs(FileIOError::from((
|
Ok(bytes) => bytes,
|
||||||
store_jobs_file,
|
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
|
||||||
e,
|
debug!("No pending jobs found on disk");
|
||||||
"Failed to load jobs from disk",
|
return Ok(());
|
||||||
)))
|
}
|
||||||
})?,
|
Err(e) => {
|
||||||
|
return Err(JobSystemError::StoredJobs(FileIOError::from((
|
||||||
|
store_jobs_file,
|
||||||
|
e,
|
||||||
|
"Failed to load jobs from disk",
|
||||||
|
))))
|
||||||
|
}
|
||||||
|
},
|
||||||
)?;
|
)?;
|
||||||
|
|
||||||
stores_jobs_by_db
|
stores_jobs_by_db
|
||||||
|
@ -254,7 +319,7 @@ async fn load_stored_job_entries<Ctx: OuterContext>(
|
||||||
.filter_map(|(ctx_id, entries)| {
|
.filter_map(|(ctx_id, entries)| {
|
||||||
previously_existing_job_contexts.get(&ctx_id).map_or_else(
|
previously_existing_job_contexts.get(&ctx_id).map_or_else(
|
||||||
|| {
|
|| {
|
||||||
warn!("Found stored jobs for a database that doesn't exist anymore: <ctx_id='{ctx_id}'>");
|
warn!(%ctx_id, "Found stored jobs for a database that doesn't exist anymore;");
|
||||||
None
|
None
|
||||||
},
|
},
|
||||||
|ctx| Some((entries, ctx.clone())),
|
|ctx| Some((entries, ctx.clone())),
|
||||||
|
@ -270,7 +335,7 @@ async fn load_stored_job_entries<Ctx: OuterContext>(
|
||||||
.await
|
.await
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.filter_map(|res| {
|
.filter_map(|res| {
|
||||||
res.map_err(|e| error!("Failed to load stored jobs: {e:#?}"))
|
res.map_err(|e| error!(?e, "Failed to load stored jobs;"))
|
||||||
.ok()
|
.ok()
|
||||||
})
|
})
|
||||||
.flat_map(|(stored_jobs, ctx)| {
|
.flat_map(|(stored_jobs, ctx)| {
|
||||||
|
@ -283,7 +348,7 @@ async fn load_stored_job_entries<Ctx: OuterContext>(
|
||||||
|
|
||||||
msgs_tx
|
msgs_tx
|
||||||
.send(RunnerMessage::ResumeStoredJob {
|
.send(RunnerMessage::ResumeStoredJob {
|
||||||
id: dyn_job.id(),
|
job_id: dyn_job.id(),
|
||||||
location_id,
|
location_id,
|
||||||
dyn_job,
|
dyn_job,
|
||||||
ctx,
|
ctx,
|
||||||
|
|
|
@ -1,14 +1,15 @@
|
||||||
use sd_prisma::prisma::{job, PrismaClient};
|
use crate::NonCriticalError;
|
||||||
|
|
||||||
|
use sd_prisma::prisma::{file_path, job, location, PrismaClient};
|
||||||
use sd_utils::db::{maybe_missing, MissingFieldError};
|
use sd_utils::db::{maybe_missing, MissingFieldError};
|
||||||
|
|
||||||
use std::{collections::HashMap, fmt, str::FromStr};
|
use std::{collections::HashMap, fmt, path::PathBuf, str::FromStr};
|
||||||
|
|
||||||
use chrono::{DateTime, Utc};
|
use chrono::{DateTime, Utc};
|
||||||
use prisma_client_rust::QueryError;
|
use prisma_client_rust::QueryError;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use specta::Type;
|
use specta::Type;
|
||||||
use strum::ParseError;
|
use strum::ParseError;
|
||||||
use tracing::error;
|
|
||||||
|
|
||||||
use super::{job::JobName, JobId};
|
use super::{job::JobName, JobId};
|
||||||
|
|
||||||
|
@ -22,10 +23,8 @@ pub enum ReportError {
|
||||||
InvalidJobStatusInt(i32),
|
InvalidJobStatusInt(i32),
|
||||||
#[error("job not found in database: <id='{0}'>")]
|
#[error("job not found in database: <id='{0}'>")]
|
||||||
MissingReport(JobId),
|
MissingReport(JobId),
|
||||||
#[error("serialization error: {0}")]
|
#[error("json error: {0}")]
|
||||||
Serialization(#[from] rmp_serde::encode::Error),
|
Json(#[from] serde_json::Error),
|
||||||
#[error("deserialization error: {0}")]
|
|
||||||
Deserialization(#[from] rmp_serde::decode::Error),
|
|
||||||
#[error(transparent)]
|
#[error(transparent)]
|
||||||
MissingField(#[from] MissingFieldError),
|
MissingField(#[from] MissingFieldError),
|
||||||
#[error("failed to parse job name from database: {0}")]
|
#[error("failed to parse job name from database: {0}")]
|
||||||
|
@ -44,10 +43,7 @@ impl From<ReportError> for rspc::Error {
|
||||||
ReportError::MissingReport(_) => {
|
ReportError::MissingReport(_) => {
|
||||||
Self::with_cause(rspc::ErrorCode::NotFound, e.to_string(), e)
|
Self::with_cause(rspc::ErrorCode::NotFound, e.to_string(), e)
|
||||||
}
|
}
|
||||||
ReportError::Serialization(_)
|
ReportError::Json(_) | ReportError::MissingField(_) | ReportError::JobNameParse(_) => {
|
||||||
| ReportError::Deserialization(_)
|
|
||||||
| ReportError::MissingField(_)
|
|
||||||
| ReportError::JobNameParse(_) => {
|
|
||||||
Self::with_cause(rspc::ErrorCode::InternalServerError, e.to_string(), e)
|
Self::with_cause(rspc::ErrorCode::InternalServerError, e.to_string(), e)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -55,21 +51,78 @@ impl From<ReportError> for rspc::Error {
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Serialize, Deserialize, Type, Clone)]
|
#[derive(Debug, Serialize, Deserialize, Type, Clone)]
|
||||||
|
#[serde(rename_all = "snake_case")]
|
||||||
|
#[serde(tag = "type", content = "metadata")]
|
||||||
pub enum ReportMetadata {
|
pub enum ReportMetadata {
|
||||||
Input(ReportInputMetadata),
|
Input(ReportInputMetadata),
|
||||||
Output(ReportOutputMetadata),
|
Output(ReportOutputMetadata),
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Serialize, Deserialize, Type, Clone)]
|
#[derive(Debug, Serialize, Deserialize, Type, Clone)]
|
||||||
|
#[serde(rename_all = "snake_case")]
|
||||||
|
#[serde(tag = "type", content = "data")]
|
||||||
pub enum ReportInputMetadata {
|
pub enum ReportInputMetadata {
|
||||||
Placeholder,
|
// TODO: Add more variants as needed
|
||||||
// TODO: Add more types
|
Location(location::Data),
|
||||||
|
SubPath(PathBuf),
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Serialize, Deserialize, Type, Clone)]
|
#[derive(Debug, Serialize, Deserialize, Type, Clone)]
|
||||||
|
#[serde(rename_all = "snake_case")]
|
||||||
|
#[serde(tag = "type", content = "data")]
|
||||||
pub enum ReportOutputMetadata {
|
pub enum ReportOutputMetadata {
|
||||||
Metrics(HashMap<String, serde_json::Value>),
|
Metrics(HashMap<String, serde_json::Value>),
|
||||||
// TODO: Add more types
|
Indexer {
|
||||||
|
total_paths: (u32, u32),
|
||||||
|
},
|
||||||
|
FileIdentifier {
|
||||||
|
total_orphan_paths: (u32, u32),
|
||||||
|
total_objects_created: (u32, u32),
|
||||||
|
total_objects_linked: (u32, u32),
|
||||||
|
},
|
||||||
|
MediaProcessor {
|
||||||
|
media_data_extracted: (u32, u32),
|
||||||
|
media_data_skipped: (u32, u32),
|
||||||
|
thumbnails_generated: (u32, u32),
|
||||||
|
thumbnails_skipped: (u32, u32),
|
||||||
|
},
|
||||||
|
Copier {
|
||||||
|
source_location_id: location::id::Type,
|
||||||
|
target_location_id: location::id::Type,
|
||||||
|
sources_file_path_ids: Vec<file_path::id::Type>,
|
||||||
|
target_location_relative_directory_path: PathBuf,
|
||||||
|
},
|
||||||
|
Mover {
|
||||||
|
source_location_id: location::id::Type,
|
||||||
|
target_location_id: location::id::Type,
|
||||||
|
sources_file_path_ids: Vec<file_path::id::Type>,
|
||||||
|
target_location_relative_directory_path: PathBuf,
|
||||||
|
},
|
||||||
|
Deleter {
|
||||||
|
location_id: location::id::Type,
|
||||||
|
file_path_ids: Vec<file_path::id::Type>,
|
||||||
|
},
|
||||||
|
Eraser {
|
||||||
|
location_id: location::id::Type,
|
||||||
|
file_path_ids: Vec<file_path::id::Type>,
|
||||||
|
passes: u32,
|
||||||
|
},
|
||||||
|
FileValidator {
|
||||||
|
location_id: location::id::Type,
|
||||||
|
sub_path: Option<PathBuf>,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<ReportInputMetadata> for ReportMetadata {
|
||||||
|
fn from(value: ReportInputMetadata) -> Self {
|
||||||
|
Self::Input(value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<ReportOutputMetadata> for ReportMetadata {
|
||||||
|
fn from(value: ReportOutputMetadata) -> Self {
|
||||||
|
Self::Output(value)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Serialize, Type, Clone)]
|
#[derive(Debug, Serialize, Type, Clone)]
|
||||||
|
@ -80,7 +133,7 @@ pub struct Report {
|
||||||
|
|
||||||
pub metadata: Vec<ReportMetadata>,
|
pub metadata: Vec<ReportMetadata>,
|
||||||
pub critical_error: Option<String>,
|
pub critical_error: Option<String>,
|
||||||
pub non_critical_errors: Vec<String>,
|
pub non_critical_errors: Vec<NonCriticalError>,
|
||||||
|
|
||||||
pub created_at: Option<DateTime<Utc>>,
|
pub created_at: Option<DateTime<Utc>>,
|
||||||
pub started_at: Option<DateTime<Utc>>,
|
pub started_at: Option<DateTime<Utc>>,
|
||||||
|
@ -111,46 +164,53 @@ impl fmt::Display for Report {
|
||||||
impl TryFrom<job::Data> for Report {
|
impl TryFrom<job::Data> for Report {
|
||||||
type Error = ReportError;
|
type Error = ReportError;
|
||||||
|
|
||||||
fn try_from(data: job::Data) -> Result<Self, Self::Error> {
|
fn try_from(
|
||||||
|
job::Data {
|
||||||
|
id,
|
||||||
|
name,
|
||||||
|
action,
|
||||||
|
status,
|
||||||
|
errors_text: _, // Deprecated
|
||||||
|
critical_error,
|
||||||
|
non_critical_errors,
|
||||||
|
data: _, // Deprecated
|
||||||
|
metadata,
|
||||||
|
parent_id,
|
||||||
|
task_count,
|
||||||
|
completed_task_count,
|
||||||
|
date_estimated_completion,
|
||||||
|
date_created,
|
||||||
|
date_started,
|
||||||
|
date_completed,
|
||||||
|
..
|
||||||
|
}: job::Data,
|
||||||
|
) -> Result<Self, Self::Error> {
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
id: JobId::from_slice(&data.id).expect("corrupted database"),
|
id: JobId::from_slice(&id).expect("corrupted database"),
|
||||||
name: JobName::from_str(&maybe_missing(data.name, "job.name")?)?,
|
name: JobName::from_str(&maybe_missing(name, "job.name")?)?,
|
||||||
action: data.action,
|
action,
|
||||||
|
metadata: if let Some(metadata) = metadata {
|
||||||
metadata: data
|
serde_json::from_slice(&metadata)?
|
||||||
.metadata
|
} else {
|
||||||
.map(|m| {
|
vec![]
|
||||||
rmp_serde::from_slice(&m).unwrap_or_else(|e| {
|
},
|
||||||
error!("Failed to deserialize job metadata: {e:#?}");
|
critical_error,
|
||||||
vec![]
|
non_critical_errors: if let Some(non_critical_errors) = non_critical_errors {
|
||||||
})
|
serde_json::from_slice(&non_critical_errors)?
|
||||||
})
|
} else {
|
||||||
.unwrap_or_default(),
|
vec![]
|
||||||
critical_error: data.critical_error,
|
},
|
||||||
non_critical_errors: data.non_critical_errors.map_or_else(
|
created_at: date_created.map(DateTime::into),
|
||||||
Default::default,
|
started_at: date_started.map(DateTime::into),
|
||||||
|non_critical_errors| {
|
completed_at: date_completed.map(DateTime::into),
|
||||||
serde_json::from_slice(&non_critical_errors).unwrap_or_else(|e| {
|
parent_id: parent_id.map(|id| JobId::from_slice(&id).expect("corrupted database")),
|
||||||
error!("Failed to deserialize job non-critical errors: {e:#?}");
|
status: Status::try_from(maybe_missing(status, "job.status")?)
|
||||||
vec![]
|
|
||||||
})
|
|
||||||
},
|
|
||||||
),
|
|
||||||
created_at: data.date_created.map(DateTime::into),
|
|
||||||
started_at: data.date_started.map(DateTime::into),
|
|
||||||
completed_at: data.date_completed.map(DateTime::into),
|
|
||||||
parent_id: data
|
|
||||||
.parent_id
|
|
||||||
.map(|id| JobId::from_slice(&id).expect("corrupted database")),
|
|
||||||
status: Status::try_from(maybe_missing(data.status, "job.status")?)
|
|
||||||
.expect("corrupted database"),
|
.expect("corrupted database"),
|
||||||
task_count: data.task_count.unwrap_or(0),
|
task_count: task_count.unwrap_or(0),
|
||||||
completed_task_count: data.completed_task_count.unwrap_or(0),
|
completed_task_count: completed_task_count.unwrap_or(0),
|
||||||
phase: String::new(),
|
phase: String::new(),
|
||||||
message: String::new(),
|
message: String::new(),
|
||||||
estimated_completion: data
|
estimated_completion: date_estimated_completion.map_or_else(Utc::now, DateTime::into),
|
||||||
.date_estimated_completion
|
|
||||||
.map_or_else(Utc::now, DateTime::into),
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -178,6 +238,10 @@ impl Report {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn push_metadata(&mut self, metadata: ReportOutputMetadata) {
|
||||||
|
self.metadata.push(metadata.into());
|
||||||
|
}
|
||||||
|
|
||||||
#[must_use]
|
#[must_use]
|
||||||
pub fn get_action_name_and_group_key(&self) -> (String, Option<String>) {
|
pub fn get_action_name_and_group_key(&self) -> (String, Option<String>) {
|
||||||
// actions are formatted like "added_location" or "added_location-1"
|
// actions are formatted like "added_location" or "added_location-1"
|
||||||
|
@ -197,9 +261,11 @@ impl Report {
|
||||||
(action_name, Some(group_key))
|
(action_name, Some(group_key))
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn create(&mut self, db: &PrismaClient) -> Result<(), ReportError> {
|
pub async fn create(
|
||||||
let now = Utc::now();
|
&mut self,
|
||||||
|
db: &PrismaClient,
|
||||||
|
created_at: DateTime<Utc>,
|
||||||
|
) -> Result<(), ReportError> {
|
||||||
db.job()
|
db.job()
|
||||||
.create(
|
.create(
|
||||||
self.id.as_bytes().to_vec(),
|
self.id.as_bytes().to_vec(),
|
||||||
|
@ -207,11 +273,11 @@ impl Report {
|
||||||
[
|
[
|
||||||
job::name::set(Some(self.name.to_string())),
|
job::name::set(Some(self.name.to_string())),
|
||||||
job::action::set(self.action.clone()),
|
job::action::set(self.action.clone()),
|
||||||
job::date_created::set(Some(now.into())),
|
job::date_created::set(Some(created_at.into())),
|
||||||
job::metadata::set(Some(rmp_serde::to_vec(&self.metadata)?)),
|
job::metadata::set(Some(serde_json::to_vec(&self.metadata)?)),
|
||||||
job::status::set(Some(self.status as i32)),
|
job::status::set(Some(self.status as i32)),
|
||||||
job::date_started::set(self.started_at.map(Into::into)),
|
job::date_started::set(self.started_at.map(Into::into)),
|
||||||
job::task_count::set(Some(1)),
|
job::task_count::set(Some(0)),
|
||||||
job::completed_task_count::set(Some(0)),
|
job::completed_task_count::set(Some(0)),
|
||||||
],
|
],
|
||||||
[self
|
[self
|
||||||
|
@ -224,7 +290,7 @@ impl Report {
|
||||||
.map_err(ReportError::Create)?;
|
.map_err(ReportError::Create)?;
|
||||||
|
|
||||||
// Only setting created_at after we successfully created the job in DB
|
// Only setting created_at after we successfully created the job in DB
|
||||||
self.created_at = Some(now);
|
self.created_at = Some(created_at);
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
@ -236,10 +302,10 @@ impl Report {
|
||||||
vec![
|
vec![
|
||||||
job::status::set(Some(self.status as i32)),
|
job::status::set(Some(self.status as i32)),
|
||||||
job::critical_error::set(self.critical_error.clone()),
|
job::critical_error::set(self.critical_error.clone()),
|
||||||
job::non_critical_errors::set(Some(rmp_serde::to_vec(
|
job::non_critical_errors::set(Some(serde_json::to_vec(
|
||||||
&self.non_critical_errors,
|
&self.non_critical_errors,
|
||||||
)?)),
|
)?)),
|
||||||
job::metadata::set(Some(rmp_serde::to_vec(&self.metadata)?)),
|
job::metadata::set(Some(serde_json::to_vec(&self.metadata)?)),
|
||||||
job::task_count::set(Some(self.task_count)),
|
job::task_count::set(Some(self.task_count)),
|
||||||
job::completed_task_count::set(Some(self.completed_task_count)),
|
job::completed_task_count::set(Some(self.completed_task_count)),
|
||||||
job::date_started::set(self.started_at.map(Into::into)),
|
job::date_started::set(self.started_at.map(Into::into)),
|
||||||
|
@ -347,7 +413,7 @@ impl ReportBuilder {
|
||||||
|
|
||||||
#[must_use]
|
#[must_use]
|
||||||
pub fn with_metadata(mut self, metadata: ReportInputMetadata) -> Self {
|
pub fn with_metadata(mut self, metadata: ReportInputMetadata) -> Self {
|
||||||
self.metadata.push(ReportMetadata::Input(metadata));
|
self.metadata.push(metadata.into());
|
||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
use crate::Error;
|
use crate::{Error, JobContext};
|
||||||
|
|
||||||
use sd_prisma::prisma::location;
|
use sd_prisma::prisma::location;
|
||||||
use sd_task_system::BaseTaskDispatcher;
|
use sd_task_system::BaseTaskDispatcher;
|
||||||
|
@ -15,19 +15,23 @@ use std::{
|
||||||
use async_channel as chan;
|
use async_channel as chan;
|
||||||
use chrono::Utc;
|
use chrono::Utc;
|
||||||
use futures::StreamExt;
|
use futures::StreamExt;
|
||||||
use futures_concurrency::{future::TryJoin, stream::Merge};
|
use futures_concurrency::{
|
||||||
|
future::{Join, TryJoin},
|
||||||
|
stream::Merge,
|
||||||
|
};
|
||||||
|
use serde_json::json;
|
||||||
use tokio::{
|
use tokio::{
|
||||||
fs,
|
fs,
|
||||||
sync::oneshot,
|
sync::oneshot,
|
||||||
time::{interval_at, Instant},
|
time::{interval_at, Instant},
|
||||||
};
|
};
|
||||||
use tokio_stream::wrappers::IntervalStream;
|
use tokio_stream::wrappers::IntervalStream;
|
||||||
use tracing::{debug, error, info, warn};
|
use tracing::{debug, error, info, instrument, trace, warn};
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
use super::{
|
use super::{
|
||||||
job::{DynJob, JobHandle, JobName, JobOutput, OuterContext, ReturnStatus},
|
job::{DynJob, JobHandle, JobName, JobOutput, OuterContext, ReturnStatus},
|
||||||
report,
|
report::{self, ReportOutputMetadata},
|
||||||
store::{StoredJob, StoredJobEntry},
|
store::{StoredJob, StoredJobEntry},
|
||||||
Command, JobId, JobSystemError, SerializedTasks,
|
Command, JobId, JobSystemError, SerializedTasks,
|
||||||
};
|
};
|
||||||
|
@ -35,61 +39,76 @@ use super::{
|
||||||
const JOBS_INITIAL_CAPACITY: usize = 32;
|
const JOBS_INITIAL_CAPACITY: usize = 32;
|
||||||
const FIVE_MINUTES: Duration = Duration::from_secs(5 * 60);
|
const FIVE_MINUTES: Duration = Duration::from_secs(5 * 60);
|
||||||
|
|
||||||
pub(super) enum RunnerMessage<Ctx: OuterContext> {
|
pub(super) enum RunnerMessage<OuterCtx: OuterContext, JobCtx: JobContext<OuterCtx>> {
|
||||||
NewJob {
|
NewJob {
|
||||||
id: JobId,
|
job_id: JobId,
|
||||||
location_id: location::id::Type,
|
location_id: location::id::Type,
|
||||||
dyn_job: Box<dyn DynJob<Ctx>>,
|
dyn_job: Box<dyn DynJob<OuterCtx, JobCtx>>,
|
||||||
ctx: Ctx,
|
ctx: OuterCtx,
|
||||||
ack_tx: oneshot::Sender<Result<(), JobSystemError>>,
|
ack_tx: oneshot::Sender<Result<(), JobSystemError>>,
|
||||||
},
|
},
|
||||||
ResumeStoredJob {
|
ResumeStoredJob {
|
||||||
id: JobId,
|
job_id: JobId,
|
||||||
location_id: location::id::Type,
|
location_id: location::id::Type,
|
||||||
dyn_job: Box<dyn DynJob<Ctx>>,
|
dyn_job: Box<dyn DynJob<OuterCtx, JobCtx>>,
|
||||||
ctx: Ctx,
|
ctx: OuterCtx,
|
||||||
serialized_tasks: Option<SerializedTasks>,
|
serialized_tasks: Option<SerializedTasks>,
|
||||||
ack_tx: oneshot::Sender<Result<(), JobSystemError>>,
|
ack_tx: oneshot::Sender<Result<(), JobSystemError>>,
|
||||||
},
|
},
|
||||||
Command {
|
Command {
|
||||||
id: JobId,
|
job_id: JobId,
|
||||||
command: Command,
|
command: Command,
|
||||||
ack_tx: oneshot::Sender<Result<(), JobSystemError>>,
|
ack_tx: oneshot::Sender<Result<(), JobSystemError>>,
|
||||||
},
|
},
|
||||||
CheckIfJobAreRunning {
|
GetActiveReports {
|
||||||
|
ack_tx: oneshot::Sender<HashMap<JobId, report::Report>>,
|
||||||
|
},
|
||||||
|
CheckIfJobsAreRunning {
|
||||||
job_names: Vec<JobName>,
|
job_names: Vec<JobName>,
|
||||||
location_id: location::id::Type,
|
location_id: location::id::Type,
|
||||||
ack_tx: oneshot::Sender<bool>,
|
ack_tx: oneshot::Sender<bool>,
|
||||||
},
|
},
|
||||||
Shutdown,
|
Shutdown,
|
||||||
|
HasActiveJobs {
|
||||||
|
ctx_id: Uuid,
|
||||||
|
ack_tx: oneshot::Sender<bool>,
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(super) struct JobSystemRunner<Ctx: OuterContext> {
|
struct JobsWorktables {
|
||||||
base_dispatcher: BaseTaskDispatcher<Error>,
|
|
||||||
handles: HashMap<JobId, JobHandle<Ctx>>,
|
|
||||||
job_hashes: HashMap<u64, JobId>,
|
job_hashes: HashMap<u64, JobId>,
|
||||||
job_hashes_by_id: HashMap<JobId, u64>,
|
job_hashes_by_id: HashMap<JobId, u64>,
|
||||||
running_jobs_by_job_id: HashMap<JobId, (JobName, location::id::Type)>,
|
running_jobs_by_job_id: HashMap<JobId, (JobName, location::id::Type)>,
|
||||||
running_jobs_set: HashSet<(JobName, location::id::Type)>,
|
running_jobs_set: HashSet<(JobName, location::id::Type)>,
|
||||||
jobs_to_store_by_ctx_id: HashMap<Uuid, Vec<StoredJobEntry>>,
|
jobs_to_store_by_ctx_id: HashMap<Uuid, Vec<StoredJobEntry>>,
|
||||||
job_return_status_tx: chan::Sender<(JobId, Result<ReturnStatus, Error>)>,
|
|
||||||
job_outputs_tx: chan::Sender<(JobId, Result<JobOutput, JobSystemError>)>,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl<Ctx: OuterContext> JobSystemRunner<Ctx> {
|
pub(super) struct JobSystemRunner<OuterCtx: OuterContext, JobCtx: JobContext<OuterCtx>> {
|
||||||
|
on_shutdown_mode: bool,
|
||||||
|
base_dispatcher: BaseTaskDispatcher<Error>,
|
||||||
|
handles: HashMap<JobId, JobHandle<OuterCtx, JobCtx>>,
|
||||||
|
worktables: JobsWorktables,
|
||||||
|
job_return_status_tx: chan::Sender<(JobId, Result<ReturnStatus, Error>)>,
|
||||||
|
job_outputs_tx: chan::Sender<(JobId, Result<JobOutput, Error>)>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<OuterCtx: OuterContext, JobCtx: JobContext<OuterCtx>> JobSystemRunner<OuterCtx, JobCtx> {
|
||||||
pub(super) fn new(
|
pub(super) fn new(
|
||||||
base_dispatcher: BaseTaskDispatcher<Error>,
|
base_dispatcher: BaseTaskDispatcher<Error>,
|
||||||
job_return_status_tx: chan::Sender<(JobId, Result<ReturnStatus, Error>)>,
|
job_return_status_tx: chan::Sender<(JobId, Result<ReturnStatus, Error>)>,
|
||||||
job_outputs_tx: chan::Sender<(JobId, Result<JobOutput, JobSystemError>)>,
|
job_outputs_tx: chan::Sender<(JobId, Result<JobOutput, Error>)>,
|
||||||
) -> Self {
|
) -> Self {
|
||||||
Self {
|
Self {
|
||||||
|
on_shutdown_mode: false,
|
||||||
base_dispatcher,
|
base_dispatcher,
|
||||||
handles: HashMap::with_capacity(JOBS_INITIAL_CAPACITY),
|
handles: HashMap::with_capacity(JOBS_INITIAL_CAPACITY),
|
||||||
job_hashes: HashMap::with_capacity(JOBS_INITIAL_CAPACITY),
|
worktables: JobsWorktables {
|
||||||
job_hashes_by_id: HashMap::with_capacity(JOBS_INITIAL_CAPACITY),
|
job_hashes: HashMap::with_capacity(JOBS_INITIAL_CAPACITY),
|
||||||
running_jobs_by_job_id: HashMap::with_capacity(JOBS_INITIAL_CAPACITY),
|
job_hashes_by_id: HashMap::with_capacity(JOBS_INITIAL_CAPACITY),
|
||||||
running_jobs_set: HashSet::with_capacity(JOBS_INITIAL_CAPACITY),
|
running_jobs_by_job_id: HashMap::with_capacity(JOBS_INITIAL_CAPACITY),
|
||||||
jobs_to_store_by_ctx_id: HashMap::new(),
|
running_jobs_set: HashSet::with_capacity(JOBS_INITIAL_CAPACITY),
|
||||||
|
jobs_to_store_by_ctx_id: HashMap::new(),
|
||||||
|
},
|
||||||
job_return_status_tx,
|
job_return_status_tx,
|
||||||
job_outputs_tx,
|
job_outputs_tx,
|
||||||
}
|
}
|
||||||
|
@ -97,42 +116,43 @@ impl<Ctx: OuterContext> JobSystemRunner<Ctx> {
|
||||||
|
|
||||||
async fn new_job(
|
async fn new_job(
|
||||||
&mut self,
|
&mut self,
|
||||||
id: JobId,
|
job_id: JobId,
|
||||||
location_id: location::id::Type,
|
location_id: location::id::Type,
|
||||||
dyn_job: Box<dyn DynJob<Ctx>>,
|
dyn_job: Box<dyn DynJob<OuterCtx, JobCtx>>,
|
||||||
ctx: Ctx,
|
ctx: OuterCtx,
|
||||||
maybe_existing_tasks: Option<SerializedTasks>,
|
maybe_existing_tasks: Option<SerializedTasks>,
|
||||||
) -> Result<(), JobSystemError> {
|
) -> Result<(), JobSystemError> {
|
||||||
let Self {
|
let Self {
|
||||||
base_dispatcher,
|
base_dispatcher,
|
||||||
handles,
|
handles,
|
||||||
job_hashes,
|
worktables:
|
||||||
job_hashes_by_id,
|
JobsWorktables {
|
||||||
|
job_hashes,
|
||||||
|
job_hashes_by_id,
|
||||||
|
running_jobs_by_job_id,
|
||||||
|
running_jobs_set,
|
||||||
|
..
|
||||||
|
},
|
||||||
job_return_status_tx,
|
job_return_status_tx,
|
||||||
running_jobs_by_job_id,
|
|
||||||
running_jobs_set,
|
|
||||||
..
|
..
|
||||||
} = self;
|
} = self;
|
||||||
|
|
||||||
let db = ctx.db();
|
|
||||||
let job_name = dyn_job.job_name();
|
let job_name = dyn_job.job_name();
|
||||||
|
|
||||||
let job_hash = dyn_job.hash();
|
let job_hash = dyn_job.hash();
|
||||||
if let Some(&already_running_id) = job_hashes.get(&job_hash) {
|
if let Some(&already_running_id) = job_hashes.get(&job_hash) {
|
||||||
return Err(JobSystemError::AlreadyRunning {
|
return Err(JobSystemError::AlreadyRunning {
|
||||||
new_id: id,
|
new_id: job_id,
|
||||||
already_running_id,
|
already_running_id,
|
||||||
job_name,
|
job_name,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
running_jobs_by_job_id.insert(id, (job_name, location_id));
|
running_jobs_by_job_id.insert(job_id, (job_name, location_id));
|
||||||
running_jobs_set.insert((job_name, location_id));
|
running_jobs_set.insert((job_name, location_id));
|
||||||
|
|
||||||
job_hashes.insert(job_hash, id);
|
job_hashes.insert(job_hash, job_id);
|
||||||
job_hashes_by_id.insert(id, job_hash);
|
job_hashes_by_id.insert(job_id, job_hash);
|
||||||
|
|
||||||
let start_time = Utc::now();
|
|
||||||
|
|
||||||
let mut handle = if maybe_existing_tasks.is_some() {
|
let mut handle = if maybe_existing_tasks.is_some() {
|
||||||
dyn_job.resume(
|
dyn_job.resume(
|
||||||
|
@ -149,174 +169,220 @@ impl<Ctx: OuterContext> JobSystemRunner<Ctx> {
|
||||||
)
|
)
|
||||||
};
|
};
|
||||||
|
|
||||||
handle.report.status = report::Status::Running;
|
handle.register_start(Utc::now()).await?;
|
||||||
if handle.report.started_at.is_none() {
|
|
||||||
handle.report.started_at = Some(start_time);
|
|
||||||
}
|
|
||||||
|
|
||||||
// If the report doesn't have a created_at date, it's a new report
|
handles.insert(job_id, handle);
|
||||||
if handle.report.created_at.is_none() {
|
|
||||||
handle.report.create(db).await?;
|
|
||||||
} else {
|
|
||||||
// Otherwise it can be a job being resumed or a children job that was already been created
|
|
||||||
handle.report.update(db).await?;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Registering children jobs
|
|
||||||
handle
|
|
||||||
.next_jobs
|
|
||||||
.iter_mut()
|
|
||||||
.map(|dyn_job| dyn_job.report_mut())
|
|
||||||
.map(|next_job_report| async {
|
|
||||||
if next_job_report.created_at.is_none() {
|
|
||||||
next_job_report.create(ctx.db()).await
|
|
||||||
} else {
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
})
|
|
||||||
.collect::<Vec<_>>()
|
|
||||||
.try_join()
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
handles.insert(id, handle);
|
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn process_command(&mut self, id: JobId, command: Command) -> Result<(), JobSystemError> {
|
async fn get_active_reports(&self) -> HashMap<JobId, report::Report> {
|
||||||
if let Some(handle) = self.handles.get_mut(&id) {
|
self.handles
|
||||||
handle.send_command(command).await?;
|
.iter()
|
||||||
Ok(())
|
.map(|(job_id, handle)| async { (*job_id, handle.ctx.report().await.clone()) })
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join()
|
||||||
|
.await
|
||||||
|
.into_iter()
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn process_command(
|
||||||
|
&mut self,
|
||||||
|
job_id: JobId,
|
||||||
|
command: Command,
|
||||||
|
ack_tx: oneshot::Sender<Result<(), JobSystemError>>,
|
||||||
|
) {
|
||||||
|
if let Some(handle) = self.handles.get_mut(&job_id) {
|
||||||
|
match (command, handle.is_running) {
|
||||||
|
(Command::Pause, false) => {
|
||||||
|
warn!("Tried to pause a job already paused");
|
||||||
|
return ack_tx.send(Ok(())).expect(
|
||||||
|
"ack channel closed before sending response to already paused job",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
(Command::Resume, true) => {
|
||||||
|
warn!("Tried to resume a job already running");
|
||||||
|
return ack_tx.send(Ok(())).expect(
|
||||||
|
"ack channel closed before sending response to already running job",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
match command {
|
||||||
|
Command::Pause | Command::Cancel | Command::Shutdown => {
|
||||||
|
handle.is_running = false;
|
||||||
|
}
|
||||||
|
Command::Resume => {
|
||||||
|
handle.is_running = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
handle.send_command(command, ack_tx).await;
|
||||||
|
handle.ctx.invalidate_query("jobs.isActive");
|
||||||
|
handle.ctx.invalidate_query("jobs.reports");
|
||||||
} else {
|
} else {
|
||||||
Err(JobSystemError::NotFound(id))
|
error!("Job not found");
|
||||||
|
ack_tx
|
||||||
|
.send(Err(JobSystemError::NotFound(job_id)))
|
||||||
|
.unwrap_or_else(|_| {
|
||||||
|
panic!("ack channel closed before sending {command:?} response")
|
||||||
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn is_empty(&self) -> bool {
|
fn is_empty(&self) -> bool {
|
||||||
self.handles.is_empty() && self.job_hashes.is_empty() && self.job_hashes_by_id.is_empty()
|
self.handles.is_empty()
|
||||||
|
&& self.worktables.job_hashes.is_empty()
|
||||||
|
&& self.worktables.job_hashes_by_id.is_empty()
|
||||||
}
|
}
|
||||||
|
|
||||||
fn check_if_job_are_running(
|
fn total_jobs(&self) -> usize {
|
||||||
|
self.handles.len()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn check_if_jobs_are_running(
|
||||||
&self,
|
&self,
|
||||||
job_names: Vec<JobName>,
|
job_names: Vec<JobName>,
|
||||||
location_id: location::id::Type,
|
location_id: location::id::Type,
|
||||||
) -> bool {
|
) -> bool {
|
||||||
job_names
|
job_names.into_iter().any(|job_name| {
|
||||||
.into_iter()
|
self.worktables
|
||||||
.any(|job_name| self.running_jobs_set.contains(&(job_name, location_id)))
|
.running_jobs_set
|
||||||
|
.contains(&(job_name, location_id))
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn process_return_status(&mut self, job_id: JobId, status: Result<ReturnStatus, Error>) {
|
#[instrument(skip_all, fields(%job_id))]
|
||||||
|
async fn process_return_status(
|
||||||
|
&mut self,
|
||||||
|
job_id: JobId,
|
||||||
|
status: Result<ReturnStatus, Error>,
|
||||||
|
) -> Result<(), JobSystemError> {
|
||||||
let Self {
|
let Self {
|
||||||
|
on_shutdown_mode,
|
||||||
handles,
|
handles,
|
||||||
job_hashes,
|
worktables,
|
||||||
job_hashes_by_id,
|
|
||||||
job_outputs_tx,
|
job_outputs_tx,
|
||||||
job_return_status_tx,
|
job_return_status_tx,
|
||||||
base_dispatcher,
|
base_dispatcher,
|
||||||
jobs_to_store_by_ctx_id,
|
|
||||||
running_jobs_by_job_id,
|
|
||||||
running_jobs_set,
|
|
||||||
..
|
..
|
||||||
} = self;
|
} = self;
|
||||||
|
|
||||||
let job_hash = job_hashes_by_id.remove(&job_id).expect("it must be here");
|
let job_hash = worktables
|
||||||
let (job_name, location_id) = running_jobs_by_job_id
|
.job_hashes_by_id
|
||||||
|
.remove(&job_id)
|
||||||
|
.expect("it must be here");
|
||||||
|
|
||||||
|
let (job_name, location_id) = worktables
|
||||||
|
.running_jobs_by_job_id
|
||||||
.remove(&job_id)
|
.remove(&job_id)
|
||||||
.expect("a JobName and location_id must've been inserted in the map with the job id");
|
.expect("a JobName and location_id must've been inserted in the map with the job id");
|
||||||
assert!(running_jobs_set.remove(&(job_name, location_id)));
|
|
||||||
|
|
||||||
assert!(job_hashes.remove(&job_hash).is_some());
|
assert!(worktables.running_jobs_set.remove(&(job_name, location_id)));
|
||||||
|
assert!(worktables.job_hashes.remove(&job_hash).is_some());
|
||||||
|
|
||||||
let mut handle = handles.remove(&job_id).expect("it must be here");
|
let mut handle = handles.remove(&job_id).expect("it must be here");
|
||||||
|
handle.run_time += handle.start_time.elapsed();
|
||||||
|
|
||||||
|
handle
|
||||||
|
.ctx
|
||||||
|
.report_mut()
|
||||||
|
.await
|
||||||
|
.push_metadata(ReportOutputMetadata::Metrics(HashMap::from([(
|
||||||
|
"job_run_time".into(),
|
||||||
|
json!(handle.run_time),
|
||||||
|
)])));
|
||||||
|
|
||||||
let res = match status {
|
let res = match status {
|
||||||
Ok(ReturnStatus::Completed(job_return)) => {
|
Ok(ReturnStatus::Completed(job_return)) => {
|
||||||
try_dispatch_next_job(
|
try_dispatch_next_job(
|
||||||
&mut handle,
|
&mut handle,
|
||||||
|
location_id,
|
||||||
base_dispatcher.clone(),
|
base_dispatcher.clone(),
|
||||||
(job_hashes, job_hashes_by_id),
|
worktables,
|
||||||
handles,
|
handles,
|
||||||
job_return_status_tx.clone(),
|
job_return_status_tx.clone(),
|
||||||
);
|
)
|
||||||
|
.await?;
|
||||||
|
|
||||||
handle.complete_job(job_return).await
|
handle.complete_job(job_return).await.map_err(Into::into)
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(ReturnStatus::Shutdown(Ok(Some(serialized_job)))) => {
|
Ok(ReturnStatus::Shutdown(res)) => {
|
||||||
let name = handle.report.name;
|
match res {
|
||||||
|
Ok(Some(serialized_job)) => {
|
||||||
|
let name = {
|
||||||
|
let db = handle.ctx.db();
|
||||||
|
let mut report = handle.ctx.report_mut().await;
|
||||||
|
if let Err(e) = report.update(db).await {
|
||||||
|
error!(?e, "Failed to update report on job shutdown;");
|
||||||
|
}
|
||||||
|
report.name
|
||||||
|
};
|
||||||
|
|
||||||
let Ok(next_jobs) = handle
|
worktables
|
||||||
.next_jobs
|
.jobs_to_store_by_ctx_id
|
||||||
.into_iter()
|
.entry(handle.ctx.id())
|
||||||
.map(|next_job| async move {
|
.or_default()
|
||||||
let next_id = next_job.id();
|
.push(StoredJobEntry {
|
||||||
let next_name = next_job.job_name();
|
location_id,
|
||||||
next_job
|
root_job: StoredJob {
|
||||||
.serialize()
|
id: job_id,
|
||||||
.await
|
run_time: handle.start_time.elapsed(),
|
||||||
.map(|maybe_serialized_job| {
|
name,
|
||||||
maybe_serialized_job.map(|serialized_job| StoredJob {
|
|
||||||
id: next_id,
|
|
||||||
name: next_name,
|
|
||||||
serialized_job,
|
serialized_job,
|
||||||
})
|
},
|
||||||
})
|
next_jobs: serialize_next_jobs_to_shutdown(
|
||||||
.map_err(|e| {
|
job_id,
|
||||||
error!(
|
job_name,
|
||||||
"Failed to serialize next job: \
|
handle.next_jobs,
|
||||||
<parent_id='{job_id}', parent_name='{name}', \
|
)
|
||||||
next_id='{next_id}', next_name='{next_name}'>: {e:#?}"
|
.await
|
||||||
);
|
.unwrap_or_default(),
|
||||||
})
|
});
|
||||||
})
|
|
||||||
.collect::<Vec<_>>()
|
|
||||||
.try_join()
|
|
||||||
.await
|
|
||||||
else {
|
|
||||||
return;
|
|
||||||
};
|
|
||||||
|
|
||||||
jobs_to_store_by_ctx_id
|
debug!(%name, "Job was shutdown and serialized;");
|
||||||
.entry(handle.ctx.id())
|
}
|
||||||
.or_default()
|
|
||||||
.push(StoredJobEntry {
|
|
||||||
location_id,
|
|
||||||
root_job: StoredJob {
|
|
||||||
id: job_id,
|
|
||||||
name,
|
|
||||||
serialized_job,
|
|
||||||
},
|
|
||||||
next_jobs: next_jobs.into_iter().flatten().collect(),
|
|
||||||
});
|
|
||||||
|
|
||||||
return;
|
Ok(None) => {
|
||||||
|
debug!(
|
||||||
|
"Job was shutdown but didn't returned any serialized data, \
|
||||||
|
probably it isn't resumable job"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
Err(e) => {
|
||||||
|
error!(?e, "Failed to serialize job;");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if *on_shutdown_mode && handles.is_empty() {
|
||||||
|
// Job system is empty and in shutdown mode so we close this channel to finish the shutdown process
|
||||||
|
job_return_status_tx.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
return Ok(());
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(ReturnStatus::Shutdown(Ok(None))) => {
|
Ok(ReturnStatus::Canceled(job_return)) => {
|
||||||
debug!(
|
handle.cancel_job(job_return).await.map_err(Into::into)
|
||||||
"Job was shutdown but didn't returned any serialized data, \
|
|
||||||
probably it isn't resumable job: <id='{job_id}'>"
|
|
||||||
);
|
|
||||||
return;
|
|
||||||
}
|
}
|
||||||
|
Err(e) => handle
|
||||||
Ok(ReturnStatus::Shutdown(Err(e))) => {
|
.failed_job(&e)
|
||||||
error!("Failed to serialize job: {e:#?}");
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(ReturnStatus::Canceled) => handle
|
|
||||||
.cancel_job()
|
|
||||||
.await
|
.await
|
||||||
.and_then(|()| Err(JobSystemError::Canceled(job_id))),
|
.map_err(Into::into)
|
||||||
|
.and_then(|()| Err(e)),
|
||||||
Err(e) => handle.failed_job(&e).await.and_then(|()| Err(e.into())),
|
|
||||||
};
|
};
|
||||||
|
|
||||||
job_outputs_tx
|
job_outputs_tx
|
||||||
.send((job_id, res))
|
.send((job_id, res))
|
||||||
.await
|
.await
|
||||||
.expect("job outputs channel unexpectedly closed on job completion");
|
.expect("job outputs channel unexpectedly closed on job completion");
|
||||||
|
|
||||||
|
handle.ctx.invalidate_query("jobs.isActive");
|
||||||
|
handle.ctx.invalidate_query("jobs.reports");
|
||||||
|
|
||||||
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn clean_memory(&mut self) {
|
fn clean_memory(&mut self) {
|
||||||
|
@ -326,28 +392,34 @@ impl<Ctx: OuterContext> JobSystemRunner<Ctx> {
|
||||||
self.handles.shrink_to(JOBS_INITIAL_CAPACITY);
|
self.handles.shrink_to(JOBS_INITIAL_CAPACITY);
|
||||||
}
|
}
|
||||||
|
|
||||||
if self.job_hashes.capacity() > JOBS_INITIAL_CAPACITY
|
if self.worktables.job_hashes.capacity() > JOBS_INITIAL_CAPACITY
|
||||||
&& self.job_hashes.len() < JOBS_INITIAL_CAPACITY
|
&& self.worktables.job_hashes.len() < JOBS_INITIAL_CAPACITY
|
||||||
{
|
{
|
||||||
self.job_hashes.shrink_to(JOBS_INITIAL_CAPACITY);
|
self.worktables.job_hashes.shrink_to(JOBS_INITIAL_CAPACITY);
|
||||||
}
|
}
|
||||||
|
|
||||||
if self.job_hashes_by_id.capacity() > JOBS_INITIAL_CAPACITY
|
if self.worktables.job_hashes_by_id.capacity() > JOBS_INITIAL_CAPACITY
|
||||||
&& self.job_hashes_by_id.len() < JOBS_INITIAL_CAPACITY
|
&& self.worktables.job_hashes_by_id.len() < JOBS_INITIAL_CAPACITY
|
||||||
{
|
{
|
||||||
self.job_hashes_by_id.shrink_to(JOBS_INITIAL_CAPACITY);
|
self.worktables
|
||||||
|
.job_hashes_by_id
|
||||||
|
.shrink_to(JOBS_INITIAL_CAPACITY);
|
||||||
}
|
}
|
||||||
|
|
||||||
if self.running_jobs_by_job_id.capacity() > JOBS_INITIAL_CAPACITY
|
if self.worktables.running_jobs_by_job_id.capacity() > JOBS_INITIAL_CAPACITY
|
||||||
&& self.running_jobs_by_job_id.len() < JOBS_INITIAL_CAPACITY
|
&& self.worktables.running_jobs_by_job_id.len() < JOBS_INITIAL_CAPACITY
|
||||||
{
|
{
|
||||||
self.running_jobs_by_job_id.shrink_to(JOBS_INITIAL_CAPACITY);
|
self.worktables
|
||||||
|
.running_jobs_by_job_id
|
||||||
|
.shrink_to(JOBS_INITIAL_CAPACITY);
|
||||||
}
|
}
|
||||||
|
|
||||||
if self.running_jobs_set.capacity() > JOBS_INITIAL_CAPACITY
|
if self.worktables.running_jobs_set.capacity() > JOBS_INITIAL_CAPACITY
|
||||||
&& self.running_jobs_set.len() < JOBS_INITIAL_CAPACITY
|
&& self.worktables.running_jobs_set.len() < JOBS_INITIAL_CAPACITY
|
||||||
{
|
{
|
||||||
self.running_jobs_set.shrink_to(JOBS_INITIAL_CAPACITY);
|
self.worktables
|
||||||
|
.running_jobs_set
|
||||||
|
.shrink_to(JOBS_INITIAL_CAPACITY);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -359,9 +431,13 @@ impl<Ctx: OuterContext> JobSystemRunner<Ctx> {
|
||||||
|
|
||||||
let Self {
|
let Self {
|
||||||
handles,
|
handles,
|
||||||
job_hashes,
|
worktables:
|
||||||
job_hashes_by_id,
|
JobsWorktables {
|
||||||
jobs_to_store_by_ctx_id,
|
job_hashes,
|
||||||
|
job_hashes_by_id,
|
||||||
|
jobs_to_store_by_ctx_id,
|
||||||
|
..
|
||||||
|
},
|
||||||
..
|
..
|
||||||
} = self;
|
} = self;
|
||||||
|
|
||||||
|
@ -382,23 +458,113 @@ impl<Ctx: OuterContext> JobSystemRunner<Ctx> {
|
||||||
.await
|
.await
|
||||||
.map_err(|e| JobSystemError::StoredJobs(FileIOError::from((store_jobs_file, e))))
|
.map_err(|e| JobSystemError::StoredJobs(FileIOError::from((store_jobs_file, e))))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn has_active_jobs(&self, ctx_id: Uuid) -> bool {
|
||||||
|
self.handles
|
||||||
|
.values()
|
||||||
|
.any(|handle| handle.ctx.id() == ctx_id && handle.is_running)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn dispatch_shutdown_command_to_jobs(&mut self) {
|
||||||
|
self.handles
|
||||||
|
.values_mut()
|
||||||
|
.map(|handle| async move {
|
||||||
|
let (tx, rx) = oneshot::channel();
|
||||||
|
|
||||||
|
handle.send_command(Command::Shutdown, tx).await;
|
||||||
|
|
||||||
|
rx.await.expect("Worker failed to ack shutdown request")
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join()
|
||||||
|
.await
|
||||||
|
.into_iter()
|
||||||
|
.for_each(|res| {
|
||||||
|
if let Err(e) = res {
|
||||||
|
error!(?e, "Failed to shutdown job;");
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn try_dispatch_next_job<Ctx: OuterContext>(
|
#[instrument(skip(next_jobs))]
|
||||||
handle: &mut JobHandle<Ctx>,
|
async fn serialize_next_jobs_to_shutdown<OuterCtx: OuterContext, JobCtx: JobContext<OuterCtx>>(
|
||||||
|
parent_job_id: JobId,
|
||||||
|
parent_job_name: JobName,
|
||||||
|
next_jobs: impl IntoIterator<Item = Box<dyn DynJob<OuterCtx, JobCtx>>> + Send,
|
||||||
|
) -> Option<Vec<StoredJob>> {
|
||||||
|
next_jobs
|
||||||
|
.into_iter()
|
||||||
|
.map(|next_job| async move {
|
||||||
|
let next_id = next_job.id();
|
||||||
|
let next_name = next_job.job_name();
|
||||||
|
next_job
|
||||||
|
.serialize()
|
||||||
|
.await
|
||||||
|
.map(|maybe_serialized_job| {
|
||||||
|
maybe_serialized_job.map(|serialized_job| StoredJob {
|
||||||
|
id: next_id,
|
||||||
|
run_time: Duration::ZERO,
|
||||||
|
name: next_name,
|
||||||
|
serialized_job,
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.map_err(|e| {
|
||||||
|
error!(%next_id, %next_name, ?e, "Failed to serialize next job;");
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.try_join()
|
||||||
|
.await
|
||||||
|
.map(|maybe_serialized_next_jobs| {
|
||||||
|
maybe_serialized_next_jobs.into_iter().flatten().collect()
|
||||||
|
})
|
||||||
|
.ok()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[instrument(
|
||||||
|
skip_all,
|
||||||
|
fields(
|
||||||
|
job_id = %handle.id,
|
||||||
|
next_jobs_count = handle.next_jobs.len(),
|
||||||
|
location_id = %location_id,
|
||||||
|
total_running_jobs = handles.len(),
|
||||||
|
)
|
||||||
|
)]
|
||||||
|
async fn try_dispatch_next_job<OuterCtx: OuterContext, JobCtx: JobContext<OuterCtx>>(
|
||||||
|
handle: &mut JobHandle<OuterCtx, JobCtx>,
|
||||||
|
location_id: location::id::Type,
|
||||||
base_dispatcher: BaseTaskDispatcher<Error>,
|
base_dispatcher: BaseTaskDispatcher<Error>,
|
||||||
(job_hashes, job_hashes_by_id): (&mut HashMap<u64, JobId>, &mut HashMap<JobId, u64>),
|
JobsWorktables {
|
||||||
handles: &mut HashMap<JobId, JobHandle<Ctx>>,
|
job_hashes,
|
||||||
|
job_hashes_by_id,
|
||||||
|
running_jobs_by_job_id,
|
||||||
|
running_jobs_set,
|
||||||
|
..
|
||||||
|
}: &mut JobsWorktables,
|
||||||
|
handles: &mut HashMap<JobId, JobHandle<OuterCtx, JobCtx>>,
|
||||||
job_return_status_tx: chan::Sender<(JobId, Result<ReturnStatus, Error>)>,
|
job_return_status_tx: chan::Sender<(JobId, Result<ReturnStatus, Error>)>,
|
||||||
) {
|
) -> Result<(), JobSystemError> {
|
||||||
if let Some(next) = handle.next_jobs.pop_front() {
|
if let Some(next) = handle.next_jobs.pop_front() {
|
||||||
let next_id = next.id();
|
let next_id = next.id();
|
||||||
let next_hash = next.hash();
|
let next_hash = next.hash();
|
||||||
|
let next_name = next.job_name();
|
||||||
|
|
||||||
if let Entry::Vacant(e) = job_hashes.entry(next_hash) {
|
if let Entry::Vacant(e) = job_hashes.entry(next_hash) {
|
||||||
e.insert(next_id);
|
e.insert(next_id);
|
||||||
|
trace!(%next_id, %next_name, "Dispatching next job;");
|
||||||
|
|
||||||
job_hashes_by_id.insert(next_id, next_hash);
|
job_hashes_by_id.insert(next_id, next_hash);
|
||||||
let mut next_handle =
|
running_jobs_by_job_id.insert(next_id, (next_name, location_id));
|
||||||
next.dispatch(base_dispatcher, handle.ctx.clone(), job_return_status_tx);
|
running_jobs_set.insert((next_name, location_id));
|
||||||
|
|
||||||
|
let mut next_handle = next.dispatch(
|
||||||
|
base_dispatcher,
|
||||||
|
handle.ctx.get_outer_ctx(),
|
||||||
|
job_return_status_tx,
|
||||||
|
);
|
||||||
|
|
||||||
|
next_handle.register_start(Utc::now()).await?;
|
||||||
|
|
||||||
assert!(
|
assert!(
|
||||||
next_handle.next_jobs.is_empty(),
|
next_handle.next_jobs.is_empty(),
|
||||||
|
@ -410,30 +576,34 @@ fn try_dispatch_next_job<Ctx: OuterContext>(
|
||||||
|
|
||||||
handles.insert(next_id, next_handle);
|
handles.insert(next_id, next_handle);
|
||||||
} else {
|
} else {
|
||||||
warn!("Unexpectedly found a job with the same hash as the next job: <id='{next_id}', name='{}'>", next.job_name());
|
warn!(%next_id, %next_name, "Unexpectedly found a job with the same hash as the next job;");
|
||||||
}
|
}
|
||||||
|
} else {
|
||||||
|
trace!("No next jobs to dispatch");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(super) async fn run<Ctx: OuterContext>(
|
pub(super) async fn run<OuterCtx: OuterContext, JobCtx: JobContext<OuterCtx>>(
|
||||||
mut runner: JobSystemRunner<Ctx>,
|
mut runner: JobSystemRunner<OuterCtx, JobCtx>,
|
||||||
store_jobs_file: impl AsRef<Path> + Send,
|
store_jobs_file: impl AsRef<Path> + Send,
|
||||||
msgs_rx: chan::Receiver<RunnerMessage<Ctx>>,
|
msgs_rx: chan::Receiver<RunnerMessage<OuterCtx, JobCtx>>,
|
||||||
job_return_status_rx: chan::Receiver<(JobId, Result<ReturnStatus, Error>)>,
|
job_done_rx: chan::Receiver<(JobId, Result<ReturnStatus, Error>)>,
|
||||||
) {
|
) {
|
||||||
enum StreamMessage<Ctx: OuterContext> {
|
enum StreamMessage<OuterCtx: OuterContext, JobCtx: JobContext<OuterCtx>> {
|
||||||
ReturnStatus((JobId, Result<ReturnStatus, Error>)),
|
ReturnStatus((JobId, Result<ReturnStatus, Error>)),
|
||||||
RunnerMessage(RunnerMessage<Ctx>),
|
RunnerMessage(RunnerMessage<OuterCtx, JobCtx>),
|
||||||
CleanMemoryTick,
|
CleanMemoryTick,
|
||||||
}
|
}
|
||||||
|
|
||||||
let memory_cleanup_interval = interval_at(Instant::now() + FIVE_MINUTES, FIVE_MINUTES);
|
let memory_cleanup_interval = interval_at(Instant::now() + FIVE_MINUTES, FIVE_MINUTES);
|
||||||
|
|
||||||
let job_return_status_rx_to_shutdown = job_return_status_rx.clone();
|
let job_return_status_rx_to_shutdown = job_done_rx.clone();
|
||||||
|
|
||||||
let mut msg_stream = pin!((
|
let mut msg_stream = pin!((
|
||||||
msgs_rx.map(StreamMessage::RunnerMessage),
|
msgs_rx.map(StreamMessage::RunnerMessage),
|
||||||
job_return_status_rx.map(StreamMessage::ReturnStatus),
|
job_done_rx.map(StreamMessage::ReturnStatus),
|
||||||
IntervalStream::new(memory_cleanup_interval).map(|_| StreamMessage::CleanMemoryTick),
|
IntervalStream::new(memory_cleanup_interval).map(|_| StreamMessage::CleanMemoryTick),
|
||||||
)
|
)
|
||||||
.merge());
|
.merge());
|
||||||
|
@ -442,24 +612,41 @@ pub(super) async fn run<Ctx: OuterContext>(
|
||||||
match msg {
|
match msg {
|
||||||
// Job return status messages
|
// Job return status messages
|
||||||
StreamMessage::ReturnStatus((job_id, status)) => {
|
StreamMessage::ReturnStatus((job_id, status)) => {
|
||||||
runner.process_return_status(job_id, status).await;
|
if let Err(e) = runner.process_return_status(job_id, status).await {
|
||||||
|
error!(?e, "Failed to process return status;");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Runner messages
|
// Runner messages
|
||||||
StreamMessage::RunnerMessage(RunnerMessage::NewJob {
|
StreamMessage::RunnerMessage(RunnerMessage::NewJob {
|
||||||
id,
|
job_id,
|
||||||
location_id,
|
location_id,
|
||||||
dyn_job,
|
dyn_job,
|
||||||
ctx,
|
ctx,
|
||||||
ack_tx,
|
ack_tx,
|
||||||
}) => {
|
}) => {
|
||||||
ack_tx
|
ack_tx
|
||||||
.send(runner.new_job(id, location_id, dyn_job, ctx, None).await)
|
.send(
|
||||||
|
runner
|
||||||
|
.new_job(job_id, location_id, dyn_job, ctx, None)
|
||||||
|
.await,
|
||||||
|
)
|
||||||
.expect("ack channel closed before sending new job response");
|
.expect("ack channel closed before sending new job response");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
StreamMessage::RunnerMessage(RunnerMessage::HasActiveJobs { ctx_id, ack_tx }) => {
|
||||||
|
ack_tx
|
||||||
|
.send(runner.has_active_jobs(ctx_id))
|
||||||
|
.expect("ack channel closed before sending has active jobs response");
|
||||||
|
}
|
||||||
|
|
||||||
|
StreamMessage::RunnerMessage(RunnerMessage::GetActiveReports { ack_tx }) => {
|
||||||
|
ack_tx
|
||||||
|
.send(runner.get_active_reports().await)
|
||||||
|
.expect("ack channel closed before sending active reports response");
|
||||||
|
}
|
||||||
StreamMessage::RunnerMessage(RunnerMessage::ResumeStoredJob {
|
StreamMessage::RunnerMessage(RunnerMessage::ResumeStoredJob {
|
||||||
id,
|
job_id,
|
||||||
location_id,
|
location_id,
|
||||||
dyn_job,
|
dyn_job,
|
||||||
ctx,
|
ctx,
|
||||||
|
@ -469,60 +656,58 @@ pub(super) async fn run<Ctx: OuterContext>(
|
||||||
ack_tx
|
ack_tx
|
||||||
.send(
|
.send(
|
||||||
runner
|
runner
|
||||||
.new_job(id, location_id, dyn_job, ctx, serialized_tasks)
|
.new_job(job_id, location_id, dyn_job, ctx, serialized_tasks)
|
||||||
.await,
|
.await,
|
||||||
)
|
)
|
||||||
.expect("ack channel closed before sending resume job response");
|
.expect("ack channel closed before sending resume job response");
|
||||||
}
|
}
|
||||||
|
|
||||||
StreamMessage::RunnerMessage(RunnerMessage::Command {
|
StreamMessage::RunnerMessage(RunnerMessage::Command {
|
||||||
id,
|
job_id: id,
|
||||||
command,
|
command,
|
||||||
ack_tx,
|
ack_tx,
|
||||||
}) => {
|
}) => runner.process_command(id, command, ack_tx).await,
|
||||||
ack_tx
|
|
||||||
.send(runner.process_command(id, command).await)
|
|
||||||
.unwrap_or_else(|_| {
|
|
||||||
panic!("ack channel closed before sending {command:?} response")
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
StreamMessage::RunnerMessage(RunnerMessage::Shutdown) => {
|
StreamMessage::RunnerMessage(RunnerMessage::Shutdown) => {
|
||||||
|
runner.on_shutdown_mode = true;
|
||||||
// Consuming all pending return status messages
|
// Consuming all pending return status messages
|
||||||
loop {
|
if !runner.is_empty() {
|
||||||
while let Ok((job_id, status)) = job_return_status_rx_to_shutdown.try_recv() {
|
let mut job_return_status_stream = pin!(job_return_status_rx_to_shutdown);
|
||||||
runner.process_return_status(job_id, status).await;
|
|
||||||
|
runner.dispatch_shutdown_command_to_jobs().await;
|
||||||
|
|
||||||
|
debug!(
|
||||||
|
total_jobs = runner.total_jobs(),
|
||||||
|
"Waiting for jobs to shutdown before shutting down the job system...;",
|
||||||
|
);
|
||||||
|
|
||||||
|
while let Some((job_id, status)) = job_return_status_stream.next().await {
|
||||||
|
if let Err(e) = runner.process_return_status(job_id, status).await {
|
||||||
|
error!(?e, "Failed to process return status before shutting down;");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if runner.is_empty() {
|
// Now the runner can shutdown
|
||||||
break;
|
if let Err(e) = runner.save_jobs(store_jobs_file).await {
|
||||||
|
error!(?e, "Failed to save jobs before shutting down;");
|
||||||
}
|
}
|
||||||
|
|
||||||
debug!("Waiting for all jobs to complete before shutting down...");
|
|
||||||
}
|
|
||||||
|
|
||||||
// Now the runner can shutdown
|
|
||||||
if let Err(e) = runner.save_jobs(store_jobs_file).await {
|
|
||||||
error!("Failed to save jobs before shutting down: {e:#?}");
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
StreamMessage::RunnerMessage(RunnerMessage::CheckIfJobAreRunning {
|
StreamMessage::RunnerMessage(RunnerMessage::CheckIfJobsAreRunning {
|
||||||
job_names,
|
job_names,
|
||||||
location_id,
|
location_id,
|
||||||
ack_tx,
|
ack_tx,
|
||||||
}) => {
|
}) => {
|
||||||
ack_tx
|
ack_tx
|
||||||
.send(runner.check_if_job_are_running(job_names, location_id))
|
.send(runner.check_if_jobs_are_running(job_names, location_id))
|
||||||
.expect("ack channel closed before sending resume job response");
|
.expect("ack channel closed before sending resume job response");
|
||||||
}
|
}
|
||||||
|
|
||||||
// Memory cleanup tick
|
// Memory cleanup tick
|
||||||
StreamMessage::CleanMemoryTick => {
|
StreamMessage::CleanMemoryTick => runner.clean_memory(),
|
||||||
runner.clean_memory();
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
use crate::{file_identifier, indexer, media_processor};
|
use crate::{file_identifier, indexer, media_processor, JobContext};
|
||||||
|
|
||||||
use sd_prisma::prisma::{job, location};
|
use sd_prisma::prisma::{job, location};
|
||||||
use sd_utils::uuid_to_bytes;
|
use sd_utils::uuid_to_bytes;
|
||||||
|
@ -8,6 +8,7 @@ use std::{
|
||||||
future::Future,
|
future::Future,
|
||||||
iter,
|
iter,
|
||||||
marker::PhantomData,
|
marker::PhantomData,
|
||||||
|
time::Duration,
|
||||||
};
|
};
|
||||||
|
|
||||||
use futures_concurrency::future::TryJoin;
|
use futures_concurrency::future::TryJoin;
|
||||||
|
@ -20,9 +21,11 @@ use super::{
|
||||||
};
|
};
|
||||||
|
|
||||||
#[derive(Debug, Serialize, Deserialize)]
|
#[derive(Debug, Serialize, Deserialize)]
|
||||||
|
#[repr(transparent)]
|
||||||
|
#[serde(transparent)]
|
||||||
pub struct SerializedTasks(pub Vec<u8>);
|
pub struct SerializedTasks(pub Vec<u8>);
|
||||||
|
|
||||||
pub trait SerializableJob<Ctx: OuterContext>: 'static
|
pub trait SerializableJob<OuterCtx: OuterContext>: 'static
|
||||||
where
|
where
|
||||||
Self: Sized,
|
Self: Sized,
|
||||||
{
|
{
|
||||||
|
@ -35,7 +38,7 @@ where
|
||||||
#[allow(unused_variables)]
|
#[allow(unused_variables)]
|
||||||
fn deserialize(
|
fn deserialize(
|
||||||
serialized_job: &[u8],
|
serialized_job: &[u8],
|
||||||
ctx: &Ctx,
|
ctx: &OuterCtx,
|
||||||
) -> impl Future<
|
) -> impl Future<
|
||||||
Output = Result<Option<(Self, Option<SerializedTasks>)>, rmp_serde::decode::Error>,
|
Output = Result<Option<(Self, Option<SerializedTasks>)>, rmp_serde::decode::Error>,
|
||||||
> + Send {
|
> + Send {
|
||||||
|
@ -47,6 +50,7 @@ where
|
||||||
pub struct StoredJob {
|
pub struct StoredJob {
|
||||||
pub(super) id: JobId,
|
pub(super) id: JobId,
|
||||||
pub(super) name: JobName,
|
pub(super) name: JobName,
|
||||||
|
pub(super) run_time: Duration,
|
||||||
pub(super) serialized_job: Vec<u8>,
|
pub(super) serialized_job: Vec<u8>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -57,13 +61,13 @@ pub struct StoredJobEntry {
|
||||||
pub(super) next_jobs: Vec<StoredJob>,
|
pub(super) next_jobs: Vec<StoredJob>,
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn load_jobs<Ctx: OuterContext>(
|
pub async fn load_jobs<OuterCtx: OuterContext, JobCtx: JobContext<OuterCtx>>(
|
||||||
entries: Vec<StoredJobEntry>,
|
entries: Vec<StoredJobEntry>,
|
||||||
ctx: &Ctx,
|
ctx: &OuterCtx,
|
||||||
) -> Result<
|
) -> Result<
|
||||||
Vec<(
|
Vec<(
|
||||||
location::id::Type,
|
location::id::Type,
|
||||||
Box<dyn DynJob<Ctx>>,
|
Box<dyn DynJob<OuterCtx, JobCtx>>,
|
||||||
Option<SerializedTasks>,
|
Option<SerializedTasks>,
|
||||||
)>,
|
)>,
|
||||||
JobSystemError,
|
JobSystemError,
|
||||||
|
@ -81,7 +85,7 @@ pub async fn load_jobs<Ctx: OuterContext>(
|
||||||
..
|
..
|
||||||
}| { iter::once(*id).chain(next_jobs.iter().map(|StoredJob { id, .. }| *id)) },
|
}| { iter::once(*id).chain(next_jobs.iter().map(|StoredJob { id, .. }| *id)) },
|
||||||
)
|
)
|
||||||
.map(uuid_to_bytes)
|
.map(|job_id| uuid_to_bytes(&job_id))
|
||||||
.collect::<Vec<_>>(),
|
.collect::<Vec<_>>(),
|
||||||
)])
|
)])
|
||||||
.exec()
|
.exec()
|
||||||
|
@ -166,50 +170,58 @@ pub async fn load_jobs<Ctx: OuterContext>(
|
||||||
}
|
}
|
||||||
|
|
||||||
macro_rules! match_deserialize_job {
|
macro_rules! match_deserialize_job {
|
||||||
($stored_job:ident, $report:ident, $ctx:ident, $ctx_type:ty, [$($job_type:ty),+ $(,)?]) => {{
|
($stored_job:ident, $report:ident, $outer_ctx:ident, $outer_ctx_type:ty, $job_ctx_type:ty, [$($job_type:ty),+ $(,)?]) => {{
|
||||||
let StoredJob {
|
let StoredJob {
|
||||||
id,
|
id,
|
||||||
name,
|
name,
|
||||||
|
run_time,
|
||||||
serialized_job,
|
serialized_job,
|
||||||
} = $stored_job;
|
} = $stored_job;
|
||||||
|
|
||||||
|
|
||||||
match name {
|
match name {
|
||||||
$(<$job_type as Job>::NAME => <$job_type as SerializableJob<$ctx_type>>::deserialize(
|
$(<$job_type as Job>::NAME => <$job_type as SerializableJob<$outer_ctx_type>>::deserialize(
|
||||||
&serialized_job,
|
&serialized_job,
|
||||||
$ctx,
|
$outer_ctx,
|
||||||
).await
|
).await
|
||||||
.map(|maybe_job| maybe_job.map(|(job, tasks)| -> (
|
.map(|maybe_job| maybe_job.map(|(job, maybe_tasks)| -> (
|
||||||
Box<dyn DynJob<$ctx_type>>,
|
Box<dyn DynJob<$outer_ctx_type, $job_ctx_type>>,
|
||||||
Option<SerializedTasks>
|
Option<SerializedTasks>
|
||||||
) {
|
) {
|
||||||
(
|
(
|
||||||
Box::new(JobHolder {
|
Box::new(JobHolder {
|
||||||
id,
|
id,
|
||||||
job,
|
job,
|
||||||
|
run_time,
|
||||||
report: $report,
|
report: $report,
|
||||||
next_jobs: VecDeque::new(),
|
next_jobs: VecDeque::new(),
|
||||||
_ctx: PhantomData,
|
_ctx: PhantomData,
|
||||||
}),
|
}),
|
||||||
tasks,
|
maybe_tasks.and_then(
|
||||||
|
|tasks| (!tasks.0.is_empty()).then_some(tasks)
|
||||||
|
),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
))
|
))
|
||||||
.map_err(Into::into),)+
|
.map_err(Into::into),)+
|
||||||
|
|
||||||
|
// TODO(fogodev): this is temporary until we can get rid of the old job system
|
||||||
|
_ => unimplemented!("Job not implemented"),
|
||||||
}
|
}
|
||||||
}};
|
}};
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn load_job<Ctx: OuterContext>(
|
async fn load_job<OuterCtx: OuterContext, JobCtx: JobContext<OuterCtx>>(
|
||||||
stored_job: StoredJob,
|
stored_job: StoredJob,
|
||||||
report: Report,
|
report: Report,
|
||||||
ctx: &Ctx,
|
ctx: &OuterCtx,
|
||||||
) -> Result<Option<(Box<dyn DynJob<Ctx>>, Option<SerializedTasks>)>, JobSystemError> {
|
) -> Result<Option<(Box<dyn DynJob<OuterCtx, JobCtx>>, Option<SerializedTasks>)>, JobSystemError> {
|
||||||
match_deserialize_job!(
|
match_deserialize_job!(
|
||||||
stored_job,
|
stored_job,
|
||||||
report,
|
report,
|
||||||
ctx,
|
ctx,
|
||||||
Ctx,
|
OuterCtx,
|
||||||
|
JobCtx,
|
||||||
[
|
[
|
||||||
indexer::job::Indexer,
|
indexer::job::Indexer,
|
||||||
file_identifier::job::FileIdentifier,
|
file_identifier::job::FileIdentifier,
|
||||||
|
|
|
@ -1,16 +1,35 @@
|
||||||
use crate::Error;
|
use crate::Error;
|
||||||
|
|
||||||
use sd_task_system::TaskHandle;
|
use sd_task_system::{TaskHandle, TaskStatus};
|
||||||
|
|
||||||
|
use futures::{stream::FuturesUnordered, StreamExt};
|
||||||
use futures_concurrency::future::Join;
|
use futures_concurrency::future::Join;
|
||||||
|
use tracing::{error, trace};
|
||||||
|
|
||||||
pub async fn cancel_pending_tasks(
|
pub async fn cancel_pending_tasks(pending_tasks: &mut FuturesUnordered<TaskHandle<Error>>) {
|
||||||
pending_tasks: impl IntoIterator<Item = &TaskHandle<Error>> + Send,
|
|
||||||
) {
|
|
||||||
pending_tasks
|
pending_tasks
|
||||||
.into_iter()
|
.iter()
|
||||||
.map(TaskHandle::cancel)
|
.map(TaskHandle::cancel)
|
||||||
.collect::<Vec<_>>()
|
.collect::<Vec<_>>()
|
||||||
.join()
|
.join()
|
||||||
.await;
|
.await;
|
||||||
|
|
||||||
|
trace!(total_tasks = %pending_tasks.len(), "canceled all pending tasks, now waiting completion");
|
||||||
|
|
||||||
|
while let Some(task_result) = pending_tasks.next().await {
|
||||||
|
match task_result {
|
||||||
|
Ok(TaskStatus::Done((task_id, _))) => trace!(
|
||||||
|
%task_id,
|
||||||
|
"tasks cancellation received a completed task;",
|
||||||
|
),
|
||||||
|
|
||||||
|
Ok(TaskStatus::Canceled | TaskStatus::ForcedAbortion | TaskStatus::Shutdown(_)) => {
|
||||||
|
// Job canceled task
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(TaskStatus::Error(e)) => error!(%e, "job canceled an errored task;"),
|
||||||
|
|
||||||
|
Err(e) => error!(%e, "task system failed to cancel a task;"),
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -44,8 +44,12 @@ pub mod utils;
|
||||||
use media_processor::ThumbKey;
|
use media_processor::ThumbKey;
|
||||||
|
|
||||||
pub use job_system::{
|
pub use job_system::{
|
||||||
job::{IntoJob, JobBuilder, JobName, JobOutput, JobOutputData, OuterContext, ProgressUpdate},
|
job::{
|
||||||
JobId, JobSystem,
|
IntoJob, JobContext, JobEnqueuer, JobName, JobOutput, JobOutputData, OuterContext,
|
||||||
|
ProgressUpdate,
|
||||||
|
},
|
||||||
|
report::Report,
|
||||||
|
JobId, JobSystem, JobSystemError,
|
||||||
};
|
};
|
||||||
|
|
||||||
#[derive(Error, Debug)]
|
#[derive(Error, Debug)]
|
||||||
|
@ -59,6 +63,9 @@ pub enum Error {
|
||||||
|
|
||||||
#[error(transparent)]
|
#[error(transparent)]
|
||||||
TaskSystem(#[from] TaskSystemError),
|
TaskSystem(#[from] TaskSystemError),
|
||||||
|
|
||||||
|
#[error(transparent)]
|
||||||
|
JobSystem(#[from] JobSystemError),
|
||||||
}
|
}
|
||||||
|
|
||||||
impl From<Error> for rspc::Error {
|
impl From<Error> for rspc::Error {
|
||||||
|
@ -70,19 +77,21 @@ impl From<Error> for rspc::Error {
|
||||||
Error::TaskSystem(e) => {
|
Error::TaskSystem(e) => {
|
||||||
Self::with_cause(rspc::ErrorCode::InternalServerError, e.to_string(), e)
|
Self::with_cause(rspc::ErrorCode::InternalServerError, e.to_string(), e)
|
||||||
}
|
}
|
||||||
|
Error::JobSystem(e) => e.into(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(thiserror::Error, Debug, Serialize, Deserialize, Type)]
|
#[derive(thiserror::Error, Debug, Serialize, Deserialize, Type, Clone)]
|
||||||
|
#[serde(rename_all = "snake_case")]
|
||||||
pub enum NonCriticalError {
|
pub enum NonCriticalError {
|
||||||
// TODO: Add variants as needed
|
// TODO: Add variants as needed
|
||||||
#[error(transparent)]
|
#[error(transparent)]
|
||||||
Indexer(#[from] indexer::NonCriticalError),
|
Indexer(#[from] indexer::NonCriticalIndexerError),
|
||||||
#[error(transparent)]
|
#[error(transparent)]
|
||||||
FileIdentifier(#[from] file_identifier::NonCriticalError),
|
FileIdentifier(#[from] file_identifier::NonCriticalFileIdentifierError),
|
||||||
#[error(transparent)]
|
#[error(transparent)]
|
||||||
MediaProcessor(#[from] media_processor::NonCriticalError),
|
MediaProcessor(#[from] media_processor::NonCriticalMediaProcessorError),
|
||||||
}
|
}
|
||||||
|
|
||||||
#[repr(i32)]
|
#[repr(i32)]
|
||||||
|
@ -96,7 +105,7 @@ pub enum LocationScanState {
|
||||||
|
|
||||||
#[derive(Debug, Serialize, Type)]
|
#[derive(Debug, Serialize, Type)]
|
||||||
pub enum UpdateEvent {
|
pub enum UpdateEvent {
|
||||||
NewThumbnailEvent {
|
NewThumbnail {
|
||||||
thumb_key: ThumbKey,
|
thumb_key: ThumbKey,
|
||||||
},
|
},
|
||||||
NewIdentifiedObjects {
|
NewIdentifiedObjects {
|
||||||
|
|
|
@ -1,12 +1,24 @@
|
||||||
use crate::media_processor::{self, media_data_extractor};
|
use crate::media_processor::{self, media_data_extractor};
|
||||||
|
|
||||||
|
use sd_core_prisma_helpers::ObjectPubId;
|
||||||
|
use sd_core_sync::Manager as SyncManager;
|
||||||
|
|
||||||
use sd_file_ext::extensions::{Extension, ImageExtension, ALL_IMAGE_EXTENSIONS};
|
use sd_file_ext::extensions::{Extension, ImageExtension, ALL_IMAGE_EXTENSIONS};
|
||||||
use sd_media_metadata::ExifMetadata;
|
use sd_media_metadata::ExifMetadata;
|
||||||
use sd_prisma::prisma::{exif_data, object, PrismaClient};
|
use sd_prisma::{
|
||||||
|
prisma::{exif_data, object, PrismaClient},
|
||||||
|
prisma_sync,
|
||||||
|
};
|
||||||
|
use sd_sync::{option_sync_db_entry, OperationFactory};
|
||||||
|
use sd_utils::chain_optional_iter;
|
||||||
|
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
|
|
||||||
|
use futures_concurrency::future::TryJoin;
|
||||||
use once_cell::sync::Lazy;
|
use once_cell::sync::Lazy;
|
||||||
|
use prisma_client_rust::QueryError;
|
||||||
|
|
||||||
|
use super::from_slice_option_to_option;
|
||||||
|
|
||||||
pub static AVAILABLE_EXTENSIONS: Lazy<Vec<Extension>> = Lazy::new(|| {
|
pub static AVAILABLE_EXTENSIONS: Lazy<Vec<Extension>> = Lazy::new(|| {
|
||||||
ALL_IMAGE_EXTENSIONS
|
ALL_IMAGE_EXTENSIONS
|
||||||
|
@ -17,6 +29,7 @@ pub static AVAILABLE_EXTENSIONS: Lazy<Vec<Extension>> = Lazy::new(|| {
|
||||||
.collect()
|
.collect()
|
||||||
});
|
});
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
pub const fn can_extract(image_extension: ImageExtension) -> bool {
|
pub const fn can_extract(image_extension: ImageExtension) -> bool {
|
||||||
use ImageExtension::{
|
use ImageExtension::{
|
||||||
Avci, Avcs, Avif, Dng, Heic, Heif, Heifs, Hif, Jpeg, Jpg, Png, Tiff, Webp,
|
Avci, Avcs, Avif, Dng, Heic, Heif, Heifs, Hif, Jpeg, Jpg, Png, Tiff, Webp,
|
||||||
|
@ -27,33 +40,62 @@ pub const fn can_extract(image_extension: ImageExtension) -> bool {
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn to_query(
|
#[must_use]
|
||||||
mdi: ExifMetadata,
|
fn to_query(
|
||||||
|
ExifMetadata {
|
||||||
|
resolution,
|
||||||
|
date_taken,
|
||||||
|
location,
|
||||||
|
camera_data,
|
||||||
|
artist,
|
||||||
|
description,
|
||||||
|
copyright,
|
||||||
|
exif_version,
|
||||||
|
}: ExifMetadata,
|
||||||
object_id: exif_data::object_id::Type,
|
object_id: exif_data::object_id::Type,
|
||||||
) -> exif_data::CreateUnchecked {
|
) -> (Vec<(&'static str, rmpv::Value)>, exif_data::Create) {
|
||||||
exif_data::CreateUnchecked {
|
let (sync_params, db_params) = chain_optional_iter(
|
||||||
object_id,
|
[],
|
||||||
_params: vec![
|
[
|
||||||
exif_data::camera_data::set(serde_json::to_vec(&mdi.camera_data).ok()),
|
option_sync_db_entry!(
|
||||||
exif_data::media_date::set(serde_json::to_vec(&mdi.date_taken).ok()),
|
serde_json::to_vec(&camera_data).ok(),
|
||||||
exif_data::resolution::set(serde_json::to_vec(&mdi.resolution).ok()),
|
exif_data::camera_data
|
||||||
exif_data::media_location::set(serde_json::to_vec(&mdi.location).ok()),
|
),
|
||||||
exif_data::artist::set(mdi.artist),
|
option_sync_db_entry!(serde_json::to_vec(&date_taken).ok(), exif_data::media_date),
|
||||||
exif_data::description::set(mdi.description),
|
option_sync_db_entry!(serde_json::to_vec(&resolution).ok(), exif_data::resolution),
|
||||||
exif_data::copyright::set(mdi.copyright),
|
option_sync_db_entry!(
|
||||||
exif_data::exif_version::set(mdi.exif_version),
|
serde_json::to_vec(&location).ok(),
|
||||||
exif_data::epoch_time::set(mdi.date_taken.map(|x| x.unix_timestamp())),
|
exif_data::media_location
|
||||||
|
),
|
||||||
|
option_sync_db_entry!(artist, exif_data::artist),
|
||||||
|
option_sync_db_entry!(description, exif_data::description),
|
||||||
|
option_sync_db_entry!(copyright, exif_data::copyright),
|
||||||
|
option_sync_db_entry!(exif_version, exif_data::exif_version),
|
||||||
|
option_sync_db_entry!(
|
||||||
|
date_taken.map(|x| x.unix_timestamp()),
|
||||||
|
exif_data::epoch_time
|
||||||
|
),
|
||||||
],
|
],
|
||||||
}
|
)
|
||||||
|
.into_iter()
|
||||||
|
.unzip();
|
||||||
|
|
||||||
|
(
|
||||||
|
sync_params,
|
||||||
|
exif_data::Create {
|
||||||
|
object: object::id::equals(object_id),
|
||||||
|
_params: db_params,
|
||||||
|
},
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn extract(
|
pub async fn extract(
|
||||||
path: impl AsRef<Path> + Send,
|
path: impl AsRef<Path> + Send,
|
||||||
) -> Result<Option<ExifMetadata>, media_processor::NonCriticalError> {
|
) -> Result<Option<ExifMetadata>, media_processor::NonCriticalMediaProcessorError> {
|
||||||
let path = path.as_ref();
|
let path = path.as_ref();
|
||||||
|
|
||||||
ExifMetadata::from_path(&path).await.map_err(|e| {
|
ExifMetadata::from_path(&path).await.map_err(|e| {
|
||||||
media_data_extractor::NonCriticalError::FailedToExtractImageMediaData(
|
media_data_extractor::NonCriticalMediaDataExtractorError::FailedToExtractImageMediaData(
|
||||||
path.to_path_buf(),
|
path.to_path_buf(),
|
||||||
e.to_string(),
|
e.to_string(),
|
||||||
)
|
)
|
||||||
|
@ -62,24 +104,62 @@ pub async fn extract(
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn save(
|
pub async fn save(
|
||||||
media_datas: Vec<(ExifMetadata, object::id::Type)>,
|
exif_datas: impl IntoIterator<Item = (ExifMetadata, object::id::Type, ObjectPubId)> + Send,
|
||||||
db: &PrismaClient,
|
db: &PrismaClient,
|
||||||
) -> Result<u64, media_processor::Error> {
|
sync: &SyncManager,
|
||||||
db.exif_data()
|
) -> Result<u64, QueryError> {
|
||||||
.create_many(
|
exif_datas
|
||||||
media_datas
|
.into_iter()
|
||||||
.into_iter()
|
.map(|(exif_data, object_id, object_pub_id)| async move {
|
||||||
.map(|(exif_data, object_id)| to_query(exif_data, object_id))
|
let (sync_params, create) = to_query(exif_data, object_id);
|
||||||
.collect(),
|
let db_params = create._params.clone();
|
||||||
)
|
|
||||||
.skip_duplicates()
|
sync.write_ops(
|
||||||
.exec()
|
db,
|
||||||
.await
|
(
|
||||||
.map(|created| {
|
sync.shared_create(
|
||||||
#[allow(clippy::cast_sign_loss)]
|
prisma_sync::exif_data::SyncId {
|
||||||
{
|
object: prisma_sync::object::SyncId {
|
||||||
created as u64
|
pub_id: object_pub_id.into(),
|
||||||
}
|
},
|
||||||
|
},
|
||||||
|
sync_params,
|
||||||
|
),
|
||||||
|
db.exif_data()
|
||||||
|
.upsert(exif_data::object_id::equals(object_id), create, db_params)
|
||||||
|
.select(exif_data::select!({ id })),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.await
|
||||||
})
|
})
|
||||||
.map_err(Into::into)
|
.collect::<Vec<_>>()
|
||||||
|
.try_join()
|
||||||
|
.await
|
||||||
|
.map(|created_vec| created_vec.len() as u64)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
|
pub fn from_prisma_data(
|
||||||
|
exif_data::Data {
|
||||||
|
resolution,
|
||||||
|
media_date,
|
||||||
|
media_location,
|
||||||
|
camera_data,
|
||||||
|
artist,
|
||||||
|
description,
|
||||||
|
copyright,
|
||||||
|
exif_version,
|
||||||
|
..
|
||||||
|
}: exif_data::Data,
|
||||||
|
) -> ExifMetadata {
|
||||||
|
ExifMetadata {
|
||||||
|
camera_data: from_slice_option_to_option(camera_data).unwrap_or_default(),
|
||||||
|
date_taken: from_slice_option_to_option(media_date).unwrap_or_default(),
|
||||||
|
resolution: from_slice_option_to_option(resolution).unwrap_or_default(),
|
||||||
|
location: from_slice_option_to_option(media_location),
|
||||||
|
artist,
|
||||||
|
description,
|
||||||
|
copyright,
|
||||||
|
exif_version,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,5 +1,7 @@
|
||||||
use crate::media_processor::{self, media_data_extractor};
|
use crate::media_processor::{self, media_data_extractor};
|
||||||
|
|
||||||
|
use sd_core_prisma_helpers::object_with_media_data;
|
||||||
|
|
||||||
use sd_file_ext::extensions::{
|
use sd_file_ext::extensions::{
|
||||||
AudioExtension, Extension, VideoExtension, ALL_AUDIO_EXTENSIONS, ALL_VIDEO_EXTENSIONS,
|
AudioExtension, Extension, VideoExtension, ALL_AUDIO_EXTENSIONS, ALL_VIDEO_EXTENSIONS,
|
||||||
};
|
};
|
||||||
|
@ -19,7 +21,10 @@ use sd_prisma::prisma::{
|
||||||
ffmpeg_data, ffmpeg_media_audio_props, ffmpeg_media_chapter, ffmpeg_media_codec,
|
ffmpeg_data, ffmpeg_media_audio_props, ffmpeg_media_chapter, ffmpeg_media_codec,
|
||||||
ffmpeg_media_program, ffmpeg_media_stream, ffmpeg_media_video_props, object, PrismaClient,
|
ffmpeg_media_program, ffmpeg_media_stream, ffmpeg_media_video_props, object, PrismaClient,
|
||||||
};
|
};
|
||||||
use sd_utils::db::ffmpeg_data_field_to_db;
|
use sd_utils::{
|
||||||
|
db::{ffmpeg_data_field_from_db, ffmpeg_data_field_to_db},
|
||||||
|
i64_to_frontend,
|
||||||
|
};
|
||||||
|
|
||||||
use std::{collections::HashMap, path::Path};
|
use std::{collections::HashMap, path::Path};
|
||||||
|
|
||||||
|
@ -28,6 +33,8 @@ use once_cell::sync::Lazy;
|
||||||
use prisma_client_rust::QueryError;
|
use prisma_client_rust::QueryError;
|
||||||
use tracing::error;
|
use tracing::error;
|
||||||
|
|
||||||
|
use super::from_slice_option_to_option;
|
||||||
|
|
||||||
pub static AVAILABLE_EXTENSIONS: Lazy<Vec<Extension>> = Lazy::new(|| {
|
pub static AVAILABLE_EXTENSIONS: Lazy<Vec<Extension>> = Lazy::new(|| {
|
||||||
ALL_AUDIO_EXTENSIONS
|
ALL_AUDIO_EXTENSIONS
|
||||||
.iter()
|
.iter()
|
||||||
|
@ -44,6 +51,7 @@ pub static AVAILABLE_EXTENSIONS: Lazy<Vec<Extension>> = Lazy::new(|| {
|
||||||
.collect()
|
.collect()
|
||||||
});
|
});
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
pub const fn can_extract_for_audio(audio_extension: AudioExtension) -> bool {
|
pub const fn can_extract_for_audio(audio_extension: AudioExtension) -> bool {
|
||||||
use AudioExtension::{
|
use AudioExtension::{
|
||||||
Aac, Adts, Aif, Aiff, Amr, Aptx, Ast, Caf, Flac, Loas, M4a, Mid, Mp2, Mp3, Oga, Ogg, Opus,
|
Aac, Adts, Aif, Aiff, Amr, Aptx, Ast, Caf, Flac, Loas, M4a, Mid, Mp2, Mp3, Oga, Ogg, Opus,
|
||||||
|
@ -63,34 +71,35 @@ pub const fn can_extract_for_audio(audio_extension: AudioExtension) -> bool {
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
pub const fn can_extract_for_video(video_extension: VideoExtension) -> bool {
|
pub const fn can_extract_for_video(video_extension: VideoExtension) -> bool {
|
||||||
use VideoExtension::{
|
use VideoExtension::{
|
||||||
Asf, Avi, Avifs, F4v, Flv, Hevc, M2ts, M2v, M4v, Mjpeg, Mkv, Mov, Mp4, Mpe, Mpeg, Mpg, Mts,
|
Asf, Avi, Avifs, F4v, Flv, Hevc, M2ts, M2v, M4v, Mjpeg, Mkv, Mov, Mp4, Mpe, Mpeg, Mpg, Mxf,
|
||||||
Mxf, Ogv, Qt, Swf, Ts, Vob, Webm, Wm, Wmv, Wtv, _3gp,
|
Ogv, Qt, Swf, Vob, Webm, Wm, Wmv, Wtv, _3gp,
|
||||||
};
|
};
|
||||||
|
|
||||||
matches!(
|
matches!(
|
||||||
video_extension,
|
video_extension,
|
||||||
Avi | Avifs
|
Avi | Avifs
|
||||||
| Qt | Mov | Swf
|
| Qt | Mov | Swf
|
||||||
| Mjpeg | Ts | Mts
|
| Mjpeg | Mpeg
|
||||||
| Mpeg | Mxf | M2v
|
| Mxf | M2v | Mpg
|
||||||
| Mpg | Mpe | M2ts
|
| Mpe | M2ts | Flv
|
||||||
| Flv | Wm | _3gp
|
| Wm | _3gp | M4v
|
||||||
| M4v | Wmv | Asf
|
| Wmv | Asf | Mp4
|
||||||
| Mp4 | Webm | Mkv
|
| Webm | Mkv | Vob
|
||||||
| Vob | Ogv | Wtv
|
| Ogv | Wtv | Hevc
|
||||||
| Hevc | F4v
|
| F4v // | Ts | Mts TODO: Uncomment when we start using magic instead of extension
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn extract(
|
pub async fn extract(
|
||||||
path: impl AsRef<Path> + Send,
|
path: impl AsRef<Path> + Send,
|
||||||
) -> Result<FFmpegMetadata, media_processor::NonCriticalError> {
|
) -> Result<FFmpegMetadata, media_processor::NonCriticalMediaProcessorError> {
|
||||||
let path = path.as_ref();
|
let path = path.as_ref();
|
||||||
|
|
||||||
FFmpegMetadata::from_path(&path).await.map_err(|e| {
|
FFmpegMetadata::from_path(&path).await.map_err(|e| {
|
||||||
media_data_extractor::NonCriticalError::FailedToExtractImageMediaData(
|
media_data_extractor::NonCriticalMediaDataExtractorError::FailedToExtractImageMediaData(
|
||||||
path.to_path_buf(),
|
path.to_path_buf(),
|
||||||
e.to_string(),
|
e.to_string(),
|
||||||
)
|
)
|
||||||
|
@ -101,7 +110,7 @@ pub async fn extract(
|
||||||
pub async fn save(
|
pub async fn save(
|
||||||
ffmpeg_datas: impl IntoIterator<Item = (FFmpegMetadata, object::id::Type)> + Send,
|
ffmpeg_datas: impl IntoIterator<Item = (FFmpegMetadata, object::id::Type)> + Send,
|
||||||
db: &PrismaClient,
|
db: &PrismaClient,
|
||||||
) -> Result<u64, media_processor::Error> {
|
) -> Result<u64, QueryError> {
|
||||||
ffmpeg_datas
|
ffmpeg_datas
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.map(
|
.map(
|
||||||
|
@ -180,9 +189,9 @@ async fn create_ffmpeg_data(
|
||||||
)),
|
)),
|
||||||
ffmpeg_data::metadata::set(
|
ffmpeg_data::metadata::set(
|
||||||
serde_json::to_vec(&metadata)
|
serde_json::to_vec(&metadata)
|
||||||
.map_err(|err| {
|
.map_err(|e| {
|
||||||
error!("Error reading FFmpegData metadata: {err:#?}");
|
error!(?e, "Error reading FFmpegData metadata;");
|
||||||
err
|
e
|
||||||
})
|
})
|
||||||
.ok(),
|
.ok(),
|
||||||
),
|
),
|
||||||
|
@ -224,9 +233,9 @@ async fn create_ffmpeg_chapters(
|
||||||
ffmpeg_data_id,
|
ffmpeg_data_id,
|
||||||
_params: vec![ffmpeg_media_chapter::metadata::set(
|
_params: vec![ffmpeg_media_chapter::metadata::set(
|
||||||
serde_json::to_vec(&metadata)
|
serde_json::to_vec(&metadata)
|
||||||
.map_err(|err| {
|
.map_err(|e| {
|
||||||
error!("Error reading FFmpegMediaChapter metadata: {err:#?}");
|
error!(?e, "Error reading FFmpegMediaChapter metadata;");
|
||||||
err
|
e
|
||||||
})
|
})
|
||||||
.ok(),
|
.ok(),
|
||||||
)],
|
)],
|
||||||
|
@ -244,37 +253,36 @@ async fn create_ffmpeg_programs(
|
||||||
programs: Vec<Program>,
|
programs: Vec<Program>,
|
||||||
db: &PrismaClient,
|
db: &PrismaClient,
|
||||||
) -> Result<Vec<(ffmpeg_media_program::program_id::Type, Vec<Stream>)>, QueryError> {
|
) -> Result<Vec<(ffmpeg_media_program::program_id::Type, Vec<Stream>)>, QueryError> {
|
||||||
let (creates, streams_by_program_id) =
|
let (creates, streams_by_program_id) = programs
|
||||||
programs
|
.into_iter()
|
||||||
.into_iter()
|
.map(
|
||||||
.map(
|
|Program {
|
||||||
|Program {
|
id: program_id,
|
||||||
id: program_id,
|
name,
|
||||||
name,
|
metadata,
|
||||||
metadata,
|
streams,
|
||||||
streams,
|
}| {
|
||||||
}| {
|
(
|
||||||
(
|
ffmpeg_media_program::CreateUnchecked {
|
||||||
ffmpeg_media_program::CreateUnchecked {
|
program_id,
|
||||||
program_id,
|
ffmpeg_data_id: data_id,
|
||||||
ffmpeg_data_id: data_id,
|
_params: vec![
|
||||||
_params: vec![
|
ffmpeg_media_program::name::set(name),
|
||||||
ffmpeg_media_program::name::set(name),
|
ffmpeg_media_program::metadata::set(
|
||||||
ffmpeg_media_program::metadata::set(
|
serde_json::to_vec(&metadata)
|
||||||
serde_json::to_vec(&metadata)
|
.map_err(|e| {
|
||||||
.map_err(|err| {
|
error!(?e, "Error reading FFmpegMediaProgram metadata;");
|
||||||
error!("Error reading FFmpegMediaProgram metadata: {err:#?}");
|
e
|
||||||
err
|
})
|
||||||
})
|
.ok(),
|
||||||
.ok(),
|
),
|
||||||
),
|
],
|
||||||
],
|
},
|
||||||
},
|
(program_id, streams),
|
||||||
(program_id, streams),
|
)
|
||||||
)
|
},
|
||||||
},
|
)
|
||||||
)
|
.unzip::<_, _, Vec<_>, Vec<_>>();
|
||||||
.unzip::<_, _, Vec<_>, Vec<_>>();
|
|
||||||
|
|
||||||
db.ffmpeg_media_program()
|
db.ffmpeg_media_program()
|
||||||
.create_many(creates)
|
.create_many(creates)
|
||||||
|
@ -333,9 +341,9 @@ async fn create_ffmpeg_streams(
|
||||||
ffmpeg_media_stream::language::set(metadata.language.clone()),
|
ffmpeg_media_stream::language::set(metadata.language.clone()),
|
||||||
ffmpeg_media_stream::metadata::set(
|
ffmpeg_media_stream::metadata::set(
|
||||||
serde_json::to_vec(&metadata)
|
serde_json::to_vec(&metadata)
|
||||||
.map_err(|err| {
|
.map_err(|e| {
|
||||||
error!("Error reading FFmpegMediaStream metadata: {err:#?}");
|
error!(?e, "Error reading FFmpegMediaStream metadata;");
|
||||||
err
|
e
|
||||||
})
|
})
|
||||||
.ok(),
|
.ok(),
|
||||||
),
|
),
|
||||||
|
@ -570,3 +578,207 @@ async fn create_ffmpeg_video_props(
|
||||||
.await
|
.await
|
||||||
.map(|_| ())
|
.map(|_| ())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn from_prisma_data(
|
||||||
|
object_with_media_data::ffmpeg_data::Data {
|
||||||
|
formats,
|
||||||
|
duration,
|
||||||
|
start_time,
|
||||||
|
bit_rate,
|
||||||
|
metadata,
|
||||||
|
chapters,
|
||||||
|
programs,
|
||||||
|
..
|
||||||
|
}: object_with_media_data::ffmpeg_data::Data,
|
||||||
|
) -> FFmpegMetadata {
|
||||||
|
FFmpegMetadata {
|
||||||
|
formats: formats.split(',').map(String::from).collect::<Vec<_>>(),
|
||||||
|
duration: duration.map(|duration| i64_to_frontend(ffmpeg_data_field_from_db(&duration))),
|
||||||
|
start_time: start_time
|
||||||
|
.map(|start_time| i64_to_frontend(ffmpeg_data_field_from_db(&start_time))),
|
||||||
|
bit_rate: i64_to_frontend(ffmpeg_data_field_from_db(&bit_rate)),
|
||||||
|
chapters: chapters_from_prisma_data(chapters),
|
||||||
|
programs: programs_from_prisma_data(programs),
|
||||||
|
metadata: from_slice_option_to_option(metadata).unwrap_or_default(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[inline]
|
||||||
|
fn chapters_from_prisma_data(chapters: Vec<ffmpeg_media_chapter::Data>) -> Vec<Chapter> {
|
||||||
|
chapters
|
||||||
|
.into_iter()
|
||||||
|
.map(
|
||||||
|
|ffmpeg_media_chapter::Data {
|
||||||
|
chapter_id,
|
||||||
|
start,
|
||||||
|
end,
|
||||||
|
time_base_den,
|
||||||
|
time_base_num,
|
||||||
|
metadata,
|
||||||
|
..
|
||||||
|
}| Chapter {
|
||||||
|
id: chapter_id,
|
||||||
|
start: i64_to_frontend(ffmpeg_data_field_from_db(&start)),
|
||||||
|
end: i64_to_frontend(ffmpeg_data_field_from_db(&end)),
|
||||||
|
time_base_den,
|
||||||
|
time_base_num,
|
||||||
|
metadata: from_slice_option_to_option(metadata).unwrap_or_default(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[inline]
|
||||||
|
fn programs_from_prisma_data(
|
||||||
|
programs: Vec<object_with_media_data::ffmpeg_data::programs::Data>,
|
||||||
|
) -> Vec<Program> {
|
||||||
|
programs
|
||||||
|
.into_iter()
|
||||||
|
.map(
|
||||||
|
|object_with_media_data::ffmpeg_data::programs::Data {
|
||||||
|
program_id,
|
||||||
|
name,
|
||||||
|
metadata,
|
||||||
|
streams,
|
||||||
|
..
|
||||||
|
}| Program {
|
||||||
|
id: program_id,
|
||||||
|
name,
|
||||||
|
streams: streams_from_prisma_data(streams),
|
||||||
|
metadata: from_slice_option_to_option(metadata).unwrap_or_default(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn streams_from_prisma_data(
|
||||||
|
streams: Vec<object_with_media_data::ffmpeg_data::programs::streams::Data>,
|
||||||
|
) -> Vec<Stream> {
|
||||||
|
streams
|
||||||
|
.into_iter()
|
||||||
|
.map(
|
||||||
|
|object_with_media_data::ffmpeg_data::programs::streams::Data {
|
||||||
|
stream_id,
|
||||||
|
name,
|
||||||
|
aspect_ratio_num,
|
||||||
|
aspect_ratio_den,
|
||||||
|
frames_per_second_num,
|
||||||
|
frames_per_second_den,
|
||||||
|
time_base_real_den,
|
||||||
|
time_base_real_num,
|
||||||
|
dispositions,
|
||||||
|
metadata,
|
||||||
|
codec,
|
||||||
|
..
|
||||||
|
}| {
|
||||||
|
Stream {
|
||||||
|
id: stream_id,
|
||||||
|
name,
|
||||||
|
codec: codec_from_prisma_data(codec),
|
||||||
|
aspect_ratio_num,
|
||||||
|
aspect_ratio_den,
|
||||||
|
frames_per_second_num,
|
||||||
|
frames_per_second_den,
|
||||||
|
time_base_real_den,
|
||||||
|
time_base_real_num,
|
||||||
|
dispositions: dispositions
|
||||||
|
.map(|dispositions| {
|
||||||
|
dispositions
|
||||||
|
.split(',')
|
||||||
|
.map(String::from)
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
})
|
||||||
|
.unwrap_or_default(),
|
||||||
|
metadata: from_slice_option_to_option(metadata).unwrap_or_default(),
|
||||||
|
}
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn codec_from_prisma_data(
|
||||||
|
codec: Option<object_with_media_data::ffmpeg_data::programs::streams::codec::Data>,
|
||||||
|
) -> Option<Codec> {
|
||||||
|
codec.map(
|
||||||
|
|object_with_media_data::ffmpeg_data::programs::streams::codec::Data {
|
||||||
|
kind,
|
||||||
|
sub_kind,
|
||||||
|
tag,
|
||||||
|
name,
|
||||||
|
profile,
|
||||||
|
bit_rate,
|
||||||
|
audio_props,
|
||||||
|
video_props,
|
||||||
|
..
|
||||||
|
}| Codec {
|
||||||
|
kind,
|
||||||
|
sub_kind,
|
||||||
|
tag,
|
||||||
|
name,
|
||||||
|
profile,
|
||||||
|
bit_rate,
|
||||||
|
props: match (audio_props, video_props) {
|
||||||
|
(
|
||||||
|
Some(ffmpeg_media_audio_props::Data {
|
||||||
|
delay,
|
||||||
|
padding,
|
||||||
|
sample_rate,
|
||||||
|
sample_format,
|
||||||
|
bit_per_sample,
|
||||||
|
channel_layout,
|
||||||
|
..
|
||||||
|
}),
|
||||||
|
None,
|
||||||
|
) => Some(Props::Audio(AudioProps {
|
||||||
|
delay,
|
||||||
|
padding,
|
||||||
|
sample_rate,
|
||||||
|
sample_format,
|
||||||
|
bit_per_sample,
|
||||||
|
channel_layout,
|
||||||
|
})),
|
||||||
|
(
|
||||||
|
None,
|
||||||
|
Some(ffmpeg_media_video_props::Data {
|
||||||
|
pixel_format,
|
||||||
|
color_range,
|
||||||
|
bits_per_channel,
|
||||||
|
color_space,
|
||||||
|
color_primaries,
|
||||||
|
color_transfer,
|
||||||
|
field_order,
|
||||||
|
chroma_location,
|
||||||
|
width,
|
||||||
|
height,
|
||||||
|
aspect_ratio_num,
|
||||||
|
aspect_ratio_den,
|
||||||
|
properties,
|
||||||
|
..
|
||||||
|
}),
|
||||||
|
) => Some(Props::Video(VideoProps {
|
||||||
|
pixel_format,
|
||||||
|
color_range,
|
||||||
|
bits_per_channel,
|
||||||
|
color_space,
|
||||||
|
color_primaries,
|
||||||
|
color_transfer,
|
||||||
|
field_order,
|
||||||
|
chroma_location,
|
||||||
|
width,
|
||||||
|
height,
|
||||||
|
aspect_ratio_num,
|
||||||
|
aspect_ratio_den,
|
||||||
|
properties: properties
|
||||||
|
.map(|dispositions| {
|
||||||
|
dispositions
|
||||||
|
.split(',')
|
||||||
|
.map(String::from)
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
})
|
||||||
|
.unwrap_or_default(),
|
||||||
|
})),
|
||||||
|
_ => None,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
|
@ -1,3 +1,12 @@
|
||||||
pub mod exif_media_data;
|
pub mod exif_media_data;
|
||||||
pub mod ffmpeg_media_data;
|
pub mod ffmpeg_media_data;
|
||||||
pub mod thumbnailer;
|
pub mod thumbnailer;
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
|
fn from_slice_option_to_option<T: serde::Serialize + serde::de::DeserializeOwned>(
|
||||||
|
value: Option<Vec<u8>>,
|
||||||
|
) -> Option<T> {
|
||||||
|
value
|
||||||
|
.map(|x| serde_json::from_slice(&x).ok())
|
||||||
|
.unwrap_or_default()
|
||||||
|
}
|
||||||
|
|
|
@ -1,16 +1,37 @@
|
||||||
use once_cell::sync::Lazy;
|
use crate::media_processor::thumbnailer;
|
||||||
|
|
||||||
|
use sd_core_prisma_helpers::CasId;
|
||||||
|
|
||||||
use sd_file_ext::extensions::{
|
use sd_file_ext::extensions::{
|
||||||
DocumentExtension, Extension, ImageExtension, ALL_DOCUMENT_EXTENSIONS, ALL_IMAGE_EXTENSIONS,
|
DocumentExtension, Extension, ImageExtension, ALL_DOCUMENT_EXTENSIONS, ALL_IMAGE_EXTENSIONS,
|
||||||
};
|
};
|
||||||
|
use sd_images::{format_image, scale_dimensions, ConvertibleExtension};
|
||||||
|
use sd_media_metadata::exif::Orientation;
|
||||||
|
use sd_utils::error::FileIOError;
|
||||||
|
|
||||||
#[cfg(feature = "ffmpeg")]
|
#[cfg(feature = "ffmpeg")]
|
||||||
use sd_file_ext::extensions::{VideoExtension, ALL_VIDEO_EXTENSIONS};
|
use sd_file_ext::extensions::{VideoExtension, ALL_VIDEO_EXTENSIONS};
|
||||||
|
|
||||||
use std::time::Duration;
|
use std::{
|
||||||
|
ops::Deref,
|
||||||
|
path::{Path, PathBuf},
|
||||||
|
str::FromStr,
|
||||||
|
time::Duration,
|
||||||
|
};
|
||||||
|
|
||||||
|
use image::{imageops, DynamicImage, GenericImageView};
|
||||||
|
use once_cell::sync::Lazy;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use specta::Type;
|
use specta::Type;
|
||||||
|
use tokio::{
|
||||||
|
fs, io,
|
||||||
|
sync::{oneshot, Mutex},
|
||||||
|
task::spawn_blocking,
|
||||||
|
time::{sleep, Instant},
|
||||||
|
};
|
||||||
|
use tracing::{error, instrument, trace};
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
use webp::Encoder;
|
||||||
|
|
||||||
// Files names constants
|
// Files names constants
|
||||||
pub const THUMBNAIL_CACHE_DIR_NAME: &str = "thumbnails";
|
pub const THUMBNAIL_CACHE_DIR_NAME: &str = "thumbnails";
|
||||||
|
@ -25,8 +46,12 @@ pub const TARGET_PX: f32 = 1_048_576.0; // 1024x1024
|
||||||
/// and is treated as a percentage (so 60% in this case, or it's the same as multiplying by `0.6`).
|
/// and is treated as a percentage (so 60% in this case, or it's the same as multiplying by `0.6`).
|
||||||
pub const TARGET_QUALITY: f32 = 60.0;
|
pub const TARGET_QUALITY: f32 = 60.0;
|
||||||
|
|
||||||
/// How much time we allow for the thumbnail generation process to complete before we give up.
|
/// How much time we allow for the thumbnailer task to complete before we give up.
|
||||||
pub const THUMBNAIL_GENERATION_TIMEOUT: Duration = Duration::from_secs(60);
|
pub const THUMBNAILER_TASK_TIMEOUT: Duration = Duration::from_secs(60 * 5);
|
||||||
|
|
||||||
|
pub fn get_thumbnails_directory(data_directory: impl AsRef<Path>) -> PathBuf {
|
||||||
|
data_directory.as_ref().join(THUMBNAIL_CACHE_DIR_NAME)
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(feature = "ffmpeg")]
|
#[cfg(feature = "ffmpeg")]
|
||||||
pub static THUMBNAILABLE_VIDEO_EXTENSIONS: Lazy<Vec<Extension>> = Lazy::new(|| {
|
pub static THUMBNAILABLE_VIDEO_EXTENSIONS: Lazy<Vec<Extension>> = Lazy::new(|| {
|
||||||
|
@ -68,25 +93,43 @@ pub static ALL_THUMBNAILABLE_EXTENSIONS: Lazy<Vec<Extension>> = Lazy::new(|| {
|
||||||
|
|
||||||
/// This type is used to pass the relevant data to the frontend so it can request the thumbnail.
|
/// This type is used to pass the relevant data to the frontend so it can request the thumbnail.
|
||||||
/// Tt supports extending the shard hex to support deeper directory structures in the future
|
/// Tt supports extending the shard hex to support deeper directory structures in the future
|
||||||
#[derive(Debug, Serialize, Deserialize, Type)]
|
#[derive(Debug, Serialize, Deserialize, Type, Clone)]
|
||||||
pub struct ThumbKey {
|
pub struct ThumbKey {
|
||||||
pub shard_hex: String,
|
pub shard_hex: String,
|
||||||
pub cas_id: String,
|
pub cas_id: CasId<'static>,
|
||||||
pub base_directory_str: String,
|
pub base_directory_str: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ThumbKey {
|
impl ThumbKey {
|
||||||
#[must_use]
|
#[must_use]
|
||||||
pub fn new(cas_id: &str, kind: &ThumbnailKind) -> Self {
|
pub fn new(cas_id: CasId<'static>, kind: &ThumbnailKind) -> Self {
|
||||||
Self {
|
Self {
|
||||||
shard_hex: get_shard_hex(cas_id).to_string(),
|
shard_hex: get_shard_hex(&cas_id).to_string(),
|
||||||
cas_id: cas_id.to_string(),
|
cas_id,
|
||||||
base_directory_str: match kind {
|
base_directory_str: match kind {
|
||||||
ThumbnailKind::Ephemeral => String::from(EPHEMERAL_DIR),
|
ThumbnailKind::Ephemeral => String::from(EPHEMERAL_DIR),
|
||||||
ThumbnailKind::Indexed(library_id) => library_id.to_string(),
|
ThumbnailKind::Indexed(library_id) => library_id.to_string(),
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
|
pub fn new_indexed(cas_id: CasId<'static>, library_id: Uuid) -> Self {
|
||||||
|
Self {
|
||||||
|
shard_hex: get_shard_hex(&cas_id).to_string(),
|
||||||
|
cas_id,
|
||||||
|
base_directory_str: library_id.to_string(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
|
pub fn new_ephemeral(cas_id: CasId<'static>) -> Self {
|
||||||
|
Self {
|
||||||
|
shard_hex: get_shard_hex(&cas_id).to_string(),
|
||||||
|
cas_id,
|
||||||
|
base_directory_str: String::from(EPHEMERAL_DIR),
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Serialize, Deserialize, Type, Clone, Copy)]
|
#[derive(Debug, Serialize, Deserialize, Type, Clone, Copy)]
|
||||||
|
@ -95,6 +138,41 @@ pub enum ThumbnailKind {
|
||||||
Indexed(Uuid),
|
Indexed(Uuid),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl ThumbnailKind {
|
||||||
|
pub fn compute_path(&self, data_directory: impl AsRef<Path>, cas_id: &CasId<'_>) -> PathBuf {
|
||||||
|
let mut thumb_path = get_thumbnails_directory(data_directory);
|
||||||
|
match self {
|
||||||
|
Self::Ephemeral => thumb_path.push(EPHEMERAL_DIR),
|
||||||
|
Self::Indexed(library_id) => {
|
||||||
|
thumb_path.push(library_id.to_string());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
thumb_path.push(get_shard_hex(cas_id));
|
||||||
|
thumb_path.push(cas_id.as_str());
|
||||||
|
thumb_path.set_extension(WEBP_EXTENSION);
|
||||||
|
|
||||||
|
thumb_path
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize, Deserialize)]
|
||||||
|
pub struct GenerateThumbnailArgs<'cas_id> {
|
||||||
|
pub extension: String,
|
||||||
|
pub cas_id: CasId<'cas_id>,
|
||||||
|
pub path: PathBuf,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<'cas_id> GenerateThumbnailArgs<'cas_id> {
|
||||||
|
#[must_use]
|
||||||
|
pub const fn new(extension: String, cas_id: CasId<'cas_id>, path: PathBuf) -> Self {
|
||||||
|
Self {
|
||||||
|
extension,
|
||||||
|
cas_id,
|
||||||
|
path,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// The practice of dividing files into hex coded folders, often called "sharding,"
|
/// The practice of dividing files into hex coded folders, often called "sharding,"
|
||||||
/// is mainly used to optimize file system performance. File systems can start to slow down
|
/// is mainly used to optimize file system performance. File systems can start to slow down
|
||||||
/// as the number of files in a directory increases. Thus, it's often beneficial to split
|
/// as the number of files in a directory increases. Thus, it's often beneficial to split
|
||||||
|
@ -105,18 +183,21 @@ pub enum ThumbnailKind {
|
||||||
/// three characters of a the hash, this will give us 4096 (16^3) possible directories,
|
/// three characters of a the hash, this will give us 4096 (16^3) possible directories,
|
||||||
/// named 000 to fff.
|
/// named 000 to fff.
|
||||||
#[inline]
|
#[inline]
|
||||||
pub fn get_shard_hex(cas_id: &str) -> &str {
|
#[must_use]
|
||||||
|
pub fn get_shard_hex<'cas_id>(cas_id: &'cas_id CasId<'cas_id>) -> &'cas_id str {
|
||||||
// Use the first three characters of the hash as the directory name
|
// Use the first three characters of the hash as the directory name
|
||||||
&cas_id[0..3]
|
&cas_id.as_str()[0..3]
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(feature = "ffmpeg")]
|
#[cfg(feature = "ffmpeg")]
|
||||||
|
#[must_use]
|
||||||
pub const fn can_generate_thumbnail_for_video(video_extension: VideoExtension) -> bool {
|
pub const fn can_generate_thumbnail_for_video(video_extension: VideoExtension) -> bool {
|
||||||
use VideoExtension::{Hevc, M2ts, M2v, Mpg, Mts, Swf, Ts};
|
use VideoExtension::{Hevc, M2ts, M2v, Mpg, Mts, Swf, Ts};
|
||||||
// File extensions that are specifically not supported by the thumbnailer
|
// File extensions that are specifically not supported by the thumbnailer
|
||||||
!matches!(video_extension, Mpg | Swf | M2v | Hevc | M2ts | Mts | Ts)
|
!matches!(video_extension, Mpg | Swf | M2v | Hevc | M2ts | Mts | Ts)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
pub const fn can_generate_thumbnail_for_image(image_extension: ImageExtension) -> bool {
|
pub const fn can_generate_thumbnail_for_image(image_extension: ImageExtension) -> bool {
|
||||||
use ImageExtension::{
|
use ImageExtension::{
|
||||||
Avif, Bmp, Gif, Heic, Heics, Heif, Heifs, Ico, Jpeg, Jpg, Png, Svg, Webp,
|
Avif, Bmp, Gif, Heic, Heics, Heif, Heifs, Ico, Jpeg, Jpg, Png, Svg, Webp,
|
||||||
|
@ -128,8 +209,291 @@ pub const fn can_generate_thumbnail_for_image(image_extension: ImageExtension) -
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
pub const fn can_generate_thumbnail_for_document(document_extension: DocumentExtension) -> bool {
|
pub const fn can_generate_thumbnail_for_document(document_extension: DocumentExtension) -> bool {
|
||||||
use DocumentExtension::Pdf;
|
use DocumentExtension::Pdf;
|
||||||
|
|
||||||
matches!(document_extension, Pdf)
|
matches!(document_extension, Pdf)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub enum GenerationStatus {
|
||||||
|
Generated,
|
||||||
|
Skipped,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[instrument(skip(thumbnails_directory, cas_id, should_regenerate, kind))]
|
||||||
|
pub async fn generate_thumbnail(
|
||||||
|
thumbnails_directory: &Path,
|
||||||
|
GenerateThumbnailArgs {
|
||||||
|
extension,
|
||||||
|
cas_id,
|
||||||
|
path,
|
||||||
|
}: &GenerateThumbnailArgs<'_>,
|
||||||
|
kind: &ThumbnailKind,
|
||||||
|
should_regenerate: bool,
|
||||||
|
) -> (
|
||||||
|
Duration,
|
||||||
|
Result<(ThumbKey, GenerationStatus), thumbnailer::NonCriticalThumbnailerError>,
|
||||||
|
) {
|
||||||
|
trace!("Generating thumbnail");
|
||||||
|
let start = Instant::now();
|
||||||
|
|
||||||
|
let mut output_path = match kind {
|
||||||
|
ThumbnailKind::Ephemeral => thumbnails_directory.join(EPHEMERAL_DIR),
|
||||||
|
ThumbnailKind::Indexed(library_id) => thumbnails_directory.join(library_id.to_string()),
|
||||||
|
};
|
||||||
|
|
||||||
|
output_path.push(get_shard_hex(cas_id));
|
||||||
|
output_path.push(cas_id.as_str());
|
||||||
|
output_path.set_extension(WEBP_EXTENSION);
|
||||||
|
|
||||||
|
if let Err(e) = fs::metadata(&*output_path).await {
|
||||||
|
if e.kind() != io::ErrorKind::NotFound {
|
||||||
|
error!(
|
||||||
|
?e,
|
||||||
|
"Failed to check if thumbnail exists, but we will try to generate it anyway;"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
// Otherwise we good, thumbnail doesn't exist so we can generate it
|
||||||
|
} else if !should_regenerate {
|
||||||
|
trace!("Skipping thumbnail generation because it already exists");
|
||||||
|
return (
|
||||||
|
start.elapsed(),
|
||||||
|
Ok((
|
||||||
|
ThumbKey::new(cas_id.to_owned(), kind),
|
||||||
|
GenerationStatus::Skipped,
|
||||||
|
)),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Ok(extension) = ImageExtension::from_str(extension) {
|
||||||
|
if can_generate_thumbnail_for_image(extension) {
|
||||||
|
trace!("Generating image thumbnail");
|
||||||
|
if let Err(e) = generate_image_thumbnail(&path, &output_path).await {
|
||||||
|
return (start.elapsed(), Err(e));
|
||||||
|
}
|
||||||
|
trace!("Generated image thumbnail");
|
||||||
|
}
|
||||||
|
} else if let Ok(extension) = DocumentExtension::from_str(extension) {
|
||||||
|
if can_generate_thumbnail_for_document(extension) {
|
||||||
|
trace!("Generating document thumbnail");
|
||||||
|
if let Err(e) = generate_image_thumbnail(&path, &output_path).await {
|
||||||
|
return (start.elapsed(), Err(e));
|
||||||
|
}
|
||||||
|
trace!("Generating document thumbnail");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "ffmpeg")]
|
||||||
|
{
|
||||||
|
use crate::media_processor::helpers::thumbnailer::can_generate_thumbnail_for_video;
|
||||||
|
use sd_file_ext::extensions::VideoExtension;
|
||||||
|
|
||||||
|
if let Ok(extension) = VideoExtension::from_str(extension) {
|
||||||
|
if can_generate_thumbnail_for_video(extension) {
|
||||||
|
trace!("Generating video thumbnail");
|
||||||
|
if let Err(e) = generate_video_thumbnail(&path, &output_path).await {
|
||||||
|
return (start.elapsed(), Err(e));
|
||||||
|
}
|
||||||
|
trace!("Generated video thumbnail");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
trace!("Generated thumbnail");
|
||||||
|
|
||||||
|
(
|
||||||
|
start.elapsed(),
|
||||||
|
Ok((
|
||||||
|
ThumbKey::new(cas_id.to_owned(), kind),
|
||||||
|
GenerationStatus::Generated,
|
||||||
|
)),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn inner_generate_image_thumbnail(
|
||||||
|
file_path: PathBuf,
|
||||||
|
) -> Result<Vec<u8>, thumbnailer::NonCriticalThumbnailerError> {
|
||||||
|
let mut img = format_image(&file_path).map_err(|e| {
|
||||||
|
thumbnailer::NonCriticalThumbnailerError::FormatImage(file_path.clone(), e.to_string())
|
||||||
|
})?;
|
||||||
|
|
||||||
|
let (w, h) = img.dimensions();
|
||||||
|
|
||||||
|
#[allow(clippy::cast_precision_loss)]
|
||||||
|
let (w_scaled, h_scaled) = scale_dimensions(w as f32, h as f32, TARGET_PX);
|
||||||
|
|
||||||
|
// Optionally, resize the existing photo and convert back into DynamicImage
|
||||||
|
if w != w_scaled && h != h_scaled {
|
||||||
|
img = DynamicImage::ImageRgba8(imageops::resize(
|
||||||
|
&img,
|
||||||
|
w_scaled,
|
||||||
|
h_scaled,
|
||||||
|
imageops::FilterType::Triangle,
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
// this corrects the rotation/flip of the image based on the *available* exif data
|
||||||
|
// not all images have exif data, so we don't error. we also don't rotate HEIF as that's against the spec
|
||||||
|
if let Some(orientation) = Orientation::from_path(&file_path) {
|
||||||
|
if ConvertibleExtension::try_from(file_path.as_ref())
|
||||||
|
.expect("we already checked if the image was convertible")
|
||||||
|
.should_rotate()
|
||||||
|
{
|
||||||
|
img = orientation.correct_thumbnail(img);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create the WebP encoder for the above image
|
||||||
|
let encoder = Encoder::from_image(&img).map_err(|reason| {
|
||||||
|
thumbnailer::NonCriticalThumbnailerError::WebPEncoding(file_path, reason.to_string())
|
||||||
|
})?;
|
||||||
|
|
||||||
|
// Type `WebPMemory` is !Send, which makes the `Future` in this function `!Send`,
|
||||||
|
// this make us `deref` to have a `&[u8]` and then `to_owned` to make a `Vec<u8>`
|
||||||
|
// which implies on a unwanted clone...
|
||||||
|
Ok(encoder.encode(TARGET_QUALITY).deref().to_owned())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[instrument(
|
||||||
|
skip_all,
|
||||||
|
fields(
|
||||||
|
input_path = %file_path.as_ref().display(),
|
||||||
|
output_path = %output_path.as_ref().display()
|
||||||
|
)
|
||||||
|
)]
|
||||||
|
async fn generate_image_thumbnail(
|
||||||
|
file_path: impl AsRef<Path> + Send,
|
||||||
|
output_path: impl AsRef<Path> + Send,
|
||||||
|
) -> Result<(), thumbnailer::NonCriticalThumbnailerError> {
|
||||||
|
let file_path = file_path.as_ref().to_path_buf();
|
||||||
|
|
||||||
|
let (tx, rx) = oneshot::channel();
|
||||||
|
|
||||||
|
// Using channel instead of waiting the JoinHandle as for some reason
|
||||||
|
// the JoinHandle can take some extra time to complete
|
||||||
|
let handle = spawn_blocking({
|
||||||
|
let file_path = file_path.clone();
|
||||||
|
|
||||||
|
move || {
|
||||||
|
// Handling error on receiver side
|
||||||
|
let _ = tx.send(inner_generate_image_thumbnail(file_path));
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
let webp = if let Ok(res) = rx.await {
|
||||||
|
res?
|
||||||
|
} else {
|
||||||
|
error!("Failed to generate thumbnail");
|
||||||
|
return Err(
|
||||||
|
thumbnailer::NonCriticalThumbnailerError::PanicWhileGeneratingThumbnail(
|
||||||
|
file_path,
|
||||||
|
handle
|
||||||
|
.await
|
||||||
|
.expect_err("as the channel was closed, then the spawned task panicked")
|
||||||
|
.to_string(),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
|
trace!("Generated thumbnail bytes");
|
||||||
|
|
||||||
|
let output_path = output_path.as_ref();
|
||||||
|
|
||||||
|
if let Some(shard_dir) = output_path.parent() {
|
||||||
|
fs::create_dir_all(shard_dir).await.map_err(|e| {
|
||||||
|
thumbnailer::NonCriticalThumbnailerError::CreateShardDirectory(
|
||||||
|
FileIOError::from((shard_dir, e)).to_string(),
|
||||||
|
)
|
||||||
|
})?;
|
||||||
|
} else {
|
||||||
|
error!("Failed to get parent directory for sharding parent directory");
|
||||||
|
}
|
||||||
|
|
||||||
|
trace!("Created shard directory and writing it to disk");
|
||||||
|
|
||||||
|
let res = fs::write(output_path, &webp).await.map_err(|e| {
|
||||||
|
thumbnailer::NonCriticalThumbnailerError::SaveThumbnail(
|
||||||
|
file_path,
|
||||||
|
FileIOError::from((output_path, e)).to_string(),
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
trace!("Wrote thumbnail to disk");
|
||||||
|
res
|
||||||
|
}
|
||||||
|
|
||||||
|
#[instrument(
|
||||||
|
skip_all,
|
||||||
|
fields(
|
||||||
|
input_path = %file_path.as_ref().display(),
|
||||||
|
output_path = %output_path.as_ref().display()
|
||||||
|
)
|
||||||
|
)]
|
||||||
|
#[cfg(feature = "ffmpeg")]
|
||||||
|
async fn generate_video_thumbnail(
|
||||||
|
file_path: impl AsRef<Path> + Send,
|
||||||
|
output_path: impl AsRef<Path> + Send,
|
||||||
|
) -> Result<(), thumbnailer::NonCriticalThumbnailerError> {
|
||||||
|
use sd_ffmpeg::{to_thumbnail, ThumbnailSize};
|
||||||
|
|
||||||
|
let file_path = file_path.as_ref();
|
||||||
|
|
||||||
|
to_thumbnail(
|
||||||
|
file_path,
|
||||||
|
output_path,
|
||||||
|
ThumbnailSize::Scale(1024),
|
||||||
|
TARGET_QUALITY,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|e| {
|
||||||
|
thumbnailer::NonCriticalThumbnailerError::VideoThumbnailGenerationFailed(
|
||||||
|
file_path.to_path_buf(),
|
||||||
|
e.to_string(),
|
||||||
|
)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
const HALF_SEC: Duration = Duration::from_millis(500);
|
||||||
|
static LAST_SINGLE_THUMB_GENERATED_LOCK: Lazy<Mutex<Instant>> =
|
||||||
|
Lazy::new(|| Mutex::new(Instant::now()));
|
||||||
|
|
||||||
|
/// WARNING!!!! DON'T USE THIS FUNCTION IN A LOOP!!!!!!!!!!!!! It will be pretty slow on purpose!
|
||||||
|
pub async fn generate_single_thumbnail(
|
||||||
|
thumbnails_directory: impl AsRef<Path> + Send,
|
||||||
|
extension: String,
|
||||||
|
cas_id: CasId<'static>,
|
||||||
|
path: impl AsRef<Path> + Send,
|
||||||
|
kind: ThumbnailKind,
|
||||||
|
) -> Result<(), thumbnailer::NonCriticalThumbnailerError> {
|
||||||
|
let mut last_single_thumb_generated_guard = LAST_SINGLE_THUMB_GENERATED_LOCK.lock().await;
|
||||||
|
|
||||||
|
let elapsed = Instant::now() - *last_single_thumb_generated_guard;
|
||||||
|
if elapsed < HALF_SEC {
|
||||||
|
// This will choke up in case someone try to use this method in a loop, otherwise
|
||||||
|
// it will consume all the machine resources like a gluton monster from hell
|
||||||
|
sleep(HALF_SEC - elapsed).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
let (_duration, res) = generate_thumbnail(
|
||||||
|
thumbnails_directory.as_ref(),
|
||||||
|
&GenerateThumbnailArgs {
|
||||||
|
extension,
|
||||||
|
cas_id,
|
||||||
|
path: path.as_ref().to_path_buf(),
|
||||||
|
},
|
||||||
|
&kind,
|
||||||
|
false,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
let (_thumb_key, status) = res?;
|
||||||
|
|
||||||
|
if matches!(status, GenerationStatus::Generated) {
|
||||||
|
*last_single_thumb_generated_guard = Instant::now();
|
||||||
|
drop(last_single_thumb_generated_guard); // Clippy was weirdly complaining about not doing an "early" drop here
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
File diff suppressed because it is too large
Load diff
|
@ -1,11 +1,15 @@
|
||||||
use crate::{utils::sub_path, OuterContext, UpdateEvent};
|
use crate::{utils::sub_path, OuterContext, UpdateEvent};
|
||||||
|
|
||||||
use sd_core_file_path_helper::FilePathError;
|
use sd_core_file_path_helper::{FilePathError, IsolatedFilePathData};
|
||||||
|
use sd_core_prisma_helpers::file_path_for_media_processor;
|
||||||
|
|
||||||
|
use sd_file_ext::extensions::Extension;
|
||||||
|
use sd_prisma::prisma::{file_path, object, PrismaClient};
|
||||||
use sd_utils::db::MissingFieldError;
|
use sd_utils::db::MissingFieldError;
|
||||||
|
|
||||||
use std::fmt;
|
use std::{collections::HashMap, fmt};
|
||||||
|
|
||||||
|
use prisma_client_rust::{raw, PrismaValue};
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use specta::Type;
|
use specta::Type;
|
||||||
|
|
||||||
|
@ -19,10 +23,22 @@ pub use tasks::{
|
||||||
thumbnailer::{self, Thumbnailer},
|
thumbnailer::{self, Thumbnailer},
|
||||||
};
|
};
|
||||||
|
|
||||||
pub use helpers::thumbnailer::{ThumbKey, ThumbnailKind};
|
pub use helpers::{
|
||||||
|
exif_media_data, ffmpeg_media_data,
|
||||||
|
thumbnailer::{
|
||||||
|
can_generate_thumbnail_for_document, can_generate_thumbnail_for_image,
|
||||||
|
generate_single_thumbnail, get_shard_hex, get_thumbnails_directory, GenerateThumbnailArgs,
|
||||||
|
ThumbKey, ThumbnailKind, WEBP_EXTENSION,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
#[cfg(feature = "ffmpeg")]
|
||||||
|
pub use helpers::thumbnailer::can_generate_thumbnail_for_video;
|
||||||
|
|
||||||
pub use shallow::shallow;
|
pub use shallow::shallow;
|
||||||
|
|
||||||
use self::thumbnailer::NewThumbnailReporter;
|
use media_data_extractor::NonCriticalMediaDataExtractorError;
|
||||||
|
use thumbnailer::{NewThumbnailReporter, NonCriticalThumbnailerError};
|
||||||
|
|
||||||
const BATCH_SIZE: usize = 10;
|
const BATCH_SIZE: usize = 10;
|
||||||
|
|
||||||
|
@ -43,31 +59,126 @@ pub enum Error {
|
||||||
|
|
||||||
impl From<Error> for rspc::Error {
|
impl From<Error> for rspc::Error {
|
||||||
fn from(e: Error) -> Self {
|
fn from(e: Error) -> Self {
|
||||||
Self::with_cause(rspc::ErrorCode::InternalServerError, e.to_string(), e)
|
match e {
|
||||||
|
Error::SubPath(sub_path_err) => sub_path_err.into(),
|
||||||
|
|
||||||
|
_ => Self::with_cause(rspc::ErrorCode::InternalServerError, e.to_string(), e),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(thiserror::Error, Debug, Serialize, Deserialize, Type)]
|
#[derive(thiserror::Error, Debug, Serialize, Deserialize, Type, Clone)]
|
||||||
pub enum NonCriticalError {
|
#[serde(rename_all = "snake_case")]
|
||||||
|
pub enum NonCriticalMediaProcessorError {
|
||||||
#[error(transparent)]
|
#[error(transparent)]
|
||||||
MediaDataExtractor(#[from] media_data_extractor::NonCriticalError),
|
MediaDataExtractor(#[from] NonCriticalMediaDataExtractorError),
|
||||||
#[error(transparent)]
|
#[error(transparent)]
|
||||||
Thumbnailer(#[from] thumbnailer::NonCriticalError),
|
Thumbnailer(#[from] NonCriticalThumbnailerError),
|
||||||
}
|
}
|
||||||
|
|
||||||
struct NewThumbnailsReporter<Ctx: OuterContext> {
|
#[derive(Clone)]
|
||||||
ctx: Ctx,
|
pub struct NewThumbnailsReporter<OuterCtx: OuterContext> {
|
||||||
|
pub ctx: OuterCtx,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl<Ctx: OuterContext> fmt::Debug for NewThumbnailsReporter<Ctx> {
|
impl<OuterCtx: OuterContext> fmt::Debug for NewThumbnailsReporter<OuterCtx> {
|
||||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||||
f.debug_struct("NewThumbnailsReporter").finish()
|
f.debug_struct("NewThumbnailsReporter").finish()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl<Ctx: OuterContext> NewThumbnailReporter for NewThumbnailsReporter<Ctx> {
|
impl<OuterCtx: OuterContext> NewThumbnailReporter for NewThumbnailsReporter<OuterCtx> {
|
||||||
fn new_thumbnail(&self, thumb_key: ThumbKey) {
|
fn new_thumbnail(&self, thumb_key: ThumbKey) {
|
||||||
self.ctx
|
self.ctx
|
||||||
.report_update(UpdateEvent::NewThumbnailEvent { thumb_key });
|
.report_update(UpdateEvent::NewThumbnail { thumb_key });
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
struct RawFilePathForMediaProcessor {
|
||||||
|
id: file_path::id::Type,
|
||||||
|
materialized_path: file_path::materialized_path::Type,
|
||||||
|
is_dir: file_path::is_dir::Type,
|
||||||
|
name: file_path::name::Type,
|
||||||
|
extension: file_path::extension::Type,
|
||||||
|
cas_id: file_path::cas_id::Type,
|
||||||
|
object_id: object::id::Type,
|
||||||
|
object_pub_id: object::pub_id::Type,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<RawFilePathForMediaProcessor> for file_path_for_media_processor::Data {
|
||||||
|
fn from(
|
||||||
|
RawFilePathForMediaProcessor {
|
||||||
|
id,
|
||||||
|
materialized_path,
|
||||||
|
is_dir,
|
||||||
|
name,
|
||||||
|
extension,
|
||||||
|
cas_id,
|
||||||
|
object_id,
|
||||||
|
object_pub_id,
|
||||||
|
}: RawFilePathForMediaProcessor,
|
||||||
|
) -> Self {
|
||||||
|
Self {
|
||||||
|
id,
|
||||||
|
materialized_path,
|
||||||
|
is_dir,
|
||||||
|
name,
|
||||||
|
extension,
|
||||||
|
cas_id,
|
||||||
|
object: Some(file_path_for_media_processor::object::Data {
|
||||||
|
id: object_id,
|
||||||
|
pub_id: object_pub_id,
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn get_direct_children_files_by_extensions(
|
||||||
|
parent_iso_file_path: &IsolatedFilePathData<'_>,
|
||||||
|
extensions: &[Extension],
|
||||||
|
db: &PrismaClient,
|
||||||
|
) -> Result<Vec<file_path_for_media_processor::Data>, Error> {
|
||||||
|
// FIXME: Had to use format! macro because PCR doesn't support IN with Vec for SQLite
|
||||||
|
// We have no data coming from the user, so this is sql injection safe
|
||||||
|
let unique_by_object_id = db
|
||||||
|
._query_raw::<RawFilePathForMediaProcessor>(raw!(
|
||||||
|
&format!(
|
||||||
|
"SELECT
|
||||||
|
file_path.id,
|
||||||
|
file_path.materialized_path,
|
||||||
|
file_path.is_dir,
|
||||||
|
file_path.name,
|
||||||
|
file_path.extension,
|
||||||
|
file_path.cas_id,
|
||||||
|
object.id as 'object_id',
|
||||||
|
object.pub_id as 'object_pub_id'
|
||||||
|
FROM file_path
|
||||||
|
INNER JOIN object ON object.id = file_path.object_id
|
||||||
|
WHERE
|
||||||
|
location_id={{}}
|
||||||
|
AND cas_id IS NOT NULL
|
||||||
|
AND LOWER(extension) IN ({})
|
||||||
|
AND materialized_path = {{}}
|
||||||
|
ORDER BY name ASC",
|
||||||
|
extensions
|
||||||
|
.iter()
|
||||||
|
.map(|ext| format!("LOWER('{ext}')"))
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join(",")
|
||||||
|
),
|
||||||
|
PrismaValue::Int(parent_iso_file_path.location_id()),
|
||||||
|
PrismaValue::String(
|
||||||
|
parent_iso_file_path
|
||||||
|
.materialized_path_for_children()
|
||||||
|
.expect("sub path iso_file_path must be a directory")
|
||||||
|
)
|
||||||
|
))
|
||||||
|
.exec()
|
||||||
|
.await?
|
||||||
|
.into_iter()
|
||||||
|
.map(|raw_file_path| (raw_file_path.object_id, raw_file_path))
|
||||||
|
.collect::<HashMap<_, _>>();
|
||||||
|
|
||||||
|
Ok(unique_by_object_id.into_values().map(Into::into).collect())
|
||||||
|
}
|
||||||
|
|
|
@ -4,9 +4,8 @@ use crate::{
|
||||||
};
|
};
|
||||||
|
|
||||||
use sd_core_file_path_helper::IsolatedFilePathData;
|
use sd_core_file_path_helper::IsolatedFilePathData;
|
||||||
use sd_core_prisma_helpers::file_path_for_media_processor;
|
use sd_core_sync::Manager as SyncManager;
|
||||||
|
|
||||||
use sd_file_ext::extensions::Extension;
|
|
||||||
use sd_prisma::prisma::{location, PrismaClient};
|
use sd_prisma::prisma::{location, PrismaClient};
|
||||||
use sd_task_system::{
|
use sd_task_system::{
|
||||||
BaseTaskDispatcher, CancelTaskOnDrop, IntoTask, TaskDispatcher, TaskHandle, TaskOutput,
|
BaseTaskDispatcher, CancelTaskOnDrop, IntoTask, TaskDispatcher, TaskHandle, TaskOutput,
|
||||||
|
@ -19,15 +18,18 @@ use std::{
|
||||||
sync::Arc,
|
sync::Arc,
|
||||||
};
|
};
|
||||||
|
|
||||||
use futures::StreamExt;
|
use futures::{stream::FuturesUnordered, StreamExt};
|
||||||
use futures_concurrency::future::{FutureGroup, TryJoin};
|
use futures_concurrency::future::TryJoin;
|
||||||
use itertools::Itertools;
|
use itertools::Itertools;
|
||||||
use prisma_client_rust::{raw, PrismaValue};
|
|
||||||
use tracing::{debug, warn};
|
use tracing::{debug, warn};
|
||||||
|
|
||||||
use super::{
|
use super::{
|
||||||
|
get_direct_children_files_by_extensions,
|
||||||
helpers::{self, exif_media_data, ffmpeg_media_data, thumbnailer::THUMBNAIL_CACHE_DIR_NAME},
|
helpers::{self, exif_media_data, ffmpeg_media_data, thumbnailer::THUMBNAIL_CACHE_DIR_NAME},
|
||||||
tasks::{self, media_data_extractor, thumbnailer},
|
tasks::{
|
||||||
|
self, media_data_extractor,
|
||||||
|
thumbnailer::{self, NewThumbnailReporter},
|
||||||
|
},
|
||||||
NewThumbnailsReporter, BATCH_SIZE,
|
NewThumbnailsReporter, BATCH_SIZE,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -35,8 +37,8 @@ use super::{
|
||||||
pub async fn shallow(
|
pub async fn shallow(
|
||||||
location: location::Data,
|
location: location::Data,
|
||||||
sub_path: impl AsRef<Path> + Send,
|
sub_path: impl AsRef<Path> + Send,
|
||||||
dispatcher: BaseTaskDispatcher<Error>,
|
dispatcher: &BaseTaskDispatcher<Error>,
|
||||||
ctx: impl OuterContext,
|
ctx: &impl OuterContext,
|
||||||
) -> Result<Vec<NonCriticalError>, Error> {
|
) -> Result<Vec<NonCriticalError>, Error> {
|
||||||
let sub_path = sub_path.as_ref();
|
let sub_path = sub_path.as_ref();
|
||||||
|
|
||||||
|
@ -47,14 +49,13 @@ pub async fn shallow(
|
||||||
|
|
||||||
let location = Arc::new(location);
|
let location = Arc::new(location);
|
||||||
|
|
||||||
let sub_iso_file_path = maybe_get_iso_file_path_from_sub_path(
|
let sub_iso_file_path = maybe_get_iso_file_path_from_sub_path::<media_processor::Error>(
|
||||||
location.id,
|
location.id,
|
||||||
&Some(sub_path),
|
Some(sub_path),
|
||||||
&*location_path,
|
&*location_path,
|
||||||
ctx.db(),
|
ctx.db(),
|
||||||
)
|
)
|
||||||
.await
|
.await?
|
||||||
.map_err(media_processor::Error::from)?
|
|
||||||
.map_or_else(
|
.map_or_else(
|
||||||
|| {
|
|| {
|
||||||
IsolatedFilePathData::new(location.id, &*location_path, &*location_path, true)
|
IsolatedFilePathData::new(location.id, &*location_path, &*location_path, true)
|
||||||
|
@ -65,37 +66,70 @@ pub async fn shallow(
|
||||||
|
|
||||||
let mut errors = vec![];
|
let mut errors = vec![];
|
||||||
|
|
||||||
let mut futures = dispatch_media_data_extractor_tasks(
|
let media_data_extraction_tasks = dispatch_media_data_extractor_tasks(
|
||||||
ctx.db(),
|
ctx.db(),
|
||||||
|
ctx.sync(),
|
||||||
&sub_iso_file_path,
|
&sub_iso_file_path,
|
||||||
&location_path,
|
&location_path,
|
||||||
&dispatcher,
|
dispatcher,
|
||||||
)
|
)
|
||||||
.await?
|
.await?;
|
||||||
.into_iter()
|
|
||||||
.map(CancelTaskOnDrop)
|
let total_media_data_extraction_tasks = media_data_extraction_tasks.len();
|
||||||
.chain(
|
|
||||||
dispatch_thumbnailer_tasks(&sub_iso_file_path, false, &location_path, &dispatcher, &ctx)
|
let thumbnailer_tasks =
|
||||||
.await?
|
dispatch_thumbnailer_tasks(&sub_iso_file_path, false, &location_path, dispatcher, ctx)
|
||||||
.into_iter()
|
.await?;
|
||||||
.map(CancelTaskOnDrop),
|
|
||||||
)
|
let total_thumbnailer_tasks = thumbnailer_tasks.len();
|
||||||
.collect::<FutureGroup<_>>();
|
|
||||||
|
let mut futures = media_data_extraction_tasks
|
||||||
|
.into_iter()
|
||||||
|
.chain(thumbnailer_tasks.into_iter())
|
||||||
|
.map(CancelTaskOnDrop::new)
|
||||||
|
.collect::<FuturesUnordered<_>>();
|
||||||
|
|
||||||
|
let mut completed_media_data_extraction_tasks = 0;
|
||||||
|
let mut completed_thumbnailer_tasks = 0;
|
||||||
|
|
||||||
while let Some(res) = futures.next().await {
|
while let Some(res) = futures.next().await {
|
||||||
match res {
|
match res {
|
||||||
Ok(TaskStatus::Done((_, TaskOutput::Out(out)))) => {
|
Ok(TaskStatus::Done((_, TaskOutput::Out(out)))) => {
|
||||||
if out.is::<media_data_extractor::Output>() {
|
if out.is::<media_data_extractor::Output>() {
|
||||||
errors.extend(
|
let media_data_extractor::Output {
|
||||||
out.downcast::<media_data_extractor::Output>()
|
db_read_time,
|
||||||
.expect("just checked")
|
filtering_time,
|
||||||
.errors,
|
extraction_time,
|
||||||
|
db_write_time,
|
||||||
|
errors: new_errors,
|
||||||
|
..
|
||||||
|
} = *out
|
||||||
|
.downcast::<media_data_extractor::Output>()
|
||||||
|
.expect("just checked");
|
||||||
|
|
||||||
|
errors.extend(new_errors);
|
||||||
|
|
||||||
|
completed_media_data_extraction_tasks += 1;
|
||||||
|
|
||||||
|
debug!(
|
||||||
|
"Media data extraction task ({completed_media_data_extraction_tasks}/\
|
||||||
|
{total_media_data_extraction_tasks}) completed in {:?};",
|
||||||
|
db_read_time + filtering_time + extraction_time + db_write_time
|
||||||
);
|
);
|
||||||
} else if out.is::<thumbnailer::Output>() {
|
} else if out.is::<thumbnailer::Output>() {
|
||||||
errors.extend(
|
let thumbnailer::Output {
|
||||||
out.downcast::<thumbnailer::Output>()
|
total_time,
|
||||||
.expect("just checked")
|
errors: new_errors,
|
||||||
.errors,
|
..
|
||||||
|
} = *out.downcast::<thumbnailer::Output>().expect("just checked");
|
||||||
|
|
||||||
|
errors.extend(new_errors);
|
||||||
|
|
||||||
|
completed_thumbnailer_tasks += 1;
|
||||||
|
|
||||||
|
debug!(
|
||||||
|
"Thumbnailer task ({completed_thumbnailer_tasks}/{total_thumbnailer_tasks}) \
|
||||||
|
completed in {total_time:?};",
|
||||||
);
|
);
|
||||||
} else {
|
} else {
|
||||||
unreachable!(
|
unreachable!(
|
||||||
|
@ -120,20 +154,21 @@ pub async fn shallow(
|
||||||
|
|
||||||
async fn dispatch_media_data_extractor_tasks(
|
async fn dispatch_media_data_extractor_tasks(
|
||||||
db: &Arc<PrismaClient>,
|
db: &Arc<PrismaClient>,
|
||||||
|
sync: &Arc<SyncManager>,
|
||||||
parent_iso_file_path: &IsolatedFilePathData<'_>,
|
parent_iso_file_path: &IsolatedFilePathData<'_>,
|
||||||
location_path: &Arc<PathBuf>,
|
location_path: &Arc<PathBuf>,
|
||||||
dispatcher: &BaseTaskDispatcher<Error>,
|
dispatcher: &BaseTaskDispatcher<Error>,
|
||||||
) -> Result<Vec<TaskHandle<Error>>, media_processor::Error> {
|
) -> Result<Vec<TaskHandle<Error>>, Error> {
|
||||||
let (extract_exif_file_paths, extract_ffmpeg_file_paths) = (
|
let (extract_exif_file_paths, extract_ffmpeg_file_paths) = (
|
||||||
get_files_by_extensions(
|
get_direct_children_files_by_extensions(
|
||||||
db,
|
|
||||||
parent_iso_file_path,
|
parent_iso_file_path,
|
||||||
&exif_media_data::AVAILABLE_EXTENSIONS,
|
&exif_media_data::AVAILABLE_EXTENSIONS,
|
||||||
),
|
|
||||||
get_files_by_extensions(
|
|
||||||
db,
|
db,
|
||||||
|
),
|
||||||
|
get_direct_children_files_by_extensions(
|
||||||
parent_iso_file_path,
|
parent_iso_file_path,
|
||||||
&ffmpeg_media_data::AVAILABLE_EXTENSIONS,
|
&ffmpeg_media_data::AVAILABLE_EXTENSIONS,
|
||||||
|
db,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
.try_join()
|
.try_join()
|
||||||
|
@ -150,6 +185,7 @@ async fn dispatch_media_data_extractor_tasks(
|
||||||
parent_iso_file_path.location_id(),
|
parent_iso_file_path.location_id(),
|
||||||
Arc::clone(location_path),
|
Arc::clone(location_path),
|
||||||
Arc::clone(db),
|
Arc::clone(db),
|
||||||
|
Arc::clone(sync),
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
.map(IntoTask::into_task)
|
.map(IntoTask::into_task)
|
||||||
|
@ -165,47 +201,20 @@ async fn dispatch_media_data_extractor_tasks(
|
||||||
parent_iso_file_path.location_id(),
|
parent_iso_file_path.location_id(),
|
||||||
Arc::clone(location_path),
|
Arc::clone(location_path),
|
||||||
Arc::clone(db),
|
Arc::clone(db),
|
||||||
|
Arc::clone(sync),
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
.map(IntoTask::into_task),
|
.map(IntoTask::into_task),
|
||||||
)
|
)
|
||||||
.collect::<Vec<_>>();
|
.collect::<Vec<_>>();
|
||||||
|
|
||||||
Ok(dispatcher.dispatch_many_boxed(tasks).await)
|
dispatcher.dispatch_many_boxed(tasks).await.map_or_else(
|
||||||
}
|
|_| {
|
||||||
|
debug!("Task system is shutting down while a shallow media processor was in progress");
|
||||||
async fn get_files_by_extensions(
|
Ok(vec![])
|
||||||
db: &PrismaClient,
|
},
|
||||||
parent_iso_file_path: &IsolatedFilePathData<'_>,
|
Ok,
|
||||||
extensions: &[Extension],
|
)
|
||||||
) -> Result<Vec<file_path_for_media_processor::Data>, media_processor::Error> {
|
|
||||||
// FIXME: Had to use format! macro because PCR doesn't support IN with Vec for SQLite
|
|
||||||
// We have no data coming from the user, so this is sql injection safe
|
|
||||||
db._query_raw(raw!(
|
|
||||||
&format!(
|
|
||||||
"SELECT id, materialized_path, is_dir, name, extension, cas_id, object_id
|
|
||||||
FROM file_path
|
|
||||||
WHERE
|
|
||||||
location_id={{}}
|
|
||||||
AND cas_id IS NOT NULL
|
|
||||||
AND LOWER(extension) IN ({})
|
|
||||||
AND materialized_path = {{}}",
|
|
||||||
extensions
|
|
||||||
.iter()
|
|
||||||
.map(|ext| format!("LOWER('{ext}')"))
|
|
||||||
.collect::<Vec<_>>()
|
|
||||||
.join(",")
|
|
||||||
),
|
|
||||||
PrismaValue::Int(parent_iso_file_path.location_id()),
|
|
||||||
PrismaValue::String(
|
|
||||||
parent_iso_file_path
|
|
||||||
.materialized_path_for_children()
|
|
||||||
.expect("sub path iso_file_path must be a directory")
|
|
||||||
)
|
|
||||||
))
|
|
||||||
.exec()
|
|
||||||
.await
|
|
||||||
.map_err(Into::into)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn dispatch_thumbnailer_tasks(
|
async fn dispatch_thumbnailer_tasks(
|
||||||
|
@ -214,18 +223,19 @@ async fn dispatch_thumbnailer_tasks(
|
||||||
location_path: &PathBuf,
|
location_path: &PathBuf,
|
||||||
dispatcher: &BaseTaskDispatcher<Error>,
|
dispatcher: &BaseTaskDispatcher<Error>,
|
||||||
ctx: &impl OuterContext,
|
ctx: &impl OuterContext,
|
||||||
) -> Result<Vec<TaskHandle<Error>>, media_processor::Error> {
|
) -> Result<Vec<TaskHandle<Error>>, Error> {
|
||||||
let thumbnails_directory_path =
|
let thumbnails_directory_path =
|
||||||
Arc::new(ctx.get_data_directory().join(THUMBNAIL_CACHE_DIR_NAME));
|
Arc::new(ctx.get_data_directory().join(THUMBNAIL_CACHE_DIR_NAME));
|
||||||
let location_id = parent_iso_file_path.location_id();
|
let location_id = parent_iso_file_path.location_id();
|
||||||
let library_id = ctx.id();
|
let library_id = ctx.id();
|
||||||
let db = ctx.db();
|
let db = ctx.db();
|
||||||
let reporter = Arc::new(NewThumbnailsReporter { ctx: ctx.clone() });
|
let reporter: Arc<dyn NewThumbnailReporter> =
|
||||||
|
Arc::new(NewThumbnailsReporter { ctx: ctx.clone() });
|
||||||
|
|
||||||
let file_paths = get_files_by_extensions(
|
let file_paths = get_direct_children_files_by_extensions(
|
||||||
db,
|
|
||||||
parent_iso_file_path,
|
parent_iso_file_path,
|
||||||
&helpers::thumbnailer::ALL_THUMBNAILABLE_EXTENSIONS,
|
&helpers::thumbnailer::ALL_THUMBNAILABLE_EXTENSIONS,
|
||||||
|
db,
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
|
@ -249,10 +259,13 @@ async fn dispatch_thumbnailer_tasks(
|
||||||
.map(IntoTask::into_task)
|
.map(IntoTask::into_task)
|
||||||
.collect::<Vec<_>>();
|
.collect::<Vec<_>>();
|
||||||
|
|
||||||
debug!(
|
debug!(%thumbs_count, priority_tasks_count = tasks.len(), "Dispatching thumbnails to be processed;");
|
||||||
"Dispatching {thumbs_count} thumbnails to be processed, in {} priority tasks",
|
|
||||||
tasks.len(),
|
|
||||||
);
|
|
||||||
|
|
||||||
Ok(dispatcher.dispatch_many_boxed(tasks).await)
|
dispatcher.dispatch_many_boxed(tasks).await.map_or_else(
|
||||||
|
|_| {
|
||||||
|
debug!("Task system is shutting down while a shallow media processor was in progress");
|
||||||
|
Ok(vec![])
|
||||||
|
},
|
||||||
|
Ok,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
|
@ -7,7 +7,8 @@ use crate::{
|
||||||
};
|
};
|
||||||
|
|
||||||
use sd_core_file_path_helper::IsolatedFilePathData;
|
use sd_core_file_path_helper::IsolatedFilePathData;
|
||||||
use sd_core_prisma_helpers::file_path_for_media_processor;
|
use sd_core_prisma_helpers::{file_path_for_media_processor, ObjectPubId};
|
||||||
|
use sd_core_sync::Manager as SyncManager;
|
||||||
|
|
||||||
use sd_media_metadata::{ExifMetadata, FFmpegMetadata};
|
use sd_media_metadata::{ExifMetadata, FFmpegMetadata};
|
||||||
use sd_prisma::prisma::{exif_data, ffmpeg_data, file_path, location, object, PrismaClient};
|
use sd_prisma::prisma::{exif_data, ffmpeg_data, file_path, location, object, PrismaClient};
|
||||||
|
@ -26,11 +27,22 @@ use std::{
|
||||||
time::Duration,
|
time::Duration,
|
||||||
};
|
};
|
||||||
|
|
||||||
use futures::{FutureExt, StreamExt};
|
use futures::{stream::FuturesUnordered, FutureExt, StreamExt};
|
||||||
use futures_concurrency::future::{FutureGroup, Race};
|
use futures_concurrency::future::Race;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use specta::Type;
|
use specta::Type;
|
||||||
use tokio::time::Instant;
|
use tokio::time::Instant;
|
||||||
|
use tracing::{debug, instrument, trace, Level};
|
||||||
|
|
||||||
|
#[derive(thiserror::Error, Debug, Serialize, Deserialize, Type, Clone)]
|
||||||
|
pub enum NonCriticalMediaDataExtractorError {
|
||||||
|
#[error("failed to extract media data from <file='{}'>: {1}", .0.display())]
|
||||||
|
FailedToExtractImageMediaData(PathBuf, String),
|
||||||
|
#[error("file path missing object id: <file_path_id='{0}'>")]
|
||||||
|
FilePathMissingObjectId(file_path::id::Type),
|
||||||
|
#[error("failed to construct isolated file path data: <file_path_id='{0}'>: {1}")]
|
||||||
|
FailedToConstructIsolatedFilePathData(file_path::id::Type, String),
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Serialize, Deserialize, Clone, Copy, PartialEq, Eq)]
|
#[derive(Debug, Serialize, Deserialize, Clone, Copy, PartialEq, Eq)]
|
||||||
enum Kind {
|
enum Kind {
|
||||||
|
@ -40,14 +52,24 @@ enum Kind {
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
pub struct MediaDataExtractor {
|
pub struct MediaDataExtractor {
|
||||||
|
// Task control
|
||||||
id: TaskId,
|
id: TaskId,
|
||||||
kind: Kind,
|
kind: Kind,
|
||||||
|
|
||||||
|
// Received input args
|
||||||
file_paths: Vec<file_path_for_media_processor::Data>,
|
file_paths: Vec<file_path_for_media_processor::Data>,
|
||||||
location_id: location::id::Type,
|
location_id: location::id::Type,
|
||||||
location_path: Arc<PathBuf>,
|
location_path: Arc<PathBuf>,
|
||||||
|
|
||||||
|
// Inner state
|
||||||
stage: Stage,
|
stage: Stage,
|
||||||
db: Arc<PrismaClient>,
|
|
||||||
|
// Out collector
|
||||||
output: Output,
|
output: Output,
|
||||||
|
|
||||||
|
// Dependencies
|
||||||
|
db: Arc<PrismaClient>,
|
||||||
|
sync: Arc<SyncManager>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Serialize, Deserialize)]
|
#[derive(Debug, Serialize, Deserialize)]
|
||||||
|
@ -55,74 +77,34 @@ enum Stage {
|
||||||
Starting,
|
Starting,
|
||||||
FetchedObjectsAlreadyWithMediaData(Vec<object::id::Type>),
|
FetchedObjectsAlreadyWithMediaData(Vec<object::id::Type>),
|
||||||
ExtractingMediaData {
|
ExtractingMediaData {
|
||||||
paths_by_id: HashMap<file_path::id::Type, (PathBuf, object::id::Type)>,
|
paths_by_id: HashMap<file_path::id::Type, (PathBuf, object::id::Type, ObjectPubId)>,
|
||||||
exif_media_datas: Vec<(ExifMetadata, object::id::Type)>,
|
exif_media_datas: Vec<(ExifMetadata, object::id::Type, ObjectPubId)>,
|
||||||
ffmpeg_media_datas: Vec<(FFmpegMetadata, object::id::Type)>,
|
ffmpeg_media_datas: Vec<(FFmpegMetadata, object::id::Type)>,
|
||||||
extract_ids_to_remove_from_map: Vec<file_path::id::Type>,
|
extract_ids_to_remove_from_map: Vec<file_path::id::Type>,
|
||||||
},
|
},
|
||||||
SaveMediaData {
|
SaveMediaData {
|
||||||
exif_media_datas: Vec<(ExifMetadata, object::id::Type)>,
|
exif_media_datas: Vec<(ExifMetadata, object::id::Type, ObjectPubId)>,
|
||||||
ffmpeg_media_datas: Vec<(FFmpegMetadata, object::id::Type)>,
|
ffmpeg_media_datas: Vec<(FFmpegMetadata, object::id::Type)>,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
impl MediaDataExtractor {
|
/// [`MediaDataExtractor`] task output
|
||||||
fn new(
|
#[derive(Serialize, Deserialize, Default, Debug)]
|
||||||
kind: Kind,
|
pub struct Output {
|
||||||
file_paths: &[file_path_for_media_processor::Data],
|
/// How many files were successfully processed
|
||||||
location_id: location::id::Type,
|
pub extracted: u64,
|
||||||
location_path: Arc<PathBuf>,
|
/// How many files were skipped
|
||||||
db: Arc<PrismaClient>,
|
pub skipped: u64,
|
||||||
) -> Self {
|
/// Time spent reading data from database
|
||||||
let mut output = Output::default();
|
pub db_read_time: Duration,
|
||||||
|
/// Time spent filtering files to extract media data and files to skip
|
||||||
Self {
|
pub filtering_time: Duration,
|
||||||
id: TaskId::new_v4(),
|
/// Time spent extracting media data
|
||||||
kind,
|
pub extraction_time: Duration,
|
||||||
file_paths: file_paths
|
/// Time spent writing media data to database
|
||||||
.iter()
|
pub db_write_time: Duration,
|
||||||
.filter(|file_path| {
|
/// Errors encountered during the task
|
||||||
if file_path.object_id.is_some() {
|
pub errors: Vec<crate::NonCriticalError>,
|
||||||
true
|
|
||||||
} else {
|
|
||||||
output.errors.push(
|
|
||||||
media_processor::NonCriticalError::from(
|
|
||||||
NonCriticalError::FilePathMissingObjectId(file_path.id),
|
|
||||||
)
|
|
||||||
.into(),
|
|
||||||
);
|
|
||||||
false
|
|
||||||
}
|
|
||||||
})
|
|
||||||
.cloned()
|
|
||||||
.collect(),
|
|
||||||
location_id,
|
|
||||||
location_path,
|
|
||||||
stage: Stage::Starting,
|
|
||||||
db,
|
|
||||||
output,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[must_use]
|
|
||||||
pub fn new_exif(
|
|
||||||
file_paths: &[file_path_for_media_processor::Data],
|
|
||||||
location_id: location::id::Type,
|
|
||||||
location_path: Arc<PathBuf>,
|
|
||||||
db: Arc<PrismaClient>,
|
|
||||||
) -> Self {
|
|
||||||
Self::new(Kind::Exif, file_paths, location_id, location_path, db)
|
|
||||||
}
|
|
||||||
|
|
||||||
#[must_use]
|
|
||||||
pub fn new_ffmpeg(
|
|
||||||
file_paths: &[file_path_for_media_processor::Data],
|
|
||||||
location_id: location::id::Type,
|
|
||||||
location_path: Arc<PathBuf>,
|
|
||||||
db: Arc<PrismaClient>,
|
|
||||||
) -> Self {
|
|
||||||
Self::new(Kind::FFmpeg, file_paths, location_id, location_path, db)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[async_trait::async_trait]
|
#[async_trait::async_trait]
|
||||||
|
@ -138,6 +120,20 @@ impl Task<Error> for MediaDataExtractor {
|
||||||
false
|
false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[instrument(
|
||||||
|
skip_all,
|
||||||
|
fields(
|
||||||
|
task_id = %self.id,
|
||||||
|
kind = ?self.kind,
|
||||||
|
location_id = %self.location_id,
|
||||||
|
location_path = %self.location_path.display(),
|
||||||
|
file_paths_count = %self.file_paths.len(),
|
||||||
|
),
|
||||||
|
ret(level = Level::TRACE),
|
||||||
|
err,
|
||||||
|
)]
|
||||||
|
#[allow(clippy::blocks_in_conditions)] // Due to `err` on `instrument` macro above
|
||||||
|
#[allow(clippy::too_many_lines)]
|
||||||
async fn run(&mut self, interrupter: &Interrupter) -> Result<ExecStatus, Error> {
|
async fn run(&mut self, interrupter: &Interrupter) -> Result<ExecStatus, Error> {
|
||||||
loop {
|
loop {
|
||||||
match &mut self.stage {
|
match &mut self.stage {
|
||||||
|
@ -150,18 +146,22 @@ impl Task<Error> for MediaDataExtractor {
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
self.output.db_read_time = db_read_start.elapsed();
|
self.output.db_read_time = db_read_start.elapsed();
|
||||||
|
trace!(
|
||||||
|
object_ids_count = object_ids.len(),
|
||||||
|
"Fetched objects already with media data;",
|
||||||
|
);
|
||||||
|
|
||||||
self.stage = Stage::FetchedObjectsAlreadyWithMediaData(object_ids);
|
self.stage = Stage::FetchedObjectsAlreadyWithMediaData(object_ids);
|
||||||
}
|
}
|
||||||
|
|
||||||
Stage::FetchedObjectsAlreadyWithMediaData(objects_already_with_media_data) => {
|
Stage::FetchedObjectsAlreadyWithMediaData(objects_already_with_media_data) => {
|
||||||
let filtering_start = Instant::now();
|
|
||||||
if self.file_paths.len() == objects_already_with_media_data.len() {
|
if self.file_paths.len() == objects_already_with_media_data.len() {
|
||||||
// All files already have media data, skipping
|
self.output.skipped = self.file_paths.len() as u64; // Files already have media data, skipping
|
||||||
self.output.skipped = self.file_paths.len() as u64;
|
debug!("Skipped all files as they already have media data");
|
||||||
|
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let filtering_start = Instant::now();
|
||||||
let paths_by_id = filter_files_to_extract_media_data(
|
let paths_by_id = filter_files_to_extract_media_data(
|
||||||
mem::take(objects_already_with_media_data),
|
mem::take(objects_already_with_media_data),
|
||||||
self.location_id,
|
self.location_id,
|
||||||
|
@ -169,9 +169,13 @@ impl Task<Error> for MediaDataExtractor {
|
||||||
&mut self.file_paths,
|
&mut self.file_paths,
|
||||||
&mut self.output,
|
&mut self.output,
|
||||||
);
|
);
|
||||||
|
|
||||||
self.output.filtering_time = filtering_start.elapsed();
|
self.output.filtering_time = filtering_start.elapsed();
|
||||||
|
|
||||||
|
trace!(
|
||||||
|
paths_needing_media_data_extraction_count = paths_by_id.len(),
|
||||||
|
"Filtered files to extract media data;",
|
||||||
|
);
|
||||||
|
|
||||||
self.stage = Stage::ExtractingMediaData {
|
self.stage = Stage::ExtractingMediaData {
|
||||||
extract_ids_to_remove_from_map: Vec::with_capacity(paths_by_id.len()),
|
extract_ids_to_remove_from_map: Vec::with_capacity(paths_by_id.len()),
|
||||||
exif_media_datas: if self.kind == Kind::Exif {
|
exif_media_datas: if self.kind == Kind::Exif {
|
||||||
|
@ -241,8 +245,14 @@ impl Task<Error> for MediaDataExtractor {
|
||||||
ffmpeg_media_datas,
|
ffmpeg_media_datas,
|
||||||
} => {
|
} => {
|
||||||
let db_write_start = Instant::now();
|
let db_write_start = Instant::now();
|
||||||
self.output.extracted =
|
self.output.extracted = save(
|
||||||
save(self.kind, exif_media_datas, ffmpeg_media_datas, &self.db).await?;
|
self.kind,
|
||||||
|
exif_media_datas,
|
||||||
|
ffmpeg_media_datas,
|
||||||
|
&self.db,
|
||||||
|
&self.sync,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
self.output.db_write_time = db_write_start.elapsed();
|
self.output.db_write_time = db_write_start.elapsed();
|
||||||
|
|
||||||
self.output.skipped += self.output.errors.len() as u64;
|
self.output.skipped += self.output.errors.len() as u64;
|
||||||
|
@ -258,91 +268,74 @@ impl Task<Error> for MediaDataExtractor {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(thiserror::Error, Debug, Serialize, Deserialize, Type)]
|
impl MediaDataExtractor {
|
||||||
pub enum NonCriticalError {
|
fn new(
|
||||||
#[error("failed to extract media data from <file='{}'>: {1}", .0.display())]
|
kind: Kind,
|
||||||
FailedToExtractImageMediaData(PathBuf, String),
|
file_paths: &[file_path_for_media_processor::Data],
|
||||||
#[error("file path missing object id: <file_path_id='{0}'>")]
|
location_id: location::id::Type,
|
||||||
FilePathMissingObjectId(file_path::id::Type),
|
location_path: Arc<PathBuf>,
|
||||||
#[error("failed to construct isolated file path data: <file_path_id='{0}'>: {1}")]
|
db: Arc<PrismaClient>,
|
||||||
FailedToConstructIsolatedFilePathData(file_path::id::Type, String),
|
sync: Arc<SyncManager>,
|
||||||
}
|
) -> Self {
|
||||||
|
let mut output = Output::default();
|
||||||
|
|
||||||
#[derive(Serialize, Deserialize, Default, Debug)]
|
Self {
|
||||||
pub struct Output {
|
id: TaskId::new_v4(),
|
||||||
pub extracted: u64,
|
|
||||||
pub skipped: u64,
|
|
||||||
pub db_read_time: Duration,
|
|
||||||
pub filtering_time: Duration,
|
|
||||||
pub extraction_time: Duration,
|
|
||||||
pub db_write_time: Duration,
|
|
||||||
pub errors: Vec<crate::NonCriticalError>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Serialize, Deserialize)]
|
|
||||||
struct SaveState {
|
|
||||||
id: TaskId,
|
|
||||||
kind: Kind,
|
|
||||||
file_paths: Vec<file_path_for_media_processor::Data>,
|
|
||||||
location_id: location::id::Type,
|
|
||||||
location_path: Arc<PathBuf>,
|
|
||||||
stage: Stage,
|
|
||||||
output: Output,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl SerializableTask<Error> for MediaDataExtractor {
|
|
||||||
type SerializeError = rmp_serde::encode::Error;
|
|
||||||
|
|
||||||
type DeserializeError = rmp_serde::decode::Error;
|
|
||||||
|
|
||||||
type DeserializeCtx = Arc<PrismaClient>;
|
|
||||||
|
|
||||||
async fn serialize(self) -> Result<Vec<u8>, Self::SerializeError> {
|
|
||||||
let Self {
|
|
||||||
id,
|
|
||||||
kind,
|
kind,
|
||||||
file_paths,
|
file_paths: file_paths
|
||||||
|
.iter()
|
||||||
|
.filter(|file_path| {
|
||||||
|
if file_path.object.is_some() {
|
||||||
|
true
|
||||||
|
} else {
|
||||||
|
output.errors.push(
|
||||||
|
media_processor::NonCriticalMediaProcessorError::from(
|
||||||
|
NonCriticalMediaDataExtractorError::FilePathMissingObjectId(
|
||||||
|
file_path.id,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.into(),
|
||||||
|
);
|
||||||
|
false
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.cloned()
|
||||||
|
.collect(),
|
||||||
location_id,
|
location_id,
|
||||||
location_path,
|
location_path,
|
||||||
stage,
|
stage: Stage::Starting,
|
||||||
|
db,
|
||||||
|
sync,
|
||||||
output,
|
output,
|
||||||
..
|
}
|
||||||
} = self;
|
|
||||||
|
|
||||||
rmp_serde::to_vec_named(&SaveState {
|
|
||||||
id,
|
|
||||||
kind,
|
|
||||||
file_paths,
|
|
||||||
location_id,
|
|
||||||
location_path,
|
|
||||||
stage,
|
|
||||||
output,
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn deserialize(
|
#[must_use]
|
||||||
data: &[u8],
|
pub fn new_exif(
|
||||||
db: Self::DeserializeCtx,
|
file_paths: &[file_path_for_media_processor::Data],
|
||||||
) -> Result<Self, Self::DeserializeError> {
|
location_id: location::id::Type,
|
||||||
rmp_serde::from_slice(data).map(
|
location_path: Arc<PathBuf>,
|
||||||
|SaveState {
|
db: Arc<PrismaClient>,
|
||||||
id,
|
sync: Arc<SyncManager>,
|
||||||
kind,
|
) -> Self {
|
||||||
file_paths,
|
Self::new(Kind::Exif, file_paths, location_id, location_path, db, sync)
|
||||||
location_id,
|
}
|
||||||
location_path,
|
|
||||||
stage,
|
#[must_use]
|
||||||
output,
|
pub fn new_ffmpeg(
|
||||||
}| Self {
|
file_paths: &[file_path_for_media_processor::Data],
|
||||||
id,
|
location_id: location::id::Type,
|
||||||
kind,
|
location_path: Arc<PathBuf>,
|
||||||
file_paths,
|
db: Arc<PrismaClient>,
|
||||||
location_id,
|
sync: Arc<SyncManager>,
|
||||||
location_path,
|
) -> Self {
|
||||||
stage,
|
Self::new(
|
||||||
db,
|
Kind::FFmpeg,
|
||||||
output,
|
file_paths,
|
||||||
},
|
location_id,
|
||||||
|
location_path,
|
||||||
|
db,
|
||||||
|
sync,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -355,7 +348,7 @@ async fn fetch_objects_already_with_media_data(
|
||||||
) -> Result<Vec<object::id::Type>, media_processor::Error> {
|
) -> Result<Vec<object::id::Type>, media_processor::Error> {
|
||||||
let object_ids = file_paths
|
let object_ids = file_paths
|
||||||
.iter()
|
.iter()
|
||||||
.filter_map(|file_path| file_path.object_id)
|
.filter_map(|file_path| file_path.object.as_ref().map(|object| object.id))
|
||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
match kind {
|
match kind {
|
||||||
|
@ -388,7 +381,7 @@ fn filter_files_to_extract_media_data(
|
||||||
Output {
|
Output {
|
||||||
skipped, errors, ..
|
skipped, errors, ..
|
||||||
}: &mut Output,
|
}: &mut Output,
|
||||||
) -> HashMap<file_path::id::Type, (PathBuf, object::id::Type)> {
|
) -> HashMap<file_path::id::Type, (PathBuf, object::id::Type, ObjectPubId)> {
|
||||||
let unique_objects_already_with_media_data = objects_already_with_media_data
|
let unique_objects_already_with_media_data = objects_already_with_media_data
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.collect::<HashSet<_>>();
|
.collect::<HashSet<_>>();
|
||||||
|
@ -397,7 +390,7 @@ fn filter_files_to_extract_media_data(
|
||||||
|
|
||||||
file_paths.retain(|file_path| {
|
file_paths.retain(|file_path| {
|
||||||
!unique_objects_already_with_media_data
|
!unique_objects_already_with_media_data
|
||||||
.contains(&file_path.object_id.expect("already checked"))
|
.contains(&file_path.object.as_ref().expect("already checked").id)
|
||||||
});
|
});
|
||||||
|
|
||||||
file_paths
|
file_paths
|
||||||
|
@ -406,8 +399,8 @@ fn filter_files_to_extract_media_data(
|
||||||
IsolatedFilePathData::try_from((location_id, file_path))
|
IsolatedFilePathData::try_from((location_id, file_path))
|
||||||
.map_err(|e| {
|
.map_err(|e| {
|
||||||
errors.push(
|
errors.push(
|
||||||
media_processor::NonCriticalError::from(
|
media_processor::NonCriticalMediaProcessorError::from(
|
||||||
NonCriticalError::FailedToConstructIsolatedFilePathData(
|
NonCriticalMediaDataExtractorError::FailedToConstructIsolatedFilePathData(
|
||||||
file_path.id,
|
file_path.id,
|
||||||
e.to_string(),
|
e.to_string(),
|
||||||
),
|
),
|
||||||
|
@ -416,11 +409,14 @@ fn filter_files_to_extract_media_data(
|
||||||
);
|
);
|
||||||
})
|
})
|
||||||
.map(|iso_file_path| {
|
.map(|iso_file_path| {
|
||||||
|
let object = file_path.object.as_ref().expect("already checked");
|
||||||
|
|
||||||
(
|
(
|
||||||
file_path.id,
|
file_path.id,
|
||||||
(
|
(
|
||||||
location_path.join(iso_file_path),
|
location_path.join(iso_file_path),
|
||||||
file_path.object_id.expect("already checked"),
|
object.id,
|
||||||
|
object.pub_id.as_slice().into(),
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
|
@ -430,13 +426,14 @@ fn filter_files_to_extract_media_data(
|
||||||
}
|
}
|
||||||
|
|
||||||
enum ExtractionOutputKind {
|
enum ExtractionOutputKind {
|
||||||
Exif(Result<Option<ExifMetadata>, media_processor::NonCriticalError>),
|
Exif(Result<Option<ExifMetadata>, media_processor::NonCriticalMediaProcessorError>),
|
||||||
FFmpeg(Result<FFmpegMetadata, media_processor::NonCriticalError>),
|
FFmpeg(Result<FFmpegMetadata, media_processor::NonCriticalMediaProcessorError>),
|
||||||
}
|
}
|
||||||
|
|
||||||
struct ExtractionOutput {
|
struct ExtractionOutput {
|
||||||
file_path_id: file_path::id::Type,
|
file_path_id: file_path::id::Type,
|
||||||
object_id: object::id::Type,
|
object_id: object::id::Type,
|
||||||
|
object_pub_id: ObjectPubId,
|
||||||
kind: ExtractionOutputKind,
|
kind: ExtractionOutputKind,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -453,23 +450,28 @@ enum InterruptRace {
|
||||||
#[inline]
|
#[inline]
|
||||||
fn prepare_extraction_futures<'a>(
|
fn prepare_extraction_futures<'a>(
|
||||||
kind: Kind,
|
kind: Kind,
|
||||||
paths_by_id: &'a HashMap<file_path::id::Type, (PathBuf, object::id::Type)>,
|
paths_by_id: &'a HashMap<file_path::id::Type, (PathBuf, object::id::Type, ObjectPubId)>,
|
||||||
interrupter: &'a Interrupter,
|
interrupter: &'a Interrupter,
|
||||||
) -> FutureGroup<impl Future<Output = InterruptRace> + 'a> {
|
) -> FuturesUnordered<impl Future<Output = InterruptRace> + 'a> {
|
||||||
paths_by_id
|
paths_by_id
|
||||||
.iter()
|
.iter()
|
||||||
.map(|(file_path_id, (path, object_id))| async move {
|
.map(
|
||||||
InterruptRace::Processed(ExtractionOutput {
|
|(file_path_id, (path, object_id, object_pub_id))| async move {
|
||||||
file_path_id: *file_path_id,
|
InterruptRace::Processed(ExtractionOutput {
|
||||||
object_id: *object_id,
|
file_path_id: *file_path_id,
|
||||||
kind: match kind {
|
object_id: *object_id,
|
||||||
Kind::Exif => ExtractionOutputKind::Exif(exif_media_data::extract(path).await),
|
object_pub_id: object_pub_id.clone(),
|
||||||
Kind::FFmpeg => {
|
kind: match kind {
|
||||||
ExtractionOutputKind::FFmpeg(ffmpeg_media_data::extract(path).await)
|
Kind::Exif => {
|
||||||
}
|
ExtractionOutputKind::Exif(exif_media_data::extract(path).await)
|
||||||
},
|
}
|
||||||
})
|
Kind::FFmpeg => {
|
||||||
})
|
ExtractionOutputKind::FFmpeg(ffmpeg_media_data::extract(path).await)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
})
|
||||||
|
},
|
||||||
|
)
|
||||||
.map(|fut| {
|
.map(|fut| {
|
||||||
(
|
(
|
||||||
fut,
|
fut,
|
||||||
|
@ -477,24 +479,28 @@ fn prepare_extraction_futures<'a>(
|
||||||
)
|
)
|
||||||
.race()
|
.race()
|
||||||
})
|
})
|
||||||
.collect::<FutureGroup<_>>()
|
.collect::<FuturesUnordered<_>>()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[instrument(skip_all, fields(%file_path_id, %object_id))]
|
||||||
#[inline]
|
#[inline]
|
||||||
fn process_output(
|
fn process_output(
|
||||||
ExtractionOutput {
|
ExtractionOutput {
|
||||||
file_path_id,
|
file_path_id,
|
||||||
object_id,
|
object_id,
|
||||||
|
object_pub_id,
|
||||||
kind,
|
kind,
|
||||||
}: ExtractionOutput,
|
}: ExtractionOutput,
|
||||||
exif_media_datas: &mut Vec<(ExifMetadata, object::id::Type)>,
|
exif_media_datas: &mut Vec<(ExifMetadata, object::id::Type, ObjectPubId)>,
|
||||||
ffmpeg_media_datas: &mut Vec<(FFmpegMetadata, object::id::Type)>,
|
ffmpeg_media_datas: &mut Vec<(FFmpegMetadata, object::id::Type)>,
|
||||||
extract_ids_to_remove_from_map: &mut Vec<file_path::id::Type>,
|
extract_ids_to_remove_from_map: &mut Vec<file_path::id::Type>,
|
||||||
output: &mut Output,
|
output: &mut Output,
|
||||||
) {
|
) {
|
||||||
|
trace!("Processing extracted media data");
|
||||||
|
|
||||||
match kind {
|
match kind {
|
||||||
ExtractionOutputKind::Exif(Ok(Some(exif_data))) => {
|
ExtractionOutputKind::Exif(Ok(Some(exif_data))) => {
|
||||||
exif_media_datas.push((exif_data, object_id));
|
exif_media_datas.push((exif_data, object_id, object_pub_id));
|
||||||
}
|
}
|
||||||
ExtractionOutputKind::Exif(Ok(None)) => {
|
ExtractionOutputKind::Exif(Ok(None)) => {
|
||||||
// No exif media data found
|
// No exif media data found
|
||||||
|
@ -514,12 +520,85 @@ fn process_output(
|
||||||
#[inline]
|
#[inline]
|
||||||
async fn save(
|
async fn save(
|
||||||
kind: Kind,
|
kind: Kind,
|
||||||
exif_media_datas: &mut Vec<(ExifMetadata, object::id::Type)>,
|
exif_media_datas: &mut Vec<(ExifMetadata, object::id::Type, ObjectPubId)>,
|
||||||
ffmpeg_media_datas: &mut Vec<(FFmpegMetadata, object::id::Type)>,
|
ffmpeg_media_datas: &mut Vec<(FFmpegMetadata, object::id::Type)>,
|
||||||
db: &PrismaClient,
|
db: &PrismaClient,
|
||||||
|
sync: &SyncManager,
|
||||||
) -> Result<u64, media_processor::Error> {
|
) -> Result<u64, media_processor::Error> {
|
||||||
|
trace!("Saving media data on database");
|
||||||
|
|
||||||
match kind {
|
match kind {
|
||||||
Kind::Exif => exif_media_data::save(mem::take(exif_media_datas), db).await,
|
Kind::Exif => exif_media_data::save(mem::take(exif_media_datas), db, sync).await,
|
||||||
Kind::FFmpeg => ffmpeg_media_data::save(mem::take(ffmpeg_media_datas), db).await,
|
Kind::FFmpeg => ffmpeg_media_data::save(mem::take(ffmpeg_media_datas), db).await,
|
||||||
}
|
}
|
||||||
|
.map_err(Into::into)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize, Deserialize)]
|
||||||
|
struct SaveState {
|
||||||
|
id: TaskId,
|
||||||
|
kind: Kind,
|
||||||
|
file_paths: Vec<file_path_for_media_processor::Data>,
|
||||||
|
location_id: location::id::Type,
|
||||||
|
location_path: Arc<PathBuf>,
|
||||||
|
stage: Stage,
|
||||||
|
output: Output,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl SerializableTask<Error> for MediaDataExtractor {
|
||||||
|
type SerializeError = rmp_serde::encode::Error;
|
||||||
|
|
||||||
|
type DeserializeError = rmp_serde::decode::Error;
|
||||||
|
|
||||||
|
type DeserializeCtx = (Arc<PrismaClient>, Arc<SyncManager>);
|
||||||
|
|
||||||
|
async fn serialize(self) -> Result<Vec<u8>, Self::SerializeError> {
|
||||||
|
let Self {
|
||||||
|
id,
|
||||||
|
kind,
|
||||||
|
file_paths,
|
||||||
|
location_id,
|
||||||
|
location_path,
|
||||||
|
stage,
|
||||||
|
output,
|
||||||
|
..
|
||||||
|
} = self;
|
||||||
|
|
||||||
|
rmp_serde::to_vec_named(&SaveState {
|
||||||
|
id,
|
||||||
|
kind,
|
||||||
|
file_paths,
|
||||||
|
location_id,
|
||||||
|
location_path,
|
||||||
|
stage,
|
||||||
|
output,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn deserialize(
|
||||||
|
data: &[u8],
|
||||||
|
(db, sync): Self::DeserializeCtx,
|
||||||
|
) -> Result<Self, Self::DeserializeError> {
|
||||||
|
rmp_serde::from_slice(data).map(
|
||||||
|
|SaveState {
|
||||||
|
id,
|
||||||
|
kind,
|
||||||
|
file_paths,
|
||||||
|
location_id,
|
||||||
|
location_path,
|
||||||
|
stage,
|
||||||
|
output,
|
||||||
|
}| Self {
|
||||||
|
id,
|
||||||
|
kind,
|
||||||
|
file_paths,
|
||||||
|
location_id,
|
||||||
|
location_path,
|
||||||
|
stage,
|
||||||
|
output,
|
||||||
|
db,
|
||||||
|
sync,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -12,8 +12,7 @@ use crate::{
|
||||||
media_processor::{
|
media_processor::{
|
||||||
self,
|
self,
|
||||||
helpers::thumbnailer::{
|
helpers::thumbnailer::{
|
||||||
can_generate_thumbnail_for_document, can_generate_thumbnail_for_image, get_shard_hex,
|
generate_thumbnail, GenerateThumbnailArgs, GenerationStatus, THUMBNAILER_TASK_TIMEOUT,
|
||||||
EPHEMERAL_DIR, TARGET_PX, TARGET_QUALITY, THUMBNAIL_GENERATION_TIMEOUT, WEBP_EXTENSION,
|
|
||||||
},
|
},
|
||||||
ThumbKey, ThumbnailKind,
|
ThumbKey, ThumbnailKind,
|
||||||
},
|
},
|
||||||
|
@ -21,61 +20,31 @@ use crate::{
|
||||||
};
|
};
|
||||||
|
|
||||||
use sd_core_file_path_helper::IsolatedFilePathData;
|
use sd_core_file_path_helper::IsolatedFilePathData;
|
||||||
use sd_core_prisma_helpers::file_path_for_media_processor;
|
use sd_core_prisma_helpers::{file_path_for_media_processor, CasId};
|
||||||
|
|
||||||
use sd_file_ext::extensions::{DocumentExtension, ImageExtension};
|
|
||||||
use sd_images::{format_image, scale_dimensions, ConvertibleExtension};
|
|
||||||
use sd_media_metadata::exif::Orientation;
|
|
||||||
use sd_prisma::prisma::{file_path, location};
|
use sd_prisma::prisma::{file_path, location};
|
||||||
use sd_task_system::{
|
use sd_task_system::{
|
||||||
ExecStatus, Interrupter, InterruptionKind, IntoAnyTaskOutput, SerializableTask, Task, TaskId,
|
ExecStatus, Interrupter, InterruptionKind, IntoAnyTaskOutput, SerializableTask, Task, TaskId,
|
||||||
};
|
};
|
||||||
use sd_utils::error::FileIOError;
|
|
||||||
|
|
||||||
use std::{
|
use std::{
|
||||||
collections::HashMap,
|
collections::HashMap,
|
||||||
fmt,
|
fmt,
|
||||||
future::IntoFuture,
|
future::IntoFuture,
|
||||||
mem,
|
mem,
|
||||||
ops::Deref,
|
|
||||||
path::{Path, PathBuf},
|
path::{Path, PathBuf},
|
||||||
pin::pin,
|
pin::pin,
|
||||||
str::FromStr,
|
|
||||||
sync::Arc,
|
sync::Arc,
|
||||||
time::Duration,
|
time::Duration,
|
||||||
};
|
};
|
||||||
|
|
||||||
use futures::{FutureExt, StreamExt};
|
use futures::{stream::FuturesUnordered, FutureExt, StreamExt};
|
||||||
use futures_concurrency::future::{FutureGroup, Race};
|
use futures_concurrency::future::Race;
|
||||||
use image::{imageops, DynamicImage, GenericImageView};
|
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use specta::Type;
|
use specta::Type;
|
||||||
use tokio::{
|
use tokio::time::Instant;
|
||||||
fs, io,
|
use tracing::{error, instrument, trace, Level};
|
||||||
task::spawn_blocking,
|
|
||||||
time::{sleep, Instant},
|
|
||||||
};
|
|
||||||
use tracing::{error, info, trace};
|
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
use webp::Encoder;
|
|
||||||
|
|
||||||
#[derive(Debug, Serialize, Deserialize)]
|
|
||||||
pub struct GenerateThumbnailArgs {
|
|
||||||
pub extension: String,
|
|
||||||
pub cas_id: String,
|
|
||||||
pub path: PathBuf,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl GenerateThumbnailArgs {
|
|
||||||
#[must_use]
|
|
||||||
pub const fn new(extension: String, cas_id: String, path: PathBuf) -> Self {
|
|
||||||
Self {
|
|
||||||
extension,
|
|
||||||
cas_id,
|
|
||||||
path,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub type ThumbnailId = u32;
|
pub type ThumbnailId = u32;
|
||||||
|
|
||||||
|
@ -84,20 +53,29 @@ pub trait NewThumbnailReporter: Send + Sync + fmt::Debug + 'static {
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
pub struct Thumbnailer<Reporter: NewThumbnailReporter> {
|
pub struct Thumbnailer {
|
||||||
|
// Task control
|
||||||
id: TaskId,
|
id: TaskId,
|
||||||
reporter: Arc<Reporter>,
|
with_priority: bool,
|
||||||
|
|
||||||
|
// Received input args
|
||||||
thumbs_kind: ThumbnailKind,
|
thumbs_kind: ThumbnailKind,
|
||||||
thumbnails_directory_path: Arc<PathBuf>,
|
thumbnails_directory_path: Arc<PathBuf>,
|
||||||
thumbnails_to_generate: HashMap<ThumbnailId, GenerateThumbnailArgs>,
|
thumbnails_to_generate: HashMap<ThumbnailId, GenerateThumbnailArgs<'static>>,
|
||||||
already_processed_ids: Vec<ThumbnailId>,
|
|
||||||
should_regenerate: bool,
|
should_regenerate: bool,
|
||||||
with_priority: bool,
|
|
||||||
|
// Inner state
|
||||||
|
already_processed_ids: Vec<ThumbnailId>,
|
||||||
|
|
||||||
|
// Out collector
|
||||||
output: Output,
|
output: Output,
|
||||||
|
|
||||||
|
// Dependencies
|
||||||
|
reporter: Arc<dyn NewThumbnailReporter>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[async_trait::async_trait]
|
#[async_trait::async_trait]
|
||||||
impl<Reporter: NewThumbnailReporter> Task<Error> for Thumbnailer<Reporter> {
|
impl Task<Error> for Thumbnailer {
|
||||||
fn id(&self) -> TaskId {
|
fn id(&self) -> TaskId {
|
||||||
self.id
|
self.id
|
||||||
}
|
}
|
||||||
|
@ -107,9 +85,23 @@ impl<Reporter: NewThumbnailReporter> Task<Error> for Thumbnailer<Reporter> {
|
||||||
}
|
}
|
||||||
|
|
||||||
fn with_timeout(&self) -> Option<Duration> {
|
fn with_timeout(&self) -> Option<Duration> {
|
||||||
Some(Duration::from_secs(60 * 5)) // The entire task must not take more than 5 minutes
|
Some(THUMBNAILER_TASK_TIMEOUT) // The entire task must not take more than this constant
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[instrument(
|
||||||
|
skip_all,
|
||||||
|
fields(
|
||||||
|
task_id = %self.id,
|
||||||
|
thumbs_kind = ?self.thumbs_kind,
|
||||||
|
should_regenerate = self.should_regenerate,
|
||||||
|
thumbnails_to_generate_count = self.thumbnails_to_generate.len(),
|
||||||
|
already_processed_ids_count = self.already_processed_ids.len(),
|
||||||
|
with_priority = self.with_priority,
|
||||||
|
),
|
||||||
|
ret(level = Level::TRACE),
|
||||||
|
err,
|
||||||
|
)]
|
||||||
|
#[allow(clippy::blocks_in_conditions)] // Due to `err` on `instrument` macro above
|
||||||
async fn run(&mut self, interrupter: &Interrupter) -> Result<ExecStatus, Error> {
|
async fn run(&mut self, interrupter: &Interrupter) -> Result<ExecStatus, Error> {
|
||||||
enum InterruptRace {
|
enum InterruptRace {
|
||||||
Interrupted(InterruptionKind),
|
Interrupted(InterruptionKind),
|
||||||
|
@ -135,38 +127,27 @@ impl<Reporter: NewThumbnailReporter> Task<Error> for Thumbnailer<Reporter> {
|
||||||
|
|
||||||
let start = Instant::now();
|
let start = Instant::now();
|
||||||
|
|
||||||
let mut futures = pin!(thumbnails_to_generate
|
let futures = thumbnails_to_generate
|
||||||
.iter()
|
.iter()
|
||||||
.map(|(id, generate_args)| {
|
.map(|(id, generate_args)| {
|
||||||
let path = &generate_args.path;
|
generate_thumbnail(
|
||||||
|
thumbnails_directory_path,
|
||||||
|
generate_args,
|
||||||
|
thumbs_kind,
|
||||||
|
*should_regenerate,
|
||||||
|
)
|
||||||
|
.map(|res| InterruptRace::Processed((*id, res)))
|
||||||
|
})
|
||||||
|
.map(|fut| {
|
||||||
(
|
(
|
||||||
generate_thumbnail(
|
fut,
|
||||||
thumbnails_directory_path,
|
interrupter.into_future().map(InterruptRace::Interrupted),
|
||||||
generate_args,
|
|
||||||
thumbs_kind,
|
|
||||||
*should_regenerate,
|
|
||||||
)
|
|
||||||
.map(|res| (*id, res)),
|
|
||||||
sleep(THUMBNAIL_GENERATION_TIMEOUT).map(|()| {
|
|
||||||
(
|
|
||||||
*id,
|
|
||||||
(
|
|
||||||
THUMBNAIL_GENERATION_TIMEOUT,
|
|
||||||
Err(NonCriticalError::ThumbnailGenerationTimeout(path.clone())),
|
|
||||||
),
|
|
||||||
)
|
|
||||||
}),
|
|
||||||
)
|
)
|
||||||
.race()
|
.race()
|
||||||
.map(InterruptRace::Processed)
|
|
||||||
})
|
})
|
||||||
.map(|fut| (
|
.collect::<FuturesUnordered<_>>();
|
||||||
fut,
|
|
||||||
interrupter.into_future().map(InterruptRace::Interrupted)
|
let mut futures = pin!(futures);
|
||||||
)
|
|
||||||
.race())
|
|
||||||
.collect::<FutureGroup<_>>());
|
|
||||||
|
|
||||||
while let Some(race_output) = futures.next().await {
|
while let Some(race_output) = futures.next().await {
|
||||||
match race_output {
|
match race_output {
|
||||||
|
@ -190,25 +171,25 @@ impl<Reporter: NewThumbnailReporter> Task<Error> for Thumbnailer<Reporter> {
|
||||||
|
|
||||||
output.total_time += start.elapsed();
|
output.total_time += start.elapsed();
|
||||||
|
|
||||||
#[allow(clippy::cast_precision_loss)]
|
if output.generated > 1 {
|
||||||
// SAFETY: we're probably won't have 2^52 thumbnails being generated on a single task for this cast to have
|
#[allow(clippy::cast_precision_loss)]
|
||||||
// a precision loss issue
|
// SAFETY: we're probably won't have 2^52 thumbnails being generated on a single task for this cast to have
|
||||||
let total = (output.generated + output.skipped) as f64;
|
// a precision loss issue
|
||||||
|
let total = (output.generated + output.skipped) as f64;
|
||||||
|
let mean_generation_time_f64 = output.mean_time_acc / total;
|
||||||
|
|
||||||
let mean_generation_time = output.mean_time_acc / total;
|
trace!(
|
||||||
|
generated = output.generated,
|
||||||
let generation_time_std_dev = Duration::from_secs_f64(
|
skipped = output.skipped,
|
||||||
(mean_generation_time.mul_add(-mean_generation_time, output.std_dev_acc / total))
|
"mean generation time: {mean_generation_time:?} ± {generation_time_std_dev:?};",
|
||||||
.sqrt(),
|
mean_generation_time = Duration::from_secs_f64(mean_generation_time_f64),
|
||||||
);
|
generation_time_std_dev = Duration::from_secs_f64(
|
||||||
|
(mean_generation_time_f64
|
||||||
info!(
|
.mul_add(-mean_generation_time_f64, output.std_dev_acc / total))
|
||||||
"{{generated: {generated}, skipped: {skipped}}} thumbnails; \
|
.sqrt(),
|
||||||
mean generation time: {mean_generation_time:?} ± {generation_time_std_dev:?}",
|
)
|
||||||
generated = output.generated,
|
);
|
||||||
skipped = output.skipped,
|
}
|
||||||
mean_generation_time = Duration::from_secs_f64(mean_generation_time)
|
|
||||||
);
|
|
||||||
|
|
||||||
Ok(ExecStatus::Done(mem::take(output).into_output()))
|
Ok(ExecStatus::Done(mem::take(output).into_output()))
|
||||||
}
|
}
|
||||||
|
@ -224,8 +205,8 @@ pub struct Output {
|
||||||
pub std_dev_acc: f64,
|
pub std_dev_acc: f64,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(thiserror::Error, Debug, Serialize, Deserialize, Type)]
|
#[derive(thiserror::Error, Debug, Serialize, Deserialize, Type, Clone)]
|
||||||
pub enum NonCriticalError {
|
pub enum NonCriticalThumbnailerError {
|
||||||
#[error("file path <id='{0}'> has no cas_id")]
|
#[error("file path <id='{0}'> has no cas_id")]
|
||||||
MissingCasId(file_path::id::Type),
|
MissingCasId(file_path::id::Type),
|
||||||
#[error("failed to extract isolated file path data from file path <id='{0}'>: {1}")]
|
#[error("failed to extract isolated file path data from file path <id='{0}'>: {1}")]
|
||||||
|
@ -242,19 +223,19 @@ pub enum NonCriticalError {
|
||||||
CreateShardDirectory(String),
|
CreateShardDirectory(String),
|
||||||
#[error("failed to save thumbnail <path='{}'>: {1}", .0.display())]
|
#[error("failed to save thumbnail <path='{}'>: {1}", .0.display())]
|
||||||
SaveThumbnail(PathBuf, String),
|
SaveThumbnail(PathBuf, String),
|
||||||
#[error("thumbnail generation timed out <path='{}'>", .0.display())]
|
#[error("task timed out: {0}")]
|
||||||
ThumbnailGenerationTimeout(PathBuf),
|
TaskTimeout(TaskId),
|
||||||
}
|
}
|
||||||
|
|
||||||
impl<Reporter: NewThumbnailReporter> Thumbnailer<Reporter> {
|
impl Thumbnailer {
|
||||||
fn new(
|
fn new(
|
||||||
thumbs_kind: ThumbnailKind,
|
thumbs_kind: ThumbnailKind,
|
||||||
thumbnails_directory_path: Arc<PathBuf>,
|
thumbnails_directory_path: Arc<PathBuf>,
|
||||||
thumbnails_to_generate: HashMap<ThumbnailId, GenerateThumbnailArgs>,
|
thumbnails_to_generate: HashMap<ThumbnailId, GenerateThumbnailArgs<'static>>,
|
||||||
errors: Vec<crate::NonCriticalError>,
|
errors: Vec<crate::NonCriticalError>,
|
||||||
should_regenerate: bool,
|
should_regenerate: bool,
|
||||||
with_priority: bool,
|
with_priority: bool,
|
||||||
reporter: Arc<Reporter>,
|
reporter: Arc<dyn NewThumbnailReporter>,
|
||||||
) -> Self {
|
) -> Self {
|
||||||
Self {
|
Self {
|
||||||
id: TaskId::new_v4(),
|
id: TaskId::new_v4(),
|
||||||
|
@ -275,8 +256,8 @@ impl<Reporter: NewThumbnailReporter> Thumbnailer<Reporter> {
|
||||||
#[must_use]
|
#[must_use]
|
||||||
pub fn new_ephemeral(
|
pub fn new_ephemeral(
|
||||||
thumbnails_directory_path: Arc<PathBuf>,
|
thumbnails_directory_path: Arc<PathBuf>,
|
||||||
thumbnails_to_generate: Vec<GenerateThumbnailArgs>,
|
thumbnails_to_generate: Vec<GenerateThumbnailArgs<'static>>,
|
||||||
reporter: Arc<Reporter>,
|
reporter: Arc<dyn NewThumbnailReporter>,
|
||||||
) -> Self {
|
) -> Self {
|
||||||
Self::new(
|
Self::new(
|
||||||
ThumbnailKind::Ephemeral,
|
ThumbnailKind::Ephemeral,
|
||||||
|
@ -308,7 +289,7 @@ impl<Reporter: NewThumbnailReporter> Thumbnailer<Reporter> {
|
||||||
library_id: Uuid,
|
library_id: Uuid,
|
||||||
should_regenerate: bool,
|
should_regenerate: bool,
|
||||||
with_priority: bool,
|
with_priority: bool,
|
||||||
reporter: Arc<Reporter>,
|
reporter: Arc<dyn NewThumbnailReporter>,
|
||||||
) -> Self {
|
) -> Self {
|
||||||
let mut errors = Vec::new();
|
let mut errors = Vec::new();
|
||||||
|
|
||||||
|
@ -318,13 +299,18 @@ impl<Reporter: NewThumbnailReporter> Thumbnailer<Reporter> {
|
||||||
file_paths
|
file_paths
|
||||||
.iter()
|
.iter()
|
||||||
.filter_map(|file_path| {
|
.filter_map(|file_path| {
|
||||||
if let Some(cas_id) = file_path.cas_id.as_ref() {
|
if let Some(cas_id) = file_path
|
||||||
|
.cas_id
|
||||||
|
.as_ref()
|
||||||
|
.map(CasId::from)
|
||||||
|
.map(CasId::into_owned)
|
||||||
|
{
|
||||||
let file_path_id = file_path.id;
|
let file_path_id = file_path.id;
|
||||||
IsolatedFilePathData::try_from((location_id, file_path))
|
IsolatedFilePathData::try_from((location_id, file_path))
|
||||||
.map_err(|e| {
|
.map_err(|e| {
|
||||||
errors.push(
|
errors.push(
|
||||||
media_processor::NonCriticalError::from(
|
media_processor::NonCriticalMediaProcessorError::from(
|
||||||
NonCriticalError::FailedToExtractIsolatedFilePathData(
|
NonCriticalThumbnailerError::FailedToExtractIsolatedFilePathData(
|
||||||
file_path_id,
|
file_path_id,
|
||||||
e.to_string(),
|
e.to_string(),
|
||||||
),
|
),
|
||||||
|
@ -336,8 +322,8 @@ impl<Reporter: NewThumbnailReporter> Thumbnailer<Reporter> {
|
||||||
.map(|iso_file_path| (file_path_id, cas_id, iso_file_path))
|
.map(|iso_file_path| (file_path_id, cas_id, iso_file_path))
|
||||||
} else {
|
} else {
|
||||||
errors.push(
|
errors.push(
|
||||||
media_processor::NonCriticalError::from(
|
media_processor::NonCriticalMediaProcessorError::from(
|
||||||
NonCriticalError::MissingCasId(file_path.id),
|
NonCriticalThumbnailerError::MissingCasId(file_path.id),
|
||||||
)
|
)
|
||||||
.into(),
|
.into(),
|
||||||
);
|
);
|
||||||
|
@ -354,7 +340,7 @@ impl<Reporter: NewThumbnailReporter> Thumbnailer<Reporter> {
|
||||||
file_path_id as u32,
|
file_path_id as u32,
|
||||||
GenerateThumbnailArgs::new(
|
GenerateThumbnailArgs::new(
|
||||||
iso_file_path.extension().to_string(),
|
iso_file_path.extension().to_string(),
|
||||||
cas_id.clone(),
|
cas_id,
|
||||||
full_path,
|
full_path,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
@ -369,23 +355,74 @@ impl<Reporter: NewThumbnailReporter> Thumbnailer<Reporter> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[instrument(skip_all, fields(thumb_id = id, %generated, %skipped, ?elapsed_time, ?res))]
|
||||||
|
fn process_thumbnail_generation_output(
|
||||||
|
(id, (elapsed_time, res)): ThumbnailGenerationOutput,
|
||||||
|
with_priority: bool,
|
||||||
|
reporter: &dyn NewThumbnailReporter,
|
||||||
|
already_processed_ids: &mut Vec<ThumbnailId>,
|
||||||
|
Output {
|
||||||
|
generated,
|
||||||
|
skipped,
|
||||||
|
errors,
|
||||||
|
mean_time_acc: mean_generation_time_accumulator,
|
||||||
|
std_dev_acc: std_dev_accumulator,
|
||||||
|
..
|
||||||
|
}: &mut Output,
|
||||||
|
) {
|
||||||
|
let elapsed_time = elapsed_time.as_secs_f64();
|
||||||
|
*mean_generation_time_accumulator += elapsed_time;
|
||||||
|
*std_dev_accumulator += elapsed_time * elapsed_time;
|
||||||
|
|
||||||
|
match res {
|
||||||
|
Ok((thumb_key, status)) => {
|
||||||
|
match status {
|
||||||
|
GenerationStatus::Generated => {
|
||||||
|
*generated += 1;
|
||||||
|
}
|
||||||
|
GenerationStatus::Skipped => {
|
||||||
|
*skipped += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// This if is REALLY needed, due to the sheer performance of the thumbnailer,
|
||||||
|
// I restricted to only send events notifying for thumbnails in the current
|
||||||
|
// opened directory, sending events for the entire location turns into a
|
||||||
|
// humongous bottleneck in the frontend lol, since it doesn't even knows
|
||||||
|
// what to do with thumbnails for inner directories lol
|
||||||
|
// - fogodev
|
||||||
|
if with_priority {
|
||||||
|
reporter.new_thumbnail(thumb_key);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
errors.push(media_processor::NonCriticalMediaProcessorError::from(e).into());
|
||||||
|
*skipped += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
already_processed_ids.push(id);
|
||||||
|
|
||||||
|
trace!("Thumbnail processed");
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Serialize, Deserialize)]
|
#[derive(Debug, Serialize, Deserialize)]
|
||||||
struct SaveState {
|
struct SaveState {
|
||||||
id: TaskId,
|
id: TaskId,
|
||||||
thumbs_kind: ThumbnailKind,
|
thumbs_kind: ThumbnailKind,
|
||||||
thumbnails_directory_path: Arc<PathBuf>,
|
thumbnails_directory_path: Arc<PathBuf>,
|
||||||
thumbnails_to_generate: HashMap<ThumbnailId, GenerateThumbnailArgs>,
|
thumbnails_to_generate: HashMap<ThumbnailId, GenerateThumbnailArgs<'static>>,
|
||||||
should_regenerate: bool,
|
should_regenerate: bool,
|
||||||
with_priority: bool,
|
with_priority: bool,
|
||||||
output: Output,
|
output: Output,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl<Reporter: NewThumbnailReporter> SerializableTask<Error> for Thumbnailer<Reporter> {
|
impl SerializableTask<Error> for Thumbnailer {
|
||||||
type SerializeError = rmp_serde::encode::Error;
|
type SerializeError = rmp_serde::encode::Error;
|
||||||
|
|
||||||
type DeserializeError = rmp_serde::decode::Error;
|
type DeserializeError = rmp_serde::decode::Error;
|
||||||
|
|
||||||
type DeserializeCtx = Arc<Reporter>;
|
type DeserializeCtx = Arc<dyn NewThumbnailReporter>;
|
||||||
|
|
||||||
async fn serialize(self) -> Result<Vec<u8>, Self::SerializeError> {
|
async fn serialize(self) -> Result<Vec<u8>, Self::SerializeError> {
|
||||||
let Self {
|
let Self {
|
||||||
|
@ -443,235 +480,10 @@ impl<Reporter: NewThumbnailReporter> SerializableTask<Error> for Thumbnailer<Rep
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
enum GenerationStatus {
|
|
||||||
Generated,
|
|
||||||
Skipped,
|
|
||||||
}
|
|
||||||
|
|
||||||
type ThumbnailGenerationOutput = (
|
type ThumbnailGenerationOutput = (
|
||||||
ThumbnailId,
|
ThumbnailId,
|
||||||
(
|
(
|
||||||
Duration,
|
Duration,
|
||||||
Result<(ThumbKey, GenerationStatus), NonCriticalError>,
|
Result<(ThumbKey, GenerationStatus), NonCriticalThumbnailerError>,
|
||||||
),
|
),
|
||||||
);
|
);
|
||||||
|
|
||||||
fn process_thumbnail_generation_output(
|
|
||||||
(id, (elapsed_time, res)): ThumbnailGenerationOutput,
|
|
||||||
with_priority: bool,
|
|
||||||
reporter: &impl NewThumbnailReporter,
|
|
||||||
already_processed_ids: &mut Vec<ThumbnailId>,
|
|
||||||
Output {
|
|
||||||
generated,
|
|
||||||
skipped,
|
|
||||||
errors,
|
|
||||||
mean_time_acc: mean_generation_time_accumulator,
|
|
||||||
std_dev_acc: std_dev_accumulator,
|
|
||||||
..
|
|
||||||
}: &mut Output,
|
|
||||||
) {
|
|
||||||
let elapsed_time = elapsed_time.as_secs_f64();
|
|
||||||
*mean_generation_time_accumulator += elapsed_time;
|
|
||||||
*std_dev_accumulator += elapsed_time * elapsed_time;
|
|
||||||
|
|
||||||
match res {
|
|
||||||
Ok((thumb_key, status)) => {
|
|
||||||
match status {
|
|
||||||
GenerationStatus::Generated => {
|
|
||||||
*generated += 1;
|
|
||||||
}
|
|
||||||
GenerationStatus::Skipped => {
|
|
||||||
*skipped += 1;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// This if is REALLY needed, due to the sheer performance of the thumbnailer,
|
|
||||||
// I restricted to only send events notifying for thumbnails in the current
|
|
||||||
// opened directory, sending events for the entire location turns into a
|
|
||||||
// humongous bottleneck in the frontend lol, since it doesn't even knows
|
|
||||||
// what to do with thumbnails for inner directories lol
|
|
||||||
// - fogodev
|
|
||||||
if with_priority {
|
|
||||||
reporter.new_thumbnail(thumb_key);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Err(e) => {
|
|
||||||
errors.push(media_processor::NonCriticalError::from(e).into());
|
|
||||||
*skipped += 1;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
already_processed_ids.push(id);
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn generate_thumbnail(
|
|
||||||
thumbnails_directory: &Path,
|
|
||||||
GenerateThumbnailArgs {
|
|
||||||
extension,
|
|
||||||
cas_id,
|
|
||||||
path,
|
|
||||||
}: &GenerateThumbnailArgs,
|
|
||||||
kind: &ThumbnailKind,
|
|
||||||
should_regenerate: bool,
|
|
||||||
) -> (
|
|
||||||
Duration,
|
|
||||||
Result<(ThumbKey, GenerationStatus), NonCriticalError>,
|
|
||||||
) {
|
|
||||||
trace!("Generating thumbnail for {}", path.display());
|
|
||||||
let start = Instant::now();
|
|
||||||
|
|
||||||
let mut output_path = match kind {
|
|
||||||
ThumbnailKind::Ephemeral => thumbnails_directory.join(EPHEMERAL_DIR),
|
|
||||||
ThumbnailKind::Indexed(library_id) => thumbnails_directory.join(library_id.to_string()),
|
|
||||||
};
|
|
||||||
|
|
||||||
output_path.push(get_shard_hex(cas_id));
|
|
||||||
output_path.push(cas_id);
|
|
||||||
output_path.set_extension(WEBP_EXTENSION);
|
|
||||||
|
|
||||||
if let Err(e) = fs::metadata(&*output_path).await {
|
|
||||||
if e.kind() != io::ErrorKind::NotFound {
|
|
||||||
error!(
|
|
||||||
"Failed to check if thumbnail exists, but we will try to generate it anyway: {e:#?}"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
// Otherwise we good, thumbnail doesn't exist so we can generate it
|
|
||||||
} else if !should_regenerate {
|
|
||||||
trace!(
|
|
||||||
"Skipping thumbnail generation for {} because it already exists",
|
|
||||||
path.display()
|
|
||||||
);
|
|
||||||
return (
|
|
||||||
start.elapsed(),
|
|
||||||
Ok((ThumbKey::new(cas_id, kind), GenerationStatus::Skipped)),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
if let Ok(extension) = ImageExtension::from_str(extension) {
|
|
||||||
if can_generate_thumbnail_for_image(extension) {
|
|
||||||
if let Err(e) = generate_image_thumbnail(&path, &output_path).await {
|
|
||||||
return (start.elapsed(), Err(e));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else if let Ok(extension) = DocumentExtension::from_str(extension) {
|
|
||||||
if can_generate_thumbnail_for_document(extension) {
|
|
||||||
if let Err(e) = generate_image_thumbnail(&path, &output_path).await {
|
|
||||||
return (start.elapsed(), Err(e));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(feature = "ffmpeg")]
|
|
||||||
{
|
|
||||||
use crate::media_processor::helpers::thumbnailer::can_generate_thumbnail_for_video;
|
|
||||||
use sd_file_ext::extensions::VideoExtension;
|
|
||||||
|
|
||||||
if let Ok(extension) = VideoExtension::from_str(extension) {
|
|
||||||
if can_generate_thumbnail_for_video(extension) {
|
|
||||||
if let Err(e) = generate_video_thumbnail(&path, &output_path).await {
|
|
||||||
return (start.elapsed(), Err(e));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
trace!("Generated thumbnail for {}", path.display());
|
|
||||||
|
|
||||||
(
|
|
||||||
start.elapsed(),
|
|
||||||
Ok((ThumbKey::new(cas_id, kind), GenerationStatus::Generated)),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn generate_image_thumbnail(
|
|
||||||
file_path: impl AsRef<Path> + Send,
|
|
||||||
output_path: impl AsRef<Path> + Send,
|
|
||||||
) -> Result<(), NonCriticalError> {
|
|
||||||
let file_path = file_path.as_ref().to_path_buf();
|
|
||||||
|
|
||||||
let webp = spawn_blocking({
|
|
||||||
let file_path = file_path.clone();
|
|
||||||
|
|
||||||
move || -> Result<_, NonCriticalError> {
|
|
||||||
let mut img = format_image(&file_path)
|
|
||||||
.map_err(|e| NonCriticalError::FormatImage(file_path.clone(), e.to_string()))?;
|
|
||||||
|
|
||||||
let (w, h) = img.dimensions();
|
|
||||||
|
|
||||||
#[allow(clippy::cast_precision_loss)]
|
|
||||||
let (w_scaled, h_scaled) = scale_dimensions(w as f32, h as f32, TARGET_PX);
|
|
||||||
|
|
||||||
// Optionally, resize the existing photo and convert back into DynamicImage
|
|
||||||
if w != w_scaled && h != h_scaled {
|
|
||||||
img = DynamicImage::ImageRgba8(imageops::resize(
|
|
||||||
&img,
|
|
||||||
w_scaled,
|
|
||||||
h_scaled,
|
|
||||||
imageops::FilterType::Triangle,
|
|
||||||
));
|
|
||||||
}
|
|
||||||
|
|
||||||
// this corrects the rotation/flip of the image based on the *available* exif data
|
|
||||||
// not all images have exif data, so we don't error. we also don't rotate HEIF as that's against the spec
|
|
||||||
if let Some(orientation) = Orientation::from_path(&file_path) {
|
|
||||||
if ConvertibleExtension::try_from(file_path.as_ref())
|
|
||||||
.expect("we already checked if the image was convertible")
|
|
||||||
.should_rotate()
|
|
||||||
{
|
|
||||||
img = orientation.correct_thumbnail(img);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Create the WebP encoder for the above image
|
|
||||||
let encoder = Encoder::from_image(&img)
|
|
||||||
.map_err(|reason| NonCriticalError::WebPEncoding(file_path, reason.to_string()))?;
|
|
||||||
|
|
||||||
// Type `WebPMemory` is !Send, which makes the `Future` in this function `!Send`,
|
|
||||||
// this make us `deref` to have a `&[u8]` and then `to_owned` to make a `Vec<u8>`
|
|
||||||
// which implies on a unwanted clone...
|
|
||||||
Ok(encoder.encode(TARGET_QUALITY).deref().to_owned())
|
|
||||||
}
|
|
||||||
})
|
|
||||||
.await
|
|
||||||
.map_err(|e| {
|
|
||||||
NonCriticalError::PanicWhileGeneratingThumbnail(file_path.clone(), e.to_string())
|
|
||||||
})??;
|
|
||||||
|
|
||||||
let output_path = output_path.as_ref();
|
|
||||||
|
|
||||||
if let Some(shard_dir) = output_path.parent() {
|
|
||||||
fs::create_dir_all(shard_dir).await.map_err(|e| {
|
|
||||||
NonCriticalError::CreateShardDirectory(FileIOError::from((shard_dir, e)).to_string())
|
|
||||||
})?;
|
|
||||||
} else {
|
|
||||||
error!(
|
|
||||||
"Failed to get parent directory of '{}' for sharding parent directory",
|
|
||||||
output_path.display()
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
fs::write(output_path, &webp).await.map_err(|e| {
|
|
||||||
NonCriticalError::SaveThumbnail(file_path, FileIOError::from((output_path, e)).to_string())
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(feature = "ffmpeg")]
|
|
||||||
async fn generate_video_thumbnail(
|
|
||||||
file_path: impl AsRef<Path> + Send,
|
|
||||||
output_path: impl AsRef<Path> + Send,
|
|
||||||
) -> Result<(), NonCriticalError> {
|
|
||||||
use sd_ffmpeg::{to_thumbnail, ThumbnailSize};
|
|
||||||
|
|
||||||
let file_path = file_path.as_ref();
|
|
||||||
|
|
||||||
to_thumbnail(
|
|
||||||
file_path,
|
|
||||||
output_path,
|
|
||||||
ThumbnailSize::Scale(1024),
|
|
||||||
TARGET_QUALITY,
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.map_err(|e| {
|
|
||||||
NonCriticalError::VideoThumbnailGenerationFailed(file_path.to_path_buf(), e.to_string())
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
|
@ -1,4 +1,3 @@
|
||||||
use rspc::ErrorCode;
|
|
||||||
use sd_core_file_path_helper::{
|
use sd_core_file_path_helper::{
|
||||||
ensure_file_path_exists, ensure_sub_path_is_directory, ensure_sub_path_is_in_location,
|
ensure_file_path_exists, ensure_sub_path_is_directory, ensure_sub_path_is_in_location,
|
||||||
FilePathError, IsolatedFilePathData,
|
FilePathError, IsolatedFilePathData,
|
||||||
|
@ -9,6 +8,7 @@ use sd_prisma::prisma::{location, PrismaClient};
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
|
|
||||||
use prisma_client_rust::QueryError;
|
use prisma_client_rust::QueryError;
|
||||||
|
use rspc::ErrorCode;
|
||||||
|
|
||||||
#[derive(thiserror::Error, Debug)]
|
#[derive(thiserror::Error, Debug)]
|
||||||
pub enum Error {
|
pub enum Error {
|
||||||
|
@ -23,66 +23,91 @@ pub enum Error {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl From<Error> for rspc::Error {
|
impl From<Error> for rspc::Error {
|
||||||
fn from(err: Error) -> Self {
|
fn from(e: Error) -> Self {
|
||||||
match err {
|
match e {
|
||||||
Error::SubPathNotFound(_) => {
|
Error::SubPathNotFound(_) => Self::with_cause(ErrorCode::NotFound, e.to_string(), e),
|
||||||
Self::with_cause(ErrorCode::NotFound, err.to_string(), err)
|
|
||||||
|
_ => Self::with_cause(ErrorCode::InternalServerError, e.to_string(), e),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn get_full_path_from_sub_path<E: From<Error>>(
|
||||||
|
location_id: location::id::Type,
|
||||||
|
sub_path: Option<impl AsRef<Path> + Send + Sync>,
|
||||||
|
location_path: impl AsRef<Path> + Send,
|
||||||
|
db: &PrismaClient,
|
||||||
|
) -> Result<PathBuf, E> {
|
||||||
|
async fn inner(
|
||||||
|
location_id: location::id::Type,
|
||||||
|
sub_path: Option<&Path>,
|
||||||
|
location_path: &Path,
|
||||||
|
db: &PrismaClient,
|
||||||
|
) -> Result<PathBuf, Error> {
|
||||||
|
match sub_path {
|
||||||
|
Some(sub_path) if sub_path != Path::new("") => {
|
||||||
|
let full_path = ensure_sub_path_is_in_location(location_path, sub_path).await?;
|
||||||
|
|
||||||
|
ensure_sub_path_is_directory(location_path, sub_path).await?;
|
||||||
|
|
||||||
|
ensure_file_path_exists(
|
||||||
|
sub_path,
|
||||||
|
&IsolatedFilePathData::new(location_id, location_path, &full_path, true)?,
|
||||||
|
db,
|
||||||
|
Error::SubPathNotFound,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
Ok(full_path)
|
||||||
}
|
}
|
||||||
|
_ => Ok(location_path.to_path_buf()),
|
||||||
_ => Self::with_cause(ErrorCode::InternalServerError, err.to_string(), err),
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
inner(
|
||||||
|
location_id,
|
||||||
|
sub_path.as_ref().map(AsRef::as_ref),
|
||||||
|
location_path.as_ref(),
|
||||||
|
db,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(E::from)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn get_full_path_from_sub_path(
|
pub async fn maybe_get_iso_file_path_from_sub_path<E: From<Error>>(
|
||||||
location_id: location::id::Type,
|
location_id: location::id::Type,
|
||||||
sub_path: &Option<impl AsRef<Path> + Send + Sync>,
|
sub_path: Option<impl AsRef<Path> + Send + Sync>,
|
||||||
location_path: impl AsRef<Path> + Send,
|
location_path: impl AsRef<Path> + Send,
|
||||||
db: &PrismaClient,
|
db: &PrismaClient,
|
||||||
) -> Result<PathBuf, Error> {
|
) -> Result<Option<IsolatedFilePathData<'static>>, E> {
|
||||||
let location_path = location_path.as_ref();
|
async fn inner(
|
||||||
|
location_id: location::id::Type,
|
||||||
|
sub_path: Option<&Path>,
|
||||||
|
location_path: &Path,
|
||||||
|
db: &PrismaClient,
|
||||||
|
) -> Result<Option<IsolatedFilePathData<'static>>, Error> {
|
||||||
|
match sub_path {
|
||||||
|
Some(sub_path) if sub_path != Path::new("") => {
|
||||||
|
let full_path = ensure_sub_path_is_in_location(location_path, sub_path).await?;
|
||||||
|
ensure_sub_path_is_directory(location_path, sub_path).await?;
|
||||||
|
|
||||||
match sub_path {
|
let sub_iso_file_path =
|
||||||
Some(sub_path) if sub_path.as_ref() != Path::new("") => {
|
IsolatedFilePathData::new(location_id, location_path, &full_path, true)?;
|
||||||
let sub_path = sub_path.as_ref();
|
|
||||||
let full_path = ensure_sub_path_is_in_location(location_path, sub_path).await?;
|
|
||||||
|
|
||||||
ensure_sub_path_is_directory(location_path, sub_path).await?;
|
ensure_file_path_exists(sub_path, &sub_iso_file_path, db, Error::SubPathNotFound)
|
||||||
|
.await
|
||||||
ensure_file_path_exists(
|
.map(|()| Some(sub_iso_file_path))
|
||||||
sub_path,
|
}
|
||||||
&IsolatedFilePathData::new(location_id, location_path, &full_path, true)?,
|
_ => Ok(None),
|
||||||
db,
|
|
||||||
Error::SubPathNotFound,
|
|
||||||
)
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
Ok(full_path)
|
|
||||||
}
|
}
|
||||||
_ => Ok(location_path.to_path_buf()),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn maybe_get_iso_file_path_from_sub_path(
|
|
||||||
location_id: location::id::Type,
|
|
||||||
sub_path: &Option<impl AsRef<Path> + Send + Sync>,
|
|
||||||
location_path: impl AsRef<Path> + Send,
|
|
||||||
db: &PrismaClient,
|
|
||||||
) -> Result<Option<IsolatedFilePathData<'static>>, Error> {
|
|
||||||
let location_path = location_path.as_ref();
|
|
||||||
|
|
||||||
match sub_path {
|
|
||||||
Some(sub_path) if sub_path.as_ref() != Path::new("") => {
|
|
||||||
let full_path = ensure_sub_path_is_in_location(location_path, sub_path).await?;
|
|
||||||
ensure_sub_path_is_directory(location_path, sub_path).await?;
|
|
||||||
|
|
||||||
let sub_iso_file_path =
|
|
||||||
IsolatedFilePathData::new(location_id, location_path, &full_path, true)?;
|
|
||||||
|
|
||||||
ensure_file_path_exists(sub_path, &sub_iso_file_path, db, Error::SubPathNotFound)
|
|
||||||
.await
|
|
||||||
.map(|()| Some(sub_iso_file_path))
|
|
||||||
}
|
|
||||||
_ => Ok(None),
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
inner(
|
||||||
|
location_id,
|
||||||
|
sub_path.as_ref().map(AsRef::as_ref),
|
||||||
|
location_path.as_ref(),
|
||||||
|
db,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(E::from)
|
||||||
}
|
}
|
||||||
|
|
|
@ -51,15 +51,15 @@ use rspc::ErrorCode;
|
||||||
|
|
||||||
use specta::Type;
|
use specta::Type;
|
||||||
use thiserror::Error;
|
use thiserror::Error;
|
||||||
use tokio::{fs, sync::RwLock};
|
use tokio::fs;
|
||||||
use tracing::debug;
|
use tracing::{debug, instrument, trace};
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
pub mod seed;
|
pub mod seed;
|
||||||
mod serde_impl;
|
mod serde_impl;
|
||||||
|
|
||||||
#[derive(Error, Debug)]
|
#[derive(Error, Debug)]
|
||||||
pub enum IndexerRuleError {
|
pub enum Error {
|
||||||
// User errors
|
// User errors
|
||||||
#[error("invalid indexer rule kind integer: {0}")]
|
#[error("invalid indexer rule kind integer: {0}")]
|
||||||
InvalidRuleKindInt(i32),
|
InvalidRuleKindInt(i32),
|
||||||
|
@ -83,16 +83,14 @@ pub enum IndexerRuleError {
|
||||||
MissingField(#[from] MissingFieldError),
|
MissingField(#[from] MissingFieldError),
|
||||||
}
|
}
|
||||||
|
|
||||||
impl From<IndexerRuleError> for rspc::Error {
|
impl From<Error> for rspc::Error {
|
||||||
fn from(err: IndexerRuleError) -> Self {
|
fn from(e: Error) -> Self {
|
||||||
match err {
|
match e {
|
||||||
IndexerRuleError::InvalidRuleKindInt(_)
|
Error::InvalidRuleKindInt(_) | Error::Glob(_) | Error::NonUtf8Path(_) => {
|
||||||
| IndexerRuleError::Glob(_)
|
Self::with_cause(ErrorCode::BadRequest, e.to_string(), e)
|
||||||
| IndexerRuleError::NonUtf8Path(_) => {
|
|
||||||
Self::with_cause(ErrorCode::BadRequest, err.to_string(), err)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
_ => Self::with_cause(ErrorCode::InternalServerError, err.to_string(), err),
|
_ => Self::with_cause(ErrorCode::InternalServerError, e.to_string(), e),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -113,21 +111,17 @@ pub struct IndexerRuleCreateArgs {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl IndexerRuleCreateArgs {
|
impl IndexerRuleCreateArgs {
|
||||||
pub async fn create(
|
#[instrument(skip_all, fields(name = %self.name, rules = ?self.rules), err)]
|
||||||
self,
|
pub async fn create(self, db: &PrismaClient) -> Result<Option<indexer_rule::Data>, Error> {
|
||||||
db: &PrismaClient,
|
|
||||||
) -> Result<Option<indexer_rule::Data>, IndexerRuleError> {
|
|
||||||
use indexer_rule::{date_created, date_modified, name, rules_per_kind};
|
use indexer_rule::{date_created, date_modified, name, rules_per_kind};
|
||||||
|
|
||||||
debug!(
|
debug!(
|
||||||
"{} a new indexer rule (name = {}, params = {:?})",
|
"{} a new indexer rule",
|
||||||
if self.dry_run {
|
if self.dry_run {
|
||||||
"Dry run: Would create"
|
"Dry run: Would create"
|
||||||
} else {
|
} else {
|
||||||
"Trying to create"
|
"Trying to create"
|
||||||
},
|
},
|
||||||
self.name,
|
|
||||||
self.rules
|
|
||||||
);
|
);
|
||||||
|
|
||||||
let rules_data = rmp_serde::to_vec_named(
|
let rules_data = rmp_serde::to_vec_named(
|
||||||
|
@ -167,7 +161,7 @@ impl IndexerRuleCreateArgs {
|
||||||
Ok(Some(
|
Ok(Some(
|
||||||
db.indexer_rule()
|
db.indexer_rule()
|
||||||
.create(
|
.create(
|
||||||
sd_utils::uuid_to_bytes(generate_pub_id()),
|
sd_utils::uuid_to_bytes(&generate_pub_id()),
|
||||||
vec![
|
vec![
|
||||||
name::set(Some(self.name)),
|
name::set(Some(self.name)),
|
||||||
rules_per_kind::set(Some(rules_data)),
|
rules_per_kind::set(Some(rules_data)),
|
||||||
|
@ -224,7 +218,7 @@ impl RulePerKind {
|
||||||
fn new_files_by_globs_str_and_kind(
|
fn new_files_by_globs_str_and_kind(
|
||||||
globs_str: impl IntoIterator<Item = impl AsRef<str>>,
|
globs_str: impl IntoIterator<Item = impl AsRef<str>>,
|
||||||
kind_fn: impl Fn(Vec<Glob>, GlobSet) -> Self,
|
kind_fn: impl Fn(Vec<Glob>, GlobSet) -> Self,
|
||||||
) -> Result<Self, IndexerRuleError> {
|
) -> Result<Self, Error> {
|
||||||
globs_str
|
globs_str
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.map(|s| s.as_ref().parse::<Glob>())
|
.map(|s| s.as_ref().parse::<Glob>())
|
||||||
|
@ -245,13 +239,13 @@ impl RulePerKind {
|
||||||
|
|
||||||
pub fn new_accept_files_by_globs_str(
|
pub fn new_accept_files_by_globs_str(
|
||||||
globs_str: impl IntoIterator<Item = impl AsRef<str>>,
|
globs_str: impl IntoIterator<Item = impl AsRef<str>>,
|
||||||
) -> Result<Self, IndexerRuleError> {
|
) -> Result<Self, Error> {
|
||||||
Self::new_files_by_globs_str_and_kind(globs_str, Self::AcceptFilesByGlob)
|
Self::new_files_by_globs_str_and_kind(globs_str, Self::AcceptFilesByGlob)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn new_reject_files_by_globs_str(
|
pub fn new_reject_files_by_globs_str(
|
||||||
globs_str: impl IntoIterator<Item = impl AsRef<str>>,
|
globs_str: impl IntoIterator<Item = impl AsRef<str>>,
|
||||||
) -> Result<Self, IndexerRuleError> {
|
) -> Result<Self, Error> {
|
||||||
Self::new_files_by_globs_str_and_kind(globs_str, Self::RejectFilesByGlob)
|
Self::new_files_by_globs_str_and_kind(globs_str, Self::RejectFilesByGlob)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -267,51 +261,19 @@ impl MetadataForIndexerRules for Metadata {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl RulePerKind {
|
impl RulePerKind {
|
||||||
#[deprecated = "Use `[apply_with_metadata]` instead"]
|
|
||||||
async fn apply(
|
async fn apply(
|
||||||
&self,
|
&self,
|
||||||
source: impl AsRef<Path> + Send,
|
source: impl AsRef<Path> + Send,
|
||||||
) -> Result<(RuleKind, bool), IndexerRuleError> {
|
|
||||||
match self {
|
|
||||||
Self::AcceptIfChildrenDirectoriesArePresent(children) => {
|
|
||||||
accept_dir_for_its_children(source, children)
|
|
||||||
.await
|
|
||||||
.map(|accepted| (RuleKind::AcceptIfChildrenDirectoriesArePresent, accepted))
|
|
||||||
}
|
|
||||||
Self::RejectIfChildrenDirectoriesArePresent(children) => {
|
|
||||||
reject_dir_for_its_children(source, children)
|
|
||||||
.await
|
|
||||||
.map(|rejected| (RuleKind::RejectIfChildrenDirectoriesArePresent, rejected))
|
|
||||||
}
|
|
||||||
|
|
||||||
Self::AcceptFilesByGlob(_globs, accept_glob_set) => Ok((
|
|
||||||
RuleKind::AcceptFilesByGlob,
|
|
||||||
accept_by_glob(source, accept_glob_set),
|
|
||||||
)),
|
|
||||||
Self::RejectFilesByGlob(_globs, reject_glob_set) => Ok((
|
|
||||||
RuleKind::RejectFilesByGlob,
|
|
||||||
reject_by_glob(source, reject_glob_set),
|
|
||||||
)),
|
|
||||||
Self::IgnoredByGit(git_repo, patterns) => Ok((
|
|
||||||
RuleKind::IgnoredByGit,
|
|
||||||
accept_by_gitpattern(source.as_ref(), git_repo, patterns),
|
|
||||||
)),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn apply_with_metadata(
|
|
||||||
&self,
|
|
||||||
source: impl AsRef<Path> + Send,
|
|
||||||
metadata: &impl MetadataForIndexerRules,
|
metadata: &impl MetadataForIndexerRules,
|
||||||
) -> Result<(RuleKind, bool), IndexerRuleError> {
|
) -> Result<(RuleKind, bool), Error> {
|
||||||
match self {
|
match self {
|
||||||
Self::AcceptIfChildrenDirectoriesArePresent(children) => {
|
Self::AcceptIfChildrenDirectoriesArePresent(children) => {
|
||||||
accept_dir_for_its_children_with_metadata(source, metadata, children)
|
accept_dir_for_its_children(source, metadata, children)
|
||||||
.await
|
.await
|
||||||
.map(|accepted| (RuleKind::AcceptIfChildrenDirectoriesArePresent, accepted))
|
.map(|accepted| (RuleKind::AcceptIfChildrenDirectoriesArePresent, accepted))
|
||||||
}
|
}
|
||||||
Self::RejectIfChildrenDirectoriesArePresent(children) => {
|
Self::RejectIfChildrenDirectoriesArePresent(children) => {
|
||||||
reject_dir_for_its_children_with_metadata(source, metadata, children)
|
reject_dir_for_its_children(source, metadata, children)
|
||||||
.await
|
.await
|
||||||
.map(|rejected| (RuleKind::RejectIfChildrenDirectoriesArePresent, rejected))
|
.map(|rejected| (RuleKind::RejectIfChildrenDirectoriesArePresent, rejected))
|
||||||
}
|
}
|
||||||
|
@ -326,24 +288,32 @@ impl RulePerKind {
|
||||||
)),
|
)),
|
||||||
Self::IgnoredByGit(base_dir, patterns) => Ok((
|
Self::IgnoredByGit(base_dir, patterns) => Ok((
|
||||||
RuleKind::IgnoredByGit,
|
RuleKind::IgnoredByGit,
|
||||||
accept_by_gitpattern(source.as_ref(), base_dir, patterns),
|
accept_by_git_pattern(source, base_dir, patterns),
|
||||||
)),
|
)),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn accept_by_gitpattern(source: &Path, base_dir: &Path, search: &Search) -> bool {
|
fn accept_by_git_pattern(
|
||||||
let relative = source
|
source: impl AsRef<Path>,
|
||||||
.strip_prefix(base_dir)
|
base_dir: impl AsRef<Path>,
|
||||||
.expect("`base_dir` should be our git repo, and `source` should be inside of it");
|
search: &Search,
|
||||||
|
) -> bool {
|
||||||
|
fn inner(source: &Path, base_dir: &Path, search: &Search) -> bool {
|
||||||
|
let relative = source
|
||||||
|
.strip_prefix(base_dir)
|
||||||
|
.expect("`base_dir` should be our git repo, and `source` should be inside of it");
|
||||||
|
|
||||||
let Some(src) = relative.to_str().map(|s| s.as_bytes().into()) else {
|
let Some(src) = relative.to_str().map(|s| s.as_bytes().into()) else {
|
||||||
return false;
|
return false;
|
||||||
};
|
};
|
||||||
|
|
||||||
search
|
search
|
||||||
.pattern_matching_relative_path(src, Some(source.is_dir()), Case::Fold)
|
.pattern_matching_relative_path(src, Some(source.is_dir()), Case::Fold)
|
||||||
.map_or(true, |rule| rule.pattern.is_negative())
|
.map_or(true, |rule| rule.pattern.is_negative())
|
||||||
|
}
|
||||||
|
|
||||||
|
inner(source.as_ref(), base_dir.as_ref(), search)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Serialize, Deserialize, Clone)]
|
#[derive(Debug, Serialize, Deserialize, Clone)]
|
||||||
|
@ -357,32 +327,19 @@ pub struct IndexerRule {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl IndexerRule {
|
impl IndexerRule {
|
||||||
#[deprecated = "Use `[apply_with_metadata]` instead"]
|
|
||||||
pub async fn apply(
|
pub async fn apply(
|
||||||
&self,
|
&self,
|
||||||
source: impl AsRef<Path> + Send,
|
source: impl AsRef<Path> + Send,
|
||||||
) -> Result<Vec<(RuleKind, bool)>, IndexerRuleError> {
|
|
||||||
self.rules
|
|
||||||
.iter()
|
|
||||||
.map(|rule| rule.apply(source.as_ref()))
|
|
||||||
.collect::<Vec<_>>()
|
|
||||||
.try_join()
|
|
||||||
.await
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn apply_with_metadata(
|
|
||||||
&self,
|
|
||||||
source: impl AsRef<Path> + Send,
|
|
||||||
metadata: &impl MetadataForIndexerRules,
|
metadata: &impl MetadataForIndexerRules,
|
||||||
) -> Result<Vec<(RuleKind, bool)>, IndexerRuleError> {
|
) -> Result<Vec<(RuleKind, bool)>, Error> {
|
||||||
async fn inner(
|
async fn inner(
|
||||||
rules: &[RulePerKind],
|
rules: &[RulePerKind],
|
||||||
source: &Path,
|
source: &Path,
|
||||||
metadata: &impl MetadataForIndexerRules,
|
metadata: &impl MetadataForIndexerRules,
|
||||||
) -> Result<Vec<(RuleKind, bool)>, IndexerRuleError> {
|
) -> Result<Vec<(RuleKind, bool)>, Error> {
|
||||||
rules
|
rules
|
||||||
.iter()
|
.iter()
|
||||||
.map(|rule| rule.apply_with_metadata(source, metadata))
|
.map(|rule| rule.apply(source, metadata))
|
||||||
.collect::<Vec<_>>()
|
.collect::<Vec<_>>()
|
||||||
.try_join()
|
.try_join()
|
||||||
.await
|
.await
|
||||||
|
@ -390,64 +347,79 @@ impl IndexerRule {
|
||||||
|
|
||||||
inner(&self.rules, source.as_ref(), metadata).await
|
inner(&self.rules, source.as_ref(), metadata).await
|
||||||
}
|
}
|
||||||
|
|
||||||
#[deprecated = "Use `[IndexerRuler::apply_all]` instead"]
|
|
||||||
pub async fn apply_all(
|
|
||||||
rules: &[Self],
|
|
||||||
source: impl AsRef<Path> + Send,
|
|
||||||
) -> Result<HashMap<RuleKind, Vec<bool>>, IndexerRuleError> {
|
|
||||||
rules
|
|
||||||
.iter()
|
|
||||||
.map(|rule| rule.apply(source.as_ref()))
|
|
||||||
.collect::<Vec<_>>()
|
|
||||||
.try_join()
|
|
||||||
.await
|
|
||||||
.map(|results| {
|
|
||||||
results.into_iter().flatten().fold(
|
|
||||||
HashMap::<_, Vec<_>>::with_capacity(RuleKind::variant_count()),
|
|
||||||
|mut map, (kind, result)| {
|
|
||||||
map.entry(kind).or_default().push(result);
|
|
||||||
map
|
|
||||||
},
|
|
||||||
)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Default)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub enum RulerDecision {
|
||||||
|
Accept,
|
||||||
|
Reject,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Default, Serialize, Deserialize)]
|
||||||
pub struct IndexerRuler {
|
pub struct IndexerRuler {
|
||||||
rules: Arc<RwLock<Vec<IndexerRule>>>,
|
base: Arc<Vec<IndexerRule>>,
|
||||||
|
extra: Vec<IndexerRule>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Clone for IndexerRuler {
|
||||||
|
fn clone(&self) -> Self {
|
||||||
|
Self {
|
||||||
|
base: Arc::clone(&self.base),
|
||||||
|
// Each instance of IndexerRules MUST have its own extra rules no clones allowed!
|
||||||
|
extra: Vec::new(),
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl IndexerRuler {
|
impl IndexerRuler {
|
||||||
#[must_use]
|
#[must_use]
|
||||||
pub fn new(rules: Vec<IndexerRule>) -> Self {
|
pub fn new(rules: Vec<IndexerRule>) -> Self {
|
||||||
Self {
|
Self {
|
||||||
rules: Arc::new(RwLock::new(rules)),
|
base: Arc::new(rules),
|
||||||
|
extra: Vec::new(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn serialize(&self) -> Result<Vec<u8>, encode::Error> {
|
pub async fn evaluate_path(
|
||||||
rmp_serde::to_vec_named(&*self.rules.read().await)
|
&self,
|
||||||
}
|
source: impl AsRef<Path> + Send,
|
||||||
|
metadata: &impl MetadataForIndexerRules,
|
||||||
|
) -> Result<RulerDecision, Error> {
|
||||||
|
async fn inner(
|
||||||
|
this: &IndexerRuler,
|
||||||
|
source: &Path,
|
||||||
|
metadata: &impl MetadataForIndexerRules,
|
||||||
|
) -> Result<RulerDecision, Error> {
|
||||||
|
Ok(
|
||||||
|
if IndexerRuler::reject_path(
|
||||||
|
source,
|
||||||
|
metadata.is_dir(),
|
||||||
|
&this.apply_all(source, metadata).await?,
|
||||||
|
) {
|
||||||
|
RulerDecision::Reject
|
||||||
|
} else {
|
||||||
|
RulerDecision::Accept
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
pub fn deserialize(data: &[u8]) -> Result<Self, decode::Error> {
|
inner(self, source.as_ref(), metadata).await
|
||||||
rmp_serde::from_slice(data).map(Self::new)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn apply_all(
|
pub async fn apply_all(
|
||||||
&self,
|
&self,
|
||||||
source: impl AsRef<Path> + Send,
|
source: impl AsRef<Path> + Send,
|
||||||
metadata: &impl MetadataForIndexerRules,
|
metadata: &impl MetadataForIndexerRules,
|
||||||
) -> Result<HashMap<RuleKind, Vec<bool>>, IndexerRuleError> {
|
) -> Result<HashMap<RuleKind, Vec<bool>>, Error> {
|
||||||
async fn inner(
|
async fn inner(
|
||||||
rules: &[IndexerRule],
|
base: &[IndexerRule],
|
||||||
|
extra: &[IndexerRule],
|
||||||
source: &Path,
|
source: &Path,
|
||||||
metadata: &impl MetadataForIndexerRules,
|
metadata: &impl MetadataForIndexerRules,
|
||||||
) -> Result<HashMap<RuleKind, Vec<bool>>, IndexerRuleError> {
|
) -> Result<HashMap<RuleKind, Vec<bool>>, Error> {
|
||||||
rules
|
base.iter()
|
||||||
.iter()
|
.chain(extra.iter())
|
||||||
.map(|rule| rule.apply_with_metadata(source, metadata))
|
.map(|rule| rule.apply(source, metadata))
|
||||||
.collect::<Vec<_>>()
|
.collect::<Vec<_>>()
|
||||||
.try_join()
|
.try_join()
|
||||||
.await
|
.await
|
||||||
|
@ -462,24 +434,99 @@ impl IndexerRuler {
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
inner(&self.rules.read().await, source.as_ref(), metadata).await
|
inner(&self.base, &self.extra, source.as_ref(), metadata).await
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Extend the indexer rules with the contents from an iterator of rules
|
/// Extend the indexer rules with the contents from an iterator of rules
|
||||||
pub async fn extend(&self, iter: impl IntoIterator<Item = IndexerRule> + Send) {
|
pub fn extend(&mut self, iter: impl IntoIterator<Item = IndexerRule> + Send) {
|
||||||
let mut indexer = self.rules.write().await;
|
self.extra.extend(iter);
|
||||||
indexer.extend(iter);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn has_system(&self, rule: &SystemIndexerRule) -> bool {
|
#[must_use]
|
||||||
let rules = self.rules.read().await;
|
pub fn has_system(&self, rule: &SystemIndexerRule) -> bool {
|
||||||
|
self.base
|
||||||
|
.iter()
|
||||||
|
.chain(self.extra.iter())
|
||||||
|
.any(|inner_rule| rule == inner_rule)
|
||||||
|
}
|
||||||
|
|
||||||
rules.iter().any(|inner_rule| rule == inner_rule)
|
#[instrument(skip_all, fields(current_path = %current_path.display()))]
|
||||||
|
fn reject_path(
|
||||||
|
current_path: &Path,
|
||||||
|
is_dir: bool,
|
||||||
|
acceptance_per_rule_kind: &HashMap<RuleKind, Vec<bool>>,
|
||||||
|
) -> bool {
|
||||||
|
Self::rejected_by_reject_glob(acceptance_per_rule_kind)
|
||||||
|
|| Self::rejected_by_git_ignore(acceptance_per_rule_kind)
|
||||||
|
|| (is_dir && Self::rejected_by_children_directories(acceptance_per_rule_kind))
|
||||||
|
|| Self::rejected_by_accept_glob(acceptance_per_rule_kind)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn rejected_by_accept_glob(
|
||||||
|
acceptance_per_rule_kind: &HashMap<RuleKind, Vec<bool>>,
|
||||||
|
) -> bool {
|
||||||
|
let res = acceptance_per_rule_kind
|
||||||
|
.get(&RuleKind::AcceptFilesByGlob)
|
||||||
|
.map_or(false, |accept_rules| {
|
||||||
|
accept_rules.iter().all(|accept| !accept)
|
||||||
|
});
|
||||||
|
|
||||||
|
if res {
|
||||||
|
trace!("Reject because it didn't passed in any `RuleKind::AcceptFilesByGlob` rules");
|
||||||
|
}
|
||||||
|
|
||||||
|
res
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn rejected_by_children_directories(
|
||||||
|
acceptance_per_rule_kind: &HashMap<RuleKind, Vec<bool>>,
|
||||||
|
) -> bool {
|
||||||
|
let res = acceptance_per_rule_kind
|
||||||
|
.get(&RuleKind::RejectIfChildrenDirectoriesArePresent)
|
||||||
|
.map_or(false, |reject_results| {
|
||||||
|
reject_results.iter().any(|reject| !reject)
|
||||||
|
});
|
||||||
|
|
||||||
|
if res {
|
||||||
|
trace!("Rejected by rule `RuleKind::RejectIfChildrenDirectoriesArePresent`");
|
||||||
|
}
|
||||||
|
|
||||||
|
res
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn rejected_by_reject_glob(
|
||||||
|
acceptance_per_rule_kind: &HashMap<RuleKind, Vec<bool>>,
|
||||||
|
) -> bool {
|
||||||
|
let res = acceptance_per_rule_kind
|
||||||
|
.get(&RuleKind::RejectFilesByGlob)
|
||||||
|
.map_or(false, |reject_results| {
|
||||||
|
reject_results.iter().any(|reject| !reject)
|
||||||
|
});
|
||||||
|
|
||||||
|
if res {
|
||||||
|
trace!("Rejected by `RuleKind::RejectFilesByGlob`");
|
||||||
|
}
|
||||||
|
|
||||||
|
res
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn rejected_by_git_ignore(acceptance_per_rule_kind: &HashMap<RuleKind, Vec<bool>>) -> bool {
|
||||||
|
let res = acceptance_per_rule_kind
|
||||||
|
.get(&RuleKind::IgnoredByGit)
|
||||||
|
.map_or(false, |reject_results| {
|
||||||
|
reject_results.iter().any(|reject| !reject)
|
||||||
|
});
|
||||||
|
|
||||||
|
if res {
|
||||||
|
trace!("Rejected by `RuleKind::IgnoredByGit`");
|
||||||
|
}
|
||||||
|
|
||||||
|
res
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl TryFrom<&indexer_rule::Data> for IndexerRule {
|
impl TryFrom<&indexer_rule::Data> for IndexerRule {
|
||||||
type Error = IndexerRuleError;
|
type Error = Error;
|
||||||
|
|
||||||
fn try_from(data: &indexer_rule::Data) -> Result<Self, Self::Error> {
|
fn try_from(data: &indexer_rule::Data) -> Result<Self, Self::Error> {
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
|
@ -497,7 +544,7 @@ impl TryFrom<&indexer_rule::Data> for IndexerRule {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl TryFrom<indexer_rule::Data> for IndexerRule {
|
impl TryFrom<indexer_rule::Data> for IndexerRule {
|
||||||
type Error = IndexerRuleError;
|
type Error = Error;
|
||||||
|
|
||||||
fn try_from(data: indexer_rule::Data) -> Result<Self, Self::Error> {
|
fn try_from(data: indexer_rule::Data) -> Result<Self, Self::Error> {
|
||||||
Self::try_from(&data)
|
Self::try_from(&data)
|
||||||
|
@ -512,140 +559,56 @@ fn reject_by_glob(source: impl AsRef<Path>, reject_glob_set: &GlobSet) -> bool {
|
||||||
!accept_by_glob(source.as_ref(), reject_glob_set)
|
!accept_by_glob(source.as_ref(), reject_glob_set)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[deprecated = "Use `[accept_dir_for_its_children_with_metadata]` instead"]
|
|
||||||
async fn accept_dir_for_its_children(
|
async fn accept_dir_for_its_children(
|
||||||
source: impl AsRef<Path> + Send,
|
|
||||||
children: &HashSet<String>,
|
|
||||||
) -> Result<bool, IndexerRuleError> {
|
|
||||||
let source = source.as_ref();
|
|
||||||
|
|
||||||
// FIXME(fogodev): Just check for io::ErrorKind::NotADirectory error instead (feature = "io_error_more", issue = "86442")
|
|
||||||
if !fs::metadata(source)
|
|
||||||
.await
|
|
||||||
.map_err(|e| IndexerRuleError::AcceptByItsChildrenFileIO(FileIOError::from((source, e))))?
|
|
||||||
.is_dir()
|
|
||||||
{
|
|
||||||
return Ok(false);
|
|
||||||
}
|
|
||||||
|
|
||||||
let mut read_dir = fs::read_dir(source)
|
|
||||||
.await // TODO: Check NotADirectory error here when available
|
|
||||||
.map_err(|e| IndexerRuleError::AcceptByItsChildrenFileIO(FileIOError::from((source, e))))?;
|
|
||||||
while let Some(entry) = read_dir
|
|
||||||
.next_entry()
|
|
||||||
.await
|
|
||||||
.map_err(|e| IndexerRuleError::AcceptByItsChildrenFileIO(FileIOError::from((source, e))))?
|
|
||||||
{
|
|
||||||
let entry_name = entry
|
|
||||||
.file_name()
|
|
||||||
.to_str()
|
|
||||||
.ok_or_else(|| NonUtf8PathError(entry.path().into()))?
|
|
||||||
.to_string();
|
|
||||||
|
|
||||||
if entry
|
|
||||||
.metadata()
|
|
||||||
.await
|
|
||||||
.map_err(|e| {
|
|
||||||
IndexerRuleError::AcceptByItsChildrenFileIO(FileIOError::from((source, e)))
|
|
||||||
})?
|
|
||||||
.is_dir() && children.contains(&entry_name)
|
|
||||||
{
|
|
||||||
return Ok(true);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(false)
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn accept_dir_for_its_children_with_metadata(
|
|
||||||
source: impl AsRef<Path> + Send,
|
source: impl AsRef<Path> + Send,
|
||||||
metadata: &impl MetadataForIndexerRules,
|
metadata: &impl MetadataForIndexerRules,
|
||||||
children: &HashSet<String>,
|
children: &HashSet<String>,
|
||||||
) -> Result<bool, IndexerRuleError> {
|
) -> Result<bool, Error> {
|
||||||
let source = source.as_ref();
|
async fn inner(
|
||||||
|
source: &Path,
|
||||||
// FIXME(fogodev): Just check for io::ErrorKind::NotADirectory error instead (feature = "io_error_more", issue = "86442")
|
metadata: &impl MetadataForIndexerRules,
|
||||||
if !metadata.is_dir() {
|
children: &HashSet<String>,
|
||||||
return Ok(false);
|
) -> Result<bool, Error> {
|
||||||
}
|
// FIXME(fogodev): Just check for io::ErrorKind::NotADirectory error instead (feature = "io_error_more", issue = "86442")
|
||||||
|
if !metadata.is_dir() {
|
||||||
let mut read_dir = fs::read_dir(source)
|
|
||||||
.await // TODO: Check NotADirectory error here when available
|
|
||||||
.map_err(|e| IndexerRuleError::AcceptByItsChildrenFileIO(FileIOError::from((source, e))))?;
|
|
||||||
while let Some(entry) = read_dir
|
|
||||||
.next_entry()
|
|
||||||
.await
|
|
||||||
.map_err(|e| IndexerRuleError::AcceptByItsChildrenFileIO(FileIOError::from((source, e))))?
|
|
||||||
{
|
|
||||||
let entry_name = entry
|
|
||||||
.file_name()
|
|
||||||
.to_str()
|
|
||||||
.ok_or_else(|| NonUtf8PathError(entry.path().into()))?
|
|
||||||
.to_string();
|
|
||||||
|
|
||||||
if entry
|
|
||||||
.metadata()
|
|
||||||
.await
|
|
||||||
.map_err(|e| {
|
|
||||||
IndexerRuleError::AcceptByItsChildrenFileIO(FileIOError::from((source, e)))
|
|
||||||
})?
|
|
||||||
.is_dir() && children.contains(&entry_name)
|
|
||||||
{
|
|
||||||
return Ok(true);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(false)
|
|
||||||
}
|
|
||||||
|
|
||||||
#[deprecated = "Use `[reject_dir_for_its_children_with_metadata]` instead"]
|
|
||||||
async fn reject_dir_for_its_children(
|
|
||||||
source: impl AsRef<Path> + Send,
|
|
||||||
children: &HashSet<String>,
|
|
||||||
) -> Result<bool, IndexerRuleError> {
|
|
||||||
let source = source.as_ref();
|
|
||||||
|
|
||||||
// FIXME(fogodev): Just check for io::ErrorKind::NotADirectory error instead (feature = "io_error_more", issue = "86442")
|
|
||||||
if !fs::metadata(source)
|
|
||||||
.await
|
|
||||||
.map_err(|e| IndexerRuleError::AcceptByItsChildrenFileIO(FileIOError::from((source, e))))?
|
|
||||||
.is_dir()
|
|
||||||
{
|
|
||||||
return Ok(true);
|
|
||||||
}
|
|
||||||
|
|
||||||
let mut read_dir = fs::read_dir(source)
|
|
||||||
.await // TODO: Check NotADirectory error here when available
|
|
||||||
.map_err(|e| IndexerRuleError::RejectByItsChildrenFileIO(FileIOError::from((source, e))))?;
|
|
||||||
while let Some(entry) = read_dir
|
|
||||||
.next_entry()
|
|
||||||
.await
|
|
||||||
.map_err(|e| IndexerRuleError::RejectByItsChildrenFileIO(FileIOError::from((source, e))))?
|
|
||||||
{
|
|
||||||
if entry
|
|
||||||
.metadata()
|
|
||||||
.await
|
|
||||||
.map_err(|e| {
|
|
||||||
IndexerRuleError::RejectByItsChildrenFileIO(FileIOError::from((source, e)))
|
|
||||||
})?
|
|
||||||
.is_dir() && children.contains(
|
|
||||||
entry
|
|
||||||
.file_name()
|
|
||||||
.to_str()
|
|
||||||
.ok_or_else(|| NonUtf8PathError(entry.path().into()))?,
|
|
||||||
) {
|
|
||||||
return Ok(false);
|
return Ok(false);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let mut read_dir = fs::read_dir(source)
|
||||||
|
.await // TODO: Check NotADirectory error here when available
|
||||||
|
.map_err(|e| Error::AcceptByItsChildrenFileIO(FileIOError::from((source, e))))?;
|
||||||
|
while let Some(entry) = read_dir
|
||||||
|
.next_entry()
|
||||||
|
.await
|
||||||
|
.map_err(|e| Error::AcceptByItsChildrenFileIO(FileIOError::from((source, e))))?
|
||||||
|
{
|
||||||
|
let entry_name = entry
|
||||||
|
.file_name()
|
||||||
|
.to_str()
|
||||||
|
.ok_or_else(|| NonUtf8PathError(entry.path().into()))?
|
||||||
|
.to_string();
|
||||||
|
|
||||||
|
if entry
|
||||||
|
.metadata()
|
||||||
|
.await
|
||||||
|
.map_err(|e| Error::AcceptByItsChildrenFileIO(FileIOError::from((source, e))))?
|
||||||
|
.is_dir() && children.contains(&entry_name)
|
||||||
|
{
|
||||||
|
return Ok(true);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(false)
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(true)
|
inner(source.as_ref(), metadata, children).await
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn reject_dir_for_its_children_with_metadata(
|
async fn reject_dir_for_its_children(
|
||||||
source: impl AsRef<Path> + Send,
|
source: impl AsRef<Path> + Send,
|
||||||
metadata: &impl MetadataForIndexerRules,
|
metadata: &impl MetadataForIndexerRules,
|
||||||
children: &HashSet<String>,
|
children: &HashSet<String>,
|
||||||
) -> Result<bool, IndexerRuleError> {
|
) -> Result<bool, Error> {
|
||||||
let source = source.as_ref();
|
let source = source.as_ref();
|
||||||
|
|
||||||
// FIXME(fogodev): Just check for io::ErrorKind::NotADirectory error instead (feature = "io_error_more", issue = "86442")
|
// FIXME(fogodev): Just check for io::ErrorKind::NotADirectory error instead (feature = "io_error_more", issue = "86442")
|
||||||
|
@ -655,18 +618,16 @@ async fn reject_dir_for_its_children_with_metadata(
|
||||||
|
|
||||||
let mut read_dir = fs::read_dir(source)
|
let mut read_dir = fs::read_dir(source)
|
||||||
.await // TODO: Check NotADirectory error here when available
|
.await // TODO: Check NotADirectory error here when available
|
||||||
.map_err(|e| IndexerRuleError::RejectByItsChildrenFileIO(FileIOError::from((source, e))))?;
|
.map_err(|e| Error::RejectByItsChildrenFileIO(FileIOError::from((source, e))))?;
|
||||||
while let Some(entry) = read_dir
|
while let Some(entry) = read_dir
|
||||||
.next_entry()
|
.next_entry()
|
||||||
.await
|
.await
|
||||||
.map_err(|e| IndexerRuleError::RejectByItsChildrenFileIO(FileIOError::from((source, e))))?
|
.map_err(|e| Error::RejectByItsChildrenFileIO(FileIOError::from((source, e))))?
|
||||||
{
|
{
|
||||||
if entry
|
if entry
|
||||||
.metadata()
|
.metadata()
|
||||||
.await
|
.await
|
||||||
.map_err(|e| {
|
.map_err(|e| Error::RejectByItsChildrenFileIO(FileIOError::from((source, e))))?
|
||||||
IndexerRuleError::RejectByItsChildrenFileIO(FileIOError::from((source, e)))
|
|
||||||
})?
|
|
||||||
.is_dir() && children.contains(
|
.is_dir() && children.contains(
|
||||||
entry
|
entry
|
||||||
.file_name()
|
.file_name()
|
||||||
|
@ -710,9 +671,37 @@ mod tests {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn check_rule(indexer_rule: &IndexerRule, path: impl AsRef<Path> + Send) -> bool {
|
fn check_rule(indexer_rule: &IndexerRule, path: impl AsRef<Path>) -> bool {
|
||||||
|
let path = path.as_ref();
|
||||||
indexer_rule
|
indexer_rule
|
||||||
.apply(path)
|
.rules
|
||||||
|
.iter()
|
||||||
|
.map(|rule| match rule {
|
||||||
|
RulePerKind::AcceptFilesByGlob(_globs, accept_glob_set) => (
|
||||||
|
RuleKind::AcceptFilesByGlob,
|
||||||
|
accept_by_glob(path, accept_glob_set),
|
||||||
|
),
|
||||||
|
RulePerKind::RejectFilesByGlob(_globs, reject_glob_set) => (
|
||||||
|
RuleKind::RejectFilesByGlob,
|
||||||
|
reject_by_glob(path, reject_glob_set),
|
||||||
|
),
|
||||||
|
RulePerKind::IgnoredByGit(git_repo, patterns) => (
|
||||||
|
RuleKind::IgnoredByGit,
|
||||||
|
accept_by_git_pattern(path, git_repo, patterns),
|
||||||
|
),
|
||||||
|
|
||||||
|
_ => unimplemented!("can't use simple `apply` for this rule: {:?}", rule),
|
||||||
|
})
|
||||||
|
.all(|(_kind, res)| res)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn check_rule_with_metadata(
|
||||||
|
indexer_rule: &IndexerRule,
|
||||||
|
path: impl AsRef<Path> + Send,
|
||||||
|
metadata: &impl MetadataForIndexerRules,
|
||||||
|
) -> bool {
|
||||||
|
indexer_rule
|
||||||
|
.apply(path.as_ref(), metadata)
|
||||||
.await
|
.await
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.into_iter()
|
.into_iter()
|
||||||
|
@ -739,12 +728,12 @@ mod tests {
|
||||||
)],
|
)],
|
||||||
);
|
);
|
||||||
|
|
||||||
assert!(!check_rule(&rule, hidden).await);
|
assert!(!check_rule(&rule, hidden));
|
||||||
assert!(check_rule(&rule, normal).await);
|
assert!(check_rule(&rule, normal));
|
||||||
assert!(!check_rule(&rule, hidden_inner_dir).await);
|
assert!(!check_rule(&rule, hidden_inner_dir));
|
||||||
assert!(!check_rule(&rule, hidden_inner_file).await);
|
assert!(!check_rule(&rule, hidden_inner_file));
|
||||||
assert!(check_rule(&rule, normal_inner_dir).await);
|
assert!(check_rule(&rule, normal_inner_dir));
|
||||||
assert!(check_rule(&rule, normal_inner_file).await);
|
assert!(check_rule(&rule, normal_inner_file));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
@ -765,9 +754,9 @@ mod tests {
|
||||||
)],
|
)],
|
||||||
);
|
);
|
||||||
|
|
||||||
assert!(check_rule(&rule, project_file).await);
|
assert!(check_rule(&rule, project_file));
|
||||||
assert!(!check_rule(&rule, project_build_dir).await);
|
assert!(!check_rule(&rule, project_build_dir));
|
||||||
assert!(!check_rule(&rule, project_build_dir_inner).await);
|
assert!(!check_rule(&rule, project_build_dir_inner));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
@ -795,16 +784,16 @@ mod tests {
|
||||||
)],
|
)],
|
||||||
);
|
);
|
||||||
|
|
||||||
assert!(!check_rule(&rule, text).await);
|
assert!(!check_rule(&rule, text));
|
||||||
assert!(check_rule(&rule, png).await);
|
assert!(check_rule(&rule, png));
|
||||||
assert!(check_rule(&rule, jpg).await);
|
assert!(check_rule(&rule, jpg));
|
||||||
assert!(check_rule(&rule, jpeg).await);
|
assert!(check_rule(&rule, jpeg));
|
||||||
assert!(!check_rule(&rule, inner_text).await);
|
assert!(!check_rule(&rule, inner_text));
|
||||||
assert!(check_rule(&rule, inner_png).await);
|
assert!(check_rule(&rule, inner_png));
|
||||||
assert!(check_rule(&rule, inner_jpg).await);
|
assert!(check_rule(&rule, inner_jpg));
|
||||||
assert!(check_rule(&rule, inner_jpeg).await);
|
assert!(check_rule(&rule, inner_jpeg));
|
||||||
assert!(!check_rule(&rule, many_inner_dirs_text).await);
|
assert!(!check_rule(&rule, many_inner_dirs_text));
|
||||||
assert!(check_rule(&rule, many_inner_dirs_png).await);
|
assert!(check_rule(&rule, many_inner_dirs_png));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
@ -833,9 +822,22 @@ mod tests {
|
||||||
)],
|
)],
|
||||||
);
|
);
|
||||||
|
|
||||||
assert!(check_rule(&rule, project1).await);
|
assert!(
|
||||||
assert!(check_rule(&rule, project2).await);
|
!check_rule_with_metadata(&rule, &project1, &fs::metadata(&project1).await.unwrap())
|
||||||
assert!(!check_rule(&rule, not_project).await);
|
.await
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
!check_rule_with_metadata(&rule, &project2, &fs::metadata(&project2).await.unwrap())
|
||||||
|
.await
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
check_rule_with_metadata(
|
||||||
|
&rule,
|
||||||
|
¬_project,
|
||||||
|
&fs::metadata(¬_project).await.unwrap()
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
@ -864,9 +866,22 @@ mod tests {
|
||||||
)],
|
)],
|
||||||
);
|
);
|
||||||
|
|
||||||
assert!(!check_rule(&rule, project1).await);
|
assert!(
|
||||||
assert!(!check_rule(&rule, project2).await);
|
!check_rule_with_metadata(&rule, &project1, &fs::metadata(&project1).await.unwrap())
|
||||||
assert!(check_rule(&rule, not_project).await);
|
.await
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
!check_rule_with_metadata(&rule, &project2, &fs::metadata(&project2).await.unwrap())
|
||||||
|
.await
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
check_rule_with_metadata(
|
||||||
|
&rule,
|
||||||
|
¬_project,
|
||||||
|
&fs::metadata(¬_project).await.unwrap()
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
impl PartialEq for RulePerKind {
|
impl PartialEq for RulePerKind {
|
||||||
|
|
|
@ -1,25 +1,24 @@
|
||||||
use std::path::{Path, PathBuf};
|
|
||||||
|
|
||||||
use futures_concurrency::future::Join;
|
|
||||||
use gix_ignore::{glob::search::pattern::List, search::Ignore, Search};
|
|
||||||
use sd_prisma::prisma::{indexer_rule, PrismaClient};
|
use sd_prisma::prisma::{indexer_rule, PrismaClient};
|
||||||
|
|
||||||
|
use std::path::{Path, PathBuf};
|
||||||
|
|
||||||
use chrono::Utc;
|
use chrono::Utc;
|
||||||
use thiserror::Error;
|
use futures_concurrency::future::Join;
|
||||||
|
use gix_ignore::{glob::search::pattern::List, search::Ignore, Search};
|
||||||
|
use once_cell::sync::Lazy;
|
||||||
use tokio::fs;
|
use tokio::fs;
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
use super::{IndexerRule, IndexerRuleError, RulePerKind};
|
use super::{Error, IndexerRule, RulePerKind};
|
||||||
use once_cell::sync::Lazy;
|
|
||||||
|
|
||||||
#[derive(Error, Debug)]
|
#[derive(thiserror::Error, Debug)]
|
||||||
pub enum SeederError {
|
pub enum SeederError {
|
||||||
#[error("Failed to run indexer rules seeder: {0}")]
|
#[error("Failed to run indexer rules seeder: {0}")]
|
||||||
IndexerRules(#[from] IndexerRuleError),
|
IndexerRules(#[from] Error),
|
||||||
#[error("An error occurred with the database while applying migrations: {0}")]
|
#[error("An error occurred with the database while applying migrations: {0}")]
|
||||||
DatabaseError(#[from] prisma_client_rust::QueryError),
|
DatabaseError(#[from] prisma_client_rust::QueryError),
|
||||||
#[error("Failed to parse indexer rules based on external system")]
|
#[error("Failed to parse indexer rules based on external system")]
|
||||||
InhirentedExternalRules,
|
InheritedExternalRules,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
|
@ -29,7 +28,7 @@ pub struct GitIgnoreRules {
|
||||||
|
|
||||||
impl GitIgnoreRules {
|
impl GitIgnoreRules {
|
||||||
pub async fn get_rules_if_in_git_repo(
|
pub async fn get_rules_if_in_git_repo(
|
||||||
library_root: &Path,
|
location_root: &Path,
|
||||||
current: &Path,
|
current: &Path,
|
||||||
) -> Option<Result<Self, SeederError>> {
|
) -> Option<Result<Self, SeederError>> {
|
||||||
let mut git_repo = None;
|
let mut git_repo = None;
|
||||||
|
@ -38,7 +37,7 @@ impl GitIgnoreRules {
|
||||||
|
|
||||||
for ancestor in current
|
for ancestor in current
|
||||||
.ancestors()
|
.ancestors()
|
||||||
.take_while(|&path| path.starts_with(library_root))
|
.take_while(|&path| path.starts_with(location_root))
|
||||||
{
|
{
|
||||||
let git_ignore = ancestor.join(".gitignore");
|
let git_ignore = ancestor.join(".gitignore");
|
||||||
|
|
||||||
|
@ -54,13 +53,16 @@ impl GitIgnoreRules {
|
||||||
}
|
}
|
||||||
|
|
||||||
let git_repo = git_repo?;
|
let git_repo = git_repo?;
|
||||||
Some(Self::parse_gitrepo(git_repo, ignores).await)
|
Some(Self::parse_git_repo(git_repo, ignores).await)
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn parse_gitrepo(git_repo: &Path, gitignores: Vec<PathBuf>) -> Result<Self, SeederError> {
|
async fn parse_git_repo(
|
||||||
|
git_repo: &Path,
|
||||||
|
git_ignores: Vec<PathBuf>,
|
||||||
|
) -> Result<Self, SeederError> {
|
||||||
let mut search = Search::default();
|
let mut search = Search::default();
|
||||||
|
|
||||||
let gitignores = gitignores
|
let git_ignores = git_ignores
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.map(Self::parse_git_ignore)
|
.map(Self::parse_git_ignore)
|
||||||
.collect::<Vec<_>>()
|
.collect::<Vec<_>>()
|
||||||
|
@ -68,7 +70,7 @@ impl GitIgnoreRules {
|
||||||
.await;
|
.await;
|
||||||
search
|
search
|
||||||
.patterns
|
.patterns
|
||||||
.extend(gitignores.into_iter().filter_map(Result::ok));
|
.extend(git_ignores.into_iter().filter_map(Result::ok));
|
||||||
|
|
||||||
let git_exclude_rules = Self::parse_git_exclude(git_repo.join(".git")).await;
|
let git_exclude_rules = Self::parse_git_exclude(git_repo.join(".git")).await;
|
||||||
if let Ok(rules) = git_exclude_rules {
|
if let Ok(rules) = git_exclude_rules {
|
||||||
|
@ -86,11 +88,11 @@ impl GitIgnoreRules {
|
||||||
if let Ok(Some(patterns)) = List::from_file(gitignore, None, true, &mut buf) {
|
if let Ok(Some(patterns)) = List::from_file(gitignore, None, true, &mut buf) {
|
||||||
Ok(patterns)
|
Ok(patterns)
|
||||||
} else {
|
} else {
|
||||||
Err(SeederError::InhirentedExternalRules)
|
Err(SeederError::InheritedExternalRules)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
.await
|
.await
|
||||||
.map_err(|_| SeederError::InhirentedExternalRules)?
|
.map_err(|_| SeederError::InheritedExternalRules)?
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn parse_git_exclude(dot_git: PathBuf) -> Result<Vec<List<Ignore>>, SeederError> {
|
async fn parse_git_exclude(dot_git: PathBuf) -> Result<Vec<List<Ignore>>, SeederError> {
|
||||||
|
@ -98,10 +100,10 @@ impl GitIgnoreRules {
|
||||||
let mut buf = Vec::new();
|
let mut buf = Vec::new();
|
||||||
Search::from_git_dir(dot_git.as_ref(), None, &mut buf)
|
Search::from_git_dir(dot_git.as_ref(), None, &mut buf)
|
||||||
.map(|search| search.patterns)
|
.map(|search| search.patterns)
|
||||||
.map_err(|_| SeederError::InhirentedExternalRules)
|
.map_err(|_| SeederError::InheritedExternalRules)
|
||||||
})
|
})
|
||||||
.await
|
.await
|
||||||
.map_err(|_| SeederError::InhirentedExternalRules)?
|
.map_err(|_| SeederError::InheritedExternalRules)?
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn is_git_repo(path: &Path) -> bool {
|
async fn is_git_repo(path: &Path) -> bool {
|
||||||
|
@ -179,8 +181,8 @@ pub async fn new_or_existing_library(db: &PrismaClient) -> Result<(), SeederErro
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.enumerate()
|
.enumerate()
|
||||||
{
|
{
|
||||||
let pub_id = sd_utils::uuid_to_bytes(Uuid::from_u128(i as u128));
|
let pub_id = sd_utils::uuid_to_bytes(&Uuid::from_u128(i as u128));
|
||||||
let rules = rmp_serde::to_vec_named(&rule.rules).map_err(IndexerRuleError::from)?;
|
let rules = rmp_serde::to_vec_named(&rule.rules).map_err(Error::from)?;
|
||||||
|
|
||||||
let data = vec![
|
let data = vec![
|
||||||
name::set(Some(rule.name.to_string())),
|
name::set(Some(rule.name.to_string())),
|
||||||
|
|
|
@ -9,7 +9,10 @@ edition = { workspace = true }
|
||||||
[dependencies]
|
[dependencies]
|
||||||
# Spacedrive Sub-crates
|
# Spacedrive Sub-crates
|
||||||
sd-prisma = { path = "../../../crates/prisma" }
|
sd-prisma = { path = "../../../crates/prisma" }
|
||||||
|
sd-utils = { path = "../../../crates/utils" }
|
||||||
|
|
||||||
# Workspace dependencies
|
# Workspace dependencies
|
||||||
prisma-client-rust = { workspace = true }
|
prisma-client-rust = { workspace = true }
|
||||||
serde = { workspace = true }
|
serde = { workspace = true, features = ["derive"] }
|
||||||
|
specta = { workspace = true }
|
||||||
|
uuid = { workspace = true, features = ["v4", "serde"] }
|
||||||
|
|
|
@ -29,8 +29,16 @@
|
||||||
#![allow(clippy::missing_errors_doc, clippy::module_name_repetitions)]
|
#![allow(clippy::missing_errors_doc, clippy::module_name_repetitions)]
|
||||||
|
|
||||||
use sd_prisma::prisma::{file_path, job, label, location, object};
|
use sd_prisma::prisma::{file_path, job, label, location, object};
|
||||||
|
use sd_utils::{from_bytes_to_uuid, uuid_to_bytes};
|
||||||
|
|
||||||
|
use std::{borrow::Cow, fmt};
|
||||||
|
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
use specta::Type;
|
||||||
|
use uuid::Uuid;
|
||||||
|
|
||||||
// File Path selectables!
|
// File Path selectables!
|
||||||
|
file_path::select!(file_path_id { id });
|
||||||
file_path::select!(file_path_pub_id { pub_id });
|
file_path::select!(file_path_pub_id { pub_id });
|
||||||
file_path::select!(file_path_pub_and_cas_ids { id pub_id cas_id });
|
file_path::select!(file_path_pub_and_cas_ids { id pub_id cas_id });
|
||||||
file_path::select!(file_path_just_pub_id_materialized_path {
|
file_path::select!(file_path_just_pub_id_materialized_path {
|
||||||
|
@ -62,7 +70,10 @@ file_path::select!(file_path_for_media_processor {
|
||||||
name
|
name
|
||||||
extension
|
extension
|
||||||
cas_id
|
cas_id
|
||||||
object_id
|
object: select {
|
||||||
|
id
|
||||||
|
pub_id
|
||||||
|
}
|
||||||
});
|
});
|
||||||
file_path::select!(file_path_to_isolate {
|
file_path::select!(file_path_to_isolate {
|
||||||
location_id
|
location_id
|
||||||
|
@ -137,6 +148,11 @@ file_path::select!(file_path_to_full_path {
|
||||||
path
|
path
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
file_path::select!(file_path_to_create_object {
|
||||||
|
id
|
||||||
|
pub_id
|
||||||
|
date_created
|
||||||
|
});
|
||||||
|
|
||||||
// File Path includes!
|
// File Path includes!
|
||||||
file_path::include!(file_path_with_object { object });
|
file_path::include!(file_path_with_object { object });
|
||||||
|
@ -157,6 +173,7 @@ file_path::include!(file_path_for_frontend {
|
||||||
});
|
});
|
||||||
|
|
||||||
// Object selectables!
|
// Object selectables!
|
||||||
|
object::select!(object_ids { id pub_id });
|
||||||
object::select!(object_for_file_identifier {
|
object::select!(object_for_file_identifier {
|
||||||
pub_id
|
pub_id
|
||||||
file_paths: select { pub_id cas_id extension is_dir materialized_path name }
|
file_paths: select { pub_id cas_id extension is_dir materialized_path name }
|
||||||
|
@ -222,6 +239,14 @@ job::select!(job_without_data {
|
||||||
date_estimated_completion
|
date_estimated_completion
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// Location selectables!
|
||||||
|
location::select!(location_ids_and_path {
|
||||||
|
id
|
||||||
|
pub_id
|
||||||
|
instance_id
|
||||||
|
path
|
||||||
|
});
|
||||||
|
|
||||||
// Location includes!
|
// Location includes!
|
||||||
location::include!(location_with_indexer_rules {
|
location::include!(location_with_indexer_rules {
|
||||||
indexer_rules: select { indexer_rule }
|
indexer_rules: select { indexer_rule }
|
||||||
|
@ -284,3 +309,220 @@ label::include!((take: i64) => label_with_objects {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize, Deserialize, Hash, PartialEq, Eq, Type)]
|
||||||
|
#[serde(transparent)]
|
||||||
|
pub struct CasId<'cas_id>(Cow<'cas_id, str>);
|
||||||
|
|
||||||
|
impl Clone for CasId<'_> {
|
||||||
|
fn clone(&self) -> CasId<'static> {
|
||||||
|
CasId(Cow::Owned(self.0.clone().into_owned()))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<'cas_id> CasId<'cas_id> {
|
||||||
|
#[must_use]
|
||||||
|
pub fn as_str(&self) -> &str {
|
||||||
|
self.0.as_ref()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
|
pub fn to_owned(&self) -> CasId<'static> {
|
||||||
|
CasId(Cow::Owned(self.0.clone().into_owned()))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
|
pub fn into_owned(self) -> CasId<'static> {
|
||||||
|
CasId(Cow::Owned(self.0.clone().into_owned()))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<&CasId<'_>> for file_path::cas_id::Type {
|
||||||
|
fn from(CasId(cas_id): &CasId<'_>) -> Self {
|
||||||
|
Some(cas_id.clone().into_owned())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<'cas_id> From<&'cas_id str> for CasId<'cas_id> {
|
||||||
|
fn from(cas_id: &'cas_id str) -> Self {
|
||||||
|
Self(Cow::Borrowed(cas_id))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<'cas_id> From<&'cas_id String> for CasId<'cas_id> {
|
||||||
|
fn from(cas_id: &'cas_id String) -> Self {
|
||||||
|
Self(Cow::Borrowed(cas_id))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<String> for CasId<'static> {
|
||||||
|
fn from(cas_id: String) -> Self {
|
||||||
|
Self(cas_id.into())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<CasId<'_>> for String {
|
||||||
|
fn from(CasId(cas_id): CasId<'_>) -> Self {
|
||||||
|
cas_id.into_owned()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<&CasId<'_>> for String {
|
||||||
|
fn from(CasId(cas_id): &CasId<'_>) -> Self {
|
||||||
|
cas_id.clone().into_owned()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize, Deserialize, Hash, PartialEq, Eq, Clone)]
|
||||||
|
#[serde(transparent)]
|
||||||
|
#[repr(transparent)]
|
||||||
|
pub struct FilePathPubId(PubId);
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize, Deserialize, Hash, PartialEq, Eq, Clone)]
|
||||||
|
#[serde(transparent)]
|
||||||
|
#[repr(transparent)]
|
||||||
|
pub struct ObjectPubId(PubId);
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize, Deserialize, Hash, PartialEq, Eq, Clone)]
|
||||||
|
enum PubId {
|
||||||
|
Uuid(Uuid),
|
||||||
|
Vec(Vec<u8>),
|
||||||
|
}
|
||||||
|
|
||||||
|
impl PubId {
|
||||||
|
fn new() -> Self {
|
||||||
|
Self::Uuid(Uuid::new_v4())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn to_db(&self) -> Vec<u8> {
|
||||||
|
match self {
|
||||||
|
Self::Uuid(uuid) => uuid_to_bytes(uuid),
|
||||||
|
Self::Vec(bytes) => bytes.clone(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for PubId {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self::new()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<Uuid> for PubId {
|
||||||
|
fn from(uuid: Uuid) -> Self {
|
||||||
|
Self::Uuid(uuid)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<Vec<u8>> for PubId {
|
||||||
|
fn from(bytes: Vec<u8>) -> Self {
|
||||||
|
Self::Vec(bytes)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<&Vec<u8>> for PubId {
|
||||||
|
fn from(bytes: &Vec<u8>) -> Self {
|
||||||
|
Self::Vec(bytes.clone())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<&[u8]> for PubId {
|
||||||
|
fn from(bytes: &[u8]) -> Self {
|
||||||
|
Self::Vec(bytes.to_vec())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<PubId> for Vec<u8> {
|
||||||
|
fn from(pub_id: PubId) -> Self {
|
||||||
|
match pub_id {
|
||||||
|
PubId::Uuid(uuid) => uuid_to_bytes(&uuid),
|
||||||
|
PubId::Vec(bytes) => bytes,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<PubId> for Uuid {
|
||||||
|
fn from(pub_id: PubId) -> Self {
|
||||||
|
match pub_id {
|
||||||
|
PubId::Uuid(uuid) => uuid,
|
||||||
|
PubId::Vec(bytes) => from_bytes_to_uuid(&bytes),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl fmt::Display for PubId {
|
||||||
|
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||||
|
match self {
|
||||||
|
Self::Uuid(uuid) => write!(f, "{uuid}"),
|
||||||
|
Self::Vec(bytes) => write!(f, "{}", from_bytes_to_uuid(bytes)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
macro_rules! delegate_pub_id {
|
||||||
|
($($type_name:ty),+ $(,)?) => {
|
||||||
|
$(
|
||||||
|
impl From<::uuid::Uuid> for $type_name {
|
||||||
|
fn from(uuid: ::uuid::Uuid) -> Self {
|
||||||
|
Self(uuid.into())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<Vec<u8>> for $type_name {
|
||||||
|
fn from(bytes: Vec<u8>) -> Self {
|
||||||
|
Self(bytes.into())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<&Vec<u8>> for $type_name {
|
||||||
|
fn from(bytes: &Vec<u8>) -> Self {
|
||||||
|
Self(bytes.into())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<&[u8]> for $type_name {
|
||||||
|
fn from(bytes: &[u8]) -> Self {
|
||||||
|
Self(bytes.into())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<$type_name> for Vec<u8> {
|
||||||
|
fn from(pub_id: $type_name) -> Self {
|
||||||
|
pub_id.0.into()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<$type_name> for ::uuid::Uuid {
|
||||||
|
fn from(pub_id: $type_name) -> Self {
|
||||||
|
pub_id.0.into()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ::std::fmt::Display for $type_name {
|
||||||
|
fn fmt(&self, f: &mut ::std::fmt::Formatter<'_>) -> ::std::fmt::Result {
|
||||||
|
write!(f, "{}", self.0)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl $type_name {
|
||||||
|
#[must_use]
|
||||||
|
pub fn new() -> Self {
|
||||||
|
Self(PubId::new())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
|
pub fn to_db(&self) -> Vec<u8> {
|
||||||
|
self.0.to_db()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for $type_name {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self::new()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
)+
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
delegate_pub_id!(FilePathPubId, ObjectPubId);
|
||||||
|
|
|
@ -114,10 +114,10 @@ impl Actor {
|
||||||
}
|
}
|
||||||
State::Ingesting(event) => {
|
State::Ingesting(event) => {
|
||||||
debug!(
|
debug!(
|
||||||
"ingesting {} operations: {} to {}",
|
messages_count = event.messages.len(),
|
||||||
event.messages.len(),
|
first_message = event.messages.first().unwrap().3.timestamp.as_u64(),
|
||||||
event.messages.first().unwrap().3.timestamp.as_u64(),
|
last_message = event.messages.last().unwrap().3.timestamp.as_u64(),
|
||||||
event.messages.last().unwrap().3.timestamp.as_u64(),
|
"Ingesting operations;",
|
||||||
);
|
);
|
||||||
|
|
||||||
for (instance, data) in event.messages.0 {
|
for (instance, data) in event.messages.0 {
|
||||||
|
|
|
@ -175,7 +175,7 @@ impl Manager {
|
||||||
.crdt_operation()
|
.crdt_operation()
|
||||||
.find_many(vec![
|
.find_many(vec![
|
||||||
crdt_operation::instance::is(vec![instance::pub_id::equals(uuid_to_bytes(
|
crdt_operation::instance::is(vec![instance::pub_id::equals(uuid_to_bytes(
|
||||||
instance_uuid,
|
&instance_uuid,
|
||||||
))]),
|
))]),
|
||||||
crdt_operation::timestamp::gt(timestamp.as_u64() as i64),
|
crdt_operation::timestamp::gt(timestamp.as_u64() as i64),
|
||||||
])
|
])
|
||||||
|
@ -204,7 +204,7 @@ impl Manager {
|
||||||
.map(|(instance_id, timestamp)| {
|
.map(|(instance_id, timestamp)| {
|
||||||
prisma_client_rust::and![
|
prisma_client_rust::and![
|
||||||
$op::instance::is(vec![instance::pub_id::equals(uuid_to_bytes(
|
$op::instance::is(vec![instance::pub_id::equals(uuid_to_bytes(
|
||||||
*instance_id
|
instance_id
|
||||||
))]),
|
))]),
|
||||||
$op::timestamp::gt(timestamp.as_u64() as i64)
|
$op::timestamp::gt(timestamp.as_u64() as i64)
|
||||||
]
|
]
|
||||||
|
@ -216,7 +216,7 @@ impl Manager {
|
||||||
.clocks
|
.clocks
|
||||||
.iter()
|
.iter()
|
||||||
.map(|(instance_id, _)| {
|
.map(|(instance_id, _)| {
|
||||||
uuid_to_bytes(*instance_id)
|
uuid_to_bytes(instance_id)
|
||||||
})
|
})
|
||||||
.collect()
|
.collect()
|
||||||
)
|
)
|
||||||
|
@ -263,7 +263,7 @@ impl Manager {
|
||||||
.map(|(instance_id, timestamp)| {
|
.map(|(instance_id, timestamp)| {
|
||||||
prisma_client_rust::and![
|
prisma_client_rust::and![
|
||||||
$op::instance::is(vec![instance::pub_id::equals(uuid_to_bytes(
|
$op::instance::is(vec![instance::pub_id::equals(uuid_to_bytes(
|
||||||
*instance_id
|
instance_id
|
||||||
))]),
|
))]),
|
||||||
$op::timestamp::gt(timestamp.as_u64() as i64)
|
$op::timestamp::gt(timestamp.as_u64() as i64)
|
||||||
]
|
]
|
||||||
|
@ -275,7 +275,7 @@ impl Manager {
|
||||||
.clocks
|
.clocks
|
||||||
.iter()
|
.iter()
|
||||||
.map(|(instance_id, _)| {
|
.map(|(instance_id, _)| {
|
||||||
uuid_to_bytes(*instance_id)
|
uuid_to_bytes(instance_id)
|
||||||
})
|
})
|
||||||
.collect()
|
.collect()
|
||||||
)
|
)
|
||||||
|
|
|
@ -30,11 +30,11 @@ async fn write_test_location(
|
||||||
(
|
(
|
||||||
instance.sync.shared_create(
|
instance.sync.shared_create(
|
||||||
prisma_sync::location::SyncId {
|
prisma_sync::location::SyncId {
|
||||||
pub_id: uuid_to_bytes(id),
|
pub_id: uuid_to_bytes(&id),
|
||||||
},
|
},
|
||||||
sync_ops,
|
sync_ops,
|
||||||
),
|
),
|
||||||
instance.db.location().create(uuid_to_bytes(id), db_ops),
|
instance.db.location().create(uuid_to_bytes(&id), db_ops),
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
.await?)
|
.await?)
|
||||||
|
|
|
@ -36,7 +36,7 @@ impl Instance {
|
||||||
|
|
||||||
db.instance()
|
db.instance()
|
||||||
.create(
|
.create(
|
||||||
uuid_to_bytes(id),
|
uuid_to_bytes(&id),
|
||||||
vec![],
|
vec![],
|
||||||
vec![],
|
vec![],
|
||||||
Utc::now().into(),
|
Utc::now().into(),
|
||||||
|
@ -73,7 +73,7 @@ impl Instance {
|
||||||
left.db
|
left.db
|
||||||
.instance()
|
.instance()
|
||||||
.create(
|
.create(
|
||||||
uuid_to_bytes(right.id),
|
uuid_to_bytes(&right.id),
|
||||||
vec![],
|
vec![],
|
||||||
vec![],
|
vec![],
|
||||||
Utc::now().into(),
|
Utc::now().into(),
|
||||||
|
|
|
@ -150,15 +150,19 @@ async fn start_backup(node: Arc<Node>, library: Arc<Library>) -> Uuid {
|
||||||
match do_backup(bkp_id, &node, &library).await {
|
match do_backup(bkp_id, &node, &library).await {
|
||||||
Ok(path) => {
|
Ok(path) => {
|
||||||
info!(
|
info!(
|
||||||
"Backup '{bkp_id}' for library '{}' created at '{path:?}'!",
|
backup_id = %bkp_id,
|
||||||
library.id
|
library_id = %library.id,
|
||||||
|
path = %path.display(),
|
||||||
|
"Backup created!;",
|
||||||
);
|
);
|
||||||
invalidate_query!(library, "backups.getAll");
|
invalidate_query!(library, "backups.getAll");
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
error!(
|
error!(
|
||||||
"Error with backup '{bkp_id}' for library '{}': {e:?}",
|
backup_id = %bkp_id,
|
||||||
library.id
|
library_id = %library.id,
|
||||||
|
?e,
|
||||||
|
"Error with backup for library;",
|
||||||
);
|
);
|
||||||
|
|
||||||
// TODO: Alert user something went wrong
|
// TODO: Alert user something went wrong
|
||||||
|
@ -282,10 +286,10 @@ async fn do_backup(id: Uuid, node: &Node, library: &Library) -> Result<PathBuf,
|
||||||
async fn start_restore(node: Arc<Node>, path: PathBuf) {
|
async fn start_restore(node: Arc<Node>, path: PathBuf) {
|
||||||
match restore_backup(&node, &path).await {
|
match restore_backup(&node, &path).await {
|
||||||
Ok(Header { id, library_id, .. }) => {
|
Ok(Header { id, library_id, .. }) => {
|
||||||
info!("Restored to '{id}' for library '{library_id}'!",);
|
info!(%id, %library_id, "Restored backup for library!");
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
error!("Error restoring backup '{}': {e:#?}", path.display());
|
error!(path = %path.display(), ?e, "Error restoring backup;");
|
||||||
|
|
||||||
// TODO: Alert user something went wrong
|
// TODO: Alert user something went wrong
|
||||||
}
|
}
|
||||||
|
|
|
@ -155,9 +155,9 @@ mod library {
|
||||||
&library.db,
|
&library.db,
|
||||||
&library.sync,
|
&library.sync,
|
||||||
&node.libraries,
|
&node.libraries,
|
||||||
instance.uuid,
|
&instance.uuid,
|
||||||
instance.identity,
|
instance.identity,
|
||||||
instance.node_id,
|
&instance.node_id,
|
||||||
RemoteIdentity::from_str(&instance.node_remote_identity)
|
RemoteIdentity::from_str(&instance.node_remote_identity)
|
||||||
.expect("malformed remote identity in the DB"),
|
.expect("malformed remote identity in the DB"),
|
||||||
instance.metadata,
|
instance.metadata,
|
||||||
|
@ -304,8 +304,8 @@ mod locations {
|
||||||
.body(ByteStream::from_body_0_4(Full::from("Hello, world!")))
|
.body(ByteStream::from_body_0_4(Full::from("Hello, world!")))
|
||||||
.send()
|
.send()
|
||||||
.await
|
.await
|
||||||
.map_err(|err| {
|
.map_err(|e| {
|
||||||
tracing::error!("S3 error: {err:?}");
|
tracing::error!(?e, "S3 error;");
|
||||||
rspc::Error::new(
|
rspc::Error::new(
|
||||||
rspc::ErrorCode::InternalServerError,
|
rspc::ErrorCode::InternalServerError,
|
||||||
"Failed to upload to S3".to_string(),
|
"Failed to upload to S3".to_string(),
|
||||||
|
|
|
@ -7,11 +7,13 @@ use crate::{
|
||||||
library::Library,
|
library::Library,
|
||||||
object::{
|
object::{
|
||||||
fs::{error::FileSystemJobsError, find_available_filename_for_duplicate},
|
fs::{error::FileSystemJobsError, find_available_filename_for_duplicate},
|
||||||
media::exif_metadata_extractor::{can_extract_exif_data_for_image, extract_exif_data},
|
// media::exif_metadata_extractor::{can_extract_exif_data_for_image, extract_exif_data},
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
use sd_core_file_path_helper::IsolatedFilePathData;
|
use sd_core_file_path_helper::IsolatedFilePathData;
|
||||||
|
use sd_core_heavy_lifting::media_processor::exif_media_data;
|
||||||
|
|
||||||
use sd_file_ext::{
|
use sd_file_ext::{
|
||||||
extensions::{Extension, ImageExtension},
|
extensions::{Extension, ImageExtension},
|
||||||
kind::ObjectKind,
|
kind::ObjectKind,
|
||||||
|
@ -64,18 +66,18 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
||||||
};
|
};
|
||||||
|
|
||||||
let image_extension = ImageExtension::from_str(extension).map_err(|e| {
|
let image_extension = ImageExtension::from_str(extension).map_err(|e| {
|
||||||
error!("Failed to parse image extension: {e:#?}");
|
error!(?e, "Failed to parse image extension;");
|
||||||
rspc::Error::new(
|
rspc::Error::new(
|
||||||
ErrorCode::BadRequest,
|
ErrorCode::BadRequest,
|
||||||
"Invalid image extension".to_string(),
|
"Invalid image extension".to_string(),
|
||||||
)
|
)
|
||||||
})?;
|
})?;
|
||||||
|
|
||||||
if !can_extract_exif_data_for_image(&image_extension) {
|
if !exif_media_data::can_extract(image_extension) {
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
}
|
}
|
||||||
|
|
||||||
let exif_data = extract_exif_data(full_path)
|
let exif_data = exif_media_data::extract(full_path)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| {
|
.map_err(|e| {
|
||||||
rspc::Error::with_cause(
|
rspc::Error::with_cause(
|
||||||
|
@ -91,7 +93,7 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
||||||
Some(v) if v == ObjectKind::Audio || v == ObjectKind::Video => {
|
Some(v) if v == ObjectKind::Audio || v == ObjectKind::Video => {
|
||||||
let ffmpeg_data = MediaData::FFmpeg(
|
let ffmpeg_data = MediaData::FFmpeg(
|
||||||
FFmpegMetadata::from_path(full_path).await.map_err(|e| {
|
FFmpegMetadata::from_path(full_path).await.map_err(|e| {
|
||||||
error!("{e:#?}");
|
error!(?e, "Failed to extract ffmpeg metadata;");
|
||||||
rspc::Error::with_cause(
|
rspc::Error::with_cause(
|
||||||
ErrorCode::InternalServerError,
|
ErrorCode::InternalServerError,
|
||||||
e.to_string(),
|
e.to_string(),
|
||||||
|
@ -206,14 +208,15 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
||||||
))
|
))
|
||||||
})?;
|
})?;
|
||||||
|
|
||||||
Ok(())
|
Ok::<_, rspc::Error>(())
|
||||||
}
|
}
|
||||||
Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(()),
|
Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(()),
|
||||||
Err(e) => Err(FileIOError::from((
|
Err(e) => Err(FileIOError::from((
|
||||||
path,
|
path,
|
||||||
e,
|
e,
|
||||||
"Failed to get file metadata for deletion",
|
"Failed to get file metadata for deletion",
|
||||||
))),
|
))
|
||||||
|
.into()),
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
.collect::<Vec<_>>()
|
.collect::<Vec<_>>()
|
||||||
|
@ -384,9 +387,10 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
||||||
|
|
||||||
fs::rename(&old_path, &new_path).await.map_err(|e| {
|
fs::rename(&old_path, &new_path).await.map_err(|e| {
|
||||||
error!(
|
error!(
|
||||||
"Failed to rename file from: '{}' to: '{}'; Error: {e:#?}",
|
old_path = %old_path.display(),
|
||||||
old_path.display(),
|
new_path = %new_path.display(),
|
||||||
new_path.display()
|
?e,
|
||||||
|
"Failed to rename file;",
|
||||||
);
|
);
|
||||||
let e = FileIOError::from((old_path, e, "Failed to rename file"));
|
let e = FileIOError::from((old_path, e, "Failed to rename file"));
|
||||||
rspc::Error::with_cause(ErrorCode::Conflict, e.to_string(), e)
|
rspc::Error::with_cause(ErrorCode::Conflict, e.to_string(), e)
|
||||||
|
@ -493,7 +497,7 @@ impl EphemeralFileSystemOps {
|
||||||
let target = target_dir.join(name);
|
let target = target_dir.join(name);
|
||||||
Some((source, target))
|
Some((source, target))
|
||||||
} else {
|
} else {
|
||||||
warn!("Skipping file with no name: '{}'", source.display());
|
warn!(source = %source.display(), "Skipping file with no name;");
|
||||||
None
|
None
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
@ -615,7 +619,7 @@ impl EphemeralFileSystemOps {
|
||||||
let target = target_dir.join(name);
|
let target = target_dir.join(name);
|
||||||
Some((source, target))
|
Some((source, target))
|
||||||
} else {
|
} else {
|
||||||
warn!("Skipping file with no name: '{}'", source.display());
|
warn!(source = %source.display(), "Skipping file with no name;");
|
||||||
None
|
None
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
|
@ -9,12 +9,13 @@ use crate::{
|
||||||
old_copy::OldFileCopierJobInit, old_cut::OldFileCutterJobInit,
|
old_copy::OldFileCopierJobInit, old_cut::OldFileCutterJobInit,
|
||||||
old_delete::OldFileDeleterJobInit, old_erase::OldFileEraserJobInit,
|
old_delete::OldFileDeleterJobInit, old_erase::OldFileEraserJobInit,
|
||||||
},
|
},
|
||||||
media::{exif_media_data_from_prisma_data, ffmpeg_data_from_prisma_data},
|
// media::{exif_media_data_from_prisma_data, ffmpeg_data_from_prisma_data},
|
||||||
},
|
},
|
||||||
old_job::Job,
|
old_job::OldJob,
|
||||||
};
|
};
|
||||||
|
|
||||||
use sd_core_file_path_helper::{FilePathError, IsolatedFilePathData};
|
use sd_core_file_path_helper::{FilePathError, IsolatedFilePathData};
|
||||||
|
use sd_core_heavy_lifting::media_processor::{exif_media_data, ffmpeg_media_data};
|
||||||
use sd_core_prisma_helpers::{
|
use sd_core_prisma_helpers::{
|
||||||
file_path_to_isolate, file_path_to_isolate_with_id, object_with_file_paths,
|
file_path_to_isolate, file_path_to_isolate_with_id, object_with_file_paths,
|
||||||
object_with_media_data,
|
object_with_media_data,
|
||||||
|
@ -127,13 +128,13 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
||||||
.and_then(|obj| {
|
.and_then(|obj| {
|
||||||
Some(match obj.kind {
|
Some(match obj.kind {
|
||||||
Some(v) if v == ObjectKind::Image as i32 => MediaData::Exif(
|
Some(v) if v == ObjectKind::Image as i32 => MediaData::Exif(
|
||||||
exif_media_data_from_prisma_data(obj.exif_data?),
|
exif_media_data::from_prisma_data(obj.exif_data?),
|
||||||
),
|
),
|
||||||
Some(v)
|
Some(v)
|
||||||
if v == ObjectKind::Audio as i32
|
if v == ObjectKind::Audio as i32
|
||||||
|| v == ObjectKind::Video as i32 =>
|
|| v == ObjectKind::Video as i32 =>
|
||||||
{
|
{
|
||||||
MediaData::FFmpeg(ffmpeg_data_from_prisma_data(
|
MediaData::FFmpeg(ffmpeg_media_data::from_prisma_data(
|
||||||
obj.ffmpeg_data?,
|
obj.ffmpeg_data?,
|
||||||
))
|
))
|
||||||
}
|
}
|
||||||
|
@ -476,8 +477,8 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
||||||
Ok(()) => Ok(()),
|
Ok(()) => Ok(()),
|
||||||
Err(e) if e.kind() == io::ErrorKind::NotFound => {
|
Err(e) if e.kind() == io::ErrorKind::NotFound => {
|
||||||
warn!(
|
warn!(
|
||||||
"File not found in the file system, will remove from database: {}",
|
path = %full_path.display(),
|
||||||
full_path.display()
|
"File not found in the file system, will remove from database;",
|
||||||
);
|
);
|
||||||
library
|
library
|
||||||
.db
|
.db
|
||||||
|
@ -495,7 +496,7 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
_ => Job::new(args)
|
_ => OldJob::new(args)
|
||||||
.spawn(&node, &library)
|
.spawn(&node, &library)
|
||||||
.await
|
.await
|
||||||
.map_err(Into::into),
|
.map_err(Into::into),
|
||||||
|
@ -560,7 +561,7 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
_ => Job::new(args)
|
_ => OldJob::new(args)
|
||||||
.spawn(&node, &library)
|
.spawn(&node, &library)
|
||||||
.await
|
.await
|
||||||
.map_err(Into::into),
|
.map_err(Into::into),
|
||||||
|
@ -642,10 +643,11 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
||||||
})
|
})
|
||||||
.await
|
.await
|
||||||
.map_err(|e| {
|
.map_err(|e| {
|
||||||
error!("{e:#?}");
|
error!(?e, "Failed to convert image;");
|
||||||
rspc::Error::new(
|
rspc::Error::with_cause(
|
||||||
ErrorCode::InternalServerError,
|
ErrorCode::InternalServerError,
|
||||||
"Had an internal problem converting image".to_string(),
|
"Had an internal problem converting image".to_string(),
|
||||||
|
e,
|
||||||
)
|
)
|
||||||
})??;
|
})??;
|
||||||
|
|
||||||
|
@ -706,7 +708,7 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
||||||
.procedure("eraseFiles", {
|
.procedure("eraseFiles", {
|
||||||
R.with2(library())
|
R.with2(library())
|
||||||
.mutation(|(node, library), args: OldFileEraserJobInit| async move {
|
.mutation(|(node, library), args: OldFileEraserJobInit| async move {
|
||||||
Job::new(args)
|
OldJob::new(args)
|
||||||
.spawn(&node, &library)
|
.spawn(&node, &library)
|
||||||
.await
|
.await
|
||||||
.map_err(Into::into)
|
.map_err(Into::into)
|
||||||
|
@ -715,7 +717,7 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
||||||
.procedure("copyFiles", {
|
.procedure("copyFiles", {
|
||||||
R.with2(library())
|
R.with2(library())
|
||||||
.mutation(|(node, library), args: OldFileCopierJobInit| async move {
|
.mutation(|(node, library), args: OldFileCopierJobInit| async move {
|
||||||
Job::new(args)
|
OldJob::new(args)
|
||||||
.spawn(&node, &library)
|
.spawn(&node, &library)
|
||||||
.await
|
.await
|
||||||
.map_err(Into::into)
|
.map_err(Into::into)
|
||||||
|
@ -724,7 +726,7 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
||||||
.procedure("cutFiles", {
|
.procedure("cutFiles", {
|
||||||
R.with2(library())
|
R.with2(library())
|
||||||
.mutation(|(node, library), args: OldFileCutterJobInit| async move {
|
.mutation(|(node, library), args: OldFileCutterJobInit| async move {
|
||||||
Job::new(args)
|
OldJob::new(args)
|
||||||
.spawn(&node, &library)
|
.spawn(&node, &library)
|
||||||
.await
|
.await
|
||||||
.map_err(Into::into)
|
.map_err(Into::into)
|
||||||
|
@ -878,10 +880,11 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
||||||
} else {
|
} else {
|
||||||
fs::rename(&from, &to).await.map_err(|e| {
|
fs::rename(&from, &to).await.map_err(|e| {
|
||||||
error!(
|
error!(
|
||||||
"Failed to rename file from: '{}' to: '{}'; Error: {e:#?}",
|
from = %from.display(),
|
||||||
from.display(),
|
to = %to.display(),
|
||||||
to.display()
|
?e,
|
||||||
);
|
"Failed to rename file;",
|
||||||
|
);
|
||||||
rspc::Error::with_cause(
|
rspc::Error::with_cause(
|
||||||
ErrorCode::Conflict,
|
ErrorCode::Conflict,
|
||||||
"Failed to rename file".to_string(),
|
"Failed to rename file".to_string(),
|
||||||
|
|
|
@ -1,21 +1,22 @@
|
||||||
use crate::{
|
use crate::{
|
||||||
|
context::NodeContext,
|
||||||
invalidate_query,
|
invalidate_query,
|
||||||
location::{find_location, LocationError},
|
location::{find_location, LocationError},
|
||||||
object::{
|
object::validation::old_validator_job::OldObjectValidatorJobInit,
|
||||||
media::OldMediaProcessorJobInit,
|
old_job::{JobStatus, OldJob, OldJobReport},
|
||||||
old_file_identifier::old_file_identifier_job::OldFileIdentifierJobInit,
|
|
||||||
validation::old_validator_job::OldObjectValidatorJobInit,
|
|
||||||
},
|
|
||||||
old_job::{Job, JobReport, JobStatus, OldJobs},
|
|
||||||
};
|
};
|
||||||
|
|
||||||
use sd_core_prisma_helpers::job_without_data;
|
use sd_core_heavy_lifting::{
|
||||||
|
file_identifier::FileIdentifier, job_system::report, media_processor::job::MediaProcessor,
|
||||||
|
JobId, JobSystemError, Report,
|
||||||
|
};
|
||||||
|
|
||||||
use sd_prisma::prisma::{job, location, SortOrder};
|
use sd_prisma::prisma::{job, location, SortOrder};
|
||||||
|
|
||||||
use std::{
|
use std::{
|
||||||
collections::{hash_map::Entry, BTreeMap, HashMap, VecDeque},
|
collections::{hash_map::Entry, BTreeMap, HashMap, VecDeque},
|
||||||
path::PathBuf,
|
path::PathBuf,
|
||||||
|
sync::Arc,
|
||||||
time::Instant,
|
time::Instant,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -30,6 +31,8 @@ use uuid::Uuid;
|
||||||
|
|
||||||
use super::{utils::library, CoreEvent, Ctx, R};
|
use super::{utils::library, CoreEvent, Ctx, R};
|
||||||
|
|
||||||
|
const TEN_MINUTES: Duration = Duration::from_secs(60 * 10);
|
||||||
|
|
||||||
pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
||||||
R.router()
|
R.router()
|
||||||
.procedure("progress", {
|
.procedure("progress", {
|
||||||
|
@ -41,7 +44,7 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
||||||
.subscription(|(node, _), _: ()| async move {
|
.subscription(|(node, _), _: ()| async move {
|
||||||
let mut event_bus_rx = node.event_bus.0.subscribe();
|
let mut event_bus_rx = node.event_bus.0.subscribe();
|
||||||
// debounce per-job
|
// debounce per-job
|
||||||
let mut intervals = BTreeMap::<Uuid, Instant>::new();
|
let mut intervals = BTreeMap::<JobId, Instant>::new();
|
||||||
|
|
||||||
async_stream::stream! {
|
async_stream::stream! {
|
||||||
loop {
|
loop {
|
||||||
|
@ -62,6 +65,9 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
||||||
yield progress_event;
|
yield progress_event;
|
||||||
|
|
||||||
*instant = Instant::now();
|
*instant = Instant::now();
|
||||||
|
|
||||||
|
// remove stale jobs that didn't receive a progress for more than 10 minutes
|
||||||
|
intervals.retain(|_, instant| instant.elapsed() < TEN_MINUTES);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
@ -73,44 +79,53 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
||||||
// this is to ensure the client will always get the correct initial state
|
// this is to ensure the client will always get the correct initial state
|
||||||
// - jobs are sorted in to groups by their action
|
// - jobs are sorted in to groups by their action
|
||||||
// - TODO: refactor grouping system to a many-to-many table
|
// - TODO: refactor grouping system to a many-to-many table
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize, Type)]
|
#[derive(Debug, Clone, Serialize, Type)]
|
||||||
pub struct JobGroup {
|
pub struct JobGroup {
|
||||||
id: Uuid,
|
id: JobId,
|
||||||
|
running_job_id: Option<JobId>,
|
||||||
action: Option<String>,
|
action: Option<String>,
|
||||||
status: JobStatus,
|
status: report::Status,
|
||||||
created_at: DateTime<Utc>,
|
created_at: DateTime<Utc>,
|
||||||
jobs: VecDeque<JobReport>,
|
jobs: VecDeque<Report>,
|
||||||
}
|
}
|
||||||
|
|
||||||
R.with2(library())
|
R.with2(library())
|
||||||
.query(|(node, library), _: ()| async move {
|
.query(|(node, library), _: ()| async move {
|
||||||
let mut groups: HashMap<String, JobGroup> = HashMap::new();
|
let mut groups: HashMap<String, JobGroup> = HashMap::new();
|
||||||
|
|
||||||
let job_reports: Vec<JobReport> = library
|
let job_reports: Vec<Report> = library
|
||||||
.db
|
.db
|
||||||
.job()
|
.job()
|
||||||
.find_many(vec![])
|
.find_many(vec![])
|
||||||
.order_by(job::date_created::order(SortOrder::Desc))
|
.order_by(job::date_created::order(SortOrder::Desc))
|
||||||
.take(100)
|
.take(100)
|
||||||
.select(job_without_data::select())
|
|
||||||
.exec()
|
.exec()
|
||||||
.await?
|
.await?
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.flat_map(JobReport::try_from)
|
.flat_map(|job| {
|
||||||
|
if let Ok(report) = Report::try_from(job.clone()) {
|
||||||
|
Some(report)
|
||||||
|
} else {
|
||||||
|
// TODO(fogodev): this is a temporary fix for the old job system
|
||||||
|
OldJobReport::try_from(job).map(Into::into).ok()
|
||||||
|
}
|
||||||
|
})
|
||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
let active_reports_by_id = node.old_jobs.get_active_reports_with_id().await;
|
let mut active_reports_by_id = node.job_system.get_active_reports().await;
|
||||||
|
active_reports_by_id.extend(
|
||||||
|
node.old_jobs
|
||||||
|
.get_active_reports_with_id()
|
||||||
|
.await
|
||||||
|
.into_iter()
|
||||||
|
.map(|(id, old_report)| (id, old_report.into())),
|
||||||
|
);
|
||||||
|
|
||||||
for job in job_reports {
|
for job in job_reports {
|
||||||
// action name and group key are computed from the job data
|
// action name and group key are computed from the job data
|
||||||
let (action_name, group_key) = job.get_meta();
|
let (action_name, group_key) = job.get_action_name_and_group_key();
|
||||||
|
|
||||||
trace!(
|
trace!(?job, %action_name, ?group_key);
|
||||||
"job {:#?}, action_name {}, group_key {:?}",
|
|
||||||
job,
|
|
||||||
action_name,
|
|
||||||
group_key
|
|
||||||
);
|
|
||||||
|
|
||||||
// if the job is running, use the in-memory report
|
// if the job is running, use the in-memory report
|
||||||
let report = active_reports_by_id.get(&job.id).unwrap_or(&job);
|
let report = active_reports_by_id.get(&job.id).unwrap_or(&job);
|
||||||
|
@ -122,7 +137,10 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
||||||
Entry::Vacant(entry) => {
|
Entry::Vacant(entry) => {
|
||||||
entry.insert(JobGroup {
|
entry.insert(JobGroup {
|
||||||
id: job.parent_id.unwrap_or(job.id),
|
id: job.parent_id.unwrap_or(job.id),
|
||||||
action: Some(action_name.clone()),
|
running_job_id: (job.status == report::Status::Running
|
||||||
|
|| job.status == report::Status::Paused)
|
||||||
|
.then_some(job.id),
|
||||||
|
action: Some(action_name),
|
||||||
status: job.status,
|
status: job.status,
|
||||||
jobs: [report.clone()].into_iter().collect(),
|
jobs: [report.clone()].into_iter().collect(),
|
||||||
created_at: job.created_at.unwrap_or(Utc::now()),
|
created_at: job.created_at.unwrap_or(Utc::now()),
|
||||||
|
@ -132,8 +150,10 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
||||||
Entry::Occupied(mut entry) => {
|
Entry::Occupied(mut entry) => {
|
||||||
let group = entry.get_mut();
|
let group = entry.get_mut();
|
||||||
|
|
||||||
// protect paused status from being overwritten
|
if report.status == report::Status::Running
|
||||||
if report.status != JobStatus::Paused {
|
|| report.status == report::Status::Paused
|
||||||
|
{
|
||||||
|
group.running_job_id = Some(report.id);
|
||||||
group.status = report.status;
|
group.status = report.status;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -146,6 +166,7 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
||||||
job.id.to_string(),
|
job.id.to_string(),
|
||||||
JobGroup {
|
JobGroup {
|
||||||
id: job.id,
|
id: job.id,
|
||||||
|
running_job_id: Some(job.id),
|
||||||
action: None,
|
action: None,
|
||||||
status: job.status,
|
status: job.status,
|
||||||
jobs: [report.clone()].into_iter().collect(),
|
jobs: [report.clone()].into_iter().collect(),
|
||||||
|
@ -164,7 +185,14 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
||||||
.procedure("isActive", {
|
.procedure("isActive", {
|
||||||
R.with2(library())
|
R.with2(library())
|
||||||
.query(|(node, library), _: ()| async move {
|
.query(|(node, library), _: ()| async move {
|
||||||
Ok(node.old_jobs.has_active_workers(library.id).await)
|
let library_id = library.id;
|
||||||
|
Ok(node
|
||||||
|
.job_system
|
||||||
|
.has_active_jobs(NodeContext {
|
||||||
|
node: Arc::clone(&node),
|
||||||
|
library,
|
||||||
|
})
|
||||||
|
.await || node.old_jobs.has_active_workers(library_id).await)
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
.procedure("clear", {
|
.procedure("clear", {
|
||||||
|
@ -204,30 +232,56 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
||||||
// pause job
|
// pause job
|
||||||
.procedure("pause", {
|
.procedure("pause", {
|
||||||
R.with2(library())
|
R.with2(library())
|
||||||
.mutation(|(node, library), id: Uuid| async move {
|
.mutation(|(node, library), job_id: JobId| async move {
|
||||||
let ret = OldJobs::pause(&node.old_jobs, id).await.map_err(Into::into);
|
if let Err(e) = node.job_system.pause(job_id).await {
|
||||||
|
if matches!(e, JobSystemError::NotFound(_)) {
|
||||||
|
// If the job is not found, it can be a job from the old job system
|
||||||
|
node.old_jobs.pause(job_id).await?;
|
||||||
|
} else {
|
||||||
|
return Err(e.into());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
invalidate_query!(library, "jobs.isActive");
|
||||||
invalidate_query!(library, "jobs.reports");
|
invalidate_query!(library, "jobs.reports");
|
||||||
ret
|
|
||||||
|
Ok(())
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
.procedure("resume", {
|
.procedure("resume", {
|
||||||
R.with2(library())
|
R.with2(library())
|
||||||
.mutation(|(node, library), id: Uuid| async move {
|
.mutation(|(node, library), job_id: JobId| async move {
|
||||||
let ret = OldJobs::resume(&node.old_jobs, id)
|
if let Err(e) = node.job_system.resume(job_id).await {
|
||||||
.await
|
if matches!(e, JobSystemError::NotFound(_)) {
|
||||||
.map_err(Into::into);
|
// If the job is not found, it can be a job from the old job system
|
||||||
|
node.old_jobs.resume(job_id).await?;
|
||||||
|
} else {
|
||||||
|
return Err(e.into());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
invalidate_query!(library, "jobs.isActive");
|
||||||
invalidate_query!(library, "jobs.reports");
|
invalidate_query!(library, "jobs.reports");
|
||||||
ret
|
|
||||||
|
Ok(())
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
.procedure("cancel", {
|
.procedure("cancel", {
|
||||||
R.with2(library())
|
R.with2(library())
|
||||||
.mutation(|(node, library), id: Uuid| async move {
|
.mutation(|(node, library), job_id: JobId| async move {
|
||||||
let ret = OldJobs::cancel(&node.old_jobs, id)
|
if let Err(e) = node.job_system.cancel(job_id).await {
|
||||||
.await
|
if matches!(e, JobSystemError::NotFound(_)) {
|
||||||
.map_err(Into::into);
|
// If the job is not found, it can be a job from the old job system
|
||||||
|
node.old_jobs.cancel(job_id).await?;
|
||||||
|
} else {
|
||||||
|
return Err(e.into());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
invalidate_query!(library, "jobs.isActive");
|
||||||
invalidate_query!(library, "jobs.reports");
|
invalidate_query!(library, "jobs.reports");
|
||||||
ret
|
|
||||||
|
Ok(())
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
.procedure("generateThumbsForLocation", {
|
.procedure("generateThumbsForLocation", {
|
||||||
|
@ -250,50 +304,50 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
||||||
return Err(LocationError::IdNotFound(id).into());
|
return Err(LocationError::IdNotFound(id).into());
|
||||||
};
|
};
|
||||||
|
|
||||||
Job::new(OldMediaProcessorJobInit {
|
node.job_system
|
||||||
location,
|
.dispatch(
|
||||||
sub_path: Some(path),
|
MediaProcessor::new(location, Some(path), regenerate)?,
|
||||||
regenerate_thumbnails: regenerate,
|
id,
|
||||||
regenerate_labels: false,
|
NodeContext {
|
||||||
})
|
node: Arc::clone(&node),
|
||||||
.spawn(&node, &library)
|
library,
|
||||||
.await
|
},
|
||||||
.map_err(Into::into)
|
)
|
||||||
},
|
.await
|
||||||
)
|
.map_err(Into::into)
|
||||||
})
|
|
||||||
.procedure("generateLabelsForLocation", {
|
|
||||||
#[derive(Type, Deserialize)]
|
|
||||||
pub struct GenerateLabelsForLocationArgs {
|
|
||||||
pub id: location::id::Type,
|
|
||||||
pub path: PathBuf,
|
|
||||||
#[serde(default)]
|
|
||||||
pub regenerate: bool,
|
|
||||||
}
|
|
||||||
|
|
||||||
R.with2(library()).mutation(
|
|
||||||
|(node, library),
|
|
||||||
GenerateLabelsForLocationArgs {
|
|
||||||
id,
|
|
||||||
path,
|
|
||||||
regenerate,
|
|
||||||
}: GenerateLabelsForLocationArgs| async move {
|
|
||||||
let Some(location) = find_location(&library, id).exec().await? else {
|
|
||||||
return Err(LocationError::IdNotFound(id).into());
|
|
||||||
};
|
|
||||||
|
|
||||||
Job::new(OldMediaProcessorJobInit {
|
|
||||||
location,
|
|
||||||
sub_path: Some(path),
|
|
||||||
regenerate_thumbnails: false,
|
|
||||||
regenerate_labels: regenerate,
|
|
||||||
})
|
|
||||||
.spawn(&node, &library)
|
|
||||||
.await
|
|
||||||
.map_err(Into::into)
|
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
|
// .procedure("generateLabelsForLocation", {
|
||||||
|
// #[derive(Type, Deserialize)]
|
||||||
|
// pub struct GenerateLabelsForLocationArgs {
|
||||||
|
// pub id: location::id::Type,
|
||||||
|
// pub path: PathBuf,
|
||||||
|
// #[serde(default)]
|
||||||
|
// pub regenerate: bool,
|
||||||
|
// }
|
||||||
|
// R.with2(library()).mutation(
|
||||||
|
// |(node, library),
|
||||||
|
// GenerateLabelsForLocationArgs {
|
||||||
|
// id,
|
||||||
|
// path,
|
||||||
|
// regenerate,
|
||||||
|
// }: GenerateLabelsForLocationArgs| async move {
|
||||||
|
// let Some(location) = find_location(&library, id).exec().await? else {
|
||||||
|
// return Err(LocationError::IdNotFound(id).into());
|
||||||
|
// };
|
||||||
|
// OldJob::new(OldMediaProcessorJobInit {
|
||||||
|
// location,
|
||||||
|
// sub_path: Some(path),
|
||||||
|
// regenerate_thumbnails: false,
|
||||||
|
// regenerate_labels: regenerate,
|
||||||
|
// })
|
||||||
|
// .spawn(&node, &library)
|
||||||
|
// .await
|
||||||
|
// .map_err(Into::into)
|
||||||
|
// },
|
||||||
|
// )
|
||||||
|
// })
|
||||||
.procedure("objectValidator", {
|
.procedure("objectValidator", {
|
||||||
#[derive(Type, Deserialize)]
|
#[derive(Type, Deserialize)]
|
||||||
pub struct ObjectValidatorArgs {
|
pub struct ObjectValidatorArgs {
|
||||||
|
@ -307,7 +361,7 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
||||||
return Err(LocationError::IdNotFound(args.id).into());
|
return Err(LocationError::IdNotFound(args.id).into());
|
||||||
};
|
};
|
||||||
|
|
||||||
Job::new(OldObjectValidatorJobInit {
|
OldJob::new(OldObjectValidatorJobInit {
|
||||||
location,
|
location,
|
||||||
sub_path: Some(args.path),
|
sub_path: Some(args.path),
|
||||||
})
|
})
|
||||||
|
@ -324,18 +378,22 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
||||||
}
|
}
|
||||||
|
|
||||||
R.with2(library()).mutation(
|
R.with2(library()).mutation(
|
||||||
|(node, library), args: IdentifyUniqueFilesArgs| async move {
|
|(node, library), IdentifyUniqueFilesArgs { id, path }: IdentifyUniqueFilesArgs| async move {
|
||||||
let Some(location) = find_location(&library, args.id).exec().await? else {
|
let Some(location) = find_location(&library, id).exec().await? else {
|
||||||
return Err(LocationError::IdNotFound(args.id).into());
|
return Err(LocationError::IdNotFound(id).into());
|
||||||
};
|
};
|
||||||
|
|
||||||
Job::new(OldFileIdentifierJobInit {
|
node.job_system
|
||||||
location,
|
.dispatch(
|
||||||
sub_path: Some(args.path),
|
FileIdentifier::new(location, Some(path))?,
|
||||||
})
|
id,
|
||||||
.spawn(&node, &library)
|
NodeContext {
|
||||||
.await
|
node: Arc::clone(&node),
|
||||||
.map_err(Into::into)
|
library,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(Into::into)
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
|
|
|
@ -1,8 +1,7 @@
|
||||||
use crate::{
|
use crate::{invalidate_query, library::Library};
|
||||||
invalidate_query, library::Library, object::media::old_thumbnail::get_indexed_thumb_key,
|
|
||||||
};
|
|
||||||
|
|
||||||
use sd_core_prisma_helpers::label_with_objects;
|
use sd_core_heavy_lifting::media_processor::ThumbKey;
|
||||||
|
use sd_core_prisma_helpers::{label_with_objects, CasId};
|
||||||
|
|
||||||
use sd_prisma::{
|
use sd_prisma::{
|
||||||
prisma::{label, label_on_object, object, SortOrder},
|
prisma::{label, label_on_object, object, SortOrder},
|
||||||
|
@ -49,7 +48,9 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
||||||
file_path_data
|
file_path_data
|
||||||
.cas_id
|
.cas_id
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.map(|cas_id| get_indexed_thumb_key(cas_id, library.id))
|
.map(CasId::from)
|
||||||
|
.map(CasId::into_owned)
|
||||||
|
.map(|cas_id| ThumbKey::new_indexed(cas_id, library.id))
|
||||||
}) // Filter out None values and transform each element to Vec<Vec<String>>
|
}) // Filter out None values and transform each element to Vec<Vec<String>>
|
||||||
.collect::<Vec<_>>(), // Collect into Vec<Vec<Vec<String>>>
|
.collect::<Vec<_>>(), // Collect into Vec<Vec<Vec<String>>>
|
||||||
})
|
})
|
||||||
|
|
|
@ -8,6 +8,7 @@ use crate::{
|
||||||
|
|
||||||
use futures::StreamExt;
|
use futures::StreamExt;
|
||||||
use prisma_client_rust::raw;
|
use prisma_client_rust::raw;
|
||||||
|
use sd_core_heavy_lifting::JobId;
|
||||||
use sd_file_ext::kind::ObjectKind;
|
use sd_file_ext::kind::ObjectKind;
|
||||||
use sd_p2p::RemoteIdentity;
|
use sd_p2p::RemoteIdentity;
|
||||||
use sd_prisma::prisma::{indexer_rule, object, statistics};
|
use sd_prisma::prisma::{indexer_rule, object, statistics};
|
||||||
|
@ -106,7 +107,7 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
||||||
match STATISTICS_UPDATERS.lock().await.entry(library.id) {
|
match STATISTICS_UPDATERS.lock().await.entry(library.id) {
|
||||||
Entry::Occupied(entry) => {
|
Entry::Occupied(entry) => {
|
||||||
if entry.get().send(Instant::now()).await.is_err() {
|
if entry.get().send(Instant::now()).await.is_err() {
|
||||||
error!("Failed to send statistics update request");
|
error!("Failed to send statistics update request;");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Entry::Vacant(entry) => {
|
Entry::Vacant(entry) => {
|
||||||
|
@ -181,13 +182,13 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
||||||
}: DefaultLocations,
|
}: DefaultLocations,
|
||||||
node: Arc<Node>,
|
node: Arc<Node>,
|
||||||
library: Arc<Library>,
|
library: Arc<Library>,
|
||||||
) -> Result<(), rspc::Error> {
|
) -> Result<Option<JobId>, rspc::Error> {
|
||||||
// If all of them are false, we skip
|
// If all of them are false, we skip
|
||||||
if [!desktop, !documents, !downloads, !pictures, !music, !videos]
|
if [!desktop, !documents, !downloads, !pictures, !music, !videos]
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.all(identity)
|
.all(identity)
|
||||||
{
|
{
|
||||||
return Ok(());
|
return Ok(None);
|
||||||
}
|
}
|
||||||
|
|
||||||
let Some(default_locations_paths) = UserDirs::new() else {
|
let Some(default_locations_paths) = UserDirs::new() else {
|
||||||
|
@ -242,7 +243,7 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
||||||
.await
|
.await
|
||||||
.map_err(rspc::Error::from)?
|
.map_err(rspc::Error::from)?
|
||||||
else {
|
else {
|
||||||
return Ok(());
|
return Ok(None);
|
||||||
};
|
};
|
||||||
|
|
||||||
let scan_state = ScanState::try_from(location.scan_state)?;
|
let scan_state = ScanState::try_from(location.scan_state)?;
|
||||||
|
@ -271,7 +272,7 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
||||||
})
|
})
|
||||||
.fold(&mut maybe_error, |maybe_error, res| {
|
.fold(&mut maybe_error, |maybe_error, res| {
|
||||||
if let Err(e) = res {
|
if let Err(e) = res {
|
||||||
error!("Failed to create default location: {e:#?}");
|
error!(?e, "Failed to create default location;");
|
||||||
*maybe_error = Some(e);
|
*maybe_error = Some(e);
|
||||||
}
|
}
|
||||||
maybe_error
|
maybe_error
|
||||||
|
@ -283,7 +284,7 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
||||||
|
|
||||||
debug!("Created default locations");
|
debug!("Created default locations");
|
||||||
|
|
||||||
Ok(())
|
Ok(None)
|
||||||
}
|
}
|
||||||
|
|
||||||
R.mutation(
|
R.mutation(
|
||||||
|
@ -296,7 +297,7 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
||||||
|
|
||||||
let library = node.libraries.create(name, None, &node).await?;
|
let library = node.libraries.create(name, None, &node).await?;
|
||||||
|
|
||||||
debug!("Created library {}", library.id);
|
debug!(%library.id, "Created library;");
|
||||||
|
|
||||||
if let Some(locations) = default_locations {
|
if let Some(locations) = default_locations {
|
||||||
create_default_locations_on_library_creation(
|
create_default_locations_on_library_creation(
|
||||||
|
@ -381,16 +382,19 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
||||||
for _ in 0..5 {
|
for _ in 0..5 {
|
||||||
match library.db._execute_raw(raw!("VACUUM;")).exec().await {
|
match library.db._execute_raw(raw!("VACUUM;")).exec().await {
|
||||||
Ok(_) => break,
|
Ok(_) => break,
|
||||||
Err(err) => {
|
Err(e) => {
|
||||||
warn!(
|
warn!(
|
||||||
"Failed to vacuum DB for library '{}', retrying...: {err:#?}",
|
%library.id,
|
||||||
library.id
|
?e,
|
||||||
|
"Failed to vacuum DB for library, retrying...;",
|
||||||
);
|
);
|
||||||
tokio::time::sleep(Duration::from_millis(500)).await;
|
tokio::time::sleep(Duration::from_millis(500)).await;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
info!("Successfully vacuumed DB for library '{}'", library.id);
|
|
||||||
|
info!(%library.id, "Successfully vacuumed DB;");
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}),
|
}),
|
||||||
)
|
)
|
||||||
|
@ -421,7 +425,7 @@ async fn update_statistics_loop(
|
||||||
Message::Tick => {
|
Message::Tick => {
|
||||||
if last_received_at.elapsed() < FIVE_MINUTES {
|
if last_received_at.elapsed() < FIVE_MINUTES {
|
||||||
if let Err(e) = update_library_statistics(&node, &library).await {
|
if let Err(e) = update_library_statistics(&node, &library).await {
|
||||||
error!("Failed to update library statistics: {e:#?}");
|
error!(?e, "Failed to update library statistics;");
|
||||||
} else {
|
} else {
|
||||||
invalidate_query!(&library, "library.statistics");
|
invalidate_query!(&library, "library.statistics");
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,16 +1,15 @@
|
||||||
use crate::{
|
use crate::{
|
||||||
invalidate_query,
|
invalidate_query,
|
||||||
location::{
|
location::{
|
||||||
delete_location, find_location, indexer::OldIndexerJobInit, light_scan_location,
|
delete_location, find_location, light_scan_location, non_indexed::NonIndexedPathItem,
|
||||||
non_indexed::NonIndexedPathItem, relink_location, scan_location, scan_location_sub_path,
|
relink_location, scan_location, scan_location_sub_path, LocationCreateArgs, LocationError,
|
||||||
LocationCreateArgs, LocationError, LocationUpdateArgs, ScanState,
|
LocationUpdateArgs, ScanState,
|
||||||
},
|
},
|
||||||
object::old_file_identifier::old_file_identifier_job::OldFileIdentifierJobInit,
|
|
||||||
old_job::StatefulJob,
|
|
||||||
p2p::PeerMetadata,
|
p2p::PeerMetadata,
|
||||||
util::AbortOnDrop,
|
util::AbortOnDrop,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
use sd_core_heavy_lifting::{media_processor::ThumbKey, JobName};
|
||||||
use sd_core_indexer_rules::IndexerRuleCreateArgs;
|
use sd_core_indexer_rules::IndexerRuleCreateArgs;
|
||||||
use sd_core_prisma_helpers::{
|
use sd_core_prisma_helpers::{
|
||||||
file_path_for_frontend, label_with_objects, location_with_indexer_rules, object_with_file_paths,
|
file_path_for_frontend, label_with_objects, location_with_indexer_rules, object_with_file_paths,
|
||||||
|
@ -29,28 +28,24 @@ use tracing::{debug, error};
|
||||||
|
|
||||||
use super::{utils::library, Ctx, R};
|
use super::{utils::library, Ctx, R};
|
||||||
|
|
||||||
// it includes the shard hex formatted as ([["f02", "cab34a76fbf3469f"]])
|
|
||||||
// Will be None if no thumbnail exists
|
|
||||||
pub type ThumbnailKey = Vec<String>;
|
|
||||||
|
|
||||||
#[derive(Serialize, Type, Debug)]
|
#[derive(Serialize, Type, Debug)]
|
||||||
#[serde(tag = "type")]
|
#[serde(tag = "type")]
|
||||||
pub enum ExplorerItem {
|
pub enum ExplorerItem {
|
||||||
Path {
|
Path {
|
||||||
// provide the frontend with the thumbnail key explicitly
|
// provide the frontend with the thumbnail key explicitly
|
||||||
thumbnail: Option<ThumbnailKey>,
|
thumbnail: Option<ThumbKey>,
|
||||||
// this tells the frontend if a thumbnail actually exists or not
|
// this tells the frontend if a thumbnail actually exists or not
|
||||||
has_created_thumbnail: bool,
|
has_created_thumbnail: bool,
|
||||||
// we can't actually modify data from PCR types, thats why computed properties are used on ExplorerItem
|
// we can't actually modify data from PCR types, thats why computed properties are used on ExplorerItem
|
||||||
item: Box<file_path_for_frontend::Data>,
|
item: Box<file_path_for_frontend::Data>,
|
||||||
},
|
},
|
||||||
Object {
|
Object {
|
||||||
thumbnail: Option<ThumbnailKey>,
|
thumbnail: Option<ThumbKey>,
|
||||||
has_created_thumbnail: bool,
|
has_created_thumbnail: bool,
|
||||||
item: object_with_file_paths::Data,
|
item: object_with_file_paths::Data,
|
||||||
},
|
},
|
||||||
NonIndexedPath {
|
NonIndexedPath {
|
||||||
thumbnail: Option<ThumbnailKey>,
|
thumbnail: Option<ThumbKey>,
|
||||||
has_created_thumbnail: bool,
|
has_created_thumbnail: bool,
|
||||||
item: NonIndexedPathItem,
|
item: NonIndexedPathItem,
|
||||||
},
|
},
|
||||||
|
@ -61,7 +56,7 @@ pub enum ExplorerItem {
|
||||||
item: PeerMetadata,
|
item: PeerMetadata,
|
||||||
},
|
},
|
||||||
Label {
|
Label {
|
||||||
thumbnails: Vec<ThumbnailKey>,
|
thumbnails: Vec<ThumbKey>,
|
||||||
item: label_with_objects::Data,
|
item: label_with_objects::Data,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
@ -347,7 +342,7 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
||||||
.exec()
|
.exec()
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
debug!("Disconnected {count} file paths from objects");
|
debug!(%count, "Disconnected file paths from objects;");
|
||||||
|
|
||||||
// library.orphan_remover.invoke().await;
|
// library.orphan_remover.invoke().await;
|
||||||
}
|
}
|
||||||
|
@ -409,13 +404,15 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
||||||
sub_path,
|
sub_path,
|
||||||
}: LightScanArgs| async move {
|
}: LightScanArgs| async move {
|
||||||
if node
|
if node
|
||||||
.old_jobs
|
.job_system
|
||||||
.has_job_running(|job_identity| {
|
.check_running_jobs(
|
||||||
job_identity.target_location == location_id
|
vec![
|
||||||
&& (job_identity.name == <OldIndexerJobInit as StatefulJob>::NAME
|
JobName::Indexer,
|
||||||
|| job_identity.name
|
JobName::FileIdentifier,
|
||||||
== <OldFileIdentifierJobInit as StatefulJob>::NAME)
|
JobName::MediaProcessor,
|
||||||
})
|
],
|
||||||
|
location_id,
|
||||||
|
)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
return Err(rspc::Error::new(
|
return Err(rspc::Error::new(
|
||||||
|
@ -433,7 +430,7 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
||||||
let handle = tokio::spawn(async move {
|
let handle = tokio::spawn(async move {
|
||||||
if let Err(e) = light_scan_location(node, library, location, sub_path).await
|
if let Err(e) = light_scan_location(node, library, location, sub_path).await
|
||||||
{
|
{
|
||||||
error!("light scan error: {e:#?}");
|
error!(?e, "Light scan error;");
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
|
@ -8,6 +8,7 @@ use crate::{
|
||||||
Node,
|
Node,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
use sd_core_heavy_lifting::media_processor::ThumbKey;
|
||||||
use sd_p2p::RemoteIdentity;
|
use sd_p2p::RemoteIdentity;
|
||||||
use sd_prisma::prisma::file_path;
|
use sd_prisma::prisma::file_path;
|
||||||
|
|
||||||
|
@ -54,7 +55,7 @@ pub type Router = rspc::Router<Ctx>;
|
||||||
#[derive(Debug, Clone, Serialize, Type)]
|
#[derive(Debug, Clone, Serialize, Type)]
|
||||||
pub enum CoreEvent {
|
pub enum CoreEvent {
|
||||||
NewThumbnail {
|
NewThumbnail {
|
||||||
thumb_key: Vec<String>,
|
thumb_key: ThumbKey,
|
||||||
},
|
},
|
||||||
NewIdentifiedObjects {
|
NewIdentifiedObjects {
|
||||||
file_path_ids: Vec<file_path::id::Type>,
|
file_path_ids: Vec<file_path::id::Type>,
|
||||||
|
@ -175,7 +176,7 @@ pub(crate) fn mount() -> Arc<Router> {
|
||||||
.await
|
.await
|
||||||
.map(|_| true)
|
.map(|_| true)
|
||||||
}
|
}
|
||||||
.map_err(|err| rspc::Error::new(ErrorCode::InternalServerError, err.to_string()))?;
|
.map_err(|e| rspc::Error::new(ErrorCode::InternalServerError, e.to_string()))?;
|
||||||
|
|
||||||
match feature {
|
match feature {
|
||||||
BackendFeature::CloudSync => {
|
BackendFeature::CloudSync => {
|
||||||
|
|
|
@ -82,8 +82,9 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
||||||
new_model = sd_ai::old_image_labeler::YoloV8::model(Some(&version))
|
new_model = sd_ai::old_image_labeler::YoloV8::model(Some(&version))
|
||||||
.map_err(|e| {
|
.map_err(|e| {
|
||||||
error!(
|
error!(
|
||||||
"Failed to crate image_detection model: '{}'; Error: {e:#?}",
|
%version,
|
||||||
&version,
|
?e,
|
||||||
|
"Failed to crate image_detection model;",
|
||||||
);
|
);
|
||||||
})
|
})
|
||||||
.ok();
|
.ok();
|
||||||
|
@ -94,8 +95,8 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
.await
|
.await
|
||||||
.map_err(|err| {
|
.map_err(|e| {
|
||||||
error!("Failed to write config: {}", err);
|
error!(?e, "Failed to write config;");
|
||||||
rspc::Error::new(
|
rspc::Error::new(
|
||||||
ErrorCode::InternalServerError,
|
ErrorCode::InternalServerError,
|
||||||
"error updating config".into(),
|
"error updating config".into(),
|
||||||
|
@ -186,21 +187,14 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
||||||
pub background_processing_percentage: u8, // 0-100
|
pub background_processing_percentage: u8, // 0-100
|
||||||
}
|
}
|
||||||
R.mutation(
|
R.mutation(
|
||||||
|node,
|
|node, UpdateThumbnailerPreferences { .. }: UpdateThumbnailerPreferences| async move {
|
||||||
UpdateThumbnailerPreferences {
|
|
||||||
background_processing_percentage,
|
|
||||||
}: UpdateThumbnailerPreferences| async move {
|
|
||||||
node.config
|
node.config
|
||||||
.update_preferences(|preferences| {
|
.update_preferences(|_| {
|
||||||
preferences
|
// TODO(fogodev): introduce configurable workers count to task system
|
||||||
.thumbnailer
|
|
||||||
.set_background_processing_percentage(
|
|
||||||
background_processing_percentage,
|
|
||||||
);
|
|
||||||
})
|
})
|
||||||
.await
|
.await
|
||||||
.map_err(|e| {
|
.map_err(|e| {
|
||||||
error!("failed to update thumbnailer preferences: {e:#?}");
|
error!(?e, "Failed to update thumbnailer preferences;");
|
||||||
rspc::Error::with_cause(
|
rspc::Error::with_cause(
|
||||||
ErrorCode::InternalServerError,
|
ErrorCode::InternalServerError,
|
||||||
"Failed to update thumbnailer preferences".to_string(),
|
"Failed to update thumbnailer preferences".to_string(),
|
||||||
|
|
|
@ -56,12 +56,12 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
||||||
.find_many(vec![])
|
.find_many(vec![])
|
||||||
.exec()
|
.exec()
|
||||||
.await
|
.await
|
||||||
.map_err(|err| {
|
.map_err(|e| {
|
||||||
rspc::Error::new(
|
rspc::Error::new(
|
||||||
ErrorCode::InternalServerError,
|
ErrorCode::InternalServerError,
|
||||||
format!(
|
format!(
|
||||||
"Failed to get notifications for library '{}': {}",
|
"Failed to get notifications for library '{}': {}",
|
||||||
library.id, err
|
library.id, e
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
})?
|
})?
|
||||||
|
@ -69,12 +69,12 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
||||||
.map(|n| {
|
.map(|n| {
|
||||||
Ok(Notification {
|
Ok(Notification {
|
||||||
id: NotificationId::Library(library.id, n.id as u32),
|
id: NotificationId::Library(library.id, n.id as u32),
|
||||||
data: rmp_serde::from_slice(&n.data).map_err(|err| {
|
data: rmp_serde::from_slice(&n.data).map_err(|e| {
|
||||||
rspc::Error::new(
|
rspc::Error::new(
|
||||||
ErrorCode::InternalServerError,
|
ErrorCode::InternalServerError,
|
||||||
format!(
|
format!(
|
||||||
"Failed to get notifications for library '{}': {}",
|
"Failed to get notifications for library '{}': {}",
|
||||||
library.id, err
|
library.id, e
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
})?,
|
})?,
|
||||||
|
@ -108,8 +108,8 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
||||||
.delete_many(vec![notification::id::equals(id as i32)])
|
.delete_many(vec![notification::id::equals(id as i32)])
|
||||||
.exec()
|
.exec()
|
||||||
.await
|
.await
|
||||||
.map_err(|err| {
|
.map_err(|e| {
|
||||||
rspc::Error::new(ErrorCode::InternalServerError, err.to_string())
|
rspc::Error::new(ErrorCode::InternalServerError, e.to_string())
|
||||||
})?;
|
})?;
|
||||||
}
|
}
|
||||||
NotificationId::Node(id) => {
|
NotificationId::Node(id) => {
|
||||||
|
@ -119,8 +119,8 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
||||||
.retain(|n| n.id != NotificationId::Node(id));
|
.retain(|n| n.id != NotificationId::Node(id));
|
||||||
})
|
})
|
||||||
.await
|
.await
|
||||||
.map_err(|err| {
|
.map_err(|e| {
|
||||||
rspc::Error::new(ErrorCode::InternalServerError, err.to_string())
|
rspc::Error::new(ErrorCode::InternalServerError, e.to_string())
|
||||||
})?;
|
})?;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -135,9 +135,7 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
||||||
cfg.notifications = vec![];
|
cfg.notifications = vec![];
|
||||||
})
|
})
|
||||||
.await
|
.await
|
||||||
.map_err(|err| {
|
.map_err(|e| rspc::Error::new(ErrorCode::InternalServerError, e.to_string()))?;
|
||||||
rspc::Error::new(ErrorCode::InternalServerError, err.to_string())
|
|
||||||
})?;
|
|
||||||
|
|
||||||
join_all(
|
join_all(
|
||||||
node.libraries
|
node.libraries
|
||||||
|
|
|
@ -89,20 +89,20 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
||||||
))?
|
))?
|
||||||
.new_stream()
|
.new_stream()
|
||||||
.await
|
.await
|
||||||
.map_err(|err| {
|
.map_err(|e| {
|
||||||
rspc::Error::new(
|
rspc::Error::new(
|
||||||
ErrorCode::InternalServerError,
|
ErrorCode::InternalServerError,
|
||||||
format!("error in peer.new_stream: {:?}", err),
|
format!("error in peer.new_stream: {:?}", e),
|
||||||
)
|
)
|
||||||
})?;
|
})?;
|
||||||
|
|
||||||
stream
|
stream
|
||||||
.write_all(&Header::Ping.to_bytes())
|
.write_all(&Header::Ping.to_bytes())
|
||||||
.await
|
.await
|
||||||
.map_err(|err| {
|
.map_err(|e| {
|
||||||
rspc::Error::new(
|
rspc::Error::new(
|
||||||
ErrorCode::InternalServerError,
|
ErrorCode::InternalServerError,
|
||||||
format!("error sending ping header: {:?}", err),
|
format!("error sending ping header: {:?}", e),
|
||||||
)
|
)
|
||||||
})?;
|
})?;
|
||||||
|
|
||||||
|
|
|
@ -2,12 +2,12 @@ use crate::{
|
||||||
api::{locations::ExplorerItem, utils::library},
|
api::{locations::ExplorerItem, utils::library},
|
||||||
library::Library,
|
library::Library,
|
||||||
location::{non_indexed, LocationError},
|
location::{non_indexed, LocationError},
|
||||||
object::media::old_thumbnail::get_indexed_thumb_key,
|
|
||||||
util::{unsafe_streamed_query, BatchedStream},
|
util::{unsafe_streamed_query, BatchedStream},
|
||||||
};
|
};
|
||||||
|
|
||||||
use prisma_client_rust::Operator;
|
use prisma_client_rust::Operator;
|
||||||
use sd_core_prisma_helpers::{file_path_for_frontend, object_with_file_paths};
|
use sd_core_heavy_lifting::media_processor::ThumbKey;
|
||||||
|
use sd_core_prisma_helpers::{file_path_for_frontend, object_with_file_paths, CasId};
|
||||||
use sd_prisma::prisma::{self, PrismaClient};
|
use sd_prisma::prisma::{self, PrismaClient};
|
||||||
|
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
|
@ -217,21 +217,23 @@ pub fn mount() -> AlphaRouter<Ctx> {
|
||||||
let mut items = Vec::with_capacity(file_paths.len());
|
let mut items = Vec::with_capacity(file_paths.len());
|
||||||
|
|
||||||
for file_path in file_paths {
|
for file_path in file_paths {
|
||||||
let has_created_thumbnail = if let Some(cas_id) = &file_path.cas_id {
|
let has_created_thumbnail =
|
||||||
library
|
if let Some(cas_id) = file_path.cas_id.as_ref().map(CasId::from) {
|
||||||
.thumbnail_exists(&node, cas_id)
|
library
|
||||||
.await
|
.thumbnail_exists(&node, &cas_id)
|
||||||
.map_err(LocationError::from)?
|
.await
|
||||||
} else {
|
.map_err(LocationError::from)?
|
||||||
false
|
} else {
|
||||||
};
|
false
|
||||||
|
};
|
||||||
|
|
||||||
items.push(ExplorerItem::Path {
|
items.push(ExplorerItem::Path {
|
||||||
thumbnail: file_path
|
thumbnail: file_path
|
||||||
.cas_id
|
.cas_id
|
||||||
.as_ref()
|
.as_ref()
|
||||||
// .filter(|_| thumbnail_exists_locally)
|
.map(CasId::from)
|
||||||
.map(|i| get_indexed_thumb_key(i, library.id)),
|
.map(CasId::into_owned)
|
||||||
|
.map(|cas_id| ThumbKey::new_indexed(cas_id, library.id)),
|
||||||
has_created_thumbnail,
|
has_created_thumbnail,
|
||||||
item: Box::new(file_path),
|
item: Box::new(file_path),
|
||||||
})
|
})
|
||||||
|
@ -332,9 +334,11 @@ pub fn mount() -> AlphaRouter<Ctx> {
|
||||||
.file_paths
|
.file_paths
|
||||||
.iter()
|
.iter()
|
||||||
.map(|fp| fp.cas_id.as_ref())
|
.map(|fp| fp.cas_id.as_ref())
|
||||||
.find_map(|c| c);
|
.find_map(|c| c)
|
||||||
|
.map(CasId::from)
|
||||||
|
.map(|cas_id| cas_id.to_owned());
|
||||||
|
|
||||||
let has_created_thumbnail = if let Some(cas_id) = cas_id {
|
let has_created_thumbnail = if let Some(cas_id) = &cas_id {
|
||||||
library.thumbnail_exists(&node, cas_id).await.map_err(|e| {
|
library.thumbnail_exists(&node, cas_id).await.map_err(|e| {
|
||||||
rspc::Error::with_cause(
|
rspc::Error::with_cause(
|
||||||
ErrorCode::InternalServerError,
|
ErrorCode::InternalServerError,
|
||||||
|
@ -348,8 +352,7 @@ pub fn mount() -> AlphaRouter<Ctx> {
|
||||||
|
|
||||||
items.push(ExplorerItem::Object {
|
items.push(ExplorerItem::Object {
|
||||||
thumbnail: cas_id
|
thumbnail: cas_id
|
||||||
// .filter(|_| thumbnail_exists_locally)
|
.map(|cas_id| ThumbKey::new_indexed(cas_id, library.id)),
|
||||||
.map(|cas_id| get_indexed_thumb_key(cas_id, library.id)),
|
|
||||||
item: object,
|
item: object,
|
||||||
has_created_thumbnail,
|
has_created_thumbnail,
|
||||||
});
|
});
|
||||||
|
|
|
@ -82,7 +82,7 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
||||||
// https://docs.rs/serde/latest/serde/de/struct.IgnoredAny.html
|
// https://docs.rs/serde/latest/serde/de/struct.IgnoredAny.html
|
||||||
|
|
||||||
if let Err(e) = serde_json::from_str::<IgnoredAny>(&s) {
|
if let Err(e) = serde_json::from_str::<IgnoredAny>(&s) {
|
||||||
error!("failed to parse filters: {e:#?}");
|
error!(?e, "Failed to parse filters;");
|
||||||
None
|
None
|
||||||
} else {
|
} else {
|
||||||
Some(s)
|
Some(s)
|
||||||
|
|
|
@ -221,7 +221,7 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
||||||
.iter()
|
.iter()
|
||||||
.filter(|fp| fp.is_dir.unwrap_or_default() && fp.object.is_none())
|
.filter(|fp| fp.is_dir.unwrap_or_default() && fp.object.is_none())
|
||||||
.map(|fp| {
|
.map(|fp| {
|
||||||
let id = uuid_to_bytes(Uuid::new_v4());
|
let id = uuid_to_bytes(&Uuid::new_v4());
|
||||||
|
|
||||||
sync_params.extend(sync.shared_create(
|
sync_params.extend(sync.shared_create(
|
||||||
prisma_sync::object::SyncId { pub_id: id.clone() },
|
prisma_sync::object::SyncId { pub_id: id.clone() },
|
||||||
|
|
|
@ -132,6 +132,19 @@ impl InvalidRequests {
|
||||||
#[macro_export]
|
#[macro_export]
|
||||||
// #[allow(clippy::crate_in_macro_def)]
|
// #[allow(clippy::crate_in_macro_def)]
|
||||||
macro_rules! invalidate_query {
|
macro_rules! invalidate_query {
|
||||||
|
|
||||||
|
($ctx:expr, $query:ident) => {{
|
||||||
|
let ctx: &$crate::library::Library = &$ctx; // Assert the context is the correct type
|
||||||
|
let query: &'static str = $query;
|
||||||
|
|
||||||
|
::tracing::trace!(target: "sd_core::invalidate-query", "invalidate_query!(\"{}\") at {}", query, concat!(file!(), ":", line!()));
|
||||||
|
|
||||||
|
// The error are ignored here because they aren't mission critical. If they fail the UI might be outdated for a bit.
|
||||||
|
ctx.emit($crate::api::CoreEvent::InvalidateOperation(
|
||||||
|
$crate::api::utils::InvalidateOperationEvent::dangerously_create(query, serde_json::Value::Null, None)
|
||||||
|
))
|
||||||
|
}};
|
||||||
|
|
||||||
($ctx:expr, $key:literal) => {{
|
($ctx:expr, $key:literal) => {{
|
||||||
let ctx: &$crate::library::Library = &$ctx; // Assert the context is the correct type
|
let ctx: &$crate::library::Library = &$ctx; // Assert the context is the correct type
|
||||||
|
|
||||||
|
@ -324,8 +337,12 @@ pub(crate) fn mount_invalidate() -> AlphaRouter<Ctx> {
|
||||||
) => {
|
) => {
|
||||||
let key = match to_key(&(key, arg)) {
|
let key = match to_key(&(key, arg)) {
|
||||||
Ok(key) => key,
|
Ok(key) => key,
|
||||||
Err(err) => {
|
Err(e) => {
|
||||||
warn!("Error deriving key for invalidate operation '{:?}': {:?}", first_event, err);
|
warn!(
|
||||||
|
?first_event,
|
||||||
|
?e,
|
||||||
|
"Error deriving key for invalidate operation;"
|
||||||
|
);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
@ -345,7 +362,10 @@ pub(crate) fn mount_invalidate() -> AlphaRouter<Ctx> {
|
||||||
}
|
}
|
||||||
event = event_bus_rx.recv() => {
|
event = event_bus_rx.recv() => {
|
||||||
let Ok(event) = event else {
|
let Ok(event) = event else {
|
||||||
warn!("Shutting down invalidation manager thread due to the core event bus being dropped!");
|
warn!(
|
||||||
|
"Shutting down invalidation manager thread \
|
||||||
|
due to the core event bus being dropped!"
|
||||||
|
);
|
||||||
break;
|
break;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -359,8 +379,12 @@ pub(crate) fn mount_invalidate() -> AlphaRouter<Ctx> {
|
||||||
Ok(key) => {
|
Ok(key) => {
|
||||||
buf.insert(key, op);
|
buf.insert(key, op);
|
||||||
},
|
},
|
||||||
Err(err) => {
|
Err(e) => {
|
||||||
warn!("Error deriving key for invalidate operation '{:?}': {:?}", op, err);
|
warn!(
|
||||||
|
?op,
|
||||||
|
?e,
|
||||||
|
"Error deriving key for invalidate operation;",
|
||||||
|
);
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
@ -383,7 +407,10 @@ pub(crate) fn mount_invalidate() -> AlphaRouter<Ctx> {
|
||||||
Ok(_) => {}
|
Ok(_) => {}
|
||||||
// All receivers are shutdown means that all clients are disconnected.
|
// All receivers are shutdown means that all clients are disconnected.
|
||||||
Err(_) => {
|
Err(_) => {
|
||||||
debug!("Shutting down invalidation manager! This is normal if all clients disconnects.");
|
debug!(
|
||||||
|
"Shutting down invalidation manager! \
|
||||||
|
This is normal if all clients disconnects."
|
||||||
|
);
|
||||||
manager_thread_active.swap(false, Ordering::Relaxed);
|
manager_thread_active.swap(false, Ordering::Relaxed);
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
|
@ -42,7 +42,6 @@ pub async fn run_actor(
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
Request::Messages { timestamps, .. } => timestamps,
|
Request::Messages { timestamps, .. } => timestamps,
|
||||||
_ => continue,
|
|
||||||
};
|
};
|
||||||
|
|
||||||
let (ops_ids, ops): (Vec<_>, Vec<_>) = err_break!(
|
let (ops_ids, ops): (Vec<_>, Vec<_>) = err_break!(
|
||||||
|
@ -60,10 +59,10 @@ pub async fn run_actor(
|
||||||
}
|
}
|
||||||
|
|
||||||
debug!(
|
debug!(
|
||||||
"Sending {} messages ({:?} to {:?}) to ingester",
|
messages_count = ops.len(),
|
||||||
ops.len(),
|
first_message = ?ops.first().map(|operation| operation.timestamp.as_u64()),
|
||||||
ops.first().map(|operation| operation.timestamp.as_u64()),
|
last_message = ?ops.last().map(|operation| operation.timestamp.as_u64()),
|
||||||
ops.last().map(|operation| operation.timestamp.as_u64()),
|
"Sending messages to ingester",
|
||||||
);
|
);
|
||||||
|
|
||||||
let (wait_tx, wait_rx) = tokio::sync::oneshot::channel::<()>();
|
let (wait_tx, wait_rx) = tokio::sync::oneshot::channel::<()>();
|
||||||
|
|
|
@ -97,7 +97,7 @@ macro_rules! err_break {
|
||||||
match $e {
|
match $e {
|
||||||
Ok(d) => d,
|
Ok(d) => d,
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
tracing::error!("{e}");
|
tracing::error!(?e);
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -56,7 +56,7 @@ pub async fn run_actor(
|
||||||
.map(|id| {
|
.map(|id| {
|
||||||
db.cloud_crdt_operation()
|
db.cloud_crdt_operation()
|
||||||
.find_first(vec![cloud_crdt_operation::instance::is(vec![
|
.find_first(vec![cloud_crdt_operation::instance::is(vec![
|
||||||
instance::pub_id::equals(uuid_to_bytes(*id)),
|
instance::pub_id::equals(uuid_to_bytes(id)),
|
||||||
])])
|
])])
|
||||||
.order_by(cloud_crdt_operation::timestamp::order(
|
.order_by(cloud_crdt_operation::timestamp::order(
|
||||||
SortOrder::Desc,
|
SortOrder::Desc,
|
||||||
|
@ -76,8 +76,10 @@ pub async fn run_actor(
|
||||||
let cloud_timestamp = d.map(|d| d.timestamp).unwrap_or_default() as u64;
|
let cloud_timestamp = d.map(|d| d.timestamp).unwrap_or_default() as u64;
|
||||||
|
|
||||||
debug!(
|
debug!(
|
||||||
"Instance {id}, Sync Timestamp {}, Cloud Timestamp {cloud_timestamp}",
|
instance_id = %id,
|
||||||
sync_timestamp.as_u64()
|
sync_timestamp = sync_timestamp.as_u64(),
|
||||||
|
%cloud_timestamp,
|
||||||
|
"Comparing sync timestamps",
|
||||||
);
|
);
|
||||||
|
|
||||||
let max_timestamp = Ord::max(cloud_timestamp, sync_timestamp.as_u64());
|
let max_timestamp = Ord::max(cloud_timestamp, sync_timestamp.as_u64());
|
||||||
|
@ -118,7 +120,10 @@ pub async fn run_actor(
|
||||||
.await
|
.await
|
||||||
);
|
);
|
||||||
|
|
||||||
info!("Received {} collections", collections.len());
|
info!(
|
||||||
|
collections_count = collections.len(),
|
||||||
|
"Received collections;",
|
||||||
|
);
|
||||||
|
|
||||||
if collections.is_empty() {
|
if collections.is_empty() {
|
||||||
break;
|
break;
|
||||||
|
@ -165,9 +170,9 @@ pub async fn run_actor(
|
||||||
&db,
|
&db,
|
||||||
&sync,
|
&sync,
|
||||||
&libraries,
|
&libraries,
|
||||||
collection.instance_uuid,
|
&collection.instance_uuid,
|
||||||
instance.identity,
|
instance.identity,
|
||||||
instance.node_id,
|
&instance.node_id,
|
||||||
RemoteIdentity::from_str(&instance.node_remote_identity)
|
RemoteIdentity::from_str(&instance.node_remote_identity)
|
||||||
.expect("malformed remote identity in the DB"),
|
.expect("malformed remote identity in the DB"),
|
||||||
node.p2p.peer_metadata(),
|
node.p2p.peer_metadata(),
|
||||||
|
@ -185,14 +190,10 @@ pub async fn run_actor(
|
||||||
let operations = compressed_operations.into_ops();
|
let operations = compressed_operations.into_ops();
|
||||||
|
|
||||||
debug!(
|
debug!(
|
||||||
"Processing collection. Instance {}, Start {:?}, End {:?}",
|
instance_id = %collection.instance_uuid,
|
||||||
&collection.instance_uuid,
|
start = ?operations.first().map(|operation| operation.timestamp.as_u64()),
|
||||||
operations
|
end = ?operations.last().map(|operation| operation.timestamp.as_u64()),
|
||||||
.first()
|
"Processing collection",
|
||||||
.map(|operation| operation.timestamp.as_u64()),
|
|
||||||
operations
|
|
||||||
.last()
|
|
||||||
.map(|operation| operation.timestamp.as_u64()),
|
|
||||||
);
|
);
|
||||||
|
|
||||||
err_break!(write_cloud_ops_to_db(operations, &db).await);
|
err_break!(write_cloud_ops_to_db(operations, &db).await);
|
||||||
|
@ -247,9 +248,9 @@ pub async fn upsert_instance(
|
||||||
db: &PrismaClient,
|
db: &PrismaClient,
|
||||||
sync: &sd_core_sync::Manager,
|
sync: &sd_core_sync::Manager,
|
||||||
libraries: &Libraries,
|
libraries: &Libraries,
|
||||||
uuid: Uuid,
|
uuid: &Uuid,
|
||||||
identity: RemoteIdentity,
|
identity: RemoteIdentity,
|
||||||
node_id: Uuid,
|
node_id: &Uuid,
|
||||||
node_remote_identity: RemoteIdentity,
|
node_remote_identity: RemoteIdentity,
|
||||||
metadata: HashMap<String, String>,
|
metadata: HashMap<String, String>,
|
||||||
) -> prisma_client_rust::Result<()> {
|
) -> prisma_client_rust::Result<()> {
|
||||||
|
@ -276,7 +277,7 @@ pub async fn upsert_instance(
|
||||||
.exec()
|
.exec()
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
sync.timestamps.write().await.entry(uuid).or_default();
|
sync.timestamps.write().await.entry(*uuid).or_default();
|
||||||
|
|
||||||
// Called again so the new instances are picked up
|
// Called again so the new instances are picked up
|
||||||
libraries.update_instances_by_id(library_id).await;
|
libraries.update_instances_by_id(library_id).await;
|
||||||
|
|
|
@ -52,8 +52,8 @@ pub async fn run_actor(
|
||||||
use sd_cloud_api::library::message_collections::do_add;
|
use sd_cloud_api::library::message_collections::do_add;
|
||||||
|
|
||||||
debug!(
|
debug!(
|
||||||
"Preparing to send {} instances' operations to cloud",
|
total_operations = req_adds.len(),
|
||||||
req_adds.len()
|
"Preparing to send instance's operations to cloud;"
|
||||||
);
|
);
|
||||||
|
|
||||||
// gets new operations for each instance to send to cloud
|
// gets new operations for each instance to send to cloud
|
||||||
|
@ -84,10 +84,7 @@ pub async fn run_actor(
|
||||||
|
|
||||||
use base64::prelude::*;
|
use base64::prelude::*;
|
||||||
|
|
||||||
debug!(
|
debug!(instance_id = %req_add.instance_uuid, %start_time, %end_time);
|
||||||
"Instance {}: {} to {}",
|
|
||||||
req_add.instance_uuid, start_time, end_time
|
|
||||||
);
|
|
||||||
|
|
||||||
instances.push(do_add::Input {
|
instances.push(do_add::Input {
|
||||||
uuid: req_add.instance_uuid,
|
uuid: req_add.instance_uuid,
|
||||||
|
|
229
core/src/context.rs
Normal file
229
core/src/context.rs
Normal file
|
@ -0,0 +1,229 @@
|
||||||
|
use crate::{api::CoreEvent, invalidate_query, library::Library, old_job::JobProgressEvent, Node};
|
||||||
|
|
||||||
|
use sd_core_heavy_lifting::{
|
||||||
|
job_system::report::{Report, Status},
|
||||||
|
OuterContext, ProgressUpdate, UpdateEvent,
|
||||||
|
};
|
||||||
|
|
||||||
|
use std::{
|
||||||
|
ops::{Deref, DerefMut},
|
||||||
|
sync::{
|
||||||
|
atomic::{AtomicU8, Ordering},
|
||||||
|
Arc,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
use chrono::{DateTime, Utc};
|
||||||
|
use tokio::{spawn, sync::RwLock};
|
||||||
|
use tracing::{error, trace};
|
||||||
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
#[derive(Clone)]
|
||||||
|
pub struct NodeContext {
|
||||||
|
pub node: Arc<Node>,
|
||||||
|
pub library: Arc<Library>,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub trait NodeContextExt: sealed::Sealed {
|
||||||
|
fn library(&self) -> &Arc<Library>;
|
||||||
|
}
|
||||||
|
|
||||||
|
mod sealed {
|
||||||
|
pub trait Sealed {}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl sealed::Sealed for NodeContext {}
|
||||||
|
|
||||||
|
impl NodeContextExt for NodeContext {
|
||||||
|
fn library(&self) -> &Arc<Library> {
|
||||||
|
&self.library
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl OuterContext for NodeContext {
|
||||||
|
fn id(&self) -> Uuid {
|
||||||
|
self.library.id
|
||||||
|
}
|
||||||
|
|
||||||
|
fn db(&self) -> &Arc<sd_prisma::prisma::PrismaClient> {
|
||||||
|
&self.library.db
|
||||||
|
}
|
||||||
|
|
||||||
|
fn sync(&self) -> &Arc<sd_core_sync::Manager> {
|
||||||
|
&self.library.sync
|
||||||
|
}
|
||||||
|
|
||||||
|
fn invalidate_query(&self, query: &'static str) {
|
||||||
|
invalidate_query!(self.library, query)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn query_invalidator(&self) -> impl Fn(&'static str) + Send + Sync {
|
||||||
|
|query| {
|
||||||
|
invalidate_query!(self.library, query);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn report_update(&self, update: UpdateEvent) {
|
||||||
|
// FIX-ME: Remove this conversion once we have a proper atomic updates system
|
||||||
|
let event = match update {
|
||||||
|
UpdateEvent::NewThumbnail { thumb_key } => CoreEvent::NewThumbnail { thumb_key },
|
||||||
|
UpdateEvent::NewIdentifiedObjects { file_path_ids } => {
|
||||||
|
CoreEvent::NewIdentifiedObjects { file_path_ids }
|
||||||
|
}
|
||||||
|
};
|
||||||
|
self.node.emit(event);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn get_data_directory(&self) -> &std::path::Path {
|
||||||
|
&self.node.data_dir
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone)]
|
||||||
|
pub struct JobContext<OuterCtx: OuterContext + NodeContextExt> {
|
||||||
|
outer_ctx: OuterCtx,
|
||||||
|
report: Arc<RwLock<Report>>,
|
||||||
|
start_time: DateTime<Utc>,
|
||||||
|
report_update_counter: Arc<AtomicU8>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<OuterCtx: OuterContext + NodeContextExt> OuterContext for JobContext<OuterCtx> {
|
||||||
|
fn id(&self) -> Uuid {
|
||||||
|
self.outer_ctx.id()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn db(&self) -> &Arc<sd_prisma::prisma::PrismaClient> {
|
||||||
|
self.outer_ctx.db()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn sync(&self) -> &Arc<sd_core_sync::Manager> {
|
||||||
|
self.outer_ctx.sync()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn invalidate_query(&self, query: &'static str) {
|
||||||
|
self.outer_ctx.invalidate_query(query);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn query_invalidator(&self) -> impl Fn(&'static str) + Send + Sync {
|
||||||
|
self.outer_ctx.query_invalidator()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn report_update(&self, update: UpdateEvent) {
|
||||||
|
self.outer_ctx.report_update(update);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn get_data_directory(&self) -> &std::path::Path {
|
||||||
|
self.outer_ctx.get_data_directory()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<OuterCtx: OuterContext + NodeContextExt> sd_core_heavy_lifting::JobContext<OuterCtx>
|
||||||
|
for JobContext<OuterCtx>
|
||||||
|
{
|
||||||
|
fn new(report: Report, outer_ctx: OuterCtx) -> Self {
|
||||||
|
Self {
|
||||||
|
report: Arc::new(RwLock::new(report)),
|
||||||
|
outer_ctx,
|
||||||
|
start_time: Utc::now(),
|
||||||
|
report_update_counter: Arc::new(AtomicU8::new(0)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn progress(&self, updates: impl IntoIterator<Item = ProgressUpdate> + Send) {
|
||||||
|
let mut report = self.report.write().await;
|
||||||
|
|
||||||
|
// protect against updates if job is not running
|
||||||
|
if report.status != Status::Running {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut changed_phase = false;
|
||||||
|
|
||||||
|
for update in updates {
|
||||||
|
match update {
|
||||||
|
ProgressUpdate::TaskCount(task_count) => {
|
||||||
|
report.task_count = task_count as i32;
|
||||||
|
}
|
||||||
|
ProgressUpdate::CompletedTaskCount(completed_task_count) => {
|
||||||
|
report.completed_task_count = completed_task_count as i32;
|
||||||
|
}
|
||||||
|
|
||||||
|
ProgressUpdate::Message(message) => {
|
||||||
|
trace!(job_id = %report.id, %message, "job message;");
|
||||||
|
report.message = message;
|
||||||
|
}
|
||||||
|
ProgressUpdate::Phase(phase) => {
|
||||||
|
trace!(
|
||||||
|
job_id = %report.id,
|
||||||
|
"changing phase: {} -> {phase};",
|
||||||
|
report.phase
|
||||||
|
);
|
||||||
|
report.phase = phase;
|
||||||
|
changed_phase = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Calculate elapsed time
|
||||||
|
let elapsed = Utc::now() - self.start_time;
|
||||||
|
|
||||||
|
// Calculate remaining time
|
||||||
|
let task_count = report.task_count as usize;
|
||||||
|
let completed_task_count = report.completed_task_count as usize;
|
||||||
|
let remaining_task_count = task_count.saturating_sub(completed_task_count);
|
||||||
|
|
||||||
|
// Adding 1 to avoid division by zero
|
||||||
|
let remaining_time_per_task = elapsed / (completed_task_count + 1) as i32;
|
||||||
|
|
||||||
|
let remaining_time = remaining_time_per_task * remaining_task_count as i32;
|
||||||
|
|
||||||
|
// Update the report with estimated remaining time
|
||||||
|
report.estimated_completion = Utc::now()
|
||||||
|
.checked_add_signed(remaining_time)
|
||||||
|
.unwrap_or(Utc::now());
|
||||||
|
|
||||||
|
let library = self.outer_ctx.library();
|
||||||
|
|
||||||
|
let counter = self.report_update_counter.fetch_add(1, Ordering::AcqRel);
|
||||||
|
|
||||||
|
if counter == 50 || counter == 0 || changed_phase {
|
||||||
|
self.report_update_counter.store(1, Ordering::Release);
|
||||||
|
|
||||||
|
spawn({
|
||||||
|
let db = Arc::clone(&library.db);
|
||||||
|
let mut report = report.clone();
|
||||||
|
async move {
|
||||||
|
if let Err(e) = report.update(&db).await {
|
||||||
|
error!(
|
||||||
|
?e,
|
||||||
|
"Failed to update job report on debounced job progress event;"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// emit a CoreEvent
|
||||||
|
library.emit(CoreEvent::JobProgress(JobProgressEvent {
|
||||||
|
id: report.id,
|
||||||
|
library_id: library.id,
|
||||||
|
task_count: report.task_count,
|
||||||
|
completed_task_count: report.completed_task_count,
|
||||||
|
estimated_completion: report.estimated_completion,
|
||||||
|
phase: report.phase.clone(),
|
||||||
|
message: report.message.clone(),
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn report(&self) -> impl Deref<Target = Report> {
|
||||||
|
Arc::clone(&self.report).read_owned().await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn report_mut(&self) -> impl DerefMut<Target = Report> {
|
||||||
|
Arc::clone(&self.report).write_owned().await
|
||||||
|
}
|
||||||
|
|
||||||
|
fn get_outer_ctx(&self) -> OuterCtx {
|
||||||
|
self.outer_ctx.clone()
|
||||||
|
}
|
||||||
|
}
|
|
@ -1,16 +1,13 @@
|
||||||
use crate::{
|
use crate::{
|
||||||
api::{utils::InvalidateOperationEvent, CoreEvent},
|
api::{utils::InvalidateOperationEvent, CoreEvent},
|
||||||
library::Library,
|
library::Library,
|
||||||
object::media::old_thumbnail::WEBP_EXTENSION,
|
|
||||||
p2p::operations::{self, request_file},
|
p2p::operations::{self, request_file},
|
||||||
util::InfallibleResponse,
|
util::InfallibleResponse,
|
||||||
Node,
|
Node,
|
||||||
};
|
};
|
||||||
|
|
||||||
use async_stream::stream;
|
|
||||||
use bytes::Bytes;
|
|
||||||
use mpsc_to_async_write::MpscToAsyncWrite;
|
|
||||||
use sd_core_file_path_helper::IsolatedFilePathData;
|
use sd_core_file_path_helper::IsolatedFilePathData;
|
||||||
|
use sd_core_heavy_lifting::media_processor::WEBP_EXTENSION;
|
||||||
use sd_core_prisma_helpers::file_path_to_handle_custom_uri;
|
use sd_core_prisma_helpers::file_path_to_handle_custom_uri;
|
||||||
|
|
||||||
use sd_file_ext::text::is_text;
|
use sd_file_ext::text::is_text;
|
||||||
|
@ -30,6 +27,7 @@ use std::{
|
||||||
sync::Arc,
|
sync::Arc,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
use async_stream::stream;
|
||||||
use axum::{
|
use axum::{
|
||||||
body::{self, Body, BoxBody, Full, StreamBody},
|
body::{self, Body, BoxBody, Full, StreamBody},
|
||||||
extract::{self, State},
|
extract::{self, State},
|
||||||
|
@ -39,6 +37,7 @@ use axum::{
|
||||||
routing::get,
|
routing::get,
|
||||||
Router,
|
Router,
|
||||||
};
|
};
|
||||||
|
use bytes::Bytes;
|
||||||
use http_body::combinators::UnsyncBoxBody;
|
use http_body::combinators::UnsyncBoxBody;
|
||||||
use hyper::{header, upgrade::OnUpgrade};
|
use hyper::{header, upgrade::OnUpgrade};
|
||||||
use mini_moka::sync::Cache;
|
use mini_moka::sync::Cache;
|
||||||
|
@ -56,6 +55,8 @@ mod mpsc_to_async_write;
|
||||||
mod serve_file;
|
mod serve_file;
|
||||||
mod utils;
|
mod utils;
|
||||||
|
|
||||||
|
use mpsc_to_async_write::MpscToAsyncWrite;
|
||||||
|
|
||||||
type CacheKey = (Uuid, file_path::id::Type);
|
type CacheKey = (Uuid, file_path::id::Type);
|
||||||
|
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
|
@ -74,8 +75,8 @@ pub enum ServeFrom {
|
||||||
Local,
|
Local,
|
||||||
/// Serve from a specific instance
|
/// Serve from a specific instance
|
||||||
Remote {
|
Remote {
|
||||||
library_identity: RemoteIdentity,
|
library_identity: Box<RemoteIdentity>,
|
||||||
node_identity: RemoteIdentity,
|
node_identity: Box<RemoteIdentity>,
|
||||||
library: Arc<Library>,
|
library: Arc<Library>,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
@ -102,8 +103,8 @@ async fn request_to_remote_node(
|
||||||
|
|
||||||
let mut response = match operations::remote_rspc(p2p.clone(), identity, request).await {
|
let mut response = match operations::remote_rspc(p2p.clone(), identity, request).await {
|
||||||
Ok(v) => v,
|
Ok(v) => v,
|
||||||
Err(err) => {
|
Err(e) => {
|
||||||
warn!("Error doing remote rspc query with '{identity}': {err:?}");
|
warn!(%identity, ?e, "Error doing remote rspc query with;");
|
||||||
return StatusCode::BAD_GATEWAY.into_response();
|
return StatusCode::BAD_GATEWAY.into_response();
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
@ -120,21 +121,21 @@ async fn request_to_remote_node(
|
||||||
};
|
};
|
||||||
|
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
let Ok(mut request_upgraded) = request_upgraded.await.map_err(|err| {
|
let Ok(mut request_upgraded) = request_upgraded.await.map_err(|e| {
|
||||||
warn!("Error upgrading websocket request: {err}");
|
warn!(?e, "Error upgrading websocket request;");
|
||||||
}) else {
|
}) else {
|
||||||
return;
|
return;
|
||||||
};
|
};
|
||||||
let Ok(mut response_upgraded) = response_upgraded.await.map_err(|err| {
|
let Ok(mut response_upgraded) = response_upgraded.await.map_err(|e| {
|
||||||
warn!("Error upgrading websocket response: {err}");
|
warn!(?e, "Error upgrading websocket response;");
|
||||||
}) else {
|
}) else {
|
||||||
return;
|
return;
|
||||||
};
|
};
|
||||||
|
|
||||||
copy_bidirectional(&mut request_upgraded, &mut response_upgraded)
|
copy_bidirectional(&mut request_upgraded, &mut response_upgraded)
|
||||||
.await
|
.await
|
||||||
.map_err(|err| {
|
.map_err(|e| {
|
||||||
warn!("Error upgrading websocket response: {err}");
|
warn!(?e, "Error upgrading websocket response;");
|
||||||
})
|
})
|
||||||
.ok();
|
.ok();
|
||||||
});
|
});
|
||||||
|
@ -204,8 +205,8 @@ async fn get_or_init_lru_entry(
|
||||||
ServeFrom::Local
|
ServeFrom::Local
|
||||||
} else {
|
} else {
|
||||||
ServeFrom::Remote {
|
ServeFrom::Remote {
|
||||||
library_identity,
|
library_identity: Box::new(library_identity),
|
||||||
node_identity,
|
node_identity: Box::new(node_identity),
|
||||||
library: library.clone(),
|
library: library.clone(),
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
@ -237,9 +238,9 @@ pub fn base_router() -> Router<LocalState> {
|
||||||
.then_some(())
|
.then_some(())
|
||||||
.ok_or_else(|| not_found(()))?;
|
.ok_or_else(|| not_found(()))?;
|
||||||
|
|
||||||
let file = File::open(&path).await.map_err(|err| {
|
let file = File::open(&path).await.map_err(|e| {
|
||||||
InfallibleResponse::builder()
|
InfallibleResponse::builder()
|
||||||
.status(if err.kind() == io::ErrorKind::NotFound {
|
.status(if e.kind() == io::ErrorKind::NotFound {
|
||||||
StatusCode::NOT_FOUND
|
StatusCode::NOT_FOUND
|
||||||
} else {
|
} else {
|
||||||
StatusCode::INTERNAL_SERVER_ERROR
|
StatusCode::INTERNAL_SERVER_ERROR
|
||||||
|
@ -270,7 +271,7 @@ pub fn base_router() -> Router<LocalState> {
|
||||||
serve_from,
|
serve_from,
|
||||||
..
|
..
|
||||||
},
|
},
|
||||||
..
|
_library,
|
||||||
) = get_or_init_lru_entry(&state, path).await?;
|
) = get_or_init_lru_entry(&state, path).await?;
|
||||||
|
|
||||||
match serve_from {
|
match serve_from {
|
||||||
|
@ -282,24 +283,23 @@ pub fn base_router() -> Router<LocalState> {
|
||||||
.then_some(())
|
.then_some(())
|
||||||
.ok_or_else(|| not_found(()))?;
|
.ok_or_else(|| not_found(()))?;
|
||||||
|
|
||||||
let mut file =
|
let mut file = File::open(&file_path_full_path).await.map_err(|e| {
|
||||||
File::open(&file_path_full_path).await.map_err(|err| {
|
InfallibleResponse::builder()
|
||||||
InfallibleResponse::builder()
|
.status(if e.kind() == io::ErrorKind::NotFound {
|
||||||
.status(if err.kind() == io::ErrorKind::NotFound {
|
StatusCode::NOT_FOUND
|
||||||
StatusCode::NOT_FOUND
|
} else {
|
||||||
} else {
|
StatusCode::INTERNAL_SERVER_ERROR
|
||||||
StatusCode::INTERNAL_SERVER_ERROR
|
})
|
||||||
})
|
.body(body::boxed(Full::from("")))
|
||||||
.body(body::boxed(Full::from("")))
|
})?;
|
||||||
})?;
|
|
||||||
|
|
||||||
let resp = InfallibleResponse::builder().header(
|
let resp = InfallibleResponse::builder().header(
|
||||||
"Content-Type",
|
"Content-Type",
|
||||||
HeaderValue::from_str(
|
HeaderValue::from_str(
|
||||||
&infer_the_mime_type(&extension, &mut file, &metadata).await?,
|
&infer_the_mime_type(&extension, &mut file, &metadata).await?,
|
||||||
)
|
)
|
||||||
.map_err(|err| {
|
.map_err(|e| {
|
||||||
error!("Error converting mime-type into header value: {}", err);
|
error!(?e, "Error converting mime-type into header value;");
|
||||||
internal_server_error(())
|
internal_server_error(())
|
||||||
})?,
|
})?,
|
||||||
);
|
);
|
||||||
|
@ -316,15 +316,20 @@ pub fn base_router() -> Router<LocalState> {
|
||||||
let (tx, mut rx) = tokio::sync::mpsc::channel::<io::Result<Bytes>>(150);
|
let (tx, mut rx) = tokio::sync::mpsc::channel::<io::Result<Bytes>>(150);
|
||||||
request_file(
|
request_file(
|
||||||
state.node.p2p.p2p.clone(),
|
state.node.p2p.p2p.clone(),
|
||||||
node_identity,
|
*node_identity,
|
||||||
&library.identity,
|
&library.identity,
|
||||||
file_path_pub_id,
|
file_path_pub_id,
|
||||||
Range::Full,
|
Range::Full,
|
||||||
MpscToAsyncWrite::new(PollSender::new(tx)),
|
MpscToAsyncWrite::new(PollSender::new(tx)),
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
.map_err(|err| {
|
.map_err(|e| {
|
||||||
error!("Error requesting file {file_path_pub_id:?} from node {:?}: {err:?}", library.identity.to_remote_identity());
|
error!(
|
||||||
|
%file_path_pub_id,
|
||||||
|
node_identity = ?library.identity.to_remote_identity(),
|
||||||
|
?e,
|
||||||
|
"Error requesting file from other node;",
|
||||||
|
);
|
||||||
internal_server_error(())
|
internal_server_error(())
|
||||||
})?;
|
})?;
|
||||||
|
|
||||||
|
@ -352,9 +357,9 @@ pub fn base_router() -> Router<LocalState> {
|
||||||
.then_some(())
|
.then_some(())
|
||||||
.ok_or_else(|| not_found(()))?;
|
.ok_or_else(|| not_found(()))?;
|
||||||
|
|
||||||
let mut file = File::open(&path).await.map_err(|err| {
|
let mut file = File::open(&path).await.map_err(|e| {
|
||||||
InfallibleResponse::builder()
|
InfallibleResponse::builder()
|
||||||
.status(if err.kind() == io::ErrorKind::NotFound {
|
.status(if e.kind() == io::ErrorKind::NotFound {
|
||||||
StatusCode::NOT_FOUND
|
StatusCode::NOT_FOUND
|
||||||
} else {
|
} else {
|
||||||
StatusCode::INTERNAL_SERVER_ERROR
|
StatusCode::INTERNAL_SERVER_ERROR
|
||||||
|
@ -368,8 +373,8 @@ pub fn base_router() -> Router<LocalState> {
|
||||||
None => "text/plain".to_string(),
|
None => "text/plain".to_string(),
|
||||||
Some(ext) => infer_the_mime_type(ext, &mut file, &metadata).await?,
|
Some(ext) => infer_the_mime_type(ext, &mut file, &metadata).await?,
|
||||||
})
|
})
|
||||||
.map_err(|err| {
|
.map_err(|e| {
|
||||||
error!("Error converting mime-type into header value: {}", err);
|
error!(?e, "Error converting mime-type into header value;");
|
||||||
internal_server_error(())
|
internal_server_error(())
|
||||||
})?,
|
})?,
|
||||||
);
|
);
|
||||||
|
@ -423,8 +428,8 @@ pub fn router(node: Arc<Node>) -> Router<()> {
|
||||||
mut request: Request<Body>| async move {
|
mut request: Request<Body>| async move {
|
||||||
let identity = match RemoteIdentity::from_str(&identity) {
|
let identity = match RemoteIdentity::from_str(&identity) {
|
||||||
Ok(identity) => identity,
|
Ok(identity) => identity,
|
||||||
Err(err) => {
|
Err(e) => {
|
||||||
warn!("Error parsing identity '{}': {}", identity, err);
|
warn!(%identity, ?e, "Error parsing identity;");
|
||||||
return (StatusCode::BAD_REQUEST, HeaderMap::new(), vec![])
|
return (StatusCode::BAD_REQUEST, HeaderMap::new(), vec![])
|
||||||
.into_response();
|
.into_response();
|
||||||
}
|
}
|
||||||
|
|
|
@ -11,8 +11,8 @@ use http_body::Full;
|
||||||
use tracing::debug;
|
use tracing::debug;
|
||||||
|
|
||||||
#[track_caller]
|
#[track_caller]
|
||||||
pub(crate) fn bad_request(err: impl Debug) -> http::Response<BoxBody> {
|
pub(crate) fn bad_request(e: impl Debug) -> http::Response<BoxBody> {
|
||||||
debug!("400: Bad Request at {}: {err:?}", Location::caller());
|
debug!(caller = %Location::caller(), ?e, "400: Bad Request;");
|
||||||
|
|
||||||
InfallibleResponse::builder()
|
InfallibleResponse::builder()
|
||||||
.status(StatusCode::BAD_REQUEST)
|
.status(StatusCode::BAD_REQUEST)
|
||||||
|
@ -20,8 +20,8 @@ pub(crate) fn bad_request(err: impl Debug) -> http::Response<BoxBody> {
|
||||||
}
|
}
|
||||||
|
|
||||||
#[track_caller]
|
#[track_caller]
|
||||||
pub(crate) fn not_found(err: impl Debug) -> http::Response<BoxBody> {
|
pub(crate) fn not_found(e: impl Debug) -> http::Response<BoxBody> {
|
||||||
debug!("404: Not Found at {}: {err:?}", Location::caller());
|
debug!(caller = %Location::caller(), ?e, "404: Not Found;");
|
||||||
|
|
||||||
InfallibleResponse::builder()
|
InfallibleResponse::builder()
|
||||||
.status(StatusCode::NOT_FOUND)
|
.status(StatusCode::NOT_FOUND)
|
||||||
|
@ -29,11 +29,8 @@ pub(crate) fn not_found(err: impl Debug) -> http::Response<BoxBody> {
|
||||||
}
|
}
|
||||||
|
|
||||||
#[track_caller]
|
#[track_caller]
|
||||||
pub(crate) fn internal_server_error(err: impl Debug) -> http::Response<BoxBody> {
|
pub(crate) fn internal_server_error(e: impl Debug) -> http::Response<BoxBody> {
|
||||||
debug!(
|
debug!(caller = %Location::caller(), ?e, "500: Internal Server Error;");
|
||||||
"500: Internal Server Error at {}: {err:?}",
|
|
||||||
Location::caller()
|
|
||||||
);
|
|
||||||
|
|
||||||
InfallibleResponse::builder()
|
InfallibleResponse::builder()
|
||||||
.status(StatusCode::INTERNAL_SERVER_ERROR)
|
.status(StatusCode::INTERNAL_SERVER_ERROR)
|
||||||
|
@ -41,8 +38,8 @@ pub(crate) fn internal_server_error(err: impl Debug) -> http::Response<BoxBody>
|
||||||
}
|
}
|
||||||
|
|
||||||
#[track_caller]
|
#[track_caller]
|
||||||
pub(crate) fn not_implemented(err: impl Debug) -> http::Response<BoxBody> {
|
pub(crate) fn not_implemented(e: impl Debug) -> http::Response<BoxBody> {
|
||||||
debug!("501: Not Implemented at {}: {err:?}", Location::caller());
|
debug!(caller = %Location::caller(), ?e, "501: Not Implemented;");
|
||||||
|
|
||||||
InfallibleResponse::builder()
|
InfallibleResponse::builder()
|
||||||
.status(StatusCode::NOT_IMPLEMENTED)
|
.status(StatusCode::NOT_IMPLEMENTED)
|
||||||
|
|
108
core/src/lib.rs
108
core/src/lib.rs
|
@ -4,18 +4,16 @@
|
||||||
use crate::{
|
use crate::{
|
||||||
api::{CoreEvent, Router},
|
api::{CoreEvent, Router},
|
||||||
location::LocationManagerError,
|
location::LocationManagerError,
|
||||||
object::media::old_thumbnail::old_actor::OldThumbnailer,
|
|
||||||
};
|
};
|
||||||
|
|
||||||
|
use sd_core_heavy_lifting::{media_processor::ThumbnailKind, JobSystem};
|
||||||
|
use sd_core_prisma_helpers::CasId;
|
||||||
|
|
||||||
#[cfg(feature = "ai")]
|
#[cfg(feature = "ai")]
|
||||||
use sd_ai::old_image_labeler::{DownloadModelError, OldImageLabeler, YoloV8};
|
use sd_ai::old_image_labeler::{DownloadModelError, OldImageLabeler, YoloV8};
|
||||||
use sd_utils::error::FileIOError;
|
|
||||||
|
|
||||||
use api::notifications::{Notification, NotificationData, NotificationId};
|
use sd_task_system::TaskSystem;
|
||||||
use chrono::{DateTime, Utc};
|
use sd_utils::error::FileIOError;
|
||||||
use node::config;
|
|
||||||
use notifications::Notifications;
|
|
||||||
use reqwest::{RequestBuilder, Response};
|
|
||||||
|
|
||||||
use std::{
|
use std::{
|
||||||
fmt,
|
fmt,
|
||||||
|
@ -23,6 +21,9 @@ use std::{
|
||||||
sync::{atomic::AtomicBool, Arc},
|
sync::{atomic::AtomicBool, Arc},
|
||||||
};
|
};
|
||||||
|
|
||||||
|
use chrono::{DateTime, Utc};
|
||||||
|
use futures_concurrency::future::Join;
|
||||||
|
use reqwest::{RequestBuilder, Response};
|
||||||
use thiserror::Error;
|
use thiserror::Error;
|
||||||
use tokio::{fs, io, sync::broadcast};
|
use tokio::{fs, io, sync::broadcast};
|
||||||
use tracing::{error, info, warn};
|
use tracing::{error, info, warn};
|
||||||
|
@ -34,6 +35,9 @@ use tracing_subscriber::{filter::FromEnvError, prelude::*, EnvFilter};
|
||||||
|
|
||||||
pub mod api;
|
pub mod api;
|
||||||
mod cloud;
|
mod cloud;
|
||||||
|
mod context;
|
||||||
|
#[cfg(feature = "crypto")]
|
||||||
|
pub(crate) mod crypto;
|
||||||
pub mod custom_uri;
|
pub mod custom_uri;
|
||||||
mod env;
|
mod env;
|
||||||
pub mod library;
|
pub mod library;
|
||||||
|
@ -50,7 +54,10 @@ pub(crate) mod volume;
|
||||||
|
|
||||||
pub use env::Env;
|
pub use env::Env;
|
||||||
|
|
||||||
use object::media::old_thumbnail::get_ephemeral_thumbnail_path;
|
use api::notifications::{Notification, NotificationData, NotificationId};
|
||||||
|
use context::{JobContext, NodeContext};
|
||||||
|
use node::config;
|
||||||
|
use notifications::Notifications;
|
||||||
|
|
||||||
pub(crate) use sd_core_sync as sync;
|
pub(crate) use sd_core_sync as sync;
|
||||||
|
|
||||||
|
@ -65,10 +72,11 @@ pub struct Node {
|
||||||
pub p2p: Arc<p2p::P2PManager>,
|
pub p2p: Arc<p2p::P2PManager>,
|
||||||
pub event_bus: (broadcast::Sender<CoreEvent>, broadcast::Receiver<CoreEvent>),
|
pub event_bus: (broadcast::Sender<CoreEvent>, broadcast::Receiver<CoreEvent>),
|
||||||
pub notifications: Notifications,
|
pub notifications: Notifications,
|
||||||
pub thumbnailer: OldThumbnailer,
|
|
||||||
pub cloud_sync_flag: Arc<AtomicBool>,
|
pub cloud_sync_flag: Arc<AtomicBool>,
|
||||||
pub env: Arc<env::Env>,
|
pub env: Arc<env::Env>,
|
||||||
pub http: reqwest::Client,
|
pub http: reqwest::Client,
|
||||||
|
pub task_system: TaskSystem<sd_core_heavy_lifting::Error>,
|
||||||
|
pub job_system: JobSystem<NodeContext, JobContext<NodeContext>>,
|
||||||
#[cfg(feature = "ai")]
|
#[cfg(feature = "ai")]
|
||||||
pub old_image_labeller: Option<OldImageLabeler>,
|
pub old_image_labeller: Option<OldImageLabeler>,
|
||||||
}
|
}
|
||||||
|
@ -88,7 +96,7 @@ impl Node {
|
||||||
) -> Result<(Arc<Node>, Arc<Router>), NodeError> {
|
) -> Result<(Arc<Node>, Arc<Router>), NodeError> {
|
||||||
let data_dir = data_dir.as_ref();
|
let data_dir = data_dir.as_ref();
|
||||||
|
|
||||||
info!("Starting core with data directory '{}'", data_dir.display());
|
info!(data_directory = %data_dir.display(), "Starting core;");
|
||||||
|
|
||||||
let env = Arc::new(env);
|
let env = Arc::new(env);
|
||||||
|
|
||||||
|
@ -117,22 +125,19 @@ impl Node {
|
||||||
let (old_jobs, jobs_actor) = old_job::OldJobs::new();
|
let (old_jobs, jobs_actor) = old_job::OldJobs::new();
|
||||||
let libraries = library::Libraries::new(data_dir.join("libraries")).await?;
|
let libraries = library::Libraries::new(data_dir.join("libraries")).await?;
|
||||||
|
|
||||||
|
let task_system = TaskSystem::new();
|
||||||
|
|
||||||
let (p2p, start_p2p) = p2p::P2PManager::new(config.clone(), libraries.clone())
|
let (p2p, start_p2p) = p2p::P2PManager::new(config.clone(), libraries.clone())
|
||||||
.await
|
.await
|
||||||
.map_err(NodeError::P2PManager)?;
|
.map_err(NodeError::P2PManager)?;
|
||||||
let node = Arc::new(Node {
|
let node = Arc::new(Node {
|
||||||
data_dir: data_dir.to_path_buf(),
|
data_dir: data_dir.to_path_buf(),
|
||||||
|
job_system: JobSystem::new(task_system.get_dispatcher(), data_dir),
|
||||||
|
task_system,
|
||||||
old_jobs,
|
old_jobs,
|
||||||
locations,
|
locations,
|
||||||
notifications: notifications::Notifications::new(),
|
notifications: notifications::Notifications::new(),
|
||||||
p2p,
|
p2p,
|
||||||
thumbnailer: OldThumbnailer::new(
|
|
||||||
data_dir,
|
|
||||||
libraries.clone(),
|
|
||||||
event_bus.0.clone(),
|
|
||||||
config.preferences_watcher(),
|
|
||||||
)
|
|
||||||
.await,
|
|
||||||
config,
|
config,
|
||||||
event_bus,
|
event_bus,
|
||||||
libraries,
|
libraries,
|
||||||
|
@ -146,7 +151,10 @@ impl Node {
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| {
|
.map_err(|e| {
|
||||||
error!("Failed to initialize image labeller. AI features will be disabled: {e:#?}");
|
error!(
|
||||||
|
?e,
|
||||||
|
"Failed to initialize image labeller. AI features will be disabled;"
|
||||||
|
);
|
||||||
})
|
})
|
||||||
.ok(),
|
.ok(),
|
||||||
});
|
});
|
||||||
|
@ -168,6 +176,27 @@ impl Node {
|
||||||
locations_actor.start(node.clone());
|
locations_actor.start(node.clone());
|
||||||
node.libraries.init(&node).await?;
|
node.libraries.init(&node).await?;
|
||||||
jobs_actor.start(node.clone());
|
jobs_actor.start(node.clone());
|
||||||
|
|
||||||
|
node.job_system
|
||||||
|
.init(
|
||||||
|
&node
|
||||||
|
.libraries
|
||||||
|
.get_all()
|
||||||
|
.await
|
||||||
|
.into_iter()
|
||||||
|
.map(|library| {
|
||||||
|
(
|
||||||
|
library.id,
|
||||||
|
NodeContext {
|
||||||
|
library,
|
||||||
|
node: Arc::clone(&node),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.collect(),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
|
||||||
start_p2p(
|
start_p2p(
|
||||||
node.clone(),
|
node.clone(),
|
||||||
axum::Router::new()
|
axum::Router::new()
|
||||||
|
@ -188,7 +217,7 @@ impl Node {
|
||||||
.into_make_service(),
|
.into_make_service(),
|
||||||
);
|
);
|
||||||
|
|
||||||
info!("Spacedrive online.");
|
info!("Spacedrive online!");
|
||||||
Ok((node, router))
|
Ok((node, router))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -212,7 +241,14 @@ impl Node {
|
||||||
|
|
||||||
std::env::set_var(
|
std::env::set_var(
|
||||||
"RUST_LOG",
|
"RUST_LOG",
|
||||||
format!("info,sd_core={level},sd_p2p=debug,sd_core::location::manager=info,sd_ai={level}"),
|
format!(
|
||||||
|
"info,\
|
||||||
|
sd_core={level},\
|
||||||
|
sd_p2p={level},\
|
||||||
|
sd_core_heavy_lifting={level},\
|
||||||
|
sd_task_system={level},\
|
||||||
|
sd_ai={level}"
|
||||||
|
),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -259,9 +295,18 @@ impl Node {
|
||||||
|
|
||||||
pub async fn shutdown(&self) {
|
pub async fn shutdown(&self) {
|
||||||
info!("Spacedrive shutting down...");
|
info!("Spacedrive shutting down...");
|
||||||
self.thumbnailer.shutdown().await;
|
|
||||||
self.old_jobs.shutdown().await;
|
// Let's shutdown the task system first, as the job system will receive tasks to save
|
||||||
self.p2p.shutdown().await;
|
self.task_system.shutdown().await;
|
||||||
|
|
||||||
|
(
|
||||||
|
self.old_jobs.shutdown(),
|
||||||
|
self.p2p.shutdown(),
|
||||||
|
self.job_system.shutdown(),
|
||||||
|
)
|
||||||
|
.join()
|
||||||
|
.await;
|
||||||
|
|
||||||
#[cfg(feature = "ai")]
|
#[cfg(feature = "ai")]
|
||||||
if let Some(image_labeller) = &self.old_image_labeller {
|
if let Some(image_labeller) = &self.old_image_labeller {
|
||||||
image_labeller.shutdown().await;
|
image_labeller.shutdown().await;
|
||||||
|
@ -271,12 +316,16 @@ impl Node {
|
||||||
|
|
||||||
pub(crate) fn emit(&self, event: CoreEvent) {
|
pub(crate) fn emit(&self, event: CoreEvent) {
|
||||||
if let Err(e) = self.event_bus.0.send(event) {
|
if let Err(e) = self.event_bus.0.send(event) {
|
||||||
warn!("Error sending event to event bus: {e:?}");
|
warn!(?e, "Error sending event to event bus;");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn ephemeral_thumbnail_exists(&self, cas_id: &str) -> Result<bool, FileIOError> {
|
pub async fn ephemeral_thumbnail_exists(
|
||||||
let thumb_path = get_ephemeral_thumbnail_path(self, cas_id);
|
&self,
|
||||||
|
cas_id: &CasId<'_>,
|
||||||
|
) -> Result<bool, FileIOError> {
|
||||||
|
let thumb_path =
|
||||||
|
ThumbnailKind::Ephemeral.compute_path(self.config.data_directory(), cas_id);
|
||||||
|
|
||||||
match fs::metadata(&thumb_path).await {
|
match fs::metadata(&thumb_path).await {
|
||||||
Ok(_) => Ok(true),
|
Ok(_) => Ok(true),
|
||||||
|
@ -301,8 +350,8 @@ impl Node {
|
||||||
Ok(_) => {
|
Ok(_) => {
|
||||||
self.notifications._internal_send(notification);
|
self.notifications._internal_send(notification);
|
||||||
}
|
}
|
||||||
Err(err) => {
|
Err(e) => {
|
||||||
error!("Error saving notification to config: {:?}", err);
|
error!(?e, "Error saving notification to config;");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -375,6 +424,9 @@ pub enum NodeError {
|
||||||
InitConfig(#[from] util::debug_initializer::InitConfigError),
|
InitConfig(#[from] util::debug_initializer::InitConfigError),
|
||||||
#[error("logger error: {0}")]
|
#[error("logger error: {0}")]
|
||||||
Logger(#[from] FromEnvError),
|
Logger(#[from] FromEnvError),
|
||||||
|
#[error(transparent)]
|
||||||
|
JobSystem(#[from] sd_core_heavy_lifting::JobSystemError),
|
||||||
|
|
||||||
#[cfg(feature = "ai")]
|
#[cfg(feature = "ai")]
|
||||||
#[error("ai error: {0}")]
|
#[error("ai error: {0}")]
|
||||||
AI(#[from] sd_ai::Error),
|
AI(#[from] sd_ai::Error),
|
||||||
|
|
|
@ -130,7 +130,7 @@ impl LibraryConfig {
|
||||||
db.indexer_rule().update_many(
|
db.indexer_rule().update_many(
|
||||||
vec![indexer_rule::name::equals(Some(name))],
|
vec![indexer_rule::name::equals(Some(name))],
|
||||||
vec![indexer_rule::pub_id::set(sd_utils::uuid_to_bytes(
|
vec![indexer_rule::pub_id::set(sd_utils::uuid_to_bytes(
|
||||||
Uuid::from_u128(i as u128),
|
&Uuid::from_u128(i as u128),
|
||||||
))],
|
))],
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
|
@ -221,7 +221,7 @@ impl LibraryConfig {
|
||||||
maybe_missing(path.size_in_bytes, "file_path.size_in_bytes")
|
maybe_missing(path.size_in_bytes, "file_path.size_in_bytes")
|
||||||
.map_or_else(
|
.map_or_else(
|
||||||
|e| {
|
|e| {
|
||||||
error!("{e:#?}");
|
error!(?e);
|
||||||
None
|
None
|
||||||
},
|
},
|
||||||
Some,
|
Some,
|
||||||
|
@ -232,9 +232,11 @@ impl LibraryConfig {
|
||||||
Some(size.to_be_bytes().to_vec())
|
Some(size.to_be_bytes().to_vec())
|
||||||
} else {
|
} else {
|
||||||
error!(
|
error!(
|
||||||
"File path <id='{}'> had invalid size: '{}'",
|
file_path_id = %path.id,
|
||||||
path.id, size_in_bytes
|
size = %size_in_bytes,
|
||||||
|
"File path had invalid size;",
|
||||||
);
|
);
|
||||||
|
|
||||||
None
|
None
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -463,7 +465,8 @@ impl LibraryConfig {
|
||||||
}
|
}
|
||||||
|
|
||||||
_ => {
|
_ => {
|
||||||
error!("Library config version is not handled: {:?}", current);
|
error!(current_version = ?current, "Library config version is not handled;");
|
||||||
|
|
||||||
return Err(VersionManagerError::UnexpectedMigration {
|
return Err(VersionManagerError::UnexpectedMigration {
|
||||||
current_version: current.int_value(),
|
current_version: current.int_value(),
|
||||||
next_version: next.int_value(),
|
next_version: next.int_value(),
|
||||||
|
|
|
@ -1,9 +1,8 @@
|
||||||
use crate::{
|
use crate::{api::CoreEvent, cloud, sync, Node};
|
||||||
api::CoreEvent, cloud, object::media::old_thumbnail::get_indexed_thumbnail_path, sync, Node,
|
|
||||||
};
|
|
||||||
|
|
||||||
use sd_core_file_path_helper::IsolatedFilePathData;
|
use sd_core_file_path_helper::IsolatedFilePathData;
|
||||||
use sd_core_prisma_helpers::file_path_to_full_path;
|
use sd_core_heavy_lifting::media_processor::ThumbnailKind;
|
||||||
|
use sd_core_prisma_helpers::{file_path_to_full_path, CasId};
|
||||||
|
|
||||||
use sd_p2p::Identity;
|
use sd_p2p::Identity;
|
||||||
use sd_prisma::prisma::{file_path, location, PrismaClient};
|
use sd_prisma::prisma::{file_path, location, PrismaClient};
|
||||||
|
@ -121,12 +120,17 @@ impl Library {
|
||||||
// TODO: Remove this once we replace the old invalidation system
|
// TODO: Remove this once we replace the old invalidation system
|
||||||
pub(crate) fn emit(&self, event: CoreEvent) {
|
pub(crate) fn emit(&self, event: CoreEvent) {
|
||||||
if let Err(e) = self.event_bus_tx.send(event) {
|
if let Err(e) = self.event_bus_tx.send(event) {
|
||||||
warn!("Error sending event to event bus: {e:?}");
|
warn!(?e, "Error sending event to event bus;");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn thumbnail_exists(&self, node: &Node, cas_id: &str) -> Result<bool, FileIOError> {
|
pub async fn thumbnail_exists(
|
||||||
let thumb_path = get_indexed_thumbnail_path(node, cas_id, self.id);
|
&self,
|
||||||
|
node: &Node,
|
||||||
|
cas_id: &CasId<'_>,
|
||||||
|
) -> Result<bool, FileIOError> {
|
||||||
|
let thumb_path =
|
||||||
|
ThumbnailKind::Indexed(self.id).compute_path(node.config.data_directory(), cas_id);
|
||||||
|
|
||||||
match fs::metadata(&thumb_path).await {
|
match fs::metadata(&thumb_path).await {
|
||||||
Ok(_) => Ok(true),
|
Ok(_) => Ok(true),
|
||||||
|
@ -182,7 +186,7 @@ impl Library {
|
||||||
|
|
||||||
pub fn do_cloud_sync(&self) {
|
pub fn do_cloud_sync(&self) {
|
||||||
if let Err(e) = self.do_cloud_sync.send(()) {
|
if let Err(e) = self.do_cloud_sync.send(()) {
|
||||||
warn!("Error sending cloud resync message: {e:?}");
|
warn!(?e, "Error sending cloud resync message;");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -36,7 +36,7 @@ use tokio::{
|
||||||
sync::{broadcast, RwLock},
|
sync::{broadcast, RwLock},
|
||||||
time::sleep,
|
time::sleep,
|
||||||
};
|
};
|
||||||
use tracing::{debug, error, info, warn};
|
use tracing::{debug, error, info, instrument, warn};
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
use super::{Library, LibraryConfig, LibraryName};
|
use super::{Library, LibraryConfig, LibraryName};
|
||||||
|
@ -113,9 +113,9 @@ impl Libraries {
|
||||||
.and_then(|v| v.to_str().map(Uuid::from_str))
|
.and_then(|v| v.to_str().map(Uuid::from_str))
|
||||||
else {
|
else {
|
||||||
warn!(
|
warn!(
|
||||||
"Attempted to load library from path '{}' \
|
config_path = %config_path.display(),
|
||||||
but it has an invalid filename. Skipping...",
|
"Attempted to load library from path \
|
||||||
config_path.display()
|
but it has an invalid filename. Skipping...;",
|
||||||
);
|
);
|
||||||
continue;
|
continue;
|
||||||
};
|
};
|
||||||
|
@ -124,7 +124,11 @@ impl Libraries {
|
||||||
match fs::metadata(&db_path).await {
|
match fs::metadata(&db_path).await {
|
||||||
Ok(_) => {}
|
Ok(_) => {}
|
||||||
Err(e) if e.kind() == io::ErrorKind::NotFound => {
|
Err(e) if e.kind() == io::ErrorKind::NotFound => {
|
||||||
warn!("Found library '{}' but no matching database file was found. Skipping...", config_path.display());
|
warn!(
|
||||||
|
config_path = %config_path.display(),
|
||||||
|
"Found library but no matching database file was found. Skipping...;",
|
||||||
|
);
|
||||||
|
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
Err(e) => return Err(FileIOError::from((db_path, e)).into()),
|
Err(e) => return Err(FileIOError::from((db_path, e)).into()),
|
||||||
|
@ -158,6 +162,7 @@ impl Libraries {
|
||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[instrument(skip(self, instance, node), err)]
|
||||||
#[allow(clippy::too_many_arguments)]
|
#[allow(clippy::too_many_arguments)]
|
||||||
pub(crate) async fn create_with_uuid(
|
pub(crate) async fn create_with_uuid(
|
||||||
self: &Arc<Self>,
|
self: &Arc<Self>,
|
||||||
|
@ -189,9 +194,8 @@ impl Libraries {
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
debug!(
|
debug!(
|
||||||
"Created library '{}' config at '{}'",
|
config_path = %config_path.display(),
|
||||||
id,
|
"Created library;",
|
||||||
config_path.display()
|
|
||||||
);
|
);
|
||||||
|
|
||||||
let node_cfg = node.config.get().await;
|
let node_cfg = node.config.get().await;
|
||||||
|
@ -225,12 +229,12 @@ impl Libraries {
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
debug!("Loaded library '{id:?}'");
|
debug!("Loaded library");
|
||||||
|
|
||||||
if should_seed {
|
if should_seed {
|
||||||
tag::seed::new_library(&library).await?;
|
tag::seed::new_library(&library).await?;
|
||||||
sd_core_indexer_rules::seed::new_or_existing_library(&library.db).await?;
|
sd_core_indexer_rules::seed::new_or_existing_library(&library.db).await?;
|
||||||
debug!("Seeded library '{id:?}'");
|
debug!("Seeded library");
|
||||||
}
|
}
|
||||||
|
|
||||||
invalidate_query!(library, "library.list");
|
invalidate_query!(library, "library.list");
|
||||||
|
@ -325,7 +329,7 @@ impl Libraries {
|
||||||
.exec()
|
.exec()
|
||||||
.await
|
.await
|
||||||
.map(|locations| locations.into_iter().filter_map(|location| location.path))
|
.map(|locations| locations.into_iter().filter_map(|location| location.path))
|
||||||
.map_err(|e| error!("Failed to fetch locations for library deletion: {e:#?}"))
|
.map_err(|e| error!(?e, "Failed to fetch locations for library deletion;"))
|
||||||
{
|
{
|
||||||
location_paths
|
location_paths
|
||||||
.map(|location_path| async move {
|
.map(|location_path| async move {
|
||||||
|
@ -343,7 +347,7 @@ impl Libraries {
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.for_each(|res| {
|
.for_each(|res| {
|
||||||
if let Err(e) = res {
|
if let Err(e) = res {
|
||||||
error!("Failed to remove library from location metadata: {e:#?}");
|
error!(?e, "Failed to remove library from location metadata;");
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
@ -371,7 +375,7 @@ impl Libraries {
|
||||||
.remove(id)
|
.remove(id)
|
||||||
.expect("we have exclusive access and checked it exists!");
|
.expect("we have exclusive access and checked it exists!");
|
||||||
|
|
||||||
info!("Removed Library <id='{}'>", library.id);
|
info!(%library.id, "Removed Library;");
|
||||||
|
|
||||||
invalidate_query!(library, "library.list");
|
invalidate_query!(library, "library.list");
|
||||||
|
|
||||||
|
@ -420,6 +424,16 @@ impl Libraries {
|
||||||
self.libraries.read().await.get(library_id).is_some()
|
self.libraries.read().await.get(library_id).is_some()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[instrument(
|
||||||
|
skip_all,
|
||||||
|
fields(
|
||||||
|
library_id = %id,
|
||||||
|
db_path = %db_path.as_ref().display(),
|
||||||
|
config_path = %config_path.as_ref().display(),
|
||||||
|
%should_seed,
|
||||||
|
),
|
||||||
|
err,
|
||||||
|
)]
|
||||||
/// load the library from a given path.
|
/// load the library from a given path.
|
||||||
pub async fn load(
|
pub async fn load(
|
||||||
self: &Arc<Self>,
|
self: &Arc<Self>,
|
||||||
|
@ -479,8 +493,9 @@ impl Libraries {
|
||||||
|| curr_metadata != Some(node.p2p.peer_metadata())
|
|| curr_metadata != Some(node.p2p.peer_metadata())
|
||||||
{
|
{
|
||||||
info!(
|
info!(
|
||||||
"Detected that the library '{}' has changed node from '{}' to '{}'. Reconciling node data...",
|
old_node_id = %instance_node_id,
|
||||||
id, instance_node_id, node_config.id
|
new_node_id = %node_config.id,
|
||||||
|
"Detected that the library has changed nodes. Reconciling node data...",
|
||||||
);
|
);
|
||||||
|
|
||||||
// ensure
|
// ensure
|
||||||
|
@ -593,12 +608,12 @@ impl Libraries {
|
||||||
.await?
|
.await?
|
||||||
{
|
{
|
||||||
if let Err(e) = node.locations.add(location.id, library.clone()).await {
|
if let Err(e) = node.locations.add(location.id, library.clone()).await {
|
||||||
error!("Failed to watch location on startup: {e}");
|
error!(?e, "Failed to watch location on startup;");
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Err(e) = node.old_jobs.clone().cold_resume(node, &library).await {
|
if let Err(e) = node.old_jobs.clone().cold_resume(node, &library).await {
|
||||||
error!("Failed to resume jobs for library. {:#?}", e);
|
error!(?e, "Failed to resume jobs for library;");
|
||||||
}
|
}
|
||||||
|
|
||||||
tokio::spawn({
|
tokio::spawn({
|
||||||
|
@ -639,20 +654,20 @@ impl Libraries {
|
||||||
if should_update {
|
if should_update {
|
||||||
warn!("Library instance on cloud is outdated. Updating...");
|
warn!("Library instance on cloud is outdated. Updating...");
|
||||||
|
|
||||||
if let Err(err) =
|
if let Err(e) = sd_cloud_api::library::update_instance(
|
||||||
sd_cloud_api::library::update_instance(
|
node.cloud_api_config().await,
|
||||||
node.cloud_api_config().await,
|
library.id,
|
||||||
library.id,
|
this_instance.uuid,
|
||||||
this_instance.uuid,
|
Some(node_config.id),
|
||||||
Some(node_config.id),
|
Some(node_config.identity.to_remote_identity()),
|
||||||
Some(node_config.identity.to_remote_identity()),
|
Some(node.p2p.peer_metadata()),
|
||||||
Some(node.p2p.peer_metadata()),
|
)
|
||||||
)
|
.await
|
||||||
.await
|
|
||||||
{
|
{
|
||||||
error!(
|
error!(
|
||||||
"Failed to updating instance '{}' on cloud: {:#?}",
|
instance_uuid = %this_instance.uuid,
|
||||||
this_instance.uuid, err
|
?e,
|
||||||
|
"Failed to updating instance on cloud;",
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -661,29 +676,26 @@ impl Libraries {
|
||||||
if lib.name != *library.config().await.name {
|
if lib.name != *library.config().await.name {
|
||||||
warn!("Library name on cloud is outdated. Updating...");
|
warn!("Library name on cloud is outdated. Updating...");
|
||||||
|
|
||||||
if let Err(err) = sd_cloud_api::library::update(
|
if let Err(e) = sd_cloud_api::library::update(
|
||||||
node.cloud_api_config().await,
|
node.cloud_api_config().await,
|
||||||
library.id,
|
library.id,
|
||||||
Some(lib.name),
|
Some(lib.name),
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
error!(
|
error!(?e, "Failed to update library name on cloud;");
|
||||||
"Failed to update library name on cloud: {:#?}",
|
|
||||||
err
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
for instance in lib.instances {
|
for instance in lib.instances {
|
||||||
if let Err(err) = cloud::sync::receive::upsert_instance(
|
if let Err(e) = cloud::sync::receive::upsert_instance(
|
||||||
library.id,
|
library.id,
|
||||||
&library.db,
|
&library.db,
|
||||||
&library.sync,
|
&library.sync,
|
||||||
&node.libraries,
|
&node.libraries,
|
||||||
instance.uuid,
|
&instance.uuid,
|
||||||
instance.identity,
|
instance.identity,
|
||||||
instance.node_id,
|
&instance.node_id,
|
||||||
RemoteIdentity::from_str(
|
RemoteIdentity::from_str(
|
||||||
&instance.node_remote_identity,
|
&instance.node_remote_identity,
|
||||||
)
|
)
|
||||||
|
@ -692,10 +704,7 @@ impl Libraries {
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
error!(
|
error!(?e, "Failed to create instance on cloud;");
|
||||||
"Failed to create instance from cloud: {:#?}",
|
|
||||||
err
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -37,8 +37,8 @@ pub async fn update_library_statistics(
|
||||||
.find_many(vec![])
|
.find_many(vec![])
|
||||||
.exec()
|
.exec()
|
||||||
.await
|
.await
|
||||||
.unwrap_or_else(|err| {
|
.unwrap_or_else(|e| {
|
||||||
error!("Failed to get locations: {:#?}", err);
|
error!(?e, "Failed to get locations;");
|
||||||
vec![]
|
vec![]
|
||||||
})
|
})
|
||||||
.into_iter()
|
.into_iter()
|
||||||
|
@ -79,7 +79,7 @@ pub async fn update_library_statistics(
|
||||||
.exec()
|
.exec()
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
info!("Updated library statistics: {:?}", stats);
|
info!(?stats, "Updated library statistics;");
|
||||||
|
|
||||||
Ok(stats)
|
Ok(stats)
|
||||||
}
|
}
|
||||||
|
|
|
@ -81,35 +81,33 @@ pub enum LocationError {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl From<LocationError> for rspc::Error {
|
impl From<LocationError> for rspc::Error {
|
||||||
fn from(err: LocationError) -> Self {
|
fn from(e: LocationError) -> Self {
|
||||||
use LocationError::*;
|
use LocationError::*;
|
||||||
|
|
||||||
match err {
|
match e {
|
||||||
// Not found errors
|
// Not found errors
|
||||||
PathNotFound(_)
|
PathNotFound(_)
|
||||||
| UuidNotFound(_)
|
| UuidNotFound(_)
|
||||||
| IdNotFound(_)
|
| IdNotFound(_)
|
||||||
| FilePath(FilePathError::IdNotFound(_) | FilePathError::NotFound(_)) => {
|
| FilePath(FilePathError::IdNotFound(_) | FilePathError::NotFound(_)) => {
|
||||||
Self::with_cause(ErrorCode::NotFound, err.to_string(), err)
|
Self::with_cause(ErrorCode::NotFound, e.to_string(), e)
|
||||||
}
|
}
|
||||||
|
|
||||||
// User's fault errors
|
// User's fault errors
|
||||||
NotDirectory(_) | NestedLocation(_) | LocationAlreadyExists(_) => {
|
NotDirectory(_) | NestedLocation(_) | LocationAlreadyExists(_) => {
|
||||||
Self::with_cause(ErrorCode::BadRequest, err.to_string(), err)
|
Self::with_cause(ErrorCode::BadRequest, e.to_string(), e)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Custom error message is used to differentiate these errors in the frontend
|
// Custom error message is used to differentiate these errors in the frontend
|
||||||
// TODO: A better solution would be for rspc to support sending custom data alongside errors
|
// TODO: A better solution would be for rspc to support sending custom data alongside errors
|
||||||
NeedRelink { .. } => {
|
NeedRelink { .. } => Self::with_cause(ErrorCode::Conflict, "NEED_RELINK".to_owned(), e),
|
||||||
Self::with_cause(ErrorCode::Conflict, "NEED_RELINK".to_owned(), err)
|
|
||||||
}
|
|
||||||
AddLibraryToMetadata(_) => {
|
AddLibraryToMetadata(_) => {
|
||||||
Self::with_cause(ErrorCode::Conflict, "ADD_LIBRARY".to_owned(), err)
|
Self::with_cause(ErrorCode::Conflict, "ADD_LIBRARY".to_owned(), e)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Internal errors
|
// Internal errors
|
||||||
MissingField(missing_error) => missing_error.into(),
|
MissingField(missing_error) => missing_error.into(),
|
||||||
_ => Self::with_cause(ErrorCode::InternalServerError, err.to_string(), err),
|
_ => Self::with_cause(ErrorCode::InternalServerError, e.to_string(), e),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,546 +0,0 @@
|
||||||
use crate::library::Library;
|
|
||||||
|
|
||||||
use sd_core_file_path_helper::{FilePathError, IsolatedFilePathData, IsolatedFilePathDataParts};
|
|
||||||
use sd_core_indexer_rules::IndexerRuleError;
|
|
||||||
use sd_core_prisma_helpers::file_path_pub_and_cas_ids;
|
|
||||||
|
|
||||||
use sd_prisma::{
|
|
||||||
prisma::{file_path, location, PrismaClient},
|
|
||||||
prisma_sync,
|
|
||||||
};
|
|
||||||
use sd_sync::*;
|
|
||||||
use sd_utils::{db::inode_to_db, error::FileIOError, from_bytes_to_uuid, msgpack};
|
|
||||||
|
|
||||||
use std::{collections::HashMap, path::Path};
|
|
||||||
|
|
||||||
use chrono::Utc;
|
|
||||||
use futures_concurrency::future::TryJoin;
|
|
||||||
use itertools::Itertools;
|
|
||||||
use prisma_client_rust::operator::or;
|
|
||||||
use rspc::ErrorCode;
|
|
||||||
use serde::{Deserialize, Serialize};
|
|
||||||
use thiserror::Error;
|
|
||||||
use tracing::{trace, warn};
|
|
||||||
|
|
||||||
use super::location_with_indexer_rules;
|
|
||||||
|
|
||||||
pub mod old_indexer_job;
|
|
||||||
mod old_shallow;
|
|
||||||
mod old_walk;
|
|
||||||
|
|
||||||
use old_walk::WalkedEntry;
|
|
||||||
|
|
||||||
pub use old_indexer_job::OldIndexerJobInit;
|
|
||||||
pub use old_shallow::*;
|
|
||||||
|
|
||||||
#[derive(Serialize, Deserialize, Debug)]
|
|
||||||
pub struct OldIndexerJobSaveStep {
|
|
||||||
chunk_idx: usize,
|
|
||||||
walked: Vec<WalkedEntry>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Serialize, Deserialize, Debug)]
|
|
||||||
pub struct OldIndexerJobUpdateStep {
|
|
||||||
chunk_idx: usize,
|
|
||||||
to_update: Vec<WalkedEntry>,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Error type for the indexer module
|
|
||||||
#[derive(Error, Debug)]
|
|
||||||
pub enum IndexerError {
|
|
||||||
// Not Found errors
|
|
||||||
#[error("indexer rule not found: <id='{0}'>")]
|
|
||||||
IndexerRuleNotFound(i32),
|
|
||||||
#[error("received sub path not in database: <path='{}'>", .0.display())]
|
|
||||||
SubPathNotFound(Box<Path>),
|
|
||||||
|
|
||||||
// Internal Errors
|
|
||||||
#[error("Database Error: {}", .0.to_string())]
|
|
||||||
Database(#[from] prisma_client_rust::QueryError),
|
|
||||||
#[error(transparent)]
|
|
||||||
FileIO(#[from] FileIOError),
|
|
||||||
#[error(transparent)]
|
|
||||||
FilePath(#[from] FilePathError),
|
|
||||||
|
|
||||||
// Mixed errors
|
|
||||||
#[error(transparent)]
|
|
||||||
IndexerRules(#[from] IndexerRuleError),
|
|
||||||
}
|
|
||||||
|
|
||||||
impl From<IndexerError> for rspc::Error {
|
|
||||||
fn from(err: IndexerError) -> Self {
|
|
||||||
match err {
|
|
||||||
IndexerError::IndexerRuleNotFound(_) | IndexerError::SubPathNotFound(_) => {
|
|
||||||
rspc::Error::with_cause(ErrorCode::NotFound, err.to_string(), err)
|
|
||||||
}
|
|
||||||
|
|
||||||
IndexerError::IndexerRules(rule_err) => rule_err.into(),
|
|
||||||
|
|
||||||
_ => rspc::Error::with_cause(ErrorCode::InternalServerError, err.to_string(), err),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn execute_indexer_save_step(
|
|
||||||
location: &location_with_indexer_rules::Data,
|
|
||||||
OldIndexerJobSaveStep { walked, .. }: &OldIndexerJobSaveStep,
|
|
||||||
library: &Library,
|
|
||||||
) -> Result<i64, IndexerError> {
|
|
||||||
let Library { sync, db, .. } = library;
|
|
||||||
|
|
||||||
let (sync_stuff, paths): (Vec<_>, Vec<_>) = walked
|
|
||||||
.iter()
|
|
||||||
.map(|entry| {
|
|
||||||
let IsolatedFilePathDataParts {
|
|
||||||
materialized_path,
|
|
||||||
is_dir,
|
|
||||||
name,
|
|
||||||
extension,
|
|
||||||
..
|
|
||||||
} = &entry.iso_file_path.to_parts();
|
|
||||||
|
|
||||||
use file_path::*;
|
|
||||||
|
|
||||||
let pub_id = sd_utils::uuid_to_bytes(entry.pub_id);
|
|
||||||
|
|
||||||
let (sync_params, db_params): (Vec<_>, Vec<_>) = [
|
|
||||||
(
|
|
||||||
(
|
|
||||||
location::NAME,
|
|
||||||
msgpack!(prisma_sync::location::SyncId {
|
|
||||||
pub_id: location.pub_id.clone()
|
|
||||||
}),
|
|
||||||
),
|
|
||||||
location_id::set(Some(location.id)),
|
|
||||||
),
|
|
||||||
sync_db_entry!(materialized_path.to_string(), materialized_path),
|
|
||||||
sync_db_entry!(name.to_string(), name),
|
|
||||||
sync_db_entry!(*is_dir, is_dir),
|
|
||||||
sync_db_entry!(extension.to_string(), extension),
|
|
||||||
sync_db_entry!(
|
|
||||||
entry.metadata.size_in_bytes.to_be_bytes().to_vec(),
|
|
||||||
size_in_bytes_bytes
|
|
||||||
),
|
|
||||||
sync_db_entry!(inode_to_db(entry.metadata.inode), inode),
|
|
||||||
{
|
|
||||||
let v = entry.metadata.created_at.into();
|
|
||||||
sync_db_entry!(v, date_created)
|
|
||||||
},
|
|
||||||
{
|
|
||||||
let v = entry.metadata.modified_at.into();
|
|
||||||
sync_db_entry!(v, date_modified)
|
|
||||||
},
|
|
||||||
{
|
|
||||||
let v = Utc::now().into();
|
|
||||||
sync_db_entry!(v, date_indexed)
|
|
||||||
},
|
|
||||||
sync_db_entry!(entry.metadata.hidden, hidden),
|
|
||||||
]
|
|
||||||
.into_iter()
|
|
||||||
.unzip();
|
|
||||||
|
|
||||||
(
|
|
||||||
sync.shared_create(
|
|
||||||
prisma_sync::file_path::SyncId {
|
|
||||||
pub_id: sd_utils::uuid_to_bytes(entry.pub_id),
|
|
||||||
},
|
|
||||||
sync_params,
|
|
||||||
),
|
|
||||||
file_path::create_unchecked(pub_id, db_params),
|
|
||||||
)
|
|
||||||
})
|
|
||||||
.unzip();
|
|
||||||
|
|
||||||
let count = sync
|
|
||||||
.write_ops(
|
|
||||||
db,
|
|
||||||
(
|
|
||||||
sync_stuff.into_iter().flatten().collect(),
|
|
||||||
db.file_path().create_many(paths).skip_duplicates(),
|
|
||||||
),
|
|
||||||
)
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
trace!("Inserted {count} records");
|
|
||||||
|
|
||||||
Ok(count)
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn execute_indexer_update_step(
|
|
||||||
update_step: &OldIndexerJobUpdateStep,
|
|
||||||
Library { sync, db, .. }: &Library,
|
|
||||||
) -> Result<i64, IndexerError> {
|
|
||||||
let (sync_stuff, paths_to_update): (Vec<_>, Vec<_>) = update_step
|
|
||||||
.to_update
|
|
||||||
.iter()
|
|
||||||
.map(|entry| async move {
|
|
||||||
let IsolatedFilePathDataParts { is_dir, .. } = &entry.iso_file_path.to_parts();
|
|
||||||
|
|
||||||
let pub_id = sd_utils::uuid_to_bytes(entry.pub_id);
|
|
||||||
|
|
||||||
let should_unlink_object = if let Some(object_id) = entry.maybe_object_id {
|
|
||||||
db.file_path()
|
|
||||||
.count(vec![file_path::object_id::equals(Some(object_id))])
|
|
||||||
.exec()
|
|
||||||
.await? > 1
|
|
||||||
} else {
|
|
||||||
false
|
|
||||||
};
|
|
||||||
|
|
||||||
use file_path::*;
|
|
||||||
|
|
||||||
let (sync_params, db_params): (Vec<_>, Vec<_>) = [
|
|
||||||
// As this file was updated while Spacedrive was offline, we mark the object_id and cas_id as null
|
|
||||||
// So this file_path will be updated at file identifier job
|
|
||||||
should_unlink_object
|
|
||||||
.then_some(((object_id::NAME, msgpack!(nil)), object::disconnect())),
|
|
||||||
Some(((cas_id::NAME, msgpack!(nil)), cas_id::set(None))),
|
|
||||||
Some(sync_db_entry!(*is_dir, is_dir)),
|
|
||||||
Some(sync_db_entry!(
|
|
||||||
entry.metadata.size_in_bytes.to_be_bytes().to_vec(),
|
|
||||||
size_in_bytes_bytes
|
|
||||||
)),
|
|
||||||
Some(sync_db_entry!(inode_to_db(entry.metadata.inode), inode)),
|
|
||||||
Some({
|
|
||||||
let v = entry.metadata.created_at.into();
|
|
||||||
sync_db_entry!(v, date_created)
|
|
||||||
}),
|
|
||||||
Some({
|
|
||||||
let v = entry.metadata.modified_at.into();
|
|
||||||
sync_db_entry!(v, date_modified)
|
|
||||||
}),
|
|
||||||
Some(sync_db_entry!(entry.metadata.hidden, hidden)),
|
|
||||||
]
|
|
||||||
.into_iter()
|
|
||||||
.flatten()
|
|
||||||
.unzip();
|
|
||||||
|
|
||||||
Ok::<_, IndexerError>((
|
|
||||||
sync_params
|
|
||||||
.into_iter()
|
|
||||||
.map(|(field, value)| {
|
|
||||||
sync.shared_update(
|
|
||||||
prisma_sync::file_path::SyncId {
|
|
||||||
pub_id: pub_id.clone(),
|
|
||||||
},
|
|
||||||
field,
|
|
||||||
value,
|
|
||||||
)
|
|
||||||
})
|
|
||||||
.collect::<Vec<_>>(),
|
|
||||||
db.file_path()
|
|
||||||
.update(file_path::pub_id::equals(pub_id), db_params)
|
|
||||||
.select(file_path::select!({ id })),
|
|
||||||
))
|
|
||||||
})
|
|
||||||
.collect::<Vec<_>>()
|
|
||||||
.try_join()
|
|
||||||
.await?
|
|
||||||
.into_iter()
|
|
||||||
.unzip();
|
|
||||||
|
|
||||||
let updated = sync
|
|
||||||
.write_ops(
|
|
||||||
db,
|
|
||||||
(sync_stuff.into_iter().flatten().collect(), paths_to_update),
|
|
||||||
)
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
trace!("Updated {updated:?} records");
|
|
||||||
|
|
||||||
Ok(updated.len() as i64)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn iso_file_path_factory(
|
|
||||||
location_id: location::id::Type,
|
|
||||||
location_path: &Path,
|
|
||||||
) -> impl Fn(&Path, bool) -> Result<IsolatedFilePathData<'static>, IndexerError> + '_ {
|
|
||||||
move |path, is_dir| {
|
|
||||||
IsolatedFilePathData::new(location_id, location_path, path, is_dir).map_err(Into::into)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn remove_non_existing_file_paths(
|
|
||||||
to_remove: impl IntoIterator<Item = file_path_pub_and_cas_ids::Data>,
|
|
||||||
db: &PrismaClient,
|
|
||||||
sync: &sd_core_sync::Manager,
|
|
||||||
) -> Result<u64, IndexerError> {
|
|
||||||
let (sync_params, db_params): (Vec<_>, Vec<_>) = to_remove
|
|
||||||
.into_iter()
|
|
||||||
.map(|d| {
|
|
||||||
(
|
|
||||||
sync.shared_delete(prisma_sync::file_path::SyncId { pub_id: d.pub_id }),
|
|
||||||
d.id,
|
|
||||||
)
|
|
||||||
})
|
|
||||||
.unzip();
|
|
||||||
|
|
||||||
sync.write_ops(
|
|
||||||
db,
|
|
||||||
(
|
|
||||||
sync_params,
|
|
||||||
db.file_path()
|
|
||||||
.delete_many(vec![file_path::id::in_vec(db_params)]),
|
|
||||||
),
|
|
||||||
)
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
Ok(0)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TODO: Change this macro to a fn when we're able to return
|
|
||||||
// `impl Fn(Vec<file_path::WhereParam>) -> impl Future<Output = Result<Vec<file_path_walker::Data>, IndexerError>>`
|
|
||||||
// Maybe when TAITs arrive
|
|
||||||
#[macro_export]
|
|
||||||
macro_rules! file_paths_db_fetcher_fn {
|
|
||||||
($db:expr) => {{
|
|
||||||
|found_paths| async {
|
|
||||||
// Each found path is a AND with 4 terms, and SQLite has a expression tree limit of 1000 terms
|
|
||||||
// so we will use chunks of 200 just to be safe
|
|
||||||
|
|
||||||
// FIXME: Can't pass this chunks variable direct to _batch because of lifetime issues
|
|
||||||
let chunks = found_paths
|
|
||||||
.into_iter()
|
|
||||||
.chunks(200)
|
|
||||||
.into_iter()
|
|
||||||
.map(|founds| {
|
|
||||||
$db.file_path()
|
|
||||||
.find_many(vec![::prisma_client_rust::operator::or(
|
|
||||||
founds.collect::<Vec<_>>(),
|
|
||||||
)])
|
|
||||||
.select(::sd_core_prisma_helpers::file_path_walker::select())
|
|
||||||
})
|
|
||||||
.collect::<Vec<_>>();
|
|
||||||
|
|
||||||
$db._batch(chunks)
|
|
||||||
.await
|
|
||||||
.map(|fetched| fetched.into_iter().flatten().collect::<Vec<_>>())
|
|
||||||
.map_err(Into::into)
|
|
||||||
}
|
|
||||||
}};
|
|
||||||
}
|
|
||||||
|
|
||||||
// TODO: Change this macro to a fn when we're able to return
|
|
||||||
// `impl Fn(&Path, Vec<file_path::WhereParam>) -> impl Future<Output = Result<Vec<file_path_just_pub_id::Data>, IndexerError>>`
|
|
||||||
// Maybe when TAITs arrive
|
|
||||||
// FIXME: (fogodev) I was receiving this error here https://github.com/rust-lang/rust/issues/74497
|
|
||||||
#[macro_export]
|
|
||||||
macro_rules! to_remove_db_fetcher_fn {
|
|
||||||
($location_id:expr, $db:expr) => {{
|
|
||||||
|parent_iso_file_path, unique_location_id_materialized_path_name_extension_params| async {
|
|
||||||
let location_id: ::sd_prisma::prisma::location::id::Type = $location_id;
|
|
||||||
let db: &::sd_prisma::prisma::PrismaClient = $db;
|
|
||||||
let parent_iso_file_path: ::sd_core_file_path_helper::IsolatedFilePathData<
|
|
||||||
'static,
|
|
||||||
> = parent_iso_file_path;
|
|
||||||
let unique_location_id_materialized_path_name_extension_params: ::std::vec::Vec<
|
|
||||||
::sd_prisma::prisma::file_path::WhereParam,
|
|
||||||
> = unique_location_id_materialized_path_name_extension_params;
|
|
||||||
|
|
||||||
// FIXME: Can't pass this chunks variable direct to _batch because of lifetime issues
|
|
||||||
let chunks = unique_location_id_materialized_path_name_extension_params
|
|
||||||
.into_iter()
|
|
||||||
.chunks(200)
|
|
||||||
.into_iter()
|
|
||||||
.map(|unique_params| {
|
|
||||||
db.file_path()
|
|
||||||
.find_many(vec![::prisma_client_rust::operator::or(
|
|
||||||
unique_params.collect(),
|
|
||||||
)])
|
|
||||||
.select(::sd_prisma::prisma::file_path::select!({ id }))
|
|
||||||
})
|
|
||||||
.collect::<::std::vec::Vec<_>>();
|
|
||||||
|
|
||||||
let founds_ids = db._batch(chunks).await.map(|founds_chunk| {
|
|
||||||
founds_chunk
|
|
||||||
.into_iter()
|
|
||||||
.map(|file_paths| file_paths.into_iter().map(|file_path| file_path.id))
|
|
||||||
.flatten()
|
|
||||||
.collect::<::std::collections::HashSet<_>>()
|
|
||||||
})?;
|
|
||||||
|
|
||||||
// NOTE: This batch size can be increased if we wish to trade memory for more performance
|
|
||||||
const BATCH_SIZE: i64 = 1000;
|
|
||||||
|
|
||||||
let mut to_remove = vec![];
|
|
||||||
let mut cursor = 1;
|
|
||||||
|
|
||||||
loop {
|
|
||||||
let found = $db.file_path()
|
|
||||||
.find_many(vec![
|
|
||||||
::sd_prisma::prisma::file_path::location_id::equals(Some(location_id)),
|
|
||||||
::sd_prisma::prisma::file_path::materialized_path::equals(Some(
|
|
||||||
parent_iso_file_path
|
|
||||||
.materialized_path_for_children()
|
|
||||||
.expect("the received isolated file path must be from a directory"),
|
|
||||||
)),
|
|
||||||
])
|
|
||||||
.order_by(::sd_prisma::prisma::file_path::id::order(::sd_prisma::prisma::SortOrder::Asc))
|
|
||||||
.take(BATCH_SIZE)
|
|
||||||
.cursor(::sd_prisma::prisma::file_path::id::equals(cursor))
|
|
||||||
.select(::sd_prisma::prisma::file_path::select!({ id pub_id cas_id }))
|
|
||||||
.exec()
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
let should_stop = (found.len() as i64) < BATCH_SIZE;
|
|
||||||
|
|
||||||
if let Some(last) = found.last() {
|
|
||||||
cursor = last.id;
|
|
||||||
} else {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
|
|
||||||
to_remove.extend(
|
|
||||||
found
|
|
||||||
.into_iter()
|
|
||||||
.filter(|file_path| !founds_ids.contains(&file_path.id))
|
|
||||||
.map(|file_path| ::sd_core_prisma_helpers::file_path_pub_and_cas_ids::Data {
|
|
||||||
id: file_path.id,
|
|
||||||
pub_id: file_path.pub_id,
|
|
||||||
cas_id: file_path.cas_id,
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
|
|
||||||
if should_stop {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(to_remove)
|
|
||||||
}
|
|
||||||
}};
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn reverse_update_directories_sizes(
|
|
||||||
base_path: impl AsRef<Path>,
|
|
||||||
location_id: location::id::Type,
|
|
||||||
location_path: impl AsRef<Path>,
|
|
||||||
library: &Library,
|
|
||||||
) -> Result<(), FilePathError> {
|
|
||||||
let base_path = base_path.as_ref();
|
|
||||||
let location_path = location_path.as_ref();
|
|
||||||
|
|
||||||
let Library { sync, db, .. } = library;
|
|
||||||
|
|
||||||
let ancestors = base_path
|
|
||||||
.ancestors()
|
|
||||||
.take_while(|&ancestor| ancestor != location_path)
|
|
||||||
.map(|ancestor| IsolatedFilePathData::new(location_id, location_path, ancestor, true))
|
|
||||||
.collect::<Result<Vec<_>, _>>()?;
|
|
||||||
|
|
||||||
let chunked_queries = ancestors
|
|
||||||
.iter()
|
|
||||||
.chunks(200)
|
|
||||||
.into_iter()
|
|
||||||
.map(|ancestors_iso_file_paths_chunk| {
|
|
||||||
db.file_path()
|
|
||||||
.find_many(vec![or(ancestors_iso_file_paths_chunk
|
|
||||||
.into_iter()
|
|
||||||
.map(file_path::WhereParam::from)
|
|
||||||
.collect::<Vec<_>>())])
|
|
||||||
.select(file_path::select!({ pub_id materialized_path name }))
|
|
||||||
})
|
|
||||||
.collect::<Vec<_>>();
|
|
||||||
|
|
||||||
let mut pub_id_by_ancestor_materialized_path = db
|
|
||||||
._batch(chunked_queries)
|
|
||||||
.await?
|
|
||||||
.into_iter()
|
|
||||||
.flatten()
|
|
||||||
.filter_map(
|
|
||||||
|file_path| match (file_path.materialized_path, file_path.name) {
|
|
||||||
(Some(materialized_path), Some(name)) => {
|
|
||||||
Some((format!("{materialized_path}{name}/"), (file_path.pub_id, 0)))
|
|
||||||
}
|
|
||||||
_ => {
|
|
||||||
warn!(
|
|
||||||
"Found a file_path missing its materialized_path or name: <pub_id='{:#?}'>",
|
|
||||||
from_bytes_to_uuid(&file_path.pub_id)
|
|
||||||
);
|
|
||||||
None
|
|
||||||
}
|
|
||||||
},
|
|
||||||
)
|
|
||||||
.collect::<HashMap<_, _>>();
|
|
||||||
|
|
||||||
db.file_path()
|
|
||||||
.find_many(vec![
|
|
||||||
file_path::location_id::equals(Some(location_id)),
|
|
||||||
file_path::materialized_path::in_vec(
|
|
||||||
ancestors
|
|
||||||
.iter()
|
|
||||||
.map(|ancestor_iso_file_path| {
|
|
||||||
ancestor_iso_file_path
|
|
||||||
.materialized_path_for_children()
|
|
||||||
.expect("each ancestor is a directory")
|
|
||||||
})
|
|
||||||
.collect(),
|
|
||||||
),
|
|
||||||
])
|
|
||||||
.select(file_path::select!({ materialized_path size_in_bytes_bytes }))
|
|
||||||
.exec()
|
|
||||||
.await?
|
|
||||||
.into_iter()
|
|
||||||
.for_each(|file_path| {
|
|
||||||
if let Some(materialized_path) = file_path.materialized_path {
|
|
||||||
if let Some((_, size)) =
|
|
||||||
pub_id_by_ancestor_materialized_path.get_mut(&materialized_path)
|
|
||||||
{
|
|
||||||
*size += file_path
|
|
||||||
.size_in_bytes_bytes
|
|
||||||
.map(|size_in_bytes_bytes| {
|
|
||||||
u64::from_be_bytes([
|
|
||||||
size_in_bytes_bytes[0],
|
|
||||||
size_in_bytes_bytes[1],
|
|
||||||
size_in_bytes_bytes[2],
|
|
||||||
size_in_bytes_bytes[3],
|
|
||||||
size_in_bytes_bytes[4],
|
|
||||||
size_in_bytes_bytes[5],
|
|
||||||
size_in_bytes_bytes[6],
|
|
||||||
size_in_bytes_bytes[7],
|
|
||||||
])
|
|
||||||
})
|
|
||||||
.unwrap_or_else(|| {
|
|
||||||
warn!("Got a directory missing its size in bytes");
|
|
||||||
0
|
|
||||||
});
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
warn!("Corrupt database possessing a file_path entry without materialized_path");
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
let to_sync_and_update = ancestors
|
|
||||||
.into_iter()
|
|
||||||
.filter_map(|ancestor_iso_file_path| {
|
|
||||||
if let Some((pub_id, size)) = pub_id_by_ancestor_materialized_path.remove(
|
|
||||||
&ancestor_iso_file_path
|
|
||||||
.materialized_path_for_children()
|
|
||||||
.expect("each ancestor is a directory"),
|
|
||||||
) {
|
|
||||||
let size_bytes = size.to_be_bytes().to_vec();
|
|
||||||
|
|
||||||
Some((
|
|
||||||
sync.shared_update(
|
|
||||||
prisma_sync::file_path::SyncId {
|
|
||||||
pub_id: pub_id.clone(),
|
|
||||||
},
|
|
||||||
file_path::size_in_bytes_bytes::NAME,
|
|
||||||
msgpack!(size_bytes.clone()),
|
|
||||||
),
|
|
||||||
db.file_path().update(
|
|
||||||
file_path::pub_id::equals(pub_id),
|
|
||||||
vec![file_path::size_in_bytes_bytes::set(Some(size_bytes))],
|
|
||||||
),
|
|
||||||
))
|
|
||||||
} else {
|
|
||||||
warn!("Got a missing ancestor for a file_path in the database, maybe we have a corruption");
|
|
||||||
None
|
|
||||||
}
|
|
||||||
})
|
|
||||||
.unzip::<_, _, Vec<_>, Vec<_>>();
|
|
||||||
|
|
||||||
sync.write_ops(db, to_sync_and_update).await?;
|
|
||||||
|
|
||||||
Ok(())
|
|
||||||
}
|
|
|
@ -1,660 +0,0 @@
|
||||||
use crate::{
|
|
||||||
file_paths_db_fetcher_fn, invalidate_query,
|
|
||||||
library::Library,
|
|
||||||
location::{location_with_indexer_rules, update_location_size, ScanState},
|
|
||||||
old_job::{
|
|
||||||
CurrentStep, JobError, JobInitOutput, JobReportUpdate, JobResult, JobRunMetadata,
|
|
||||||
JobStepOutput, StatefulJob, WorkerContext,
|
|
||||||
},
|
|
||||||
to_remove_db_fetcher_fn,
|
|
||||||
};
|
|
||||||
|
|
||||||
use sd_core_file_path_helper::{
|
|
||||||
ensure_file_path_exists, ensure_sub_path_is_directory, ensure_sub_path_is_in_location,
|
|
||||||
IsolatedFilePathData,
|
|
||||||
};
|
|
||||||
use sd_core_indexer_rules::IndexerRule;
|
|
||||||
|
|
||||||
use sd_prisma::{
|
|
||||||
prisma::{file_path, location},
|
|
||||||
prisma_sync,
|
|
||||||
};
|
|
||||||
use sd_sync::*;
|
|
||||||
use sd_utils::{db::maybe_missing, from_bytes_to_uuid, msgpack};
|
|
||||||
|
|
||||||
use std::{
|
|
||||||
collections::HashMap,
|
|
||||||
hash::{Hash, Hasher},
|
|
||||||
path::{Path, PathBuf},
|
|
||||||
sync::Arc,
|
|
||||||
time::Duration,
|
|
||||||
};
|
|
||||||
|
|
||||||
use itertools::Itertools;
|
|
||||||
use prisma_client_rust::operator::or;
|
|
||||||
use serde::{Deserialize, Serialize};
|
|
||||||
use serde_json::json;
|
|
||||||
use tokio::time::Instant;
|
|
||||||
use tracing::{debug, info, warn};
|
|
||||||
|
|
||||||
use super::{
|
|
||||||
execute_indexer_save_step, execute_indexer_update_step, iso_file_path_factory,
|
|
||||||
old_walk::{keep_walking, walk, ToWalkEntry, WalkResult},
|
|
||||||
remove_non_existing_file_paths, reverse_update_directories_sizes, IndexerError,
|
|
||||||
OldIndexerJobSaveStep, OldIndexerJobUpdateStep,
|
|
||||||
};
|
|
||||||
|
|
||||||
/// BATCH_SIZE is the number of files to index at each step, writing the chunk of files metadata in the database.
|
|
||||||
const BATCH_SIZE: usize = 1000;
|
|
||||||
|
|
||||||
/// `IndexerJobInit` receives a `location::Data` object to be indexed
|
|
||||||
/// and possibly a `sub_path` to be indexed. The `sub_path` is used when
|
|
||||||
/// we want do index just a part of a location.
|
|
||||||
#[derive(Serialize, Deserialize, Debug)]
|
|
||||||
pub struct OldIndexerJobInit {
|
|
||||||
pub location: location_with_indexer_rules::Data,
|
|
||||||
pub sub_path: Option<PathBuf>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Hash for OldIndexerJobInit {
|
|
||||||
fn hash<H: Hasher>(&self, state: &mut H) {
|
|
||||||
self.location.id.hash(state);
|
|
||||||
if let Some(ref sub_path) = self.sub_path {
|
|
||||||
sub_path.hash(state);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// `IndexerJobData` contains the state of the indexer job, which includes a `location_path` that
|
|
||||||
/// is cached and casted on `PathBuf` from `local_path` column in the `location` table. It also
|
|
||||||
/// contains some metadata for logging purposes.
|
|
||||||
#[derive(Serialize, Deserialize, Debug)]
|
|
||||||
pub struct OldIndexerJobData {
|
|
||||||
location_path: PathBuf,
|
|
||||||
indexed_path: PathBuf,
|
|
||||||
indexer_rules: Vec<IndexerRule>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Serialize, Deserialize, Default, Debug)]
|
|
||||||
pub struct OldIndexerJobRunMetadata {
|
|
||||||
db_write_time: Duration,
|
|
||||||
scan_read_time: Duration,
|
|
||||||
total_paths: u64,
|
|
||||||
total_updated_paths: u64,
|
|
||||||
total_save_steps: u64,
|
|
||||||
total_update_steps: u64,
|
|
||||||
indexed_count: u64,
|
|
||||||
updated_count: u64,
|
|
||||||
removed_count: u64,
|
|
||||||
paths_and_sizes: HashMap<PathBuf, u64>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl JobRunMetadata for OldIndexerJobRunMetadata {
|
|
||||||
fn update(&mut self, new_data: Self) {
|
|
||||||
self.db_write_time += new_data.db_write_time;
|
|
||||||
self.scan_read_time += new_data.scan_read_time;
|
|
||||||
self.total_paths += new_data.total_paths;
|
|
||||||
self.total_updated_paths += new_data.total_updated_paths;
|
|
||||||
self.total_save_steps += new_data.total_save_steps;
|
|
||||||
self.total_update_steps += new_data.total_update_steps;
|
|
||||||
self.indexed_count += new_data.indexed_count;
|
|
||||||
self.removed_count += new_data.removed_count;
|
|
||||||
|
|
||||||
for (path, size) in new_data.paths_and_sizes {
|
|
||||||
*self.paths_and_sizes.entry(path).or_default() += size;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone)]
|
|
||||||
pub enum ScanProgress {
|
|
||||||
ChunkCount(usize),
|
|
||||||
SavedChunks(usize),
|
|
||||||
UpdatedChunks(usize),
|
|
||||||
Message(String),
|
|
||||||
}
|
|
||||||
|
|
||||||
impl OldIndexerJobData {
|
|
||||||
fn on_scan_progress(ctx: &WorkerContext, progress: Vec<ScanProgress>) {
|
|
||||||
ctx.progress(
|
|
||||||
progress
|
|
||||||
.into_iter()
|
|
||||||
.map(|p| match p {
|
|
||||||
ScanProgress::ChunkCount(c) => JobReportUpdate::TaskCount(c),
|
|
||||||
ScanProgress::SavedChunks(p) | ScanProgress::UpdatedChunks(p) => {
|
|
||||||
JobReportUpdate::CompletedTaskCount(p)
|
|
||||||
}
|
|
||||||
ScanProgress::Message(m) => JobReportUpdate::Message(m),
|
|
||||||
})
|
|
||||||
.collect(),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// `IndexerJobStepInput` defines the action that should be executed in the current step
|
|
||||||
#[derive(Serialize, Deserialize, Debug)]
|
|
||||||
pub enum OldIndexerJobStepInput {
|
|
||||||
Save(OldIndexerJobSaveStep),
|
|
||||||
Walk(ToWalkEntry),
|
|
||||||
Update(OldIndexerJobUpdateStep),
|
|
||||||
}
|
|
||||||
|
|
||||||
/// A `IndexerJob` is a stateful job that walks a directory and indexes all files.
|
|
||||||
/// First it walks the directory and generates a list of files to index, chunked into
|
|
||||||
/// batches of [`BATCH_SIZE`]. Then for each chunk it write the file metadata to the database.
|
|
||||||
#[async_trait::async_trait]
|
|
||||||
impl StatefulJob for OldIndexerJobInit {
|
|
||||||
type Data = OldIndexerJobData;
|
|
||||||
type Step = OldIndexerJobStepInput;
|
|
||||||
type RunMetadata = OldIndexerJobRunMetadata;
|
|
||||||
|
|
||||||
const NAME: &'static str = "indexer";
|
|
||||||
const IS_BATCHED: bool = true;
|
|
||||||
|
|
||||||
fn target_location(&self) -> location::id::Type {
|
|
||||||
self.location.id
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Creates a vector of valid path buffers from a directory, chunked into batches of `BATCH_SIZE`.
|
|
||||||
async fn init(
|
|
||||||
&self,
|
|
||||||
ctx: &WorkerContext,
|
|
||||||
data: &mut Option<Self::Data>,
|
|
||||||
) -> Result<JobInitOutput<Self::RunMetadata, Self::Step>, JobError> {
|
|
||||||
let init = self;
|
|
||||||
let location_id = init.location.id;
|
|
||||||
let location_path = maybe_missing(&init.location.path, "location.path").map(Path::new)?;
|
|
||||||
|
|
||||||
let db = Arc::clone(&ctx.library.db);
|
|
||||||
let sync = &ctx.library.sync;
|
|
||||||
|
|
||||||
let indexer_rules = init
|
|
||||||
.location
|
|
||||||
.indexer_rules
|
|
||||||
.iter()
|
|
||||||
.map(|rule| IndexerRule::try_from(&rule.indexer_rule))
|
|
||||||
.collect::<Result<Vec<_>, _>>()
|
|
||||||
.map_err(IndexerError::from)?;
|
|
||||||
|
|
||||||
let to_walk_path = match &init.sub_path {
|
|
||||||
Some(sub_path) if sub_path != Path::new("") => {
|
|
||||||
let full_path = ensure_sub_path_is_in_location(location_path, sub_path)
|
|
||||||
.await
|
|
||||||
.map_err(IndexerError::from)?;
|
|
||||||
ensure_sub_path_is_directory(location_path, sub_path)
|
|
||||||
.await
|
|
||||||
.map_err(IndexerError::from)?;
|
|
||||||
|
|
||||||
ensure_file_path_exists(
|
|
||||||
sub_path,
|
|
||||||
&IsolatedFilePathData::new(location_id, location_path, &full_path, true)
|
|
||||||
.map_err(IndexerError::from)?,
|
|
||||||
&db,
|
|
||||||
IndexerError::SubPathNotFound,
|
|
||||||
)
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
full_path
|
|
||||||
}
|
|
||||||
_ => location_path.to_path_buf(),
|
|
||||||
};
|
|
||||||
|
|
||||||
let scan_start = Instant::now();
|
|
||||||
let WalkResult {
|
|
||||||
walked,
|
|
||||||
to_update,
|
|
||||||
to_walk,
|
|
||||||
to_remove,
|
|
||||||
errors,
|
|
||||||
paths_and_sizes,
|
|
||||||
} = walk(
|
|
||||||
&location_path,
|
|
||||||
&to_walk_path,
|
|
||||||
&indexer_rules,
|
|
||||||
update_notifier_fn(ctx),
|
|
||||||
file_paths_db_fetcher_fn!(&db),
|
|
||||||
to_remove_db_fetcher_fn!(location_id, &db),
|
|
||||||
iso_file_path_factory(location_id, location_path),
|
|
||||||
50_000,
|
|
||||||
)
|
|
||||||
.await?;
|
|
||||||
let scan_read_time = scan_start.elapsed();
|
|
||||||
let to_remove = to_remove.collect::<Vec<_>>();
|
|
||||||
|
|
||||||
debug!(
|
|
||||||
"Walker at indexer job found {} file_paths to be removed",
|
|
||||||
to_remove.len()
|
|
||||||
);
|
|
||||||
|
|
||||||
ctx.node
|
|
||||||
.thumbnailer
|
|
||||||
.remove_indexed_cas_ids(
|
|
||||||
to_remove
|
|
||||||
.iter()
|
|
||||||
.filter_map(|file_path| file_path.cas_id.clone())
|
|
||||||
.collect::<Vec<_>>(),
|
|
||||||
ctx.library.id,
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
|
|
||||||
let db_delete_start = Instant::now();
|
|
||||||
// TODO pass these uuids to sync system
|
|
||||||
let removed_count = remove_non_existing_file_paths(to_remove, &db, sync).await?;
|
|
||||||
let db_delete_time = db_delete_start.elapsed();
|
|
||||||
|
|
||||||
let total_new_paths = &mut 0;
|
|
||||||
let total_updated_paths = &mut 0;
|
|
||||||
let to_walk_count = to_walk.len();
|
|
||||||
let to_save_chunks = &mut 0;
|
|
||||||
let to_update_chunks = &mut 0;
|
|
||||||
|
|
||||||
let steps = walked
|
|
||||||
.chunks(BATCH_SIZE)
|
|
||||||
.into_iter()
|
|
||||||
.enumerate()
|
|
||||||
.map(|(i, chunk)| {
|
|
||||||
let chunk_steps = chunk.collect::<Vec<_>>();
|
|
||||||
|
|
||||||
*total_new_paths += chunk_steps.len() as u64;
|
|
||||||
*to_save_chunks += 1;
|
|
||||||
|
|
||||||
OldIndexerJobStepInput::Save(OldIndexerJobSaveStep {
|
|
||||||
chunk_idx: i,
|
|
||||||
walked: chunk_steps,
|
|
||||||
})
|
|
||||||
})
|
|
||||||
.chain(
|
|
||||||
to_update
|
|
||||||
.chunks(BATCH_SIZE)
|
|
||||||
.into_iter()
|
|
||||||
.enumerate()
|
|
||||||
.map(|(i, chunk)| {
|
|
||||||
let chunk_updates = chunk.collect::<Vec<_>>();
|
|
||||||
|
|
||||||
*total_updated_paths += chunk_updates.len() as u64;
|
|
||||||
*to_update_chunks += 1;
|
|
||||||
|
|
||||||
OldIndexerJobStepInput::Update(OldIndexerJobUpdateStep {
|
|
||||||
chunk_idx: i,
|
|
||||||
to_update: chunk_updates,
|
|
||||||
})
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
.chain(to_walk.into_iter().map(OldIndexerJobStepInput::Walk))
|
|
||||||
.collect::<Vec<_>>();
|
|
||||||
|
|
||||||
debug!("Walker at indexer job found {total_updated_paths} file_paths to be updated");
|
|
||||||
|
|
||||||
OldIndexerJobData::on_scan_progress(
|
|
||||||
ctx,
|
|
||||||
vec![
|
|
||||||
ScanProgress::ChunkCount(*to_save_chunks + *to_update_chunks),
|
|
||||||
ScanProgress::Message(format!(
|
|
||||||
"Starting saving {total_new_paths} files or directories, \
|
|
||||||
{total_updated_paths} files or directories to update, \
|
|
||||||
there still {to_walk_count} directories to index",
|
|
||||||
)),
|
|
||||||
],
|
|
||||||
);
|
|
||||||
|
|
||||||
*data = Some(OldIndexerJobData {
|
|
||||||
location_path: location_path.to_path_buf(),
|
|
||||||
indexed_path: to_walk_path,
|
|
||||||
indexer_rules,
|
|
||||||
});
|
|
||||||
|
|
||||||
Ok((
|
|
||||||
OldIndexerJobRunMetadata {
|
|
||||||
db_write_time: db_delete_time,
|
|
||||||
scan_read_time,
|
|
||||||
total_paths: *total_new_paths,
|
|
||||||
total_updated_paths: *total_updated_paths,
|
|
||||||
indexed_count: 0,
|
|
||||||
updated_count: 0,
|
|
||||||
removed_count,
|
|
||||||
total_save_steps: *to_save_chunks as u64,
|
|
||||||
total_update_steps: *to_update_chunks as u64,
|
|
||||||
paths_and_sizes,
|
|
||||||
},
|
|
||||||
steps,
|
|
||||||
errors
|
|
||||||
.into_iter()
|
|
||||||
.map(|e| format!("{e}"))
|
|
||||||
.collect::<Vec<_>>()
|
|
||||||
.into(),
|
|
||||||
)
|
|
||||||
.into())
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Process each chunk of entries in the indexer job, writing to the `file_path` table
|
|
||||||
async fn execute_step(
|
|
||||||
&self,
|
|
||||||
ctx: &WorkerContext,
|
|
||||||
CurrentStep { step, .. }: CurrentStep<'_, Self::Step>,
|
|
||||||
data: &Self::Data,
|
|
||||||
run_metadata: &Self::RunMetadata,
|
|
||||||
) -> Result<JobStepOutput<Self::Step, Self::RunMetadata>, JobError> {
|
|
||||||
let init = self;
|
|
||||||
let mut new_metadata = Self::RunMetadata::default();
|
|
||||||
match step {
|
|
||||||
OldIndexerJobStepInput::Save(step) => {
|
|
||||||
let start_time = Instant::now();
|
|
||||||
|
|
||||||
OldIndexerJobData::on_scan_progress(
|
|
||||||
ctx,
|
|
||||||
vec![
|
|
||||||
ScanProgress::SavedChunks(step.chunk_idx + 1),
|
|
||||||
ScanProgress::Message(format!(
|
|
||||||
"Writing chunk {} of {} to database",
|
|
||||||
step.chunk_idx, run_metadata.total_save_steps
|
|
||||||
)),
|
|
||||||
],
|
|
||||||
);
|
|
||||||
|
|
||||||
let count = execute_indexer_save_step(&init.location, step, &ctx.library).await?;
|
|
||||||
|
|
||||||
new_metadata.indexed_count = count as u64;
|
|
||||||
new_metadata.db_write_time = start_time.elapsed();
|
|
||||||
|
|
||||||
Ok(new_metadata.into())
|
|
||||||
}
|
|
||||||
OldIndexerJobStepInput::Update(to_update) => {
|
|
||||||
let start_time = Instant::now();
|
|
||||||
OldIndexerJobData::on_scan_progress(
|
|
||||||
ctx,
|
|
||||||
vec![
|
|
||||||
ScanProgress::UpdatedChunks(to_update.chunk_idx + 1),
|
|
||||||
ScanProgress::Message(format!(
|
|
||||||
"Updating chunk {} of {} to database",
|
|
||||||
to_update.chunk_idx, run_metadata.total_save_steps
|
|
||||||
)),
|
|
||||||
],
|
|
||||||
);
|
|
||||||
|
|
||||||
let count = execute_indexer_update_step(to_update, &ctx.library).await?;
|
|
||||||
|
|
||||||
new_metadata.updated_count = count as u64;
|
|
||||||
new_metadata.db_write_time = start_time.elapsed();
|
|
||||||
|
|
||||||
Ok(new_metadata.into())
|
|
||||||
}
|
|
||||||
|
|
||||||
OldIndexerJobStepInput::Walk(to_walk_entry) => {
|
|
||||||
let location_id = init.location.id;
|
|
||||||
let location_path =
|
|
||||||
maybe_missing(&init.location.path, "location.path").map(Path::new)?;
|
|
||||||
|
|
||||||
let db = Arc::clone(&ctx.library.db);
|
|
||||||
let sync = &ctx.library.sync;
|
|
||||||
|
|
||||||
let scan_start = Instant::now();
|
|
||||||
|
|
||||||
let WalkResult {
|
|
||||||
walked,
|
|
||||||
to_update,
|
|
||||||
to_walk,
|
|
||||||
to_remove,
|
|
||||||
errors,
|
|
||||||
paths_and_sizes,
|
|
||||||
} = keep_walking(
|
|
||||||
location_path,
|
|
||||||
to_walk_entry,
|
|
||||||
&data.indexer_rules,
|
|
||||||
update_notifier_fn(ctx),
|
|
||||||
file_paths_db_fetcher_fn!(&db),
|
|
||||||
to_remove_db_fetcher_fn!(location_id, &db),
|
|
||||||
iso_file_path_factory(location_id, location_path),
|
|
||||||
)
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
new_metadata.paths_and_sizes = paths_and_sizes;
|
|
||||||
|
|
||||||
new_metadata.scan_read_time = scan_start.elapsed();
|
|
||||||
|
|
||||||
let db_delete_time = Instant::now();
|
|
||||||
// TODO pass these uuids to sync system
|
|
||||||
new_metadata.removed_count =
|
|
||||||
remove_non_existing_file_paths(to_remove, &db, sync).await?;
|
|
||||||
new_metadata.db_write_time = db_delete_time.elapsed();
|
|
||||||
|
|
||||||
let to_walk_count = to_walk.len();
|
|
||||||
|
|
||||||
let more_steps = walked
|
|
||||||
.chunks(BATCH_SIZE)
|
|
||||||
.into_iter()
|
|
||||||
.enumerate()
|
|
||||||
.map(|(i, chunk)| {
|
|
||||||
let chunk_steps = chunk.collect::<Vec<_>>();
|
|
||||||
new_metadata.total_paths += chunk_steps.len() as u64;
|
|
||||||
new_metadata.total_save_steps += 1;
|
|
||||||
|
|
||||||
OldIndexerJobStepInput::Save(OldIndexerJobSaveStep {
|
|
||||||
chunk_idx: i,
|
|
||||||
walked: chunk_steps,
|
|
||||||
})
|
|
||||||
})
|
|
||||||
.chain(to_update.chunks(BATCH_SIZE).into_iter().enumerate().map(
|
|
||||||
|(i, chunk)| {
|
|
||||||
let chunk_updates = chunk.collect::<Vec<_>>();
|
|
||||||
new_metadata.total_updated_paths += chunk_updates.len() as u64;
|
|
||||||
new_metadata.total_update_steps += 1;
|
|
||||||
|
|
||||||
OldIndexerJobStepInput::Update(OldIndexerJobUpdateStep {
|
|
||||||
chunk_idx: i,
|
|
||||||
to_update: chunk_updates,
|
|
||||||
})
|
|
||||||
},
|
|
||||||
))
|
|
||||||
.chain(to_walk.into_iter().map(OldIndexerJobStepInput::Walk))
|
|
||||||
.collect::<Vec<_>>();
|
|
||||||
|
|
||||||
OldIndexerJobData::on_scan_progress(
|
|
||||||
ctx,
|
|
||||||
vec![
|
|
||||||
ScanProgress::ChunkCount(more_steps.len() - to_walk_count),
|
|
||||||
ScanProgress::Message(format!(
|
|
||||||
"Scanned {} more files or directories; \
|
|
||||||
{} more directories to scan and {} more entries to update",
|
|
||||||
new_metadata.total_paths,
|
|
||||||
to_walk_count,
|
|
||||||
new_metadata.total_updated_paths
|
|
||||||
)),
|
|
||||||
],
|
|
||||||
);
|
|
||||||
|
|
||||||
Ok((
|
|
||||||
more_steps,
|
|
||||||
new_metadata,
|
|
||||||
errors
|
|
||||||
.into_iter()
|
|
||||||
.map(|e| format!("{e}"))
|
|
||||||
.collect::<Vec<_>>()
|
|
||||||
.into(),
|
|
||||||
)
|
|
||||||
.into())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn finalize(
|
|
||||||
&self,
|
|
||||||
ctx: &WorkerContext,
|
|
||||||
data: &Option<Self::Data>,
|
|
||||||
run_metadata: &Self::RunMetadata,
|
|
||||||
) -> JobResult {
|
|
||||||
let init = self;
|
|
||||||
let indexed_path_str = data
|
|
||||||
.as_ref()
|
|
||||||
.map(|data| Ok(data.indexed_path.to_string_lossy().to_string()))
|
|
||||||
.unwrap_or_else(|| maybe_missing(&init.location.path, "location.path").cloned())?;
|
|
||||||
|
|
||||||
info!(
|
|
||||||
"Scan of {indexed_path_str} completed in {:?}. {} new files found, \
|
|
||||||
indexed {} files in db, updated {} entries. db write completed in {:?}",
|
|
||||||
run_metadata.scan_read_time,
|
|
||||||
run_metadata.total_paths,
|
|
||||||
run_metadata.indexed_count,
|
|
||||||
run_metadata.total_updated_paths,
|
|
||||||
run_metadata.db_write_time,
|
|
||||||
);
|
|
||||||
|
|
||||||
if run_metadata.indexed_count > 0 || run_metadata.removed_count > 0 {
|
|
||||||
invalidate_query!(ctx.library, "search.paths");
|
|
||||||
}
|
|
||||||
|
|
||||||
if run_metadata.total_updated_paths > 0 {
|
|
||||||
// Invoking orphan remover here as we probably have some orphans objects due to updates
|
|
||||||
// ctx.library.orphan_remover.invoke().await;
|
|
||||||
}
|
|
||||||
|
|
||||||
if run_metadata.indexed_count > 0
|
|
||||||
|| run_metadata.removed_count > 0
|
|
||||||
|| run_metadata.updated_count > 0
|
|
||||||
{
|
|
||||||
if let Some(data) = data {
|
|
||||||
update_directories_sizes(
|
|
||||||
&run_metadata.paths_and_sizes,
|
|
||||||
init.location.id,
|
|
||||||
&data.indexed_path,
|
|
||||||
&ctx.library,
|
|
||||||
)
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
if data.indexed_path != data.location_path {
|
|
||||||
reverse_update_directories_sizes(
|
|
||||||
&data.indexed_path,
|
|
||||||
init.location.id,
|
|
||||||
&data.location_path,
|
|
||||||
&ctx.library,
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.map_err(IndexerError::from)?;
|
|
||||||
}
|
|
||||||
|
|
||||||
update_location_size(init.location.id, &ctx.library)
|
|
||||||
.await
|
|
||||||
.map_err(IndexerError::from)?;
|
|
||||||
|
|
||||||
ctx.library
|
|
||||||
.db
|
|
||||||
.location()
|
|
||||||
.update(
|
|
||||||
location::id::equals(init.location.id),
|
|
||||||
vec![location::scan_state::set(ScanState::Indexed as i32)],
|
|
||||||
)
|
|
||||||
.exec()
|
|
||||||
.await
|
|
||||||
.map_err(IndexerError::from)?;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// FIXME(fogodev): This is currently a workaround to don't save paths and sizes in the
|
|
||||||
// metadata after a job is completed, as it's pretty heavy. A proper fix isn't needed
|
|
||||||
// right now as I already changed it in the new indexer job. And this old one
|
|
||||||
// will be removed eventually.
|
|
||||||
let run_metadata = Self::RunMetadata {
|
|
||||||
db_write_time: run_metadata.db_write_time,
|
|
||||||
scan_read_time: run_metadata.scan_read_time,
|
|
||||||
total_paths: run_metadata.total_paths,
|
|
||||||
total_updated_paths: run_metadata.total_updated_paths,
|
|
||||||
total_save_steps: run_metadata.total_save_steps,
|
|
||||||
total_update_steps: run_metadata.total_update_steps,
|
|
||||||
indexed_count: run_metadata.indexed_count,
|
|
||||||
updated_count: run_metadata.updated_count,
|
|
||||||
removed_count: run_metadata.removed_count,
|
|
||||||
paths_and_sizes: HashMap::new(),
|
|
||||||
};
|
|
||||||
|
|
||||||
Ok(Some(json!({"init: ": init, "run_metadata": run_metadata})))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn update_notifier_fn(ctx: &WorkerContext) -> impl FnMut(&Path, usize) + '_ {
|
|
||||||
move |path, total_entries| {
|
|
||||||
OldIndexerJobData::on_scan_progress(
|
|
||||||
ctx,
|
|
||||||
vec![ScanProgress::Message(format!(
|
|
||||||
"{total_entries} entries found at {}",
|
|
||||||
path.display()
|
|
||||||
))],
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn update_directories_sizes(
|
|
||||||
paths_and_sizes: &HashMap<PathBuf, u64>,
|
|
||||||
location_id: location::id::Type,
|
|
||||||
location_path: impl AsRef<Path>,
|
|
||||||
library: &Library,
|
|
||||||
) -> Result<(), IndexerError> {
|
|
||||||
let location_path = location_path.as_ref();
|
|
||||||
|
|
||||||
let Library { db, sync, .. } = library;
|
|
||||||
|
|
||||||
let chunked_queries = paths_and_sizes
|
|
||||||
.keys()
|
|
||||||
.chunks(200)
|
|
||||||
.into_iter()
|
|
||||||
.map(|paths_chunk| {
|
|
||||||
paths_chunk
|
|
||||||
.into_iter()
|
|
||||||
.map(|path| {
|
|
||||||
IsolatedFilePathData::new(location_id, location_path, path, true)
|
|
||||||
.map(file_path::WhereParam::from)
|
|
||||||
})
|
|
||||||
.collect::<Result<Vec<_>, _>>()
|
|
||||||
.map(|params| {
|
|
||||||
db.file_path()
|
|
||||||
.find_many(vec![or(params)])
|
|
||||||
.select(file_path::select!({ pub_id materialized_path name }))
|
|
||||||
})
|
|
||||||
})
|
|
||||||
.collect::<Result<Vec<_>, _>>()?;
|
|
||||||
|
|
||||||
let to_sync_and_update = db
|
|
||||||
._batch(chunked_queries)
|
|
||||||
.await?
|
|
||||||
.into_iter()
|
|
||||||
.flatten()
|
|
||||||
.filter_map(
|
|
||||||
|file_path| match (file_path.materialized_path, file_path.name) {
|
|
||||||
(Some(materialized_path), Some(name)) => {
|
|
||||||
let mut directory_full_path = location_path.join(&materialized_path[1..]);
|
|
||||||
directory_full_path.push(name);
|
|
||||||
|
|
||||||
if let Some(size) = paths_and_sizes.get(&directory_full_path) {
|
|
||||||
let size_bytes = size.to_be_bytes().to_vec();
|
|
||||||
|
|
||||||
Some((
|
|
||||||
sync.shared_update(
|
|
||||||
prisma_sync::file_path::SyncId {
|
|
||||||
pub_id: file_path.pub_id.clone(),
|
|
||||||
},
|
|
||||||
file_path::size_in_bytes_bytes::NAME,
|
|
||||||
msgpack!(size_bytes.clone()),
|
|
||||||
),
|
|
||||||
db.file_path().update(
|
|
||||||
file_path::pub_id::equals(file_path.pub_id),
|
|
||||||
vec![file_path::size_in_bytes_bytes::set(Some(size_bytes))],
|
|
||||||
),
|
|
||||||
))
|
|
||||||
} else {
|
|
||||||
warn!("Found a file_path without ancestor in the database, possible corruption");
|
|
||||||
None
|
|
||||||
}
|
|
||||||
}
|
|
||||||
_ => {
|
|
||||||
warn!(
|
|
||||||
"Found a file_path missing its materialized_path or name: <pub_id='{:#?}'>",
|
|
||||||
from_bytes_to_uuid(&file_path.pub_id)
|
|
||||||
);
|
|
||||||
None
|
|
||||||
}
|
|
||||||
},
|
|
||||||
)
|
|
||||||
.unzip::<_, _, Vec<_>, Vec<_>>();
|
|
||||||
|
|
||||||
sync.write_ops(db, to_sync_and_update).await?;
|
|
||||||
|
|
||||||
Ok(())
|
|
||||||
}
|
|
|
@ -1,197 +0,0 @@
|
||||||
use crate::{
|
|
||||||
file_paths_db_fetcher_fn, invalidate_query,
|
|
||||||
library::Library,
|
|
||||||
location::{
|
|
||||||
indexer::{
|
|
||||||
execute_indexer_update_step, reverse_update_directories_sizes, OldIndexerJobUpdateStep,
|
|
||||||
},
|
|
||||||
scan_location_sub_path, update_location_size,
|
|
||||||
},
|
|
||||||
old_job::JobError,
|
|
||||||
to_remove_db_fetcher_fn, Node,
|
|
||||||
};
|
|
||||||
|
|
||||||
use sd_core_file_path_helper::{
|
|
||||||
check_file_path_exists, ensure_sub_path_is_directory, ensure_sub_path_is_in_location,
|
|
||||||
IsolatedFilePathData,
|
|
||||||
};
|
|
||||||
use sd_core_indexer_rules::IndexerRule;
|
|
||||||
|
|
||||||
use sd_utils::db::maybe_missing;
|
|
||||||
|
|
||||||
use std::{
|
|
||||||
collections::HashSet,
|
|
||||||
path::{Path, PathBuf},
|
|
||||||
sync::Arc,
|
|
||||||
};
|
|
||||||
|
|
||||||
use futures::future::join_all;
|
|
||||||
use itertools::Itertools;
|
|
||||||
use tracing::{debug, error};
|
|
||||||
|
|
||||||
use super::{
|
|
||||||
execute_indexer_save_step, iso_file_path_factory, location_with_indexer_rules,
|
|
||||||
old_walk::walk_single_dir, remove_non_existing_file_paths, IndexerError, OldIndexerJobSaveStep,
|
|
||||||
};
|
|
||||||
|
|
||||||
/// BATCH_SIZE is the number of files to index at each step, writing the chunk of files metadata in the database.
|
|
||||||
const BATCH_SIZE: usize = 1000;
|
|
||||||
|
|
||||||
pub async fn old_shallow(
|
|
||||||
location: &location_with_indexer_rules::Data,
|
|
||||||
sub_path: &PathBuf,
|
|
||||||
node: &Arc<Node>,
|
|
||||||
library: &Arc<Library>,
|
|
||||||
) -> Result<(), JobError> {
|
|
||||||
let location_id = location.id;
|
|
||||||
let location_path = maybe_missing(&location.path, "location.path").map(Path::new)?;
|
|
||||||
|
|
||||||
let db = library.db.clone();
|
|
||||||
let sync = &library.sync;
|
|
||||||
|
|
||||||
let indexer_rules = location
|
|
||||||
.indexer_rules
|
|
||||||
.iter()
|
|
||||||
.map(|rule| IndexerRule::try_from(&rule.indexer_rule))
|
|
||||||
.collect::<Result<Vec<_>, _>>()
|
|
||||||
.map_err(IndexerError::from)?;
|
|
||||||
|
|
||||||
let (add_root, to_walk_path) = if sub_path != Path::new("") && sub_path != Path::new("/") {
|
|
||||||
let full_path = ensure_sub_path_is_in_location(&location_path, &sub_path)
|
|
||||||
.await
|
|
||||||
.map_err(IndexerError::from)?;
|
|
||||||
ensure_sub_path_is_directory(&location_path, &sub_path)
|
|
||||||
.await
|
|
||||||
.map_err(IndexerError::from)?;
|
|
||||||
|
|
||||||
(
|
|
||||||
!check_file_path_exists::<IndexerError>(
|
|
||||||
&IsolatedFilePathData::new(location_id, location_path, &full_path, true)
|
|
||||||
.map_err(IndexerError::from)?,
|
|
||||||
&db,
|
|
||||||
)
|
|
||||||
.await?,
|
|
||||||
full_path,
|
|
||||||
)
|
|
||||||
} else {
|
|
||||||
(false, location_path.to_path_buf())
|
|
||||||
};
|
|
||||||
|
|
||||||
let (walked, to_update, to_remove, errors, _s) = {
|
|
||||||
walk_single_dir(
|
|
||||||
location_path,
|
|
||||||
&to_walk_path,
|
|
||||||
&indexer_rules,
|
|
||||||
file_paths_db_fetcher_fn!(&db),
|
|
||||||
to_remove_db_fetcher_fn!(location_id, &db),
|
|
||||||
iso_file_path_factory(location_id, location_path),
|
|
||||||
add_root,
|
|
||||||
)
|
|
||||||
.await?
|
|
||||||
};
|
|
||||||
|
|
||||||
let to_remove_count = to_remove.len();
|
|
||||||
|
|
||||||
node.thumbnailer
|
|
||||||
.remove_indexed_cas_ids(
|
|
||||||
to_remove
|
|
||||||
.iter()
|
|
||||||
.filter_map(|file_path| file_path.cas_id.clone())
|
|
||||||
.collect::<Vec<_>>(),
|
|
||||||
library.id,
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
|
|
||||||
errors.into_iter().for_each(|e| error!("{e}"));
|
|
||||||
|
|
||||||
remove_non_existing_file_paths(to_remove, &db, sync).await?;
|
|
||||||
|
|
||||||
let mut new_directories_to_scan = HashSet::new();
|
|
||||||
|
|
||||||
let mut to_create_count = 0;
|
|
||||||
|
|
||||||
let save_steps = walked
|
|
||||||
.chunks(BATCH_SIZE)
|
|
||||||
.into_iter()
|
|
||||||
.enumerate()
|
|
||||||
.map(|(i, chunk)| {
|
|
||||||
let walked = chunk.collect::<Vec<_>>();
|
|
||||||
to_create_count += walked.len();
|
|
||||||
|
|
||||||
walked
|
|
||||||
.iter()
|
|
||||||
.filter_map(|walked_entry| {
|
|
||||||
walked_entry.iso_file_path.materialized_path_for_children()
|
|
||||||
})
|
|
||||||
.for_each(|new_dir| {
|
|
||||||
new_directories_to_scan.insert(new_dir);
|
|
||||||
});
|
|
||||||
|
|
||||||
OldIndexerJobSaveStep {
|
|
||||||
chunk_idx: i,
|
|
||||||
walked,
|
|
||||||
}
|
|
||||||
})
|
|
||||||
.collect::<Vec<_>>();
|
|
||||||
|
|
||||||
for step in save_steps {
|
|
||||||
execute_indexer_save_step(location, &step, library).await?;
|
|
||||||
}
|
|
||||||
|
|
||||||
for scan in join_all(
|
|
||||||
new_directories_to_scan
|
|
||||||
.into_iter()
|
|
||||||
.map(|sub_path| scan_location_sub_path(node, library, location.clone(), sub_path)),
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
{
|
|
||||||
if let Err(e) = scan {
|
|
||||||
error!("{e}");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
let mut to_update_count = 0;
|
|
||||||
|
|
||||||
let update_steps = to_update
|
|
||||||
.chunks(BATCH_SIZE)
|
|
||||||
.into_iter()
|
|
||||||
.enumerate()
|
|
||||||
.map(|(i, chunk)| {
|
|
||||||
let to_update = chunk.collect::<Vec<_>>();
|
|
||||||
to_update_count += to_update.len();
|
|
||||||
|
|
||||||
OldIndexerJobUpdateStep {
|
|
||||||
chunk_idx: i,
|
|
||||||
to_update,
|
|
||||||
}
|
|
||||||
})
|
|
||||||
.collect::<Vec<_>>();
|
|
||||||
|
|
||||||
for step in update_steps {
|
|
||||||
execute_indexer_update_step(&step, library).await?;
|
|
||||||
}
|
|
||||||
|
|
||||||
debug!(
|
|
||||||
"Walker at shallow indexer found: \
|
|
||||||
To create: {to_create_count}; To update: {to_update_count}; To remove: {to_remove_count};"
|
|
||||||
);
|
|
||||||
|
|
||||||
if to_create_count > 0 || to_update_count > 0 || to_remove_count > 0 {
|
|
||||||
if to_walk_path != location_path {
|
|
||||||
reverse_update_directories_sizes(to_walk_path, location_id, location_path, library)
|
|
||||||
.await
|
|
||||||
.map_err(IndexerError::from)?;
|
|
||||||
}
|
|
||||||
|
|
||||||
update_location_size(location.id, library)
|
|
||||||
.await
|
|
||||||
.map_err(IndexerError::from)?;
|
|
||||||
|
|
||||||
invalidate_query!(library, "search.paths");
|
|
||||||
invalidate_query!(library, "search.objects");
|
|
||||||
}
|
|
||||||
|
|
||||||
// library.orphan_remover.invoke().await;
|
|
||||||
|
|
||||||
Ok(())
|
|
||||||
}
|
|
File diff suppressed because it is too large
Load diff
|
@ -1,281 +0,0 @@
|
||||||
use crate::{
|
|
||||||
library::{Library, LibraryId},
|
|
||||||
Node,
|
|
||||||
};
|
|
||||||
|
|
||||||
use sd_prisma::prisma::location;
|
|
||||||
use sd_utils::db::maybe_missing;
|
|
||||||
|
|
||||||
use std::{
|
|
||||||
collections::{HashMap, HashSet},
|
|
||||||
path::{Path, PathBuf},
|
|
||||||
sync::Arc,
|
|
||||||
time::Duration,
|
|
||||||
};
|
|
||||||
|
|
||||||
use tokio::{fs, io::ErrorKind, sync::oneshot, time::sleep};
|
|
||||||
use tracing::{error, warn};
|
|
||||||
use uuid::Uuid;
|
|
||||||
|
|
||||||
use super::{watcher::LocationWatcher, LocationManagerError};
|
|
||||||
|
|
||||||
type LocationAndLibraryKey = (location::id::Type, LibraryId);
|
|
||||||
|
|
||||||
const LOCATION_CHECK_INTERVAL: Duration = Duration::from_secs(5);
|
|
||||||
|
|
||||||
pub(super) async fn check_online(
|
|
||||||
location: &location::Data,
|
|
||||||
node: &Node,
|
|
||||||
library: &Library,
|
|
||||||
) -> Result<bool, LocationManagerError> {
|
|
||||||
let pub_id = Uuid::from_slice(&location.pub_id)?;
|
|
||||||
|
|
||||||
let location_path = maybe_missing(&location.path, "location.path").map(Path::new)?;
|
|
||||||
|
|
||||||
// TODO(N): This isn't gonna work with removable media and this will likely permanently break if the DB is restored from a backup.
|
|
||||||
if location.instance_id == Some(library.config().await.instance_id) {
|
|
||||||
match fs::metadata(&location_path).await {
|
|
||||||
Ok(_) => {
|
|
||||||
node.locations.add_online(pub_id).await;
|
|
||||||
Ok(true)
|
|
||||||
}
|
|
||||||
Err(e) if e.kind() == ErrorKind::NotFound => {
|
|
||||||
node.locations.remove_online(&pub_id).await;
|
|
||||||
Ok(false)
|
|
||||||
}
|
|
||||||
Err(e) => {
|
|
||||||
error!("Failed to check if location is online: {:#?}", e);
|
|
||||||
Ok(false)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
// In this case, we don't have a `local_path`, but this location was marked as online
|
|
||||||
node.locations.remove_online(&pub_id).await;
|
|
||||||
Err(LocationManagerError::NonLocalLocation(location.id))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(super) async fn location_check_sleep(
|
|
||||||
location_id: location::id::Type,
|
|
||||||
library: Arc<Library>,
|
|
||||||
) -> (location::id::Type, Arc<Library>) {
|
|
||||||
sleep(LOCATION_CHECK_INTERVAL).await;
|
|
||||||
(location_id, library)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(super) fn watch_location(
|
|
||||||
location: location::Data,
|
|
||||||
library_id: LibraryId,
|
|
||||||
locations_watched: &mut HashMap<LocationAndLibraryKey, LocationWatcher>,
|
|
||||||
locations_unwatched: &mut HashMap<LocationAndLibraryKey, LocationWatcher>,
|
|
||||||
) {
|
|
||||||
let location_id = location.id;
|
|
||||||
let location_path = location.path.as_ref();
|
|
||||||
let Some(location_path) = location_path.map(Path::new) else {
|
|
||||||
return;
|
|
||||||
};
|
|
||||||
|
|
||||||
if let Some(mut watcher) = locations_unwatched.remove(&(location_id, library_id)) {
|
|
||||||
if watcher.check_path(location_path) {
|
|
||||||
watcher.watch();
|
|
||||||
}
|
|
||||||
|
|
||||||
locations_watched.insert((location_id, library_id), watcher);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(super) fn unwatch_location(
|
|
||||||
location: location::Data,
|
|
||||||
library_id: LibraryId,
|
|
||||||
locations_watched: &mut HashMap<LocationAndLibraryKey, LocationWatcher>,
|
|
||||||
locations_unwatched: &mut HashMap<LocationAndLibraryKey, LocationWatcher>,
|
|
||||||
) {
|
|
||||||
let location_id = location.id;
|
|
||||||
let location_path = location.path.as_ref();
|
|
||||||
let Some(location_path) = location_path.map(Path::new) else {
|
|
||||||
return;
|
|
||||||
};
|
|
||||||
|
|
||||||
if let Some(mut watcher) = locations_watched.remove(&(location_id, library_id)) {
|
|
||||||
if watcher.check_path(location_path) {
|
|
||||||
watcher.unwatch();
|
|
||||||
}
|
|
||||||
|
|
||||||
locations_unwatched.insert((location_id, library_id), watcher);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(super) fn drop_location(
|
|
||||||
location_id: location::id::Type,
|
|
||||||
library_id: LibraryId,
|
|
||||||
message: &str,
|
|
||||||
locations_watched: &mut HashMap<LocationAndLibraryKey, LocationWatcher>,
|
|
||||||
locations_unwatched: &mut HashMap<LocationAndLibraryKey, LocationWatcher>,
|
|
||||||
) {
|
|
||||||
warn!("{message}: <id='{location_id}', library_id='{library_id}'>",);
|
|
||||||
if let Some(mut watcher) = locations_watched.remove(&(location_id, library_id)) {
|
|
||||||
watcher.unwatch();
|
|
||||||
} else {
|
|
||||||
locations_unwatched.remove(&(location_id, library_id));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(super) async fn get_location(
|
|
||||||
location_id: location::id::Type,
|
|
||||||
library: &Library,
|
|
||||||
) -> Option<location::Data> {
|
|
||||||
library
|
|
||||||
.db
|
|
||||||
.location()
|
|
||||||
.find_unique(location::id::equals(location_id))
|
|
||||||
.exec()
|
|
||||||
.await
|
|
||||||
.unwrap_or_else(|err| {
|
|
||||||
error!("Failed to get location data from location_id: {:#?}", err);
|
|
||||||
None
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(super) async fn handle_remove_location_request(
|
|
||||||
location_id: location::id::Type,
|
|
||||||
library: Arc<Library>,
|
|
||||||
response_tx: oneshot::Sender<Result<(), LocationManagerError>>,
|
|
||||||
forced_unwatch: &mut HashSet<LocationAndLibraryKey>,
|
|
||||||
locations_watched: &mut HashMap<LocationAndLibraryKey, LocationWatcher>,
|
|
||||||
locations_unwatched: &mut HashMap<LocationAndLibraryKey, LocationWatcher>,
|
|
||||||
to_remove: &mut HashSet<LocationAndLibraryKey>,
|
|
||||||
) {
|
|
||||||
let key = (location_id, library.id);
|
|
||||||
if let Some(location) = get_location(location_id, &library).await {
|
|
||||||
// TODO(N): This isn't gonna work with removable media and this will likely permanently break if the DB is restored from a backup.
|
|
||||||
if location.instance_id == Some(library.config().await.instance_id) {
|
|
||||||
unwatch_location(location, library.id, locations_watched, locations_unwatched);
|
|
||||||
locations_unwatched.remove(&key);
|
|
||||||
forced_unwatch.remove(&key);
|
|
||||||
} else {
|
|
||||||
drop_location(
|
|
||||||
location_id,
|
|
||||||
library.id,
|
|
||||||
"Dropping location from location manager, because we don't have a `local_path` anymore",
|
|
||||||
locations_watched,
|
|
||||||
locations_unwatched
|
|
||||||
);
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
drop_location(
|
|
||||||
location_id,
|
|
||||||
library.id,
|
|
||||||
"Removing location from manager, as we failed to fetch from db",
|
|
||||||
locations_watched,
|
|
||||||
locations_unwatched,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Marking location as removed, so we don't try to check it when the time comes
|
|
||||||
to_remove.insert(key);
|
|
||||||
|
|
||||||
let _ = response_tx.send(Ok(())); // ignore errors, we handle errors on receiver
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(super) async fn handle_stop_watcher_request(
|
|
||||||
location_id: location::id::Type,
|
|
||||||
library: Arc<Library>,
|
|
||||||
response_tx: oneshot::Sender<Result<(), LocationManagerError>>,
|
|
||||||
forced_unwatch: &mut HashSet<LocationAndLibraryKey>,
|
|
||||||
locations_watched: &mut HashMap<LocationAndLibraryKey, LocationWatcher>,
|
|
||||||
locations_unwatched: &mut HashMap<LocationAndLibraryKey, LocationWatcher>,
|
|
||||||
) {
|
|
||||||
async fn inner(
|
|
||||||
location_id: location::id::Type,
|
|
||||||
library: Arc<Library>,
|
|
||||||
forced_unwatch: &mut HashSet<LocationAndLibraryKey>,
|
|
||||||
locations_watched: &mut HashMap<LocationAndLibraryKey, LocationWatcher>,
|
|
||||||
locations_unwatched: &mut HashMap<LocationAndLibraryKey, LocationWatcher>,
|
|
||||||
) -> Result<(), LocationManagerError> {
|
|
||||||
let key = (location_id, library.id);
|
|
||||||
if !forced_unwatch.contains(&key) && locations_watched.contains_key(&key) {
|
|
||||||
get_location(location_id, &library)
|
|
||||||
.await
|
|
||||||
.ok_or_else(|| LocationManagerError::FailedToStopOrReinitWatcher {
|
|
||||||
reason: String::from("failed to fetch location from db"),
|
|
||||||
})
|
|
||||||
.map(|location| {
|
|
||||||
unwatch_location(location, library.id, locations_watched, locations_unwatched);
|
|
||||||
forced_unwatch.insert(key);
|
|
||||||
})
|
|
||||||
} else {
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
let _ = response_tx.send(
|
|
||||||
inner(
|
|
||||||
location_id,
|
|
||||||
library,
|
|
||||||
forced_unwatch,
|
|
||||||
locations_watched,
|
|
||||||
locations_unwatched,
|
|
||||||
)
|
|
||||||
.await,
|
|
||||||
); // ignore errors, we handle errors on receiver
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(super) async fn handle_reinit_watcher_request(
|
|
||||||
location_id: location::id::Type,
|
|
||||||
library: Arc<Library>,
|
|
||||||
response_tx: oneshot::Sender<Result<(), LocationManagerError>>,
|
|
||||||
forced_unwatch: &mut HashSet<LocationAndLibraryKey>,
|
|
||||||
locations_watched: &mut HashMap<LocationAndLibraryKey, LocationWatcher>,
|
|
||||||
locations_unwatched: &mut HashMap<LocationAndLibraryKey, LocationWatcher>,
|
|
||||||
) {
|
|
||||||
async fn inner(
|
|
||||||
location_id: location::id::Type,
|
|
||||||
library: Arc<Library>,
|
|
||||||
forced_unwatch: &mut HashSet<LocationAndLibraryKey>,
|
|
||||||
locations_watched: &mut HashMap<LocationAndLibraryKey, LocationWatcher>,
|
|
||||||
locations_unwatched: &mut HashMap<LocationAndLibraryKey, LocationWatcher>,
|
|
||||||
) -> Result<(), LocationManagerError> {
|
|
||||||
let key = (location_id, library.id);
|
|
||||||
if forced_unwatch.contains(&key) && locations_unwatched.contains_key(&key) {
|
|
||||||
get_location(location_id, &library)
|
|
||||||
.await
|
|
||||||
.ok_or_else(|| LocationManagerError::FailedToStopOrReinitWatcher {
|
|
||||||
reason: String::from("failed to fetch location from db"),
|
|
||||||
})
|
|
||||||
.map(|location| {
|
|
||||||
watch_location(location, library.id, locations_watched, locations_unwatched);
|
|
||||||
forced_unwatch.remove(&key);
|
|
||||||
})
|
|
||||||
} else {
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
let _ = response_tx.send(
|
|
||||||
inner(
|
|
||||||
location_id,
|
|
||||||
library,
|
|
||||||
forced_unwatch,
|
|
||||||
locations_watched,
|
|
||||||
locations_unwatched,
|
|
||||||
)
|
|
||||||
.await,
|
|
||||||
); // ignore errors, we handle errors on receiver
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(super) fn handle_ignore_path_request(
|
|
||||||
location_id: location::id::Type,
|
|
||||||
library: Arc<Library>,
|
|
||||||
path: PathBuf,
|
|
||||||
ignore: bool,
|
|
||||||
response_tx: oneshot::Sender<Result<(), LocationManagerError>>,
|
|
||||||
locations_watched: &HashMap<LocationAndLibraryKey, LocationWatcher>,
|
|
||||||
) {
|
|
||||||
let _ = response_tx.send(
|
|
||||||
if let Some(watcher) = locations_watched.get(&(location_id, library.id)) {
|
|
||||||
watcher.ignore_path(path, ignore)
|
|
||||||
} else {
|
|
||||||
Ok(())
|
|
||||||
},
|
|
||||||
); // ignore errors, we handle errors on receiver
|
|
||||||
}
|
|
|
@ -1,6 +1,5 @@
|
||||||
use crate::{
|
use crate::{
|
||||||
library::{Library, LibraryManagerEvent},
|
library::{Library, LibraryManagerEvent},
|
||||||
old_job::JobManagerError,
|
|
||||||
Node,
|
Node,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -15,19 +14,22 @@ use std::{
|
||||||
sync::Arc,
|
sync::Arc,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
use async_channel as chan;
|
||||||
use futures::executor::block_on;
|
use futures::executor::block_on;
|
||||||
use thiserror::Error;
|
use thiserror::Error;
|
||||||
use tokio::sync::{
|
use tokio::{
|
||||||
broadcast::{self, Receiver},
|
spawn,
|
||||||
mpsc, oneshot, RwLock,
|
sync::{
|
||||||
|
broadcast::{self, Receiver},
|
||||||
|
oneshot, RwLock,
|
||||||
|
},
|
||||||
};
|
};
|
||||||
use tracing::{debug, error};
|
use tracing::{debug, error, instrument, trace};
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
mod runner;
|
||||||
mod watcher;
|
mod watcher;
|
||||||
|
|
||||||
mod helpers;
|
|
||||||
|
|
||||||
#[derive(Clone, Copy, Debug)]
|
#[derive(Clone, Copy, Debug)]
|
||||||
enum ManagementMessageAction {
|
enum ManagementMessageAction {
|
||||||
Add,
|
Add,
|
||||||
|
@ -39,13 +41,13 @@ pub struct LocationManagementMessage {
|
||||||
location_id: location::id::Type,
|
location_id: location::id::Type,
|
||||||
library: Arc<Library>,
|
library: Arc<Library>,
|
||||||
action: ManagementMessageAction,
|
action: ManagementMessageAction,
|
||||||
response_tx: oneshot::Sender<Result<(), LocationManagerError>>,
|
ack: oneshot::Sender<Result<(), LocationManagerError>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
enum WatcherManagementMessageAction {
|
enum WatcherManagementMessageAction {
|
||||||
Stop,
|
Pause,
|
||||||
Reinit,
|
Resume,
|
||||||
IgnoreEventsForPath { path: PathBuf, ignore: bool },
|
IgnoreEventsForPath { path: PathBuf, ignore: bool },
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -54,51 +56,42 @@ pub struct WatcherManagementMessage {
|
||||||
location_id: location::id::Type,
|
location_id: location::id::Type,
|
||||||
library: Arc<Library>,
|
library: Arc<Library>,
|
||||||
action: WatcherManagementMessageAction,
|
action: WatcherManagementMessageAction,
|
||||||
response_tx: oneshot::Sender<Result<(), LocationManagerError>>,
|
ack: oneshot::Sender<Result<(), LocationManagerError>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Error, Debug)]
|
#[derive(Error, Debug)]
|
||||||
pub enum LocationManagerError {
|
pub enum LocationManagerError {
|
||||||
#[error("Unable to send location management message to location manager actor: (error: {0})")]
|
#[error("location not found in database: <id={0}>")]
|
||||||
ActorSendLocationError(#[from] mpsc::error::SendError<LocationManagementMessage>),
|
LocationNotFound(location::id::Type),
|
||||||
|
|
||||||
#[error("Unable to send path to be ignored by watcher actor: (error: {0})")]
|
#[error("watcher error: {0}")]
|
||||||
ActorIgnorePathError(#[from] mpsc::error::SendError<watcher::IgnorePath>),
|
Watcher(#[from] notify::Error),
|
||||||
|
|
||||||
#[error("Unable to watcher management message to watcher manager actor: (error: {0})")]
|
#[error("non local location: <id='{0}'>")]
|
||||||
ActorIgnorePathMessageError(#[from] mpsc::error::SendError<WatcherManagementMessage>),
|
|
||||||
|
|
||||||
#[error("Unable to receive actor response: (error: {0})")]
|
|
||||||
ActorResponseError(#[from] oneshot::error::RecvError),
|
|
||||||
|
|
||||||
#[error("Watcher error: (error: {0})")]
|
|
||||||
WatcherError(#[from] notify::Error),
|
|
||||||
|
|
||||||
#[error("Failed to stop or reinit a watcher: {reason}")]
|
|
||||||
FailedToStopOrReinitWatcher { reason: String },
|
|
||||||
|
|
||||||
#[error("Missing location from database: <id='{0}'>")]
|
|
||||||
MissingLocation(location::id::Type),
|
|
||||||
|
|
||||||
#[error("Non local location: <id='{0}'>")]
|
|
||||||
NonLocalLocation(location::id::Type),
|
NonLocalLocation(location::id::Type),
|
||||||
|
|
||||||
#[error("failed to move file '{}' for reason: {reason}", .path.display())]
|
#[error("file still exists on disk after remove event received: <path='{}'>", .0.display())]
|
||||||
MoveError { path: Box<Path>, reason: String },
|
FileStillExistsOnDisk(Box<Path>),
|
||||||
|
|
||||||
#[error("Tried to update a non-existing file: <path='{0}'>")]
|
#[error("failed to move file '{}' for reason: {reason}", .path.display())]
|
||||||
UpdateNonExistingFile(PathBuf),
|
MoveError {
|
||||||
#[error("Database error: {0}")]
|
path: Box<Path>,
|
||||||
|
reason: &'static str,
|
||||||
|
},
|
||||||
|
|
||||||
|
#[error("database error: {0}")]
|
||||||
Database(#[from] prisma_client_rust::QueryError),
|
Database(#[from] prisma_client_rust::QueryError),
|
||||||
#[error("File path related error (error: {0})")]
|
#[error("corrupted location pub_id on database: {0}")]
|
||||||
FilePath(#[from] FilePathError),
|
|
||||||
#[error("Corrupted location pub_id on database: (error: {0})")]
|
|
||||||
CorruptedLocationPubId(#[from] uuid::Error),
|
CorruptedLocationPubId(#[from] uuid::Error),
|
||||||
#[error("Job Manager error: (error: {0})")]
|
#[error("missing field: {0}")]
|
||||||
JobManager(#[from] JobManagerError),
|
|
||||||
#[error("missing-field")]
|
|
||||||
MissingField(#[from] MissingFieldError),
|
MissingField(#[from] MissingFieldError),
|
||||||
|
|
||||||
|
#[error(transparent)]
|
||||||
|
FilePath(#[from] FilePathError),
|
||||||
|
#[error(transparent)]
|
||||||
|
IndexerRuler(#[from] sd_core_indexer_rules::Error),
|
||||||
|
#[error(transparent)]
|
||||||
|
JobSystem(#[from] sd_core_heavy_lifting::Error),
|
||||||
#[error(transparent)]
|
#[error(transparent)]
|
||||||
FileIO(#[from] FileIOError),
|
FileIO(#[from] FileIOError),
|
||||||
}
|
}
|
||||||
|
@ -107,20 +100,18 @@ type OnlineLocations = BTreeSet<Vec<u8>>;
|
||||||
|
|
||||||
#[must_use = "'LocationManagerActor::start' must be used to start the actor"]
|
#[must_use = "'LocationManagerActor::start' must be used to start the actor"]
|
||||||
pub struct LocationManagerActor {
|
pub struct LocationManagerActor {
|
||||||
location_management_rx: mpsc::Receiver<LocationManagementMessage>,
|
location_management_rx: chan::Receiver<LocationManagementMessage>,
|
||||||
|
watcher_management_rx: chan::Receiver<WatcherManagementMessage>,
|
||||||
watcher_management_rx: mpsc::Receiver<WatcherManagementMessage>,
|
stop_rx: chan::Receiver<()>,
|
||||||
|
|
||||||
stop_rx: oneshot::Receiver<()>,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl LocationManagerActor {
|
impl LocationManagerActor {
|
||||||
pub fn start(self, node: Arc<Node>) {
|
pub fn start(self, node: Arc<Node>) {
|
||||||
tokio::spawn({
|
spawn({
|
||||||
let node = node.clone();
|
let node = node.clone();
|
||||||
let rx = node.libraries.rx.clone();
|
let rx = node.libraries.rx.clone();
|
||||||
async move {
|
async move {
|
||||||
if let Err(err) = rx
|
if let Err(e) = rx
|
||||||
.subscribe(|event| {
|
.subscribe(|event| {
|
||||||
let node = node.clone();
|
let node = node.clone();
|
||||||
async move {
|
async move {
|
||||||
|
@ -134,17 +125,18 @@ impl LocationManagerActor {
|
||||||
.await
|
.await
|
||||||
.unwrap_or_else(|e| {
|
.unwrap_or_else(|e| {
|
||||||
error!(
|
error!(
|
||||||
"Failed to get locations from database for location manager: {:#?}",
|
?e,
|
||||||
e
|
"Failed to get locations from database for location manager;",
|
||||||
);
|
);
|
||||||
|
|
||||||
vec![]
|
vec![]
|
||||||
}) {
|
}) {
|
||||||
if let Err(e) =
|
if let Err(e) =
|
||||||
node.locations.add(location.id, library.clone()).await
|
node.locations.add(location.id, library.clone()).await
|
||||||
{
|
{
|
||||||
error!(
|
error!(
|
||||||
"Failed to add location to location manager: {:#?}",
|
?e,
|
||||||
e
|
"Failed to add location to location manager;",
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -160,17 +152,46 @@ impl LocationManagerActor {
|
||||||
})
|
})
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
error!("Core may become unstable! LocationManager's library manager subscription aborted with error: {err:?}");
|
error!(
|
||||||
|
?e,
|
||||||
|
"Core may become unstable! LocationManager's \
|
||||||
|
library manager subscription aborted with error;",
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
tokio::spawn(Locations::run_locations_checker(
|
spawn({
|
||||||
self.location_management_rx,
|
let node = Arc::clone(&node);
|
||||||
self.watcher_management_rx,
|
let Self {
|
||||||
self.stop_rx,
|
location_management_rx,
|
||||||
node,
|
watcher_management_rx,
|
||||||
));
|
stop_rx,
|
||||||
|
} = self;
|
||||||
|
|
||||||
|
async move {
|
||||||
|
while let Err(e) = spawn({
|
||||||
|
runner::run(
|
||||||
|
location_management_rx.clone(),
|
||||||
|
watcher_management_rx.clone(),
|
||||||
|
stop_rx.clone(),
|
||||||
|
Arc::clone(&node),
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
if e.is_panic() {
|
||||||
|
error!(?e, "Location manager panicked;");
|
||||||
|
} else {
|
||||||
|
trace!("Location manager received shutdown signal and will exit...");
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
trace!("Restarting location manager processing task...");
|
||||||
|
}
|
||||||
|
|
||||||
|
debug!("Location manager gracefully shutdown");
|
||||||
|
}
|
||||||
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -178,64 +199,62 @@ pub struct Locations {
|
||||||
online_locations: RwLock<OnlineLocations>,
|
online_locations: RwLock<OnlineLocations>,
|
||||||
pub online_tx: broadcast::Sender<OnlineLocations>,
|
pub online_tx: broadcast::Sender<OnlineLocations>,
|
||||||
|
|
||||||
location_management_tx: mpsc::Sender<LocationManagementMessage>,
|
location_management_tx: chan::Sender<LocationManagementMessage>,
|
||||||
|
|
||||||
watcher_management_tx: mpsc::Sender<WatcherManagementMessage>,
|
watcher_management_tx: chan::Sender<WatcherManagementMessage>,
|
||||||
stop_tx: Option<oneshot::Sender<()>>,
|
stop_tx: chan::Sender<()>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Locations {
|
impl Locations {
|
||||||
pub fn new() -> (Self, LocationManagerActor) {
|
pub fn new() -> (Self, LocationManagerActor) {
|
||||||
let online_tx = broadcast::channel(16).0;
|
let (location_management_tx, location_management_rx) = chan::bounded(128);
|
||||||
|
let (watcher_management_tx, watcher_management_rx) = chan::bounded(128);
|
||||||
|
let (stop_tx, stop_rx) = chan::bounded(1);
|
||||||
|
|
||||||
{
|
debug!("Starting location manager actor");
|
||||||
let (location_management_tx, location_management_rx) = mpsc::channel(128);
|
|
||||||
let (watcher_management_tx, watcher_management_rx) = mpsc::channel(128);
|
|
||||||
let (stop_tx, stop_rx) = oneshot::channel();
|
|
||||||
debug!("Starting location manager actor");
|
|
||||||
|
|
||||||
(
|
(
|
||||||
Self {
|
Self {
|
||||||
online_locations: Default::default(),
|
online_locations: Default::default(),
|
||||||
online_tx,
|
online_tx: broadcast::channel(16).0,
|
||||||
location_management_tx,
|
location_management_tx,
|
||||||
watcher_management_tx,
|
watcher_management_tx,
|
||||||
stop_tx: Some(stop_tx),
|
stop_tx,
|
||||||
},
|
},
|
||||||
LocationManagerActor {
|
LocationManagerActor {
|
||||||
location_management_rx,
|
location_management_rx,
|
||||||
watcher_management_rx,
|
watcher_management_rx,
|
||||||
stop_rx,
|
stop_rx,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[instrument(skip(self, library), fields(library_id = %library.id), err)]
|
||||||
#[inline]
|
#[inline]
|
||||||
#[allow(unused_variables)]
|
|
||||||
async fn location_management_message(
|
async fn location_management_message(
|
||||||
&self,
|
&self,
|
||||||
location_id: location::id::Type,
|
location_id: location::id::Type,
|
||||||
library: Arc<Library>,
|
library: Arc<Library>,
|
||||||
action: ManagementMessageAction,
|
action: ManagementMessageAction,
|
||||||
) -> Result<(), LocationManagerError> {
|
) -> Result<(), LocationManagerError> {
|
||||||
{
|
let (tx, rx) = oneshot::channel();
|
||||||
let (tx, rx) = oneshot::channel();
|
trace!("Sending location management message to location manager actor");
|
||||||
debug!("Sending location management message to location manager actor: {action:?}");
|
|
||||||
|
|
||||||
self.location_management_tx
|
self.location_management_tx
|
||||||
.send(LocationManagementMessage {
|
.send(LocationManagementMessage {
|
||||||
location_id,
|
location_id,
|
||||||
library,
|
library,
|
||||||
action,
|
action,
|
||||||
response_tx: tx,
|
ack: tx,
|
||||||
})
|
})
|
||||||
.await?;
|
.await
|
||||||
|
.expect("Location manager actor channel closed sending new location message");
|
||||||
|
|
||||||
rx.await?
|
rx.await
|
||||||
}
|
.expect("Ack channel closed for location management message response")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[instrument(skip(self, library), fields(library_id = %library.id), err)]
|
||||||
#[inline]
|
#[inline]
|
||||||
#[allow(unused_variables)]
|
#[allow(unused_variables)]
|
||||||
async fn watcher_management_message(
|
async fn watcher_management_message(
|
||||||
|
@ -244,22 +263,21 @@ impl Locations {
|
||||||
library: Arc<Library>,
|
library: Arc<Library>,
|
||||||
action: WatcherManagementMessageAction,
|
action: WatcherManagementMessageAction,
|
||||||
) -> Result<(), LocationManagerError> {
|
) -> Result<(), LocationManagerError> {
|
||||||
{
|
let (tx, rx) = oneshot::channel();
|
||||||
let (tx, rx) = oneshot::channel();
|
trace!("Sending watcher management message to location manager actor");
|
||||||
|
|
||||||
debug!("Sending watcher management message to location manager actor: {action:?}");
|
self.watcher_management_tx
|
||||||
|
.send(WatcherManagementMessage {
|
||||||
|
location_id,
|
||||||
|
library,
|
||||||
|
action,
|
||||||
|
ack: tx,
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("Location manager actor channel closed sending new watcher message");
|
||||||
|
|
||||||
self.watcher_management_tx
|
rx.await
|
||||||
.send(WatcherManagementMessage {
|
.expect("Ack channel closed for watcher management message response")
|
||||||
location_id,
|
|
||||||
library,
|
|
||||||
action,
|
|
||||||
response_tx: tx,
|
|
||||||
})
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
rx.await?
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn add(
|
pub async fn add(
|
||||||
|
@ -280,16 +298,16 @@ impl Locations {
|
||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn stop_watcher(
|
pub async fn pause_watcher(
|
||||||
&self,
|
&self,
|
||||||
location_id: location::id::Type,
|
location_id: location::id::Type,
|
||||||
library: Arc<Library>,
|
library: Arc<Library>,
|
||||||
) -> Result<(), LocationManagerError> {
|
) -> Result<(), LocationManagerError> {
|
||||||
self.watcher_management_message(location_id, library, WatcherManagementMessageAction::Stop)
|
self.watcher_management_message(location_id, library, WatcherManagementMessageAction::Pause)
|
||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn reinit_watcher(
|
pub async fn resume_watcher(
|
||||||
&self,
|
&self,
|
||||||
location_id: location::id::Type,
|
location_id: location::id::Type,
|
||||||
library: Arc<Library>,
|
library: Arc<Library>,
|
||||||
|
@ -297,19 +315,19 @@ impl Locations {
|
||||||
self.watcher_management_message(
|
self.watcher_management_message(
|
||||||
location_id,
|
location_id,
|
||||||
library,
|
library,
|
||||||
WatcherManagementMessageAction::Reinit,
|
WatcherManagementMessageAction::Resume,
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn temporary_stop(
|
pub async fn temporary_watcher_pause(
|
||||||
&self,
|
&self,
|
||||||
location_id: location::id::Type,
|
location_id: location::id::Type,
|
||||||
library: Arc<Library>,
|
library: Arc<Library>,
|
||||||
) -> Result<StopWatcherGuard, LocationManagerError> {
|
) -> Result<PauseWatcherGuard<'_>, LocationManagerError> {
|
||||||
self.stop_watcher(location_id, library.clone()).await?;
|
self.pause_watcher(location_id, library.clone()).await?;
|
||||||
|
|
||||||
Ok(StopWatcherGuard {
|
Ok(PauseWatcherGuard {
|
||||||
location_id,
|
location_id,
|
||||||
library: Some(library),
|
library: Some(library),
|
||||||
manager: self,
|
manager: self,
|
||||||
|
@ -320,8 +338,8 @@ impl Locations {
|
||||||
&self,
|
&self,
|
||||||
location_id: location::id::Type,
|
location_id: location::id::Type,
|
||||||
library: Arc<Library>,
|
library: Arc<Library>,
|
||||||
path: impl AsRef<Path>,
|
path: impl AsRef<Path> + Send,
|
||||||
) -> Result<IgnoreEventsForPathGuard, LocationManagerError> {
|
) -> Result<IgnoreEventsForPathGuard<'_>, LocationManagerError> {
|
||||||
let path = path.as_ref().to_path_buf();
|
let path = path.as_ref().to_path_buf();
|
||||||
|
|
||||||
self.watcher_management_message(
|
self.watcher_management_message(
|
||||||
|
@ -342,217 +360,6 @@ impl Locations {
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn run_locations_checker(
|
|
||||||
mut location_management_rx: mpsc::Receiver<LocationManagementMessage>,
|
|
||||||
mut watcher_management_rx: mpsc::Receiver<WatcherManagementMessage>,
|
|
||||||
mut stop_rx: oneshot::Receiver<()>,
|
|
||||||
node: Arc<Node>,
|
|
||||||
) -> Result<(), LocationManagerError> {
|
|
||||||
use std::collections::{HashMap, HashSet};
|
|
||||||
|
|
||||||
use futures::stream::{FuturesUnordered, StreamExt};
|
|
||||||
use tokio::select;
|
|
||||||
use tracing::warn;
|
|
||||||
|
|
||||||
use helpers::{
|
|
||||||
check_online, drop_location, get_location, handle_ignore_path_request,
|
|
||||||
handle_reinit_watcher_request, handle_remove_location_request,
|
|
||||||
handle_stop_watcher_request, location_check_sleep, unwatch_location, watch_location,
|
|
||||||
};
|
|
||||||
use watcher::LocationWatcher;
|
|
||||||
|
|
||||||
let mut to_check_futures = FuturesUnordered::new();
|
|
||||||
let mut to_remove = HashSet::new();
|
|
||||||
let mut locations_watched = HashMap::new();
|
|
||||||
let mut locations_unwatched = HashMap::new();
|
|
||||||
let mut forced_unwatch = HashSet::new();
|
|
||||||
|
|
||||||
loop {
|
|
||||||
select! {
|
|
||||||
// Location management messages
|
|
||||||
Some(LocationManagementMessage{
|
|
||||||
location_id,
|
|
||||||
library,
|
|
||||||
action,
|
|
||||||
response_tx
|
|
||||||
}) = location_management_rx.recv() => {
|
|
||||||
match action {
|
|
||||||
|
|
||||||
// To add a new location
|
|
||||||
ManagementMessageAction::Add => {
|
|
||||||
response_tx.send(
|
|
||||||
if let Some(location) = get_location(location_id, &library).await {
|
|
||||||
match check_online(&location, &node, &library).await {
|
|
||||||
Ok(is_online) => {
|
|
||||||
|
|
||||||
LocationWatcher::new(location, library.clone(), node.clone())
|
|
||||||
.await
|
|
||||||
.map(|mut watcher| {
|
|
||||||
if is_online {
|
|
||||||
watcher.watch();
|
|
||||||
locations_watched.insert(
|
|
||||||
(location_id, library.id),
|
|
||||||
watcher
|
|
||||||
);
|
|
||||||
debug!("Location {location_id} is online, watching it");
|
|
||||||
// info!("Locations watched: {:#?}", locations_watched);
|
|
||||||
} else {
|
|
||||||
locations_unwatched.insert(
|
|
||||||
(location_id, library.id),
|
|
||||||
watcher
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
to_check_futures.push(
|
|
||||||
location_check_sleep(location_id, library)
|
|
||||||
);
|
|
||||||
}
|
|
||||||
)
|
|
||||||
},
|
|
||||||
Err(e) => {
|
|
||||||
error!("Error while checking online status of location {location_id}: {e}");
|
|
||||||
Ok(()) // TODO: Probs should be error but that will break startup when location is offline
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
warn!(
|
|
||||||
"Location not found in database to be watched: {}",
|
|
||||||
location_id
|
|
||||||
);
|
|
||||||
Ok(()) // TODO: Probs should be error but that will break startup when location is offline
|
|
||||||
}).ok(); // ignore errors, we handle errors on receiver
|
|
||||||
},
|
|
||||||
|
|
||||||
// To remove an location
|
|
||||||
ManagementMessageAction::Remove => {
|
|
||||||
handle_remove_location_request(
|
|
||||||
location_id,
|
|
||||||
library,
|
|
||||||
response_tx,
|
|
||||||
&mut forced_unwatch,
|
|
||||||
&mut locations_watched,
|
|
||||||
&mut locations_unwatched,
|
|
||||||
&mut to_remove,
|
|
||||||
).await;
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Watcher management messages
|
|
||||||
Some(WatcherManagementMessage{
|
|
||||||
location_id,
|
|
||||||
library,
|
|
||||||
action,
|
|
||||||
response_tx,
|
|
||||||
}) = watcher_management_rx.recv() => {
|
|
||||||
match action {
|
|
||||||
// To stop a watcher
|
|
||||||
WatcherManagementMessageAction::Stop => {
|
|
||||||
handle_stop_watcher_request(
|
|
||||||
location_id,
|
|
||||||
library,
|
|
||||||
response_tx,
|
|
||||||
&mut forced_unwatch,
|
|
||||||
&mut locations_watched,
|
|
||||||
&mut locations_unwatched,
|
|
||||||
).await;
|
|
||||||
},
|
|
||||||
|
|
||||||
// To reinit a stopped watcher
|
|
||||||
WatcherManagementMessageAction::Reinit => {
|
|
||||||
handle_reinit_watcher_request(
|
|
||||||
location_id,
|
|
||||||
library,
|
|
||||||
response_tx,
|
|
||||||
&mut forced_unwatch,
|
|
||||||
&mut locations_watched,
|
|
||||||
&mut locations_unwatched,
|
|
||||||
).await;
|
|
||||||
},
|
|
||||||
|
|
||||||
// To ignore or not events for a path
|
|
||||||
WatcherManagementMessageAction::IgnoreEventsForPath { path, ignore } => {
|
|
||||||
handle_ignore_path_request(
|
|
||||||
location_id,
|
|
||||||
library,
|
|
||||||
path,
|
|
||||||
ignore,
|
|
||||||
response_tx,
|
|
||||||
&locations_watched,
|
|
||||||
);
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Periodically checking locations
|
|
||||||
Some((location_id, library)) = to_check_futures.next() => {
|
|
||||||
let key = (location_id, library.id);
|
|
||||||
|
|
||||||
if to_remove.contains(&key) {
|
|
||||||
// The time to check came for an already removed library, so we just ignore it
|
|
||||||
to_remove.remove(&key);
|
|
||||||
} else if let Some(location) = get_location(location_id, &library).await {
|
|
||||||
// TODO(N): This isn't gonna work with removable media and this will likely permanently break if the DB is restored from a backup.
|
|
||||||
if location.instance_id == Some(library.config().await.instance_id) {
|
|
||||||
let is_online = match check_online(&location, &node, &library).await {
|
|
||||||
Ok(is_online) => is_online,
|
|
||||||
Err(e) => {
|
|
||||||
error!("Error while checking online status of location {location_id}: {e}");
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
if is_online
|
|
||||||
&& !forced_unwatch.contains(&key)
|
|
||||||
{
|
|
||||||
watch_location(
|
|
||||||
location,
|
|
||||||
library.id,
|
|
||||||
&mut locations_watched,
|
|
||||||
&mut locations_unwatched,
|
|
||||||
);
|
|
||||||
} else {
|
|
||||||
unwatch_location(
|
|
||||||
location,
|
|
||||||
library.id,
|
|
||||||
&mut locations_watched,
|
|
||||||
&mut locations_unwatched,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
to_check_futures.push(location_check_sleep(location_id, library));
|
|
||||||
} else {
|
|
||||||
drop_location(
|
|
||||||
location_id,
|
|
||||||
library.id,
|
|
||||||
"Dropping location from location manager, because \
|
|
||||||
it isn't a location in the current node",
|
|
||||||
&mut locations_watched,
|
|
||||||
&mut locations_unwatched
|
|
||||||
);
|
|
||||||
forced_unwatch.remove(&key);
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
drop_location(
|
|
||||||
location_id,
|
|
||||||
library.id,
|
|
||||||
"Removing location from manager, as we failed to fetch from db",
|
|
||||||
&mut locations_watched,
|
|
||||||
&mut locations_unwatched,
|
|
||||||
);
|
|
||||||
forced_unwatch.remove(&key);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
_ = &mut stop_rx => {
|
|
||||||
debug!("Stopping location manager");
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn is_online(&self, id: &Uuid) -> bool {
|
pub async fn is_online(&self, id: &Uuid) -> bool {
|
||||||
let online_locations = self.online_locations.read().await;
|
let online_locations = self.online_locations.read().await;
|
||||||
online_locations.iter().any(|v| v == id.as_bytes())
|
online_locations.iter().any(|v| v == id.as_bytes())
|
||||||
|
@ -591,29 +398,28 @@ impl Locations {
|
||||||
|
|
||||||
impl Drop for Locations {
|
impl Drop for Locations {
|
||||||
fn drop(&mut self) {
|
fn drop(&mut self) {
|
||||||
if let Some(stop_tx) = self.stop_tx.take() {
|
// SAFETY: This will never block as we only have 1 sender and this channel has 1 slot
|
||||||
if stop_tx.send(()).is_err() {
|
if self.stop_tx.send_blocking(()).is_err() {
|
||||||
error!("Failed to send stop signal to location manager");
|
error!("Failed to send stop signal to location manager");
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[must_use = "this `StopWatcherGuard` must be held for some time, so the watcher is stopped"]
|
#[must_use = "this `StopWatcherGuard` must be held for some time, so the watcher is stopped"]
|
||||||
pub struct StopWatcherGuard<'m> {
|
pub struct PauseWatcherGuard<'m> {
|
||||||
manager: &'m Locations,
|
manager: &'m Locations,
|
||||||
location_id: location::id::Type,
|
location_id: location::id::Type,
|
||||||
library: Option<Arc<Library>>,
|
library: Option<Arc<Library>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Drop for StopWatcherGuard<'_> {
|
impl Drop for PauseWatcherGuard<'_> {
|
||||||
fn drop(&mut self) {
|
fn drop(&mut self) {
|
||||||
// FIXME: change this Drop to async drop in the future
|
// FIXME: change this Drop to async drop in the future
|
||||||
if let Err(e) = block_on(self.manager.reinit_watcher(
|
if let Err(e) = block_on(self.manager.resume_watcher(
|
||||||
self.location_id,
|
self.location_id,
|
||||||
self.library.take().expect("library should be set"),
|
self.library.take().expect("library should be set"),
|
||||||
)) {
|
)) {
|
||||||
error!("Failed to reinit watcher on stop watcher guard drop: {e}");
|
error!(?e, "Failed to resume watcher on stop watcher guard drop;");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -637,7 +443,7 @@ impl Drop for IgnoreEventsForPathGuard<'_> {
|
||||||
ignore: false,
|
ignore: false,
|
||||||
},
|
},
|
||||||
)) {
|
)) {
|
||||||
error!("Failed to un-ignore path on watcher guard drop: {e}");
|
error!(?e, "Failed to un-ignore path on watcher guard drop;");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
449
core/src/location/manager/runner.rs
Normal file
449
core/src/location/manager/runner.rs
Normal file
|
@ -0,0 +1,449 @@
|
||||||
|
use crate::{
|
||||||
|
library::{Library, LibraryId},
|
||||||
|
Node,
|
||||||
|
};
|
||||||
|
|
||||||
|
use sd_core_prisma_helpers::location_ids_and_path;
|
||||||
|
|
||||||
|
use sd_prisma::prisma::location;
|
||||||
|
use sd_utils::db::maybe_missing;
|
||||||
|
|
||||||
|
use std::{
|
||||||
|
collections::{HashMap, HashSet},
|
||||||
|
io::ErrorKind,
|
||||||
|
path::PathBuf,
|
||||||
|
pin::pin,
|
||||||
|
sync::Arc,
|
||||||
|
time::Duration,
|
||||||
|
};
|
||||||
|
|
||||||
|
use async_channel as chan;
|
||||||
|
use futures::stream::StreamExt;
|
||||||
|
use futures_concurrency::stream::Merge;
|
||||||
|
use tokio::{
|
||||||
|
fs,
|
||||||
|
sync::oneshot,
|
||||||
|
time::{interval, MissedTickBehavior},
|
||||||
|
};
|
||||||
|
use tokio_stream::wrappers::IntervalStream;
|
||||||
|
use tracing::{debug, error, instrument, trace, warn};
|
||||||
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
use super::{
|
||||||
|
watcher::LocationWatcher, LocationManagementMessage, LocationManagerError,
|
||||||
|
ManagementMessageAction, WatcherManagementMessage, WatcherManagementMessageAction,
|
||||||
|
};
|
||||||
|
|
||||||
|
type LocationIdAndLibraryId = (location::id::Type, LibraryId);
|
||||||
|
|
||||||
|
struct Runner {
|
||||||
|
node: Arc<Node>,
|
||||||
|
locations_to_check: HashMap<location::id::Type, Arc<Library>>,
|
||||||
|
locations_watched: HashMap<LocationIdAndLibraryId, LocationWatcher>,
|
||||||
|
locations_unwatched: HashMap<LocationIdAndLibraryId, LocationWatcher>,
|
||||||
|
forced_unwatch: HashSet<LocationIdAndLibraryId>,
|
||||||
|
}
|
||||||
|
impl Runner {
|
||||||
|
fn new(node: Arc<Node>) -> Self {
|
||||||
|
Self {
|
||||||
|
node,
|
||||||
|
locations_to_check: HashMap::new(),
|
||||||
|
locations_watched: HashMap::new(),
|
||||||
|
locations_unwatched: HashMap::new(),
|
||||||
|
forced_unwatch: HashSet::new(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn add_location(
|
||||||
|
&mut self,
|
||||||
|
location_id: i32,
|
||||||
|
library: Arc<Library>,
|
||||||
|
) -> Result<(), LocationManagerError> {
|
||||||
|
if let Some(location) = get_location(location_id, &library).await? {
|
||||||
|
check_online(&location, &self.node, &library)
|
||||||
|
.await
|
||||||
|
.and_then(|is_online| {
|
||||||
|
LocationWatcher::new(location, Arc::clone(&library), Arc::clone(&self.node))
|
||||||
|
.map(|mut watcher| {
|
||||||
|
if is_online {
|
||||||
|
trace!(%location_id, "Location is online, watching it!;");
|
||||||
|
watcher.watch();
|
||||||
|
self.locations_watched
|
||||||
|
.insert((location_id, library.id), watcher);
|
||||||
|
} else {
|
||||||
|
self.locations_unwatched
|
||||||
|
.insert((location_id, library.id), watcher);
|
||||||
|
}
|
||||||
|
|
||||||
|
self.locations_to_check
|
||||||
|
.insert(location_id, Arc::clone(&library));
|
||||||
|
})
|
||||||
|
})
|
||||||
|
} else {
|
||||||
|
Err(LocationManagerError::LocationNotFound(location_id))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn remove_location(
|
||||||
|
&mut self,
|
||||||
|
location_id: i32,
|
||||||
|
library: Arc<Library>,
|
||||||
|
) -> Result<(), LocationManagerError> {
|
||||||
|
let key = (location_id, library.id);
|
||||||
|
|
||||||
|
if let Some(location) = get_location(location_id, &library).await? {
|
||||||
|
// TODO(N): This isn't gonna work with removable media and this will likely permanently break if the DB is restored from a backup.
|
||||||
|
if location.instance_id == Some(library.config().await.instance_id) {
|
||||||
|
self.unwatch_location(location, library.id);
|
||||||
|
self.locations_unwatched.remove(&key);
|
||||||
|
self.forced_unwatch.remove(&key);
|
||||||
|
} else {
|
||||||
|
self.drop_location(
|
||||||
|
location_id,
|
||||||
|
library.id,
|
||||||
|
"Dropping location from location manager, because we don't have a `local_path` anymore",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
self.drop_location(
|
||||||
|
location_id,
|
||||||
|
library.id,
|
||||||
|
"Removing location from location manager, as we failed to fetch from db",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Removing location from checker
|
||||||
|
self.locations_to_check.remove(&location_id);
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[instrument(skip(self, reason))]
|
||||||
|
fn drop_location(
|
||||||
|
&mut self,
|
||||||
|
location_id: location::id::Type,
|
||||||
|
library_id: LibraryId,
|
||||||
|
reason: &'static str,
|
||||||
|
) {
|
||||||
|
warn!(%reason);
|
||||||
|
if let Some(mut watcher) = self.locations_watched.remove(&(location_id, library_id)) {
|
||||||
|
watcher.unwatch();
|
||||||
|
} else {
|
||||||
|
self.locations_unwatched.remove(&(location_id, library_id));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn watch_location(
|
||||||
|
&mut self,
|
||||||
|
location_ids_and_path::Data {
|
||||||
|
id: location_id,
|
||||||
|
path: maybe_location_path,
|
||||||
|
..
|
||||||
|
}: location_ids_and_path::Data,
|
||||||
|
library_id: LibraryId,
|
||||||
|
) {
|
||||||
|
if let Some(location_path) = maybe_location_path {
|
||||||
|
if let Some(mut watcher) = self.locations_unwatched.remove(&(location_id, library_id)) {
|
||||||
|
if watcher.check_path(location_path) {
|
||||||
|
watcher.watch();
|
||||||
|
}
|
||||||
|
|
||||||
|
self.locations_watched
|
||||||
|
.insert((location_id, library_id), watcher);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn unwatch_location(
|
||||||
|
&mut self,
|
||||||
|
location_ids_and_path::Data {
|
||||||
|
id: location_id,
|
||||||
|
path: maybe_location_path,
|
||||||
|
..
|
||||||
|
}: location_ids_and_path::Data,
|
||||||
|
library_id: LibraryId,
|
||||||
|
) {
|
||||||
|
if let Some(location_path) = maybe_location_path {
|
||||||
|
if let Some(mut watcher) = self.locations_watched.remove(&(location_id, library_id)) {
|
||||||
|
if watcher.check_path(location_path) {
|
||||||
|
watcher.unwatch();
|
||||||
|
}
|
||||||
|
|
||||||
|
self.locations_unwatched
|
||||||
|
.insert((location_id, library_id), watcher);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[instrument(skip(self, library), fields(library_id = %library.id), err)]
|
||||||
|
async fn pause_watcher(
|
||||||
|
&mut self,
|
||||||
|
location_id: location::id::Type,
|
||||||
|
library: Arc<Library>,
|
||||||
|
) -> Result<(), LocationManagerError> {
|
||||||
|
let key = (location_id, library.id);
|
||||||
|
|
||||||
|
if !self.forced_unwatch.contains(&key) && self.locations_watched.contains_key(&key) {
|
||||||
|
get_location(location_id, &library)
|
||||||
|
.await?
|
||||||
|
.ok_or(LocationManagerError::LocationNotFound(location_id))
|
||||||
|
.map(|location| {
|
||||||
|
self.unwatch_location(location, library.id);
|
||||||
|
self.forced_unwatch.insert(key);
|
||||||
|
})
|
||||||
|
} else {
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[instrument(skip(self, library), fields(library_id = %library.id), err)]
|
||||||
|
async fn resume_watcher(
|
||||||
|
&mut self,
|
||||||
|
location_id: location::id::Type,
|
||||||
|
library: Arc<Library>,
|
||||||
|
) -> Result<(), LocationManagerError> {
|
||||||
|
let key = (location_id, library.id);
|
||||||
|
|
||||||
|
if self.forced_unwatch.contains(&key) && self.locations_unwatched.contains_key(&key) {
|
||||||
|
get_location(location_id, &library)
|
||||||
|
.await?
|
||||||
|
.ok_or(LocationManagerError::LocationNotFound(location_id))
|
||||||
|
.map(|location| {
|
||||||
|
self.watch_location(location, library.id);
|
||||||
|
self.forced_unwatch.remove(&key);
|
||||||
|
})
|
||||||
|
} else {
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn ignore_events_for_path(
|
||||||
|
&self,
|
||||||
|
location_id: location::id::Type,
|
||||||
|
library: Arc<Library>,
|
||||||
|
path: PathBuf,
|
||||||
|
ignore: bool,
|
||||||
|
) {
|
||||||
|
if let Some(watcher) = self.locations_watched.get(&(location_id, library.id)) {
|
||||||
|
watcher.ignore_path(path, ignore).await
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn handle_location_management_message(
|
||||||
|
&mut self,
|
||||||
|
location_id: location::id::Type,
|
||||||
|
library: Arc<Library>,
|
||||||
|
action: ManagementMessageAction,
|
||||||
|
ack: oneshot::Sender<Result<(), LocationManagerError>>,
|
||||||
|
) {
|
||||||
|
ack.send(match action {
|
||||||
|
ManagementMessageAction::Add => self.add_location(location_id, library).await,
|
||||||
|
ManagementMessageAction::Remove => self.remove_location(location_id, library).await,
|
||||||
|
})
|
||||||
|
.expect("Ack channel closed")
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn handle_watcher_management_message(
|
||||||
|
&mut self,
|
||||||
|
location_id: location::id::Type,
|
||||||
|
library: Arc<Library>,
|
||||||
|
action: WatcherManagementMessageAction,
|
||||||
|
ack: oneshot::Sender<Result<(), LocationManagerError>>,
|
||||||
|
) {
|
||||||
|
ack.send(match action {
|
||||||
|
WatcherManagementMessageAction::Pause => self.pause_watcher(location_id, library).await,
|
||||||
|
WatcherManagementMessageAction::Resume => {
|
||||||
|
self.resume_watcher(location_id, library).await
|
||||||
|
}
|
||||||
|
WatcherManagementMessageAction::IgnoreEventsForPath { path, ignore } => {
|
||||||
|
self.ignore_events_for_path(location_id, library, path, ignore)
|
||||||
|
.await;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.expect("Ack channel closed")
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn check_locations(
|
||||||
|
&mut self,
|
||||||
|
locations_to_check_buffer: &mut Vec<(location::id::Type, Arc<Library>)>,
|
||||||
|
) -> Result<(), Vec<LocationManagerError>> {
|
||||||
|
let mut errors = vec![];
|
||||||
|
locations_to_check_buffer.clear();
|
||||||
|
locations_to_check_buffer.extend(self.locations_to_check.drain());
|
||||||
|
|
||||||
|
for (location_id, library) in locations_to_check_buffer.drain(..) {
|
||||||
|
if let Err(e) = self
|
||||||
|
.check_single_location(location_id, Arc::clone(&library))
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
self.drop_location(
|
||||||
|
location_id,
|
||||||
|
library.id,
|
||||||
|
"Removing location from manager, as we failed to check if it was online",
|
||||||
|
);
|
||||||
|
self.forced_unwatch.remove(&(location_id, library.id));
|
||||||
|
errors.push(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn check_single_location(
|
||||||
|
&mut self,
|
||||||
|
location_id: i32,
|
||||||
|
library: Arc<Library>,
|
||||||
|
) -> Result<(), LocationManagerError> {
|
||||||
|
let key = (location_id, library.id);
|
||||||
|
|
||||||
|
if let Some(location) = get_location(location_id, &library).await? {
|
||||||
|
// TODO(N): This isn't gonna work with removable media and this will likely permanently break if the DB is restored from a backup.
|
||||||
|
if location.instance_id == Some(library.config().await.instance_id) {
|
||||||
|
if check_online(&location, &self.node, &library).await?
|
||||||
|
&& !self.forced_unwatch.contains(&key)
|
||||||
|
{
|
||||||
|
self.watch_location(location, library.id);
|
||||||
|
} else {
|
||||||
|
self.unwatch_location(location, library.id);
|
||||||
|
}
|
||||||
|
|
||||||
|
self.locations_to_check.insert(location_id, library);
|
||||||
|
} else {
|
||||||
|
self.drop_location(
|
||||||
|
location_id,
|
||||||
|
library.id,
|
||||||
|
"Dropping location from location manager, because \
|
||||||
|
it isn't a location in the current node",
|
||||||
|
);
|
||||||
|
self.forced_unwatch.remove(&key);
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
} else {
|
||||||
|
Err(LocationManagerError::LocationNotFound(location_id))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) async fn run(
|
||||||
|
location_management_rx: chan::Receiver<LocationManagementMessage>,
|
||||||
|
watcher_management_rx: chan::Receiver<WatcherManagementMessage>,
|
||||||
|
stop_rx: chan::Receiver<()>,
|
||||||
|
node: Arc<Node>,
|
||||||
|
) {
|
||||||
|
enum StreamMessage {
|
||||||
|
LocationManagementMessage(LocationManagementMessage),
|
||||||
|
WatcherManagementMessage(WatcherManagementMessage),
|
||||||
|
CheckLocations,
|
||||||
|
Stop,
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut locations_to_check_buffer = vec![];
|
||||||
|
|
||||||
|
let mut check_locations_interval = interval(Duration::from_secs(2));
|
||||||
|
check_locations_interval.set_missed_tick_behavior(MissedTickBehavior::Skip);
|
||||||
|
|
||||||
|
let mut runner = Runner::new(node);
|
||||||
|
|
||||||
|
let mut msg_stream = pin!((
|
||||||
|
location_management_rx.map(StreamMessage::LocationManagementMessage),
|
||||||
|
watcher_management_rx.map(StreamMessage::WatcherManagementMessage),
|
||||||
|
IntervalStream::new(check_locations_interval).map(|_| StreamMessage::CheckLocations),
|
||||||
|
stop_rx.map(|()| StreamMessage::Stop),
|
||||||
|
)
|
||||||
|
.merge());
|
||||||
|
|
||||||
|
while let Some(msg) = msg_stream.next().await {
|
||||||
|
match msg {
|
||||||
|
StreamMessage::LocationManagementMessage(LocationManagementMessage {
|
||||||
|
location_id,
|
||||||
|
library,
|
||||||
|
action,
|
||||||
|
ack,
|
||||||
|
}) => {
|
||||||
|
runner
|
||||||
|
.handle_location_management_message(location_id, library, action, ack)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
// Watcher management messages
|
||||||
|
StreamMessage::WatcherManagementMessage(WatcherManagementMessage {
|
||||||
|
location_id,
|
||||||
|
library,
|
||||||
|
action,
|
||||||
|
ack,
|
||||||
|
}) => {
|
||||||
|
runner
|
||||||
|
.handle_watcher_management_message(location_id, library, action, ack)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
StreamMessage::CheckLocations => {
|
||||||
|
if let Err(errors) = runner.check_locations(&mut locations_to_check_buffer).await {
|
||||||
|
warn!(?errors, "Errors while checking locations;");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
StreamMessage::Stop => {
|
||||||
|
debug!("Stopping location manager");
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[instrument(skip(library), fields(library_id = %library.id), err)]
|
||||||
|
async fn get_location(
|
||||||
|
location_id: location::id::Type,
|
||||||
|
library: &Library,
|
||||||
|
) -> Result<Option<location_ids_and_path::Data>, LocationManagerError> {
|
||||||
|
library
|
||||||
|
.db
|
||||||
|
.location()
|
||||||
|
.find_unique(location::id::equals(location_id))
|
||||||
|
.select(location_ids_and_path::select())
|
||||||
|
.exec()
|
||||||
|
.await
|
||||||
|
.map_err(Into::into)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[instrument(
|
||||||
|
skip_all,
|
||||||
|
fields(%location_id, library_id = %library.id),
|
||||||
|
err,
|
||||||
|
)]
|
||||||
|
pub(super) async fn check_online(
|
||||||
|
location_ids_and_path::Data {
|
||||||
|
id: location_id,
|
||||||
|
pub_id,
|
||||||
|
instance_id,
|
||||||
|
path,
|
||||||
|
}: &location_ids_and_path::Data,
|
||||||
|
node: &Node,
|
||||||
|
library: &Library,
|
||||||
|
) -> Result<bool, LocationManagerError> {
|
||||||
|
let pub_id = Uuid::from_slice(pub_id)?;
|
||||||
|
|
||||||
|
// TODO(N): This isn't gonna work with removable media and this will likely permanently break if the DB is restored from a backup.
|
||||||
|
if *instance_id == Some(library.config().await.instance_id) {
|
||||||
|
match fs::metadata(maybe_missing(path, "location.path")?).await {
|
||||||
|
Ok(_) => {
|
||||||
|
node.locations.add_online(pub_id).await;
|
||||||
|
Ok(true)
|
||||||
|
}
|
||||||
|
Err(e) if e.kind() == ErrorKind::NotFound => {
|
||||||
|
node.locations.remove_online(&pub_id).await;
|
||||||
|
Ok(false)
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
error!(
|
||||||
|
?e,
|
||||||
|
"Failed to check if location is online, will consider as offline;"
|
||||||
|
);
|
||||||
|
Ok(false)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// In this case, we don't have a `local_path`, but this location was marked as online
|
||||||
|
node.locations.remove_online(&pub_id).await;
|
||||||
|
Err(LocationManagerError::NonLocalLocation(*location_id))
|
||||||
|
}
|
||||||
|
}
|
|
@ -12,40 +12,35 @@ use std::{
|
||||||
sync::Arc,
|
sync::Arc,
|
||||||
};
|
};
|
||||||
|
|
||||||
use async_trait::async_trait;
|
|
||||||
use notify::{
|
use notify::{
|
||||||
event::{CreateKind, DataChange, ModifyKind, RenameMode},
|
event::{CreateKind, DataChange, ModifyKind, RenameMode},
|
||||||
Event, EventKind,
|
Event, EventKind,
|
||||||
};
|
};
|
||||||
use tokio::{fs, time::Instant};
|
use tokio::{fs, time::Instant};
|
||||||
use tracing::{debug, error, trace};
|
use tracing::{error, instrument, trace};
|
||||||
|
|
||||||
use super::{
|
use super::{
|
||||||
utils::{create_dir, recalculate_directories_size, remove, rename, update_file},
|
utils::{create_dir, recalculate_directories_size, remove, rename, update_file},
|
||||||
EventHandler, HUNDRED_MILLIS, ONE_SECOND,
|
HUNDRED_MILLIS, ONE_SECOND,
|
||||||
};
|
};
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
pub(super) struct AndroidEventHandler<'lib> {
|
pub(super) struct EventHandler {
|
||||||
location_id: location::id::Type,
|
location_id: location::id::Type,
|
||||||
library: &'lib Arc<Library>,
|
library: Arc<Library>,
|
||||||
node: &'lib Arc<Node>,
|
node: Arc<Node>,
|
||||||
last_events_eviction_check: Instant,
|
last_events_eviction_check: Instant,
|
||||||
rename_from: HashMap<PathBuf, Instant>,
|
rename_from: HashMap<PathBuf, Instant>,
|
||||||
recently_renamed_from: BTreeMap<PathBuf, Instant>,
|
recently_renamed_from: BTreeMap<PathBuf, Instant>,
|
||||||
files_to_update: HashMap<PathBuf, Instant>,
|
files_to_update: HashMap<PathBuf, Instant>,
|
||||||
reincident_to_update_files: HashMap<PathBuf, Instant>,
|
reincident_to_update_files: HashMap<PathBuf, Instant>,
|
||||||
to_recalculate_size: HashMap<PathBuf, Instant>,
|
to_recalculate_size: HashMap<PathBuf, Instant>,
|
||||||
|
|
||||||
path_and_instant_buffer: Vec<(PathBuf, Instant)>,
|
path_and_instant_buffer: Vec<(PathBuf, Instant)>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[async_trait]
|
impl super::EventHandler for EventHandler {
|
||||||
impl<'lib> EventHandler<'lib> for AndroidEventHandler<'lib> {
|
fn new(location_id: location::id::Type, library: Arc<Library>, node: Arc<Node>) -> Self {
|
||||||
fn new(
|
|
||||||
location_id: location::id::Type,
|
|
||||||
library: &'lib Arc<Library>,
|
|
||||||
node: &'lib Arc<Node>,
|
|
||||||
) -> Self {
|
|
||||||
Self {
|
Self {
|
||||||
location_id,
|
location_id,
|
||||||
library,
|
library,
|
||||||
|
@ -60,8 +55,19 @@ impl<'lib> EventHandler<'lib> for AndroidEventHandler<'lib> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[instrument(
|
||||||
|
skip_all,
|
||||||
|
fields(
|
||||||
|
location_id = %self.location_id,
|
||||||
|
library_id = %self.library.id,
|
||||||
|
waiting_rename_count = %self.recently_renamed_from.len(),
|
||||||
|
waiting_update_count = %self.files_to_update.len(),
|
||||||
|
reincident_to_update_files_count = %self.reincident_to_update_files.len(),
|
||||||
|
waiting_size_count = %self.to_recalculate_size.len(),
|
||||||
|
),
|
||||||
|
)]
|
||||||
async fn handle_event(&mut self, event: Event) -> Result<(), LocationManagerError> {
|
async fn handle_event(&mut self, event: Event) -> Result<(), LocationManagerError> {
|
||||||
debug!("Received Android event: {:#?}", event);
|
trace!("Received Android event");
|
||||||
|
|
||||||
let Event {
|
let Event {
|
||||||
kind, mut paths, ..
|
kind, mut paths, ..
|
||||||
|
@ -70,7 +76,7 @@ impl<'lib> EventHandler<'lib> for AndroidEventHandler<'lib> {
|
||||||
match kind {
|
match kind {
|
||||||
EventKind::Create(CreateKind::File)
|
EventKind::Create(CreateKind::File)
|
||||||
| EventKind::Modify(ModifyKind::Data(DataChange::Any)) => {
|
| EventKind::Modify(ModifyKind::Data(DataChange::Any)) => {
|
||||||
// When we receive a create, modify data or metadata events of the abore kinds
|
// When we receive a create, modify data or metadata events of the above kinds
|
||||||
// we just mark the file to be updated in a near future
|
// we just mark the file to be updated in a near future
|
||||||
// each consecutive event of these kinds that we receive for the same file
|
// each consecutive event of these kinds that we receive for the same file
|
||||||
// we just store the path again in the map below, with a new instant
|
// we just store the path again in the map below, with a new instant
|
||||||
|
@ -101,13 +107,14 @@ impl<'lib> EventHandler<'lib> for AndroidEventHandler<'lib> {
|
||||||
&fs::metadata(path)
|
&fs::metadata(path)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| FileIOError::from((path, e)))?,
|
.map_err(|e| FileIOError::from((path, e)))?,
|
||||||
self.node,
|
&self.node,
|
||||||
self.library,
|
&self.library,
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
}
|
}
|
||||||
|
|
||||||
EventKind::Modify(ModifyKind::Name(RenameMode::From)) => {
|
EventKind::Modify(ModifyKind::Name(RenameMode::From)) => {
|
||||||
// Just in case we can't garantee that we receive the Rename From event before the
|
// Just in case we can't guarantee that we receive the Rename From event before the
|
||||||
// Rename Both event. Just a safeguard
|
// Rename Both event. Just a safeguard
|
||||||
if self.recently_renamed_from.remove(&paths[0]).is_none() {
|
if self.recently_renamed_from.remove(&paths[0]).is_none() {
|
||||||
self.rename_from.insert(paths.remove(0), Instant::now());
|
self.rename_from.insert(paths.remove(0), Instant::now());
|
||||||
|
@ -115,23 +122,25 @@ impl<'lib> EventHandler<'lib> for AndroidEventHandler<'lib> {
|
||||||
}
|
}
|
||||||
|
|
||||||
EventKind::Modify(ModifyKind::Name(RenameMode::Both)) => {
|
EventKind::Modify(ModifyKind::Name(RenameMode::Both)) => {
|
||||||
let from_path = &paths[0];
|
let to_path = paths.remove(1);
|
||||||
let to_path = &paths[1];
|
let from_path = paths.remove(0);
|
||||||
|
|
||||||
|
self.rename_from.remove(&from_path);
|
||||||
|
|
||||||
self.rename_from.remove(from_path);
|
|
||||||
rename(
|
rename(
|
||||||
self.location_id,
|
self.location_id,
|
||||||
to_path,
|
&to_path,
|
||||||
from_path,
|
&from_path,
|
||||||
fs::metadata(to_path)
|
fs::metadata(&to_path)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| FileIOError::from((to_path, e)))?,
|
.map_err(|e| FileIOError::from((&to_path, e)))?,
|
||||||
self.library,
|
&self.library,
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
self.recently_renamed_from
|
|
||||||
.insert(paths.swap_remove(0), Instant::now());
|
self.recently_renamed_from.insert(from_path, Instant::now());
|
||||||
}
|
}
|
||||||
|
|
||||||
EventKind::Remove(_) => {
|
EventKind::Remove(_) => {
|
||||||
let path = paths.remove(0);
|
let path = paths.remove(0);
|
||||||
if let Some(parent) = path.parent() {
|
if let Some(parent) = path.parent() {
|
||||||
|
@ -141,10 +150,11 @@ impl<'lib> EventHandler<'lib> for AndroidEventHandler<'lib> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
remove(self.location_id, &path, self.library).await?;
|
remove(self.location_id, &path, &self.library).await?;
|
||||||
}
|
}
|
||||||
other_event_kind => {
|
|
||||||
trace!("Other Linux event that we don't handle for now: {other_event_kind:#?}");
|
_ => {
|
||||||
|
trace!("Other Android event that we don't handle for now");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -154,11 +164,14 @@ impl<'lib> EventHandler<'lib> for AndroidEventHandler<'lib> {
|
||||||
async fn tick(&mut self) {
|
async fn tick(&mut self) {
|
||||||
if self.last_events_eviction_check.elapsed() > HUNDRED_MILLIS {
|
if self.last_events_eviction_check.elapsed() > HUNDRED_MILLIS {
|
||||||
if let Err(e) = self.handle_to_update_eviction().await {
|
if let Err(e) = self.handle_to_update_eviction().await {
|
||||||
error!("Error while handling recently created or update files eviction: {e:#?}");
|
error!(
|
||||||
|
?e,
|
||||||
|
"Error while handling recently created or update files eviction;"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Err(e) = self.handle_rename_from_eviction().await {
|
if let Err(e) = self.handle_rename_from_eviction().await {
|
||||||
error!("Failed to remove file_path: {e:#?}");
|
error!(?e, "Failed to remove file_path;");
|
||||||
}
|
}
|
||||||
|
|
||||||
self.recently_renamed_from
|
self.recently_renamed_from
|
||||||
|
@ -169,11 +182,11 @@ impl<'lib> EventHandler<'lib> for AndroidEventHandler<'lib> {
|
||||||
&mut self.to_recalculate_size,
|
&mut self.to_recalculate_size,
|
||||||
&mut self.path_and_instant_buffer,
|
&mut self.path_and_instant_buffer,
|
||||||
self.location_id,
|
self.location_id,
|
||||||
self.library,
|
&self.library,
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
error!("Failed to recalculate directories size: {e:#?}");
|
error!(?e, "Failed to recalculate directories size;");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -182,9 +195,10 @@ impl<'lib> EventHandler<'lib> for AndroidEventHandler<'lib> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl AndroidEventHandler<'_> {
|
impl EventHandler {
|
||||||
async fn handle_to_update_eviction(&mut self) -> Result<(), LocationManagerError> {
|
async fn handle_to_update_eviction(&mut self) -> Result<(), LocationManagerError> {
|
||||||
self.path_and_instant_buffer.clear();
|
self.path_and_instant_buffer.clear();
|
||||||
|
|
||||||
let mut should_invalidate = false;
|
let mut should_invalidate = false;
|
||||||
|
|
||||||
for (path, created_at) in self.files_to_update.drain() {
|
for (path, created_at) in self.files_to_update.drain() {
|
||||||
|
@ -197,8 +211,11 @@ impl AndroidEventHandler<'_> {
|
||||||
.insert(parent.to_path_buf(), Instant::now());
|
.insert(parent.to_path_buf(), Instant::now());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
self.reincident_to_update_files.remove(&path);
|
self.reincident_to_update_files.remove(&path);
|
||||||
update_file(self.location_id, &path, self.node, self.library).await?;
|
|
||||||
|
update_file(self.location_id, &path, &self.node, &self.library).await?;
|
||||||
|
|
||||||
should_invalidate = true;
|
should_invalidate = true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -221,14 +238,17 @@ impl AndroidEventHandler<'_> {
|
||||||
.insert(parent.to_path_buf(), Instant::now());
|
.insert(parent.to_path_buf(), Instant::now());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
self.files_to_update.remove(&path);
|
self.files_to_update.remove(&path);
|
||||||
update_file(self.location_id, &path, self.node, self.library).await?;
|
|
||||||
|
update_file(self.location_id, &path, &self.node, &self.library).await?;
|
||||||
|
|
||||||
should_invalidate = true;
|
should_invalidate = true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if should_invalidate {
|
if should_invalidate {
|
||||||
invalidate_query!(self.library, "search.paths");
|
invalidate_query!(&self.library, "search.paths");
|
||||||
}
|
}
|
||||||
|
|
||||||
self.reincident_to_update_files
|
self.reincident_to_update_files
|
||||||
|
@ -249,21 +269,23 @@ impl AndroidEventHandler<'_> {
|
||||||
.insert(parent.to_path_buf(), Instant::now());
|
.insert(parent.to_path_buf(), Instant::now());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
remove(self.location_id, &path, self.library).await?;
|
|
||||||
|
remove(self.location_id, &path, &self.library).await?;
|
||||||
|
|
||||||
should_invalidate = true;
|
should_invalidate = true;
|
||||||
trace!("Removed file_path due timeout: {}", path.display());
|
|
||||||
|
trace!(path = %path.display(), "Removed file_path due timeout;");
|
||||||
} else {
|
} else {
|
||||||
self.path_and_instant_buffer.push((path, instant));
|
self.path_and_instant_buffer.push((path, instant));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if should_invalidate {
|
if should_invalidate {
|
||||||
invalidate_query!(self.library, "search.paths");
|
invalidate_query!(&self.library, "search.paths");
|
||||||
}
|
}
|
||||||
|
|
||||||
for (path, instant) in self.path_and_instant_buffer.drain(..) {
|
self.rename_from
|
||||||
self.rename_from.insert(path, instant);
|
.extend(self.path_and_instant_buffer.drain(..));
|
||||||
}
|
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
|
@ -15,45 +15,40 @@ use std::{
|
||||||
sync::Arc,
|
sync::Arc,
|
||||||
};
|
};
|
||||||
|
|
||||||
use async_trait::async_trait;
|
|
||||||
use notify::{
|
use notify::{
|
||||||
event::{CreateKind, DataChange, MetadataKind, ModifyKind, RenameMode},
|
event::{CreateKind, DataChange, MetadataKind, ModifyKind, RenameMode},
|
||||||
Event, EventKind,
|
Event, EventKind,
|
||||||
};
|
};
|
||||||
use tokio::{fs, io, time::Instant};
|
use tokio::{fs, io, time::Instant};
|
||||||
use tracing::{debug, error, trace, warn};
|
use tracing::{error, instrument, trace, warn};
|
||||||
|
|
||||||
use super::{
|
use super::{
|
||||||
utils::{
|
utils::{
|
||||||
create_dir, create_file, extract_inode_from_path, extract_location_path,
|
create_dir, create_file, extract_inode_from_path, extract_location_path,
|
||||||
recalculate_directories_size, remove, rename, update_file,
|
recalculate_directories_size, remove, rename, update_file,
|
||||||
},
|
},
|
||||||
EventHandler, INode, InstantAndPath, HUNDRED_MILLIS, ONE_SECOND,
|
INode, InstantAndPath, HUNDRED_MILLIS, ONE_SECOND,
|
||||||
};
|
};
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
pub(super) struct IosEventHandler<'lib> {
|
pub(super) struct EventHandler {
|
||||||
location_id: location::id::Type,
|
location_id: location::id::Type,
|
||||||
library: &'lib Arc<Library>,
|
library: Arc<Library>,
|
||||||
node: &'lib Arc<Node>,
|
node: Arc<Node>,
|
||||||
files_to_update: HashMap<PathBuf, Instant>,
|
|
||||||
reincident_to_update_files: HashMap<PathBuf, Instant>,
|
|
||||||
last_events_eviction_check: Instant,
|
last_events_eviction_check: Instant,
|
||||||
latest_created_dir: Option<PathBuf>,
|
latest_created_dir: Option<PathBuf>,
|
||||||
old_paths_map: HashMap<INode, InstantAndPath>,
|
old_paths_map: HashMap<INode, InstantAndPath>,
|
||||||
new_paths_map: HashMap<INode, InstantAndPath>,
|
new_paths_map: HashMap<INode, InstantAndPath>,
|
||||||
paths_map_buffer: Vec<(INode, InstantAndPath)>,
|
files_to_update: HashMap<PathBuf, Instant>,
|
||||||
|
reincident_to_update_files: HashMap<PathBuf, Instant>,
|
||||||
to_recalculate_size: HashMap<PathBuf, Instant>,
|
to_recalculate_size: HashMap<PathBuf, Instant>,
|
||||||
|
|
||||||
path_and_instant_buffer: Vec<(PathBuf, Instant)>,
|
path_and_instant_buffer: Vec<(PathBuf, Instant)>,
|
||||||
|
paths_map_buffer: Vec<(INode, InstantAndPath)>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[async_trait]
|
impl super::EventHandler for EventHandler {
|
||||||
impl<'lib> EventHandler<'lib> for IosEventHandler<'lib> {
|
fn new(location_id: location::id::Type, library: Arc<Library>, node: Arc<Node>) -> Self
|
||||||
fn new(
|
|
||||||
location_id: location::id::Type,
|
|
||||||
library: &'lib Arc<Library>,
|
|
||||||
node: &'lib Arc<Node>,
|
|
||||||
) -> Self
|
|
||||||
where
|
where
|
||||||
Self: Sized,
|
Self: Sized,
|
||||||
{
|
{
|
||||||
|
@ -61,38 +56,54 @@ impl<'lib> EventHandler<'lib> for IosEventHandler<'lib> {
|
||||||
location_id,
|
location_id,
|
||||||
library,
|
library,
|
||||||
node,
|
node,
|
||||||
files_to_update: HashMap::new(),
|
|
||||||
reincident_to_update_files: HashMap::new(),
|
|
||||||
last_events_eviction_check: Instant::now(),
|
last_events_eviction_check: Instant::now(),
|
||||||
latest_created_dir: None,
|
latest_created_dir: None,
|
||||||
old_paths_map: HashMap::new(),
|
old_paths_map: HashMap::new(),
|
||||||
new_paths_map: HashMap::new(),
|
new_paths_map: HashMap::new(),
|
||||||
paths_map_buffer: Vec::new(),
|
files_to_update: HashMap::new(),
|
||||||
|
reincident_to_update_files: HashMap::new(),
|
||||||
to_recalculate_size: HashMap::new(),
|
to_recalculate_size: HashMap::new(),
|
||||||
path_and_instant_buffer: Vec::new(),
|
path_and_instant_buffer: Vec::new(),
|
||||||
|
paths_map_buffer: Vec::new(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[instrument(
|
||||||
|
skip_all,
|
||||||
|
fields(
|
||||||
|
location_id = %self.location_id,
|
||||||
|
library_id = %self.library.id,
|
||||||
|
latest_created_dir = ?self.latest_created_dir,
|
||||||
|
old_paths_map_count = %self.old_paths_map.len(),
|
||||||
|
new_paths_map = %self.new_paths_map.len(),
|
||||||
|
waiting_update_count = %self.files_to_update.len(),
|
||||||
|
reincident_to_update_files_count = %self.reincident_to_update_files.len(),
|
||||||
|
waiting_size_count = %self.to_recalculate_size.len(),
|
||||||
|
),
|
||||||
|
)]
|
||||||
async fn handle_event(&mut self, event: Event) -> Result<(), LocationManagerError> {
|
async fn handle_event(&mut self, event: Event) -> Result<(), LocationManagerError> {
|
||||||
|
trace!("Received iOS event");
|
||||||
|
|
||||||
let Event {
|
let Event {
|
||||||
kind, mut paths, ..
|
kind, mut paths, ..
|
||||||
} = event;
|
} = event;
|
||||||
|
|
||||||
match kind {
|
match kind {
|
||||||
EventKind::Create(CreateKind::Folder) => {
|
EventKind::Create(CreateKind::Folder) => {
|
||||||
let path = &paths[0];
|
let path = paths.remove(0);
|
||||||
|
|
||||||
create_dir(
|
create_dir(
|
||||||
self.location_id,
|
self.location_id,
|
||||||
path,
|
&path,
|
||||||
&fs::metadata(path)
|
&fs::metadata(&path)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| FileIOError::from((path, e)))?,
|
.map_err(|e| FileIOError::from((&path, e)))?,
|
||||||
self.node,
|
&self.node,
|
||||||
self.library,
|
&self.library,
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
self.latest_created_dir = Some(paths.remove(0));
|
|
||||||
|
self.latest_created_dir = Some(path);
|
||||||
}
|
}
|
||||||
|
|
||||||
EventKind::Create(CreateKind::File)
|
EventKind::Create(CreateKind::File)
|
||||||
|
@ -100,12 +111,13 @@ impl<'lib> EventHandler<'lib> for IosEventHandler<'lib> {
|
||||||
| EventKind::Modify(ModifyKind::Metadata(
|
| EventKind::Modify(ModifyKind::Metadata(
|
||||||
MetadataKind::WriteTime | MetadataKind::Extended,
|
MetadataKind::WriteTime | MetadataKind::Extended,
|
||||||
)) => {
|
)) => {
|
||||||
// When we receive a create, modify data or metadata events of the abore kinds
|
// When we receive a create, modify data or metadata events of the above kinds
|
||||||
// we just mark the file to be updated in a near future
|
// we just mark the file to be updated in a near future
|
||||||
// each consecutive event of these kinds that we receive for the same file
|
// each consecutive event of these kinds that we receive for the same file
|
||||||
// we just store the path again in the map below, with a new instant
|
// we just store the path again in the map below, with a new instant
|
||||||
// that effectively resets the timer for the file to be updated <- Copied from macos.rs
|
// that effectively resets the timer for the file to be updated <- Copied from macos.rs
|
||||||
let path = paths.remove(0);
|
let path = paths.remove(0);
|
||||||
|
|
||||||
if self.files_to_update.contains_key(&path) {
|
if self.files_to_update.contains_key(&path) {
|
||||||
if let Some(old_instant) =
|
if let Some(old_instant) =
|
||||||
self.files_to_update.insert(path.clone(), Instant::now())
|
self.files_to_update.insert(path.clone(), Instant::now())
|
||||||
|
@ -118,6 +130,7 @@ impl<'lib> EventHandler<'lib> for IosEventHandler<'lib> {
|
||||||
self.files_to_update.insert(path, Instant::now());
|
self.files_to_update.insert(path, Instant::now());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
EventKind::Modify(ModifyKind::Name(RenameMode::Any)) => {
|
EventKind::Modify(ModifyKind::Name(RenameMode::Any)) => {
|
||||||
self.handle_single_rename_event(paths.remove(0)).await?;
|
self.handle_single_rename_event(paths.remove(0)).await?;
|
||||||
}
|
}
|
||||||
|
@ -125,18 +138,22 @@ impl<'lib> EventHandler<'lib> for IosEventHandler<'lib> {
|
||||||
// For some reason, iOS doesn't have a Delete Event, so the vent type comes up as this.
|
// For some reason, iOS doesn't have a Delete Event, so the vent type comes up as this.
|
||||||
// Delete Event
|
// Delete Event
|
||||||
EventKind::Modify(ModifyKind::Metadata(MetadataKind::Any)) => {
|
EventKind::Modify(ModifyKind::Metadata(MetadataKind::Any)) => {
|
||||||
debug!("File has been deleted: {:#?}", paths);
|
|
||||||
let path = paths.remove(0);
|
let path = paths.remove(0);
|
||||||
|
|
||||||
|
trace!(path = %path.display(), "File has been deleted;");
|
||||||
|
|
||||||
if let Some(parent) = path.parent() {
|
if let Some(parent) = path.parent() {
|
||||||
if parent != Path::new("") {
|
if parent != Path::new("") {
|
||||||
self.to_recalculate_size
|
self.to_recalculate_size
|
||||||
.insert(parent.to_path_buf(), Instant::now());
|
.insert(parent.to_path_buf(), Instant::now());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
remove(self.location_id, &path, self.library).await?; //FIXME: Find out why this freezes the watcher
|
|
||||||
|
remove(self.location_id, &path, &self.library).await?; //FIXME: Find out why this freezes the watcher
|
||||||
}
|
}
|
||||||
other_event_kind => {
|
|
||||||
trace!("Other iOS event that we don't handle for now: {other_event_kind:#?}");
|
_ => {
|
||||||
|
trace!("Other iOS event that we don't handle for now");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -146,16 +163,19 @@ impl<'lib> EventHandler<'lib> for IosEventHandler<'lib> {
|
||||||
async fn tick(&mut self) {
|
async fn tick(&mut self) {
|
||||||
if self.last_events_eviction_check.elapsed() > HUNDRED_MILLIS {
|
if self.last_events_eviction_check.elapsed() > HUNDRED_MILLIS {
|
||||||
if let Err(e) = self.handle_to_update_eviction().await {
|
if let Err(e) = self.handle_to_update_eviction().await {
|
||||||
error!("Error while handling recently created or update files eviction: {e:#?}");
|
error!(
|
||||||
|
?e,
|
||||||
|
"Error while handling recently created or update files eviction;"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Cleaning out recently renamed files that are older than 100 milliseconds
|
// Cleaning out recently renamed files that are older than 100 milliseconds
|
||||||
if let Err(e) = self.handle_rename_create_eviction().await {
|
if let Err(e) = self.handle_rename_create_eviction().await {
|
||||||
error!("Failed to create file_path on iOS : {e:#?}");
|
error!(?e, "Failed to create file_path on iOS;");
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Err(e) = self.handle_rename_remove_eviction().await {
|
if let Err(e) = self.handle_rename_remove_eviction().await {
|
||||||
error!("Failed to remove file_path: {e:#?}");
|
error!(?e, "Failed to remove file_path;");
|
||||||
}
|
}
|
||||||
|
|
||||||
if !self.to_recalculate_size.is_empty() {
|
if !self.to_recalculate_size.is_empty() {
|
||||||
|
@ -163,11 +183,11 @@ impl<'lib> EventHandler<'lib> for IosEventHandler<'lib> {
|
||||||
&mut self.to_recalculate_size,
|
&mut self.to_recalculate_size,
|
||||||
&mut self.path_and_instant_buffer,
|
&mut self.path_and_instant_buffer,
|
||||||
self.location_id,
|
self.location_id,
|
||||||
self.library,
|
&self.library,
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
error!("Failed to recalculate directories size: {e:#?}");
|
error!(?e, "Failed to recalculate directories size;");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -176,7 +196,7 @@ impl<'lib> EventHandler<'lib> for IosEventHandler<'lib> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl IosEventHandler<'_> {
|
impl EventHandler {
|
||||||
async fn handle_to_update_eviction(&mut self) -> Result<(), LocationManagerError> {
|
async fn handle_to_update_eviction(&mut self) -> Result<(), LocationManagerError> {
|
||||||
self.path_and_instant_buffer.clear();
|
self.path_and_instant_buffer.clear();
|
||||||
let mut should_invalidate = false;
|
let mut should_invalidate = false;
|
||||||
|
@ -191,8 +211,11 @@ impl IosEventHandler<'_> {
|
||||||
.insert(parent.to_path_buf(), Instant::now());
|
.insert(parent.to_path_buf(), Instant::now());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
self.reincident_to_update_files.remove(&path);
|
self.reincident_to_update_files.remove(&path);
|
||||||
update_file(self.location_id, &path, self.node, self.library).await?;
|
|
||||||
|
update_file(self.location_id, &path, &self.node, &self.library).await?;
|
||||||
|
|
||||||
should_invalidate = true;
|
should_invalidate = true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -215,8 +238,11 @@ impl IosEventHandler<'_> {
|
||||||
.insert(parent.to_path_buf(), Instant::now());
|
.insert(parent.to_path_buf(), Instant::now());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
self.files_to_update.remove(&path);
|
self.files_to_update.remove(&path);
|
||||||
update_file(self.location_id, &path, self.node, self.library).await?;
|
|
||||||
|
update_file(self.location_id, &path, &self.node, &self.library).await?;
|
||||||
|
|
||||||
should_invalidate = true;
|
should_invalidate = true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -246,8 +272,14 @@ impl IosEventHandler<'_> {
|
||||||
if metadata.is_dir() {
|
if metadata.is_dir() {
|
||||||
// Don't need to dispatch a recalculate directory event as `create_dir` dispatches
|
// Don't need to dispatch a recalculate directory event as `create_dir` dispatches
|
||||||
// a `scan_location_sub_path` function, which recalculates the size already
|
// a `scan_location_sub_path` function, which recalculates the size already
|
||||||
create_dir(self.location_id, &path, &metadata, self.node, self.library)
|
create_dir(
|
||||||
.await?;
|
self.location_id,
|
||||||
|
&path,
|
||||||
|
&metadata,
|
||||||
|
&self.node,
|
||||||
|
&self.library,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
} else {
|
} else {
|
||||||
if let Some(parent) = path.parent() {
|
if let Some(parent) = path.parent() {
|
||||||
if parent != Path::new("") {
|
if parent != Path::new("") {
|
||||||
|
@ -255,11 +287,19 @@ impl IosEventHandler<'_> {
|
||||||
.insert(parent.to_path_buf(), Instant::now());
|
.insert(parent.to_path_buf(), Instant::now());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
create_file(self.location_id, &path, &metadata, self.node, self.library)
|
|
||||||
.await?;
|
create_file(
|
||||||
|
self.location_id,
|
||||||
|
&path,
|
||||||
|
&metadata,
|
||||||
|
&self.node,
|
||||||
|
&self.library,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
}
|
}
|
||||||
|
|
||||||
trace!("Created file_path due timeout: {}", path.display());
|
trace!(path = %path.display(), "Created file_path due timeout;");
|
||||||
|
|
||||||
should_invalidate = true;
|
should_invalidate = true;
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
|
@ -289,8 +329,11 @@ impl IosEventHandler<'_> {
|
||||||
.insert(parent.to_path_buf(), Instant::now());
|
.insert(parent.to_path_buf(), Instant::now());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
remove(self.location_id, &path, self.library).await?;
|
|
||||||
trace!("Removed file_path due timeout: {}", path.display());
|
remove(self.location_id, &path, &self.library).await?;
|
||||||
|
|
||||||
|
trace!(path = %path.display(), "Removed file_path due timeout;");
|
||||||
|
|
||||||
should_invalidate = true;
|
should_invalidate = true;
|
||||||
} else {
|
} else {
|
||||||
self.paths_map_buffer.push((inode, (instant, path)));
|
self.paths_map_buffer.push((inode, (instant, path)));
|
||||||
|
@ -313,10 +356,10 @@ impl IosEventHandler<'_> {
|
||||||
match fs::metadata(&path).await {
|
match fs::metadata(&path).await {
|
||||||
Ok(meta) => {
|
Ok(meta) => {
|
||||||
// File or directory exists, so this can be a "new path" to an actual rename/move or a creation
|
// File or directory exists, so this can be a "new path" to an actual rename/move or a creation
|
||||||
trace!("Path exists: {}", path.display());
|
trace!(path = %path.display(), "Path exists;");
|
||||||
|
|
||||||
let inode = get_inode(&meta);
|
let inode = get_inode(&meta);
|
||||||
let location_path = extract_location_path(self.location_id, self.library).await?;
|
let location_path = extract_location_path(self.location_id, &self.library).await?;
|
||||||
|
|
||||||
if !check_file_path_exists::<FilePathError>(
|
if !check_file_path_exists::<FilePathError>(
|
||||||
&IsolatedFilePathData::new(
|
&IsolatedFilePathData::new(
|
||||||
|
@ -331,21 +374,22 @@ impl IosEventHandler<'_> {
|
||||||
{
|
{
|
||||||
if let Some((_, old_path)) = self.old_paths_map.remove(&inode) {
|
if let Some((_, old_path)) = self.old_paths_map.remove(&inode) {
|
||||||
trace!(
|
trace!(
|
||||||
"Got a match new -> old: {} -> {}",
|
old_path = %old_path.display(),
|
||||||
path.display(),
|
new_path = %path.display(),
|
||||||
old_path.display()
|
"Got a match new -> old;",
|
||||||
);
|
);
|
||||||
|
|
||||||
// We found a new path for this old path, so we can rename it
|
// We found a new path for this old path, so we can rename it
|
||||||
rename(self.location_id, &path, &old_path, meta, self.library).await?;
|
rename(self.location_id, &path, &old_path, meta, &self.library).await?;
|
||||||
} else {
|
} else {
|
||||||
trace!("No match for new path yet: {}", path.display());
|
trace!(path = %path.display(), "No match for new path yet;");
|
||||||
|
|
||||||
self.new_paths_map.insert(inode, (Instant::now(), path));
|
self.new_paths_map.insert(inode, (Instant::now(), path));
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
warn!(
|
warn!(
|
||||||
"Received rename event for a file that already exists in the database: {}",
|
path = %path.display(),
|
||||||
path.display()
|
"Received rename event for a file that already exists in the database;",
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -353,23 +397,25 @@ impl IosEventHandler<'_> {
|
||||||
// File or directory does not exist in the filesystem, if it exists in the database,
|
// File or directory does not exist in the filesystem, if it exists in the database,
|
||||||
// then we try pairing it with the old path from our map
|
// then we try pairing it with the old path from our map
|
||||||
|
|
||||||
trace!("Path doesn't exists: {}", path.display());
|
trace!(path = %path.display(), "Path doesn't exists;");
|
||||||
|
|
||||||
let inode =
|
let inode =
|
||||||
match extract_inode_from_path(self.location_id, &path, self.library).await {
|
match extract_inode_from_path(self.location_id, &path, &self.library).await {
|
||||||
Ok(inode) => inode,
|
Ok(inode) => inode,
|
||||||
|
|
||||||
Err(LocationManagerError::FilePath(FilePathError::NotFound(_))) => {
|
Err(LocationManagerError::FilePath(FilePathError::NotFound(_))) => {
|
||||||
// temporary file, we can ignore it
|
// temporary file, we can ignore it
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
|
|
||||||
Err(e) => return Err(e),
|
Err(e) => return Err(e),
|
||||||
};
|
};
|
||||||
|
|
||||||
if let Some((_, new_path)) = self.new_paths_map.remove(&inode) {
|
if let Some((_, new_path)) = self.new_paths_map.remove(&inode) {
|
||||||
trace!(
|
trace!(
|
||||||
"Got a match old -> new: {} -> {}",
|
old_path = %path.display(),
|
||||||
path.display(),
|
new_path = %new_path.display(),
|
||||||
new_path.display()
|
"Got a match old -> new;",
|
||||||
);
|
);
|
||||||
|
|
||||||
// We found a new path for this old path, so we can rename it
|
// We found a new path for this old path, so we can rename it
|
||||||
|
@ -380,11 +426,12 @@ impl IosEventHandler<'_> {
|
||||||
fs::metadata(&new_path)
|
fs::metadata(&new_path)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| FileIOError::from((&new_path, e)))?,
|
.map_err(|e| FileIOError::from((&new_path, e)))?,
|
||||||
self.library,
|
&self.library,
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
} else {
|
} else {
|
||||||
trace!("No match for old path yet: {}", path.display());
|
trace!(path = %path.display(), "No match for old path yet;");
|
||||||
|
|
||||||
// We didn't find a new path for this old path, so we store ir for later
|
// We didn't find a new path for this old path, so we store ir for later
|
||||||
self.old_paths_map.insert(inode, (Instant::now(), path));
|
self.old_paths_map.insert(inode, (Instant::now(), path));
|
||||||
}
|
}
|
||||||
|
|
|
@ -17,40 +17,35 @@ use std::{
|
||||||
sync::Arc,
|
sync::Arc,
|
||||||
};
|
};
|
||||||
|
|
||||||
use async_trait::async_trait;
|
|
||||||
use notify::{
|
use notify::{
|
||||||
event::{CreateKind, DataChange, ModifyKind, RenameMode},
|
event::{CreateKind, DataChange, ModifyKind, RenameMode},
|
||||||
Event, EventKind,
|
Event, EventKind,
|
||||||
};
|
};
|
||||||
use tokio::{fs, time::Instant};
|
use tokio::{fs, time::Instant};
|
||||||
use tracing::{error, trace};
|
use tracing::{error, instrument, trace};
|
||||||
|
|
||||||
use super::{
|
use super::{
|
||||||
utils::{create_dir, recalculate_directories_size, remove, rename, update_file},
|
utils::{create_dir, recalculate_directories_size, remove, rename, update_file},
|
||||||
EventHandler, HUNDRED_MILLIS, ONE_SECOND,
|
HUNDRED_MILLIS, ONE_SECOND,
|
||||||
};
|
};
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
pub(super) struct LinuxEventHandler<'lib> {
|
pub(super) struct EventHandler {
|
||||||
location_id: location::id::Type,
|
location_id: location::id::Type,
|
||||||
library: &'lib Arc<Library>,
|
library: Arc<Library>,
|
||||||
node: &'lib Arc<Node>,
|
node: Arc<Node>,
|
||||||
last_events_eviction_check: Instant,
|
last_events_eviction_check: Instant,
|
||||||
rename_from: HashMap<PathBuf, Instant>,
|
rename_from: HashMap<PathBuf, Instant>,
|
||||||
recently_renamed_from: BTreeMap<PathBuf, Instant>,
|
recently_renamed_from: BTreeMap<PathBuf, Instant>,
|
||||||
files_to_update: HashMap<PathBuf, Instant>,
|
files_to_update: HashMap<PathBuf, Instant>,
|
||||||
reincident_to_update_files: HashMap<PathBuf, Instant>,
|
reincident_to_update_files: HashMap<PathBuf, Instant>,
|
||||||
to_recalculate_size: HashMap<PathBuf, Instant>,
|
to_recalculate_size: HashMap<PathBuf, Instant>,
|
||||||
|
|
||||||
path_and_instant_buffer: Vec<(PathBuf, Instant)>,
|
path_and_instant_buffer: Vec<(PathBuf, Instant)>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[async_trait]
|
impl super::EventHandler for EventHandler {
|
||||||
impl<'lib> EventHandler<'lib> for LinuxEventHandler<'lib> {
|
fn new(location_id: location::id::Type, library: Arc<Library>, node: Arc<Node>) -> Self {
|
||||||
fn new(
|
|
||||||
location_id: location::id::Type,
|
|
||||||
library: &'lib Arc<Library>,
|
|
||||||
node: &'lib Arc<Node>,
|
|
||||||
) -> Self {
|
|
||||||
Self {
|
Self {
|
||||||
location_id,
|
location_id,
|
||||||
library,
|
library,
|
||||||
|
@ -65,8 +60,19 @@ impl<'lib> EventHandler<'lib> for LinuxEventHandler<'lib> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[instrument(
|
||||||
|
skip_all,
|
||||||
|
fields(
|
||||||
|
location_id = %self.location_id,
|
||||||
|
library_id = %self.library.id,
|
||||||
|
waiting_rename_count = %self.recently_renamed_from.len(),
|
||||||
|
waiting_update_count = %self.files_to_update.len(),
|
||||||
|
reincident_to_update_files_count = %self.reincident_to_update_files.len(),
|
||||||
|
waiting_size_count = %self.to_recalculate_size.len(),
|
||||||
|
),
|
||||||
|
)]
|
||||||
async fn handle_event(&mut self, event: Event) -> Result<(), LocationManagerError> {
|
async fn handle_event(&mut self, event: Event) -> Result<(), LocationManagerError> {
|
||||||
trace!("Received Linux event: {:#?}", event);
|
trace!("Received Linux event");
|
||||||
|
|
||||||
let Event {
|
let Event {
|
||||||
kind, mut paths, ..
|
kind, mut paths, ..
|
||||||
|
@ -81,6 +87,7 @@ impl<'lib> EventHandler<'lib> for LinuxEventHandler<'lib> {
|
||||||
// we just store the path again in the map below, with a new instant
|
// we just store the path again in the map below, with a new instant
|
||||||
// that effectively resets the timer for the file to be updated
|
// that effectively resets the timer for the file to be updated
|
||||||
let path = paths.remove(0);
|
let path = paths.remove(0);
|
||||||
|
|
||||||
if self.files_to_update.contains_key(&path) {
|
if self.files_to_update.contains_key(&path) {
|
||||||
if let Some(old_instant) =
|
if let Some(old_instant) =
|
||||||
self.files_to_update.insert(path.clone(), Instant::now())
|
self.files_to_update.insert(path.clone(), Instant::now())
|
||||||
|
@ -95,22 +102,23 @@ impl<'lib> EventHandler<'lib> for LinuxEventHandler<'lib> {
|
||||||
}
|
}
|
||||||
|
|
||||||
EventKind::Create(CreateKind::Folder) => {
|
EventKind::Create(CreateKind::Folder) => {
|
||||||
let path = &paths[0];
|
let path = paths.remove(0);
|
||||||
|
|
||||||
// Don't need to dispatch a recalculate directory event as `create_dir` dispatches
|
// Don't need to dispatch a recalculate directory event as `create_dir` dispatches
|
||||||
// a `scan_location_sub_path` function, which recalculates the size already
|
// a `scan_location_sub_path` function, which recalculates the size already
|
||||||
|
|
||||||
create_dir(
|
create_dir(
|
||||||
self.location_id,
|
self.location_id,
|
||||||
path,
|
&path,
|
||||||
&fs::metadata(path)
|
&fs::metadata(&path)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| FileIOError::from((path, e)))?,
|
.map_err(|e| FileIOError::from((&path, e)))?,
|
||||||
self.node,
|
&self.node,
|
||||||
self.library,
|
&self.library,
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
}
|
}
|
||||||
|
|
||||||
EventKind::Modify(ModifyKind::Name(RenameMode::From)) => {
|
EventKind::Modify(ModifyKind::Name(RenameMode::From)) => {
|
||||||
// Just in case we can't guarantee that we receive the Rename From event before the
|
// Just in case we can't guarantee that we receive the Rename From event before the
|
||||||
// Rename Both event. Just a safeguard
|
// Rename Both event. Just a safeguard
|
||||||
|
@ -120,23 +128,24 @@ impl<'lib> EventHandler<'lib> for LinuxEventHandler<'lib> {
|
||||||
}
|
}
|
||||||
|
|
||||||
EventKind::Modify(ModifyKind::Name(RenameMode::Both)) => {
|
EventKind::Modify(ModifyKind::Name(RenameMode::Both)) => {
|
||||||
let from_path = &paths[0];
|
let to_path = paths.remove(1);
|
||||||
let to_path = &paths[1];
|
let from_path = paths.remove(0);
|
||||||
|
|
||||||
self.rename_from.remove(from_path);
|
self.rename_from.remove(&from_path);
|
||||||
rename(
|
rename(
|
||||||
self.location_id,
|
self.location_id,
|
||||||
to_path,
|
&to_path,
|
||||||
from_path,
|
&from_path,
|
||||||
fs::metadata(to_path)
|
fs::metadata(&to_path)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| FileIOError::from((to_path, e)))?,
|
.map_err(|e| FileIOError::from((&to_path, e)))?,
|
||||||
self.library,
|
&self.library,
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
self.recently_renamed_from
|
|
||||||
.insert(paths.swap_remove(0), Instant::now());
|
self.recently_renamed_from.insert(from_path, Instant::now());
|
||||||
}
|
}
|
||||||
|
|
||||||
EventKind::Remove(_) => {
|
EventKind::Remove(_) => {
|
||||||
let path = paths.remove(0);
|
let path = paths.remove(0);
|
||||||
if let Some(parent) = path.parent() {
|
if let Some(parent) = path.parent() {
|
||||||
|
@ -146,10 +155,11 @@ impl<'lib> EventHandler<'lib> for LinuxEventHandler<'lib> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
remove(self.location_id, &path, self.library).await?;
|
remove(self.location_id, &path, &self.library).await?;
|
||||||
}
|
}
|
||||||
other_event_kind => {
|
|
||||||
trace!("Other Linux event that we don't handle for now: {other_event_kind:#?}");
|
_ => {
|
||||||
|
trace!("Other Linux event that we don't handle for now");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -159,11 +169,14 @@ impl<'lib> EventHandler<'lib> for LinuxEventHandler<'lib> {
|
||||||
async fn tick(&mut self) {
|
async fn tick(&mut self) {
|
||||||
if self.last_events_eviction_check.elapsed() > HUNDRED_MILLIS {
|
if self.last_events_eviction_check.elapsed() > HUNDRED_MILLIS {
|
||||||
if let Err(e) = self.handle_to_update_eviction().await {
|
if let Err(e) = self.handle_to_update_eviction().await {
|
||||||
error!("Error while handling recently created or update files eviction: {e:#?}");
|
error!(
|
||||||
|
?e,
|
||||||
|
"Error while handling recently created or update files eviction;"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Err(e) = self.handle_rename_from_eviction().await {
|
if let Err(e) = self.handle_rename_from_eviction().await {
|
||||||
error!("Failed to remove file_path: {e:#?}");
|
error!(?e, "Failed to remove file_path;");
|
||||||
}
|
}
|
||||||
|
|
||||||
self.recently_renamed_from
|
self.recently_renamed_from
|
||||||
|
@ -174,11 +187,11 @@ impl<'lib> EventHandler<'lib> for LinuxEventHandler<'lib> {
|
||||||
&mut self.to_recalculate_size,
|
&mut self.to_recalculate_size,
|
||||||
&mut self.path_and_instant_buffer,
|
&mut self.path_and_instant_buffer,
|
||||||
self.location_id,
|
self.location_id,
|
||||||
self.library,
|
&self.library,
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
error!("Failed to recalculate directories size: {e:#?}");
|
error!(?e, "Failed to recalculate directories size;");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -187,9 +200,10 @@ impl<'lib> EventHandler<'lib> for LinuxEventHandler<'lib> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl LinuxEventHandler<'_> {
|
impl EventHandler {
|
||||||
async fn handle_to_update_eviction(&mut self) -> Result<(), LocationManagerError> {
|
async fn handle_to_update_eviction(&mut self) -> Result<(), LocationManagerError> {
|
||||||
self.path_and_instant_buffer.clear();
|
self.path_and_instant_buffer.clear();
|
||||||
|
|
||||||
let mut should_invalidate = false;
|
let mut should_invalidate = false;
|
||||||
|
|
||||||
for (path, created_at) in self.files_to_update.drain() {
|
for (path, created_at) in self.files_to_update.drain() {
|
||||||
|
@ -202,8 +216,11 @@ impl LinuxEventHandler<'_> {
|
||||||
.insert(parent.to_path_buf(), Instant::now());
|
.insert(parent.to_path_buf(), Instant::now());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
self.reincident_to_update_files.remove(&path);
|
self.reincident_to_update_files.remove(&path);
|
||||||
update_file(self.location_id, &path, self.node, self.library).await?;
|
|
||||||
|
update_file(self.location_id, &path, &self.node, &self.library).await?;
|
||||||
|
|
||||||
should_invalidate = true;
|
should_invalidate = true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -226,8 +243,11 @@ impl LinuxEventHandler<'_> {
|
||||||
.insert(parent.to_path_buf(), Instant::now());
|
.insert(parent.to_path_buf(), Instant::now());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
self.files_to_update.remove(&path);
|
self.files_to_update.remove(&path);
|
||||||
update_file(self.location_id, &path, self.node, self.library).await?;
|
|
||||||
|
update_file(self.location_id, &path, &self.node, &self.library).await?;
|
||||||
|
|
||||||
should_invalidate = true;
|
should_invalidate = true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -244,6 +264,7 @@ impl LinuxEventHandler<'_> {
|
||||||
|
|
||||||
async fn handle_rename_from_eviction(&mut self) -> Result<(), LocationManagerError> {
|
async fn handle_rename_from_eviction(&mut self) -> Result<(), LocationManagerError> {
|
||||||
self.path_and_instant_buffer.clear();
|
self.path_and_instant_buffer.clear();
|
||||||
|
|
||||||
let mut should_invalidate = false;
|
let mut should_invalidate = false;
|
||||||
|
|
||||||
for (path, instant) in self.rename_from.drain() {
|
for (path, instant) in self.rename_from.drain() {
|
||||||
|
@ -254,9 +275,12 @@ impl LinuxEventHandler<'_> {
|
||||||
.insert(parent.to_path_buf(), Instant::now());
|
.insert(parent.to_path_buf(), Instant::now());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
remove(self.location_id, &path, self.library).await?;
|
|
||||||
|
remove(self.location_id, &path, &self.library).await?;
|
||||||
|
|
||||||
should_invalidate = true;
|
should_invalidate = true;
|
||||||
trace!("Removed file_path due timeout: {}", path.display());
|
|
||||||
|
trace!(path = %path.display(), "Removed file_path due timeout;");
|
||||||
} else {
|
} else {
|
||||||
self.path_and_instant_buffer.push((path, instant));
|
self.path_and_instant_buffer.push((path, instant));
|
||||||
}
|
}
|
||||||
|
@ -266,9 +290,8 @@ impl LinuxEventHandler<'_> {
|
||||||
invalidate_query!(self.library, "search.paths");
|
invalidate_query!(self.library, "search.paths");
|
||||||
}
|
}
|
||||||
|
|
||||||
for (path, instant) in self.path_and_instant_buffer.drain(..) {
|
self.rename_from
|
||||||
self.rename_from.insert(path, instant);
|
.extend(self.path_and_instant_buffer.drain(..));
|
||||||
}
|
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
|
@ -24,45 +24,40 @@ use std::{
|
||||||
sync::Arc,
|
sync::Arc,
|
||||||
};
|
};
|
||||||
|
|
||||||
use async_trait::async_trait;
|
|
||||||
use notify::{
|
use notify::{
|
||||||
event::{CreateKind, DataChange, MetadataKind, ModifyKind, RenameMode},
|
event::{CreateKind, DataChange, MetadataKind, ModifyKind, RenameMode},
|
||||||
Event, EventKind,
|
Event, EventKind,
|
||||||
};
|
};
|
||||||
use tokio::{fs, io, time::Instant};
|
use tokio::{fs, io, time::Instant};
|
||||||
use tracing::{error, trace, warn};
|
use tracing::{error, instrument, trace, warn};
|
||||||
|
|
||||||
use super::{
|
use super::{
|
||||||
utils::{
|
utils::{
|
||||||
create_dir, create_file, extract_inode_from_path, extract_location_path,
|
create_dir, create_file, extract_inode_from_path, extract_location_path,
|
||||||
recalculate_directories_size, remove, rename, update_file,
|
recalculate_directories_size, remove, rename, update_file,
|
||||||
},
|
},
|
||||||
EventHandler, INode, InstantAndPath, HUNDRED_MILLIS, ONE_SECOND,
|
INode, InstantAndPath, HUNDRED_MILLIS, ONE_SECOND,
|
||||||
};
|
};
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
pub(super) struct MacOsEventHandler<'lib> {
|
pub(super) struct EventHandler {
|
||||||
location_id: location::id::Type,
|
location_id: location::id::Type,
|
||||||
library: &'lib Arc<Library>,
|
library: Arc<Library>,
|
||||||
node: &'lib Arc<Node>,
|
node: Arc<Node>,
|
||||||
files_to_update: HashMap<PathBuf, Instant>,
|
|
||||||
reincident_to_update_files: HashMap<PathBuf, Instant>,
|
|
||||||
last_events_eviction_check: Instant,
|
last_events_eviction_check: Instant,
|
||||||
latest_created_dir: Option<PathBuf>,
|
latest_created_dir: Option<PathBuf>,
|
||||||
old_paths_map: HashMap<INode, InstantAndPath>,
|
old_paths_map: HashMap<INode, InstantAndPath>,
|
||||||
new_paths_map: HashMap<INode, InstantAndPath>,
|
new_paths_map: HashMap<INode, InstantAndPath>,
|
||||||
paths_map_buffer: Vec<(INode, InstantAndPath)>,
|
files_to_update: HashMap<PathBuf, Instant>,
|
||||||
|
reincident_to_update_files: HashMap<PathBuf, Instant>,
|
||||||
to_recalculate_size: HashMap<PathBuf, Instant>,
|
to_recalculate_size: HashMap<PathBuf, Instant>,
|
||||||
|
|
||||||
path_and_instant_buffer: Vec<(PathBuf, Instant)>,
|
path_and_instant_buffer: Vec<(PathBuf, Instant)>,
|
||||||
|
paths_map_buffer: Vec<(INode, InstantAndPath)>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[async_trait]
|
impl super::EventHandler for EventHandler {
|
||||||
impl<'lib> EventHandler<'lib> for MacOsEventHandler<'lib> {
|
fn new(location_id: location::id::Type, library: Arc<Library>, node: Arc<Node>) -> Self
|
||||||
fn new(
|
|
||||||
location_id: location::id::Type,
|
|
||||||
library: &'lib Arc<Library>,
|
|
||||||
node: &'lib Arc<Node>,
|
|
||||||
) -> Self
|
|
||||||
where
|
where
|
||||||
Self: Sized,
|
Self: Sized,
|
||||||
{
|
{
|
||||||
|
@ -70,20 +65,33 @@ impl<'lib> EventHandler<'lib> for MacOsEventHandler<'lib> {
|
||||||
location_id,
|
location_id,
|
||||||
library,
|
library,
|
||||||
node,
|
node,
|
||||||
files_to_update: HashMap::new(),
|
|
||||||
reincident_to_update_files: HashMap::new(),
|
|
||||||
last_events_eviction_check: Instant::now(),
|
last_events_eviction_check: Instant::now(),
|
||||||
latest_created_dir: None,
|
latest_created_dir: None,
|
||||||
old_paths_map: HashMap::new(),
|
old_paths_map: HashMap::new(),
|
||||||
new_paths_map: HashMap::new(),
|
new_paths_map: HashMap::new(),
|
||||||
paths_map_buffer: Vec::new(),
|
files_to_update: HashMap::new(),
|
||||||
|
reincident_to_update_files: HashMap::new(),
|
||||||
to_recalculate_size: HashMap::new(),
|
to_recalculate_size: HashMap::new(),
|
||||||
path_and_instant_buffer: Vec::new(),
|
path_and_instant_buffer: Vec::new(),
|
||||||
|
paths_map_buffer: Vec::new(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[instrument(
|
||||||
|
skip_all,
|
||||||
|
fields(
|
||||||
|
location_id = %self.location_id,
|
||||||
|
library_id = %self.library.id,
|
||||||
|
latest_created_dir = ?self.latest_created_dir,
|
||||||
|
old_paths_map_count = %self.old_paths_map.len(),
|
||||||
|
new_paths_map = %self.new_paths_map.len(),
|
||||||
|
waiting_update_count = %self.files_to_update.len(),
|
||||||
|
reincident_to_update_files_count = %self.reincident_to_update_files.len(),
|
||||||
|
waiting_size_count = %self.to_recalculate_size.len(),
|
||||||
|
),
|
||||||
|
)]
|
||||||
async fn handle_event(&mut self, event: Event) -> Result<(), LocationManagerError> {
|
async fn handle_event(&mut self, event: Event) -> Result<(), LocationManagerError> {
|
||||||
trace!("Received MacOS event: {:#?}", event);
|
trace!("Received MacOS event");
|
||||||
|
|
||||||
let Event {
|
let Event {
|
||||||
kind, mut paths, ..
|
kind, mut paths, ..
|
||||||
|
@ -91,8 +99,9 @@ impl<'lib> EventHandler<'lib> for MacOsEventHandler<'lib> {
|
||||||
|
|
||||||
match kind {
|
match kind {
|
||||||
EventKind::Create(CreateKind::Folder) => {
|
EventKind::Create(CreateKind::Folder) => {
|
||||||
let path = &paths[0];
|
let path = paths.remove(0);
|
||||||
if let Some(ref latest_created_dir) = self.latest_created_dir.take() {
|
|
||||||
|
if let Some(latest_created_dir) = self.latest_created_dir.take() {
|
||||||
if path == latest_created_dir {
|
if path == latest_created_dir {
|
||||||
// NOTE: This is a MacOS specific event that happens when a folder is created
|
// NOTE: This is a MacOS specific event that happens when a folder is created
|
||||||
// trough Finder. It creates a folder but 2 events are triggered in
|
// trough Finder. It creates a folder but 2 events are triggered in
|
||||||
|
@ -105,18 +114,27 @@ impl<'lib> EventHandler<'lib> for MacOsEventHandler<'lib> {
|
||||||
// Don't need to dispatch a recalculate directory event as `create_dir` dispatches
|
// Don't need to dispatch a recalculate directory event as `create_dir` dispatches
|
||||||
// a `scan_location_sub_path` function, which recalculates the size already
|
// a `scan_location_sub_path` function, which recalculates the size already
|
||||||
|
|
||||||
|
let metadata = match fs::metadata(&path).await {
|
||||||
|
Ok(metadata) => metadata,
|
||||||
|
Err(e) if e.kind() == io::ErrorKind::NotFound => {
|
||||||
|
// temporary file, bailing out
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
Err(e) => return Err(FileIOError::from((&path, e)).into()),
|
||||||
|
};
|
||||||
|
|
||||||
create_dir(
|
create_dir(
|
||||||
self.location_id,
|
self.location_id,
|
||||||
path,
|
&path,
|
||||||
&fs::metadata(path)
|
&metadata,
|
||||||
.await
|
&self.node,
|
||||||
.map_err(|e| FileIOError::from((path, e)))?,
|
&self.library,
|
||||||
self.node,
|
|
||||||
self.library,
|
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
self.latest_created_dir = Some(paths.remove(0));
|
|
||||||
|
self.latest_created_dir = Some(path);
|
||||||
}
|
}
|
||||||
|
|
||||||
EventKind::Create(CreateKind::File)
|
EventKind::Create(CreateKind::File)
|
||||||
| EventKind::Modify(ModifyKind::Data(DataChange::Content))
|
| EventKind::Modify(ModifyKind::Data(DataChange::Content))
|
||||||
| EventKind::Modify(ModifyKind::Metadata(
|
| EventKind::Modify(ModifyKind::Metadata(
|
||||||
|
@ -128,6 +146,7 @@ impl<'lib> EventHandler<'lib> for MacOsEventHandler<'lib> {
|
||||||
// we just store the path again in the map below, with a new instant
|
// we just store the path again in the map below, with a new instant
|
||||||
// that effectively resets the timer for the file to be updated
|
// that effectively resets the timer for the file to be updated
|
||||||
let path = paths.remove(0);
|
let path = paths.remove(0);
|
||||||
|
|
||||||
if self.files_to_update.contains_key(&path) {
|
if self.files_to_update.contains_key(&path) {
|
||||||
if let Some(old_instant) =
|
if let Some(old_instant) =
|
||||||
self.files_to_update.insert(path.clone(), Instant::now())
|
self.files_to_update.insert(path.clone(), Instant::now())
|
||||||
|
@ -140,22 +159,24 @@ impl<'lib> EventHandler<'lib> for MacOsEventHandler<'lib> {
|
||||||
self.files_to_update.insert(path, Instant::now());
|
self.files_to_update.insert(path, Instant::now());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
EventKind::Modify(ModifyKind::Name(RenameMode::Any)) => {
|
EventKind::Modify(ModifyKind::Name(RenameMode::Any)) => {
|
||||||
self.handle_single_rename_event(paths.remove(0)).await?;
|
self.handle_single_rename_event(paths.remove(0)).await?;
|
||||||
}
|
}
|
||||||
|
|
||||||
EventKind::Remove(_) => {
|
EventKind::Remove(_) => {
|
||||||
let path = paths.remove(0);
|
let path = paths.remove(0);
|
||||||
|
|
||||||
if let Some(parent) = path.parent() {
|
if let Some(parent) = path.parent() {
|
||||||
if parent != Path::new("") {
|
if parent != Path::new("") {
|
||||||
self.to_recalculate_size
|
self.to_recalculate_size
|
||||||
.insert(parent.to_path_buf(), Instant::now());
|
.insert(parent.to_path_buf(), Instant::now());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
remove(self.location_id, &path, self.library).await?;
|
remove(self.location_id, &path, &self.library).await?;
|
||||||
}
|
}
|
||||||
other_event_kind => {
|
_ => {
|
||||||
trace!("Other MacOS event that we don't handle for now: {other_event_kind:#?}");
|
trace!("Other MacOS event that we don't handle for now");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -165,16 +186,19 @@ impl<'lib> EventHandler<'lib> for MacOsEventHandler<'lib> {
|
||||||
async fn tick(&mut self) {
|
async fn tick(&mut self) {
|
||||||
if self.last_events_eviction_check.elapsed() > HUNDRED_MILLIS {
|
if self.last_events_eviction_check.elapsed() > HUNDRED_MILLIS {
|
||||||
if let Err(e) = self.handle_to_update_eviction().await {
|
if let Err(e) = self.handle_to_update_eviction().await {
|
||||||
error!("Error while handling recently created or update files eviction: {e:#?}");
|
error!(
|
||||||
|
?e,
|
||||||
|
"Error while handling recently created or update files eviction;"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Cleaning out recently renamed files that are older than 100 milliseconds
|
// Cleaning out recently renamed files that are older than 100 milliseconds
|
||||||
if let Err(e) = self.handle_rename_create_eviction().await {
|
if let Err(e) = self.handle_rename_create_eviction().await {
|
||||||
error!("Failed to create file_path on MacOS : {e:#?}");
|
error!(?e, "Failed to create file_path on MacOS;");
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Err(e) = self.handle_rename_remove_eviction().await {
|
if let Err(e) = self.handle_rename_remove_eviction().await {
|
||||||
error!("Failed to remove file_path: {e:#?}");
|
error!(?e, "Failed to remove file_path;");
|
||||||
}
|
}
|
||||||
|
|
||||||
if !self.to_recalculate_size.is_empty() {
|
if !self.to_recalculate_size.is_empty() {
|
||||||
|
@ -182,11 +206,11 @@ impl<'lib> EventHandler<'lib> for MacOsEventHandler<'lib> {
|
||||||
&mut self.to_recalculate_size,
|
&mut self.to_recalculate_size,
|
||||||
&mut self.path_and_instant_buffer,
|
&mut self.path_and_instant_buffer,
|
||||||
self.location_id,
|
self.location_id,
|
||||||
self.library,
|
&self.library,
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
error!("Failed to recalculate directories size: {e:#?}");
|
error!(?e, "Failed to recalculate directories size;");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -195,9 +219,10 @@ impl<'lib> EventHandler<'lib> for MacOsEventHandler<'lib> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl MacOsEventHandler<'_> {
|
impl EventHandler {
|
||||||
async fn handle_to_update_eviction(&mut self) -> Result<(), LocationManagerError> {
|
async fn handle_to_update_eviction(&mut self) -> Result<(), LocationManagerError> {
|
||||||
self.path_and_instant_buffer.clear();
|
self.path_and_instant_buffer.clear();
|
||||||
|
|
||||||
let mut should_invalidate = false;
|
let mut should_invalidate = false;
|
||||||
|
|
||||||
for (path, created_at) in self.files_to_update.drain() {
|
for (path, created_at) in self.files_to_update.drain() {
|
||||||
|
@ -211,7 +236,7 @@ impl MacOsEventHandler<'_> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
self.reincident_to_update_files.remove(&path);
|
self.reincident_to_update_files.remove(&path);
|
||||||
update_file(self.location_id, &path, self.node, self.library).await?;
|
update_file(self.location_id, &path, &self.node, &self.library).await?;
|
||||||
should_invalidate = true;
|
should_invalidate = true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -235,7 +260,7 @@ impl MacOsEventHandler<'_> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
self.files_to_update.remove(&path);
|
self.files_to_update.remove(&path);
|
||||||
update_file(self.location_id, &path, self.node, self.library).await?;
|
update_file(self.location_id, &path, &self.node, &self.library).await?;
|
||||||
should_invalidate = true;
|
should_invalidate = true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -253,20 +278,32 @@ impl MacOsEventHandler<'_> {
|
||||||
async fn handle_rename_create_eviction(&mut self) -> Result<(), LocationManagerError> {
|
async fn handle_rename_create_eviction(&mut self) -> Result<(), LocationManagerError> {
|
||||||
// Just to make sure that our buffer is clean
|
// Just to make sure that our buffer is clean
|
||||||
self.paths_map_buffer.clear();
|
self.paths_map_buffer.clear();
|
||||||
|
|
||||||
let mut should_invalidate = false;
|
let mut should_invalidate = false;
|
||||||
|
|
||||||
for (inode, (instant, path)) in self.new_paths_map.drain() {
|
for (inode, (instant, path)) in self.new_paths_map.drain() {
|
||||||
if instant.elapsed() > HUNDRED_MILLIS {
|
if instant.elapsed() > HUNDRED_MILLIS {
|
||||||
if !self.files_to_update.contains_key(&path) {
|
if !self.files_to_update.contains_key(&path) {
|
||||||
let metadata = fs::metadata(&path)
|
let metadata = match fs::metadata(&path).await {
|
||||||
.await
|
Ok(metadata) => metadata,
|
||||||
.map_err(|e| FileIOError::from((&path, e)))?;
|
Err(e) if e.kind() == io::ErrorKind::NotFound => {
|
||||||
|
// temporary file, bailing out
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
Err(e) => return Err(FileIOError::from((&path, e)).into()),
|
||||||
|
};
|
||||||
|
|
||||||
if metadata.is_dir() {
|
if metadata.is_dir() {
|
||||||
// Don't need to dispatch a recalculate directory event as `create_dir` dispatches
|
// Don't need to dispatch a recalculate directory event as `create_dir` dispatches
|
||||||
// a `scan_location_sub_path` function, which recalculates the size already
|
// a `scan_location_sub_path` function, which recalculates the size already
|
||||||
create_dir(self.location_id, &path, &metadata, self.node, self.library)
|
create_dir(
|
||||||
.await?;
|
self.location_id,
|
||||||
|
&path,
|
||||||
|
&metadata,
|
||||||
|
&self.node,
|
||||||
|
&self.library,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
} else {
|
} else {
|
||||||
if let Some(parent) = path.parent() {
|
if let Some(parent) = path.parent() {
|
||||||
if parent != Path::new("") {
|
if parent != Path::new("") {
|
||||||
|
@ -274,11 +311,18 @@ impl MacOsEventHandler<'_> {
|
||||||
.insert(parent.to_path_buf(), Instant::now());
|
.insert(parent.to_path_buf(), Instant::now());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
create_file(self.location_id, &path, &metadata, self.node, self.library)
|
create_file(
|
||||||
.await?;
|
self.location_id,
|
||||||
|
&path,
|
||||||
|
&metadata,
|
||||||
|
&self.node,
|
||||||
|
&self.library,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
}
|
}
|
||||||
|
|
||||||
trace!("Created file_path due timeout: {}", path.display());
|
trace!(path = %path.display(), "Created file_path due timeout;");
|
||||||
|
|
||||||
should_invalidate = true;
|
should_invalidate = true;
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
|
@ -298,6 +342,7 @@ impl MacOsEventHandler<'_> {
|
||||||
async fn handle_rename_remove_eviction(&mut self) -> Result<(), LocationManagerError> {
|
async fn handle_rename_remove_eviction(&mut self) -> Result<(), LocationManagerError> {
|
||||||
// Just to make sure that our buffer is clean
|
// Just to make sure that our buffer is clean
|
||||||
self.paths_map_buffer.clear();
|
self.paths_map_buffer.clear();
|
||||||
|
|
||||||
let mut should_invalidate = false;
|
let mut should_invalidate = false;
|
||||||
|
|
||||||
for (inode, (instant, path)) in self.old_paths_map.drain() {
|
for (inode, (instant, path)) in self.old_paths_map.drain() {
|
||||||
|
@ -308,8 +353,11 @@ impl MacOsEventHandler<'_> {
|
||||||
.insert(parent.to_path_buf(), Instant::now());
|
.insert(parent.to_path_buf(), Instant::now());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
remove(self.location_id, &path, self.library).await?;
|
|
||||||
trace!("Removed file_path due timeout: {}", path.display());
|
remove(self.location_id, &path, &self.library).await?;
|
||||||
|
|
||||||
|
trace!(path = %path.display(), "Removed file_path due timeout;");
|
||||||
|
|
||||||
should_invalidate = true;
|
should_invalidate = true;
|
||||||
} else {
|
} else {
|
||||||
self.paths_map_buffer.push((inode, (instant, path)));
|
self.paths_map_buffer.push((inode, (instant, path)));
|
||||||
|
@ -332,10 +380,10 @@ impl MacOsEventHandler<'_> {
|
||||||
match fs::metadata(&path).await {
|
match fs::metadata(&path).await {
|
||||||
Ok(meta) => {
|
Ok(meta) => {
|
||||||
// File or directory exists, so this can be a "new path" to an actual rename/move or a creation
|
// File or directory exists, so this can be a "new path" to an actual rename/move or a creation
|
||||||
trace!("Path exists: {}", path.display());
|
trace!(path = %path.display(), "Path exists;");
|
||||||
|
|
||||||
let inode = get_inode(&meta);
|
let inode = get_inode(&meta);
|
||||||
let location_path = extract_location_path(self.location_id, self.library).await?;
|
let location_path = extract_location_path(self.location_id, &self.library).await?;
|
||||||
|
|
||||||
if !check_file_path_exists::<FilePathError>(
|
if !check_file_path_exists::<FilePathError>(
|
||||||
&IsolatedFilePathData::new(
|
&IsolatedFilePathData::new(
|
||||||
|
@ -350,45 +398,49 @@ impl MacOsEventHandler<'_> {
|
||||||
{
|
{
|
||||||
if let Some((_, old_path)) = self.old_paths_map.remove(&inode) {
|
if let Some((_, old_path)) = self.old_paths_map.remove(&inode) {
|
||||||
trace!(
|
trace!(
|
||||||
"Got a match new -> old: {} -> {}",
|
new_path = %path.display(),
|
||||||
path.display(),
|
old_path = %old_path.display(),
|
||||||
old_path.display()
|
"Got a match new -> old;",
|
||||||
);
|
);
|
||||||
|
|
||||||
// We found a new path for this old path, so we can rename it
|
// We found a new path for this old path, so we can rename it
|
||||||
rename(self.location_id, &path, &old_path, meta, self.library).await?;
|
rename(self.location_id, &path, &old_path, meta, &self.library).await?;
|
||||||
} else {
|
} else {
|
||||||
trace!("No match for new path yet: {}", path.display());
|
trace!(path = %path.display(), "No match for new path yet;");
|
||||||
|
|
||||||
self.new_paths_map.insert(inode, (Instant::now(), path));
|
self.new_paths_map.insert(inode, (Instant::now(), path));
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
warn!(
|
warn!(
|
||||||
"Received rename event for a file that already exists in the database: {}",
|
path = %path.display(),
|
||||||
path.display()
|
"Received rename event for a file that already exists in the database;",
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Err(e) if e.kind() == io::ErrorKind::NotFound => {
|
Err(e) if e.kind() == io::ErrorKind::NotFound => {
|
||||||
// File or directory does not exist in the filesystem, if it exists in the database,
|
// File or directory does not exist in the filesystem, if it exists in the database,
|
||||||
// then we try pairing it with the old path from our map
|
// then we try pairing it with the old path from our map
|
||||||
|
|
||||||
trace!("Path doesn't exists: {}", path.display());
|
trace!(path = %path.display(), "Path doesn't exists;");
|
||||||
|
|
||||||
let inode =
|
let inode =
|
||||||
match extract_inode_from_path(self.location_id, &path, self.library).await {
|
match extract_inode_from_path(self.location_id, &path, &self.library).await {
|
||||||
Ok(inode) => inode,
|
Ok(inode) => inode,
|
||||||
|
|
||||||
Err(LocationManagerError::FilePath(FilePathError::NotFound(_))) => {
|
Err(LocationManagerError::FilePath(FilePathError::NotFound(_))) => {
|
||||||
// temporary file, we can ignore it
|
// temporary file, we can ignore it
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
|
|
||||||
Err(e) => return Err(e),
|
Err(e) => return Err(e),
|
||||||
};
|
};
|
||||||
|
|
||||||
if let Some((_, new_path)) = self.new_paths_map.remove(&inode) {
|
if let Some((_, new_path)) = self.new_paths_map.remove(&inode) {
|
||||||
trace!(
|
trace!(
|
||||||
"Got a match old -> new: {} -> {}",
|
old_path = %path.display(),
|
||||||
path.display(),
|
new_path = %new_path.display(),
|
||||||
new_path.display()
|
"Got a match old -> new;",
|
||||||
);
|
);
|
||||||
|
|
||||||
// We found a new path for this old path, so we can rename it
|
// We found a new path for this old path, so we can rename it
|
||||||
|
@ -399,15 +451,17 @@ impl MacOsEventHandler<'_> {
|
||||||
fs::metadata(&new_path)
|
fs::metadata(&new_path)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| FileIOError::from((&new_path, e)))?,
|
.map_err(|e| FileIOError::from((&new_path, e)))?,
|
||||||
self.library,
|
&self.library,
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
} else {
|
} else {
|
||||||
trace!("No match for old path yet: {}", path.display());
|
trace!(path = %path.display(), "No match for old path yet;");
|
||||||
|
|
||||||
// We didn't find a new path for this old path, so we store ir for later
|
// We didn't find a new path for this old path, so we store ir for later
|
||||||
self.old_paths_map.insert(inode, (Instant::now(), path));
|
self.old_paths_map.insert(inode, (Instant::now(), path));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Err(e) => return Err(FileIOError::from((path, e)).into()),
|
Err(e) => return Err(FileIOError::from((path, e)).into()),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -1,25 +1,31 @@
|
||||||
use crate::{library::Library, Node};
|
use crate::{library::Library, Node};
|
||||||
|
|
||||||
use sd_prisma::prisma::location;
|
use sd_core_indexer_rules::{IndexerRule, IndexerRuler};
|
||||||
|
use sd_core_prisma_helpers::{location_ids_and_path, location_with_indexer_rules};
|
||||||
|
|
||||||
|
use sd_prisma::prisma::{location, PrismaClient};
|
||||||
use sd_utils::db::maybe_missing;
|
use sd_utils::db::maybe_missing;
|
||||||
|
|
||||||
use std::{
|
use std::{
|
||||||
collections::HashSet,
|
collections::HashSet,
|
||||||
|
future::Future,
|
||||||
path::{Path, PathBuf},
|
path::{Path, PathBuf},
|
||||||
|
pin::pin,
|
||||||
sync::Arc,
|
sync::Arc,
|
||||||
time::Duration,
|
time::Duration,
|
||||||
};
|
};
|
||||||
|
|
||||||
use async_trait::async_trait;
|
use async_channel as chan;
|
||||||
|
use futures::StreamExt;
|
||||||
|
use futures_concurrency::stream::Merge;
|
||||||
use notify::{Config, Event, RecommendedWatcher, RecursiveMode, Watcher};
|
use notify::{Config, Event, RecommendedWatcher, RecursiveMode, Watcher};
|
||||||
use tokio::{
|
use tokio::{
|
||||||
runtime::Handle,
|
spawn,
|
||||||
select,
|
task::JoinHandle,
|
||||||
sync::{mpsc, oneshot},
|
|
||||||
task::{block_in_place, JoinHandle},
|
|
||||||
time::{interval_at, Instant, MissedTickBehavior},
|
time::{interval_at, Instant, MissedTickBehavior},
|
||||||
};
|
};
|
||||||
use tracing::{debug, error, warn};
|
use tokio_stream::wrappers::IntervalStream;
|
||||||
|
use tracing::{debug, error, info, instrument, trace, warn, Instrument};
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
use super::LocationManagerError;
|
use super::LocationManagerError;
|
||||||
|
@ -32,22 +38,22 @@ mod windows;
|
||||||
|
|
||||||
mod utils;
|
mod utils;
|
||||||
|
|
||||||
use utils::check_event;
|
use utils::reject_event;
|
||||||
|
|
||||||
#[cfg(target_os = "linux")]
|
#[cfg(target_os = "linux")]
|
||||||
type Handler<'lib> = linux::LinuxEventHandler<'lib>;
|
type Handler = linux::EventHandler;
|
||||||
|
|
||||||
#[cfg(target_os = "macos")]
|
#[cfg(target_os = "macos")]
|
||||||
type Handler<'lib> = macos::MacOsEventHandler<'lib>;
|
type Handler = macos::EventHandler;
|
||||||
|
|
||||||
#[cfg(target_os = "windows")]
|
#[cfg(target_os = "windows")]
|
||||||
type Handler<'lib> = windows::WindowsEventHandler<'lib>;
|
type Handler = windows::EventHandler;
|
||||||
|
|
||||||
#[cfg(target_os = "android")]
|
#[cfg(target_os = "android")]
|
||||||
type Handler<'lib> = android::AndroidEventHandler<'lib>;
|
type Handler = android::EventHandler;
|
||||||
|
|
||||||
#[cfg(target_os = "ios")]
|
#[cfg(target_os = "ios")]
|
||||||
type Handler<'lib> = ios::IosEventHandler<'lib>;
|
type Handler = ios::EventHandler;
|
||||||
|
|
||||||
pub(super) type IgnorePath = (PathBuf, bool);
|
pub(super) type IgnorePath = (PathBuf, bool);
|
||||||
|
|
||||||
|
@ -55,82 +61,115 @@ type INode = u64;
|
||||||
type InstantAndPath = (Instant, PathBuf);
|
type InstantAndPath = (Instant, PathBuf);
|
||||||
|
|
||||||
const ONE_SECOND: Duration = Duration::from_secs(1);
|
const ONE_SECOND: Duration = Duration::from_secs(1);
|
||||||
|
const THIRTY_SECONDS: Duration = Duration::from_secs(30);
|
||||||
const HUNDRED_MILLIS: Duration = Duration::from_millis(100);
|
const HUNDRED_MILLIS: Duration = Duration::from_millis(100);
|
||||||
|
|
||||||
#[async_trait]
|
trait EventHandler: 'static {
|
||||||
trait EventHandler<'lib> {
|
fn new(location_id: location::id::Type, library: Arc<Library>, node: Arc<Node>) -> Self
|
||||||
fn new(
|
|
||||||
location_id: location::id::Type,
|
|
||||||
library: &'lib Arc<Library>,
|
|
||||||
node: &'lib Arc<Node>,
|
|
||||||
) -> Self
|
|
||||||
where
|
where
|
||||||
Self: Sized;
|
Self: Sized;
|
||||||
|
|
||||||
/// Handle a file system event.
|
/// Handle a file system event.
|
||||||
async fn handle_event(&mut self, event: Event) -> Result<(), LocationManagerError>;
|
fn handle_event(
|
||||||
|
&mut self,
|
||||||
|
event: Event,
|
||||||
|
) -> impl Future<Output = Result<(), LocationManagerError>> + Send;
|
||||||
|
|
||||||
/// As Event Handlers have some inner state, from time to time we need to call this tick method
|
/// As Event Handlers have some inner state, from time to time we need to call this tick method
|
||||||
/// so the event handler can update its state.
|
/// so the event handler can update its state.
|
||||||
async fn tick(&mut self);
|
fn tick(&mut self) -> impl Future<Output = ()> + Send;
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
pub(super) struct LocationWatcher {
|
pub(super) struct LocationWatcher {
|
||||||
id: i32,
|
location_id: location::id::Type,
|
||||||
path: String,
|
location_path: PathBuf,
|
||||||
watcher: RecommendedWatcher,
|
watcher: RecommendedWatcher,
|
||||||
ignore_path_tx: mpsc::UnboundedSender<IgnorePath>,
|
ignore_path_tx: chan::Sender<IgnorePath>,
|
||||||
handle: Option<JoinHandle<()>>,
|
handle: Option<JoinHandle<()>>,
|
||||||
stop_tx: Option<oneshot::Sender<()>>,
|
stop_tx: chan::Sender<()>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl LocationWatcher {
|
impl LocationWatcher {
|
||||||
pub(super) async fn new(
|
#[instrument(
|
||||||
location: location::Data,
|
name = "location_watcher",
|
||||||
|
skip(pub_id, maybe_location_path, library, node),
|
||||||
|
fields(
|
||||||
|
library_id = %library.id,
|
||||||
|
location_path = ?maybe_location_path,
|
||||||
|
),
|
||||||
|
)]
|
||||||
|
pub(super) fn new(
|
||||||
|
location_ids_and_path::Data {
|
||||||
|
id: location_id,
|
||||||
|
pub_id,
|
||||||
|
path: maybe_location_path,
|
||||||
|
..
|
||||||
|
}: location_ids_and_path::Data,
|
||||||
library: Arc<Library>,
|
library: Arc<Library>,
|
||||||
node: Arc<Node>,
|
node: Arc<Node>,
|
||||||
) -> Result<Self, LocationManagerError> {
|
) -> Result<Self, LocationManagerError> {
|
||||||
let (events_tx, events_rx) = mpsc::unbounded_channel();
|
let location_pub_id = Uuid::from_slice(&pub_id)?;
|
||||||
let (ignore_path_tx, ignore_path_rx) = mpsc::unbounded_channel();
|
let location_path = maybe_missing(maybe_location_path, "location.path")?.into();
|
||||||
let (stop_tx, stop_rx) = oneshot::channel();
|
|
||||||
|
let (events_tx, events_rx) = chan::unbounded();
|
||||||
|
let (ignore_path_tx, ignore_path_rx) = chan::bounded(8);
|
||||||
|
let (stop_tx, stop_rx) = chan::bounded(1);
|
||||||
|
|
||||||
let watcher = RecommendedWatcher::new(
|
let watcher = RecommendedWatcher::new(
|
||||||
move |result| {
|
move |result| {
|
||||||
if !events_tx.is_closed() {
|
if !events_tx.is_closed() {
|
||||||
if events_tx.send(result).is_err() {
|
// SAFETY: we are not blocking the thread as this is an unbounded channel
|
||||||
error!(
|
if events_tx.send_blocking(result).is_err() {
|
||||||
"Unable to send watcher event to location manager for location: <id='{}'>",
|
error!(%location_id, "Unable to send watcher event to location manager;");
|
||||||
location.id
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
error!(
|
error!(%location_id, "Tried to send file system events to a closed channel;");
|
||||||
"Tried to send location file system events to a closed channel: <id='{}'",
|
|
||||||
location.id
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
Config::default(),
|
Config::default(),
|
||||||
)?;
|
)?;
|
||||||
|
|
||||||
let handle = tokio::spawn(Self::handle_watch_events(
|
let handle = spawn({
|
||||||
location.id,
|
let events_rx = events_rx.clone();
|
||||||
Uuid::from_slice(&location.pub_id)?,
|
let ignore_path_rx = ignore_path_rx.clone();
|
||||||
node,
|
let stop_rx = stop_rx.clone();
|
||||||
library,
|
async move {
|
||||||
events_rx,
|
while let Err(e) = spawn(
|
||||||
ignore_path_rx,
|
Self::handle_watch_events(
|
||||||
stop_rx,
|
location_id,
|
||||||
));
|
location_pub_id,
|
||||||
|
Arc::clone(&node),
|
||||||
|
Arc::clone(&library),
|
||||||
|
events_rx.clone(),
|
||||||
|
ignore_path_rx.clone(),
|
||||||
|
stop_rx.clone(),
|
||||||
|
)
|
||||||
|
.in_current_span(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
if e.is_panic() {
|
||||||
|
error!(?e, "Location watcher panicked;");
|
||||||
|
} else {
|
||||||
|
trace!("Location watcher received shutdown signal and will exit...");
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
trace!("Restarting location watcher processing task...");
|
||||||
|
}
|
||||||
|
|
||||||
|
info!("Location watcher gracefully shutdown");
|
||||||
|
}
|
||||||
|
.in_current_span()
|
||||||
|
});
|
||||||
|
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
id: location.id,
|
location_id,
|
||||||
path: maybe_missing(location.path, "location.path")?,
|
location_path,
|
||||||
watcher,
|
watcher,
|
||||||
ignore_path_tx,
|
ignore_path_tx,
|
||||||
handle: Some(handle),
|
handle: Some(handle),
|
||||||
stop_tx: Some(stop_tx),
|
stop_tx,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -139,157 +178,226 @@ impl LocationWatcher {
|
||||||
location_pub_id: Uuid,
|
location_pub_id: Uuid,
|
||||||
node: Arc<Node>,
|
node: Arc<Node>,
|
||||||
library: Arc<Library>,
|
library: Arc<Library>,
|
||||||
mut events_rx: mpsc::UnboundedReceiver<notify::Result<Event>>,
|
events_rx: chan::Receiver<notify::Result<Event>>,
|
||||||
mut ignore_path_rx: mpsc::UnboundedReceiver<IgnorePath>,
|
ignore_path_rx: chan::Receiver<IgnorePath>,
|
||||||
mut stop_rx: oneshot::Receiver<()>,
|
stop_rx: chan::Receiver<()>,
|
||||||
) {
|
) {
|
||||||
let mut event_handler = Handler::new(location_id, &library, &node);
|
enum StreamMessage {
|
||||||
|
NewEvent(notify::Result<Event>),
|
||||||
|
NewIgnorePath(IgnorePath),
|
||||||
|
Tick,
|
||||||
|
Stop,
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut event_handler = Handler::new(location_id, Arc::clone(&library), Arc::clone(&node));
|
||||||
|
|
||||||
|
let mut last_event_at = Instant::now();
|
||||||
|
|
||||||
|
let mut cached_indexer_ruler = None;
|
||||||
|
let mut cached_location_path = None;
|
||||||
|
|
||||||
let mut paths_to_ignore = HashSet::new();
|
let mut paths_to_ignore = HashSet::new();
|
||||||
|
|
||||||
let mut handler_interval = interval_at(Instant::now() + HUNDRED_MILLIS, HUNDRED_MILLIS);
|
let mut handler_tick_interval =
|
||||||
|
interval_at(Instant::now() + HUNDRED_MILLIS, HUNDRED_MILLIS);
|
||||||
// In case of doubt check: https://docs.rs/tokio/latest/tokio/time/enum.MissedTickBehavior.html
|
// In case of doubt check: https://docs.rs/tokio/latest/tokio/time/enum.MissedTickBehavior.html
|
||||||
handler_interval.set_missed_tick_behavior(MissedTickBehavior::Delay);
|
handler_tick_interval.set_missed_tick_behavior(MissedTickBehavior::Delay);
|
||||||
loop {
|
|
||||||
select! {
|
let mut msg_stream = pin!((
|
||||||
Some(event) = events_rx.recv() => {
|
events_rx.map(StreamMessage::NewEvent),
|
||||||
match event {
|
ignore_path_rx.map(StreamMessage::NewIgnorePath),
|
||||||
Ok(event) => {
|
IntervalStream::new(handler_tick_interval).map(|_| StreamMessage::Tick),
|
||||||
debug!("[Debug - handle_watch_events] Received event: {:#?}", event);
|
stop_rx.map(|()| StreamMessage::Stop),
|
||||||
if let Err(e) = Self::handle_single_event(
|
)
|
||||||
location_id,
|
.merge());
|
||||||
location_pub_id,
|
|
||||||
event,
|
while let Some(msg) = msg_stream.next().await {
|
||||||
&mut event_handler,
|
match msg {
|
||||||
&node,
|
StreamMessage::NewEvent(Ok(event)) => {
|
||||||
&library,
|
if let Err(e) = get_cached_indexer_ruler_and_location_path(
|
||||||
&paths_to_ignore,
|
location_id,
|
||||||
).await {
|
&mut cached_indexer_ruler,
|
||||||
error!("Failed to handle location file system event: \
|
&mut cached_location_path,
|
||||||
<id='{location_id}', error='{e:#?}'>",
|
&last_event_at,
|
||||||
);
|
&library.db,
|
||||||
}
|
)
|
||||||
}
|
.await
|
||||||
Err(e) => {
|
{
|
||||||
error!("watch error: {:#?}", e);
|
error!(?e, "Failed to get indexer ruler;");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
last_event_at = Instant::now();
|
||||||
|
|
||||||
|
if let Err(e) = Self::handle_single_event(
|
||||||
|
location_pub_id,
|
||||||
|
cached_location_path.as_deref(),
|
||||||
|
event,
|
||||||
|
&mut event_handler,
|
||||||
|
&node,
|
||||||
|
&paths_to_ignore,
|
||||||
|
cached_indexer_ruler.as_ref(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
error!(?e, "Failed to handle location file system event;");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Some((path, ignore)) = ignore_path_rx.recv() => {
|
StreamMessage::NewEvent(Err(e)) => error!(?e, "Watcher error;"),
|
||||||
if ignore {
|
|
||||||
|
StreamMessage::NewIgnorePath((path, should_ignore)) => {
|
||||||
|
if should_ignore {
|
||||||
paths_to_ignore.insert(path);
|
paths_to_ignore.insert(path);
|
||||||
} else {
|
} else {
|
||||||
paths_to_ignore.remove(&path);
|
paths_to_ignore.remove(&path);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
_ = handler_interval.tick() => {
|
StreamMessage::Tick => event_handler.tick().await,
|
||||||
event_handler.tick().await;
|
|
||||||
}
|
|
||||||
|
|
||||||
_ = &mut stop_rx => {
|
StreamMessage::Stop => {
|
||||||
debug!("Stop Location Manager event handler for location: <id='{}'>", location_id);
|
debug!("Stopping Location Manager event handler for location");
|
||||||
break
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn handle_single_event<'lib>(
|
#[instrument(skip_all, fields(?event, ?ignore_paths, ?location_path))]
|
||||||
location_id: location::id::Type,
|
async fn handle_single_event(
|
||||||
location_pub_id: Uuid,
|
location_pub_id: Uuid,
|
||||||
|
location_path: Option<&Path>,
|
||||||
event: Event,
|
event: Event,
|
||||||
event_handler: &mut impl EventHandler<'lib>,
|
event_handler: &mut impl EventHandler,
|
||||||
node: &'lib Node,
|
node: &Node,
|
||||||
_library: &'lib Library,
|
|
||||||
ignore_paths: &HashSet<PathBuf>,
|
ignore_paths: &HashSet<PathBuf>,
|
||||||
|
indexer_ruler: Option<&IndexerRuler>,
|
||||||
) -> Result<(), LocationManagerError> {
|
) -> Result<(), LocationManagerError> {
|
||||||
debug!("Event: {:#?}", event);
|
if reject_event(&event, ignore_paths, location_path, indexer_ruler).await {
|
||||||
if !check_event(&event, ignore_paths) {
|
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
|
|
||||||
// let Some(location) = find_location(library, location_id)
|
|
||||||
// .include(location_with_indexer_rules::include())
|
|
||||||
// .exec()
|
|
||||||
// .await?
|
|
||||||
// else {
|
|
||||||
// warn!("Tried to handle event for unknown location: <id='{location_id}'>");
|
|
||||||
// return Ok(());
|
|
||||||
// };
|
|
||||||
|
|
||||||
if !node.locations.is_online(&location_pub_id).await {
|
if !node.locations.is_online(&location_pub_id).await {
|
||||||
warn!("Tried to handle event for offline location: <id='{location_id}'>");
|
warn!("Tried to handle event for offline location");
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
|
|
||||||
// debug!("Handling event: {:#?}", event);
|
|
||||||
|
|
||||||
event_handler.handle_event(event).await
|
event_handler.handle_event(event).await
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(super) fn ignore_path(
|
#[instrument(
|
||||||
&self,
|
skip(self, path),
|
||||||
path: PathBuf,
|
fields(
|
||||||
ignore: bool,
|
location_id = %self.location_id,
|
||||||
) -> Result<(), LocationManagerError> {
|
location_path = %self.location_path.display(),
|
||||||
self.ignore_path_tx.send((path, ignore)).map_err(Into::into)
|
path = %path.display(),
|
||||||
|
),
|
||||||
|
)]
|
||||||
|
pub(super) async fn ignore_path(&self, path: PathBuf, ignore: bool) {
|
||||||
|
self.ignore_path_tx
|
||||||
|
.send((path, ignore))
|
||||||
|
.await
|
||||||
|
.expect("Location watcher ignore path channel closed");
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(super) fn check_path(&self, path: impl AsRef<Path>) -> bool {
|
pub(super) fn check_path(&self, path: impl AsRef<Path>) -> bool {
|
||||||
Path::new(&self.path) == path.as_ref()
|
self.location_path == path.as_ref()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[instrument(
|
||||||
|
skip(self),
|
||||||
|
fields(
|
||||||
|
location_id = %self.location_id,
|
||||||
|
location_path = %self.location_path.display(),
|
||||||
|
),
|
||||||
|
)]
|
||||||
pub(super) fn watch(&mut self) {
|
pub(super) fn watch(&mut self) {
|
||||||
let path = &self.path;
|
trace!("Start watching location");
|
||||||
debug!("Start watching location: (path: {path})");
|
|
||||||
|
|
||||||
if let Err(e) = self
|
if let Err(e) = self
|
||||||
.watcher
|
.watcher
|
||||||
.watch(Path::new(path), RecursiveMode::Recursive)
|
.watch(self.location_path.as_path(), RecursiveMode::Recursive)
|
||||||
{
|
{
|
||||||
error!("Unable to watch location: (path: {path}, error: {e:#?})");
|
error!(?e, "Unable to watch location;");
|
||||||
} else {
|
} else {
|
||||||
debug!("Now watching location: (path: {path})");
|
trace!("Now watching location");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[instrument(
|
||||||
|
skip(self),
|
||||||
|
fields(
|
||||||
|
location_id = %self.location_id,
|
||||||
|
location_path = %self.location_path.display(),
|
||||||
|
),
|
||||||
|
)]
|
||||||
pub(super) fn unwatch(&mut self) {
|
pub(super) fn unwatch(&mut self) {
|
||||||
let path = &self.path;
|
if let Err(e) = self.watcher.unwatch(self.location_path.as_path()) {
|
||||||
if let Err(e) = self.watcher.unwatch(Path::new(path)) {
|
|
||||||
/**************************************** TODO: ****************************************
|
/**************************************** TODO: ****************************************
|
||||||
* According to an unit test, this error may occur when a subdirectory is removed *
|
* According to an unit test, this error may occur when a subdirectory is removed *
|
||||||
* and we try to unwatch the parent directory then we have to check the implications *
|
* and we try to unwatch the parent directory then we have to check the implications *
|
||||||
* of unwatch error for this case. *
|
* of unwatch error for this case. *
|
||||||
**************************************************************************************/
|
**************************************************************************************/
|
||||||
error!("Unable to unwatch location: (path: {path}, error: {e:#?})",);
|
error!(?e, "Unable to unwatch location;");
|
||||||
} else {
|
} else {
|
||||||
debug!("Stop watching location: (path: {path})");
|
trace!("Stop watching location");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Drop for LocationWatcher {
|
impl Drop for LocationWatcher {
|
||||||
fn drop(&mut self) {
|
fn drop(&mut self) {
|
||||||
if let Some(stop_tx) = self.stop_tx.take() {
|
// FIXME: change this Drop to async drop in the future
|
||||||
if stop_tx.send(()).is_err() {
|
if let Some(handle) = self.handle.take() {
|
||||||
error!(
|
let stop_tx = self.stop_tx.clone();
|
||||||
"Failed to send stop signal to location watcher: <id='{}'>",
|
spawn(async move {
|
||||||
self.id
|
stop_tx
|
||||||
);
|
.send(())
|
||||||
}
|
.await
|
||||||
|
.expect("Location watcher stop channel closed");
|
||||||
|
|
||||||
// FIXME: change this Drop to async drop in the future
|
if let Err(e) = handle.await {
|
||||||
if let Some(handle) = self.handle.take() {
|
error!(?e, "Failed to join watcher task;");
|
||||||
if let Err(e) = block_in_place(move || Handle::current().block_on(handle)) {
|
|
||||||
error!("Failed to join watcher task: {e:#?}")
|
|
||||||
}
|
}
|
||||||
}
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn get_cached_indexer_ruler_and_location_path(
|
||||||
|
location_id: location::id::Type,
|
||||||
|
cached_indexer_ruler: &mut Option<IndexerRuler>,
|
||||||
|
location_path: &mut Option<PathBuf>,
|
||||||
|
last_event_at: &Instant,
|
||||||
|
db: &PrismaClient,
|
||||||
|
) -> Result<(), LocationManagerError> {
|
||||||
|
if cached_indexer_ruler.is_none() || last_event_at.elapsed() > THIRTY_SECONDS {
|
||||||
|
if let Some(location_with_indexer_rules::Data {
|
||||||
|
path,
|
||||||
|
indexer_rules,
|
||||||
|
..
|
||||||
|
}) = db
|
||||||
|
.location()
|
||||||
|
.find_unique(location::id::equals(location_id))
|
||||||
|
.include(location_with_indexer_rules::include())
|
||||||
|
.exec()
|
||||||
|
.await?
|
||||||
|
{
|
||||||
|
*cached_indexer_ruler = Some(
|
||||||
|
indexer_rules
|
||||||
|
.iter()
|
||||||
|
.map(|rule| IndexerRule::try_from(&rule.indexer_rule))
|
||||||
|
.collect::<Result<Vec<_>, _>>()
|
||||||
|
.map(IndexerRuler::new)?,
|
||||||
|
);
|
||||||
|
|
||||||
|
*location_path = path.map(Into::into);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
/***************************************************************************************************
|
/***************************************************************************************************
|
||||||
* Some tests to validate our assumptions of events through different file systems *
|
* Some tests to validate our assumptions of events through different file systems *
|
||||||
****************************************************************************************************
|
****************************************************************************************************
|
||||||
|
@ -412,26 +520,23 @@ mod tests {
|
||||||
expected_event: EventKind,
|
expected_event: EventKind,
|
||||||
) {
|
) {
|
||||||
let path = path.as_ref();
|
let path = path.as_ref();
|
||||||
debug!(
|
debug!(?expected_event, path = %path.display());
|
||||||
"Expecting event: {expected_event:#?} at path: {}",
|
|
||||||
path.display()
|
|
||||||
);
|
|
||||||
let mut tries = 0;
|
let mut tries = 0;
|
||||||
loop {
|
loop {
|
||||||
match events_rx.try_recv() {
|
match events_rx.try_recv() {
|
||||||
Ok(maybe_event) => {
|
Ok(maybe_event) => {
|
||||||
let event = maybe_event.expect("Failed to receive event");
|
let event = maybe_event.expect("Failed to receive event");
|
||||||
debug!("Received event: {event:#?}");
|
debug!(?event, "Received event;");
|
||||||
// Using `ends_with` and removing root path here due to a weird edge case on CI tests at MacOS
|
// Using `ends_with` and removing root path here due to a weird edge case on CI tests at MacOS
|
||||||
if event.paths[0].ends_with(path.iter().skip(1).collect::<PathBuf>())
|
if event.paths[0].ends_with(path.iter().skip(1).collect::<PathBuf>())
|
||||||
&& event.kind == expected_event
|
&& event.kind == expected_event
|
||||||
{
|
{
|
||||||
debug!("Received expected event: {expected_event:#?}");
|
debug!("Received expected event");
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
debug!("No event yet: {e:#?}");
|
debug!(?e, "No event yet;");
|
||||||
tries += 1;
|
tries += 1;
|
||||||
sleep(Duration::from_millis(100)).await;
|
sleep(Duration::from_millis(100)).await;
|
||||||
}
|
}
|
||||||
|
@ -451,7 +556,7 @@ mod tests {
|
||||||
watcher
|
watcher
|
||||||
.watch(root_dir.path(), notify::RecursiveMode::Recursive)
|
.watch(root_dir.path(), notify::RecursiveMode::Recursive)
|
||||||
.expect("Failed to watch root directory");
|
.expect("Failed to watch root directory");
|
||||||
debug!("Now watching {}", root_dir.path().display());
|
debug!(root = %root_dir.path().display(), "Now watching;");
|
||||||
|
|
||||||
let file_path = root_dir.path().join("test.txt");
|
let file_path = root_dir.path().join("test.txt");
|
||||||
fs::write(&file_path, "test").await.unwrap();
|
fs::write(&file_path, "test").await.unwrap();
|
||||||
|
@ -475,9 +580,9 @@ mod tests {
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
|
|
||||||
debug!("Unwatching root directory: {}", root_dir.path().display());
|
debug!(root = %root_dir.path().display(), "Unwatching root directory;");
|
||||||
if let Err(e) = watcher.unwatch(root_dir.path()) {
|
if let Err(e) = watcher.unwatch(root_dir.path()) {
|
||||||
error!("Failed to unwatch root directory: {e:#?}");
|
error!(?e, "Failed to unwatch root directory;");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -489,7 +594,7 @@ mod tests {
|
||||||
watcher
|
watcher
|
||||||
.watch(root_dir.path(), notify::RecursiveMode::Recursive)
|
.watch(root_dir.path(), notify::RecursiveMode::Recursive)
|
||||||
.expect("Failed to watch root directory");
|
.expect("Failed to watch root directory");
|
||||||
debug!("Now watching {}", root_dir.path().display());
|
debug!(root = %root_dir.path().display(), "Now watching;");
|
||||||
|
|
||||||
let dir_path = root_dir.path().join("inner");
|
let dir_path = root_dir.path().join("inner");
|
||||||
fs::create_dir(&dir_path)
|
fs::create_dir(&dir_path)
|
||||||
|
@ -505,9 +610,9 @@ mod tests {
|
||||||
#[cfg(target_os = "linux")]
|
#[cfg(target_os = "linux")]
|
||||||
expect_event(events_rx, &dir_path, EventKind::Create(CreateKind::Folder)).await;
|
expect_event(events_rx, &dir_path, EventKind::Create(CreateKind::Folder)).await;
|
||||||
|
|
||||||
debug!("Unwatching root directory: {}", root_dir.path().display());
|
debug!(root = %root_dir.path().display(), "Unwatching root directory;");
|
||||||
if let Err(e) = watcher.unwatch(root_dir.path()) {
|
if let Err(e) = watcher.unwatch(root_dir.path()) {
|
||||||
error!("Failed to unwatch root directory: {e:#?}");
|
error!(?e, "Failed to unwatch root directory;");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -522,7 +627,7 @@ mod tests {
|
||||||
watcher
|
watcher
|
||||||
.watch(root_dir.path(), notify::RecursiveMode::Recursive)
|
.watch(root_dir.path(), notify::RecursiveMode::Recursive)
|
||||||
.expect("Failed to watch root directory");
|
.expect("Failed to watch root directory");
|
||||||
debug!("Now watching {}", root_dir.path().display());
|
debug!(root = %root_dir.path().display(), "Now watching;");
|
||||||
|
|
||||||
let mut file = fs::OpenOptions::new()
|
let mut file = fs::OpenOptions::new()
|
||||||
.append(true)
|
.append(true)
|
||||||
|
@ -556,9 +661,9 @@ mod tests {
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
|
|
||||||
debug!("Unwatching root directory: {}", root_dir.path().display());
|
debug!(root = %root_dir.path().display(), "Unwatching root directory;");
|
||||||
if let Err(e) = watcher.unwatch(root_dir.path()) {
|
if let Err(e) = watcher.unwatch(root_dir.path()) {
|
||||||
error!("Failed to unwatch root directory: {e:#?}");
|
error!(?e, "Failed to unwatch root directory;");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -573,7 +678,7 @@ mod tests {
|
||||||
watcher
|
watcher
|
||||||
.watch(root_dir.path(), notify::RecursiveMode::Recursive)
|
.watch(root_dir.path(), notify::RecursiveMode::Recursive)
|
||||||
.expect("Failed to watch root directory");
|
.expect("Failed to watch root directory");
|
||||||
debug!("Now watching {}", root_dir.path().display());
|
debug!(root = %root_dir.path().display(), "Now watching;");
|
||||||
|
|
||||||
let new_file_name = root_dir.path().join("test2.txt");
|
let new_file_name = root_dir.path().join("test2.txt");
|
||||||
|
|
||||||
|
@ -605,9 +710,9 @@ mod tests {
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
|
|
||||||
debug!("Unwatching root directory: {}", root_dir.path().display());
|
debug!(root = %root_dir.path().display(), "Unwatching root directory;");
|
||||||
if let Err(e) = watcher.unwatch(root_dir.path()) {
|
if let Err(e) = watcher.unwatch(root_dir.path()) {
|
||||||
error!("Failed to unwatch root directory: {e:#?}");
|
error!(?e, "Failed to unwatch root directory;");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -624,7 +729,7 @@ mod tests {
|
||||||
watcher
|
watcher
|
||||||
.watch(root_dir.path(), notify::RecursiveMode::Recursive)
|
.watch(root_dir.path(), notify::RecursiveMode::Recursive)
|
||||||
.expect("Failed to watch root directory");
|
.expect("Failed to watch root directory");
|
||||||
debug!("Now watching {}", root_dir.path().display());
|
debug!(root = %root_dir.path().display(), "Now watching;");
|
||||||
|
|
||||||
let new_dir_name = root_dir.path().join("inner2");
|
let new_dir_name = root_dir.path().join("inner2");
|
||||||
|
|
||||||
|
@ -656,9 +761,9 @@ mod tests {
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
|
|
||||||
debug!("Unwatching root directory: {}", root_dir.path().display());
|
debug!(root = %root_dir.path().display(), "Unwatching root directory;");
|
||||||
if let Err(e) = watcher.unwatch(root_dir.path()) {
|
if let Err(e) = watcher.unwatch(root_dir.path()) {
|
||||||
error!("Failed to unwatch root directory: {e:#?}");
|
error!(?e, "Failed to unwatch root directory;");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -673,7 +778,7 @@ mod tests {
|
||||||
watcher
|
watcher
|
||||||
.watch(root_dir.path(), notify::RecursiveMode::Recursive)
|
.watch(root_dir.path(), notify::RecursiveMode::Recursive)
|
||||||
.expect("Failed to watch root directory");
|
.expect("Failed to watch root directory");
|
||||||
debug!("Now watching {}", root_dir.path().display());
|
debug!(root = %root_dir.path().display(), "Now watching;");
|
||||||
|
|
||||||
fs::remove_file(&file_path)
|
fs::remove_file(&file_path)
|
||||||
.await
|
.await
|
||||||
|
@ -696,9 +801,9 @@ mod tests {
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
|
|
||||||
debug!("Unwatching root directory: {}", root_dir.path().display());
|
debug!(root = %root_dir.path().display(), "Unwatching root directory;");
|
||||||
if let Err(e) = watcher.unwatch(root_dir.path()) {
|
if let Err(e) = watcher.unwatch(root_dir.path()) {
|
||||||
error!("Failed to unwatch root directory: {e:#?}");
|
error!(?e, "Failed to unwatch root directory;");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -723,11 +828,11 @@ mod tests {
|
||||||
watcher
|
watcher
|
||||||
.watch(root_dir.path(), notify::RecursiveMode::Recursive)
|
.watch(root_dir.path(), notify::RecursiveMode::Recursive)
|
||||||
.expect("Failed to watch root directory");
|
.expect("Failed to watch root directory");
|
||||||
debug!("Now watching {}", root_dir.path().display());
|
debug!(root = %root_dir.path().display(), "Now watching;");
|
||||||
|
|
||||||
debug!("First unwatching the inner directory before removing it");
|
debug!("First unwatching the inner directory before removing it");
|
||||||
if let Err(e) = watcher.unwatch(&dir_path) {
|
if let Err(e) = watcher.unwatch(&dir_path) {
|
||||||
error!("Failed to unwatch inner directory: {e:#?}");
|
error!(?e, "Failed to unwatch inner directory;");
|
||||||
}
|
}
|
||||||
|
|
||||||
fs::remove_dir(&dir_path)
|
fs::remove_dir(&dir_path)
|
||||||
|
@ -751,9 +856,9 @@ mod tests {
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
|
|
||||||
debug!("Unwatching root directory: {}", root_dir.path().display());
|
debug!(root = %root_dir.path().display(), "Unwatching root directory;");
|
||||||
if let Err(e) = watcher.unwatch(root_dir.path()) {
|
if let Err(e) = watcher.unwatch(root_dir.path()) {
|
||||||
error!("Failed to unwatch root directory: {e:#?}");
|
error!(?e, "Failed to unwatch root directory;");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -6,19 +6,7 @@ use crate::{
|
||||||
indexer::reverse_update_directories_sizes, location_with_indexer_rules,
|
indexer::reverse_update_directories_sizes, location_with_indexer_rules,
|
||||||
manager::LocationManagerError, scan_location_sub_path, update_location_size,
|
manager::LocationManagerError, scan_location_sub_path, update_location_size,
|
||||||
},
|
},
|
||||||
object::{
|
object::validation::hash::file_checksum,
|
||||||
media::{
|
|
||||||
exif_data_image_to_query_params,
|
|
||||||
exif_metadata_extractor::{can_extract_exif_data_for_image, extract_exif_data},
|
|
||||||
ffmpeg_metadata_extractor::{
|
|
||||||
can_extract_ffmpeg_data_for_audio, can_extract_ffmpeg_data_for_video,
|
|
||||||
extract_ffmpeg_data, save_ffmpeg_data,
|
|
||||||
},
|
|
||||||
old_thumbnail::get_indexed_thumbnail_path,
|
|
||||||
},
|
|
||||||
old_file_identifier::FileMetadata,
|
|
||||||
validation::hash::file_checksum,
|
|
||||||
},
|
|
||||||
Node,
|
Node,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -28,21 +16,32 @@ use sd_core_file_path_helper::{
|
||||||
loose_find_existing_file_path_params, path_is_hidden, FilePathError, FilePathMetadata,
|
loose_find_existing_file_path_params, path_is_hidden, FilePathError, FilePathMetadata,
|
||||||
IsolatedFilePathData, MetadataExt,
|
IsolatedFilePathData, MetadataExt,
|
||||||
};
|
};
|
||||||
use sd_core_prisma_helpers::file_path_with_object;
|
use sd_core_heavy_lifting::{
|
||||||
|
file_identifier::FileMetadata,
|
||||||
|
media_processor::{
|
||||||
|
exif_media_data, ffmpeg_media_data, generate_single_thumbnail, get_thumbnails_directory,
|
||||||
|
ThumbnailKind,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
use sd_core_indexer_rules::{
|
||||||
|
seed::{GitIgnoreRules, GITIGNORE},
|
||||||
|
IndexerRuler, RulerDecision,
|
||||||
|
};
|
||||||
|
use sd_core_prisma_helpers::{file_path_with_object, object_ids, CasId, ObjectPubId};
|
||||||
|
|
||||||
use sd_file_ext::{
|
use sd_file_ext::{
|
||||||
extensions::{AudioExtension, ImageExtension, VideoExtension},
|
extensions::{AudioExtension, ImageExtension, VideoExtension},
|
||||||
kind::ObjectKind,
|
kind::ObjectKind,
|
||||||
};
|
};
|
||||||
use sd_prisma::{
|
use sd_prisma::{
|
||||||
prisma::{exif_data, file_path, location, object},
|
prisma::{file_path, location, object},
|
||||||
prisma_sync,
|
prisma_sync,
|
||||||
};
|
};
|
||||||
use sd_sync::OperationFactory;
|
use sd_sync::OperationFactory;
|
||||||
use sd_utils::{
|
use sd_utils::{
|
||||||
db::{inode_from_db, inode_to_db, maybe_missing},
|
db::{inode_from_db, inode_to_db, maybe_missing},
|
||||||
error::FileIOError,
|
error::FileIOError,
|
||||||
msgpack, uuid_to_bytes,
|
msgpack,
|
||||||
};
|
};
|
||||||
|
|
||||||
#[cfg(target_family = "unix")]
|
#[cfg(target_family = "unix")]
|
||||||
|
@ -61,31 +60,107 @@ use std::{
|
||||||
};
|
};
|
||||||
|
|
||||||
use chrono::{DateTime, FixedOffset, Local, Utc};
|
use chrono::{DateTime, FixedOffset, Local, Utc};
|
||||||
|
use futures_concurrency::future::Join;
|
||||||
use notify::Event;
|
use notify::Event;
|
||||||
use tokio::{
|
use tokio::{
|
||||||
fs,
|
fs,
|
||||||
io::{self, ErrorKind},
|
io::{self, ErrorKind},
|
||||||
spawn,
|
spawn,
|
||||||
time::Instant,
|
time::{sleep, Instant},
|
||||||
};
|
};
|
||||||
use tracing::{debug, error, trace, warn};
|
use tracing::{error, instrument, trace, warn};
|
||||||
use uuid::Uuid;
|
|
||||||
|
|
||||||
use super::{INode, HUNDRED_MILLIS};
|
use super::{INode, HUNDRED_MILLIS, ONE_SECOND};
|
||||||
|
|
||||||
pub(super) fn check_event(event: &Event, ignore_paths: &HashSet<PathBuf>) -> bool {
|
pub(super) async fn reject_event(
|
||||||
|
event: &Event,
|
||||||
|
ignore_paths: &HashSet<PathBuf>,
|
||||||
|
location_path: Option<&Path>,
|
||||||
|
indexer_ruler: Option<&IndexerRuler>,
|
||||||
|
) -> bool {
|
||||||
// if path includes .DS_Store, .spacedrive file creation or is in the `ignore_paths` set, we ignore
|
// if path includes .DS_Store, .spacedrive file creation or is in the `ignore_paths` set, we ignore
|
||||||
!event.paths.iter().any(|p| {
|
if event.paths.iter().any(|p| {
|
||||||
p.file_name()
|
p.file_name()
|
||||||
.and_then(OsStr::to_str)
|
.and_then(OsStr::to_str)
|
||||||
.map_or(false, |name| name == ".DS_Store" || name == ".spacedrive")
|
.map_or(false, |name| name == ".DS_Store" || name == ".spacedrive")
|
||||||
|| ignore_paths.contains(p)
|
|| ignore_paths.contains(p)
|
||||||
})
|
}) {
|
||||||
|
trace!("Rejected by ignored paths");
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Some(indexer_ruler) = indexer_ruler {
|
||||||
|
let ruler_decisions = event
|
||||||
|
.paths
|
||||||
|
.iter()
|
||||||
|
.map(|path| async move { (path, fs::metadata(path).await) })
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join()
|
||||||
|
.await
|
||||||
|
.into_iter()
|
||||||
|
.filter_map(|(path, res)| {
|
||||||
|
res.map(|metadata| (path, metadata))
|
||||||
|
.map_err(|e| {
|
||||||
|
if e.kind() != ErrorKind::NotFound {
|
||||||
|
error!(?e, path = %path.display(), "Failed to get metadata for path;");
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.ok()
|
||||||
|
})
|
||||||
|
.map(|(path, metadata)| {
|
||||||
|
let mut independent_ruler = indexer_ruler.clone();
|
||||||
|
|
||||||
|
async move {
|
||||||
|
let path_to_check_gitignore = if metadata.is_dir() {
|
||||||
|
Some(path.as_path())
|
||||||
|
} else {
|
||||||
|
path.parent()
|
||||||
|
};
|
||||||
|
|
||||||
|
if let (Some(path_to_check_gitignore), Some(location_path)) =
|
||||||
|
(path_to_check_gitignore, location_path.as_ref())
|
||||||
|
{
|
||||||
|
if independent_ruler.has_system(&GITIGNORE) {
|
||||||
|
if let Some(rules) = GitIgnoreRules::get_rules_if_in_git_repo(
|
||||||
|
location_path,
|
||||||
|
path_to_check_gitignore,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
trace!("Found gitignore rules to follow");
|
||||||
|
independent_ruler.extend(rules.map(Into::into));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
independent_ruler.evaluate_path(path, &metadata).await
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join()
|
||||||
|
.await;
|
||||||
|
|
||||||
|
if !ruler_decisions.is_empty()
|
||||||
|
&& ruler_decisions.into_iter().all(|res| {
|
||||||
|
matches!(
|
||||||
|
res.map_err(|e| trace!(?e, "Failed to evaluate path;"))
|
||||||
|
// In case of error, we accept the path as a safe default
|
||||||
|
.unwrap_or(RulerDecision::Accept),
|
||||||
|
RulerDecision::Reject
|
||||||
|
)
|
||||||
|
}) {
|
||||||
|
trace!("Rejected by indexer ruler");
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[instrument(skip_all, fields(path = %path.as_ref().display()), err)]
|
||||||
pub(super) async fn create_dir(
|
pub(super) async fn create_dir(
|
||||||
location_id: location::id::Type,
|
location_id: location::id::Type,
|
||||||
path: impl AsRef<Path>,
|
path: impl AsRef<Path> + Send,
|
||||||
metadata: &Metadata,
|
metadata: &Metadata,
|
||||||
node: &Arc<Node>,
|
node: &Arc<Node>,
|
||||||
library: &Arc<Library>,
|
library: &Arc<Library>,
|
||||||
|
@ -94,17 +169,13 @@ pub(super) async fn create_dir(
|
||||||
.include(location_with_indexer_rules::include())
|
.include(location_with_indexer_rules::include())
|
||||||
.exec()
|
.exec()
|
||||||
.await?
|
.await?
|
||||||
.ok_or(LocationManagerError::MissingLocation(location_id))?;
|
.ok_or(LocationManagerError::LocationNotFound(location_id))?;
|
||||||
|
|
||||||
let path = path.as_ref();
|
let path = path.as_ref();
|
||||||
|
|
||||||
let location_path = maybe_missing(&location.path, "location.path")?;
|
let location_path = maybe_missing(&location.path, "location.path")?;
|
||||||
|
|
||||||
trace!(
|
trace!(new_directory = %path.display(), "Creating directory;");
|
||||||
"Location: <root_path ='{}'> creating directory: {}",
|
|
||||||
location_path,
|
|
||||||
path.display()
|
|
||||||
);
|
|
||||||
|
|
||||||
let iso_file_path = IsolatedFilePathData::new(location.id, location_path, path, true)?;
|
let iso_file_path = IsolatedFilePathData::new(location.id, location_path, path, true)?;
|
||||||
|
|
||||||
|
@ -112,10 +183,8 @@ pub(super) async fn create_dir(
|
||||||
if !parent_iso_file_path.is_root()
|
if !parent_iso_file_path.is_root()
|
||||||
&& !check_file_path_exists::<FilePathError>(&parent_iso_file_path, &library.db).await?
|
&& !check_file_path_exists::<FilePathError>(&parent_iso_file_path, &library.db).await?
|
||||||
{
|
{
|
||||||
warn!(
|
warn!(%iso_file_path, "Watcher found a directory without parent;");
|
||||||
"Watcher found a directory without parent: {}",
|
|
||||||
&iso_file_path
|
|
||||||
);
|
|
||||||
return Ok(());
|
return Ok(());
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -123,8 +192,6 @@ pub(super) async fn create_dir(
|
||||||
.materialized_path_for_children()
|
.materialized_path_for_children()
|
||||||
.expect("We're in the create dir function lol");
|
.expect("We're in the create dir function lol");
|
||||||
|
|
||||||
debug!("Creating path: {}", iso_file_path);
|
|
||||||
|
|
||||||
create_file_path(
|
create_file_path(
|
||||||
library,
|
library,
|
||||||
iso_file_path.to_parts(),
|
iso_file_path.to_parts(),
|
||||||
|
@ -133,8 +200,24 @@ pub(super) async fn create_dir(
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
// scan the new directory
|
spawn({
|
||||||
scan_location_sub_path(node, library, location, &children_materialized_path).await?;
|
let node = Arc::clone(node);
|
||||||
|
let library = Arc::clone(library);
|
||||||
|
|
||||||
|
async move {
|
||||||
|
// Wait a bit for any files being moved into the new directory to be indexed by the watcher
|
||||||
|
sleep(ONE_SECOND).await;
|
||||||
|
|
||||||
|
trace!(%iso_file_path, "Scanning new directory;");
|
||||||
|
|
||||||
|
// scan the new directory
|
||||||
|
if let Err(e) =
|
||||||
|
scan_location_sub_path(&node, &library, location, &children_materialized_path).await
|
||||||
|
{
|
||||||
|
error!(?e, "Failed to scan new directory;");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
invalidate_query!(library, "search.paths");
|
invalidate_query!(library, "search.paths");
|
||||||
invalidate_query!(library, "search.objects");
|
invalidate_query!(library, "search.objects");
|
||||||
|
@ -142,9 +225,10 @@ pub(super) async fn create_dir(
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[instrument(skip_all, fields(path = %path.as_ref().display()), err)]
|
||||||
pub(super) async fn create_file(
|
pub(super) async fn create_file(
|
||||||
location_id: location::id::Type,
|
location_id: location::id::Type,
|
||||||
path: impl AsRef<Path>,
|
path: impl AsRef<Path> + Send,
|
||||||
metadata: &Metadata,
|
metadata: &Metadata,
|
||||||
node: &Arc<Node>,
|
node: &Arc<Node>,
|
||||||
library: &Arc<Library>,
|
library: &Arc<Library>,
|
||||||
|
@ -162,8 +246,8 @@ pub(super) async fn create_file(
|
||||||
|
|
||||||
async fn inner_create_file(
|
async fn inner_create_file(
|
||||||
location_id: location::id::Type,
|
location_id: location::id::Type,
|
||||||
location_path: impl AsRef<Path>,
|
location_path: impl AsRef<Path> + Send,
|
||||||
path: impl AsRef<Path>,
|
path: impl AsRef<Path> + Send,
|
||||||
metadata: &Metadata,
|
metadata: &Metadata,
|
||||||
node: &Arc<Node>,
|
node: &Arc<Node>,
|
||||||
library @ Library {
|
library @ Library {
|
||||||
|
@ -176,11 +260,7 @@ async fn inner_create_file(
|
||||||
let path = path.as_ref();
|
let path = path.as_ref();
|
||||||
let location_path = location_path.as_ref();
|
let location_path = location_path.as_ref();
|
||||||
|
|
||||||
trace!(
|
trace!(new_file = %path.display(), "Creating file;");
|
||||||
"Location: <root_path ='{}'> creating file: {}",
|
|
||||||
location_path.display(),
|
|
||||||
path.display()
|
|
||||||
);
|
|
||||||
|
|
||||||
let iso_file_path = IsolatedFilePathData::new(location_id, location_path, path, false)?;
|
let iso_file_path = IsolatedFilePathData::new(location_id, location_path, path, false)?;
|
||||||
let iso_file_path_parts = iso_file_path.to_parts();
|
let iso_file_path_parts = iso_file_path.to_parts();
|
||||||
|
@ -200,7 +280,8 @@ async fn inner_create_file(
|
||||||
.exec()
|
.exec()
|
||||||
.await?
|
.await?
|
||||||
{
|
{
|
||||||
trace!("File already exists with that inode: {}", iso_file_path);
|
trace!(%iso_file_path, "File already exists with that inode;");
|
||||||
|
|
||||||
return inner_update_file(location_path, &file_path, path, node, library, None).await;
|
return inner_update_file(location_path, &file_path, path, node, library, None).await;
|
||||||
|
|
||||||
// If we can't find an existing file with the same inode, we check if there is a file with the same path
|
// If we can't find an existing file with the same inode, we check if there is a file with the same path
|
||||||
|
@ -216,10 +297,8 @@ async fn inner_create_file(
|
||||||
.exec()
|
.exec()
|
||||||
.await?
|
.await?
|
||||||
{
|
{
|
||||||
trace!(
|
trace!(%iso_file_path, "File already exists with that iso_file_path;");
|
||||||
"File already exists with that iso_file_path: {}",
|
|
||||||
iso_file_path
|
|
||||||
);
|
|
||||||
return inner_update_file(
|
return inner_update_file(
|
||||||
location_path,
|
location_path,
|
||||||
&file_path,
|
&file_path,
|
||||||
|
@ -235,7 +314,8 @@ async fn inner_create_file(
|
||||||
if !parent_iso_file_path.is_root()
|
if !parent_iso_file_path.is_root()
|
||||||
&& !check_file_path_exists::<FilePathError>(&parent_iso_file_path, db).await?
|
&& !check_file_path_exists::<FilePathError>(&parent_iso_file_path, db).await?
|
||||||
{
|
{
|
||||||
warn!("Watcher found a file without parent: {}", &iso_file_path);
|
warn!(%iso_file_path, "Watcher found a file without parent;");
|
||||||
|
|
||||||
return Ok(());
|
return Ok(());
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -246,17 +326,13 @@ async fn inner_create_file(
|
||||||
fs_metadata,
|
fs_metadata,
|
||||||
} = FileMetadata::new(&location_path, &iso_file_path).await?;
|
} = FileMetadata::new(&location_path, &iso_file_path).await?;
|
||||||
|
|
||||||
debug!("Creating path: {}", iso_file_path);
|
|
||||||
|
|
||||||
let created_file =
|
let created_file =
|
||||||
create_file_path(library, iso_file_path_parts, cas_id.clone(), metadata).await?;
|
create_file_path(library, iso_file_path_parts, cas_id.clone(), metadata).await?;
|
||||||
|
|
||||||
object::select!(object_ids { id pub_id });
|
|
||||||
|
|
||||||
let existing_object = db
|
let existing_object = db
|
||||||
.object()
|
.object()
|
||||||
.find_first(vec![object::file_paths::some(vec![
|
.find_first(vec![object::file_paths::some(vec![
|
||||||
file_path::cas_id::equals(cas_id.clone()),
|
file_path::cas_id::equals(cas_id.clone().map(Into::into)),
|
||||||
file_path::pub_id::not(created_file.pub_id.clone()),
|
file_path::pub_id::not(created_file.pub_id.clone()),
|
||||||
])])
|
])])
|
||||||
.select(object_ids::select())
|
.select(object_ids::select())
|
||||||
|
@ -269,16 +345,17 @@ async fn inner_create_file(
|
||||||
} = if let Some(object) = existing_object {
|
} = if let Some(object) = existing_object {
|
||||||
object
|
object
|
||||||
} else {
|
} else {
|
||||||
let pub_id = uuid_to_bytes(Uuid::new_v4());
|
let pub_id: ObjectPubId = ObjectPubId::new();
|
||||||
let date_created: DateTime<FixedOffset> =
|
let date_created: DateTime<FixedOffset> =
|
||||||
DateTime::<Local>::from(fs_metadata.created_or_now()).into();
|
DateTime::<Local>::from(fs_metadata.created_or_now()).into();
|
||||||
let int_kind = kind as i32;
|
let int_kind = kind as i32;
|
||||||
|
|
||||||
sync.write_ops(
|
sync.write_ops(
|
||||||
db,
|
db,
|
||||||
(
|
(
|
||||||
sync.shared_create(
|
sync.shared_create(
|
||||||
prisma_sync::object::SyncId {
|
prisma_sync::object::SyncId {
|
||||||
pub_id: pub_id.clone(),
|
pub_id: pub_id.to_db(),
|
||||||
},
|
},
|
||||||
[
|
[
|
||||||
(object::date_created::NAME, msgpack!(date_created)),
|
(object::date_created::NAME, msgpack!(date_created)),
|
||||||
|
@ -287,7 +364,7 @@ async fn inner_create_file(
|
||||||
),
|
),
|
||||||
db.object()
|
db.object()
|
||||||
.create(
|
.create(
|
||||||
pub_id.to_vec(),
|
pub_id.into(),
|
||||||
vec![
|
vec![
|
||||||
object::date_created::set(Some(date_created)),
|
object::date_created::set(Some(date_created)),
|
||||||
object::kind::set(Some(int_kind)),
|
object::kind::set(Some(int_kind)),
|
||||||
|
@ -330,16 +407,21 @@ async fn inner_create_file(
|
||||||
spawn({
|
spawn({
|
||||||
let extension = extension.clone();
|
let extension = extension.clone();
|
||||||
let path = path.to_path_buf();
|
let path = path.to_path_buf();
|
||||||
let node = node.clone();
|
let thumbnails_directory =
|
||||||
|
get_thumbnails_directory(node.config.data_directory());
|
||||||
let library_id = *library_id;
|
let library_id = *library_id;
|
||||||
|
|
||||||
async move {
|
async move {
|
||||||
if let Err(e) = node
|
if let Err(e) = generate_single_thumbnail(
|
||||||
.thumbnailer
|
&thumbnails_directory,
|
||||||
.generate_single_indexed_thumbnail(&extension, cas_id, path, library_id)
|
extension,
|
||||||
.await
|
cas_id,
|
||||||
|
path,
|
||||||
|
ThumbnailKind::Indexed(library_id),
|
||||||
|
)
|
||||||
|
.await
|
||||||
{
|
{
|
||||||
error!("Failed to generate thumbnail in the watcher: {e:#?}");
|
error!(?e, "Failed to generate thumbnail in the watcher;");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
@ -349,34 +431,15 @@ async fn inner_create_file(
|
||||||
match kind {
|
match kind {
|
||||||
ObjectKind::Image => {
|
ObjectKind::Image => {
|
||||||
if let Ok(image_extension) = ImageExtension::from_str(&extension) {
|
if let Ok(image_extension) = ImageExtension::from_str(&extension) {
|
||||||
if can_extract_exif_data_for_image(&image_extension) {
|
if exif_media_data::can_extract(image_extension) {
|
||||||
if let Ok(Some(exif_data)) = extract_exif_data(path)
|
if let Ok(Some(exif_data)) = exif_media_data::extract(path)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| error!("Failed to extract media data: {e:#?}"))
|
.map_err(|e| error!(?e, "Failed to extract image media data;"))
|
||||||
{
|
{
|
||||||
let (sync_params, db_params) =
|
exif_media_data::save(
|
||||||
exif_data_image_to_query_params(exif_data);
|
[(exif_data, object_id, object_pub_id.into())],
|
||||||
|
|
||||||
sync.write_ops(
|
|
||||||
db,
|
db,
|
||||||
(
|
sync,
|
||||||
sync.shared_create(
|
|
||||||
prisma_sync::exif_data::SyncId {
|
|
||||||
object: prisma_sync::object::SyncId {
|
|
||||||
pub_id: object_pub_id.clone(),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
sync_params,
|
|
||||||
),
|
|
||||||
db.exif_data().upsert(
|
|
||||||
exif_data::object_id::equals(object_id),
|
|
||||||
exif_data::create(
|
|
||||||
object::id::equals(object_id),
|
|
||||||
db_params.clone(),
|
|
||||||
),
|
|
||||||
db_params,
|
|
||||||
),
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
}
|
}
|
||||||
|
@ -386,12 +449,12 @@ async fn inner_create_file(
|
||||||
|
|
||||||
ObjectKind::Audio => {
|
ObjectKind::Audio => {
|
||||||
if let Ok(audio_extension) = AudioExtension::from_str(&extension) {
|
if let Ok(audio_extension) = AudioExtension::from_str(&extension) {
|
||||||
if can_extract_ffmpeg_data_for_audio(&audio_extension) {
|
if ffmpeg_media_data::can_extract_for_audio(audio_extension) {
|
||||||
if let Ok(ffmpeg_data) = extract_ffmpeg_data(path)
|
if let Ok(ffmpeg_data) = ffmpeg_media_data::extract(path)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| error!("Failed to extract media data: {e:#?}"))
|
.map_err(|e| error!(?e, "Failed to extract audio media data;"))
|
||||||
{
|
{
|
||||||
save_ffmpeg_data([(ffmpeg_data, object_id)], db).await?;
|
ffmpeg_media_data::save([(ffmpeg_data, object_id)], db).await?;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -399,12 +462,12 @@ async fn inner_create_file(
|
||||||
|
|
||||||
ObjectKind::Video => {
|
ObjectKind::Video => {
|
||||||
if let Ok(video_extension) = VideoExtension::from_str(&extension) {
|
if let Ok(video_extension) = VideoExtension::from_str(&extension) {
|
||||||
if can_extract_ffmpeg_data_for_video(&video_extension) {
|
if ffmpeg_media_data::can_extract_for_video(video_extension) {
|
||||||
if let Ok(ffmpeg_data) = extract_ffmpeg_data(path)
|
if let Ok(ffmpeg_data) = ffmpeg_media_data::extract(path)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| error!("Failed to extract media data: {e:#?}"))
|
.map_err(|e| error!(?e, "Failed to extract video media data;"))
|
||||||
{
|
{
|
||||||
save_ffmpeg_data([(ffmpeg_data, object_id)], db).await?;
|
ffmpeg_media_data::save([(ffmpeg_data, object_id)], db).await?;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -422,13 +485,14 @@ async fn inner_create_file(
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[instrument(skip_all, fields(path = %path.as_ref().display()), err)]
|
||||||
pub(super) async fn update_file(
|
pub(super) async fn update_file(
|
||||||
location_id: location::id::Type,
|
location_id: location::id::Type,
|
||||||
full_path: impl AsRef<Path>,
|
path: impl AsRef<Path> + Send,
|
||||||
node: &Arc<Node>,
|
node: &Arc<Node>,
|
||||||
library: &Arc<Library>,
|
library: &Arc<Library>,
|
||||||
) -> Result<(), LocationManagerError> {
|
) -> Result<(), LocationManagerError> {
|
||||||
let full_path = full_path.as_ref();
|
let full_path = path.as_ref();
|
||||||
|
|
||||||
let metadata = match fs::metadata(full_path).await {
|
let metadata = match fs::metadata(full_path).await {
|
||||||
Ok(metadata) => metadata,
|
Ok(metadata) => metadata,
|
||||||
|
@ -464,16 +528,16 @@ pub(super) async fn update_file(
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
.map(|_| {
|
.map(|()| {
|
||||||
invalidate_query!(library, "search.paths");
|
invalidate_query!(library, "search.paths");
|
||||||
invalidate_query!(library, "search.objects");
|
invalidate_query!(library, "search.objects");
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn inner_update_file(
|
async fn inner_update_file(
|
||||||
location_path: impl AsRef<Path>,
|
location_path: impl AsRef<Path> + Send,
|
||||||
file_path: &file_path_with_object::Data,
|
file_path: &file_path_with_object::Data,
|
||||||
full_path: impl AsRef<Path>,
|
full_path: impl AsRef<Path> + Send,
|
||||||
node: &Arc<Node>,
|
node: &Arc<Node>,
|
||||||
library @ Library { db, sync, .. }: &Library,
|
library @ Library { db, sync, .. }: &Library,
|
||||||
maybe_new_inode: Option<INode>,
|
maybe_new_inode: Option<INode>,
|
||||||
|
@ -485,9 +549,9 @@ async fn inner_update_file(
|
||||||
inode_from_db(&maybe_missing(file_path.inode.as_ref(), "file_path.inode")?[0..8]);
|
inode_from_db(&maybe_missing(file_path.inode.as_ref(), "file_path.inode")?[0..8]);
|
||||||
|
|
||||||
trace!(
|
trace!(
|
||||||
"Location: <root_path ='{}'> updating file: {}",
|
location_path = %location_path.display(),
|
||||||
location_path.display(),
|
path = %full_path.display(),
|
||||||
full_path.display()
|
"Updating file;",
|
||||||
);
|
);
|
||||||
|
|
||||||
let iso_file_path = IsolatedFilePathData::try_from(file_path)?;
|
let iso_file_path = IsolatedFilePathData::try_from(file_path)?;
|
||||||
|
@ -514,7 +578,7 @@ async fn inner_update_file(
|
||||||
};
|
};
|
||||||
|
|
||||||
let is_hidden = path_is_hidden(full_path, &fs_metadata);
|
let is_hidden = path_is_hidden(full_path, &fs_metadata);
|
||||||
if file_path.cas_id != cas_id {
|
if file_path.cas_id.as_deref() != cas_id.as_ref().map(CasId::as_str) {
|
||||||
let (sync_params, db_params): (Vec<_>, Vec<_>) = {
|
let (sync_params, db_params): (Vec<_>, Vec<_>) = {
|
||||||
use file_path::*;
|
use file_path::*;
|
||||||
|
|
||||||
|
@ -637,7 +701,7 @@ async fn inner_update_file(
|
||||||
.await?;
|
.await?;
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
let pub_id = uuid_to_bytes(Uuid::new_v4());
|
let pub_id = ObjectPubId::new();
|
||||||
let date_created: DateTime<FixedOffset> =
|
let date_created: DateTime<FixedOffset> =
|
||||||
DateTime::<Local>::from(fs_metadata.created_or_now()).into();
|
DateTime::<Local>::from(fs_metadata.created_or_now()).into();
|
||||||
|
|
||||||
|
@ -646,7 +710,7 @@ async fn inner_update_file(
|
||||||
(
|
(
|
||||||
sync.shared_create(
|
sync.shared_create(
|
||||||
prisma_sync::object::SyncId {
|
prisma_sync::object::SyncId {
|
||||||
pub_id: pub_id.clone(),
|
pub_id: pub_id.to_db(),
|
||||||
},
|
},
|
||||||
[
|
[
|
||||||
(object::date_created::NAME, msgpack!(date_created)),
|
(object::date_created::NAME, msgpack!(date_created)),
|
||||||
|
@ -654,7 +718,7 @@ async fn inner_update_file(
|
||||||
],
|
],
|
||||||
),
|
),
|
||||||
db.object().create(
|
db.object().create(
|
||||||
pub_id.to_vec(),
|
pub_id.to_db(),
|
||||||
vec![
|
vec![
|
||||||
object::date_created::set(Some(date_created)),
|
object::date_created::set(Some(date_created)),
|
||||||
object::kind::set(Some(int_kind)),
|
object::kind::set(Some(int_kind)),
|
||||||
|
@ -672,49 +736,57 @@ async fn inner_update_file(
|
||||||
},
|
},
|
||||||
file_path::object::NAME,
|
file_path::object::NAME,
|
||||||
msgpack!(prisma_sync::object::SyncId {
|
msgpack!(prisma_sync::object::SyncId {
|
||||||
pub_id: pub_id.clone()
|
pub_id: pub_id.to_db()
|
||||||
}),
|
}),
|
||||||
),
|
),
|
||||||
db.file_path().update(
|
db.file_path().update(
|
||||||
file_path::pub_id::equals(file_path.pub_id.clone()),
|
file_path::pub_id::equals(file_path.pub_id.clone()),
|
||||||
vec![file_path::object::connect(object::pub_id::equals(pub_id))],
|
vec![file_path::object::connect(object::pub_id::equals(
|
||||||
|
pub_id.into(),
|
||||||
|
))],
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some(old_cas_id) = &file_path.cas_id {
|
if let Some(old_cas_id) = file_path.cas_id.as_ref().map(CasId::from) {
|
||||||
// if this file had a thumbnail previously, we update it to match the new content
|
// if this file had a thumbnail previously, we update it to match the new content
|
||||||
if library.thumbnail_exists(node, old_cas_id).await? {
|
if library.thumbnail_exists(node, &old_cas_id).await? {
|
||||||
if let Some(ext) = file_path.extension.clone() {
|
if let Some(ext) = file_path.extension.clone() {
|
||||||
// Running in a detached task as thumbnail generation can take a while and we don't want to block the watcher
|
// Running in a detached task as thumbnail generation can take a while and we don't want to block the watcher
|
||||||
if let Some(cas_id) = cas_id {
|
if let Some(cas_id) = cas_id {
|
||||||
let node = Arc::clone(node);
|
let node = Arc::clone(node);
|
||||||
let path = full_path.to_path_buf();
|
let path = full_path.to_path_buf();
|
||||||
let library_id = library.id;
|
let library_id = library.id;
|
||||||
let old_cas_id = old_cas_id.clone();
|
let old_cas_id = old_cas_id.to_owned();
|
||||||
|
|
||||||
spawn(async move {
|
spawn(async move {
|
||||||
|
let thumbnails_directory =
|
||||||
|
get_thumbnails_directory(node.config.data_directory());
|
||||||
|
|
||||||
let was_overwritten = old_cas_id == cas_id;
|
let was_overwritten = old_cas_id == cas_id;
|
||||||
if let Err(e) = node
|
if let Err(e) = generate_single_thumbnail(
|
||||||
.thumbnailer
|
&thumbnails_directory,
|
||||||
.generate_single_indexed_thumbnail(
|
ext.clone(),
|
||||||
&ext, cas_id, path, library_id,
|
cas_id,
|
||||||
)
|
path,
|
||||||
.await
|
ThumbnailKind::Indexed(library_id),
|
||||||
|
)
|
||||||
|
.await
|
||||||
{
|
{
|
||||||
error!("Failed to generate thumbnail in the watcher: {e:#?}");
|
error!(?e, "Failed to generate thumbnail in the watcher;");
|
||||||
}
|
}
|
||||||
|
|
||||||
// If only a few bytes changed, cas_id will probably remains intact
|
// If only a few bytes changed, cas_id will probably remains intact
|
||||||
// so we overwrote our previous thumbnail, so we can't remove it
|
// so we overwrote our previous thumbnail, so we can't remove it
|
||||||
if !was_overwritten {
|
if !was_overwritten {
|
||||||
// remove the old thumbnail as we're generating a new one
|
// remove the old thumbnail as we're generating a new one
|
||||||
let thumb_path =
|
let thumb_path = ThumbnailKind::Indexed(library_id)
|
||||||
get_indexed_thumbnail_path(&node, &old_cas_id, library_id);
|
.compute_path(node.config.data_directory(), &old_cas_id);
|
||||||
if let Err(e) = fs::remove_file(&thumb_path).await {
|
if let Err(e) = fs::remove_file(&thumb_path).await {
|
||||||
error!(
|
error!(
|
||||||
"Failed to remove old thumbnail: {:#?}",
|
e = ?FileIOError::from((thumb_path, e)),
|
||||||
FileIOError::from((thumb_path, e))
|
"Failed to remove old thumbnail;",
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -728,34 +800,15 @@ async fn inner_update_file(
|
||||||
match kind {
|
match kind {
|
||||||
ObjectKind::Image => {
|
ObjectKind::Image => {
|
||||||
if let Ok(image_extension) = ImageExtension::from_str(extension) {
|
if let Ok(image_extension) = ImageExtension::from_str(extension) {
|
||||||
if can_extract_exif_data_for_image(&image_extension) {
|
if exif_media_data::can_extract(image_extension) {
|
||||||
if let Ok(Some(exif_data)) = extract_exif_data(full_path)
|
if let Ok(Some(exif_data)) = exif_media_data::extract(full_path)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| error!("Failed to extract media data: {e:#?}"))
|
.map_err(|e| error!(?e, "Failed to extract media data;"))
|
||||||
{
|
{
|
||||||
let (sync_params, db_params) =
|
exif_media_data::save(
|
||||||
exif_data_image_to_query_params(exif_data);
|
[(exif_data, object.id, object.pub_id.as_slice().into())],
|
||||||
|
|
||||||
sync.write_ops(
|
|
||||||
db,
|
db,
|
||||||
(
|
sync,
|
||||||
sync.shared_create(
|
|
||||||
prisma_sync::exif_data::SyncId {
|
|
||||||
object: prisma_sync::object::SyncId {
|
|
||||||
pub_id: object.pub_id.clone(),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
sync_params,
|
|
||||||
),
|
|
||||||
db.exif_data().upsert(
|
|
||||||
exif_data::object_id::equals(object.id),
|
|
||||||
exif_data::create(
|
|
||||||
object::id::equals(object.id),
|
|
||||||
db_params.clone(),
|
|
||||||
),
|
|
||||||
db_params,
|
|
||||||
),
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
}
|
}
|
||||||
|
@ -765,12 +818,12 @@ async fn inner_update_file(
|
||||||
|
|
||||||
ObjectKind::Audio => {
|
ObjectKind::Audio => {
|
||||||
if let Ok(audio_extension) = AudioExtension::from_str(extension) {
|
if let Ok(audio_extension) = AudioExtension::from_str(extension) {
|
||||||
if can_extract_ffmpeg_data_for_audio(&audio_extension) {
|
if ffmpeg_media_data::can_extract_for_audio(audio_extension) {
|
||||||
if let Ok(ffmpeg_data) = extract_ffmpeg_data(full_path)
|
if let Ok(ffmpeg_data) = ffmpeg_media_data::extract(full_path)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| error!("Failed to extract media data: {e:#?}"))
|
.map_err(|e| error!(?e, "Failed to extract media data;"))
|
||||||
{
|
{
|
||||||
save_ffmpeg_data([(ffmpeg_data, object.id)], db).await?;
|
ffmpeg_media_data::save([(ffmpeg_data, object.id)], db).await?;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -778,12 +831,12 @@ async fn inner_update_file(
|
||||||
|
|
||||||
ObjectKind::Video => {
|
ObjectKind::Video => {
|
||||||
if let Ok(video_extension) = VideoExtension::from_str(extension) {
|
if let Ok(video_extension) = VideoExtension::from_str(extension) {
|
||||||
if can_extract_ffmpeg_data_for_video(&video_extension) {
|
if ffmpeg_media_data::can_extract_for_video(video_extension) {
|
||||||
if let Ok(ffmpeg_data) = extract_ffmpeg_data(full_path)
|
if let Ok(ffmpeg_data) = ffmpeg_media_data::extract(full_path)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| error!("Failed to extract media data: {e:#?}"))
|
.map_err(|e| error!(?e, "Failed to extract media data;"))
|
||||||
{
|
{
|
||||||
save_ffmpeg_data([(ffmpeg_data, object.id)], db).await?;
|
ffmpeg_media_data::save([(ffmpeg_data, object.id)], db).await?;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -823,10 +876,15 @@ async fn inner_update_file(
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[instrument(
|
||||||
|
skip_all,
|
||||||
|
fields(new_path = %new_path.as_ref().display(), old_path = %old_path.as_ref().display()),
|
||||||
|
err,
|
||||||
|
)]
|
||||||
pub(super) async fn rename(
|
pub(super) async fn rename(
|
||||||
location_id: location::id::Type,
|
location_id: location::id::Type,
|
||||||
new_path: impl AsRef<Path>,
|
new_path: impl AsRef<Path> + Send,
|
||||||
old_path: impl AsRef<Path>,
|
old_path: impl AsRef<Path> + Send,
|
||||||
new_path_metadata: Metadata,
|
new_path_metadata: Metadata,
|
||||||
library: &Library,
|
library: &Library,
|
||||||
) -> Result<(), LocationManagerError> {
|
) -> Result<(), LocationManagerError> {
|
||||||
|
@ -841,7 +899,8 @@ pub(super) async fn rename(
|
||||||
let new_path_materialized_str =
|
let new_path_materialized_str =
|
||||||
extract_normalized_materialized_path_str(location_id, &location_path, new_path)?;
|
extract_normalized_materialized_path_str(location_id, &location_path, new_path)?;
|
||||||
|
|
||||||
// Renaming a file could potentially be a move to another directory, so we check if our parent changed
|
// Renaming a file could potentially be a move to another directory,
|
||||||
|
// so we check if our parent changed
|
||||||
if old_path_materialized_str != new_path_materialized_str
|
if old_path_materialized_str != new_path_materialized_str
|
||||||
&& !check_file_path_exists::<FilePathError>(
|
&& !check_file_path_exists::<FilePathError>(
|
||||||
&IsolatedFilePathData::new(location_id, &location_path, new_path, true)?.parent(),
|
&IsolatedFilePathData::new(location_id, &location_path, new_path, true)?.parent(),
|
||||||
|
@ -851,7 +910,7 @@ pub(super) async fn rename(
|
||||||
{
|
{
|
||||||
return Err(LocationManagerError::MoveError {
|
return Err(LocationManagerError::MoveError {
|
||||||
path: new_path.into(),
|
path: new_path.into(),
|
||||||
reason: "parent directory does not exist".into(),
|
reason: "parent directory does not exist",
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -890,7 +949,7 @@ pub(super) async fn rename(
|
||||||
.exec()
|
.exec()
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
let len = paths.len();
|
let total_paths_count = paths.len();
|
||||||
let (sync_params, db_params): (Vec<_>, Vec<_>) = paths
|
let (sync_params, db_params): (Vec<_>, Vec<_>) = paths
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.filter_map(|path| path.materialized_path.map(|mp| (path.id, path.pub_id, mp)))
|
.filter_map(|path| path.materialized_path.map(|mp| (path.id, path.pub_id, mp)))
|
||||||
|
@ -916,7 +975,7 @@ pub(super) async fn rename(
|
||||||
|
|
||||||
sync.write_ops(db, (sync_params, db_params)).await?;
|
sync.write_ops(db, (sync_params, db_params)).await?;
|
||||||
|
|
||||||
trace!("Updated {len} file_paths");
|
trace!(%total_paths_count, "Updated file_paths;");
|
||||||
}
|
}
|
||||||
|
|
||||||
let is_hidden = path_is_hidden(new_path, &new_path_metadata);
|
let is_hidden = path_is_hidden(new_path, &new_path_metadata);
|
||||||
|
@ -979,12 +1038,13 @@ pub(super) async fn rename(
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[instrument(skip_all, fields(path = %path.as_ref().display()), err)]
|
||||||
pub(super) async fn remove(
|
pub(super) async fn remove(
|
||||||
location_id: location::id::Type,
|
location_id: location::id::Type,
|
||||||
full_path: impl AsRef<Path>,
|
path: impl AsRef<Path> + Send,
|
||||||
library: &Library,
|
library: &Library,
|
||||||
) -> Result<(), LocationManagerError> {
|
) -> Result<(), LocationManagerError> {
|
||||||
let full_path = full_path.as_ref();
|
let full_path = path.as_ref();
|
||||||
let location_path = extract_location_path(location_id, library).await?;
|
let location_path = extract_location_path(location_id, library).await?;
|
||||||
|
|
||||||
// if it doesn't exist either way, then we don't care
|
// if it doesn't exist either way, then we don't care
|
||||||
|
@ -1005,16 +1065,22 @@ pub(super) async fn remove(
|
||||||
remove_by_file_path(location_id, full_path, &file_path, library).await
|
remove_by_file_path(location_id, full_path, &file_path, library).await
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(super) async fn remove_by_file_path(
|
async fn remove_by_file_path(
|
||||||
location_id: location::id::Type,
|
location_id: location::id::Type,
|
||||||
path: impl AsRef<Path>,
|
path: impl AsRef<Path> + Send,
|
||||||
file_path: &file_path::Data,
|
file_path: &file_path::Data,
|
||||||
library: &Library,
|
library: &Library,
|
||||||
) -> Result<(), LocationManagerError> {
|
) -> Result<(), LocationManagerError> {
|
||||||
// check file still exists on disk
|
// check file still exists on disk
|
||||||
match fs::metadata(path.as_ref()).await {
|
match fs::metadata(path.as_ref()).await {
|
||||||
Ok(_) => {
|
Ok(_) => {
|
||||||
todo!("file has changed in some way, re-identify it")
|
// It's possible that in the interval of time between the removal file event being
|
||||||
|
// received and we reaching this point, the file has been created again for some
|
||||||
|
// external reason, so we just error out and hope to receive this new create event
|
||||||
|
// later
|
||||||
|
return Err(LocationManagerError::FileStillExistsOnDisk(
|
||||||
|
path.as_ref().into(),
|
||||||
|
));
|
||||||
}
|
}
|
||||||
Err(e) if e.kind() == ErrorKind::NotFound => {
|
Err(e) if e.kind() == ErrorKind::NotFound => {
|
||||||
let Library { sync, db, .. } = library;
|
let Library { sync, db, .. } = library;
|
||||||
|
@ -1060,9 +1126,10 @@ pub(super) async fn remove_by_file_path(
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[instrument(skip_all, fields(path = %path.as_ref().display()), err)]
|
||||||
pub(super) async fn extract_inode_from_path(
|
pub(super) async fn extract_inode_from_path(
|
||||||
location_id: location::id::Type,
|
location_id: location::id::Type,
|
||||||
path: impl AsRef<Path>,
|
path: impl AsRef<Path> + Send,
|
||||||
library: &Library,
|
library: &Library,
|
||||||
) -> Result<INode, LocationManagerError> {
|
) -> Result<INode, LocationManagerError> {
|
||||||
let path = path.as_ref();
|
let path = path.as_ref();
|
||||||
|
@ -1070,7 +1137,7 @@ pub(super) async fn extract_inode_from_path(
|
||||||
.select(location::select!({ path }))
|
.select(location::select!({ path }))
|
||||||
.exec()
|
.exec()
|
||||||
.await?
|
.await?
|
||||||
.ok_or(LocationManagerError::MissingLocation(location_id))?;
|
.ok_or(LocationManagerError::LocationNotFound(location_id))?;
|
||||||
|
|
||||||
let location_path = maybe_missing(&location.path, "location.path")?;
|
let location_path = maybe_missing(&location.path, "location.path")?;
|
||||||
|
|
||||||
|
@ -1095,6 +1162,7 @@ pub(super) async fn extract_inode_from_path(
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[instrument(skip_all, err)]
|
||||||
pub(super) async fn extract_location_path(
|
pub(super) async fn extract_location_path(
|
||||||
location_id: location::id::Type,
|
location_id: location::id::Type,
|
||||||
library: &Library,
|
library: &Library,
|
||||||
|
@ -1104,12 +1172,12 @@ pub(super) async fn extract_location_path(
|
||||||
.exec()
|
.exec()
|
||||||
.await?
|
.await?
|
||||||
.map_or(
|
.map_or(
|
||||||
Err(LocationManagerError::MissingLocation(location_id)),
|
Err(LocationManagerError::LocationNotFound(location_id)),
|
||||||
// NOTE: The following usage of `PathBuf` doesn't incur a new allocation so it's fine
|
// NOTE: The following usage of `PathBuf` doesn't incur a new allocation so it's fine
|
||||||
|location| Ok(maybe_missing(location.path, "location.path")?.into()),
|
|location| Ok(maybe_missing(location.path, "location.path")?.into()),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
#[instrument(skip_all, err)]
|
||||||
pub(super) async fn recalculate_directories_size(
|
pub(super) async fn recalculate_directories_size(
|
||||||
candidates: &mut HashMap<PathBuf, Instant>,
|
candidates: &mut HashMap<PathBuf, Instant>,
|
||||||
buffer: &mut Vec<(PathBuf, Instant)>,
|
buffer: &mut Vec<(PathBuf, Instant)>,
|
||||||
|
@ -1129,7 +1197,7 @@ pub(super) async fn recalculate_directories_size(
|
||||||
.select(location::select!({ path }))
|
.select(location::select!({ path }))
|
||||||
.exec()
|
.exec()
|
||||||
.await?
|
.await?
|
||||||
.ok_or(LocationManagerError::MissingLocation(location_id))?
|
.ok_or(LocationManagerError::LocationNotFound(location_id))?
|
||||||
.path,
|
.path,
|
||||||
"location.path",
|
"location.path",
|
||||||
)?))
|
)?))
|
||||||
|
@ -1138,12 +1206,29 @@ pub(super) async fn recalculate_directories_size(
|
||||||
if let Some(location_path) = &location_path_cache {
|
if let Some(location_path) = &location_path_cache {
|
||||||
if path != *location_path {
|
if path != *location_path {
|
||||||
trace!(
|
trace!(
|
||||||
"Reverse calculating directory sizes starting at {} until {}",
|
start_directory = %path.display(),
|
||||||
path.display(),
|
end_directory = %location_path.display(),
|
||||||
location_path.display(),
|
"Reverse calculating directory sizes;",
|
||||||
);
|
);
|
||||||
reverse_update_directories_sizes(path, location_id, location_path, library)
|
let mut non_critical_errors = vec![];
|
||||||
.await?;
|
reverse_update_directories_sizes(
|
||||||
|
path,
|
||||||
|
location_id,
|
||||||
|
location_path,
|
||||||
|
&library.db,
|
||||||
|
&library.sync,
|
||||||
|
&mut non_critical_errors,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(sd_core_heavy_lifting::Error::from)?;
|
||||||
|
|
||||||
|
if !non_critical_errors.is_empty() {
|
||||||
|
error!(
|
||||||
|
?non_critical_errors,
|
||||||
|
"Reverse calculating directory sizes finished errors;",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
should_invalidate = true;
|
should_invalidate = true;
|
||||||
} else {
|
} else {
|
||||||
should_update_location_size = true;
|
should_update_location_size = true;
|
||||||
|
|
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue