mirror of
https://github.com/spacedriveapp/spacedrive
synced 2024-07-07 00:53:28 +00:00
Compare commits
9 commits
7c00d2ef50
...
a4db125c96
Author | SHA1 | Date | |
---|---|---|---|
a4db125c96 | |||
cee3389eb7 | |||
aa9a117a22 | |||
6cdaed47ec | |||
f72166fe70 | |||
7f6f6d2067 | |||
7ea836ebb2 | |||
3f05cec895 | |||
128b38c007 |
73
.vscode/tasks.json
vendored
73
.vscode/tasks.json
vendored
|
@ -60,7 +60,6 @@
|
||||||
],
|
],
|
||||||
"env": {
|
"env": {
|
||||||
"RUST_BACKTRACE": "short"
|
"RUST_BACKTRACE": "short"
|
||||||
// "RUST_LOG": "sd_core::invalidate-query=trace"
|
|
||||||
},
|
},
|
||||||
"problemMatcher": ["$rustc"],
|
"problemMatcher": ["$rustc"],
|
||||||
"group": "build",
|
"group": "build",
|
||||||
|
@ -78,6 +77,78 @@
|
||||||
"group": "build",
|
"group": "build",
|
||||||
"label": "rust: run spacedrive release",
|
"label": "rust: run spacedrive release",
|
||||||
"dependsOn": ["ui:build"]
|
"dependsOn": ["ui:build"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "cargo",
|
||||||
|
"command": "test",
|
||||||
|
"args": [
|
||||||
|
"--package",
|
||||||
|
"sd-core-sync",
|
||||||
|
"--test",
|
||||||
|
"lib",
|
||||||
|
"--",
|
||||||
|
"writes_operations_and_rows_together",
|
||||||
|
"--exact",
|
||||||
|
"--show-output"
|
||||||
|
],
|
||||||
|
"env": {
|
||||||
|
"RUST_BACKTRACE": "short",
|
||||||
|
"BROWSER": "open",
|
||||||
|
"COMMAND_MODE": "unix2003",
|
||||||
|
"EDITOR": "vim",
|
||||||
|
"GOPATH": "/Users/ericson/.go",
|
||||||
|
"GREP_COLOR": "37;45",
|
||||||
|
"GREP_COLORS": "mt=37;45",
|
||||||
|
"HOME": "/Users/ericson",
|
||||||
|
"HOMEBREW_CELLAR": "/opt/homebrew/Cellar",
|
||||||
|
"HOMEBREW_PREFIX": "/opt/homebrew",
|
||||||
|
"HOMEBREW_REPOSITORY": "/opt/homebrew",
|
||||||
|
"INFOPATH": "/opt/homebrew/share/info:/usr/local/share/info:",
|
||||||
|
"LANG": "en_US.UTF-8",
|
||||||
|
"LDFLAGS": "-L/opt/homebrew/opt/llvm/lib/c++ -Wl,-rpath,/opt/homebrew/opt/llvm/lib/c++",
|
||||||
|
"LESS": "-g -i -M -R -S -w -X -z-4",
|
||||||
|
"LESS_TERMCAP_mb": "\u001b[01;31m",
|
||||||
|
"LESS_TERMCAP_md": "\u001b[01;31m",
|
||||||
|
"LESS_TERMCAP_me": "\u001b[0m",
|
||||||
|
"LESS_TERMCAP_se": "\u001b[0m",
|
||||||
|
"LESS_TERMCAP_so": "\u001b[00;47;30m",
|
||||||
|
"LESS_TERMCAP_ue": "\u001b[0m",
|
||||||
|
"LESS_TERMCAP_us": "\u001b[01;32m",
|
||||||
|
"LOGNAME": "ericson",
|
||||||
|
"LSCOLORS": "exfxcxdxbxGxDxabagacad",
|
||||||
|
"MANPATH": "/opt/homebrew/share/man:/usr/local/share/man:::",
|
||||||
|
"MallocNanoZone": "0",
|
||||||
|
"ORIGINAL_XDG_CURRENT_DESKTOP": "undefined",
|
||||||
|
"P9K_SSH": "0",
|
||||||
|
"PAGER": "less",
|
||||||
|
"PATH": "/usr/local/opt/openjdk/bin:/usr/local/opt/game-porting-toolkit/bin:/opt/homebrew/bin:/opt/homebrew/sbin:/Users/ericson/.pnpm:/usr/local/bin:/usr/local/sbin:/opt/homebrew/bin:/opt/homebrew/sbin:/usr/local/bin:/usr/local/sbin:/System/Cryptexes/App/usr/bin:/usr/bin:/bin:/usr/sbin:/sbin:/var/run/com.apple.security.cryptexd/codex.system/bootstrap/usr/local/bin:/var/run/com.apple.security.cryptexd/codex.system/bootstrap/usr/bin:/var/run/com.apple.security.cryptexd/codex.system/bootstrap/usr/appleinternal/bin:/Library/Apple/usr/bin:/usr/local/share/dotnet:~/.dotnet/tools:/Library/Frameworks/Mono.framework/Versions/Current/Commands:/Users/ericson/.cargo/bin:/Users/ericson/Library/Application Support/JetBrains/Toolbox/scripts:/Users/ericson/.local/bin:/Users/ericson/.go/bin:/Users/ericson/.local/share/containers/podman-desktop/extensions-storage/podman-desktop.compose/bin",
|
||||||
|
"PNPM_HOME": "/Users/ericson/.pnpm",
|
||||||
|
"PWD": "/",
|
||||||
|
"SHELL": "/bin/zsh",
|
||||||
|
"SHLVL": "0",
|
||||||
|
"SSH_AUTH_SOCK": "/private/tmp/com.apple.launchd.a62yq49fKe/Listeners",
|
||||||
|
"TMPDIR": "/var/folders/k5/pvf6cfbd05s_prpwdl0h03rr0000gn/T/",
|
||||||
|
"USER": "ericson",
|
||||||
|
"VISUAL": "vim",
|
||||||
|
"VSCODE_AMD_ENTRYPOINT": "vs/workbench/api/node/extensionHostProcess",
|
||||||
|
"VSCODE_CODE_CACHE_PATH": "/Users/ericson/Library/Application Support/Code/CachedData/611f9bfce64f25108829dd295f54a6894e87339d",
|
||||||
|
"VSCODE_CRASH_REPORTER_PROCESS_TYPE": "extensionHost",
|
||||||
|
"VSCODE_CWD": "/",
|
||||||
|
"VSCODE_HANDLES_UNCAUGHT_ERRORS": "true",
|
||||||
|
"VSCODE_IPC_HOOK": "/Users/ericson/Library/Application Support/Code/1.90-main.sock",
|
||||||
|
"VSCODE_NLS_CONFIG": "{\"locale\":\"pt-br\",\"osLocale\":\"pt-br\",\"availableLanguages\":{},\"_languagePackSupport\":true}",
|
||||||
|
"VSCODE_PID": "79712",
|
||||||
|
"XPC_FLAGS": "0x0",
|
||||||
|
"XPC_SERVICE_NAME": "application.com.microsoft.VSCode.81888144.81888150",
|
||||||
|
"_": "/Applications/Visual Studio Code.app/Contents/MacOS/Electron",
|
||||||
|
"__CFBundleIdentifier": "com.microsoft.VSCode",
|
||||||
|
"__CF_USER_TEXT_ENCODING": "0x1F5:0x0:0x47",
|
||||||
|
"ELECTRON_RUN_AS_NODE": "1",
|
||||||
|
"VSCODE_L10N_BUNDLE_LOCATION": ""
|
||||||
|
},
|
||||||
|
"problemMatcher": ["$rustc"],
|
||||||
|
"group": "build",
|
||||||
|
"label": "rust: test writes_operations_and_rows_together"
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|
|
@ -89,7 +89,7 @@ To run the landing page:
|
||||||
|
|
||||||
If you encounter any issues, ensure that you are using the following versions of Rust, Node and Pnpm:
|
If you encounter any issues, ensure that you are using the following versions of Rust, Node and Pnpm:
|
||||||
|
|
||||||
- Rust version: **1.78**
|
- Rust version: **1.79**
|
||||||
- Node version: **18.18**
|
- Node version: **18.18**
|
||||||
- Pnpm version: **9.1.1**
|
- Pnpm version: **9.1.1**
|
||||||
|
|
||||||
|
|
13
Cargo.lock
generated
13
Cargo.lock
generated
|
@ -8888,8 +8888,11 @@ dependencies = [
|
||||||
name = "sd-actors"
|
name = "sd-actors"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
|
"async-channel",
|
||||||
"futures",
|
"futures",
|
||||||
|
"pin-project-lite",
|
||||||
"tokio",
|
"tokio",
|
||||||
|
"tracing",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
|
@ -9047,6 +9050,7 @@ dependencies = [
|
||||||
"prisma-client-rust",
|
"prisma-client-rust",
|
||||||
"regex",
|
"regex",
|
||||||
"sd-core-prisma-helpers",
|
"sd-core-prisma-helpers",
|
||||||
|
"sd-core-sync",
|
||||||
"sd-prisma",
|
"sd-prisma",
|
||||||
"sd-utils",
|
"sd-utils",
|
||||||
"serde",
|
"serde",
|
||||||
|
@ -9141,17 +9145,24 @@ dependencies = [
|
||||||
name = "sd-core-sync"
|
name = "sd-core-sync"
|
||||||
version = "0.0.0"
|
version = "0.0.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
|
"async-channel",
|
||||||
|
"futures",
|
||||||
|
"futures-concurrency",
|
||||||
"prisma-client-rust",
|
"prisma-client-rust",
|
||||||
"rmp-serde",
|
"rmp-serde",
|
||||||
"rmpv",
|
"rmpv",
|
||||||
|
"rspc",
|
||||||
"sd-actors",
|
"sd-actors",
|
||||||
"sd-prisma",
|
"sd-prisma",
|
||||||
"sd-sync",
|
"sd-sync",
|
||||||
"sd-utils",
|
"sd-utils",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
|
"thiserror",
|
||||||
"tokio",
|
"tokio",
|
||||||
"tracing",
|
"tracing",
|
||||||
|
"tracing-subscriber",
|
||||||
|
"tracing-test",
|
||||||
"uhlc",
|
"uhlc",
|
||||||
"uuid",
|
"uuid",
|
||||||
]
|
]
|
||||||
|
@ -9470,7 +9481,7 @@ dependencies = [
|
||||||
"futures",
|
"futures",
|
||||||
"futures-concurrency",
|
"futures-concurrency",
|
||||||
"lending-stream",
|
"lending-stream",
|
||||||
"pin-project",
|
"pin-project-lite",
|
||||||
"rand 0.8.5",
|
"rand 0.8.5",
|
||||||
"rmp-serde",
|
"rmp-serde",
|
||||||
"serde",
|
"serde",
|
||||||
|
|
|
@ -11,6 +11,7 @@ edition = { workspace = true }
|
||||||
[dependencies]
|
[dependencies]
|
||||||
# Inner Core Sub-crates
|
# Inner Core Sub-crates
|
||||||
sd-core-prisma-helpers = { path = "../prisma-helpers" }
|
sd-core-prisma-helpers = { path = "../prisma-helpers" }
|
||||||
|
sd-core-sync = { path = "../sync" }
|
||||||
|
|
||||||
# Spacedrive Sub-crates
|
# Spacedrive Sub-crates
|
||||||
sd-prisma = { path = "../../../crates/prisma" }
|
sd-prisma = { path = "../../../crates/prisma" }
|
||||||
|
|
|
@ -173,6 +173,8 @@ pub enum FilePathError {
|
||||||
NonUtf8Path(#[from] NonUtf8PathError),
|
NonUtf8Path(#[from] NonUtf8PathError),
|
||||||
#[error("received an invalid filename and extension: <filename_and_extension='{0}'>")]
|
#[error("received an invalid filename and extension: <filename_and_extension='{0}'>")]
|
||||||
InvalidFilenameAndExtension(String),
|
InvalidFilenameAndExtension(String),
|
||||||
|
#[error(transparent)]
|
||||||
|
Sync(#[from] sd_core_sync::Error),
|
||||||
}
|
}
|
||||||
|
|
||||||
#[must_use]
|
#[must_use]
|
||||||
|
|
|
@ -51,6 +51,8 @@ pub enum Error {
|
||||||
FilePathError(#[from] FilePathError),
|
FilePathError(#[from] FilePathError),
|
||||||
#[error(transparent)]
|
#[error(transparent)]
|
||||||
SubPath(#[from] sub_path::Error),
|
SubPath(#[from] sub_path::Error),
|
||||||
|
#[error(transparent)]
|
||||||
|
Sync(#[from] sd_core_sync::Error),
|
||||||
}
|
}
|
||||||
|
|
||||||
impl From<Error> for rspc::Error {
|
impl From<Error> for rspc::Error {
|
||||||
|
|
|
@ -60,6 +60,8 @@ pub enum Error {
|
||||||
NonUtf8Path(#[from] NonUtf8PathError),
|
NonUtf8Path(#[from] NonUtf8PathError),
|
||||||
#[error(transparent)]
|
#[error(transparent)]
|
||||||
IsoFilePath(#[from] FilePathError),
|
IsoFilePath(#[from] FilePathError),
|
||||||
|
#[error(transparent)]
|
||||||
|
Sync(#[from] sd_core_sync::Error),
|
||||||
#[error("missing field on database: {0}")]
|
#[error("missing field on database: {0}")]
|
||||||
MissingField(#[from] MissingFieldError),
|
MissingField(#[from] MissingFieldError),
|
||||||
#[error("failed to deserialized stored tasks for job resume: {0}")]
|
#[error("failed to deserialized stored tasks for job resume: {0}")]
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
use crate::{indexer, Error};
|
use crate::{indexer, Error};
|
||||||
|
|
||||||
use sd_core_file_path_helper::IsolatedFilePathDataParts;
|
use sd_core_file_path_helper::{FilePathMetadata, IsolatedFilePathDataParts};
|
||||||
use sd_core_sync::Manager as SyncManager;
|
use sd_core_sync::Manager as SyncManager;
|
||||||
|
|
||||||
use sd_prisma::{
|
use sd_prisma::{
|
||||||
|
@ -9,7 +9,10 @@ use sd_prisma::{
|
||||||
};
|
};
|
||||||
use sd_sync::{sync_db_entry, OperationFactory};
|
use sd_sync::{sync_db_entry, OperationFactory};
|
||||||
use sd_task_system::{ExecStatus, Interrupter, IntoAnyTaskOutput, SerializableTask, Task, TaskId};
|
use sd_task_system::{ExecStatus, Interrupter, IntoAnyTaskOutput, SerializableTask, Task, TaskId};
|
||||||
use sd_utils::{db::inode_to_db, msgpack};
|
use sd_utils::{
|
||||||
|
db::{inode_to_db, size_in_bytes_to_db},
|
||||||
|
msgpack,
|
||||||
|
};
|
||||||
|
|
||||||
use std::{sync::Arc, time::Duration};
|
use std::{sync::Arc, time::Duration};
|
||||||
|
|
||||||
|
@ -92,7 +95,14 @@ impl Task<Error> for Saver {
|
||||||
pub_id,
|
pub_id,
|
||||||
maybe_object_id,
|
maybe_object_id,
|
||||||
iso_file_path,
|
iso_file_path,
|
||||||
metadata,
|
metadata:
|
||||||
|
FilePathMetadata {
|
||||||
|
inode,
|
||||||
|
size_in_bytes,
|
||||||
|
created_at,
|
||||||
|
modified_at,
|
||||||
|
hidden,
|
||||||
|
},
|
||||||
}| {
|
}| {
|
||||||
let IsolatedFilePathDataParts {
|
let IsolatedFilePathDataParts {
|
||||||
materialized_path,
|
materialized_path,
|
||||||
|
@ -118,19 +128,16 @@ impl Task<Error> for Saver {
|
||||||
),
|
),
|
||||||
location_id::set(Some(*location_id)),
|
location_id::set(Some(*location_id)),
|
||||||
),
|
),
|
||||||
sync_db_entry!(materialized_path.to_string(), materialized_path),
|
sync_db_entry!(materialized_path, materialized_path),
|
||||||
sync_db_entry!(name.to_string(), name),
|
sync_db_entry!(name, name),
|
||||||
sync_db_entry!(is_dir, is_dir),
|
sync_db_entry!(is_dir, is_dir),
|
||||||
sync_db_entry!(extension.to_string(), extension),
|
sync_db_entry!(extension, extension),
|
||||||
sync_db_entry!(
|
sync_db_entry!(size_in_bytes_to_db(size_in_bytes), size_in_bytes_bytes),
|
||||||
metadata.size_in_bytes.to_be_bytes().to_vec(),
|
sync_db_entry!(inode_to_db(inode), inode),
|
||||||
size_in_bytes_bytes
|
sync_db_entry!(created_at, date_created),
|
||||||
),
|
sync_db_entry!(modified_at, date_modified),
|
||||||
sync_db_entry!(inode_to_db(metadata.inode), inode),
|
sync_db_entry!(Utc::now(), date_indexed),
|
||||||
sync_db_entry!(metadata.created_at.into(), date_created),
|
sync_db_entry!(hidden, hidden),
|
||||||
sync_db_entry!(metadata.modified_at.into(), date_modified),
|
|
||||||
sync_db_entry!(Utc::now().into(), date_indexed),
|
|
||||||
sync_db_entry!(metadata.hidden, hidden),
|
|
||||||
]
|
]
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.unzip();
|
.unzip();
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
use crate::{indexer, Error};
|
use crate::{indexer, Error};
|
||||||
|
|
||||||
use sd_core_file_path_helper::IsolatedFilePathDataParts;
|
use sd_core_file_path_helper::{FilePathMetadata, IsolatedFilePathDataParts};
|
||||||
use sd_core_sync::Manager as SyncManager;
|
use sd_core_sync::Manager as SyncManager;
|
||||||
|
|
||||||
use sd_prisma::{
|
use sd_prisma::{
|
||||||
|
@ -11,7 +11,11 @@ use sd_sync::{sync_db_entry, OperationFactory};
|
||||||
use sd_task_system::{
|
use sd_task_system::{
|
||||||
check_interruption, ExecStatus, Interrupter, IntoAnyTaskOutput, SerializableTask, Task, TaskId,
|
check_interruption, ExecStatus, Interrupter, IntoAnyTaskOutput, SerializableTask, Task, TaskId,
|
||||||
};
|
};
|
||||||
use sd_utils::{chain_optional_iter, db::inode_to_db, msgpack};
|
use sd_utils::{
|
||||||
|
chain_optional_iter,
|
||||||
|
db::{inode_to_db, size_in_bytes_to_db},
|
||||||
|
msgpack,
|
||||||
|
};
|
||||||
|
|
||||||
use std::{collections::HashSet, sync::Arc, time::Duration};
|
use std::{collections::HashSet, sync::Arc, time::Duration};
|
||||||
|
|
||||||
|
@ -96,7 +100,14 @@ impl Task<Error> for Updater {
|
||||||
pub_id,
|
pub_id,
|
||||||
maybe_object_id,
|
maybe_object_id,
|
||||||
iso_file_path,
|
iso_file_path,
|
||||||
metadata,
|
metadata:
|
||||||
|
FilePathMetadata {
|
||||||
|
inode,
|
||||||
|
size_in_bytes,
|
||||||
|
created_at,
|
||||||
|
modified_at,
|
||||||
|
hidden,
|
||||||
|
},
|
||||||
}| {
|
}| {
|
||||||
let IsolatedFilePathDataParts { is_dir, .. } = &iso_file_path.to_parts();
|
let IsolatedFilePathDataParts { is_dir, .. } = &iso_file_path.to_parts();
|
||||||
|
|
||||||
|
@ -108,20 +119,11 @@ impl Task<Error> for Updater {
|
||||||
[
|
[
|
||||||
((cas_id::NAME, msgpack!(nil)), cas_id::set(None)),
|
((cas_id::NAME, msgpack!(nil)), cas_id::set(None)),
|
||||||
sync_db_entry!(*is_dir, is_dir),
|
sync_db_entry!(*is_dir, is_dir),
|
||||||
sync_db_entry!(
|
sync_db_entry!(size_in_bytes_to_db(size_in_bytes), size_in_bytes_bytes),
|
||||||
metadata.size_in_bytes.to_be_bytes().to_vec(),
|
sync_db_entry!(inode_to_db(inode), inode),
|
||||||
size_in_bytes_bytes
|
sync_db_entry!(created_at, date_created),
|
||||||
),
|
sync_db_entry!(modified_at, date_modified),
|
||||||
sync_db_entry!(inode_to_db(metadata.inode), inode),
|
sync_db_entry!(hidden, hidden),
|
||||||
{
|
|
||||||
let v = metadata.created_at.into();
|
|
||||||
sync_db_entry!(v, date_created)
|
|
||||||
},
|
|
||||||
{
|
|
||||||
let v = metadata.modified_at.into();
|
|
||||||
sync_db_entry!(v, date_modified)
|
|
||||||
},
|
|
||||||
sync_db_entry!(metadata.hidden, hidden),
|
|
||||||
],
|
],
|
||||||
[
|
[
|
||||||
// As this file was updated while Spacedrive was offline, we mark the object_id and cas_id as null
|
// As this file was updated while Spacedrive was offline, we mark the object_id and cas_id as null
|
||||||
|
|
|
@ -16,7 +16,6 @@ use std::path::Path;
|
||||||
|
|
||||||
use futures_concurrency::future::TryJoin;
|
use futures_concurrency::future::TryJoin;
|
||||||
use once_cell::sync::Lazy;
|
use once_cell::sync::Lazy;
|
||||||
use prisma_client_rust::QueryError;
|
|
||||||
|
|
||||||
use super::from_slice_option_to_option;
|
use super::from_slice_option_to_option;
|
||||||
|
|
||||||
|
@ -107,7 +106,7 @@ pub async fn save(
|
||||||
exif_datas: impl IntoIterator<Item = (ExifMetadata, object::id::Type, ObjectPubId)> + Send,
|
exif_datas: impl IntoIterator<Item = (ExifMetadata, object::id::Type, ObjectPubId)> + Send,
|
||||||
db: &PrismaClient,
|
db: &PrismaClient,
|
||||||
sync: &SyncManager,
|
sync: &SyncManager,
|
||||||
) -> Result<u64, QueryError> {
|
) -> Result<u64, sd_core_sync::Error> {
|
||||||
exif_datas
|
exif_datas
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.map(|(exif_data, object_id, object_pub_id)| async move {
|
.map(|(exif_data, object_id, object_pub_id)| async move {
|
||||||
|
|
|
@ -110,7 +110,7 @@ pub async fn extract(
|
||||||
pub async fn save(
|
pub async fn save(
|
||||||
ffmpeg_datas: impl IntoIterator<Item = (FFmpegMetadata, object::id::Type)> + Send,
|
ffmpeg_datas: impl IntoIterator<Item = (FFmpegMetadata, object::id::Type)> + Send,
|
||||||
db: &PrismaClient,
|
db: &PrismaClient,
|
||||||
) -> Result<u64, QueryError> {
|
) -> Result<u64, sd_core_sync::Error> {
|
||||||
ffmpeg_datas
|
ffmpeg_datas
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.map(
|
.map(
|
||||||
|
|
|
@ -55,6 +55,8 @@ pub enum Error {
|
||||||
FilePathError(#[from] FilePathError),
|
FilePathError(#[from] FilePathError),
|
||||||
#[error(transparent)]
|
#[error(transparent)]
|
||||||
SubPath(#[from] sub_path::Error),
|
SubPath(#[from] sub_path::Error),
|
||||||
|
#[error(transparent)]
|
||||||
|
Sync(#[from] sd_core_sync::Error),
|
||||||
}
|
}
|
||||||
|
|
||||||
impl From<Error> for rspc::Error {
|
impl From<Error> for rspc::Error {
|
||||||
|
|
|
@ -14,12 +14,21 @@ sd-utils = { path = "../../../crates/utils" }
|
||||||
sd-actors = { path = "../../../crates/actors" }
|
sd-actors = { path = "../../../crates/actors" }
|
||||||
|
|
||||||
# Workspace dependencies
|
# Workspace dependencies
|
||||||
prisma-client-rust = { workspace = true }
|
async-channel = { workspace = true }
|
||||||
|
futures = { workspace = true }
|
||||||
|
futures-concurrency = { workspace = true }
|
||||||
|
prisma-client-rust = { workspace = true, features = ["rspc"] }
|
||||||
rmpv = { workspace = true }
|
rmpv = { workspace = true }
|
||||||
rmp-serde = { workspace = true }
|
rmp-serde = { workspace = true }
|
||||||
|
rspc = { workspace = true }
|
||||||
serde = { workspace = true }
|
serde = { workspace = true }
|
||||||
serde_json = { workspace = true }
|
serde_json = { workspace = true }
|
||||||
tokio = { workspace = true }
|
tokio = { workspace = true }
|
||||||
|
thiserror = { workspace = true }
|
||||||
tracing = { workspace = true }
|
tracing = { workspace = true }
|
||||||
uhlc = { workspace = true }
|
uhlc = { workspace = true }
|
||||||
uuid = { workspace = true }
|
uuid = { workspace = true }
|
||||||
|
|
||||||
|
[dev-dependencies]
|
||||||
|
tracing-test = { workspace = true }
|
||||||
|
tracing-subscriber = { workspace = true, features = ["env-filter"] }
|
||||||
|
|
|
@ -1,16 +1,14 @@
|
||||||
use std::sync::Arc;
|
use async_channel as chan;
|
||||||
|
|
||||||
use tokio::sync::{mpsc, Mutex};
|
|
||||||
|
|
||||||
pub trait ActorTypes {
|
pub trait ActorTypes {
|
||||||
type Event;
|
type Event: Send;
|
||||||
type Request;
|
type Request: Send;
|
||||||
type Handler;
|
type Handler;
|
||||||
}
|
}
|
||||||
|
|
||||||
pub struct ActorIO<T: ActorTypes> {
|
pub struct ActorIO<T: ActorTypes> {
|
||||||
pub event_rx: Arc<Mutex<mpsc::Receiver<T::Event>>>,
|
pub event_rx: chan::Receiver<T::Event>,
|
||||||
pub req_tx: mpsc::Sender<T::Request>,
|
pub req_tx: chan::Sender<T::Request>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl<T: ActorTypes> Clone for ActorIO<T> {
|
impl<T: ActorTypes> Clone for ActorIO<T> {
|
||||||
|
@ -23,33 +21,19 @@ impl<T: ActorTypes> Clone for ActorIO<T> {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl<T: ActorTypes> ActorIO<T> {
|
impl<T: ActorTypes> ActorIO<T> {
|
||||||
pub async fn send(&self, value: T::Request) -> Result<(), mpsc::error::SendError<T::Request>> {
|
pub async fn send(&self, value: T::Request) -> Result<(), chan::SendError<T::Request>> {
|
||||||
self.req_tx.send(value).await
|
self.req_tx.send(value).await
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub struct HandlerIO<T: ActorTypes> {
|
pub struct HandlerIO<T: ActorTypes> {
|
||||||
pub event_tx: mpsc::Sender<T::Event>,
|
pub event_tx: chan::Sender<T::Event>,
|
||||||
pub req_rx: mpsc::Receiver<T::Request>,
|
pub req_rx: chan::Receiver<T::Request>,
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn create_actor_io<T: ActorTypes>() -> (ActorIO<T>, HandlerIO<T>) {
|
pub fn create_actor_io<T: ActorTypes>() -> (ActorIO<T>, HandlerIO<T>) {
|
||||||
let (req_tx, req_rx) = mpsc::channel(20);
|
let (req_tx, req_rx) = chan::bounded(32);
|
||||||
let (event_tx, event_rx) = mpsc::channel(20);
|
let (event_tx, event_rx) = chan::bounded(32);
|
||||||
|
|
||||||
let event_rx = Arc::new(Mutex::new(event_rx));
|
|
||||||
|
|
||||||
(ActorIO { event_rx, req_tx }, HandlerIO { event_tx, req_rx })
|
(ActorIO { event_rx, req_tx }, HandlerIO { event_tx, req_rx })
|
||||||
}
|
}
|
||||||
|
|
||||||
#[macro_export]
|
|
||||||
macro_rules! wait {
|
|
||||||
($rx:expr, $pattern:pat $(=> $expr:expr)?) => {
|
|
||||||
loop {
|
|
||||||
match $rx.recv().await {
|
|
||||||
Some($pattern) => break $($expr)?,
|
|
||||||
_ => continue
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
|
@ -1,447 +1,74 @@
|
||||||
use std::future::Future;
|
|
||||||
|
|
||||||
use sd_prisma::{
|
use sd_prisma::{
|
||||||
prisma::{
|
prisma::{
|
||||||
crdt_operation, exif_data, file_path, label, label_on_object, location, object, tag,
|
crdt_operation, exif_data, file_path, instance, label, label_on_object, location, object,
|
||||||
tag_on_object, PrismaClient, SortOrder,
|
tag, tag_on_object, PrismaClient, SortOrder,
|
||||||
},
|
},
|
||||||
prisma_sync,
|
prisma_sync,
|
||||||
};
|
};
|
||||||
use sd_sync::{option_sync_entry, OperationFactory};
|
use sd_sync::{option_sync_entry, sync_entry, OperationFactory};
|
||||||
use sd_utils::{chain_optional_iter, msgpack};
|
use sd_utils::chain_optional_iter;
|
||||||
|
|
||||||
use crate::crdt_op_unchecked_db;
|
use std::future::Future;
|
||||||
|
|
||||||
/// Takes all the syncable data in the database and generates CRDTOperations for it.
|
use tokio::time::Instant;
|
||||||
|
use tracing::{debug, instrument};
|
||||||
|
|
||||||
|
use super::{crdt_op_unchecked_db, Error};
|
||||||
|
|
||||||
|
/// Takes all the syncable data in the database and generates [`CRDTOperations`] for it.
|
||||||
/// This is a requirement before the library can sync.
|
/// This is a requirement before the library can sync.
|
||||||
pub async fn backfill_operations(db: &PrismaClient, sync: &crate::Manager, instance_id: i32) {
|
pub async fn backfill_operations(
|
||||||
let lock = sync.timestamp_lock.acquire().await;
|
db: &PrismaClient,
|
||||||
|
sync: &crate::Manager,
|
||||||
|
instance_id: instance::id::Type,
|
||||||
|
) -> Result<(), Error> {
|
||||||
|
let lock = sync.timestamp_lock.lock().await;
|
||||||
|
|
||||||
db._transaction()
|
let res = db
|
||||||
.with_timeout(9999999999)
|
._transaction()
|
||||||
|
.with_timeout(9_999_999_999)
|
||||||
.run(|db| async move {
|
.run(|db| async move {
|
||||||
println!("backfill started");
|
debug!("backfill started");
|
||||||
|
let start = Instant::now();
|
||||||
db.crdt_operation()
|
db.crdt_operation()
|
||||||
.delete_many(vec![crdt_operation::instance_id::equals(instance_id)])
|
.delete_many(vec![crdt_operation::instance_id::equals(instance_id)])
|
||||||
.exec()
|
.exec()
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
paginate(
|
paginate_tags(&db, sync, instance_id).await?;
|
||||||
|cursor| {
|
paginate_locations(&db, sync, instance_id).await?;
|
||||||
db.tag()
|
paginate_objects(&db, sync, instance_id).await?;
|
||||||
.find_many(vec![tag::id::gt(cursor)])
|
paginate_exif_datas(&db, sync, instance_id).await?;
|
||||||
.order_by(tag::id::order(SortOrder::Asc))
|
paginate_file_paths(&db, sync, instance_id).await?;
|
||||||
.exec()
|
paginate_tags_on_objects(&db, sync, instance_id).await?;
|
||||||
},
|
paginate_labels(&db, sync, instance_id).await?;
|
||||||
|tag| tag.id,
|
paginate_labels_on_objects(&db, sync, instance_id).await?;
|
||||||
|tags| {
|
|
||||||
db.crdt_operation()
|
|
||||||
.create_many(
|
|
||||||
tags.into_iter()
|
|
||||||
.flat_map(|t| {
|
|
||||||
sync.shared_create(
|
|
||||||
prisma_sync::tag::SyncId { pub_id: t.pub_id },
|
|
||||||
chain_optional_iter(
|
|
||||||
[],
|
|
||||||
[
|
|
||||||
t.name.map(|v| (tag::name::NAME, msgpack!(v))),
|
|
||||||
t.color.map(|v| (tag::color::NAME, msgpack!(v))),
|
|
||||||
t.date_created.map(|v| {
|
|
||||||
(tag::date_created::NAME, msgpack!(v))
|
|
||||||
}),
|
|
||||||
t.date_modified.map(|v| {
|
|
||||||
(tag::date_modified::NAME, msgpack!(v))
|
|
||||||
}),
|
|
||||||
],
|
|
||||||
),
|
|
||||||
)
|
|
||||||
})
|
|
||||||
.map(|o| crdt_op_unchecked_db(&o, instance_id))
|
|
||||||
.collect(),
|
|
||||||
)
|
|
||||||
.exec()
|
|
||||||
},
|
|
||||||
)
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
paginate(
|
debug!(elapsed = ?start.elapsed(), "backfill ended");
|
||||||
|cursor| {
|
|
||||||
db.location()
|
|
||||||
.find_many(vec![location::id::gt(cursor)])
|
|
||||||
.order_by(location::id::order(SortOrder::Asc))
|
|
||||||
.take(1000)
|
|
||||||
.include(location::include!({
|
|
||||||
instance: select {
|
|
||||||
id
|
|
||||||
pub_id
|
|
||||||
}
|
|
||||||
}))
|
|
||||||
.exec()
|
|
||||||
},
|
|
||||||
|location| location.id,
|
|
||||||
|locations| {
|
|
||||||
db.crdt_operation()
|
|
||||||
.create_many(
|
|
||||||
locations
|
|
||||||
.into_iter()
|
|
||||||
.flat_map(|l| {
|
|
||||||
use location::*;
|
|
||||||
|
|
||||||
sync.shared_create(
|
Ok(())
|
||||||
prisma_sync::location::SyncId { pub_id: l.pub_id },
|
|
||||||
chain_optional_iter(
|
|
||||||
[],
|
|
||||||
[
|
|
||||||
option_sync_entry!(l.name, name),
|
|
||||||
option_sync_entry!(l.path, path),
|
|
||||||
option_sync_entry!(
|
|
||||||
l.total_capacity,
|
|
||||||
total_capacity
|
|
||||||
),
|
|
||||||
option_sync_entry!(
|
|
||||||
l.available_capacity,
|
|
||||||
available_capacity
|
|
||||||
),
|
|
||||||
option_sync_entry!(l.size_in_bytes, size_in_bytes),
|
|
||||||
option_sync_entry!(l.is_archived, is_archived),
|
|
||||||
option_sync_entry!(
|
|
||||||
l.generate_preview_media,
|
|
||||||
generate_preview_media
|
|
||||||
),
|
|
||||||
option_sync_entry!(
|
|
||||||
l.sync_preview_media,
|
|
||||||
sync_preview_media
|
|
||||||
),
|
|
||||||
option_sync_entry!(l.hidden, hidden),
|
|
||||||
option_sync_entry!(l.date_created, date_created),
|
|
||||||
option_sync_entry!(
|
|
||||||
l.instance.map(|i| {
|
|
||||||
prisma_sync::instance::SyncId {
|
|
||||||
pub_id: i.pub_id,
|
|
||||||
}
|
|
||||||
}),
|
|
||||||
instance
|
|
||||||
),
|
|
||||||
],
|
|
||||||
),
|
|
||||||
)
|
|
||||||
})
|
|
||||||
.map(|o| crdt_op_unchecked_db(&o, instance_id))
|
|
||||||
.collect(),
|
|
||||||
)
|
|
||||||
.exec()
|
|
||||||
},
|
|
||||||
)
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
paginate(
|
|
||||||
|cursor| {
|
|
||||||
db.object()
|
|
||||||
.find_many(vec![object::id::gt(cursor)])
|
|
||||||
.order_by(object::id::order(SortOrder::Asc))
|
|
||||||
.take(1000)
|
|
||||||
.exec()
|
|
||||||
},
|
|
||||||
|object| object.id,
|
|
||||||
|objects| {
|
|
||||||
db.crdt_operation()
|
|
||||||
.create_many(
|
|
||||||
objects
|
|
||||||
.into_iter()
|
|
||||||
.flat_map(|o| {
|
|
||||||
use object::*;
|
|
||||||
|
|
||||||
sync.shared_create(
|
|
||||||
prisma_sync::object::SyncId { pub_id: o.pub_id },
|
|
||||||
chain_optional_iter(
|
|
||||||
[],
|
|
||||||
[
|
|
||||||
option_sync_entry!(o.kind, kind),
|
|
||||||
option_sync_entry!(o.hidden, hidden),
|
|
||||||
option_sync_entry!(o.favorite, favorite),
|
|
||||||
option_sync_entry!(o.important, important),
|
|
||||||
option_sync_entry!(o.note, note),
|
|
||||||
option_sync_entry!(o.date_created, date_created),
|
|
||||||
option_sync_entry!(o.date_accessed, date_accessed),
|
|
||||||
],
|
|
||||||
),
|
|
||||||
)
|
|
||||||
})
|
|
||||||
.map(|o| crdt_op_unchecked_db(&o, instance_id))
|
|
||||||
.collect(),
|
|
||||||
)
|
|
||||||
.exec()
|
|
||||||
},
|
|
||||||
)
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
paginate(
|
|
||||||
|cursor| {
|
|
||||||
db.exif_data()
|
|
||||||
.find_many(vec![exif_data::id::gt(cursor)])
|
|
||||||
.order_by(exif_data::id::order(SortOrder::Asc))
|
|
||||||
.take(1000)
|
|
||||||
.include(exif_data::include!({
|
|
||||||
object: select { pub_id }
|
|
||||||
}))
|
|
||||||
.exec()
|
|
||||||
},
|
|
||||||
|o| o.id,
|
|
||||||
|media_datas| {
|
|
||||||
db.crdt_operation()
|
|
||||||
.create_many(
|
|
||||||
media_datas
|
|
||||||
.into_iter()
|
|
||||||
.flat_map(|md| {
|
|
||||||
use exif_data::*;
|
|
||||||
|
|
||||||
sync.shared_create(
|
|
||||||
prisma_sync::exif_data::SyncId {
|
|
||||||
object: prisma_sync::object::SyncId {
|
|
||||||
pub_id: md.object.pub_id,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
chain_optional_iter(
|
|
||||||
[],
|
|
||||||
[
|
|
||||||
option_sync_entry!(md.resolution, resolution),
|
|
||||||
option_sync_entry!(md.media_date, media_date),
|
|
||||||
option_sync_entry!(
|
|
||||||
md.media_location,
|
|
||||||
media_location
|
|
||||||
),
|
|
||||||
option_sync_entry!(md.camera_data, camera_data),
|
|
||||||
option_sync_entry!(md.artist, artist),
|
|
||||||
option_sync_entry!(md.description, description),
|
|
||||||
option_sync_entry!(md.copyright, copyright),
|
|
||||||
option_sync_entry!(md.exif_version, exif_version),
|
|
||||||
option_sync_entry!(md.epoch_time, epoch_time),
|
|
||||||
],
|
|
||||||
),
|
|
||||||
)
|
|
||||||
})
|
|
||||||
.map(|o| crdt_op_unchecked_db(&o, instance_id))
|
|
||||||
.collect(),
|
|
||||||
)
|
|
||||||
.exec()
|
|
||||||
},
|
|
||||||
)
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
paginate(
|
|
||||||
|cursor| {
|
|
||||||
db.file_path()
|
|
||||||
.find_many(vec![file_path::id::gt(cursor)])
|
|
||||||
.order_by(file_path::id::order(SortOrder::Asc))
|
|
||||||
.include(file_path::include!({
|
|
||||||
location: select { pub_id }
|
|
||||||
object: select { pub_id }
|
|
||||||
}))
|
|
||||||
.exec()
|
|
||||||
},
|
|
||||||
|o| o.id,
|
|
||||||
|file_paths| {
|
|
||||||
db.crdt_operation()
|
|
||||||
.create_many(
|
|
||||||
file_paths
|
|
||||||
.into_iter()
|
|
||||||
.flat_map(|fp| {
|
|
||||||
use file_path::*;
|
|
||||||
|
|
||||||
sync.shared_create(
|
|
||||||
prisma_sync::file_path::SyncId { pub_id: fp.pub_id },
|
|
||||||
chain_optional_iter(
|
|
||||||
[],
|
|
||||||
[
|
|
||||||
option_sync_entry!(fp.is_dir, is_dir),
|
|
||||||
option_sync_entry!(fp.cas_id, cas_id),
|
|
||||||
option_sync_entry!(
|
|
||||||
fp.integrity_checksum,
|
|
||||||
integrity_checksum
|
|
||||||
),
|
|
||||||
option_sync_entry!(
|
|
||||||
fp.location.map(|l| {
|
|
||||||
prisma_sync::location::SyncId {
|
|
||||||
pub_id: l.pub_id,
|
|
||||||
}
|
|
||||||
}),
|
|
||||||
location
|
|
||||||
),
|
|
||||||
option_sync_entry!(
|
|
||||||
fp.object.map(|o| {
|
|
||||||
prisma_sync::object::SyncId {
|
|
||||||
pub_id: o.pub_id,
|
|
||||||
}
|
|
||||||
}),
|
|
||||||
object
|
|
||||||
),
|
|
||||||
option_sync_entry!(
|
|
||||||
fp.materialized_path,
|
|
||||||
materialized_path
|
|
||||||
),
|
|
||||||
option_sync_entry!(fp.name, name),
|
|
||||||
option_sync_entry!(fp.extension, extension),
|
|
||||||
option_sync_entry!(fp.hidden, hidden),
|
|
||||||
option_sync_entry!(
|
|
||||||
fp.size_in_bytes_bytes,
|
|
||||||
size_in_bytes_bytes
|
|
||||||
),
|
|
||||||
option_sync_entry!(fp.inode, inode),
|
|
||||||
option_sync_entry!(fp.date_created, date_created),
|
|
||||||
option_sync_entry!(fp.date_modified, date_modified),
|
|
||||||
option_sync_entry!(fp.date_indexed, date_indexed),
|
|
||||||
],
|
|
||||||
),
|
|
||||||
)
|
|
||||||
})
|
|
||||||
.map(|o| crdt_op_unchecked_db(&o, instance_id))
|
|
||||||
.collect(),
|
|
||||||
)
|
|
||||||
.exec()
|
|
||||||
},
|
|
||||||
)
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
paginate_relation(
|
|
||||||
|group_id, item_id| {
|
|
||||||
db.tag_on_object()
|
|
||||||
.find_many(vec![
|
|
||||||
tag_on_object::tag_id::gt(group_id),
|
|
||||||
tag_on_object::object_id::gt(item_id),
|
|
||||||
])
|
|
||||||
.order_by(tag_on_object::tag_id::order(SortOrder::Asc))
|
|
||||||
.order_by(tag_on_object::object_id::order(SortOrder::Asc))
|
|
||||||
.include(tag_on_object::include!({
|
|
||||||
tag: select { pub_id }
|
|
||||||
object: select { pub_id }
|
|
||||||
}))
|
|
||||||
.exec()
|
|
||||||
},
|
|
||||||
|t_o| (t_o.tag_id, t_o.object_id),
|
|
||||||
|tag_on_objects| {
|
|
||||||
db.crdt_operation()
|
|
||||||
.create_many(
|
|
||||||
tag_on_objects
|
|
||||||
.into_iter()
|
|
||||||
.flat_map(|t_o| {
|
|
||||||
sync.relation_create(
|
|
||||||
prisma_sync::tag_on_object::SyncId {
|
|
||||||
tag: prisma_sync::tag::SyncId {
|
|
||||||
pub_id: t_o.tag.pub_id,
|
|
||||||
},
|
|
||||||
object: prisma_sync::object::SyncId {
|
|
||||||
pub_id: t_o.object.pub_id,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
chain_optional_iter(
|
|
||||||
[],
|
|
||||||
[option_sync_entry!(
|
|
||||||
t_o.date_created,
|
|
||||||
tag_on_object::date_created
|
|
||||||
)],
|
|
||||||
),
|
|
||||||
)
|
|
||||||
})
|
|
||||||
.map(|o| crdt_op_unchecked_db(&o, instance_id))
|
|
||||||
.collect(),
|
|
||||||
)
|
|
||||||
.exec()
|
|
||||||
},
|
|
||||||
)
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
paginate(
|
|
||||||
|cursor| {
|
|
||||||
db.label()
|
|
||||||
.find_many(vec![label::id::gt(cursor)])
|
|
||||||
.order_by(label::id::order(SortOrder::Asc))
|
|
||||||
.exec()
|
|
||||||
},
|
|
||||||
|label| label.id,
|
|
||||||
|labels| {
|
|
||||||
db.crdt_operation()
|
|
||||||
.create_many(
|
|
||||||
labels
|
|
||||||
.into_iter()
|
|
||||||
.flat_map(|l| {
|
|
||||||
sync.shared_create(
|
|
||||||
prisma_sync::label::SyncId { name: l.name },
|
|
||||||
[
|
|
||||||
(label::date_created::NAME, msgpack!(l.date_created)),
|
|
||||||
(label::date_modified::NAME, msgpack!(l.date_modified)),
|
|
||||||
],
|
|
||||||
)
|
|
||||||
})
|
|
||||||
.map(|o| crdt_op_unchecked_db(&o, instance_id))
|
|
||||||
.collect(),
|
|
||||||
)
|
|
||||||
.exec()
|
|
||||||
},
|
|
||||||
)
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
let res = paginate_relation(
|
|
||||||
|group_id, item_id| {
|
|
||||||
db.label_on_object()
|
|
||||||
.find_many(vec![
|
|
||||||
label_on_object::label_id::gt(group_id),
|
|
||||||
label_on_object::object_id::gt(item_id),
|
|
||||||
])
|
|
||||||
.order_by(label_on_object::label_id::order(SortOrder::Asc))
|
|
||||||
.order_by(label_on_object::object_id::order(SortOrder::Asc))
|
|
||||||
.include(label_on_object::include!({
|
|
||||||
object: select { pub_id }
|
|
||||||
label: select { name }
|
|
||||||
}))
|
|
||||||
.exec()
|
|
||||||
},
|
|
||||||
|l_o| (l_o.label_id, l_o.object_id),
|
|
||||||
|label_on_objects| {
|
|
||||||
db.crdt_operation()
|
|
||||||
.create_many(
|
|
||||||
label_on_objects
|
|
||||||
.into_iter()
|
|
||||||
.flat_map(|l_o| {
|
|
||||||
sync.relation_create(
|
|
||||||
prisma_sync::label_on_object::SyncId {
|
|
||||||
label: prisma_sync::label::SyncId {
|
|
||||||
name: l_o.label.name,
|
|
||||||
},
|
|
||||||
object: prisma_sync::object::SyncId {
|
|
||||||
pub_id: l_o.object.pub_id,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
[],
|
|
||||||
)
|
|
||||||
})
|
|
||||||
.map(|o| crdt_op_unchecked_db(&o, instance_id))
|
|
||||||
.collect(),
|
|
||||||
)
|
|
||||||
.exec()
|
|
||||||
},
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
|
|
||||||
println!("backfill ended");
|
|
||||||
|
|
||||||
res
|
|
||||||
})
|
})
|
||||||
.await
|
.await;
|
||||||
.unwrap();
|
|
||||||
|
|
||||||
drop(lock);
|
drop(lock);
|
||||||
|
|
||||||
|
res
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn paginate<
|
async fn paginate<T, E1, E2, E3, GetterFut, OperationsFut>(
|
||||||
T,
|
getter: impl Fn(i32) -> GetterFut + Send,
|
||||||
E: std::fmt::Debug,
|
id: impl Fn(&T) -> i32 + Send,
|
||||||
TGetter: Future<Output = Result<Vec<T>, E>>,
|
operations: impl Fn(Vec<T>) -> Result<OperationsFut, E3> + Send,
|
||||||
TOperations: Future<Output = Result<i64, E>>,
|
) -> Result<(), Error>
|
||||||
>(
|
where
|
||||||
getter: impl Fn(i32) -> TGetter,
|
T: Send,
|
||||||
id: impl Fn(&T) -> i32,
|
E1: Send,
|
||||||
operations: impl Fn(Vec<T>) -> TOperations,
|
E2: Send,
|
||||||
) -> Result<(), E> {
|
E3: Send,
|
||||||
|
Error: From<E1> + From<E2> + From<E3> + Send,
|
||||||
|
GetterFut: Future<Output = Result<Vec<T>, E1>> + Send,
|
||||||
|
OperationsFut: Future<Output = Result<i64, E2>> + Send,
|
||||||
|
{
|
||||||
let mut next_cursor = Some(-1);
|
let mut next_cursor = Some(-1);
|
||||||
loop {
|
loop {
|
||||||
let Some(cursor) = next_cursor else {
|
let Some(cursor) = next_cursor else {
|
||||||
|
@ -450,22 +77,26 @@ async fn paginate<
|
||||||
|
|
||||||
let items = getter(cursor).await?;
|
let items = getter(cursor).await?;
|
||||||
next_cursor = items.last().map(&id);
|
next_cursor = items.last().map(&id);
|
||||||
operations(items).await?;
|
operations(items)?.await?;
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn paginate_relation<
|
async fn paginate_relation<T, E1, E2, E3, GetterFut, OperationsFut>(
|
||||||
T,
|
getter: impl Fn(i32, i32) -> GetterFut + Send,
|
||||||
E: std::fmt::Debug,
|
id: impl Fn(&T) -> (i32, i32) + Send,
|
||||||
TGetter: Future<Output = Result<Vec<T>, E>>,
|
operations: impl Fn(Vec<T>) -> Result<OperationsFut, E3> + Send,
|
||||||
TOperations: Future<Output = Result<i64, E>>,
|
) -> Result<(), Error>
|
||||||
>(
|
where
|
||||||
getter: impl Fn(i32, i32) -> TGetter,
|
T: Send,
|
||||||
id: impl Fn(&T) -> (i32, i32),
|
E1: Send,
|
||||||
operations: impl Fn(Vec<T>) -> TOperations,
|
E2: Send,
|
||||||
) -> Result<(), E> {
|
E3: Send,
|
||||||
|
Error: From<E1> + From<E2> + From<E3> + Send,
|
||||||
|
GetterFut: Future<Output = Result<Vec<T>, E1>> + Send,
|
||||||
|
OperationsFut: Future<Output = Result<i64, E2>> + Send,
|
||||||
|
{
|
||||||
let mut next_cursor = Some((-1, -1));
|
let mut next_cursor = Some((-1, -1));
|
||||||
loop {
|
loop {
|
||||||
let Some(cursor) = next_cursor else {
|
let Some(cursor) = next_cursor else {
|
||||||
|
@ -474,8 +105,416 @@ async fn paginate_relation<
|
||||||
|
|
||||||
let items = getter(cursor.0, cursor.1).await?;
|
let items = getter(cursor.0, cursor.1).await?;
|
||||||
next_cursor = items.last().map(&id);
|
next_cursor = items.last().map(&id);
|
||||||
operations(items).await?;
|
operations(items)?.await?;
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[instrument(skip(db, sync), err)]
|
||||||
|
async fn paginate_tags(
|
||||||
|
db: &PrismaClient,
|
||||||
|
sync: &crate::Manager,
|
||||||
|
instance_id: instance::id::Type,
|
||||||
|
) -> Result<(), Error> {
|
||||||
|
use tag::{color, date_created, date_modified, id, name};
|
||||||
|
|
||||||
|
paginate(
|
||||||
|
|cursor| {
|
||||||
|
db.tag()
|
||||||
|
.find_many(vec![id::gt(cursor)])
|
||||||
|
.order_by(id::order(SortOrder::Asc))
|
||||||
|
.exec()
|
||||||
|
},
|
||||||
|
|tag| tag.id,
|
||||||
|
|tags| {
|
||||||
|
tags.into_iter()
|
||||||
|
.flat_map(|t| {
|
||||||
|
sync.shared_create(
|
||||||
|
prisma_sync::tag::SyncId { pub_id: t.pub_id },
|
||||||
|
chain_optional_iter(
|
||||||
|
[],
|
||||||
|
[
|
||||||
|
option_sync_entry!(t.name, name),
|
||||||
|
option_sync_entry!(t.color, color),
|
||||||
|
option_sync_entry!(t.date_created, date_created),
|
||||||
|
option_sync_entry!(t.date_modified, date_modified),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.map(|o| crdt_op_unchecked_db(&o, instance_id))
|
||||||
|
.collect::<Result<Vec<_>, _>>()
|
||||||
|
.map(|creates| db.crdt_operation().create_many(creates).exec())
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
#[instrument(skip(db, sync), err)]
|
||||||
|
async fn paginate_locations(
|
||||||
|
db: &PrismaClient,
|
||||||
|
sync: &crate::Manager,
|
||||||
|
instance_id: instance::id::Type,
|
||||||
|
) -> Result<(), Error> {
|
||||||
|
use location::{
|
||||||
|
available_capacity, date_created, generate_preview_media, hidden, id, include, instance,
|
||||||
|
is_archived, name, path, size_in_bytes, sync_preview_media, total_capacity,
|
||||||
|
};
|
||||||
|
|
||||||
|
paginate(
|
||||||
|
|cursor| {
|
||||||
|
db.location()
|
||||||
|
.find_many(vec![id::gt(cursor)])
|
||||||
|
.order_by(id::order(SortOrder::Asc))
|
||||||
|
.take(1000)
|
||||||
|
.include(include!({
|
||||||
|
instance: select {
|
||||||
|
id
|
||||||
|
pub_id
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
.exec()
|
||||||
|
},
|
||||||
|
|location| location.id,
|
||||||
|
|locations| {
|
||||||
|
locations
|
||||||
|
.into_iter()
|
||||||
|
.flat_map(|l| {
|
||||||
|
sync.shared_create(
|
||||||
|
prisma_sync::location::SyncId { pub_id: l.pub_id },
|
||||||
|
chain_optional_iter(
|
||||||
|
[],
|
||||||
|
[
|
||||||
|
option_sync_entry!(l.name, name),
|
||||||
|
option_sync_entry!(l.path, path),
|
||||||
|
option_sync_entry!(l.total_capacity, total_capacity),
|
||||||
|
option_sync_entry!(l.available_capacity, available_capacity),
|
||||||
|
option_sync_entry!(l.size_in_bytes, size_in_bytes),
|
||||||
|
option_sync_entry!(l.is_archived, is_archived),
|
||||||
|
option_sync_entry!(
|
||||||
|
l.generate_preview_media,
|
||||||
|
generate_preview_media
|
||||||
|
),
|
||||||
|
option_sync_entry!(l.sync_preview_media, sync_preview_media),
|
||||||
|
option_sync_entry!(l.hidden, hidden),
|
||||||
|
option_sync_entry!(l.date_created, date_created),
|
||||||
|
option_sync_entry!(
|
||||||
|
l.instance.map(|i| {
|
||||||
|
prisma_sync::instance::SyncId { pub_id: i.pub_id }
|
||||||
|
}),
|
||||||
|
instance
|
||||||
|
),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.map(|o| crdt_op_unchecked_db(&o, instance_id))
|
||||||
|
.collect::<Result<Vec<_>, _>>()
|
||||||
|
.map(|creates| db.crdt_operation().create_many(creates).exec())
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
#[instrument(skip(db, sync), err)]
|
||||||
|
async fn paginate_objects(
|
||||||
|
db: &PrismaClient,
|
||||||
|
sync: &crate::Manager,
|
||||||
|
instance_id: instance::id::Type,
|
||||||
|
) -> Result<(), Error> {
|
||||||
|
use object::{date_accessed, date_created, favorite, hidden, id, important, kind, note};
|
||||||
|
|
||||||
|
paginate(
|
||||||
|
|cursor| {
|
||||||
|
db.object()
|
||||||
|
.find_many(vec![id::gt(cursor)])
|
||||||
|
.order_by(id::order(SortOrder::Asc))
|
||||||
|
.take(1000)
|
||||||
|
.exec()
|
||||||
|
},
|
||||||
|
|object| object.id,
|
||||||
|
|objects| {
|
||||||
|
objects
|
||||||
|
.into_iter()
|
||||||
|
.flat_map(|o| {
|
||||||
|
sync.shared_create(
|
||||||
|
prisma_sync::object::SyncId { pub_id: o.pub_id },
|
||||||
|
chain_optional_iter(
|
||||||
|
[],
|
||||||
|
[
|
||||||
|
option_sync_entry!(o.kind, kind),
|
||||||
|
option_sync_entry!(o.hidden, hidden),
|
||||||
|
option_sync_entry!(o.favorite, favorite),
|
||||||
|
option_sync_entry!(o.important, important),
|
||||||
|
option_sync_entry!(o.note, note),
|
||||||
|
option_sync_entry!(o.date_created, date_created),
|
||||||
|
option_sync_entry!(o.date_accessed, date_accessed),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.map(|o| crdt_op_unchecked_db(&o, instance_id))
|
||||||
|
.collect::<Result<Vec<_>, _>>()
|
||||||
|
.map(|creates| db.crdt_operation().create_many(creates).exec())
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
#[instrument(skip(db, sync), err)]
|
||||||
|
async fn paginate_exif_datas(
|
||||||
|
db: &PrismaClient,
|
||||||
|
sync: &crate::Manager,
|
||||||
|
instance_id: instance::id::Type,
|
||||||
|
) -> Result<(), Error> {
|
||||||
|
use exif_data::{
|
||||||
|
artist, camera_data, copyright, description, epoch_time, exif_version, id, include,
|
||||||
|
media_date, media_location, resolution,
|
||||||
|
};
|
||||||
|
|
||||||
|
paginate(
|
||||||
|
|cursor| {
|
||||||
|
db.exif_data()
|
||||||
|
.find_many(vec![id::gt(cursor)])
|
||||||
|
.order_by(id::order(SortOrder::Asc))
|
||||||
|
.take(1000)
|
||||||
|
.include(include!({
|
||||||
|
object: select { pub_id }
|
||||||
|
}))
|
||||||
|
.exec()
|
||||||
|
},
|
||||||
|
|ed| ed.id,
|
||||||
|
|exif_datas| {
|
||||||
|
exif_datas
|
||||||
|
.into_iter()
|
||||||
|
.flat_map(|ed| {
|
||||||
|
sync.shared_create(
|
||||||
|
prisma_sync::exif_data::SyncId {
|
||||||
|
object: prisma_sync::object::SyncId {
|
||||||
|
pub_id: ed.object.pub_id,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
chain_optional_iter(
|
||||||
|
[],
|
||||||
|
[
|
||||||
|
option_sync_entry!(ed.resolution, resolution),
|
||||||
|
option_sync_entry!(ed.media_date, media_date),
|
||||||
|
option_sync_entry!(ed.media_location, media_location),
|
||||||
|
option_sync_entry!(ed.camera_data, camera_data),
|
||||||
|
option_sync_entry!(ed.artist, artist),
|
||||||
|
option_sync_entry!(ed.description, description),
|
||||||
|
option_sync_entry!(ed.copyright, copyright),
|
||||||
|
option_sync_entry!(ed.exif_version, exif_version),
|
||||||
|
option_sync_entry!(ed.epoch_time, epoch_time),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.map(|o| crdt_op_unchecked_db(&o, instance_id))
|
||||||
|
.collect::<Result<Vec<_>, _>>()
|
||||||
|
.map(|creates| db.crdt_operation().create_many(creates).exec())
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
#[instrument(skip(db, sync), err)]
|
||||||
|
async fn paginate_file_paths(
|
||||||
|
db: &PrismaClient,
|
||||||
|
sync: &crate::Manager,
|
||||||
|
instance_id: instance::id::Type,
|
||||||
|
) -> Result<(), Error> {
|
||||||
|
use file_path::{
|
||||||
|
cas_id, date_created, date_indexed, date_modified, extension, hidden, id, include, inode,
|
||||||
|
integrity_checksum, is_dir, location, materialized_path, name, object, size_in_bytes_bytes,
|
||||||
|
};
|
||||||
|
|
||||||
|
paginate(
|
||||||
|
|cursor| {
|
||||||
|
db.file_path()
|
||||||
|
.find_many(vec![id::gt(cursor)])
|
||||||
|
.order_by(id::order(SortOrder::Asc))
|
||||||
|
.include(include!({
|
||||||
|
location: select { pub_id }
|
||||||
|
object: select { pub_id }
|
||||||
|
}))
|
||||||
|
.exec()
|
||||||
|
},
|
||||||
|
|o| o.id,
|
||||||
|
|file_paths| {
|
||||||
|
file_paths
|
||||||
|
.into_iter()
|
||||||
|
.flat_map(|fp| {
|
||||||
|
sync.shared_create(
|
||||||
|
prisma_sync::file_path::SyncId { pub_id: fp.pub_id },
|
||||||
|
chain_optional_iter(
|
||||||
|
[],
|
||||||
|
[
|
||||||
|
option_sync_entry!(fp.is_dir, is_dir),
|
||||||
|
option_sync_entry!(fp.cas_id, cas_id),
|
||||||
|
option_sync_entry!(fp.integrity_checksum, integrity_checksum),
|
||||||
|
option_sync_entry!(
|
||||||
|
fp.location.map(|l| {
|
||||||
|
prisma_sync::location::SyncId { pub_id: l.pub_id }
|
||||||
|
}),
|
||||||
|
location
|
||||||
|
),
|
||||||
|
option_sync_entry!(
|
||||||
|
fp.object.map(|o| {
|
||||||
|
prisma_sync::object::SyncId { pub_id: o.pub_id }
|
||||||
|
}),
|
||||||
|
object
|
||||||
|
),
|
||||||
|
option_sync_entry!(fp.materialized_path, materialized_path),
|
||||||
|
option_sync_entry!(fp.name, name),
|
||||||
|
option_sync_entry!(fp.extension, extension),
|
||||||
|
option_sync_entry!(fp.hidden, hidden),
|
||||||
|
option_sync_entry!(fp.size_in_bytes_bytes, size_in_bytes_bytes),
|
||||||
|
option_sync_entry!(fp.inode, inode),
|
||||||
|
option_sync_entry!(fp.date_created, date_created),
|
||||||
|
option_sync_entry!(fp.date_modified, date_modified),
|
||||||
|
option_sync_entry!(fp.date_indexed, date_indexed),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.map(|o| crdt_op_unchecked_db(&o, instance_id))
|
||||||
|
.collect::<Result<Vec<_>, _>>()
|
||||||
|
.map(|creates| db.crdt_operation().create_many(creates).exec())
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
#[instrument(skip(db, sync), err)]
|
||||||
|
async fn paginate_tags_on_objects(
|
||||||
|
db: &PrismaClient,
|
||||||
|
sync: &crate::Manager,
|
||||||
|
instance_id: instance::id::Type,
|
||||||
|
) -> Result<(), Error> {
|
||||||
|
use tag_on_object::{date_created, include, object_id, tag_id};
|
||||||
|
|
||||||
|
paginate_relation(
|
||||||
|
|group_id, item_id| {
|
||||||
|
db.tag_on_object()
|
||||||
|
.find_many(vec![tag_id::gt(group_id), object_id::gt(item_id)])
|
||||||
|
.order_by(tag_id::order(SortOrder::Asc))
|
||||||
|
.order_by(object_id::order(SortOrder::Asc))
|
||||||
|
.include(include!({
|
||||||
|
tag: select { pub_id }
|
||||||
|
object: select { pub_id }
|
||||||
|
}))
|
||||||
|
.exec()
|
||||||
|
},
|
||||||
|
|t_o| (t_o.tag_id, t_o.object_id),
|
||||||
|
|tag_on_objects| {
|
||||||
|
tag_on_objects
|
||||||
|
.into_iter()
|
||||||
|
.flat_map(|t_o| {
|
||||||
|
sync.relation_create(
|
||||||
|
prisma_sync::tag_on_object::SyncId {
|
||||||
|
tag: prisma_sync::tag::SyncId {
|
||||||
|
pub_id: t_o.tag.pub_id,
|
||||||
|
},
|
||||||
|
object: prisma_sync::object::SyncId {
|
||||||
|
pub_id: t_o.object.pub_id,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
chain_optional_iter(
|
||||||
|
[],
|
||||||
|
[option_sync_entry!(t_o.date_created, date_created)],
|
||||||
|
),
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.map(|o| crdt_op_unchecked_db(&o, instance_id))
|
||||||
|
.collect::<Result<Vec<_>, _>>()
|
||||||
|
.map(|creates| db.crdt_operation().create_many(creates).exec())
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
#[instrument(skip(db, sync), err)]
|
||||||
|
async fn paginate_labels(
|
||||||
|
db: &PrismaClient,
|
||||||
|
sync: &crate::Manager,
|
||||||
|
instance_id: instance::id::Type,
|
||||||
|
) -> Result<(), Error> {
|
||||||
|
use label::{date_created, date_modified, id};
|
||||||
|
|
||||||
|
paginate(
|
||||||
|
|cursor| {
|
||||||
|
db.label()
|
||||||
|
.find_many(vec![id::gt(cursor)])
|
||||||
|
.order_by(id::order(SortOrder::Asc))
|
||||||
|
.exec()
|
||||||
|
},
|
||||||
|
|label| label.id,
|
||||||
|
|labels| {
|
||||||
|
labels
|
||||||
|
.into_iter()
|
||||||
|
.flat_map(|l| {
|
||||||
|
sync.shared_create(
|
||||||
|
prisma_sync::label::SyncId { name: l.name },
|
||||||
|
chain_optional_iter(
|
||||||
|
[],
|
||||||
|
[
|
||||||
|
option_sync_entry!(l.date_created, date_created),
|
||||||
|
option_sync_entry!(l.date_modified, date_modified),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.map(|o| crdt_op_unchecked_db(&o, instance_id))
|
||||||
|
.collect::<Result<Vec<_>, _>>()
|
||||||
|
.map(|creates| db.crdt_operation().create_many(creates).exec())
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
#[instrument(skip(db, sync), err)]
|
||||||
|
async fn paginate_labels_on_objects(
|
||||||
|
db: &PrismaClient,
|
||||||
|
sync: &crate::Manager,
|
||||||
|
instance_id: instance::id::Type,
|
||||||
|
) -> Result<(), Error> {
|
||||||
|
use label_on_object::{date_created, include, label_id, object_id};
|
||||||
|
|
||||||
|
paginate_relation(
|
||||||
|
|group_id, item_id| {
|
||||||
|
db.label_on_object()
|
||||||
|
.find_many(vec![label_id::gt(group_id), object_id::gt(item_id)])
|
||||||
|
.order_by(label_id::order(SortOrder::Asc))
|
||||||
|
.order_by(object_id::order(SortOrder::Asc))
|
||||||
|
.include(include!({
|
||||||
|
object: select { pub_id }
|
||||||
|
label: select { name }
|
||||||
|
}))
|
||||||
|
.exec()
|
||||||
|
},
|
||||||
|
|l_o| (l_o.label_id, l_o.object_id),
|
||||||
|
|label_on_objects| {
|
||||||
|
label_on_objects
|
||||||
|
.into_iter()
|
||||||
|
.flat_map(|l_o| {
|
||||||
|
sync.relation_create(
|
||||||
|
prisma_sync::label_on_object::SyncId {
|
||||||
|
label: prisma_sync::label::SyncId {
|
||||||
|
name: l_o.label.name,
|
||||||
|
},
|
||||||
|
object: prisma_sync::object::SyncId {
|
||||||
|
pub_id: l_o.object.pub_id,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
[sync_entry!(l_o.date_created, date_created)],
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.map(|o| crdt_op_unchecked_db(&o, instance_id))
|
||||||
|
.collect::<Result<Vec<_>, _>>()
|
||||||
|
.map(|creates| db.crdt_operation().create_many(creates).exec())
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
|
@ -1,77 +1,102 @@
|
||||||
use rmp_serde::to_vec;
|
|
||||||
use sd_prisma::prisma::{cloud_crdt_operation, crdt_operation, instance, PrismaClient};
|
use sd_prisma::prisma::{cloud_crdt_operation, crdt_operation, instance, PrismaClient};
|
||||||
use sd_sync::CRDTOperation;
|
use sd_sync::CRDTOperation;
|
||||||
|
use sd_utils::from_bytes_to_uuid;
|
||||||
|
|
||||||
|
use tracing::instrument;
|
||||||
use uhlc::NTP64;
|
use uhlc::NTP64;
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
crdt_operation::include!(crdt_include {
|
use super::Error;
|
||||||
|
|
||||||
|
crdt_operation::include!(crdt_with_instance {
|
||||||
instance: select { pub_id }
|
instance: select { pub_id }
|
||||||
});
|
});
|
||||||
|
|
||||||
cloud_crdt_operation::include!(cloud_crdt_include {
|
cloud_crdt_operation::include!(cloud_crdt_with_instance {
|
||||||
instance: select { pub_id }
|
instance: select { pub_id }
|
||||||
});
|
});
|
||||||
|
|
||||||
impl crdt_include::Data {
|
impl crdt_with_instance::Data {
|
||||||
pub fn timestamp(&self) -> NTP64 {
|
#[allow(clippy::cast_sign_loss)] // SAFETY: we had to store using i64 due to SQLite limitations
|
||||||
|
pub const fn timestamp(&self) -> NTP64 {
|
||||||
NTP64(self.timestamp as u64)
|
NTP64(self.timestamp as u64)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn instance(&self) -> Uuid {
|
pub fn instance(&self) -> Uuid {
|
||||||
Uuid::from_slice(&self.instance.pub_id).unwrap()
|
from_bytes_to_uuid(&self.instance.pub_id)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn into_operation(self) -> CRDTOperation {
|
pub fn into_operation(self) -> Result<CRDTOperation, Error> {
|
||||||
CRDTOperation {
|
Ok(CRDTOperation {
|
||||||
instance: self.instance(),
|
instance: self.instance(),
|
||||||
timestamp: self.timestamp(),
|
timestamp: self.timestamp(),
|
||||||
record_id: rmp_serde::from_slice(&self.record_id).unwrap(),
|
record_id: rmp_serde::from_slice(&self.record_id)?,
|
||||||
model: self.model as u16,
|
|
||||||
data: rmp_serde::from_slice(&self.data).unwrap(),
|
model: {
|
||||||
}
|
#[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)]
|
||||||
|
// SAFETY: we will not have more than 2^16 models and we had to store using signed
|
||||||
|
// integers due to SQLite limitations
|
||||||
|
{
|
||||||
|
self.model as u16
|
||||||
|
}
|
||||||
|
},
|
||||||
|
data: rmp_serde::from_slice(&self.data)?,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl cloud_crdt_include::Data {
|
impl cloud_crdt_with_instance::Data {
|
||||||
pub fn timestamp(&self) -> NTP64 {
|
#[allow(clippy::cast_sign_loss)] // SAFETY: we had to store using i64 due to SQLite limitations
|
||||||
|
pub const fn timestamp(&self) -> NTP64 {
|
||||||
NTP64(self.timestamp as u64)
|
NTP64(self.timestamp as u64)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn instance(&self) -> Uuid {
|
pub fn instance(&self) -> Uuid {
|
||||||
Uuid::from_slice(&self.instance.pub_id).unwrap()
|
from_bytes_to_uuid(&self.instance.pub_id)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn into_operation(self) -> (i32, CRDTOperation) {
|
#[instrument(skip(self), err)]
|
||||||
(
|
pub fn into_operation(self) -> Result<(i32, CRDTOperation), Error> {
|
||||||
|
Ok((
|
||||||
self.id,
|
self.id,
|
||||||
CRDTOperation {
|
CRDTOperation {
|
||||||
instance: self.instance(),
|
instance: self.instance(),
|
||||||
timestamp: self.timestamp(),
|
timestamp: self.timestamp(),
|
||||||
record_id: rmp_serde::from_slice(&self.record_id).unwrap(),
|
record_id: rmp_serde::from_slice(&self.record_id)?,
|
||||||
model: self.model as u16,
|
model: {
|
||||||
data: serde_json::from_slice(&self.data).unwrap(),
|
#[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)]
|
||||||
|
// SAFETY: we will not have more than 2^16 models and we had to store using signed
|
||||||
|
// integers due to SQLite limitations
|
||||||
|
{
|
||||||
|
self.model as u16
|
||||||
|
}
|
||||||
|
},
|
||||||
|
data: rmp_serde::from_slice(&self.data)?,
|
||||||
},
|
},
|
||||||
)
|
))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn write_crdt_op_to_db(
|
#[instrument(skip(op, db), err)]
|
||||||
op: &CRDTOperation,
|
pub async fn write_crdt_op_to_db(op: &CRDTOperation, db: &PrismaClient) -> Result<(), Error> {
|
||||||
db: &PrismaClient,
|
|
||||||
) -> Result<(), prisma_client_rust::QueryError> {
|
|
||||||
crdt_op_db(op).to_query(db).exec().await?;
|
|
||||||
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
fn crdt_op_db(op: &CRDTOperation) -> crdt_operation::Create {
|
|
||||||
crdt_operation::Create {
|
crdt_operation::Create {
|
||||||
timestamp: op.timestamp.0 as i64,
|
timestamp: {
|
||||||
|
#[allow(clippy::cast_possible_wrap)]
|
||||||
|
// SAFETY: we have to store using i64 due to SQLite limitations
|
||||||
|
{
|
||||||
|
op.timestamp.0 as i64
|
||||||
|
}
|
||||||
|
},
|
||||||
instance: instance::pub_id::equals(op.instance.as_bytes().to_vec()),
|
instance: instance::pub_id::equals(op.instance.as_bytes().to_vec()),
|
||||||
kind: op.kind().to_string(),
|
kind: op.kind().to_string(),
|
||||||
data: to_vec(&op.data).unwrap(),
|
data: rmp_serde::to_vec(&op.data)?,
|
||||||
model: op.model as i32,
|
model: i32::from(op.model),
|
||||||
record_id: rmp_serde::to_vec(&op.record_id).unwrap(),
|
record_id: rmp_serde::to_vec(&op.record_id)?,
|
||||||
_params: vec![],
|
_params: vec![],
|
||||||
}
|
}
|
||||||
|
.to_query(db)
|
||||||
|
.select(crdt_operation::select!({ id })) // To don't fetch the whole object for nothing
|
||||||
|
.exec()
|
||||||
|
.await
|
||||||
|
.map_or_else(|e| Err(e.into()), |_| Ok(()))
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,27 +1,38 @@
|
||||||
use std::{
|
|
||||||
collections::BTreeMap,
|
|
||||||
num::NonZeroU128,
|
|
||||||
ops::Deref,
|
|
||||||
sync::{atomic::Ordering, Arc},
|
|
||||||
};
|
|
||||||
|
|
||||||
use sd_prisma::{
|
use sd_prisma::{
|
||||||
prisma::{crdt_operation, SortOrder},
|
prisma::{crdt_operation, PrismaClient, SortOrder},
|
||||||
prisma_sync::ModelSyncData,
|
prisma_sync::ModelSyncData,
|
||||||
};
|
};
|
||||||
use sd_sync::{
|
use sd_sync::{
|
||||||
CRDTOperation, CRDTOperationData, CompressedCRDTOperation, CompressedCRDTOperations,
|
CRDTOperation, CRDTOperationData, CompressedCRDTOperation, CompressedCRDTOperations,
|
||||||
OperationKind,
|
OperationKind,
|
||||||
};
|
};
|
||||||
use tokio::sync::{mpsc, oneshot, Mutex};
|
|
||||||
use tracing::debug;
|
use std::{
|
||||||
|
collections::BTreeMap,
|
||||||
|
future::IntoFuture,
|
||||||
|
num::NonZeroU128,
|
||||||
|
ops::Deref,
|
||||||
|
pin::pin,
|
||||||
|
sync::{atomic::Ordering, Arc},
|
||||||
|
time::SystemTime,
|
||||||
|
};
|
||||||
|
|
||||||
|
use async_channel as chan;
|
||||||
|
use futures::{stream, FutureExt, StreamExt};
|
||||||
|
use futures_concurrency::{
|
||||||
|
future::{Race, TryJoin},
|
||||||
|
stream::Merge,
|
||||||
|
};
|
||||||
|
use prisma_client_rust::chrono::{DateTime, Utc};
|
||||||
|
use tokio::sync::oneshot;
|
||||||
|
use tracing::{debug, error, instrument, trace, warn};
|
||||||
use uhlc::{Timestamp, NTP64};
|
use uhlc::{Timestamp, NTP64};
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
use crate::{
|
use super::{
|
||||||
actor::{create_actor_io, ActorIO, ActorTypes},
|
actor::{create_actor_io, ActorIO, ActorTypes, HandlerIO},
|
||||||
db_operation::write_crdt_op_to_db,
|
db_operation::write_crdt_op_to_db,
|
||||||
wait, SharedState,
|
Error, SharedState,
|
||||||
};
|
};
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
|
@ -32,7 +43,6 @@ pub enum Request {
|
||||||
timestamps: Vec<(Uuid, NTP64)>,
|
timestamps: Vec<(Uuid, NTP64)>,
|
||||||
tx: oneshot::Sender<()>,
|
tx: oneshot::Sender<()>,
|
||||||
},
|
},
|
||||||
// Ingested,
|
|
||||||
FinishedIngesting,
|
FinishedIngesting,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -53,7 +63,7 @@ pub enum State {
|
||||||
|
|
||||||
/// The single entrypoint for sync operation ingestion.
|
/// The single entrypoint for sync operation ingestion.
|
||||||
/// Requests sync operations in a given timestamp range,
|
/// Requests sync operations in a given timestamp range,
|
||||||
/// and attempts to write them to the syn coperations table along with
|
/// and attempts to write them to the sync operations table along with
|
||||||
/// the actual cell that the operation points to.
|
/// the actual cell that the operation points to.
|
||||||
///
|
///
|
||||||
/// If this actor stops running, no sync operations will
|
/// If this actor stops running, no sync operations will
|
||||||
|
@ -66,133 +76,203 @@ pub struct Actor {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Actor {
|
impl Actor {
|
||||||
async fn tick(mut self) -> Option<Self> {
|
#[instrument(skip(self), fields(old_state = ?self.state))]
|
||||||
let state = match self.state.take()? {
|
async fn tick(&mut self) {
|
||||||
State::WaitingForNotification => {
|
let state = match self
|
||||||
self.shared.active.store(false, Ordering::Relaxed);
|
.state
|
||||||
self.shared.active_notify.notify_waiters();
|
.take()
|
||||||
|
.expect("ingest actor in inconsistent state")
|
||||||
wait!(self.io.event_rx.lock().await, Event::Notification);
|
{
|
||||||
|
State::WaitingForNotification => self.waiting_for_notification_state_transition().await,
|
||||||
self.shared.active.store(true, Ordering::Relaxed);
|
State::RetrievingMessages => self.retrieving_messages_state_transition().await,
|
||||||
self.shared.active_notify.notify_waiters();
|
State::Ingesting(event) => self.ingesting_state_transition(event).await,
|
||||||
|
|
||||||
State::RetrievingMessages
|
|
||||||
}
|
|
||||||
State::RetrievingMessages => {
|
|
||||||
let (tx, mut rx) = oneshot::channel::<()>();
|
|
||||||
|
|
||||||
let timestamps = self
|
|
||||||
.timestamps
|
|
||||||
.read()
|
|
||||||
.await
|
|
||||||
.iter()
|
|
||||||
.map(|(&k, &v)| (k, v))
|
|
||||||
.collect();
|
|
||||||
|
|
||||||
self.io
|
|
||||||
.send(Request::Messages { timestamps, tx })
|
|
||||||
.await
|
|
||||||
.ok();
|
|
||||||
|
|
||||||
let mut event_rx = self.io.event_rx.lock().await;
|
|
||||||
|
|
||||||
loop {
|
|
||||||
tokio::select! {
|
|
||||||
biased;
|
|
||||||
res = event_rx.recv() => {
|
|
||||||
if let Some(Event::Messages(event)) = res { break State::Ingesting(event) }
|
|
||||||
}
|
|
||||||
res = &mut rx => {
|
|
||||||
if res.is_err() {
|
|
||||||
debug!("messages request ignored");
|
|
||||||
break State::WaitingForNotification
|
|
||||||
}
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
State::Ingesting(event) => {
|
|
||||||
debug!(
|
|
||||||
messages_count = event.messages.len(),
|
|
||||||
first_message = event.messages.first().unwrap().3.timestamp.as_u64(),
|
|
||||||
last_message = event.messages.last().unwrap().3.timestamp.as_u64(),
|
|
||||||
"Ingesting operations;",
|
|
||||||
);
|
|
||||||
|
|
||||||
for (instance, data) in event.messages.0 {
|
|
||||||
for (model, data) in data {
|
|
||||||
for (record, ops) in data {
|
|
||||||
self.receive_crdt_operations(instance, model, record, ops)
|
|
||||||
.await
|
|
||||||
.expect("sync ingest failed");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if let Some(tx) = event.wait_tx {
|
|
||||||
tx.send(()).ok();
|
|
||||||
}
|
|
||||||
|
|
||||||
match event.has_more {
|
|
||||||
true => State::RetrievingMessages,
|
|
||||||
false => {
|
|
||||||
self.io.send(Request::FinishedIngesting).await.ok();
|
|
||||||
|
|
||||||
State::WaitingForNotification
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
};
|
||||||
|
|
||||||
Some(Self {
|
trace!(?state, "Actor state transitioned;");
|
||||||
state: Some(state),
|
|
||||||
..self
|
self.state = Some(state);
|
||||||
})
|
}
|
||||||
|
|
||||||
|
async fn waiting_for_notification_state_transition(&self) -> State {
|
||||||
|
self.shared.active.store(false, Ordering::Relaxed);
|
||||||
|
self.shared.active_notify.notify_waiters();
|
||||||
|
|
||||||
|
loop {
|
||||||
|
match self
|
||||||
|
.io
|
||||||
|
.event_rx
|
||||||
|
.recv()
|
||||||
|
.await
|
||||||
|
.expect("sync actor receiver unexpectedly closed")
|
||||||
|
{
|
||||||
|
Event::Notification => {
|
||||||
|
trace!("Received notification");
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
Event::Messages(event) => {
|
||||||
|
trace!(
|
||||||
|
?event,
|
||||||
|
"Ignored event message as we're waiting for a `Event::Notification`"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
self.shared.active.store(true, Ordering::Relaxed);
|
||||||
|
self.shared.active_notify.notify_waiters();
|
||||||
|
|
||||||
|
State::RetrievingMessages
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn retrieving_messages_state_transition(&self) -> State {
|
||||||
|
enum StreamMessage {
|
||||||
|
NewEvent(Event),
|
||||||
|
AckedRequest(Result<(), oneshot::error::RecvError>),
|
||||||
|
}
|
||||||
|
|
||||||
|
let (tx, rx) = oneshot::channel::<()>();
|
||||||
|
|
||||||
|
let timestamps = self
|
||||||
|
.timestamps
|
||||||
|
.read()
|
||||||
|
.await
|
||||||
|
.iter()
|
||||||
|
.map(|(&uid, ×tamp)| (uid, timestamp))
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
if self
|
||||||
|
.io
|
||||||
|
.send(Request::Messages { timestamps, tx })
|
||||||
|
.await
|
||||||
|
.is_err()
|
||||||
|
{
|
||||||
|
warn!("Failed to send messages request");
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut msg_stream = pin!((
|
||||||
|
self.io.event_rx.clone().map(StreamMessage::NewEvent),
|
||||||
|
stream::once(rx.map(StreamMessage::AckedRequest)),
|
||||||
|
)
|
||||||
|
.merge());
|
||||||
|
|
||||||
|
loop {
|
||||||
|
if let Some(msg) = msg_stream.next().await {
|
||||||
|
match msg {
|
||||||
|
StreamMessage::NewEvent(event) => {
|
||||||
|
if let Event::Messages(messages) = event {
|
||||||
|
trace!(?messages, "Received messages;");
|
||||||
|
break State::Ingesting(messages);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
StreamMessage::AckedRequest(res) => {
|
||||||
|
if res.is_err() {
|
||||||
|
debug!("messages request ignored");
|
||||||
|
break State::WaitingForNotification;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
break State::WaitingForNotification;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn ingesting_state_transition(&mut self, event: MessagesEvent) -> State {
|
||||||
|
debug!(
|
||||||
|
messages_count = event.messages.len(),
|
||||||
|
first_message = ?DateTime::<Utc>::from(
|
||||||
|
event.messages
|
||||||
|
.first()
|
||||||
|
.map_or(SystemTime::UNIX_EPOCH, |m| m.3.timestamp.to_system_time())
|
||||||
|
),
|
||||||
|
last_message = ?DateTime::<Utc>::from(
|
||||||
|
event.messages
|
||||||
|
.last()
|
||||||
|
.map_or(SystemTime::UNIX_EPOCH, |m| m.3.timestamp.to_system_time())
|
||||||
|
),
|
||||||
|
"Ingesting operations;",
|
||||||
|
);
|
||||||
|
|
||||||
|
for (instance, data) in event.messages.0 {
|
||||||
|
for (model, data) in data {
|
||||||
|
for (record, ops) in data {
|
||||||
|
if let Err(e) = self
|
||||||
|
.process_crdt_operations(instance, model, record, ops)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
error!(?e, "Failed to ingest CRDT operations;");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Some(tx) = event.wait_tx {
|
||||||
|
if tx.send(()).is_err() {
|
||||||
|
warn!("Failed to send wait_tx signal");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if event.has_more {
|
||||||
|
State::RetrievingMessages
|
||||||
|
} else {
|
||||||
|
{
|
||||||
|
if self.io.send(Request::FinishedIngesting).await.is_err() {
|
||||||
|
error!("Failed to send finished ingesting request");
|
||||||
|
}
|
||||||
|
|
||||||
|
State::WaitingForNotification
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn declare(shared: Arc<SharedState>) -> Handler {
|
pub async fn declare(shared: Arc<SharedState>) -> Handler {
|
||||||
let (actor_io, handler_io) = create_actor_io::<Self>();
|
let (io, HandlerIO { event_tx, req_rx }) = create_actor_io::<Self>();
|
||||||
|
|
||||||
shared
|
shared
|
||||||
.actors
|
.actors
|
||||||
.declare(
|
.declare(
|
||||||
"Sync Ingest",
|
"Sync Ingest",
|
||||||
{
|
{
|
||||||
let shared = shared.clone();
|
let shared = Arc::clone(&shared);
|
||||||
move || async move {
|
move |stop| async move {
|
||||||
|
enum Race {
|
||||||
|
Ticked,
|
||||||
|
Stopped,
|
||||||
|
}
|
||||||
|
|
||||||
let mut this = Self {
|
let mut this = Self {
|
||||||
state: Some(Default::default()),
|
state: Some(State::default()),
|
||||||
io: actor_io,
|
io,
|
||||||
shared,
|
shared,
|
||||||
};
|
};
|
||||||
|
|
||||||
loop {
|
while matches!(
|
||||||
this = match this.tick().await {
|
(
|
||||||
Some(this) => this,
|
this.tick().map(|()| Race::Ticked),
|
||||||
None => break,
|
stop.into_future().map(|()| Race::Stopped),
|
||||||
};
|
)
|
||||||
}
|
.race()
|
||||||
|
.await,
|
||||||
|
Race::Ticked
|
||||||
|
) { /* Everything is Awesome! */ }
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
true,
|
true,
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
|
|
||||||
Handler {
|
Handler { event_tx, req_rx }
|
||||||
event_tx: handler_io.event_tx,
|
|
||||||
req_rx: Arc::new(Mutex::new(handler_io.req_rx)),
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// where the magic happens
|
// where the magic happens
|
||||||
async fn receive_crdt_operations(
|
#[instrument(skip(self, ops), fields(operations_count = %ops.len()), err)]
|
||||||
|
async fn process_crdt_operations(
|
||||||
&mut self,
|
&mut self,
|
||||||
instance: Uuid,
|
instance: Uuid,
|
||||||
model: u16,
|
model: u16,
|
||||||
record_id: rmpv::Value,
|
record_id: rmpv::Value,
|
||||||
mut ops: Vec<CompressedCRDTOperation>,
|
mut ops: Vec<CompressedCRDTOperation>,
|
||||||
) -> prisma_client_rust::Result<()> {
|
) -> Result<(), Error> {
|
||||||
let db = &self.db;
|
let db = &self.db;
|
||||||
|
|
||||||
ops.sort_by_key(|op| op.timestamp);
|
ops.sort_by_key(|op| op.timestamp);
|
||||||
|
@ -209,52 +289,31 @@ impl Actor {
|
||||||
.expect("timestamp has too much drift!");
|
.expect("timestamp has too much drift!");
|
||||||
|
|
||||||
// read the timestamp for the operation's instance, or insert one if it doesn't exist
|
// read the timestamp for the operation's instance, or insert one if it doesn't exist
|
||||||
let timestamp = self.timestamps.read().await.get(&instance).cloned();
|
let timestamp = self.timestamps.read().await.get(&instance).copied();
|
||||||
|
|
||||||
// Delete - ignores all other messages
|
// Delete - ignores all other messages
|
||||||
if let Some(delete_op) = ops
|
if let Some(delete_op) = ops
|
||||||
.iter()
|
.iter()
|
||||||
.rev()
|
.rev()
|
||||||
.find(|op| matches!(op.data, sd_sync::CRDTOperationData::Delete))
|
.find(|op| matches!(op.data, CRDTOperationData::Delete))
|
||||||
{
|
{
|
||||||
// deletes are the be all and end all, no need to check anything
|
trace!("Deleting operation");
|
||||||
|
handle_crdt_deletion(db, instance, model, record_id, delete_op).await?;
|
||||||
let op = CRDTOperation {
|
|
||||||
instance,
|
|
||||||
model,
|
|
||||||
record_id,
|
|
||||||
timestamp: delete_op.timestamp,
|
|
||||||
data: CRDTOperationData::Delete,
|
|
||||||
};
|
|
||||||
|
|
||||||
self.db
|
|
||||||
._transaction()
|
|
||||||
.with_timeout(30 * 1000)
|
|
||||||
.run(|db| async move {
|
|
||||||
ModelSyncData::from_op(op.clone())
|
|
||||||
.unwrap()
|
|
||||||
.exec(&db)
|
|
||||||
.await?;
|
|
||||||
write_crdt_op_to_db(&op, &db).await?;
|
|
||||||
|
|
||||||
Ok(())
|
|
||||||
})
|
|
||||||
.await?;
|
|
||||||
}
|
}
|
||||||
// Create + > 0 Update - overwrites the create's data with the updates
|
// Create + > 0 Update - overwrites the create's data with the updates
|
||||||
else if let Some(timestamp) = ops.iter().rev().find_map(|op| {
|
else if let Some(timestamp) = ops
|
||||||
if let sd_sync::CRDTOperationData::Create(_) = &op.data {
|
.iter()
|
||||||
return Some(op.timestamp);
|
.rev()
|
||||||
}
|
.find_map(|op| matches!(&op.data, CRDTOperationData::Create(_)).then_some(op.timestamp))
|
||||||
|
{
|
||||||
|
trace!("Create + Updates operations");
|
||||||
|
|
||||||
None
|
|
||||||
}) {
|
|
||||||
// conflict resolution
|
// conflict resolution
|
||||||
let delete = db
|
let delete = db
|
||||||
.crdt_operation()
|
.crdt_operation()
|
||||||
.find_first(vec![
|
.find_first(vec![
|
||||||
crdt_operation::model::equals(model as i32),
|
crdt_operation::model::equals(i32::from(model)),
|
||||||
crdt_operation::record_id::equals(rmp_serde::to_vec(&record_id).unwrap()),
|
crdt_operation::record_id::equals(rmp_serde::to_vec(&record_id)?),
|
||||||
crdt_operation::kind::equals(OperationKind::Delete.to_string()),
|
crdt_operation::kind::equals(OperationKind::Delete.to_string()),
|
||||||
])
|
])
|
||||||
.order_by(crdt_operation::timestamp::order(SortOrder::Desc))
|
.order_by(crdt_operation::timestamp::order(SortOrder::Desc))
|
||||||
|
@ -262,73 +321,16 @@ impl Actor {
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
if delete.is_some() {
|
if delete.is_some() {
|
||||||
|
debug!("Found a previous delete operation with the same SyncId, will ignore these operations");
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut data = BTreeMap::new();
|
handle_crdt_create_and_updates(db, instance, model, record_id, ops, timestamp).await?;
|
||||||
|
|
||||||
let mut applied_ops = vec![];
|
|
||||||
|
|
||||||
// search for all Updates until a Create is found
|
|
||||||
for op in ops.iter().rev() {
|
|
||||||
match &op.data {
|
|
||||||
CRDTOperationData::Delete => unreachable!("Delete can't exist here!"),
|
|
||||||
CRDTOperationData::Create(create_data) => {
|
|
||||||
for (k, v) in create_data {
|
|
||||||
data.entry(k).or_insert(v);
|
|
||||||
}
|
|
||||||
|
|
||||||
applied_ops.push(op);
|
|
||||||
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
CRDTOperationData::Update { field, value } => {
|
|
||||||
applied_ops.push(op);
|
|
||||||
data.insert(field, value);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
self.db
|
|
||||||
._transaction()
|
|
||||||
.with_timeout(30 * 1000)
|
|
||||||
.run(|db| async move {
|
|
||||||
// fake a create with a bunch of data rather than individual insert
|
|
||||||
ModelSyncData::from_op(CRDTOperation {
|
|
||||||
instance,
|
|
||||||
model,
|
|
||||||
record_id: record_id.clone(),
|
|
||||||
timestamp,
|
|
||||||
data: CRDTOperationData::Create(
|
|
||||||
data.into_iter()
|
|
||||||
.map(|(k, v)| (k.clone(), v.clone()))
|
|
||||||
.collect(),
|
|
||||||
),
|
|
||||||
})
|
|
||||||
.unwrap()
|
|
||||||
.exec(&db)
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
for op in applied_ops {
|
|
||||||
write_crdt_op_to_db(
|
|
||||||
&CRDTOperation {
|
|
||||||
instance,
|
|
||||||
model,
|
|
||||||
record_id: record_id.clone(),
|
|
||||||
timestamp: op.timestamp,
|
|
||||||
data: op.data.clone(),
|
|
||||||
},
|
|
||||||
&db,
|
|
||||||
)
|
|
||||||
.await?;
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(())
|
|
||||||
})
|
|
||||||
.await?;
|
|
||||||
}
|
}
|
||||||
// > 0 Update - batches updates with a fake Create op
|
// > 0 Update - batches updates with a fake Create op
|
||||||
else {
|
else {
|
||||||
|
trace!("Updates operation");
|
||||||
|
|
||||||
let mut data = BTreeMap::new();
|
let mut data = BTreeMap::new();
|
||||||
|
|
||||||
for op in ops.into_iter().rev() {
|
for op in ops.into_iter().rev() {
|
||||||
|
@ -344,84 +346,43 @@ impl Actor {
|
||||||
._batch((
|
._batch((
|
||||||
db.crdt_operation()
|
db.crdt_operation()
|
||||||
.find_first(vec![
|
.find_first(vec![
|
||||||
crdt_operation::model::equals(model as i32),
|
crdt_operation::model::equals(i32::from(model)),
|
||||||
crdt_operation::record_id::equals(
|
crdt_operation::record_id::equals(rmp_serde::to_vec(&record_id)?),
|
||||||
rmp_serde::to_vec(&record_id).unwrap(),
|
|
||||||
),
|
|
||||||
crdt_operation::kind::equals(OperationKind::Create.to_string()),
|
crdt_operation::kind::equals(OperationKind::Create.to_string()),
|
||||||
])
|
])
|
||||||
.order_by(crdt_operation::timestamp::order(SortOrder::Desc)),
|
.order_by(crdt_operation::timestamp::order(SortOrder::Desc)),
|
||||||
data.iter()
|
data.iter()
|
||||||
.map(|(k, (_, timestamp))| {
|
.map(|(k, (_, timestamp))| {
|
||||||
db.crdt_operation()
|
Ok(db
|
||||||
|
.crdt_operation()
|
||||||
.find_first(vec![
|
.find_first(vec![
|
||||||
crdt_operation::timestamp::gt(timestamp.as_u64() as i64),
|
crdt_operation::timestamp::gt({
|
||||||
crdt_operation::model::equals(model as i32),
|
#[allow(clippy::cast_possible_wrap)]
|
||||||
crdt_operation::record_id::equals(
|
// SAFETY: we had to store using i64 due to SQLite limitations
|
||||||
rmp_serde::to_vec(&record_id).unwrap(),
|
{
|
||||||
),
|
timestamp.as_u64() as i64
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
crdt_operation::model::equals(i32::from(model)),
|
||||||
|
crdt_operation::record_id::equals(rmp_serde::to_vec(
|
||||||
|
&record_id,
|
||||||
|
)?),
|
||||||
crdt_operation::kind::equals(
|
crdt_operation::kind::equals(
|
||||||
OperationKind::Update(k).to_string(),
|
OperationKind::Update(k).to_string(),
|
||||||
),
|
),
|
||||||
])
|
])
|
||||||
.order_by(crdt_operation::timestamp::order(SortOrder::Desc))
|
.order_by(crdt_operation::timestamp::order(SortOrder::Desc)))
|
||||||
})
|
})
|
||||||
.collect::<Vec<_>>(),
|
.collect::<Result<Vec<_>, Error>>()?,
|
||||||
))
|
))
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
if create.is_none() {
|
if create.is_none() {
|
||||||
|
warn!("Failed to find a previous create operation with the same SyncId");
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
|
|
||||||
// does the same thing as processing ops one-by-one and returning early if a newer op was found
|
handle_crdt_updates(db, instance, model, record_id, data, updates).await?;
|
||||||
for (update, key) in updates
|
|
||||||
.into_iter()
|
|
||||||
.zip(data.keys().cloned().collect::<Vec<_>>())
|
|
||||||
{
|
|
||||||
if update.is_some() {
|
|
||||||
data.remove(&key);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
self.db
|
|
||||||
._transaction()
|
|
||||||
.with_timeout(30 * 1000)
|
|
||||||
.run(|db| async move {
|
|
||||||
// fake operation to batch them all at once
|
|
||||||
ModelSyncData::from_op(CRDTOperation {
|
|
||||||
instance,
|
|
||||||
model,
|
|
||||||
record_id: record_id.clone(),
|
|
||||||
timestamp: NTP64(0),
|
|
||||||
data: CRDTOperationData::Create(
|
|
||||||
data.iter()
|
|
||||||
.map(|(k, (data, _))| (k.to_string(), data.clone()))
|
|
||||||
.collect(),
|
|
||||||
),
|
|
||||||
})
|
|
||||||
.unwrap()
|
|
||||||
.exec(&db)
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
// need to only apply ops that haven't been filtered out
|
|
||||||
for (field, (value, timestamp)) in data {
|
|
||||||
write_crdt_op_to_db(
|
|
||||||
&CRDTOperation {
|
|
||||||
instance,
|
|
||||||
model,
|
|
||||||
record_id: record_id.clone(),
|
|
||||||
timestamp,
|
|
||||||
data: CRDTOperationData::Update { field, value },
|
|
||||||
},
|
|
||||||
&db,
|
|
||||||
)
|
|
||||||
.await?;
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(())
|
|
||||||
})
|
|
||||||
.await?;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// update the stored timestamp for this instance - will be derived from the crdt operations table on restart
|
// update the stored timestamp for this instance - will be derived from the crdt operations table on restart
|
||||||
|
@ -429,12 +390,179 @@ impl Actor {
|
||||||
|
|
||||||
self.timestamps.write().await.insert(instance, new_ts);
|
self.timestamps.write().await.insert(instance, new_ts);
|
||||||
|
|
||||||
// self.io.req_tx.send(Request::Ingested).await.ok();
|
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn handle_crdt_updates(
|
||||||
|
db: &PrismaClient,
|
||||||
|
instance: Uuid,
|
||||||
|
model: u16,
|
||||||
|
record_id: rmpv::Value,
|
||||||
|
mut data: BTreeMap<String, (rmpv::Value, NTP64)>,
|
||||||
|
updates: Vec<Option<crdt_operation::Data>>,
|
||||||
|
) -> Result<(), Error> {
|
||||||
|
let keys = data.keys().cloned().collect::<Vec<_>>();
|
||||||
|
|
||||||
|
// does the same thing as processing ops one-by-one and returning early if a newer op was found
|
||||||
|
for (update, key) in updates.into_iter().zip(keys) {
|
||||||
|
if update.is_some() {
|
||||||
|
data.remove(&key);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
db._transaction()
|
||||||
|
.with_timeout(30 * 1000)
|
||||||
|
.run(|db| async move {
|
||||||
|
// fake operation to batch them all at once
|
||||||
|
ModelSyncData::from_op(CRDTOperation {
|
||||||
|
instance,
|
||||||
|
model,
|
||||||
|
record_id: record_id.clone(),
|
||||||
|
timestamp: NTP64(0),
|
||||||
|
data: CRDTOperationData::Create(
|
||||||
|
data.iter()
|
||||||
|
.map(|(k, (data, _))| (k.clone(), data.clone()))
|
||||||
|
.collect(),
|
||||||
|
),
|
||||||
|
})
|
||||||
|
.ok_or(Error::InvalidModelId(model))?
|
||||||
|
.exec(&db)
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
// need to only apply ops that haven't been filtered out
|
||||||
|
data.into_iter()
|
||||||
|
.map(|(field, (value, timestamp))| {
|
||||||
|
let record_id = record_id.clone();
|
||||||
|
let db = &db;
|
||||||
|
|
||||||
|
async move {
|
||||||
|
write_crdt_op_to_db(
|
||||||
|
&CRDTOperation {
|
||||||
|
instance,
|
||||||
|
model,
|
||||||
|
record_id,
|
||||||
|
timestamp,
|
||||||
|
data: CRDTOperationData::Update { field, value },
|
||||||
|
},
|
||||||
|
db,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.try_join()
|
||||||
|
.await
|
||||||
|
.map(|_| ())
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn handle_crdt_create_and_updates(
|
||||||
|
db: &PrismaClient,
|
||||||
|
instance: Uuid,
|
||||||
|
model: u16,
|
||||||
|
record_id: rmpv::Value,
|
||||||
|
ops: Vec<CompressedCRDTOperation>,
|
||||||
|
timestamp: NTP64,
|
||||||
|
) -> Result<(), Error> {
|
||||||
|
let mut data = BTreeMap::new();
|
||||||
|
|
||||||
|
let mut applied_ops = vec![];
|
||||||
|
|
||||||
|
// search for all Updates until a Create is found
|
||||||
|
for op in ops.iter().rev() {
|
||||||
|
match &op.data {
|
||||||
|
CRDTOperationData::Delete => unreachable!("Delete can't exist here!"),
|
||||||
|
CRDTOperationData::Create(create_data) => {
|
||||||
|
for (k, v) in create_data {
|
||||||
|
data.entry(k).or_insert(v);
|
||||||
|
}
|
||||||
|
|
||||||
|
applied_ops.push(op);
|
||||||
|
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
CRDTOperationData::Update { field, value } => {
|
||||||
|
applied_ops.push(op);
|
||||||
|
data.insert(field, value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
db._transaction()
|
||||||
|
.with_timeout(30 * 1000)
|
||||||
|
.run(|db| async move {
|
||||||
|
// fake a create with a bunch of data rather than individual insert
|
||||||
|
ModelSyncData::from_op(CRDTOperation {
|
||||||
|
instance,
|
||||||
|
model,
|
||||||
|
record_id: record_id.clone(),
|
||||||
|
timestamp,
|
||||||
|
data: CRDTOperationData::Create(
|
||||||
|
data.into_iter()
|
||||||
|
.map(|(k, v)| (k.clone(), v.clone()))
|
||||||
|
.collect(),
|
||||||
|
),
|
||||||
|
})
|
||||||
|
.ok_or(Error::InvalidModelId(model))?
|
||||||
|
.exec(&db)
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
applied_ops
|
||||||
|
.into_iter()
|
||||||
|
.map(|op| {
|
||||||
|
let record_id = record_id.clone();
|
||||||
|
let db = &db;
|
||||||
|
async move {
|
||||||
|
let operation = CRDTOperation {
|
||||||
|
instance,
|
||||||
|
model,
|
||||||
|
record_id,
|
||||||
|
timestamp: op.timestamp,
|
||||||
|
data: op.data.clone(),
|
||||||
|
};
|
||||||
|
|
||||||
|
write_crdt_op_to_db(&operation, db).await
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.try_join()
|
||||||
|
.await
|
||||||
|
.map(|_| ())
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn handle_crdt_deletion(
|
||||||
|
db: &PrismaClient,
|
||||||
|
instance: Uuid,
|
||||||
|
model: u16,
|
||||||
|
record_id: rmpv::Value,
|
||||||
|
delete_op: &CompressedCRDTOperation,
|
||||||
|
) -> Result<(), Error> {
|
||||||
|
// deletes are the be all and end all, no need to check anything
|
||||||
|
let op = CRDTOperation {
|
||||||
|
instance,
|
||||||
|
model,
|
||||||
|
record_id,
|
||||||
|
timestamp: delete_op.timestamp,
|
||||||
|
data: CRDTOperationData::Delete,
|
||||||
|
};
|
||||||
|
|
||||||
|
db._transaction()
|
||||||
|
.with_timeout(30 * 1000)
|
||||||
|
.run(|db| async move {
|
||||||
|
ModelSyncData::from_op(op.clone())
|
||||||
|
.ok_or(Error::InvalidModelId(model))?
|
||||||
|
.exec(&db)
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
write_crdt_op_to_db(&op, &db).await
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
impl Deref for Actor {
|
impl Deref for Actor {
|
||||||
type Target = SharedState;
|
type Target = SharedState;
|
||||||
|
|
||||||
|
@ -444,8 +572,8 @@ impl Deref for Actor {
|
||||||
}
|
}
|
||||||
|
|
||||||
pub struct Handler {
|
pub struct Handler {
|
||||||
pub event_tx: mpsc::Sender<Event>,
|
pub event_tx: chan::Sender<Event>,
|
||||||
pub req_rx: Arc<Mutex<mpsc::Receiver<Request>>>,
|
pub req_rx: chan::Receiver<Request>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
|
@ -466,12 +594,13 @@ impl ActorTypes for Actor {
|
||||||
mod test {
|
mod test {
|
||||||
use std::{sync::atomic::AtomicBool, time::Duration};
|
use std::{sync::atomic::AtomicBool, time::Duration};
|
||||||
|
|
||||||
|
use tokio::sync::Notify;
|
||||||
use uhlc::HLCBuilder;
|
use uhlc::HLCBuilder;
|
||||||
|
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
async fn new_actor() -> (Handler, Arc<SharedState>) {
|
async fn new_actor() -> (Handler, Arc<SharedState>) {
|
||||||
let instance = uuid::Uuid::new_v4();
|
let instance = Uuid::new_v4();
|
||||||
let shared = Arc::new(SharedState {
|
let shared = Arc::new(SharedState {
|
||||||
db: sd_prisma::test_db().await,
|
db: sd_prisma::test_db().await,
|
||||||
instance,
|
instance,
|
||||||
|
@ -480,14 +609,14 @@ mod test {
|
||||||
NonZeroU128::new(instance.to_u128_le()).expect("Non zero id"),
|
NonZeroU128::new(instance.to_u128_le()).expect("Non zero id"),
|
||||||
))
|
))
|
||||||
.build(),
|
.build(),
|
||||||
timestamps: Default::default(),
|
timestamps: Arc::default(),
|
||||||
emit_messages_flag: Arc::new(AtomicBool::new(true)),
|
emit_messages_flag: Arc::new(AtomicBool::new(true)),
|
||||||
active: Default::default(),
|
active: AtomicBool::default(),
|
||||||
active_notify: Default::default(),
|
active_notify: Notify::default(),
|
||||||
actors: Default::default(),
|
actors: Arc::default(),
|
||||||
});
|
});
|
||||||
|
|
||||||
(Actor::declare(shared.clone()).await, shared)
|
(Actor::declare(Arc::clone(&shared)).await, shared)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// If messages tx is dropped, actor should reset and assume no further messages
|
/// If messages tx is dropped, actor should reset and assume no further messages
|
||||||
|
@ -497,11 +626,9 @@ mod test {
|
||||||
let (ingest, _) = new_actor().await;
|
let (ingest, _) = new_actor().await;
|
||||||
|
|
||||||
for _ in 0..10 {
|
for _ in 0..10 {
|
||||||
let mut rx = ingest.req_rx.lock().await;
|
|
||||||
|
|
||||||
ingest.event_tx.send(Event::Notification).await.unwrap();
|
ingest.event_tx.send(Event::Notification).await.unwrap();
|
||||||
|
|
||||||
let Some(Request::Messages { .. }) = rx.recv().await else {
|
let Ok(Request::Messages { .. }) = ingest.req_rx.recv().await else {
|
||||||
panic!("bruh")
|
panic!("bruh")
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
|
@ -1,10 +1,31 @@
|
||||||
#![allow(clippy::unwrap_used, clippy::panic)] // TODO: Brendan remove this once you've got error handling here
|
#![warn(
|
||||||
|
clippy::all,
|
||||||
mod actor;
|
clippy::pedantic,
|
||||||
pub mod backfill;
|
clippy::correctness,
|
||||||
mod db_operation;
|
clippy::perf,
|
||||||
pub mod ingest;
|
clippy::style,
|
||||||
mod manager;
|
clippy::suspicious,
|
||||||
|
clippy::complexity,
|
||||||
|
clippy::nursery,
|
||||||
|
clippy::unwrap_used,
|
||||||
|
unused_qualifications,
|
||||||
|
rust_2018_idioms,
|
||||||
|
trivial_casts,
|
||||||
|
trivial_numeric_casts,
|
||||||
|
unused_allocation,
|
||||||
|
clippy::unnecessary_cast,
|
||||||
|
clippy::cast_lossless,
|
||||||
|
clippy::cast_possible_truncation,
|
||||||
|
clippy::cast_possible_wrap,
|
||||||
|
clippy::cast_precision_loss,
|
||||||
|
clippy::cast_sign_loss,
|
||||||
|
clippy::dbg_macro,
|
||||||
|
clippy::deprecated_cfg_attr,
|
||||||
|
clippy::separated_literal_suffix,
|
||||||
|
deprecated
|
||||||
|
)]
|
||||||
|
#![forbid(deprecated_in_future)]
|
||||||
|
#![allow(clippy::missing_errors_doc, clippy::module_name_repetitions)]
|
||||||
|
|
||||||
use sd_prisma::prisma::{crdt_operation, instance, PrismaClient};
|
use sd_prisma::prisma::{crdt_operation, instance, PrismaClient};
|
||||||
use sd_sync::CRDTOperation;
|
use sd_sync::CRDTOperation;
|
||||||
|
@ -14,6 +35,15 @@ use std::{
|
||||||
sync::{atomic::AtomicBool, Arc},
|
sync::{atomic::AtomicBool, Arc},
|
||||||
};
|
};
|
||||||
|
|
||||||
|
use tokio::sync::{Notify, RwLock};
|
||||||
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
mod actor;
|
||||||
|
pub mod backfill;
|
||||||
|
mod db_operation;
|
||||||
|
pub mod ingest;
|
||||||
|
mod manager;
|
||||||
|
|
||||||
pub use ingest::*;
|
pub use ingest::*;
|
||||||
pub use manager::*;
|
pub use manager::*;
|
||||||
pub use uhlc::NTP64;
|
pub use uhlc::NTP64;
|
||||||
|
@ -24,44 +54,83 @@ pub enum SyncMessage {
|
||||||
Created,
|
Created,
|
||||||
}
|
}
|
||||||
|
|
||||||
pub type Timestamps = Arc<tokio::sync::RwLock<HashMap<uuid::Uuid, NTP64>>>;
|
pub type Timestamps = Arc<RwLock<HashMap<Uuid, NTP64>>>;
|
||||||
|
|
||||||
pub struct SharedState {
|
pub struct SharedState {
|
||||||
pub db: Arc<PrismaClient>,
|
pub db: Arc<PrismaClient>,
|
||||||
pub emit_messages_flag: Arc<AtomicBool>,
|
pub emit_messages_flag: Arc<AtomicBool>,
|
||||||
pub instance: uuid::Uuid,
|
pub instance: Uuid,
|
||||||
pub timestamps: Timestamps,
|
pub timestamps: Timestamps,
|
||||||
pub clock: uhlc::HLC,
|
pub clock: uhlc::HLC,
|
||||||
pub active: AtomicBool,
|
pub active: AtomicBool,
|
||||||
pub active_notify: tokio::sync::Notify,
|
pub active_notify: Notify,
|
||||||
pub actors: Arc<sd_actors::Actors>,
|
pub actors: Arc<sd_actors::Actors>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[must_use]
|
#[derive(thiserror::Error, Debug)]
|
||||||
pub fn crdt_op_db(op: &CRDTOperation) -> crdt_operation::Create {
|
pub enum Error {
|
||||||
crdt_operation::Create {
|
#[error("serialization error: {0}")]
|
||||||
timestamp: op.timestamp.0 as i64,
|
Serialization(#[from] rmp_serde::encode::Error),
|
||||||
instance: instance::pub_id::equals(op.instance.as_bytes().to_vec()),
|
#[error("deserialization error: {0}")]
|
||||||
kind: op.kind().to_string(),
|
Deserialization(#[from] rmp_serde::decode::Error),
|
||||||
data: rmp_serde::to_vec(&op.data).unwrap(),
|
#[error("database error: {0}")]
|
||||||
model: op.model as i32,
|
Database(#[from] prisma_client_rust::QueryError),
|
||||||
record_id: rmp_serde::to_vec(&op.record_id).unwrap(),
|
#[error("invalid model id: {0}")]
|
||||||
_params: vec![],
|
InvalidModelId(u16),
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<Error> for rspc::Error {
|
||||||
|
fn from(e: Error) -> Self {
|
||||||
|
match e {
|
||||||
|
Error::Database(e) => e.into(),
|
||||||
|
Error::InvalidModelId(id) => Self::new(
|
||||||
|
rspc::ErrorCode::BadRequest,
|
||||||
|
format!("Invalid model id <id={id}>"),
|
||||||
|
),
|
||||||
|
_ => Self::with_cause(
|
||||||
|
rspc::ErrorCode::InternalServerError,
|
||||||
|
"Internal sync error".to_string(),
|
||||||
|
e,
|
||||||
|
),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[must_use]
|
pub fn crdt_op_db(op: &CRDTOperation) -> Result<crdt_operation::Create, Error> {
|
||||||
|
Ok(crdt_operation::Create {
|
||||||
|
timestamp: {
|
||||||
|
#[allow(clippy::cast_possible_wrap)]
|
||||||
|
// SAFETY: we had to store using i64 due to SQLite limitations
|
||||||
|
{
|
||||||
|
op.timestamp.as_u64() as i64
|
||||||
|
}
|
||||||
|
},
|
||||||
|
instance: instance::pub_id::equals(op.instance.as_bytes().to_vec()),
|
||||||
|
kind: op.kind().to_string(),
|
||||||
|
data: rmp_serde::to_vec(&op.data)?,
|
||||||
|
model: i32::from(op.model),
|
||||||
|
record_id: rmp_serde::to_vec(&op.record_id)?,
|
||||||
|
_params: vec![],
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
pub fn crdt_op_unchecked_db(
|
pub fn crdt_op_unchecked_db(
|
||||||
op: &CRDTOperation,
|
op: &CRDTOperation,
|
||||||
instance_id: i32,
|
instance_id: i32,
|
||||||
) -> crdt_operation::CreateUnchecked {
|
) -> Result<crdt_operation::CreateUnchecked, Error> {
|
||||||
crdt_operation::CreateUnchecked {
|
Ok(crdt_operation::CreateUnchecked {
|
||||||
timestamp: op.timestamp.0 as i64,
|
timestamp: {
|
||||||
|
#[allow(clippy::cast_possible_wrap)]
|
||||||
|
// SAFETY: we had to store using i64 due to SQLite limitations
|
||||||
|
{
|
||||||
|
op.timestamp.as_u64() as i64
|
||||||
|
}
|
||||||
|
},
|
||||||
instance_id,
|
instance_id,
|
||||||
kind: op.kind().to_string(),
|
kind: op.kind().to_string(),
|
||||||
data: rmp_serde::to_vec(&op.data).unwrap(),
|
data: rmp_serde::to_vec(&op.data)?,
|
||||||
model: op.model as i32,
|
model: i32::from(op.model),
|
||||||
record_id: rmp_serde::to_vec(&op.record_id).unwrap(),
|
record_id: rmp_serde::to_vec(&op.record_id)?,
|
||||||
_params: vec![],
|
_params: vec![],
|
||||||
}
|
})
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,13 +1,10 @@
|
||||||
use crate::{crdt_op_db, db_operation::*, ingest, SharedState, SyncMessage, NTP64};
|
|
||||||
|
|
||||||
use sd_prisma::prisma::{cloud_crdt_operation, crdt_operation, instance, PrismaClient, SortOrder};
|
use sd_prisma::prisma::{cloud_crdt_operation, crdt_operation, instance, PrismaClient, SortOrder};
|
||||||
use sd_sync::{CRDTOperation, OperationFactory};
|
use sd_sync::{CRDTOperation, OperationFactory};
|
||||||
use sd_utils::uuid_to_bytes;
|
use sd_utils::{from_bytes_to_uuid, uuid_to_bytes};
|
||||||
|
use tracing::warn;
|
||||||
|
|
||||||
use std::{
|
use std::{
|
||||||
cmp::Ordering,
|
cmp, fmt,
|
||||||
collections::HashMap,
|
|
||||||
fmt,
|
|
||||||
num::NonZeroU128,
|
num::NonZeroU128,
|
||||||
ops::Deref,
|
ops::Deref,
|
||||||
sync::{
|
sync::{
|
||||||
|
@ -16,16 +13,23 @@ use std::{
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
use tokio::sync::{broadcast, RwLock};
|
use prisma_client_rust::{and, operator::or};
|
||||||
|
use tokio::sync::{broadcast, Mutex, Notify, RwLock};
|
||||||
use uhlc::{HLCBuilder, HLC};
|
use uhlc::{HLCBuilder, HLC};
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
use super::{
|
||||||
|
crdt_op_db,
|
||||||
|
db_operation::{cloud_crdt_with_instance, crdt_with_instance},
|
||||||
|
ingest, Error, SharedState, SyncMessage, NTP64,
|
||||||
|
};
|
||||||
|
|
||||||
/// Wrapper that spawns the ingest actor and provides utilities for reading and writing sync operations.
|
/// Wrapper that spawns the ingest actor and provides utilities for reading and writing sync operations.
|
||||||
pub struct Manager {
|
pub struct Manager {
|
||||||
pub tx: broadcast::Sender<SyncMessage>,
|
pub tx: broadcast::Sender<SyncMessage>,
|
||||||
pub ingest: ingest::Handler,
|
pub ingest: ingest::Handler,
|
||||||
pub shared: Arc<SharedState>,
|
pub shared: Arc<SharedState>,
|
||||||
pub timestamp_lock: tokio::sync::Semaphore,
|
pub timestamp_lock: Mutex<()>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl fmt::Debug for Manager {
|
impl fmt::Debug for Manager {
|
||||||
|
@ -40,74 +44,122 @@ pub struct GetOpsArgs {
|
||||||
pub count: u32,
|
pub count: u32,
|
||||||
}
|
}
|
||||||
|
|
||||||
pub struct New {
|
|
||||||
pub manager: Manager,
|
|
||||||
pub rx: broadcast::Receiver<SyncMessage>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Manager {
|
impl Manager {
|
||||||
#[allow(clippy::new_ret_no_self)]
|
/// Creates a new manager that can be used to read and write CRDT operations.
|
||||||
|
/// Sync messages are received on the returned [`broadcast::Receiver<SyncMessage>`].
|
||||||
pub async fn new(
|
pub async fn new(
|
||||||
db: &Arc<PrismaClient>,
|
db: Arc<PrismaClient>,
|
||||||
instance: Uuid,
|
current_instance_uuid: Uuid,
|
||||||
emit_messages_flag: &Arc<AtomicBool>,
|
emit_messages_flag: Arc<AtomicBool>,
|
||||||
timestamps: HashMap<Uuid, NTP64>,
|
actors: Arc<sd_actors::Actors>,
|
||||||
actors: &Arc<sd_actors::Actors>,
|
) -> Result<(Self, broadcast::Receiver<SyncMessage>), Error> {
|
||||||
) -> New {
|
let existing_instances = db.instance().find_many(vec![]).exec().await?;
|
||||||
|
|
||||||
|
Self::with_existing_instances(
|
||||||
|
db,
|
||||||
|
current_instance_uuid,
|
||||||
|
emit_messages_flag,
|
||||||
|
&existing_instances,
|
||||||
|
actors,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Creates a new manager that can be used to read and write CRDT operations from a list of existing instances.
|
||||||
|
/// Sync messages are received on the returned [`broadcast::Receiver<SyncMessage>`].
|
||||||
|
///
|
||||||
|
/// # Panics
|
||||||
|
/// Panics if the `current_instance_id` UUID is zeroed.
|
||||||
|
pub async fn with_existing_instances(
|
||||||
|
db: Arc<PrismaClient>,
|
||||||
|
current_instance_uuid: Uuid,
|
||||||
|
emit_messages_flag: Arc<AtomicBool>,
|
||||||
|
existing_instances: &[instance::Data],
|
||||||
|
actors: Arc<sd_actors::Actors>,
|
||||||
|
) -> Result<(Self, broadcast::Receiver<SyncMessage>), Error> {
|
||||||
|
let timestamps = db
|
||||||
|
._batch(
|
||||||
|
existing_instances
|
||||||
|
.iter()
|
||||||
|
.map(|i| {
|
||||||
|
db.crdt_operation()
|
||||||
|
.find_first(vec![crdt_operation::instance::is(vec![
|
||||||
|
instance::id::equals(i.id),
|
||||||
|
])])
|
||||||
|
.order_by(crdt_operation::timestamp::order(SortOrder::Desc))
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>(),
|
||||||
|
)
|
||||||
|
.await?
|
||||||
|
.into_iter()
|
||||||
|
.zip(existing_instances)
|
||||||
|
.map(|(op, i)| {
|
||||||
|
(
|
||||||
|
from_bytes_to_uuid(&i.pub_id),
|
||||||
|
#[allow(clippy::cast_sign_loss)]
|
||||||
|
// SAFETY: we had to store using i64 due to SQLite limitations
|
||||||
|
NTP64(op.map(|o| o.timestamp).unwrap_or_default() as u64),
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
|
||||||
let (tx, rx) = broadcast::channel(64);
|
let (tx, rx) = broadcast::channel(64);
|
||||||
|
|
||||||
let clock = HLCBuilder::new()
|
let clock = HLCBuilder::new()
|
||||||
.with_id(uhlc::ID::from(
|
.with_id(uhlc::ID::from(
|
||||||
NonZeroU128::new(instance.to_u128_le()).expect("Non zero id"),
|
NonZeroU128::new(current_instance_uuid.to_u128_le()).expect("Non zero id"),
|
||||||
))
|
))
|
||||||
.build();
|
.build();
|
||||||
|
|
||||||
let shared = Arc::new(SharedState {
|
let shared = Arc::new(SharedState {
|
||||||
db: db.clone(),
|
db,
|
||||||
instance,
|
instance: current_instance_uuid,
|
||||||
clock,
|
clock,
|
||||||
timestamps: Arc::new(RwLock::new(timestamps)),
|
timestamps: Arc::new(RwLock::new(timestamps)),
|
||||||
emit_messages_flag: emit_messages_flag.clone(),
|
emit_messages_flag,
|
||||||
active: Default::default(),
|
active: AtomicBool::default(),
|
||||||
active_notify: Default::default(),
|
active_notify: Notify::default(),
|
||||||
actors: actors.clone(),
|
actors,
|
||||||
});
|
});
|
||||||
|
|
||||||
let ingest = ingest::Actor::declare(shared.clone()).await;
|
let ingest = ingest::Actor::declare(shared.clone()).await;
|
||||||
|
|
||||||
New {
|
Ok((
|
||||||
manager: Self {
|
Self {
|
||||||
tx,
|
tx,
|
||||||
ingest,
|
ingest,
|
||||||
shared,
|
shared,
|
||||||
timestamp_lock: tokio::sync::Semaphore::new(1),
|
timestamp_lock: Mutex::default(),
|
||||||
},
|
},
|
||||||
rx,
|
rx,
|
||||||
}
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn subscribe(&self) -> broadcast::Receiver<SyncMessage> {
|
pub fn subscribe(&self) -> broadcast::Receiver<SyncMessage> {
|
||||||
self.tx.subscribe()
|
self.tx.subscribe()
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn write_ops<'item, I: prisma_client_rust::BatchItem<'item>>(
|
pub async fn write_ops<'item, Q>(
|
||||||
&self,
|
&self,
|
||||||
tx: &PrismaClient,
|
tx: &PrismaClient,
|
||||||
(mut ops, queries): (Vec<CRDTOperation>, I),
|
(mut ops, queries): (Vec<CRDTOperation>, Q),
|
||||||
) -> prisma_client_rust::Result<<I as prisma_client_rust::BatchItemParent>::ReturnValue> {
|
) -> Result<Q::ReturnValue, Error>
|
||||||
|
where
|
||||||
|
Q: prisma_client_rust::BatchItem<'item, ReturnValue: Send> + Send,
|
||||||
|
{
|
||||||
let ret = if self.emit_messages_flag.load(atomic::Ordering::Relaxed) {
|
let ret = if self.emit_messages_flag.load(atomic::Ordering::Relaxed) {
|
||||||
let lock = self.timestamp_lock.acquire().await;
|
let lock = self.timestamp_lock.lock().await;
|
||||||
|
|
||||||
ops.iter_mut().for_each(|op| {
|
for op in &mut ops {
|
||||||
op.timestamp = *self.get_clock().new_timestamp().get_time();
|
op.timestamp = *self.get_clock().new_timestamp().get_time();
|
||||||
});
|
}
|
||||||
|
|
||||||
let (res, _) = tx
|
let (res, _) = tx
|
||||||
._batch((
|
._batch((
|
||||||
queries,
|
queries,
|
||||||
ops.iter()
|
ops.iter()
|
||||||
.map(|op| crdt_op_db(op).to_query(tx))
|
.map(|op| crdt_op_db(op).map(|q| q.to_query(tx)))
|
||||||
.collect::<Vec<_>>(),
|
.collect::<Result<Vec<_>, _>>()?,
|
||||||
))
|
))
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
|
@ -119,7 +171,9 @@ impl Manager {
|
||||||
.insert(self.instance, last.timestamp);
|
.insert(self.instance, last.timestamp);
|
||||||
}
|
}
|
||||||
|
|
||||||
self.tx.send(SyncMessage::Created).ok();
|
if self.tx.send(SyncMessage::Created).is_err() {
|
||||||
|
warn!("failed to send created message on `write_ops`");
|
||||||
|
}
|
||||||
|
|
||||||
drop(lock);
|
drop(lock);
|
||||||
|
|
||||||
|
@ -131,21 +185,25 @@ impl Manager {
|
||||||
Ok(ret)
|
Ok(ret)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(unused_variables)]
|
pub async fn write_op<'item, Q>(
|
||||||
pub async fn write_op<'item, Q: prisma_client_rust::BatchItem<'item>>(
|
|
||||||
&self,
|
&self,
|
||||||
tx: &PrismaClient,
|
tx: &PrismaClient,
|
||||||
mut op: CRDTOperation,
|
mut op: CRDTOperation,
|
||||||
query: Q,
|
query: Q,
|
||||||
) -> prisma_client_rust::Result<<Q as prisma_client_rust::BatchItemParent>::ReturnValue> {
|
) -> Result<Q::ReturnValue, Error>
|
||||||
|
where
|
||||||
|
Q: prisma_client_rust::BatchItem<'item, ReturnValue: Send> + Send,
|
||||||
|
{
|
||||||
let ret = if self.emit_messages_flag.load(atomic::Ordering::Relaxed) {
|
let ret = if self.emit_messages_flag.load(atomic::Ordering::Relaxed) {
|
||||||
let lock = self.timestamp_lock.acquire().await;
|
let lock = self.timestamp_lock.lock().await;
|
||||||
|
|
||||||
op.timestamp = *self.get_clock().new_timestamp().get_time();
|
op.timestamp = *self.get_clock().new_timestamp().get_time();
|
||||||
|
|
||||||
let ret = tx._batch((crdt_op_db(&op).to_query(tx), query)).await?.1;
|
let ret = tx._batch((crdt_op_db(&op)?.to_query(tx), query)).await?.1;
|
||||||
|
|
||||||
self.tx.send(SyncMessage::Created).ok();
|
if self.tx.send(SyncMessage::Created).is_err() {
|
||||||
|
warn!("failed to send created message on `write_op`");
|
||||||
|
}
|
||||||
|
|
||||||
drop(lock);
|
drop(lock);
|
||||||
|
|
||||||
|
@ -168,143 +226,121 @@ impl Manager {
|
||||||
count: u32,
|
count: u32,
|
||||||
instance_uuid: Uuid,
|
instance_uuid: Uuid,
|
||||||
timestamp: NTP64,
|
timestamp: NTP64,
|
||||||
) -> prisma_client_rust::Result<Vec<CRDTOperation>> {
|
) -> Result<Vec<CRDTOperation>, Error> {
|
||||||
let db = &self.db;
|
self.db
|
||||||
|
|
||||||
Ok(db
|
|
||||||
.crdt_operation()
|
.crdt_operation()
|
||||||
.find_many(vec![
|
.find_many(vec![
|
||||||
crdt_operation::instance::is(vec![instance::pub_id::equals(uuid_to_bytes(
|
crdt_operation::instance::is(vec![instance::pub_id::equals(uuid_to_bytes(
|
||||||
&instance_uuid,
|
&instance_uuid,
|
||||||
))]),
|
))]),
|
||||||
|
#[allow(clippy::cast_possible_wrap)]
|
||||||
crdt_operation::timestamp::gt(timestamp.as_u64() as i64),
|
crdt_operation::timestamp::gt(timestamp.as_u64() as i64),
|
||||||
])
|
])
|
||||||
.take(i64::from(count))
|
.take(i64::from(count))
|
||||||
.order_by(crdt_operation::timestamp::order(SortOrder::Asc))
|
.order_by(crdt_operation::timestamp::order(SortOrder::Asc))
|
||||||
.include(crdt_include::include())
|
.include(crdt_with_instance::include())
|
||||||
.exec()
|
.exec()
|
||||||
.await?
|
.await?
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.map(|o| o.into_operation())
|
.map(crdt_with_instance::Data::into_operation)
|
||||||
.collect())
|
.collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn get_ops(
|
pub async fn get_ops(&self, args: GetOpsArgs) -> Result<Vec<CRDTOperation>, Error> {
|
||||||
&self,
|
let mut ops = self
|
||||||
args: GetOpsArgs,
|
.db
|
||||||
) -> prisma_client_rust::Result<Vec<CRDTOperation>> {
|
|
||||||
let db = &self.db;
|
|
||||||
|
|
||||||
macro_rules! db_args {
|
|
||||||
($args:ident, $op:ident) => {
|
|
||||||
vec![prisma_client_rust::operator::or(
|
|
||||||
$args
|
|
||||||
.clocks
|
|
||||||
.iter()
|
|
||||||
.map(|(instance_id, timestamp)| {
|
|
||||||
prisma_client_rust::and![
|
|
||||||
$op::instance::is(vec![instance::pub_id::equals(uuid_to_bytes(
|
|
||||||
instance_id
|
|
||||||
))]),
|
|
||||||
$op::timestamp::gt(timestamp.as_u64() as i64)
|
|
||||||
]
|
|
||||||
})
|
|
||||||
.chain([
|
|
||||||
$op::instance::is_not(vec![
|
|
||||||
instance::pub_id::in_vec(
|
|
||||||
$args
|
|
||||||
.clocks
|
|
||||||
.iter()
|
|
||||||
.map(|(instance_id, _)| {
|
|
||||||
uuid_to_bytes(instance_id)
|
|
||||||
})
|
|
||||||
.collect()
|
|
||||||
)
|
|
||||||
])
|
|
||||||
])
|
|
||||||
.collect(),
|
|
||||||
)]
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
let mut ops = db
|
|
||||||
.crdt_operation()
|
.crdt_operation()
|
||||||
.find_many(db_args!(args, crdt_operation))
|
.find_many(vec![or(args
|
||||||
|
.clocks
|
||||||
|
.iter()
|
||||||
|
.map(|(instance_id, timestamp)| {
|
||||||
|
and![
|
||||||
|
crdt_operation::instance::is(vec![instance::pub_id::equals(
|
||||||
|
uuid_to_bytes(instance_id)
|
||||||
|
)]),
|
||||||
|
crdt_operation::timestamp::gt({
|
||||||
|
#[allow(clippy::cast_possible_wrap)]
|
||||||
|
// SAFETY: we had to store using i64 due to SQLite limitations
|
||||||
|
{
|
||||||
|
timestamp.as_u64() as i64
|
||||||
|
}
|
||||||
|
})
|
||||||
|
]
|
||||||
|
})
|
||||||
|
.chain([crdt_operation::instance::is_not(vec![
|
||||||
|
instance::pub_id::in_vec(
|
||||||
|
args.clocks
|
||||||
|
.iter()
|
||||||
|
.map(|(instance_id, _)| uuid_to_bytes(instance_id))
|
||||||
|
.collect(),
|
||||||
|
),
|
||||||
|
])])
|
||||||
|
.collect())])
|
||||||
.take(i64::from(args.count))
|
.take(i64::from(args.count))
|
||||||
.order_by(crdt_operation::timestamp::order(SortOrder::Asc))
|
.order_by(crdt_operation::timestamp::order(SortOrder::Asc))
|
||||||
.include(crdt_include::include())
|
.include(crdt_with_instance::include())
|
||||||
.exec()
|
.exec()
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
ops.sort_by(|a, b| match a.timestamp().cmp(&b.timestamp()) {
|
ops.sort_by(|a, b| match a.timestamp().cmp(&b.timestamp()) {
|
||||||
Ordering::Equal => a.instance().cmp(&b.instance()),
|
cmp::Ordering::Equal => a.instance().cmp(&b.instance()),
|
||||||
o => o,
|
o => o,
|
||||||
});
|
});
|
||||||
|
|
||||||
Ok(ops
|
ops.into_iter()
|
||||||
.into_iter()
|
|
||||||
.take(args.count as usize)
|
.take(args.count as usize)
|
||||||
.map(|o| o.into_operation())
|
.map(crdt_with_instance::Data::into_operation)
|
||||||
.collect())
|
.collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn get_cloud_ops(
|
pub async fn get_cloud_ops(
|
||||||
&self,
|
&self,
|
||||||
args: GetOpsArgs,
|
args: GetOpsArgs,
|
||||||
) -> prisma_client_rust::Result<Vec<(i32, CRDTOperation)>> {
|
) -> Result<Vec<(i32, CRDTOperation)>, Error> {
|
||||||
let db = &self.db;
|
let mut ops = self
|
||||||
|
.db
|
||||||
macro_rules! db_args {
|
|
||||||
($args:ident, $op:ident) => {
|
|
||||||
vec![prisma_client_rust::operator::or(
|
|
||||||
$args
|
|
||||||
.clocks
|
|
||||||
.iter()
|
|
||||||
.map(|(instance_id, timestamp)| {
|
|
||||||
prisma_client_rust::and![
|
|
||||||
$op::instance::is(vec![instance::pub_id::equals(uuid_to_bytes(
|
|
||||||
instance_id
|
|
||||||
))]),
|
|
||||||
$op::timestamp::gt(timestamp.as_u64() as i64)
|
|
||||||
]
|
|
||||||
})
|
|
||||||
.chain([
|
|
||||||
$op::instance::is_not(vec![
|
|
||||||
instance::pub_id::in_vec(
|
|
||||||
$args
|
|
||||||
.clocks
|
|
||||||
.iter()
|
|
||||||
.map(|(instance_id, _)| {
|
|
||||||
uuid_to_bytes(instance_id)
|
|
||||||
})
|
|
||||||
.collect()
|
|
||||||
)
|
|
||||||
])
|
|
||||||
])
|
|
||||||
.collect(),
|
|
||||||
)]
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
let mut ops = db
|
|
||||||
.cloud_crdt_operation()
|
.cloud_crdt_operation()
|
||||||
.find_many(db_args!(args, cloud_crdt_operation))
|
.find_many(vec![or(args
|
||||||
|
.clocks
|
||||||
|
.iter()
|
||||||
|
.map(|(instance_id, timestamp)| {
|
||||||
|
and![
|
||||||
|
cloud_crdt_operation::instance::is(vec![instance::pub_id::equals(
|
||||||
|
uuid_to_bytes(instance_id)
|
||||||
|
)]),
|
||||||
|
cloud_crdt_operation::timestamp::gt({
|
||||||
|
#[allow(clippy::cast_possible_wrap)]
|
||||||
|
// SAFETY: we had to store using i64 due to SQLite limitations
|
||||||
|
{
|
||||||
|
timestamp.as_u64() as i64
|
||||||
|
}
|
||||||
|
})
|
||||||
|
]
|
||||||
|
})
|
||||||
|
.chain([cloud_crdt_operation::instance::is_not(vec![
|
||||||
|
instance::pub_id::in_vec(
|
||||||
|
args.clocks
|
||||||
|
.iter()
|
||||||
|
.map(|(instance_id, _)| uuid_to_bytes(instance_id))
|
||||||
|
.collect(),
|
||||||
|
),
|
||||||
|
])])
|
||||||
|
.collect())])
|
||||||
.take(i64::from(args.count))
|
.take(i64::from(args.count))
|
||||||
.order_by(cloud_crdt_operation::timestamp::order(SortOrder::Asc))
|
.order_by(cloud_crdt_operation::timestamp::order(SortOrder::Asc))
|
||||||
.include(cloud_crdt_include::include())
|
.include(cloud_crdt_with_instance::include())
|
||||||
.exec()
|
.exec()
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
ops.sort_by(|a, b| match a.timestamp().cmp(&b.timestamp()) {
|
ops.sort_by(|a, b| match a.timestamp().cmp(&b.timestamp()) {
|
||||||
Ordering::Equal => a.instance().cmp(&b.instance()),
|
cmp::Ordering::Equal => a.instance().cmp(&b.instance()),
|
||||||
o => o,
|
o => o,
|
||||||
});
|
});
|
||||||
|
|
||||||
Ok(ops
|
ops.into_iter()
|
||||||
.into_iter()
|
|
||||||
.take(args.count as usize)
|
.take(args.count as usize)
|
||||||
.map(|o| o.into_operation())
|
.map(cloud_crdt_with_instance::Data::into_operation)
|
||||||
.collect())
|
.collect()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -2,27 +2,27 @@ mod mock_instance;
|
||||||
|
|
||||||
use sd_core_sync::*;
|
use sd_core_sync::*;
|
||||||
|
|
||||||
use sd_prisma::{prisma, prisma_sync};
|
use sd_prisma::{prisma::location, prisma_sync};
|
||||||
use sd_sync::*;
|
use sd_sync::*;
|
||||||
use sd_utils::{msgpack, uuid_to_bytes};
|
use sd_utils::{msgpack, uuid_to_bytes};
|
||||||
|
|
||||||
use mock_instance::Instance;
|
use mock_instance::Instance;
|
||||||
|
use tracing::info;
|
||||||
|
use tracing_test::traced_test;
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
async fn write_test_location(
|
const MOCK_LOCATION_NAME: &str = "Location 0";
|
||||||
instance: &Instance,
|
const MOCK_LOCATION_PATH: &str = "/User/Anon/Documents";
|
||||||
) -> Result<prisma::location::Data, Box<dyn std::error::Error>> {
|
|
||||||
Ok(instance
|
async fn write_test_location(instance: &Instance) -> location::Data {
|
||||||
|
let location_pub_id = Uuid::new_v4();
|
||||||
|
|
||||||
|
let location = instance
|
||||||
.sync
|
.sync
|
||||||
.write_ops(&instance.db, {
|
.write_ops(&instance.db, {
|
||||||
let id = Uuid::new_v4();
|
|
||||||
|
|
||||||
let (sync_ops, db_ops): (Vec<_>, Vec<_>) = [
|
let (sync_ops, db_ops): (Vec<_>, Vec<_>) = [
|
||||||
sync_db_entry!("Location 0".to_string(), prisma::location::name),
|
sync_db_entry!(MOCK_LOCATION_NAME, location::name),
|
||||||
sync_db_entry!(
|
sync_db_entry!(MOCK_LOCATION_PATH, location::path),
|
||||||
"/User/Brendan/Documents".to_string(),
|
|
||||||
prisma::location::path
|
|
||||||
),
|
|
||||||
]
|
]
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.unzip();
|
.unzip();
|
||||||
|
@ -30,21 +30,60 @@ async fn write_test_location(
|
||||||
(
|
(
|
||||||
instance.sync.shared_create(
|
instance.sync.shared_create(
|
||||||
prisma_sync::location::SyncId {
|
prisma_sync::location::SyncId {
|
||||||
pub_id: uuid_to_bytes(&id),
|
pub_id: uuid_to_bytes(&location_pub_id),
|
||||||
},
|
},
|
||||||
sync_ops,
|
sync_ops,
|
||||||
),
|
),
|
||||||
instance.db.location().create(uuid_to_bytes(&id), db_ops),
|
instance
|
||||||
|
.db
|
||||||
|
.location()
|
||||||
|
.create(uuid_to_bytes(&location_pub_id), db_ops),
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
.await?)
|
.await
|
||||||
|
.expect("failed to create mock location");
|
||||||
|
|
||||||
|
instance
|
||||||
|
.sync
|
||||||
|
.write_ops(&instance.db, {
|
||||||
|
let (sync_ops, db_ops): (Vec<_>, Vec<_>) = [
|
||||||
|
sync_db_entry!(1024, location::total_capacity),
|
||||||
|
sync_db_entry!(512, location::available_capacity),
|
||||||
|
]
|
||||||
|
.into_iter()
|
||||||
|
.unzip();
|
||||||
|
|
||||||
|
(
|
||||||
|
sync_ops
|
||||||
|
.into_iter()
|
||||||
|
.map(|(k, v)| {
|
||||||
|
instance.sync.shared_update(
|
||||||
|
prisma_sync::location::SyncId {
|
||||||
|
pub_id: uuid_to_bytes(&location_pub_id),
|
||||||
|
},
|
||||||
|
k,
|
||||||
|
v,
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>(),
|
||||||
|
instance
|
||||||
|
.db
|
||||||
|
.location()
|
||||||
|
.update(location::id::equals(location.id), db_ops),
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("failed to create mock location");
|
||||||
|
|
||||||
|
location
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[traced_test]
|
||||||
async fn writes_operations_and_rows_together() -> Result<(), Box<dyn std::error::Error>> {
|
async fn writes_operations_and_rows_together() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
let instance = Instance::new(Uuid::new_v4()).await;
|
let instance = Instance::new(Uuid::new_v4()).await;
|
||||||
|
|
||||||
write_test_location(&instance).await?;
|
write_test_location(&instance).await;
|
||||||
|
|
||||||
let operations = instance
|
let operations = instance
|
||||||
.db
|
.db
|
||||||
|
@ -57,27 +96,46 @@ async fn writes_operations_and_rows_together() -> Result<(), Box<dyn std::error:
|
||||||
assert_eq!(operations.len(), 3);
|
assert_eq!(operations.len(), 3);
|
||||||
assert_eq!(operations[0].model, prisma_sync::location::MODEL_ID as i32);
|
assert_eq!(operations[0].model, prisma_sync::location::MODEL_ID as i32);
|
||||||
|
|
||||||
|
let out = instance
|
||||||
|
.sync
|
||||||
|
.get_ops(GetOpsArgs {
|
||||||
|
clocks: vec![],
|
||||||
|
count: 100,
|
||||||
|
})
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
assert_eq!(out.len(), 3);
|
||||||
|
|
||||||
let locations = instance.db.location().find_many(vec![]).exec().await?;
|
let locations = instance.db.location().find_many(vec![]).exec().await?;
|
||||||
|
|
||||||
assert_eq!(locations.len(), 1);
|
assert_eq!(locations.len(), 1);
|
||||||
let location = locations.first().unwrap();
|
let location = locations.first().unwrap();
|
||||||
assert_eq!(location.name, Some("Location 0".to_string()));
|
assert_eq!(location.name.as_deref(), Some(MOCK_LOCATION_NAME));
|
||||||
assert_eq!(location.path, Some("/User/Brendan/Documents".to_string()));
|
assert_eq!(location.path.as_deref(), Some(MOCK_LOCATION_PATH));
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[traced_test]
|
||||||
async fn operations_send_and_ingest() -> Result<(), Box<dyn std::error::Error>> {
|
async fn operations_send_and_ingest() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
let instance1 = Instance::new(Uuid::new_v4()).await;
|
let instance1 = Instance::new(Uuid::new_v4()).await;
|
||||||
let instance2 = Instance::new(Uuid::new_v4()).await;
|
let instance2 = Instance::new(Uuid::new_v4()).await;
|
||||||
|
|
||||||
|
let mut instance2_sync_rx = instance2.sync_rx.resubscribe();
|
||||||
|
|
||||||
|
info!("Created instances!");
|
||||||
|
|
||||||
Instance::pair(&instance1, &instance2).await;
|
Instance::pair(&instance1, &instance2).await;
|
||||||
|
|
||||||
write_test_location(&instance1).await?;
|
info!("Paired instances!");
|
||||||
|
|
||||||
|
write_test_location(&instance1).await;
|
||||||
|
|
||||||
|
info!("Created mock location!");
|
||||||
|
|
||||||
assert!(matches!(
|
assert!(matches!(
|
||||||
instance2.sync_rx.resubscribe().recv().await?,
|
instance2_sync_rx.recv().await?,
|
||||||
SyncMessage::Ingested
|
SyncMessage::Ingested
|
||||||
));
|
));
|
||||||
|
|
||||||
|
@ -89,6 +147,11 @@ async fn operations_send_and_ingest() -> Result<(), Box<dyn std::error::Error>>
|
||||||
})
|
})
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
|
assert_locations_equality(
|
||||||
|
&instance1.db.location().find_many(vec![]).exec().await?[0],
|
||||||
|
&instance2.db.location().find_many(vec![]).exec().await?[0],
|
||||||
|
);
|
||||||
|
|
||||||
assert_eq!(out.len(), 3);
|
assert_eq!(out.len(), 3);
|
||||||
|
|
||||||
instance1.teardown().await;
|
instance1.teardown().await;
|
||||||
|
@ -102,12 +165,14 @@ async fn no_update_after_delete() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
let instance1 = Instance::new(Uuid::new_v4()).await;
|
let instance1 = Instance::new(Uuid::new_v4()).await;
|
||||||
let instance2 = Instance::new(Uuid::new_v4()).await;
|
let instance2 = Instance::new(Uuid::new_v4()).await;
|
||||||
|
|
||||||
|
let mut instance2_sync_rx = instance2.sync_rx.resubscribe();
|
||||||
|
|
||||||
Instance::pair(&instance1, &instance2).await;
|
Instance::pair(&instance1, &instance2).await;
|
||||||
|
|
||||||
let location = write_test_location(&instance1).await?;
|
let location = write_test_location(&instance1).await;
|
||||||
|
|
||||||
assert!(matches!(
|
assert!(matches!(
|
||||||
instance2.sync_rx.resubscribe().recv().await?,
|
instance2_sync_rx.recv().await?,
|
||||||
SyncMessage::Ingested
|
SyncMessage::Ingested
|
||||||
));
|
));
|
||||||
|
|
||||||
|
@ -140,8 +205,7 @@ async fn no_update_after_delete() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
),
|
),
|
||||||
instance1.db.location().find_many(vec![]),
|
instance1.db.location().find_many(vec![]),
|
||||||
)
|
)
|
||||||
.await
|
.await?;
|
||||||
.ok();
|
|
||||||
|
|
||||||
// one spare update operation that actually gets ignored by instance 2
|
// one spare update operation that actually gets ignored by instance 2
|
||||||
assert_eq!(instance1.db.crdt_operation().count(vec![]).exec().await?, 5);
|
assert_eq!(instance1.db.crdt_operation().count(vec![]).exec().await?, 5);
|
||||||
|
@ -156,3 +220,28 @@ async fn no_update_after_delete() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn assert_locations_equality(l1: &location::Data, l2: &location::Data) {
|
||||||
|
assert_eq!(l1.pub_id, l2.pub_id, "pub id");
|
||||||
|
assert_eq!(l1.name, l2.name, "name");
|
||||||
|
assert_eq!(l1.path, l2.path, "path");
|
||||||
|
assert_eq!(l1.total_capacity, l2.total_capacity, "total capacity");
|
||||||
|
assert_eq!(
|
||||||
|
l1.available_capacity, l2.available_capacity,
|
||||||
|
"available capacity"
|
||||||
|
);
|
||||||
|
assert_eq!(l1.size_in_bytes, l2.size_in_bytes, "size in bytes");
|
||||||
|
assert_eq!(l1.is_archived, l2.is_archived, "is archived");
|
||||||
|
assert_eq!(
|
||||||
|
l1.generate_preview_media, l2.generate_preview_media,
|
||||||
|
"generate preview media"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
l1.sync_preview_media, l2.sync_preview_media,
|
||||||
|
"sync preview media"
|
||||||
|
);
|
||||||
|
assert_eq!(l1.hidden, l2.hidden, "hidden");
|
||||||
|
assert_eq!(l1.date_created, l2.date_created, "date created");
|
||||||
|
assert_eq!(l1.scan_state, l2.scan_state, "scan state");
|
||||||
|
assert_eq!(l1.instance_id, l2.instance_id, "instance id");
|
||||||
|
}
|
||||||
|
|
|
@ -1,11 +1,14 @@
|
||||||
use sd_core_sync::*;
|
use sd_core_sync::*;
|
||||||
use sd_prisma::prisma::{self};
|
|
||||||
|
use sd_prisma::prisma;
|
||||||
use sd_sync::CompressedCRDTOperations;
|
use sd_sync::CompressedCRDTOperations;
|
||||||
use sd_utils::uuid_to_bytes;
|
use sd_utils::uuid_to_bytes;
|
||||||
|
|
||||||
use prisma_client_rust::chrono::Utc;
|
|
||||||
use std::sync::{atomic::AtomicBool, Arc};
|
use std::sync::{atomic::AtomicBool, Arc};
|
||||||
use tokio::sync::broadcast;
|
|
||||||
|
use prisma_client_rust::chrono::Utc;
|
||||||
|
use tokio::{fs, spawn, sync::broadcast};
|
||||||
|
use tracing::{info, instrument, warn, Instrument};
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
fn db_path(id: Uuid) -> String {
|
fn db_path(id: Uuid) -> String {
|
||||||
|
@ -47,29 +50,30 @@ impl Instance {
|
||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
let sync = sd_core_sync::Manager::new(
|
let (sync, sync_rx) = sd_core_sync::Manager::new(
|
||||||
&db,
|
Arc::clone(&db),
|
||||||
id,
|
id,
|
||||||
&Arc::new(AtomicBool::new(true)),
|
Arc::new(AtomicBool::new(true)),
|
||||||
Default::default(),
|
Default::default(),
|
||||||
&Default::default(),
|
|
||||||
)
|
)
|
||||||
.await;
|
.await
|
||||||
|
.expect("failed to create sync manager");
|
||||||
|
|
||||||
Arc::new(Self {
|
Arc::new(Self {
|
||||||
id,
|
id,
|
||||||
db,
|
db,
|
||||||
sync: Arc::new(sync.manager),
|
sync: Arc::new(sync),
|
||||||
sync_rx: Arc::new(sync.rx),
|
sync_rx: Arc::new(sync_rx),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn teardown(&self) {
|
pub async fn teardown(&self) {
|
||||||
tokio::fs::remove_file(db_path(self.id)).await.unwrap();
|
fs::remove_file(db_path(self.id)).await.unwrap();
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn pair(left: &Arc<Self>, right: &Arc<Self>) {
|
pub async fn pair(instance1: &Arc<Self>, instance2: &Arc<Self>) {
|
||||||
async fn half(left: &Arc<Instance>, right: &Arc<Instance>) {
|
#[instrument(skip(left, right))]
|
||||||
|
async fn half(left: &Arc<Instance>, right: &Arc<Instance>, context: &'static str) {
|
||||||
left.db
|
left.db
|
||||||
.instance()
|
.instance()
|
||||||
.create(
|
.create(
|
||||||
|
@ -84,34 +88,38 @@ impl Instance {
|
||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
tokio::spawn({
|
spawn({
|
||||||
let mut sync_rx_1 = left.sync_rx.resubscribe();
|
let mut sync_rx_left = left.sync_rx.resubscribe();
|
||||||
let instance2 = right.clone();
|
let right = Arc::clone(right);
|
||||||
|
|
||||||
async move {
|
async move {
|
||||||
while let Ok(msg) = sync_rx_1.recv().await {
|
while let Ok(msg) = sync_rx_left.recv().await {
|
||||||
|
info!(?msg, "sync_rx_left received message");
|
||||||
if matches!(msg, SyncMessage::Created) {
|
if matches!(msg, SyncMessage::Created) {
|
||||||
instance2
|
right
|
||||||
.sync
|
.sync
|
||||||
.ingest
|
.ingest
|
||||||
.event_tx
|
.event_tx
|
||||||
.send(ingest::Event::Notification)
|
.send(ingest::Event::Notification)
|
||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
info!("sent notification to instance 2");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
.in_current_span()
|
||||||
});
|
});
|
||||||
|
|
||||||
tokio::spawn({
|
spawn({
|
||||||
let instance1 = left.clone();
|
let left = Arc::clone(left);
|
||||||
let instance2 = right.clone();
|
let right = Arc::clone(right);
|
||||||
|
|
||||||
async move {
|
async move {
|
||||||
while let Some(msg) = instance2.sync.ingest.req_rx.lock().await.recv().await {
|
while let Ok(msg) = right.sync.ingest.req_rx.recv().await {
|
||||||
|
info!(?msg, "right instance received request");
|
||||||
match msg {
|
match msg {
|
||||||
ingest::Request::Messages { timestamps, .. } => {
|
ingest::Request::Messages { timestamps, tx } => {
|
||||||
let messages = instance1
|
let messages = left
|
||||||
.sync
|
.sync
|
||||||
.get_ops(GetOpsArgs {
|
.get_ops(GetOpsArgs {
|
||||||
clocks: timestamps,
|
clocks: timestamps,
|
||||||
|
@ -120,30 +128,34 @@ impl Instance {
|
||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
let ingest = &instance2.sync.ingest;
|
let ingest = &right.sync.ingest;
|
||||||
|
|
||||||
ingest
|
ingest
|
||||||
.event_tx
|
.event_tx
|
||||||
.send(ingest::Event::Messages(ingest::MessagesEvent {
|
.send(ingest::Event::Messages(ingest::MessagesEvent {
|
||||||
messages: CompressedCRDTOperations::new(messages),
|
messages: CompressedCRDTOperations::new(messages),
|
||||||
has_more: false,
|
has_more: false,
|
||||||
instance_id: instance1.id,
|
instance_id: left.id,
|
||||||
wait_tx: None,
|
wait_tx: None,
|
||||||
}))
|
}))
|
||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
|
if tx.send(()).is_err() {
|
||||||
|
warn!("failed to send ack to instance 1");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ingest::Request::FinishedIngesting => {
|
||||||
|
right.sync.tx.send(SyncMessage::Ingested).unwrap();
|
||||||
}
|
}
|
||||||
// ingest::Request::Ingested => {
|
|
||||||
// instance2.sync.tx.send(SyncMessage::Ingested).ok();
|
|
||||||
// }
|
|
||||||
ingest::Request::FinishedIngesting => {}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
.in_current_span()
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
half(left, right).await;
|
half(instance1, instance2, "instance1 -> instance2").await;
|
||||||
half(right, left).await;
|
half(instance2, instance1, "instance2 -> instance1").await;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -28,6 +28,7 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
||||||
pub p2p_discovery: Option<P2PDiscoveryState>,
|
pub p2p_discovery: Option<P2PDiscoveryState>,
|
||||||
pub p2p_remote_access: Option<bool>,
|
pub p2p_remote_access: Option<bool>,
|
||||||
pub p2p_manual_peers: Option<HashSet<String>>,
|
pub p2p_manual_peers: Option<HashSet<String>>,
|
||||||
|
#[cfg(feature = "ai")]
|
||||||
pub image_labeler_version: Option<String>,
|
pub image_labeler_version: Option<String>,
|
||||||
}
|
}
|
||||||
R.mutation(|node, args: ChangeNodeNameArgs| async move {
|
R.mutation(|node, args: ChangeNodeNameArgs| async move {
|
||||||
|
@ -115,30 +116,31 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
||||||
if let Some(model) = new_model {
|
if let Some(model) = new_model {
|
||||||
let version = model.version().to_string();
|
let version = model.version().to_string();
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
let notification =
|
let notification = if let Some(image_labeller) =
|
||||||
if let Some(image_labeller) = node.old_image_labeller.as_ref() {
|
node.old_image_labeller.as_ref()
|
||||||
if let Err(e) = image_labeller.change_model(model).await {
|
{
|
||||||
NotificationData {
|
if let Err(e) = image_labeller.change_model(model).await {
|
||||||
title: String::from(
|
NotificationData {
|
||||||
"Failed to change image detection model",
|
title: String::from(
|
||||||
),
|
"Failed to change image detection model",
|
||||||
content: format!("Error: {e}"),
|
),
|
||||||
kind: NotificationKind::Error,
|
content: format!("Error: {e}"),
|
||||||
}
|
kind: NotificationKind::Error,
|
||||||
} else {
|
|
||||||
NotificationData {
|
|
||||||
title: String::from("Model download completed"),
|
|
||||||
content: format!("Sucessfuly loaded model: {version}"),
|
|
||||||
kind: NotificationKind::Success,
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
NotificationData {
|
NotificationData {
|
||||||
|
title: String::from("Model download completed"),
|
||||||
|
content: format!("Successfully loaded model: {version}"),
|
||||||
|
kind: NotificationKind::Success,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
NotificationData {
|
||||||
title: String::from("Failed to change image detection model"),
|
title: String::from("Failed to change image detection model"),
|
||||||
content: "The AI system is disabled due to a previous error. Contact support for help.".to_string(),
|
content: "The AI system is disabled due to a previous error. Contact support for help.".to_string(),
|
||||||
kind: NotificationKind::Success,
|
kind: NotificationKind::Success,
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
node.emit_notification(notification, None).await;
|
node.emit_notification(notification, None).await;
|
||||||
});
|
});
|
||||||
|
@ -184,7 +186,7 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
||||||
.procedure("updateThumbnailerPreferences", {
|
.procedure("updateThumbnailerPreferences", {
|
||||||
#[derive(Deserialize, Type)]
|
#[derive(Deserialize, Type)]
|
||||||
pub struct UpdateThumbnailerPreferences {
|
pub struct UpdateThumbnailerPreferences {
|
||||||
pub background_processing_percentage: u8, // 0-100
|
// pub background_processing_percentage: u8, // 0-100
|
||||||
}
|
}
|
||||||
R.mutation(
|
R.mutation(
|
||||||
|node, UpdateThumbnailerPreferences { .. }: UpdateThumbnailerPreferences| async move {
|
|node, UpdateThumbnailerPreferences { .. }: UpdateThumbnailerPreferences| async move {
|
||||||
|
|
|
@ -152,7 +152,7 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
||||||
|
|
||||||
|(_, library), (id, args): (saved_search::id::Type, Args)| async move {
|
|(_, library), (id, args): (saved_search::id::Type, Args)| async move {
|
||||||
let Library { db, sync, .. } = library.as_ref();
|
let Library { db, sync, .. } = library.as_ref();
|
||||||
let updated_at = Utc::now().into();
|
let updated_at = Utc::now();
|
||||||
|
|
||||||
let search = db
|
let search = db
|
||||||
.saved_search()
|
.saved_search()
|
||||||
|
|
|
@ -51,7 +51,7 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
|
||||||
&library.sync,
|
&library.sync,
|
||||||
library.config().await.instance_id,
|
library.config().await.instance_id,
|
||||||
)
|
)
|
||||||
.await;
|
.await?;
|
||||||
|
|
||||||
node.libraries
|
node.libraries
|
||||||
.edit(
|
.edit(
|
||||||
|
|
|
@ -1,14 +1,20 @@
|
||||||
|
use crate::cloud::sync::err_break;
|
||||||
|
|
||||||
use sd_prisma::prisma::cloud_crdt_operation;
|
use sd_prisma::prisma::cloud_crdt_operation;
|
||||||
use sd_sync::CompressedCRDTOperations;
|
use sd_sync::CompressedCRDTOperations;
|
||||||
use std::sync::{
|
|
||||||
atomic::{AtomicBool, Ordering},
|
use std::{
|
||||||
Arc,
|
pin::pin,
|
||||||
|
sync::{
|
||||||
|
atomic::{AtomicBool, Ordering},
|
||||||
|
Arc,
|
||||||
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
|
use futures::StreamExt;
|
||||||
use tokio::sync::Notify;
|
use tokio::sync::Notify;
|
||||||
use tracing::debug;
|
use tracing::debug;
|
||||||
|
|
||||||
use crate::cloud::sync::err_break;
|
|
||||||
|
|
||||||
// Responsible for taking sync operations received from the cloud,
|
// Responsible for taking sync operations received from the cloud,
|
||||||
// and applying them to the local database via the sync system's ingest actor.
|
// and applying them to the local database via the sync system's ingest actor.
|
||||||
|
|
||||||
|
@ -23,7 +29,7 @@ pub async fn run_actor(
|
||||||
state_notify.notify_waiters();
|
state_notify.notify_waiters();
|
||||||
|
|
||||||
{
|
{
|
||||||
let mut rx = sync.ingest.req_rx.lock().await;
|
let mut rx = pin!(sync.ingest.req_rx.clone());
|
||||||
|
|
||||||
if sync
|
if sync
|
||||||
.ingest
|
.ingest
|
||||||
|
@ -32,9 +38,13 @@ pub async fn run_actor(
|
||||||
.await
|
.await
|
||||||
.is_ok()
|
.is_ok()
|
||||||
{
|
{
|
||||||
while let Some(req) = rx.recv().await {
|
while let Some(req) = rx.next().await {
|
||||||
const OPS_PER_REQUEST: u32 = 1000;
|
const OPS_PER_REQUEST: u32 = 1000;
|
||||||
|
|
||||||
|
// FIXME: If there are exactly a multiple of OPS_PER_REQUEST operations,
|
||||||
|
// then this will bug, as we sent `has_more` as true, but we don't have
|
||||||
|
// more operations to send.
|
||||||
|
|
||||||
use sd_core_sync::*;
|
use sd_core_sync::*;
|
||||||
|
|
||||||
let timestamps = match req {
|
let timestamps = match req {
|
||||||
|
|
|
@ -42,7 +42,10 @@ pub async fn declare_actors(
|
||||||
let active = state.send_active.clone();
|
let active = state.send_active.clone();
|
||||||
let active_notifier = state.notifier.clone();
|
let active_notifier = state.notifier.clone();
|
||||||
|
|
||||||
move || send::run_actor(library_id, sync, node, active, active_notifier)
|
move |_stop| {
|
||||||
|
// FIXME: Properly use the stop actor
|
||||||
|
send::run_actor(library_id, sync, node, active, active_notifier)
|
||||||
|
}
|
||||||
},
|
},
|
||||||
autorun,
|
autorun,
|
||||||
)
|
)
|
||||||
|
@ -58,7 +61,8 @@ pub async fn declare_actors(
|
||||||
let active_notifier = state.notifier.clone();
|
let active_notifier = state.notifier.clone();
|
||||||
let active = state.receive_active.clone();
|
let active = state.receive_active.clone();
|
||||||
|
|
||||||
move || {
|
move |_stop| {
|
||||||
|
// FIXME: Properly use the stop actor
|
||||||
receive::run_actor(
|
receive::run_actor(
|
||||||
node.libraries.clone(),
|
node.libraries.clone(),
|
||||||
db.clone(),
|
db.clone(),
|
||||||
|
@ -83,7 +87,10 @@ pub async fn declare_actors(
|
||||||
let active = state.ingest_active.clone();
|
let active = state.ingest_active.clone();
|
||||||
let active_notifier = state.notifier.clone();
|
let active_notifier = state.notifier.clone();
|
||||||
|
|
||||||
move || ingest::run_actor(sync.clone(), ingest_notify, active, active_notifier)
|
move |_stop| {
|
||||||
|
// FIXME: Properly use the stop actor
|
||||||
|
ingest::run_actor(sync.clone(), ingest_notify, active, active_notifier)
|
||||||
|
}
|
||||||
},
|
},
|
||||||
autorun,
|
autorun,
|
||||||
)
|
)
|
||||||
|
|
|
@ -46,6 +46,8 @@ pub enum LibraryManagerError {
|
||||||
FileIO(#[from] FileIOError),
|
FileIO(#[from] FileIOError),
|
||||||
#[error(transparent)]
|
#[error(transparent)]
|
||||||
LibraryConfig(#[from] LibraryConfigError),
|
LibraryConfig(#[from] LibraryConfigError),
|
||||||
|
#[error(transparent)]
|
||||||
|
Sync(#[from] sd_core_sync::Error),
|
||||||
}
|
}
|
||||||
|
|
||||||
impl From<LibraryManagerError> for rspc::Error {
|
impl From<LibraryManagerError> for rspc::Error {
|
||||||
|
|
|
@ -8,14 +8,12 @@ use crate::{
|
||||||
Node,
|
Node,
|
||||||
};
|
};
|
||||||
|
|
||||||
use futures::future::join_all;
|
|
||||||
use sd_core_sync::SyncMessage;
|
use sd_core_sync::SyncMessage;
|
||||||
use sd_p2p::{Identity, RemoteIdentity};
|
use sd_p2p::{Identity, RemoteIdentity};
|
||||||
use sd_prisma::prisma::{crdt_operation, instance, location, SortOrder};
|
use sd_prisma::prisma::{instance, location};
|
||||||
use sd_utils::{
|
use sd_utils::{
|
||||||
db,
|
db,
|
||||||
error::{FileIOError, NonUtf8PathError},
|
error::{FileIOError, NonUtf8PathError},
|
||||||
from_bytes_to_uuid,
|
|
||||||
};
|
};
|
||||||
|
|
||||||
use std::{
|
use std::{
|
||||||
|
@ -32,7 +30,7 @@ use std::{
|
||||||
use chrono::Utc;
|
use chrono::Utc;
|
||||||
use futures_concurrency::future::{Join, TryJoin};
|
use futures_concurrency::future::{Join, TryJoin};
|
||||||
use tokio::{
|
use tokio::{
|
||||||
fs, io,
|
fs, io, spawn,
|
||||||
sync::{broadcast, RwLock},
|
sync::{broadcast, RwLock},
|
||||||
time::sleep,
|
time::sleep,
|
||||||
};
|
};
|
||||||
|
@ -392,31 +390,31 @@ impl Libraries {
|
||||||
&self,
|
&self,
|
||||||
instance: &RemoteIdentity,
|
instance: &RemoteIdentity,
|
||||||
) -> Option<Arc<Library>> {
|
) -> Option<Arc<Library>> {
|
||||||
join_all(
|
self.libraries
|
||||||
self.libraries
|
.read()
|
||||||
.read()
|
.await
|
||||||
.await
|
.iter()
|
||||||
.iter()
|
.map(|(_, library)| async move {
|
||||||
.map(|(_, library)| async move {
|
library
|
||||||
library
|
.db
|
||||||
.db
|
.instance()
|
||||||
.instance()
|
.find_many(vec![instance::remote_identity::equals(
|
||||||
.find_many(vec![instance::remote_identity::equals(
|
instance.get_bytes().to_vec(),
|
||||||
instance.get_bytes().to_vec(),
|
)])
|
||||||
)])
|
.exec()
|
||||||
.exec()
|
.await
|
||||||
.await
|
.ok()
|
||||||
.ok()
|
.iter()
|
||||||
.iter()
|
.flatten()
|
||||||
.flatten()
|
.filter_map(|i| RemoteIdentity::from_bytes(&i.remote_identity).ok())
|
||||||
.filter_map(|i| RemoteIdentity::from_bytes(&i.remote_identity).ok())
|
.any(|i| i == *instance)
|
||||||
.any(|i| i == *instance)
|
.then(|| Arc::clone(library))
|
||||||
.then(|| Arc::clone(library))
|
})
|
||||||
}),
|
.collect::<Vec<_>>()
|
||||||
)
|
.join()
|
||||||
.await
|
.await
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.find_map(|v| v)
|
.find_map(|v| v)
|
||||||
}
|
}
|
||||||
|
|
||||||
// get_ctx will return the library context for the given library id.
|
// get_ctx will return the library context for the given library id.
|
||||||
|
@ -529,38 +527,15 @@ impl Libraries {
|
||||||
|
|
||||||
let actors = Default::default();
|
let actors = Default::default();
|
||||||
|
|
||||||
let sync = sync::Manager::new(
|
let (sync, sync_rx) = sync::Manager::with_existing_instances(
|
||||||
&db,
|
Arc::clone(&db),
|
||||||
instance_id,
|
instance_id,
|
||||||
&config.generate_sync_operations,
|
Arc::clone(&config.generate_sync_operations),
|
||||||
{
|
&instances,
|
||||||
db._batch(
|
Arc::clone(&actors),
|
||||||
instances
|
|
||||||
.iter()
|
|
||||||
.map(|i| {
|
|
||||||
db.crdt_operation()
|
|
||||||
.find_first(vec![crdt_operation::instance::is(vec![
|
|
||||||
instance::id::equals(i.id),
|
|
||||||
])])
|
|
||||||
.order_by(crdt_operation::timestamp::order(SortOrder::Desc))
|
|
||||||
})
|
|
||||||
.collect::<Vec<_>>(),
|
|
||||||
)
|
|
||||||
.await?
|
|
||||||
.into_iter()
|
|
||||||
.zip(&instances)
|
|
||||||
.map(|(op, i)| {
|
|
||||||
(
|
|
||||||
from_bytes_to_uuid(&i.pub_id),
|
|
||||||
sd_sync::NTP64(op.map(|o| o.timestamp).unwrap_or_default() as u64),
|
|
||||||
)
|
|
||||||
})
|
|
||||||
.collect()
|
|
||||||
},
|
|
||||||
&actors,
|
|
||||||
)
|
)
|
||||||
.await;
|
.await?;
|
||||||
let sync_manager = Arc::new(sync.manager);
|
let sync_manager = Arc::new(sync);
|
||||||
|
|
||||||
let cloud = crate::cloud::start(node, &actors, id, instance_id, &sync_manager, &db).await;
|
let cloud = crate::cloud::start(node, &actors, id, instance_id, &sync_manager, &db).await;
|
||||||
|
|
||||||
|
@ -581,7 +556,7 @@ impl Libraries {
|
||||||
.await;
|
.await;
|
||||||
|
|
||||||
// This is an exception. Generally subscribe to this by `self.tx.subscribe`.
|
// This is an exception. Generally subscribe to this by `self.tx.subscribe`.
|
||||||
tokio::spawn(sync_rx_actor(library.clone(), node.clone(), sync.rx));
|
spawn(sync_rx_actor(library.clone(), node.clone(), sync_rx));
|
||||||
|
|
||||||
self.tx
|
self.tx
|
||||||
.emit(LibraryManagerEvent::Load(library.clone()))
|
.emit(LibraryManagerEvent::Load(library.clone()))
|
||||||
|
@ -616,7 +591,7 @@ impl Libraries {
|
||||||
error!(?e, "Failed to resume jobs for library;");
|
error!(?e, "Failed to resume jobs for library;");
|
||||||
}
|
}
|
||||||
|
|
||||||
tokio::spawn({
|
spawn({
|
||||||
let this = self.clone();
|
let this = self.clone();
|
||||||
let node = node.clone();
|
let node = node.clone();
|
||||||
let library = library.clone();
|
let library = library.clone();
|
||||||
|
|
|
@ -78,6 +78,8 @@ pub enum LocationError {
|
||||||
MissingField(#[from] MissingFieldError),
|
MissingField(#[from] MissingFieldError),
|
||||||
#[error("invalid location scan state value: {0}")]
|
#[error("invalid location scan state value: {0}")]
|
||||||
InvalidScanStateValue(i32),
|
InvalidScanStateValue(i32),
|
||||||
|
#[error(transparent)]
|
||||||
|
Sync(#[from] sd_core_sync::Error),
|
||||||
}
|
}
|
||||||
|
|
||||||
impl From<LocationError> for rspc::Error {
|
impl From<LocationError> for rspc::Error {
|
||||||
|
|
|
@ -94,6 +94,8 @@ pub enum LocationManagerError {
|
||||||
JobSystem(#[from] sd_core_heavy_lifting::Error),
|
JobSystem(#[from] sd_core_heavy_lifting::Error),
|
||||||
#[error(transparent)]
|
#[error(transparent)]
|
||||||
FileIO(#[from] FileIOError),
|
FileIO(#[from] FileIOError),
|
||||||
|
#[error(transparent)]
|
||||||
|
Sync(#[from] sd_core_sync::Error),
|
||||||
}
|
}
|
||||||
|
|
||||||
type OnlineLocations = BTreeSet<Vec<u8>>;
|
type OnlineLocations = BTreeSet<Vec<u8>>;
|
||||||
|
|
|
@ -3,7 +3,7 @@ use crate::library::Library;
|
||||||
use sd_prisma::{prisma::tag, prisma_sync};
|
use sd_prisma::{prisma::tag, prisma_sync};
|
||||||
use sd_sync::*;
|
use sd_sync::*;
|
||||||
|
|
||||||
use chrono::{DateTime, FixedOffset, Utc};
|
use chrono::Utc;
|
||||||
use serde::Deserialize;
|
use serde::Deserialize;
|
||||||
use specta::Type;
|
use specta::Type;
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
@ -20,15 +20,14 @@ impl TagCreateArgs {
|
||||||
pub async fn exec(
|
pub async fn exec(
|
||||||
self,
|
self,
|
||||||
Library { db, sync, .. }: &Library,
|
Library { db, sync, .. }: &Library,
|
||||||
) -> prisma_client_rust::Result<tag::Data> {
|
) -> Result<tag::Data, sd_core_sync::Error> {
|
||||||
let pub_id = Uuid::new_v4().as_bytes().to_vec();
|
let pub_id = Uuid::new_v4().as_bytes().to_vec();
|
||||||
let date_created: DateTime<FixedOffset> = Utc::now().into();
|
|
||||||
|
|
||||||
let (sync_params, db_params): (Vec<_>, Vec<_>) = [
|
let (sync_params, db_params): (Vec<_>, Vec<_>) = [
|
||||||
sync_db_entry!(self.name, tag::name),
|
sync_db_entry!(self.name, tag::name),
|
||||||
sync_db_entry!(self.color, tag::color),
|
sync_db_entry!(self.color, tag::color),
|
||||||
sync_db_entry!(false, tag::is_hidden),
|
sync_db_entry!(false, tag::is_hidden),
|
||||||
sync_db_entry!(date_created, tag::date_created),
|
sync_db_entry!(Utc::now(), tag::date_created),
|
||||||
]
|
]
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.unzip();
|
.unzip();
|
||||||
|
|
|
@ -4,7 +4,7 @@ use super::TagCreateArgs;
|
||||||
|
|
||||||
/// Seeds tags in a new library.
|
/// Seeds tags in a new library.
|
||||||
/// Shouldn't be called more than once!
|
/// Shouldn't be called more than once!
|
||||||
pub async fn new_library(library: &Library) -> prisma_client_rust::Result<()> {
|
pub async fn new_library(library: &Library) -> Result<(), sd_core_sync::Error> {
|
||||||
// remove type after tags are added
|
// remove type after tags are added
|
||||||
|
|
||||||
let tags = [
|
let tags = [
|
||||||
|
|
|
@ -54,6 +54,8 @@ pub enum JobError {
|
||||||
Timeout(Duration),
|
Timeout(Duration),
|
||||||
#[error("critical job error: {0}")]
|
#[error("critical job error: {0}")]
|
||||||
Critical(&'static str),
|
Critical(&'static str),
|
||||||
|
#[error(transparent)]
|
||||||
|
Sync(#[from] sd_core_sync::Error),
|
||||||
|
|
||||||
// Specific job errors
|
// Specific job errors
|
||||||
#[error(transparent)]
|
#[error(transparent)]
|
||||||
|
|
|
@ -135,7 +135,10 @@ mod originator {
|
||||||
|
|
||||||
pub use responder::run as responder;
|
pub use responder::run as responder;
|
||||||
mod responder {
|
mod responder {
|
||||||
|
use std::pin::pin;
|
||||||
|
|
||||||
use super::*;
|
use super::*;
|
||||||
|
use futures::StreamExt;
|
||||||
use originator::tx as rx;
|
use originator::tx as rx;
|
||||||
|
|
||||||
pub mod tx {
|
pub mod tx {
|
||||||
|
@ -196,30 +199,15 @@ mod responder {
|
||||||
stream: &mut (impl AsyncRead + AsyncWrite + Unpin),
|
stream: &mut (impl AsyncRead + AsyncWrite + Unpin),
|
||||||
library: Arc<Library>,
|
library: Arc<Library>,
|
||||||
) -> Result<(), ()> {
|
) -> Result<(), ()> {
|
||||||
let ingest = &library.sync.ingest;
|
|
||||||
|
|
||||||
async fn early_return(stream: &mut (impl AsyncRead + AsyncWrite + Unpin)) {
|
|
||||||
// TODO: Proper error returned to remote instead of this.
|
|
||||||
// TODO: We can't just abort the connection when the remote is expecting data.
|
|
||||||
stream
|
|
||||||
.write_all(&tx::MainRequest::Done.to_bytes())
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
stream.flush().await.unwrap();
|
|
||||||
}
|
|
||||||
|
|
||||||
let Ok(mut rx) = ingest.req_rx.try_lock() else {
|
|
||||||
warn!("Rejected sync due to libraries lock being held!");
|
|
||||||
|
|
||||||
early_return(stream).await;
|
|
||||||
return Ok(());
|
|
||||||
};
|
|
||||||
|
|
||||||
use sync::ingest::*;
|
use sync::ingest::*;
|
||||||
|
|
||||||
|
let ingest = &library.sync.ingest;
|
||||||
|
|
||||||
ingest.event_tx.send(Event::Notification).await.unwrap();
|
ingest.event_tx.send(Event::Notification).await.unwrap();
|
||||||
|
|
||||||
while let Some(req) = rx.recv().await {
|
let mut rx = pin!(ingest.req_rx.clone());
|
||||||
|
|
||||||
|
while let Some(req) = rx.next().await {
|
||||||
const OPS_PER_REQUEST: u32 = 1000;
|
const OPS_PER_REQUEST: u32 = 1000;
|
||||||
|
|
||||||
let timestamps = match req {
|
let timestamps = match req {
|
||||||
|
@ -245,6 +233,10 @@ mod responder {
|
||||||
|
|
||||||
let (wait_tx, wait_rx) = tokio::sync::oneshot::channel::<()>();
|
let (wait_tx, wait_rx) = tokio::sync::oneshot::channel::<()>();
|
||||||
|
|
||||||
|
// FIXME: If there are exactly a multiple of OPS_PER_REQUEST operations,
|
||||||
|
// then this will bug, as we sent `has_more` as true, but we don't have
|
||||||
|
// more operations to send.
|
||||||
|
|
||||||
ingest
|
ingest
|
||||||
.event_tx
|
.event_tx
|
||||||
.send(Event::Messages(MessagesEvent {
|
.send(Event::Messages(MessagesEvent {
|
||||||
|
|
|
@ -6,5 +6,8 @@ edition.workspace = true
|
||||||
repository.workspace = true
|
repository.workspace = true
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
|
async-channel = { workspace = true }
|
||||||
futures = { workspace = true }
|
futures = { workspace = true }
|
||||||
|
pin-project-lite = { workspace = true }
|
||||||
tokio = { workspace = true }
|
tokio = { workspace = true }
|
||||||
|
tracing = { workspace = true }
|
||||||
|
|
|
@ -1,34 +1,93 @@
|
||||||
use futures::Future;
|
#![warn(
|
||||||
use std::{collections::HashMap, pin::Pin, sync::Arc};
|
clippy::all,
|
||||||
use tokio::{
|
clippy::pedantic,
|
||||||
sync::{broadcast, oneshot, Mutex},
|
clippy::correctness,
|
||||||
task::AbortHandle,
|
clippy::perf,
|
||||||
|
clippy::style,
|
||||||
|
clippy::suspicious,
|
||||||
|
clippy::complexity,
|
||||||
|
clippy::nursery,
|
||||||
|
clippy::unwrap_used,
|
||||||
|
unused_qualifications,
|
||||||
|
rust_2018_idioms,
|
||||||
|
trivial_casts,
|
||||||
|
trivial_numeric_casts,
|
||||||
|
unused_allocation,
|
||||||
|
clippy::unnecessary_cast,
|
||||||
|
clippy::cast_lossless,
|
||||||
|
clippy::cast_possible_truncation,
|
||||||
|
clippy::cast_possible_wrap,
|
||||||
|
clippy::cast_precision_loss,
|
||||||
|
clippy::cast_sign_loss,
|
||||||
|
clippy::dbg_macro,
|
||||||
|
clippy::deprecated_cfg_attr,
|
||||||
|
clippy::separated_literal_suffix,
|
||||||
|
deprecated
|
||||||
|
)]
|
||||||
|
#![forbid(deprecated_in_future)]
|
||||||
|
#![allow(clippy::missing_errors_doc, clippy::module_name_repetitions)]
|
||||||
|
|
||||||
|
use std::{
|
||||||
|
collections::HashMap,
|
||||||
|
future::{Future, IntoFuture},
|
||||||
|
panic::{panic_any, AssertUnwindSafe},
|
||||||
|
pin::Pin,
|
||||||
|
sync::{
|
||||||
|
atomic::{AtomicBool, Ordering},
|
||||||
|
Arc,
|
||||||
|
},
|
||||||
|
task::{Context, Poll},
|
||||||
|
time::Duration,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
use async_channel as chan;
|
||||||
|
use futures::FutureExt;
|
||||||
|
use tokio::{
|
||||||
|
spawn,
|
||||||
|
sync::{broadcast, RwLock},
|
||||||
|
task::JoinHandle,
|
||||||
|
time::timeout,
|
||||||
|
};
|
||||||
|
use tracing::{error, instrument, warn};
|
||||||
|
|
||||||
|
const ONE_MINUTE: Duration = Duration::from_secs(60);
|
||||||
|
|
||||||
|
type ActorFn = dyn Fn(Stopper) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync;
|
||||||
|
|
||||||
pub struct Actor {
|
pub struct Actor {
|
||||||
pub abort_handle: Mutex<Option<AbortHandle>>,
|
spawn_fn: Arc<ActorFn>,
|
||||||
pub spawn_fn: Arc<dyn Fn() -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync>,
|
maybe_handle: Option<JoinHandle<()>>,
|
||||||
|
is_running: Arc<AtomicBool>,
|
||||||
|
stop_tx: chan::Sender<()>,
|
||||||
|
stop_rx: chan::Receiver<()>,
|
||||||
}
|
}
|
||||||
|
|
||||||
pub struct Actors {
|
pub struct Actors {
|
||||||
pub invalidate_rx: broadcast::Receiver<()>,
|
pub invalidate_rx: broadcast::Receiver<()>,
|
||||||
invalidate_tx: broadcast::Sender<()>,
|
invalidate_tx: broadcast::Sender<()>,
|
||||||
actors: Arc<Mutex<HashMap<String, Arc<Actor>>>>,
|
actors: Arc<RwLock<HashMap<&'static str, Actor>>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Actors {
|
impl Actors {
|
||||||
pub async fn declare<F: Future<Output = ()> + Send + 'static>(
|
pub async fn declare<Fut>(
|
||||||
self: &Arc<Self>,
|
self: &Arc<Self>,
|
||||||
name: &str,
|
name: &'static str,
|
||||||
actor_fn: impl FnOnce() -> F + Send + Sync + Clone + 'static,
|
actor_fn: impl FnOnce(Stopper) -> Fut + Send + Sync + Clone + 'static,
|
||||||
autostart: bool,
|
autostart: bool,
|
||||||
) {
|
) where
|
||||||
self.actors.lock().await.insert(
|
Fut: Future<Output = ()> + Send + 'static,
|
||||||
name.to_string(),
|
{
|
||||||
Arc::new(Actor {
|
let (stop_tx, stop_rx) = chan::bounded(1);
|
||||||
abort_handle: Default::default(),
|
|
||||||
spawn_fn: Arc::new(move || Box::pin((actor_fn.clone())()) as Pin<Box<_>>),
|
self.actors.write().await.insert(
|
||||||
}),
|
name,
|
||||||
|
Actor {
|
||||||
|
spawn_fn: Arc::new(move |stop| Box::pin((actor_fn.clone())(stop))),
|
||||||
|
maybe_handle: None,
|
||||||
|
is_running: Arc::new(AtomicBool::new(false)),
|
||||||
|
stop_tx,
|
||||||
|
stop_rx,
|
||||||
|
},
|
||||||
);
|
);
|
||||||
|
|
||||||
if autostart {
|
if autostart {
|
||||||
|
@ -36,86 +95,162 @@ impl Actors {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[instrument(skip(self))]
|
||||||
pub async fn start(self: &Arc<Self>, name: &str) {
|
pub async fn start(self: &Arc<Self>, name: &str) {
|
||||||
let name = name.to_string();
|
if let Some(actor) = self.actors.write().await.get_mut(name) {
|
||||||
let actors = self.actors.lock().await;
|
if actor.is_running.load(Ordering::Acquire) {
|
||||||
|
warn!("Actor already running!");
|
||||||
let Some(actor) = actors.get(&name).cloned() else {
|
return;
|
||||||
return;
|
|
||||||
};
|
|
||||||
|
|
||||||
let mut abort_handle = actor.abort_handle.lock().await;
|
|
||||||
if abort_handle.is_some() {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
let (tx, rx) = oneshot::channel();
|
|
||||||
|
|
||||||
let invalidate_tx = self.invalidate_tx.clone();
|
|
||||||
|
|
||||||
let spawn_fn = actor.spawn_fn.clone();
|
|
||||||
|
|
||||||
let task = tokio::spawn(async move {
|
|
||||||
(spawn_fn)().await;
|
|
||||||
|
|
||||||
tx.send(()).ok();
|
|
||||||
});
|
|
||||||
|
|
||||||
*abort_handle = Some(task.abort_handle());
|
|
||||||
invalidate_tx.send(()).ok();
|
|
||||||
|
|
||||||
tokio::spawn({
|
|
||||||
let actor = actor.clone();
|
|
||||||
async move {
|
|
||||||
#[allow(clippy::match_single_binding)]
|
|
||||||
match rx.await {
|
|
||||||
_ => {}
|
|
||||||
};
|
|
||||||
|
|
||||||
actor.abort_handle.lock().await.take();
|
|
||||||
invalidate_tx.send(()).ok();
|
|
||||||
}
|
}
|
||||||
});
|
|
||||||
|
let invalidate_tx = self.invalidate_tx.clone();
|
||||||
|
|
||||||
|
let is_running = Arc::clone(&actor.is_running);
|
||||||
|
|
||||||
|
is_running.store(true, Ordering::Release);
|
||||||
|
|
||||||
|
if invalidate_tx.send(()).is_err() {
|
||||||
|
warn!("Failed to send invalidate signal");
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Some(handle) = actor.maybe_handle.take() {
|
||||||
|
if handle.await.is_err() {
|
||||||
|
// This should never happen, as we're trying to catch the panic below with
|
||||||
|
// `catch_unwind`.
|
||||||
|
error!("Actor unexpectedly panicked");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
actor.maybe_handle = Some(spawn({
|
||||||
|
let spawn_fn = Arc::clone(&actor.spawn_fn);
|
||||||
|
|
||||||
|
let stop_actor = Stopper(actor.stop_rx.clone());
|
||||||
|
|
||||||
|
async move {
|
||||||
|
if (AssertUnwindSafe((spawn_fn)(stop_actor)))
|
||||||
|
.catch_unwind()
|
||||||
|
.await
|
||||||
|
.is_err()
|
||||||
|
{
|
||||||
|
error!("Actor unexpectedly panicked");
|
||||||
|
}
|
||||||
|
|
||||||
|
is_running.store(false, Ordering::Release);
|
||||||
|
|
||||||
|
if invalidate_tx.send(()).is_err() {
|
||||||
|
warn!("Failed to send invalidate signal");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[instrument(skip(self))]
|
||||||
pub async fn stop(self: &Arc<Self>, name: &str) {
|
pub async fn stop(self: &Arc<Self>, name: &str) {
|
||||||
let name = name.to_string();
|
if let Some(actor) = self.actors.write().await.get_mut(name) {
|
||||||
let actors = self.actors.lock().await;
|
if !actor.is_running.load(Ordering::Acquire) {
|
||||||
|
warn!("Actor already stopped!");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
let Some(actor) = actors.get(&name).cloned() else {
|
if actor.stop_tx.send(()).await.is_ok() {
|
||||||
return;
|
wait_stop_or_abort(actor.maybe_handle.take()).await;
|
||||||
};
|
|
||||||
|
|
||||||
let mut abort_handle = actor.abort_handle.lock().await;
|
assert!(
|
||||||
|
!actor.is_running.load(Ordering::Acquire),
|
||||||
if let Some(abort_handle) = abort_handle.take() {
|
"actor handle finished without setting actor to stopped"
|
||||||
abort_handle.abort();
|
);
|
||||||
|
} else {
|
||||||
|
error!("Failed to send stop signal to actor, will check if it's already stopped or abort otherwise");
|
||||||
|
wait_stop_or_abort(actor.maybe_handle.take()).await;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn get_state(&self) -> HashMap<String, bool> {
|
pub async fn get_state(&self) -> HashMap<String, bool> {
|
||||||
let actors = self.actors.lock().await;
|
self.actors
|
||||||
|
.read()
|
||||||
let mut state = HashMap::new();
|
.await
|
||||||
|
.iter()
|
||||||
for (name, actor) in &*actors {
|
.map(|(&name, actor)| (name.to_string(), actor.is_running.load(Ordering::Relaxed)))
|
||||||
state.insert(name.to_string(), actor.abort_handle.lock().await.is_some());
|
.collect()
|
||||||
}
|
|
||||||
|
|
||||||
state
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Default for Actors {
|
impl Default for Actors {
|
||||||
fn default() -> Self {
|
fn default() -> Self {
|
||||||
let actors = Default::default();
|
|
||||||
|
|
||||||
let (invalidate_tx, invalidate_rx) = broadcast::channel(1);
|
let (invalidate_tx, invalidate_rx) = broadcast::channel(1);
|
||||||
|
|
||||||
Self {
|
Self {
|
||||||
actors,
|
actors: Arc::default(),
|
||||||
invalidate_rx,
|
invalidate_rx,
|
||||||
invalidate_tx,
|
invalidate_tx,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub struct Stopper(chan::Receiver<()>);
|
||||||
|
|
||||||
|
impl Stopper {
|
||||||
|
#[must_use]
|
||||||
|
pub fn check_stop(&self) -> bool {
|
||||||
|
self.0.try_recv().is_ok()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pin_project_lite::pin_project! {
|
||||||
|
pub struct StopActorFuture<'recv> {
|
||||||
|
#[pin]
|
||||||
|
fut: chan::Recv<'recv, ()>,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Future for StopActorFuture<'_> {
|
||||||
|
type Output = ();
|
||||||
|
|
||||||
|
fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
|
||||||
|
let this = self.project();
|
||||||
|
|
||||||
|
match this.fut.poll(cx) {
|
||||||
|
Poll::Ready(res) => {
|
||||||
|
if res.is_err() {
|
||||||
|
warn!("StopActor channel closed, will stop actor");
|
||||||
|
}
|
||||||
|
Poll::Ready(())
|
||||||
|
}
|
||||||
|
Poll::Pending => Poll::Pending,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<'recv> IntoFuture for &'recv Stopper {
|
||||||
|
type Output = ();
|
||||||
|
type IntoFuture = StopActorFuture<'recv>;
|
||||||
|
|
||||||
|
fn into_future(self) -> Self::IntoFuture {
|
||||||
|
Self::IntoFuture { fut: self.0.recv() }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn wait_stop_or_abort(maybe_handle: Option<JoinHandle<()>>) {
|
||||||
|
if let Some(handle) = maybe_handle {
|
||||||
|
let abort_handle = handle.abort_handle();
|
||||||
|
|
||||||
|
match timeout(ONE_MINUTE, handle).await {
|
||||||
|
Ok(Ok(())) => { /* Everything is Awesome! */ }
|
||||||
|
Ok(Err(e)) => {
|
||||||
|
// This should never happen, as we're trying to catch the panic with
|
||||||
|
// `catch_unwind`.
|
||||||
|
if e.is_panic() {
|
||||||
|
let p = e.into_panic();
|
||||||
|
error!("Actor unexpectedly panicked, we will pop up the panic!");
|
||||||
|
panic_any(p);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(_) => {
|
||||||
|
error!("Actor failed to gracefully stop in the allotted time, will force abortion");
|
||||||
|
abort_handle.abort();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
@ -51,4 +51,6 @@ pub enum ImageLabelerError {
|
||||||
DownloadModel(#[from] DownloadModelError),
|
DownloadModel(#[from] DownloadModelError),
|
||||||
#[error(transparent)]
|
#[error(transparent)]
|
||||||
FileIO(#[from] FileIOError),
|
FileIO(#[from] FileIOError),
|
||||||
|
#[error(transparent)]
|
||||||
|
Sync(#[from] sd_core_sync::Error),
|
||||||
}
|
}
|
||||||
|
|
|
@ -30,7 +30,7 @@ pub const HEIF_EXTENSIONS: [&str; 8] = [
|
||||||
/// This is the target pixel count for all SVG images to be rendered at.
|
/// This is the target pixel count for all SVG images to be rendered at.
|
||||||
///
|
///
|
||||||
/// It is 512x512, but if the SVG has a non-1:1 aspect ratio we need to account for that.
|
/// It is 512x512, but if the SVG has a non-1:1 aspect ratio we need to account for that.
|
||||||
pub const SVG_TARGET_PX: f32 = 262_144_f32;
|
pub const SVG_TARGET_PX: f32 = 262_144f32;
|
||||||
|
|
||||||
/// The size that PDF pages are rendered at.
|
/// The size that PDF pages are rendered at.
|
||||||
///
|
///
|
||||||
|
|
|
@ -10,14 +10,21 @@
|
||||||
clippy::unwrap_used,
|
clippy::unwrap_used,
|
||||||
unused_qualifications,
|
unused_qualifications,
|
||||||
rust_2018_idioms,
|
rust_2018_idioms,
|
||||||
clippy::expect_used,
|
|
||||||
trivial_casts,
|
trivial_casts,
|
||||||
trivial_numeric_casts,
|
trivial_numeric_casts,
|
||||||
unused_allocation,
|
unused_allocation,
|
||||||
clippy::as_conversions,
|
clippy::unnecessary_cast,
|
||||||
clippy::dbg_macro
|
clippy::cast_lossless,
|
||||||
|
clippy::cast_possible_truncation,
|
||||||
|
clippy::cast_possible_wrap,
|
||||||
|
clippy::cast_precision_loss,
|
||||||
|
clippy::cast_sign_loss,
|
||||||
|
clippy::dbg_macro,
|
||||||
|
clippy::deprecated_cfg_attr,
|
||||||
|
clippy::separated_literal_suffix,
|
||||||
|
deprecated
|
||||||
)]
|
)]
|
||||||
#![forbid(unsafe_code)]
|
#![forbid(deprecated_in_future)]
|
||||||
#![allow(clippy::missing_errors_doc, clippy::module_name_repetitions)]
|
#![allow(clippy::missing_errors_doc, clippy::module_name_repetitions)]
|
||||||
|
|
||||||
use std::{fs, path::Path};
|
use std::{fs, path::Path};
|
||||||
|
|
|
@ -10,17 +10,19 @@ pub enum AttributeFieldValue<'a> {
|
||||||
|
|
||||||
#[allow(unused)]
|
#[allow(unused)]
|
||||||
impl AttributeFieldValue<'_> {
|
impl AttributeFieldValue<'_> {
|
||||||
pub fn as_single(&self) -> Option<&str> {
|
pub const fn as_single(&self) -> Option<&str> {
|
||||||
match self {
|
if let AttributeFieldValue::Single(field) = self {
|
||||||
AttributeFieldValue::Single(field) => Some(field),
|
Some(field)
|
||||||
_ => None,
|
} else {
|
||||||
|
None
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn as_list(&self) -> Option<&Vec<&str>> {
|
pub const fn as_list(&self) -> Option<&Vec<&str>> {
|
||||||
match self {
|
if let AttributeFieldValue::List(fields) = self {
|
||||||
AttributeFieldValue::List(fields) => Some(fields),
|
Some(fields)
|
||||||
_ => None,
|
} else {
|
||||||
|
None
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -36,12 +38,14 @@ impl<'a> Attribute<'a> {
|
||||||
parser::parse(input).map(|(_, a)| a).map_err(|_| ())
|
parser::parse(input).map(|(_, a)| a).map_err(|_| ())
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn field(&self, name: &str) -> Option<&AttributeFieldValue> {
|
pub fn field(&self, name: &str) -> Option<&AttributeFieldValue<'_>> {
|
||||||
self.fields.iter().find(|(n, _)| *n == name).map(|(_, v)| v)
|
self.fields
|
||||||
|
.iter()
|
||||||
|
.find_map(|(n, v)| (*n == name).then_some(v))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn model_attributes(model: ModelWalker) -> Vec<Attribute> {
|
pub fn model_attributes(model: ModelWalker<'_>) -> Vec<Attribute<'_>> {
|
||||||
model
|
model
|
||||||
.ast_model()
|
.ast_model()
|
||||||
.documentation()
|
.documentation()
|
||||||
|
|
|
@ -1,11 +1,11 @@
|
||||||
use nom::{
|
use nom::{
|
||||||
branch::alt,
|
branch::alt,
|
||||||
bytes::complete::*,
|
bytes::complete::{is_not, tag},
|
||||||
character::complete::*,
|
character::complete::{alpha1, char, multispace0},
|
||||||
combinator::*,
|
combinator::{map, opt},
|
||||||
error::{ErrorKind, ParseError},
|
error::{ErrorKind, ParseError},
|
||||||
multi::*,
|
multi::separated_list1,
|
||||||
sequence::*,
|
sequence::{delimited, separated_pair},
|
||||||
AsChar, IResult, InputTakeAtPosition,
|
AsChar, IResult, InputTakeAtPosition,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -24,7 +24,7 @@ fn parens(input: &str) -> IResult<&str, &str> {
|
||||||
delimited(char('('), is_not(")"), char(')'))(input)
|
delimited(char('('), is_not(")"), char(')'))(input)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn single_value<T, E: ParseError<T>>(i: T) -> IResult<T, T, E>
|
fn single_value<T, E: ParseError<T>>(i: &T) -> IResult<T, T, E>
|
||||||
where
|
where
|
||||||
T: InputTakeAtPosition,
|
T: InputTakeAtPosition,
|
||||||
<T as InputTakeAtPosition>::Item: AsChar,
|
<T as InputTakeAtPosition>::Item: AsChar,
|
||||||
|
@ -41,19 +41,19 @@ where
|
||||||
fn list_value(input: &str) -> IResult<&str, Vec<&str>> {
|
fn list_value(input: &str) -> IResult<&str, Vec<&str>> {
|
||||||
delimited(
|
delimited(
|
||||||
char('['),
|
char('['),
|
||||||
separated_list1(char(','), remove_ws(single_value)),
|
separated_list1(char(','), remove_ws(|a| single_value(&a))),
|
||||||
char(']'),
|
char(']'),
|
||||||
)(input)
|
)(input)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn attribute_field_value(input: &str) -> IResult<&str, AttributeFieldValue> {
|
fn attribute_field_value(input: &str) -> IResult<&str, AttributeFieldValue<'_>> {
|
||||||
remove_ws(alt((
|
remove_ws(alt((
|
||||||
map(list_value, AttributeFieldValue::List),
|
map(|a| list_value(a), AttributeFieldValue::List),
|
||||||
map(single_value, AttributeFieldValue::Single),
|
map(|a| single_value(&a), AttributeFieldValue::Single),
|
||||||
)))(input)
|
)))(input)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn attribute_field(input: &str) -> IResult<&str, (&str, AttributeFieldValue)> {
|
fn attribute_field(input: &str) -> IResult<&str, (&str, AttributeFieldValue<'_>)> {
|
||||||
remove_ws(separated_pair(
|
remove_ws(separated_pair(
|
||||||
remove_ws(is_not(":")),
|
remove_ws(is_not(":")),
|
||||||
char(':'),
|
char(':'),
|
||||||
|
@ -61,11 +61,11 @@ fn attribute_field(input: &str) -> IResult<&str, (&str, AttributeFieldValue)> {
|
||||||
))(input)
|
))(input)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn attribute_fields(input: &str) -> IResult<&str, Vec<(&str, AttributeFieldValue)>> {
|
fn attribute_fields(input: &str) -> IResult<&str, Vec<(&str, AttributeFieldValue<'_>)>> {
|
||||||
separated_list1(char(','), attribute_field)(input)
|
separated_list1(char(','), attribute_field)(input)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn parse(input: &str) -> IResult<&str, Attribute> {
|
pub fn parse(input: &str) -> IResult<&str, Attribute<'_>> {
|
||||||
let (input, _) = remove_ws(tag("@"))(input)?;
|
let (input, _) = remove_ws(tag("@"))(input)?;
|
||||||
let (input, name) = alpha1(input)?;
|
let (input, name) = alpha1(input)?;
|
||||||
let (input, values_str) = opt(remove_ws(parens))(input)?;
|
let (input, values_str) = opt(remove_ws(parens))(input)?;
|
||||||
|
@ -86,7 +86,7 @@ mod test {
|
||||||
fn marker() {
|
fn marker() {
|
||||||
let s = "@local";
|
let s = "@local";
|
||||||
|
|
||||||
let (remaining, attribute) = super::parse(s).unwrap();
|
let (remaining, attribute) = parse(s).unwrap();
|
||||||
|
|
||||||
assert_eq!(remaining, "");
|
assert_eq!(remaining, "");
|
||||||
assert_eq!(attribute.name, "local");
|
assert_eq!(attribute.name, "local");
|
||||||
|
@ -97,7 +97,7 @@ mod test {
|
||||||
fn single() {
|
fn single() {
|
||||||
let s = "@local(foo: bar)";
|
let s = "@local(foo: bar)";
|
||||||
|
|
||||||
let (remaining, attribute) = super::parse(s).unwrap();
|
let (remaining, attribute) = parse(s).unwrap();
|
||||||
|
|
||||||
assert_eq!(remaining, "");
|
assert_eq!(remaining, "");
|
||||||
assert_eq!(attribute.name, "local");
|
assert_eq!(attribute.name, "local");
|
||||||
|
@ -113,7 +113,7 @@ mod test {
|
||||||
fn list() {
|
fn list() {
|
||||||
let s = "@local(foo: [bar, baz])";
|
let s = "@local(foo: [bar, baz])";
|
||||||
|
|
||||||
let (remaining, attribute) = match super::parse(s) {
|
let (remaining, attribute) = match parse(s) {
|
||||||
Ok(v) => v,
|
Ok(v) => v,
|
||||||
Err(e) => panic!("{}", e),
|
Err(e) => panic!("{}", e),
|
||||||
};
|
};
|
||||||
|
@ -136,7 +136,7 @@ mod test {
|
||||||
fn multiple() {
|
fn multiple() {
|
||||||
let s = "@local(foo: bar, baz: qux)";
|
let s = "@local(foo: bar, baz: qux)";
|
||||||
|
|
||||||
let (remaining, attribute) = super::parse(s).unwrap();
|
let (remaining, attribute) = parse(s).unwrap();
|
||||||
|
|
||||||
assert_eq!(remaining, "");
|
assert_eq!(remaining, "");
|
||||||
assert_eq!(attribute.name, "local");
|
assert_eq!(attribute.name, "local");
|
||||||
|
|
|
@ -1,8 +1,31 @@
|
||||||
mod attribute;
|
#![warn(
|
||||||
mod model;
|
clippy::all,
|
||||||
mod sync_data;
|
clippy::pedantic,
|
||||||
|
clippy::correctness,
|
||||||
use attribute::*;
|
clippy::perf,
|
||||||
|
clippy::style,
|
||||||
|
clippy::suspicious,
|
||||||
|
clippy::complexity,
|
||||||
|
clippy::nursery,
|
||||||
|
clippy::unwrap_used,
|
||||||
|
unused_qualifications,
|
||||||
|
rust_2018_idioms,
|
||||||
|
trivial_casts,
|
||||||
|
trivial_numeric_casts,
|
||||||
|
unused_allocation,
|
||||||
|
clippy::unnecessary_cast,
|
||||||
|
clippy::cast_lossless,
|
||||||
|
clippy::cast_possible_truncation,
|
||||||
|
clippy::cast_possible_wrap,
|
||||||
|
clippy::cast_precision_loss,
|
||||||
|
clippy::cast_sign_loss,
|
||||||
|
clippy::dbg_macro,
|
||||||
|
clippy::deprecated_cfg_attr,
|
||||||
|
clippy::separated_literal_suffix,
|
||||||
|
deprecated
|
||||||
|
)]
|
||||||
|
#![forbid(deprecated_in_future)]
|
||||||
|
#![allow(clippy::missing_errors_doc, clippy::module_name_repetitions)]
|
||||||
|
|
||||||
use prisma_client_rust_sdk::{
|
use prisma_client_rust_sdk::{
|
||||||
prelude::*,
|
prelude::*,
|
||||||
|
@ -11,6 +34,12 @@ use prisma_client_rust_sdk::{
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
|
mod attribute;
|
||||||
|
mod model;
|
||||||
|
mod sync_data;
|
||||||
|
|
||||||
|
use attribute::{model_attributes, Attribute, AttributeFieldValue};
|
||||||
|
|
||||||
#[derive(Debug, serde::Serialize, thiserror::Error)]
|
#[derive(Debug, serde::Serialize, thiserror::Error)]
|
||||||
enum Error {}
|
enum Error {}
|
||||||
|
|
||||||
|
@ -38,7 +67,7 @@ pub enum ModelSyncType<'a> {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl<'a> ModelSyncType<'a> {
|
impl<'a> ModelSyncType<'a> {
|
||||||
fn from_attribute(attr: Attribute, model: ModelWalker<'a>) -> Option<Self> {
|
fn from_attribute(attr: &Attribute<'_>, model: ModelWalker<'a>) -> Option<Self> {
|
||||||
Some(match attr.name {
|
Some(match attr.name {
|
||||||
"local" | "shared" => {
|
"local" | "shared" => {
|
||||||
let id = attr
|
let id = attr
|
||||||
|
@ -69,14 +98,15 @@ impl<'a> ModelSyncType<'a> {
|
||||||
AttributeFieldValue::List(_) => None,
|
AttributeFieldValue::List(_) => None,
|
||||||
})
|
})
|
||||||
.and_then(|name| {
|
.and_then(|name| {
|
||||||
match model
|
if let RefinedFieldWalker::Relation(r) = model
|
||||||
.fields()
|
.fields()
|
||||||
.find(|f| f.name() == name)
|
.find(|f| f.name() == name)
|
||||||
.unwrap_or_else(|| panic!("'{name}' field not found"))
|
.unwrap_or_else(|| panic!("'{name}' field not found"))
|
||||||
.refine()
|
.refine()
|
||||||
{
|
{
|
||||||
RefinedFieldWalker::Relation(r) => Some(r),
|
Some(r)
|
||||||
_ => None,
|
} else {
|
||||||
|
None
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
.unwrap_or_else(|| panic!("'{name}' must be a relation field"))
|
.unwrap_or_else(|| panic!("'{name}' must be a relation field"))
|
||||||
|
@ -96,11 +126,10 @@ impl<'a> ModelSyncType<'a> {
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
fn sync_id(&self) -> Vec<FieldWalker> {
|
fn sync_id(&self) -> Vec<FieldWalker<'_>> {
|
||||||
match self {
|
match self {
|
||||||
// Self::Owned { id } => id.clone(),
|
// Self::Owned { id } => id.clone(),
|
||||||
Self::Local { id, .. } => vec![*id],
|
Self::Local { id, .. } | Self::Shared { id, .. } => vec![*id],
|
||||||
Self::Shared { id, .. } => vec![*id],
|
|
||||||
Self::Relation { group, item, .. } => vec![(*group).into(), (*item).into()],
|
Self::Relation { group, item, .. } => vec![(*group).into(), (*item).into()],
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -127,7 +156,7 @@ impl PrismaGenerator for SDSyncGenerator {
|
||||||
|
|
||||||
type Error = Error;
|
type Error = Error;
|
||||||
|
|
||||||
fn generate(self, args: GenerateArgs) -> Result<Module, Self::Error> {
|
fn generate(self, args: GenerateArgs<'_>) -> Result<Module, Self::Error> {
|
||||||
let db = &args.schema.db;
|
let db = &args.schema.db;
|
||||||
|
|
||||||
let models_with_sync_types = db
|
let models_with_sync_types = db
|
||||||
|
@ -136,13 +165,13 @@ impl PrismaGenerator for SDSyncGenerator {
|
||||||
.map(|(model, attributes)| {
|
.map(|(model, attributes)| {
|
||||||
let sync_type = attributes
|
let sync_type = attributes
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.find_map(|a| ModelSyncType::from_attribute(a, model));
|
.find_map(|a| ModelSyncType::from_attribute(&a, model));
|
||||||
|
|
||||||
(model, sync_type)
|
(model, sync_type)
|
||||||
})
|
})
|
||||||
.collect::<Vec<_>>();
|
.collect::<Vec<_>>();
|
||||||
|
|
||||||
let model_sync_data = sync_data::r#enum(models_with_sync_types.clone());
|
let model_sync_data = sync_data::enumerate(&models_with_sync_types);
|
||||||
|
|
||||||
let mut module = Module::new(
|
let mut module = Module::new(
|
||||||
"root",
|
"root",
|
||||||
|
|
|
@ -1,13 +1,14 @@
|
||||||
use prisma_client_rust_sdk::{prelude::*, prisma::prisma_models::walkers::RefinedFieldWalker};
|
use prisma_client_rust_sdk::{prelude::*, prisma::prisma_models::walkers::RefinedFieldWalker};
|
||||||
|
use prisma_models::{ast::ModelId, walkers::Walker};
|
||||||
|
|
||||||
use crate::{ModelSyncType, ModelWithSyncType};
|
use crate::{ModelSyncType, ModelWithSyncType};
|
||||||
|
|
||||||
pub fn module((model, sync_type): ModelWithSyncType) -> Module {
|
pub fn module((model, sync_type): ModelWithSyncType<'_>) -> Module {
|
||||||
let model_name_snake = snake_ident(model.name());
|
let model_name_snake = snake_ident(model.name());
|
||||||
|
|
||||||
let sync_id = sync_type.as_ref().map(|sync_type| {
|
let sync_id = sync_type.as_ref().map(|sync_type| {
|
||||||
let fields = sync_type.sync_id();
|
let fields = sync_type.sync_id();
|
||||||
let fields = fields.iter().flat_map(|field| {
|
let fields = fields.iter().map(|field| {
|
||||||
let name_snake = snake_ident(field.name());
|
let name_snake = snake_ident(field.name());
|
||||||
|
|
||||||
let typ = match field.refine() {
|
let typ = match field.refine() {
|
||||||
|
@ -18,58 +19,10 @@ pub fn module((model, sync_type): ModelWithSyncType) -> Module {
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
Some(quote!(pub #name_snake: #typ))
|
quote!(pub #name_snake: #typ)
|
||||||
});
|
});
|
||||||
|
|
||||||
let model_stuff = match sync_type {
|
let model_stuff = parse_model(sync_type, &model_name_snake);
|
||||||
ModelSyncType::Relation {
|
|
||||||
item,
|
|
||||||
group,
|
|
||||||
model_id,
|
|
||||||
} => {
|
|
||||||
let item_name_snake = snake_ident(item.name());
|
|
||||||
let item_model_name_snake = snake_ident(item.related_model().name());
|
|
||||||
|
|
||||||
let group_name_snake = snake_ident(group.name());
|
|
||||||
let group_model_name_snake = snake_ident(group.related_model().name());
|
|
||||||
|
|
||||||
Some(quote! {
|
|
||||||
impl sd_sync::RelationSyncId for SyncId {
|
|
||||||
type ItemSyncId = super::#item_model_name_snake::SyncId;
|
|
||||||
type GroupSyncId = super::#group_model_name_snake::SyncId;
|
|
||||||
|
|
||||||
fn split(&self) -> (&Self::ItemSyncId, &Self::GroupSyncId) {
|
|
||||||
(
|
|
||||||
&self.#item_name_snake,
|
|
||||||
&self.#group_name_snake
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub const MODEL_ID: u16 = #model_id;
|
|
||||||
|
|
||||||
impl sd_sync::SyncModel for #model_name_snake::Types {
|
|
||||||
const MODEL_ID: u16 = MODEL_ID;
|
|
||||||
}
|
|
||||||
|
|
||||||
impl sd_sync::RelationSyncModel for #model_name_snake::Types {
|
|
||||||
type SyncId = SyncId;
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
ModelSyncType::Shared { model_id, .. } => Some(quote! {
|
|
||||||
pub const MODEL_ID: u16 = #model_id;
|
|
||||||
|
|
||||||
impl sd_sync::SyncModel for #model_name_snake::Types {
|
|
||||||
const MODEL_ID: u16 = MODEL_ID;
|
|
||||||
}
|
|
||||||
|
|
||||||
impl sd_sync::SharedSyncModel for #model_name_snake::Types {
|
|
||||||
type SyncId = SyncId;
|
|
||||||
}
|
|
||||||
}),
|
|
||||||
_ => None,
|
|
||||||
};
|
|
||||||
|
|
||||||
quote! {
|
quote! {
|
||||||
#[derive(serde::Serialize, serde::Deserialize, Clone, Debug)]
|
#[derive(serde::Serialize, serde::Deserialize, Clone, Debug)]
|
||||||
|
@ -101,8 +54,9 @@ pub fn module((model, sync_type): ModelWithSyncType) -> Module {
|
||||||
let relation_model_name_snake =
|
let relation_model_name_snake =
|
||||||
snake_ident(relation_field.related_model().name());
|
snake_ident(relation_field.related_model().name());
|
||||||
|
|
||||||
match relation_field.referenced_fields() {
|
relation_field.referenced_fields().map_or_else(
|
||||||
Some(i) => {
|
|| None,
|
||||||
|
|i| {
|
||||||
if i.count() == 1 {
|
if i.count() == 1 {
|
||||||
Some(quote! {{
|
Some(quote! {{
|
||||||
let val: std::collections::HashMap<String, rmpv::Value> = ::rmpv::ext::from_value(val).unwrap();
|
let val: std::collections::HashMap<String, rmpv::Value> = ::rmpv::ext::from_value(val).unwrap();
|
||||||
|
@ -115,17 +69,17 @@ pub fn module((model, sync_type): ModelWithSyncType) -> Module {
|
||||||
} else {
|
} else {
|
||||||
None
|
None
|
||||||
}
|
}
|
||||||
}
|
},
|
||||||
_ => None,
|
)
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
.map(|body| quote!(#model_name_snake::#field_name_snake::NAME => #body))
|
.map(|body| quote!(#model_name_snake::#field_name_snake::NAME => #body))
|
||||||
});
|
});
|
||||||
|
|
||||||
match field_matches.clone().count() {
|
if field_matches.clone().count() == 0 {
|
||||||
0 => quote!(),
|
quote!()
|
||||||
_ => quote! {
|
} else {
|
||||||
|
quote! {
|
||||||
impl #model_name_snake::SetParam {
|
impl #model_name_snake::SetParam {
|
||||||
pub fn deserialize(field: &str, val: ::rmpv::Value) -> Option<Self> {
|
pub fn deserialize(field: &str, val: ::rmpv::Value) -> Option<Self> {
|
||||||
Some(match field {
|
Some(match field {
|
||||||
|
@ -134,41 +88,11 @@ pub fn module((model, sync_type): ModelWithSyncType) -> Module {
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
}
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
let unique_param_impl = {
|
let unique_param_impl = process_unique_params(model, &model_name_snake);
|
||||||
let field_matches = model
|
|
||||||
.unique_criterias()
|
|
||||||
.flat_map(|criteria| match &criteria.fields().next() {
|
|
||||||
Some(field) if criteria.fields().len() == 1 => {
|
|
||||||
let field_name_snake = snake_ident(field.name());
|
|
||||||
|
|
||||||
Some(quote!(#model_name_snake::#field_name_snake::NAME =>
|
|
||||||
#model_name_snake::#field_name_snake::equals(
|
|
||||||
::rmpv::ext::from_value(val).unwrap()
|
|
||||||
),
|
|
||||||
))
|
|
||||||
}
|
|
||||||
_ => None,
|
|
||||||
})
|
|
||||||
.collect::<Vec<_>>();
|
|
||||||
|
|
||||||
match field_matches.len() {
|
|
||||||
0 => quote!(),
|
|
||||||
_ => quote! {
|
|
||||||
impl #model_name_snake::UniqueWhereParam {
|
|
||||||
pub fn deserialize(field: &str, val: ::rmpv::Value) -> Option<Self> {
|
|
||||||
Some(match field {
|
|
||||||
#(#field_matches)*
|
|
||||||
_ => return None
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
Module::new(
|
Module::new(
|
||||||
model.name(),
|
model.name(),
|
||||||
|
@ -184,3 +108,90 @@ pub fn module((model, sync_type): ModelWithSyncType) -> Module {
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[inline]
|
||||||
|
fn parse_model(sync_type: &ModelSyncType<'_>, model_name_snake: &Ident) -> Option<TokenStream> {
|
||||||
|
match sync_type {
|
||||||
|
ModelSyncType::Relation {
|
||||||
|
item,
|
||||||
|
group,
|
||||||
|
model_id,
|
||||||
|
} => {
|
||||||
|
let item_name_snake = snake_ident(item.name());
|
||||||
|
let item_model_name_snake = snake_ident(item.related_model().name());
|
||||||
|
|
||||||
|
let group_name_snake = snake_ident(group.name());
|
||||||
|
let group_model_name_snake = snake_ident(group.related_model().name());
|
||||||
|
|
||||||
|
Some(quote! {
|
||||||
|
impl sd_sync::RelationSyncId for SyncId {
|
||||||
|
type ItemSyncId = super::#item_model_name_snake::SyncId;
|
||||||
|
type GroupSyncId = super::#group_model_name_snake::SyncId;
|
||||||
|
|
||||||
|
fn split(&self) -> (&Self::ItemSyncId, &Self::GroupSyncId) {
|
||||||
|
(
|
||||||
|
&self.#item_name_snake,
|
||||||
|
&self.#group_name_snake
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub const MODEL_ID: u16 = #model_id;
|
||||||
|
|
||||||
|
impl sd_sync::SyncModel for #model_name_snake::Types {
|
||||||
|
const MODEL_ID: u16 = MODEL_ID;
|
||||||
|
}
|
||||||
|
|
||||||
|
impl sd_sync::RelationSyncModel for #model_name_snake::Types {
|
||||||
|
type SyncId = SyncId;
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
ModelSyncType::Shared { model_id, .. } => Some(quote! {
|
||||||
|
pub const MODEL_ID: u16 = #model_id;
|
||||||
|
|
||||||
|
impl sd_sync::SyncModel for #model_name_snake::Types {
|
||||||
|
const MODEL_ID: u16 = MODEL_ID;
|
||||||
|
}
|
||||||
|
|
||||||
|
impl sd_sync::SharedSyncModel for #model_name_snake::Types {
|
||||||
|
type SyncId = SyncId;
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
ModelSyncType::Local { .. } => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[inline]
|
||||||
|
fn process_unique_params(model: Walker<'_, ModelId>, model_name_snake: &Ident) -> TokenStream {
|
||||||
|
let field_matches = model
|
||||||
|
.unique_criterias()
|
||||||
|
.filter_map(|criteria| match &criteria.fields().next() {
|
||||||
|
Some(field) if criteria.fields().len() == 1 => {
|
||||||
|
let field_name_snake = snake_ident(field.name());
|
||||||
|
|
||||||
|
Some(quote!(#model_name_snake::#field_name_snake::NAME =>
|
||||||
|
#model_name_snake::#field_name_snake::equals(
|
||||||
|
::rmpv::ext::from_value(val).unwrap()
|
||||||
|
),
|
||||||
|
))
|
||||||
|
}
|
||||||
|
_ => None,
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
|
||||||
|
if field_matches.is_empty() {
|
||||||
|
quote!()
|
||||||
|
} else {
|
||||||
|
quote! {
|
||||||
|
impl #model_name_snake::UniqueWhereParam {
|
||||||
|
pub fn deserialize(field: &str, val: ::rmpv::Value) -> Option<Self> {
|
||||||
|
Some(match field {
|
||||||
|
#(#field_matches)*
|
||||||
|
_ => return None
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
@ -2,10 +2,11 @@ use prisma_client_rust_sdk::{
|
||||||
prelude::*,
|
prelude::*,
|
||||||
prisma::prisma_models::walkers::{RefinedFieldWalker, RelationFieldWalker},
|
prisma::prisma_models::walkers::{RefinedFieldWalker, RelationFieldWalker},
|
||||||
};
|
};
|
||||||
|
use prisma_models::walkers::{FieldWalker, ScalarFieldWalker};
|
||||||
|
|
||||||
use crate::{ModelSyncType, ModelWithSyncType};
|
use crate::{ModelSyncType, ModelWithSyncType};
|
||||||
|
|
||||||
pub fn r#enum(models: Vec<ModelWithSyncType>) -> TokenStream {
|
pub fn enumerate(models: &[ModelWithSyncType<'_>]) -> TokenStream {
|
||||||
let (variants, matches): (Vec<_>, Vec<_>) = models
|
let (variants, matches): (Vec<_>, Vec<_>) = models
|
||||||
.iter()
|
.iter()
|
||||||
.filter_map(|(model, sync_type)| {
|
.filter_map(|(model, sync_type)| {
|
||||||
|
@ -38,193 +39,12 @@ pub fn r#enum(models: Vec<ModelWithSyncType>) -> TokenStream {
|
||||||
|
|
||||||
let match_arms = match sync_type.as_ref()? {
|
let match_arms = match sync_type.as_ref()? {
|
||||||
ModelSyncType::Shared { id, model_id } => {
|
ModelSyncType::Shared { id, model_id } => {
|
||||||
let (get_id, equals_value, id_name_snake, create_id) = match id.refine() {
|
handle_crdt_ops_shared(id, *model_id, &model_name_snake)
|
||||||
RefinedFieldWalker::Relation(rel) => {
|
|
||||||
let scalar_field = rel.fields().unwrap().next().unwrap();
|
|
||||||
let id_name_snake = snake_ident(scalar_field.name());
|
|
||||||
let field_name_snake = snake_ident(rel.name());
|
|
||||||
let opposite_model_name_snake =
|
|
||||||
snake_ident(rel.opposite_relation_field().unwrap().model().name());
|
|
||||||
|
|
||||||
let relation_equals_condition = quote!(prisma::#opposite_model_name_snake::pub_id::equals(
|
|
||||||
id.#field_name_snake.pub_id.clone()
|
|
||||||
));
|
|
||||||
|
|
||||||
let rel_fetch = quote! {
|
|
||||||
let rel = db.#opposite_model_name_snake()
|
|
||||||
.find_unique(#relation_equals_condition)
|
|
||||||
.exec()
|
|
||||||
.await?
|
|
||||||
.unwrap();
|
|
||||||
};
|
|
||||||
|
|
||||||
(
|
|
||||||
Some(rel_fetch),
|
|
||||||
quote!(rel.id),
|
|
||||||
id_name_snake,
|
|
||||||
relation_equals_condition,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
RefinedFieldWalker::Scalar(s) => {
|
|
||||||
let field_name_snake = snake_ident(s.name());
|
|
||||||
let thing = quote!(id.#field_name_snake.clone());
|
|
||||||
|
|
||||||
(None, thing.clone(), field_name_snake, thing)
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
quote! {
|
|
||||||
#get_id
|
|
||||||
|
|
||||||
match data {
|
|
||||||
sd_sync::CRDTOperationData::Create(data) => {
|
|
||||||
let data: Vec<_> = data.into_iter().map(|(field, value)| {
|
|
||||||
prisma::#model_name_snake::SetParam::deserialize(&field, value).unwrap()
|
|
||||||
}).collect();
|
|
||||||
|
|
||||||
db.#model_name_snake()
|
|
||||||
.upsert(
|
|
||||||
prisma::#model_name_snake::#id_name_snake::equals(#equals_value),
|
|
||||||
prisma::#model_name_snake::create(#create_id, data.clone()),
|
|
||||||
data
|
|
||||||
)
|
|
||||||
.exec()
|
|
||||||
.await?;
|
|
||||||
},
|
|
||||||
sd_sync::CRDTOperationData::Update { field, value } => {
|
|
||||||
let data = vec![
|
|
||||||
prisma::#model_name_snake::SetParam::deserialize(&field, value).unwrap()
|
|
||||||
];
|
|
||||||
|
|
||||||
db.#model_name_snake()
|
|
||||||
.upsert(
|
|
||||||
prisma::#model_name_snake::#id_name_snake::equals(#equals_value),
|
|
||||||
prisma::#model_name_snake::create(#create_id, data.clone()),
|
|
||||||
data,
|
|
||||||
)
|
|
||||||
.exec()
|
|
||||||
.await?;
|
|
||||||
},
|
|
||||||
sd_sync::CRDTOperationData::Delete => {
|
|
||||||
db.#model_name_snake()
|
|
||||||
.delete(prisma::#model_name_snake::#id_name_snake::equals(#equals_value))
|
|
||||||
.exec()
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
db.crdt_operation()
|
|
||||||
.delete_many(vec![
|
|
||||||
prisma::crdt_operation::model::equals(#model_id as i32),
|
|
||||||
prisma::crdt_operation::record_id::equals(rmp_serde::to_vec(&id).unwrap()),
|
|
||||||
prisma::crdt_operation::kind::equals(sd_sync::OperationKind::Create.to_string())
|
|
||||||
])
|
|
||||||
.exec()
|
|
||||||
.await?;
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
ModelSyncType::Relation { item, group, .. } => {
|
ModelSyncType::Relation { item, group, .. } => {
|
||||||
let compound_id = format_ident!(
|
handle_crdt_ops_relation(models, item, group, &model_name_snake)
|
||||||
"{}",
|
|
||||||
group
|
|
||||||
.fields()
|
|
||||||
.unwrap()
|
|
||||||
.chain(item.fields().unwrap())
|
|
||||||
.map(|f| f.name())
|
|
||||||
.collect::<Vec<_>>()
|
|
||||||
.join("_")
|
|
||||||
);
|
|
||||||
|
|
||||||
let db_batch_items = {
|
|
||||||
let batch_item = |item: &RelationFieldWalker| {
|
|
||||||
let item_model_sync_id_field_name_snake = models
|
|
||||||
.iter()
|
|
||||||
.find(|m| m.0.name() == item.related_model().name())
|
|
||||||
.and_then(|(_m, sync)| sync.as_ref())
|
|
||||||
.map(|sync| snake_ident(sync.sync_id()[0].name()))
|
|
||||||
.unwrap();
|
|
||||||
let item_model_name_snake = snake_ident(item.related_model().name());
|
|
||||||
let item_field_name_snake = snake_ident(item.name());
|
|
||||||
|
|
||||||
quote! {
|
|
||||||
db.#item_model_name_snake()
|
|
||||||
.find_unique(
|
|
||||||
prisma::#item_model_name_snake::#item_model_sync_id_field_name_snake::equals(
|
|
||||||
id.#item_field_name_snake.#item_model_sync_id_field_name_snake.clone()
|
|
||||||
)
|
|
||||||
)
|
|
||||||
.select(prisma::#item_model_name_snake::select!({ id }))
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
[batch_item(group), batch_item(item)]
|
|
||||||
};
|
|
||||||
|
|
||||||
let create_items = {
|
|
||||||
let create_item = |item: &RelationFieldWalker, var: TokenStream| {
|
|
||||||
let item_model_name_snake = snake_ident(item.related_model().name());
|
|
||||||
|
|
||||||
quote!(
|
|
||||||
prisma::#item_model_name_snake::id::equals(#var.id)
|
|
||||||
)
|
|
||||||
};
|
|
||||||
|
|
||||||
[
|
|
||||||
create_item(item, quote!(item)),
|
|
||||||
create_item(group, quote!(group)),
|
|
||||||
]
|
|
||||||
};
|
|
||||||
|
|
||||||
quote! {
|
|
||||||
let (Some(group), Some(item)) =
|
|
||||||
(#(#db_batch_items.exec().await?),*) else {
|
|
||||||
panic!("item and group not found!");
|
|
||||||
};
|
|
||||||
|
|
||||||
let id = prisma::#model_name_snake::#compound_id(group.id, item.id);
|
|
||||||
|
|
||||||
match data {
|
|
||||||
sd_sync::CRDTOperationData::Create(_) => {
|
|
||||||
db.#model_name_snake()
|
|
||||||
.upsert(
|
|
||||||
id,
|
|
||||||
prisma::#model_name_snake::create(
|
|
||||||
#(#create_items),*,
|
|
||||||
vec![]
|
|
||||||
),
|
|
||||||
vec![],
|
|
||||||
)
|
|
||||||
.exec()
|
|
||||||
.await
|
|
||||||
.ok();
|
|
||||||
},
|
|
||||||
sd_sync::CRDTOperationData::Update { field, value } => {
|
|
||||||
let data = vec![prisma::#model_name_snake::SetParam::deserialize(&field, value).unwrap()];
|
|
||||||
|
|
||||||
db.#model_name_snake()
|
|
||||||
.upsert(
|
|
||||||
id,
|
|
||||||
prisma::#model_name_snake::create(
|
|
||||||
#(#create_items),*,
|
|
||||||
data.clone(),
|
|
||||||
),
|
|
||||||
data,
|
|
||||||
)
|
|
||||||
.exec()
|
|
||||||
.await
|
|
||||||
.ok();
|
|
||||||
},
|
|
||||||
sd_sync::CRDTOperationData::Delete => {
|
|
||||||
db.#model_name_snake()
|
|
||||||
.delete(id)
|
|
||||||
.exec()
|
|
||||||
.await
|
|
||||||
.ok();
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
_ => return None,
|
ModelSyncType::Local { .. } => return None,
|
||||||
};
|
};
|
||||||
|
|
||||||
Some(quote! {
|
Some(quote! {
|
||||||
|
@ -257,3 +77,210 @@ pub fn r#enum(models: Vec<ModelWithSyncType>) -> TokenStream {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn handle_crdt_ops_relation(
|
||||||
|
models: &[ModelWithSyncType<'_>],
|
||||||
|
item: &RelationFieldWalker<'_>,
|
||||||
|
group: &RelationFieldWalker<'_>,
|
||||||
|
model_name_snake: &Ident,
|
||||||
|
) -> TokenStream {
|
||||||
|
let compound_id = format_ident!(
|
||||||
|
"{}",
|
||||||
|
group
|
||||||
|
.fields()
|
||||||
|
.expect("missing group fields")
|
||||||
|
.chain(item.fields().expect("missing item fields"))
|
||||||
|
.map(ScalarFieldWalker::name)
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join("_")
|
||||||
|
);
|
||||||
|
|
||||||
|
let db_batch_items = {
|
||||||
|
let batch_item = |item: &RelationFieldWalker<'_>| {
|
||||||
|
let item_model_sync_id_field_name_snake = models
|
||||||
|
.iter()
|
||||||
|
.find(|m| m.0.name() == item.related_model().name())
|
||||||
|
.and_then(|(_m, sync)| sync.as_ref())
|
||||||
|
.map(|sync| snake_ident(sync.sync_id()[0].name()))
|
||||||
|
.expect("missing sync id field name for relation");
|
||||||
|
let item_model_name_snake = snake_ident(item.related_model().name());
|
||||||
|
let item_field_name_snake = snake_ident(item.name());
|
||||||
|
|
||||||
|
quote! {
|
||||||
|
db.#item_model_name_snake()
|
||||||
|
.find_unique(
|
||||||
|
prisma::#item_model_name_snake::#item_model_sync_id_field_name_snake::equals(
|
||||||
|
id.#item_field_name_snake.#item_model_sync_id_field_name_snake.clone()
|
||||||
|
)
|
||||||
|
)
|
||||||
|
.select(prisma::#item_model_name_snake::select!({ id }))
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
[batch_item(group), batch_item(item)]
|
||||||
|
};
|
||||||
|
|
||||||
|
let create_items = {
|
||||||
|
let create_item = |item: &RelationFieldWalker<'_>, var: TokenStream| {
|
||||||
|
let item_model_name_snake = snake_ident(item.related_model().name());
|
||||||
|
|
||||||
|
quote!(
|
||||||
|
prisma::#item_model_name_snake::id::equals(#var.id)
|
||||||
|
)
|
||||||
|
};
|
||||||
|
|
||||||
|
[
|
||||||
|
create_item(item, quote!(item)),
|
||||||
|
create_item(group, quote!(group)),
|
||||||
|
]
|
||||||
|
};
|
||||||
|
|
||||||
|
quote! {
|
||||||
|
let (Some(group), Some(item)) =
|
||||||
|
(#(#db_batch_items.exec().await?),*) else {
|
||||||
|
panic!("item and group not found!");
|
||||||
|
};
|
||||||
|
|
||||||
|
let id = prisma::#model_name_snake::#compound_id(group.id, item.id);
|
||||||
|
|
||||||
|
match data {
|
||||||
|
sd_sync::CRDTOperationData::Create(_) => {
|
||||||
|
db.#model_name_snake()
|
||||||
|
.upsert(
|
||||||
|
id,
|
||||||
|
prisma::#model_name_snake::create(
|
||||||
|
#(#create_items),*,
|
||||||
|
vec![]
|
||||||
|
),
|
||||||
|
vec![],
|
||||||
|
)
|
||||||
|
.exec()
|
||||||
|
.await
|
||||||
|
.ok();
|
||||||
|
},
|
||||||
|
sd_sync::CRDTOperationData::Update { field, value } => {
|
||||||
|
let data = vec![prisma::#model_name_snake::SetParam::deserialize(&field, value).unwrap()];
|
||||||
|
|
||||||
|
db.#model_name_snake()
|
||||||
|
.upsert(
|
||||||
|
id,
|
||||||
|
prisma::#model_name_snake::create(
|
||||||
|
#(#create_items),*,
|
||||||
|
data.clone(),
|
||||||
|
),
|
||||||
|
data,
|
||||||
|
)
|
||||||
|
.exec()
|
||||||
|
.await
|
||||||
|
.ok();
|
||||||
|
},
|
||||||
|
sd_sync::CRDTOperationData::Delete => {
|
||||||
|
db.#model_name_snake()
|
||||||
|
.delete(id)
|
||||||
|
.exec()
|
||||||
|
.await
|
||||||
|
.ok();
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[inline]
|
||||||
|
fn handle_crdt_ops_shared(
|
||||||
|
id: &FieldWalker<'_>,
|
||||||
|
model_id: u16,
|
||||||
|
model_name_snake: &Ident,
|
||||||
|
) -> TokenStream {
|
||||||
|
let (get_id, equals_value, id_name_snake, create_id) = match id.refine() {
|
||||||
|
RefinedFieldWalker::Relation(rel) => {
|
||||||
|
let scalar_field = rel
|
||||||
|
.fields()
|
||||||
|
.expect("missing fields")
|
||||||
|
.next()
|
||||||
|
.expect("empty fields");
|
||||||
|
let id_name_snake = snake_ident(scalar_field.name());
|
||||||
|
let field_name_snake = snake_ident(rel.name());
|
||||||
|
let opposite_model_name_snake = snake_ident(
|
||||||
|
rel.opposite_relation_field()
|
||||||
|
.expect("missing opposite relation field")
|
||||||
|
.model()
|
||||||
|
.name(),
|
||||||
|
);
|
||||||
|
|
||||||
|
let relation_equals_condition = quote!(prisma::#opposite_model_name_snake::pub_id::equals(
|
||||||
|
id.#field_name_snake.pub_id.clone()
|
||||||
|
));
|
||||||
|
|
||||||
|
let rel_fetch = quote! {
|
||||||
|
let rel = db.#opposite_model_name_snake()
|
||||||
|
.find_unique(#relation_equals_condition)
|
||||||
|
.exec()
|
||||||
|
.await?
|
||||||
|
.unwrap();
|
||||||
|
};
|
||||||
|
|
||||||
|
(
|
||||||
|
Some(rel_fetch),
|
||||||
|
quote!(rel.id),
|
||||||
|
id_name_snake,
|
||||||
|
relation_equals_condition,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
RefinedFieldWalker::Scalar(s) => {
|
||||||
|
let field_name_snake = snake_ident(s.name());
|
||||||
|
let thing = quote!(id.#field_name_snake.clone());
|
||||||
|
|
||||||
|
(None, thing.clone(), field_name_snake, thing)
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
quote! {
|
||||||
|
#get_id
|
||||||
|
|
||||||
|
match data {
|
||||||
|
sd_sync::CRDTOperationData::Create(data) => {
|
||||||
|
let data: Vec<_> = data.into_iter().map(|(field, value)| {
|
||||||
|
prisma::#model_name_snake::SetParam::deserialize(&field, value).unwrap()
|
||||||
|
}).collect();
|
||||||
|
|
||||||
|
db.#model_name_snake()
|
||||||
|
.upsert(
|
||||||
|
prisma::#model_name_snake::#id_name_snake::equals(#equals_value),
|
||||||
|
prisma::#model_name_snake::create(#create_id, data.clone()),
|
||||||
|
data
|
||||||
|
)
|
||||||
|
.exec()
|
||||||
|
.await?;
|
||||||
|
},
|
||||||
|
sd_sync::CRDTOperationData::Update { field, value } => {
|
||||||
|
let data = vec![
|
||||||
|
prisma::#model_name_snake::SetParam::deserialize(&field, value).unwrap()
|
||||||
|
];
|
||||||
|
|
||||||
|
db.#model_name_snake()
|
||||||
|
.upsert(
|
||||||
|
prisma::#model_name_snake::#id_name_snake::equals(#equals_value),
|
||||||
|
prisma::#model_name_snake::create(#create_id, data.clone()),
|
||||||
|
data,
|
||||||
|
)
|
||||||
|
.exec()
|
||||||
|
.await?;
|
||||||
|
},
|
||||||
|
sd_sync::CRDTOperationData::Delete => {
|
||||||
|
db.#model_name_snake()
|
||||||
|
.delete(prisma::#model_name_snake::#id_name_snake::equals(#equals_value))
|
||||||
|
.exec()
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
db.crdt_operation()
|
||||||
|
.delete_many(vec![
|
||||||
|
prisma::crdt_operation::model::equals(#model_id as i32),
|
||||||
|
prisma::crdt_operation::record_id::equals(rmp_serde::to_vec(&id).unwrap()),
|
||||||
|
prisma::crdt_operation::kind::equals(sd_sync::OperationKind::Create.to_string())
|
||||||
|
])
|
||||||
|
.exec()
|
||||||
|
.await?;
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
@ -1,3 +1,5 @@
|
||||||
|
use std::mem;
|
||||||
|
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use uhlc::NTP64;
|
use uhlc::NTP64;
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
@ -6,11 +8,12 @@ use crate::{CRDTOperation, CRDTOperationData};
|
||||||
|
|
||||||
pub type CompressedCRDTOperationsForModel = Vec<(rmpv::Value, Vec<CompressedCRDTOperation>)>;
|
pub type CompressedCRDTOperationsForModel = Vec<(rmpv::Value, Vec<CompressedCRDTOperation>)>;
|
||||||
|
|
||||||
/// Stores a bunch of CRDTOperations in a more memory-efficient form for sending to the cloud.
|
/// Stores a bunch of [`CRDTOperation`]s in a more memory-efficient form for sending to the cloud.
|
||||||
#[derive(Serialize, Deserialize, Debug, PartialEq)]
|
#[derive(Serialize, Deserialize, Debug, PartialEq)]
|
||||||
pub struct CompressedCRDTOperations(pub Vec<(Uuid, Vec<(u16, CompressedCRDTOperationsForModel)>)>);
|
pub struct CompressedCRDTOperations(pub Vec<(Uuid, Vec<(u16, CompressedCRDTOperationsForModel)>)>);
|
||||||
|
|
||||||
impl CompressedCRDTOperations {
|
impl CompressedCRDTOperations {
|
||||||
|
#[must_use]
|
||||||
pub fn new(ops: Vec<CRDTOperation>) -> Self {
|
pub fn new(ops: Vec<CRDTOperation>) -> Self {
|
||||||
let mut compressed = vec![];
|
let mut compressed = vec![];
|
||||||
|
|
||||||
|
@ -32,34 +35,34 @@ impl CompressedCRDTOperations {
|
||||||
for op in ops_iter {
|
for op in ops_iter {
|
||||||
if instance_id != op.instance {
|
if instance_id != op.instance {
|
||||||
model.push((
|
model.push((
|
||||||
std::mem::replace(&mut record_id, op.record_id.clone()),
|
mem::replace(&mut record_id, op.record_id.clone()),
|
||||||
std::mem::take(&mut record),
|
mem::take(&mut record),
|
||||||
));
|
));
|
||||||
instance.push((
|
instance.push((
|
||||||
std::mem::replace(&mut model_str, op.model),
|
mem::replace(&mut model_str, op.model),
|
||||||
std::mem::take(&mut model),
|
mem::take(&mut model),
|
||||||
));
|
));
|
||||||
compressed.push((
|
compressed.push((
|
||||||
std::mem::replace(&mut instance_id, op.instance),
|
mem::replace(&mut instance_id, op.instance),
|
||||||
std::mem::take(&mut instance),
|
mem::take(&mut instance),
|
||||||
));
|
));
|
||||||
} else if model_str != op.model {
|
} else if model_str != op.model {
|
||||||
model.push((
|
model.push((
|
||||||
std::mem::replace(&mut record_id, op.record_id.clone()),
|
mem::replace(&mut record_id, op.record_id.clone()),
|
||||||
std::mem::take(&mut record),
|
mem::take(&mut record),
|
||||||
));
|
));
|
||||||
instance.push((
|
instance.push((
|
||||||
std::mem::replace(&mut model_str, op.model),
|
mem::replace(&mut model_str, op.model),
|
||||||
std::mem::take(&mut model),
|
mem::take(&mut model),
|
||||||
));
|
));
|
||||||
} else if record_id != op.record_id {
|
} else if record_id != op.record_id {
|
||||||
model.push((
|
model.push((
|
||||||
std::mem::replace(&mut record_id, op.record_id.clone()),
|
mem::replace(&mut record_id, op.record_id.clone()),
|
||||||
std::mem::take(&mut record),
|
mem::take(&mut record),
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
record.push(CompressedCRDTOperation::from(op))
|
record.push(CompressedCRDTOperation::from(op));
|
||||||
}
|
}
|
||||||
|
|
||||||
model.push((record_id, record));
|
model.push((record_id, record));
|
||||||
|
@ -69,6 +72,7 @@ impl CompressedCRDTOperations {
|
||||||
Self(compressed)
|
Self(compressed)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
pub fn first(&self) -> Option<(Uuid, u16, &rmpv::Value, &CompressedCRDTOperation)> {
|
pub fn first(&self) -> Option<(Uuid, u16, &rmpv::Value, &CompressedCRDTOperation)> {
|
||||||
self.0.first().and_then(|(instance, data)| {
|
self.0.first().and_then(|(instance, data)| {
|
||||||
data.first().and_then(|(model, data)| {
|
data.first().and_then(|(model, data)| {
|
||||||
|
@ -78,6 +82,7 @@ impl CompressedCRDTOperations {
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
pub fn last(&self) -> Option<(Uuid, u16, &rmpv::Value, &CompressedCRDTOperation)> {
|
pub fn last(&self) -> Option<(Uuid, u16, &rmpv::Value, &CompressedCRDTOperation)> {
|
||||||
self.0.last().and_then(|(instance, data)| {
|
self.0.last().and_then(|(instance, data)| {
|
||||||
data.last().and_then(|(model, data)| {
|
data.last().and_then(|(model, data)| {
|
||||||
|
@ -104,6 +109,7 @@ impl CompressedCRDTOperations {
|
||||||
self.len() == 0
|
self.len() == 0
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
pub fn into_ops(self) -> Vec<CRDTOperation> {
|
pub fn into_ops(self) -> Vec<CRDTOperation> {
|
||||||
let mut ops = vec![];
|
let mut ops = vec![];
|
||||||
|
|
||||||
|
@ -117,7 +123,7 @@ impl CompressedCRDTOperations {
|
||||||
record_id: record_id.clone(),
|
record_id: record_id.clone(),
|
||||||
timestamp: op.timestamp,
|
timestamp: op.timestamp,
|
||||||
data: op.data,
|
data: op.data,
|
||||||
})
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
use std::{collections::BTreeMap, fmt::Debug};
|
use std::{collections::BTreeMap, fmt};
|
||||||
|
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use specta::Type;
|
use specta::Type;
|
||||||
|
@ -11,8 +11,8 @@ pub enum OperationKind<'a> {
|
||||||
Delete,
|
Delete,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl std::fmt::Display for OperationKind<'_> {
|
impl fmt::Display for OperationKind<'_> {
|
||||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||||
match self {
|
match self {
|
||||||
OperationKind::Create => write!(f, "c"),
|
OperationKind::Create => write!(f, "c"),
|
||||||
OperationKind::Update(field) => write!(f, "u:{field}"),
|
OperationKind::Update(field) => write!(f, "u:{field}"),
|
||||||
|
@ -36,11 +36,13 @@ pub enum CRDTOperationData {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl CRDTOperationData {
|
impl CRDTOperationData {
|
||||||
|
#[must_use]
|
||||||
pub fn create() -> Self {
|
pub fn create() -> Self {
|
||||||
Self::Create(Default::default())
|
Self::Create(BTreeMap::default())
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn as_kind(&self) -> OperationKind {
|
#[must_use]
|
||||||
|
pub fn as_kind(&self) -> OperationKind<'_> {
|
||||||
match self {
|
match self {
|
||||||
Self::Create(_) => OperationKind::Create,
|
Self::Create(_) => OperationKind::Create,
|
||||||
Self::Update { field, .. } => OperationKind::Update(field),
|
Self::Update { field, .. } => OperationKind::Update(field),
|
||||||
|
@ -62,17 +64,17 @@ pub struct CRDTOperation {
|
||||||
|
|
||||||
impl CRDTOperation {
|
impl CRDTOperation {
|
||||||
#[must_use]
|
#[must_use]
|
||||||
pub fn kind(&self) -> OperationKind {
|
pub fn kind(&self) -> OperationKind<'_> {
|
||||||
self.data.as_kind()
|
self.data.as_kind()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Debug for CRDTOperation {
|
impl fmt::Debug for CRDTOperation {
|
||||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||||
f.debug_struct("CRDTOperation")
|
f.debug_struct("CRDTOperation")
|
||||||
.field("data", &self.data)
|
.field("data", &self.data)
|
||||||
.field("model", &self.model)
|
.field("model", &self.model)
|
||||||
.field("record_id", &self.record_id.to_string())
|
.field("record_id", &self.record_id.to_string())
|
||||||
.finish()
|
.finish_non_exhaustive()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -126,7 +126,7 @@ macro_rules! option_sync_entry {
|
||||||
#[macro_export]
|
#[macro_export]
|
||||||
macro_rules! sync_db_entry {
|
macro_rules! sync_db_entry {
|
||||||
($v:expr, $($m:tt)*) => {{
|
($v:expr, $($m:tt)*) => {{
|
||||||
let v = $v;
|
let v = $v.into();
|
||||||
($crate::sync_entry!(&v, $($m)*), $($m)*::set(Some(v)))
|
($crate::sync_entry!(&v, $($m)*), $($m)*::set(Some(v)))
|
||||||
}}
|
}}
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,3 +1,32 @@
|
||||||
|
#![warn(
|
||||||
|
clippy::all,
|
||||||
|
clippy::pedantic,
|
||||||
|
clippy::correctness,
|
||||||
|
clippy::perf,
|
||||||
|
clippy::style,
|
||||||
|
clippy::suspicious,
|
||||||
|
clippy::complexity,
|
||||||
|
clippy::nursery,
|
||||||
|
clippy::unwrap_used,
|
||||||
|
unused_qualifications,
|
||||||
|
rust_2018_idioms,
|
||||||
|
trivial_casts,
|
||||||
|
trivial_numeric_casts,
|
||||||
|
unused_allocation,
|
||||||
|
clippy::unnecessary_cast,
|
||||||
|
clippy::cast_lossless,
|
||||||
|
clippy::cast_possible_truncation,
|
||||||
|
clippy::cast_possible_wrap,
|
||||||
|
clippy::cast_precision_loss,
|
||||||
|
clippy::cast_sign_loss,
|
||||||
|
clippy::dbg_macro,
|
||||||
|
clippy::deprecated_cfg_attr,
|
||||||
|
clippy::separated_literal_suffix,
|
||||||
|
deprecated
|
||||||
|
)]
|
||||||
|
#![forbid(deprecated_in_future)]
|
||||||
|
#![allow(clippy::missing_errors_doc, clippy::module_name_repetitions)]
|
||||||
|
|
||||||
mod compressed;
|
mod compressed;
|
||||||
mod crdt;
|
mod crdt;
|
||||||
mod factory;
|
mod factory;
|
||||||
|
|
|
@ -13,6 +13,7 @@ async-channel = { workspace = true }
|
||||||
async-trait = { workspace = true }
|
async-trait = { workspace = true }
|
||||||
futures = { workspace = true }
|
futures = { workspace = true }
|
||||||
futures-concurrency = { workspace = true }
|
futures-concurrency = { workspace = true }
|
||||||
|
pin-project-lite = { workspace = true }
|
||||||
thiserror = { workspace = true }
|
thiserror = { workspace = true }
|
||||||
tokio = { workspace = true, features = [
|
tokio = { workspace = true, features = [
|
||||||
"sync",
|
"sync",
|
||||||
|
@ -26,7 +27,6 @@ uuid = { workspace = true, features = ["v4"] }
|
||||||
|
|
||||||
# Specific Task System dependencies
|
# Specific Task System dependencies
|
||||||
downcast-rs = "1.2"
|
downcast-rs = "1.2"
|
||||||
pin-project = "1.1"
|
|
||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
lending-stream = { workspace = true }
|
lending-stream = { workspace = true }
|
||||||
|
|
|
@ -188,12 +188,13 @@ where
|
||||||
) -> impl Future<Output = Result<Self, Self::DeserializeError>> + Send;
|
) -> impl Future<Output = Result<Self, Self::DeserializeError>> + Send;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Intermediate struct to wait until a pause or a cancel commands are sent by the user.
|
pin_project_lite::pin_project! {
|
||||||
#[must_use = "`InterrupterFuture` does nothing unless polled"]
|
/// Intermediate struct to wait until a pause or a cancel commands are sent by the user.
|
||||||
#[pin_project::pin_project]
|
#[must_use = "`InterrupterFuture` does nothing unless polled"]
|
||||||
pub struct InterrupterFuture<'recv> {
|
pub struct InterrupterFuture<'recv> {
|
||||||
#[pin]
|
#[pin]
|
||||||
fut: chan::Recv<'recv, InterruptionRequest>,
|
fut: chan::Recv<'recv, InterruptionRequest>,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Future for InterrupterFuture<'_> {
|
impl Future for InterrupterFuture<'_> {
|
||||||
|
|
|
@ -338,7 +338,17 @@ impl Task<SampleError> for SampleActorTask {
|
||||||
|
|
||||||
async fn run(&mut self, interrupter: &Interrupter) -> Result<ExecStatus, SampleError> {
|
async fn run(&mut self, interrupter: &Interrupter) -> Result<ExecStatus, SampleError> {
|
||||||
info!("Actor data: {:#?}", self.actor_data);
|
info!("Actor data: {:#?}", self.actor_data);
|
||||||
self.timed_task.run(interrupter).await
|
let out = self.timed_task.run(interrupter).await?;
|
||||||
|
if let ExecStatus::Done(TaskOutput::Out(out)) = &out {
|
||||||
|
info!(
|
||||||
|
"Task completed with {} pauses",
|
||||||
|
out.downcast_ref::<TimedTaskOutput>()
|
||||||
|
.expect("we know the task type")
|
||||||
|
.pauses_count
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(out)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn with_priority(&self) -> bool {
|
fn with_priority(&self) -> bool {
|
||||||
|
|
|
@ -69,7 +69,7 @@ export const Component = () => {
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
const watchBackgroundProcessingPercentage = form.watch('background_processing_percentage');
|
// const watchBackgroundProcessingPercentage = form.watch('background_processing_percentage');
|
||||||
|
|
||||||
useDebouncedFormWatch(form, async (value) => {
|
useDebouncedFormWatch(form, async (value) => {
|
||||||
if (await form.trigger()) {
|
if (await form.trigger()) {
|
||||||
|
@ -85,9 +85,9 @@ export const Component = () => {
|
||||||
image_labeler_version: value.image_labeler_version ?? null
|
image_labeler_version: value.image_labeler_version ?? null
|
||||||
});
|
});
|
||||||
|
|
||||||
if (value.background_processing_percentage != undefined) {
|
if (value.background_processing_percentage != null) {
|
||||||
await updateThumbnailerPreferences.mutateAsync({
|
await updateThumbnailerPreferences.mutateAsync({
|
||||||
background_processing_percentage: value.background_processing_percentage
|
// background_processing_percentage: value.background_processing_percentage
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -169,7 +169,7 @@ export type CameraData = { device_make: string | null; device_model: string | nu
|
||||||
|
|
||||||
export type CasId = string
|
export type CasId = string
|
||||||
|
|
||||||
export type ChangeNodeNameArgs = { name: string | null; p2p_port: Port | null; p2p_disabled: boolean | null; p2p_ipv6_disabled: boolean | null; p2p_relay_disabled: boolean | null; p2p_discovery: P2PDiscoveryState | null; p2p_remote_access: boolean | null; p2p_manual_peers: string[] | null; image_labeler_version: string | null }
|
export type ChangeNodeNameArgs = { name: string | null; p2p_port: Port | null; p2p_disabled: boolean | null; p2p_ipv6_disabled: boolean | null; p2p_relay_disabled: boolean | null; p2p_discovery: P2PDiscoveryState | null; p2p_remote_access: boolean | null; p2p_manual_peers: string[] | null }
|
||||||
|
|
||||||
export type Chapter = { id: number; start: [number, number]; end: [number, number]; time_base_den: number; time_base_num: number; metadata: Metadata }
|
export type Chapter = { id: number; start: [number, number]; end: [number, number]; time_base_den: number; time_base_num: number; metadata: Metadata }
|
||||||
|
|
||||||
|
@ -657,7 +657,7 @@ export type TextMatch = { contains: string } | { startsWith: string } | { endsWi
|
||||||
*/
|
*/
|
||||||
export type ThumbKey = { shard_hex: string; cas_id: CasId; base_directory_str: string }
|
export type ThumbKey = { shard_hex: string; cas_id: CasId; base_directory_str: string }
|
||||||
|
|
||||||
export type UpdateThumbnailerPreferences = { background_processing_percentage: number }
|
export type UpdateThumbnailerPreferences = Record<string, never>
|
||||||
|
|
||||||
export type VideoProps = { pixel_format: string | null; color_range: string | null; bits_per_channel: number | null; color_space: string | null; color_primaries: string | null; color_transfer: string | null; field_order: string | null; chroma_location: string | null; width: number; height: number; aspect_ratio_num: number | null; aspect_ratio_den: number | null; properties: string[] }
|
export type VideoProps = { pixel_format: string | null; color_range: string | null; bits_per_channel: number | null; color_space: string | null; color_primaries: string | null; color_transfer: string | null; field_order: string | null; chroma_location: string | null; width: number; height: number; aspect_ratio_num: number | null; aspect_ratio_den: number | null; properties: string[] }
|
||||||
|
|
||||||
|
|
|
@ -1,2 +1,2 @@
|
||||||
[toolchain]
|
[toolchain]
|
||||||
channel = "1.78"
|
channel = "1.79"
|
||||||
|
|
Loading…
Reference in a new issue