Software-Mac/spacedrive
Software-Mac/spacedrive
1
0
Fork 0
mirror of https://github.com/spacedriveapp/spacedrive synced 2024-07-04 12:13:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

[ENG-1722] Numeric sync model IDs (#2298)

Browse source 
* numeric sync model ids

* migration

* fix test compilation
This commit is contained in:
Brendan Allan 2024-04-11 11:46:30 +08:00 committed by GitHub
parent 73a9b41c2a
commit 40fa3380e5
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
16 changed files with 435 additions and 377 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
core/crates/sync/src/db_operation.rs
View file

@ -26,7 +26,7 @@ impl crdt_include::Data {
instance: self.instance(), instance: self.instance(),
timestamp: self.timestamp(), timestamp: self.timestamp(),
record_id: rmp_serde::from_slice(&self.record_id).unwrap(), record_id: rmp_serde::from_slice(&self.record_id).unwrap(),
model: self.model, model: self.model as u16,
data: rmp_serde::from_slice(&self.data).unwrap(), data: rmp_serde::from_slice(&self.data).unwrap(),
} }
} }
@ -46,7 +46,7 @@ impl cloud_crdt_include::Data {
instance: self.instance(), instance: self.instance(),
timestamp: self.timestamp(), timestamp: self.timestamp(),
record_id: rmp_serde::from_slice(&self.record_id).unwrap(), record_id: rmp_serde::from_slice(&self.record_id).unwrap(),
model: self.model, model: self.model as u16,
data: serde_json::from_slice(&self.data).unwrap(), data: serde_json::from_slice(&self.data).unwrap(),
} }
} }
@ -67,7 +67,7 @@ fn crdt_op_db(op: &CRDTOperation) -> crdt_operation::Create {
instance: instance::pub_id::equals(op.instance.as_bytes().to_vec()), instance: instance::pub_id::equals(op.instance.as_bytes().to_vec()),
kind: op.kind().to_string(), kind: op.kind().to_string(),
data: to_vec(&op.data).unwrap(), data: to_vec(&op.data).unwrap(),
model: op.model.to_string(), model: op.model as i32,
record_id: rmp_serde::to_vec(&op.record_id).unwrap(), record_id: rmp_serde::to_vec(&op.record_id).unwrap(),
_params: vec![], _params: vec![],
} }

2
core/crates/sync/src/ingest.rs
View file

@ -219,7 +219,7 @@ impl Actor {
.crdt_operation() .crdt_operation()
.find_first(vec![ .find_first(vec![
crdt_operation::timestamp::gte(op.timestamp.as_u64() as i64), crdt_operation::timestamp::gte(op.timestamp.as_u64() as i64),
crdt_operation::model::equals(op.model.to_string()), crdt_operation::model::equals(op.model as i32),
crdt_operation::record_id::equals(serde_json::to_vec(&op.record_id).unwrap()), crdt_operation::record_id::equals(serde_json::to_vec(&op.record_id).unwrap()),
crdt_operation::kind::equals(op.kind().to_string()), crdt_operation::kind::equals(op.kind().to_string()),
]) ])

4
core/crates/sync/src/lib.rs
View file

@ -43,7 +43,7 @@ pub fn crdt_op_db(op: &CRDTOperation) -> crdt_operation::Create {
instance: instance::pub_id::equals(op.instance.as_bytes().to_vec()), instance: instance::pub_id::equals(op.instance.as_bytes().to_vec()),
kind: op.kind().to_string(), kind: op.kind().to_string(),
data: rmp_serde::to_vec(&op.data).unwrap(), data: rmp_serde::to_vec(&op.data).unwrap(),
model: op.model.to_string(), model: op.model as i32,
record_id: rmp_serde::to_vec(&op.record_id).unwrap(), record_id: rmp_serde::to_vec(&op.record_id).unwrap(),
_params: vec![], _params: vec![],
} }
@ -59,7 +59,7 @@ pub fn crdt_op_unchecked_db(
instance_id, instance_id,
kind: op.kind().to_string(), kind: op.kind().to_string(),
data: rmp_serde::to_vec(&op.data).unwrap(), data: rmp_serde::to_vec(&op.data).unwrap(),
model: op.model.to_string(), model: op.model as i32,
record_id: rmp_serde::to_vec(&op.record_id).unwrap(), record_id: rmp_serde::to_vec(&op.record_id).unwrap(),
_params: vec![], _params: vec![],
} }

2
core/crates/sync/tests/lib.rs
View file

@ -55,7 +55,7 @@ async fn writes_operations_and_rows_together() -> Result<(), Box<dyn std::error:
// 1 create, 2 update // 1 create, 2 update
assert_eq!(operations.len(), 3); assert_eq!(operations.len(), 3);
assert_eq!(operations[0].model, prisma::location::NAME); assert_eq!(operations[0].model, prisma_sync::location::MODEL_ID as i32);
let locations = instance.db.location().find_many(vec![]).exec().await?; let locations = instance.db.location().find_many(vec![]).exec().await?;

37
core/prisma/migrations/20240408081222_numeric_sync_model_id/migration.sql Normal file
View file

@ -0,0 +1,37 @@
/*
Warnings:
- You are about to alter the column `model` on the `cloud_crdt_operation` table. The data in that column could be lost. The data in that column will be cast from `String` to `Int`.
- You are about to alter the column `model` on the `crdt_operation` table. The data in that column could be lost. The data in that column will be cast from `String` to `Int`.
*/
-- RedefineTables
PRAGMA foreign_keys=OFF;
CREATE TABLE "new_cloud_crdt_operation" (
"id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
"timestamp" BIGINT NOT NULL,
"model" INTEGER NOT NULL,
"record_id" BLOB NOT NULL,
"kind" TEXT NOT NULL,
"data" BLOB NOT NULL,
"instance_id" INTEGER NOT NULL,
CONSTRAINT "cloud_crdt_operation_instance_id_fkey" FOREIGN KEY ("instance_id") REFERENCES "instance" ("id") ON DELETE RESTRICT ON UPDATE CASCADE
);
INSERT INTO "new_cloud_crdt_operation" ("data", "id", "instance_id", "kind", "model", "record_id", "timestamp") SELECT "data", "id", "instance_id", "kind", "model", "record_id", "timestamp" FROM "cloud_crdt_operation";
DROP TABLE "cloud_crdt_operation";
ALTER TABLE "new_cloud_crdt_operation" RENAME TO "cloud_crdt_operation";
CREATE TABLE "new_crdt_operation" (
"id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
"timestamp" BIGINT NOT NULL,
"model" INTEGER NOT NULL,
"record_id" BLOB NOT NULL,
"kind" TEXT NOT NULL,
"data" BLOB NOT NULL,
"instance_id" INTEGER NOT NULL,
CONSTRAINT "crdt_operation_instance_id_fkey" FOREIGN KEY ("instance_id") REFERENCES "instance" ("id") ON DELETE RESTRICT ON UPDATE CASCADE
);
INSERT INTO "new_crdt_operation" ("data", "id", "instance_id", "kind", "model", "record_id", "timestamp") SELECT "data", "id", "instance_id", "kind", "model", "record_id", "timestamp" FROM "crdt_operation";
DROP TABLE "crdt_operation";
ALTER TABLE "new_crdt_operation" RENAME TO "crdt_operation";
PRAGMA foreign_key_check;
PRAGMA foreign_keys=ON;

635
core/prisma/schema.prisma
View file

@ -1,226 +1,242 @@
datasource db { datasource db {
provider = "sqlite" provider = "sqlite"
url = "file:dev.db" url = "file:dev.db"
} }
generator client { generator client {
provider = "cargo prisma" provider = "cargo prisma"
output = "../../crates/prisma/src/prisma" output = "../../crates/prisma/src/prisma"
module_path = "prisma" module_path = "prisma"
client_format = "folder" client_format = "folder"
} }
generator sync { generator sync {
provider = "cargo prisma-sync" provider = "cargo prisma-sync"
output = "../../crates/prisma/src/prisma_sync" output = "../../crates/prisma/src/prisma_sync"
client_format = "folder" client_format = "folder"
} }
model CRDTOperation { model CRDTOperation {
id Int @id @default(autoincrement()) id Int @id @default(autoincrement())
timestamp BigInt timestamp BigInt
model String model Int
record_id Bytes record_id Bytes
// Enum: ?? // Enum: ??
kind String kind String
data Bytes data Bytes
instance_id Int instance_id Int
instance Instance @relation(fields: [instance_id], references: [id]) instance Instance @relation(fields: [instance_id], references: [id])
@@map("crdt_operation") @@map("crdt_operation")
}
model CloudCRDTOperation {
id Int @id @default(autoincrement())
timestamp BigInt
model Int
record_id Bytes
// Enum: ??
kind String
data Bytes
instance_id Int
instance Instance @relation(fields: [instance_id], references: [id])
@@map("cloud_crdt_operation")
} }
/// @deprecated: This model has to exist solely for backwards compatibility. /// @deprecated: This model has to exist solely for backwards compatibility.
model Node { model Node {
id Int @id @default(autoincrement()) id Int @id @default(autoincrement())
pub_id Bytes @unique pub_id Bytes @unique
name String name String
// Enum: sd_core::node::Platform // Enum: sd_core::node::Platform
platform Int platform Int
date_created DateTime date_created DateTime
identity Bytes? // TODO: Change to required field in future identity Bytes? // TODO: Change to required field in future
@@map("node") @@map("node")
} }
/// @local(id: pub_id)
// represents a single `.db` file (SQLite DB) that is paired to the current library. // represents a single `.db` file (SQLite DB) that is paired to the current library.
// A `LibraryInstance` is always owned by a single `Node` but it's possible for that node to change (or two to be owned by a single node). // A `LibraryInstance` is always owned by a single `Node` but it's possible for that node to change (or two to be owned by a single node).
model Instance { model Instance {
id Int @id @default(autoincrement()) // This is is NOT globally unique id Int @id @default(autoincrement()) // This is is NOT globally unique
pub_id Bytes @unique // This UUID is meaningless and exists soley cause the `uhlc::ID` must be 16-bit. Really this should be derived from the `identity` field. pub_id Bytes @unique // This UUID is meaningless and exists soley cause the `uhlc::ID` must be 16-bit. Really this should be derived from the `identity` field.
// Enum: sd_p2p::Identity (or sd_core::p2p::IdentityOrRemoteIdentity in early versions) // Enum: sd_p2p::Identity (or sd_core::p2p::IdentityOrRemoteIdentity in early versions)
identity Bytes? identity Bytes?
// Enum: sd_core::node::RemoteIdentity // Enum: sd_core::node::RemoteIdentity
remote_identity Bytes remote_identity Bytes
node_id Bytes node_id Bytes
metadata Bytes? // TODO: This should not be optional metadata Bytes? // TODO: This should not be optional
last_seen DateTime // Time core started for owner, last P2P message for P2P node last_seen DateTime // Time core started for owner, last P2P message for P2P node
date_created DateTime date_created DateTime
// clock timestamp for sync // clock timestamp for sync
timestamp BigInt? timestamp BigInt?
locations Location[] locations Location[]
CRDTOperation CRDTOperation[] CRDTOperation CRDTOperation[]
CloudCRDTOperation CloudCRDTOperation[] CloudCRDTOperation CloudCRDTOperation[]
@@map("instance") @@map("instance")
} }
model Statistics { model Statistics {
id Int @id @default(autoincrement()) id Int @id @default(autoincrement())
date_captured DateTime @default(now()) date_captured DateTime @default(now())
total_object_count Int @default(0) total_object_count Int @default(0)
library_db_size String @default("0") library_db_size String @default("0")
total_bytes_used String @default("0") total_bytes_used String @default("0")
total_bytes_capacity String @default("0") total_bytes_capacity String @default("0")
total_unique_bytes String @default("0") total_unique_bytes String @default("0")
total_bytes_free String @default("0") total_bytes_free String @default("0")
preview_media_bytes String @default("0") preview_media_bytes String @default("0")
@@map("statistics") @@map("statistics")
} }
/// @local /// @local
model Volume { model Volume {
id Int @id @default(autoincrement()) id Int @id @default(autoincrement())
name String name String
mount_point String mount_point String
total_bytes_capacity String @default("0") total_bytes_capacity String @default("0")
total_bytes_available String @default("0") total_bytes_available String @default("0")
disk_type String? disk_type String?
filesystem String? filesystem String?
is_system Boolean @default(false) is_system Boolean @default(false)
date_modified DateTime @default(now()) date_modified DateTime @default(now())
@@unique([mount_point, name]) @@unique([mount_point, name])
@@map("volume") @@map("volume")
} }
/// @shared(id: pub_id) /// @shared(id: pub_id, modelId: 1)
model Location { model Location {
id Int @id @default(autoincrement()) id Int @id @default(autoincrement())
pub_id Bytes @unique pub_id Bytes @unique
name String? name String?
path String? path String?
total_capacity Int? total_capacity Int?
available_capacity Int? available_capacity Int?
size_in_bytes Bytes? size_in_bytes Bytes?
is_archived Boolean? is_archived Boolean?
generate_preview_media Boolean? generate_preview_media Boolean?
sync_preview_media Boolean? sync_preview_media Boolean?
hidden Boolean? hidden Boolean?
date_created DateTime? date_created DateTime?
scan_state Int @default(0) // Enum: sd_core::location::ScanState scan_state Int @default(0) // Enum: sd_core::location::ScanState
/// @local /// @local
// this is just a client side cache which is annoying but oh well (@brendan) // this is just a client side cache which is annoying but oh well (@brendan)
instance_id Int? instance_id Int?
instance Instance? @relation(fields: [instance_id], references: [id], onDelete: SetNull) instance Instance? @relation(fields: [instance_id], references: [id], onDelete: SetNull)
file_paths FilePath[] file_paths FilePath[]
indexer_rules IndexerRulesInLocation[] indexer_rules IndexerRulesInLocation[]
@@map("location") @@map("location")
} }
/// @shared(id: pub_id) /// @shared(id: pub_id, modelId: 2)
model FilePath { model FilePath {
id Int @id @default(autoincrement()) id Int @id @default(autoincrement())
pub_id Bytes @unique pub_id Bytes @unique
is_dir Boolean? is_dir Boolean?
// content addressable storage id - blake3 sampled checksum // content addressable storage id - blake3 sampled checksum
cas_id String? cas_id String?
// full byte contents digested into blake3 checksum // full byte contents digested into blake3 checksum
integrity_checksum String? integrity_checksum String?
// location that owns this path // location that owns this path
location_id Int? location_id Int?
location Location? @relation(fields: [location_id], references: [id], onDelete: SetNull) location Location? @relation(fields: [location_id], references: [id], onDelete: SetNull)
// the path of the file relative to its location // the path of the file relative to its location
materialized_path String? materialized_path String?
// the name and extension, MUST have 'COLLATE NOCASE' in migration // the name and extension, MUST have 'COLLATE NOCASE' in migration
name String? name String?
extension String? extension String?
hidden Boolean? hidden Boolean?
size_in_bytes String? // deprecated size_in_bytes String? // deprecated
size_in_bytes_bytes Bytes? size_in_bytes_bytes Bytes?
inode Bytes? // This is actually an unsigned 64 bit integer, but we don't have this type in SQLite inode Bytes? // This is actually an unsigned 64 bit integer, but we don't have this type in SQLite
// the unique Object for this file path // the unique Object for this file path
object_id Int? object_id Int?
object Object? @relation(fields: [object_id], references: [id], onDelete: SetNull) object Object? @relation(fields: [object_id], references: [id], onDelete: SetNull)
key_id Int? // replacement for encryption key_id Int? // replacement for encryption
// permissions String? // permissions String?
date_created DateTime? date_created DateTime?
date_modified DateTime? date_modified DateTime?
date_indexed DateTime? date_indexed DateTime?
// key Key? @relation(fields: [key_id], references: [id]) // key Key? @relation(fields: [key_id], references: [id])
@@unique([location_id, materialized_path, name, extension]) @@unique([location_id, materialized_path, name, extension])
@@unique([location_id, inode]) @@unique([location_id, inode])
@@index([location_id]) @@index([location_id])
@@index([location_id, materialized_path]) @@index([location_id, materialized_path])
@@map("file_path") @@map("file_path")
} }
/// @shared(id: pub_id) /// @shared(id: pub_id, modelId: 3)
model Object { model Object {
id Int @id @default(autoincrement()) id Int @id @default(autoincrement())
pub_id Bytes @unique pub_id Bytes @unique
// Enum: sd_file_ext::kind::ObjectKind // Enum: sd_file_ext::kind::ObjectKind
kind Int? kind Int?
key_id Int? key_id Int?
// handy ways to mark an object // handy ways to mark an object
hidden Boolean? hidden Boolean?
favorite Boolean? favorite Boolean?
important Boolean? important Boolean?
// if we have generated preview media for this object on at least one Node // if we have generated preview media for this object on at least one Node
// commented out for now by @brendonovich since they they're irrelevant to the sync system // commented out for now by @brendonovich since they they're irrelevant to the sync system
// has_thumbnail Boolean? // has_thumbnail Boolean?
// has_thumbstrip Boolean? // has_thumbstrip Boolean?
// has_video_preview Boolean? // has_video_preview Boolean?
// TODO: change above to: // TODO: change above to:
// has_generated_thumbnail Boolean @default(false) // has_generated_thumbnail Boolean @default(false)
// has_generated_thumbstrip Boolean @default(false) // has_generated_thumbstrip Boolean @default(false)
// has_generated_video_preview Boolean @default(false) // has_generated_video_preview Boolean @default(false)
// integration with ipfs // integration with ipfs
// ipfs_id String? // ipfs_id String?
// plain text note // plain text note
note String? note String?
// the original known creation date of this object // the original known creation date of this object
date_created DateTime? date_created DateTime?
date_accessed DateTime? date_accessed DateTime?
tags TagOnObject[] tags TagOnObject[]
labels LabelOnObject[] labels LabelOnObject[]
albums ObjectInAlbum[] albums ObjectInAlbum[]
spaces ObjectInSpace[] spaces ObjectInSpace[]
file_paths FilePath[] file_paths FilePath[]
// comments Comment[] // comments Comment[]
media_data MediaData? media_data MediaData?
// key Key? @relation(fields: [key_id], references: [id]) // key Key? @relation(fields: [key_id], references: [id])
@@map("object") @@map("object")
} }
// if there is a conflicting cas_id, the conficting file should be updated to have a larger cas_id as // if there is a conflicting cas_id, the conficting file should be updated to have a larger cas_id as
@ -276,186 +292,185 @@ model Object {
// @@map("key") // @@map("key")
// } // }
/// @shared(id: object) /// @shared(id: object, modelId: 4)
model MediaData { model MediaData {
id Int @id @default(autoincrement()) id Int @id @default(autoincrement())
resolution Bytes? resolution Bytes?
media_date Bytes? media_date Bytes?
media_location Bytes? media_location Bytes?
camera_data Bytes? camera_data Bytes?
artist String? artist String?
description String? description String?
copyright String? copyright String?
exif_version String? exif_version String?
// purely for sorting/ordering, never sent to the frontend as they'd be useless // purely for sorting/ordering, never sent to the frontend as they'd be useless
// these are also usually one-way, and not reversible // these are also usually one-way, and not reversible
// (e.g. we can't get `MediaDate::Utc(2023-09-26T22:04:37+01:00)` from `1695758677` as we don't store the TZ) // (e.g. we can't get `MediaDate::Utc(2023-09-26T22:04:37+01:00)` from `1695758677` as we don't store the TZ)
epoch_time BigInt? // time since unix epoch epoch_time BigInt? // time since unix epoch
// video-specific // video-specific
// duration Int? // duration Int?
// fps Int? // fps Int?
// streams Int? // streams Int?
// video_codec String? // eg: "h264, h265, av1" // video_codec String? // eg: "h264, h265, av1"
// audio_codec String? // eg: "opus" // audio_codec String? // eg: "opus"
object_id Int @unique object_id Int @unique
object Object @relation(fields: [object_id], references: [id], onDelete: Cascade) object Object @relation(fields: [object_id], references: [id], onDelete: Cascade)
@@map("media_data") @@map("media_data")
} }
//// Tag //// //// Tag ////
/// @shared(id: pub_id) /// @shared(id: pub_id, modelId: 5)
model Tag { model Tag {
id Int @id @default(autoincrement()) id Int @id @default(autoincrement())
pub_id Bytes @unique pub_id Bytes @unique
name String? name String?
color String? color String?
is_hidden Boolean? // user hidden entire tag is_hidden Boolean? // user hidden entire tag
date_created DateTime? date_created DateTime?
date_modified DateTime? date_modified DateTime?
tag_objects TagOnObject[] tag_objects TagOnObject[]
@@map("tag") @@map("tag")
} }
/// @relation(item: object, group: tag) /// @relation(item: object, group: tag, modelId: 6)
model TagOnObject { model TagOnObject {
object_id Int object_id Int
object Object @relation(fields: [object_id], references: [id], onDelete: Restrict) object Object @relation(fields: [object_id], references: [id], onDelete: Restrict)
tag_id Int tag_id Int
tag Tag @relation(fields: [tag_id], references: [id], onDelete: Restrict) tag Tag @relation(fields: [tag_id], references: [id], onDelete: Restrict)
date_created DateTime? date_created DateTime?
@@id([tag_id, object_id]) @@id([tag_id, object_id])
@@map("tag_on_object") @@map("tag_on_object")
} }
//// Label //// //// Label ////
/// @shared(id: name) /// @shared(id: name, modelId: 7)
model Label { model Label {
id Int @id @default(autoincrement()) id Int @id @default(autoincrement())
name String @unique name String @unique
date_created DateTime? date_created DateTime?
date_modified DateTime? date_modified DateTime?
label_objects LabelOnObject[] label_objects LabelOnObject[]
@@map("label") @@map("label")
} }
/// @relation(item: object, group: label) /// @relation(item: object, group: label, modelId: 8)
model LabelOnObject { model LabelOnObject {
date_created DateTime @default(now()) date_created DateTime @default(now())
object_id Int object_id Int
object Object @relation(fields: [object_id], references: [id], onDelete: Restrict) object Object @relation(fields: [object_id], references: [id], onDelete: Restrict)
label_id Int label_id Int
label Label @relation(fields: [label_id], references: [id], onDelete: Restrict) label Label @relation(fields: [label_id], references: [id], onDelete: Restrict)
@@id([label_id, object_id]) @@id([label_id, object_id])
@@map("label_on_object") @@map("label_on_object")
} }
//// Space //// //// Space ////
model Space { model Space {
id Int @id @default(autoincrement()) id Int @id @default(autoincrement())
pub_id Bytes @unique pub_id Bytes @unique
name String? name String?
description String? description String?
date_created DateTime? date_created DateTime?
date_modified DateTime? date_modified DateTime?
objects ObjectInSpace[] objects ObjectInSpace[]
@@map("space") @@map("space")
} }
model ObjectInSpace { model ObjectInSpace {
space_id Int space_id Int
space Space @relation(fields: [space_id], references: [id], onDelete: Restrict) space Space @relation(fields: [space_id], references: [id], onDelete: Restrict)
object_id Int object_id Int
object Object @relation(fields: [object_id], references: [id], onDelete: Restrict) object Object @relation(fields: [object_id], references: [id], onDelete: Restrict)
@@id([space_id, object_id]) @@id([space_id, object_id])
@@map("object_in_space") @@map("object_in_space")
} }
//// Job //// //// Job ////
model Job { model Job {
id Bytes @id id Bytes @id
name String? name String?
action String? // Will be composed of "{action_description}(-{children_order})*" action String? // Will be composed of "{action_description}(-{children_order})*"
// Enum: sd_core::job::job_manager:JobStatus // Enum: sd_core::job::job_manager:JobStatus
status Int? // 0 = Queued status Int? // 0 = Queued
// List of errors, separated by "\n\n" in case of failed jobs or completed with errors
errors_text String? // Deprecated, use `critical_error` or `non_critical_errors` instead
critical_error String? // Serialized error field with info about the failed job after completion
non_critical_errors Bytes? // Serialized non-critical errors field with info about the completed job with errors after completion
// List of errors, separated by "\n\n" in case of failed jobs or completed with errors data Bytes? // Deprecated
errors_text String? // Deprecated, use `critical_error` or `non_critical_errors` instead metadata Bytes? // Serialized metadata field with info about the job after completion
critical_error String? // Serialized error field with info about the failed job after completion
non_critical_errors Bytes? // Serialized non-critical errors field with info about the completed job with errors after completion
data Bytes? // Deprecated parent_id Bytes?
metadata Bytes? // Serialized metadata field with info about the job after completion
parent_id Bytes? task_count Int?
completed_task_count Int?
date_estimated_completion DateTime? // Estimated timestamp that the job will be complete at
task_count Int? date_created DateTime?
completed_task_count Int? date_started DateTime? // Started execution
date_estimated_completion DateTime? // Estimated timestamp that the job will be complete at date_completed DateTime? // Finished execution
date_created DateTime? parent Job? @relation("jobs_dependency", fields: [parent_id], references: [id], onDelete: SetNull)
date_started DateTime? // Started execution children Job[] @relation("jobs_dependency")
date_completed DateTime? // Finished execution
parent Job? @relation("jobs_dependency", fields: [parent_id], references: [id], onDelete: SetNull) @@map("job")
children Job[] @relation("jobs_dependency")
@@map("job")
} }
//// Album //// //// Album ////
model Album { model Album {
id Int @id id Int @id
pub_id Bytes @unique pub_id Bytes @unique
name String? name String?
is_hidden Boolean? is_hidden Boolean?
date_created DateTime? date_created DateTime?
date_modified DateTime? date_modified DateTime?
objects ObjectInAlbum[] objects ObjectInAlbum[]
@@map("album") @@map("album")
} }
model ObjectInAlbum { model ObjectInAlbum {
date_created DateTime? date_created DateTime?
album_id Int album_id Int
album Album @relation(fields: [album_id], references: [id], onDelete: NoAction) album Album @relation(fields: [album_id], references: [id], onDelete: NoAction)
object_id Int object_id Int
object Object @relation(fields: [object_id], references: [id], onDelete: NoAction) object Object @relation(fields: [object_id], references: [id], onDelete: NoAction)
@@id([album_id, object_id]) @@id([album_id, object_id])
@@map("object_in_album") @@map("object_in_album")
} }
//// Comment //// //// Comment ////
@ -475,84 +490,66 @@ model ObjectInAlbum {
//// Indexer Rules //// //// Indexer Rules ////
model IndexerRule { model IndexerRule {
id Int @id @default(autoincrement()) id Int @id @default(autoincrement())
pub_id Bytes @unique pub_id Bytes @unique
name String? name String?
default Boolean? default Boolean?
rules_per_kind Bytes? rules_per_kind Bytes?
date_created DateTime? date_created DateTime?
date_modified DateTime? date_modified DateTime?
locations IndexerRulesInLocation[] locations IndexerRulesInLocation[]
@@map("indexer_rule") @@map("indexer_rule")
} }
model IndexerRulesInLocation { model IndexerRulesInLocation {
location_id Int location_id Int
location Location @relation(fields: [location_id], references: [id], onDelete: Restrict) location Location @relation(fields: [location_id], references: [id], onDelete: Restrict)
indexer_rule_id Int indexer_rule_id Int
indexer_rule IndexerRule @relation(fields: [indexer_rule_id], references: [id], onDelete: Restrict) indexer_rule IndexerRule @relation(fields: [indexer_rule_id], references: [id], onDelete: Restrict)
@@id([location_id, indexer_rule_id]) @@id([location_id, indexer_rule_id])
@@map("indexer_rule_in_location") @@map("indexer_rule_in_location")
} }
/// @shared(id: key) /// @shared(id: key, modelId: 9)
model Preference { model Preference {
key String @id key String @id
value Bytes? value Bytes?
@@map("preference") @@map("preference")
} }
model Notification { model Notification {
id Int @id @default(autoincrement()) id Int @id @default(autoincrement())
read Boolean @default(false) read Boolean @default(false)
// Enum: crate::api::notifications::NotificationData // Enum: crate::api::notifications::NotificationData
data Bytes data Bytes
expires_at DateTime? expires_at DateTime?
@@map("notification") @@map("notification")
} }
/// @shared(id: pub_id) /// @shared(id: pub_id, modelId: 10)
model SavedSearch { model SavedSearch {
id Int @id @default(autoincrement()) id Int @id @default(autoincrement())
pub_id Bytes @unique pub_id Bytes @unique
// enum: crate::api::search::saved::SearchTarget // enum: crate::api::search::saved::SearchTarget
target String? target String?
search String? search String?
filters String? filters String?
name String? name String?
icon String? icon String?
description String? description String?
// order Int? // Add this line to include ordering // order Int? // Add this line to include ordering
date_created DateTime? date_created DateTime?
date_modified DateTime? date_modified DateTime?
@@map("saved_search") @@map("saved_search")
}
/// @local(id: id)
model CloudCRDTOperation {
id Int @id @default(autoincrement())
timestamp BigInt
model String
record_id Bytes
// Enum: ??
kind String
data Bytes
instance_id Int
instance Instance @relation(fields: [instance_id], references: [id])
@@map("cloud_crdt_operation")
} }

2
core/src/cloud/sync/receive.rs
View file

@ -232,7 +232,7 @@ fn crdt_op_db(op: &CRDTOperation) -> cloud_crdt_operation::Create {
instance: instance::pub_id::equals(op.instance.as_bytes().to_vec()), instance: instance::pub_id::equals(op.instance.as_bytes().to_vec()),
kind: op.data.as_kind().to_string(), kind: op.data.as_kind().to_string(),
data: to_vec(&op.data).expect("unable to serialize data"), data: to_vec(&op.data).expect("unable to serialize data"),
model: op.model.to_string(), model: op.model as i32,
record_id: rmp_serde::to_vec(&op.record_id).expect("unable to serialize record id"), record_id: rmp_serde::to_vec(&op.record_id).expect("unable to serialize record id"),
_params: vec![], _params: vec![],
} }

12
core/src/location/mod.rs
View file

@ -747,12 +747,12 @@ async fn create_location(
(location::name::NAME, msgpack!(&name)), (location::name::NAME, msgpack!(&name)),
(location::path::NAME, msgpack!(&path)), (location::path::NAME, msgpack!(&path)),
(location::date_created::NAME, msgpack!(date_created)), (location::date_created::NAME, msgpack!(date_created)),
( // (
location::instance::NAME, // location::instance::NAME,
msgpack!(prisma_sync::instance::SyncId { // msgpack!(prisma_sync::instance::SyncId {
pub_id: uuid_to_bytes(sync.instance) // pub_id: uuid_to_bytes(sync.instance)
}), // }),
), // ),
], ],
), ),
db.location() db.location()

2
core/src/p2p/sync/mod.rs
View file

@ -69,7 +69,7 @@ mod originator {
instance: Uuid::new_v4(), instance: Uuid::new_v4(),
timestamp: sync::NTP64(0), timestamp: sync::NTP64(0),
record_id: rmpv::Value::Nil, record_id: rmpv::Value::Nil,
model: "name".to_string(), model: 0,
data: sd_sync::CRDTOperationData::Create, data: sd_sync::CRDTOperationData::Create,
}]); }]);

21
crates/sync-generator/src/lib.rs
View file

@ -28,10 +28,13 @@ pub enum ModelSyncType<'a> {
// }, // },
Shared { Shared {
id: FieldWalker<'a>, id: FieldWalker<'a>,
// model ids help reduce storage cost of sync messages
model_id: u16,
}, },
Relation { Relation {
group: RelationFieldWalker<'a>, group: RelationFieldWalker<'a>,
item: RelationFieldWalker<'a>, item: RelationFieldWalker<'a>,
model_id: u16,
}, },
} }
@ -49,7 +52,13 @@ impl<'a> ModelSyncType<'a> {
match attr.name { match attr.name {
"local" => Self::Local { id }, "local" => Self::Local { id },
"shared" => Self::Shared { id }, "shared" => Self::Shared {
id,
model_id: attr
.field("modelId")
.and_then(|a| a.as_single())
.and_then(|s| s.parse().ok())?,
},
_ => return None, _ => return None,
} }
} }
@ -77,6 +86,10 @@ impl<'a> ModelSyncType<'a> {
Self::Relation { Self::Relation {
item: get_field("item"), item: get_field("item"),
group: get_field("group"), group: get_field("group"),
model_id: attr
.field("modelId")
.and_then(|a| a.as_single())
.and_then(|s| s.parse().ok())?,
} }
} }
// "owned" => Self::Owned { id }, // "owned" => Self::Owned { id },
@ -87,9 +100,9 @@ impl<'a> ModelSyncType<'a> {
fn sync_id(&self) -> Vec<FieldWalker> { fn sync_id(&self) -> Vec<FieldWalker> {
match self { match self {
// Self::Owned { id } => id.clone(), // Self::Owned { id } => id.clone(),
Self::Local { id } => vec![*id], Self::Local { id, .. } => vec![*id],
Self::Shared { id } => vec![*id], Self::Shared { id, .. } => vec![*id],
Self::Relation { group, item } => vec![(*group).into(), (*item).into()], Self::Relation { group, item, .. } => vec![(*group).into(), (*item).into()],
} }
} }
} }

26
crates/sync-generator/src/model.rs
View file

@ -22,7 +22,11 @@ pub fn module((model, sync_type): ModelWithSyncType) -> Module {
}); });
let model_stuff = match sync_type { let model_stuff = match sync_type {
ModelSyncType::Relation { item, group } => { ModelSyncType::Relation {
item,
group,
model_id,
} => {
let item_name_snake = snake_ident(item.name()); let item_name_snake = snake_ident(item.name());
let item_model_name_snake = snake_ident(item.related_model().name()); let item_model_name_snake = snake_ident(item.related_model().name());
@ -42,15 +46,27 @@ pub fn module((model, sync_type): ModelWithSyncType) -> Module {
} }
} }
pub const MODEL_ID: u16 = #model_id;
impl sd_sync::SyncModel for #model_name_snake::Types {
const MODEL_ID: u16 = MODEL_ID;
}
impl sd_sync::RelationSyncModel for #model_name_snake::Types { impl sd_sync::RelationSyncModel for #model_name_snake::Types {
type SyncId = SyncId; type SyncId = SyncId;
} }
}) })
} }
ModelSyncType::Shared { .. } => Some(quote! { ModelSyncType::Shared { model_id, .. } => Some(quote! {
impl sd_sync::SharedSyncModel for #model_name_snake::Types { pub const MODEL_ID: u16 = #model_id;
type SyncId = SyncId;
} impl sd_sync::SyncModel for #model_name_snake::Types {
const MODEL_ID: u16 = MODEL_ID;
}
impl sd_sync::SharedSyncModel for #model_name_snake::Types {
type SyncId = SyncId;
}
}), }),
_ => None, _ => None,
}; };

8
crates/sync-generator/src/sync_data.rs
View file

@ -24,7 +24,7 @@ pub fn r#enum(models: Vec<ModelWithSyncType>) -> TokenStream {
( (
quote!(#model_name_pascal(#model_name_snake::SyncId, sd_sync::CRDTOperationData)), quote!(#model_name_pascal(#model_name_snake::SyncId, sd_sync::CRDTOperationData)),
quote! { quote! {
prisma::#model_name_snake::NAME => #model_name_snake::MODEL_ID =>
Self::#model_name_pascal(rmpv::ext::from_value(op.record_id).ok()?, op.data) Self::#model_name_pascal(rmpv::ext::from_value(op.record_id).ok()?, op.data)
}, },
) )
@ -37,7 +37,7 @@ pub fn r#enum(models: Vec<ModelWithSyncType>) -> TokenStream {
let model_name_snake = snake_ident(model.name()); let model_name_snake = snake_ident(model.name());
let match_arms = match sync_type.as_ref()? { let match_arms = match sync_type.as_ref()? {
ModelSyncType::Shared { id } => { ModelSyncType::Shared { id, .. } => {
let (get_id, equals_value, id_name_snake, create_id) = match id.refine() { let (get_id, equals_value, id_name_snake, create_id) = match id.refine() {
RefinedFieldWalker::Relation(rel) => { RefinedFieldWalker::Relation(rel) => {
let scalar_field = rel.referenced_fields().unwrap().next().unwrap(); let scalar_field = rel.referenced_fields().unwrap().next().unwrap();
@ -110,7 +110,7 @@ pub fn r#enum(models: Vec<ModelWithSyncType>) -> TokenStream {
} }
} }
} }
ModelSyncType::Relation { item, group } => { ModelSyncType::Relation { item, group, .. } => {
let compound_id = format_ident!( let compound_id = format_ident!(
"{}", "{}",
group group
@ -228,7 +228,7 @@ pub fn r#enum(models: Vec<ModelWithSyncType>) -> TokenStream {
impl ModelSyncData { impl ModelSyncData {
pub fn from_op(op: sd_sync::CRDTOperation) -> Option<Self> { pub fn from_op(op: sd_sync::CRDTOperation) -> Option<Self> {
Some(match op.model.as_str() { Some(match op.model {
#(#matches),*, #(#matches),*,
_ => return None _ => return None
}) })

34
crates/sync/src/compressed.rs
View file

@ -9,7 +9,7 @@ pub type CompressedCRDTOperationsForModel = Vec<(rmpv::Value, Vec<CompressedCRDT
/// Stores a bunch of CRDTOperations in a more memory-efficient form for sending to the cloud. /// Stores a bunch of CRDTOperations in a more memory-efficient form for sending to the cloud.
#[derive(Serialize, Deserialize)] #[derive(Serialize, Deserialize)]
pub struct CompressedCRDTOperations( pub struct CompressedCRDTOperations(
pub(self) Vec<(Uuid, Vec<(String, CompressedCRDTOperationsForModel)>)>, pub(self) Vec<(Uuid, Vec<(u16, CompressedCRDTOperationsForModel)>)>,
); );
impl CompressedCRDTOperations { impl CompressedCRDTOperations {
@ -121,49 +121,49 @@ mod test {
CRDTOperation { CRDTOperation {
instance, instance,
timestamp: NTP64(0), timestamp: NTP64(0),
model: "FilePath".to_string(), model: 0,
record_id: rmpv::Value::Nil, record_id: rmpv::Value::Nil,
data: CRDTOperationData::Create, data: CRDTOperationData::Create,
}, },
CRDTOperation { CRDTOperation {
instance, instance,
timestamp: NTP64(0), timestamp: NTP64(0),
model: "FilePath".to_string(), model: 0,
record_id: rmpv::Value::Nil, record_id: rmpv::Value::Nil,
data: CRDTOperationData::Create, data: CRDTOperationData::Create,
}, },
CRDTOperation { CRDTOperation {
instance, instance,
timestamp: NTP64(0), timestamp: NTP64(0),
model: "FilePath".to_string(), model: 0,
record_id: rmpv::Value::Nil, record_id: rmpv::Value::Nil,
data: CRDTOperationData::Create, data: CRDTOperationData::Create,
}, },
CRDTOperation { CRDTOperation {
instance, instance,
timestamp: NTP64(0), timestamp: NTP64(0),
model: "Object".to_string(), model: 1,
record_id: rmpv::Value::Nil, record_id: rmpv::Value::Nil,
data: CRDTOperationData::Create, data: CRDTOperationData::Create,
}, },
CRDTOperation { CRDTOperation {
instance, instance,
timestamp: NTP64(0), timestamp: NTP64(0),
model: "Object".to_string(), model: 1,
record_id: rmpv::Value::Nil, record_id: rmpv::Value::Nil,
data: CRDTOperationData::Create, data: CRDTOperationData::Create,
}, },
CRDTOperation { CRDTOperation {
instance, instance,
timestamp: NTP64(0), timestamp: NTP64(0),
model: "FilePath".to_string(), model: 0,
record_id: rmpv::Value::Nil, record_id: rmpv::Value::Nil,
data: CRDTOperationData::Create, data: CRDTOperationData::Create,
}, },
CRDTOperation { CRDTOperation {
instance, instance,
timestamp: NTP64(0), timestamp: NTP64(0),
model: "FilePath".to_string(), model: 0,
record_id: rmpv::Value::Nil, record_id: rmpv::Value::Nil,
data: CRDTOperationData::Create, data: CRDTOperationData::Create,
}, },
@ -171,9 +171,9 @@ mod test {
let CompressedCRDTOperations(compressed) = CompressedCRDTOperations::new(uncompressed); let CompressedCRDTOperations(compressed) = CompressedCRDTOperations::new(uncompressed);
assert_eq!(&compressed[0].1[0].0, "FilePath"); assert_eq!(compressed[0].1[0].0, 0);
assert_eq!(&compressed[0].1[1].0, "Object"); assert_eq!(compressed[0].1[1].0, 1);
assert_eq!(&compressed[0].1[2].0, "FilePath"); assert_eq!(compressed[0].1[2].0, 0);
assert_eq!(compressed[0].1[0].1[0].1.len(), 3); assert_eq!(compressed[0].1[0].1[0].1.len(), 3);
assert_eq!(compressed[0].1[1].1[0].1.len(), 2); assert_eq!(compressed[0].1[1].1[0].1.len(), 2);
@ -186,7 +186,7 @@ mod test {
Uuid::new_v4(), Uuid::new_v4(),
vec![ vec![
( (
"FilePath".to_string(), 0,
vec![( vec![(
rmpv::Value::Nil, rmpv::Value::Nil,
vec![ vec![
@ -206,7 +206,7 @@ mod test {
)], )],
), ),
( (
"Object".to_string(), 1,
vec![( vec![(
rmpv::Value::Nil, rmpv::Value::Nil,
vec![ vec![
@ -222,7 +222,7 @@ mod test {
)], )],
), ),
( (
"FilePath".to_string(), 0,
vec![( vec![(
rmpv::Value::Nil, rmpv::Value::Nil,
vec![ vec![
@ -243,8 +243,8 @@ mod test {
let uncompressed = compressed.into_ops(); let uncompressed = compressed.into_ops();
assert_eq!(uncompressed.len(), 7); assert_eq!(uncompressed.len(), 7);
assert_eq!(uncompressed[2].model, "FilePath"); assert_eq!(uncompressed[2].model, 0);
assert_eq!(uncompressed[4].model, "Object"); assert_eq!(uncompressed[4].model, 1);
assert_eq!(uncompressed[6].model, "FilePath"); assert_eq!(uncompressed[6].model, 0);
} }
} }

2
crates/sync/src/crdt.rs
View file

@ -50,7 +50,7 @@ pub struct CRDTOperation {
pub instance: Uuid, pub instance: Uuid,
#[specta(type = u32)] #[specta(type = u32)]
pub timestamp: NTP64, pub timestamp: NTP64,
pub model: String, pub model: u16,
#[specta(type = serde_json::Value)] #[specta(type = serde_json::Value)]
pub record_id: rmpv::Value, pub record_id: rmpv::Value,
pub data: CRDTOperationData, pub data: CRDTOperationData,

9
crates/sync/src/factory.rs
View file

@ -1,4 +1,3 @@
use prisma_client_rust::ModelTypes;
use uhlc::HLC; use uhlc::HLC;
use uuid::Uuid; use uuid::Uuid;
@ -22,17 +21,13 @@ pub trait OperationFactory {
fn get_clock(&self) -> &HLC; fn get_clock(&self) -> &HLC;
fn get_instance(&self) -> Uuid; fn get_instance(&self) -> Uuid;
fn new_op<TSyncId: SyncId<Model = TModel>, TModel: ModelTypes>( fn new_op<TSyncId: SyncId>(&self, id: &TSyncId, data: CRDTOperationData) -> CRDTOperation {
&self,
id: &TSyncId,
data: CRDTOperationData,
) -> CRDTOperation {
let timestamp = self.get_clock().new_timestamp(); let timestamp = self.get_clock().new_timestamp();
CRDTOperation { CRDTOperation {
instance: self.get_instance(), instance: self.get_instance(),
timestamp: *timestamp.get_time(), timestamp: *timestamp.get_time(),
model: TModel::MODEL.to_string(), model: <TSyncId::Model as crate::SyncModel>::MODEL_ID,
record_id: msgpack!(id), record_id: msgpack!(id),
data, data,
} }

10
crates/sync/src/model_traits.rs
View file

@ -2,14 +2,14 @@ use prisma_client_rust::ModelTypes;
use serde::{de::DeserializeOwned, Serialize}; use serde::{de::DeserializeOwned, Serialize};
pub trait SyncId: Serialize + DeserializeOwned { pub trait SyncId: Serialize + DeserializeOwned {
type Model: ModelTypes; type Model: SyncModel;
} }
pub trait LocalSyncModel: ModelTypes { pub trait SyncModel: ModelTypes {
type SyncId: SyncId; const MODEL_ID: u16;
} }
pub trait SharedSyncModel: ModelTypes { pub trait SharedSyncModel: SyncModel {
type SyncId: SyncId; type SyncId: SyncId;
} }
@ -20,6 +20,6 @@ pub trait RelationSyncId: SyncId {
fn split(&self) -> (&Self::ItemSyncId, &Self::GroupSyncId); fn split(&self) -> (&Self::ItemSyncId, &Self::GroupSyncId);
} }
pub trait RelationSyncModel: ModelTypes { pub trait RelationSyncModel: SyncModel {
type SyncId: RelationSyncId; type SyncId: RelationSyncId;
} }