mirror of
https://github.com/spacedriveapp/spacedrive
synced 2024-07-02 11:13:29 +00:00
cleanup + update to prisma-client-rust 0.5.0
still waiting to rebase for changes to identifier
This commit is contained in:
Brendan Allan
parent
7a51c93a3a
commit
d164db7fc3
2
.vscode/settings.json
vendored
2
.vscode/settings.json
vendored
|
|
@ -19,7 +19,7 @@
|
|||
"upsert"
|
||||
],
|
||||
"[rust]": {
|
||||
"editor.defaultFormatter": "matklad.rust-analyzer"
|
||||
"editor.defaultFormatter": "rust-lang.rust-analyzer"
|
||||
},
|
||||
"rust-analyzer.procMacro.enable": true,
|
||||
"rust-analyzer.diagnostics.experimental.enable": false
|
||||
|
|
|
|||
58
Cargo.lock
generated
58
Cargo.lock
generated
|
|
@ -1628,7 +1628,7 @@ checksum = "3ee2393c4a91429dffb4bedf19f4d6abf27d8a732c8ce4980305d782e5426d57"
|
|||
[[package]]
|
||||
name = "datamodel"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/Brendonovich/prisma-engines?rev=8b94a80a2d8f12a2916fe28e3218162aa71e8539#8b94a80a2d8f12a2916fe28e3218162aa71e8539"
|
||||
source = "git+https://github.com/Brendonovich/prisma-engines?rev=5d34c9a567aaec4065a48d8cf6c2ee885e6ca281#5d34c9a567aaec4065a48d8cf6c2ee885e6ca281"
|
||||
dependencies = [
|
||||
"bigdecimal 0.2.2",
|
||||
"chrono",
|
||||
|
|
@ -1651,7 +1651,7 @@ dependencies = [
|
|||
[[package]]
|
||||
name = "datamodel-connector"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/Brendonovich/prisma-engines?rev=8b94a80a2d8f12a2916fe28e3218162aa71e8539#8b94a80a2d8f12a2916fe28e3218162aa71e8539"
|
||||
source = "git+https://github.com/Brendonovich/prisma-engines?rev=5d34c9a567aaec4065a48d8cf6c2ee885e6ca281#5d34c9a567aaec4065a48d8cf6c2ee885e6ca281"
|
||||
dependencies = [
|
||||
"diagnostics",
|
||||
"enumflags2",
|
||||
|
|
@ -1713,7 +1713,7 @@ dependencies = [
|
|||
[[package]]
|
||||
name = "diagnostics"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/Brendonovich/prisma-engines?rev=8b94a80a2d8f12a2916fe28e3218162aa71e8539#8b94a80a2d8f12a2916fe28e3218162aa71e8539"
|
||||
source = "git+https://github.com/Brendonovich/prisma-engines?rev=5d34c9a567aaec4065a48d8cf6c2ee885e6ca281#5d34c9a567aaec4065a48d8cf6c2ee885e6ca281"
|
||||
dependencies = [
|
||||
"colored",
|
||||
"pest",
|
||||
|
|
@ -1799,7 +1799,7 @@ checksum = "bd0c93bb4b0c6d9b77f4435b0ae98c24d17f1c45b2ff844c6151a07256ca923b"
|
|||
[[package]]
|
||||
name = "dml"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/Brendonovich/prisma-engines?rev=8b94a80a2d8f12a2916fe28e3218162aa71e8539#8b94a80a2d8f12a2916fe28e3218162aa71e8539"
|
||||
source = "git+https://github.com/Brendonovich/prisma-engines?rev=5d34c9a567aaec4065a48d8cf6c2ee885e6ca281#5d34c9a567aaec4065a48d8cf6c2ee885e6ca281"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"cuid",
|
||||
|
|
@ -1815,7 +1815,7 @@ dependencies = [
|
|||
[[package]]
|
||||
name = "dmmf"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/Brendonovich/prisma-engines?rev=8b94a80a2d8f12a2916fe28e3218162aa71e8539#8b94a80a2d8f12a2916fe28e3218162aa71e8539"
|
||||
source = "git+https://github.com/Brendonovich/prisma-engines?rev=5d34c9a567aaec4065a48d8cf6c2ee885e6ca281#5d34c9a567aaec4065a48d8cf6c2ee885e6ca281"
|
||||
dependencies = [
|
||||
"bigdecimal 0.2.2",
|
||||
"datamodel",
|
||||
|
|
@ -4480,7 +4480,7 @@ dependencies = [
|
|||
[[package]]
|
||||
name = "mongodb-client"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/Brendonovich/prisma-engines?rev=8b94a80a2d8f12a2916fe28e3218162aa71e8539#8b94a80a2d8f12a2916fe28e3218162aa71e8539"
|
||||
source = "git+https://github.com/Brendonovich/prisma-engines?rev=5d34c9a567aaec4065a48d8cf6c2ee885e6ca281#5d34c9a567aaec4065a48d8cf6c2ee885e6ca281"
|
||||
dependencies = [
|
||||
"mongodb",
|
||||
"once_cell",
|
||||
|
|
@ -4491,7 +4491,7 @@ dependencies = [
|
|||
[[package]]
|
||||
name = "mongodb-datamodel-connector"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/Brendonovich/prisma-engines?rev=8b94a80a2d8f12a2916fe28e3218162aa71e8539#8b94a80a2d8f12a2916fe28e3218162aa71e8539"
|
||||
source = "git+https://github.com/Brendonovich/prisma-engines?rev=5d34c9a567aaec4065a48d8cf6c2ee885e6ca281#5d34c9a567aaec4065a48d8cf6c2ee885e6ca281"
|
||||
dependencies = [
|
||||
"datamodel-connector",
|
||||
"enumflags2",
|
||||
|
|
@ -4503,7 +4503,7 @@ dependencies = [
|
|||
[[package]]
|
||||
name = "mongodb-query-connector"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/Brendonovich/prisma-engines?rev=8b94a80a2d8f12a2916fe28e3218162aa71e8539#8b94a80a2d8f12a2916fe28e3218162aa71e8539"
|
||||
source = "git+https://github.com/Brendonovich/prisma-engines?rev=5d34c9a567aaec4065a48d8cf6c2ee885e6ca281#5d34c9a567aaec4065a48d8cf6c2ee885e6ca281"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-trait",
|
||||
|
|
@ -4696,7 +4696,7 @@ dependencies = [
|
|||
[[package]]
|
||||
name = "native-types"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/Brendonovich/prisma-engines?rev=8b94a80a2d8f12a2916fe28e3218162aa71e8539#8b94a80a2d8f12a2916fe28e3218162aa71e8539"
|
||||
source = "git+https://github.com/Brendonovich/prisma-engines?rev=5d34c9a567aaec4065a48d8cf6c2ee885e6ca281#5d34c9a567aaec4065a48d8cf6c2ee885e6ca281"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
|
|
@ -5328,7 +5328,7 @@ dependencies = [
|
|||
[[package]]
|
||||
name = "parser-database"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/Brendonovich/prisma-engines?rev=8b94a80a2d8f12a2916fe28e3218162aa71e8539#8b94a80a2d8f12a2916fe28e3218162aa71e8539"
|
||||
source = "git+https://github.com/Brendonovich/prisma-engines?rev=5d34c9a567aaec4065a48d8cf6c2ee885e6ca281#5d34c9a567aaec4065a48d8cf6c2ee885e6ca281"
|
||||
dependencies = [
|
||||
"diagnostics",
|
||||
"enumflags2",
|
||||
|
|
@ -5734,32 +5734,36 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "prisma-client-rust"
|
||||
version = "0.4.0"
|
||||
source = "git+https://github.com/Brendonovich/prisma-client-rust.git?tag=0.4.0#4c6c4bf93c8eb5f3a16efca161983a9537e5e53b"
|
||||
version = "0.4.1"
|
||||
source = "git+https://github.com/Brendonovich/prisma-client-rust.git?tag=0.5.0#31a864ae09e835ff046469345bf6e413ae54a30f"
|
||||
dependencies = [
|
||||
"bigdecimal 0.2.2",
|
||||
"chrono",
|
||||
"datamodel",
|
||||
"graphql-parser",
|
||||
"prisma-models",
|
||||
"query-connector",
|
||||
"query-core",
|
||||
"request-handlers",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"thiserror",
|
||||
"user-facing-errors",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "prisma-client-rust-cli"
|
||||
version = "0.4.0"
|
||||
source = "git+https://github.com/Brendonovich/prisma-client-rust.git?tag=0.4.0#4c6c4bf93c8eb5f3a16efca161983a9537e5e53b"
|
||||
version = "0.4.1"
|
||||
source = "git+https://github.com/Brendonovich/prisma-client-rust.git?tag=0.5.0#31a864ae09e835ff046469345bf6e413ae54a30f"
|
||||
dependencies = [
|
||||
"convert_case 0.5.0",
|
||||
"datamodel",
|
||||
"directories",
|
||||
"flate2",
|
||||
"http",
|
||||
"prisma-models",
|
||||
"query-core",
|
||||
"quote",
|
||||
"regex",
|
||||
"request-handlers",
|
||||
"reqwest",
|
||||
"serde",
|
||||
"serde_json",
|
||||
|
|
@ -5770,7 +5774,7 @@ dependencies = [
|
|||
[[package]]
|
||||
name = "prisma-inflector"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/Brendonovich/prisma-engines?rev=8b94a80a2d8f12a2916fe28e3218162aa71e8539#8b94a80a2d8f12a2916fe28e3218162aa71e8539"
|
||||
source = "git+https://github.com/Brendonovich/prisma-engines?rev=5d34c9a567aaec4065a48d8cf6c2ee885e6ca281#5d34c9a567aaec4065a48d8cf6c2ee885e6ca281"
|
||||
dependencies = [
|
||||
"once_cell",
|
||||
"regex",
|
||||
|
|
@ -5780,7 +5784,7 @@ dependencies = [
|
|||
[[package]]
|
||||
name = "prisma-models"
|
||||
version = "0.0.0"
|
||||
source = "git+https://github.com/Brendonovich/prisma-engines?rev=8b94a80a2d8f12a2916fe28e3218162aa71e8539#8b94a80a2d8f12a2916fe28e3218162aa71e8539"
|
||||
source = "git+https://github.com/Brendonovich/prisma-engines?rev=5d34c9a567aaec4065a48d8cf6c2ee885e6ca281#5d34c9a567aaec4065a48d8cf6c2ee885e6ca281"
|
||||
dependencies = [
|
||||
"bigdecimal 0.2.2",
|
||||
"chrono",
|
||||
|
|
@ -5801,7 +5805,7 @@ dependencies = [
|
|||
[[package]]
|
||||
name = "prisma-value"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/Brendonovich/prisma-engines?rev=8b94a80a2d8f12a2916fe28e3218162aa71e8539#8b94a80a2d8f12a2916fe28e3218162aa71e8539"
|
||||
source = "git+https://github.com/Brendonovich/prisma-engines?rev=5d34c9a567aaec4065a48d8cf6c2ee885e6ca281#5d34c9a567aaec4065a48d8cf6c2ee885e6ca281"
|
||||
dependencies = [
|
||||
"base64 0.12.3",
|
||||
"bigdecimal 0.2.2",
|
||||
|
|
@ -5991,7 +5995,7 @@ dependencies = [
|
|||
[[package]]
|
||||
name = "query-connector"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/Brendonovich/prisma-engines?rev=8b94a80a2d8f12a2916fe28e3218162aa71e8539#8b94a80a2d8f12a2916fe28e3218162aa71e8539"
|
||||
source = "git+https://github.com/Brendonovich/prisma-engines?rev=5d34c9a567aaec4065a48d8cf6c2ee885e6ca281#5d34c9a567aaec4065a48d8cf6c2ee885e6ca281"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-trait",
|
||||
|
|
@ -6011,7 +6015,7 @@ dependencies = [
|
|||
[[package]]
|
||||
name = "query-core"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/Brendonovich/prisma-engines?rev=8b94a80a2d8f12a2916fe28e3218162aa71e8539#8b94a80a2d8f12a2916fe28e3218162aa71e8539"
|
||||
source = "git+https://github.com/Brendonovich/prisma-engines?rev=5d34c9a567aaec4065a48d8cf6c2ee885e6ca281#5d34c9a567aaec4065a48d8cf6c2ee885e6ca281"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64 0.12.3",
|
||||
|
|
@ -6258,7 +6262,7 @@ dependencies = [
|
|||
[[package]]
|
||||
name = "request-handlers"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/Brendonovich/prisma-engines?rev=8b94a80a2d8f12a2916fe28e3218162aa71e8539#8b94a80a2d8f12a2916fe28e3218162aa71e8539"
|
||||
source = "git+https://github.com/Brendonovich/prisma-engines?rev=5d34c9a567aaec4065a48d8cf6c2ee885e6ca281#5d34c9a567aaec4065a48d8cf6c2ee885e6ca281"
|
||||
dependencies = [
|
||||
"bigdecimal 0.2.2",
|
||||
"connection-string",
|
||||
|
|
@ -6549,7 +6553,7 @@ dependencies = [
|
|||
[[package]]
|
||||
name = "schema-ast"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/Brendonovich/prisma-engines?rev=8b94a80a2d8f12a2916fe28e3218162aa71e8539#8b94a80a2d8f12a2916fe28e3218162aa71e8539"
|
||||
source = "git+https://github.com/Brendonovich/prisma-engines?rev=5d34c9a567aaec4065a48d8cf6c2ee885e6ca281#5d34c9a567aaec4065a48d8cf6c2ee885e6ca281"
|
||||
dependencies = [
|
||||
"diagnostics",
|
||||
"pest",
|
||||
|
|
@ -7094,7 +7098,7 @@ dependencies = [
|
|||
[[package]]
|
||||
name = "sql-datamodel-connector"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/Brendonovich/prisma-engines?rev=8b94a80a2d8f12a2916fe28e3218162aa71e8539#8b94a80a2d8f12a2916fe28e3218162aa71e8539"
|
||||
source = "git+https://github.com/Brendonovich/prisma-engines?rev=5d34c9a567aaec4065a48d8cf6c2ee885e6ca281#5d34c9a567aaec4065a48d8cf6c2ee885e6ca281"
|
||||
dependencies = [
|
||||
"datamodel-connector",
|
||||
"enumflags2",
|
||||
|
|
@ -7107,7 +7111,7 @@ dependencies = [
|
|||
[[package]]
|
||||
name = "sql-query-connector"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/Brendonovich/prisma-engines?rev=8b94a80a2d8f12a2916fe28e3218162aa71e8539#8b94a80a2d8f12a2916fe28e3218162aa71e8539"
|
||||
source = "git+https://github.com/Brendonovich/prisma-engines?rev=5d34c9a567aaec4065a48d8cf6c2ee885e6ca281#5d34c9a567aaec4065a48d8cf6c2ee885e6ca281"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-trait",
|
||||
|
|
@ -8217,7 +8221,7 @@ dependencies = [
|
|||
[[package]]
|
||||
name = "user-facing-error-macros"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/Brendonovich/prisma-engines?rev=8b94a80a2d8f12a2916fe28e3218162aa71e8539#8b94a80a2d8f12a2916fe28e3218162aa71e8539"
|
||||
source = "git+https://github.com/Brendonovich/prisma-engines?rev=5d34c9a567aaec4065a48d8cf6c2ee885e6ca281#5d34c9a567aaec4065a48d8cf6c2ee885e6ca281"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
|
|
@ -8227,7 +8231,7 @@ dependencies = [
|
|||
[[package]]
|
||||
name = "user-facing-errors"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/Brendonovich/prisma-engines?rev=8b94a80a2d8f12a2916fe28e3218162aa71e8539#8b94a80a2d8f12a2916fe28e3218162aa71e8539"
|
||||
source = "git+https://github.com/Brendonovich/prisma-engines?rev=5d34c9a567aaec4065a48d8cf6c2ee885e6ca281#5d34c9a567aaec4065a48d8cf6c2ee885e6ca281"
|
||||
dependencies = [
|
||||
"backtrace",
|
||||
"indoc",
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ int-enum = "0.4.0"
|
|||
|
||||
# Project dependencies
|
||||
ts-rs = "6.1"
|
||||
prisma-client-rust = { git = "https://github.com/Brendonovich/prisma-client-rust.git", tag = "0.4.0" }
|
||||
prisma-client-rust = { git = "https://github.com/Brendonovich/prisma-client-rust.git", tag = "0.5.0" }
|
||||
walkdir = "^2.3.2"
|
||||
libp2p = { version = "0.43.0", optional = true }
|
||||
lazy_static = "1.4.0"
|
||||
|
|
|
|||
|
|
@ -4,4 +4,4 @@ version = "0.1.0"
|
|||
edition = "2021"
|
||||
|
||||
[dependencies]
|
||||
prisma-client-rust-cli = { git = "https://github.com/Brendonovich/prisma-client-rust", tag = "0.4.0" }
|
||||
prisma-client-rust-cli = { git = "https://github.com/Brendonovich/prisma-client-rust", tag = "0.5.0" }
|
||||
|
|
@ -1,14 +1,15 @@
|
|||
use crate::prisma::{self, migration};
|
||||
use crate::prisma::migration;
|
||||
use crate::CoreContext;
|
||||
use anyhow::Result;
|
||||
use data_encoding::HEXLOWER;
|
||||
use include_dir::{include_dir, Dir};
|
||||
use prisma_client_rust::raw;
|
||||
use ring::digest::{Context, Digest, SHA256};
|
||||
use std::ffi::OsStr;
|
||||
use std::io::{BufReader, Read};
|
||||
|
||||
const INIT_MIGRATION: &str =
|
||||
include_str!("../../prisma/migrations/0_migration_table/migration.sql");
|
||||
include_str!("../../prisma/migrations/migration_table/migration.sql");
|
||||
static MIGRATIONS_DIR: Dir = include_dir!("$CARGO_MANIFEST_DIR/prisma/migrations");
|
||||
|
||||
pub fn sha256_digest<R: Read>(mut reader: R) -> Result<Digest> {
|
||||
|
|
@ -28,9 +29,9 @@ pub async fn run_migrations(ctx: &CoreContext) -> Result<()> {
|
|||
let client = &ctx.database;
|
||||
|
||||
match client
|
||||
._query_raw::<serde_json::Value>(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name='_migrations'",
|
||||
)
|
||||
._query_raw::<serde_json::Value>(raw!(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name='_migrations'"
|
||||
))
|
||||
.await
|
||||
{
|
||||
Ok(data) => {
|
||||
|
|
@ -38,7 +39,7 @@ pub async fn run_migrations(ctx: &CoreContext) -> Result<()> {
|
|||
#[cfg(debug_assertions)]
|
||||
println!("Migration table does not exist");
|
||||
// execute migration
|
||||
match client._execute_raw(INIT_MIGRATION).await {
|
||||
match client._execute_raw(raw!(INIT_MIGRATION)).await {
|
||||
Ok(_) => {}
|
||||
Err(e) => {
|
||||
println!("Failed to create migration table: {}", e);
|
||||
|
|
@ -46,9 +47,9 @@ pub async fn run_migrations(ctx: &CoreContext) -> Result<()> {
|
|||
};
|
||||
|
||||
let value: Vec<serde_json::Value> = client
|
||||
._query_raw(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name='_migrations'",
|
||||
)
|
||||
._query_raw(raw!(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name='_migrations'"
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
|
|
@ -70,15 +71,15 @@ pub async fn run_migrations(ctx: &CoreContext) -> Result<()> {
|
|||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
// migration_subdirs.sort_by(|a, b| {
|
||||
// let a_name = a.path().file_name().unwrap().to_str().unwrap();
|
||||
// let b_name = b.path().file_name().unwrap().to_str().unwrap();
|
||||
migration_subdirs.sort_by(|a, b| {
|
||||
let a_name = a.path().file_name().unwrap().to_str().unwrap();
|
||||
let b_name = b.path().file_name().unwrap().to_str().unwrap();
|
||||
|
||||
// let a_time = a_name[..14].parse::<i64>().unwrap();
|
||||
// let b_time = b_name[..14].parse::<i64>().unwrap();
|
||||
let a_time = a_name[..14].parse::<i64>().unwrap();
|
||||
let b_time = b_name[..14].parse::<i64>().unwrap();
|
||||
|
||||
// a_time.cmp(&b_time)
|
||||
// });
|
||||
a_time.cmp(&b_time)
|
||||
});
|
||||
|
||||
for subdir in migration_subdirs {
|
||||
println!("{:?}", subdir.path());
|
||||
|
|
@ -117,7 +118,7 @@ pub async fn run_migrations(ctx: &CoreContext) -> Result<()> {
|
|||
.await?;
|
||||
|
||||
for (i, step) in steps.iter().enumerate() {
|
||||
match client._execute_raw(&format!("{};", step)).await {
|
||||
match client._execute_raw(raw!(*step)).await {
|
||||
Ok(_) => {
|
||||
#[cfg(debug_assertions)]
|
||||
println!("Step {} ran successfully", i);
|
||||
|
|
|
|||
|
|
@ -10,11 +10,11 @@ use crate::{
|
|||
};
|
||||
use anyhow::Result;
|
||||
use futures::executor::block_on;
|
||||
use prisma_client_rust::Direction;
|
||||
use prisma_client_rust::prisma_models::PrismaValue;
|
||||
use prisma_client_rust::{raw, Direction};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::checksum::generate_cas_id;
|
||||
|
||||
#[derive(Deserialize, Serialize, Debug)]
|
||||
pub struct FileCreated {
|
||||
pub id: i32,
|
||||
|
|
@ -51,62 +51,74 @@ impl Job for FileIdentifierJob {
|
|||
let location_path = location.path.unwrap_or("".to_string());
|
||||
|
||||
let ctx = tokio::task::spawn_blocking(move || {
|
||||
let mut completed: usize = 0;
|
||||
let mut cursor: i32 = 1;
|
||||
let completed: usize = 0;
|
||||
let cursor: i32 = 1;
|
||||
|
||||
while completed < task_count {
|
||||
let file_paths = block_on(get_orphan_file_paths(&ctx.core_ctx, cursor)).unwrap();
|
||||
println!("Processing {:?} orphan files. ({} completed of {})", file_paths.len(), completed, task_count);
|
||||
while completed < task_count {
|
||||
let file_paths = block_on(get_orphan_file_paths(&ctx.core_ctx, cursor)).unwrap();
|
||||
println!(
|
||||
"Processing {:?} orphan files. ({} completed of {})",
|
||||
file_paths.len(),
|
||||
completed,
|
||||
task_count
|
||||
);
|
||||
|
||||
let mut rows: Vec<String> = Vec::new();
|
||||
// only rows that have a valid cas_id to be inserted
|
||||
for file_path in file_paths.iter() {
|
||||
match prepare_file_values(&location_path, file_path) {
|
||||
Ok(data) => {
|
||||
rows.push(data);
|
||||
}
|
||||
Err(e) => {
|
||||
|
||||
println!("Error processing file: {}", e);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
}
|
||||
if rows.len() == 0 {
|
||||
println!("No orphan files to process, finishing...");
|
||||
break;
|
||||
}
|
||||
let insert_files = format!(
|
||||
r#"INSERT INTO files (cas_id, size_in_bytes) VALUES {} ON CONFLICT (cas_id) DO NOTHING RETURNING id, cas_id"#,
|
||||
rows.join(", ")
|
||||
);
|
||||
|
||||
let files: Vec<FileCreated> = block_on(db._query_raw(&insert_files)).unwrap();
|
||||
let mut rows = Vec::new();
|
||||
// only rows that have a valid cas_id to be inserted
|
||||
for file_path in file_paths.iter() {
|
||||
match prepare_file_values(&location_path, file_path) {
|
||||
Ok(data) => {
|
||||
rows.push(PrismaValue::List(data));
|
||||
}
|
||||
Err(e) => {
|
||||
println!("Error processing file: {}", e);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
}
|
||||
if rows.len() == 0 {
|
||||
println!("No orphan files to process, finishing...");
|
||||
break;
|
||||
}
|
||||
|
||||
for file in files.iter() {
|
||||
let update_file_path = format!(
|
||||
r#"UPDATE file_paths SET file_id = "{}" WHERE temp_cas_id = "{}""#,
|
||||
file.id, file.cas_id
|
||||
);
|
||||
block_on(db._execute_raw(&update_file_path)).unwrap();
|
||||
}
|
||||
panic!("temp_cas_id no longer exists. please fix this code!");
|
||||
|
||||
let last_row = file_paths.last().unwrap();
|
||||
// let files: Vec<FileCreated> = block_on(db._query_raw(raw!(
|
||||
// &format!(
|
||||
// "INSERT INTO files (cas_id, size_in_bytes) VALUES {} ON CONFLICT (cas_id) DO NOTHING RETURNING id, cas_id",
|
||||
// vec!["({}, {}, {})"; rows.len()].join(",")
|
||||
// ),
|
||||
// PrismaValue::List(rows)
|
||||
// ))).unwrap();
|
||||
|
||||
cursor = last_row.id;
|
||||
|
||||
completed += 1;
|
||||
ctx.progress(vec![
|
||||
JobReportUpdate::CompletedTaskCount(completed),
|
||||
JobReportUpdate::Message(format!(
|
||||
"Processed {} of {} orphan files",
|
||||
completed,
|
||||
task_count
|
||||
)),
|
||||
]);
|
||||
}
|
||||
ctx
|
||||
}).await?;
|
||||
// for file in files.iter() {
|
||||
// block_on(
|
||||
// db.file_path()
|
||||
// .find_many(vec![file_path::temp_cas_id::equals(Some(file.cas_id.clone()))])
|
||||
// .update(vec![
|
||||
// file_path::id::set(file.id)
|
||||
// ])
|
||||
// .exec()
|
||||
// ).unwrap();
|
||||
// }
|
||||
|
||||
// let last_row = file_paths.last().unwrap();
|
||||
|
||||
// cursor = last_row.id;
|
||||
|
||||
// completed += 1;
|
||||
// ctx.progress(vec![
|
||||
// JobReportUpdate::CompletedTaskCount(completed),
|
||||
// JobReportUpdate::Message(format!(
|
||||
// "Processed {} of {} orphan files",
|
||||
// completed,
|
||||
// task_count
|
||||
// )),
|
||||
// ]);
|
||||
}
|
||||
ctx
|
||||
})
|
||||
.await?;
|
||||
|
||||
let remaining = count_orphan_file_paths(&ctx.core_ctx).await?;
|
||||
|
||||
|
|
@ -131,11 +143,10 @@ struct CountRes {
|
|||
pub async fn count_orphan_file_paths(ctx: &CoreContext) -> Result<usize, FileError> {
|
||||
let db = &ctx.database;
|
||||
let files_count = db
|
||||
._query_raw::<CountRes>(
|
||||
r#"SELECT COUNT(*) AS count FROM file_paths WHERE file_id IS NULL AND is_dir IS FALSE"#,
|
||||
)
|
||||
._query_raw::<CountRes>(raw!(
|
||||
"SELECT COUNT(*) AS count FROM file_paths WHERE file_id IS NULL AND is_dir IS FALSE"
|
||||
))
|
||||
.await?;
|
||||
println!("files: {:?}", files_count);
|
||||
Ok(files_count[0].count.unwrap_or(0))
|
||||
}
|
||||
|
||||
|
|
@ -159,19 +170,21 @@ pub async fn get_orphan_file_paths(
|
|||
Ok(files)
|
||||
}
|
||||
|
||||
pub fn prepare_file_values(location_path: &str, file_path: &file_path::Data) -> Result<String> {
|
||||
pub fn prepare_file_values(
|
||||
location_path: &str,
|
||||
file_path: &file_path::Data,
|
||||
) -> Result<Vec<PrismaValue>> {
|
||||
let path = format!("{}/{}", location_path, file_path.materialized_path);
|
||||
let metadata = fs::metadata(&path)?;
|
||||
let cas_id = {
|
||||
if !file_path.is_dir {
|
||||
// TODO: remove unwrap
|
||||
let mut x = generate_cas_id(&path, metadata.len()).unwrap();
|
||||
x.truncate(16);
|
||||
x
|
||||
let mut ret = generate_cas_id(&path, metadata.len()).unwrap();
|
||||
ret.truncate(16);
|
||||
ret
|
||||
} else {
|
||||
"".to_string()
|
||||
}
|
||||
};
|
||||
// TODO: add all metadata
|
||||
Ok(format!("(\"{}\",\"{}\")", cas_id, "0"))
|
||||
|
||||
Ok(vec![PrismaValue::String(cas_id), PrismaValue::Int(0)])
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,8 +1,9 @@
|
|||
use crate::file::cas::checksum::generate_cas_id;
|
||||
use crate::sys::locations::{create_location, LocationResource};
|
||||
use crate::CoreContext;
|
||||
use anyhow::{anyhow, Result};
|
||||
use chrono::{DateTime, SecondsFormat, Utc};
|
||||
use prisma_client_rust::prisma_models::PrismaValue;
|
||||
use prisma_client_rust::raw;
|
||||
use prisma_client_rust::raw::Raw;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::ffi::OsStr;
|
||||
use std::{collections::HashMap, fs, path::Path, path::PathBuf, time::Instant};
|
||||
|
|
@ -35,7 +36,7 @@ pub async fn scan_path(
|
|||
}
|
||||
// grab the next id so we can increment in memory for batch inserting
|
||||
let first_file_id = match db
|
||||
._query_raw::<QueryRes>(r#"SELECT MAX(id) id FROM file_paths"#)
|
||||
._query_raw::<QueryRes>(raw!("SELECT MAX(id) id FROM file_paths"))
|
||||
.await
|
||||
{
|
||||
Ok(rows) => rows[0].id.unwrap_or(0),
|
||||
|
|
@ -87,8 +88,8 @@ pub async fn scan_path(
|
|||
.unwrap_or("");
|
||||
let parent_dir_id = dirs.get(&*parent_path);
|
||||
|
||||
let str = match path.as_os_str().to_str() {
|
||||
Some(str) => str,
|
||||
let path_str = match path.as_os_str().to_str() {
|
||||
Some(path_str) => path_str,
|
||||
None => {
|
||||
println!("Error reading file {}", &path.display());
|
||||
continue;
|
||||
|
|
@ -96,7 +97,7 @@ pub async fn scan_path(
|
|||
};
|
||||
|
||||
on_progress(vec![
|
||||
ScanProgress::Message(format!("{}", str)),
|
||||
ScanProgress::Message(format!("{}", path_str)),
|
||||
ScanProgress::ChunkCount(paths.len() / BATCH_SIZE),
|
||||
]);
|
||||
|
||||
|
|
@ -128,18 +129,19 @@ pub async fn scan_path(
|
|||
on_progress(vec![
|
||||
ScanProgress::SavedChunks(i as usize),
|
||||
ScanProgress::Message(format!(
|
||||
"Writing {} of {} to library",
|
||||
"Writing {} of {} to db",
|
||||
i * chunk.len(),
|
||||
paths.len(),
|
||||
)),
|
||||
]);
|
||||
|
||||
// vector to store active models
|
||||
let mut files: Vec<String> = Vec::new();
|
||||
let mut files: Vec<PrismaValue> = Vec::new();
|
||||
|
||||
for (file_path, file_id, parent_dir_id, is_dir) in chunk {
|
||||
files.push(
|
||||
files.extend(
|
||||
match prepare_values(&file_path, *file_id, &location, parent_dir_id, *is_dir) {
|
||||
Ok(file) => file,
|
||||
Ok(values) => values.to_vec(),
|
||||
Err(e) => {
|
||||
println!("Error creating file model from path {:?}: {}", file_path, e);
|
||||
continue;
|
||||
|
|
@ -147,15 +149,19 @@ pub async fn scan_path(
|
|||
},
|
||||
);
|
||||
}
|
||||
let raw_sql = format!(
|
||||
r#"
|
||||
INSERT INTO file_paths (id, is_dir, location_id, materialized_path, name, extension, parent_id)
|
||||
VALUES {}
|
||||
"#,
|
||||
files.join(", ")
|
||||
|
||||
let raw = Raw::new(
|
||||
&format!("
|
||||
INSERT INTO file_paths (id, is_dir, location_id, materialized_path, name, extension, parent_id)
|
||||
VALUES {}
|
||||
",
|
||||
vec!["({}, {}, {}, {}, {}, {}, {})"; chunk.len()].join(", ")
|
||||
),
|
||||
files
|
||||
);
|
||||
// println!("{}", raw_sql);
|
||||
let count = db._execute_raw(&raw_sql).await;
|
||||
|
||||
let count = db._execute_raw(raw).await;
|
||||
|
||||
println!("Inserted {:?} records", count);
|
||||
}
|
||||
println!(
|
||||
|
|
@ -175,8 +181,8 @@ fn prepare_values(
|
|||
location: &LocationResource,
|
||||
parent_id: &Option<i32>,
|
||||
is_dir: bool,
|
||||
) -> Result<String> {
|
||||
// let metadata = fs::metadata(&file_path)?;
|
||||
) -> Result<[PrismaValue; 7]> {
|
||||
let metadata = fs::metadata(&file_path)?;
|
||||
let location_path = location.path.as_ref().unwrap().as_str();
|
||||
// let size = metadata.len();
|
||||
let name;
|
||||
|
|
@ -203,37 +209,18 @@ fn prepare_values(
|
|||
None => return Err(anyhow!("{}", file_path.to_str().unwrap_or_default())),
|
||||
};
|
||||
|
||||
// let cas_id = {
|
||||
// if !metadata.is_dir() {
|
||||
// // TODO: remove unwrap, skip and make sure to continue loop
|
||||
// let mut x = generate_cas_id(&file_path.to_str().unwrap(), metadata.len()).unwrap();
|
||||
// x.truncate(16);
|
||||
// x
|
||||
// } else {
|
||||
// "".to_string()
|
||||
// }
|
||||
// };
|
||||
|
||||
// let date_created: DateTime<Utc> = metadata.created().unwrap().into();
|
||||
// let parsed_date_created = date_created.to_rfc3339_opts(SecondsFormat::Millis, true);
|
||||
|
||||
let values = format!(
|
||||
"({}, {}, {}, \"{}\", \"{}\", \"{}\", {})",
|
||||
id,
|
||||
is_dir,
|
||||
location.id,
|
||||
materialized_path,
|
||||
name,
|
||||
extension.to_lowercase(),
|
||||
let values = [
|
||||
PrismaValue::Int(id as i64),
|
||||
PrismaValue::Boolean(metadata.is_dir()),
|
||||
PrismaValue::Int(location.id as i64),
|
||||
PrismaValue::String(materialized_path.to_string()),
|
||||
PrismaValue::String(name),
|
||||
PrismaValue::String(extension.to_lowercase()),
|
||||
parent_id
|
||||
.clone()
|
||||
.map(|id| format!("\"{}\"", &id))
|
||||
.unwrap_or("NULL".to_string()),
|
||||
// parsed_date_created,
|
||||
// cas_id
|
||||
);
|
||||
|
||||
println!("{}", values);
|
||||
.map(|id| PrismaValue::Int(id as i64))
|
||||
.unwrap_or(PrismaValue::Null),
|
||||
];
|
||||
|
||||
Ok(values)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,7 +4,6 @@ use thiserror::Error;
|
|||
use ts_rs::TS;
|
||||
|
||||
use crate::{
|
||||
crypto::encryption::EncryptionAlgorithm,
|
||||
prisma::{self, file, file_path},
|
||||
sys::SysError,
|
||||
};
|
||||
|
|
@ -91,7 +90,7 @@ impl Into<File> for file::Data {
|
|||
integrity_checksum: self.integrity_checksum,
|
||||
kind: IntEnum::from_int(self.kind).unwrap(),
|
||||
size_in_bytes: self.size_in_bytes.to_string(),
|
||||
// encryption: EncryptionAlgorithm::from_int(self.encryption).unwrap(),
|
||||
// encryption: EncryptionAlgorithm::from_int(self.encryption).unwrap(),
|
||||
ipfs_id: self.ipfs_id,
|
||||
hidden: self.hidden,
|
||||
favorite: self.favorite,
|
||||
|
|
@ -100,9 +99,9 @@ impl Into<File> for file::Data {
|
|||
has_thumbstrip: self.has_thumbstrip,
|
||||
has_video_preview: self.has_video_preview,
|
||||
comment: self.comment,
|
||||
date_created: self.date_created,
|
||||
date_modified: self.date_modified,
|
||||
date_indexed: self.date_indexed,
|
||||
date_created: self.date_created.into(),
|
||||
date_modified: self.date_modified.into(),
|
||||
date_indexed: self.date_indexed.into(),
|
||||
paths: vec![],
|
||||
}
|
||||
}
|
||||
|
|
@ -117,14 +116,14 @@ impl Into<FilePath> for file_path::Data {
|
|||
file_id: self.file_id,
|
||||
parent_id: self.parent_id,
|
||||
location_id: self.location_id,
|
||||
date_indexed: self.date_indexed,
|
||||
// permissions: self.permissions,
|
||||
date_indexed: self.date_indexed.into(),
|
||||
// permissions: self.permissions,
|
||||
has_local_thumbnail: false,
|
||||
name: self.name,
|
||||
extension: self.extension,
|
||||
// temp_cas_id: self.temp_cas_id,
|
||||
date_created: self.date_created,
|
||||
date_modified: self.date_modified,
|
||||
// temp_cas_id: self.temp_cas_id,
|
||||
date_created: self.date_created.into(),
|
||||
date_modified: self.date_modified.into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -52,7 +52,7 @@ impl Jobs {
|
|||
self.job_queue.push(job);
|
||||
}
|
||||
}
|
||||
pub fn ingest_queue(&mut self, ctx: &CoreContext, job: Box<dyn Job>) {
|
||||
pub fn ingest_queue(&mut self, _ctx: &CoreContext, job: Box<dyn Job>) {
|
||||
self.job_queue.push(job);
|
||||
}
|
||||
pub async fn complete(&mut self, ctx: &CoreContext, job_id: String) {
|
||||
|
|
@ -124,8 +124,8 @@ impl Into<JobReport> for job::Data {
|
|||
status: JobStatus::from_int(self.status).unwrap(),
|
||||
task_count: self.task_count,
|
||||
completed_task_count: self.completed_task_count,
|
||||
date_created: self.date_created,
|
||||
date_modified: self.date_modified,
|
||||
date_created: self.date_created.into(),
|
||||
date_modified: self.date_modified.into(),
|
||||
message: String::new(),
|
||||
seconds_elapsed: self.seconds_elapsed,
|
||||
}
|
||||
|
|
@ -170,7 +170,7 @@ impl JobReport {
|
|||
job::status::set(self.status.int_value()),
|
||||
job::task_count::set(self.task_count),
|
||||
job::completed_task_count::set(self.completed_task_count),
|
||||
job::date_modified::set(chrono::Utc::now()),
|
||||
job::date_modified::set(chrono::Utc::now().into()),
|
||||
job::seconds_elapsed::set(self.seconds_elapsed),
|
||||
])
|
||||
.exec()
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
use crate::{
|
||||
node::state,
|
||||
prisma::{library, library_statistics::*},
|
||||
sys::{self, volumes::Volume},
|
||||
sys::volumes::Volume,
|
||||
CoreContext,
|
||||
};
|
||||
use fs_extra::dir::get_size;
|
||||
|
|
@ -69,6 +69,7 @@ impl Statistics {
|
|||
};
|
||||
Ok(library_statistics_db.into())
|
||||
}
|
||||
|
||||
pub async fn calculate(ctx: &CoreContext) -> Result<Statistics, LibraryError> {
|
||||
let config = state::get();
|
||||
let db = &ctx.database;
|
||||
|
|
@ -135,20 +136,22 @@ impl Statistics {
|
|||
};
|
||||
|
||||
db.library_statistics()
|
||||
.upsert(library_id::equals(library_local_id))
|
||||
.create(
|
||||
library_id::set(library_local_id),
|
||||
vec![library_db_size::set(statistics.library_db_size.clone())],
|
||||
.upsert(
|
||||
library_id::equals(library_local_id),
|
||||
(
|
||||
library_id::set(library_local_id),
|
||||
vec![library_db_size::set(statistics.library_db_size.clone())],
|
||||
),
|
||||
vec![
|
||||
total_file_count::set(statistics.total_file_count.clone()),
|
||||
total_bytes_used::set(statistics.total_bytes_used.clone()),
|
||||
total_bytes_capacity::set(statistics.total_bytes_capacity.clone()),
|
||||
total_bytes_free::set(statistics.total_bytes_free.clone()),
|
||||
total_unique_bytes::set(statistics.total_unique_bytes.clone()),
|
||||
preview_media_bytes::set(statistics.preview_media_bytes.clone()),
|
||||
library_db_size::set(statistics.library_db_size.clone()),
|
||||
],
|
||||
)
|
||||
.update(vec![
|
||||
total_file_count::set(statistics.total_file_count.clone()),
|
||||
total_bytes_used::set(statistics.total_bytes_used.clone()),
|
||||
total_bytes_capacity::set(statistics.total_bytes_capacity.clone()),
|
||||
total_bytes_free::set(statistics.total_bytes_free.clone()),
|
||||
total_unique_bytes::set(statistics.total_unique_bytes.clone()),
|
||||
preview_media_bytes::set(statistics.preview_media_bytes.clone()),
|
||||
library_db_size::set(statistics.library_db_size.clone()),
|
||||
])
|
||||
.exec()
|
||||
.await?;
|
||||
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@ impl Into<LocationResource> for location::Data {
|
|||
available_capacity: self.available_capacity,
|
||||
is_removable: self.is_removable,
|
||||
is_online: self.is_online,
|
||||
date_created: self.date_created,
|
||||
date_created: self.date_created.into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -35,28 +35,30 @@ impl Volume {
|
|||
// enter all volumes associate with this client add to db
|
||||
for volume in volumes {
|
||||
db.volume()
|
||||
.upsert(node_id_mount_point_name(
|
||||
config.node_id.clone(),
|
||||
volume.mount_point.to_string(),
|
||||
volume.name.to_string(),
|
||||
))
|
||||
.create(
|
||||
node_id::set(config.node_id),
|
||||
name::set(volume.name),
|
||||
mount_point::set(volume.mount_point),
|
||||
.upsert(
|
||||
node_id_mount_point_name(
|
||||
config.node_id.clone(),
|
||||
volume.mount_point.to_string(),
|
||||
volume.name.to_string(),
|
||||
),
|
||||
(
|
||||
node_id::set(config.node_id),
|
||||
name::set(volume.name),
|
||||
mount_point::set(volume.mount_point),
|
||||
vec![
|
||||
disk_type::set(volume.disk_type.clone()),
|
||||
filesystem::set(volume.file_system.clone()),
|
||||
total_bytes_capacity::set(volume.total_capacity.to_string()),
|
||||
total_bytes_available::set(volume.available_capacity.to_string()),
|
||||
],
|
||||
),
|
||||
vec![
|
||||
disk_type::set(volume.disk_type.clone()),
|
||||
filesystem::set(volume.file_system.clone()),
|
||||
disk_type::set(volume.disk_type),
|
||||
filesystem::set(volume.file_system),
|
||||
total_bytes_capacity::set(volume.total_capacity.to_string()),
|
||||
total_bytes_available::set(volume.available_capacity.to_string()),
|
||||
],
|
||||
)
|
||||
.update(vec![
|
||||
disk_type::set(volume.disk_type),
|
||||
filesystem::set(volume.file_system),
|
||||
total_bytes_capacity::set(volume.total_capacity.to_string()),
|
||||
total_bytes_available::set(volume.available_capacity.to_string()),
|
||||
])
|
||||
.exec()
|
||||
.await?;
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue