From b620e1e174599913e5548eac0772592f06e5f595 Mon Sep 17 00:00:00 2001 From: Brendan Allan Date: Mon, 1 May 2023 17:45:25 +0800 Subject: [PATCH] paginated location explorer (#774) * paginated location explorer * remove props from GridView * formatting --- Cargo.lock | 8 +- Cargo.toml | 6 +- core/src/api/locations.rs | 46 +++++--- core/src/api/tags.rs | 1 + .../app/$libraryId/Explorer/GridView.tsx | 3 +- interface/app/$libraryId/Explorer/View.tsx | 6 ++ interface/app/$libraryId/Explorer/index.tsx | 5 +- interface/app/$libraryId/location/$id.tsx | 102 +++++++++++------- packages/client/src/core.ts | 40 +++---- 9 files changed, 133 insertions(+), 84 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d764d43ff..2825facb0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5497,7 +5497,7 @@ dependencies = [ [[package]] name = "prisma-client-rust" version = "0.6.8" -source = "git+https://github.com/Brendonovich/prisma-client-rust?rev=4eba61cafa31b5ac3638fee9e0d7e7d8304d6e47#4eba61cafa31b5ac3638fee9e0d7e7d8304d6e47" +source = "git+https://github.com/Brendonovich/prisma-client-rust?rev=5d8029e0a0b590e1b8f674339ba880114a1becc8#5d8029e0a0b590e1b8f674339ba880114a1becc8" dependencies = [ "base64 0.13.1", "bigdecimal", @@ -5530,7 +5530,7 @@ dependencies = [ [[package]] name = "prisma-client-rust-cli" version = "0.6.8" -source = "git+https://github.com/Brendonovich/prisma-client-rust?rev=4eba61cafa31b5ac3638fee9e0d7e7d8304d6e47#4eba61cafa31b5ac3638fee9e0d7e7d8304d6e47" +source = "git+https://github.com/Brendonovich/prisma-client-rust?rev=5d8029e0a0b590e1b8f674339ba880114a1becc8#5d8029e0a0b590e1b8f674339ba880114a1becc8" dependencies = [ "directories", "flate2", @@ -5550,7 +5550,7 @@ dependencies = [ [[package]] name = "prisma-client-rust-macros" version = "0.6.8" -source = "git+https://github.com/Brendonovich/prisma-client-rust?rev=4eba61cafa31b5ac3638fee9e0d7e7d8304d6e47#4eba61cafa31b5ac3638fee9e0d7e7d8304d6e47" +source = "git+https://github.com/Brendonovich/prisma-client-rust?rev=5d8029e0a0b590e1b8f674339ba880114a1becc8#5d8029e0a0b590e1b8f674339ba880114a1becc8" dependencies = [ "convert_case 0.6.0", "proc-macro2", @@ -5562,7 +5562,7 @@ dependencies = [ [[package]] name = "prisma-client-rust-sdk" version = "0.6.8" -source = "git+https://github.com/Brendonovich/prisma-client-rust?rev=4eba61cafa31b5ac3638fee9e0d7e7d8304d6e47#4eba61cafa31b5ac3638fee9e0d7e7d8304d6e47" +source = "git+https://github.com/Brendonovich/prisma-client-rust?rev=5d8029e0a0b590e1b8f674339ba880114a1becc8#5d8029e0a0b590e1b8f674339ba880114a1becc8" dependencies = [ "convert_case 0.5.0", "dmmf", diff --git a/Cargo.toml b/Cargo.toml index cdb616d74..7a75a6f15 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,19 +13,19 @@ members = [ ] [workspace.dependencies] -prisma-client-rust = { git = "https://github.com/Brendonovich/prisma-client-rust", rev = "4eba61cafa31b5ac3638fee9e0d7e7d8304d6e47", features = [ +prisma-client-rust = { git = "https://github.com/Brendonovich/prisma-client-rust", rev = "5d8029e0a0b590e1b8f674339ba880114a1becc8", features = [ "rspc", "sqlite-create-many", "migrations", "sqlite", ] } -prisma-client-rust-cli = { git = "https://github.com/Brendonovich/prisma-client-rust", rev = "4eba61cafa31b5ac3638fee9e0d7e7d8304d6e47", features = [ +prisma-client-rust-cli = { git = "https://github.com/Brendonovich/prisma-client-rust", rev = "5d8029e0a0b590e1b8f674339ba880114a1becc8", features = [ "rspc", "sqlite-create-many", "migrations", "sqlite", ] } -prisma-client-rust-sdk = { git = "https://github.com/Brendonovich/prisma-client-rust", rev = "4eba61cafa31b5ac3638fee9e0d7e7d8304d6e47", features = [ +prisma-client-rust-sdk = { git = "https://github.com/Brendonovich/prisma-client-rust", rev = "5d8029e0a0b590e1b8f674339ba880114a1becc8", features = [ "sqlite", ] } diff --git a/core/src/api/locations.rs b/core/src/api/locations.rs index 610fef6ae..2019f0a06 100644 --- a/core/src/api/locations.rs +++ b/core/src/api/locations.rs @@ -7,6 +7,7 @@ use crate::{ LocationError, LocationUpdateArgs, }, prisma::{file_path, indexer_rule, indexer_rules_in_location, location, object, tag}, + util::db::{chain_optional_iter, uuid_to_bytes}, }; use std::{ @@ -17,6 +18,7 @@ use std::{ use rspc::{self, alpha::AlphaRouter, ErrorCode}; use serde::{Deserialize, Serialize}; use specta::Type; +use uuid::Uuid; use super::{utils::library, Ctx, R}; @@ -46,6 +48,7 @@ pub enum ExplorerItem { pub struct ExplorerData { pub context: ExplorerContext, pub items: Vec, + pub cursor: Option>, } file_path::include!(file_path_with_object { object }); @@ -80,9 +83,12 @@ pub(crate) fn mount() -> AlphaRouter { #[derive(Clone, Serialize, Deserialize, Type, Debug)] pub struct LocationExplorerArgs { pub location_id: i32, + #[specta(optional)] pub path: Option, pub limit: i32, - pub cursor: Option, + #[specta(optional)] + pub cursor: Option>, + #[specta(optional)] pub kind: Option>, } @@ -90,6 +96,8 @@ pub(crate) fn mount() -> AlphaRouter { .query(|(_, library), args: LocationExplorerArgs| async move { let Library { db, .. } = &library; + dbg!(&args); + let location = find_location(&library, args.location_id) .exec() .await? @@ -127,19 +135,28 @@ pub(crate) fn mount() -> AlphaRouter { .map(|kinds| kinds.into_iter().collect::>()) .unwrap_or_default(); - let mut file_paths = db - .file_path() - .find_many(if directory_id.is_some() { - vec![ - file_path::location_id::equals(location.id), - file_path::parent_id::equals(directory_id), - ] - } else { - vec![file_path::location_id::equals(location.id)] - }) - .include(file_path_with_object::include()) - .exec() - .await?; + let (mut file_paths, cursor) = { + let mut query = db + .file_path() + .find_many(chain_optional_iter( + [file_path::location_id::equals(location.id)], + [directory_id.map(Some).map(file_path::parent_id::equals)], + )) + .take((args.limit + 1) as i64); + + if let Some(cursor) = args.cursor { + query = query.cursor(file_path::pub_id::equals(cursor)); + } + + let mut results = query + .include(file_path_with_object::include()) + .exec() + .await?; + + let cursor = results.pop().map(|r| r.pub_id); + + (results, cursor) + }; if !expected_kinds.is_empty() { file_paths = file_paths @@ -174,6 +191,7 @@ pub(crate) fn mount() -> AlphaRouter { Ok(ExplorerData { context: ExplorerContext::Location(location), items, + cursor, }) }) }) diff --git a/core/src/api/tags.rs b/core/src/api/tags.rs index 00083fbdb..a0f709707 100644 --- a/core/src/api/tags.rs +++ b/core/src/api/tags.rs @@ -83,6 +83,7 @@ pub(crate) fn mount() -> AlphaRouter { Ok(ExplorerData { context: ExplorerContext::Tag(tag), items, + cursor: None, }) }) }) diff --git a/interface/app/$libraryId/Explorer/GridView.tsx b/interface/app/$libraryId/Explorer/GridView.tsx index b9c8b34aa..efb5719e3 100644 --- a/interface/app/$libraryId/Explorer/GridView.tsx +++ b/interface/app/$libraryId/Explorer/GridView.tsx @@ -1,8 +1,9 @@ import { useVirtualizer } from '@tanstack/react-virtual'; import clsx from 'clsx'; -import { memo, useEffect, useLayoutEffect, useMemo, useState } from 'react'; +import { memo, useEffect, useMemo, useState } from 'react'; import { useKey, useOnWindowResize } from 'rooks'; import { ExplorerItem, formatBytes } from '@sd/client'; +import { Button } from '@sd/ui'; import { getExplorerStore, useExplorerStore } from '~/hooks/useExplorerStore'; import RenameTextBox from './File/RenameTextBox'; import Thumb from './File/Thumb'; diff --git a/interface/app/$libraryId/Explorer/View.tsx b/interface/app/$libraryId/Explorer/View.tsx index 90fc8a299..1a278b055 100644 --- a/interface/app/$libraryId/Explorer/View.tsx +++ b/interface/app/$libraryId/Explorer/View.tsx @@ -10,6 +10,7 @@ import { } from 'react'; import { createSearchParams, useNavigate } from 'react-router-dom'; import { ExplorerItem, isPath, useLibraryContext } from '@sd/client'; +import { Button } from '@sd/ui'; import { getExplorerStore, useExplorerStore } from '~/hooks/useExplorerStore'; import { TOP_BAR_HEIGHT } from '../TopBar'; import DismissibleNotice from './DismissibleNotice'; @@ -71,6 +72,8 @@ export const ViewItem = ({ interface Props { data: ExplorerItem[]; + onLoadMore?(): void; + hasNextPage?: boolean; } interface ExplorerView { @@ -101,6 +104,9 @@ export default memo((props: Props) => { {layoutMode === 'grid' && } {layoutMode === 'rows' && } {layoutMode === 'media' && } + {props.hasNextPage && ( + + )} ); diff --git a/interface/app/$libraryId/Explorer/index.tsx b/interface/app/$libraryId/Explorer/index.tsx index a3fc483a1..dc40da6a7 100644 --- a/interface/app/$libraryId/Explorer/index.tsx +++ b/interface/app/$libraryId/Explorer/index.tsx @@ -1,5 +1,4 @@ import { useEffect } from 'react'; -import { useParams } from 'react-router'; import { useKey } from 'rooks'; import { ExplorerData, rspc, useLibraryContext } from '@sd/client'; import { getExplorerStore, useExplorerStore } from '~/hooks/useExplorerStore'; @@ -13,6 +12,8 @@ interface Props { // and it's not exactly compatible with search // data?: ExplorerData; items?: ExplorerData['items']; + onLoadMore?(): void; + hasNextPage?: boolean; } export default function Explorer(props: Props) { @@ -43,7 +44,7 @@ export default function Explorer(props: Props) {
- {props.items && } + {props.items && }
diff --git a/interface/app/$libraryId/location/$id.tsx b/interface/app/$libraryId/location/$id.tsx index 8c6678524..860471a7e 100644 --- a/interface/app/$libraryId/location/$id.tsx +++ b/interface/app/$libraryId/location/$id.tsx @@ -1,7 +1,8 @@ +import { useInfiniteQuery } from '@tanstack/react-query'; import { ArrowClockwise, Key, Tag } from 'phosphor-react'; -import { useEffect } from 'react'; +import { useEffect, useMemo } from 'react'; import { useParams, useSearchParams } from 'react-router-dom'; -import { useLibraryMutation, useLibraryQuery } from '@sd/client'; +import { ExplorerData, rspc, useLibraryContext, useLibraryMutation } from '@sd/client'; import { getExplorerStore, useExplorerStore } from '~/hooks/useExplorerStore'; import { useExplorerTopBarOptions } from '~/hooks/useExplorerTopBarOptions'; import Explorer from '../Explorer'; @@ -21,9 +22,66 @@ export function useExplorerParams() { } export const Component = () => { + const { location_id, path } = useExplorerParams(); + // we destructure this since `mutate` is a stable reference but the object it's in is not + const quickRescan = useLibraryMutation('locations.quickRescan'); + + const explorerStore = useExplorerStore(); + const explorerState = getExplorerStore(); + + useEffect(() => { + explorerState.locationId = location_id; + if (location_id !== null) quickRescan.mutate({ location_id, sub_path: path }); + }, [explorerState, location_id, path, quickRescan.mutate]); + + if (location_id === null) throw new Error(`location_id is null!`); + + const ctx = rspc.useContext(); + const { library } = useLibraryContext(); + + const query = useInfiniteQuery({ + queryKey: [ + 'locations.getExplorerData', + { + library_id: library.uuid, + arg: { + location_id, + path: explorerStore.layoutMode === 'media' ? null : path, + limit: 100, + kind: explorerStore.layoutMode === 'media' ? [5, 7] : null + } + } + ] as const, + queryFn: async ({ pageParam: cursor, queryKey }): Promise => { + const arg = queryKey[1]; + (arg.arg as any).cursor = cursor; + + return await ctx.client.query(['locations.getExplorerData', arg]); + }, + getNextPageParam: (lastPage) => lastPage.cursor ?? undefined + }); + + const items = useMemo(() => query.data?.pages.flatMap((d) => d.items), [query.data]); + + return ( + <> + +
+ +
+ + ); +}; + +const useToolBarOptions = () => { const store = useExplorerStore(); const { explorerViewOptions, explorerControlOptions } = useExplorerTopBarOptions(); - const toolBarOptions: ToolOption[][] = [ + + return [ explorerViewOptions, [ { @@ -54,41 +112,5 @@ export const Component = () => { } ], explorerControlOptions - ]; - - const { location_id, path, limit } = useExplorerParams(); - // we destructure this since `mutate` is a stable reference but the object it's in is not - const { mutate: mutateQuickRescan, ...quickRescan } = - useLibraryMutation('locations.quickRescan'); - - const explorerStore = useExplorerStore(); - - const explorerState = getExplorerStore(); - - useEffect(() => { - explorerState.locationId = location_id; - if (location_id !== null) mutateQuickRescan({ location_id, sub_path: path }); - }, [explorerState, location_id, path, mutateQuickRescan]); - - if (location_id === null) throw new Error(`location_id is null!`); - - const explorerData = useLibraryQuery([ - 'locations.getExplorerData', - { - location_id, - path: explorerStore.layoutMode === 'media' ? null : path, - limit, - cursor: null, - kind: explorerStore.layoutMode === 'media' ? [5, 7] : null - } - ]); - - return ( - <> - -
- -
- - ); + ] satisfies ToolOption[][]; }; diff --git a/packages/client/src/core.ts b/packages/client/src/core.ts index 6e4ca8118..5c3492901 100644 --- a/packages/client/src/core.ts +++ b/packages/client/src/core.ts @@ -118,9 +118,7 @@ export type GenerateThumbsForLocationArgs = { id: number; path: string } export type LibraryConfigWrapped = { uuid: string; config: LibraryConfig } -export type TagUpdateArgs = { id: number; name: string | null; color: string | null } - -export type FilePathWithObject = { id: number; pub_id: number[]; is_dir: boolean; cas_id: string | null; integrity_checksum: string | null; location_id: number; materialized_path: string; name: string; extension: string; size_in_bytes: string; inode: number[]; device: number[]; object_id: number | null; parent_id: number[] | null; key_id: number | null; date_created: string; date_modified: string; date_indexed: string; object: Object | null } +export type LocationExplorerArgs = { location_id: number; path?: string | null; limit: number; cursor?: number[] | null; kind?: number[] | null } /** * These parameters define the password-hashing level. @@ -139,6 +137,8 @@ export type Params = "Standard" | "Hardened" | "Paranoid" */ export type LocationUpdateArgs = { id: number; name: string | null; generate_preview_media: boolean | null; sync_preview_media: boolean | null; hidden: boolean | null; indexer_rules_ids: number[] } +export type RenameFileArgs = { location_id: number; file_name: string; new_file_name: string } + /** * Represents the operating system which the remote peer is running. * This is not used internally and predominantly is designed to be used for display purposes by the embedding application. @@ -154,6 +154,8 @@ export type StoredKey = { uuid: string; version: StoredKeyVersion; key_type: Sto export type OnboardingConfig = { password: Protected; algorithm: Algorithm; hashing_algorithm: HashingAlgorithm } +export type ExplorerContext = ({ type: "Location" } & Location) | ({ type: "Tag" } & Tag) + export type Volume = { name: string; mount_point: string; total_capacity: string; available_capacity: string; is_removable: boolean; disk_type: string | null; file_system: string | null; is_root_filesystem: boolean } /** @@ -170,6 +172,10 @@ export type IndexerRuleCreateArgs = { kind: RuleKind; name: string; dry_run: boo export type EditLibraryArgs = { id: string; name: string | null; description: string | null } +export type ObjectWithFilePaths = { id: number; pub_id: number[]; kind: number; key_id: number | null; hidden: boolean; favorite: boolean; important: boolean; has_thumbnail: boolean; has_thumbstrip: boolean; has_video_preview: boolean; ipfs_id: string | null; note: string | null; date_created: string; file_paths: FilePath[] } + +export type LightScanArgs = { location_id: number; sub_path: string } + export type BuildInfo = { version: string; commit: string } /** @@ -181,7 +187,7 @@ export type Nonce = { XChaCha20Poly1305: number[] } | { Aes256Gcm: number[] } export type UnlockKeyManagerArgs = { password: Protected; secret_key: Protected } -export type SetNoteArgs = { id: number; note: string | null } +export type TagCreateArgs = { name: string; color: string } export type FileEncryptorJobInit = { location_id: number; path_id: number; key_uuid: string; algorithm: Algorithm; metadata: boolean; preview_media: boolean; output_path: string | null } @@ -198,23 +204,23 @@ export type CRDTOperation = { node: string; timestamp: number; id: string; typ: */ export type Salt = number[] -export type LightScanArgs = { location_id: number; sub_path: string } +export type TagUpdateArgs = { id: number; name: string | null; color: string | null } export type GetArgs = { id: number } export type FileCutterJobInit = { source_location_id: number; source_path_id: number; target_location_id: number; target_path: string } -export type ExplorerItem = { type: "Path"; has_thumbnail: boolean; item: FilePathWithObject } | { type: "Object"; has_thumbnail: boolean; item: ObjectWithFilePaths } - export type JobStatus = "Queued" | "Running" | "Completed" | "Canceled" | "Failed" | "Paused" export type ObjectValidatorArgs = { id: number; path: string } export type FileEraserJobInit = { location_id: number; path_id: number; passes: string } -export type SetFavoriteArgs = { id: number; favorite: boolean } +export type ExplorerItem = { type: "Path"; has_thumbnail: boolean; item: FilePathWithObject } | { type: "Object"; has_thumbnail: boolean; item: ObjectWithFilePaths } -export type RenameFileArgs = { location_id: number; file_name: string; new_file_name: string } +export type FilePathWithObject = { id: number; pub_id: number[]; is_dir: boolean; cas_id: string | null; integrity_checksum: string | null; location_id: number; materialized_path: string; name: string; extension: string; size_in_bytes: string; inode: number[]; device: number[]; object_id: number | null; parent_id: number[] | null; key_id: number | null; date_created: string; date_modified: string; date_indexed: string; object: Object | null } + +export type TagAssignArgs = { object_id: number; tag_id: number; unassign: boolean } export type FileDeleterJobInit = { location_id: number; path_id: number } @@ -225,12 +231,12 @@ export type FilePath = { id: number; pub_id: number[]; is_dir: boolean; cas_id: */ export type Algorithm = "XChaCha20Poly1305" | "Aes256Gcm" -export type LocationExplorerArgs = { location_id: number; path: string | null; limit: number; cursor: string | null; kind: number[] | null } - export type JobReport = { id: string; name: string; action: string | null; data: number[] | null; metadata: any | null; is_background: boolean; created_at: string | null; started_at: string | null; completed_at: string | null; parent_id: string | null; status: JobStatus; task_count: number; completed_task_count: number; message: string } export type OwnedOperationItem = { id: any; data: OwnedOperationData } +export type SetFavoriteArgs = { id: number; favorite: boolean } + export type CRDTOperationType = SharedOperation | RelationOperation | OwnedOperation /** @@ -242,10 +248,6 @@ export type SpacedropArgs = { peer_id: PeerId; file_path: string[] } export type NodeState = ({ id: string; name: string; p2p_port: number | null; p2p_email: string | null; p2p_img_url: string | null }) & { data_path: string } -export type TagCreateArgs = { name: string; color: string } - -export type ExplorerContext = ({ type: "Location" } & Location) | ({ type: "Tag" } & Tag) - export type OwnedOperation = { model: string; items: OwnedOperationItem[] } export type SharedOperation = { record_id: any; model: string; data: SharedOperationData } @@ -260,8 +262,6 @@ export type KeyAddArgs = { algorithm: Algorithm; hashing_algorithm: HashingAlgor export type RuleKind = "AcceptFilesByGlob" | "RejectFilesByGlob" | "AcceptIfChildrenDirectoriesArePresent" | "RejectIfChildrenDirectoriesArePresent" -export type TagAssignArgs = { object_id: number; tag_id: number; unassign: boolean } - /** * `LocationCreateArgs` is the argument received from the client using `rspc` to create a new location. * It has the actual path and a vector of indexer rules ids, to create many-to-many relationships @@ -282,6 +282,8 @@ export type SharedOperationData = SharedOperationCreateData | { field: string; v export type MediaData = { id: number; pixel_width: number | null; pixel_height: number | null; longitude: number | null; latitude: number | null; fps: number | null; capture_device_make: string | null; capture_device_model: string | null; capture_device_software: string | null; duration_seconds: number | null; codecs: string | null; streams: number | null } +export type ExplorerData = { context: ExplorerContext; items: ExplorerItem[]; cursor: number[] | null } + export type IndexerRule = { id: number; kind: number; name: string; default: boolean; parameters: number[]; date_created: string; date_modified: string } export type FileCopierJobInit = { source_location_id: number; source_path_id: number; target_location_id: number; target_path: string; target_file_name_suffix: string | null } @@ -295,7 +297,7 @@ export type Object = { id: number; pub_id: number[]; kind: number; key_id: numbe */ export type HashingAlgorithm = { name: "Argon2id"; params: Params } | { name: "BalloonBlake3"; params: Params } -export type ExplorerData = { context: ExplorerContext; items: ExplorerItem[] } +export type SetNoteArgs = { id: number; note: string | null } export type LocationWithIndexerRules = { id: number; pub_id: number[]; node_id: number; name: string; path: string; total_capacity: number | null; available_capacity: number | null; is_archived: boolean; generate_preview_media: boolean; sync_preview_media: boolean; hidden: boolean; date_created: string; indexer_rules: ({ indexer_rule: IndexerRule })[] } @@ -314,8 +316,6 @@ export type Protected = T export type Statistics = { id: number; date_captured: string; total_object_count: number; library_db_size: string; total_bytes_used: string; total_bytes_capacity: string; total_unique_bytes: string; total_bytes_free: string; preview_media_bytes: string } -export type ObjectWithFilePaths = { id: number; pub_id: number[]; kind: number; key_id: number | null; hidden: boolean; favorite: boolean; important: boolean; has_thumbnail: boolean; has_thumbstrip: boolean; has_video_preview: boolean; ipfs_id: string | null; note: string | null; date_created: string; file_paths: FilePath[] } - export type RestoreBackupArgs = { password: Protected; secret_key: Protected; path: string } export type Tag = { id: number; pub_id: number[]; name: string | null; color: string | null; total_objects: number | null; redundancy_goal: number | null; date_created: string; date_modified: string }