paginated location explorer (#774)

* paginated location explorer * remove props from GridView * formatting
2024-07-04 11:03:27 +00:00 · 2023-05-01 17:45:25 +08:00 · 2023-05-01 17:45:25 +08:00 · b620e1e174
parent 7cee2b2651
commit b620e1e174
9 changed files with 133 additions and 84 deletions
--- a/Cargo.lock
+++ b/Cargo.lock
@ -5497,7 +5497,7 @@ dependencies = [
 [[package]]
 name = "prisma-client-rust"
 version = "0.6.8"
-source = "git+https://github.com/Brendonovich/prisma-client-rust?rev=4eba61cafa31b5ac3638fee9e0d7e7d8304d6e47#4eba61cafa31b5ac3638fee9e0d7e7d8304d6e47"
+source = "git+https://github.com/Brendonovich/prisma-client-rust?rev=5d8029e0a0b590e1b8f674339ba880114a1becc8#5d8029e0a0b590e1b8f674339ba880114a1becc8"
 dependencies = [
 "base64 0.13.1",
 "bigdecimal",
@ -5530,7 +5530,7 @@ dependencies = [
 [[package]]
 name = "prisma-client-rust-cli"
 version = "0.6.8"
-source = "git+https://github.com/Brendonovich/prisma-client-rust?rev=4eba61cafa31b5ac3638fee9e0d7e7d8304d6e47#4eba61cafa31b5ac3638fee9e0d7e7d8304d6e47"
+source = "git+https://github.com/Brendonovich/prisma-client-rust?rev=5d8029e0a0b590e1b8f674339ba880114a1becc8#5d8029e0a0b590e1b8f674339ba880114a1becc8"
 dependencies = [
 "directories",
 "flate2",
@ -5550,7 +5550,7 @@ dependencies = [
 [[package]]
 name = "prisma-client-rust-macros"
 version = "0.6.8"
-source = "git+https://github.com/Brendonovich/prisma-client-rust?rev=4eba61cafa31b5ac3638fee9e0d7e7d8304d6e47#4eba61cafa31b5ac3638fee9e0d7e7d8304d6e47"
+source = "git+https://github.com/Brendonovich/prisma-client-rust?rev=5d8029e0a0b590e1b8f674339ba880114a1becc8#5d8029e0a0b590e1b8f674339ba880114a1becc8"
 dependencies = [
 "convert_case 0.6.0",
 "proc-macro2",
@ -5562,7 +5562,7 @@ dependencies = [
 [[package]]
 name = "prisma-client-rust-sdk"
 version = "0.6.8"
-source = "git+https://github.com/Brendonovich/prisma-client-rust?rev=4eba61cafa31b5ac3638fee9e0d7e7d8304d6e47#4eba61cafa31b5ac3638fee9e0d7e7d8304d6e47"
+source = "git+https://github.com/Brendonovich/prisma-client-rust?rev=5d8029e0a0b590e1b8f674339ba880114a1becc8#5d8029e0a0b590e1b8f674339ba880114a1becc8"
 dependencies = [
 "convert_case 0.5.0",
 "dmmf",
--- a/Cargo.toml
+++ b/Cargo.toml
@ -13,19 +13,19 @@ members = [
 ]

 [workspace.dependencies]
-prisma-client-rust =  { git = "https://github.com/Brendonovich/prisma-client-rust", rev = "4eba61cafa31b5ac3638fee9e0d7e7d8304d6e47", features = [
+prisma-client-rust =  { git = "https://github.com/Brendonovich/prisma-client-rust", rev = "5d8029e0a0b590e1b8f674339ba880114a1becc8", features = [
  "rspc",
  "sqlite-create-many",
  "migrations",
  "sqlite",
 ] }
-prisma-client-rust-cli = { git = "https://github.com/Brendonovich/prisma-client-rust", rev = "4eba61cafa31b5ac3638fee9e0d7e7d8304d6e47", features = [
+prisma-client-rust-cli = { git = "https://github.com/Brendonovich/prisma-client-rust", rev = "5d8029e0a0b590e1b8f674339ba880114a1becc8", features = [
  "rspc",
  "sqlite-create-many",
  "migrations",
  "sqlite",
 ] }
-prisma-client-rust-sdk = { git = "https://github.com/Brendonovich/prisma-client-rust", rev = "4eba61cafa31b5ac3638fee9e0d7e7d8304d6e47", features = [
+prisma-client-rust-sdk = { git = "https://github.com/Brendonovich/prisma-client-rust", rev = "5d8029e0a0b590e1b8f674339ba880114a1becc8", features = [
  "sqlite",
 ] }

--- a/core/src/api/locations.rs
+++ b/core/src/api/locations.rs
@ -7,6 +7,7 @@ use crate::{
 		LocationError, LocationUpdateArgs,
 	},
 	prisma::{file_path, indexer_rule, indexer_rules_in_location, location, object, tag},
+	util::db::{chain_optional_iter, uuid_to_bytes},
 };

 use std::{
@ -17,6 +18,7 @@ use std::{
 use rspc::{self, alpha::AlphaRouter, ErrorCode};
 use serde::{Deserialize, Serialize};
 use specta::Type;
+use uuid::Uuid;

 use super::{utils::library, Ctx, R};

@ -46,6 +48,7 @@ pub enum ExplorerItem {
 pub struct ExplorerData {
 	pub context: ExplorerContext,
 	pub items: Vec<ExplorerItem>,
+	pub cursor: Option<Vec<u8>>,
 }

 file_path::include!(file_path_with_object { object });
@ -80,9 +83,12 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
 			#[derive(Clone, Serialize, Deserialize, Type, Debug)]
 			pub struct LocationExplorerArgs {
 				pub location_id: i32,
+				#[specta(optional)]
 				pub path: Option<String>,
 				pub limit: i32,
-				pub cursor: Option<String>,
+				#[specta(optional)]
+				pub cursor: Option<Vec<u8>>,
+				#[specta(optional)]
 				pub kind: Option<Vec<i32>>,
 			}

@ -90,6 +96,8 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
 				.query(|(_, library), args: LocationExplorerArgs| async move {
 					let Library { db, .. } = &library;

+					dbg!(&args);
+
 					let location = find_location(&library, args.location_id)
 						.exec()
 						.await?
@ -127,19 +135,28 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
 						.map(|kinds| kinds.into_iter().collect::<BTreeSet<_>>())
 						.unwrap_or_default();

-					let mut file_paths = db
-						.file_path()
-						.find_many(if directory_id.is_some() {
-							vec![
-								file_path::location_id::equals(location.id),
-								file_path::parent_id::equals(directory_id),
-							]
-						} else {
-							vec![file_path::location_id::equals(location.id)]
-						})
-						.include(file_path_with_object::include())
-						.exec()
-						.await?;
+					let (mut file_paths, cursor) = {
+						let mut query = db
+							.file_path()
+							.find_many(chain_optional_iter(
+								[file_path::location_id::equals(location.id)],
+								[directory_id.map(Some).map(file_path::parent_id::equals)],
+							))
+							.take((args.limit + 1) as i64);
+
+						if let Some(cursor) = args.cursor {
+							query = query.cursor(file_path::pub_id::equals(cursor));
+						}
+
+						let mut results = query
+							.include(file_path_with_object::include())
+							.exec()
+							.await?;
+
+						let cursor = results.pop().map(|r| r.pub_id);
+
+						(results, cursor)
+					};

 					if !expected_kinds.is_empty() {
 						file_paths = file_paths
@ -174,6 +191,7 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
 					Ok(ExplorerData {
 						context: ExplorerContext::Location(location),
 						items,
+						cursor,
 					})
 				})
 		})
--- a/core/src/api/tags.rs
+++ b/core/src/api/tags.rs
@ -83,6 +83,7 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
 					Ok(ExplorerData {
 						context: ExplorerContext::Tag(tag),
 						items,
+						cursor: None,
 					})
 				})
 		})
--- a/interface/app/$libraryId/Explorer/GridView.tsx
+++ b/interface/app/$libraryId/Explorer/GridView.tsx
@ -1,8 +1,9 @@
 import { useVirtualizer } from '@tanstack/react-virtual';
 import clsx from 'clsx';
-import { memo, useEffect, useLayoutEffect, useMemo, useState } from 'react';
+import { memo, useEffect, useMemo, useState } from 'react';
 import { useKey, useOnWindowResize } from 'rooks';
 import { ExplorerItem, formatBytes } from '@sd/client';
+import { Button } from '@sd/ui';
 import { getExplorerStore, useExplorerStore } from '~/hooks/useExplorerStore';
 import RenameTextBox from './File/RenameTextBox';
 import Thumb from './File/Thumb';
--- a/interface/app/$libraryId/Explorer/View.tsx
+++ b/interface/app/$libraryId/Explorer/View.tsx
@ -10,6 +10,7 @@ import {
 } from 'react';
 import { createSearchParams, useNavigate } from 'react-router-dom';
 import { ExplorerItem, isPath, useLibraryContext } from '@sd/client';
+import { Button } from '@sd/ui';
 import { getExplorerStore, useExplorerStore } from '~/hooks/useExplorerStore';
 import { TOP_BAR_HEIGHT } from '../TopBar';
 import DismissibleNotice from './DismissibleNotice';
@ -71,6 +72,8 @@ export const ViewItem = ({

 interface Props {
 	data: ExplorerItem[];
+	onLoadMore?(): void;
+	hasNextPage?: boolean;
 }

 interface ExplorerView {
@ -101,6 +104,9 @@ export default memo((props: Props) => {
 				{layoutMode === 'grid' && <GridView />}
 				{layoutMode === 'rows' && <ListView />}
 				{layoutMode === 'media' && <MediaView />}
+				{props.hasNextPage && (
+					<Button onClick={() => props.onLoadMore?.()}>Load More</Button>
+				)}
 			</context.Provider>
 		</div>
 	);
--- a/interface/app/$libraryId/Explorer/index.tsx
+++ b/interface/app/$libraryId/Explorer/index.tsx
@ -1,5 +1,4 @@
 import { useEffect } from 'react';
-import { useParams } from 'react-router';
 import { useKey } from 'rooks';
 import { ExplorerData, rspc, useLibraryContext } from '@sd/client';
 import { getExplorerStore, useExplorerStore } from '~/hooks/useExplorerStore';
@ -13,6 +12,8 @@ interface Props {
 	// and it's not exactly compatible with search
 	// data?: ExplorerData;
 	items?: ExplorerData['items'];
+	onLoadMore?(): void;
+	hasNextPage?: boolean;
 }

 export default function Explorer(props: Props) {
@ -43,7 +44,7 @@ export default function Explorer(props: Props) {
 			<div className="flex flex-1">
 				<ExplorerContextMenu>
 					<div className="flex-1 overflow-hidden">
-						{props.items && <View data={props.items} />}
+						{props.items && <View data={props.items} onLoadMore={props.onLoadMore} />}
 					</div>
 				</ExplorerContextMenu>

--- a/interface/app/$libraryId/location/$id.tsx
+++ b/interface/app/$libraryId/location/$id.tsx
@ -1,7 +1,8 @@
+import { useInfiniteQuery } from '@tanstack/react-query';
 import { ArrowClockwise, Key, Tag } from 'phosphor-react';
-import { useEffect } from 'react';
+import { useEffect, useMemo } from 'react';
 import { useParams, useSearchParams } from 'react-router-dom';
-import { useLibraryMutation, useLibraryQuery } from '@sd/client';
+import { ExplorerData, rspc, useLibraryContext, useLibraryMutation } from '@sd/client';
 import { getExplorerStore, useExplorerStore } from '~/hooks/useExplorerStore';
 import { useExplorerTopBarOptions } from '~/hooks/useExplorerTopBarOptions';
 import Explorer from '../Explorer';
@ -21,9 +22,66 @@ export function useExplorerParams() {
 }

 export const Component = () => {
+	const { location_id, path } = useExplorerParams();
+	// we destructure this since `mutate` is a stable reference but the object it's in is not
+	const quickRescan = useLibraryMutation('locations.quickRescan');
+
+	const explorerStore = useExplorerStore();
+	const explorerState = getExplorerStore();
+
+	useEffect(() => {
+		explorerState.locationId = location_id;
+		if (location_id !== null) quickRescan.mutate({ location_id, sub_path: path });
+	}, [explorerState, location_id, path, quickRescan.mutate]);
+
+	if (location_id === null) throw new Error(`location_id is null!`);
+
+	const ctx = rspc.useContext();
+	const { library } = useLibraryContext();
+
+	const query = useInfiniteQuery({
+		queryKey: [
+			'locations.getExplorerData',
+			{
+				library_id: library.uuid,
+				arg: {
+					location_id,
+					path: explorerStore.layoutMode === 'media' ? null : path,
+					limit: 100,
+					kind: explorerStore.layoutMode === 'media' ? [5, 7] : null
+				}
+			}
+		] as const,
+		queryFn: async ({ pageParam: cursor, queryKey }): Promise<ExplorerData> => {
+			const arg = queryKey[1];
+			(arg.arg as any).cursor = cursor;
+
+			return await ctx.client.query(['locations.getExplorerData', arg]);
+		},
+		getNextPageParam: (lastPage) => lastPage.cursor ?? undefined
+	});
+
+	const items = useMemo(() => query.data?.pages.flatMap((d) => d.items), [query.data]);
+
+	return (
+		<>
+			<TopBarChildren toolOptions={useToolBarOptions()} />
+			<div className="relative flex w-full flex-col">
+				<Explorer
+					items={items}
+					onLoadMore={query.fetchNextPage}
+					hasNextPage={query.hasNextPage}
+				/>
+			</div>
+		</>
+	);
+};
+
+const useToolBarOptions = () => {
 	const store = useExplorerStore();
 	const { explorerViewOptions, explorerControlOptions } = useExplorerTopBarOptions();
-	const toolBarOptions: ToolOption[][] = [
+
+	return [
 		explorerViewOptions,
 		[
 			{
@ -54,41 +112,5 @@ export const Component = () => {
 			}
 		],
 		explorerControlOptions
-	];
-
-	const { location_id, path, limit } = useExplorerParams();
-	// we destructure this since `mutate` is a stable reference but the object it's in is not
-	const { mutate: mutateQuickRescan, ...quickRescan } =
-		useLibraryMutation('locations.quickRescan');
-
-	const explorerStore = useExplorerStore();
-
-	const explorerState = getExplorerStore();
-
-	useEffect(() => {
-		explorerState.locationId = location_id;
-		if (location_id !== null) mutateQuickRescan({ location_id, sub_path: path });
-	}, [explorerState, location_id, path, mutateQuickRescan]);
-
-	if (location_id === null) throw new Error(`location_id is null!`);
-
-	const explorerData = useLibraryQuery([
-		'locations.getExplorerData',
-		{
-			location_id,
-			path: explorerStore.layoutMode === 'media' ? null : path,
-			limit,
-			cursor: null,
-			kind: explorerStore.layoutMode === 'media' ? [5, 7] : null
-		}
-	]);
-
-	return (
-		<>
-			<TopBarChildren toolOptions={toolBarOptions} />
-			<div className="relative flex w-full flex-col">
-				<Explorer items={explorerData.data?.items} />
-			</div>
-		</>
-	);
+	] satisfies ToolOption[][];
 };
--- a/packages/client/src/core.ts
+++ b/packages/client/src/core.ts
@ -118,9 +118,7 @@ export type GenerateThumbsForLocationArgs = { id: number; path: string }

 export type LibraryConfigWrapped = { uuid: string; config: LibraryConfig }

-export type TagUpdateArgs = { id: number; name: string | null; color: string | null }
-
-export type FilePathWithObject = { id: number; pub_id: number[]; is_dir: boolean; cas_id: string | null; integrity_checksum: string | null; location_id: number; materialized_path: string; name: string; extension: string; size_in_bytes: string; inode: number[]; device: number[]; object_id: number | null; parent_id: number[] | null; key_id: number | null; date_created: string; date_modified: string; date_indexed: string; object: Object | null }
+export type LocationExplorerArgs = { location_id: number; path?: string | null; limit: number; cursor?: number[] | null; kind?: number[] | null }

 /**
 * These parameters define the password-hashing level.
@ -139,6 +137,8 @@ export type Params = "Standard" | "Hardened" | "Paranoid"
 */
 export type LocationUpdateArgs = { id: number; name: string | null; generate_preview_media: boolean | null; sync_preview_media: boolean | null; hidden: boolean | null; indexer_rules_ids: number[] }

+export type RenameFileArgs = { location_id: number; file_name: string; new_file_name: string }
+
 /**
 * Represents the operating system which the remote peer is running.
 * This is not used internally and predominantly is designed to be used for display purposes by the embedding application.
@ -154,6 +154,8 @@ export type StoredKey = { uuid: string; version: StoredKeyVersion; key_type: Sto

 export type OnboardingConfig = { password: Protected<string>; algorithm: Algorithm; hashing_algorithm: HashingAlgorithm }

+export type ExplorerContext = ({ type: "Location" } & Location) | ({ type: "Tag" } & Tag)
+
 export type Volume = { name: string; mount_point: string; total_capacity: string; available_capacity: string; is_removable: boolean; disk_type: string | null; file_system: string | null; is_root_filesystem: boolean }

 /**
@ -170,6 +172,10 @@ export type IndexerRuleCreateArgs = { kind: RuleKind; name: string; dry_run: boo

 export type EditLibraryArgs = { id: string; name: string | null; description: string | null }

+export type ObjectWithFilePaths = { id: number; pub_id: number[]; kind: number; key_id: number | null; hidden: boolean; favorite: boolean; important: boolean; has_thumbnail: boolean; has_thumbstrip: boolean; has_video_preview: boolean; ipfs_id: string | null; note: string | null; date_created: string; file_paths: FilePath[] }
+
+export type LightScanArgs = { location_id: number; sub_path: string }
+
 export type BuildInfo = { version: string; commit: string }

 /**
@ -181,7 +187,7 @@ export type Nonce = { XChaCha20Poly1305: number[] } | { Aes256Gcm: number[] }

 export type UnlockKeyManagerArgs = { password: Protected<string>; secret_key: Protected<string> }

-export type SetNoteArgs = { id: number; note: string | null }
+export type TagCreateArgs = { name: string; color: string }

 export type FileEncryptorJobInit = { location_id: number; path_id: number; key_uuid: string; algorithm: Algorithm; metadata: boolean; preview_media: boolean; output_path: string | null }

@ -198,23 +204,23 @@ export type CRDTOperation = { node: string; timestamp: number; id: string; typ:
 */
 export type Salt = number[]

-export type LightScanArgs = { location_id: number; sub_path: string }
+export type TagUpdateArgs = { id: number; name: string | null; color: string | null }

 export type GetArgs = { id: number }

 export type FileCutterJobInit = { source_location_id: number; source_path_id: number; target_location_id: number; target_path: string }

-export type ExplorerItem = { type: "Path"; has_thumbnail: boolean; item: FilePathWithObject } | { type: "Object"; has_thumbnail: boolean; item: ObjectWithFilePaths }
-
 export type JobStatus = "Queued" | "Running" | "Completed" | "Canceled" | "Failed" | "Paused"

 export type ObjectValidatorArgs = { id: number; path: string }

 export type FileEraserJobInit = { location_id: number; path_id: number; passes: string }

-export type SetFavoriteArgs = { id: number; favorite: boolean }
+export type ExplorerItem = { type: "Path"; has_thumbnail: boolean; item: FilePathWithObject } | { type: "Object"; has_thumbnail: boolean; item: ObjectWithFilePaths }

-export type RenameFileArgs = { location_id: number; file_name: string; new_file_name: string }
+export type FilePathWithObject = { id: number; pub_id: number[]; is_dir: boolean; cas_id: string | null; integrity_checksum: string | null; location_id: number; materialized_path: string; name: string; extension: string; size_in_bytes: string; inode: number[]; device: number[]; object_id: number | null; parent_id: number[] | null; key_id: number | null; date_created: string; date_modified: string; date_indexed: string; object: Object | null }
+
+export type TagAssignArgs = { object_id: number; tag_id: number; unassign: boolean }

 export type FileDeleterJobInit = { location_id: number; path_id: number }

@ -225,12 +231,12 @@ export type FilePath = { id: number; pub_id: number[]; is_dir: boolean; cas_id:
 */
 export type Algorithm = "XChaCha20Poly1305" | "Aes256Gcm"

-export type LocationExplorerArgs = { location_id: number; path: string | null; limit: number; cursor: string | null; kind: number[] | null }
-
 export type JobReport = { id: string; name: string; action: string | null; data: number[] | null; metadata: any | null; is_background: boolean; created_at: string | null; started_at: string | null; completed_at: string | null; parent_id: string | null; status: JobStatus; task_count: number; completed_task_count: number; message: string }

 export type OwnedOperationItem = { id: any; data: OwnedOperationData }

+export type SetFavoriteArgs = { id: number; favorite: boolean }
+
 export type CRDTOperationType = SharedOperation | RelationOperation | OwnedOperation

 /**
@ -242,10 +248,6 @@ export type SpacedropArgs = { peer_id: PeerId; file_path: string[] }

 export type NodeState = ({ id: string; name: string; p2p_port: number | null; p2p_email: string | null; p2p_img_url: string | null }) & { data_path: string }

-export type TagCreateArgs = { name: string; color: string }
-
-export type ExplorerContext = ({ type: "Location" } & Location) | ({ type: "Tag" } & Tag)
-
 export type OwnedOperation = { model: string; items: OwnedOperationItem[] }

 export type SharedOperation = { record_id: any; model: string; data: SharedOperationData }
@ -260,8 +262,6 @@ export type KeyAddArgs = { algorithm: Algorithm; hashing_algorithm: HashingAlgor

 export type RuleKind = "AcceptFilesByGlob" | "RejectFilesByGlob" | "AcceptIfChildrenDirectoriesArePresent" | "RejectIfChildrenDirectoriesArePresent"

-export type TagAssignArgs = { object_id: number; tag_id: number; unassign: boolean }
-
 /**
 * `LocationCreateArgs` is the argument received from the client using `rspc` to create a new location.
 * It has the actual path and a vector of indexer rules ids, to create many-to-many relationships
@ -282,6 +282,8 @@ export type SharedOperationData = SharedOperationCreateData | { field: string; v

 export type MediaData = { id: number; pixel_width: number | null; pixel_height: number | null; longitude: number | null; latitude: number | null; fps: number | null; capture_device_make: string | null; capture_device_model: string | null; capture_device_software: string | null; duration_seconds: number | null; codecs: string | null; streams: number | null }

+export type ExplorerData = { context: ExplorerContext; items: ExplorerItem[]; cursor: number[] | null }
+
 export type IndexerRule = { id: number; kind: number; name: string; default: boolean; parameters: number[]; date_created: string; date_modified: string }

 export type FileCopierJobInit = { source_location_id: number; source_path_id: number; target_location_id: number; target_path: string; target_file_name_suffix: string | null }
@ -295,7 +297,7 @@ export type Object = { id: number; pub_id: number[]; kind: number; key_id: numbe
 */
 export type HashingAlgorithm = { name: "Argon2id"; params: Params } | { name: "BalloonBlake3"; params: Params }

-export type ExplorerData = { context: ExplorerContext; items: ExplorerItem[] }
+export type SetNoteArgs = { id: number; note: string | null }

 export type LocationWithIndexerRules = { id: number; pub_id: number[]; node_id: number; name: string; path: string; total_capacity: number | null; available_capacity: number | null; is_archived: boolean; generate_preview_media: boolean; sync_preview_media: boolean; hidden: boolean; date_created: string; indexer_rules: ({ indexer_rule: IndexerRule })[] }

@ -314,8 +316,6 @@ export type Protected<T> = T

 export type Statistics = { id: number; date_captured: string; total_object_count: number; library_db_size: string; total_bytes_used: string; total_bytes_capacity: string; total_unique_bytes: string; total_bytes_free: string; preview_media_bytes: string }

-export type ObjectWithFilePaths = { id: number; pub_id: number[]; kind: number; key_id: number | null; hidden: boolean; favorite: boolean; important: boolean; has_thumbnail: boolean; has_thumbstrip: boolean; has_video_preview: boolean; ipfs_id: string | null; note: string | null; date_created: string; file_paths: FilePath[] }
-
 export type RestoreBackupArgs = { password: Protected<string>; secret_key: Protected<string>; path: string }

 export type Tag = { id: number; pub_id: number[]; name: string | null; color: string | null; total_objects: number | null; redundancy_goal: number | null; date_created: string; date_modified: string }