Reactive file identification (#2396)

* yes

* Explain mysterious if

* Use id alias just for consistency reasons

* yes

* Rust fmt

* fix ts

---------

Co-authored-by: Ericson "Fogo" Soares <ericson.ds999@gmail.com>
Co-authored-by: Utku Bakir <74243531+utkubakir@users.noreply.github.com>

core/src/api/jobs.rs

@@ -343,7 +343,6 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
 			R.with2(library())
 				.subscription(|(node, _), _: ()| async move {
 					// TODO: Only return event for the library that was subscribed to
-
 					let mut event_bus_rx = node.event_bus.0.subscribe();
 					async_stream::stream! {
 						while let Ok(event) = event_bus_rx.recv().await {
@@ -355,4 +354,19 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
 					}
 				})
 		})
+		.procedure("newFilePathIdentified", {
+			R.with2(library())
+				.subscription(|(node, _), _: ()| async move {
+					// TODO: Only return event for the library that was subscribed to
+					let mut event_bus_rx = node.event_bus.0.subscribe();
+					async_stream::stream! {
+						while let Ok(event) = event_bus_rx.recv().await {
+							match event {
+								CoreEvent::NewIdentifiedObjects { file_path_ids } => yield file_path_ids,
+								_ => {}
+							}
+						}
+					}
+				})
+		})
 }

core/src/api/locations.rs

@@ -39,8 +39,11 @@ pub type ThumbnailKey = Vec<String>;
 #[serde(tag = "type")]
 pub enum ExplorerItem {
 	Path {
+		// provide the frontend with the thumbnail key explicitly
 		thumbnail: Option<ThumbnailKey>,
-		has_created_thumbnail: bool, // this is important
+		// this tells the frontend if a thumbnail actually exists or not
+		has_created_thumbnail: bool,
+		// we can't actually modify data from PCR types, thats why computed properties are used on ExplorerItem
 		item: file_path_with_object::Data,
 	},
 	Object {

core/src/api/mod.rs

@@ -10,6 +10,8 @@ use crate::{
 
 use sd_cache::patch_typedef;
 use sd_p2p::RemoteIdentity;
+use sd_prisma::prisma::file_path;
+
 use std::sync::{atomic::Ordering, Arc};
 
 use itertools::Itertools;
@@ -52,7 +54,12 @@ pub type Router = rspc::Router<Ctx>;
 /// Represents an internal core event, these are exposed to client via a rspc subscription.
 #[derive(Debug, Clone, Serialize, Type)]
 pub enum CoreEvent {
-	NewThumbnail { thumb_key: Vec<String> },
+	NewThumbnail {
+		thumb_key: Vec<String>,
+	},
+	NewIdentifiedObjects {
+		file_path_ids: Vec<file_path::id::Type>,
+	},
 	JobProgress(JobProgressEvent),
 	InvalidateOperation(InvalidateOperationEvent),
 }

core/src/object/media/old_thumbnail/process.rs

@@ -373,7 +373,12 @@ pub(super) async fn generate_thumbnail(
 			}
 		}
 	}
-
+	// This if is REALLY needed, due to the sheer performance of the thumbnailer,
+	// I restricted to only send events notifying for thumbnails in the current
+	// opened directory, sending events for the entire location turns into a
+	// humongous bottleneck in the frontend lol, since it doesn't even knows
+	// what to do with thumbnails for inner directories lol
+	// - fogodev
 	if !in_background {
 		trace!("Emitting new thumbnail event");
 		if reporter

core/src/object/old_file_identifier/old_file_identifier_job.rs

@@ -1,4 +1,5 @@
 use crate::{
+	api::CoreEvent,
 	library::Library,
 	location::ScanState,
 	old_job::{
@@ -226,6 +227,11 @@ impl StatefulJob for OldFileIdentifierJobInit {
 		new_metadata.total_objects_linked = total_objects_linked;
 		new_metadata.cursor = new_cursor;
 
+		// send an array of ids to let clients know new objects were identified
+		ctx.node.emit(CoreEvent::NewIdentifiedObjects {
+			file_path_ids: file_paths.iter().map(|fp| fp.id).collect(),
+		});
+
 		ctx.progress(vec![
 			JobReportUpdate::CompletedTaskCount(step_number * CHUNK_SIZE + file_paths.len()),
 			JobReportUpdate::Message(format!(

interface/app/$libraryId/Explorer/FilePath/Original.tsx

@@ -17,7 +17,7 @@ import { usePlatform } from '~/util/Platform';
 
 import { useExplorerContext } from '../Context';
 import { explorerStore } from '../store';
-import { ExplorerItemData } from '../util';
+import { ExplorerItemData } from '../useExplorerItemData';
 import { Image } from './Image';
 import { useBlackBars, useSize } from './utils';
 

interface/app/$libraryId/Explorer/FilePath/Thumb.tsx

@@ -15,7 +15,7 @@ import { useIsDark } from '~/hooks';
 import { pdfViewerEnabled } from '~/util/pdfViewer';
 import { usePlatform } from '~/util/Platform';
 
-import { useExplorerItemData } from '../util';
+import { useExplorerItemData } from '../useExplorerItemData';
 import { Image, ImageProps } from './Image';
 import LayeredFileIcon from './LayeredFileIcon';
 import { Original } from './Original';

interface/app/$libraryId/Explorer/Inspector/index.tsx

@@ -55,7 +55,8 @@ import AssignTagMenuItems from '../ContextMenu/AssignTagMenuItems';
 import { FileThumb } from '../FilePath/Thumb';
 import { useQuickPreviewStore } from '../QuickPreview/store';
 import { explorerStore } from '../store';
-import { uniqueId, useExplorerItemData } from '../util';
+import { useExplorerItemData } from '../useExplorerItemData';
+import { uniqueId } from '../util';
 import { RenamableItemText } from '../View/RenamableItemText';
 import FavoriteButton from './FavoriteButton';
 import MediaData from './MediaData';

interface/app/$libraryId/Explorer/index.tsx

@@ -4,6 +4,7 @@ import {
 	explorerLayout,
 	useExplorerLayoutStore,
 	useLibrarySubscription,
+	useRspcLibraryContext,
 	useSelector
 } from '@sd/client';
 import { useShortcut } from '~/hooks';
@@ -40,19 +41,23 @@ export default function Explorer(props: PropsWithChildren<Props>) {
 	const showInspector = useSelector(explorerStore, (s) => s.showInspector);
 
 	const showPathBar = explorer.showPathBar && layoutStore.showPathBar;
-
+	const rspc = useRspcLibraryContext();
 	// Can we put this somewhere else -_-
 	useLibrarySubscription(['jobs.newThumbnail'], {
-		onStarted: () => {
-			console.log('Started RSPC subscription new thumbnail');
-		},
-		onError: (err) => {
-			console.error('Error in RSPC subscription new thumbnail', err);
-		},
 		onData: (thumbKey) => {
 			explorerStore.addNewThumbnail(thumbKey);
 		}
 	});
+	useLibrarySubscription(['jobs.newFilePathIdentified'], {
+		onData: (ids) => {
+			if (ids?.length > 0) {
+				// I had planned to somehow fetch the Object, but its a lot more work than its worth given
+				// id have to fetch the file_path explicitly and patch the query
+				// for now, it seems to work a treat just invalidating the whole query
+				rspc.queryClient.invalidateQueries(['search.paths']);
+			}
+		}
+	});
 
 	useShortcut('showPathBar', (e) => {
 		e.stopPropagation();

interface/app/$libraryId/Explorer/store.ts

@@ -122,10 +122,9 @@ export const explorerStore = proxy({
 	addNewThumbnail: (thumbKey: string[]) => {
 		explorerStore.newThumbnails.add(flattenThumbnailKey(thumbKey));
 	},
-	// this should be done when the explorer query is refreshed
-	// prevents memory leak
-	resetNewThumbnails: () => {
+	resetCache: () => {
 		explorerStore.newThumbnails.clear();
+		// explorerStore.newFilePathsIdentified.clear();
 	}
 });
 

interface/app/$libraryId/Explorer/useExplorerItemData.tsx

@@ -0,0 +1,34 @@
+import { useQueryClient } from '@tanstack/react-query';
+import { useEffect, useMemo } from 'react';
+import { getExplorerItemData, useSelector, type ExplorerItem } from '@sd/client';
+
+import { explorerStore, flattenThumbnailKey } from './store';
+
+// This is where we intercept the state of the explorer item to determine if we should rerender
+// This hook is used inside every thumbnail in the explorer
+export function useExplorerItemData(explorerItem: ExplorerItem) {
+	const newThumbnail = useSelector(explorerStore, (s) => {
+		const thumbnailKey =
+			explorerItem.type === 'Label'
+				? // labels have .thumbnails, plural
+					explorerItem.thumbnails?.[0]
+				: // all other explorer items have .thumbnail singular
+					'thumbnail' in explorerItem && explorerItem.thumbnail;
+
+		return !!(thumbnailKey && s.newThumbnails.has(flattenThumbnailKey(thumbnailKey)));
+	});
+
+	return useMemo(() => {
+		const itemData = getExplorerItemData(explorerItem);
+
+		if (!itemData.hasLocalThumbnail) {
+			itemData.hasLocalThumbnail = newThumbnail;
+		}
+
+		return itemData;
+		// whatever goes here, is what can cause an atomic re-render of an explorer item
+		// this is used for when new thumbnails are generated, and files identified
+	}, [explorerItem, newThumbnail]);
+}
+
+export type ExplorerItemData = ReturnType<typeof useExplorerItemData>;

interface/app/$libraryId/Explorer/util.ts

@@ -1,39 +1,11 @@
-import { useMemo } from 'react';
-import { getExplorerItemData, useSelector, type ExplorerItem } from '@sd/client';
+import { type ExplorerItem } from '@sd/client';
 import { ExplorerParamsSchema } from '~/app/route-schemas';
 import { useZodSearchParams } from '~/hooks';
 
-import { explorerStore, flattenThumbnailKey } from './store';
-
 export function useExplorerSearchParams() {
 	return useZodSearchParams(ExplorerParamsSchema);
 }
 
-export function useExplorerItemData(explorerItem: ExplorerItem) {
-	const newThumbnail = useSelector(explorerStore, (s) => {
-		const thumbnailKey =
-			explorerItem.type === 'Label'
-				? // labels have .thumbnails, plural
-					explorerItem.thumbnails?.[0]
-				: // all other explorer items have .thumbnail singular
-					'thumbnail' in explorerItem && explorerItem.thumbnail;
-
-		return !!(thumbnailKey && s.newThumbnails.has(flattenThumbnailKey(thumbnailKey)));
-	});
-
-	return useMemo(() => {
-		const itemData = getExplorerItemData(explorerItem);
-
-		if (!itemData.hasLocalThumbnail) {
-			itemData.hasLocalThumbnail = newThumbnail;
-		}
-
-		return itemData;
-	}, [explorerItem, newThumbnail]);
-}
-
-export type ExplorerItemData = ReturnType<typeof useExplorerItemData>;
-
 export const pubIdToString = (pub_id: number[]) =>
 	pub_id.map((b) => b.toString(16).padStart(2, '0')).join('');
 

interface/app/$libraryId/ephemeral.tsx

@@ -204,7 +204,7 @@ const EphemeralExplorer = memo((props: { args: PathParams }) => {
 		{
 			enabled: path != null,
 			suspense: true,
-			onSuccess: () => explorerStore.resetNewThumbnails(),
+			onSuccess: () => explorerStore.resetCache(),
 			onBatch: (item) => {
 				cache.withNodes(item.nodes);
 			}

interface/app/$libraryId/location/$id.tsx

@@ -104,7 +104,7 @@ const LocationExplorer = ({ location }: { location: Location; path?: string }) =
 		],
 		take,
 		paths: { order: explorerSettings.useSettingsSnapshot().order },
-		onSuccess: () => explorerStore.resetNewThumbnails()
+		onSuccess: () => explorerStore.resetCache()
 	});
 
 	const explorer = useExplorer({

interface/app/$libraryId/saved-search/$id.tsx

@@ -77,7 +77,7 @@ function Inner({ id }: { id: number }) {
 		filters: search.allFilters,
 		take: 50,
 		paths: { order: explorerSettings.useSettingsSnapshot().order },
-		onSuccess: () => explorerStore.resetNewThumbnails()
+		onSuccess: () => explorerStore.resetCache()
 	});
 
 	const explorer = useExplorer({

packages/client/src/core.ts

@@ -134,6 +134,7 @@ export type Procedures = {
     subscriptions: 
         { key: "auth.loginSession", input: never, result: Response } | 
         { key: "invalidation.listen", input: never, result: InvalidateOperationEvent[] } | 
+        { key: "jobs.newFilePathIdentified", input: LibraryArgs<null>, result: number[] } | 
         { key: "jobs.newThumbnail", input: LibraryArgs<null>, result: string[] } | 
         { key: "jobs.progress", input: LibraryArgs<null>, result: JobProgressEvent } | 
         { key: "library.actors", input: LibraryArgs<null>, result: { [key in string]: boolean } } |