[ENG-941] Jobs running in another library do not appear in the job manager (#1306)

* Fetching job reports from all libraries
Some clippy warnings
pnpm format

* Reverting expects to unwraps

apps/mobile/src/components/modal/job/JobManagerModal.tsx

@@ -1,7 +1,7 @@
 import { useQueryClient } from '@tanstack/react-query';
 import { forwardRef } from 'react';
 import { FlatList, Text, View } from 'react-native';
-import { useJobProgress, useLibraryQuery } from '@sd/client';
+import { useBridgeQuery, useJobProgress } from '@sd/client';
 import JobGroup from '~/components/job/JobGroup';
 import { Modal, ModalRef } from '~/components/layout/Modal';
 import { tw } from '~/lib/tailwind';
@@ -11,10 +11,13 @@ import { tw } from '~/lib/tailwind';
 // - Add clear all jobs button
 
 export const JobManagerModal = forwardRef<ModalRef, unknown>((_, ref) => {
-	const queryClient = useQueryClient();
+	const jobGroupsById = useBridgeQuery(['jobs.reports']);
 
-	const jobGroups = useLibraryQuery(['jobs.reports']);
-	const progress = useJobProgress(jobGroups.data);
+	// TODO: Currently we're only clustering togheter all job reports from all libraries without any distinction.
+	// TODO: We should probably cluster them by library in the job manager UI
+	const jobGroups = jobGroupsById.data ? Object.values(jobGroupsById.data).flat() : [];
+
+	const progress = useJobProgress(jobGroups);
 
 	// const clearAllJobs = useLibraryMutation(['jobs.clearAll'], {
 	// 	onError: () => {
@@ -28,7 +31,7 @@ export const JobManagerModal = forwardRef<ModalRef, unknown>((_, ref) => {
 	return (
 		<Modal ref={ref} snapPoints={['60']} title="Recent Jobs" showCloseButton>
 			<FlatList
-				data={jobGroups.data}
+				data={jobGroups}
 				style={tw`flex-1`}
 				keyExtractor={(i) => i.id}
 				contentContainerStyle={tw`mt-4`}

apps/storybook/postcss.config.js

@@ -1,10 +1,10 @@
 module.exports = {
 	plugins: {
-		tailwindcss: {},
-		autoprefixer: {},
-		"postcss-pseudo-companion-classes": {
-			prefix: "sb-pseudo--",
-			restrictTo: [":hover", ":focus"]
-		},
-	},
+		'tailwindcss': {},
+		'autoprefixer': {},
+		'postcss-pseudo-companion-classes': {
+			prefix: 'sb-pseudo--',
+			restrictTo: [':hover', ':focus']
+		}
+	}
 };

core/src/api/jobs.rs

@@ -1,6 +1,7 @@
 use crate::{
 	invalidate_query,
 	job::{job_without_data, Job, JobReport, JobStatus, Jobs},
+	library::Library,
 	location::{find_location, LocationError},
 	object::{
 		file_identifier::file_identifier_job::FileIdentifierJobInit, media::MediaProcessorJobInit,
@@ -16,6 +17,7 @@ use std::{
 };
 
 use chrono::{DateTime, Utc};
+use futures_concurrency::future::TryJoin;
 use prisma_client_rust::or;
 use rspc::alpha::AlphaRouter;
 use serde::{Deserialize, Serialize};
@@ -78,86 +80,109 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
 				jobs: VecDeque<JobReport>,
 			}
 
-			R.with2(library())
-				.query(|(node, library), _: ()| async move {
-					let mut groups: HashMap<String, JobGroup> = HashMap::new();
+			async fn group_jobs_by_library(
+				library: &Library,
+				active_job_reports_by_id: &HashMap<Uuid, JobReport>,
+			) -> Result<Vec<JobGroup>, rspc::Error> {
+				let mut groups: HashMap<String, JobGroup> = HashMap::new();
 
-					let job_reports: Vec<JobReport> = library
-						.db
-						.job()
-						.find_many(vec![])
-						.order_by(job::date_created::order(SortOrder::Desc))
-						.take(100)
-						.select(job_without_data::select())
-						.exec()
-						.await?
-						.into_iter()
-						.flat_map(JobReport::try_from)
-						.collect();
+				let job_reports: Vec<JobReport> = library
+					.db
+					.job()
+					.find_many(vec![])
+					.order_by(job::date_created::order(SortOrder::Desc))
+					.take(100)
+					.select(job_without_data::select())
+					.exec()
+					.await?
+					.into_iter()
+					.flat_map(JobReport::try_from)
+					.collect();
 
-					let active_reports_by_id = node.jobs.get_active_reports_with_id().await;
+				for job in job_reports {
+					// action name and group key are computed from the job data
+					let (action_name, group_key) = job.get_meta();
 
-					for job in job_reports {
-						// action name and group key are computed from the job data
-						let (action_name, group_key) = job.get_meta();
+					trace!(
+						"job {:#?}, action_name {}, group_key {:?}",
+						job,
+						action_name,
+						group_key
+					);
 
-						trace!(
-							"job {:#?}, action_name {}, group_key {:?}",
-							job,
-							action_name,
-							group_key
-						);
+					// if the job is running, use the in-memory report
+					let report = active_job_reports_by_id.get(&job.id).unwrap_or(&job);
 
-						// if the job is running, use the in-memory report
-						let report = active_reports_by_id.get(&job.id).unwrap_or(&job);
-
-						// if we have a group key, handle grouping
-						if let Some(group_key) = group_key {
-							match groups.entry(group_key) {
-								// Create new job group with metadata
-								Entry::Vacant(entry) => {
-									entry.insert(JobGroup {
-										id: job.parent_id.unwrap_or(job.id),
-										action: Some(action_name.clone()),
-										status: job.status,
-										jobs: [report.clone()].into_iter().collect(),
-										created_at: job.created_at.unwrap_or(Utc::now()),
-									});
-								}
-								// Add to existing job group
-								Entry::Occupied(mut entry) => {
-									let group = entry.get_mut();
-
-									// protect paused status from being overwritten
-									if report.status != JobStatus::Paused {
-										group.status = report.status;
-									}
-
-									// if group.status.is_finished() && !report.status.is_finished() {
-									// }
-									group.jobs.push_front(report.clone());
-								}
-							}
-						} else {
-							// insert individual job as group
-							groups.insert(
-								job.id.to_string(),
-								JobGroup {
-									id: job.id,
-									action: None,
+					// if we have a group key, handle grouping
+					if let Some(group_key) = group_key {
+						match groups.entry(group_key) {
+							// Create new job group with metadata
+							Entry::Vacant(entry) => {
+								entry.insert(JobGroup {
+									id: job.parent_id.unwrap_or(job.id),
+									action: Some(action_name.clone()),
 									status: job.status,
 									jobs: [report.clone()].into_iter().collect(),
 									created_at: job.created_at.unwrap_or(Utc::now()),
-								},
-							);
+								});
+							}
+							// Add to existing job group
+							Entry::Occupied(mut entry) => {
+								let group = entry.get_mut();
+
+								// protect paused status from being overwritten
+								if report.status != JobStatus::Paused {
+									group.status = report.status;
+								}
+
+								// if group.status.is_finished() && !report.status.is_finished() {
+								// }
+								group.jobs.push_front(report.clone());
+							}
 						}
+					} else {
+						// insert individual job as group
+						groups.insert(
+							job.id.to_string(),
+							JobGroup {
+								id: job.id,
+								action: None,
+								status: job.status,
+								jobs: [report.clone()].into_iter().collect(),
+								created_at: job.created_at.unwrap_or(Utc::now()),
+							},
+						);
 					}
+				}
 
-					let mut groups_vec = groups.into_values().collect::<Vec<_>>();
-					groups_vec.sort_by(|a, b| b.created_at.cmp(&a.created_at));
+				let mut groups_vec = groups.into_values().collect::<Vec<_>>();
+				groups_vec.sort_by(|a, b| b.created_at.cmp(&a.created_at));
 
-					Ok(groups_vec)
-				})
+				Ok(groups_vec)
+			}
+
+			R.query(|node, _: ()| async move {
+				// WARN: We really need the borrow in this line, this way each async move in the map below
+				// received a copy of the reference to the active job reports,
+				// I feel like I'm conquering the borrow checker
+				let active_job_reports_by_id = &node.jobs.get_active_reports_with_id().await;
+
+				node.libraries
+					.get_all()
+					.await
+					.into_iter()
+					.map(|library| async move {
+						group_jobs_by_library(&library, active_job_reports_by_id)
+							.await
+							.map(|groups| (library.id, groups))
+					})
+					.collect::<Vec<_>>()
+					.try_join()
+					.await
+					.map(|groups_by_library_id| {
+						groups_by_library_id.into_iter().collect::<HashMap<_, _>>()
+					})
+			})
 		})
 		.procedure("isActive", {
 			R.with2(library())

core/src/custom_uri.rs

@@ -164,7 +164,11 @@ pub fn router(node: Arc<Node>) -> Router<()> {
 							name: path,
 							ext: maybe_missing(file_path.extension, "extension").map_err(not_found)?,
 							file_path_pub_id: Uuid::from_slice(&file_path.pub_id).map_err(internal_server_error)?,
-							serve_from: (identity == library.identity.to_remote_identity()).then_some(ServeFrom::Local).unwrap_or_else(|| ServeFrom::Remote(identity)),
+							serve_from: if identity == library.identity.to_remote_identity() {
+								ServeFrom::Local
+							} else {
+								ServeFrom::Remote(identity)
+							},
 						};
 
 						state
@@ -202,8 +206,8 @@ pub fn router(node: Arc<Node>) -> Router<()> {
 							}
 
 							// TODO: Support `Range` requests and `ETag` headers
-
-							match state.node.nlm.state().await.get(&library_id).unwrap().instances.get(&identity).unwrap().clone() {
+							#[allow(clippy::unwrap_used)]
+							match *state.node.nlm.state().await.get(&library_id).unwrap().instances.get(&identity).unwrap() {
 								InstanceState::Discovered(_) | InstanceState::Unavailable => Ok(not_found(())),
 								InstanceState::Connected(peer_id) => {
 									let (tx, mut rx) = tokio::sync::mpsc::channel::<io::Result<Bytes>>(150);
@@ -620,6 +624,7 @@ impl AsyncWrite for MpscToAsyncWrite {
 		cx: &mut Context<'_>,
 		buf: &[u8],
 	) -> Poll<Result<usize, io::Error>> {
+		#[allow(clippy::unwrap_used)]
 		match self.0.poll_reserve(cx) {
 			Poll::Ready(Ok(())) => {
 				self.0.send_item(Ok(Bytes::from(buf.to_vec()))).unwrap();

core/src/p2p/p2p_manager.rs

@@ -628,7 +628,7 @@ impl P2PManager {
 			.write_all(
 				&Header::File {
 					library_id: library.id,
-					file_path_id: file_path_id.clone(),
+					file_path_id,
 					range: range.clone(),
 				}
 				.to_bytes(),

interface/app/$libraryId/Explorer/ContextMenu/FilePath/Items.tsx

@@ -209,7 +209,7 @@ export const ParentFolderActions = new ConditionalItem({
 							await generateThumbnails.mutateAsync({
 								id: parent.location.id,
 								path: selectedFilePaths[0]?.materialized_path ?? '/',
-								regenerate: true,
+								regenerate: true
 							});
 						} catch (error) {
 							toast.error({

interface/app/$libraryId/Explorer/ParentContextMenu.tsx

@@ -130,7 +130,7 @@ export default (props: PropsWithChildren) => {
 									await generateThumbsForLocation.mutateAsync({
 										id: parent.location.id,
 										path: currentPath ?? '/',
-										regenerate: true,
+										regenerate: true
 									});
 								} catch (error) {
 									toast.error({

interface/app/$libraryId/Layout/Sidebar/FeedbackDialog.tsx

@@ -46,7 +46,7 @@ export default function FeedbackDialog(props: UseDialogProps) {
 			ctaLabel="Submit"
 			closeLabel="Cancel"
 			buttonsSideContent={
-				<div className="flex items-center justify-center w-full gap-1">
+				<div className="flex w-full items-center justify-center gap-1">
 					{EMOJIS.map((emoji, i) => (
 						<div
 							onClick={() => emojiSelectHandler(i)}

interface/app/$libraryId/Layout/Sidebar/JobManager/index.tsx

@@ -1,7 +1,7 @@
 import { useQueryClient } from '@tanstack/react-query';
 import { Check, Trash, X } from 'phosphor-react';
 import { useState } from 'react';
-import { useJobProgress, useLibraryMutation, useLibraryQuery } from '@sd/client';
+import { useBridgeQuery, useJobProgress, useLibraryMutation } from '@sd/client';
 import { Button, PopoverClose, Tooltip, toast } from '@sd/ui';
 import IsRunningJob from './IsRunningJob';
 import JobGroup from './JobGroup';
@@ -10,9 +10,13 @@ export function JobManager() {
 	const queryClient = useQueryClient();
 	const [toggleConfirmation, setToggleConfirmation] = useState(false);
 
-	const jobGroups = useLibraryQuery(['jobs.reports']);
+	const jobGroupsById = useBridgeQuery(['jobs.reports']);
 
-	const progress = useJobProgress(jobGroups.data);
+	// TODO: Currently we're only clustering togheter all job reports from all libraries without any distinction.
+	// TODO: We should probably cluster them by library in the job manager UI
+	const jobGroups = jobGroupsById.data ? Object.values(jobGroupsById.data).flat() : [];
+
+	const progress = useJobProgress(jobGroups);
 
 	const clearAllJobs = useLibraryMutation(['jobs.clearAll'], {
 		onError: () => {
@@ -76,16 +80,15 @@ export function JobManager() {
 			</div>
 			<div className="custom-scroll job-manager-scroll h-full overflow-x-hidden">
 				<div className="h-full border-r border-app-line/50">
-					{jobGroups.data &&
-						(jobGroups.data.length === 0 ? (
-							<div className="flex h-32 items-center justify-center text-sidebar-inkDull">
-								No jobs.
-							</div>
-						) : (
-							jobGroups.data.map((group) => (
-								<JobGroup key={group.id} group={group} progress={progress} />
-							))
-						))}
+					{jobGroups.length === 0 ? (
+						<div className="flex h-32 items-center justify-center text-sidebar-inkDull">
+							No jobs.
+						</div>
+					) : (
+						jobGroups.map((group) => (
+							<JobGroup key={group.id} group={group} progress={progress} />
+						))
+					)}
 				</div>
 			</div>
 		</div>

interface/app/$libraryId/settings/library/locations/AddLocationDialog.tsx

@@ -216,7 +216,7 @@ export const AddLocationDialog = ({
 					: ''
 			}
 		>
-			<ErrorMessage name={REMOTE_ERROR_FORM_FIELD} variant="large" className="mt-2 mb-4" />
+			<ErrorMessage name={REMOTE_ERROR_FORM_FIELD} variant="large" className="mb-4 mt-2" />
 
 			<InputField
 				size="md"

packages/client/src/core.ts

@@ -12,7 +12,7 @@ export type Procedures = {
         { key: "files.getPath", input: LibraryArgs<number>, result: string | null } | 
         { key: "invalidation.test-invalidate", input: never, result: number } | 
         { key: "jobs.isActive", input: LibraryArgs<null>, result: boolean } | 
-        { key: "jobs.reports", input: LibraryArgs<null>, result: JobGroup[] } | 
+        { key: "jobs.reports", input: never, result: { [key: string]: JobGroup[] } } | 
         { key: "library.list", input: never, result: LibraryConfigWrapped[] } | 
         { key: "library.statistics", input: LibraryArgs<null>, result: Statistics } | 
         { key: "locations.get", input: LibraryArgs<number>, result: Location | null } | 

packages/ui/src/Button.stories.tsx

@@ -30,7 +30,7 @@ export const AllVariants = () => {
 		'subtle'
 	];
 	return (
-		<div className="w-full h-screen p-10 bg-app">
+		<div className="h-screen w-full bg-app p-10">
 			<h1 className="text-[20px] font-bold text-white">Buttons</h1>
 			<div className="mb-6 ml-[90px] mt-5 flex flex-col gap-8 text-sm">
 				<div className="ml-[100px] grid w-full max-w-[850px] grid-cols-9 items-center gap-6">