From 1b1a8c42585f23006c2cd36414b88b72714b06a0 Mon Sep 17 00:00:00 2001 From: Jamie Pine Date: Wed, 22 Jun 2022 04:40:45 -0700 Subject: [PATCH] (fix) job queue - adding a location performs all jobs - better logging for thumbnailer --- .vscode/settings.json | 1 + core/src/encode/thumb.rs | 18 ++++++++++++------ core/src/job/jobs.rs | 16 ++++++++++------ core/src/sys/locations.rs | 5 +---- .../src/components/file/Inspector.tsx | 6 ++---- 5 files changed, 26 insertions(+), 20 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index b7bc53d65..421272d60 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -5,6 +5,7 @@ "consts", "countup", "creationdate", + "Deque", "dotenv", "dotenvy", "fontsource", diff --git a/core/src/encode/thumb.rs b/core/src/encode/thumb.rs index f349920eb..73b3249f7 100644 --- a/core/src/encode/thumb.rs +++ b/core/src/encode/thumb.rs @@ -8,6 +8,7 @@ use crate::{ use crate::{sys, CoreEvent}; use futures::executor::block_on; use image::*; +use log::{error, info}; use std::fs; use std::path::{Path, PathBuf}; use webp::*; @@ -34,6 +35,11 @@ impl Job for ThumbnailJob { let location = sys::get_location(&core_ctx, self.location_id).await?; + info!( + "Searching for images in location {} at path {}", + location.id, self.path + ); + // create all necessary directories if they don't exist fs::create_dir_all( Path::new(&config.data_path) @@ -44,7 +50,7 @@ impl Job for ThumbnailJob { // query database for all files in this location that need thumbnails let image_files = get_images(&core_ctx, self.location_id, &self.path).await?; - println!("Found {:?} files", image_files.len()); + info!("Found {:?} files", image_files.len()); let is_background = self.background.clone(); @@ -65,7 +71,7 @@ impl Job for ThumbnailJob { // assemble the file path let path = Path::new(&root_path).join(&image_file.materialized_path); - println!("image_file {:?}", image_file); + error!("image_file {:?}", image_file); // get cas_id, if none found skip let cas_id = match image_file.file() { @@ -77,7 +83,7 @@ impl Job for ThumbnailJob { } } Err(_) => { - println!("Error getting cas_id {:?}", image_file.materialized_path); + error!("Error getting cas_id {:?}", image_file.materialized_path); continue; } }; @@ -91,10 +97,10 @@ impl Job for ThumbnailJob { // check if file exists at output path if !output_path.exists() { - println!("writing {:?} to {:?}", path, output_path); + info!("Writing {:?} to {:?}", path, output_path); generate_thumbnail(&path, &output_path) .map_err(|e| { - println!("error generating thumb {:?}", e); + info!("Error generating thumb {:?}", e); }) .unwrap_or(()); @@ -104,7 +110,7 @@ impl Job for ThumbnailJob { block_on(ctx.core_ctx.emit(CoreEvent::NewThumbnail { cas_id })); }; } else { - println!("Thumb exists, skipping... {}", output_path.display()); + info!("Thumb exists, skipping... {}", output_path.display()); } } }) diff --git a/core/src/job/jobs.rs b/core/src/job/jobs.rs index 24b5c1095..befc88042 100644 --- a/core/src/job/jobs.rs +++ b/core/src/job/jobs.rs @@ -10,7 +10,11 @@ use crate::{ use int_enum::IntEnum; use log::info; use serde::{Deserialize, Serialize}; -use std::{collections::HashMap, fmt::Debug, sync::Arc}; +use std::{ + collections::{HashMap, VecDeque}, + fmt::Debug, + sync::Arc, +}; use tokio::sync::Mutex; use ts_rs::TS; @@ -24,7 +28,7 @@ pub trait Job: Send + Sync + Debug { // jobs struct is maintained by the core pub struct Jobs { - job_queue: Vec>, + job_queue: VecDeque>, // workers are spawned when jobs are picked off the queue running_workers: HashMap>>, } @@ -32,7 +36,7 @@ pub struct Jobs { impl Jobs { pub fn new() -> Self { Self { - job_queue: vec![], + job_queue: VecDeque::new(), running_workers: HashMap::new(), } } @@ -50,17 +54,17 @@ impl Jobs { self.running_workers.insert(id, wrapped_worker); } else { - self.job_queue.push(job); + self.job_queue.push_back(job); } } pub fn ingest_queue(&mut self, _ctx: &CoreContext, job: Box) { - self.job_queue.push(job); + self.job_queue.push_back(job); } pub async fn complete(&mut self, ctx: &CoreContext, job_id: String) { // remove worker from running workers self.running_workers.remove(&job_id); // continue queue - let job = self.job_queue.pop(); + let job = self.job_queue.pop_front(); if let Some(job) = job { self.ingest(ctx, job).await; } diff --git a/core/src/sys/locations.rs b/core/src/sys/locations.rs index 3bf291e8a..cf2b52578 100644 --- a/core/src/sys/locations.rs +++ b/core/src/sys/locations.rs @@ -78,9 +78,6 @@ pub async fn get_location( Some(location) => location, None => Err(LocationError::NotFound(location_id.to_string()))?, }; - - println!("Retrieved location: {:?}", location); - Ok(location.into()) } @@ -101,7 +98,7 @@ pub async fn new_location_and_scan( ctx.queue_job(Box::new(ThumbnailJob { location_id: location.id, - path: path.to_string(), + path: "".to_string(), background: false, })); diff --git a/packages/interface/src/components/file/Inspector.tsx b/packages/interface/src/components/file/Inspector.tsx index 913c2b745..c139ac596 100644 --- a/packages/interface/src/components/file/Inspector.tsx +++ b/packages/interface/src/components/file/Inspector.tsx @@ -1,15 +1,13 @@ import { Transition } from '@headlessui/react'; import { ShareIcon } from '@heroicons/react/solid'; -import { useBridgeCommand } from '@sd/client'; import { FilePath, LocationResource } from '@sd/core'; import { Button, TextArea } from '@sd/ui'; import moment from 'moment'; import { Heart, Link } from 'phosphor-react'; -import React, { useCallback, useEffect, useState } from 'react'; -import { useDebounce } from 'rooks'; +import React, { useEffect } from 'react'; import { default as types } from '../../constants/file-types.json'; -import { updateNote, useInspectorState } from '../../hooks/useInspectorState'; +import { useInspectorState } from '../../hooks/useInspectorState'; import FileThumb from './FileThumb'; interface MetaItemProps {