mirror of
https://github.com/spacedriveapp/spacedrive
synced 2024-07-08 07:12:49 +00:00
Merge pull request #517 from spacedriveapp/eng-304-too-many-files-macos-unhappy-with-prod
[ENG-304] "Too many files" macOS unhappy with prod!
This commit is contained in:
Ericson "Fogo" Soares
GitHub
commit
31b5c4b1db
10
Cargo.lock
generated
10
Cargo.lock
generated
|
|
@ -1795,6 +1795,15 @@ version = "1.2.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2022715d62ab30faffd124d40b76f4134a550a87792276512b18d63272333394"
|
||||
|
||||
[[package]]
|
||||
name = "fsevent-sys"
|
||||
version = "4.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "76ee7a02da4d231650c7cea31349b889be2f45ddb3ef3032d2ec8185f6313fd2"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fuchsia-cprng"
|
||||
version = "0.1.1"
|
||||
|
|
@ -3574,6 +3583,7 @@ checksum = "ed2c66da08abae1c024c01d635253e402341b4060a12e99b31c7594063bf490a"
|
|||
dependencies = [
|
||||
"bitflags",
|
||||
"filetime",
|
||||
"fsevent-sys",
|
||||
"inotify",
|
||||
"kqueue",
|
||||
"libc",
|
||||
|
|
|
|||
|
|
@ -70,7 +70,7 @@ ctor = "0.1.23"
|
|||
globset = { version = "^0.4.9", features = ["serde1"] }
|
||||
itertools = "^0.10.5"
|
||||
enumflags2 = "0.7.5"
|
||||
notify = { version = "5.0.0", default-features = false, features = ["macos_kqueue"], optional = true }
|
||||
notify = { version = "5.0.0", default-features = false, features = ["macos_fsevent"], optional = true }
|
||||
uhlc = "0.5.1"
|
||||
|
||||
[dev-dependencies]
|
||||
|
|
|
|||
|
|
@ -3,24 +3,21 @@ use crate::{
|
|||
location::{indexer::indexer_job::indexer_job_location, manager::LocationManagerError},
|
||||
};
|
||||
|
||||
use std::{future::Future, time::Duration};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use notify::{
|
||||
event::{CreateKind, DataChange, ModifyKind, RenameMode},
|
||||
Event, EventKind,
|
||||
};
|
||||
use tokio::{fs, select, spawn, sync::oneshot, time::sleep};
|
||||
use tracing::{trace, warn};
|
||||
use tracing::trace;
|
||||
|
||||
use super::{
|
||||
utils::{create_dir, create_file, remove_event, rename, update_file},
|
||||
utils::{create_dir, file_creation_or_update, remove_event, rename},
|
||||
EventHandler,
|
||||
};
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub(super) struct MacOsEventHandler {
|
||||
maybe_rename_sender: Option<oneshot::Sender<Event>>,
|
||||
rename_stack: Option<Event>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
|
|
@ -41,60 +38,27 @@ impl EventHandler for MacOsEventHandler {
|
|||
trace!("Received MacOS event: {:#?}", event);
|
||||
|
||||
match event.kind {
|
||||
EventKind::Create(create_kind) => match create_kind {
|
||||
CreateKind::File => {
|
||||
let (maybe_rename_tx, maybe_rename_rx) = oneshot::channel();
|
||||
spawn(wait_to_create(
|
||||
location,
|
||||
event,
|
||||
library_ctx.clone(),
|
||||
create_file,
|
||||
maybe_rename_rx,
|
||||
));
|
||||
self.maybe_rename_sender = Some(maybe_rename_tx);
|
||||
}
|
||||
CreateKind::Folder => {
|
||||
let (maybe_rename_tx, maybe_rename_rx) = oneshot::channel();
|
||||
spawn(wait_to_create(
|
||||
location,
|
||||
event,
|
||||
library_ctx.clone(),
|
||||
create_dir,
|
||||
maybe_rename_rx,
|
||||
));
|
||||
self.maybe_rename_sender = Some(maybe_rename_tx);
|
||||
}
|
||||
other => {
|
||||
trace!("Ignoring other create event: {:#?}", other);
|
||||
}
|
||||
},
|
||||
EventKind::Modify(ref modify_kind) => match modify_kind {
|
||||
ModifyKind::Data(DataChange::Any) => {
|
||||
if fs::metadata(&event.paths[0]).await?.is_file() {
|
||||
update_file(location, event, library_ctx).await?;
|
||||
} else {
|
||||
trace!("Unexpected MacOS modify event on a directory");
|
||||
EventKind::Create(CreateKind::Folder) => {
|
||||
create_dir(location, event, library_ctx.clone()).await?;
|
||||
}
|
||||
EventKind::Modify(ModifyKind::Data(DataChange::Content)) => {
|
||||
// If a file had its content modified, then it was updated or created
|
||||
file_creation_or_update(location, event, library_ctx).await?;
|
||||
}
|
||||
EventKind::Modify(ModifyKind::Name(RenameMode::Any)) => {
|
||||
match self.rename_stack.take() {
|
||||
None => {
|
||||
self.rename_stack = Some(event);
|
||||
}
|
||||
Some(from_event) => {
|
||||
rename(&event.paths[0], &from_event.paths[0], location, library_ctx)
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// We ignore EventKind::Modify(ModifyKind::Data(DataChange::Any)) for directories
|
||||
// as they're also used for removing files and directories, being emitted
|
||||
// on the parent directory in this case
|
||||
}
|
||||
ModifyKind::Name(RenameMode::Any) => {
|
||||
if let Some(rename_sender) = self.maybe_rename_sender.take() {
|
||||
if !rename_sender.is_closed() && rename_sender.send(event).is_err() {
|
||||
warn!("Failed to send rename event");
|
||||
}
|
||||
}
|
||||
}
|
||||
other => {
|
||||
trace!("Ignoring other modify event: {:#?}", other);
|
||||
}
|
||||
},
|
||||
EventKind::Remove(remove_kind) => {
|
||||
remove_event(location, event, remove_kind, library_ctx).await?;
|
||||
// An EventKind::Modify(ModifyKind::Data(DataChange::Any)) - On parent directory
|
||||
// is also emitted, but we can ignore it.
|
||||
}
|
||||
other_event_kind => {
|
||||
trace!("Other MacOS event that we don't handle for now: {other_event_kind:#?}");
|
||||
|
|
@ -104,27 +68,3 @@ impl EventHandler for MacOsEventHandler {
|
|||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
// FIX-ME: Had some troubles with borrowck, to receive a
|
||||
// impl FnOnce(indexer_job_location::Data, Event, &LibraryContext) -> Fut
|
||||
// as a parameter, had to move LibraryContext into the functions
|
||||
async fn wait_to_create<Fut>(
|
||||
location: indexer_job_location::Data,
|
||||
event: Event,
|
||||
library_ctx: LibraryContext,
|
||||
create_fn: impl FnOnce(indexer_job_location::Data, Event, LibraryContext) -> Fut,
|
||||
maybe_rename_rx: oneshot::Receiver<Event>,
|
||||
) -> Result<(), LocationManagerError>
|
||||
where
|
||||
Fut: for<'r> Future<Output = Result<(), LocationManagerError>>,
|
||||
{
|
||||
select! {
|
||||
() = sleep(Duration::from_secs(1)) => {
|
||||
create_fn(location, event, library_ctx).await
|
||||
},
|
||||
Ok(rename_event) = maybe_rename_rx => {
|
||||
trace!("Renaming file or directory instead of creating a new one");
|
||||
rename(&event.paths[0], &rename_event.paths[0], location, &library_ctx).await
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -252,78 +252,77 @@ impl Drop for LocationWatcher {
|
|||
}
|
||||
|
||||
/***************************************************************************************************
|
||||
* Some tests to validate our assumptions of events through different file systems *
|
||||
***************************************************************************************************
|
||||
* Events dispatched on Linux: *
|
||||
* Create File: *
|
||||
* 1) EventKind::Create(CreateKind::File) *
|
||||
* 2) EventKind::Modify(ModifyKind::Metadata(MetadataKind::Any)) *
|
||||
* or EventKind::Modify(ModifyKind::Data(DataChange::Any)) *
|
||||
* 3) EventKind::Access(AccessKind::Close(AccessMode::Write))) *
|
||||
* Create Directory: *
|
||||
* 1) EventKind::Create(CreateKind::Folder) *
|
||||
* Update File: *
|
||||
* 1) EventKind::Modify(ModifyKind::Data(DataChange::Any)) *
|
||||
* 2) EventKind::Access(AccessKind::Close(AccessMode::Write))) *
|
||||
* Update File (rename): *
|
||||
* 1) EventKind::Modify(ModifyKind::Name(RenameMode::From)) *
|
||||
* 2) EventKind::Modify(ModifyKind::Name(RenameMode::To)) *
|
||||
* 3) EventKind::Modify(ModifyKind::Name(RenameMode::Both)) *
|
||||
* Update Directory (rename): *
|
||||
* 1) EventKind::Modify(ModifyKind::Name(RenameMode::From)) *
|
||||
* 2) EventKind::Modify(ModifyKind::Name(RenameMode::To)) *
|
||||
* 3) EventKind::Modify(ModifyKind::Name(RenameMode::Both)) *
|
||||
* Delete File: *
|
||||
* 1) EventKind::Remove(RemoveKind::File) *
|
||||
* Delete Directory: *
|
||||
* 1) EventKind::Remove(RemoveKind::Folder) *
|
||||
* *
|
||||
* Events dispatched on MacOS: *
|
||||
* Create File: *
|
||||
* 1) EventKind::Create(CreateKind::File) *
|
||||
* Create Directory: *
|
||||
* 1) EventKind::Create(CreateKind::Folder) *
|
||||
* Update File: *
|
||||
* 1) EventKind::Modify(ModifyKind::Data(DataChange::Any)) *
|
||||
* Update File (rename): *
|
||||
* 1) EventKind::Create(CreateKind::File) *
|
||||
* 2) EventKind::Modify(ModifyKind::Name(RenameMode::Any)) *
|
||||
* Update Directory (rename): *
|
||||
* 1) EventKind::Create(CreateKind::Folder) *
|
||||
* 2) EventKind::Modify(ModifyKind::Name(RenameMode::Any)) *
|
||||
* Delete File: *
|
||||
* 1) EventKind::Remove(RemoveKind::Any) *
|
||||
* 2) EventKind::Modify(ModifyKind::Data(DataChange::Any)) - On parent directory *
|
||||
* Delete Directory: *
|
||||
* 1) EventKind::Remove(RemoveKind::Any) *
|
||||
* 2) EventKind::Modify(ModifyKind::Data(DataChange::Any)) - On parent directory *
|
||||
* *
|
||||
* Events dispatched on Windows: *
|
||||
* Create File: *
|
||||
* 1) EventKind::Create(CreateKind::Any) *
|
||||
* 2) EventKind::Modify(ModifyKind::Any) *
|
||||
* Create Directory: *
|
||||
* 1) EventKind::Create(CreateKind::Any) *
|
||||
* Update File: *
|
||||
* 1) EventKind::Modify(ModifyKind::Any) *
|
||||
* Update File (rename): *
|
||||
* 1) EventKind::Modify(ModifyKind::Name(RenameMode::From)) *
|
||||
* 2) EventKind::Modify(ModifyKind::Name(RenameMode::To)) *
|
||||
* Update Directory (rename): *
|
||||
* 1) EventKind::Modify(ModifyKind::Name(RenameMode::From)) *
|
||||
* 2) EventKind::Modify(ModifyKind::Name(RenameMode::To)) *
|
||||
* Delete File: *
|
||||
* 1) EventKind::Remove(RemoveKind::Any) *
|
||||
* Delete Directory: *
|
||||
* 1) EventKind::Remove(RemoveKind::Any) *
|
||||
* *
|
||||
* Events dispatched on Android: *
|
||||
* TODO *
|
||||
* *
|
||||
* Events dispatched on iOS: *
|
||||
* TODO *
|
||||
* *
|
||||
**************************************************************************************************/
|
||||
* Some tests to validate our assumptions of events through different file systems *
|
||||
***************************************************************************************************
|
||||
* Events dispatched on Linux: *
|
||||
* Create File: *
|
||||
* 1) EventKind::Create(CreateKind::File) *
|
||||
* 2) EventKind::Modify(ModifyKind::Metadata(MetadataKind::Any)) *
|
||||
* or EventKind::Modify(ModifyKind::Data(DataChange::Any)) *
|
||||
* 3) EventKind::Access(AccessKind::Close(AccessMode::Write))) *
|
||||
* Create Directory: *
|
||||
* 1) EventKind::Create(CreateKind::Folder) *
|
||||
* Update File: *
|
||||
* 1) EventKind::Modify(ModifyKind::Data(DataChange::Any)) *
|
||||
* 2) EventKind::Access(AccessKind::Close(AccessMode::Write))) *
|
||||
* Update File (rename): *
|
||||
* 1) EventKind::Modify(ModifyKind::Name(RenameMode::From)) *
|
||||
* 2) EventKind::Modify(ModifyKind::Name(RenameMode::To)) *
|
||||
* 3) EventKind::Modify(ModifyKind::Name(RenameMode::Both)) *
|
||||
* Update Directory (rename): *
|
||||
* 1) EventKind::Modify(ModifyKind::Name(RenameMode::From)) *
|
||||
* 2) EventKind::Modify(ModifyKind::Name(RenameMode::To)) *
|
||||
* 3) EventKind::Modify(ModifyKind::Name(RenameMode::Both)) *
|
||||
* Delete File: *
|
||||
* 1) EventKind::Remove(RemoveKind::File) *
|
||||
* Delete Directory: *
|
||||
* 1) EventKind::Remove(RemoveKind::Folder) *
|
||||
* *
|
||||
* Events dispatched on MacOS: *
|
||||
* Create File: *
|
||||
* 1) EventKind::Create(CreateKind::File) *
|
||||
* 2) EventKind::Modify(ModifyKind::Data(DataChange::Content)) *
|
||||
* Create Directory: *
|
||||
* 1) EventKind::Create(CreateKind::Folder) *
|
||||
* Update File: *
|
||||
* 1) EventKind::Modify(ModifyKind::Data(DataChange::Content)) *
|
||||
* Update File (rename): *
|
||||
* 1) EventKind::Modify(ModifyKind::Name(RenameMode::Any)) -- From *
|
||||
* 2) EventKind::Modify(ModifyKind::Name(RenameMode::Any)) -- To *
|
||||
* Update Directory (rename): *
|
||||
* 1) EventKind::Modify(ModifyKind::Name(RenameMode::Any)) -- From *
|
||||
* 2) EventKind::Modify(ModifyKind::Name(RenameMode::Any)) -- To *
|
||||
* Delete File: *
|
||||
* 1) EventKind::Remove(RemoveKind::File) *
|
||||
* Delete Directory: *
|
||||
* 1) EventKind::Remove(RemoveKind::Folder) *
|
||||
* *
|
||||
* Events dispatched on Windows: *
|
||||
* Create File: *
|
||||
* 1) EventKind::Create(CreateKind::Any) *
|
||||
* 2) EventKind::Modify(ModifyKind::Any) *
|
||||
* Create Directory: *
|
||||
* 1) EventKind::Create(CreateKind::Any) *
|
||||
* Update File: *
|
||||
* 1) EventKind::Modify(ModifyKind::Any) *
|
||||
* Update File (rename): *
|
||||
* 1) EventKind::Modify(ModifyKind::Name(RenameMode::From)) *
|
||||
* 2) EventKind::Modify(ModifyKind::Name(RenameMode::To)) *
|
||||
* Update Directory (rename): *
|
||||
* 1) EventKind::Modify(ModifyKind::Name(RenameMode::From)) *
|
||||
* 2) EventKind::Modify(ModifyKind::Name(RenameMode::To)) *
|
||||
* Delete File: *
|
||||
* 1) EventKind::Remove(RemoveKind::Any) *
|
||||
* Delete Directory: *
|
||||
* 1) EventKind::Remove(RemoveKind::Any) *
|
||||
* *
|
||||
* Events dispatched on Android: *
|
||||
* TODO *
|
||||
* *
|
||||
* Events dispatched on iOS: *
|
||||
* TODO *
|
||||
* *
|
||||
**************************************************************************************************/
|
||||
#[cfg(test)]
|
||||
#[allow(unused)]
|
||||
mod tests {
|
||||
|
|
@ -334,7 +333,10 @@ mod tests {
|
|||
Config, Event, EventKind, RecommendedWatcher, Watcher,
|
||||
};
|
||||
use std::io::ErrorKind;
|
||||
use std::{path::Path, time::Duration};
|
||||
use std::{
|
||||
path::{Path, PathBuf},
|
||||
time::Duration,
|
||||
};
|
||||
use tempfile::{tempdir, TempDir};
|
||||
use tokio::{fs, io::AsyncWriteExt, sync::mpsc, time::sleep};
|
||||
use tracing::{debug, error};
|
||||
|
|
@ -365,16 +367,21 @@ mod tests {
|
|||
path: impl AsRef<Path>,
|
||||
expected_event: EventKind,
|
||||
) {
|
||||
debug!("Expecting event: {expected_event:#?}");
|
||||
let path = path.as_ref();
|
||||
debug!(
|
||||
"Expecting event: {expected_event:#?} at path: {}",
|
||||
path.display()
|
||||
);
|
||||
let mut tries = 0;
|
||||
loop {
|
||||
match events_rx.try_recv() {
|
||||
Ok(maybe_event) => {
|
||||
let event = maybe_event.expect("Failed to receive event");
|
||||
debug!("Received event: {event:#?}");
|
||||
// In case of file creation, we expect to see an close event on write mode
|
||||
if event.paths[0] == path && event.kind == expected_event {
|
||||
// Using `ends_with` and removing root path here due to a weird edge case on CI tests at MacOS
|
||||
if event.paths[0].ends_with(path.iter().skip(1).collect::<PathBuf>())
|
||||
&& event.kind == expected_event
|
||||
{
|
||||
debug!("Received expected event: {expected_event:#?}");
|
||||
break;
|
||||
}
|
||||
|
|
@ -409,7 +416,12 @@ mod tests {
|
|||
expect_event(events_rx, &file_path, EventKind::Modify(ModifyKind::Any)).await;
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
expect_event(events_rx, &file_path, EventKind::Create(CreateKind::File)).await;
|
||||
expect_event(
|
||||
events_rx,
|
||||
&file_path,
|
||||
EventKind::Modify(ModifyKind::Data(DataChange::Content)),
|
||||
)
|
||||
.await;
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
expect_event(
|
||||
|
|
@ -488,7 +500,7 @@ mod tests {
|
|||
expect_event(
|
||||
events_rx,
|
||||
&file_path,
|
||||
EventKind::Modify(ModifyKind::Data(DataChange::Any)),
|
||||
EventKind::Modify(ModifyKind::Data(DataChange::Content)),
|
||||
)
|
||||
.await;
|
||||
|
||||
|
|
@ -627,12 +639,7 @@ mod tests {
|
|||
expect_event(events_rx, &file_path, EventKind::Remove(RemoveKind::Any)).await;
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
expect_event(
|
||||
events_rx,
|
||||
&root_dir.path(),
|
||||
EventKind::Modify(ModifyKind::Data(DataChange::Any)),
|
||||
)
|
||||
.await;
|
||||
expect_event(events_rx, &file_path, EventKind::Remove(RemoveKind::File)).await;
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
expect_event(events_rx, &file_path, EventKind::Remove(RemoveKind::File)).await;
|
||||
|
|
@ -679,12 +686,7 @@ mod tests {
|
|||
expect_event(events_rx, &dir_path, EventKind::Remove(RemoveKind::Any)).await;
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
expect_event(
|
||||
events_rx,
|
||||
&root_dir.path(),
|
||||
EventKind::Modify(ModifyKind::Data(DataChange::Any)),
|
||||
)
|
||||
.await;
|
||||
expect_event(events_rx, &dir_path, EventKind::Remove(RemoveKind::Folder)).await;
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
expect_event(events_rx, &dir_path, EventKind::Remove(RemoveKind::Folder)).await;
|
||||
|
|
|
|||
Loading…
Reference in a new issue