Software-Mac/spacedrive
Software-Mac/spacedrive
1
0
Fork 0
mirror of https://github.com/spacedriveapp/spacedrive synced 2024-07-07 03:13:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Improve QuickPreview (#1350)

Browse source 
* Handle large text files

* wip

* nit

* Fix syntax highlighting

* Requiring an API call for my dev builds, no shot

* backend for line counting + wip frontend

* 600 lines is too much for this file, ngl

* wip: `LimitedByLinesBody` & some more restructuring

* Virtualised list for QuickPreview

* yeet bad ideas

* general cleanup + hack to fix broken toml

* fix

---------

Co-authored-by: Utku <74243531+utkubakir@users.noreply.github.com>
This commit is contained in:
Oscar Beaumont 2023-09-29 14:58:29 +10:00 committed by GitHub
parent 478ebfce64
commit 02f03f5351
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
19 changed files with 945 additions and 817 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
apps/desktop/package.json
View file

@ -22,7 +22,6 @@
"@sentry/vite-plugin": "^2.7.0",
"@tanstack/react-query": "^4.24.4",
"@tauri-apps/api": "1.3.0",
"comlink": "^4.4.1",
"react": "^18.2.0",
"react-dom": "^18.2.0",
"react-router-dom": "6.9.0"

1
apps/desktop/src/vite-env.d.ts vendored
View file

@ -1,5 +1,4 @@
/// <reference types="vite/client" />
/// <reference types="vite-plugin-comlink/client" />
declare interface ImportMetaEnv {
VITE_OS: string;

11
apps/desktop/vite.config.ts
View file

@ -27,11 +27,12 @@ export default defineConfig(({ mode }) => {
},
plugins: [
devtoolsPlugin,
sentryVitePlugin({
authToken: process.env.SENTRY_AUTH_TOKEN,
org: 'spacedriveapp',
project: 'desktop'
})
process.env.SENTRY_AUTH_TOKEN &&
sentryVitePlugin({
authToken: process.env.SENTRY_AUTH_TOKEN,
org: 'spacedriveapp',
project: 'desktop'
})
]
});
});

645
core/src/custom_uri.rs
View file

@ -1,645 +0,0 @@
use crate::{
location::file_path_helper::{file_path_to_handle_custom_uri, IsolatedFilePathData},
p2p::{sync::InstanceState, IdentityOrRemoteIdentity},
prisma::{file_path, location},
util::{db::*, InfallibleResponse},
Node,
};
use std::{
cmp::min,
ffi::OsStr,
fmt::Debug,
fs::Metadata,
io::{self, SeekFrom},
panic::Location,
path::{Path, PathBuf},
pin::Pin,
str::FromStr,
sync::{atomic::Ordering, Arc},
task::{Context, Poll},
time::UNIX_EPOCH,
};
use async_stream::stream;
use axum::{
body::{self, Body, BoxBody, Full, HttpBody, StreamBody},
extract::{self, State},
http::{self, header, request, HeaderMap, HeaderValue, Method, Request, Response, StatusCode},
middleware::{self, Next},
routing::get,
Router,
};
use bytes::Bytes;
use futures::Stream;
use http_range::HttpRange;
use mini_moka::sync::Cache;
use pin_project_lite::pin_project;
use sd_file_ext::text::is_text;
use sd_p2p::{spaceblock::Range, spacetunnel::RemoteIdentity};
use tokio::{
fs::File,
io::{AsyncRead, AsyncReadExt, AsyncSeekExt, AsyncWrite, Take},
};
use tokio_util::{io::ReaderStream, sync::PollSender};
use tracing::{debug, error};
use uuid::Uuid;
type CacheKey = (Uuid, file_path::id::Type);
#[derive(Debug, Clone)]
struct CacheValue {
name: PathBuf,
ext: String,
file_path_pub_id: Uuid,
serve_from: ServeFrom,
}
const MAX_TEXT_READ_LENGTH: usize = 10 * 1024; // 10KB
// default capacity 64KiB
const DEFAULT_CAPACITY: usize = 65536;
#[derive(Debug, Clone)]
pub enum ServeFrom {
/// Serve from the local filesystem
Local,
/// Serve from a specific instance
Remote(RemoteIdentity),
}
#[derive(Clone)]
struct LocalState {
node: Arc<Node>,
// This LRU cache allows us to avoid doing a DB lookup on every request.
// The main advantage of this LRU Cache is for video files. Video files are fetch in multiple chunks and the cache prevents a DB lookup on every chunk reducing the request time from 15-25ms to 1-10ms.
// TODO: We should listen to events when deleting or moving a location and evict the cache accordingly.
file_metadata_cache: Cache<CacheKey, CacheValue>,
}
// We are using Axum on all platforms because Tauri's custom URI protocols can't be async!
// TODO(@Oscar): Long-term hopefully this can be moved into rspc but streaming files is a hard thing for rspc to solve (Eg. how does batching work, dyn-safe handler, etc).
pub fn router(node: Arc<Node>) -> Router<()> {
Router::new()
.route(
"/thumbnail/*path",
get(
|State(state): State<LocalState>,
extract::Path(path): extract::Path<String>,
request: Request<Body>| async move {
let thumbnail_path = state.node.config.data_directory().join("thumbnails");
let path = thumbnail_path.join(path);
// Prevent directory traversal attacks (Eg. requesting `../../../etc/passwd`)
// For now we only support `webp` thumbnails.
(path.starts_with(&thumbnail_path) && path.extension() == Some(OsStr::new("webp"))).then_some(()).ok_or_else(|| not_found(()))?;
let file = File::open(&path).await.map_err(|err| {
InfallibleResponse::builder()
.status(if err.kind() == io::ErrorKind::NotFound {
StatusCode::NOT_FOUND
} else {
StatusCode::INTERNAL_SERVER_ERROR
})
.body(body::boxed(Full::from("")))
})?;
let metadata = file.metadata().await;
serve_file(
file,
metadata,
request.into_parts().0,
InfallibleResponse::builder().header("Content-Type", HeaderValue::from_static("image/webp")),
)
.await
},
),
)
.route(
"/file/:lib_id/:loc_id/:path_id",
get(
|State(state): State<LocalState>,
extract::Path((lib_id, loc_id, path_id)): extract::Path<(
String,
String,
String,
)>,
request: Request<Body>| async move {
let library_id = Uuid::from_str(&lib_id).map_err(bad_request)?;
let location_id = loc_id.parse::<location::id::Type>().map_err(bad_request)?;
let file_path_id = path_id.parse::<file_path::id::Type>().map_err(bad_request)?;
let lru_cache_key = (library_id, file_path_id);
let library = state.node.libraries.get_library(&library_id).await.ok_or_else(|| internal_server_error(()))?;
let CacheValue { name: file_path_full_path, ext: extension, file_path_pub_id, serve_from } = if let Some(entry) =
state.file_metadata_cache.get(&lru_cache_key)
{
entry
} else {
let file_path = library
.db
.file_path()
.find_unique(file_path::id::equals(file_path_id))
// TODO: This query could be seen as a security issue as it could load the private key (`identity`) when we 100% don't need it. We are gonna wanna fix that!
.select(file_path_to_handle_custom_uri::select())
.exec()
.await
.map_err(internal_server_error)?
.ok_or_else(|| not_found(()))?;
let location =
maybe_missing(&file_path.location, "file_path.location").map_err(internal_server_error)?;
let path =
maybe_missing(&location.path, "file_path.location.path").map_err(internal_server_error)?;
let instance =
maybe_missing(&location.instance, "file_path.location.instance").map_err(internal_server_error)?;
let path = Path::new(path).join(
IsolatedFilePathData::try_from((location_id, &file_path)).map_err(not_found)?
);
let identity = IdentityOrRemoteIdentity::from_bytes(&instance.identity).map_err(internal_server_error)?.remote_identity();
let lru_entry = CacheValue {
name: path,
ext: maybe_missing(file_path.extension, "extension").map_err(not_found)?,
file_path_pub_id: Uuid::from_slice(&file_path.pub_id).map_err(internal_server_error)?,
serve_from: if identity == library.identity.to_remote_identity() {
ServeFrom::Local
} else {
ServeFrom::Remote(identity)
},
};
state
.file_metadata_cache
.insert(lru_cache_key, lru_entry.clone());
lru_entry
};
match serve_from {
ServeFrom::Local => {
let metadata = file_path_full_path.metadata().map_err(internal_server_error)?;
(!metadata.is_dir()).then_some(()).ok_or_else(|| not_found(()))?;
let mut file = File::open(&file_path_full_path).await.map_err(|err| {
InfallibleResponse::builder()
.status(if err.kind() == io::ErrorKind::NotFound {
StatusCode::NOT_FOUND
} else {
StatusCode::INTERNAL_SERVER_ERROR
})
.body(body::boxed(Full::from("")))
})?;
let resp = InfallibleResponse::builder().header("Content-Type", HeaderValue::from_str(&plz_for_the_love_of_all_that_is_good_replace_this_with_the_db_instead_of_adding_variants_to_it(&extension, &mut file, &metadata).await?).map_err(|err| {
error!("Error converting mime-type into header value: {}", err);
internal_server_error(())
})?);
serve_file(file, Ok(metadata), request.into_parts().0, resp).await
}
ServeFrom::Remote(identity) => {
if !state.node.files_over_p2p_flag.load(Ordering::Relaxed) {
return Ok(not_found(()))
}
// TODO: Support `Range` requests and `ETag` headers
#[allow(clippy::unwrap_used)]
match *state.node.nlm.state().await.get(&library_id).unwrap().instances.get(&identity).unwrap() {
InstanceState::Discovered(_) | InstanceState::Unavailable => Ok(not_found(())),
InstanceState::Connected(peer_id) => {
let (tx, mut rx) = tokio::sync::mpsc::channel::<io::Result<Bytes>>(150);
// TODO: We only start a thread because of stupid `ManagerStreamAction2` and libp2p's `!Send/!Sync` bounds on a stream.
let node = state.node.clone();
tokio::spawn(async move {
node.p2p
.request_file(
peer_id,
&library,
file_path_pub_id,
Range::Full,
MpscToAsyncWrite(PollSender::new(tx)),
)
.await;
});
// TODO: Content Type
Ok(InfallibleResponse::builder()
.status(StatusCode::OK)
.body(body::boxed(StreamBody::new(stream! {
while let Some(item) = rx.recv().await {
yield item;
}
}))))
}
}
},
}
},
),
)
.route_layer(middleware::from_fn(cors_middleware))
.with_state(LocalState {
node,
file_metadata_cache: Cache::new(100),
})
}
#[track_caller]
fn bad_request(err: impl Debug) -> http::Response<BoxBody> {
debug!("400: Bad Request at {}: {err:?}", Location::caller());
InfallibleResponse::builder()
.status(StatusCode::BAD_REQUEST)
.body(body::boxed(Full::from("")))
}
#[track_caller]
fn not_found(err: impl Debug) -> http::Response<BoxBody> {
debug!("404: Not Found at {}: {err:?}", Location::caller());
InfallibleResponse::builder()
.status(StatusCode::NOT_FOUND)
.body(body::boxed(Full::from("")))
}
#[track_caller]
fn internal_server_error(err: impl Debug) -> http::Response<BoxBody> {
debug!(
"500 - Internal Server Error at {}: {err:?}",
Location::caller()
);
InfallibleResponse::builder()
.status(StatusCode::INTERNAL_SERVER_ERROR)
.body(body::boxed(Full::from("")))
}
async fn cors_middleware<B>(req: Request<B>, next: Next<B>) -> Response<BoxBody> {
if req.method() == Method::OPTIONS {
return Response::builder()
.header("Access-Control-Allow-Methods", "GET, HEAD, POST, OPTIONS")
.header("Access-Control-Allow-Headers", "*")
.header("Access-Control-Max-Age", "86400")
.status(StatusCode::OK)
.body(body::boxed(Full::from("")))
.expect("Invalid static response!");
}
let mut response = next.run(req).await;
response
.headers_mut()
.insert("Access-Control-Allow-Origin", HeaderValue::from_static("*"));
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Connection
response
.headers_mut()
.insert("Connection", HeaderValue::from_static("Keep-Alive"));
response
.headers_mut()
.insert("Server", HeaderValue::from_static("Spacedrive"));
response
}
/// Serve a Tokio file as a HTTP response.
///
/// This function takes care of:
/// - 304 Not Modified using ETag's
/// - Range requests for partial content
///
/// BE AWARE this function does not do any path traversal protection so that's up to the caller!
async fn serve_file(
mut file: File,
metadata: io::Result<Metadata>,
req: request::Parts,
mut resp: InfallibleResponse,
) -> Result<Response<BoxBody>, Response<BoxBody>> {
if let Ok(metadata) = metadata {
// We only accept range queries if `files.metadata() == Ok(_)`
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Accept-Ranges
resp = resp
.header("Accept-Ranges", HeaderValue::from_static("bytes"))
.header(
"Content-Length",
HeaderValue::from_str(&metadata.len().to_string())
.expect("number won't fail conversion"),
);
// Empty files
if metadata.len() == 0 {
return Ok(resp
.status(StatusCode::OK)
.header("Content-Length", HeaderValue::from_static("0"))
.body(body::boxed(Full::from(""))));
}
// ETag
let mut status_code = StatusCode::PARTIAL_CONTENT;
if let Ok(time) = metadata.modified() {
let etag_header = format!(
r#""{}""#,
// The ETag's can be any value so we just use the modified time to make it easy.
time.duration_since(UNIX_EPOCH)
.expect("are you a time traveller? cause that's the only explanation for this error")
.as_millis()
);
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ETag
if let Ok(etag_header) = HeaderValue::from_str(&etag_header) {
resp = resp.header("etag", etag_header);
} else {
error!("Failed to convert ETag into header value!");
}
// Used for normal requests
if let Some(etag) = req.headers.get("If-None-Match") {
if etag.as_bytes() == etag_header.as_bytes() {
return Ok(resp
.status(StatusCode::NOT_MODIFIED)
.body(body::boxed(Full::from(""))));
}
}
// Used checking if the resource has been modified since starting the download
if let Some(if_range) = req.headers.get("If-Range") {
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/If-Range
if if_range.as_bytes() != etag_header.as_bytes() {
status_code = StatusCode::OK
}
}
};
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Range_requests
if req.method == Method::GET {
if let Some(range) = req.headers.get("range") {
// TODO: Error handling
let ranges = HttpRange::parse(range.to_str().map_err(bad_request)?, metadata.len())
.map_err(bad_request)?;
// TODO: Multipart requests are not support, yet
if ranges.len() != 1 {
return Ok(resp
.header(
header::CONTENT_RANGE,
HeaderValue::from_str(&format!("bytes */{}", metadata.len()))
.map_err(internal_server_error)?,
)
.status(StatusCode::RANGE_NOT_SATISFIABLE)
.body(body::boxed(Full::from(""))));
}
let range = ranges.first().expect("checked above");
if (range.start + range.length) > metadata.len() {
return Ok(resp
.header(
header::CONTENT_RANGE,
HeaderValue::from_str(&format!("bytes */{}", metadata.len()))
.map_err(internal_server_error)?,
)
.status(StatusCode::RANGE_NOT_SATISFIABLE)
.body(body::boxed(Full::from(""))));
}
file.seek(SeekFrom::Start(range.start))
.await
.map_err(internal_server_error)?;
return Ok(resp
.status(status_code)
.header(
"Content-Range",
HeaderValue::from_str(&format!(
"bytes {}-{}/{}",
range.start,
range.start + range.length - 1,
metadata.len()
))
.map_err(internal_server_error)?,
)
.header(
"Content-Length",
HeaderValue::from_str(&range.length.to_string())
.map_err(internal_server_error)?,
)
.body(body::boxed(AsyncReadBody::with_capacity_limited(
file,
DEFAULT_CAPACITY,
range.length,
))));
}
}
}
Ok(resp.body(body::boxed(StreamBody::new(ReaderStream::new(file)))))
}
// TODO: This should be determined from magic bytes when the file is indexed and stored it in the DB on the file path
async fn plz_for_the_love_of_all_that_is_good_replace_this_with_the_db_instead_of_adding_variants_to_it(
ext: &str,
file: &mut File,
metadata: &Metadata,
) -> Result<String, Response<BoxBody>> {
let mime_type = match ext {
// AAC audio
"aac" => "audio/aac",
// Musical Instrument Digital Interface (MIDI)
"mid" | "midi" => "audio/midi, audio/x-midi",
// MP3 audio
"mp3" => "audio/mpeg",
// MP4 audio
"m4a" => "audio/mp4",
// OGG audio
"oga" => "audio/ogg",
// Opus audio
"opus" => "audio/opus",
// Waveform Audio Format
"wav" => "audio/wav",
// WEBM audio
"weba" => "audio/webm",
// AVI: Audio Video Interleave
"avi" => "video/x-msvideo",
// MP4 video
"mp4" | "m4v" => "video/mp4",
// TODO: Bruh
#[cfg(not(target_os = "macos"))]
// TODO: Bruh
// FIX-ME: This media types break macOS video rendering
// MPEG transport stream
"ts" => "video/mp2t",
// TODO: Bruh
#[cfg(not(target_os = "macos"))]
// FIX-ME: This media types break macOS video rendering
// MPEG Video
"mpeg" => "video/mpeg",
// OGG video
"ogv" => "video/ogg",
// WEBM video
"webm" => "video/webm",
// 3GPP audio/video container (TODO: audio/3gpp if it doesn't contain video)
"3gp" => "video/3gpp",
// 3GPP2 audio/video container (TODO: audio/3gpp2 if it doesn't contain video)
"3g2" => "video/3gpp2",
// Quicktime movies
"mov" => "video/quicktime",
// Windows OS/2 Bitmap Graphics
"bmp" => "image/bmp",
// Graphics Interchange Format (GIF)
"gif" => "image/gif",
// Icon format
"ico" => "image/vnd.microsoft.icon",
// JPEG images
"jpeg" | "jpg" => "image/jpeg",
// Portable Network Graphics
"png" => "image/png",
// Scalable Vector Graphics (SVG)
"svg" => "image/svg+xml",
// Tagged Image File Format (TIFF)
"tif" | "tiff" => "image/tiff",
// WEBP image
"webp" => "image/webp",
// PDF document
"pdf" => "application/pdf",
// HEIF/HEIC images
"heif" | "heifs" => "image/heif,image/heif-sequence",
"heic" | "heics" => "image/heic,image/heic-sequence",
// AVIF images
"avif" | "avci" | "avcs" => "image/avif",
_ => "text/plain",
};
Ok(if mime_type == "text/plain" {
let mut text_buf = vec![
0;
min(
metadata.len().try_into().unwrap_or(usize::MAX),
MAX_TEXT_READ_LENGTH
)
];
if !text_buf.is_empty() {
file.read_exact(&mut text_buf)
.await
.map_err(internal_server_error)?;
file.seek(SeekFrom::Start(0))
.await
.map_err(internal_server_error)?;
}
let charset = is_text(&text_buf, text_buf.len() == (metadata.len() as usize)).unwrap_or("");
// Only browser recognized types, everything else should be text/plain
// https://www.iana.org/assignments/media-types/media-types.xhtml#table-text
let mime_type = match ext {
// HyperText Markup Language
"html" | "htm" => "text/html",
// Cascading Style Sheets
"css" => "text/css",
// Javascript
"js" | "mjs" => "text/javascript",
// Comma-separated values
"csv" => "text/csv",
// Markdown
"md" | "markdown" => "text/markdown",
// Rich text format
"rtf" => "text/rtf",
// Web Video Text Tracks
"vtt" => "text/vtt",
// Extensible Markup Language
"xml" => "text/xml",
// Text
"txt" => "text/plain",
_ => {
if charset.is_empty() {
todo!();
// "TODO: This filetype is not supported because of the missing mime type!",
};
mime_type
}
};
format!("{mime_type}; charset={charset}")
} else {
mime_type.to_string()
})
}
// This code was taken from: https://github.com/tower-rs/tower-http/blob/e8eb54966604ea7fa574a2a25e55232f5cfe675b/tower-http/src/services/fs/mod.rs#L30
pin_project! {
// NOTE: This could potentially be upstreamed to `http-body`.
/// Adapter that turns an [`impl AsyncRead`][tokio::io::AsyncRead] to an [`impl Body`][http_body::Body].
#[derive(Debug)]
pub struct AsyncReadBody<T> {
#[pin]
reader: ReaderStream<T>,
}
}
impl<T> AsyncReadBody<T>
where
T: AsyncRead,
{
fn with_capacity_limited(
read: T,
capacity: usize,
max_read_bytes: u64,
) -> AsyncReadBody<Take<T>> {
AsyncReadBody {
reader: ReaderStream::with_capacity(read.take(max_read_bytes), capacity),
}
}
}
impl<T> HttpBody for AsyncReadBody<T>
where
T: AsyncRead,
{
type Data = Bytes;
type Error = io::Error;
fn poll_data(
self: Pin<&mut Self>,
cx: &mut Context<'_>,
) -> Poll<Option<Result<Self::Data, Self::Error>>> {
self.project().reader.poll_next(cx)
}
fn poll_trailers(
self: Pin<&mut Self>,
_cx: &mut Context<'_>,
) -> Poll<Result<Option<HeaderMap>, Self::Error>> {
Poll::Ready(Ok(None))
}
}
/// Allowing wrapping an `mpsc::Sender` into an `AsyncWrite`
pub struct MpscToAsyncWrite(PollSender<io::Result<Bytes>>);
impl AsyncWrite for MpscToAsyncWrite {
fn poll_write(
mut self: Pin<&mut Self>,
cx: &mut Context<'_>,
buf: &[u8],
) -> Poll<Result<usize, io::Error>> {
#[allow(clippy::unwrap_used)]
match self.0.poll_reserve(cx) {
Poll::Ready(Ok(())) => {
self.0.send_item(Ok(Bytes::from(buf.to_vec()))).unwrap();
Poll::Ready(Ok(buf.len()))
}
Poll::Ready(Err(_)) => todo!(),
Poll::Pending => Poll::Pending,
}
}
fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Result<(), io::Error>> {
Poll::Ready(Ok(()))
}
fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Result<(), io::Error>> {
Poll::Ready(Ok(()))
}
}

61
core/src/custom_uri/async_read_body.rs Normal file
View file

@ -0,0 +1,61 @@
use std::{
io,
pin::Pin,
task::{Context, Poll},
};
use axum::http::HeaderMap;
use bytes::Bytes;
use futures::Stream;
use http_body::Body;
use pin_project_lite::pin_project;
use tokio::io::{AsyncRead, AsyncReadExt, Take};
use tokio_util::io::ReaderStream;
// This code was taken from: https://github.com/tower-rs/tower-http/blob/e8eb54966604ea7fa574a2a25e55232f5cfe675b/tower-http/src/services/fs/mod.rs#L30
pin_project! {
// NOTE: This could potentially be upstreamed to `http-body`.
/// Adapter that turns an [`impl AsyncRead`][tokio::io::AsyncRead] to an [`impl Body`][http_body::Body].
#[derive(Debug)]
pub struct AsyncReadBody<T> {
#[pin]
reader: ReaderStream<T>,
}
}
impl<T> AsyncReadBody<T>
where
T: AsyncRead,
{
pub(crate) fn with_capacity_limited(
read: T,
capacity: usize,
max_read_bytes: u64,
) -> AsyncReadBody<Take<T>> {
AsyncReadBody {
reader: ReaderStream::with_capacity(read.take(max_read_bytes), capacity),
}
}
}
impl<T> Body for AsyncReadBody<T>
where
T: AsyncRead,
{
type Data = Bytes;
type Error = io::Error;
fn poll_data(
self: Pin<&mut Self>,
cx: &mut Context<'_>,
) -> Poll<Option<Result<Self::Data, Self::Error>>> {
self.project().reader.poll_next(cx)
}
fn poll_trailers(
self: Pin<&mut Self>,
_cx: &mut Context<'_>,
) -> Poll<Result<Option<HeaderMap>, Self::Error>> {
Poll::Ready(Ok(None))
}
}

422
core/src/custom_uri/mod.rs Normal file
View file

@ -0,0 +1,422 @@
use crate::{
library::Library,
location::file_path_helper::{file_path_to_handle_custom_uri, IsolatedFilePathData},
p2p::{sync::InstanceState, IdentityOrRemoteIdentity},
prisma::{file_path, location},
util::{db::*, InfallibleResponse},
Node,
};
use std::{
cmp::min,
ffi::OsStr,
fmt::Debug,
fs::Metadata,
io::{self, SeekFrom},
path::{Path, PathBuf},
str::FromStr,
sync::{atomic::Ordering, Arc},
};
use async_stream::stream;
use axum::{
body::{self, Body, BoxBody, Full, StreamBody},
extract::{self, State},
http::{HeaderValue, Request, Response, StatusCode},
middleware,
routing::get,
Router,
};
use bytes::Bytes;
use mini_moka::sync::Cache;
use sd_file_ext::text::is_text;
use sd_p2p::{spaceblock::Range, spacetunnel::RemoteIdentity};
use tokio::{
fs::File,
io::{AsyncReadExt, AsyncSeekExt},
};
use tokio_util::sync::PollSender;
use tracing::error;
use uuid::Uuid;
use self::{mpsc_to_async_write::MpscToAsyncWrite, serve_file::serve_file, utils::*};
mod async_read_body;
mod mpsc_to_async_write;
mod serve_file;
mod utils;
type CacheKey = (Uuid, file_path::id::Type);
#[derive(Debug, Clone)]
struct CacheValue {
name: PathBuf,
ext: String,
file_path_pub_id: Uuid,
serve_from: ServeFrom,
}
const MAX_TEXT_READ_LENGTH: usize = 10 * 1024; // 10KB
#[derive(Debug, Clone)]
pub enum ServeFrom {
/// Serve from the local filesystem
Local,
/// Serve from a specific instance
Remote(RemoteIdentity),
}
#[derive(Clone)]
struct LocalState {
node: Arc<Node>,
// This LRU cache allows us to avoid doing a DB lookup on every request.
// The main advantage of this LRU Cache is for video files. Video files are fetch in multiple chunks and the cache prevents a DB lookup on every chunk reducing the request time from 15-25ms to 1-10ms.
// TODO: We should listen to events when deleting or moving a location and evict the cache accordingly.
file_metadata_cache: Cache<CacheKey, CacheValue>,
}
type ExtractedPath = extract::Path<(String, String, String)>;
async fn get_or_init_lru_entry(
state: &LocalState,
extract::Path((lib_id, loc_id, path_id)): ExtractedPath,
) -> Result<(CacheValue, Arc<Library>), Response<BoxBody>> {
let library_id = Uuid::from_str(&lib_id).map_err(bad_request)?;
let location_id = loc_id.parse::<location::id::Type>().map_err(bad_request)?;
let file_path_id = path_id
.parse::<file_path::id::Type>()
.map_err(bad_request)?;
let lru_cache_key = (library_id, file_path_id);
let library = state
.node
.libraries
.get_library(&library_id)
.await
.ok_or_else(|| internal_server_error(()))?;
if let Some(entry) = state.file_metadata_cache.get(&lru_cache_key) {
Ok((entry, library))
} else {
let file_path = library
.db
.file_path()
.find_unique(file_path::id::equals(file_path_id))
// TODO: This query could be seen as a security issue as it could load the private key (`identity`) when we 100% don't need it. We are gonna wanna fix that!
.select(file_path_to_handle_custom_uri::select())
.exec()
.await
.map_err(internal_server_error)?
.ok_or_else(|| not_found(()))?;
let location = maybe_missing(&file_path.location, "file_path.location")
.map_err(internal_server_error)?;
let path = maybe_missing(&location.path, "file_path.location.path")
.map_err(internal_server_error)?;
let instance = maybe_missing(&location.instance, "file_path.location.instance")
.map_err(internal_server_error)?;
let path = Path::new(path)
.join(IsolatedFilePathData::try_from((location_id, &file_path)).map_err(not_found)?);
let identity = IdentityOrRemoteIdentity::from_bytes(&instance.identity)
.map_err(internal_server_error)?
.remote_identity();
let lru_entry = CacheValue {
name: path,
ext: maybe_missing(file_path.extension, "extension").map_err(not_found)?,
file_path_pub_id: Uuid::from_slice(&file_path.pub_id).map_err(internal_server_error)?,
serve_from: if identity == library.identity.to_remote_identity() {
ServeFrom::Local
} else {
ServeFrom::Remote(identity)
},
};
state
.file_metadata_cache
.insert(lru_cache_key, lru_entry.clone());
Ok((lru_entry, library))
}
}
// We are using Axum on all platforms because Tauri's custom URI protocols can't be async!
pub fn router(node: Arc<Node>) -> Router<()> {
Router::new()
.route(
"/thumbnail/*path",
get(
|State(state): State<LocalState>,
extract::Path(path): extract::Path<String>,
request: Request<Body>| async move {
let thumbnail_path = state.node.config.data_directory().join("thumbnails");
let path = thumbnail_path.join(path);
// Prevent directory traversal attacks (Eg. requesting `../../../etc/passwd`)
// For now we only support `webp` thumbnails.
(path.starts_with(&thumbnail_path)
&& path.extension() == Some(OsStr::new("webp")))
.then_some(())
.ok_or_else(|| not_found(()))?;
let file = File::open(&path).await.map_err(|err| {
InfallibleResponse::builder()
.status(if err.kind() == io::ErrorKind::NotFound {
StatusCode::NOT_FOUND
} else {
StatusCode::INTERNAL_SERVER_ERROR
})
.body(body::boxed(Full::from("")))
})?;
let metadata = file.metadata().await;
serve_file(
file,
metadata,
request.into_parts().0,
InfallibleResponse::builder()
.header("Content-Type", HeaderValue::from_static("image/webp")),
)
.await
},
),
)
.route(
"/file/:lib_id/:loc_id/:path_id",
get(
|State(state): State<LocalState>, path: ExtractedPath, request: Request<Body>| async move {
let (
CacheValue {
name: file_path_full_path,
ext: extension,
file_path_pub_id,
serve_from,
..
},
library,
) = get_or_init_lru_entry(&state, path).await?;
match serve_from {
ServeFrom::Local => {
let metadata = file_path_full_path
.metadata()
.map_err(internal_server_error)?;
(!metadata.is_dir())
.then_some(())
.ok_or_else(|| not_found(()))?;
let mut file =
File::open(&file_path_full_path).await.map_err(|err| {
InfallibleResponse::builder()
.status(if err.kind() == io::ErrorKind::NotFound {
StatusCode::NOT_FOUND
} else {
StatusCode::INTERNAL_SERVER_ERROR
})
.body(body::boxed(Full::from("")))
})?;
let resp = InfallibleResponse::builder().header(
"Content-Type",
HeaderValue::from_str(
&infer_the_mime_type(&extension, &mut file, &metadata).await?,
)
.map_err(|err| {
error!("Error converting mime-type into header value: {}", err);
internal_server_error(())
})?,
);
serve_file(file, Ok(metadata), request.into_parts().0, resp).await
}
ServeFrom::Remote(identity) => {
if !state.node.files_over_p2p_flag.load(Ordering::Relaxed) {
return Ok(not_found(()));
}
// TODO: Support `Range` requests and `ETag` headers
#[allow(clippy::unwrap_used)]
match *state
.node
.nlm
.state()
.await
.get(&library.id)
.unwrap()
.instances
.get(&identity)
.unwrap()
{
InstanceState::Discovered(_) | InstanceState::Unavailable => {
Ok(not_found(()))
}
InstanceState::Connected(peer_id) => {
let (tx, mut rx) =
tokio::sync::mpsc::channel::<io::Result<Bytes>>(150);
// TODO: We only start a thread because of stupid `ManagerStreamAction2` and libp2p's `!Send/!Sync` bounds on a stream.
let node = state.node.clone();
tokio::spawn(async move {
node.p2p
.request_file(
peer_id,
&library,
file_path_pub_id,
Range::Full,
MpscToAsyncWrite::new(PollSender::new(tx)),
)
.await;
});
// TODO: Content Type
Ok(InfallibleResponse::builder().status(StatusCode::OK).body(
body::boxed(StreamBody::new(stream! {
while let Some(item) = rx.recv().await {
yield item;
}
})),
))
}
}
}
}
},
),
)
.route_layer(middleware::from_fn(cors_middleware))
.with_state(LocalState {
node,
file_metadata_cache: Cache::new(150),
})
}
// TODO: This should possibly be determined from magic bytes when the file is indexed and stored it in the DB on the file path
async fn infer_the_mime_type(
ext: &str,
file: &mut File,
metadata: &Metadata,
) -> Result<String, Response<BoxBody>> {
let mime_type = match ext {
// AAC audio
"aac" => "audio/aac",
// Musical Instrument Digital Interface (MIDI)
"mid" | "midi" => "audio/midi, audio/x-midi",
// MP3 audio
"mp3" => "audio/mpeg",
// MP4 audio
"m4a" => "audio/mp4",
// OGG audio
"oga" => "audio/ogg",
// Opus audio
"opus" => "audio/opus",
// Waveform Audio Format
"wav" => "audio/wav",
// WEBM audio
"weba" => "audio/webm",
// AVI: Audio Video Interleave
"avi" => "video/x-msvideo",
// MP4 video
"mp4" | "m4v" => "video/mp4",
// TODO: Bruh
#[cfg(not(target_os = "macos"))]
// TODO: Bruh
// FIX-ME: This media types break macOS video rendering
// MPEG transport stream
"ts" => "video/mp2t",
// TODO: Bruh
#[cfg(not(target_os = "macos"))]
// FIX-ME: This media types break macOS video rendering
// MPEG Video
"mpeg" => "video/mpeg",
// OGG video
"ogv" => "video/ogg",
// WEBM video
"webm" => "video/webm",
// 3GPP audio/video container (TODO: audio/3gpp if it doesn't contain video)
"3gp" => "video/3gpp",
// 3GPP2 audio/video container (TODO: audio/3gpp2 if it doesn't contain video)
"3g2" => "video/3gpp2",
// Quicktime movies
"mov" => "video/quicktime",
// Windows OS/2 Bitmap Graphics
"bmp" => "image/bmp",
// Graphics Interchange Format (GIF)
"gif" => "image/gif",
// Icon format
"ico" => "image/vnd.microsoft.icon",
// JPEG images
"jpeg" | "jpg" => "image/jpeg",
// Portable Network Graphics
"png" => "image/png",
// Scalable Vector Graphics (SVG)
"svg" => "image/svg+xml",
// Tagged Image File Format (TIFF)
"tif" | "tiff" => "image/tiff",
// WEBP image
"webp" => "image/webp",
// PDF document
"pdf" => "application/pdf",
// HEIF/HEIC images
"heif" | "heifs" => "image/heif,image/heif-sequence",
"heic" | "heics" => "image/heic,image/heic-sequence",
// AVIF images
"avif" | "avci" | "avcs" => "image/avif",
_ => "text/plain",
};
Ok(if mime_type == "text/plain" {
let mut text_buf = vec![
0;
min(
metadata.len().try_into().unwrap_or(usize::MAX),
MAX_TEXT_READ_LENGTH
)
];
if !text_buf.is_empty() {
file.read_exact(&mut text_buf)
.await
.map_err(internal_server_error)?;
file.seek(SeekFrom::Start(0))
.await
.map_err(internal_server_error)?;
}
let charset = is_text(&text_buf, text_buf.len() == (metadata.len() as usize)).unwrap_or("");
// Only browser recognized types, everything else should be text/plain
// https://www.iana.org/assignments/media-types/media-types.xhtml#table-text
let mime_type = match ext {
// HyperText Markup Language
"html" | "htm" => "text/html",
// Cascading Style Sheets
"css" => "text/css",
// Javascript
"js" | "mjs" => "text/javascript",
// Comma-separated values
"csv" => "text/csv",
// Markdown
"md" | "markdown" => "text/markdown",
// Rich text format
"rtf" => "text/rtf",
// Web Video Text Tracks
"vtt" => "text/vtt",
// Extensible Markup Language
"xml" => "text/xml",
// Text
"txt" => "text/plain",
_ => {
if charset.is_empty() {
todo!();
// "TODO: This filetype is not supported because of the missing mime type!",
};
mime_type
}
};
format!("{mime_type}; charset={charset}")
} else {
mime_type.to_string()
})
}

44
core/src/custom_uri/mpsc_to_async_write.rs Normal file
View file

@ -0,0 +1,44 @@
use std::{
io,
pin::Pin,
task::{Context, Poll},
};
use bytes::Bytes;
use tokio::io::AsyncWrite;
use tokio_util::sync::PollSender;
/// Allowing wrapping an `mpsc::Sender` into an `AsyncWrite`
pub struct MpscToAsyncWrite(PollSender<io::Result<Bytes>>);
impl MpscToAsyncWrite {
pub fn new(sender: PollSender<io::Result<Bytes>>) -> Self {
Self(sender)
}
}
impl AsyncWrite for MpscToAsyncWrite {
fn poll_write(
mut self: Pin<&mut Self>,
cx: &mut Context<'_>,
buf: &[u8],
) -> Poll<Result<usize, io::Error>> {
#[allow(clippy::unwrap_used)]
match self.0.poll_reserve(cx) {
Poll::Ready(Ok(())) => {
self.0.send_item(Ok(Bytes::from(buf.to_vec()))).unwrap();
Poll::Ready(Ok(buf.len()))
}
Poll::Ready(Err(_)) => todo!(),
Poll::Pending => Poll::Pending,
}
}
fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Result<(), io::Error>> {
Poll::Ready(Ok(()))
}
fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Result<(), io::Error>> {
Poll::Ready(Ok(()))
}
}

153
core/src/custom_uri/serve_file.rs Normal file
View file

@ -0,0 +1,153 @@
use crate::util::InfallibleResponse;
use std::{
fs::Metadata,
io::{self, SeekFrom},
time::UNIX_EPOCH,
};
use axum::{
body::{self, BoxBody, Full, StreamBody},
http::{header, request, HeaderValue, Method, Response, StatusCode},
};
use http_range::HttpRange;
use tokio::{fs::File, io::AsyncSeekExt};
use tokio_util::io::ReaderStream;
use tracing::error;
use super::{async_read_body::AsyncReadBody, utils::*};
// default capacity 64KiB
const DEFAULT_CAPACITY: usize = 65536;
/// Serve a Tokio file as a HTTP response.
///
/// This function takes care of:
/// - 304 Not Modified using ETag's
/// - Range requests for partial content
///
/// BE AWARE this function does not do any path traversal protection so that's up to the caller!
pub(crate) async fn serve_file(
mut file: File,
metadata: io::Result<Metadata>,
req: request::Parts,
mut resp: InfallibleResponse,
) -> Result<Response<BoxBody>, Response<BoxBody>> {
if let Ok(metadata) = metadata {
// We only accept range queries if `files.metadata() == Ok(_)`
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Accept-Ranges
resp = resp
.header("Accept-Ranges", HeaderValue::from_static("bytes"))
.header(
"Content-Length",
HeaderValue::from_str(&metadata.len().to_string())
.expect("number won't fail conversion"),
);
// Empty files
if metadata.len() == 0 {
return Ok(resp
.status(StatusCode::OK)
.header("Content-Length", HeaderValue::from_static("0"))
.body(body::boxed(Full::from(""))));
}
// ETag
let mut status_code = StatusCode::PARTIAL_CONTENT;
if let Ok(time) = metadata.modified() {
let etag_header = format!(
r#""{}""#,
// The ETag's can be any value so we just use the modified time to make it easy.
time.duration_since(UNIX_EPOCH)
.expect("are you a time traveller? cause that's the only explanation for this error")
.as_millis()
);
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ETag
if let Ok(etag_header) = HeaderValue::from_str(&etag_header) {
resp = resp.header("etag", etag_header);
} else {
error!("Failed to convert ETag into header value!");
}
// Used for normal requests
if let Some(etag) = req.headers.get("If-None-Match") {
if etag.as_bytes() == etag_header.as_bytes() {
return Ok(resp
.status(StatusCode::NOT_MODIFIED)
.body(body::boxed(Full::from(""))));
}
}
// Used checking if the resource has been modified since starting the download
if let Some(if_range) = req.headers.get("If-Range") {
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/If-Range
if if_range.as_bytes() != etag_header.as_bytes() {
status_code = StatusCode::OK
}
}
};
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Range_requests
if req.method == Method::GET {
if let Some(range) = req.headers.get("range") {
// TODO: Error handling
let ranges = HttpRange::parse(range.to_str().map_err(bad_request)?, metadata.len())
.map_err(bad_request)?;
// TODO: Multipart requests are not support, yet
if ranges.len() != 1 {
return Ok(resp
.header(
header::CONTENT_RANGE,
HeaderValue::from_str(&format!("bytes */{}", metadata.len()))
.map_err(internal_server_error)?,
)
.status(StatusCode::RANGE_NOT_SATISFIABLE)
.body(body::boxed(Full::from(""))));
}
let range = ranges.first().expect("checked above");
if (range.start + range.length) > metadata.len() {
return Ok(resp
.header(
header::CONTENT_RANGE,
HeaderValue::from_str(&format!("bytes */{}", metadata.len()))
.map_err(internal_server_error)?,
)
.status(StatusCode::RANGE_NOT_SATISFIABLE)
.body(body::boxed(Full::from(""))));
}
file.seek(SeekFrom::Start(range.start))
.await
.map_err(internal_server_error)?;
return Ok(resp
.status(status_code)
.header(
"Content-Range",
HeaderValue::from_str(&format!(
"bytes {}-{}/{}",
range.start,
range.start + range.length - 1,
metadata.len()
))
.map_err(internal_server_error)?,
)
.header(
"Content-Length",
HeaderValue::from_str(&range.length.to_string())
.map_err(internal_server_error)?,
)
.body(body::boxed(AsyncReadBody::with_capacity_limited(
file,
DEFAULT_CAPACITY,
range.length,
))));
}
}
}
Ok(resp.body(body::boxed(StreamBody::new(ReaderStream::new(file)))))
}

74
core/src/custom_uri/utils.rs Normal file
View file

@ -0,0 +1,74 @@
use std::{fmt::Debug, panic::Location};
use axum::{
body::{self, BoxBody},
http::{self, HeaderValue, Method, Request, Response, StatusCode},
middleware::Next,
};
use http_body::Full;
use tracing::debug;
use crate::util::InfallibleResponse;
#[track_caller]
pub(crate) fn bad_request(err: impl Debug) -> http::Response<BoxBody> {
debug!("400: Bad Request at {}: {err:?}", Location::caller());
InfallibleResponse::builder()
.status(StatusCode::BAD_REQUEST)
.body(body::boxed(Full::from("")))
}
#[track_caller]
pub(crate) fn not_found(err: impl Debug) -> http::Response<BoxBody> {
debug!("404: Not Found at {}: {err:?}", Location::caller());
InfallibleResponse::builder()
.status(StatusCode::NOT_FOUND)
.body(body::boxed(Full::from("")))
}
#[track_caller]
pub(crate) fn internal_server_error(err: impl Debug) -> http::Response<BoxBody> {
debug!(
"500 - Internal Server Error at {}: {err:?}",
Location::caller()
);
InfallibleResponse::builder()
.status(StatusCode::INTERNAL_SERVER_ERROR)
.body(body::boxed(Full::from("")))
}
pub(crate) async fn cors_middleware<B>(req: Request<B>, next: Next<B>) -> Response<BoxBody> {
if req.method() == Method::OPTIONS {
return Response::builder()
.header("Access-Control-Allow-Methods", "GET, HEAD, POST, OPTIONS")
.header("Access-Control-Allow-Origin", "*")
.header("Access-Control-Allow-Headers", "*")
.header("Access-Control-Max-Age", "86400")
.status(StatusCode::OK)
.body(body::boxed(Full::from("")))
.expect("Invalid static response!");
}
let mut response = next.run(req).await;
{
let headers = response.headers_mut();
headers.insert("Access-Control-Allow-Origin", HeaderValue::from_static("*"));
headers.insert(
"Access-Control-Allow-Headers",
HeaderValue::from_static("*"),
);
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Connection
headers.insert("Connection", HeaderValue::from_static("Keep-Alive"));
headers.insert("Server", HeaderValue::from_static("Spacedrive"));
}
response
}

9
interface/app/$libraryId/Explorer/FilePath/Thumb.tsx
View file

@ -2,6 +2,7 @@ import { getIcon, iconNames } from '@sd/assets/util';
import clsx from 'clsx';
import {
memo,
useCallback,
useEffect,
useLayoutEffect,
useMemo,
@ -44,6 +45,7 @@ export interface ThumbProps {
className?: string;
frameClassName?: string;
childClassName?: string | ((type: ThumbType | `${ThumbType}`) => string | undefined);
isSidebarPreview?: boolean;
}
export const FileThumb = memo((props: ThumbProps) => {
@ -67,16 +69,16 @@ export const FileThumb = memo((props: ThumbProps) => {
isDark ? classes.checkers : classes.checkersLight
);
const onLoad = () => setLoaded(true);
const onLoad = useCallback(() => setLoaded(true), []);
const onError = () => {
const onError = useCallback(() => {
setLoaded(false);
setThumbType((prevThumbType) =>
prevThumbType === ThumbType.Original && itemData.hasLocalThumbnail
? ThumbType.Thumbnail
: ThumbType.Icon
);
};
}, [itemData.hasLocalThumbnail]);
// useLayoutEffect is required to ensure the thumbType is always updated before the onError listener can execute,
// thus avoiding improper thumb types changes
@ -197,6 +199,7 @@ export const FileThumb = memo((props: ThumbProps) => {
itemData.extension) ||
''
}
isSidebarPreview={props.isSidebarPreview}
/>
);

1
interface/app/$libraryId/Explorer/Inspector/index.tsx
View file

@ -148,6 +148,7 @@ const Thumbnails = ({ items }: { items: ExplorerItem[] }) => {
? 'shadow-md shadow-app-shade'
: undefined
}
isSidebarPreview={true}
/>
))}
</>

183
interface/components/TextViewer/index.tsx
View file

@ -1,62 +1,59 @@
import { useVirtualizer, VirtualItem } from '@tanstack/react-virtual';
import clsx from 'clsx';
import Prism from 'prismjs';
import { memo, useEffect, useRef, useState } from 'react';
import './prism.css';
import * as prism from './prism';
export interface TextViewerProps {
src: string;
className?: string;
onLoad?: (event: HTMLElementEventMap['load']) => void;
onError?: (event: HTMLElementEventMap['error']) => void;
className?: string;
codeExtension?: string;
isSidebarPreview?: boolean;
}
// prettier-ignore
type Worker = typeof import('./worker')
export const worker = new ComlinkWorker<Worker>(new URL('./worker', import.meta.url));
const NEW_LINE_EXP = /\n(?!$)/g;
// TODO: ANSI support
export const TextViewer = memo(
({ src, onLoad, onError, className, codeExtension }: TextViewerProps) => {
const ref = useRef<HTMLPreElement>(null);
const [highlight, setHighlight] = useState<{
code: string;
length: number;
language: string;
}>();
const [textContent, setTextContent] = useState('');
({ src, className, onLoad, onError, codeExtension, isSidebarPreview }: TextViewerProps) => {
const [lines, setLines] = useState<string[]>([]);
const parentRef = useRef<HTMLPreElement>(null);
const rowVirtualizer = useVirtualizer({
count: lines.length,
getScrollElement: () => parentRef.current,
estimateSize: () => 25
});
useEffect(() => {
// Ignore empty urls
if (!src || src === '#') return;
const controller = new AbortController();
fetch(src, { mode: 'cors', signal: controller.signal })
fetch(src, {
mode: 'cors',
signal: controller.signal
})
.then(async (response) => {
if (!response.ok) throw new Error(`Invalid response: ${response.statusText}`);
const text = await response.text();
if (controller.signal.aborted) return;
if (!response.body) return;
onLoad?.(new UIEvent('load', {}));
setTextContent(text);
if (codeExtension) {
try {
const env = await worker.highlight(text, codeExtension);
if (env && !controller.signal.aborted) {
const match = text.match(NEW_LINE_EXP);
setHighlight({
...env,
length: (match ? match.length + 1 : 1) + 1
});
}
} catch (error) {
console.error(error);
}
}
const reader = response.body.pipeThrough(new TextDecoderStream()).getReader();
const ingestLines = async () => {
const { done, value } = await reader.read();
if (done) return;
const chunks = value.split('\n');
setLines((lines) => [...lines, ...chunks]);
if (isSidebarPreview) return;
await ingestLines();
};
ingestLines();
})
.catch((error) => {
if (!controller.signal.aborted)
@ -64,40 +61,94 @@ export const TextViewer = memo(
});
return () => controller.abort();
}, [src, onError, onLoad, codeExtension]);
}, [src, onError, onLoad, codeExtension, isSidebarPreview]);
return (
<pre
ref={ref}
tabIndex={0}
className={clsx(
'text-ink',
className,
highlight && ['relative !pl-[3.8em]', `language-${highlight.language}`]
)}
>
{highlight ? (
<>
<span className="pointer-events-none absolute left-0 top-[1em] w-[3em] select-none text-[100%] tracking-[-1px] text-ink-dull">
{Array.from(highlight, (_, i) => (
<span
key={i}
className={clsx('token block text-end', i % 2 && 'bg-black/40')}
>
{i + 1}
</span>
))}
</span>
<code
style={{ whiteSpace: 'inherit' }}
className={clsx('relative', `language-${highlight.language}`)}
dangerouslySetInnerHTML={{ __html: highlight.code }}
<pre ref={parentRef} tabIndex={0} className={className}>
<div
tabIndex={0}
className={clsx(
'relative w-full whitespace-pre text-ink',
codeExtension &&
`language-${prism.languageMapping.get(codeExtension) ?? codeExtension}`
)}
style={{
height: `${rowVirtualizer.getTotalSize()}px`
}}
>
{rowVirtualizer.getVirtualItems().map((row) => (
<TextRow
key={row.key}
codeExtension={codeExtension}
row={row}
content={lines[row.index]!}
/>
</>
) : (
textContent
)}
))}
</div>
</pre>
);
}
);
function TextRow({
codeExtension,
row,
content
}: {
codeExtension?: string;
row: VirtualItem;
content: string;
}) {
const contentRef = useRef<HTMLSpanElement>(null);
useEffect(() => {
if (contentRef.current) {
const cb: IntersectionObserverCallback = (events) => {
for (const event of events) {
if (
!event.isIntersecting ||
contentRef.current?.getAttribute('data-highlighted') === 'true'
)
continue;
contentRef.current?.setAttribute('data-highlighted', 'true');
Prism.highlightElement(event.target, false); // Prism's async seems to be broken
// With this class present TOML headers are broken Eg. `[dependencies]` will format over multiple lines
const children = contentRef.current?.children;
if (children) {
for (const elem of children) {
elem.classList.remove('table');
}
}
}
};
new IntersectionObserver(cb).observe(contentRef.current);
}
}, []);
return (
<div
className={clsx('absolute left-0 top-0 flex w-full whitespace-pre')}
style={{
height: `${row.size}px`,
transform: `translateY(${row.start}px)`
}}
>
{codeExtension && (
<div
key={row.key}
className={clsx(
'token block w-[3.8em] shrink-0 whitespace-pre pl-1 text-end',
row.index % 2 && 'bg-black/40'
)}
>
{row.index + 1}
</div>
)}
<span ref={contentRef} className="flex-1 pl-2">
{content}
</span>
</div>
);
}

33
interface/components/TextViewer/prism.ts
View file

@ -2,6 +2,12 @@
// WARNING: Import order matters
window.Prism = window.Prism || {};
Prism.manual = true;
import "prismjs";
import './prism.css';
// Languages
// Do not include default ones: markup, html, xml, svg, mathml, ssml, atom, rss, css, clike, javascript, js
import 'prismjs/components/prism-applescript.js';
@ -52,3 +58,30 @@ import 'prismjs/components/prism-typoscript.js';
import 'prismjs/components/prism-vala.js';
import 'prismjs/components/prism-yaml.js';
import 'prismjs/components/prism-zig.js';
// Mapping between extensions and prismjs language identifier
// Only for those that are not already internally resolved by prismjs
// https://prismjs.com/#supported-languages
export const languageMapping = Object.entries({
applescript: ['scpt', 'scptd'],
// This is not entirely correct, but better than nothing:
// https://github.com/PrismJS/prism/issues/3656
// https://github.com/PrismJS/prism/issues/3660
sh: ['zsh', 'fish'],
c: ['h'],
cpp: ['hpp'],
js: ['mjs'],
crystal: ['cr'],
cs: ['csx'],
makefile: ['make'],
nim: ['nims'],
objc: ['m', 'mm'],
ocaml: ['ml', 'mli', 'mll', 'mly'],
perl: ['pl'],
php: ['php', 'php1', 'php2', 'php3', 'php4', 'php5', 'php6', 'phps', 'phpt', 'phtml'],
powershell: ['ps1', 'psd1', 'psm1'],
rust: ['rs']
}).reduce<Map<string, string>>((mapping, [id, exts]) => {
for (const ext of exts) mapping.set(ext, id);
return mapping;
}, new Map());

45
interface/components/TextViewer/worker.ts
View file

@ -1,45 +0,0 @@
import Prism from 'prismjs';
import './prism';
// if you are intending to use Prism functions manually, you will need to set:
Prism.manual = true;
// Mapping between extensions and prismjs language identifier
// Only for those that are not already internally resolved by prismjs
// https://prismjs.com/#supported-languages
const languageMapping = Object.entries({
applescript: ['scpt', 'scptd'],
// This is not entirely correct, but better than nothing:
// https://github.com/PrismJS/prism/issues/3656
// https://github.com/PrismJS/prism/issues/3660
sh: ['zsh', 'fish'],
c: ['h'],
cpp: ['hpp'],
js: ['mjs'],
crystal: ['cr'],
cs: ['csx'],
makefile: ['make'],
nim: ['nims'],
objc: ['m', 'mm'],
ocaml: ['ml', 'mli', 'mll', 'mly'],
perl: ['pl'],
php: ['php', 'php1', 'php2', 'php3', 'php4', 'php5', 'php6', 'phps', 'phpt', 'phtml'],
powershell: ['ps1', 'psd1', 'psm1'],
rust: ['rs']
}).reduce<Map<string, string>>((mapping, [id, exts]) => {
for (const ext of exts) mapping.set(ext, id);
return mapping;
}, new Map());
export const highlight = (code: string, ext: string) => {
const language = languageMapping.get(ext) ?? ext;
const grammar = Prism.languages[language];
return grammar
? {
code: Prism.highlight(code, grammar, language),
language
}
: null;
};

5
interface/package.json
View file

@ -34,8 +34,9 @@
"@tailwindcss/forms": "^0.5.3",
"@tanstack/react-query": "^4.12.0",
"@tanstack/react-query-devtools": "^4.22.0",
"@tanstack/react-table": "^8.10.0",
"@tanstack/react-virtual": "3.0.0-beta.54",
"@tanstack/react-table": "^8.8.5",
"@tanstack/react-virtual": "3.0.0-beta.61",
"@types/react-scroll-sync": "^0.8.4",
"@types/uuid": "^9.0.2",
"@vitejs/plugin-react": "^2.1.0",
"autoprefixer": "^10.4.12",

2
interface/tsconfig.json
View file

@ -5,7 +5,7 @@
"paths": {
"~/*": ["./*"]
},
"types": ["vite-plugin-comlink/client", "vite-plugin-svgr/client", "vite/client", "node"]
"types": ["vite-plugin-svgr/client", "vite/client", "node"]
},
"include": ["./**/*"],
"exclude": ["dist"],

1
packages/config/package.json
View file

@ -23,7 +23,6 @@
"eslint-plugin-tailwindcss": "^3.12.0",
"eslint-utils": "^3.0.0",
"regexpp": "^3.2.0",
"vite-plugin-comlink": "^3.0.5",
"vite-plugin-html": "^3.2.0",
"vite-plugin-svgr": "^2.2.1"
}

7
packages/config/vite/index.ts
View file

@ -1,6 +1,5 @@
import react from '@vitejs/plugin-react';
import { defineConfig } from 'vite';
import { comlink } from 'vite-plugin-comlink';
import { createHtmlPlugin } from 'vite-plugin-html';
import svg from 'vite-plugin-svgr';
import tsconfigPaths from 'vite-tsconfig-paths';
@ -12,8 +11,7 @@ export default defineConfig({
svg({ svgrOptions: { icon: true } }),
createHtmlPlugin({
minify: true
}),
comlink()
})
],
css: {
modules: {
@ -25,8 +23,5 @@ export default defineConfig({
sourcemap: true,
outDir: '../dist',
assetsDir: '.'
},
worker: {
plugins: [comlink()]
}
});

64
pnpm-lock.yaml
View file

@ -83,9 +83,6 @@ importers:
'@tauri-apps/api':
specifier: 1.3.0
version: 1.3.0
comlink:
specifier: ^4.4.1
version: 4.4.1
react:
specifier: ^18.2.0
version: 18.2.0
@ -677,11 +674,14 @@ importers:
specifier: ^4.22.0
version: 4.22.0(@tanstack/react-query@4.29.1)(react-dom@18.2.0)(react@18.2.0)
'@tanstack/react-table':
specifier: ^8.10.0
version: 8.10.0(react-dom@18.2.0)(react@18.2.0)
specifier: ^8.8.5
version: 8.8.5(react-dom@18.2.0)(react@18.2.0)
'@tanstack/react-virtual':
specifier: 3.0.0-beta.54
version: 3.0.0-beta.54(react@18.2.0)
specifier: 3.0.0-beta.61
version: 3.0.0-beta.61(react@18.2.0)
'@types/react-scroll-sync':
specifier: ^0.8.4
version: 0.8.4
'@types/uuid':
specifier: ^9.0.2
version: 9.0.2
@ -914,9 +914,6 @@ importers:
regexpp:
specifier: ^3.2.0
version: 3.2.0
vite-plugin-comlink:
specifier: ^3.0.5
version: 3.0.5(comlink@4.4.1)(vite@3.2.7)
vite-plugin-html:
specifier: ^3.2.0
version: 3.2.0(vite@3.2.7)
@ -10081,34 +10078,34 @@ packages:
react-native: 0.72.4(@babel/core@7.22.11)(@babel/preset-env@7.22.10)(react@18.2.0)
use-sync-external-store: 1.2.0(react@18.2.0)
/@tanstack/react-table@8.10.0(react-dom@18.2.0)(react@18.2.0):
resolution: {integrity: sha512-FNhKE3525hryvuWw90xRbP16qNiq7OLJkDZopOKcwyktErLi1ibJzAN9DFwA/gR1br9SK4StXZh9JPvp9izrrQ==}
/@tanstack/react-table@8.8.5(react-dom@18.2.0)(react@18.2.0):
resolution: {integrity: sha512-g/t21E/ICHvaCOJOhsDNr5QaB/6aDQEHFbx/YliwwU/CJThMqG+dS28vnToIBV/5MBgpeXoGRi2waDJVJlZrtg==}
engines: {node: '>=12'}
peerDependencies:
react: '>=16'
react-dom: '>=16'
dependencies:
'@tanstack/table-core': 8.10.0
'@tanstack/table-core': 8.8.5
react: 18.2.0
react-dom: 18.2.0(react@18.2.0)
dev: false
/@tanstack/react-virtual@3.0.0-beta.54(react@18.2.0):
resolution: {integrity: sha512-D1mDMf4UPbrtHRZZriCly5bXTBMhylslm4dhcHqTtDJ6brQcgGmk8YD9JdWBGWfGSWPKoh2x1H3e7eh+hgPXtQ==}
/@tanstack/react-virtual@3.0.0-beta.61(react@18.2.0):
resolution: {integrity: sha512-ElaOqL3FvJE075/6mRbv8VBHhiF5kHjGA7eo1O+x7sPWO7+fzkTr7t7AhNaWzRLxhUpQfbb6XG3ltp8mpfFLQg==}
peerDependencies:
react: ^16.8.0 || ^17.0.0 || ^18.0.0
dependencies:
'@tanstack/virtual-core': 3.0.0-beta.54
'@tanstack/virtual-core': 3.0.0-beta.61
react: 18.2.0
dev: false
/@tanstack/table-core@8.10.0:
resolution: {integrity: sha512-e701yAJ18aGDP6mzVworlFAmQ+gi3Wtqx5mGZUe2BUv4W4D80dJxUfkHdtEGJ6GryAnlCCNTej7eaJiYmPhyYg==}
/@tanstack/table-core@8.8.5:
resolution: {integrity: sha512-Xnwa1qxpgvSW7ozLiexmKp2PIYcLBiY/IizbdGriYCL6OOHuZ9baRhrrH51zjyz+61ly6K59rmt6AI/3RR+97Q==}
engines: {node: '>=12'}
dev: false
/@tanstack/virtual-core@3.0.0-beta.54:
resolution: {integrity: sha512-jtkwqdP2rY2iCCDVAFuaNBH3fiEi29aTn2RhtIoky8DTTiCdc48plpHHreLwmv1PICJ4AJUUESaq3xa8fZH8+g==}
/@tanstack/virtual-core@3.0.0-beta.61:
resolution: {integrity: sha512-C3iu9rkWvjJ/KbYAjuwLTjqiR9fcf9lXjRBLHDzsL/M6cR7/HLSU94abSdVrzuiqRvP7u7e0FeqPK0O3VbvWyg==}
dev: false
/@tauri-apps/api@1.3.0:
@ -10578,6 +10575,12 @@ packages:
'@types/react': 18.0.38
dev: true
/@types/react-scroll-sync@0.8.4:
resolution: {integrity: sha512-88N2vgZdVqlwr5ayH/5GNAAjfdlzhted/qPTyXgT/DzQwsuIWkwFpZtoOhyGCRmxUC3w5wA+ZhkpbzagIXWNaQ==}
dependencies:
'@types/react': 18.0.38
dev: false
/@types/react@18.0.38:
resolution: {integrity: sha512-ExsidLLSzYj4cvaQjGnQCk4HFfVT9+EZ9XZsQ8Hsrcn8QNgXtpZ3m9vSIC2MWtx7jHictK6wYhQgGh6ic58oOw==}
dependencies:
@ -12240,9 +12243,6 @@ packages:
dependencies:
delayed-stream: 1.0.0
/comlink@4.4.1:
resolution: {integrity: sha512-+1dlx0aY5Jo1vHy/tSsIGpSkN4tS9rZSW8FIhG0JH/crs9wwweswIo/POr451r7bZww3hFbPAKnTpimzL/mm4Q==}
/comma-separated-tokens@2.0.3:
resolution: {integrity: sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg==}
dev: false
@ -16521,12 +16521,6 @@ packages:
minimist: 1.2.8
dev: true
/json5@2.2.1:
resolution: {integrity: sha512-1hqLFMSrGHRHxav9q9gNjJ5EXznIxGVO09xQRrwplcS8qs28pZ8s8hupZAmqDwZUmVZ2Qb2jnyPOWcDH8m8dlA==}
engines: {node: '>=6'}
hasBin: true
dev: true
/json5@2.2.3:
resolution: {integrity: sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==}
engines: {node: '>=6'}
@ -23387,18 +23381,6 @@ packages:
vfile-message: 3.1.4
dev: false
/vite-plugin-comlink@3.0.5(comlink@4.4.1)(vite@3.2.7):
resolution: {integrity: sha512-my8BE9GFJEaLc7l3e2SfRUL8JJsN9On8PiW7q4Eyq3g6DHUsNqo5WlS7Butuzc8ngrs24Tf1RC8Xfdda+E5T9w==}
peerDependencies:
comlink: ^4.3.1
vite: '>=2.9.6'
dependencies:
comlink: 4.4.1
json5: 2.2.1
magic-string: 0.26.7
vite: 3.2.7
dev: true
/vite-plugin-html@3.2.0(vite@3.2.7):
resolution: {integrity: sha512-2VLCeDiHmV/BqqNn5h2V+4280KRgQzCFN47cst3WiNK848klESPQnzuC3okH5XHtgwHH/6s1Ho/YV6yIO0pgoQ==}
peerDependencies: