Cloud connection (#1842)

* actual cloud connection wow

* Preliminary cloud library joining

* remove dev overrides

* add back library caching

* re-enable vendored openssl

Cargo.lock

@@ -6807,6 +6807,7 @@ dependencies = [
  "async-stream",
  "async-trait",
  "axum",
+ "base64 0.21.5",
  "blake3",
  "bytes",
  "chrono",

core/Cargo.toml

@@ -107,6 +107,7 @@ bytes = "1.5.0"
 reqwest = { version = "0.11.22", features = ["json", "native-tls-vendored"] }
 directories = "5.0.1"
 async-recursion = "1.0.5"
+base64 = "0.21.5"
 
 # Override features of transitive dependencies
 [dependencies.openssl]

core/src/api/cloud.rs

@@ -0,0 +1,205 @@
+use base64::prelude::*;
+use reqwest::Response;
+use rspc::alpha::AlphaRouter;
+use sd_prisma::prisma::instance;
+use sd_utils::uuid_to_bytes;
+use serde::{de::DeserializeOwned, Deserialize, Serialize};
+use serde_json::json;
+use specta::Type;
+use uuid::Uuid;
+
+use crate::{invalidate_query, library::LibraryName};
+
+use crate::util::http::ensure_response;
+
+use super::{utils::library, Ctx, R};
+
+async fn parse_json_body<T: DeserializeOwned>(response: Response) -> Result<T, rspc::Error> {
+	response.json().await.map_err(|_| {
+		rspc::Error::new(
+			rspc::ErrorCode::InternalServerError,
+			"JSON conversion failed".to_string(),
+		)
+	})
+}
+
+pub(crate) fn mount() -> AlphaRouter<Ctx> {
+	R.router().merge("library.", library::mount())
+}
+
+mod library {
+	use chrono::{DateTime, Utc};
+
+	use crate::api::libraries::LibraryConfigWrapped;
+
+	use super::*;
+
+	#[derive(Serialize, Deserialize, Type)]
+	#[specta(inline)]
+	#[serde(rename_all = "camelCase")]
+	struct Response {
+		// id: String,
+		uuid: Uuid,
+		name: String,
+		owner_id: String,
+		instances: Vec<Instance>,
+	}
+
+	#[derive(Serialize, Deserialize, Type)]
+	#[specta(inline)]
+	#[serde(rename_all = "camelCase")]
+	struct Instance {
+		id: String,
+		uuid: Uuid,
+		identity: String,
+	}
+
+	pub fn mount() -> AlphaRouter<Ctx> {
+		R.router()
+			.procedure("get", {
+				R.with2(library())
+					.query(|(node, library), _: ()| async move {
+						let library_id = library.id;
+						let api_url = &node.env.api_url;
+
+						node.authed_api_request(
+							node.http
+								.get(&format!("{api_url}/api/v1/libraries/{library_id}")),
+						)
+						.await
+						.and_then(ensure_response)
+						.map(parse_json_body::<Option<Response>>)?
+						.await
+					})
+			})
+			.procedure("list", {
+				#[derive(Serialize, Deserialize, Type)]
+				#[specta(inline)]
+				#[serde(rename_all = "camelCase")]
+				struct Response {
+					// id: String,
+					uuid: Uuid,
+					name: String,
+					owner_id: String,
+					instances: Vec<Instance>,
+				}
+
+				#[derive(Serialize, Deserialize, Type)]
+				#[specta(inline)]
+				#[serde(rename_all = "camelCase")]
+				struct Instance {
+					id: String,
+					uuid: Uuid,
+				}
+
+				R.query(|node, _: ()| async move {
+					let api_url = &node.env.api_url;
+
+					node.authed_api_request(node.http.get(&format!("{api_url}/api/v1/libraries")))
+						.await
+						.and_then(ensure_response)
+						.map(parse_json_body::<Vec<Response>>)?
+						.await
+				})
+			})
+			.procedure("create", {
+				R.with2(library())
+					.mutation(|(node, library), _: ()| async move {
+						let api_url = &node.env.api_url;
+						let library_id = library.id;
+						let instance_uuid = library.instance_uuid;
+
+						node.authed_api_request(
+							node.http
+								.post(&format!("{api_url}/api/v1/libraries/{library_id}"))
+								.json(&json!({
+									"name": library.config().await.name,
+									"instanceUuid": library.instance_uuid,
+									"instanceIdentity": library.identity.to_remote_identity()
+								})),
+						)
+						.await
+						.and_then(ensure_response)?;
+
+						invalidate_query!(library, "cloud.library.get");
+
+						Ok(())
+					})
+			})
+			.procedure("join", {
+				R.mutation(|node, library_id: Uuid| async move {
+					let api_url = &node.env.api_url;
+
+					let Some(cloud_library) = node
+						.authed_api_request(
+							node.http
+								.get(&format!("{api_url}/api/v1/libraries/{library_id}")),
+						)
+						.await
+						.and_then(ensure_response)
+						.map(parse_json_body::<Option<Response>>)?
+						.await?
+					else {
+						return Err(rspc::Error::new(
+							rspc::ErrorCode::NotFound,
+							"Library not found".to_string(),
+						));
+					};
+
+					let library = node
+						.libraries
+						.create_with_uuid(
+							library_id,
+							LibraryName::new(cloud_library.name).unwrap(),
+							None,
+							false,
+							None,
+							&node,
+						)
+						.await?;
+
+					let instance_uuid = library.instance_uuid;
+
+					node.authed_api_request(
+						node.http
+							.post(&format!(
+								"{api_url}/api/v1/libraries/{library_id}/instances/{instance_uuid}"
+							))
+							.json(&json!({
+								"instanceIdentity": library.identity.to_remote_identity()
+							})),
+					)
+					.await
+					.and_then(ensure_response)?;
+
+					library
+						.db
+						.instance()
+						.create_many(
+							cloud_library
+								.instances
+								.into_iter()
+								.map(|instance| {
+									instance::create_unchecked(
+										uuid_to_bytes(instance.uuid),
+										BASE64_STANDARD.decode(instance.identity).unwrap(),
+										vec![],
+										"".to_string(),
+										0,
+										Utc::now().into(),
+										Utc::now().into(),
+										vec![],
+									)
+								})
+								.collect(),
+						)
+						.exec()
+						.await?;
+
+					invalidate_query!(library, "cloud.library.get");
+
+					Ok(LibraryConfigWrapped::from_library(&library).await)
+				})
+			})
+	}
+}

core/src/api/libraries.rs

@@ -35,6 +35,17 @@ pub struct LibraryConfigWrapped {
 	pub config: LibraryConfig,
 }
 
+impl LibraryConfigWrapped {
+	pub async fn from_library(library: &Library) -> Self {
+		Self {
+			uuid: library.id,
+			instance_id: library.instance_uuid,
+			instance_public_key: library.identity.to_remote_identity(),
+			config: library.config().await,
+		}
+	}
+}
+
 pub(crate) fn mount() -> AlphaRouter<Ctx> {
 	R.router()
 		.procedure("list", {
@@ -268,12 +279,7 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
 						.await?;
 					}
 
-					Ok(LibraryConfigWrapped {
-						uuid: library.id,
-						instance_id: library.instance_uuid,
-						instance_public_key: library.identity.to_remote_identity(),
-						config: library.config().await,
-					})
+					Ok(LibraryConfigWrapped::from_library(&library).await)
 				},
 			)
 		})

core/src/api/mod.rs

@@ -17,6 +17,7 @@ use uuid::Uuid;
 
 mod auth;
 mod backups;
+mod cloud;
 // mod categories;
 mod ephemeral_files;
 mod files;
@@ -179,6 +180,7 @@ pub(crate) fn mount() -> Arc<Router> {
 		})
 		.merge("api.", web_api::mount())
 		.merge("auth.", auth::mount())
+		.merge("cloud.", cloud::mount())
 		.merge("search.", search::mount())
 		.merge("library.", libraries::mount())
 		.merge("volumes.", volumes::mount())

core/src/lib.rs

@@ -275,6 +275,15 @@ impl Node {
 			)
 		})
 	}
+
+	pub async fn api_request(&self, req: RequestBuilder) -> Result<Response, rspc::Error> {
+		req.send().await.map_err(|_| {
+			rspc::Error::new(
+				rspc::ErrorCode::InternalServerError,
+				"Request failed".to_string(),
+			)
+		})
+	}
 }
 
 /// Error type for Node related errors.

interface/app/$libraryId/Layout/Sidebar/Contents.tsx

@@ -1,4 +1,4 @@
-import { ArrowsClockwise, Planet } from '@phosphor-icons/react';
+import { ArrowsClockwise, Cloud, Planet } from '@phosphor-icons/react';
 import { useNavigate } from 'react-router';
 import { LibraryContextProvider, useClientContext, useFeatureFlag } from '@sd/client';
 import { Tooltip } from '@sd/ui';
@@ -21,7 +21,6 @@ export default () => {
 
 	return (
 		<div className="no-scrollbar mask-fade-out flex grow flex-col space-y-5 overflow-x-hidden overflow-y-scroll pb-10">
-			{/* <div className="space-y-0.5"> */}
 			{/* <SidebarLink to="spacedrop">
 					<Icon component={Broadcast} />
 					Spacedrop
@@ -31,13 +30,20 @@ export default () => {
 					<Icon component={ArchiveBox} />
 					Imports
 				</SidebarLink> */}
-			{useFeatureFlag('syncRoute') && (
-				<SidebarLink to="sync">
-					<Icon component={ArrowsClockwise} />
-					Sync
-				</SidebarLink>
-			)}
-			{/* </div> */}
+			<div className="space-y-0.5">
+				{useFeatureFlag('syncRoute') && (
+					<SidebarLink to="sync">
+						<Icon component={ArrowsClockwise} />
+						Sync
+					</SidebarLink>
+				)}
+				{useFeatureFlag('cloud') && (
+					<SidebarLink to="cloud">
+						<Icon component={Cloud} />
+						Cloud
+					</SidebarLink>
+				)}
+			</div>
 			<EphemeralSection />
 			{library && (
 				<LibraryContextProvider library={library}>

interface/app/$libraryId/cloud.tsx

@@ -0,0 +1,61 @@
+import { auth, useLibraryMutation, useLibraryQuery } from '@sd/client';
+import { Button } from '@sd/ui';
+import { AuthRequiredOverlay } from '~/components/AuthRequiredOverlay';
+import { LoginButton } from '~/components/LoginButton';
+import { useRouteTitle } from '~/hooks';
+
+export const Component = () => {
+	useRouteTitle('Cloud');
+
+	const authState = auth.useStateSnapshot();
+
+	if (authState.status === 'loggedIn') return <Authenticated />;
+	if (authState.status === 'notLoggedIn')
+		return (
+			<div className="flex flex-row p-4">
+				<LoginButton />
+			</div>
+		);
+
+	return null;
+};
+
+function Authenticated() {
+	const cloudLibrary = useLibraryQuery(['cloud.library.get'], { suspense: true, retry: false });
+
+	const createLibrary = useLibraryMutation(['cloud.library.create']);
+
+	return (
+		<div className="flex flex-row p-4">
+			{cloudLibrary.data ? (
+				<div className="flex flex-col">
+					<p>Library: {cloudLibrary.data.name}</p>
+					<p>Instances</p>
+					<ul className="space-y-4 pl-4">
+						{cloudLibrary.data.instances.map((instance) => (
+							<li key={instance.id}>
+								<p>Id: {instance.id}</p>
+								<p>UUID: {instance.uuid}</p>
+								<p>Public Key: {instance.identity}</p>
+							</li>
+						))}
+					</ul>
+				</div>
+			) : (
+				<div className="relative">
+					<AuthRequiredOverlay />
+					<Button
+						disabled={createLibrary.isLoading}
+						onClick={() => {
+							createLibrary.mutateAsync(null);
+						}}
+					>
+						{createLibrary.isLoading
+							? 'Connecting library to Spacedrive Cloud...'
+							: 'Connect library to Spacedrive Cloud'}
+					</Button>
+				</div>
+			)}
+		</div>
+	);
+}

interface/app/$libraryId/index.tsx

@@ -13,7 +13,8 @@ const pageRoutes: RouteObject = {
 		{ path: 'media', lazy: () => import('./media') },
 		{ path: 'spaces', lazy: () => import('./spaces') },
 		{ path: 'debug', lazy: () => import('./debug') },
-		{ path: 'sync', lazy: () => import('./sync') }
+		{ path: 'sync', lazy: () => import('./sync') },
+		{ path: 'cloud', lazy: () => import('./cloud') }
 	]
 };
 

interface/app/onboarding/index.tsx

@@ -5,6 +5,7 @@ import Alpha from './alpha';
 import { useOnboardingContext } from './context';
 import CreatingLibrary from './creating-library';
 import { FullDisk } from './full-disk';
+import { JoinLibrary } from './join-library';
 import Locations from './locations';
 import NewLibrary from './new-library';
 import Privacy from './privacy';
@@ -36,6 +37,7 @@ export default [
 	// 	path: 'login'
 	// },
 	{ Component: NewLibrary, path: 'new-library' },
+	{ Component: JoinLibrary, path: 'join-library' },
 	{ Component: FullDisk, path: 'full-disk' },
 	{ Component: Locations, path: 'locations' },
 	{ Component: Privacy, path: 'privacy' },

interface/app/onboarding/join-library.tsx

@@ -0,0 +1,76 @@
+import { useQueryClient } from '@tanstack/react-query';
+import { useNavigate } from 'react-router';
+import { resetOnboardingStore, useBridgeMutation, useBridgeQuery } from '@sd/client';
+import { Button } from '@sd/ui';
+import { Icon } from '~/components';
+import { AuthRequiredOverlay } from '~/components/AuthRequiredOverlay';
+import { useRouteTitle } from '~/hooks';
+import { usePlatform } from '~/util/Platform';
+
+import { OnboardingContainer, OnboardingDescription, OnboardingTitle } from './components';
+
+export function JoinLibrary() {
+	useRouteTitle('Join Library');
+
+	return (
+		<OnboardingContainer>
+			<Icon name="Database" size={80} />
+			<OnboardingTitle>Join a Library</OnboardingTitle>
+			<OnboardingDescription>
+				Libraries are a secure, on-device database. Your files remain where they are, the
+				Library catalogs them and stores all Spacedrive related data.
+			</OnboardingDescription>
+
+			<div className="mt-2">
+				<span>Cloud Libraries</span>
+				<ul className="relative flex h-32 w-48 flex-col rounded border border-app-frame p-2">
+					<CloudLibraries />
+					<AuthRequiredOverlay />
+				</ul>
+			</div>
+		</OnboardingContainer>
+	);
+}
+
+function CloudLibraries() {
+	const cloudLibraries = useBridgeQuery(['cloud.library.list']);
+	const joinLibrary = useBridgeMutation(['cloud.library.join']);
+
+	const navigate = useNavigate();
+	const queryClient = useQueryClient();
+	const platform = usePlatform();
+
+	if (cloudLibraries.isLoading) return <span>Loading...</span>;
+
+	return (
+		<>
+			{cloudLibraries.data?.map((cloudLibrary) => (
+				<li key={cloudLibrary.uuid} className="flex flex-row gap-2">
+					<span>{cloudLibrary.name}</span>
+					<Button
+						variant="accent"
+						disabled={joinLibrary.isLoading}
+						onClick={async () => {
+							const library = await joinLibrary.mutateAsync(cloudLibrary.uuid);
+
+							queryClient.setQueryData(['library.list'], (libraries: any) => {
+								// The invalidation system beat us to it
+								if (libraries.find((l: any) => l.uuid === library.uuid))
+									return libraries;
+
+								return [...(libraries || []), library];
+							});
+
+							platform.refreshMenuBar && platform.refreshMenuBar();
+
+							resetOnboardingStore();
+							navigate(`/${library.uuid}`, { replace: true });
+						}}
+					>
+						{joinLibrary.isLoading ? 'Joining...' : 'Join'}
+					</Button>
+				</li>
+			))}
+		</>
+	);
+}

interface/app/onboarding/joining-library.tsx

@@ -0,0 +1,14 @@
+import { Loader } from '@sd/ui';
+
+import { OnboardingContainer, OnboardingDescription, OnboardingTitle } from './components';
+
+export default function OnboardingCreatingLibrary() {
+	return (
+		<OnboardingContainer>
+			<span className="text-6xl">🛠</span>
+			<OnboardingTitle>Joining library</OnboardingTitle>
+			<OnboardingDescription>Joing library...</OnboardingDescription>
+			<Loader className="mt-5" />
+		</OnboardingContainer>
+	);
+}

interface/app/onboarding/new-library.tsx

@@ -67,6 +67,14 @@ export default function OnboardingNewLibrary() {
 								Import library
 							</Button> */}
 						</div>
+						<span className="my-4 text-sm text-ink-faint">OR</span>
+						<Button
+							onClick={() => {
+								navigate('../join-library');
+							}}
+						>
+							Join a Library
+						</Button>
 					</>
 				)}
 			</OnboardingContainer>

package.json

@@ -21,6 +21,7 @@
 		"client": "pnpm --filter @sd/client -- ",
 		"storybook": "pnpm --filter @sd/storybook -- ",
 		"prisma": "cd core && cargo prisma",
+		"dev:desktop": "pnpm run --filter @sd/desktop tauri dev",
 		"dev:web": "turbo run dev --filter @sd/web --filter @sd/server",
 		"bootstrap:desktop": "cargo clean && ./scripts/setup.sh && pnpm i && pnpm prep && pnpm tauri dev",
 		"codegen": "cargo test -p sd-core api::tests::test_and_export_rspc_bindings -- --exact",

packages/client/src/core.ts

@@ -6,6 +6,8 @@ export type Procedures = {
         { key: "auth.me", input: never, result: { id: string; email: string } } | 
         { key: "backups.getAll", input: never, result: GetAll } | 
         { key: "buildInfo", input: never, result: BuildInfo } | 
+        { key: "cloud.library.get", input: LibraryArgs<null>, result: { uuid: string; name: string; ownerId: string; instances: { id: string; uuid: string; identity: string }[] } | null } | 
+        { key: "cloud.library.list", input: never, result: { uuid: string; name: string; ownerId: string; instances: { id: string; uuid: string }[] }[] } | 
         { key: "ephemeralFiles.getMediaData", input: string, result: MediaMetadata | null } | 
         { key: "files.get", input: LibraryArgs<GetArgs>, result: { id: number; pub_id: number[]; kind: number | null; key_id: number | null; hidden: boolean | null; favorite: boolean | null; important: boolean | null; note: string | null; date_created: string | null; date_accessed: string | null; file_paths: FilePath[] } | null } | 
         { key: "files.getConvertableImageExtensions", input: never, result: string[] } | 
@@ -49,6 +51,8 @@ export type Procedures = {
         { key: "backups.backup", input: LibraryArgs<null>, result: string } | 
         { key: "backups.delete", input: string, result: null } | 
         { key: "backups.restore", input: string, result: null } | 
+        { key: "cloud.library.create", input: LibraryArgs<null>, result: null } | 
+        { key: "cloud.library.join", input: string, result: LibraryConfigWrapped } | 
         { key: "ephemeralFiles.copyFiles", input: LibraryArgs<EphemeralFileSystemOps>, result: null } | 
         { key: "ephemeralFiles.createFolder", input: LibraryArgs<CreateEphemeralFolderArgs>, result: string } | 
         { key: "ephemeralFiles.cutFiles", input: LibraryArgs<EphemeralFileSystemOps>, result: null } | 

packages/client/src/hooks/useFeatureFlag.tsx

@@ -5,7 +5,7 @@ import type { BackendFeature } from '../core';
 import { valtioPersist } from '../lib/valito';
 import { nonLibraryClient, useBridgeQuery } from '../rspc';
 
-export const features = ['spacedrop', 'p2pPairing', 'syncRoute', 'backups'] as const;
+export const features = ['spacedrop', 'p2pPairing', 'syncRoute', 'backups', 'cloud'] as const;
 
 // This defines which backend feature flags show up in the UI.
 // This is kinda a hack to not having the runtime array of possible features as Specta only exports the types.