Linted Files according to the .prettierrc.

This commit is contained in:
Harry Hopkinson 2022-04-24 21:16:55 +01:00
parent 980109408d
commit d7cbba7a12
36 changed files with 6823 additions and 1578 deletions

View file

@ -3,8 +3,8 @@ on:
pull_request: pull_request:
push: push:
branches: branches:
- main - main
- ci - ci
workflow_dispatch: workflow_dispatch:
jobs: jobs:
@ -18,115 +18,115 @@ jobs:
RUST_CACHE_VERSION: 0 RUST_CACHE_VERSION: 0
runs-on: ${{ matrix.platform }} runs-on: ${{ matrix.platform }}
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v2
# from https://github.com/zmwangx/rust-ffmpeg/blob/master/.github/workflows/build.yml # from https://github.com/zmwangx/rust-ffmpeg/blob/master/.github/workflows/build.yml
- name: Install ffmpeg (Windows) - name: Install ffmpeg (Windows)
if: matrix.platform == 'windows-latest' if: matrix.platform == 'windows-latest'
run: | run: |
$VCINSTALLDIR = $(& "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe" -latest -property installationPath) $VCINSTALLDIR = $(& "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe" -latest -property installationPath)
Add-Content $env:GITHUB_ENV "LIBCLANG_PATH=${VCINSTALLDIR}\VC\Tools\LLVM\x64\bin`n" Add-Content $env:GITHUB_ENV "LIBCLANG_PATH=${VCINSTALLDIR}\VC\Tools\LLVM\x64\bin`n"
Invoke-WebRequest "${env:FFMPEG_DOWNLOAD_URL}" -OutFile ffmpeg-release-full-shared.7z Invoke-WebRequest "${env:FFMPEG_DOWNLOAD_URL}" -OutFile ffmpeg-release-full-shared.7z
7z x ffmpeg-release-full-shared.7z 7z x ffmpeg-release-full-shared.7z
mkdir ffmpeg mkdir ffmpeg
mv ffmpeg-*/* ffmpeg/ mv ffmpeg-*/* ffmpeg/
Add-Content $env:GITHUB_ENV "FFMPEG_DIR=${pwd}\ffmpeg`n" Add-Content $env:GITHUB_ENV "FFMPEG_DIR=${pwd}\ffmpeg`n"
Add-Content $env:GITHUB_PATH "${pwd}\ffmpeg\bin`n" Add-Content $env:GITHUB_PATH "${pwd}\ffmpeg\bin`n"
- name: Install CMake (Windows) - name: Install CMake (Windows)
uses: lukka/get-cmake@latest uses: lukka/get-cmake@latest
if: matrix.platform == 'windows-latest' if: matrix.platform == 'windows-latest'
# Optimisation for windows # Optimisation for windows
- name: Rename existing rust toolchain (Windows) - name: Rename existing rust toolchain (Windows)
if: matrix.platform == 'windows-latest' if: matrix.platform == 'windows-latest'
run: Rename-Item C:\Users\runneradmin\.rustup\toolchains\stable-x86_64-pc-windows-msvc C:\Users\runneradmin\.rustup\toolchains\stable-x86_64-pc-windows-msvc.old run: Rename-Item C:\Users\runneradmin\.rustup\toolchains\stable-x86_64-pc-windows-msvc C:\Users\runneradmin\.rustup\toolchains\stable-x86_64-pc-windows-msvc.old
- name: Setup Node - name: Setup Node
uses: actions/setup-node@v1 uses: actions/setup-node@v1
with: with:
node-version: 16 node-version: 16
- name: Install Rust stable - name: Install Rust stable
uses: actions-rs/toolchain@v1 uses: actions-rs/toolchain@v1
with: with:
toolchain: stable toolchain: stable
profile: minimal profile: minimal
override: true override: true
components: rustfmt, rust-src components: rustfmt, rust-src
- name: Cache Rust Dependencies - name: Cache Rust Dependencies
uses: Swatinem/rust-cache@cb2cf0cc7c5198d3364b9630e2c3d457f160790c uses: Swatinem/rust-cache@cb2cf0cc7c5198d3364b9630e2c3d457f160790c
with: with:
sharedKey: ${{ env.RUST_CACHE_VERSION }} sharedKey: ${{ env.RUST_CACHE_VERSION }}
- name: Cache pnpm dependencies - name: Cache pnpm dependencies
uses: actions/cache@v2 uses: actions/cache@v2
with: with:
path: ~/.pnpm-store path: ~/.pnpm-store
key: ${{ runner.os }}-${{ hashFiles('**/pnpm-lock.yaml') }} key: ${{ runner.os }}-${{ hashFiles('**/pnpm-lock.yaml') }}
restore-keys: | restore-keys: |
${{ runner.os }}- ${{ runner.os }}-
- name: Install pnpm - name: Install pnpm
uses: pnpm/action-setup@v2.2.1 uses: pnpm/action-setup@v2.2.1
with: with:
version: 6.32.6 version: 6.32.6
- name: Install dependencies (Ubuntu) - name: Install dependencies (Ubuntu)
if: matrix.platform == 'ubuntu-latest' if: matrix.platform == 'ubuntu-latest'
run: | run: |
sudo apt-get update sudo apt-get update
sudo apt-get install -y --no-install-recommends \ sudo apt-get install -y --no-install-recommends \
libgtk-3-dev \ libgtk-3-dev \
webkit2gtk-4.0 \ webkit2gtk-4.0 \
libappindicator3-dev \ libappindicator3-dev \
librsvg2-dev \ librsvg2-dev \
patchelf \ patchelf \
libssl-dev \ libssl-dev \
libavcodec-dev \ libavcodec-dev \
libavdevice-dev \ libavdevice-dev \
libavfilter-dev \ libavfilter-dev \
libavformat-dev \ libavformat-dev \
libavresample-dev \ libavresample-dev \
libavutil-dev \ libavutil-dev \
libswscale-dev \ libswscale-dev \
libswresample-dev \ libswresample-dev \
pkg-config \ pkg-config \
ffmpeg ffmpeg
- name: Install dependencies (macOS) - name: Install dependencies (macOS)
if: matrix.platform == 'macos-latest' if: matrix.platform == 'macos-latest'
run: | run: |
brew install ffmpeg brew install ffmpeg
- name: Install pnpm dependencies - name: Install pnpm dependencies
run: pnpm i run: pnpm i
- name: Build codegen - name: Build codegen
run: pnpm prep:ci run: pnpm prep:ci
- name: Build frontend - name: Build frontend
run: pnpm desktop build:vite run: pnpm desktop build:vite
- name: Build Tauri app - name: Build Tauri app
uses: tauri-apps/tauri-action@v0 uses: tauri-apps/tauri-action@v0
with: with:
projectPath: apps/desktop projectPath: apps/desktop
env: env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Build and publish server - name: Build and publish server
if: matrix.platform == 'ubuntu-latest' if: matrix.platform == 'ubuntu-latest'
uses: ./.github/actions/build-and-publish-server uses: ./.github/actions/build-and-publish-server
with: with:
gh_token: ${{ secrets.GITHUB_TOKEN }} gh_token: ${{ secrets.GITHUB_TOKEN }}
- name: Deploy Spacedrive Server to Kubernetes - name: Deploy Spacedrive Server to Kubernetes
if: matrix.platform == 'ubuntu-latest' if: matrix.platform == 'ubuntu-latest'
env: env:
K8S_KUBECONFIG: ${{ secrets.K8S_KUBECONFIG }} K8S_KUBECONFIG: ${{ secrets.K8S_KUBECONFIG }}
run: | run: |
mkdir -p ~/.kube mkdir -p ~/.kube
echo "$K8S_KUBECONFIG" > ~/.kube/config 2>&1 echo "$K8S_KUBECONFIG" > ~/.kube/config 2>&1
kubectl rollout restart deployment/sdserver-deployment kubectl rollout restart deployment/sdserver-deployment

View file

@ -3,9 +3,9 @@ name: Update Org README
on: on:
push: push:
branches: branches:
- main - main
paths: paths:
- README.md - README.md
workflow_dispatch: workflow_dispatch:
jobs: jobs:

View file

@ -58,36 +58,41 @@ For independent creatives, hoarders and those that want to own their digital foo
</p> </p>
> NOTE: Spacedrive is under active development, most of the listed features are still experimental and subject to change. Additionally, most of the links on this page are broken but will be working once the repository is made public. > NOTE: Spacedrive is under active development, most of the listed features are still experimental and subject to change. Additionally, most of the links on this page are broken but will be working once the repository is made public.
>
# What is a VDFS? # What is a VDFS?
A VDFS (virtual distributed filesystem) is a filesystem designed to work atop a variety of storage layers. It is not restricted to a single machine, with a uniform API to manipulate and access content across many devices. It achieves this by maintaining a virtual index of all storage locations, synchronizing the database between clients in realtime. This implementation also uses [CAS](https://en.wikipedia.org/wiki/Content-addressable_storage) (Content-addressable storage) to uniquely identify files, while keeping record of logical file paths relative to the storage locations. A VDFS (virtual distributed filesystem) is a filesystem designed to work atop a variety of storage layers. It is not restricted to a single machine, with a uniform API to manipulate and access content across many devices. It achieves this by maintaining a virtual index of all storage locations, synchronizing the database between clients in realtime. This implementation also uses [CAS](https://en.wikipedia.org/wiki/Content-addressable_storage) (Content-addressable storage) to uniquely identify files, while keeping record of logical file paths relative to the storage locations.
The first implementation of a VDFS can be found in this UC Berkeley [paper](https://www2.eecs.berkeley.edu/Pubs/TechRpts/2018/EECS-2018-29.pdf) by Haoyuan Li. This paper describes its use for cloud computing, however the underlying concepts can be translated to open consumer software. The first implementation of a VDFS can be found in this UC Berkeley [paper](https://www2.eecs.berkeley.edu/Pubs/TechRpts/2018/EECS-2018-29.pdf) by Haoyuan Li. This paper describes its use for cloud computing, however the underlying concepts can be translated to open consumer software.
# Motivation # Motivation
Many of us have multiple cloud accounts, drives that arent backed up and data at risk of loss. We depend on cloud services like Google Photos and iCloud, but are locked in with limited capacity and almost zero interoperability between services and operating systems. Photo albums shouldnt be suck in a device ecosystem, or harvested for advertising data. They should be OS agnostic, permanent and personally owned. Data we create is our legacy, that will long outlive us—open source technology is the only way to ensure we retain absolute control over the data that defines our lives, at unlimited scale. Many of us have multiple cloud accounts, drives that arent backed up and data at risk of loss. We depend on cloud services like Google Photos and iCloud, but are locked in with limited capacity and almost zero interoperability between services and operating systems. Photo albums shouldnt be suck in a device ecosystem, or harvested for advertising data. They should be OS agnostic, permanent and personally owned. Data we create is our legacy, that will long outlive us—open source technology is the only way to ensure we retain absolute control over the data that defines our lives, at unlimited scale.
# Features # Features
_Note: Links are for highlight purposes only until feature specific documentation is complete._ _Note: Links are for highlight purposes only until feature specific documentation is complete._
**Complete:** *(in testing)* **Complete:** _(in testing)_
- **[File discovery](#features)** - Scan devices, drives and cloud accounts to build a directory of all files with metadata. - **[File discovery](#features)** - Scan devices, drives and cloud accounts to build a directory of all files with metadata.
- **[Preview generation](#features)** - Auto generate lower resolution stand-ins for image and video. - **[Preview generation](#features)** - Auto generate lower resolution stand-ins for image and video.
- **[Statistics](#features)** - Total capacity, index size, preview media size, free space etc. - **[Statistics](#features)** - Total capacity, index size, preview media size, free space etc.
**In progress:** **In progress:**
- **[File Explorer](#features)** - Browse online/offline storage locations, view files with metadata, perform basic CRUD. - **[File Explorer](#features)** - Browse online/offline storage locations, view files with metadata, perform basic CRUD.
- **[Realtime synchronization](#features)** - Data index synchronized in realtime between devices, prioritizing peer-to-peer LAN connections (WiFi sync). - **[Realtime synchronization](#features)** - Data index synchronized in realtime between devices, prioritizing peer-to-peer LAN connections (WiFi sync).
**To be developed (MVP):** **To be developed (MVP):**
- **[Photos](#features)** - Photo and video albums similar to Apple/Google photos. - **[Photos](#features)** - Photo and video albums similar to Apple/Google photos.
- **[Search](#features)** - Deep search into your filesystem with a keybind, including offline locations. - **[Search](#features)** - Deep search into your filesystem with a keybind, including offline locations.
- **[Tags](#features)** - Define routines on custom tags to automate workflows, easily tag files individually, in bulk and automatically via rules. - **[Tags](#features)** - Define routines on custom tags to automate workflows, easily tag files individually, in bulk and automatically via rules.
- **[Extensions](#features)** - Build tools on top of Spacedrive, extend functionality and integrate third party services. Extension directory on [spacedrive.app/extensions](#features). - **[Extensions](#features)** - Build tools on top of Spacedrive, extend functionality and integrate third party services. Extension directory on [spacedrive.app/extensions](#features).
**To be developed (Post-MVP):** **To be developed (Post-MVP):**
- **[Cloud integration](#features)** - Index & backup to Apple Photos, Google Drive, Dropbox, OneDrive & Mega + easy API for the community to add more. - **[Cloud integration](#features)** - Index & backup to Apple Photos, Google Drive, Dropbox, OneDrive & Mega + easy API for the community to add more.
- **[Encrypted vault(s)](#features)** - Effortlessly manage & encrypt sensitive files, built on top of VeraCrypt. Encrypt individual files or create flexible-size vaults. - **[Encrypted vault(s)](#features)** - Effortlessly manage & encrypt sensitive files, built on top of VeraCrypt. Encrypt individual files or create flexible-size vaults.
- **[Key manager](#features)** - View, mount, dismount and hide keys. Mounted keys automatically unlock respective areas of your filesystem. - **[Key manager](#features)** - View, mount, dismount and hide keys. Mounted keys automatically unlock respective areas of your filesystem.
@ -99,27 +104,33 @@ _Note: Links are for highlight purposes only until feature specific documentatio
- **[Self hosted](#features)** - Spacedrive can be deployed as a service, behaving as just another device powering your personal cloud. - **[Self hosted](#features)** - Spacedrive can be deployed as a service, behaving as just another device powering your personal cloud.
# Developer Installation Instructions # Developer Installation Instructions
This environment uses [Cargo](https://doc.rust-lang.org/cargo/getting-started/installation.html) and [pnpm](https://pnpm.io/installation). Ensure you have them installed before continuing. This environment uses [Cargo](https://doc.rust-lang.org/cargo/getting-started/installation.html) and [pnpm](https://pnpm.io/installation). Ensure you have them installed before continuing.
- `$ git clone https://github.com/spacedriveapp/spacedrive` - `$ git clone https://github.com/spacedriveapp/spacedrive`
- IMPORTANT: *Install [FFMPEG](https://www.ffmpeg.org/download.html) if you don't have it already* - IMPORTANT: _Install [FFMPEG](https://www.ffmpeg.org/download.html) if you don't have it already_
- `$ cd spacedrive` - `$ cd spacedrive`
- `$ pnpm i` - `$ pnpm i`
- `$ pnpm prep` - Runs all necessary codegen & builds required dependencies. - `$ pnpm prep` - Runs all necessary codegen & builds required dependencies.
To quickly run only the desktop app after `prep` you can use: To quickly run only the desktop app after `prep` you can use:
- `$ pnpm desktop dev` - `$ pnpm desktop dev`
To run the landing page To run the landing page
- `$ pnpm web dev` - runs the web app for the embed - `$ pnpm web dev` - runs the web app for the embed
- `$ pnpm landing dev` - `$ pnpm landing dev`
If you are having issues ensure you are using the following versions of Rust and Node: If you are having issues ensure you are using the following versions of Rust and Node:
- Rust version: **1.58.1** - Rust version: **1.58.1**
- Node version: **17** - Node version: **17**
# Architecture # Architecture
This project is using what I'm calling the **"PRRTT"** stack (Prisma, Rust, React, TypeScript, Tauri). This project is using what I'm calling the **"PRRTT"** stack (Prisma, Rust, React, TypeScript, Tauri).
- Prisma on the front-end? 🤯 Made possible thanks to [prisma-client-rust](https://github.com/brendonovich/prisma-client-rust), developed by [Brendonovich](https://github.com/brendonovich). Gives us access to the powerful migration CLI in development, along with the Prisma syntax for our schema. The application bundles with the Prisma query engine and codegen for a beautiful Rust API. Our lightweight migration runner is custom built for a desktop app context. - Prisma on the front-end? 🤯 Made possible thanks to [prisma-client-rust](https://github.com/brendonovich/prisma-client-rust), developed by [Brendonovich](https://github.com/brendonovich). Gives us access to the powerful migration CLI in development, along with the Prisma syntax for our schema. The application bundles with the Prisma query engine and codegen for a beautiful Rust API. Our lightweight migration runner is custom built for a desktop app context.
- Tauri allows us to create a pure Rust native OS webview, without the overhead of your average Electron app. This brings the bundle size and average memory usage down dramatically. It also contributes to a more native feel, especially on macOS due to Safari's close integration with the OS. - Tauri allows us to create a pure Rust native OS webview, without the overhead of your average Electron app. This brings the bundle size and average memory usage down dramatically. It also contributes to a more native feel, especially on macOS due to Safari's close integration with the OS.
- The core (`sdcore`) is written in pure Rust. - The core (`sdcore`) is written in pure Rust.
@ -127,17 +138,20 @@ This project is using what I'm calling the **"PRRTT"** stack (Prisma, Rust, Reac
## Monorepo structure: ## Monorepo structure:
### Apps: ### Apps:
- `desktop`: A [Tauri](https://tauri.studio) app. - `desktop`: A [Tauri](https://tauri.studio) app.
- `mobile`: A [React Native](https://reactnative.dev/) app. - `mobile`: A [React Native](https://reactnative.dev/) app.
- `web`: A [React](https://reactjs.org) webapp. - `web`: A [React](https://reactjs.org) webapp.
- `landing`: A [React](https://reactjs.org) app using Vite SSR & Vite pages. - `landing`: A [React](https://reactjs.org) app using Vite SSR & Vite pages.
### Core: ### Core:
- `core`: The [Rust](#) core, referred to internally as `sdcore`. Contains filesystem, database and networking logic. Can be deployed in a variety of host applications. - `core`: The [Rust](#) core, referred to internally as `sdcore`. Contains filesystem, database and networking logic. Can be deployed in a variety of host applications.
### Packages: ### Packages:
- `client`: A [TypeScript](#) client library to handle dataflow via RPC between UI and the Rust core. - `client`: A [TypeScript](#) client library to handle dataflow via RPC between UI and the Rust core.
- `ui`: A [React]([#](https://reactjs.org)) Shared component library. - `ui`: A [React](<[#](https://reactjs.org)>) Shared component library.
- `interface`: The complete user interface in React (used by apps `desktop`, `web` and `landing`) - `interface`: The complete user interface in React (used by apps `desktop`, `web` and `landing`)
- `config`: `eslint` configurations (includes `eslint-config-next`, `eslint-config-prettier` and all `tsconfig.json` configs used throughout the monorepo. - `config`: `eslint` configurations (includes `eslint-config-next`, `eslint-config-prettier` and all `tsconfig.json` configs used throughout the monorepo.
- `macos`: A [Swift](#) Native binary for MacOS system extensions. - `macos`: A [Swift](#) Native binary for MacOS system extensions.

View file

@ -19,8 +19,8 @@
"paths": { "paths": {
"@sd/interface": ["../../packages/interface/src/index.ts"], "@sd/interface": ["../../packages/interface/src/index.ts"],
"@sd/ui": ["../../packages/ui/src/index.ts"], "@sd/ui": ["../../packages/ui/src/index.ts"],
"@sd/client": ["../../packages/client/src/index.ts"], "@sd/client": ["../../packages/client/src/index.ts"]
} }
}, },
"include": ["src"], "include": ["src"]
} }

View file

@ -5,8 +5,14 @@
<link rel="icon" type="image/svg+xml" href="/favicon.ico" /> <link rel="icon" type="image/svg+xml" href="/favicon.ico" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Spacedrive — A file manager from the future.</title> <title>Spacedrive — A file manager from the future.</title>
<meta name="description" content="Combine your drives and clouds into one database that you can organize and explore from any device."/> <meta
<meta name="keywords" content="files,file manager,spacedrive,file explorer,vdfs,distributed filesystem,cas,content addressable storage,virtual filesystem,photos app, video organizer,video encoder,tags,tag based filesystem" /> name="description"
content="Combine your drives and clouds into one database that you can organize and explore from any device."
/>
<meta
name="keywords"
content="files,file manager,spacedrive,file explorer,vdfs,distributed filesystem,cas,content addressable storage,virtual filesystem,photos app, video organizer,video encoder,tags,tag based filesystem"
/>
<meta name="author" content="Jamie Pine" /> <meta name="author" content="Jamie Pine" />
<meta name="robots" content="index, follow" /> <meta name="robots" content="index, follow" />
</head> </head>

View file

@ -1,9 +1,8 @@
html { html {
background-color: black; background-color: black;
-ms-overflow-style: none; /* IE and Edge */ -ms-overflow-style: none; /* IE and Edge */
scrollbar-width: none; /* Firefox */ scrollbar-width: none; /* Firefox */
&::-webkit-scrollbar { &::-webkit-scrollbar {
display: none; display: none;
}
} }
}

View file

@ -18,7 +18,7 @@
"paths": { "paths": {
"@sd/interface": ["../../packages/interface/src/index.ts"], "@sd/interface": ["../../packages/interface/src/index.ts"],
"@sd/ui": ["../../packages/ui/src/index.ts"], "@sd/ui": ["../../packages/ui/src/index.ts"],
"@sd/client": ["../../packages/client/src/index.ts"], "@sd/client": ["../../packages/client/src/index.ts"]
} }
}, },
"ts-node": { "ts-node": {
@ -27,6 +27,14 @@
"module": "CommonJS" "module": "CommonJS"
} }
}, },
"include": ["src", "env.d.ts", "src/vite-env.d.ts", "src/components", "src/pages", "renderer", "server"], "include": [
"src",
"env.d.ts",
"src/vite-env.d.ts",
"src/components",
"src/pages",
"renderer",
"server"
],
"references": [{ "path": "./tsconfig.node.json" }] "references": [{ "path": "./tsconfig.node.json" }]
} }

View file

@ -1,7 +1,6 @@
{ {
"name": "mobile", "name": "mobile",
"version": "0.0.0", "version": "0.0.0",
"main": "index.js", "main": "index.js",
"license": "MIT" "license": "MIT"
} }

View file

@ -23,9 +23,9 @@ metadata:
name: spacedrive-ns-full name: spacedrive-ns-full
namespace: spacedrive namespace: spacedrive
rules: rules:
- apiGroups: ["apps"] - apiGroups: ['apps']
resources: ["deployments"] resources: ['deployments']
verbs: ["get", "patch"] verbs: ['get', 'patch']
--- ---
apiVersion: rbac.authorization.k8s.io/v1 apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding kind: RoleBinding
@ -33,9 +33,9 @@ metadata:
name: spacedrive-ci-rb name: spacedrive-ci-rb
namespace: spacedrive namespace: spacedrive
subjects: subjects:
- kind: ServiceAccount - kind: ServiceAccount
name: spacedrive-ci name: spacedrive-ci
namespace: spacedrive namespace: spacedrive
roleRef: roleRef:
apiGroup: rbac.authorization.k8s.io apiGroup: rbac.authorization.k8s.io
kind: Role kind: Role

View file

@ -13,16 +13,16 @@ metadata:
traefik.ingress.kubernetes.io/router.middlewares: kube-system-antiseo@kubernetescrd traefik.ingress.kubernetes.io/router.middlewares: kube-system-antiseo@kubernetescrd
spec: spec:
rules: rules:
- host: spacedrive.otbeaumont.me - host: spacedrive.otbeaumont.me
http: http:
paths: paths:
- path: / - path: /
pathType: Prefix pathType: Prefix
backend: backend:
service: service:
name: sdserver-service name: sdserver-service
port: port:
number: 8080 number: 8080
--- ---
apiVersion: v1 apiVersion: v1
kind: Service kind: Service

View file

@ -4,9 +4,7 @@ import SpacedriveInterface from '@sd/interface';
import { ClientCommand, ClientQuery, CoreEvent } from '@sd/core'; import { ClientCommand, ClientQuery, CoreEvent } from '@sd/core';
import { BaseTransport } from '@sd/client'; import { BaseTransport } from '@sd/client';
const websocket = new WebSocket( const websocket = new WebSocket(import.meta.env.VITE_SDSERVER_BASE_URL || 'ws://localhost:8080/ws');
import.meta.env.VITE_SDSERVER_BASE_URL || 'ws://localhost:8080/ws'
);
const randomId = () => Math.random().toString(36).slice(2); const randomId = () => Math.random().toString(36).slice(2);

View file

@ -18,8 +18,8 @@
"paths": { "paths": {
"@sd/interface": ["../../packages/interface/src/index.ts"], "@sd/interface": ["../../packages/interface/src/index.ts"],
"@sd/ui": ["../../packages/ui/src/index.ts"], "@sd/ui": ["../../packages/ui/src/index.ts"],
"@sd/client": ["../../packages/client/src/index.ts"], "@sd/client": ["../../packages/client/src/index.ts"]
} }
}, },
"include": ["src"], "include": ["src"]
} }

View file

@ -28,33 +28,29 @@ mod sync {
} }
``` ```
## Data Types ## Data Types
Data is divided into several kinds, Shared and Owned. Data is divided into several kinds, Shared and Owned.
- **Shared data** - Can be created and modified by any client. Has a `uuid`. - **Shared data** - Can be created and modified by any client. Has a `uuid`.
*Sync Method:* `Property operation*` _Sync Method:_ `Property operation*`
> Shared resources could be,`files`, `tags`, `comments`, `albums` and `labels`. Since these can be created, updated or deleted by any client at any time. > Shared resources could be,`files`, `tags`, `comments`, `albums` and `labels`. Since these can be created, updated or deleted by any client at any time.
- **Owned data** - Can only be modified by the client that created it. Has a `client_id` and `uuid`. - **Owned data** - Can only be modified by the client that created it. Has a `client_id` and `uuid`.
*Sync Method:* `Replicate` _Sync Method:_ `Replicate`
> Owned resources would be `file_paths`, `jobs`, `locations` and `media_data`, since a client is the single source of truth for this data. This means we can perform conflict free synchronization. > Owned resources would be `file_paths`, `jobs`, `locations` and `media_data`, since a client is the single source of truth for this data. This means we can perform conflict free synchronization.
\*_Shared data doesn't always use this method, in some cases we can create shared resources in bulk, where conflicts are handled by simply merging. More on that in [Synchronization Strategy]()_.
**Shared data doesn't always use this method, in some cases we can create shared resources in bulk, where conflicts are handled by simply merging. More on that in [Synchronization Strategy]()*.
## Client Pool ## Client Pool
The client pool maintains record of all clients in your network. The client pool maintains record of all clients in your network.
An exact replica of the client pool is synchronized on each client. When a given client has a state change, it will notify every other client in the pool via the `connection` struct. An exact replica of the client pool is synchronized on each client. When a given client has a state change, it will notify every other client in the pool via the `connection` struct.
The `ClientConnection` is maintained in memory and is established on startup. The `ClientConnection` is maintained in memory and is established on startup.
@ -70,16 +66,13 @@ struct Client {
connection: Option<ClientConnection> connection: Option<ClientConnection>
} }
``` ```
Clients will ping-pong to ensure their connection stays alive, this logic is contained within the `ClientConnection` instance. Clients will ping-pong to ensure their connection stays alive, this logic is contained within the `ClientConnection` instance.
**Handling stale clients** **Handling stale clients**
If a client has not been seen in X amount of time, other clients will not persist pending operations for them. Clients take care of flushing the pending operation queue once all non-stale clients have received the pending operations. If a client has not been seen in X amount of time, other clients will not persist pending operations for them. Clients take care of flushing the pending operation queue once all non-stale clients have received the pending operations.
## Clock ## Clock
With realtime synchronization it is important to maintain the true order of events, we can timestamp each operation, but have to account for time drift; there is no way to guarantee two machines have synchronized system clocks. With realtime synchronization it is important to maintain the true order of events, we can timestamp each operation, but have to account for time drift; there is no way to guarantee two machines have synchronized system clocks.
@ -92,20 +85,15 @@ We can solve this with a Unique Hybrid Logical Clock ([UHLC]()): a globally-uniq
Each client combines their hybrid time with a unique identifier. When receiving new [Sync Events](), a client will update its own clock with the incoming timestamp. Each client combines their hybrid time with a unique identifier. When receiving new [Sync Events](), a client will update its own clock with the incoming timestamp.
A client will reject operations with a timestamp drift greater than 100ms (can be adjusted). A client will reject operations with a timestamp drift greater than 100ms (can be adjusted).
This allows us to entirely avoid the need to synchronize time between clients, as each client controls its own order of operations, never producing a conflicting timestamp with another system in the network. This allows us to entirely avoid the need to synchronize time between clients, as each client controls its own order of operations, never producing a conflicting timestamp with another system in the network.
## Synchronization Strategy ## Synchronization Strategy
Sync happens in the following order: Sync happens in the following order:
Owned data → Bulk shared data → Shared data → Relational data Owned data → Bulk shared data → Shared data → Relational data
### Types of CRDT: ### Types of CRDT:
@ -120,13 +108,8 @@ trait Replicate;
- ~~**Last Write Win** - The most recent event will always be applied, used for many-to-many datasets.~~ - ~~**Last Write Win** - The most recent event will always be applied, used for many-to-many datasets.~~
## Operations ## Operations
Operations perform a Shared data change, they are cached in the database as `pending_operations`. Operations perform a Shared data change, they are cached in the database as `pending_operations`.
Operations are removed once all online clients have received the payload. Operations are removed once all online clients have received the payload.
@ -152,8 +135,6 @@ enum OperationMethod {
``` ```
## Pending operations ## Pending operations
Here are some examples of how operations are stored to minimize disk usage and data duplication. Here are some examples of how operations are stored to minimize disk usage and data duplication.
@ -162,29 +143,25 @@ Here are some examples of how operations are stored to minimize disk usage and d
In the next case we're handling the creation of a Shared resource. The `method` is marked `Create` and the value is `NULL`. This is because we can also use the actual database record in the `tags` table as it was newly created. In the next case we're handling the creation of a Shared resource. The `method` is marked `Create` and the value is `NULL`. This is because we can also use the actual database record in the `tags` table as it was newly created.
| `client_uuid` | `uhlc_timestamp` | `method` | `resource_key` | `resource_uuid` | `resource_property` | `value` | | `client_uuid` | `uhlc_timestamp` | `method` | `resource_key` | `resource_uuid` | `resource_property` | `value` |
|----------|-------------|------|----------|----------|----------|----------| | ------------- | ---------------------- | -------- | -------------- | --------------- | ------------------- | ------- |
| 2e8f85bf... | 2022-04-09T06:53:36... | Create | tags | 2e8f85bf... | NULL | NULL | | 2e8f85bf... | 2022-04-09T06:53:36... | Create | tags | 2e8f85bf... | NULL | NULL |
**Update operation for Shared data** **Update operation for Shared data**
Shared data works at a property level Shared data works at a property level
| `client_uuid` | `uhlc_timestamp` | `method` | `resource_key` | `resource_uuid` | `resource_property` | `value` | | `client_uuid` | `uhlc_timestamp` | `method` | `resource_key` | `resource_uuid` | `resource_property` | `value` |
|----------|-------------|------|----------|----------|----------|----------| | ------------- | ---------------------- | -------- | -------------- | --------------- | ------------------- | ------- |
| 2e8f85bf... | 2022-04-09T06:53:36... | Update | albums | 2e8f85bf... | name | "jeff" | | 2e8f85bf... | 2022-04-09T06:53:36... | Update | albums | 2e8f85bf... | name | "jeff" |
## Owned Data Synchronization ## Owned Data Synchronization
Owned data does not use the Operation system, it is queried dynamically by the `updated_at` column on Owned datasets. Owned data does not use the Operation system, it is queried dynamically by the `updated_at` column on Owned datasets.
For the sake of compatibility with local relations, some resource properties can be ignored*, such as `file_id` and `parent_id` on the `file_paths` resource, these are re-calculated on bulk ingest. For the sake of compatibility with local relations, some resource properties can be ignored\*, such as `file_id` and `parent_id` on the `file_paths` resource, these are re-calculated on bulk ingest.
*_This will require some form of definition when creating an owned data resource_.
\*_This will require some form of definition when creating an owned data resource_.
## Bulk Shared Data Synchronization ## Bulk Shared Data Synchronization
@ -196,10 +173,6 @@ As `files` are created in abundance (hundreds of thousands at a time), it would
We handle this by using `SyncMethod::Merge`, simply merging the data where the oldest resource properties are prioritized. We handle this by using `SyncMethod::Merge`, simply merging the data where the oldest resource properties are prioritized.
## Combining CRDTs ## Combining CRDTs
Combining CRDT types allow for some tailored functionality for particular resources. Combining CRDT types allow for some tailored functionality for particular resources.
@ -220,12 +193,6 @@ impl Replicate for Job {
} }
``` ```
## Creating Sync Events ## Creating Sync Events
We have a simple Rust syntax for creating sync events in the core. We have a simple Rust syntax for creating sync events in the core.
@ -261,34 +228,6 @@ Then inside the `sync` function we send the event to the
Files also impempent `OperationalMerge` would use Files also impempent `OperationalMerge` would use
# Resources # Resources
- https://archive.jlongster.com/using-crdts-in-the-wild - https://archive.jlongster.com/using-crdts-in-the-wild

View file

@ -2,8 +2,6 @@
Spacedrive maintains a virtual filesystem comprised of storage locations through various clients. It records important metadata about a given file as well as a unique checksum for content based addressing [CAS](). Spacedrive maintains a virtual filesystem comprised of storage locations through various clients. It records important metadata about a given file as well as a unique checksum for content based addressing [CAS]().
### File — `Shared data` ### File — `Shared data`
Represents a unique file across the virtual filesystem, all Spacedrive metadata is tied to this resource through local data relations. Represents a unique file across the virtual filesystem, all Spacedrive metadata is tied to this resource through local data relations.
@ -36,7 +34,7 @@ struct File {
} }
``` ```
- `partial_checksum ` - A SHA256 checksum generated from 5 samples of 10,000 bytes throughout the file data, including the begining and end + total byte count. This is used to identify a file as *likely* unique in under 100µs. - `partial_checksum ` - A SHA256 checksum generated from 5 samples of 10,000 bytes throughout the file data, including the begining and end + total byte count. This is used to identify a file as _likely_ unique in under 100µs.
> ~~It is impossible to have a unique constraint at a database level for the `partial_checksum` however we can asyncronously resolve conflicts by querying for duplicates and generating full checksums at a later date.~~ > ~~It is impossible to have a unique constraint at a database level for the `partial_checksum` however we can asyncronously resolve conflicts by querying for duplicates and generating full checksums at a later date.~~
> >
@ -44,8 +42,6 @@ struct File {
- `checksum` - A full SHA256 checksum of the file data used to verify uniqueness should a `partial_checksum` conflict occur. - `checksum` - A full SHA256 checksum of the file data used to verify uniqueness should a `partial_checksum` conflict occur.
### FilePath — `Owned data` ### FilePath — `Owned data`
This represents a logical file path within a [Location](), used to derive `file` records. This represents a logical file path within a [Location](), used to derive `file` records.
@ -70,8 +66,6 @@ struct FilePath {
} }
``` ```
```typescript ```typescript
export function useBridgeCommand< export function useBridgeCommand<
K extends CommandKeyType, K extends CommandKeyType,
@ -85,6 +79,3 @@ export function useBridgeCommand<
); );
} }
``` ```

View file

@ -1,5 +1,3 @@
# Todo # Todo
- Landing sections - Landing sections

View file

@ -4,22 +4,16 @@ hello@spacedrive.app`
**Made with ♥️ in Canada** **Made with ♥️ in Canada**
*&copy; Copyright 2022-Present Jamie Pine* _&copy; Copyright 2022-Present Jamie Pine_
**Developers:** **Developers:**
Jamie Pine, Brendonovich, Oscar Beaumont, Haden Fletcher Jamie Pine, Brendonovich, Oscar Beaumont, Haden Fletcher
**Founding Users:** **Founding Users:**
People, Who, Buy, Founder Pack People, Who, Buy, Founder Pack
**Sponsors:** **Sponsors:**
Jamie Pine Jamie Pine

View file

@ -1,4 +1,4 @@
## What is it? ## What is it?
Spacedrive is a cross platform app that allows you to manage files across all devices, drives and clouds at once. Check out the [readme](https://github.com/spacedriveapp) for more detailed info. Spacedrive is a cross platform app that allows you to manage files across all devices, drives and clouds at once. Check out the [readme](https://github.com/spacedriveapp) for more detailed info.
@ -51,5 +51,3 @@ We call it "PRRTT"; [Prisma](), [Rust](), [React](), [TypeScript]() and [Tauri](
## Why Rust? ## Why Rust?
Rust is a powerful, fast, memory safe, cross-platform, low-level language and thanks to Tauri, the perfect tool to power a desktop application. Rust is a powerful, fast, memory safe, cross-platform, low-level language and thanks to Tauri, the perfect tool to power a desktop application.

View file

@ -5,4 +5,3 @@
### The file explorer of the future. ### The file explorer of the future.
### All your files in one space. ### All your files in one space.

View file

@ -32,6 +32,7 @@ core.register_context(IndexerContext {
``` ```
For Apple Photos we need: For Apple Photos we need:
- Hidden/Favorite items - Hidden/Favorite items
- Live photo support - Live photo support
- Original creation date - Original creation date

View file

@ -1 +1 @@
module.exports = require("scripts/eslint-preset"); module.exports = require('scripts/eslint-preset');

View file

@ -1,11 +1,11 @@
module.exports = { module.exports = {
extends: ["next", "prettier"], extends: ['next', 'prettier'],
settings: { settings: {
next: { next: {
rootDir: ["apps/*/", "packages/*/"], rootDir: ['apps/*/', 'packages/*/']
}, }
}, },
rules: { rules: {
"no-html-link-for-pages": "off", 'no-html-link-for-pages': 'off'
}, }
}; };

View file

@ -171,7 +171,6 @@ import { ReactComponent as Yarn } from './yarn.svg';
import { ReactComponent as Yarnerror } from './yarnerror.svg'; import { ReactComponent as Yarnerror } from './yarnerror.svg';
import { ReactComponent as Zip } from './zip.svg'; import { ReactComponent as Zip } from './zip.svg';
export default { export default {
ai: Ai, ai: Ai,
angular: Angular, angular: Angular,
@ -344,5 +343,5 @@ export default {
yaml: Yaml, yaml: Yaml,
yarn: Yarn, yarn: Yarn,
yarnerror: Yarnerror, yarnerror: Yarnerror,
zip: Zip, zip: Zip
} };

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View file

@ -3,11 +3,16 @@ import React from 'react';
import { DefaultProps } from '../primitive/types'; import { DefaultProps } from '../primitive/types';
export interface DriveListItemProps extends DefaultProps { export interface DriveListItemProps extends DefaultProps {
name: string; name: string;
} }
export const DriveListItem: React.FC<DriveListItemProps> = (props) => { export const DriveListItem: React.FC<DriveListItemProps> = (props) => {
return <div className={clsx('rounded px-1.5 py-1 text-xs font-medium inline-block cursor-default', props.className)}> return (
<div
</div>; className={clsx(
'rounded px-1.5 py-1 text-xs font-medium inline-block cursor-default',
props.className
)}
></div>
);
}; };

View file

@ -22,5 +22,5 @@
"@sd/client": ["../client/src/index.ts"] "@sd/client": ["../client/src/index.ts"]
} }
}, },
"include": ["src", "scripts"], "include": ["src", "scripts"]
} }

View file

@ -2,8 +2,6 @@
@tailwind components; @tailwind components;
@tailwind utilities; @tailwind utilities;
.html { .html {
background: #404040; background: #404040;
} }

View file

@ -2,6 +2,6 @@
"extends": "../config/tsconfig/react-library.json", "extends": "../config/tsconfig/react-library.json",
"include": ["src"], "include": ["src"],
"compilerOptions": { "compilerOptions": {
"outDir": "dist", "outDir": "dist"
} }
} }

File diff suppressed because it is too large Load diff