mirror of
https://github.com/spacedriveapp/spacedrive
synced 2024-07-02 08:53:32 +00:00
Linted Files according to the .prettierrc.
This commit is contained in:
Harry Hopkinson
parent
980109408d
commit
d7cbba7a12
196
.github/workflows/ci.yml
vendored
196
.github/workflows/ci.yml
vendored
|
|
@ -3,8 +3,8 @@ on:
|
|||
pull_request:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- ci
|
||||
- main
|
||||
- ci
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
|
|
@ -18,115 +18,115 @@ jobs:
|
|||
RUST_CACHE_VERSION: 0
|
||||
runs-on: ${{ matrix.platform }}
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
# from https://github.com/zmwangx/rust-ffmpeg/blob/master/.github/workflows/build.yml
|
||||
- name: Install ffmpeg (Windows)
|
||||
if: matrix.platform == 'windows-latest'
|
||||
run: |
|
||||
$VCINSTALLDIR = $(& "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe" -latest -property installationPath)
|
||||
Add-Content $env:GITHUB_ENV "LIBCLANG_PATH=${VCINSTALLDIR}\VC\Tools\LLVM\x64\bin`n"
|
||||
Invoke-WebRequest "${env:FFMPEG_DOWNLOAD_URL}" -OutFile ffmpeg-release-full-shared.7z
|
||||
7z x ffmpeg-release-full-shared.7z
|
||||
mkdir ffmpeg
|
||||
mv ffmpeg-*/* ffmpeg/
|
||||
Add-Content $env:GITHUB_ENV "FFMPEG_DIR=${pwd}\ffmpeg`n"
|
||||
Add-Content $env:GITHUB_PATH "${pwd}\ffmpeg\bin`n"
|
||||
# from https://github.com/zmwangx/rust-ffmpeg/blob/master/.github/workflows/build.yml
|
||||
- name: Install ffmpeg (Windows)
|
||||
if: matrix.platform == 'windows-latest'
|
||||
run: |
|
||||
$VCINSTALLDIR = $(& "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe" -latest -property installationPath)
|
||||
Add-Content $env:GITHUB_ENV "LIBCLANG_PATH=${VCINSTALLDIR}\VC\Tools\LLVM\x64\bin`n"
|
||||
Invoke-WebRequest "${env:FFMPEG_DOWNLOAD_URL}" -OutFile ffmpeg-release-full-shared.7z
|
||||
7z x ffmpeg-release-full-shared.7z
|
||||
mkdir ffmpeg
|
||||
mv ffmpeg-*/* ffmpeg/
|
||||
Add-Content $env:GITHUB_ENV "FFMPEG_DIR=${pwd}\ffmpeg`n"
|
||||
Add-Content $env:GITHUB_PATH "${pwd}\ffmpeg\bin`n"
|
||||
|
||||
- name: Install CMake (Windows)
|
||||
uses: lukka/get-cmake@latest
|
||||
if: matrix.platform == 'windows-latest'
|
||||
- name: Install CMake (Windows)
|
||||
uses: lukka/get-cmake@latest
|
||||
if: matrix.platform == 'windows-latest'
|
||||
|
||||
# Optimisation for windows
|
||||
- name: Rename existing rust toolchain (Windows)
|
||||
if: matrix.platform == 'windows-latest'
|
||||
run: Rename-Item C:\Users\runneradmin\.rustup\toolchains\stable-x86_64-pc-windows-msvc C:\Users\runneradmin\.rustup\toolchains\stable-x86_64-pc-windows-msvc.old
|
||||
# Optimisation for windows
|
||||
- name: Rename existing rust toolchain (Windows)
|
||||
if: matrix.platform == 'windows-latest'
|
||||
run: Rename-Item C:\Users\runneradmin\.rustup\toolchains\stable-x86_64-pc-windows-msvc C:\Users\runneradmin\.rustup\toolchains\stable-x86_64-pc-windows-msvc.old
|
||||
|
||||
- name: Setup Node
|
||||
uses: actions/setup-node@v1
|
||||
with:
|
||||
node-version: 16
|
||||
- name: Setup Node
|
||||
uses: actions/setup-node@v1
|
||||
with:
|
||||
node-version: 16
|
||||
|
||||
- name: Install Rust stable
|
||||
uses: actions-rs/toolchain@v1
|
||||
with:
|
||||
toolchain: stable
|
||||
profile: minimal
|
||||
override: true
|
||||
components: rustfmt, rust-src
|
||||
- name: Install Rust stable
|
||||
uses: actions-rs/toolchain@v1
|
||||
with:
|
||||
toolchain: stable
|
||||
profile: minimal
|
||||
override: true
|
||||
components: rustfmt, rust-src
|
||||
|
||||
- name: Cache Rust Dependencies
|
||||
uses: Swatinem/rust-cache@cb2cf0cc7c5198d3364b9630e2c3d457f160790c
|
||||
with:
|
||||
sharedKey: ${{ env.RUST_CACHE_VERSION }}
|
||||
- name: Cache Rust Dependencies
|
||||
uses: Swatinem/rust-cache@cb2cf0cc7c5198d3364b9630e2c3d457f160790c
|
||||
with:
|
||||
sharedKey: ${{ env.RUST_CACHE_VERSION }}
|
||||
|
||||
- name: Cache pnpm dependencies
|
||||
uses: actions/cache@v2
|
||||
with:
|
||||
path: ~/.pnpm-store
|
||||
key: ${{ runner.os }}-${{ hashFiles('**/pnpm-lock.yaml') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-
|
||||
- name: Cache pnpm dependencies
|
||||
uses: actions/cache@v2
|
||||
with:
|
||||
path: ~/.pnpm-store
|
||||
key: ${{ runner.os }}-${{ hashFiles('**/pnpm-lock.yaml') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-
|
||||
|
||||
- name: Install pnpm
|
||||
uses: pnpm/action-setup@v2.2.1
|
||||
with:
|
||||
version: 6.32.6
|
||||
- name: Install pnpm
|
||||
uses: pnpm/action-setup@v2.2.1
|
||||
with:
|
||||
version: 6.32.6
|
||||
|
||||
- name: Install dependencies (Ubuntu)
|
||||
if: matrix.platform == 'ubuntu-latest'
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y --no-install-recommends \
|
||||
libgtk-3-dev \
|
||||
webkit2gtk-4.0 \
|
||||
libappindicator3-dev \
|
||||
librsvg2-dev \
|
||||
patchelf \
|
||||
libssl-dev \
|
||||
libavcodec-dev \
|
||||
libavdevice-dev \
|
||||
libavfilter-dev \
|
||||
libavformat-dev \
|
||||
libavresample-dev \
|
||||
libavutil-dev \
|
||||
libswscale-dev \
|
||||
libswresample-dev \
|
||||
pkg-config \
|
||||
ffmpeg
|
||||
- name: Install dependencies (Ubuntu)
|
||||
if: matrix.platform == 'ubuntu-latest'
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y --no-install-recommends \
|
||||
libgtk-3-dev \
|
||||
webkit2gtk-4.0 \
|
||||
libappindicator3-dev \
|
||||
librsvg2-dev \
|
||||
patchelf \
|
||||
libssl-dev \
|
||||
libavcodec-dev \
|
||||
libavdevice-dev \
|
||||
libavfilter-dev \
|
||||
libavformat-dev \
|
||||
libavresample-dev \
|
||||
libavutil-dev \
|
||||
libswscale-dev \
|
||||
libswresample-dev \
|
||||
pkg-config \
|
||||
ffmpeg
|
||||
|
||||
- name: Install dependencies (macOS)
|
||||
if: matrix.platform == 'macos-latest'
|
||||
run: |
|
||||
brew install ffmpeg
|
||||
- name: Install dependencies (macOS)
|
||||
if: matrix.platform == 'macos-latest'
|
||||
run: |
|
||||
brew install ffmpeg
|
||||
|
||||
- name: Install pnpm dependencies
|
||||
run: pnpm i
|
||||
- name: Install pnpm dependencies
|
||||
run: pnpm i
|
||||
|
||||
- name: Build codegen
|
||||
run: pnpm prep:ci
|
||||
- name: Build codegen
|
||||
run: pnpm prep:ci
|
||||
|
||||
- name: Build frontend
|
||||
run: pnpm desktop build:vite
|
||||
- name: Build frontend
|
||||
run: pnpm desktop build:vite
|
||||
|
||||
- name: Build Tauri app
|
||||
uses: tauri-apps/tauri-action@v0
|
||||
with:
|
||||
projectPath: apps/desktop
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
- name: Build Tauri app
|
||||
uses: tauri-apps/tauri-action@v0
|
||||
with:
|
||||
projectPath: apps/desktop
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Build and publish server
|
||||
if: matrix.platform == 'ubuntu-latest'
|
||||
uses: ./.github/actions/build-and-publish-server
|
||||
with:
|
||||
gh_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
- name: Build and publish server
|
||||
if: matrix.platform == 'ubuntu-latest'
|
||||
uses: ./.github/actions/build-and-publish-server
|
||||
with:
|
||||
gh_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Deploy Spacedrive Server to Kubernetes
|
||||
if: matrix.platform == 'ubuntu-latest'
|
||||
env:
|
||||
K8S_KUBECONFIG: ${{ secrets.K8S_KUBECONFIG }}
|
||||
run: |
|
||||
mkdir -p ~/.kube
|
||||
echo "$K8S_KUBECONFIG" > ~/.kube/config 2>&1
|
||||
kubectl rollout restart deployment/sdserver-deployment
|
||||
- name: Deploy Spacedrive Server to Kubernetes
|
||||
if: matrix.platform == 'ubuntu-latest'
|
||||
env:
|
||||
K8S_KUBECONFIG: ${{ secrets.K8S_KUBECONFIG }}
|
||||
run: |
|
||||
mkdir -p ~/.kube
|
||||
echo "$K8S_KUBECONFIG" > ~/.kube/config 2>&1
|
||||
kubectl rollout restart deployment/sdserver-deployment
|
||||
|
|
|
|||
4
.github/workflows/org-readme.yml
vendored
4
.github/workflows/org-readme.yml
vendored
|
|
@ -3,9 +3,9 @@ name: Update Org README
|
|||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- main
|
||||
paths:
|
||||
- README.md
|
||||
- README.md
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
|
|
|
|||
24
README.md
24
README.md
|
|
@ -58,36 +58,41 @@ For independent creatives, hoarders and those that want to own their digital foo
|
|||
</p>
|
||||
|
||||
> NOTE: Spacedrive is under active development, most of the listed features are still experimental and subject to change. Additionally, most of the links on this page are broken but will be working once the repository is made public.
|
||||
>
|
||||
|
||||
# What is a VDFS?
|
||||
|
||||
A VDFS (virtual distributed filesystem) is a filesystem designed to work atop a variety of storage layers. It is not restricted to a single machine, with a uniform API to manipulate and access content across many devices. It achieves this by maintaining a virtual index of all storage locations, synchronizing the database between clients in realtime. This implementation also uses [CAS](https://en.wikipedia.org/wiki/Content-addressable_storage) (Content-addressable storage) to uniquely identify files, while keeping record of logical file paths relative to the storage locations.
|
||||
|
||||
The first implementation of a VDFS can be found in this UC Berkeley [paper](https://www2.eecs.berkeley.edu/Pubs/TechRpts/2018/EECS-2018-29.pdf) by Haoyuan Li. This paper describes its use for cloud computing, however the underlying concepts can be translated to open consumer software.
|
||||
|
||||
# Motivation
|
||||
|
||||
Many of us have multiple cloud accounts, drives that aren’t backed up and data at risk of loss. We depend on cloud services like Google Photos and iCloud, but are locked in with limited capacity and almost zero interoperability between services and operating systems. Photo albums shouldn’t be suck in a device ecosystem, or harvested for advertising data. They should be OS agnostic, permanent and personally owned. Data we create is our legacy, that will long outlive us—open source technology is the only way to ensure we retain absolute control over the data that defines our lives, at unlimited scale.
|
||||
|
||||
|
||||
# Features
|
||||
|
||||
_Note: Links are for highlight purposes only until feature specific documentation is complete._
|
||||
|
||||
**Complete:** *(in testing)*
|
||||
**Complete:** _(in testing)_
|
||||
|
||||
- **[File discovery](#features)** - Scan devices, drives and cloud accounts to build a directory of all files with metadata.
|
||||
- **[Preview generation](#features)** - Auto generate lower resolution stand-ins for image and video.
|
||||
- **[Statistics](#features)** - Total capacity, index size, preview media size, free space etc.
|
||||
|
||||
**In progress:**
|
||||
|
||||
- **[File Explorer](#features)** - Browse online/offline storage locations, view files with metadata, perform basic CRUD.
|
||||
- **[Realtime synchronization](#features)** - Data index synchronized in realtime between devices, prioritizing peer-to-peer LAN connections (WiFi sync).
|
||||
|
||||
**To be developed (MVP):**
|
||||
|
||||
- **[Photos](#features)** - Photo and video albums similar to Apple/Google photos.
|
||||
- **[Search](#features)** - Deep search into your filesystem with a keybind, including offline locations.
|
||||
- **[Tags](#features)** - Define routines on custom tags to automate workflows, easily tag files individually, in bulk and automatically via rules.
|
||||
- **[Extensions](#features)** - Build tools on top of Spacedrive, extend functionality and integrate third party services. Extension directory on [spacedrive.app/extensions](#features).
|
||||
|
||||
**To be developed (Post-MVP):**
|
||||
|
||||
- **[Cloud integration](#features)** - Index & backup to Apple Photos, Google Drive, Dropbox, OneDrive & Mega + easy API for the community to add more.
|
||||
- **[Encrypted vault(s)](#features)** - Effortlessly manage & encrypt sensitive files, built on top of VeraCrypt. Encrypt individual files or create flexible-size vaults.
|
||||
- **[Key manager](#features)** - View, mount, dismount and hide keys. Mounted keys automatically unlock respective areas of your filesystem.
|
||||
|
|
@ -99,27 +104,33 @@ _Note: Links are for highlight purposes only until feature specific documentatio
|
|||
- **[Self hosted](#features)** - Spacedrive can be deployed as a service, behaving as just another device powering your personal cloud.
|
||||
|
||||
# Developer Installation Instructions
|
||||
|
||||
This environment uses [Cargo](https://doc.rust-lang.org/cargo/getting-started/installation.html) and [pnpm](https://pnpm.io/installation). Ensure you have them installed before continuing.
|
||||
|
||||
- `$ git clone https://github.com/spacedriveapp/spacedrive`
|
||||
- IMPORTANT: *Install [FFMPEG](https://www.ffmpeg.org/download.html) if you don't have it already*
|
||||
- IMPORTANT: _Install [FFMPEG](https://www.ffmpeg.org/download.html) if you don't have it already_
|
||||
- `$ cd spacedrive`
|
||||
- `$ pnpm i`
|
||||
- `$ pnpm prep` - Runs all necessary codegen & builds required dependencies.
|
||||
|
||||
To quickly run only the desktop app after `prep` you can use:
|
||||
|
||||
- `$ pnpm desktop dev`
|
||||
|
||||
To run the landing page
|
||||
|
||||
- `$ pnpm web dev` - runs the web app for the embed
|
||||
- `$ pnpm landing dev`
|
||||
|
||||
If you are having issues ensure you are using the following versions of Rust and Node:
|
||||
|
||||
- Rust version: **1.58.1**
|
||||
- Node version: **17**
|
||||
|
||||
# Architecture
|
||||
|
||||
This project is using what I'm calling the **"PRRTT"** stack (Prisma, Rust, React, TypeScript, Tauri).
|
||||
|
||||
- Prisma on the front-end? 🤯 Made possible thanks to [prisma-client-rust](https://github.com/brendonovich/prisma-client-rust), developed by [Brendonovich](https://github.com/brendonovich). Gives us access to the powerful migration CLI in development, along with the Prisma syntax for our schema. The application bundles with the Prisma query engine and codegen for a beautiful Rust API. Our lightweight migration runner is custom built for a desktop app context.
|
||||
- Tauri allows us to create a pure Rust native OS webview, without the overhead of your average Electron app. This brings the bundle size and average memory usage down dramatically. It also contributes to a more native feel, especially on macOS due to Safari's close integration with the OS.
|
||||
- The core (`sdcore`) is written in pure Rust.
|
||||
|
|
@ -127,17 +138,20 @@ This project is using what I'm calling the **"PRRTT"** stack (Prisma, Rust, Reac
|
|||
## Monorepo structure:
|
||||
|
||||
### Apps:
|
||||
|
||||
- `desktop`: A [Tauri](https://tauri.studio) app.
|
||||
- `mobile`: A [React Native](https://reactnative.dev/) app.
|
||||
- `web`: A [React](https://reactjs.org) webapp.
|
||||
- `landing`: A [React](https://reactjs.org) app using Vite SSR & Vite pages.
|
||||
|
||||
### Core:
|
||||
|
||||
- `core`: The [Rust](#) core, referred to internally as `sdcore`. Contains filesystem, database and networking logic. Can be deployed in a variety of host applications.
|
||||
|
||||
### Packages:
|
||||
|
||||
- `client`: A [TypeScript](#) client library to handle dataflow via RPC between UI and the Rust core.
|
||||
- `ui`: A [React]([#](https://reactjs.org)) Shared component library.
|
||||
- `ui`: A [React](<[#](https://reactjs.org)>) Shared component library.
|
||||
- `interface`: The complete user interface in React (used by apps `desktop`, `web` and `landing`)
|
||||
- `config`: `eslint` configurations (includes `eslint-config-next`, `eslint-config-prettier` and all `tsconfig.json` configs used throughout the monorepo.
|
||||
- `macos`: A [Swift](#) Native binary for MacOS system extensions.
|
||||
|
|
|
|||
|
|
@ -19,8 +19,8 @@
|
|||
"paths": {
|
||||
"@sd/interface": ["../../packages/interface/src/index.ts"],
|
||||
"@sd/ui": ["../../packages/ui/src/index.ts"],
|
||||
"@sd/client": ["../../packages/client/src/index.ts"],
|
||||
"@sd/client": ["../../packages/client/src/index.ts"]
|
||||
}
|
||||
},
|
||||
"include": ["src"],
|
||||
"include": ["src"]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,8 +5,14 @@
|
|||
<link rel="icon" type="image/svg+xml" href="/favicon.ico" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Spacedrive — A file manager from the future.</title>
|
||||
<meta name="description" content="Combine your drives and clouds into one database that you can organize and explore from any device."/>
|
||||
<meta name="keywords" content="files,file manager,spacedrive,file explorer,vdfs,distributed filesystem,cas,content addressable storage,virtual filesystem,photos app, video organizer,video encoder,tags,tag based filesystem" />
|
||||
<meta
|
||||
name="description"
|
||||
content="Combine your drives and clouds into one database that you can organize and explore from any device."
|
||||
/>
|
||||
<meta
|
||||
name="keywords"
|
||||
content="files,file manager,spacedrive,file explorer,vdfs,distributed filesystem,cas,content addressable storage,virtual filesystem,photos app, video organizer,video encoder,tags,tag based filesystem"
|
||||
/>
|
||||
<meta name="author" content="Jamie Pine" />
|
||||
<meta name="robots" content="index, follow" />
|
||||
</head>
|
||||
|
|
|
|||
|
|
@ -1,9 +1,8 @@
|
|||
html {
|
||||
background-color: black;
|
||||
-ms-overflow-style: none; /* IE and Edge */
|
||||
scrollbar-width: none; /* Firefox */
|
||||
&::-webkit-scrollbar {
|
||||
display: none;
|
||||
}
|
||||
background-color: black;
|
||||
-ms-overflow-style: none; /* IE and Edge */
|
||||
scrollbar-width: none; /* Firefox */
|
||||
&::-webkit-scrollbar {
|
||||
display: none;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@
|
|||
"paths": {
|
||||
"@sd/interface": ["../../packages/interface/src/index.ts"],
|
||||
"@sd/ui": ["../../packages/ui/src/index.ts"],
|
||||
"@sd/client": ["../../packages/client/src/index.ts"],
|
||||
"@sd/client": ["../../packages/client/src/index.ts"]
|
||||
}
|
||||
},
|
||||
"ts-node": {
|
||||
|
|
@ -27,6 +27,14 @@
|
|||
"module": "CommonJS"
|
||||
}
|
||||
},
|
||||
"include": ["src", "env.d.ts", "src/vite-env.d.ts", "src/components", "src/pages", "renderer", "server"],
|
||||
"include": [
|
||||
"src",
|
||||
"env.d.ts",
|
||||
"src/vite-env.d.ts",
|
||||
"src/components",
|
||||
"src/pages",
|
||||
"renderer",
|
||||
"server"
|
||||
],
|
||||
"references": [{ "path": "./tsconfig.node.json" }]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
{
|
||||
"name": "mobile",
|
||||
"version": "0.0.0",
|
||||
"main": "index.js",
|
||||
"license": "MIT"
|
||||
}
|
||||
|
||||
"name": "mobile",
|
||||
"version": "0.0.0",
|
||||
"main": "index.js",
|
||||
"license": "MIT"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -23,9 +23,9 @@ metadata:
|
|||
name: spacedrive-ns-full
|
||||
namespace: spacedrive
|
||||
rules:
|
||||
- apiGroups: ["apps"]
|
||||
resources: ["deployments"]
|
||||
verbs: ["get", "patch"]
|
||||
- apiGroups: ['apps']
|
||||
resources: ['deployments']
|
||||
verbs: ['get', 'patch']
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
|
|
@ -33,9 +33,9 @@ metadata:
|
|||
name: spacedrive-ci-rb
|
||||
namespace: spacedrive
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: spacedrive-ci
|
||||
namespace: spacedrive
|
||||
- kind: ServiceAccount
|
||||
name: spacedrive-ci
|
||||
namespace: spacedrive
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: Role
|
||||
|
|
|
|||
|
|
@ -13,16 +13,16 @@ metadata:
|
|||
traefik.ingress.kubernetes.io/router.middlewares: kube-system-antiseo@kubernetescrd
|
||||
spec:
|
||||
rules:
|
||||
- host: spacedrive.otbeaumont.me
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: sdserver-service
|
||||
port:
|
||||
number: 8080
|
||||
- host: spacedrive.otbeaumont.me
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: sdserver-service
|
||||
port:
|
||||
number: 8080
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
|
|
|
|||
|
|
@ -4,9 +4,7 @@ import SpacedriveInterface from '@sd/interface';
|
|||
import { ClientCommand, ClientQuery, CoreEvent } from '@sd/core';
|
||||
import { BaseTransport } from '@sd/client';
|
||||
|
||||
const websocket = new WebSocket(
|
||||
import.meta.env.VITE_SDSERVER_BASE_URL || 'ws://localhost:8080/ws'
|
||||
);
|
||||
const websocket = new WebSocket(import.meta.env.VITE_SDSERVER_BASE_URL || 'ws://localhost:8080/ws');
|
||||
|
||||
const randomId = () => Math.random().toString(36).slice(2);
|
||||
|
||||
|
|
|
|||
|
|
@ -18,8 +18,8 @@
|
|||
"paths": {
|
||||
"@sd/interface": ["../../packages/interface/src/index.ts"],
|
||||
"@sd/ui": ["../../packages/ui/src/index.ts"],
|
||||
"@sd/client": ["../../packages/client/src/index.ts"],
|
||||
"@sd/client": ["../../packages/client/src/index.ts"]
|
||||
}
|
||||
},
|
||||
"include": ["src"],
|
||||
"include": ["src"]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -28,33 +28,29 @@ mod sync {
|
|||
}
|
||||
```
|
||||
|
||||
|
||||
|
||||
## Data Types
|
||||
|
||||
Data is divided into several kinds, Shared and Owned.
|
||||
|
||||
- **Shared data** - Can be created and modified by any client. Has a `uuid`.
|
||||
|
||||
*Sync Method:* `Property operation*`
|
||||
_Sync Method:_ `Property operation*`
|
||||
|
||||
> Shared resources could be,`files`, `tags`, `comments`, `albums` and `labels`. Since these can be created, updated or deleted by any client at any time.
|
||||
|
||||
- **Owned data** - Can only be modified by the client that created it. Has a `client_id` and `uuid`.
|
||||
|
||||
*Sync Method:* `Replicate`
|
||||
_Sync Method:_ `Replicate`
|
||||
|
||||
> Owned resources would be `file_paths`, `jobs`, `locations` and `media_data`, since a client is the single source of truth for this data. This means we can perform conflict free synchronization.
|
||||
|
||||
|
||||
|
||||
**Shared data doesn't always use this method, in some cases we can create shared resources in bulk, where conflicts are handled by simply merging. More on that in [Synchronization Strategy]()*.
|
||||
|
||||
|
||||
|
||||
\*_Shared data doesn't always use this method, in some cases we can create shared resources in bulk, where conflicts are handled by simply merging. More on that in [Synchronization Strategy]()_.
|
||||
|
||||
## Client Pool
|
||||
|
||||
The client pool maintains record of all clients in your network.
|
||||
|
||||
An exact replica of the client pool is synchronized on each client. When a given client has a state change, it will notify every other client in the pool via the `connection` struct.
|
||||
An exact replica of the client pool is synchronized on each client. When a given client has a state change, it will notify every other client in the pool via the `connection` struct.
|
||||
|
||||
The `ClientConnection` is maintained in memory and is established on startup.
|
||||
|
||||
|
|
@ -70,16 +66,13 @@ struct Client {
|
|||
connection: Option<ClientConnection>
|
||||
}
|
||||
```
|
||||
|
||||
Clients will ping-pong to ensure their connection stays alive, this logic is contained within the `ClientConnection` instance.
|
||||
|
||||
**Handling stale clients**
|
||||
|
||||
If a client has not been seen in X amount of time, other clients will not persist pending operations for them. Clients take care of flushing the pending operation queue once all non-stale clients have received the pending operations.
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
## Clock
|
||||
|
||||
With realtime synchronization it is important to maintain the true order of events, we can timestamp each operation, but have to account for time drift; there is no way to guarantee two machines have synchronized system clocks.
|
||||
|
|
@ -92,20 +85,15 @@ We can solve this with a Unique Hybrid Logical Clock ([UHLC]()): a globally-uniq
|
|||
|
||||
Each client combines their hybrid time with a unique identifier. When receiving new [Sync Events](), a client will update its own clock with the incoming timestamp.
|
||||
|
||||
|
||||
A client will reject operations with a timestamp drift greater than 100ms (can be adjusted).
|
||||
|
||||
This allows us to entirely avoid the need to synchronize time between clients, as each client controls its own order of operations, never producing a conflicting timestamp with another system in the network.
|
||||
|
||||
|
||||
|
||||
## Synchronization Strategy
|
||||
|
||||
Sync happens in the following order:
|
||||
|
||||
Owned data → Bulk shared data → Shared data → Relational data
|
||||
|
||||
|
||||
Owned data → Bulk shared data → Shared data → Relational data
|
||||
|
||||
### Types of CRDT:
|
||||
|
||||
|
|
@ -120,13 +108,8 @@ trait Replicate;
|
|||
|
||||
- ~~**Last Write Win** - The most recent event will always be applied, used for many-to-many datasets.~~
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
## Operations
|
||||
|
||||
Operations perform a Shared data change, they are cached in the database as `pending_operations`.
|
||||
|
||||
Operations are removed once all online clients have received the payload.
|
||||
|
|
@ -152,8 +135,6 @@ enum OperationMethod {
|
|||
|
||||
```
|
||||
|
||||
|
||||
|
||||
## Pending operations
|
||||
|
||||
Here are some examples of how operations are stored to minimize disk usage and data duplication.
|
||||
|
|
@ -162,29 +143,25 @@ Here are some examples of how operations are stored to minimize disk usage and d
|
|||
|
||||
In the next case we're handling the creation of a Shared resource. The `method` is marked `Create` and the value is `NULL`. This is because we can also use the actual database record in the `tags` table as it was newly created.
|
||||
|
||||
| `client_uuid` | `uhlc_timestamp` | `method` | `resource_key` | `resource_uuid` | `resource_property` | `value` |
|
||||
|----------|-------------|------|----------|----------|----------|----------|
|
||||
| 2e8f85bf... | 2022-04-09T06:53:36... | Create | tags | 2e8f85bf... | NULL | NULL |
|
||||
| `client_uuid` | `uhlc_timestamp` | `method` | `resource_key` | `resource_uuid` | `resource_property` | `value` |
|
||||
| ------------- | ---------------------- | -------- | -------------- | --------------- | ------------------- | ------- |
|
||||
| 2e8f85bf... | 2022-04-09T06:53:36... | Create | tags | 2e8f85bf... | NULL | NULL |
|
||||
|
||||
**Update operation for Shared data**
|
||||
|
||||
Shared data works at a property level
|
||||
|
||||
| `client_uuid` | `uhlc_timestamp` | `method` | `resource_key` | `resource_uuid` | `resource_property` | `value` |
|
||||
|----------|-------------|------|----------|----------|----------|----------|
|
||||
| 2e8f85bf... | 2022-04-09T06:53:36... | Update | albums | 2e8f85bf... | name | "jeff" |
|
||||
|
||||
|
||||
| `client_uuid` | `uhlc_timestamp` | `method` | `resource_key` | `resource_uuid` | `resource_property` | `value` |
|
||||
| ------------- | ---------------------- | -------- | -------------- | --------------- | ------------------- | ------- |
|
||||
| 2e8f85bf... | 2022-04-09T06:53:36... | Update | albums | 2e8f85bf... | name | "jeff" |
|
||||
|
||||
## Owned Data Synchronization
|
||||
|
||||
Owned data does not use the Operation system, it is queried dynamically by the `updated_at` column on Owned datasets.
|
||||
|
||||
For the sake of compatibility with local relations, some resource properties can be ignored*, such as `file_id` and `parent_id` on the `file_paths` resource, these are re-calculated on bulk ingest.
|
||||
|
||||
*_This will require some form of definition when creating an owned data resource_.
|
||||
|
||||
For the sake of compatibility with local relations, some resource properties can be ignored\*, such as `file_id` and `parent_id` on the `file_paths` resource, these are re-calculated on bulk ingest.
|
||||
|
||||
\*_This will require some form of definition when creating an owned data resource_.
|
||||
|
||||
## Bulk Shared Data Synchronization
|
||||
|
||||
|
|
@ -196,10 +173,6 @@ As `files` are created in abundance (hundreds of thousands at a time), it would
|
|||
|
||||
We handle this by using `SyncMethod::Merge`, simply merging the data where the oldest resource properties are prioritized.
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
## Combining CRDTs
|
||||
|
||||
Combining CRDT types allow for some tailored functionality for particular resources.
|
||||
|
|
@ -220,12 +193,6 @@ impl Replicate for Job {
|
|||
}
|
||||
```
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
## Creating Sync Events
|
||||
|
||||
We have a simple Rust syntax for creating sync events in the core.
|
||||
|
|
@ -261,34 +228,6 @@ Then inside the `sync` function we send the event to the
|
|||
|
||||
Files also impempent `OperationalMerge` would use
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
# Resources
|
||||
|
||||
- https://archive.jlongster.com/using-crdts-in-the-wild
|
||||
|
|
|
|||
|
|
@ -2,8 +2,6 @@
|
|||
|
||||
Spacedrive maintains a virtual filesystem comprised of storage locations through various clients. It records important metadata about a given file as well as a unique checksum for content based addressing [CAS]().
|
||||
|
||||
|
||||
|
||||
### File — `Shared data`
|
||||
|
||||
Represents a unique file across the virtual filesystem, all Spacedrive metadata is tied to this resource through local data relations.
|
||||
|
|
@ -36,7 +34,7 @@ struct File {
|
|||
}
|
||||
```
|
||||
|
||||
- `partial_checksum ` - A SHA256 checksum generated from 5 samples of 10,000 bytes throughout the file data, including the begining and end + total byte count. This is used to identify a file as *likely* unique in under 100µs.
|
||||
- `partial_checksum ` - A SHA256 checksum generated from 5 samples of 10,000 bytes throughout the file data, including the begining and end + total byte count. This is used to identify a file as _likely_ unique in under 100µs.
|
||||
|
||||
> ~~It is impossible to have a unique constraint at a database level for the `partial_checksum` however we can asyncronously resolve conflicts by querying for duplicates and generating full checksums at a later date.~~
|
||||
>
|
||||
|
|
@ -44,8 +42,6 @@ struct File {
|
|||
|
||||
- `checksum` - A full SHA256 checksum of the file data used to verify uniqueness should a `partial_checksum` conflict occur.
|
||||
|
||||
|
||||
|
||||
### FilePath — `Owned data`
|
||||
|
||||
This represents a logical file path within a [Location](), used to derive `file` records.
|
||||
|
|
@ -70,8 +66,6 @@ struct FilePath {
|
|||
}
|
||||
```
|
||||
|
||||
|
||||
|
||||
```typescript
|
||||
export function useBridgeCommand<
|
||||
K extends CommandKeyType,
|
||||
|
|
@ -85,6 +79,3 @@ export function useBridgeCommand<
|
|||
);
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,3 @@
|
|||
|
||||
|
||||
# Todo
|
||||
|
||||
- Landing sections
|
||||
|
|
|
|||
|
|
@ -4,22 +4,16 @@ hello@spacedrive.app`
|
|||
|
||||
**Made with ♥️ in Canada**
|
||||
|
||||
*© Copyright 2022-Present Jamie Pine*
|
||||
|
||||
|
||||
_© Copyright 2022-Present Jamie Pine_
|
||||
|
||||
**Developers:**
|
||||
|
||||
Jamie Pine, Brendonovich, Oscar Beaumont, Haden Fletcher
|
||||
|
||||
|
||||
|
||||
**Founding Users:**
|
||||
|
||||
People, Who, Buy, Founder Pack
|
||||
|
||||
|
||||
|
||||
**Sponsors:**
|
||||
|
||||
Jamie Pine
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
## What is it?
|
||||
## What is it?
|
||||
|
||||
Spacedrive is a cross platform app that allows you to manage files across all devices, drives and clouds at once. Check out the [readme](https://github.com/spacedriveapp) for more detailed info.
|
||||
|
||||
|
|
@ -51,5 +51,3 @@ We call it "PRRTT"; [Prisma](), [Rust](), [React](), [TypeScript]() and [Tauri](
|
|||
## Why Rust?
|
||||
|
||||
Rust is a powerful, fast, memory safe, cross-platform, low-level language and thanks to Tauri, the perfect tool to power a desktop application.
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -5,4 +5,3 @@
|
|||
### The file explorer of the future.
|
||||
|
||||
### All your files in one space.
|
||||
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@ core.register_context(IndexerContext {
|
|||
```
|
||||
|
||||
For Apple Photos we need:
|
||||
|
||||
- Hidden/Favorite items
|
||||
- Live photo support
|
||||
- Original creation date
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
module.exports = require("scripts/eslint-preset");
|
||||
module.exports = require('scripts/eslint-preset');
|
||||
|
|
|
|||
|
|
@ -1,11 +1,11 @@
|
|||
module.exports = {
|
||||
extends: ["next", "prettier"],
|
||||
extends: ['next', 'prettier'],
|
||||
settings: {
|
||||
next: {
|
||||
rootDir: ["apps/*/", "packages/*/"],
|
||||
},
|
||||
rootDir: ['apps/*/', 'packages/*/']
|
||||
}
|
||||
},
|
||||
rules: {
|
||||
"no-html-link-for-pages": "off",
|
||||
},
|
||||
'no-html-link-for-pages': 'off'
|
||||
}
|
||||
};
|
||||
|
|
|
|||
|
|
@ -171,7 +171,6 @@ import { ReactComponent as Yarn } from './yarn.svg';
|
|||
import { ReactComponent as Yarnerror } from './yarnerror.svg';
|
||||
import { ReactComponent as Zip } from './zip.svg';
|
||||
|
||||
|
||||
export default {
|
||||
ai: Ai,
|
||||
angular: Angular,
|
||||
|
|
@ -344,5 +343,5 @@ export default {
|
|||
yaml: Yaml,
|
||||
yarn: Yarn,
|
||||
yarnerror: Yarnerror,
|
||||
zip: Zip,
|
||||
}
|
||||
zip: Zip
|
||||
};
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -3,11 +3,16 @@ import React from 'react';
|
|||
import { DefaultProps } from '../primitive/types';
|
||||
|
||||
export interface DriveListItemProps extends DefaultProps {
|
||||
name: string;
|
||||
name: string;
|
||||
}
|
||||
|
||||
export const DriveListItem: React.FC<DriveListItemProps> = (props) => {
|
||||
return <div className={clsx('rounded px-1.5 py-1 text-xs font-medium inline-block cursor-default', props.className)}>
|
||||
|
||||
</div>;
|
||||
return (
|
||||
<div
|
||||
className={clsx(
|
||||
'rounded px-1.5 py-1 text-xs font-medium inline-block cursor-default',
|
||||
props.className
|
||||
)}
|
||||
></div>
|
||||
);
|
||||
};
|
||||
|
|
|
|||
|
|
@ -22,5 +22,5 @@
|
|||
"@sd/client": ["../client/src/index.ts"]
|
||||
}
|
||||
},
|
||||
"include": ["src", "scripts"],
|
||||
"include": ["src", "scripts"]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,8 +2,6 @@
|
|||
@tailwind components;
|
||||
@tailwind utilities;
|
||||
|
||||
|
||||
|
||||
.html {
|
||||
background: #404040;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,6 +2,6 @@
|
|||
"extends": "../config/tsconfig/react-library.json",
|
||||
"include": ["src"],
|
||||
"compilerOptions": {
|
||||
"outDir": "dist",
|
||||
"outDir": "dist"
|
||||
}
|
||||
}
|
||||
4627
pnpm-lock.yaml
4627
pnpm-lock.yaml
File diff suppressed because it is too large
Load diff
Loading…
Reference in a new issue