mirror of
https://github.com/spacedriveapp/spacedrive
synced 2024-07-07 04:23:29 +00:00
(fix) add new team members
This commit is contained in:
Jamie Pine
parent
c1bdf541a1
commit
9583efb6d9
|
|
@ -1,52 +1,53 @@
|
|||
{
|
||||
"name": "@sd/server",
|
||||
"scripts": {
|
||||
"dev": "npm run server",
|
||||
"prod": "npm run build && npm run server:prod",
|
||||
"build": "vite build",
|
||||
"server": "ts-node ./server",
|
||||
"server:prod": "cross-env NODE_ENV=production ts-node ./server"
|
||||
},
|
||||
"dependencies": {
|
||||
"@heroicons/react": "^1.0.6",
|
||||
"@icons-pack/react-simple-icons": "^5.2.0",
|
||||
"@sd/interface": "link:../../packages/interface",
|
||||
"@sd/ui": "link:../../packages/ui",
|
||||
"@tryghost/content-api": "^1.11.0",
|
||||
"@types/compression": "^1.7.2",
|
||||
"@types/express": "^4.17.13",
|
||||
"@types/node": "^17.0.31",
|
||||
"@types/react": "^18.0.8",
|
||||
"@types/react-dom": "^18.0.3",
|
||||
"@vitejs/plugin-react": "^1.3.2",
|
||||
"clsx": "^1.2.1",
|
||||
"compression": "^1.7.4",
|
||||
"cross-env": "^7.0.3",
|
||||
"express": "^4.18.1",
|
||||
"phosphor-react": "^1.4.1",
|
||||
"prismjs": "^1.28.0",
|
||||
"react": "^18.1.0",
|
||||
"react-dom": "^18.1.0",
|
||||
"react-helmet": "^6.1.0",
|
||||
"react-hook-form": "^7.33.1",
|
||||
"react-tsparticles": "^2.1.3",
|
||||
"sirv": "^2.0.2",
|
||||
"ts-node": "^10.7.0",
|
||||
"tsparticles": "^2.1.3",
|
||||
"typescript": "^4.6.4",
|
||||
"vite": "^2.9.14",
|
||||
"vite-plugin-ssr": "^0.4.15"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@tailwindcss/line-clamp": "^0.4.0",
|
||||
"@tailwindcss/typography": "^0.5.4",
|
||||
"@types/prismjs": "^1.26.0",
|
||||
"@types/react-helmet": "^6.1.5",
|
||||
"@types/tryghost__content-api": "^1.3.11",
|
||||
"postcss": "^8.4.14",
|
||||
"sass": "^1.54.0",
|
||||
"tailwind": "^4.0.0",
|
||||
"vite-plugin-markdown": "^2.0.2",
|
||||
"vite-plugin-svgr": "^2.2.1"
|
||||
}
|
||||
"name": "@sd/landing",
|
||||
"scripts": {
|
||||
"dev": "pnpm run server",
|
||||
"prod": "pnpm run build && pnpm run server:prod",
|
||||
"vercel-build": "./vercel/deploy.sh",
|
||||
"build": "vite build && vite build --ssr && vite-plugin-ssr prerender",
|
||||
"server": "ts-node ./server",
|
||||
"server:prod": "cross-env NODE_ENV=production ts-node ./server"
|
||||
},
|
||||
"dependencies": {
|
||||
"@heroicons/react": "^1.0.6",
|
||||
"@icons-pack/react-simple-icons": "^5.2.0",
|
||||
"@sd/interface": "link:../../packages/interface",
|
||||
"@sd/ui": "link:../../packages/ui",
|
||||
"@tryghost/content-api": "^1.11.0",
|
||||
"@types/compression": "^1.7.2",
|
||||
"@types/express": "^4.17.13",
|
||||
"@types/node": "^17.0.31",
|
||||
"@types/react": "^18.0.8",
|
||||
"@types/react-dom": "^18.0.3",
|
||||
"@vitejs/plugin-react": "^1.3.2",
|
||||
"clsx": "^1.2.1",
|
||||
"compression": "^1.7.4",
|
||||
"cross-env": "^7.0.3",
|
||||
"express": "^4.18.1",
|
||||
"phosphor-react": "^1.4.1",
|
||||
"prismjs": "^1.28.0",
|
||||
"react": "^18.1.0",
|
||||
"react-dom": "^18.1.0",
|
||||
"react-helmet": "^6.1.0",
|
||||
"react-hook-form": "^7.33.1",
|
||||
"react-tsparticles": "^2.1.3",
|
||||
"sirv": "^2.0.2",
|
||||
"ts-node": "^10.7.0",
|
||||
"tsparticles": "^2.1.3",
|
||||
"typescript": "^4.6.4",
|
||||
"vite": "^2.9.14",
|
||||
"vite-plugin-ssr": "^0.4.15"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@tailwindcss/line-clamp": "^0.4.0",
|
||||
"@tailwindcss/typography": "^0.5.4",
|
||||
"@types/prismjs": "^1.26.0",
|
||||
"@types/react-helmet": "^6.1.5",
|
||||
"@types/tryghost__content-api": "^1.3.11",
|
||||
"postcss": "^8.4.14",
|
||||
"sass": "^1.54.0",
|
||||
"tailwind": "^4.0.0",
|
||||
"vite-plugin-markdown": "^2.0.2",
|
||||
"vite-plugin-svgr": "^2.2.1"
|
||||
}
|
||||
}
|
||||
|
|
|
|||
BIN
apps/landing/src/assets/images/team/ericson.jpg
Normal file
BIN
apps/landing/src/assets/images/team/ericson.jpg
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 35 KiB |
BIN
apps/landing/src/assets/images/team/utku.jpg
Normal file
BIN
apps/landing/src/assets/images/team/utku.jpg
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 43 KiB |
|
|
@ -41,6 +41,23 @@ const teamMembers: Array<TeamMemberProps> = [
|
|||
github: 'https://github.com/oscartbeaumont'
|
||||
}
|
||||
},
|
||||
{
|
||||
name: 'Ericson Soares',
|
||||
role: 'Rust Backend Engineer',
|
||||
image: teamImages['ericson.jpg'],
|
||||
socials: {
|
||||
twitter: 'https://twitter.com/fogodev',
|
||||
github: 'https://github.com/fogodev'
|
||||
}
|
||||
},
|
||||
{
|
||||
name: 'Utku Bakir',
|
||||
role: 'React Native Engineer',
|
||||
image: teamImages['utku.jpg'],
|
||||
socials: {
|
||||
github: 'https://github.com/utkubakir'
|
||||
}
|
||||
},
|
||||
{
|
||||
name: 'Haden Fletcher',
|
||||
role: 'Engineer & Designer',
|
||||
|
|
|
|||
|
|
@ -70,7 +70,7 @@ impl InvalidRequests {
|
|||
}
|
||||
|
||||
/// invalidate_query is a macro which stores a list of all of it's invocations so it can ensure all of the queries match the queries attached to the router.
|
||||
/// This allows invalidate to the type safe even when the router keys are stringly typed.
|
||||
/// This allows invalidate to be type-safe even when the router keys are stringly typed.
|
||||
/// ```ignore
|
||||
/// invalidate_query!(
|
||||
/// library, // crate::library::LibraryContext
|
||||
|
|
|
|||
|
|
@ -1,225 +0,0 @@
|
|||
# Distributed Data Sync
|
||||
|
||||
Synchronizing data between clients in a Spacedrive network is accomplished using various forms of [CRDTs](https://en.wikipedia.org/wiki/Conflict-free_replicated_data_type) combined with a hybrid logical clock, ensuring eventual constancy.
|
||||
|
||||
Designed for synchronizing data in realtime between [SQLite](https://www.sqlite.org/) databases potentially in the gigabytes.
|
||||
|
||||
```rust
|
||||
// we can now impl specific CRDT traits to given resources
|
||||
enum SyncResource {
|
||||
FilePath(dyn Replicate),
|
||||
File(dyn PropertyOperation),
|
||||
Tag(dyn PropertyOperation),
|
||||
TagOnFile(dyn LastWriteWin),
|
||||
Jobs(dyn Replicate + OperationalTransform)
|
||||
}
|
||||
```
|
||||
|
||||
## Data Types
|
||||
|
||||
Data is divided into several kinds, Shared and Owned.
|
||||
|
||||
- **Shared data** - Can be created and modified by any client. Has a `uuid`.
|
||||
|
||||
_Sync Method:_ `Property operation*`
|
||||
|
||||
> Shared resources could be,`files`, `tags`, `notes`, `albums` and `labels`. Since these can be created, updated or deleted by any client at any time.
|
||||
|
||||
- **Owned data** - Can only be modified by the client that created it. Has a `client_id` and `uuid`.
|
||||
|
||||
_Sync Method:_ `Replicate`
|
||||
|
||||
> Owned resources would be `file_paths`, `jobs`, `locations` and `media_data`, since a client is the single source of truth for this data. This means we can perform conflict free synchronization.
|
||||
|
||||
\*_Shared data doesn't always use this method, in some cases we can create shared resources in bulk, where conflicts are handled by simply merging. More on that in [Synchronization Strategy]()_.
|
||||
|
||||
## Node Pool
|
||||
|
||||
The node pool maintains record of all nodes in your network.
|
||||
|
||||
An exact replica of the client pool is synchronized on each client. When a given client has a state change, it will notify every other client in the pool via the `connection` struct.
|
||||
|
||||
The `ClientConnection` is maintained in memory and is established on startup.
|
||||
|
||||
```rust
|
||||
struct NodePool {
|
||||
clients: Vec<Client>
|
||||
}
|
||||
|
||||
struct Node {
|
||||
uuid: String,
|
||||
last_seen: DateTime<Utc>,
|
||||
last_synchronized: DateTime<Utc>,
|
||||
connection: Option<NodeConnection>
|
||||
}
|
||||
```
|
||||
|
||||
Nodes will ping-pong to ensure their connection stays alive, this logic is contained within the `NodeConnection` instance.
|
||||
|
||||
**Handling stale nodes**
|
||||
|
||||
If a node has not been seen in X amount of time, other nodes will not persist pending operations for them. Nodes take care of flushing the pending operation queue once all non-stale nodes have received the pending operations.
|
||||
|
||||
## Clock
|
||||
|
||||
With realtime synchronization it is important to maintain the true order of events, we can timestamp each operation, but have to account for time drift; there is no way to guarantee two machines have synchronized system clocks.
|
||||
|
||||
We can solve this with a Unique Hybrid Logical Clock ([UHLC]()): a globally-unique, monotonic timestamp.
|
||||
|
||||
```
|
||||
2022-04-09T06:53:36.397295996Z/89F9DD8514914648989315B2D30D7BE5
|
||||
```
|
||||
|
||||
Each client combines their hybrid time with a unique identifier. When receiving new [Sync Events](), a client will update its own clock with the incoming timestamp.
|
||||
|
||||
A client will reject operations with a timestamp drift greater than 100ms (can be adjusted).
|
||||
|
||||
This allows us to entirely avoid the need to synchronize time between clients, as each client controls its own order of operations, never producing a conflicting timestamp with another system in the network.
|
||||
|
||||
## Synchronization Strategy
|
||||
|
||||
Sync happens in the following order:
|
||||
|
||||
Owned data → Bulk shared data → Shared data
|
||||
|
||||
### Types of CRDT:
|
||||
|
||||
```rust
|
||||
trait PropertyOperation;
|
||||
|
||||
trait Replicate;
|
||||
```
|
||||
|
||||
- **PropertyOperation** - Update Shared resources at a property level. Operations stored in `pending_operations` table.
|
||||
- **Replicate** - Used exclusively for Owned data, clients will replicate with no questions asked.
|
||||
|
||||
- ~~**Last Write Win** - The most recent event will always be applied, used for many-to-many datasets.~~
|
||||
|
||||
## Operations
|
||||
|
||||
Operations perform a Shared data change, they are cached in the database as `pending_operations`.
|
||||
|
||||
Operations are removed once all online clients have received the payload.
|
||||
|
||||
```rust
|
||||
struct PropertyOperation<V> {
|
||||
method: OperationMethod,
|
||||
// the name of the database table
|
||||
resource_type: String,
|
||||
// the unique identifier of the resource (None for batched)
|
||||
resource_uuid: String,
|
||||
// the property on the resource whose value shall be affected
|
||||
resource_property: Option<String>
|
||||
// optional value for operation
|
||||
value: Option<Box<V>>,
|
||||
}
|
||||
|
||||
enum OperationMethod {
|
||||
Create,
|
||||
Update
|
||||
Delete
|
||||
}
|
||||
|
||||
```
|
||||
|
||||
## Pending operations
|
||||
|
||||
Here are some examples of how operations are stored to minimize disk usage and data duplication.
|
||||
|
||||
**Create operation for Shared data**
|
||||
|
||||
In the next case we're handling the creation of a Shared resource. The `method` is marked `Create` and the value is `NULL`. This is because we can also use the actual database record in the `tags` table as it was newly created.
|
||||
|
||||
| `client_uuid` | `uhlc_timestamp` | `method` | `resource_key` | `resource_uuid` | `resource_property` | `value` |
|
||||
| ------------- | ---------------------- | -------- | -------------- | --------------- | ------------------- | ------- |
|
||||
| 2e8f85bf... | 2022-04-09T06:53:36... | Create | tags | 2e8f85bf... | NULL | NULL |
|
||||
|
||||
**Update operation for Shared data**
|
||||
|
||||
Shared data works at a property level
|
||||
|
||||
| `client_uuid` | `uhlc_timestamp` | `method` | `resource_key` | `resource_uuid` | `resource_property` | `value` |
|
||||
| ------------- | ---------------------- | -------- | -------------- | --------------- | ------------------- | ------- |
|
||||
| 2e8f85bf... | 2022-04-09T06:53:36... | Update | albums | 2e8f85bf... | name | "jeff" |
|
||||
|
||||
## Owned Data Synchronization
|
||||
|
||||
Owned data does not use the Operation system, it is queried dynamically by the `updated_at` column on Owned datasets.
|
||||
|
||||
For the sake of compatibility with local relations, some resource properties can be ignored\*, such as `file_id` and `parent_id` on the `file_paths` resource, these are re-calculated on bulk ingest.
|
||||
|
||||
\*_This will require some form of definition when creating an owned data resource_.
|
||||
|
||||
## Bulk Shared Data Synchronization
|
||||
|
||||
In some cases we are able to create many shared data resources at once and resolve conflicts on the fly by merging where the oldest resource takes priority.
|
||||
|
||||
This is intended for the `files` resource. It requires Shared data behavior as most other shared resources are related at a database level and user defined metadata can be assigned, however it is initially derived from `file_paths` which is Owned data.
|
||||
|
||||
As `files` are created in abundance (hundreds of thousands at a time), it would be inefficient to record these changes in the `pending_operations` table. But we are also unable to sync in the same way as Owned data due to the possibility of conflicts.
|
||||
|
||||
We handle this by using `SyncMethod::Merge`, simply merging the data where the oldest resource properties are prioritized.
|
||||
|
||||
## Combining CRDTs
|
||||
|
||||
Combining CRDT types allow for some tailored functionality for particular resources.
|
||||
|
||||
Looking at the `jobs` resource let look how `OperationalTransform + Replicate` might work.
|
||||
|
||||
Jobs are unique in that they have frequent updates to some properties and
|
||||
|
||||
```rust
|
||||
impl OperationalTransform for Job {
|
||||
pub fn create () {}
|
||||
pub fn update () {}
|
||||
pub fn delete () {}
|
||||
}
|
||||
|
||||
impl Replicate for Job {
|
||||
|
||||
}
|
||||
```
|
||||
|
||||
## Creating Sync Events
|
||||
|
||||
We have a simple Rust syntax for creating sync events in the core.
|
||||
|
||||
```rust
|
||||
async fn my_core_function(&ctx: CoreContext) -> Result<()> {
|
||||
let mut file = File::get_unique(1).await?;
|
||||
|
||||
ctx.sync.operation(file.id,
|
||||
SyncResource::File(
|
||||
Operation::Update(
|
||||
FileUpdate::HasThumbnail(true)
|
||||
)
|
||||
)
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
```
|
||||
|
||||
Then inside the `sync` function we send the event to the
|
||||
|
||||
```rust
|
||||
impl SyncEngine {
|
||||
pub fn operation(&self, uuid: &str, sync_resource: SyncResource) {
|
||||
self.perform_operation(
|
||||
uuid.clone(),
|
||||
SyncTransport::Message(sync_resource)
|
||||
);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Files also implement `OperationalMerge` would use
|
||||
|
||||
# Resources
|
||||
|
||||
- https://archive.jlongster.com/using-crdts-in-the-wild
|
||||
- https://cse.buffalo.edu/tech-reports/2014-04.pdf
|
||||
- https://sergeiturukin.com/2017/06/26/hybrid-logical-clocks.html
|
||||
- https://github.com/atolab/uhlc-rs
|
||||
- https://github.com/alangibson/awesome-crdt
|
||||
- https://blog.logrocket.com/libp2p-tutorial-build-a-peer-to-peer-app-in-rust/
|
||||
|
|
@ -1,77 +0,0 @@
|
|||
# Virtual Filesystem
|
||||
|
||||
Spacedrive maintains a virtual filesystem comprised of storage locations through various clients. It records important metadata about a given file as well as a unique checksum for content based addressing [CAS]().
|
||||
|
||||
### File — `Shared data`
|
||||
|
||||
Represents a unique file across the virtual filesystem, all Spacedrive metadata is tied to this resource through local data relations.
|
||||
|
||||
```rust
|
||||
struct File {
|
||||
id: i32,
|
||||
cas_id: str,
|
||||
integrity_checksum: Option<str>,
|
||||
kind: FileKind,
|
||||
hidden: bool,
|
||||
favorite: bool,
|
||||
has_thumbnail: bool,
|
||||
has_thumbstrip: bool,
|
||||
has_video_preview: bool,
|
||||
key: Key,
|
||||
ipfs_id: Option<str>,
|
||||
paths: Vec<FilePath>,
|
||||
tags: Vec<Tag>,
|
||||
labels: Vec<Label>,
|
||||
notes: Vec<Note>,
|
||||
albums: Vec<Album>,
|
||||
media_data: Option<MediaData>,
|
||||
date_created: DateTime<Utc>,
|
||||
date_modified: DateTime<Utc>,
|
||||
}
|
||||
```
|
||||
|
||||
- `cas_id ` - A SHA256 checksum generated from 5 samples of 10,000 bytes throughout the file data, including the beginning and end + total byte count. This is used to identify a file as _likely_ unique in under 100µs.
|
||||
|
||||
> ~~It is impossible to have a unique constraint at a database level for the `partial_checksum` however we can asynchronously resolve conflicts by querying for duplicates and generating full checksums at a later date.~~
|
||||
>
|
||||
> For synchronization of this resource we can tolerate temporary duplicates, any client can calculate that two files resources are duplicate and merge them into a single resource. In turn, triggering a shared data merge operation, whereby the older record is prioritized at a property level during the merge.
|
||||
|
||||
- `integrity_checksum` - A full SHA256 checksum of the file data used to verify uniqueness should a `cas_id` conflict occur.
|
||||
|
||||
### FilePath — `Owned data`
|
||||
|
||||
This represents a logical file path within a [Location](), used to derive `file` records.
|
||||
|
||||
```rust
|
||||
struct FilePath {
|
||||
uuid: String,
|
||||
is_dir: bool,
|
||||
location_id: i32,
|
||||
path: String,
|
||||
name: String,
|
||||
extension: Option<String>,
|
||||
size_in_bytes: String,
|
||||
permissions: Option<String>,
|
||||
|
||||
parent_id: Option<i32>,
|
||||
file_id: Option<i32>,
|
||||
|
||||
date_created: DateTime<Utc>,
|
||||
date_modified: DateTime<Utc>,
|
||||
date_indexed: DateTime<Utc>,
|
||||
}
|
||||
```
|
||||
|
||||
```typescript
|
||||
export function useBridgeCommand<
|
||||
K extends CommandKeyType,
|
||||
CC extends CCType<K>,
|
||||
CR extends CRType<K>
|
||||
>(key: K, options: UseMutationOptions<ExtractData<CC>> = {}) {
|
||||
return useMutation<ExtractData<CR>, unknown, any>(
|
||||
[key],
|
||||
async (vars: ExtractParams<CC>) => await commandBridge(key, vars),
|
||||
options
|
||||
);
|
||||
}
|
||||
```
|
||||
|
|
@ -16,7 +16,7 @@ export function ErrorFallback({ error, resetErrorBoundary }: FallbackProps) {
|
|||
<Button variant="primary" className="mt-2" onClick={resetErrorBoundary}>
|
||||
Reload
|
||||
</Button>
|
||||
<Button className="mt-2" onClick={resetErrorBoundary}>
|
||||
<Button variant="gray" className="mt-2" onClick={resetErrorBoundary}>
|
||||
Send report
|
||||
</Button>
|
||||
</div>
|
||||
|
|
|
|||
Loading…
Reference in a new issue