CI Cache Factory (#593)

* introduce cache factory

run cache factory on all branches

add names

use single quotes

add setup and prisma generation

add pnpm install

don't install cargo-watch in ci

make composite setup script

use env var for windows ci

undo all powershell stuff

* better windows ci

* don't install cargo-watch in ci

* only run cache factory on main

* i'm silly

* debug only ci

* check runner instead of matrix in setup action

* don't save cache output most of the time

* no more os-specific features

* cache entire workspace

* only cache on main again

* test instead of buidl in cache factory

* smaller ci + testing

* don't individually test each crate

* remove unnecessary needs

* smaller formatting and more clippy

* undo rustfmt changes

* disable tests

.github/actions/generate-prisma-client/action.yaml

@@ -14,4 +14,4 @@ runs:
       working-directory: core
       if: steps.cache-prisma.outputs.cache-hit != 'true'
       shell: bash
-      run: cargo run -p prisma-cli --bin prisma --release -- generate
+      run: cargo run -p prisma-cli --bin prisma -- generate

.github/actions/publish-artifacts/action.yaml

@@ -1,5 +1,8 @@
 name: Publish artifacts
 description: Publishes artifacts after CI process
+inputs:
+  profile:
+    description: "'debug' or 'release'"
 runs:
   using: 'composite'
   steps:
@@ -13,8 +16,8 @@ runs:
       if: matrix.platform == 'ubuntu-latest'
       shell: bash
       run: |
-        chmod +x ./target/release/bundle/appimage/spacedrive*.AppImage
-        tar -czvf ./Spacedrive-AppImage-${{ env.GITHUB_SHA_SHORT }}.tar.gz ./target/release/bundle/appimage/spacedrive*.AppImage
+        chmod +x ./target/${{ inputs.profile }}/bundle/appimage/spacedrive*.AppImage
+        tar -czvf ./Spacedrive-AppImage-${{ env.GITHUB_SHA_SHORT }}.tar.gz ./target/${{ inputs.profile }}/bundle/appimage/spacedrive*.AppImage
 
     - name: Publish artifacts (AppImage)
       if: matrix.platform == 'ubuntu-latest'
@@ -28,25 +31,25 @@ runs:
       uses: actions/upload-artifact@v3
       with:
         name: Spacedrive-deb-${{ env.GITHUB_SHA_SHORT }}
-        path: ./target/release/bundle/deb/spacedrive*.deb
+        path: ./target/${{ inputs.profile }}/bundle/deb/spacedrive*.deb
 
     - name: Publish artifacts (server)
       if: matrix.platform == 'ubuntu-latest'
       uses: actions/upload-artifact@v3
       with:
         name: Spacedrive-server-${{ env.GITHUB_SHA_SHORT }}
-        path: ./target/release/server
+        path: ./target/${{ inputs.profile }}/server
 
     - name: Publish artifacts (Windows)
       if: matrix.platform == 'windows-latest'
       uses: actions/upload-artifact@v3
       with:
         name: Spacedrive-Windows-${{ env.GITHUB_SHA_SHORT }}
-        path: .\target\release\bundle\msi\*.msi
+        path: .\target\${{ inputs.profile }}\bundle\msi\*.msi
 
     - name: Publish artifacts (macOS)
       if: matrix.platform == 'macos-latest'
       uses: actions/upload-artifact@v3
       with:
         name: Spacedrive-macOS-${{ env.GITHUB_SHA_SHORT }}
-        path: ./target/release/bundle/macos/*.app
+        path: ./target/${{ inputs.profile }}/bundle/macos/*.app

.github/actions/setup/action.yml

@@ -0,0 +1,38 @@
+name: Setup
+description: Sets up runner, Rust and pnpm
+inputs:
+  save-cache:
+    description: Whether to save the Rust cache
+    required: false
+    default: "false"
+runs:
+  using: 'composite'
+  steps:
+    - name: Install Rust stable
+      uses: actions-rs/toolchain@v1
+      with:
+        toolchain: stable
+        profile: minimal
+
+    - name: Install pnpm
+      uses: pnpm/action-setup@v2.2.2
+      with:
+        version: 7.x.x
+
+    - name: Cache Rust deps
+      uses: Swatinem/rust-cache@v2
+      with:
+        save-if: ${{ inputs.save-cache }} == "true"
+
+    - name: Run 'setup-system.sh' script
+      shell: bash
+      if: runner.os == 'Linux' || runner.os == 'macOS'
+      run: ./.github/scripts/setup-system.sh
+
+    - name: Run 'setup-system.ps1' script
+      shell: powershell
+      if: runner.os == 'Windows'
+      run: ./.github/scripts/setup-system.ps1
+
+    - name: Generate Prisma client
+      uses: ./.github/actions/generate-prisma-client

.github/scripts/setup-system.ps1

@@ -1,9 +1,3 @@
-# Get ci parameter to check if running with ci
-param(
-    [Parameter()]
-    [Switch]$ci
-)
-
 # Get temp folder
 $temp = [System.IO.Path]::GetTempPath()
 
@@ -68,8 +62,10 @@ else {
    Write-Host "Cargo is installed."
 }
 
-Write-Host "Installing Rust tools" -ForegroundColor Yellow
-cargo install cargo-watch
+if ($env:CI -ne $True) {
+	Write-Host "Installing Rust tools" -ForegroundColor Yellow
+	cargo install cargo-watch
+}
 
 Write-Host
 Write-Host "Checking for pnpm..." -ForegroundColor Yellow
@@ -94,7 +90,7 @@ else {
 }
 
 # A GitHub Action takes care of installing node, so this isn't necessary if running in the ci.
-if ($ci -eq $True) {
+if ($env:CI -eq $True) {
    Write-Host
    Write-Host "Running with Ci, skipping Node install." -ForegroundColor Yellow
 }
@@ -111,7 +107,7 @@ else {
 
 
 # The ci has LLVM installed already, so we instead just set the env variables.
-if ($ci -eq $True) {
+if ($env:CI -eq $True) {
    Write-Host
    Write-Host "Running with Ci, skipping LLVM install." -ForegroundColor Yellow
 
@@ -161,7 +157,7 @@ Remove-Item "$temp\ffmpeg.zip"
 Write-Host
 Write-Host "Setting environment variables..." -ForegroundColor Yellow
 
-if ($ci -eq $True) {
+if ($env:CI -eq $True) {
    # If running in ci, we need to use GITHUB_ENV and GITHUB_PATH instead of the normal PATH env variables, so we set them here
    Add-Content $env:GITHUB_ENV "FFMPEG_DIR=$HOME\$foldername`n"
    Add-Content $env:GITHUB_PATH "$HOME\$foldername\bin`n" 

.github/scripts/setup-system.sh

@@ -37,8 +37,10 @@ else
 	echo "Skipping pnpm check."
 fi
 
-echo "Installing Rust tools"
-cargo install cargo-watch
+if [ "$CI" != "true" ]; then
+	echo "Installing Rust tools"
+	cargo install cargo-watch
+fi
 
 echo
 

.github/workflows/cache-factory.yaml

@@ -0,0 +1,31 @@
+# This workflow _produces_ caches which are used to speed up pull request builds.
+# Originally from https://github.com/libp2p/rust-libp2p/blob/master/.github/workflows/cache-factory.yml
+
+name: Cache Factory
+
+on:
+  push:
+    branches:
+      - main
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  make_cache:
+    name: 'Make Cache'
+    runs-on: ${{ matrix.platform }}
+    strategy:
+      fail-fast: true
+      matrix:
+        platform: [ubuntu-latest, macos-latest, windows-latest]
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: ./.github/actions/setup
+        with:
+          save-cache: "true"
+
+      - name: Compile workspace with stable Rust
+        run: cargo test --workspace --all-features --no-run

.github/workflows/ci.yml

@@ -62,7 +62,7 @@ jobs:
         run: pnpm lint
 
   rustfmt:
-    name: rustfmt
+    name: Rust Formatting
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
@@ -74,8 +74,8 @@ jobs:
           toolchain: stable
           components: rustfmt
 
-      - name: Cache Rust dependencies
-        uses: ./.github/actions/cache-rust-deps
+      - name: Cache Rust deps
+        uses: Swatinem/rust-cache@v2
 
       - name: Generate Prisma client
         uses: ./.github/actions/generate-prisma-client
@@ -83,120 +83,37 @@ jobs:
       - name: Run rustfmt
         run: cargo fmt --all -- --check
 
-  clippy_check:
-    name: Clippy
-    runs-on: ubuntu-latest
+  clippy:
+    name: Clippy (${{ matrix.platform }})
+    runs-on: ${{ matrix.platform }}
+    strategy:
+      matrix:
+        platform: [ubuntu-latest, macos-latest, windows-latest]
     steps:
       - name: Checkout repository
         uses: actions/checkout@v2
 
-      - name: Install Rust
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          profile: minimal
-          override: true
-          components: clippy
-
-      - name: Cache Rust dependencies
-        uses: ./.github/actions/cache-rust-deps
-
-      - name: Install Node.js
-        uses: actions/setup-node@v3
-        with:
-          node-version: 18
-
-      - name: Install pnpm
-        uses: pnpm/action-setup@v2.0.1
-        with:
-          version: 7
-          run_install: false
-
-      - name: Run 'setup-system.sh' script
-        run: ./.github/scripts/setup-system.sh
-
-      - name: Generate Prisma client
-        uses: ./.github/actions/generate-prisma-client
+      - name: Setup
+        uses: ./.github/actions/setup
 
       - name: Run Clippy
         uses: actions-rs/clippy-check@v1
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
-          args: --all-features --release
+          args: --workspace --all-features
 
-  build-and-publish:
-    name: Build and Publish (${{ matrix.platform }})
-    runs-on: ${{ matrix.platform }}
-    strategy:
-      fail-fast: true
-      matrix:
-        platform: [ubuntu-latest, macos-latest, windows-latest]
-    steps:
-      - name: Log in to the Container registry
-        if: matrix.platform == 'ubuntu-latest' && github.event_name != 'pull_request'
-        uses: docker/login-action@v1
-        with:
-          registry: ghcr.io
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
+  # test:
+  #   name: Test (${{ matrix.platform }})
+  #   runs-on: ${{ matrix.platform }}
+  #   strategy:
+  #     matrix:
+  #       platform: [ubuntu-latest, macos-latest, windows-latest]
+  #   steps:
+  #     - name: Checkout repository
+  #       uses: actions/checkout@v3
 
-      - name: Checkout repository
-        uses: actions/checkout@v3
+  #     - name: Setup
+  #       uses: ./.github/actions/setup
 
-      - name: Install Rust stable
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          profile: minimal
-          override: true
-          components: rustfmt, rust-src
-
-      - name: Cache Rust dependencies
-        uses: ./.github/actions/cache-rust-deps
-
-      - name: Install pnpm
-        uses: pnpm/action-setup@v2.2.2
-        with:
-          version: 7.x.x
-
-      - name: Install Node.js
-        uses: actions/setup-node@v3
-        with:
-          node-version: 18
-          cache: 'pnpm'
-
-      - name: Install pnpm dependencies
-        run: pnpm i --frozen-lockfile
-
-      - name: Run 'setup-system.sh' script
-        if: matrix.platform == 'ubuntu-latest' || matrix.platform == 'macos-latest'
-        run: ./.github/scripts/setup-system.sh
-
-      - name: Run 'setup-system.ps1' script
-        if: matrix.platform == 'windows-latest'
-        run: ./.github/scripts/setup-system.ps1 -ci
-
-      - name: Generate Prisma client
-        uses: ./.github/actions/generate-prisma-client
-
-      - name: Cargo fetch
-        run: cargo fetch
-
-      - name: Cargo test core
-        run: cargo test -p sd-core -F location-watcher
-
-      - name: Check core
-        run: cargo check -p sd-core --release
-
-      - name: Cargo test sd-crypto
-        run: cargo test -p sd-crypto --release --lib --all-features
-
-      - name: Bundle Desktop
-        run: pnpm desktop tauri build
-
-      - name: Publish server Docker image
-        if: matrix.platform == 'ubuntu-latest' && github.event_name != 'pull_request'
-        uses: ./.github/actions/publish-server-image
-
-      - name: Publish artifacts
-        uses: ./.github/actions/publish-artifacts
+  #     - name: Test
+  #       run: cargo test --workspace --all-features

Cargo.lock

@@ -105,12 +105,6 @@ dependencies = [
  "alloc-no-stdlib",
 ]
 
-[[package]]
-name = "android_log-sys"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "85965b6739a430150bdd138e2374a98af0c3ee0d030b3bb7fc3bddff58d0102e"
-
 [[package]]
 name = "android_system_properties"
 version = "0.1.5"
@@ -5627,7 +5621,6 @@ dependencies = [
  "thiserror",
  "tokio",
  "tracing",
- "tracing-android",
  "tracing-subscriber",
  "tracing-test",
  "uhlc",
@@ -7161,17 +7154,6 @@ dependencies = [
  "tracing-core",
 ]
 
-[[package]]
-name = "tracing-android"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "12612be8f868a09c0ceae7113ff26afe79d81a24473a393cb9120ece162e86c0"
-dependencies = [
- "android_log-sys",
- "tracing",
- "tracing-subscriber",
-]
-
 [[package]]
 name = "tracing-attributes"
 version = "0.1.23"

apps/mobile/crates/android/Cargo.toml

@@ -13,7 +13,7 @@ crate-type = ["cdylib"]
 jni = "0.19.0"
 
 # Core mobile handling stuff
-sd-mobile-core = { path = "../core", features = ["android"] }
+sd-mobile-core = { path = "../core" }
 
 # Other 
 tracing = "0.1.37"

apps/mobile/crates/core/Cargo.toml

@@ -4,9 +4,6 @@ version = "0.1.0"
 edition = "2021"
 rust-version = "1.64.0"
 
-[features]
-android = ["sd-core/android"]
-
 [dependencies]
 once_cell = "1.15.0"
 sd-core = { path = "../../../../core", features = [

core/Cargo.toml

@@ -12,7 +12,6 @@ rust-version = "1.67.0"
 default = ["p2p"]
 p2p = [] # This feature controls whether the Spacedrive Core contains the Peer to Peer syncing engine (It isn't required for the hosted core so we can disable it).
 mobile = [] # This feature allows features to be disabled when the Core is running on mobile.
-android = ["dep:tracing-android"]
 ffmpeg = ["dep:ffmpeg-next", "dep:sd-ffmpeg"] # This feature controls whether the Spacedrive Core contains functionality which requires FFmpeg.
 location-watcher = ["dep:notify"]
 
@@ -57,7 +56,6 @@ sd-file-ext = { path = "../crates/file-ext" }
 sd-sync = { path = "../crates/sync" }
 tracing = "0.1.36"
 tracing-subscriber = { version = "0.3.15", features = ["env-filter"] }
-tracing-android = { version = "0.2.0", optional = true }
 async-stream = "0.3.3"
 once_cell = "1.15.0"
 ctor = "0.1.23"

core/src/lib.rs

@@ -41,7 +41,7 @@ pub struct Node {
 	secure_temp_keystore: Arc<SecureTempKeystore>,
 }
 
-#[cfg(not(feature = "android"))]
+#[cfg(not(target_os = "android"))]
 const CONSOLE_LOG_FILTER: tracing_subscriber::filter::LevelFilter = {
 	use tracing_subscriber::filter::LevelFilter;
 
@@ -99,9 +99,9 @@ impl Node {
 			    // 		.expect("Error invalid tracing directive!"),
 			    // ),
 		);
-		#[cfg(not(feature = "android"))]
+		#[cfg(not(target_os = "android"))]
 		let subscriber = subscriber.with(tracing_subscriber::fmt::layer().with_filter(CONSOLE_LOG_FILTER));
-		#[cfg(feature = "android")]
+		#[cfg(target_os = "android")]
 		let subscriber = subscriber.with(tracing_android::layer("com.spacedrive.app").unwrap()); // TODO: This is not working
 		subscriber
 			// .with(