diff --git a/.cargo/config.toml.mustache b/.cargo/config.toml.mustache index 16e2445b5..8236e42c8 100644 --- a/.cargo/config.toml.mustache +++ b/.cargo/config.toml.mustache @@ -1,3 +1,4 @@ +{{#nativeDeps}} [env] PROTOC = { force = true, value = "{{{protoc}}}" } FFMPEG_DIR = { force = true, value = "{{{nativeDeps}}}" } @@ -7,17 +8,18 @@ ORT_LIB_LOCATION = { force = true, value = "{{{nativeDeps}}}/lib" } OPENSSL_STATIC = { force = true, value = "1" } OPENSSL_NO_VENDOR = { force = true, value = "0" } OPENSSL_RUST_USE_NASM = { force = true, value = "1" } +{{/nativeDeps}} {{#isMacOS}} [target.x86_64-apple-darwin] -rustflags = ["-L", "{{{nativeDeps}}}/lib"] +rustflags = ["-L", "{{{nativeDeps}}}/lib", "-Csplit-debuginfo=unpacked"] [target.x86_64-apple-darwin.heif] rustc-link-search = ["{{{nativeDeps}}}/lib"] rustc-link-lib = ["heif"] [target.aarch64-apple-darwin] -rustflags = ["-L", "{{{nativeDeps}}}/lib"] +rustflags = ["-L", "{{{nativeDeps}}}/lib", "-Csplit-debuginfo=unpacked"] [target.aarch64-apple-darwin.heif] rustc-link-search = ["{{{nativeDeps}}}/lib"] @@ -26,6 +28,9 @@ rustc-link-lib = ["heif"] {{#isWin}} [target.x86_64-pc-windows-msvc] +{{#hasLLD}} +linker = "lld-link.exe" +{{/hasLLD}} rustflags = ["-L", "{{{nativeDeps}}}\\lib"] [target.x86_64-pc-windows-msvc.heif] @@ -35,14 +40,30 @@ rustc-link-lib = ["heif"] {{#isLinux}} [target.x86_64-unknown-linux-gnu] -rustflags = ["-L", "{{{nativeDeps}}}/lib", "-C", "link-arg=-Wl,-rpath=${ORIGIN}/../lib/spacedrive"] +{{#hasLLD}} +linker = "clang" +{{/hasLLD}} +rustflags = [ + "-L", "{{{nativeDeps}}}/lib", "-C", "link-arg=-Wl,-rpath=${ORIGIN}/../lib/spacedrive", + {{#hasLLD}} + "-C", "link-arg=-fuse-ld={{{linker}}}", + {{/hasLLD}} +] [target.x86_64-unknown-linux-gnu.heif] rustc-link-search = ["{{{nativeDeps}}}/lib"] rustc-link-lib = ["heif"] [target.aarch64-unknown-linux-gnu] -rustflags = ["-L", "{{{nativeDeps}}}/lib", "-C", "link-arg=-Wl,-rpath=${ORIGIN}/../lib/spacedrive"] +{{#hasLLD}} +linker = "clang" +{{/hasLLD}} +rustflags = [ + "-L", "{{{nativeDeps}}}/lib", "-C", "link-arg=-Wl,-rpath=${ORIGIN}/../lib/spacedrive", + {{#hasLLD}} + "-C", "link-arg=-fuse-ld={{{linker}}}", + {{/hasLLD}} +] [target.aarch64-unknown-linux-gnu.heif] rustc-link-search = ["{{{nativeDeps}}}/lib"] diff --git a/.github/actions/setup-pnpm/action.yml b/.github/actions/setup-pnpm/action.yml index 61b77048c..2f9630a65 100644 --- a/.github/actions/setup-pnpm/action.yml +++ b/.github/actions/setup-pnpm/action.yml @@ -9,12 +9,12 @@ runs: using: 'composite' steps: - name: Install pnpm - uses: pnpm/action-setup@v2 + uses: pnpm/action-setup@v3 with: version: 8.x.x - name: Install Node.js - uses: actions/setup-node@v3 + uses: actions/setup-node@v4 with: token: ${{ inputs.token }} check-latest: true diff --git a/.github/actions/setup-rust/action.yaml b/.github/actions/setup-rust/action.yaml index 53b2222b1..6e3f078f1 100644 --- a/.github/actions/setup-rust/action.yaml +++ b/.github/actions/setup-rust/action.yaml @@ -8,6 +8,10 @@ inputs: description: Whether to save the Rust cache required: false default: 'false' + restore-cache: + description: Whether to restore the Rust cache + required: false + default: 'true' runs: using: 'composite' steps: @@ -20,8 +24,10 @@ runs: components: clippy, rustfmt - name: Cache Rust Dependencies + if: ${{ inputs.restore-cache == 'true' }} uses: Swatinem/rust-cache@v2 with: + key: ${{ inputs.target }} save-if: ${{ inputs.save-cache }} shared-key: stable-cache @@ -29,9 +35,30 @@ runs: shell: bash run: echo '{}' | npx -y mustache - .cargo/config.toml.mustache .cargo/config.toml + - name: Turn Off Debuginfo and bump opt-level + shell: bash + if: ${{ runner.os != 'Windows' }} + run: | + sed '/\[profile.dev]/a\ + debug = 0 + ' Cargo.toml > Cargo.toml.tmp && mv Cargo.toml.tmp Cargo.toml + + sed '/\[profile.dev]/a\ + opt-level=1 + ' Cargo.toml > Cargo.toml.tmp && mv Cargo.toml.tmp Cargo.toml + + - name: Turn Off Debuginfo and bump opt-level + if: ${{ runner.os == 'Windows' }} + shell: powershell + run: | + (Get-Content Cargo.toml) -replace '\[profile.dev\]', '[profile.dev] + debug = 0' | Set-Content Cargo.toml + (Get-Content Cargo.toml) -replace '\[profile.dev\]', '[profile.dev] + opt-level=1' | Set-Content Cargo.toml + - name: Restore cached Prisma codegen id: cache-prisma-restore - uses: actions/cache/restore@v3 + uses: actions/cache/restore@v4 with: key: prisma-1-${{ runner.os }}-${{ hashFiles('./core/prisma/*', './crates/sync-generator/*', './Cargo.*') }} path: crates/prisma/src/**/*.rs @@ -44,8 +71,8 @@ runs: - name: Save Prisma codegen id: cache-prisma-save - if: ${{ inputs.save-cache == 'true' }} - uses: actions/cache/save@v3 + if: ${{ steps.cache-prisma-restore.outputs.cache-hit != 'true' }} + uses: actions/cache/save@v4 with: key: ${{ steps.cache-prisma-restore.outputs.cache-primary-key }} path: crates/prisma/src/**/*.rs diff --git a/.github/actions/setup-system/action.yml b/.github/actions/setup-system/action.yml index f14eb6275..f6b46dfab 100644 --- a/.github/actions/setup-system/action.yml +++ b/.github/actions/setup-system/action.yml @@ -22,7 +22,7 @@ runs: - name: Restore cached LLVM and Clang if: ${{ runner.os == 'Windows' }} id: cache-llvm-restore - uses: actions/cache/restore@v3 + uses: actions/cache/restore@v4 with: key: llvm-15 path: C:/Program Files/LLVM @@ -37,7 +37,7 @@ runs: - name: Save LLVM and Clang if: ${{ runner.os == 'Windows' && inputs.save-cache == 'true' }} id: cache-llvm-save - uses: actions/cache/save@v3 + uses: actions/cache/save@v4 with: key: ${{ steps.cache-llvm-restore.outputs.cache-primary-key }} path: C:/Program Files/LLVM @@ -46,6 +46,13 @@ runs: if: ${{ runner.os != 'Linux' }} uses: ilammy/setup-nasm@v1 + - name: Install Mold + shell: bash + if: ${{ runner.os == 'Linux' }} + run: | + curl -L# 'https://github.com/rui314/mold/releases/download/v2.4.0/mold-2.4.0-x86_64-linux.tar.gz' \ + | sudo tar -xzf- -C /usr/local + - name: Setup Rust and Dependencies uses: ./.github/actions/setup-rust with: diff --git a/.github/workflows/cache-factory.yaml b/.github/workflows/cache-factory.yaml index 35b1ccf79..c5ddccc2a 100644 --- a/.github/workflows/cache-factory.yaml +++ b/.github/workflows/cache-factory.yaml @@ -9,12 +9,23 @@ on: - main schedule: - cron: '0 0 * * *' + pull_request: + paths: + - '.github/workflows/cache-factory.yaml' + workflow_dispatch: # Cancel previous runs of the same workflow on the same branch. concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +# From: https://github.com/rust-lang/rust-analyzer/blob/master/.github/workflows/ci.yaml +env: + CARGO_INCREMENTAL: 0 + CARGO_NET_RETRY: 10 + RUST_BACKTRACE: short + RUSTUP_MAX_RETRIES: 10 + jobs: make_cache: strategy: @@ -53,9 +64,6 @@ jobs: remove-haskell: 'true' remove-docker-images: 'true' - - name: Checkout repository - uses: actions/checkout@v3 - - name: Symlink target to C:\ if: ${{ runner.os == 'Windows' }} shell: powershell @@ -63,6 +71,9 @@ jobs: New-Item -ItemType Directory -Force -Path C:\spacedrive_target New-Item -Path target -ItemType Junction -Value C:\spacedrive_target + - name: Checkout repository + uses: actions/checkout@v4 + - name: Setup System and Rust uses: ./.github/actions/setup-system with: @@ -70,11 +81,18 @@ jobs: target: ${{ matrix.settings.target }} save-cache: 'true' - - name: Clippy - run: cargo clippy --workspace --all-features --target ${{ matrix.settings.target }} + - name: Compile tests (debug) + run: cargo test --workspace --all-features --no-run --locked --target ${{ matrix.settings.target }} + - name: Compile tests (release) + run: cargo test --workspace --all-features --no-run --locked --release --target ${{ matrix.settings.target }} + + # It's faster to `test` before `build` ¯\_(ツ)_/¯ - name: Compile (debug) - run: cargo test --workspace --all-features --no-run --target ${{ matrix.settings.target }} + run: cargo build --quiet --workspace --all-features --target ${{ matrix.settings.target }} - name: Compile (release) - run: cargo test --workspace --all-features --no-run --release --target ${{ matrix.settings.target }} + run: cargo build --quiet --workspace --all-features --release --target ${{ matrix.settings.target }} + + - name: Clippy + run: cargo clippy --workspace --all-features --target ${{ matrix.settings.target }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0d754dc47..03cdd321f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup Node.js, pnpm and dependencies uses: ./.github/actions/setup-pnpm @@ -33,7 +33,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup Node.js, pnpm and dependencies uses: ./.github/actions/setup-pnpm @@ -58,9 +58,6 @@ jobs: remove-haskell: 'true' remove-docker-images: 'true' - - name: Checkout repository - uses: actions/checkout@v3 - - name: Symlink target to C:\ if: ${{ runner.os == 'Windows' }} shell: powershell @@ -68,7 +65,10 @@ jobs: New-Item -ItemType Directory -Force -Path C:\spacedrive_target New-Item -Path target -ItemType Junction -Value C:\spacedrive_target - - uses: dorny/paths-filter@v2 + - name: Checkout repository + uses: actions/checkout@v4 + + - uses: dorny/paths-filter@v3 id: filter with: filters: | @@ -87,6 +87,8 @@ jobs: - name: Setup Rust and Prisma if: steps.filter.outputs.changes == 'true' uses: ./.github/actions/setup-rust + with: + restore-cache: 'false' - name: Run rustfmt if: steps.filter.outputs.changes == 'true' @@ -110,9 +112,6 @@ jobs: remove-haskell: 'true' remove-docker-images: 'true' - - name: Checkout repository - uses: actions/checkout@v3 - - name: Symlink target to C:\ if: ${{ runner.os == 'Windows' }} shell: powershell @@ -120,7 +119,10 @@ jobs: New-Item -ItemType Directory -Force -Path C:\spacedrive_target New-Item -Path target -ItemType Junction -Value C:\spacedrive_target - - uses: dorny/paths-filter@v2 + - name: Checkout repository + uses: actions/checkout@v4 + + - uses: dorny/paths-filter@v3 id: filter with: filters: | @@ -146,10 +148,9 @@ jobs: - name: Run Clippy if: steps.filter.outputs.changes == 'true' - uses: actions-rs/clippy-check@v1 + uses: actions-rs-plus/clippy-check@v2 with: args: --workspace --all-features - token: ${{ secrets.GITHUB_TOKEN }} # test: # name: Test (${{ matrix.platform }}) @@ -159,7 +160,7 @@ jobs: # platform: [ubuntu-20.04, macos-latest, windows-latest] # steps: # - name: Checkout repository - # uses: actions/checkout@v3 + # uses: actions/checkout@v4 # # - name: Setup # uses: ./.github/actions/setup diff --git a/.github/workflows/mobile-ci.yml b/.github/workflows/mobile-ci.yml index 598fa7cd9..df7ca272f 100644 --- a/.github/workflows/mobile-ci.yml +++ b/.github/workflows/mobile-ci.yml @@ -139,7 +139,7 @@ jobs: ios: name: iOS - runs-on: macos-12 + runs-on: macos-14 steps: - name: Checkout repository uses: actions/checkout@v3 @@ -172,7 +172,7 @@ jobs: version: latest # - name: Cache Pods - # uses: actions/cache@v3 + # uses: actions/cache@v4 # with: # path: | # ./apps/mobile/ios/Pods @@ -187,7 +187,7 @@ jobs: - name: Build iOS working-directory: ./apps/mobile/ios - run: xcodebuild -workspace ./Spacedrive.xcworkspace -scheme Spacedrive -configuration Release -sdk iphonesimulator -derivedDataPath build -arch x86_64 + run: xcodebuild -workspace ./Spacedrive.xcworkspace -scheme Spacedrive -configuration Release -sdk iphonesimulator -derivedDataPath build -arch "$(uname -m)" - name: Install Maestro run: | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 733194384..a11358236 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -52,9 +52,6 @@ jobs: remove-haskell: 'true' remove-docker-images: 'true' - - name: Checkout repository - uses: actions/checkout@v3 - - name: Symlink target to C:\ if: ${{ runner.os == 'Windows' }} shell: powershell @@ -62,6 +59,9 @@ jobs: New-Item -ItemType Directory -Force -Path C:\spacedrive_target New-Item -Path target -ItemType Junction -Value C:\spacedrive_target + - name: Checkout repository + uses: actions/checkout@v4 + - name: Remove 32-bit libs if: ${{ runner.os == 'Linux' }} run: | @@ -140,10 +140,11 @@ jobs: contents: write steps: - name: Download artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 - name: Create Release - uses: softprops/action-gh-release@v1 + # TODO: Change to stable version when available + uses: softprops/action-gh-release@4634c16 with: draft: true files: '*/**' diff --git a/.github/workflows/server.yml b/.github/workflows/server.yml index c0f191bc4..43218520b 100644 --- a/.github/workflows/server.yml +++ b/.github/workflows/server.yml @@ -25,7 +25,7 @@ jobs: remove-docker-images: 'true' - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Update buildah shell: bash @@ -54,7 +54,8 @@ jobs: - name: Build image id: build-image - uses: redhat-actions/buildah-build@v2 + # TODO: Change to stable version when available + uses: redhat-actions/buildah-build@c79846f with: tags: ${{ steps.image_info.outputs.tag }} ${{ github.event_name == 'release' && 'latest' || 'staging' }} archs: amd64 @@ -68,7 +69,9 @@ jobs: ./apps/server/docker/Dockerfile - name: Push image to ghcr.io - uses: redhat-actions/push-to-registry@v2 + # TODO: Restore redhat-actions/push-to-registry after PR is merged: + # https://github.com/redhat-actions/push-to-registry/pull/93 + uses: Eusebiotrigo/push-to-registry@5acfa47 with: tags: ${{ steps.build-image.outputs.tags }} image: ${{ steps.build-image.outputs.image }} diff --git a/.prettierignore b/.prettierignore index f9766e9ca..c07516039 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1,6 +1,9 @@ # built product/cache target/ dist/ + +# Mobile build artifacts +apps/mobile/.expo apps/mobile/android/app/build apps/mobile/modules/sd-core/android/build diff --git a/Cargo.toml b/Cargo.toml index f54358834..f960ff040 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -100,6 +100,10 @@ libp2p-core = { git = "https://github.com/spacedriveapp/rust-libp2p.git", rev = libp2p-swarm = { git = "https://github.com/spacedriveapp/rust-libp2p.git", rev = "a005656df7e82059a0eb2e333ebada4731d23f8c" } libp2p-stream = { git = "https://github.com/spacedriveapp/rust-libp2p.git", rev = "a005656df7e82059a0eb2e333ebada4731d23f8c" } +[profile.dev] +# Make compilation faster on macOS +split-debuginfo = "unpacked" + # Set the settings for build scripts and proc-macros. [profile.dev.build-override] opt-level = 3 diff --git a/apps/mobile/modules/sd-core/ios/build-rust.sh b/apps/mobile/modules/sd-core/ios/build-rust.sh index 4f0394071..32f8c9e32 100755 --- a/apps/mobile/modules/sd-core/ios/build-rust.sh +++ b/apps/mobile/modules/sd-core/ios/build-rust.sh @@ -6,6 +6,13 @@ if [ "${CI:-}" = "true" ]; then set -x fi +err() { + for _line in "$@"; do + echo "$_line" >&2 + done + exit 1 +} + if [ -z "${HOME:-}" ]; then HOME="$(CDPATH='' cd -- "$(osascript -e 'set output to (POSIX path of (path to home folder))')" && pwd -P)" export HOME @@ -28,23 +35,21 @@ fi # Required for CI and for everyone I guess? export PATH="${CARGO_HOME:-"${HOME}/.cargo"}/bin:$PATH" -# TODO: Also do this for non-Apple Silicon Macs -if [ "${SPACEDRIVE_CI:-}" = "1" ]; then - - cargo build -p sd-mobile-ios --target x86_64-apple-ios - - if [ "${PLATFORM_NAME:-}" = "iphonesimulator" ]; then - lipo -create -output "$TARGET_DIRECTORY"/libsd_mobile_iossim.a "$TARGET_DIRECTORY"/x86_64-apple-ios/debug/libsd_mobile_ios.a - else - lipo -create -output "$TARGET_DIRECTORY"/libsd_mobile_ios.a "$TARGET_DIRECTORY"/x86_64-apple-ios/debug/libsd_mobile_ios.a - fi - exit 0 -fi - if [ "${PLATFORM_NAME:-}" = "iphonesimulator" ]; then - cargo build -p sd-mobile-ios --target aarch64-apple-ios-sim - lipo -create -output "$TARGET_DIRECTORY"/libsd_mobile_iossim.a "$TARGET_DIRECTORY"/aarch64-apple-ios-sim/debug/libsd_mobile_ios.a + case "$(uname -m)" in + "arm64" | "aarch64") # M series + cargo build -p sd-mobile-ios --target aarch64-apple-ios-sim + lipo -create -output "$TARGET_DIRECTORY"/libsd_mobile_iossim.a "$TARGET_DIRECTORY"/aarch64-apple-ios-sim/debug/libsd_mobile_ios.a + ;; + "x86_64") # Intel + cargo build -p sd-mobile-ios --target x86_64-apple-ios + lipo -create -output "$TARGET_DIRECTORY"/libsd_mobile_iossim.a "$TARGET_DIRECTORY"/x86_64-apple-ios/debug/libsd_mobile_ios.a + ;; + *) + err 'Unsupported architecture.' + ;; + esac else - cargo build -p sd-mobile-ios --target aarch64-apple-ios - lipo -create -output "$TARGET_DIRECTORY"/libsd_mobile_ios.a "$TARGET_DIRECTORY"/aarch64-apple-ios/debug/libsd_mobile_ios.a + cargo build -p sd-mobile-ios --target aarch64-apple-ios --release + lipo -create -output "$TARGET_DIRECTORY"/libsd_mobile_ios.a "$TARGET_DIRECTORY"/aarch64-apple-ios/release/libsd_mobile_ios.a fi diff --git a/scripts/preprep.mjs b/scripts/preprep.mjs index fe37a5e9a..cc253c6d4 100755 --- a/scripts/preprep.mjs +++ b/scripts/preprep.mjs @@ -101,6 +101,31 @@ try { // Generate .cargo/config.toml console.log('Generating cargo config...') try { + let isWin = false + let isMacOS = false + let isLinux = false + /** @type {boolean | { linker: string }} */ + let hasLLD = false + switch (machineId[0]) { + case 'Linux': + isLinux = true + if (await which('clang')) { + if (await which('mold')) { + hasLLD = { linker: 'mold' } + } else if (await which('lld')) { + hasLLD = { linker: 'lld' } + } + } + break + case 'Darwin': + isMacOS = true + break + case 'Windows_NT': + isWin = true + hasLLD = await which('lld-link') + break + } + await fs.writeFile( path.join(__root, '.cargo', 'config.toml'), mustache @@ -109,9 +134,9 @@ try { encoding: 'utf8', }), { - isWin: machineId[0] === 'Windows_NT', - isMacOS: machineId[0] === 'Darwin', - isLinux: machineId[0] === 'Linux', + isWin, + isMacOS, + isLinux, // Escape windows path separator to be compatible with TOML parsing protoc: path .join( @@ -121,6 +146,7 @@ try { ) .replaceAll('\\', '\\\\'), nativeDeps: nativeDeps.replaceAll('\\', '\\\\'), + hasLLD, } ) .replace(/\n\n+/g, '\n'),