Make mobile tests more resilient (#2236)

* Attempt at making mobile tests more resilient

* Forgot shell property

* Remove debug flag from run-maestro-tests.sh

* Fix kill command

* Improve device emulator launch code
 - Remove maestro version lock (expo is not using it anymore on their repo)
 - Use some of the same command expo uses to ensure device emulator launched

* ...

* ...

* Make grep timeout less strict

* Increase timeouts

* Downgrade macOS and Xcode

Following Expo changes:
cb7c90dac4

.github/actions/setup-system/action.yml

@@ -42,6 +42,11 @@ runs:
         key: ${{ steps.cache-llvm-restore.outputs.cache-primary-key }}
         path: C:/Program Files/LLVM
 
+    - name: Install current Bash on macOS
+      shell: bash
+      if: runner.os == 'macOS'
+      run: brew install bash
+
     - name: Install Nasm
       if: ${{ runner.os != 'Linux' }}
       uses: ilammy/setup-nasm@v1

.github/workflows/mobile-ci.yml

@@ -138,15 +138,15 @@ jobs:
 
   ios:
     name: iOS
-    runs-on: macos-14
+    runs-on: macos-12
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
 
-      - name: Install Xcode
-        uses: maxim-lobanov/setup-xcode@v1
-        with:
-          xcode-version: latest-stable
+      # - name: Install Xcode
+      #   uses: maxim-lobanov/setup-xcode@v1
+      #   with:
+      #     xcode-version: latest-stable
 
       - name: Setup System and Rust
         uses: ./.github/actions/setup-system
@@ -189,9 +189,6 @@ jobs:
         run: xcodebuild -workspace ./Spacedrive.xcworkspace -scheme Spacedrive -configuration Release -sdk iphonesimulator -derivedDataPath build -arch "$(uname -m)"
 
       - name: Install Maestro
-        env:
-          # Workaround: https://github.com/mobile-dev-inc/maestro/issues/1585
-          MAESTRO_VERSION: '1.33.1'
         run: |
           curl -Ls "https://get.maestro.mobile.dev" | bash
           brew tap facebook/fb
@@ -199,16 +196,12 @@ jobs:
           echo "${HOME}/.maestro/bin" >> $GITHUB_PATH
 
       - name: Run Simulator
+        id: run_simulator
         uses: futureware-tech/simulator-action@v3
         with:
-          model: 'iPhone 15'
-          os_version: 17
+          model: 'iPhone 14'
+          os_version: 16
           erase_before_boot: false
 
       - name: Run Tests
-        env:
-          # https://github.com/expo/expo/blob/339fa68/apps/bare-expo/scripts/start-ios-e2e-test.ts#L12
-          MAESTRO_DRIVER_STARTUP_TIMEOUT: 120000
-        run: |
-          xcrun simctl install booted apps/mobile/ios/build/Build/Products/Release-iphonesimulator/Spacedrive.app
-          ./apps/mobile/scripts/run-maestro-tests.sh ios
+        run: ./apps/mobile/scripts/run-maestro-tests.sh ios ${{ steps.run_simulator.outputs.udid }}

apps/mobile/scripts/run-maestro-tests.sh

@@ -2,23 +2,70 @@
 
 set -eEuo pipefail
 
-if [ "${CI:-}" = "true" ]; then
-  set -x
-fi
-
 # Script root
 _root="$(CDPATH='' cd -- "$(dirname "$0")" && pwd -P)"
 _test_dir="$(CDPATH='' cd -- "${_root}/../tests" && pwd -P)"
 
-PLATFORM=${1:-}
+PLATFORM="${1:-}"
+DEVICE_ID=""
+IOS_APP_BIN_PATH="${_root}/../ios/build/Build/Products/Release-iphonesimulator/Spacedrive.app"
 case $PLATFORM in
-  ios | android) ;;
+  ios)
+    DEVICE_ID="${2:-}"
+    if [ -z "$DEVICE_ID" ]; then
+      echo "Empty IOS emulator UUID" >&2
+      exit 1
+    fi
+
+    if ! [ -e "$IOS_APP_BIN_PATH" ]; then
+      echo "Invalid IOS app binary path" >&2
+      exit 1
+    fi
+    ;;
+  android)
+    echo 'Android tests are not implemented yet' >&2
+    exit 1
+    ;;
   *)
     echo "Usage: run-maestro-tests.sh <android|ios>" >&2
     exit 1
     ;;
 esac
 
+start_app() {
+  case $PLATFORM in
+    ios)
+      xcrun simctl bootstatus "$DEVICE_ID" -b
+      open -a Simulator --args -CurrentDeviceUDID "$DEVICE_ID"
+      xcrun simctl install "$DEVICE_ID" "${_root}/../ios/build/Build/Products/Release-iphonesimulator/Spacedrive.app"
+      # ¯\_(ツ)_/¯
+      sleep 10
+      ;;
+    android)
+      echo 'Android tests are not implemented yet' >&2
+      exit 1
+      ;;
+  esac
+}
+
+# https://stackoverflow.com/q/11027679#answer-59592881
+# SYNTAX:
+#   catch STDOUT_VARIABLE STDERR_VARIABLE COMMAND [ARG1[ ARG2[ ...[ ARGN]]]]
+catch() {
+  {
+    IFS=$'\n' read -r -d '' "${1}"
+    IFS=$'\n' read -r -d '' "${2}"
+    (
+      IFS=$'\n' read -r -d '' _ERRNO_
+      return "$_ERRNO_"
+    )
+  } < <((printf '\0%s\0%d\0' "$( ( ( ({
+    shift 2
+    "${@}"
+    echo "${?}" 1>&3-
+  } | tr -d '\0' 1>&4-) 4>&2- 2>&1- | tr -d '\0' 1>&4-) 3>&1- | exit "$(cat)") 4>&1-)" "${?}" 1>&2) 2>&1)
+}
+
 run_maestro_test() {
   if [ $# -ne 1 ]; then
     echo "Usage: run_maestro_test <test_file>" >&2
@@ -28,13 +75,43 @@ run_maestro_test() {
   local i
   local retry_seconds
   for i in {1..6}; do
-    if maestro test "$1"; then
+    _maestro_out=''
+    _maestro_err=''
+
+    # https://github.com/expo/expo/blob/339fa68/apps/bare-expo/scripts/start-ios-e2e-test.ts#L12
+    if catch _maestro_out _maestro_err \
+      env MAESTRO_DRIVER_STARTUP_TIMEOUT=120000 maestro --device "$DEVICE_ID" test "$1"; then
       # Test succeeded
+      printf '%s' "$_maestro_out"
+      printf '%s' "$_maestro_err" >&2
       return
-    else
+    elif echo "$_maestro_err" | grep 'TimeoutException'; then
+      # Test timed out
+      # Kill maestro processes
+      pgrep -fi maestro | xargs kill -KILL
+
+      # Restart app if necessary
+      case $PLATFORM in
+        ios)
+          if ! { xcrun simctl listapps booted | grep CFBundleIdentifier | grep Spacedrive; }; then
+            start_app
+          fi
+          ;;
+        android)
+          echo 'Android tests are not implemented yet' >&2
+          exit 1
+          ;;
+      esac
+
+      # Retry
       retry_seconds=$((20 * i))
-      echo "Test $1 failed. Retrying in $retry_seconds seconds..."
+      echo "Test $1 timed out. Retrying in $retry_seconds seconds..."
       sleep $retry_seconds
+    else
+      # Test failed
+      printf '%s' "$_maestro_out"
+      printf '%s' "$_maestro_err" >&2
+      return 1
     fi
   done
 
@@ -57,6 +134,9 @@ else
   )
 fi
 
+# Start Spacedrive in the device emulator
+start_app
+
 # Run onboarding first
 onboardingFile="${_test_dir}/onboarding.yml"
 if ! run_maestro_test "$onboardingFile"; then