Build, Sign & Notarise macOS builds (#486)

.github/workflows/build_and_deploy.yaml

@@ -0,0 +1,77 @@
+name: Build and Deploy
+on:
+  # Nightly build
+  schedule:
+    - cron: '0 9 * * *'
+  # Manual nightly & release
+  workflow_dispatch:
+    inputs:
+      mode:
+        description: What type of build to trigger. Release builds should be ran from the `master` branch.
+        required: true
+        default: nightly
+        type: choice
+        options:
+          - nightly
+          - release
+      macos:
+        description: Whether to build macOS
+        required: true
+        type: boolean
+        default: true
+      deploy:
+        description: Whether to deploy artifacts
+        required: true
+        type: boolean
+        default: true
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+env:
+  # XXX: UPDATE THIS BEFORE WHEN GOING LIVE
+  R2_BUCKET: 'packages-element-io-test'
+jobs:
+  prepare:
+    uses: ./.github/workflows/build_prepare.yaml
+    with:
+      config: element.io/${{ inputs.mode || 'nightly' }}
+      version: ${{ inputs.mode == 'release' && '' || 'develop' }}
+      calculate-nightly-versions: ${{ inputs.mode != 'release' }}
+    secrets:
+      CF_R2_ACCESS_KEY_ID: ${{ secrets.CF_R2_ACCESS_KEY_ID }}
+      CF_R2_TOKEN: ${{ secrets.CF_R2_TOKEN }}
+      CF_R2_S3_API: ${{ secrets.CF_R2_S3_API }}
+
+  macos:
+    if: github.event_name != 'workflow_dispatch' || inputs.macos
+    needs: prepare
+    name: macOS
+    uses: ./.github/workflows/build_macos.yaml
+    secrets: inherit
+    with:
+      sign: true
+      deploy-mode: true
+      base-url: https://packages.element.io/${{ inputs.mode == 'release' && 'desktop' || 'nightly' }}
+      version: ${{ needs.prepare.outputs.macos-version }}
+
+  deploy:
+    needs:
+      - macos
+    runs-on: ubuntu-latest
+    name: Deploy
+    if: always() && (github.event != 'workflow_dispatch' || inputs.deploy)
+    environment: packages.element.io
+    steps:
+      - name: Download artifacts
+        uses: actions/download-artifact@v3
+        with:
+          name: packages.element.io
+          path: packages.element.io
+
+      - name: Deploy artifacts
+        run: aws s3 cp --recursive packages.element.io/ s3://$R2_BUCKET/$DEPLOYMENT_DIR --endpoint-url $R2_URL --region auto
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.CF_R2_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_TOKEN }}
+          R2_URL: ${{ secrets.CF_R2_S3_API }}
+          DEPLOYMENT_DIR: ${{ inputs.mode == 'release' && 'desktop' || 'nightly' }}

.github/workflows/build_and_test.yaml

@@ -2,7 +2,7 @@ name: Build and Test
 on:
   pull_request: {}
   push:
-    branches: [develop, master]
+    branches: [develop, staging, master]
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true

.github/workflows/build_macos.yaml

@@ -3,9 +3,38 @@
 # the correct cache scoping, and additional care must be taken to not run untrusted actions on the develop branch.
 on:
     workflow_call:
+        secrets:
+            APPLE_ID:
+                required: false
+            APPLE_ID_PASSWORD:
+                required: false
+            APPLE_TEAM_ID:
+                required: false
+            APPLE_CSC_KEY_PASSWORD:
+                required: false
+            APPLE_CSC_LINK:
+                required: false
+        inputs:
+            version:
+                type: string
+                required: false
+                description: "Version string to override the one in package.json, used for non-release builds"
+            sign:
+                type: string
+                required: false
+                description: "Whether to sign & notarise the build, requires 'packages.element.io' environment"
+            deploy-mode:
+                type: string
+                required: false
+                description: "Whether to arrange artifacts in the arrangement needed for deployment, skipping unrelated ones"
+            base-url:
+                type: string
+                required: false
+                description: "The URL to which the output will be deployed, required if deploy-mode is enabled."
 jobs:
     build:
         runs-on: macos-latest
+        environment: ${{ inputs.sign && 'packages.element.io' || '' }}
         steps:
             - uses: actions/checkout@v3
 
@@ -40,12 +69,68 @@ jobs:
               if: steps.cache.outputs.cache-hit != 'true'
               run: "yarn build:native:universal"
 
+            - name: '[Nightly] Resolve version'
+              id: nightly
+              if: inputs.version != ''
+              run: |
+                  echo "config-args=--nightly '${{ inputs.version }}'" >> $GITHUB_OUTPUT
+
             - name: Build App
-              run: "yarn build:universal --publish never"
+              run: |
+                  scripts/generate-builder-config.ts ${{ steps.nightly.outputs.config-args }}
+                  yarn build:universal --publish never --config electron-builder.json
+              env:
+                  NOTARIZE_APPLE_ID: ${{ secrets.APPLE_ID }}
+                  NOTARIZE_APPLE_ID_PASSWORD: ${{ secrets.APPLE_ID_PASSWORD }}
+                  NOTARIZE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }}
+                  CSC_KEY_PASSWORD: ${{ secrets.APPLE_CSC_KEY_PASSWORD }}
+                  CSC_LINK: ${{ secrets.APPLE_CSC_LINK }}
+
+            - name: Prepare artifacts for deployment
+              if: inputs.deploy-mode
+              run: |
+                  mv dist _dist
+                  mkdir -p dist/install/macos dist/update/macos
+                  mv _dist/*-mac.zip dist/update/macos/
+                  mv _dist/*.dmg dist/install/macos/
+        
+                  PKG_JSON_VERSION=$(cat package.json | jq -r .version)
+                  LATEST=$(find dist -type f -iname "*-mac.zip" | xargs -0 -n1 -- basename)
+                  URL="${{ inputs.base-url }}/update/macos/$LATEST"
+        
+                  jq -n --arg version "${VERSION:-$PKG_JSON_VERSION}" --arg url "$URL" '
+                    {
+                      currentRelease: $version,
+                      releases: [{
+                        version: $version,
+                        updateTo: {
+                          version: $version,
+                          url: $url,
+                        },
+                      }],
+                    }
+                  ' > dist/update/macos/releases.json
+                  jq -n --arg url "$URL" '
+                    { url: $url }
+                  ' > dist/update/macos/releases-legacy.json
+              env:
+                  VERSION: ${{ inputs.version }}
+
+            # We don't wish to store the installer for every nightly ever, so we only keep the latest
+            - name: '[Nightly] Strip version from installer file'
+              if: inputs.deploy-mode && inputs.version != ''
+              run: |
+                  mv dist/install/macos/*.dmg "dist/install/macos/Element Nightly.dmg"
+
+            - name: '[Release] Prepare release latest symlink'
+              if: inputs.deploy-mode && inputs.version == ''
+              run: |
+                  LATEST=$(find dist -type f -iname "*.dmg" | xargs -0 -n1 -- basename)
+                  ln -s "dist/install/macos/$LATEST" dist/install/macos/Element.dmg
 
             - name: Upload Artifacts
               uses: actions/upload-artifact@v3
               with:
-                  name: macos
+                  name: ${{ inputs.deploy-mode && 'packages.element.io' || 'macos' }}
                   path: dist
                   retention-days: 1

.github/workflows/build_prepare.yaml

@@ -1,3 +1,4 @@
+# This action helps perform common actions before the build_* actions are started in parallel.
 on:
     workflow_call:
         inputs:
@@ -9,10 +10,31 @@ on:
                 type: string
                 required: false
                 description: "The version tag to fetch, or 'develop', will pick automatically if not passed"
+            calculate-nightly-versions:
+                type: string
+                required: false
+                description: "Whether to calculate the version strings new Nightly builds should use"
+        secrets:
+            # Required if `calculate-nightly-versions` is set
+            CF_R2_ACCESS_KEY_ID:
+                required: false
+            # Required if `calculate-nightly-versions` is set
+            CF_R2_TOKEN:
+                required: false
+            # Required if `calculate-nightly-versions` is set
+            CF_R2_S3_API:
+                required: false
+        outputs:
+            macos-version:
+                description: "The version string the next macOS Nightly should use, only output for calculate-nightly-versions"
+                value: ${{ jobs.prepare.outputs.macos-version }}
 jobs:
     prepare:
         name: Prepare
+        environment: ${{ inputs.calculate-nightly-versions && 'packages.element.io' || '' }}
         runs-on: ubuntu-latest
+        outputs:
+            macos-version: ${{ steps.macos.outputs.version }}
         steps:
             - uses: actions/checkout@v3
 
@@ -41,3 +63,14 @@ jobs:
                       package.json
                       electronVersion
                       hakDependencies.json
+
+            - name: Calculate macOS Nightly version
+              id: macos
+              if: inputs.calculate-nightly-versions
+              run: |
+                  LATEST=$(aws s3 cp s3://$R2_BUCKET/nightly/update/macos/releases.json - --endpoint-url $R2_URL --region auto | jq -r .currentRelease)
+                  echo "version=$(scripts/generate-nightly-version.ts --latest $LATEST)" >> $GITHUB_OUTPUT
+              env:
+                  AWS_ACCESS_KEY_ID: ${{ secrets.CF_R2_ACCESS_KEY_ID }}
+                  AWS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_TOKEN }}
+                  R2_URL: ${{ secrets.CF_R2_S3_API }}

.github/workflows/packages_index.yaml

@@ -5,14 +5,20 @@ on:
         branches: [develop]
         paths:
             - "packages.element.io/**"
-    # Trigger a daily rebuild for nightlies
+    # Trigger a daily rebuild for (mac-mini built) Nightly builds
     schedule:
         - cron: "0 11 * * *"
+  # Trigger after Nightly builds are deployed
+    workflow_run:
+        workflows: [ "Build and Deploy" ]
+        types:
+            - completed
     # Manual trigger for rebuilding for releases
     workflow_dispatch: {}
 jobs:
     deploy:
         name: "Deploy"
+        if: github.event != 'workflow_run' || github.event.workflow_run.conclusion == 'success'
         runs-on: ubuntu-latest
         environment: packages.element.io
         env:

scripts/electron_afterSign.js

@@ -8,12 +8,21 @@ exports.default = async function (context) {
     if (electronPlatformName === "darwin") {
         const appName = context.packager.appInfo.productFilename;
 
-        const keychainProfile = process.env.NOTARIZE_KEYCHAIN_PROFILE;
-        if (keychainProfile === undefined) {
+        const notarizeToolCredentials = {};
+        if (process.env.NOTARIZE_KEYCHAIN_PROFILE) {
+            notarizeToolCredentials.keychainProfile = process.env.NOTARIZE_KEYCHAIN_PROFILE;
+            notarizeToolCredentials.keychain = process.env.NOTARIZE_KEYCHAIN;
+        } if (process.env.NOTARIZE_APPLE_ID && process.env.NOTARIZE_APPLE_ID_PASSWORD && process.env.NOTARIZE_TEAM_ID) {
+            notarizeToolCredentials.appleId = process.env.NOTARIZE_APPLE_ID;
+            notarizeToolCredentials.appleIdPassword = process.env.NOTARIZE_APPLE_ID_PASSWORD;
+            notarizeToolCredentials.teamId = process.env.NOTARIZE_TEAM_ID;
+        } else {
             if (!warned) {
                 console.log("*****************************************");
-                console.log("* NOTARIZE_KEYCHAIN_PROFILE is not set. *");
                 console.log("*   This build will NOT be notarised.   *");
+                console.log("* Provide NOTARIZE_KEYCHAIN_PROFILE or  *");
+                console.log("* NOTARIZE_APPLE_ID, NOTARIZE_TEAM_ID   *");
+                console.log("* and NOTARIZE_APPLE_ID_PASSWORD        *");
                 console.log("*****************************************");
                 warned = true;
             }
@@ -25,8 +34,7 @@ exports.default = async function (context) {
             tool: "notarytool",
             appBundleId: appId,
             appPath: `${appOutDir}/${appName}.app`,
-            keychainProfile,
-            keychain: process.env.NOTARIZE_KEYCHAIN,
+            ...notarizeToolCredentials,
         });
     }
 };

scripts/generate-builder-config.ts

@@ -0,0 +1,131 @@
+#!/usr/bin/env -S npx ts-node
+
+/**
+ * Script to generate electron-builder.json config files for builds which don't match package.json, e.g. nightlies
+ * This script has different outputs depending on your os platform.
+ *
+ * On Windows:
+ *  Prefixes the nightly version with `0.0.1-nightly.` as it breaks if it is not semver
+ *
+ * On Linux:
+ *  Replaces spaces in the product name with dashes as spaces in paths can cause issues
+ *  Passes --deb-custom-control to build.deb.fpm if specified
+ */
+
+import parseArgs from "minimist";
+import fsProm from "fs/promises";
+import * as os from "os";
+
+const ELECTRON_BUILDER_CFG_FILE = "electron-builder.json";
+
+const NIGHTLY_APP_ID = "im.riot.nightly";
+const NIGHTLY_APP_NAME = "element-desktop-nightly";
+
+const argv = parseArgs<{
+    nightly?: string;
+    "deb-custom-control"?: string;
+}>(process.argv.slice(2), {
+    string: ["nightly", "deb-custom-control"],
+});
+
+interface File {
+    from: string;
+    to: string;
+}
+
+interface PackageBuild {
+    appId: string;
+    asarUnpack: string;
+    files: Array<string | File>;
+    extraResources: Array<string | File>;
+    linux: {
+        target: string;
+        category: string;
+        maintainer: string;
+        desktop: {
+            StartupWMClass: string;
+        };
+    };
+    mac: {
+        category: string;
+        darkModeSupport: boolean;
+    };
+    win: {
+        target: {
+            target: string;
+        };
+        sign: string;
+    };
+    deb?: {
+        fpm?: string[];
+    };
+    directories: {
+        output: string;
+    };
+    afterPack: string;
+    afterSign: string;
+    protocols: Array<{
+        name: string;
+        schemes: string[];
+    }>;
+    extraMetadata?: {
+        productName?: string;
+        name?: string;
+        version?: string;
+    };
+}
+
+interface Package {
+    build: PackageBuild;
+    productName: string;
+}
+
+async function main(): Promise<number | void> {
+    // Electron builder doesn't overlay with the config in package.json, so load it here
+    const pkg: Package = JSON.parse(await fsProm.readFile("package.json", "utf8"));
+
+    const cfg: PackageBuild = {
+        ...pkg.build,
+        extraMetadata: {
+            productName: pkg.productName,
+        },
+    };
+
+    if (argv.nightly) {
+        cfg.appId = NIGHTLY_APP_ID;
+        cfg.extraMetadata!.productName += " Nightly";
+        cfg.extraMetadata!.name = NIGHTLY_APP_NAME;
+
+        let version = argv.nightly;
+        if (os.platform() === "win32") {
+            // The windows packager relies on parsing this as semver, so we have to make it look like one.
+            // This will give our update packages really stupid names, but we probably can't change that either
+            // because squirrel windows parses them for the version too. We don't really care: nobody sees them.
+            // We just give the installer a static name, so you'll just see this in the 'about' dialog.
+            // Turns out if you use 0.0.0 here it makes Squirrel windows crash, so we use 0.0.1.
+            version = "0.0.1-nightly." + version;
+        }
+        cfg.extraMetadata!.version = version;
+    }
+
+    if (os.platform() === "linux") {
+        // Electron crashes on debian if there's a space in the path.
+        // https://github.com/vector-im/element-web/issues/13171
+        cfg.extraMetadata!.productName = cfg.extraMetadata!.productName!.replace(/ /g, "-");
+
+        if (argv["deb-custom-control"]) {
+            cfg.deb = {
+                fpm: [`--deb-custom-control=${argv["deb-custom-control"]}`],
+            };
+        }
+    }
+
+    await fsProm.writeFile(ELECTRON_BUILDER_CFG_FILE, JSON.stringify(cfg, null, 4));
+}
+
+main().then((ret) => {
+    process.exit(ret!);
+}).catch((e) => {
+    console.error(e);
+    process.exit(1);
+});

scripts/generate-nightly-version.ts

@@ -0,0 +1,41 @@
+#!/usr/bin/env -S npx ts-node
+
+/**
+ * Script to generate incremental Nightly build versions, based on the latest Nightly build version of that kind.
+ * The version format is YYYYMMDDNN where NN is in case we need to do multiple versions in a day.
+ *
+ * NB. on windows, squirrel will try to parse the version number parts, including this string, into 32-bit integers,
+ * which is fine as long as we only add two digits to the end...
+ */
+
+import parseArgs from "minimist";
+
+const argv = parseArgs<{
+    latest?: string;
+}>(process.argv.slice(2), {
+    string: ["latest"],
+});
+
+function parseVersion(version: string): [Date, number] {
+    const year = parseInt(version.slice(0, 4), 10);
+    const month = parseInt(version.slice(4, 2), 10);
+    const day = parseInt(version.slice(6, 2), 10);
+    const num = parseInt(version.slice(8, 2), 10);
+    return [new Date(year, month - 1, day), num];
+}
+
+const [latestDate, latestNum] = argv.latest ? parseVersion(argv.latest) : [];
+
+const now = new Date();
+const month = (now.getMonth() + 1).toString().padStart(2, '0');
+const date = now.getDate().toString().padStart(2, '0');
+let buildNum = 1;
+if (latestDate && new Date(latestDate).getDate().toString().padStart(2, '0') === date) {
+    buildNum = latestNum! + 1;
+}
+
+if (buildNum > 99) {
+    throw new Error("Maximum number of Nightlies exceeded on this day.");
+}
+
+console.log(now.getFullYear() + month + date + buildNum.toString().padStart(2, '0') + buildNum);