From e5117f97365af45ebff541d3031eeea3276d8c9f Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Tue, 31 Jan 2023 13:22:30 +0000 Subject: [PATCH] Build, Sign & Notarise macOS builds (#486) --- .github/workflows/build_and_deploy.yaml | 77 ++++++++++ .../{build.yaml => build_and_test.yaml} | 2 +- .github/workflows/build_macos.yaml | 89 +++++++++++- .github/workflows/build_prepare.yaml | 33 +++++ .github/workflows/packages_index.yaml | 8 +- scripts/electron_afterSign.js | 18 ++- scripts/generate-builder-config.ts | 131 ++++++++++++++++++ scripts/generate-nightly-version.ts | 41 ++++++ 8 files changed, 390 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/build_and_deploy.yaml rename .github/workflows/{build.yaml => build_and_test.yaml} (98%) create mode 100755 scripts/generate-builder-config.ts create mode 100755 scripts/generate-nightly-version.ts diff --git a/.github/workflows/build_and_deploy.yaml b/.github/workflows/build_and_deploy.yaml new file mode 100644 index 0000000..fc551cd --- /dev/null +++ b/.github/workflows/build_and_deploy.yaml @@ -0,0 +1,77 @@ +name: Build and Deploy +on: + # Nightly build + schedule: + - cron: '0 9 * * *' + # Manual nightly & release + workflow_dispatch: + inputs: + mode: + description: What type of build to trigger. Release builds should be ran from the `master` branch. + required: true + default: nightly + type: choice + options: + - nightly + - release + macos: + description: Whether to build macOS + required: true + type: boolean + default: true + deploy: + description: Whether to deploy artifacts + required: true + type: boolean + default: true +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true +env: + # XXX: UPDATE THIS BEFORE WHEN GOING LIVE + R2_BUCKET: 'packages-element-io-test' +jobs: + prepare: + uses: ./.github/workflows/build_prepare.yaml + with: + config: element.io/${{ inputs.mode || 'nightly' }} + version: ${{ inputs.mode == 'release' && '' || 'develop' }} + calculate-nightly-versions: ${{ inputs.mode != 'release' }} + secrets: + CF_R2_ACCESS_KEY_ID: ${{ secrets.CF_R2_ACCESS_KEY_ID }} + CF_R2_TOKEN: ${{ secrets.CF_R2_TOKEN }} + CF_R2_S3_API: ${{ secrets.CF_R2_S3_API }} + + macos: + if: github.event_name != 'workflow_dispatch' || inputs.macos + needs: prepare + name: macOS + uses: ./.github/workflows/build_macos.yaml + secrets: inherit + with: + sign: true + deploy-mode: true + base-url: https://packages.element.io/${{ inputs.mode == 'release' && 'desktop' || 'nightly' }} + version: ${{ needs.prepare.outputs.macos-version }} + + deploy: + needs: + - macos + runs-on: ubuntu-latest + name: Deploy + if: always() && (github.event != 'workflow_dispatch' || inputs.deploy) + environment: packages.element.io + steps: + - name: Download artifacts + uses: actions/download-artifact@v3 + with: + name: packages.element.io + path: packages.element.io + + - name: Deploy artifacts + run: aws s3 cp --recursive packages.element.io/ s3://$R2_BUCKET/$DEPLOYMENT_DIR --endpoint-url $R2_URL --region auto + env: + AWS_ACCESS_KEY_ID: ${{ secrets.CF_R2_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_TOKEN }} + R2_URL: ${{ secrets.CF_R2_S3_API }} + DEPLOYMENT_DIR: ${{ inputs.mode == 'release' && 'desktop' || 'nightly' }} diff --git a/.github/workflows/build.yaml b/.github/workflows/build_and_test.yaml similarity index 98% rename from .github/workflows/build.yaml rename to .github/workflows/build_and_test.yaml index 77400c0..9c5ec04 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build_and_test.yaml @@ -2,7 +2,7 @@ name: Build and Test on: pull_request: {} push: - branches: [develop, master] + branches: [develop, staging, master] concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true diff --git a/.github/workflows/build_macos.yaml b/.github/workflows/build_macos.yaml index 1a82a07..3830e90 100644 --- a/.github/workflows/build_macos.yaml +++ b/.github/workflows/build_macos.yaml @@ -3,9 +3,38 @@ # the correct cache scoping, and additional care must be taken to not run untrusted actions on the develop branch. on: workflow_call: + secrets: + APPLE_ID: + required: false + APPLE_ID_PASSWORD: + required: false + APPLE_TEAM_ID: + required: false + APPLE_CSC_KEY_PASSWORD: + required: false + APPLE_CSC_LINK: + required: false + inputs: + version: + type: string + required: false + description: "Version string to override the one in package.json, used for non-release builds" + sign: + type: string + required: false + description: "Whether to sign & notarise the build, requires 'packages.element.io' environment" + deploy-mode: + type: string + required: false + description: "Whether to arrange artifacts in the arrangement needed for deployment, skipping unrelated ones" + base-url: + type: string + required: false + description: "The URL to which the output will be deployed, required if deploy-mode is enabled." jobs: build: runs-on: macos-latest + environment: ${{ inputs.sign && 'packages.element.io' || '' }} steps: - uses: actions/checkout@v3 @@ -40,12 +69,68 @@ jobs: if: steps.cache.outputs.cache-hit != 'true' run: "yarn build:native:universal" + - name: '[Nightly] Resolve version' + id: nightly + if: inputs.version != '' + run: | + echo "config-args=--nightly '${{ inputs.version }}'" >> $GITHUB_OUTPUT + - name: Build App - run: "yarn build:universal --publish never" + run: | + scripts/generate-builder-config.ts ${{ steps.nightly.outputs.config-args }} + yarn build:universal --publish never --config electron-builder.json + env: + NOTARIZE_APPLE_ID: ${{ secrets.APPLE_ID }} + NOTARIZE_APPLE_ID_PASSWORD: ${{ secrets.APPLE_ID_PASSWORD }} + NOTARIZE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }} + CSC_KEY_PASSWORD: ${{ secrets.APPLE_CSC_KEY_PASSWORD }} + CSC_LINK: ${{ secrets.APPLE_CSC_LINK }} + + - name: Prepare artifacts for deployment + if: inputs.deploy-mode + run: | + mv dist _dist + mkdir -p dist/install/macos dist/update/macos + mv _dist/*-mac.zip dist/update/macos/ + mv _dist/*.dmg dist/install/macos/ + + PKG_JSON_VERSION=$(cat package.json | jq -r .version) + LATEST=$(find dist -type f -iname "*-mac.zip" | xargs -0 -n1 -- basename) + URL="${{ inputs.base-url }}/update/macos/$LATEST" + + jq -n --arg version "${VERSION:-$PKG_JSON_VERSION}" --arg url "$URL" ' + { + currentRelease: $version, + releases: [{ + version: $version, + updateTo: { + version: $version, + url: $url, + }, + }], + } + ' > dist/update/macos/releases.json + jq -n --arg url "$URL" ' + { url: $url } + ' > dist/update/macos/releases-legacy.json + env: + VERSION: ${{ inputs.version }} + + # We don't wish to store the installer for every nightly ever, so we only keep the latest + - name: '[Nightly] Strip version from installer file' + if: inputs.deploy-mode && inputs.version != '' + run: | + mv dist/install/macos/*.dmg "dist/install/macos/Element Nightly.dmg" + + - name: '[Release] Prepare release latest symlink' + if: inputs.deploy-mode && inputs.version == '' + run: | + LATEST=$(find dist -type f -iname "*.dmg" | xargs -0 -n1 -- basename) + ln -s "dist/install/macos/$LATEST" dist/install/macos/Element.dmg - name: Upload Artifacts uses: actions/upload-artifact@v3 with: - name: macos + name: ${{ inputs.deploy-mode && 'packages.element.io' || 'macos' }} path: dist retention-days: 1 diff --git a/.github/workflows/build_prepare.yaml b/.github/workflows/build_prepare.yaml index c785a2a..c974a91 100644 --- a/.github/workflows/build_prepare.yaml +++ b/.github/workflows/build_prepare.yaml @@ -1,3 +1,4 @@ +# This action helps perform common actions before the build_* actions are started in parallel. on: workflow_call: inputs: @@ -9,10 +10,31 @@ on: type: string required: false description: "The version tag to fetch, or 'develop', will pick automatically if not passed" + calculate-nightly-versions: + type: string + required: false + description: "Whether to calculate the version strings new Nightly builds should use" + secrets: + # Required if `calculate-nightly-versions` is set + CF_R2_ACCESS_KEY_ID: + required: false + # Required if `calculate-nightly-versions` is set + CF_R2_TOKEN: + required: false + # Required if `calculate-nightly-versions` is set + CF_R2_S3_API: + required: false + outputs: + macos-version: + description: "The version string the next macOS Nightly should use, only output for calculate-nightly-versions" + value: ${{ jobs.prepare.outputs.macos-version }} jobs: prepare: name: Prepare + environment: ${{ inputs.calculate-nightly-versions && 'packages.element.io' || '' }} runs-on: ubuntu-latest + outputs: + macos-version: ${{ steps.macos.outputs.version }} steps: - uses: actions/checkout@v3 @@ -41,3 +63,14 @@ jobs: package.json electronVersion hakDependencies.json + + - name: Calculate macOS Nightly version + id: macos + if: inputs.calculate-nightly-versions + run: | + LATEST=$(aws s3 cp s3://$R2_BUCKET/nightly/update/macos/releases.json - --endpoint-url $R2_URL --region auto | jq -r .currentRelease) + echo "version=$(scripts/generate-nightly-version.ts --latest $LATEST)" >> $GITHUB_OUTPUT + env: + AWS_ACCESS_KEY_ID: ${{ secrets.CF_R2_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_TOKEN }} + R2_URL: ${{ secrets.CF_R2_S3_API }} diff --git a/.github/workflows/packages_index.yaml b/.github/workflows/packages_index.yaml index 4e82a8c..61b55fd 100644 --- a/.github/workflows/packages_index.yaml +++ b/.github/workflows/packages_index.yaml @@ -5,14 +5,20 @@ on: branches: [develop] paths: - "packages.element.io/**" - # Trigger a daily rebuild for nightlies + # Trigger a daily rebuild for (mac-mini built) Nightly builds schedule: - cron: "0 11 * * *" + # Trigger after Nightly builds are deployed + workflow_run: + workflows: [ "Build and Deploy" ] + types: + - completed # Manual trigger for rebuilding for releases workflow_dispatch: {} jobs: deploy: name: "Deploy" + if: github.event != 'workflow_run' || github.event.workflow_run.conclusion == 'success' runs-on: ubuntu-latest environment: packages.element.io env: diff --git a/scripts/electron_afterSign.js b/scripts/electron_afterSign.js index aa3c1d4..5b2a2a1 100644 --- a/scripts/electron_afterSign.js +++ b/scripts/electron_afterSign.js @@ -8,12 +8,21 @@ exports.default = async function (context) { if (electronPlatformName === "darwin") { const appName = context.packager.appInfo.productFilename; - const keychainProfile = process.env.NOTARIZE_KEYCHAIN_PROFILE; - if (keychainProfile === undefined) { + const notarizeToolCredentials = {}; + if (process.env.NOTARIZE_KEYCHAIN_PROFILE) { + notarizeToolCredentials.keychainProfile = process.env.NOTARIZE_KEYCHAIN_PROFILE; + notarizeToolCredentials.keychain = process.env.NOTARIZE_KEYCHAIN; + } if (process.env.NOTARIZE_APPLE_ID && process.env.NOTARIZE_APPLE_ID_PASSWORD && process.env.NOTARIZE_TEAM_ID) { + notarizeToolCredentials.appleId = process.env.NOTARIZE_APPLE_ID; + notarizeToolCredentials.appleIdPassword = process.env.NOTARIZE_APPLE_ID_PASSWORD; + notarizeToolCredentials.teamId = process.env.NOTARIZE_TEAM_ID; + } else { if (!warned) { console.log("*****************************************"); - console.log("* NOTARIZE_KEYCHAIN_PROFILE is not set. *"); console.log("* This build will NOT be notarised. *"); + console.log("* Provide NOTARIZE_KEYCHAIN_PROFILE or *"); + console.log("* NOTARIZE_APPLE_ID, NOTARIZE_TEAM_ID *"); + console.log("* and NOTARIZE_APPLE_ID_PASSWORD *"); console.log("*****************************************"); warned = true; } @@ -25,8 +34,7 @@ exports.default = async function (context) { tool: "notarytool", appBundleId: appId, appPath: `${appOutDir}/${appName}.app`, - keychainProfile, - keychain: process.env.NOTARIZE_KEYCHAIN, + ...notarizeToolCredentials, }); } }; diff --git a/scripts/generate-builder-config.ts b/scripts/generate-builder-config.ts new file mode 100755 index 0000000..adaaddb --- /dev/null +++ b/scripts/generate-builder-config.ts @@ -0,0 +1,131 @@ +#!/usr/bin/env -S npx ts-node + +/** + * Script to generate electron-builder.json config files for builds which don't match package.json, e.g. nightlies + * This script has different outputs depending on your os platform. + * + * On Windows: + * Prefixes the nightly version with `0.0.1-nightly.` as it breaks if it is not semver + * + * On Linux: + * Replaces spaces in the product name with dashes as spaces in paths can cause issues + * Passes --deb-custom-control to build.deb.fpm if specified + */ + +import parseArgs from "minimist"; +import fsProm from "fs/promises"; +import * as os from "os"; + +const ELECTRON_BUILDER_CFG_FILE = "electron-builder.json"; + +const NIGHTLY_APP_ID = "im.riot.nightly"; +const NIGHTLY_APP_NAME = "element-desktop-nightly"; + +const argv = parseArgs<{ + nightly?: string; + "deb-custom-control"?: string; +}>(process.argv.slice(2), { + string: ["nightly", "deb-custom-control"], +}); + +interface File { + from: string; + to: string; +} + +interface PackageBuild { + appId: string; + asarUnpack: string; + files: Array; + extraResources: Array; + linux: { + target: string; + category: string; + maintainer: string; + desktop: { + StartupWMClass: string; + }; + }; + mac: { + category: string; + darkModeSupport: boolean; + }; + win: { + target: { + target: string; + }; + sign: string; + }; + deb?: { + fpm?: string[]; + }; + directories: { + output: string; + }; + afterPack: string; + afterSign: string; + protocols: Array<{ + name: string; + schemes: string[]; + }>; + extraMetadata?: { + productName?: string; + name?: string; + version?: string; + }; +} + +interface Package { + build: PackageBuild; + productName: string; +} + +async function main(): Promise { + // Electron builder doesn't overlay with the config in package.json, so load it here + const pkg: Package = JSON.parse(await fsProm.readFile("package.json", "utf8")); + + const cfg: PackageBuild = { + ...pkg.build, + extraMetadata: { + productName: pkg.productName, + }, + }; + + if (argv.nightly) { + cfg.appId = NIGHTLY_APP_ID; + cfg.extraMetadata!.productName += " Nightly"; + cfg.extraMetadata!.name = NIGHTLY_APP_NAME; + + let version = argv.nightly; + if (os.platform() === "win32") { + // The windows packager relies on parsing this as semver, so we have to make it look like one. + // This will give our update packages really stupid names, but we probably can't change that either + // because squirrel windows parses them for the version too. We don't really care: nobody sees them. + // We just give the installer a static name, so you'll just see this in the 'about' dialog. + // Turns out if you use 0.0.0 here it makes Squirrel windows crash, so we use 0.0.1. + version = "0.0.1-nightly." + version; + } + cfg.extraMetadata!.version = version; + } + + if (os.platform() === "linux") { + // Electron crashes on debian if there's a space in the path. + // https://github.com/vector-im/element-web/issues/13171 + cfg.extraMetadata!.productName = cfg.extraMetadata!.productName!.replace(/ /g, "-"); + + if (argv["deb-custom-control"]) { + cfg.deb = { + fpm: [`--deb-custom-control=${argv["deb-custom-control"]}`], + }; + } + } + + await fsProm.writeFile(ELECTRON_BUILDER_CFG_FILE, JSON.stringify(cfg, null, 4)); +} + +main().then((ret) => { + process.exit(ret!); +}).catch((e) => { + console.error(e); + process.exit(1); +}); diff --git a/scripts/generate-nightly-version.ts b/scripts/generate-nightly-version.ts new file mode 100755 index 0000000..6cc7686 --- /dev/null +++ b/scripts/generate-nightly-version.ts @@ -0,0 +1,41 @@ +#!/usr/bin/env -S npx ts-node + +/** + * Script to generate incremental Nightly build versions, based on the latest Nightly build version of that kind. + * The version format is YYYYMMDDNN where NN is in case we need to do multiple versions in a day. + * + * NB. on windows, squirrel will try to parse the version number parts, including this string, into 32-bit integers, + * which is fine as long as we only add two digits to the end... + */ + +import parseArgs from "minimist"; + +const argv = parseArgs<{ + latest?: string; +}>(process.argv.slice(2), { + string: ["latest"], +}); + +function parseVersion(version: string): [Date, number] { + const year = parseInt(version.slice(0, 4), 10); + const month = parseInt(version.slice(4, 2), 10); + const day = parseInt(version.slice(6, 2), 10); + const num = parseInt(version.slice(8, 2), 10); + return [new Date(year, month - 1, day), num]; +} + +const [latestDate, latestNum] = argv.latest ? parseVersion(argv.latest) : []; + +const now = new Date(); +const month = (now.getMonth() + 1).toString().padStart(2, '0'); +const date = now.getDate().toString().padStart(2, '0'); +let buildNum = 1; +if (latestDate && new Date(latestDate).getDate().toString().padStart(2, '0') === date) { + buildNum = latestNum! + 1; +} + +if (buildNum > 99) { + throw new Error("Maximum number of Nightlies exceeded on this day."); +} + +console.log(now.getFullYear() + month + date + buildNum.toString().padStart(2, '0') + buildNum);