Merge pull request #216 from vector-im/jryans/aarch64-apple-darwin

Add update and native build support for Apple silicon

hak/matrix-seshat/build.js

@@ -1,5 +1,5 @@
 /*
-Copyright 2020 The Matrix.org Foundation C.I.C.
+Copyright 2020-2021 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -182,9 +182,36 @@ async function buildSqlCipherUnix(hakEnv, moduleInfo) {
     if (hakEnv.isMac()) {
         args.push('--with-crypto-lib=commoncrypto');
     }
-    args.push('CFLAGS=-DSQLITE_HAS_CODEC');
+
+    if (!hakEnv.isHost()) {
+        // In the nonsense world of `configure`, it is assumed you are building
+        // a compiler like `gcc`, so the `host` option actually means the target
+        // the build output runs on.
+        args.push(`--host=${hakEnv.getTargetId()}`);
+    }
+
+    const cflags = [
+        '-DSQLITE_HAS_CODEC',
+    ];
+
+    if (!hakEnv.isHost()) {
+        // `clang` uses more logical option naming.
+        cflags.push(`--target=${hakEnv.getTargetId()}`);
+    }
+
+    if (cflags.length) {
+        args.push(`CFLAGS=${cflags.join(' ')}`);
+    }
+
+    const ldflags = [];
+
     if (hakEnv.isMac()) {
-        args.push('LDFLAGS=-framework Security -framework Foundation');
+        ldflags.push('-framework Security');
+        ldflags.push('-framework Foundation');
+    }
+
+    if (ldflags.length) {
+        args.push(`LDFLAGS=${ldflags.join(' ')}`);
     }
 
     await new Promise((resolve, reject) => {
@@ -251,6 +278,10 @@ async function buildMatrixSeshat(hakEnv, moduleInfo) {
         env.RUSTUP_TOOLCHAIN = hakEnv.arch == 'x64' ? 'stable-x86_64-pc-windows-msvc' : 'stable-i686-pc-windows-msvc';
     }
 
+    if (!hakEnv.isHost()) {
+        env.CARGO_BUILD_TARGET = hakEnv.getTargetId();
+    }
+
     console.log("Running neon with env", env);
     await new Promise((resolve, reject) => {
         const proc = childProcess.spawn(

hak/matrix-seshat/check.js

@@ -1,5 +1,5 @@
 /*
-Copyright 2020 The Matrix.org Foundation C.I.C.
+Copyright 2020-2021 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -34,7 +34,10 @@ module.exports = async function(hakEnv, moduleInfo) {
         });
     }
 
-    const tools = [['python', '--version']]; // node-gyp uses python for reasons beyond comprehension
+    const tools = [
+        ['rustc', '--version'],
+        ['python', '--version'], // node-gyp uses python for reasons beyond comprehension
+    ];
     if (hakEnv.isWin()) {
         tools.push(['perl', '--version']); // for openssl configure
         tools.push(['patch', '--version']); // to patch sqlcipher Makefile.msc
@@ -57,4 +60,18 @@ module.exports = async function(hakEnv, moduleInfo) {
             });
         });
     }
+
+    // Ensure Rust target exists
+    await new Promise((resolve, reject) => {
+        childProcess.execFile('rustup', ['target', 'list', '--installed'], (err, out) => {
+            if (err) {
+                reject("Can't find rustup");
+            }
+            const target = hakEnv.getTargetId();
+            if (!out.includes(target)) {
+                reject(`Rust target ${target} not installed`);
+            }
+            resolve();
+        });
+    });
 };

scripts/hak/copy.js

@@ -1,5 +1,5 @@
 /*
-Copyright 2020 The Matrix.org Foundation C.I.C.
+Copyright 2020-2021 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -41,8 +41,8 @@ async function copy(hakEnv, moduleInfo) {
 
     if (moduleInfo.cfg.copy) {
         console.log(
-            "Copying " + moduleInfo.cfg.prune + " from " +
-            moduleInfo.moduleOutDir + " to " + moduleInfo.moduleOutDir,
+            "Copying files from " +
+            moduleInfo.moduleBuildDir + " to " + moduleInfo.moduleOutDir,
         );
         const files = await new Promise(async (resolve, reject) => {
             glob(moduleInfo.cfg.copy, {

scripts/hak/hakEnv.js

@@ -1,5 +1,5 @@
 /*
-Copyright 2020 The Matrix.org Foundation C.I.C.
+Copyright 2020-2021 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -18,6 +18,7 @@ const path = require('path');
 const os = require('os');
 
 const nodePreGypVersioning = require('node-pre-gyp/lib/util/versioning');
+const { TARGETS, getHost, isHostId } = require('./target');
 
 function getElectronVersion(packageJson) {
     // should we pick the version of an installed electron
@@ -33,7 +34,7 @@ function getRuntime(packageJson) {
     return electronVersion ? 'electron' : 'node-webkit';
 }
 
-function getTarget(packageJson) {
+function getRuntimeVersion(packageJson) {
     const electronVersion = getElectronVersion(packageJson);
     if (electronVersion) {
         return electronVersion;
@@ -42,30 +43,24 @@ function getTarget(packageJson) {
     }
 }
 
-function detectArch() {
-    if (process.platform === 'win32') {
-        // vcvarsall.bat (the script that sets up the environment for
-        // visual studio build tools) sets an env var to tell us what
-        // architecture the active build tools target, so we auto-detect
-        // this.
-        const targetArch = process.env.VSCMD_ARG_TGT_ARCH;
-        if (targetArch === 'x86') {
-            return 'ia32';
-        } else if (targetArch === 'x64') {
-            return 'x64';
-        }
-    }
-    return process.arch;
-}
-
 module.exports = class HakEnv {
-    constructor(prefix, packageJson) {
+    constructor(prefix, packageJson, targetId) {
+        let target;
+        if (targetId) {
+            target = TARGETS[targetId];
+        } else {
+            target = getHost();
+        }
+
+        if (!target) {
+            throw new Error(`Unknown target ${targetId}!`);
+        }
+
         Object.assign(this, {
             // what we're targeting
             runtime: getRuntime(packageJson),
-            target: getTarget(packageJson),
-            platform: process.platform,
-            arch: detectArch(),
+            runtimeVersion: getRuntimeVersion(packageJson),
+            target,
 
             // paths
             projectRoot: prefix,
@@ -76,34 +71,42 @@ module.exports = class HakEnv {
     getRuntimeAbi() {
         return nodePreGypVersioning.get_runtime_abi(
             this.runtime,
-            this.target,
+            this.runtimeVersion,
         );
     }
 
     // {node_abi}-{platform}-{arch}
     getNodeTriple() {
-        return this.getRuntimeAbi() + '-' + this.platform + '-' + this.arch;
+        return this.getRuntimeAbi() + '-' + this.target.platform + '-' + this.target.arch;
+    }
+
+    getTargetId() {
+        return this.target.id;
     }
 
     isWin() {
-        return this.platform === 'win32';
+        return this.target.platform === 'win32';
     }
 
     isMac() {
-        return this.platform === 'darwin';
+        return this.target.platform === 'darwin';
     }
 
     isLinux() {
-        return this.platform === 'linux';
+        return this.target.platform === 'linux';
+    }
+
+    isHost() {
+        return isHostId(this.target.id);
     }
 
     makeGypEnv() {
         return Object.assign({}, process.env, {
-            npm_config_target: this.target,
-            npm_config_arch: this.arch,
-            npm_config_target_arch: this.arch,
+            npm_config_arch: this.target.arch,
+            npm_config_target_arch: this.target.arch,
             npm_config_disturl: 'https://atom.io/download/electron',
             npm_config_runtime: this.runtime,
+            npm_config_target: this.runtimeVersion,
             npm_config_build_from_source: true,
             npm_config_devdir: path.join(os.homedir(), ".electron-gyp"),
         });

scripts/hak/index.js

@@ -1,5 +1,5 @@
 /*
-Copyright 2020 The Matrix.org Foundation C.I.C.
+Copyright 2020-2021 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -53,7 +53,19 @@ async function main() {
         process.exit(1);
     }
 
-    const hakEnv = new HakEnv(prefix, packageJson);
+    // Apply `--target <target>` option if specified
+    const targetIndex = process.argv.indexOf('--target');
+    let targetId;
+    if (targetIndex >= 0) {
+        if ((targetIndex + 1) >= process.argv.length) {
+            console.error("--target option specified without a target");
+            process.exit(1);
+        }
+        // Extract target ID and remove from args
+        targetId = process.argv.splice(targetIndex, 2)[1];
+    }
+
+    const hakEnv = new HakEnv(prefix, packageJson, targetId);
 
     const deps = {};
 
@@ -133,4 +145,7 @@ async function main() {
     }
 }
 
-main().catch(() => process.exit(1));
+main().catch(err => {
+    console.error(err);
+    process.exit(1);
+});

scripts/hak/target.js

@@ -0,0 +1,82 @@
+"use strict";
+/*
+Copyright 2021 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/*
+ * THIS FILE IS GENERATED, NOT MEANT FOR EDITING DIRECTLY
+ * The original source is `target.ts` in the `element-builder` repo. You can
+ * edit it over there, run `yarn build`, and paste the changes here. It is
+ * currently assumed this file will rarely change, so a spearate package is not
+ * yet warranted.
+ */
+
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.isHost = exports.isHostId = exports.getHost = exports.ENABLED_TARGETS = exports.TARGETS = void 0;
+const aarch64AppleDarwin = {
+    id: 'aarch64-apple-darwin',
+    platform: 'darwin',
+    arch: 'arm64',
+};
+const i686PcWindowsMsvc = {
+    id: 'i686-pc-windows-msvc',
+    platform: 'win32',
+    arch: 'ia32',
+    vcVarsArch: 'x86',
+};
+const x8664PcWindowsMsvc = {
+    id: 'x86_64-pc-windows-msvc',
+    platform: 'win32',
+    arch: 'x64',
+    vcVarsArch: 'amd64',
+};
+const x8664AppleDarwin = {
+    id: 'x86_64-apple-darwin',
+    platform: 'darwin',
+    arch: 'x64',
+};
+const x8664UnknownLinuxGnu = {
+    id: 'x86_64-unknown-linux-gnu',
+    platform: 'linux',
+    arch: 'x64',
+};
+exports.TARGETS = {
+    'aarch64-apple-darwin': aarch64AppleDarwin,
+    'i686-pc-windows-msvc': i686PcWindowsMsvc,
+    'x86_64-pc-windows-msvc': x8664PcWindowsMsvc,
+    'x86_64-apple-darwin': x8664AppleDarwin,
+    'x86_64-unknown-linux-gnu': x8664UnknownLinuxGnu,
+};
+// The set of targets we build by default, sorted by increasing complexity so
+// that we fail fast when the native host target fails.
+exports.ENABLED_TARGETS = [
+    exports.TARGETS['x86_64-apple-darwin'],
+    exports.TARGETS['aarch64-apple-darwin'],
+    exports.TARGETS['x86_64-unknown-linux-gnu'],
+    exports.TARGETS['i686-pc-windows-msvc'],
+];
+function getHost() {
+    return Object.values(exports.TARGETS).find(target => (target.platform === process.platform &&
+        target.arch === process.arch));
+}
+exports.getHost = getHost;
+function isHostId(id) {
+    return getHost()?.id === id;
+}
+exports.isHostId = isHostId;
+function isHost(target) {
+    return getHost()?.id === target.id;
+}
+exports.isHost = isHost;

src/updater.js

@@ -1,3 +1,19 @@
+/*
+Copyright 2016-2021 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
 const { app, autoUpdater, ipcMain } = require('electron');
 
 const UPDATE_POLL_INTERVAL_MS = 60 * 60 * 1000;
@@ -37,7 +53,7 @@ module.exports.start = function startAutoUpdate(updateBaseUrl) {
             // rely on NSURLConnection setting the User-Agent to what we expect,
             // and also acts as a convenient cache-buster to ensure that when the
             // app updates it always gets a fresh value to avoid update-looping.
-            url = `${updateBaseUrl}macos/?localVersion=${encodeURIComponent(app.getVersion())}`;
+            url = `${updateBaseUrl}macos/${process.arch}/?localVersion=${encodeURIComponent(app.getVersion())}`;
         } else if (process.platform === 'win32') {
             url = `${updateBaseUrl}win32/${process.arch}/`;
         } else {
@@ -48,6 +64,7 @@ module.exports.start = function startAutoUpdate(updateBaseUrl) {
         }
 
         if (url) {
+            console.log(`Update URL: ${url}`);
             autoUpdater.setFeedURL(url);
             // We check for updates ourselves rather than using 'updater' because we need to
             // do it in the main process (and we don't really need to check every 10 minutes: