Matrix/element-desktop
Matrix/element-desktop
1
0
Fork 0
mirror of https://github.com/vector-im/element-desktop.git synced 2024-07-02 04:23:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

Use a fully static seshat build (#631)

Browse source 
Co-authored-by: Michael Telatynski <7t3chguy@gmail.com>
This commit is contained in:
Mathieu Velten 2023-04-24 14:19:10 +02:00 committed by GitHub
parent b6815b2731
commit cf94a1e49a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 40 additions and 474 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
.github/workflows/build_linux.yaml vendored
View file

@ -83,7 +83,7 @@ jobs:
- name: Prepare for static sqlcipher build
if: inputs.sqlcipher == 'static'
run: |
echo "SQLCIPHER_STATIC=1" >> $GITHUB_ENV
echo "SQLCIPHER_BUNDLED=1" >> $GITHUB_ENV
# Ideally the docker image would be ready for cross-compilation but libsqlcipher-dev is not Multi-Arch compatible
# https://unix.stackexchange.com/a/349359
@ -150,17 +150,13 @@ jobs:
LIBS=$(readelf -d dist/**/*.node | grep NEEDED)
echo "$LIBS"
if [ "$SQLCIPHER_STATIC" == "1" ]; then
if grep -q "libsqlcipher.so.0" <<< "$LIBS" ; then
exit 2
fi
set +x
assert_contains_string() { [[ "$1" == *"$2"* ]]; }
! assert_contains_string "$LIBS" "libcrypto.so.1.1"
if [ "$SQLCIPHER_BUNDLED" == "1" ]; then
! assert_contains_string "$LIBS" "libsqlcipher.so.0"
else
if grep -q "libcrypto.so.1.1" <<< "$LIBS" ; then
exit 3
fi
if ! grep -q "libsqlcipher.so.0" <<< "$LIBS" ; then
exit 4
fi
assert_contains_string "$LIBS" "libsqlcipher.so.0"
fi
env:
ARCH: ${{ steps.config.outputs.arch }}

6
docs/native-node-modules.md
View file

@ -71,9 +71,9 @@ as usual using:
On Windows & macOS we always statically link libsqlcipher for it is not generally available.
On Linux by default we will use a system package, on debian & ubuntu this is `libsqlcipher0`,
but this is problematic for some other packages.
By including `SQLCIPHER_STATIC=1` in the build environment, the build scripts will statically link sqlcipher,
note that this will want a `libcrypto1.1` shared library available in the system.
but this is problematic for some other packages, and we found that it may crashes for unknown reasons.
By including `SQLCIPHER_BUNDLED=1` in the build environment, the build scripts will fully statically
link sqlcipher, including a static build of OpenSSL.
More info can be found at https://github.com/matrix-org/seshat/issues/102
and https://github.com/vector-im/element-web/issues/20926.

305
hak/matrix-seshat/build.ts
View file

@ -14,305 +14,46 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
import path from "path";
import childProcess from "child_process";
import { mkdirp } from "mkdirp";
import fsExtra from "fs-extra";
import HakEnv from "../../scripts/hak/hakEnv";
import { DependencyInfo } from "../../scripts/hak/dep";
type WinConfiguration =
| "VC-WIN32"
| "VC-WIN64A"
| "VC-WIN64-ARM"
| "VC-WIN64-CLANGASM-ARM"
| "VC-CLANG-WIN64-CLANGASM-ARM"
| "VC-WIN32-HYBRIDCRT"
| "VC-WIN64A-HYBRIDCRT";
export default async function (hakEnv: HakEnv, moduleInfo: DependencyInfo): Promise<void> {
if (hakEnv.isWin()) {
await buildOpenSslWin(hakEnv, moduleInfo);
await buildSqlCipherWin(hakEnv, moduleInfo);
} else if (hakEnv.wantsStaticSqlCipherUnix()) {
await buildSqlCipherUnix(hakEnv, moduleInfo);
}
await buildMatrixSeshat(hakEnv, moduleInfo);
}
async function buildOpenSslWin(hakEnv: HakEnv, moduleInfo: DependencyInfo): Promise<void> {
const version = moduleInfo.cfg.dependencies.openssl;
const openSslDir = path.join(moduleInfo.moduleTargetDotHakDir, `openssl-${version}`);
let openSslArch: WinConfiguration;
switch (hakEnv.getTargetArch()) {
case "x64":
openSslArch = "VC-WIN64A";
break;
case "ia32":
openSslArch = "VC-WIN32";
break;
case "arm64":
openSslArch = "VC-WIN64-ARM";
break;
}
console.log("Building openssl in " + openSslDir);
await new Promise<void>((resolve, reject) => {
const proc = childProcess.spawn(
"perl",
[
"Configure",
"--prefix=" + moduleInfo.depPrefix,
// sqlcipher only uses about a tiny part of openssl. We link statically
// so will only pull in the symbols we use, but we may as well turn off
// as much as possible to save on build time.
"no-afalgeng",
"no-capieng",
"no-cms",
"no-ct",
"no-deprecated",
"no-dgram",
"no-dso",
"no-ec",
"no-ec2m",
"no-gost",
"no-nextprotoneg",
"no-ocsp",
"no-sock",
"no-srp",
"no-srtp",
"no-tests",
"no-ssl",
"no-tls",
"no-dtls",
"no-shared",
"no-aria",
"no-camellia",
"no-cast",
"no-chacha",
"no-cmac",
"no-des",
"no-dh",
"no-dsa",
"no-ecdh",
"no-ecdsa",
"no-idea",
"no-md4",
"no-mdc2",
"no-ocb",
"no-poly1305",
"no-rc2",
"no-rc4",
"no-rmd160",
"no-scrypt",
"no-seed",
"no-siphash",
"no-sm2",
"no-sm3",
"no-sm4",
"no-whirlpool",
openSslArch,
],
{
cwd: openSslDir,
stdio: "inherit",
},
);
proc.on("exit", (code) => {
code ? reject(code) : resolve();
});
});
await new Promise<void>((resolve, reject) => {
const proc = childProcess.spawn("nmake", ["build_libs"], {
cwd: openSslDir,
stdio: "inherit",
});
proc.on("exit", (code) => {
code ? reject(code) : resolve();
});
});
await new Promise<void>((resolve, reject) => {
const proc = childProcess.spawn("nmake", ["install_dev"], {
cwd: openSslDir,
stdio: "inherit",
});
proc.on("exit", (code) => {
code ? reject(code) : resolve();
});
});
}
async function buildSqlCipherWin(hakEnv: HakEnv, moduleInfo: DependencyInfo): Promise<void> {
const version = moduleInfo.cfg.dependencies.sqlcipher;
const sqlCipherDir = path.join(moduleInfo.moduleTargetDotHakDir, `sqlcipher-${version}`);
const buildDir = path.join(sqlCipherDir, "bld");
await mkdirp(buildDir);
await new Promise<void>((resolve, reject) => {
const proc = childProcess.spawn("nmake", ["/f", path.join("..", "Makefile.msc"), "libsqlite3.lib", "TOP=.."], {
cwd: buildDir,
stdio: "inherit",
env: Object.assign({}, process.env, {
CCOPTS: "-DSQLITE_HAS_CODEC -I" + path.join(moduleInfo.depPrefix, "include"),
LTLIBPATHS: "/LIBPATH:" + path.join(moduleInfo.depPrefix, "lib"),
LTLIBS: "libcrypto.lib",
}),
});
proc.on("exit", (code) => {
code ? reject(code) : resolve();
});
});
await fsExtra.copy(path.join(buildDir, "libsqlite3.lib"), path.join(moduleInfo.depPrefix, "lib", "sqlcipher.lib"));
await fsExtra.copy(path.join(buildDir, "sqlite3.h"), path.join(moduleInfo.depPrefix, "include", "sqlcipher.h"));
}
async function buildSqlCipherUnix(hakEnv: HakEnv, moduleInfo: DependencyInfo): Promise<void> {
const version = moduleInfo.cfg.dependencies.sqlcipher;
const sqlCipherDir = path.join(moduleInfo.moduleTargetDotHakDir, `sqlcipher-${version}`);
const args = [
"--prefix=" + moduleInfo.depPrefix + "",
"--enable-tempstore=yes",
"--enable-shared=no",
"--enable-tcl=no",
];
if (hakEnv.isMac()) {
args.push("--with-crypto-lib=commoncrypto");
}
if (hakEnv.wantsStaticSqlCipherUnix()) {
args.push("--enable-tcl=no");
if (hakEnv.isLinux()) {
args.push("--with-pic=yes");
}
}
if (!hakEnv.isHost()) {
// In the nonsense world of `configure`, it is assumed you are building
// a compiler like `gcc`, so the `host` option actually means the target
// the build output runs on.
args.push(`--host=${hakEnv.getTargetId()}`);
}
const cflags = ["-DSQLITE_HAS_CODEC"];
// If the caller has specified CFLAGS then we shouldn't specify target
// as their compiler may be incompatible (gcc)
if (!hakEnv.isHost() && !process.env.CFLAGS) {
// `clang` uses more logical option naming.
cflags.push(`--target=${hakEnv.getTargetId()}`);
}
if (process.env.CFLAGS) cflags.unshift(process.env.CFLAGS);
args.push(`CFLAGS=${cflags.join(" ")}`);
const ldflags: string[] = [];
if (hakEnv.isMac()) {
ldflags.push("-framework Security");
ldflags.push("-framework Foundation");
}
if (ldflags.length) {
if (process.env.LDFLAGS) ldflags.unshift(process.env.LDFLAGS);
args.push(`LDFLAGS=${ldflags.join(" ")}`);
}
await new Promise<void>((resolve, reject) => {
const proc = childProcess.spawn(path.join(sqlCipherDir, "configure"), args, {
cwd: sqlCipherDir,
stdio: "inherit",
});
proc.on("exit", (code) => {
code ? reject(code) : resolve();
});
});
await new Promise<void>((resolve, reject) => {
const proc = childProcess.spawn("make", [], {
cwd: sqlCipherDir,
stdio: "inherit",
});
proc.on("exit", (code) => {
code ? reject(code) : resolve();
});
});
await new Promise<void>((resolve, reject) => {
const proc = childProcess.spawn("make", ["install"], {
cwd: sqlCipherDir,
stdio: "inherit",
});
proc.on("exit", (code) => {
code ? reject(code) : resolve();
});
});
}
async function buildMatrixSeshat(hakEnv: HakEnv, moduleInfo: DependencyInfo): Promise<void> {
// seshat now uses n-api so we shouldn't need to specify a node version to
// build against, but it does seems to still need something in here, so leaving
// it for now: we should confirm how much of this it still actually needs.
const env = hakEnv.makeGypEnv();
if (!hakEnv.isLinux() || hakEnv.wantsStaticSqlCipherUnix()) {
Object.assign(env, {
SQLCIPHER_STATIC: 1,
SQLCIPHER_LIB_DIR: path.join(moduleInfo.depPrefix, "lib"),
SQLCIPHER_INCLUDE_DIR: path.join(moduleInfo.depPrefix, "include"),
});
}
if (hakEnv.isLinux() && hakEnv.wantsStaticSqlCipherUnix()) {
// Ensure Element uses the statically-linked seshat build, and prevent other applications
// from attempting to use this one. Detailed explanation:
//
// RUSTFLAGS
// An environment variable containing a list of arguments to pass to rustc.
// -Clink-arg=VALUE
// A rustc argument to pass a single argument to the linker.
// -Wl,
// gcc syntax to pass an argument (from gcc) to the linker (ld).
// -Bsymbolic:
// Prefer local/statically linked symbols over those in the environment.
// Prevent overriding native libraries by LD_PRELOAD etc.
// --exclude-libs ALL
// Prevent symbols from being exported by any archive libraries.
// Reduces output filesize and prevents being dynamically linked against.
env.RUSTFLAGS = "-Clink-arg=-Wl,-Bsymbolic -Clink-arg=-Wl,--exclude-libs,ALL";
}
if (hakEnv.isWin()) {
env.RUSTFLAGS = "-Ctarget-feature=+crt-static -Clink-args=libcrypto.lib";
// Note that in general, you can specify targets in Rust without having to have
// the matching toolchain, however for this, cargo gets confused when building
// the build scripts since they run on the host, but vcvarsall.bat sets the c
// compiler in the path to be the one for the target, so we just use the matching
// toolchain for the target architecture which makes everything happy.
env.RUSTUP_TOOLCHAIN = `stable-${hakEnv.getTargetId()}`;
}
if (!hakEnv.isHost()) {
env.CARGO_BUILD_TARGET = hakEnv.getTargetId();
}
console.log("Running neon with env", env);
console.log("Running yarn install");
await new Promise<void>((resolve, reject) => {
const proc = childProcess.spawn(
path.join(moduleInfo.nodeModuleBinDir, "neon" + (hakEnv.isWin() ? ".cmd" : "")),
["build", "--release"],
"yarn" + (hakEnv.isWin() ? ".cmd" : ""),
["install"],
{
cwd: moduleInfo.moduleBuildDir,
env,
shell: true,
stdio: "inherit",
},
);
proc.on("exit", (code) => {
code ? reject(code) : resolve();
});
});
const buildTarget = hakEnv.wantsStaticSqlCipher() ? "build-bundled" : "build";
console.log("Running yarn build");
await new Promise<void>((resolve, reject) => {
const proc = childProcess.spawn(
"yarn" + (hakEnv.isWin() ? ".cmd" : ""),
["run", buildTarget],
{
cwd: moduleInfo.moduleBuildDir,
env,
shell: true,
stdio: "inherit",
},
);

17
hak/matrix-seshat/check.ts
View file

@ -21,23 +21,6 @@ import HakEnv from "../../scripts/hak/hakEnv";
import { DependencyInfo } from "../../scripts/hak/dep";
export default async function (hakEnv: HakEnv, moduleInfo: DependencyInfo): Promise<void> {
if (hakEnv.wantsStaticSqlCipher()) {
// of course tcl doesn't have a --version
await new Promise<void>((resolve, reject) => {
const proc = childProcess.spawn("tclsh", [], {
stdio: ["pipe", "ignore", "ignore"],
});
proc.on("exit", (code) => {
if (code !== 0) {
reject("Can't find tclsh - have you installed TCL?");
} else {
resolve();
}
});
proc.stdin.end();
});
}
const tools = [
["rustc", "--version"],
["python", "--version"], // node-gyp uses python for reasons beyond comprehension

129
hak/matrix-seshat/fetchDeps.ts
View file

@ -1,129 +0,0 @@
/*
Copyright 2020 The Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import path from "path";
import childProcess from "child_process";
import fs from "fs";
import fsProm from "fs/promises";
import tar from "tar";
import fetch from "node-fetch";
import { promises as stream } from "stream";
import HakEnv from "../../scripts/hak/hakEnv";
import { DependencyInfo } from "../../scripts/hak/dep";
async function download(url: string, filename: string): Promise<void> {
const resp = await fetch(url);
if (!resp.ok) throw new Error(`unexpected response ${resp.statusText}`);
if (!resp.body) throw new Error(`unexpected response has no body ${resp.statusText}`);
await stream.pipeline(resp.body, fs.createWriteStream(filename));
}
export default async function (hakEnv: HakEnv, moduleInfo: DependencyInfo): Promise<void> {
if (hakEnv.wantsStaticSqlCipher()) {
await getSqlCipher(hakEnv, moduleInfo);
}
if (hakEnv.isWin()) {
await getOpenSsl(hakEnv, moduleInfo);
}
}
async function getSqlCipher(hakEnv: HakEnv, moduleInfo: DependencyInfo): Promise<void> {
const version = moduleInfo.cfg.dependencies.sqlcipher;
const sqlCipherDir = path.join(moduleInfo.moduleTargetDotHakDir, `sqlcipher-${version}`);
let haveSqlcipher: boolean;
try {
await fsProm.stat(sqlCipherDir);
haveSqlcipher = true;
} catch (e) {
haveSqlcipher = false;
}
if (haveSqlcipher) return;
const sqlCipherTarball = path.join(moduleInfo.moduleDotHakDir, `sqlcipher-${version}.tar.gz`);
let haveSqlcipherTar: boolean;
try {
await fsProm.stat(sqlCipherTarball);
haveSqlcipherTar = true;
} catch (e) {
haveSqlcipherTar = false;
}
if (!haveSqlcipherTar) {
await download(`https://github.com/sqlcipher/sqlcipher/archive/v${version}.tar.gz`, sqlCipherTarball);
}
// Extract the tarball to per-target directories, then we avoid cross-contaiminating archs
await tar.x({
file: sqlCipherTarball,
cwd: moduleInfo.moduleTargetDotHakDir,
});
if (hakEnv.isWin()) {
// On Windows, we need to patch the makefile because it forces TEMP_STORE to
// default to files (1) but the README specifically says you '*must*' set it
// set it to 2 (default to memory).
const patchFile = path.join(moduleInfo.moduleHakDir, `sqlcipher-${version}-win.patch`);
await new Promise<void>((resolve, reject) => {
const readStream = fs.createReadStream(patchFile);
const proc = childProcess.spawn("patch", ["-p1"], {
cwd: sqlCipherDir,
stdio: ["pipe", "inherit", "inherit"],
});
proc.on("exit", (code) => {
code ? reject(code) : resolve();
});
readStream.pipe(proc.stdin);
});
}
}
async function getOpenSsl(hakEnv: HakEnv, moduleInfo: DependencyInfo): Promise<void> {
const version = moduleInfo.cfg.dependencies.openssl;
const openSslDir = path.join(moduleInfo.moduleTargetDotHakDir, `openssl-${version}`);
let haveOpenSsl: boolean;
try {
await fsProm.stat(openSslDir);
haveOpenSsl = true;
} catch (e) {
haveOpenSsl = false;
}
if (haveOpenSsl) return;
const openSslTarball = path.join(moduleInfo.moduleDotHakDir, `openssl-${version}.tar.gz`);
let haveOpenSslTar: boolean;
try {
await fsProm.stat(openSslTarball);
haveOpenSslTar = true;
} catch (e) {
haveOpenSslTar = false;
}
if (!haveOpenSslTar) {
await download(`https://www.openssl.org/source/openssl-${version}.tar.gz`, openSslTarball);
}
console.log("extracting " + openSslTarball + " in " + moduleInfo.moduleTargetDotHakDir);
await tar.x({
file: openSslTarball,
cwd: moduleInfo.moduleTargetDotHakDir,
});
}

8
hak/matrix-seshat/hak.json
View file

@ -1,13 +1,7 @@
{
"scripts": {
"check": "check.ts",
"fetchDeps": "fetchDeps.ts",
"build": "build.ts"
},
"prune": "native",
"copy": "native/index.node",
"dependencies": {
"openssl": "1.1.1f",
"sqlcipher": "4.3.0"
}
"copy": "index.node"
}

14
hak/matrix-seshat/sqlcipher-4.3.0-win.patch
View file

@ -1,14 +0,0 @@
diff -Nur sqlcipher-4.3.0-orig/Makefile.msc sqlcipher-4.3.0-mod/Makefile.msc
--- sqlcipher-4.3.0-orig/Makefile.msc 2019-12-20 16:40:26.000000000 +0000
+++ sqlcipher-4.3.0-mod/Makefile.msc 2020-02-14 11:31:39.000000000 +0000
@@ -985,8 +985,8 @@
# default to file, 2 to default to memory, and 3 to force temporary
# tables to always be in memory.
#
-TCC = $(TCC) -DSQLITE_TEMP_STORE=1
-RCC = $(RCC) -DSQLITE_TEMP_STORE=1
+TCC = $(TCC) -DSQLITE_TEMP_STORE=2
+RCC = $(RCC) -DSQLITE_TEMP_STORE=2
# Enable/disable loadable extensions, and other optional features
# based on configuration. (-DSQLITE_OMIT*, -DSQLITE_ENABLE*).

2
package.json
View file

@ -115,7 +115,7 @@
"typescript": "5.0.4"
},
"hakDependencies": {
"matrix-seshat": "^2.3.3",
"matrix-seshat": "^3.0.0",
"keytar": "^7.9.0"
},
"resolutions": {

1
scripts/hak/README.md
View file

@ -62,7 +62,6 @@ Hak is divided into lifecycle stages, in order:
- fetch - Download and extract the source of the dependency
- link - Link the copy of the dependency into your node_modules directory
- fetchDeps - Fetch & extract any native dependencies required to build the module.
- build - The Good Stuff. Configure and build any native dependencies, then the module itself.
- copy - Copy the built artifact from the module build directory to the module output directory.

6
scripts/hak/hakEnv.ts
View file

@ -101,11 +101,7 @@ export default class HakEnv {
});
}
public wantsStaticSqlCipherUnix(): boolean {
return this.isMac() || process.env.SQLCIPHER_STATIC == "1";
}
public wantsStaticSqlCipher(): boolean {
return this.isWin() || this.wantsStaticSqlCipherUnix();
return !this.isLinux() || process.env.SQLCIPHER_BUNDLED == "1";
}
}

8
scripts/hak/index.ts
View file

@ -24,17 +24,17 @@ import { DependencyInfo } from "./dep";
const GENERALCOMMANDS = ["target"];
// These can only be run on specific modules
const MODULECOMMANDS = ["check", "fetch", "link", "fetchDeps", "build", "copy", "clean"];
const MODULECOMMANDS = ["check", "fetch", "link", "build", "copy", "clean"];
// Shortcuts for multiple commands at once (useful for building universal binaries
// because you can run the fetch/fetchDeps/build for each arch and then copy/link once)
const METACOMMANDS: Record<string, string[]> = {
fetchandbuild: ["check", "fetch", "fetchDeps", "build"],
fetchandbuild: ["check", "fetch", "build"],
copyandlink: ["copy", "link"],
};
// Scripts valid in a hak.json 'scripts' section
const HAKSCRIPTS = ["check", "fetch", "fetchDeps", "build"];
const HAKSCRIPTS = ["check", "fetch", "build"];
async function main(): Promise<void> {
const prefix = await findNpmPrefix(process.cwd());
@ -113,7 +113,7 @@ async function main(): Promise<void> {
let cmds: string[];
if (process.argv.length < 3) {
cmds = ["check", "fetch", "fetchDeps", "build", "copy", "link"];
cmds = ["check", "fetch", "build", "copy", "link"];
} else if (METACOMMANDS[process.argv[2]]) {
cmds = METACOMMANDS[process.argv[2]];
} else {