Distribute aarch64 Linux tarball builds (#1664)

2024-06-30 03:23:28 +00:00 · 2024-05-14 23:13:03 +01:00 · 2024-05-14 23:13:03 +01:00 · a83003e0bd
parent d81521e5ab
commit a83003e0bd
9 changed files with 140 additions and 139 deletions
--- a/.github/workflows/build_and_deploy.yaml
+++ b/.github/workflows/build_and_deploy.yaml
@ -86,9 +86,6 @@ jobs:
            matrix:
                arch: [amd64, arm64]
                sqlcipher: [system, static]
-                exclude:
-                    - arch: arm64
-                      sqlcipher: static
        uses: ./.github/workflows/build_linux.yaml
        with:
            arch: ${{ matrix.arch }}
@ -136,6 +133,10 @@ jobs:
                      mkdir -p packages.element.io/install/linux/glibc-x86-64
                      mv linux-amd64-sqlcipher-static/*.tar.gz packages.element.io/install/linux/glibc-x86-64
                  fi
+                  if [ -d linux-arm64-sqlcipher-static ]; then
+                      mkdir -p packages.element.io/install/linux/glibc-aarch64
+                      mv linux-arm64-sqlcipher-static/*.tar.gz packages.element.io/install/linux/glibc-aarch64
+                  fi

            # We don't wish to store the installer for every nightly ever, so we only keep the latest
            - name: "[Nightly] Strip version from installer file"
@ -152,6 +153,7 @@ jobs:

                  # Linux
                  [ -d linux-amd64-sqlcipher-static ] && mv packages.element.io/install/linux/glibc-x86-64/{*,element-desktop-nightly}.tar.gz
+                  [ -d linux-arm64-sqlcipher-static ] && mv packages.element.io/install/linux/glibc-aarch64/{*,element-desktop-nightly}.tar.gz

            - name: "[Release] Prepare release latest symlink"
              if: needs.prepare.outputs.nightly-version == ''
@ -179,6 +181,11 @@ jobs:
                      ln -s "$(find . -type f -iname "*.tar.gz" | xargs -0 -n1 -- basename)" "element-desktop.tar.gz"
                      popd
                  fi
+                  if [ -d linux-arm64-sqlcipher-static ]; then
+                      pushd packages.element.io/install/linux/glibc-aarch64
+                      ln -s "$(find . -type f -iname "*.tar.gz" | xargs -0 -n1 -- basename)" "element-desktop.tar.gz"
+                      popd
+                  fi

            - name: Stash packages.element.io
              if: needs.prepare.outputs.deploy == 'false'
--- a/.github/workflows/build_and_test.yaml
+++ b/.github/workflows/build_and_test.yaml
@ -23,71 +23,17 @@ jobs:
        with:
            arch: ${{ matrix.arch }}

-    # This allows core contributors to test changes to the dockerbuild image within a pull request
-    linux_docker:
-        name: Linux docker
-        runs-on: ubuntu-latest
-        if: github.event_name == 'pull_request'
-        outputs:
-            docker-image: ${{ steps.docker.outputs.image }}
-        permissions:
-            contents: read
-            packages: write
-        env:
-            REGISTRY: ghcr.io
-            IMAGE_NAME: ${{ github.repository }}-dockerbuild-pr
-        steps:
-            - uses: actions/checkout@v4
-
-            - name: "Get modified files"
-              id: changed_files
-              uses: tj-actions/changed-files@635f118699dd888d737c15018cd30aff2e0274f8 # v44
-              with:
-                  files: |
-                      dockerbuild/**
-            - name: Log in to the Container registry
-              if: steps.changed_files.outputs.any_modified == 'true'
-              uses: docker/login-action@5f4866a30a54f16a52d2ecb4a3898e9e424939cf
-              with:
-                  registry: ${{ env.REGISTRY }}
-                  username: ${{ github.actor }}
-                  password: ${{ secrets.GITHUB_TOKEN }}
-
-            - id: docker
-              if: steps.changed_files.outputs.any_modified == 'true'
-              run: |
-                  echo "image=$IMAGE:$PR" >> $GITHUB_OUTPUT
-              env:
-                  IMAGE: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
-                  PR: ${{ github.event.pull_request.number }}
-
-            - name: Build and push Docker image
-              if: steps.changed_files.outputs.any_modified == 'true'
-              uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5
-              with:
-                  context: dockerbuild
-                  push: true
-                  tags: ${{ steps.docker.outputs.image }}
-
    linux:
-        needs:
-            - fetch
-            - linux_docker
+        needs: fetch
        name: "Linux (${{ matrix.arch }}) (sqlcipher: ${{ matrix.sqlcipher }})"
        uses: ./.github/workflows/build_linux.yaml
        strategy:
            matrix:
                sqlcipher: [system, static]
                arch: [amd64, arm64]
-                exclude:
-                    # FIXME: This combination yields a broken Seshat at this time
-                    # Errors at launch with `undefined symbol: PKCS5_PBKDF2_HMAC
-                    - arch: arm64
-                      sqlcipher: static
        with:
            config: ${{ github.event.pull_request.base.ref == 'develop' && 'element.io/nightly' || 'element.io/release' }}
            sqlcipher: ${{ matrix.sqlcipher }}
-            docker-image: ${{ needs.linux_docker.outputs.docker-image }}
            arch: ${{ matrix.arch }}

    macos:
@ -128,6 +74,11 @@ jobs:
                      artifact: linux-arm64-sqlcipher-system
                      executable: "/opt/Element/element-desktop"
                      prepare_cmd: "sudo apt install -y ./dist/*.deb"
+                    - name: "Linux (arm64) (sqlcipher: static)"
+                      os: dind-l-arm64
+                      artifact: linux-arm64-sqlcipher-static
+                      executable: "/opt/Element/element-desktop"
+                      prepare_cmd: "sudo apt install -y ./dist/*.deb"
                    - name: Windows (x86)
                      os: windows-latest
                      artifact: win-ia32
--- a/.github/workflows/build_linux.yaml
+++ b/.github/workflows/build_linux.yaml
@ -20,19 +20,31 @@ on:
                type: string
                required: true
                description: "How to link sqlcipher, one of 'system' | 'static'"
-            docker-image:
-                type: string
-                required: false
-                description: "The docker image to use for the build, defaults to ghcr.io/element-hq/element-desktop-dockerbuild"
+env:
+    SQLCIPHER_BUNDLED: ${{ inputs.sqlcipher == 'static' && '1' || '' }}
 jobs:
-    build:
-        runs-on: ubuntu-latest
-        container:
-            image: ${{ inputs.docker-image || format('ghcr.io/element-hq/element-desktop-dockerbuild:{0}', github.ref_name == 'master' && 'master' || 'develop') }}
-        defaults:
-            run:
-                shell: bash
+    # We build the hak files on native infrastructure as matrix-seshat fails to cross-compile properly
+    # https://github.com/matrix-org/seshat/issues/135
+    hak:
+        runs-on: ${{ inputs.arch == 'arm64' && 'dind-l-arm64' || 'ubuntu-latest' }}
+        env:
+            HAK_DOCKER_IMAGE: ${{ format('ghcr.io/element-hq/element-desktop-dockerbuild:{0}', github.event_name == 'pull_request' && 'develop' || github.ref_name) }}
+        outputs:
+            cache-key: ${{ steps.cache-key.outputs.key }}
+            arch: ${{ steps.config.outputs.arch }}
+            build-args: ${{ steps.config.outputs.build-args }}
        steps:
+            # Workaround for self-hosted runners lacking tools
+            - name: Install missing tools
+              if: runner.environment == 'self-hosted'
+              run: |
+                  sudo apt-get -qq update
+                  # curl for yarn download, git for tj-actions/changed-files, zstd for actions/cache
+                  sudo apt-get install -y curl git zstd
+                  curl -fsSL --create-dirs -o $HOME/bin/yarn https://github.com/yarnpkg/yarn/releases/download/v1.22.19/yarn-1.22.19.js
+                  chmod +x $HOME/bin/yarn
+                  echo "$HOME/bin" >> $GITHUB_PATH
+
            - uses: nbucic/variable-mapper@0673f6891a0619ba7c002ecfed0f9f4f39017b6f
              id: config
              with:
@ -57,11 +69,18 @@ jobs:
              with:
                  name: webapp

+            - name: Calculate cache key
+              id: cache-key
+              run: |
+                  echo "key=$CACHE_KEY" >> $GITHUB_OUTPUT
+              env:
+                  CACHE_KEY: ${{ runner.os }}-${{ github.ref_name }}-${{ inputs.sqlcipher }}-${{ inputs.arch }}-${{ hashFiles('hakHash', 'electronVersion', 'dockerbuild/*') }}
+
            - name: Cache .hak
              id: cache
              uses: actions/cache@v4
              with:
-                  key: ${{ runner.os }}-${{ inputs.docker-image || github.ref_name }}-${{ inputs.sqlcipher }}-${{ inputs.arch }}-${{ hashFiles('hakHash', 'electronVersion') }}
+                  key: ${{ steps.cache-key.outputs.key }}
                  path: |
                      ./.hak

@ -77,27 +96,73 @@ jobs:
            - name: Install Deps
              run: "yarn install --frozen-lockfile"

-            - name: Prepare for static sqlcipher build
-              if: inputs.sqlcipher == 'static'
-              run: |
-                  echo "SQLCIPHER_BUNDLED=1" >> $GITHUB_ENV
+            - name: "Get modified files"
+              id: changed_files
+              if: steps.cache.outputs.cache-hit != 'true' && github.event_name == 'pull_request'
+              uses: tj-actions/changed-files@635f118699dd888d737c15018cd30aff2e0274f8 # v44
+              with:
+                  files: |
+                      dockerbuild/**

-            # Ideally the docker image would be ready for cross-compilation but libsqlcipher-dev is not Multi-Arch compatible
-            # https://unix.stackexchange.com/a/349359
-            - name: Prepare for cross compilation
-              if: steps.cache.outputs.cache-hit != 'true' && inputs.arch == 'arm64'
-              run: |
-                  set -x
-                  dpkg --add-architecture arm64
-                  apt-get -qq update
-                  apt-get -qq install --no-install-recommends crossbuild-essential-arm64 libsqlcipher-dev:arm64 libssl-dev:arm64 libsecret-1-dev:arm64
-                  rustup target add aarch64-unknown-linux-gnu
-                  mv dockerbuild/aarch64/.cargo .
-                  cat dockerbuild/aarch64/.env >> $GITHUB_ENV
+            # This allows contributors to test changes to the dockerbuild image within a pull request
+            - name: Build docker image
+              uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5
+              if: steps.changed_files.outputs.any_modified == 'true'
+              with:
+                  context: dockerbuild
+                  load: true
+                  platforms: linux/${{ inputs.arch }}
+                  tags: ${{ env.HAK_DOCKER_IMAGE }}

            - name: Build Natives
              if: steps.cache.outputs.cache-hit != 'true'
-              run: "yarn build:native --target ${{ steps.config.outputs.target }}"
+              run: |
+                  docker run \
+                    -v ${{ github.workspace }}:/work -w /work \
+                    -e SQLCIPHER_BUNDLED \
+                    $HAK_DOCKER_IMAGE \
+                    yarn build:native
+
+            - name: Check native libraries
+              run: |
+                  shopt -s globstar
+
+                  for filename in ./.hak/hakModules/**/*.node; do
+                      ./scripts/glibc-check.sh $filename
+                  done
+              env:
+                  MAX_VER: 2.28 # buster-era glibc
+
+    build:
+        needs: hak
+        runs-on: ubuntu-latest
+        steps:
+            - uses: actions/checkout@v4
+
+            - uses: actions/download-artifact@v4
+              with:
+                  name: webapp
+
+            - name: Load .hak
+              id: cache
+              uses: actions/cache/restore@v4
+              with:
+                  key: ${{ needs.hak.outputs.cache-key }}
+                  fail-on-cache-miss: true
+                  path: |
+                      ./.hak
+
+            - uses: actions/setup-node@v4
+              with:
+                  node-version-file: package.json
+                  cache: "yarn"
+              env:
+                  # Workaround for https://github.com/actions/setup-node/issues/317
+                  FORCE_COLOR: 0
+
+            # Does not need branch matching as only analyses this layer
+            - name: Install Deps
+              run: "yarn install --frozen-lockfile"

            - name: "[Nightly] Resolve version"
              if: inputs.version != ''
@ -111,8 +176,7 @@ jobs:
                  fi

            - name: Build App
-              run: |
-                  yarn build --publish never -l ${{ steps.config.outputs.build-args }}
+              run: yarn build --publish never -l ${{ needs.hak.outputs.build-args }}

            - name: Check native libraries
              run: |
@ -139,9 +203,8 @@ jobs:
                  fi

                  ./scripts/glibc-check.sh dist/linux-*unpacked/element-desktop*
-                  ./scripts/glibc-check.sh dist/linux-*unpacked/resources/app.asar.unpacked/node_modules/matrix-seshat/index.node
              env:
-                  ARCH: ${{ steps.config.outputs.arch }}
+                  ARCH: ${{ needs.hak.outputs.arch }}

            # We exclude *-unpacked as it loses permissions and the tarball contains it with correct permissions
            - name: Upload Artifacts
--- a/.github/workflows/dockerbuild.yaml
+++ b/.github/workflows/dockerbuild.yaml
@ -2,7 +2,7 @@ name: Dockerbuild
 on:
    workflow_dispatch: {}
    push:
-        branches: [master, develop]
+        branches: [master, staging, develop]
        paths:
            - "dockerbuild/**"
 concurrency: ${{ github.workflow }}-${{ github.ref_name }}
@ -19,6 +19,14 @@ jobs:
        steps:
            - uses: actions/checkout@v4

+            - name: Set up QEMU
+              uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3
+
+            - name: Set up Docker Buildx
+              uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3
+              with:
+                  install: true
+
            - name: Log in to the Container registry
              uses: docker/login-action@5f4866a30a54f16a52d2ecb4a3898e9e424939cf
              with:
@ -41,3 +49,4 @@ jobs:
                  push: true
                  tags: ${{ steps.meta.outputs.tags }}
                  labels: ${{ steps.meta.outputs.labels }}
+                  platforms: linux/amd64,linux/arm64
--- a/dockerbuild/Dockerfile
+++ b/dockerbuild/Dockerfile
@ -1,47 +1,25 @@
-# Docker image to facilitate building Element Desktop with native bits using a glibc version with broader compatibility
-FROM buildpack-deps:buster-curl
+# Docker image to facilitate building Element Desktop's native bits using a glibc version with broader compatibility
+FROM rust:buster

 ENV DEBIAN_FRONTEND noninteractive

 RUN curl --proto "=https" -L https://yarnpkg.com/latest.tar.gz | tar xvz && mv yarn-* /yarn && ln -s /yarn/bin/yarn /usr/bin/yarn
 RUN apt-get -qq update && apt-get -qq dist-upgrade && \
-  # add repo for git-lfs
-  curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash && \
-  # git ssh for using as docker image on CircleCI
-  # python for node-gyp
-  # rpm is required for FPM to build rpm package
+  apt-get -qq install --no-install-recommends \
  # tclsh is required for building SQLite as part of SQLCipher
+  tcl \
  # libsecret-1-dev is required even for prebuild keytar
-  apt-get -qq install --no-install-recommends qtbase5-dev bsdtar build-essential autoconf libssl-dev gcc-multilib g++-multilib lzip rpm python libcurl4 git git-lfs ssh unzip tcl \
  libsecret-1-dev \
-  libopenjp2-tools \
-  # Used by github actions \
-  jq grep file \
  # Used by seshat (when not SQLCIPHER_STATIC) \
  libsqlcipher-dev && \
-  # git-lfs
-  git lfs install && \
  apt-get purge -y --auto-remove && rm -rf /var/lib/apt/lists/*
-
-WORKDIR /project
-
-# fix error /usr/local/bundle/gems/fpm-1.5.0/lib/fpm/package/freebsd.rb:72:in `encode': "\xE2" from ASCII-8BIT to UTF-8 (Encoding::UndefinedConversionError)
-# http://jaredmarkell.com/docker-and-locales/
-# http://askubuntu.com/a/601498
-ENV LANG C.UTF-8
-ENV LANGUAGE C.UTF-8
-ENV LC_ALL C.UTF-8
+RUN ln -s /usr/bin/python3 /usr/bin/python & ln -s /usr/bin/pip3 /usr/bin/pip

 ENV DEBUG_COLORS true
 ENV FORCE_COLOR true
+
 ENV NODE_VERSION 18.19.0
-
-# this package is used for snapcraft and we should not clear apt list - to avoid apt-get update during snap build
-RUN curl --proto "=https" -L https://nodejs.org/dist/v$NODE_VERSION/node-v$NODE_VERSION-linux-x64.tar.gz | tar xz -C /usr/local --strip-components=1 && \
-  unlink /usr/local/CHANGELOG.md && unlink /usr/local/LICENSE && unlink /usr/local/README.md
-
-ENV RUSTUP_HOME=/usr/local/rustup \
-    CARGO_HOME=/usr/local/cargo \
-    PATH=/usr/local/cargo/bin:$PATH
-
-RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path --profile minimal
+ARG TARGETOS
+ARG TARGETARCH
+COPY setup.sh /setup.sh
+RUN /setup.sh
--- a/dockerbuild/aarch64/.cargo/config.toml
+++ b/dockerbuild/aarch64/.cargo/config.toml
@ -1,3 +0,0 @@
-[target.aarch64-unknown-linux-gnu]
-linker = "aarch64-linux-gnu-gcc"
-rustflags = ["-L/usr/lib/aarch64-linux-gnu"]
--- a/dockerbuild/aarch64/.env
+++ b/dockerbuild/aarch64/.env
@ -1,11 +0,0 @@
-AS=/usr/bin/aarch64-linux-gnu-as
-STRIP=/usr/bin/aarch64-linux-gnu-strip
-AR=/usr/bin/aarch64-linux-gnu-ar
-CC=/usr/bin/aarch64-linux-gnu-gcc
-CPP=/usr/bin/aarch64-linux-gnu-cpp
-CXX=/usr/bin/aarch64-linux-gnu-g++
-LD=/usr/bin/aarch64-linux-gnu-ld
-FC=/usr/bin/aarch64-linux-gnu-gfortran
-PKG_CONFIG_PATH=/usr/lib/aarch64-linux-gnu/pkgconfig
-CFLAGS=-L/usr/lib/aarch64-linux-gnu
-RUSTFLAGS=-L/usr/lib/aarch64-linux-gnu
--- a/dockerbuild/setup.sh
+++ b/dockerbuild/setup.sh
@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -x
+declare -A archMap=(["amd64"]="x64" ["arm64"]="arm64")
+ARCH="${archMap["$TARGETARCH"]}"
+curl --proto "=https" -L "https://nodejs.org/dist/v$NODE_VERSION/node-v$NODE_VERSION-$TARGETOS-$ARCH.tar.gz" | tar xz -C /usr/local --strip-components=1 && \
+  unlink /usr/local/CHANGELOG.md && unlink /usr/local/LICENSE && unlink /usr/local/README.md
--- a/scripts/glibc-check.sh
+++ b/scripts/glibc-check.sh
@ -36,7 +36,7 @@ vercomp() {

 IFS="
 "
-VERS=$(objdump -T "$BINARY" | grep GLIBC | sed 's/.*GLIBC_\([.0-9]*\).*/\1/g' | sort -u)
+VERS=$(objdump -T "$BINARY" | grep GLIBC_ | sed 's/.*GLIBC_\([.0-9]*\).*/\1/g' | sort -u)

 for VER in $VERS; do
  vercomp "$VER" "$MAX_VER"