Enable docker image caching for the deb build (#10431)

This commit is contained in:
Richard van der Hoff 2021-07-26 11:36:01 +01:00 committed by GitHub
parent ab82fd6ed1
commit f22252d4f9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 65 additions and 13 deletions

View file

@ -48,12 +48,43 @@ jobs:
distro: ${{ fromJson(needs.get-distros.outputs.distros) }} distro: ${{ fromJson(needs.get-distros.outputs.distros) }}
steps: steps:
- uses: actions/checkout@v2 - name: Checkout
uses: actions/checkout@v2
with: with:
path: src path: src
- uses: actions/setup-python@v2
- run: ./src/scripts-dev/build_debian_packages "${{ matrix.distro }}" - name: Set up Docker Buildx
- uses: actions/upload-artifact@v2 id: buildx
uses: docker/setup-buildx-action@v1
with:
install: true
- name: Set up docker layer caching
uses: actions/cache@v2
with:
path: /tmp/.buildx-cache
key: ${{ runner.os }}-buildx-${{ github.sha }}
restore-keys: |
${{ runner.os }}-buildx-
- name: Set up python
uses: actions/setup-python@v2
- name: Build the packages
# see https://github.com/docker/build-push-action/issues/252
# for the cache magic here
run: |
./src/scripts-dev/build_debian_packages \
--docker-build-arg=--cache-from=type=local,src=/tmp/.buildx-cache \
--docker-build-arg=--cache-to=type=local,mode=max,dest=/tmp/.buildx-cache-new \
--docker-build-arg=--progress=plain \
--docker-build-arg=--load \
"${{ matrix.distro }}"
rm -rf /tmp/.buildx-cache
mv /tmp/.buildx-cache-new /tmp/.buildx-cache
- name: Upload debs as artifacts
uses: actions/upload-artifact@v2
with: with:
name: debs name: debs
path: debs/* path: debs/*

1
changelog.d/10431.misc Normal file
View file

@ -0,0 +1 @@
Use a docker image cache for the prerequisites for the debian package build.

View file

@ -17,6 +17,7 @@ import subprocess
import sys import sys
import threading import threading
from concurrent.futures import ThreadPoolExecutor from concurrent.futures import ThreadPoolExecutor
from typing import Optional, Sequence
DISTS = ( DISTS = (
"debian:buster", "debian:buster",
@ -39,8 +40,11 @@ projdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
class Builder(object): class Builder(object):
def __init__(self, redirect_stdout=False): def __init__(
self, redirect_stdout=False, docker_build_args: Optional[Sequence[str]] = None
):
self.redirect_stdout = redirect_stdout self.redirect_stdout = redirect_stdout
self._docker_build_args = tuple(docker_build_args or ())
self.active_containers = set() self.active_containers = set()
self._lock = threading.Lock() self._lock = threading.Lock()
self._failed = False self._failed = False
@ -79,8 +83,8 @@ class Builder(object):
stdout = None stdout = None
# first build a docker image for the build environment # first build a docker image for the build environment
subprocess.check_call( build_args = (
[ (
"docker", "docker",
"build", "build",
"--tag", "--tag",
@ -89,8 +93,13 @@ class Builder(object):
"distro=" + dist, "distro=" + dist,
"-f", "-f",
"docker/Dockerfile-dhvirtualenv", "docker/Dockerfile-dhvirtualenv",
"docker", )
], + self._docker_build_args
+ ("docker",)
)
subprocess.check_call(
build_args,
stdout=stdout, stdout=stdout,
stderr=subprocess.STDOUT, stderr=subprocess.STDOUT,
cwd=projdir, cwd=projdir,
@ -147,9 +156,7 @@ class Builder(object):
self.active_containers.remove(c) self.active_containers.remove(c)
def run_builds(dists, jobs=1, skip_tests=False): def run_builds(builder, dists, jobs=1, skip_tests=False):
builder = Builder(redirect_stdout=(jobs > 1))
def sig(signum, _frame): def sig(signum, _frame):
print("Caught SIGINT") print("Caught SIGINT")
builder.kill_containers() builder.kill_containers()
@ -180,6 +187,11 @@ if __name__ == "__main__":
action="store_true", action="store_true",
help="skip running tests after building", help="skip running tests after building",
) )
parser.add_argument(
"--docker-build-arg",
action="append",
help="specify an argument to pass to docker build",
)
parser.add_argument( parser.add_argument(
"--show-dists-json", "--show-dists-json",
action="store_true", action="store_true",
@ -195,4 +207,12 @@ if __name__ == "__main__":
if args.show_dists_json: if args.show_dists_json:
print(json.dumps(DISTS)) print(json.dumps(DISTS))
else: else:
run_builds(dists=args.dist, jobs=args.jobs, skip_tests=args.no_check) builder = Builder(
redirect_stdout=(args.jobs > 1), docker_build_args=args.docker_build_arg
)
run_builds(
builder,
dists=args.dist,
jobs=args.jobs,
skip_tests=args.no_check,
)