From b94c52c2444fe35cfb3d304092fba7daec23bf2f Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 21 Feb 2020 18:17:31 +0000 Subject: [PATCH] Support fetching the latest develop build Frustratingly, this requires a buildkite API token, even though the builds & artifacts are available without auth on the buildkite website. --- package.json | 1 - scripts/fetch-package.js | 117 +++++++++++++++++++++++++++++++-------- yarn.lock | 9 +-- 3 files changed, 94 insertions(+), 33 deletions(-) diff --git a/package.json b/package.json index 5b9ffe7..a52119c 100644 --- a/package.json +++ b/package.json @@ -46,7 +46,6 @@ "eslint-config-google": "^0.7.1", "eslint-plugin-babel": "^4.1.2", "find-npm-prefix": "^1.0.2", - "follow-redirects": "^1.9.0", "fs-extra": "^8.1.0", "glob": "^7.1.6", "matrix-js-sdk": "^2.4.6-rc.1", diff --git a/scripts/fetch-package.js b/scripts/fetch-package.js index 621cf0e..9462d98 100755 --- a/scripts/fetch-package.js +++ b/scripts/fetch-package.js @@ -4,10 +4,10 @@ const process = require('process'); const path = require('path'); const fs = require('fs'); const fsPromises = require('fs').promises; -const { https } = require('follow-redirects'); const childProcess = require('child_process'); const tar = require('tar'); const asar = require('asar'); +const needle = require('needle'); const riotDesktopPackageJson = require('../package.json'); @@ -15,30 +15,84 @@ const PUB_KEY_URL = "https://packages.riot.im/riot-release-key.asc"; const PACKAGE_URL_PREFIX = "https://github.com/vector-im/riot-web/releases/download/"; const ASAR_PATH = 'webapp.asar'; +async function getLatestDevelopUrl(bkToken) { + const buildsResult = await needle('get', + "https://api.buildkite.com/v2/organizations/matrix-dot-org/pipelines/riot-web/builds", + { + branch: 'develop', + state: 'passed', + per_page: 1, + }, + { + headers: { + authorization: "Bearer " + bkToken, + }, + }, + ); + const latestBuild = buildsResult.body[0]; + console.log("Latest build is " + latestBuild.number); + let artifactUrl; + for (const job of latestBuild.jobs) { + // Strip any colon-form emoji from the build name + if (job.name && job.name.replace(/:\w*:\s*/, '') === 'Package') { + artifactUrl = job.artifacts_url; + break; + } + } + if (artifactUrl === undefined) { + throw new Error("Couldn't find artifact URL - has the name of the package job changed?"); + } + + const artifactsResult = await needle('get', artifactUrl, {}, + { + headers: { + authorization: "Bearer " + bkToken, + }, + }, + ); + let dlUrl; + let dlFilename; + for (const artifact of artifactsResult.body) { + if (artifact.filename && /^riot-.*\.tar.gz$/.test(artifact.filename)) { + dlUrl = artifact.download_url; + dlFilename = artifact.filename; + break; + } + } + if (dlUrl === undefined) { + throw new Error("Couldn't find artifact download URL - has the artifact filename changed?"); + } + console.log("Fetching artifact URL..."); + const dlResult = await needle('get', dlUrl, {}, + { + headers: { + authorization: "Bearer " + bkToken, + }, + // This URL will give us a Location header, but will also give us + // a JSON object with the direct URL. We'll take the URL and pass it + // back, then we can easily support specifying a URL directly. + follow_max: 0, + }, + ); + return [dlFilename, dlResult.body.url]; +} + async function downloadToFile(url, filename) { console.log("Downloading " + url + "..."); - const outStream = await fs.createWriteStream(filename); - return new Promise((resolve, reject) => { - https.get(url, (resp) => { - if (resp.statusCode / 100 !== 2) { - reject("Download failed: " + resp.statusCode); - return; - } - - resp.on('data', (chunk) => { - outStream.write(chunk); - }); - resp.on('end', (chunk) => { - outStream.end(); - resolve(); - }); - }); - }).catch(async (e) => { - outStream.end(); - await fsPromises.unlink(filename); + try { + const bob = await needle('get', url, null, + { + follow_max: 5, + output: filename, + }, + ); + } catch (e) { + try { + await fsPromises.unlink(filename); + } catch (_) {} throw e; - }); + } } async function verifyFile(filename) { @@ -60,6 +114,8 @@ async function main() { let deployDir = 'deploys'; let cfgDir; let targetVersion; + let filename; + let url; while (process.argv.length > 2) { switch (process.argv[2]) { @@ -90,6 +146,21 @@ async function main() { if (targetVersion === undefined) { targetVersion = 'v' + riotDesktopPackageJson.version; + } else if (targetVersion === 'develop') { + const buildKiteApiKey = process.env.BUILDKITE_API_KEY; + if (buildKiteApiKey === undefined) { + console.log("Set BUILDKITE_API_KEY to fetch latest develop version"); + console.log( + "Sorry - Buildkite's API requires authentication to access builds, " + + "even if those builds are accessible on the web with no auth.", + ); + process.exit(1); + } + [filename, url] = await getLatestDevelopUrl(buildKiteApiKey); + verify = false; // develop builds aren't signed + } else { + filename = 'riot-' + targetVersion + '.tar.gz'; + url = PACKAGE_URL_PREFIX + targetVersion + '/' + filename; } const haveGpg = await new Promise((resolve) => { @@ -138,7 +209,7 @@ async function main() { } let haveDeploy = false; - const expectedDeployDir = path.join(deployDir, 'riot-' + targetVersion); + const expectedDeployDir = path.join(deployDir, path.basename(filename).replace(/\.tar\.gz/, '')); try { await fs.opendir(expectedDeployDir); console.log(expectedDeployDir + "already exists"); @@ -147,9 +218,7 @@ async function main() { } if (!haveDeploy) { - const filename = 'riot-' + targetVersion + '.tar.gz'; const outPath = path.join(pkgDir, filename); - const url = PACKAGE_URL_PREFIX + targetVersion + '/' + filename; try { await fsPromises.stat(outPath); console.log("Already have " + filename + ": not redownloading"); diff --git a/yarn.lock b/yarn.lock index 537800b..12fec42 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1065,7 +1065,7 @@ debug@3.1.0: dependencies: ms "2.0.0" -debug@^3.0.0, debug@^3.1.0, debug@^3.2.6: +debug@^3.1.0, debug@^3.2.6: version "3.2.6" resolved "https://registry.yarnpkg.com/debug/-/debug-3.2.6.tgz#e83d17de16d8a7efb7717edbe5fb10135eee629b" integrity sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ== @@ -1713,13 +1713,6 @@ flush-write-stream@^1.0.0: inherits "^2.0.3" readable-stream "^2.3.6" -follow-redirects@^1.9.0: - version "1.9.0" - resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.9.0.tgz#8d5bcdc65b7108fe1508649c79c12d732dcedb4f" - integrity sha512-CRcPzsSIbXyVDl0QI01muNDu69S8trU4jArW9LpOt2WtC6LyUJetcIrmfHsRBx7/Jb6GHJUiuqyYxPooFfNt6A== - dependencies: - debug "^3.0.0" - forever-agent@~0.6.1: version "0.6.1" resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91"