First attempt at automated Playwright tests (#402)

* First attempt at automated Playwright tests

* Fix deb installation in CI

* Try again with .deb

* Try xvfb

* Fix paths

* Prepare macOS runner for UI tests

* test_artifacts

* Increase test timeout to 30s

* Add timeout

* Split out build & test stages for easier retrying

* Use a single test matrix job

* Fix testing

* Yarn install

* Install deps

* Make prep more generic

* Update build.yaml

* Update build.yaml

* Update build.yaml

.github/workflows/build.yaml

@@ -1,4 +1,4 @@
-name: Build
+name: Build and Test
 on:
   pull_request: { }
   push:
@@ -154,6 +154,9 @@ jobs:
       - name: Build App
         run: "yarn build --publish never"
 
+      - name: Install .deb
+        run: "sudo apt install ./dist/*.deb"
+
       - name: Upload Artifacts
         uses: actions/upload-artifact@v3
         with:
@@ -205,3 +208,71 @@ jobs:
           name: macos
           path: dist
           retention-days: 1
+
+  test:
+    needs:
+      - macos
+      - linux
+      - windows
+    strategy:
+      matrix:
+        include:
+# Disable macOS tests for now, they fail to run in CI, needs investigation.
+#          - name: macOS Universal
+#            os: macos
+#            artifact: macos
+#            executable: "./dist/mac-universal/Element.app/Contents/MacOS/Element"
+#            prepare_cmd: "chmod +x ./dist/mac-universal/Element.app/Contents/MacOS/Element"
+          - name: 'Linux (sqlcipher: system)'
+            os: ubuntu
+            artifact: linux-sqlcipher-system
+            executable: "element-desktop"
+            prepare_cmd: "sudo apt install ./dist/*.deb"
+          - name: 'Linux (sqlcipher: static)'
+            os: ubuntu
+            artifact: linux-sqlcipher-static
+            executable: "element-desktop"
+            prepare_cmd: "sudo apt install ./dist/*.deb"
+          - name: Windows (x86)
+            os: windows
+            artifact: win-x86
+            executable: "./dist/win-ia32-unpacked/Element.exe"
+          - name: Windows (x64)
+            os: windows
+            artifact: win-x64
+            executable: "./dist/win-unpacked/Element.exe"
+    name: Test ${{ matrix.name }}
+    runs-on: ${{ matrix.os }}-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-node@v3
+        with:
+          cache: "yarn"
+
+      - name: Install Deps
+        run: "yarn install --pure-lockfile"
+
+      - uses: actions/download-artifact@v3
+        with:
+          name: ${{ matrix.artifact }}
+          path: dist
+
+      - name: Prepare for tests
+        run: ${{ matrix.prepare_cmd }}
+        if: matrix.prepare_cmd
+
+      - name: Run tests
+        uses: GabrielBB/xvfb-action@v1
+        timeout-minutes: 5
+        with:
+          run: "yarn test"
+        env:
+          ELEMENT_DESKTOP_EXECUTABLE: ${{ matrix.executable }}
+
+      - name: Upload Artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ matrix.artifact }}
+          path: test_artifacts
+          retention-days: 1

.gitignore

@@ -13,3 +13,5 @@
 /.npmrc
 .vscode
 .vscode/
+/test_artifacts/
+/coverage/

babel.config.js

@@ -0,0 +1,6 @@
+module.exports = {
+    presets: [
+        ['@babel/preset-env', { targets: { node: 'current' } }],
+        '@babel/preset-typescript',
+    ],
+};

package.json

@@ -37,7 +37,8 @@
     "docker:install": "scripts/in-docker.sh yarn install",
     "debrepo": "scripts/mkrepo.sh",
     "clean": "rimraf webapp.asar dist packages deploys lib",
-    "hak": "ts-node scripts/hak/index.ts"
+    "hak": "ts-node scripts/hak/index.ts",
+    "test": "jest"
   },
   "dependencies": {
     "auto-launch": "^5.0.5",
@@ -49,8 +50,12 @@
     "request": "^2.88.2"
   },
   "devDependencies": {
+    "@babel/core": "^7.18.10",
+    "@babel/preset-env": "^7.18.10",
+    "@babel/preset-typescript": "^7.18.6",
     "@types/auto-launch": "^5.0.1",
     "@types/counterpart": "^0.18.1",
+    "@types/jest": "^28",
     "@types/minimist": "^1.2.1",
     "@types/mkdirp": "^1.0.2",
     "@types/pacote": "^11.1.1",
@@ -60,6 +65,7 @@
     "allchange": "^1.0.6",
     "app-builder-lib": "^22.14.10",
     "asar": "^2.0.1",
+    "babel-jest": "^28.1.3",
     "chokidar": "^3.5.2",
     "detect-libc": "^1.0.3",
     "electron": "^20",
@@ -71,16 +77,20 @@
     "eslint-config-google": "^0.14.0",
     "eslint-plugin-import": "^2.25.4",
     "eslint-plugin-matrix-org": "^0.4.0",
+    "expect-playwright": "^0.8.0",
     "find-npm-prefix": "^1.0.2",
     "fs-extra": "^8.1.0",
     "glob": "^7.1.6",
+    "jest": "^28",
     "matrix-web-i18n": "^1.3.0",
     "mkdirp": "^1.0.3",
     "needle": "^2.5.0",
     "node-pre-gyp": "^0.15.0",
     "pacote": "^11.3.5",
+    "playwright": "^1.25.0",
     "rimraf": "^3.0.2",
     "tar": "^6.1.2",
+    "ts-jest": "^28.0.8",
     "ts-node": "^10.4.0",
     "typescript": "4.5.5"
   },
@@ -144,5 +154,12 @@
         ]
       }
     ]
+  },
+  "jest": {
+    "testEnvironment": "node",
+    "testMatch": [
+      "<rootDir>/test/**/*-test.[jt]s?(x)"
+    ],
+    "setupFilesAfterEnv": ["expect-playwright"]
   }
 }

test/launch-test.ts

@@ -0,0 +1,64 @@
+/*
+Copyright 2022 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import * as os from "os";
+import * as fs from "fs";
+import * as path from "path";
+import "expect-playwright";
+import { _electron as electron } from "playwright";
+import { ElectronApplication, Page } from "playwright-core";
+
+describe("App launch", () => {
+    const artifactsPath = path.join(process.cwd(), "test_artifacts");
+    fs.mkdirSync(artifactsPath);
+
+    const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "element-desktop-tests"));
+    console.log("Using temp profile directory: ", tmpDir);
+
+    let app: ElectronApplication;
+    let window: Page;
+
+    beforeAll(async () => {
+        const args = ["--profile-dir", tmpDir];
+
+        const executablePath = process.env["ELEMENT_DESKTOP_EXECUTABLE"];
+        if (!executablePath) {
+            // Unpackaged mode testing
+            args.unshift("./lib/electron-main.js");
+        }
+
+        app = await electron.launch({
+            executablePath,
+            args,
+            recordVideo: {
+                dir: artifactsPath,
+            }
+        });
+        window = await app.firstWindow();
+    });
+
+    afterAll(async () => {
+        await app?.close();
+        fs.rmSync(tmpDir, { recursive: true });
+    });
+
+    it("should launch and render the welcome view successfully", async () => {
+        await window.locator("#matrixchat").waitFor();
+        await window.locator(".mx_Welcome").waitFor();
+        await expect(window).toMatchURL("vector://vector/webapp/#/welcome");
+        await window.screenshot({ path: path.join(artifactsPath, "welcome.png") });
+    }, 30000);
+});

tsconfig.json

@@ -14,9 +14,11 @@
     "lib": [
       "es2019",
       "dom"
-    ]
+    ],
+    "types": ["jest", "node"],
   },
   "include": [
-    "./src/**/*.ts"
+    "./src/**/*.ts",
+    "./tests/**/*.ts"
   ]
 }