diff --git a/docs/labs.md b/docs/labs.md index 8889f929a0..cb797b61ab 100644 --- a/docs/labs.md +++ b/docs/labs.md @@ -73,3 +73,11 @@ improves the device verification experience by allowing you to verify a user instead of verifying each of their devices. This feature is still in development and will be landing in several chunks. + +## Event indexing and E2EE search support using Seshat (`feature_event_indexing`) + +Adds support for search in E2E encrypted rooms. This enables an event indexer +that downloads, stores, and indexes room messages for E2E encrypted rooms. + +The existing search will transparently work for encrypted rooms just like it +does for non-encrypted. diff --git a/docs/native-node-modules.md b/docs/native-node-modules.md new file mode 100644 index 0000000000..6e3dce4525 --- /dev/null +++ b/docs/native-node-modules.md @@ -0,0 +1,59 @@ +# Native Node Modules + +For some features, the desktop version of Riot can make use of native Node +modules. These allow Riot to integrate with the desktop in ways that a browser +cannot. + +While native modules enable powerful new features, they must be complied for +each operating system. For official Riot releases, we will always build these +modules from source to ensure we can trust the compiled output. In the future, +we may offer a pre-compiled path for those who want to use these features in a +custom build of Riot without installing the various build tools required. + +Do note that compiling a module for a particular operating system +(Linux/macOS/Windows) will need to be done on that operating system. +Cross-compiling from a host OS for a different target OS may be possible, but +we don't support this flow with Riot dependencies at this time. + +At the moment, we need to make some changes to the Riot release process before +we can support native Node modules at release time, so these features are +currently disabled by default until that is resolved. The following sections +explain the manual steps you can use with a custom build of Riot to enable +these features if you'd like to try them out. + +## Adding Seshat for search in E2E encrypted rooms + +Seshat is a native Node module that adds support for local event indexing and +full text search in E2E encrypted rooms. + +Since Seshat is written in Rust, the Rust compiler and related tools need to be +installed before installing Seshat itself. To install Rust please consult the +official Rust [documentation](https://www.rust-lang.org/tools/install). + +Seshat also depends on the SQLCipher library to store its data in encrypted form +on disk. You'll need to install it via your OS package manager. + +After installing the Rust compiler and SQLCipher, Seshat support can be added +using yarn inside the `electron_app/` directory: + + yarn add matrix-seshat + +After this is done the Electron version of Riot can be run from the main folder +as usual using: + + yarn electron + +If for some reason recompilation of Seshat is needed, e.g. when using a +development version of Seshat using `yarn link`, or if the initial compilation was +done for the wrong electron version, Seshat can be recompiled with the +`electron-build-env` tool. Again from the `electron_app/` directory: + + yarn add electron-build-env + +Recompiling Seshat itself can be done like so: + + yarn run electron-build-env -- --electron 6.1.1 -- neon build matrix-seshat --release` + +Please make sure to include all the `--` as well as the `--release` command line +switch at the end. Modify your electron version accordingly depending on the +version that is installed on your system. diff --git a/electron_app/src/electron-main.js b/electron_app/src/electron-main.js index b928900410..0c4b6fd510 100644 --- a/electron_app/src/electron-main.js +++ b/electron_app/src/electron-main.js @@ -40,6 +40,16 @@ const { migrateFromOldOrigin } = require('./originMigrator'); const windowStateKeeper = require('electron-window-state'); const Store = require('electron-store'); +const fs = require('fs'); +const afs = fs.promises; + +let Seshat = null; + +try { + Seshat = require('matrix-seshat'); +} catch (e) { +} + if (argv["help"]) { console.log("Options:"); console.log(" --profile-dir {path}: Path to where to store the profile."); @@ -94,8 +104,11 @@ try { // Could not load local config, this is expected in most cases. } +const eventStorePath = path.join(app.getPath('userData'), 'EventStore'); const store = new Store({ name: "electron-config" }); +let eventIndex = null; + let mainWindow = null; global.appQuitting = false; @@ -225,6 +238,7 @@ ipcMain.on('ipcCall', async function(ev, payload) { case 'getConfig': ret = vectorConfig; break; + default: mainWindow.webContents.send('ipcReply', { id: payload.id, @@ -239,6 +253,154 @@ ipcMain.on('ipcCall', async function(ev, payload) { }); }); +ipcMain.on('seshat', async function(ev, payload) { + if (!mainWindow) return; + + const sendError = (id, e) => { + const error = { + message: e.message + } + + mainWindow.webContents.send('seshatReply', { + id:id, + error: error + }); + } + + const args = payload.args || []; + let ret; + + switch (payload.name) { + case 'supportsEventIndexing': + if (Seshat === null) ret = false; + else ret = true; + break; + + case 'initEventIndex': + if (eventIndex === null) { + try { + await afs.mkdir(eventStorePath, {recursive: true}); + eventIndex = new Seshat(eventStorePath, {passphrase: "DEFAULT_PASSPHRASE"}); + } catch (e) { + sendError(payload.id, e); + return; + } + } + break; + + case 'closeEventIndex': + eventIndex = null; + break; + + case 'deleteEventIndex': + const deleteFolderRecursive = async(p) => { + for (let entry of await afs.readdir(p)) { + const curPath = path.join(p, entry); + await afs.unlink(curPath); + } + } + + try { + await deleteFolderRecursive(eventStorePath); + } catch (e) { + } + + break; + + case 'isEventIndexEmpty': + if (eventIndex === null) ret = true; + else ret = await eventIndex.isEmpty(); + break; + + case 'addEventToIndex': + try { + eventIndex.addEvent(args[0], args[1]); + } catch (e) { + sendError(payload.id, e); + return; + } + break; + + case 'commitLiveEvents': + try { + ret = await eventIndex.commit(); + } catch (e) { + sendError(payload.id, e); + return; + } + break; + + case 'searchEventIndex': + try { + ret = await eventIndex.search(args[0]); + } catch (e) { + sendError(payload.id, e); + return; + } + break; + + case 'addHistoricEvents': + if (eventIndex === null) ret = false; + else { + try { + ret = await eventIndex.addHistoricEvents( + args[0], args[1], args[2]); + } catch (e) { + sendError(payload.id, e); + return; + } + } + break; + + case 'removeCrawlerCheckpoint': + if (eventIndex === null) ret = false; + else { + try { + ret = await eventIndex.removeCrawlerCheckpoint(args[0]); + } catch (e) { + sendError(payload.id, e); + return; + } + } + break; + + case 'addCrawlerCheckpoint': + if (eventIndex === null) ret = false; + else { + try { + ret = await eventIndex.addCrawlerCheckpoint(args[0]); + } catch (e) { + sendError(payload.id, e); + return; + } + } + break; + + case 'loadCheckpoints': + if (eventIndex === null) ret = []; + else { + try { + ret = await eventIndex.loadCheckpoints(); + } catch (e) { + ret = []; + } + } + break; + + default: + mainWindow.webContents.send('seshatReply', { + id: payload.id, + error: "Unknown IPC Call: " + payload.name, + }); + return; + } + + mainWindow.webContents.send('seshatReply', { + id: payload.id, + reply: ret, + }); +}); + app.commandLine.appendSwitch('--enable-usermedia-screen-capturing'); const gotLock = app.requestSingleInstanceLock(); diff --git a/riot.im/develop/config.json b/riot.im/develop/config.json index 1d68c6ca5f..40c3d29698 100644 --- a/riot.im/develop/config.json +++ b/riot.im/develop/config.json @@ -20,7 +20,8 @@ "feature_many_integration_managers": "labs", "feature_mjolnir": "labs", "feature_dm_verification": "labs", - "feature_cross_signing": "labs" + "feature_cross_signing": "labs", + "feature_event_indexing": "labs" }, "welcomeUserId": "@riot-bot:matrix.org", "piwik": { diff --git a/src/vector/platform/ElectronPlatform.js b/src/vector/platform/ElectronPlatform.js index 66f81a3582..eb5b2a86a5 100644 --- a/src/vector/platform/ElectronPlatform.js +++ b/src/vector/platform/ElectronPlatform.js @@ -20,6 +20,7 @@ limitations under the License. */ import VectorBasePlatform, {updateCheckStatusEnum} from './VectorBasePlatform'; +import BaseEventIndexManager from 'matrix-react-sdk/lib/indexing/BaseEventIndexManager'; import dis from 'matrix-react-sdk/lib/dispatcher'; import { _t } from 'matrix-react-sdk/lib/languageHandler'; import rageshake from 'matrix-react-sdk/lib/rageshake/rageshake'; @@ -65,12 +66,104 @@ function getUpdateCheckStatus(status) { } } +class SeshatIndexManager extends BaseEventIndexManager { + constructor() { + super(); + + this._pendingIpcCalls = {}; + this._nextIpcCallId = 0; + ipcRenderer.on('seshatReply', this._onIpcReply.bind(this)); + } + + async _ipcCall(name: string, ...args: []): Promise<{}> { + // TODO this should be moved into the preload.js file. + const ipcCallId = ++this._nextIpcCallId; + return new Promise((resolve, reject) => { + this._pendingIpcCalls[ipcCallId] = {resolve, reject}; + window.ipcRenderer.send('seshat', {id: ipcCallId, name, args}); + }); + } + + _onIpcReply(ev: {}, payload: {}) { + if (payload.id === undefined) { + console.warn("Ignoring IPC reply with no ID"); + return; + } + + if (this._pendingIpcCalls[payload.id] === undefined) { + console.warn("Unknown IPC payload ID: " + payload.id); + return; + } + + const callbacks = this._pendingIpcCalls[payload.id]; + delete this._pendingIpcCalls[payload.id]; + if (payload.error) { + callbacks.reject(payload.error); + } else { + callbacks.resolve(payload.reply); + } + } + + async supportsEventIndexing(): Promise { + return this._ipcCall('supportsEventIndexing'); + } + + async initEventIndex(): Promise<> { + return this._ipcCall('initEventIndex'); + } + + async addEventToIndex(ev: MatrixEvent, profile: MatrixProfile): Promise<> { + return this._ipcCall('addEventToIndex', ev, profile); + } + + async isEventIndexEmpty(): Promise { + return this._ipcCall('isEventIndexEmpty'); + } + + async commitLiveEvents(): Promise<> { + return this._ipcCall('commitLiveEvents'); + } + + async searchEventIndex(searchConfig: SearchConfig): Promise { + return this._ipcCall('searchEventIndex', searchConfig); + } + + async addHistoricEvents( + events: [HistoricEvent], + checkpoint: CrawlerCheckpoint | null, + oldCheckpoint: CrawlerCheckpoint | null, + ): Promise<> { + return this._ipcCall('addHistoricEvents', events, checkpoint, oldCheckpoint); + } + + async addCrawlerCheckpoint(checkpoint: CrawlerCheckpoint): Promise<> { + return this._ipcCall('addCrawlerCheckpoint', checkpoint); + } + + async removeCrawlerCheckpoint(checkpoint: CrawlerCheckpoint): Promise<> { + return this._ipcCall('removeCrawlerCheckpoint', checkpoint); + } + + async loadCheckpoints(): Promise<[CrawlerCheckpoint]> { + return this._ipcCall('loadCheckpoints'); + } + + async closeEventIndex(): Promise<> { + return this._ipcCall('closeEventIndex'); + } + + async deleteEventIndex(): Promise<> { + return this._ipcCall('deleteEventIndex'); + } +} + export default class ElectronPlatform extends VectorBasePlatform { constructor() { super(); this._pendingIpcCalls = {}; this._nextIpcCallId = 0; + this.eventIndexManager = new SeshatIndexManager(); dis.register(_onAction); /* @@ -294,4 +387,8 @@ export default class ElectronPlatform extends VectorBasePlatform { callbacks.resolve(payload.reply); } } + + getEventIndexingManager(): BaseEventIndexManager | null { + return this.eventIndexManager; + } }