chore: unify desktop and mobile databases (#764)

* chore: unify desktop and mobile databases

- migrate `package:sqflite_flutter` to `sqlcipher_flutter_libs`
- use FFI for all SQLite operations
- use `SQfLiteEncryptionHelper` for database encryption
- enforce encryption for new SQLite datbase implementation
- migrate existing SQLite databases
 - encrypt unencrypted ones
 - migrate database locations to unified approach
- drop dependency on sqlite

Signed-off-by: The one with the braid <info@braid.business>

* chore: add sqlcipher to macos CI

Signed-off-by: The one with the braid <info@braid.business>

---------

Signed-off-by: The one with the braid <info@braid.business>

.github/workflows/integrate.yaml

@@ -67,7 +67,7 @@ jobs:
           flutter-version: ${{ env.FLUTTER_VERSION }}
           cache: true
       - name: Install dependencies
-        run: sudo apt-get update && sudo apt-get install curl clang cmake ninja-build pkg-config libgtk-3-dev libblkid-dev liblzma-dev libjsoncpp-dev cmake-data libsecret-1-dev libsecret-1-0 librhash0 -y
+        run: sudo apt-get update && sudo apt-get install curl clang cmake ninja-build pkg-config libgtk-3-dev libblkid-dev liblzma-dev libjsoncpp-dev cmake-data libsecret-1-dev libsecret-1-0 librhash0 libssl-dev -y
       - run: flutter pub get
       - run: flutter build linux --target-platform linux-x64
 
@@ -84,5 +84,6 @@ jobs:
         uses: maxim-lobanov/setup-xcode@v1.6.0
         with:
           xcode-version: latest
+      - run: brew install sqlcipher
       - run: flutter pub get
       - run: flutter build ios --no-codesign

.github/workflows/release.yaml

@@ -113,7 +113,7 @@ jobs:
           flutter-version: ${{ env.FLUTTER_VERSION }}
           cache: true
       - name: Install dependencies
-        run: sudo apt-get update && sudo apt-get install curl clang cmake ninja-build pkg-config libgtk-3-dev libblkid-dev liblzma-dev libjsoncpp-dev cmake-data libsecret-1-dev libsecret-1-0 librhash0 -y
+        run: sudo apt-get update && sudo apt-get install curl clang cmake ninja-build pkg-config libgtk-3-dev libblkid-dev liblzma-dev libjsoncpp-dev cmake-data libsecret-1-dev libsecret-1-0 librhash0 libssl-dev -y
       - run: flutter pub get
       - run: flutter build linux --release --target-platform linux-x64
       - name: Create archive

ios/Podfile

@@ -38,6 +38,12 @@ post_install do |installer|
   installer.pods_project.targets.each do |target|
     flutter_additional_ios_build_settings(target)
     target.build_configurations.each do |config|
+      # ensure all dependencies are using SQLCipher instead of SQLite
+      xcconfig_path = config.base_configuration_reference.real_path
+      xcconfig = File.read(xcconfig_path)
+      new_xcconfig = xcconfig.sub(' -l"sqlite3"', '')
+      File.open(xcconfig_path, "w") { |file| file << new_xcconfig }
+
       config.build_settings['ENABLE_BITCODE'] = 'NO'
 
       # see https://github.com/flutter-webrtc/flutter-webrtc/issues/1054

lib/utils/client_manager.dart

@@ -18,7 +18,7 @@ import 'package:fluffychat/utils/custom_image_resizer.dart';
 import 'package:fluffychat/utils/init_with_restore.dart';
 import 'package:fluffychat/utils/matrix_sdk_extensions/flutter_hive_collections_database.dart';
 import 'package:fluffychat/utils/platform_infos.dart';
-import 'matrix_sdk_extensions/flutter_matrix_sdk_database_builder.dart';
+import 'matrix_sdk_extensions/flutter_matrix_dart_sdk_database/builder.dart';
 
 abstract class ClientManager {
   static const String clientNamespace = 'im.fluffychat.store.clients';

lib/utils/matrix_sdk_extensions/flutter_matrix_dart_sdk_database/builder.dart

@@ -0,0 +1,125 @@
+import 'dart:io';
+
+import 'package:flutter/foundation.dart';
+
+import 'package:flutter_gen/gen_l10n/l10n.dart';
+import 'package:matrix/matrix.dart';
+import 'package:path/path.dart';
+import 'package:path_provider/path_provider.dart';
+import 'package:sqflite_common_ffi/sqflite_ffi.dart';
+import 'package:universal_html/html.dart' as html;
+
+import 'package:fluffychat/config/app_config.dart';
+import 'package:fluffychat/utils/client_manager.dart';
+import 'package:fluffychat/utils/matrix_sdk_extensions/flutter_hive_collections_database.dart';
+import 'package:fluffychat/utils/platform_infos.dart';
+import 'cipher.dart';
+
+import 'sqlcipher_stub.dart'
+    if (dart.library.io) 'package:sqlcipher_flutter_libs/sqlcipher_flutter_libs.dart';
+
+Future<DatabaseApi> flutterMatrixSdkDatabaseBuilder(Client client) async {
+  MatrixSdkDatabase? database;
+  try {
+    database = await _constructDatabase(client);
+    await database.open();
+    return database;
+  } catch (e) {
+    // Try to delete database so that it can created again on next init:
+    database?.delete().catchError(
+          (e, s) => Logs().w(
+            'Unable to delete database, after failed construction',
+            e,
+            s,
+          ),
+        );
+
+    // Send error notification:
+    final l10n = lookupL10n(PlatformDispatcher.instance.locale);
+    ClientManager.sendInitNotification(
+      l10n.initAppError,
+      l10n.databaseBuildErrorBody(
+        AppConfig.newIssueUrl.toString(),
+        e.toString(),
+      ),
+    );
+
+    return FlutterHiveCollectionsDatabase.databaseBuilder(client);
+  }
+}
+
+Future<MatrixSdkDatabase> _constructDatabase(Client client) async {
+  if (kIsWeb) {
+    html.window.navigator.storage?.persist();
+    return MatrixSdkDatabase(client.clientName);
+  }
+
+  final cipher = await getDatabaseCipher();
+
+  final fileStoragePath = PlatformInfos.isIOS || PlatformInfos.isMacOS
+      ? await getLibraryDirectory()
+      : await getApplicationSupportDirectory();
+
+  final path = join(fileStoragePath.path, '${client.clientName}.sqlite');
+
+  // fix dlopen for old Android
+  await applyWorkaroundToOpenSqlCipherOnOldAndroidVersions();
+  // import the SQLite / SQLCipher shared objects / dynamic libraries
+  final factory =
+      createDatabaseFactoryFfi(ffiInit: SQfLiteEncryptionHelper.ffiInit);
+
+  // migrate from potential previous SQLite database path to current one
+  await _migrateLegacyLocation(path, client.clientName);
+
+  // required for [getDatabasesPath]
+  databaseFactory = factory;
+
+  // in case we got a cipher, we use the encryption helper
+  // to manage SQLite encryption
+  final helper = SQfLiteEncryptionHelper(
+    factory: factory,
+    path: path,
+    cipher: cipher,
+  );
+
+  // check whether the DB is already encrypted and otherwise do so
+  await helper.ensureDatabaseFileEncrypted();
+
+  final database = await factory.openDatabase(
+    path,
+    options: OpenDatabaseOptions(
+      version: 1,
+      // most important : apply encryption when opening the DB
+      onConfigure: helper.applyPragmaKey,
+    ),
+  );
+
+  return MatrixSdkDatabase(
+    client.clientName,
+    database: database,
+    maxFileSize: 1024 * 1024 * 10,
+    fileStoragePath: fileStoragePath,
+    deleteFilesAfterDuration: const Duration(days: 30),
+  );
+}
+
+Future<void> _migrateLegacyLocation(
+  String sqlFilePath,
+  String clientName,
+) async {
+  final oldPath = PlatformInfos.isDesktop
+      ? (await getApplicationSupportDirectory()).path
+      : await getDatabasesPath();
+
+  final oldFilePath = join(oldPath, clientName);
+  if (oldFilePath == sqlFilePath) return;
+
+  final maybeOldFile = File(oldFilePath);
+  if (await maybeOldFile.exists()) {
+    Logs().i(
+      'Migrate legacy location for database from "$oldFilePath" to "$sqlFilePath"',
+    );
+    await maybeOldFile.copy(sqlFilePath);
+    await maybeOldFile.delete();
+  }
+}

lib/utils/matrix_sdk_extensions/flutter_matrix_dart_sdk_database/cipher.dart

@@ -0,0 +1,48 @@
+import 'dart:convert';
+import 'dart:math';
+
+import 'package:flutter/services.dart';
+
+import 'package:flutter_secure_storage/flutter_secure_storage.dart';
+import 'package:matrix/matrix.dart';
+
+const _passwordStorageKey = 'database_password';
+
+Future<String> getDatabaseCipher() async {
+  String? password;
+
+  try {
+    const secureStorage = FlutterSecureStorage();
+    final containsEncryptionKey =
+        await secureStorage.read(key: _passwordStorageKey) != null;
+    if (!containsEncryptionKey) {
+      final rng = Random.secure();
+      final list = Uint8List(32);
+      list.setAll(0, Iterable.generate(list.length, (i) => rng.nextInt(256)));
+      final newPassword = base64UrlEncode(list);
+      await secureStorage.write(
+        key: _passwordStorageKey,
+        value: newPassword,
+      );
+    }
+    // workaround for if we just wrote to the key and it still doesn't exist
+    password = await secureStorage.read(key: _passwordStorageKey);
+    if (password == null) throw MissingPluginException();
+  } on MissingPluginException catch (_) {
+    const FlutterSecureStorage()
+        .delete(key: _passwordStorageKey)
+        .catchError((_) {});
+    Logs().i('Database encryption is not supported on this platform');
+  } catch (e, s) {
+    const FlutterSecureStorage()
+        .delete(key: _passwordStorageKey)
+        .catchError((_) {});
+    Logs().w('Unable to init database encryption', e, s);
+  }
+
+  // with the new database, we should no longer allow unencrypted storage
+  // secure_storage now supports all platforms we support
+  assert(password != null);
+
+  return password!;
+}

lib/utils/matrix_sdk_extensions/flutter_matrix_dart_sdk_database/sqlcipher_stub.dart

@@ -0,0 +1 @@
+Future<void> applyWorkaroundToOpenSqlCipherOnOldAndroidVersions() async {}

lib/utils/matrix_sdk_extensions/flutter_matrix_sdk_database_builder.dart

@@ -1,107 +0,0 @@
-import 'dart:convert';
-import 'dart:math';
-
-import 'package:flutter/foundation.dart';
-import 'package:flutter/services.dart';
-
-import 'package:flutter_gen/gen_l10n/l10n.dart';
-import 'package:flutter_secure_storage/flutter_secure_storage.dart';
-import 'package:matrix/matrix.dart';
-import 'package:path_provider/path_provider.dart';
-import 'package:sqflite_common_ffi/sqflite_ffi.dart' as ffi;
-import 'package:sqflite_sqlcipher/sqflite.dart';
-import 'package:universal_html/html.dart' as html;
-
-import 'package:fluffychat/config/app_config.dart';
-import 'package:fluffychat/utils/client_manager.dart';
-import 'package:fluffychat/utils/matrix_sdk_extensions/flutter_hive_collections_database.dart';
-import 'package:fluffychat/utils/platform_infos.dart';
-
-Future<DatabaseApi> flutterMatrixSdkDatabaseBuilder(Client client) async {
-  MatrixSdkDatabase? database;
-  try {
-    database = await _constructDatabase(client);
-    await database.open();
-    return database;
-  } catch (e, s) {
-    Logs().wtf('Unable to build database!', e, s);
-
-    // Send error notification:
-    final l10n = lookupL10n(PlatformDispatcher.instance.locale);
-    ClientManager.sendInitNotification(
-      l10n.initAppError,
-      l10n.databaseBuildErrorBody(
-        AppConfig.newIssueUrl.toString(),
-        e.toString(),
-      ),
-    );
-
-    return FlutterHiveCollectionsDatabase.databaseBuilder(client);
-  }
-}
-
-Future<MatrixSdkDatabase> _constructDatabase(Client client) async {
-  if (kIsWeb) {
-    html.window.navigator.storage?.persist();
-    return MatrixSdkDatabase(client.clientName);
-  }
-  if (PlatformInfos.isDesktop) {
-    final path = await getApplicationSupportDirectory();
-    return MatrixSdkDatabase(
-      client.clientName,
-      database: await ffi.databaseFactoryFfi.openDatabase(
-        '${path.path}/${client.clientName}',
-      ),
-      maxFileSize: 1024 * 1024 * 10,
-      fileStoragePath: path,
-      deleteFilesAfterDuration: const Duration(days: 30),
-    );
-  }
-
-  final path = await getDatabasesPath();
-  const passwordStorageKey = 'database_password';
-  String? password;
-
-  try {
-    // Workaround for secure storage is calling Platform.operatingSystem on web
-    if (kIsWeb) throw MissingPluginException();
-
-    const secureStorage = FlutterSecureStorage();
-    final containsEncryptionKey =
-        await secureStorage.read(key: passwordStorageKey) != null;
-    if (!containsEncryptionKey) {
-      final rng = Random.secure();
-      final list = Uint8List(32);
-      list.setAll(0, Iterable.generate(list.length, (i) => rng.nextInt(256)));
-      final newPassword = base64UrlEncode(list);
-      await secureStorage.write(
-        key: passwordStorageKey,
-        value: newPassword,
-      );
-    }
-    // workaround for if we just wrote to the key and it still doesn't exist
-    password = await secureStorage.read(key: passwordStorageKey);
-    if (password == null) throw MissingPluginException();
-  } on MissingPluginException catch (_) {
-    const FlutterSecureStorage()
-        .delete(key: passwordStorageKey)
-        .catchError((_) {});
-    Logs().i('Database encryption is not supported on this platform');
-  } catch (e, s) {
-    const FlutterSecureStorage()
-        .delete(key: passwordStorageKey)
-        .catchError((_) {});
-    Logs().w('Unable to init database encryption', e, s);
-  }
-
-  return MatrixSdkDatabase(
-    client.clientName,
-    database: await openDatabase(
-      '$path/${client.clientName}',
-      password: password,
-    ),
-    maxFileSize: 1024 * 1024 * 10,
-    fileStoragePath: await getTemporaryDirectory(),
-    deleteFilesAfterDuration: const Duration(days: 30),
-  );
-}

linux/flutter/generated_plugin_registrant.cc

@@ -14,6 +14,7 @@
 #include <flutter_webrtc/flutter_web_r_t_c_plugin.h>
 #include <pasteboard/pasteboard_plugin.h>
 #include <record_linux/record_linux_plugin.h>
+#include <sqlcipher_flutter_libs/sqlite3_flutter_libs_plugin.h>
 #include <url_launcher_linux/url_launcher_plugin.h>
 #include <window_to_front/window_to_front_plugin.h>
 
@@ -42,6 +43,9 @@ void fl_register_plugins(FlPluginRegistry* registry) {
   g_autoptr(FlPluginRegistrar) record_linux_registrar =
       fl_plugin_registry_get_registrar_for_plugin(registry, "RecordLinuxPlugin");
   record_linux_plugin_register_with_registrar(record_linux_registrar);
+  g_autoptr(FlPluginRegistrar) sqlcipher_flutter_libs_registrar =
+      fl_plugin_registry_get_registrar_for_plugin(registry, "Sqlite3FlutterLibsPlugin");
+  sqlite3_flutter_libs_plugin_register_with_registrar(sqlcipher_flutter_libs_registrar);
   g_autoptr(FlPluginRegistrar) url_launcher_linux_registrar =
       fl_plugin_registry_get_registrar_for_plugin(registry, "UrlLauncherPlugin");
   url_launcher_plugin_register_with_registrar(url_launcher_linux_registrar);

linux/flutter/generated_plugins.cmake

@@ -11,6 +11,7 @@ list(APPEND FLUTTER_PLUGIN_LIST
   flutter_webrtc
   pasteboard
   record_linux
+  sqlcipher_flutter_libs
   url_launcher_linux
   window_to_front
 )

macos/Flutter/GeneratedPluginRegistrant.swift

@@ -28,7 +28,7 @@ import record_macos
 import share_plus
 import shared_preferences_foundation
 import sqflite
-import sqflite_sqlcipher
+import sqlcipher_flutter_libs
 import url_launcher_macos
 import video_compress
 import video_player_avfoundation
@@ -59,7 +59,7 @@ func RegisterGeneratedPlugins(registry: FlutterPluginRegistry) {
   SharePlusMacosPlugin.register(with: registry.registrar(forPlugin: "SharePlusMacosPlugin"))
   SharedPreferencesPlugin.register(with: registry.registrar(forPlugin: "SharedPreferencesPlugin"))
   SqflitePlugin.register(with: registry.registrar(forPlugin: "SqflitePlugin"))
-  SqfliteSqlCipherPlugin.register(with: registry.registrar(forPlugin: "SqfliteSqlCipherPlugin"))
+  Sqlite3FlutterLibsPlugin.register(with: registry.registrar(forPlugin: "Sqlite3FlutterLibsPlugin"))
   UrlLauncherPlugin.register(with: registry.registrar(forPlugin: "UrlLauncherPlugin"))
   VideoCompressPlugin.register(with: registry.registrar(forPlugin: "VideoCompressPlugin"))
   FVPVideoPlayerPlugin.register(with: registry.registrar(forPlugin: "FVPVideoPlayerPlugin"))

pubspec.lock

@@ -1287,7 +1287,7 @@ packages:
     source: hosted
     version: "0.2.0"
   path:
-    dependency: transitive
+    dependency: "direct main"
     description:
       name: path
       sha256: "087ce49c3f0dc39180befefc60fdb4acd8f8620e5682fe2476afd0b3688bb4af"
@@ -1772,7 +1772,7 @@ packages:
     source: hosted
     version: "1.10.0"
   sqflite:
-    dependency: "direct main"
+    dependency: transitive
     description:
       name: sqflite
       sha256: "591f1602816e9c31377d5f008c2d9ef7b8aca8941c3f89cc5fd9d84da0c38a9a"
@@ -1795,22 +1795,22 @@ packages:
       url: "https://pub.dev"
     source: hosted
     version: "2.3.0+4"
-  sqflite_sqlcipher:
+  sqlcipher_flutter_libs:
     dependency: "direct main"
     description:
-      name: sqflite_sqlcipher
-      sha256: e1dfb55bf21ee5a18c43f28faa4291272a801da4ab34a6ba9973b6c0e1ed77da
+      name: sqlcipher_flutter_libs
+      sha256: e2d4dde4288c7fd1fd1cc0b1d39a9cf537ec236fed0c6dcd54b577543872a19d
       url: "https://pub.dev"
     source: hosted
-    version: "2.2.1"
+    version: "0.6.0"
   sqlite3:
     dependency: transitive
     description:
       name: sqlite3
-      sha256: db65233e6b99e99b2548932f55a987961bc06d82a31a0665451fa0b4fff4c3fb
+      sha256: c4a4c5a4b2a32e2d0f6837b33d7c91a67903891a5b7dbe706cf4b1f6b0c798c5
       url: "https://pub.dev"
     source: hosted
-    version: "2.1.0"
+    version: "2.3.0"
   stack_trace:
     dependency: transitive
     description:

pubspec.yaml

@@ -69,6 +69,7 @@ dependencies:
   native_imaging: ^0.1.0
   package_info_plus: ^5.0.1
   pasteboard: ^0.2.0
+  path: ^1.9.0
   path_provider: ^2.0.9
   permission_handler: ^11.0.1
   pretty_qr_code: ^3.2.1
@@ -81,9 +82,8 @@ dependencies:
   share_plus: ^7.2.1
   shared_preferences: ^2.2.0 # Pinned because https://github.com/flutter/flutter/issues/118401
   slugify: ^2.0.0
-  sqflite: ^2.3.0
   sqflite_common_ffi: ^2.3.0+4
-  sqflite_sqlcipher: ^2.2.1
+  sqlcipher_flutter_libs: ^0.6.0
   swipe_to_action: ^0.2.0
   tor_detector_web: ^1.1.0
   uni_links: ^0.5.1

snap/snapcraft.yaml

@@ -77,8 +77,7 @@ parts:
     stage-packages:
       - libsecret-1-dev
       - libjsoncpp-dev
-      - libsqlite3-0
-      - libsqlite3-dev
+      - libssl-dev
 
 slots:
   dbus-svc:
@@ -89,7 +88,7 @@ slots:
 apps:
   fluffychat:
     command: fluffychat
-    extensions: [gnome]
+    extensions: [ gnome ]
     plugs:
       - audio-playback
       - desktop

windows/flutter/generated_plugin_registrant.cc

@@ -16,6 +16,7 @@
 #include <permission_handler_windows/permission_handler_windows_plugin.h>
 #include <record_windows/record_windows_plugin_c_api.h>
 #include <share_plus/share_plus_windows_plugin_c_api.h>
+#include <sqlcipher_flutter_libs/sqlite3_flutter_libs_plugin.h>
 #include <url_launcher_windows/url_launcher_windows.h>
 #include <window_to_front/window_to_front_plugin.h>
 
@@ -40,6 +41,8 @@ void RegisterPlugins(flutter::PluginRegistry* registry) {
       registry->GetRegistrarForPlugin("RecordWindowsPluginCApi"));
   SharePlusWindowsPluginCApiRegisterWithRegistrar(
       registry->GetRegistrarForPlugin("SharePlusWindowsPluginCApi"));
+  Sqlite3FlutterLibsPluginRegisterWithRegistrar(
+      registry->GetRegistrarForPlugin("Sqlite3FlutterLibsPlugin"));
   UrlLauncherWindowsRegisterWithRegistrar(
       registry->GetRegistrarForPlugin("UrlLauncherWindows"));
   WindowToFrontPluginRegisterWithRegistrar(

windows/flutter/generated_plugins.cmake

@@ -13,6 +13,7 @@ list(APPEND FLUTTER_PLUGIN_LIST
   permission_handler_windows
   record_windows
   share_plus
+  sqlcipher_flutter_libs
   url_launcher_windows
   window_to_front
 )