diff --git a/fediverse_archive_browser/lib/src/mastodon/serializers/mastodon_contact_serializer.dart b/fediverse_archive_browser/lib/src/mastodon/serializers/mastodon_contact_serializer.dart new file mode 100644 index 0000000..e69de29 diff --git a/fediverse_archive_browser/lib/src/mastodon/serializers/mastodon_media_attachment_serializer.dart b/fediverse_archive_browser/lib/src/mastodon/serializers/mastodon_media_attachment_serializer.dart new file mode 100644 index 0000000..1cc0873 --- /dev/null +++ b/fediverse_archive_browser/lib/src/mastodon/serializers/mastodon_media_attachment_serializer.dart @@ -0,0 +1,23 @@ +import '../../models/media_attachment.dart'; + +MediaAttachment mediaAttachmentfromMastodonJson(Map json) { + final uri = Uri.parse(json['url']); + const creationTimestamp = 0; + final explicitType = (json['mediaType'] ?? '').startsWith('image') + ? AttachmentMediaType.image + : (json['mimetype'] ?? '').startsWith('video') + ? AttachmentMediaType.video + : AttachmentMediaType.unknown; + final thumbnailUri = Uri(); + final title = json['name'] ?? ''; + final description = json['blurhash'] ?? ''; + + return MediaAttachment( + uri: uri, + creationTimestamp: creationTimestamp, + metadata: {}, + thumbnailUri: thumbnailUri, + title: title, + explicitType: explicitType, + description: description); +} diff --git a/fediverse_archive_browser/lib/src/mastodon/serializers/mastodon_timeline_entry_serializer.dart b/fediverse_archive_browser/lib/src/mastodon/serializers/mastodon_timeline_entry_serializer.dart new file mode 100644 index 0000000..3670dba --- /dev/null +++ b/fediverse_archive_browser/lib/src/mastodon/serializers/mastodon_timeline_entry_serializer.dart @@ -0,0 +1,53 @@ +import 'package:fediverse_archive_browser/src/mastodon/serializers/mastodon_media_attachment_serializer.dart'; +import 'package:logging/logging.dart'; + +import '../../models/location_data.dart'; +import '../../models/timeline_entry.dart'; +import '../../utils/offsetdatetime_utils.dart'; + +final _logger = Logger('timelineEntryFromMastodonJson'); +TimelineEntry timelineEntryFromMastodonJson(Map json) { + final int timestamp = json.containsKey('published') + ? OffsetDateTimeUtils.epochSecTimeFromTimeZoneString(json['published']) + .fold( + onSuccess: (value) => value, + onError: (error) { + _logger.severe("Couldn't read date time string: $error"); + return 0; + }) + : 0; + final id = json['id'] ?? ''; + final isReshare = json.containsKey('reblogged'); + final parentId = json['inReplyTo'] ?? ''; + final parentAuthor = json['in_reply_to_account_id'] ?? ''; + final parentAuthorId = json['in_reply_to_account_id'] ?? ''; + final body = json['content'] ?? ''; + final author = json['attributedTo'] ?? ''; + final authorId = json['attributedTo'] ?? ''; + const title = ''; + final externalLink = json['url'] ?? ''; + final actualLocationData = LocationData(); + final modificationTimestamp = timestamp; + final backdatedTimestamp = timestamp; + final mediaAttachments = (json['attachment'] as List? ?? []) + .map((json) => mediaAttachmentfromMastodonJson(json)) + .toList(); + return TimelineEntry( + creationTimestamp: timestamp, + modificationTimestamp: modificationTimestamp, + backdatedTimestamp: backdatedTimestamp, + locationData: actualLocationData, + body: body, + isReshare: isReshare, + id: id, + parentId: parentId, + parentAuthorId: parentAuthorId, + externalLink: externalLink, + author: author, + authorId: authorId, + parentAuthor: parentAuthor, + title: title, + links: [], + mediaAttachments: mediaAttachments, + ); +} diff --git a/fediverse_archive_browser/lib/src/mastodon/services/mastodon_archive_service.dart b/fediverse_archive_browser/lib/src/mastodon/services/mastodon_archive_service.dart new file mode 100644 index 0000000..e2f6e2d --- /dev/null +++ b/fediverse_archive_browser/lib/src/mastodon/services/mastodon_archive_service.dart @@ -0,0 +1,145 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'package:path/path.dart' as p; +import 'package:result_monad/result_monad.dart'; + +import '../../models/entry_tree_item.dart'; +import '../../models/local_image_archive_entry.dart'; +import '../../services/archive_service_interface.dart'; +import '../../services/connections_manager.dart'; +import '../../utils/exec_error.dart'; +import '../serializers/mastodon_timeline_entry_serializer.dart'; +import 'mastodon_path_mapping_service.dart'; + +class MastodonArchiveService implements ArchiveService { + @override + final MastodonPathMappingService pathMappingService; + + final Map _imagesByRequestUrl = {}; + final List _postEntries = []; + final List _orphanedCommentEntries = []; + final List _allComments = []; + @override + final ConnectionsManager connectionsManager = ConnectionsManager(); + + MastodonArchiveService({required this.pathMappingService}); + + @override + // TODO: implement ownersName + String get ownersName => throw UnimplementedError(); + + @override + void clearCaches() { + connectionsManager.clearCaches(); + _imagesByRequestUrl.clear(); + _orphanedCommentEntries.clear(); + _allComments.clear(); + _postEntries.clear(); + } + + @override + FutureResult, ExecError> getPosts() async { + if (_postEntries.isEmpty && _allComments.isEmpty) { + _loadEntries(); + } + + return Result.ok(_postEntries); + } + + @override + FutureResult, ExecError> getAllComments() async { + if (_postEntries.isEmpty && _allComments.isEmpty) { + _loadEntries(); + } + + return Result.ok(_allComments); + } + + @override + FutureResult, ExecError> getOrphanedComments() async { + if (_postEntries.isEmpty && _allComments.isEmpty) { + _loadEntries(); + } + + return Result.ok(_orphanedCommentEntries); + } + + @override + Result getImageByUrl(String url) { + if (_imagesByRequestUrl.isEmpty) { + _loadImages(); + } + + final result = _imagesByRequestUrl[url]; + return result == null + ? Result.error(ExecError(errorMessage: '$url not found')) + : Result.ok(result); + } + + String get _baseArchiveFolder => pathMappingService.rootFolder; + + void _loadEntries() { + final entriesJsonPath = p.join(_baseArchiveFolder, 'outbox.json'); + final jsonFile = File(entriesJsonPath); + try { + if (jsonFile.existsSync()) { + final jsonText = jsonFile.readAsStringSync(); + final json = jsonDecode(jsonText) as Map; + final entriesJson = json['orderedItems'] as List; + final entries = entriesJson + .where((e) => 'Create' == e['type']) + .map((e) => e['object']) + .map((e) => timelineEntryFromMastodonJson(e)) + .toList(); + final topLevelEntries = + entries.where((element) => element.parentId.isEmpty); + final commentEntries = + entries.where((element) => element.parentId.isNotEmpty).toList(); + final entryTrees = {}; + + final postTreeEntries = []; + for (final entry in topLevelEntries) { + final treeEntry = EntryTreeItem(entry, false); + entryTrees[entry.id] = treeEntry; + postTreeEntries.add(treeEntry); + } + + final commentTreeEntries = []; + commentEntries.sort( + (c1, c2) => c1.creationTimestamp.compareTo(c2.creationTimestamp)); + for (final entry in commentEntries) { + final parent = entryTrees[entry.parentId]; + final treeEntry = EntryTreeItem(entry, parent == null); + parent?.addChild(treeEntry); + entryTrees[entry.id] = treeEntry; + commentTreeEntries.add(treeEntry); + } + + _postEntries.clear(); + _postEntries.addAll(postTreeEntries); + + _allComments.clear(); + _allComments.addAll(commentTreeEntries); + + _orphanedCommentEntries.clear(); + _orphanedCommentEntries + .addAll(entryTrees.values.where((element) => element.isOrphaned)); + } + } catch (e) { + print(e); + } + } + + void _loadImages() { + final imageJsonPath = p.join(_baseArchiveFolder, 'images.json'); + final jsonFile = File(imageJsonPath); + if (jsonFile.existsSync()) { + final json = jsonDecode(jsonFile.readAsStringSync()) as List; + final imageEntries = json.map((j) => ImageEntry.fromJson(j)); + for (final entry in imageEntries) { + _imagesByRequestUrl[entry.url] = entry; + } + } + } +} diff --git a/fediverse_archive_browser/lib/src/mastodon/services/mastodon_path_mapping_service.dart b/fediverse_archive_browser/lib/src/mastodon/services/mastodon_path_mapping_service.dart new file mode 100644 index 0000000..521e181 --- /dev/null +++ b/fediverse_archive_browser/lib/src/mastodon/services/mastodon_path_mapping_service.dart @@ -0,0 +1,73 @@ +import 'dart:io'; + +import 'package:fediverse_archive_browser/src/settings/settings_controller.dart'; +import 'package:logging/logging.dart'; +import 'package:path/path.dart' as p; + +import '../../services/path_mapper_service_interface.dart'; + +class MastodonPathMappingService implements PathMappingService { + static final _logger = Logger('$MastodonPathMappingService'); + final SettingsController settings; + final _archiveDirectories = []; + + MastodonPathMappingService(this.settings) { + refresh(); + } + + String get rootFolder => settings.rootFolder; + + List get archiveDirectories => + List.unmodifiable(_archiveDirectories); + + void refresh() { + _logger.fine('Refreshing path mapping service directory data.'); + if (!Directory(settings.rootFolder).existsSync()) { + _logger.severe( + "Base directory does not exist! can't do mapping of ${settings.rootFolder}"); + return; + } + _archiveDirectories.clear(); + + final recursive = !_calcRootIsSingleArchiveFolder(); + _archiveDirectories.addAll(Directory(settings.rootFolder) + .listSync(recursive: recursive) + .where((element) => + element.statSync().type == FileSystemEntityType.directory && + p.basename(element.path) == 'media_attachments') + .map((d) => d.parent)); + } + + String toFullPath(String relPath) { + for (final file in _archiveDirectories) { + final fullPath = + p.join(file.path, relPath[0] == '/' ? relPath.substring(1) : relPath); + if (File(fullPath).existsSync()) { + return fullPath; + } + } + + _logger.fine( + 'Did not find a file with this relPath anywhere therefore returning the relPath'); + return relPath; + } + + bool _calcRootIsSingleArchiveFolder() { + for (final entity in Directory(rootFolder).listSync(recursive: false)) { + if (_knownRootFilesAndFolders.contains(entity.uri.pathSegments + .where((element) => element.isNotEmpty) + .last)) { + return true; + } + } + + return false; + } + + static final _knownRootFilesAndFolders = [ + 'media_attachments', + 'actor.json', + 'likes.json', + 'outbox.json', + ]; +} diff --git a/fediverse_archive_browser/lib/src/models/archive_types_enum.dart b/fediverse_archive_browser/lib/src/models/archive_types_enum.dart index 5801882..a95983b 100644 --- a/fediverse_archive_browser/lib/src/models/archive_types_enum.dart +++ b/fediverse_archive_browser/lib/src/models/archive_types_enum.dart @@ -2,4 +2,5 @@ enum ArchiveType { unknown, diaspora, friendica, + mastodon, } diff --git a/fediverse_archive_browser/lib/src/services/archive_service_provider.dart b/fediverse_archive_browser/lib/src/services/archive_service_provider.dart index 0f49c38..1389031 100644 --- a/fediverse_archive_browser/lib/src/services/archive_service_provider.dart +++ b/fediverse_archive_browser/lib/src/services/archive_service_provider.dart @@ -1,12 +1,14 @@ -import 'package:flutter/cupertino.dart'; import 'package:fediverse_archive_browser/src/diaspora/services/diaspora_archive_service.dart'; import 'package:fediverse_archive_browser/src/diaspora/services/diaspora_path_mapping_service.dart'; import 'package:fediverse_archive_browser/src/friendica/services/friendica_archive_service.dart'; import 'package:fediverse_archive_browser/src/friendica/services/friendica_path_mapping_service.dart'; +import 'package:fediverse_archive_browser/src/mastodon/services/mastodon_archive_service.dart'; +import 'package:fediverse_archive_browser/src/mastodon/services/mastodon_path_mapping_service.dart'; import 'package:fediverse_archive_browser/src/services/archive_service_interface.dart'; import 'package:fediverse_archive_browser/src/services/connections_manager.dart'; import 'package:fediverse_archive_browser/src/services/path_mapper_service_interface.dart'; import 'package:fediverse_archive_browser/src/settings/settings_controller.dart'; +import 'package:flutter/cupertino.dart'; import 'package:result_monad/result_monad.dart'; import '../models/archive_types_enum.dart'; @@ -18,6 +20,7 @@ class ArchiveServiceProvider extends ChangeNotifier implements ArchiveService { final SettingsController settings; late DiasporaArchiveService _diasporaArchiveService; late FriendicaArchiveService _friendicaArchiveService; + late MastodonArchiveService _mastodonArchiveService; @override ConnectionsManager get connectionsManager => @@ -32,6 +35,7 @@ class ArchiveServiceProvider extends ChangeNotifier implements ArchiveService { void clearCaches() { _friendicaArchiveService.clearCaches(); _diasporaArchiveService.clearCaches(); + _mastodonArchiveService.clearCaches(); _buildArchiveServices(); } @@ -61,17 +65,19 @@ class ArchiveServiceProvider extends ChangeNotifier implements ArchiveService { return _diasporaArchiveService; case ArchiveType.friendica: return _friendicaArchiveService; + case ArchiveType.mastodon: + return _mastodonArchiveService; default: throw Exception('Unknown archive type'); } } - void _buildArchiveServices() { _diasporaArchiveService = DiasporaArchiveService( pathMappingService: DiasporaPathMappingService(settings)); _friendicaArchiveService = FriendicaArchiveService( pathMappingService: FriendicaPathMappingService(settings)); + _mastodonArchiveService = MastodonArchiveService( + pathMappingService: MastodonPathMappingService(settings)); } - } diff --git a/fediverse_archive_browser/pubspec.lock b/fediverse_archive_browser/pubspec.lock index 3b28716..f006cc9 100644 --- a/fediverse_archive_browser/pubspec.lock +++ b/fediverse_archive_browser/pubspec.lock @@ -552,7 +552,7 @@ packages: name: result_monad url: "https://pub.dartlang.org" source: hosted - version: "1.0.2" + version: "2.0.2" rxdart: dependency: transitive description: diff --git a/fediverse_archive_browser/pubspec.yaml b/fediverse_archive_browser/pubspec.yaml index 7636dbc..5003936 100644 --- a/fediverse_archive_browser/pubspec.yaml +++ b/fediverse_archive_browser/pubspec.yaml @@ -28,7 +28,7 @@ dependencies: path: ^1.8.0 path_provider: ^2.0.6 provider: ^6.0.0 - result_monad: ^1.0.2 + result_monad: ^2.0.2 scrollable_positioned_list: ^0.2.2 shared_preferences: ^2.0.8 time_machine: