mirror of
https://gitlab.com/mysocialportal/fediverse-archiving-tools.git
synced 2024-10-18 08:53:31 +00:00
First pass at adding Mastodon capabilities...
This commit is contained in:
parent
3e25f5b6c1
commit
21fbcdfbdd
9 changed files with 306 additions and 5 deletions
|
@ -0,0 +1,23 @@
|
||||||
|
import '../../models/media_attachment.dart';
|
||||||
|
|
||||||
|
MediaAttachment mediaAttachmentfromMastodonJson(Map<String, dynamic> json) {
|
||||||
|
final uri = Uri.parse(json['url']);
|
||||||
|
const creationTimestamp = 0;
|
||||||
|
final explicitType = (json['mediaType'] ?? '').startsWith('image')
|
||||||
|
? AttachmentMediaType.image
|
||||||
|
: (json['mimetype'] ?? '').startsWith('video')
|
||||||
|
? AttachmentMediaType.video
|
||||||
|
: AttachmentMediaType.unknown;
|
||||||
|
final thumbnailUri = Uri();
|
||||||
|
final title = json['name'] ?? '';
|
||||||
|
final description = json['blurhash'] ?? '';
|
||||||
|
|
||||||
|
return MediaAttachment(
|
||||||
|
uri: uri,
|
||||||
|
creationTimestamp: creationTimestamp,
|
||||||
|
metadata: {},
|
||||||
|
thumbnailUri: thumbnailUri,
|
||||||
|
title: title,
|
||||||
|
explicitType: explicitType,
|
||||||
|
description: description);
|
||||||
|
}
|
|
@ -0,0 +1,53 @@
|
||||||
|
import 'package:fediverse_archive_browser/src/mastodon/serializers/mastodon_media_attachment_serializer.dart';
|
||||||
|
import 'package:logging/logging.dart';
|
||||||
|
|
||||||
|
import '../../models/location_data.dart';
|
||||||
|
import '../../models/timeline_entry.dart';
|
||||||
|
import '../../utils/offsetdatetime_utils.dart';
|
||||||
|
|
||||||
|
final _logger = Logger('timelineEntryFromMastodonJson');
|
||||||
|
TimelineEntry timelineEntryFromMastodonJson(Map<String, dynamic> json) {
|
||||||
|
final int timestamp = json.containsKey('published')
|
||||||
|
? OffsetDateTimeUtils.epochSecTimeFromTimeZoneString(json['published'])
|
||||||
|
.fold(
|
||||||
|
onSuccess: (value) => value,
|
||||||
|
onError: (error) {
|
||||||
|
_logger.severe("Couldn't read date time string: $error");
|
||||||
|
return 0;
|
||||||
|
})
|
||||||
|
: 0;
|
||||||
|
final id = json['id'] ?? '';
|
||||||
|
final isReshare = json.containsKey('reblogged');
|
||||||
|
final parentId = json['inReplyTo'] ?? '';
|
||||||
|
final parentAuthor = json['in_reply_to_account_id'] ?? '';
|
||||||
|
final parentAuthorId = json['in_reply_to_account_id'] ?? '';
|
||||||
|
final body = json['content'] ?? '';
|
||||||
|
final author = json['attributedTo'] ?? '';
|
||||||
|
final authorId = json['attributedTo'] ?? '';
|
||||||
|
const title = '';
|
||||||
|
final externalLink = json['url'] ?? '';
|
||||||
|
final actualLocationData = LocationData();
|
||||||
|
final modificationTimestamp = timestamp;
|
||||||
|
final backdatedTimestamp = timestamp;
|
||||||
|
final mediaAttachments = (json['attachment'] as List<dynamic>? ?? [])
|
||||||
|
.map((json) => mediaAttachmentfromMastodonJson(json))
|
||||||
|
.toList();
|
||||||
|
return TimelineEntry(
|
||||||
|
creationTimestamp: timestamp,
|
||||||
|
modificationTimestamp: modificationTimestamp,
|
||||||
|
backdatedTimestamp: backdatedTimestamp,
|
||||||
|
locationData: actualLocationData,
|
||||||
|
body: body,
|
||||||
|
isReshare: isReshare,
|
||||||
|
id: id,
|
||||||
|
parentId: parentId,
|
||||||
|
parentAuthorId: parentAuthorId,
|
||||||
|
externalLink: externalLink,
|
||||||
|
author: author,
|
||||||
|
authorId: authorId,
|
||||||
|
parentAuthor: parentAuthor,
|
||||||
|
title: title,
|
||||||
|
links: [],
|
||||||
|
mediaAttachments: mediaAttachments,
|
||||||
|
);
|
||||||
|
}
|
|
@ -0,0 +1,145 @@
|
||||||
|
import 'dart:convert';
|
||||||
|
import 'dart:io';
|
||||||
|
|
||||||
|
import 'package:path/path.dart' as p;
|
||||||
|
import 'package:result_monad/result_monad.dart';
|
||||||
|
|
||||||
|
import '../../models/entry_tree_item.dart';
|
||||||
|
import '../../models/local_image_archive_entry.dart';
|
||||||
|
import '../../services/archive_service_interface.dart';
|
||||||
|
import '../../services/connections_manager.dart';
|
||||||
|
import '../../utils/exec_error.dart';
|
||||||
|
import '../serializers/mastodon_timeline_entry_serializer.dart';
|
||||||
|
import 'mastodon_path_mapping_service.dart';
|
||||||
|
|
||||||
|
class MastodonArchiveService implements ArchiveService {
|
||||||
|
@override
|
||||||
|
final MastodonPathMappingService pathMappingService;
|
||||||
|
|
||||||
|
final Map<String, ImageEntry> _imagesByRequestUrl = {};
|
||||||
|
final List<EntryTreeItem> _postEntries = [];
|
||||||
|
final List<EntryTreeItem> _orphanedCommentEntries = [];
|
||||||
|
final List<EntryTreeItem> _allComments = [];
|
||||||
|
@override
|
||||||
|
final ConnectionsManager connectionsManager = ConnectionsManager();
|
||||||
|
|
||||||
|
MastodonArchiveService({required this.pathMappingService});
|
||||||
|
|
||||||
|
@override
|
||||||
|
// TODO: implement ownersName
|
||||||
|
String get ownersName => throw UnimplementedError();
|
||||||
|
|
||||||
|
@override
|
||||||
|
void clearCaches() {
|
||||||
|
connectionsManager.clearCaches();
|
||||||
|
_imagesByRequestUrl.clear();
|
||||||
|
_orphanedCommentEntries.clear();
|
||||||
|
_allComments.clear();
|
||||||
|
_postEntries.clear();
|
||||||
|
}
|
||||||
|
|
||||||
|
@override
|
||||||
|
FutureResult<List<EntryTreeItem>, ExecError> getPosts() async {
|
||||||
|
if (_postEntries.isEmpty && _allComments.isEmpty) {
|
||||||
|
_loadEntries();
|
||||||
|
}
|
||||||
|
|
||||||
|
return Result.ok(_postEntries);
|
||||||
|
}
|
||||||
|
|
||||||
|
@override
|
||||||
|
FutureResult<List<EntryTreeItem>, ExecError> getAllComments() async {
|
||||||
|
if (_postEntries.isEmpty && _allComments.isEmpty) {
|
||||||
|
_loadEntries();
|
||||||
|
}
|
||||||
|
|
||||||
|
return Result.ok(_allComments);
|
||||||
|
}
|
||||||
|
|
||||||
|
@override
|
||||||
|
FutureResult<List<EntryTreeItem>, ExecError> getOrphanedComments() async {
|
||||||
|
if (_postEntries.isEmpty && _allComments.isEmpty) {
|
||||||
|
_loadEntries();
|
||||||
|
}
|
||||||
|
|
||||||
|
return Result.ok(_orphanedCommentEntries);
|
||||||
|
}
|
||||||
|
|
||||||
|
@override
|
||||||
|
Result<ImageEntry, ExecError> getImageByUrl(String url) {
|
||||||
|
if (_imagesByRequestUrl.isEmpty) {
|
||||||
|
_loadImages();
|
||||||
|
}
|
||||||
|
|
||||||
|
final result = _imagesByRequestUrl[url];
|
||||||
|
return result == null
|
||||||
|
? Result.error(ExecError(errorMessage: '$url not found'))
|
||||||
|
: Result.ok(result);
|
||||||
|
}
|
||||||
|
|
||||||
|
String get _baseArchiveFolder => pathMappingService.rootFolder;
|
||||||
|
|
||||||
|
void _loadEntries() {
|
||||||
|
final entriesJsonPath = p.join(_baseArchiveFolder, 'outbox.json');
|
||||||
|
final jsonFile = File(entriesJsonPath);
|
||||||
|
try {
|
||||||
|
if (jsonFile.existsSync()) {
|
||||||
|
final jsonText = jsonFile.readAsStringSync();
|
||||||
|
final json = jsonDecode(jsonText) as Map<String, dynamic>;
|
||||||
|
final entriesJson = json['orderedItems'] as List<dynamic>;
|
||||||
|
final entries = entriesJson
|
||||||
|
.where((e) => 'Create' == e['type'])
|
||||||
|
.map((e) => e['object'])
|
||||||
|
.map((e) => timelineEntryFromMastodonJson(e))
|
||||||
|
.toList();
|
||||||
|
final topLevelEntries =
|
||||||
|
entries.where((element) => element.parentId.isEmpty);
|
||||||
|
final commentEntries =
|
||||||
|
entries.where((element) => element.parentId.isNotEmpty).toList();
|
||||||
|
final entryTrees = <String, EntryTreeItem>{};
|
||||||
|
|
||||||
|
final postTreeEntries = <EntryTreeItem>[];
|
||||||
|
for (final entry in topLevelEntries) {
|
||||||
|
final treeEntry = EntryTreeItem(entry, false);
|
||||||
|
entryTrees[entry.id] = treeEntry;
|
||||||
|
postTreeEntries.add(treeEntry);
|
||||||
|
}
|
||||||
|
|
||||||
|
final commentTreeEntries = <EntryTreeItem>[];
|
||||||
|
commentEntries.sort(
|
||||||
|
(c1, c2) => c1.creationTimestamp.compareTo(c2.creationTimestamp));
|
||||||
|
for (final entry in commentEntries) {
|
||||||
|
final parent = entryTrees[entry.parentId];
|
||||||
|
final treeEntry = EntryTreeItem(entry, parent == null);
|
||||||
|
parent?.addChild(treeEntry);
|
||||||
|
entryTrees[entry.id] = treeEntry;
|
||||||
|
commentTreeEntries.add(treeEntry);
|
||||||
|
}
|
||||||
|
|
||||||
|
_postEntries.clear();
|
||||||
|
_postEntries.addAll(postTreeEntries);
|
||||||
|
|
||||||
|
_allComments.clear();
|
||||||
|
_allComments.addAll(commentTreeEntries);
|
||||||
|
|
||||||
|
_orphanedCommentEntries.clear();
|
||||||
|
_orphanedCommentEntries
|
||||||
|
.addAll(entryTrees.values.where((element) => element.isOrphaned));
|
||||||
|
}
|
||||||
|
} catch (e) {
|
||||||
|
print(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void _loadImages() {
|
||||||
|
final imageJsonPath = p.join(_baseArchiveFolder, 'images.json');
|
||||||
|
final jsonFile = File(imageJsonPath);
|
||||||
|
if (jsonFile.existsSync()) {
|
||||||
|
final json = jsonDecode(jsonFile.readAsStringSync()) as List<dynamic>;
|
||||||
|
final imageEntries = json.map((j) => ImageEntry.fromJson(j));
|
||||||
|
for (final entry in imageEntries) {
|
||||||
|
_imagesByRequestUrl[entry.url] = entry;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,73 @@
|
||||||
|
import 'dart:io';
|
||||||
|
|
||||||
|
import 'package:fediverse_archive_browser/src/settings/settings_controller.dart';
|
||||||
|
import 'package:logging/logging.dart';
|
||||||
|
import 'package:path/path.dart' as p;
|
||||||
|
|
||||||
|
import '../../services/path_mapper_service_interface.dart';
|
||||||
|
|
||||||
|
class MastodonPathMappingService implements PathMappingService {
|
||||||
|
static final _logger = Logger('$MastodonPathMappingService');
|
||||||
|
final SettingsController settings;
|
||||||
|
final _archiveDirectories = <FileSystemEntity>[];
|
||||||
|
|
||||||
|
MastodonPathMappingService(this.settings) {
|
||||||
|
refresh();
|
||||||
|
}
|
||||||
|
|
||||||
|
String get rootFolder => settings.rootFolder;
|
||||||
|
|
||||||
|
List<FileSystemEntity> get archiveDirectories =>
|
||||||
|
List.unmodifiable(_archiveDirectories);
|
||||||
|
|
||||||
|
void refresh() {
|
||||||
|
_logger.fine('Refreshing path mapping service directory data.');
|
||||||
|
if (!Directory(settings.rootFolder).existsSync()) {
|
||||||
|
_logger.severe(
|
||||||
|
"Base directory does not exist! can't do mapping of ${settings.rootFolder}");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
_archiveDirectories.clear();
|
||||||
|
|
||||||
|
final recursive = !_calcRootIsSingleArchiveFolder();
|
||||||
|
_archiveDirectories.addAll(Directory(settings.rootFolder)
|
||||||
|
.listSync(recursive: recursive)
|
||||||
|
.where((element) =>
|
||||||
|
element.statSync().type == FileSystemEntityType.directory &&
|
||||||
|
p.basename(element.path) == 'media_attachments')
|
||||||
|
.map((d) => d.parent));
|
||||||
|
}
|
||||||
|
|
||||||
|
String toFullPath(String relPath) {
|
||||||
|
for (final file in _archiveDirectories) {
|
||||||
|
final fullPath =
|
||||||
|
p.join(file.path, relPath[0] == '/' ? relPath.substring(1) : relPath);
|
||||||
|
if (File(fullPath).existsSync()) {
|
||||||
|
return fullPath;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.fine(
|
||||||
|
'Did not find a file with this relPath anywhere therefore returning the relPath');
|
||||||
|
return relPath;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool _calcRootIsSingleArchiveFolder() {
|
||||||
|
for (final entity in Directory(rootFolder).listSync(recursive: false)) {
|
||||||
|
if (_knownRootFilesAndFolders.contains(entity.uri.pathSegments
|
||||||
|
.where((element) => element.isNotEmpty)
|
||||||
|
.last)) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
static final _knownRootFilesAndFolders = [
|
||||||
|
'media_attachments',
|
||||||
|
'actor.json',
|
||||||
|
'likes.json',
|
||||||
|
'outbox.json',
|
||||||
|
];
|
||||||
|
}
|
|
@ -2,4 +2,5 @@ enum ArchiveType {
|
||||||
unknown,
|
unknown,
|
||||||
diaspora,
|
diaspora,
|
||||||
friendica,
|
friendica,
|
||||||
|
mastodon,
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,12 +1,14 @@
|
||||||
import 'package:flutter/cupertino.dart';
|
|
||||||
import 'package:fediverse_archive_browser/src/diaspora/services/diaspora_archive_service.dart';
|
import 'package:fediverse_archive_browser/src/diaspora/services/diaspora_archive_service.dart';
|
||||||
import 'package:fediverse_archive_browser/src/diaspora/services/diaspora_path_mapping_service.dart';
|
import 'package:fediverse_archive_browser/src/diaspora/services/diaspora_path_mapping_service.dart';
|
||||||
import 'package:fediverse_archive_browser/src/friendica/services/friendica_archive_service.dart';
|
import 'package:fediverse_archive_browser/src/friendica/services/friendica_archive_service.dart';
|
||||||
import 'package:fediverse_archive_browser/src/friendica/services/friendica_path_mapping_service.dart';
|
import 'package:fediverse_archive_browser/src/friendica/services/friendica_path_mapping_service.dart';
|
||||||
|
import 'package:fediverse_archive_browser/src/mastodon/services/mastodon_archive_service.dart';
|
||||||
|
import 'package:fediverse_archive_browser/src/mastodon/services/mastodon_path_mapping_service.dart';
|
||||||
import 'package:fediverse_archive_browser/src/services/archive_service_interface.dart';
|
import 'package:fediverse_archive_browser/src/services/archive_service_interface.dart';
|
||||||
import 'package:fediverse_archive_browser/src/services/connections_manager.dart';
|
import 'package:fediverse_archive_browser/src/services/connections_manager.dart';
|
||||||
import 'package:fediverse_archive_browser/src/services/path_mapper_service_interface.dart';
|
import 'package:fediverse_archive_browser/src/services/path_mapper_service_interface.dart';
|
||||||
import 'package:fediverse_archive_browser/src/settings/settings_controller.dart';
|
import 'package:fediverse_archive_browser/src/settings/settings_controller.dart';
|
||||||
|
import 'package:flutter/cupertino.dart';
|
||||||
import 'package:result_monad/result_monad.dart';
|
import 'package:result_monad/result_monad.dart';
|
||||||
|
|
||||||
import '../models/archive_types_enum.dart';
|
import '../models/archive_types_enum.dart';
|
||||||
|
@ -18,6 +20,7 @@ class ArchiveServiceProvider extends ChangeNotifier implements ArchiveService {
|
||||||
final SettingsController settings;
|
final SettingsController settings;
|
||||||
late DiasporaArchiveService _diasporaArchiveService;
|
late DiasporaArchiveService _diasporaArchiveService;
|
||||||
late FriendicaArchiveService _friendicaArchiveService;
|
late FriendicaArchiveService _friendicaArchiveService;
|
||||||
|
late MastodonArchiveService _mastodonArchiveService;
|
||||||
|
|
||||||
@override
|
@override
|
||||||
ConnectionsManager get connectionsManager =>
|
ConnectionsManager get connectionsManager =>
|
||||||
|
@ -32,6 +35,7 @@ class ArchiveServiceProvider extends ChangeNotifier implements ArchiveService {
|
||||||
void clearCaches() {
|
void clearCaches() {
|
||||||
_friendicaArchiveService.clearCaches();
|
_friendicaArchiveService.clearCaches();
|
||||||
_diasporaArchiveService.clearCaches();
|
_diasporaArchiveService.clearCaches();
|
||||||
|
_mastodonArchiveService.clearCaches();
|
||||||
_buildArchiveServices();
|
_buildArchiveServices();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -61,17 +65,19 @@ class ArchiveServiceProvider extends ChangeNotifier implements ArchiveService {
|
||||||
return _diasporaArchiveService;
|
return _diasporaArchiveService;
|
||||||
case ArchiveType.friendica:
|
case ArchiveType.friendica:
|
||||||
return _friendicaArchiveService;
|
return _friendicaArchiveService;
|
||||||
|
case ArchiveType.mastodon:
|
||||||
|
return _mastodonArchiveService;
|
||||||
default:
|
default:
|
||||||
throw Exception('Unknown archive type');
|
throw Exception('Unknown archive type');
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
void _buildArchiveServices() {
|
void _buildArchiveServices() {
|
||||||
_diasporaArchiveService = DiasporaArchiveService(
|
_diasporaArchiveService = DiasporaArchiveService(
|
||||||
pathMappingService: DiasporaPathMappingService(settings));
|
pathMappingService: DiasporaPathMappingService(settings));
|
||||||
_friendicaArchiveService = FriendicaArchiveService(
|
_friendicaArchiveService = FriendicaArchiveService(
|
||||||
pathMappingService: FriendicaPathMappingService(settings));
|
pathMappingService: FriendicaPathMappingService(settings));
|
||||||
|
_mastodonArchiveService = MastodonArchiveService(
|
||||||
|
pathMappingService: MastodonPathMappingService(settings));
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -552,7 +552,7 @@ packages:
|
||||||
name: result_monad
|
name: result_monad
|
||||||
url: "https://pub.dartlang.org"
|
url: "https://pub.dartlang.org"
|
||||||
source: hosted
|
source: hosted
|
||||||
version: "1.0.2"
|
version: "2.0.2"
|
||||||
rxdart:
|
rxdart:
|
||||||
dependency: transitive
|
dependency: transitive
|
||||||
description:
|
description:
|
||||||
|
|
|
@ -28,7 +28,7 @@ dependencies:
|
||||||
path: ^1.8.0
|
path: ^1.8.0
|
||||||
path_provider: ^2.0.6
|
path_provider: ^2.0.6
|
||||||
provider: ^6.0.0
|
provider: ^6.0.0
|
||||||
result_monad: ^1.0.2
|
result_monad: ^2.0.2
|
||||||
scrollable_positioned_list: ^0.2.2
|
scrollable_positioned_list: ^0.2.2
|
||||||
shared_preferences: ^2.0.8
|
shared_preferences: ^2.0.8
|
||||||
time_machine:
|
time_machine:
|
||||||
|
|
Loading…
Reference in a new issue