mirror of
https://gitlab.com/mysocialportal/fediverse-archiving-tools.git
synced 2024-10-18 08:53:31 +00:00
Add initial archive reader CLI, client, with output
This commit is contained in:
parent
1daef343b4
commit
799b14e020
13 changed files with 171 additions and 82 deletions
6
.gitignore
vendored
6
.gitignore
vendored
|
@ -4,3 +4,9 @@
|
||||||
|
|
||||||
# Conventional directory for build output.
|
# Conventional directory for build output.
|
||||||
build/
|
build/
|
||||||
|
|
||||||
|
# IntelliJ Files
|
||||||
|
.idea
|
||||||
|
|
||||||
|
# Executables
|
||||||
|
*.exe
|
8
.idea/.gitignore
vendored
8
.idea/.gitignore
vendored
|
@ -1,8 +0,0 @@
|
||||||
# Default ignored files
|
|
||||||
/shelf/
|
|
||||||
/workspace.xml
|
|
||||||
# Datasource local storage ignored files
|
|
||||||
/dataSources/
|
|
||||||
/dataSources.local.xml
|
|
||||||
# Editor-based HTTP Client requests
|
|
||||||
/httpRequests/
|
|
|
@ -1,20 +0,0 @@
|
||||||
<component name="libraryTable">
|
|
||||||
<library name="Dart Packages" type="DartPackagesLibraryType">
|
|
||||||
<properties>
|
|
||||||
<option name="packageNameToDirsMap">
|
|
||||||
<entry key="lints">
|
|
||||||
<value>
|
|
||||||
<list>
|
|
||||||
<option value="$USER_HOME$/.pub-cache/hosted/pub.dartlang.org/lints-1.0.1/lib" />
|
|
||||||
</list>
|
|
||||||
</value>
|
|
||||||
</entry>
|
|
||||||
</option>
|
|
||||||
</properties>
|
|
||||||
<CLASSES>
|
|
||||||
<root url="file://$USER_HOME$/.pub-cache/hosted/pub.dartlang.org/lints-1.0.1/lib" />
|
|
||||||
</CLASSES>
|
|
||||||
<JAVADOC />
|
|
||||||
<SOURCES />
|
|
||||||
</library>
|
|
||||||
</component>
|
|
|
@ -1,27 +0,0 @@
|
||||||
<component name="libraryTable">
|
|
||||||
<library name="Dart SDK">
|
|
||||||
<CLASSES>
|
|
||||||
<root url="file://$USER_HOME$/snap/flutter/common/flutter/bin/cache/dart-sdk/lib/async" />
|
|
||||||
<root url="file://$USER_HOME$/snap/flutter/common/flutter/bin/cache/dart-sdk/lib/cli" />
|
|
||||||
<root url="file://$USER_HOME$/snap/flutter/common/flutter/bin/cache/dart-sdk/lib/collection" />
|
|
||||||
<root url="file://$USER_HOME$/snap/flutter/common/flutter/bin/cache/dart-sdk/lib/convert" />
|
|
||||||
<root url="file://$USER_HOME$/snap/flutter/common/flutter/bin/cache/dart-sdk/lib/core" />
|
|
||||||
<root url="file://$USER_HOME$/snap/flutter/common/flutter/bin/cache/dart-sdk/lib/developer" />
|
|
||||||
<root url="file://$USER_HOME$/snap/flutter/common/flutter/bin/cache/dart-sdk/lib/ffi" />
|
|
||||||
<root url="file://$USER_HOME$/snap/flutter/common/flutter/bin/cache/dart-sdk/lib/html" />
|
|
||||||
<root url="file://$USER_HOME$/snap/flutter/common/flutter/bin/cache/dart-sdk/lib/indexed_db" />
|
|
||||||
<root url="file://$USER_HOME$/snap/flutter/common/flutter/bin/cache/dart-sdk/lib/io" />
|
|
||||||
<root url="file://$USER_HOME$/snap/flutter/common/flutter/bin/cache/dart-sdk/lib/isolate" />
|
|
||||||
<root url="file://$USER_HOME$/snap/flutter/common/flutter/bin/cache/dart-sdk/lib/js" />
|
|
||||||
<root url="file://$USER_HOME$/snap/flutter/common/flutter/bin/cache/dart-sdk/lib/js_util" />
|
|
||||||
<root url="file://$USER_HOME$/snap/flutter/common/flutter/bin/cache/dart-sdk/lib/math" />
|
|
||||||
<root url="file://$USER_HOME$/snap/flutter/common/flutter/bin/cache/dart-sdk/lib/mirrors" />
|
|
||||||
<root url="file://$USER_HOME$/snap/flutter/common/flutter/bin/cache/dart-sdk/lib/svg" />
|
|
||||||
<root url="file://$USER_HOME$/snap/flutter/common/flutter/bin/cache/dart-sdk/lib/typed_data" />
|
|
||||||
<root url="file://$USER_HOME$/snap/flutter/common/flutter/bin/cache/dart-sdk/lib/web_audio" />
|
|
||||||
<root url="file://$USER_HOME$/snap/flutter/common/flutter/bin/cache/dart-sdk/lib/web_gl" />
|
|
||||||
</CLASSES>
|
|
||||||
<JAVADOC />
|
|
||||||
<SOURCES />
|
|
||||||
</library>
|
|
||||||
</component>
|
|
|
@ -1,6 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<project version="4">
|
|
||||||
<component name="ProjectRootManager">
|
|
||||||
<output url="file://$PROJECT_DIR$/out" />
|
|
||||||
</component>
|
|
||||||
</project>
|
|
|
@ -1,8 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<project version="4">
|
|
||||||
<component name="ProjectModuleManager">
|
|
||||||
<modules>
|
|
||||||
<module fileurl="file://$PROJECT_DIR$/friendica_archiver.iml" filepath="$PROJECT_DIR$/friendica_archiver.iml" />
|
|
||||||
</modules>
|
|
||||||
</component>
|
|
||||||
</project>
|
|
|
@ -1,10 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<project version="4">
|
|
||||||
<component name="RunConfigurationProducerService">
|
|
||||||
<option name="ignoredProducers">
|
|
||||||
<set>
|
|
||||||
<option value="com.android.tools.idea.compose.preview.runconfiguration.ComposePreviewRunConfigurationProducer" />
|
|
||||||
</set>
|
|
||||||
</option>
|
|
||||||
</component>
|
|
||||||
</project>
|
|
11
bin/exec_error.dart
Normal file
11
bin/exec_error.dart
Normal file
|
@ -0,0 +1,11 @@
|
||||||
|
class ExecError {
|
||||||
|
final ErrorType type;
|
||||||
|
final String message;
|
||||||
|
|
||||||
|
ExecError({required this.type, this.message = ''});
|
||||||
|
}
|
||||||
|
|
||||||
|
enum ErrorType {
|
||||||
|
authentication,
|
||||||
|
missingEndpoint,
|
||||||
|
}
|
|
@ -1,3 +1,85 @@
|
||||||
void main(List<String> arguments) {
|
import 'dart:io';
|
||||||
print('Hello world!');
|
|
||||||
|
import 'package:args/args.dart';
|
||||||
|
|
||||||
|
import 'friendica_client.dart';
|
||||||
|
import 'json_printer.dart';
|
||||||
|
|
||||||
|
const defaultRequestDelayMilliseconds = 5000;
|
||||||
|
const defaultMaxPosts = 1;
|
||||||
|
const defaultReadComments = false;
|
||||||
|
const defaultReadImages = false;
|
||||||
|
|
||||||
|
void main(List<String> arguments) async {
|
||||||
|
final argParser = ArgParser()
|
||||||
|
..addOption('archive-folder',
|
||||||
|
abbr: 'a',
|
||||||
|
help:
|
||||||
|
'Specifies the local folder all data files pulled from the server will be stored',
|
||||||
|
mandatory: true)
|
||||||
|
..addOption('username',
|
||||||
|
abbr: 'u', help: 'Username on your Friendica instance', mandatory: true)
|
||||||
|
..addOption('server-name',
|
||||||
|
abbr: 's',
|
||||||
|
help:
|
||||||
|
'The server name for your instance. (e.g. if the URL in your browser is "https://friendica.com/" then this would be "friendica.com',
|
||||||
|
mandatory: true)
|
||||||
|
..addOption('delay',
|
||||||
|
abbr: 'd',
|
||||||
|
help:
|
||||||
|
'Delay in milliseconds between requests to try not to stress the server (thousands of API calls can be made)',
|
||||||
|
defaultsTo: '$defaultRequestDelayMilliseconds')
|
||||||
|
..addOption('max-post-requests',
|
||||||
|
abbr: 'm',
|
||||||
|
help: 'The maximum number of times to query for posts',
|
||||||
|
defaultsTo: '$defaultMaxPosts')
|
||||||
|
..addFlag('read-comments',
|
||||||
|
abbr: 'c',
|
||||||
|
help:
|
||||||
|
'Whether to read comments on posts (defaults to $defaultReadComments)',
|
||||||
|
defaultsTo: defaultReadComments)
|
||||||
|
..addFlag('download-images',
|
||||||
|
abbr: 'i',
|
||||||
|
help:
|
||||||
|
'Whether to download images from posts when those images are stored on the server (not links to other sites) (defaults to $defaultReadImages)',
|
||||||
|
defaultsTo: defaultReadComments);
|
||||||
|
|
||||||
|
late ArgResults settings;
|
||||||
|
try {
|
||||||
|
settings = argParser.parse(arguments);
|
||||||
|
} on ArgParserException catch (e) {
|
||||||
|
print("Error with arguments: ${e.message}");
|
||||||
|
print(argParser.usage);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
stdout.write('Enter Password: ');
|
||||||
|
_setEcho(false);
|
||||||
|
final password = stdin.readLineSync() ?? '';
|
||||||
|
_setEcho(true);
|
||||||
|
print('');
|
||||||
|
|
||||||
|
final username = settings['username'];
|
||||||
|
final client = FriendicaClient(
|
||||||
|
username: username,
|
||||||
|
password: password,
|
||||||
|
serverName: settings['server-name']);
|
||||||
|
final timelineResult = await client.getTimeline(username, 1, 3);
|
||||||
|
timelineResult.match(
|
||||||
|
onSuccess: (posts) => File('/tmp/test.json')
|
||||||
|
.writeAsStringSync(PrettyJsonEncoder().convert(posts)),
|
||||||
|
onError: (error) => print('Error getting posts: $error'));
|
||||||
|
print("Done processing API requests");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Seems in IntelliJ and release build mode setting echo fails
|
||||||
|
void _setEcho(bool value) {
|
||||||
|
try {
|
||||||
|
stdin.echoMode = value;
|
||||||
|
// ignore: empty_catches
|
||||||
|
} catch (e) {
|
||||||
|
print('');
|
||||||
|
print('Error toggling echo to $value, so will stay current value...');
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
44
bin/friendica_client.dart
Normal file
44
bin/friendica_client.dart
Normal file
|
@ -0,0 +1,44 @@
|
||||||
|
import 'dart:convert';
|
||||||
|
import 'dart:io';
|
||||||
|
|
||||||
|
import 'package:result_monad/result_monad.dart';
|
||||||
|
|
||||||
|
class FriendicaClient {
|
||||||
|
final String username;
|
||||||
|
final String password;
|
||||||
|
final String serverName;
|
||||||
|
final _client = HttpClient();
|
||||||
|
late final String _authHeader;
|
||||||
|
|
||||||
|
FriendicaClient(
|
||||||
|
{required this.username,
|
||||||
|
required this.password,
|
||||||
|
required this.serverName}) {
|
||||||
|
final authenticationString = "$username:$password";
|
||||||
|
final encodedAuthString = base64Encode(utf8.encode(authenticationString));
|
||||||
|
_authHeader = "Basic $encodedAuthString";
|
||||||
|
}
|
||||||
|
|
||||||
|
FutureResult<List<dynamic>, String> getTimeline(
|
||||||
|
String userId, int page, int count) async {
|
||||||
|
final request = Uri.parse(
|
||||||
|
'https://$serverName/api/statuses/user_timelineuser_id=$userId&count=$count&page=$page');
|
||||||
|
return await _getApiRequest(request);
|
||||||
|
}
|
||||||
|
|
||||||
|
FutureResult<List<dynamic>, String> getPostComments(int postId) async {
|
||||||
|
return Result.error("Not Implemented");
|
||||||
|
}
|
||||||
|
|
||||||
|
FutureResult<List<dynamic>, String> _getApiRequest(Uri url) async {
|
||||||
|
// TODO Error mode against: bad server URL, bad auth, bad path, empty response
|
||||||
|
final request = await _client.getUrl(url);
|
||||||
|
request.headers.add('authorization', _authHeader);
|
||||||
|
request.headers.contentType =
|
||||||
|
ContentType('application', 'json', charset: 'utf-8');
|
||||||
|
final response = await request.close();
|
||||||
|
final body = await response.transform(utf8.decoder).join('');
|
||||||
|
final bodyJson = jsonDecode(body) as List<dynamic>;
|
||||||
|
return Result.ok(bodyJson);
|
||||||
|
}
|
||||||
|
}
|
11
bin/json_printer.dart
Normal file
11
bin/json_printer.dart
Normal file
|
@ -0,0 +1,11 @@
|
||||||
|
import 'dart:convert';
|
||||||
|
|
||||||
|
class PrettyJsonEncoder {
|
||||||
|
late JsonEncoder encoder;
|
||||||
|
|
||||||
|
PrettyJsonEncoder() {
|
||||||
|
encoder = JsonEncoder.withIndent('\t');
|
||||||
|
}
|
||||||
|
|
||||||
|
String convert(Object json) => encoder.convert(json);
|
||||||
|
}
|
14
pubspec.lock
14
pubspec.lock
|
@ -1,6 +1,13 @@
|
||||||
# Generated by pub
|
# Generated by pub
|
||||||
# See https://dart.dev/tools/pub/glossary#lockfile
|
# See https://dart.dev/tools/pub/glossary#lockfile
|
||||||
packages:
|
packages:
|
||||||
|
args:
|
||||||
|
dependency: "direct main"
|
||||||
|
description:
|
||||||
|
name: args
|
||||||
|
url: "https://pub.dartlang.org"
|
||||||
|
source: hosted
|
||||||
|
version: "2.3.0"
|
||||||
lints:
|
lints:
|
||||||
dependency: "direct dev"
|
dependency: "direct dev"
|
||||||
description:
|
description:
|
||||||
|
@ -8,5 +15,12 @@ packages:
|
||||||
url: "https://pub.dartlang.org"
|
url: "https://pub.dartlang.org"
|
||||||
source: hosted
|
source: hosted
|
||||||
version: "1.0.1"
|
version: "1.0.1"
|
||||||
|
result_monad:
|
||||||
|
dependency: "direct main"
|
||||||
|
description:
|
||||||
|
name: result_monad
|
||||||
|
url: "https://pub.dartlang.org"
|
||||||
|
source: hosted
|
||||||
|
version: "1.0.2"
|
||||||
sdks:
|
sdks:
|
||||||
dart: ">=2.15.1 <3.0.0"
|
dart: ">=2.15.1 <3.0.0"
|
||||||
|
|
|
@ -6,9 +6,9 @@ version: 1.0.0
|
||||||
environment:
|
environment:
|
||||||
sdk: '>=2.15.1 <3.0.0'
|
sdk: '>=2.15.1 <3.0.0'
|
||||||
|
|
||||||
|
|
||||||
dependencies:
|
dependencies:
|
||||||
args: ^2.3.0
|
args: ^2.3.0
|
||||||
|
result_monad: ^1.0.2
|
||||||
|
|
||||||
dev_dependencies:
|
dev_dependencies:
|
||||||
lints: ^1.0.0
|
lints: ^1.0.0
|
||||||
|
|
Loading…
Reference in a new issue