mirror of
https://gitlab.com/mysocialportal/fediverse-archiving-tools.git
synced 2024-10-18 08:53:31 +00:00
Add initial archive reader CLI, client, with output
This commit is contained in:
parent
1daef343b4
commit
799b14e020
13 changed files with 171 additions and 82 deletions
6
.gitignore
vendored
6
.gitignore
vendored
|
@ -4,3 +4,9 @@
|
|||
|
||||
# Conventional directory for build output.
|
||||
build/
|
||||
|
||||
# IntelliJ Files
|
||||
.idea
|
||||
|
||||
# Executables
|
||||
*.exe
|
8
.idea/.gitignore
vendored
8
.idea/.gitignore
vendored
|
@ -1,8 +0,0 @@
|
|||
# Default ignored files
|
||||
/shelf/
|
||||
/workspace.xml
|
||||
# Datasource local storage ignored files
|
||||
/dataSources/
|
||||
/dataSources.local.xml
|
||||
# Editor-based HTTP Client requests
|
||||
/httpRequests/
|
|
@ -1,20 +0,0 @@
|
|||
<component name="libraryTable">
|
||||
<library name="Dart Packages" type="DartPackagesLibraryType">
|
||||
<properties>
|
||||
<option name="packageNameToDirsMap">
|
||||
<entry key="lints">
|
||||
<value>
|
||||
<list>
|
||||
<option value="$USER_HOME$/.pub-cache/hosted/pub.dartlang.org/lints-1.0.1/lib" />
|
||||
</list>
|
||||
</value>
|
||||
</entry>
|
||||
</option>
|
||||
</properties>
|
||||
<CLASSES>
|
||||
<root url="file://$USER_HOME$/.pub-cache/hosted/pub.dartlang.org/lints-1.0.1/lib" />
|
||||
</CLASSES>
|
||||
<JAVADOC />
|
||||
<SOURCES />
|
||||
</library>
|
||||
</component>
|
|
@ -1,27 +0,0 @@
|
|||
<component name="libraryTable">
|
||||
<library name="Dart SDK">
|
||||
<CLASSES>
|
||||
<root url="file://$USER_HOME$/snap/flutter/common/flutter/bin/cache/dart-sdk/lib/async" />
|
||||
<root url="file://$USER_HOME$/snap/flutter/common/flutter/bin/cache/dart-sdk/lib/cli" />
|
||||
<root url="file://$USER_HOME$/snap/flutter/common/flutter/bin/cache/dart-sdk/lib/collection" />
|
||||
<root url="file://$USER_HOME$/snap/flutter/common/flutter/bin/cache/dart-sdk/lib/convert" />
|
||||
<root url="file://$USER_HOME$/snap/flutter/common/flutter/bin/cache/dart-sdk/lib/core" />
|
||||
<root url="file://$USER_HOME$/snap/flutter/common/flutter/bin/cache/dart-sdk/lib/developer" />
|
||||
<root url="file://$USER_HOME$/snap/flutter/common/flutter/bin/cache/dart-sdk/lib/ffi" />
|
||||
<root url="file://$USER_HOME$/snap/flutter/common/flutter/bin/cache/dart-sdk/lib/html" />
|
||||
<root url="file://$USER_HOME$/snap/flutter/common/flutter/bin/cache/dart-sdk/lib/indexed_db" />
|
||||
<root url="file://$USER_HOME$/snap/flutter/common/flutter/bin/cache/dart-sdk/lib/io" />
|
||||
<root url="file://$USER_HOME$/snap/flutter/common/flutter/bin/cache/dart-sdk/lib/isolate" />
|
||||
<root url="file://$USER_HOME$/snap/flutter/common/flutter/bin/cache/dart-sdk/lib/js" />
|
||||
<root url="file://$USER_HOME$/snap/flutter/common/flutter/bin/cache/dart-sdk/lib/js_util" />
|
||||
<root url="file://$USER_HOME$/snap/flutter/common/flutter/bin/cache/dart-sdk/lib/math" />
|
||||
<root url="file://$USER_HOME$/snap/flutter/common/flutter/bin/cache/dart-sdk/lib/mirrors" />
|
||||
<root url="file://$USER_HOME$/snap/flutter/common/flutter/bin/cache/dart-sdk/lib/svg" />
|
||||
<root url="file://$USER_HOME$/snap/flutter/common/flutter/bin/cache/dart-sdk/lib/typed_data" />
|
||||
<root url="file://$USER_HOME$/snap/flutter/common/flutter/bin/cache/dart-sdk/lib/web_audio" />
|
||||
<root url="file://$USER_HOME$/snap/flutter/common/flutter/bin/cache/dart-sdk/lib/web_gl" />
|
||||
</CLASSES>
|
||||
<JAVADOC />
|
||||
<SOURCES />
|
||||
</library>
|
||||
</component>
|
|
@ -1,6 +0,0 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="ProjectRootManager">
|
||||
<output url="file://$PROJECT_DIR$/out" />
|
||||
</component>
|
||||
</project>
|
|
@ -1,8 +0,0 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="ProjectModuleManager">
|
||||
<modules>
|
||||
<module fileurl="file://$PROJECT_DIR$/friendica_archiver.iml" filepath="$PROJECT_DIR$/friendica_archiver.iml" />
|
||||
</modules>
|
||||
</component>
|
||||
</project>
|
|
@ -1,10 +0,0 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="RunConfigurationProducerService">
|
||||
<option name="ignoredProducers">
|
||||
<set>
|
||||
<option value="com.android.tools.idea.compose.preview.runconfiguration.ComposePreviewRunConfigurationProducer" />
|
||||
</set>
|
||||
</option>
|
||||
</component>
|
||||
</project>
|
11
bin/exec_error.dart
Normal file
11
bin/exec_error.dart
Normal file
|
@ -0,0 +1,11 @@
|
|||
class ExecError {
|
||||
final ErrorType type;
|
||||
final String message;
|
||||
|
||||
ExecError({required this.type, this.message = ''});
|
||||
}
|
||||
|
||||
enum ErrorType {
|
||||
authentication,
|
||||
missingEndpoint,
|
||||
}
|
|
@ -1,3 +1,85 @@
|
|||
void main(List<String> arguments) {
|
||||
print('Hello world!');
|
||||
import 'dart:io';
|
||||
|
||||
import 'package:args/args.dart';
|
||||
|
||||
import 'friendica_client.dart';
|
||||
import 'json_printer.dart';
|
||||
|
||||
const defaultRequestDelayMilliseconds = 5000;
|
||||
const defaultMaxPosts = 1;
|
||||
const defaultReadComments = false;
|
||||
const defaultReadImages = false;
|
||||
|
||||
void main(List<String> arguments) async {
|
||||
final argParser = ArgParser()
|
||||
..addOption('archive-folder',
|
||||
abbr: 'a',
|
||||
help:
|
||||
'Specifies the local folder all data files pulled from the server will be stored',
|
||||
mandatory: true)
|
||||
..addOption('username',
|
||||
abbr: 'u', help: 'Username on your Friendica instance', mandatory: true)
|
||||
..addOption('server-name',
|
||||
abbr: 's',
|
||||
help:
|
||||
'The server name for your instance. (e.g. if the URL in your browser is "https://friendica.com/" then this would be "friendica.com',
|
||||
mandatory: true)
|
||||
..addOption('delay',
|
||||
abbr: 'd',
|
||||
help:
|
||||
'Delay in milliseconds between requests to try not to stress the server (thousands of API calls can be made)',
|
||||
defaultsTo: '$defaultRequestDelayMilliseconds')
|
||||
..addOption('max-post-requests',
|
||||
abbr: 'm',
|
||||
help: 'The maximum number of times to query for posts',
|
||||
defaultsTo: '$defaultMaxPosts')
|
||||
..addFlag('read-comments',
|
||||
abbr: 'c',
|
||||
help:
|
||||
'Whether to read comments on posts (defaults to $defaultReadComments)',
|
||||
defaultsTo: defaultReadComments)
|
||||
..addFlag('download-images',
|
||||
abbr: 'i',
|
||||
help:
|
||||
'Whether to download images from posts when those images are stored on the server (not links to other sites) (defaults to $defaultReadImages)',
|
||||
defaultsTo: defaultReadComments);
|
||||
|
||||
late ArgResults settings;
|
||||
try {
|
||||
settings = argParser.parse(arguments);
|
||||
} on ArgParserException catch (e) {
|
||||
print("Error with arguments: ${e.message}");
|
||||
print(argParser.usage);
|
||||
return;
|
||||
}
|
||||
|
||||
stdout.write('Enter Password: ');
|
||||
_setEcho(false);
|
||||
final password = stdin.readLineSync() ?? '';
|
||||
_setEcho(true);
|
||||
print('');
|
||||
|
||||
final username = settings['username'];
|
||||
final client = FriendicaClient(
|
||||
username: username,
|
||||
password: password,
|
||||
serverName: settings['server-name']);
|
||||
final timelineResult = await client.getTimeline(username, 1, 3);
|
||||
timelineResult.match(
|
||||
onSuccess: (posts) => File('/tmp/test.json')
|
||||
.writeAsStringSync(PrettyJsonEncoder().convert(posts)),
|
||||
onError: (error) => print('Error getting posts: $error'));
|
||||
print("Done processing API requests");
|
||||
return;
|
||||
}
|
||||
|
||||
// Seems in IntelliJ and release build mode setting echo fails
|
||||
void _setEcho(bool value) {
|
||||
try {
|
||||
stdin.echoMode = value;
|
||||
// ignore: empty_catches
|
||||
} catch (e) {
|
||||
print('');
|
||||
print('Error toggling echo to $value, so will stay current value...');
|
||||
}
|
||||
}
|
||||
|
|
44
bin/friendica_client.dart
Normal file
44
bin/friendica_client.dart
Normal file
|
@ -0,0 +1,44 @@
|
|||
import 'dart:convert';
|
||||
import 'dart:io';
|
||||
|
||||
import 'package:result_monad/result_monad.dart';
|
||||
|
||||
class FriendicaClient {
|
||||
final String username;
|
||||
final String password;
|
||||
final String serverName;
|
||||
final _client = HttpClient();
|
||||
late final String _authHeader;
|
||||
|
||||
FriendicaClient(
|
||||
{required this.username,
|
||||
required this.password,
|
||||
required this.serverName}) {
|
||||
final authenticationString = "$username:$password";
|
||||
final encodedAuthString = base64Encode(utf8.encode(authenticationString));
|
||||
_authHeader = "Basic $encodedAuthString";
|
||||
}
|
||||
|
||||
FutureResult<List<dynamic>, String> getTimeline(
|
||||
String userId, int page, int count) async {
|
||||
final request = Uri.parse(
|
||||
'https://$serverName/api/statuses/user_timelineuser_id=$userId&count=$count&page=$page');
|
||||
return await _getApiRequest(request);
|
||||
}
|
||||
|
||||
FutureResult<List<dynamic>, String> getPostComments(int postId) async {
|
||||
return Result.error("Not Implemented");
|
||||
}
|
||||
|
||||
FutureResult<List<dynamic>, String> _getApiRequest(Uri url) async {
|
||||
// TODO Error mode against: bad server URL, bad auth, bad path, empty response
|
||||
final request = await _client.getUrl(url);
|
||||
request.headers.add('authorization', _authHeader);
|
||||
request.headers.contentType =
|
||||
ContentType('application', 'json', charset: 'utf-8');
|
||||
final response = await request.close();
|
||||
final body = await response.transform(utf8.decoder).join('');
|
||||
final bodyJson = jsonDecode(body) as List<dynamic>;
|
||||
return Result.ok(bodyJson);
|
||||
}
|
||||
}
|
11
bin/json_printer.dart
Normal file
11
bin/json_printer.dart
Normal file
|
@ -0,0 +1,11 @@
|
|||
import 'dart:convert';
|
||||
|
||||
class PrettyJsonEncoder {
|
||||
late JsonEncoder encoder;
|
||||
|
||||
PrettyJsonEncoder() {
|
||||
encoder = JsonEncoder.withIndent('\t');
|
||||
}
|
||||
|
||||
String convert(Object json) => encoder.convert(json);
|
||||
}
|
14
pubspec.lock
14
pubspec.lock
|
@ -1,6 +1,13 @@
|
|||
# Generated by pub
|
||||
# See https://dart.dev/tools/pub/glossary#lockfile
|
||||
packages:
|
||||
args:
|
||||
dependency: "direct main"
|
||||
description:
|
||||
name: args
|
||||
url: "https://pub.dartlang.org"
|
||||
source: hosted
|
||||
version: "2.3.0"
|
||||
lints:
|
||||
dependency: "direct dev"
|
||||
description:
|
||||
|
@ -8,5 +15,12 @@ packages:
|
|||
url: "https://pub.dartlang.org"
|
||||
source: hosted
|
||||
version: "1.0.1"
|
||||
result_monad:
|
||||
dependency: "direct main"
|
||||
description:
|
||||
name: result_monad
|
||||
url: "https://pub.dartlang.org"
|
||||
source: hosted
|
||||
version: "1.0.2"
|
||||
sdks:
|
||||
dart: ">=2.15.1 <3.0.0"
|
||||
|
|
|
@ -6,9 +6,9 @@ version: 1.0.0
|
|||
environment:
|
||||
sdk: '>=2.15.1 <3.0.0'
|
||||
|
||||
|
||||
dependencies:
|
||||
args: ^2.3.0
|
||||
result_monad: ^1.0.2
|
||||
|
||||
dev_dependencies:
|
||||
lints: ^1.0.0
|
||||
|
|
Loading…
Reference in a new issue