Friendica/fediverse-archiving-tools
1
0
Fork 0
mirror of https://gitlab.com/mysocialportal/fediverse-archiving-tools.git synced 2024-10-18 08:53:31 +00:00
Code Issues Projects Releases Packages Wiki Activity

Add items per page and properly wire in whether to download images

Browse source
This commit is contained in:
Hank Grabowski 2022-01-14 20:25:08 -05:00
parent 743006df35
commit 299638ea74
1 changed files with 33 additions and 19 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

52
bin/friendica_archiver.dart
View file

@ -10,8 +10,8 @@ import 'models.dart';
const defaultRequestDelayMilliseconds = 5000;
const defaultMaxPostsQuery = 1000000000;
const defaultReadComments = false;
const defaultReadImages = false;
const defaultItemsPerPage = 20;
const defaultDownloadImages = true;
void main(List<String> arguments) async {
final argParser = _buildArgs();
@ -38,9 +38,12 @@ void main(List<String> arguments) async {
final maxQueries = int.parse(settings['max-post-requests']);
final queryDelayMillis = int.parse(settings['delay']);
final sleepDuration = Duration(milliseconds: queryDelayMillis);
final itemsPerPage = 20;
final itemsPerPage = int.parse(settings['items-per-page']);
final allEntries = <FriendicaEntry>[];
print(
"Max number of queries will be $maxQueries with $itemsPerPage items per page");
for (var page = 0; page < maxQueries; page++) {
final timelineResult =
await client.getTimeline(username, page, itemsPerPage);
@ -51,13 +54,16 @@ void main(List<String> arguments) async {
final entries = timelineResult.value;
print('# Post/Comments returned for Page $page: ${entries.length}');
allEntries.addAll(entries);
for (final entry in entries) {
final imageEntryResults = await imageArchive.addEntryImages(entry);
if (entry.images.isNotEmpty) {
print(
'${imageEntryResults.length} new images of ${entry.images.length} in entry retrieved');
if (settings['download-images']) {
for (final entry in entries) {
final imageEntryResults = await imageArchive.addEntryImages(entry);
if (entry.images.isNotEmpty) {
print(
'${imageEntryResults.length} new images of ${entry.images.length} in entry retrieved');
}
}
}
if (entries.length != itemsPerPage) {
print(
'Returned less than a full page, assuming at end of timeline and quiting');
@ -66,13 +72,21 @@ void main(List<String> arguments) async {
print("Sleeping for $queryDelayMillis milliseconds before next query");
final postsJsonFile = p.join(baseDirectory.path, 'postsAndComments.json');
final postsJson = allEntries.map((e) => e.originalJson).toList();
// Yes we are rewriting the entire file every time to preserve the results
// over time.
File(postsJsonFile)
.writeAsStringSync(PrettyJsonEncoder().convert(postsJson));
print("Posts written to JSON file: $postsJsonFile");
final imageArchiveJsonFilePath = p.join(baseDirectory.path, 'images.json');
File(imageArchiveJsonFilePath)
.writeAsStringSync(PrettyJsonEncoder().convert(imageArchive.images));
print('Images directory saved to: $imageArchiveJsonFilePath');
if (settings['download-images']) {
final imageArchiveJsonFilePath =
p.join(baseDirectory.path, 'images.json');
File(imageArchiveJsonFilePath)
.writeAsStringSync(PrettyJsonEncoder().convert(imageArchive.images));
print('Images directory saved to: $imageArchiveJsonFilePath');
}
sleep(sleepDuration);
}
@ -101,13 +115,13 @@ ArgParser _buildArgs() => ArgParser()
abbr: 'm',
help: 'The maximum number of times to query for posts',
defaultsTo: '$defaultMaxPostsQuery')
..addFlag('read-comments',
abbr: 'c',
help:
'Whether to read comments on posts (defaults to $defaultReadComments)',
defaultsTo: defaultReadComments)
..addOption('items-per-page',
abbr: 'p',
help: 'The requested number of items per page',
allowed: ['1', '5', '10', '20', '50', '100'],
defaultsTo: '$defaultItemsPerPage')
..addFlag('download-images',
abbr: 'i',
help:
'Whether to download images from posts when those images are stored on the server (not links to other sites) (defaults to $defaultReadImages)',
defaultsTo: defaultReadComments);
'Whether to download images from posts when those images are stored on the server (not links to other sites) (defaults to $defaultDownloadImages)',
defaultsTo: defaultDownloadImages);