From 86105635ca3b0e096d128718b778b313b5eaa88e Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 27 Jul 2022 17:39:00 +0000 Subject: [PATCH] Improved message handling / new activity relay handling --- database.sql | 31 +-- doc/database.md | 2 +- doc/database/db_apcontact.md | 1 + doc/database/db_post-activity.md | 29 ++ mod/item.php | 10 - src/Database/DBStructure.php | 2 +- src/Model/APContact.php | 7 +- src/Model/Conversation.php | 62 ----- src/Model/GServer.php | 7 +- src/Model/Item.php | 8 +- src/Model/Post/Activity.php | 74 +++++ src/Model/Tag.php | 2 +- src/Module/Admin/Item/Source.php | 6 - src/Module/DFRN/Notify.php | 2 +- .../Notifications/Factory/Notification.php | 5 +- src/Protocol/ActivityPub/Delivery.php | 5 +- src/Protocol/ActivityPub/Processor.php | 16 +- src/Protocol/ActivityPub/Queue.php | 56 ++++ src/Protocol/ActivityPub/Receiver.php | 49 ++-- src/Protocol/ActivityPub/Transmitter.php | 6 +- src/Protocol/DFRN.php | 27 +- src/Protocol/OStatus.php | 253 +----------------- src/Worker/Cron.php | 2 +- ...Conversations.php => ExpireActivities.php} | 12 +- src/Worker/Notifier.php | 2 +- static/dbstructure.config.php | 32 +-- 26 files changed, 280 insertions(+), 428 deletions(-) create mode 100644 doc/database/db_post-activity.md create mode 100644 src/Model/Post/Activity.php rename src/Worker/{ExpireConversations.php => ExpireActivities.php} (76%) diff --git a/database.sql b/database.sql index 6b8d7df2ff..02e21f137e 100644 --- a/database.sql +++ b/database.sql @@ -1,6 +1,6 @@ -- ------------------------------------------ -- Friendica 2022.09-dev (Giant Rhubarb) --- DB_UPDATE_VERSION 1474 +-- DB_UPDATE_VERSION 1475 -- ------------------------------------------ @@ -338,6 +338,7 @@ CREATE TABLE IF NOT EXISTS `apcontact` ( `featured-tags` varchar(255) COMMENT 'Address for the collection of featured tags', `manually-approve` boolean COMMENT '', `discoverable` boolean COMMENT 'Mastodon extension: true if profile is published in their directory', + `suspended` boolean COMMENT 'Mastodon extension: true if profile is suspended', `nick` varchar(255) NOT NULL DEFAULT '' COMMENT '', `name` varchar(255) COMMENT '', `about` text COMMENT '', @@ -503,23 +504,6 @@ CREATE TABLE IF NOT EXISTS `conv` ( FOREIGN KEY (`uid`) REFERENCES `user` (`uid`) ON UPDATE RESTRICT ON DELETE CASCADE ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='private messages'; --- --- TABLE conversation --- -CREATE TABLE IF NOT EXISTS `conversation` ( - `item-uri` varbinary(255) NOT NULL COMMENT 'Original URI of the item - unrelated to the table with the same name', - `reply-to-uri` varbinary(255) NOT NULL DEFAULT '' COMMENT 'URI to which this item is a reply', - `conversation-uri` varbinary(255) NOT NULL DEFAULT '' COMMENT 'GNU Social conversation URI', - `conversation-href` varbinary(255) NOT NULL DEFAULT '' COMMENT 'GNU Social conversation link', - `protocol` tinyint unsigned NOT NULL DEFAULT 255 COMMENT 'The protocol of the item', - `direction` tinyint unsigned NOT NULL DEFAULT 0 COMMENT 'How the message arrived here: 1=push, 2=pull', - `source` mediumtext COMMENT 'Original source', - `received` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Receiving date', - PRIMARY KEY(`item-uri`), - INDEX `conversation-uri` (`conversation-uri`), - INDEX `received` (`received`) -) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Raw data and structure information for messages'; - -- -- TABLE workerqueue -- @@ -1117,6 +1101,17 @@ CREATE TABLE IF NOT EXISTS `post` ( FOREIGN KEY (`vid`) REFERENCES `verb` (`id`) ON UPDATE RESTRICT ON DELETE RESTRICT ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Structure for all posts'; +-- +-- TABLE post-activity +-- +CREATE TABLE IF NOT EXISTS `post-activity` ( + `uri-id` int unsigned NOT NULL COMMENT 'Id of the item-uri table entry that contains the item uri', + `activity` mediumtext COMMENT 'Original activity', + `received` datetime COMMENT '', + PRIMARY KEY(`uri-id`), + FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE +) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Original remote activity'; + -- -- TABLE post-category -- diff --git a/doc/database.md b/doc/database.md index 0effb1754a..9932ee307a 100644 --- a/doc/database.md +++ b/doc/database.md @@ -19,7 +19,6 @@ Database Tables | [contact](help/database/db_contact) | contact table | | [contact-relation](help/database/db_contact-relation) | Contact relations | | [conv](help/database/db_conv) | private messages | -| [conversation](help/database/db_conversation) | Raw data and structure information for messages | | [delayed-post](help/database/db_delayed-post) | Posts that are about to be distributed at a later time | | [diaspora-interaction](help/database/db_diaspora-interaction) | Signed Diaspora Interaction | | [endpoint](help/database/db_endpoint) | ActivityPub endpoints - used in the ActivityPub implementation | @@ -50,6 +49,7 @@ Database Tables | [permissionset](help/database/db_permissionset) | | | [photo](help/database/db_photo) | photo storage | | [post](help/database/db_post) | Structure for all posts | +| [post-activity](help/database/db_post-activity) | Original remote activity | | [post-category](help/database/db_post-category) | post relation to categories | | [post-collection](help/database/db_post-collection) | Collection of posts | | [post-content](help/database/db_post-content) | Content for all posts | diff --git a/doc/database/db_apcontact.md b/doc/database/db_apcontact.md index 34cf4ec18b..578df57bfd 100644 --- a/doc/database/db_apcontact.md +++ b/doc/database/db_apcontact.md @@ -21,6 +21,7 @@ Fields | featured-tags | Address for the collection of featured tags | varchar(255) | YES | | NULL | | | manually-approve | | boolean | YES | | NULL | | | discoverable | Mastodon extension: true if profile is published in their directory | boolean | YES | | NULL | | +| suspended | Mastodon extension: true if profile is suspended | boolean | YES | | NULL | | | nick | | varchar(255) | NO | | | | | name | | varchar(255) | YES | | NULL | | | about | | text | YES | | NULL | | diff --git a/doc/database/db_post-activity.md b/doc/database/db_post-activity.md new file mode 100644 index 0000000000..56d04d37d7 --- /dev/null +++ b/doc/database/db_post-activity.md @@ -0,0 +1,29 @@ +Table post-activity +=========== + +Original remote activity + +Fields +------ + +| Field | Description | Type | Null | Key | Default | Extra | +| -------- | --------------------------------------------------------- | ------------ | ---- | --- | ------- | ----- | +| uri-id | Id of the item-uri table entry that contains the item uri | int unsigned | NO | PRI | NULL | | +| activity | Original activity | mediumtext | YES | | NULL | | +| received | | datetime | YES | | NULL | | + +Indexes +------------ + +| Name | Fields | +| ------- | ------ | +| PRIMARY | uri-id | + +Foreign Keys +------------ + +| Field | Target Table | Target Field | +|-------|--------------|--------------| +| uri-id | [item-uri](help/database/db_item-uri) | id | + +Return to [database documentation](help/database) diff --git a/mod/item.php b/mod/item.php index a840ea9ad9..ce4cd45ca7 100644 --- a/mod/item.php +++ b/mod/item.php @@ -595,16 +595,6 @@ function item_post(App $a) { $datarray['protocol'] = Conversation::PARCEL_DIRECT; $datarray['direction'] = Conversation::PUSH; - $conversation = DBA::selectFirst('conversation', ['conversation-uri', 'conversation-href'], ['item-uri' => $datarray['thr-parent']]); - if (DBA::isResult($conversation)) { - if ($conversation['conversation-uri'] != '') { - $datarray['conversation-uri'] = $conversation['conversation-uri']; - } - if ($conversation['conversation-href'] != '') { - $datarray['conversation-href'] = $conversation['conversation-href']; - } - } - if ($orig_post) { $datarray['edit'] = true; } else { diff --git a/src/Database/DBStructure.php b/src/Database/DBStructure.php index 441116d167..d14e791e2c 100644 --- a/src/Database/DBStructure.php +++ b/src/Database/DBStructure.php @@ -74,7 +74,7 @@ class DBStructure $old_tables = ['fserver', 'gcign', 'gcontact', 'gcontact-relation', 'gfollower' ,'glink', 'item-delivery-data', 'item-activity', 'item-content', 'item_id', 'participation', 'poll', 'poll_result', 'queue', 'retriever_rule', 'deliverq', 'dsprphotoq', 'ffinder', 'sign', 'spam', 'term', 'user-item', 'thread', 'item', 'challenge', - 'auth_codes', 'tokens', 'clients', 'profile_check', 'host']; + 'auth_codes', 'tokens', 'clients', 'profile_check', 'host', 'conversation']; $tables = DBA::selectToArray('INFORMATION_SCHEMA.TABLES', ['TABLE_NAME'], ['TABLE_SCHEMA' => DBA::databaseName(), 'TABLE_TYPE' => 'BASE TABLE']); diff --git a/src/Model/APContact.php b/src/Model/APContact.php index 841c028909..bb6f12e10c 100644 --- a/src/Model/APContact.php +++ b/src/Model/APContact.php @@ -38,6 +38,7 @@ use Friendica\Util\DateTimeFormat; use Friendica\Util\HTTPSignature; use Friendica\Util\JsonLD; use Friendica\Util\Network; +use GuzzleHttp\Psr7\Uri; class APContact { @@ -310,6 +311,8 @@ class APContact $apcontact['manually-approve'] = (int)JsonLD::fetchElement($compacted, 'as:manuallyApprovesFollowers'); + $apcontact['suspended'] = (int)JsonLD::fetchElement($compacted, 'toot:suspended'); + if (!empty($compacted['as:generator'])) { $apcontact['baseurl'] = JsonLD::fetchElement($compacted['as:generator'], 'as:url', '@id'); $apcontact['generator'] = JsonLD::fetchElement($compacted['as:generator'], 'as:name', '@value'); @@ -380,11 +383,11 @@ class APContact if (strlen($apcontact['photo']) > 255) { $parts = parse_url($apcontact['photo']); unset($parts['fragment']); - $apcontact['photo'] = Network::unparseURL($parts); + $apcontact['photo'] = Uri::fromParts($parts); if (strlen($apcontact['photo']) > 255) { unset($parts['query']); - $apcontact['photo'] = Network::unparseURL($parts); + $apcontact['photo'] = Uri::fromParts($parts); } if (strlen($apcontact['photo']) > 255) { diff --git a/src/Model/Conversation.php b/src/Model/Conversation.php index 7d8b8058f1..4739b2f0a1 100644 --- a/src/Model/Conversation.php +++ b/src/Model/Conversation.php @@ -21,11 +21,6 @@ namespace Friendica\Model; -use Friendica\Core\Protocol; -use Friendica\Database\Database; -use Friendica\Database\DBA; -use Friendica\Util\DateTimeFormat; - class Conversation { /* @@ -62,61 +57,4 @@ class Conversation */ const RELAY = 3; - public static function getByItemUri(string $item_uri) - { - return DBA::selectFirst('conversation', [], ['item-uri' => $item_uri]); - } - - /** - * Store the conversation data - * - * @param array $arr Item array with conversation data - * @return array Item array with removed conversation data - * @throws \Exception - */ - public static function insert(array $arr): array - { - if (in_array(($arr['network'] ?? '') ?: Protocol::PHANTOM, - [Protocol::ACTIVITYPUB, Protocol::DFRN, Protocol::DIASPORA, Protocol::OSTATUS, Protocol::TWITTER]) && !empty($arr['uri'])) { - $conversation = ['item-uri' => $arr['uri'], 'received' => DateTimeFormat::utcNow()]; - - if (isset($arr['parent-uri']) && ($arr['parent-uri'] != $arr['uri'])) { - $conversation['reply-to-uri'] = $arr['parent-uri']; - } - - if (isset($arr['thr-parent']) && ($arr['thr-parent'] != $arr['uri'])) { - $conversation['reply-to-uri'] = $arr['thr-parent']; - } - - if (isset($arr['conversation-uri'])) { - $conversation['conversation-uri'] = $arr['conversation-uri']; - } - - if (isset($arr['conversation-href'])) { - $conversation['conversation-href'] = $arr['conversation-href']; - } - - if (isset($arr['protocol'])) { - $conversation['protocol'] = $arr['protocol']; - } - - if (isset($arr['direction'])) { - $conversation['direction'] = $arr['direction']; - } - - if (isset($arr['source'])) { - $conversation['source'] = $arr['source']; - } - - if (!DBA::exists('conversation', ['item-uri' => $conversation['item-uri']])) { - DBA::insert('conversation', $conversation, Database::INSERT_IGNORE); - } - } - - unset($arr['conversation-uri']); - unset($arr['conversation-href']); - unset($arr['source']); - - return $arr; - } } diff --git a/src/Model/GServer.php b/src/Model/GServer.php index 8305a3c965..d0993141a8 100644 --- a/src/Model/GServer.php +++ b/src/Model/GServer.php @@ -563,8 +563,13 @@ class GServer $serverdata['registered-users'] = $serverdata['registered-users'] ?? 0; + // Numbers above a reasonable value (10 millions) are ignored + if ($serverdata['registered-users'] > 10000000) { + $serverdata['registered-users'] = 0; + } + // On an active server there has to be at least a single user - if (!in_array($serverdata['network'], [Protocol::PHANTOM, Protocol::FEED]) && ($serverdata['registered-users'] == 0)) { + if (!in_array($serverdata['network'], [Protocol::PHANTOM, Protocol::FEED]) && ($serverdata['registered-users'] <= 0)) { $serverdata['registered-users'] = 1; } elseif (in_array($serverdata['network'], [Protocol::PHANTOM, Protocol::FEED])) { $serverdata['registered-users'] = 0; diff --git a/src/Model/Item.php b/src/Model/Item.php index 3ae23412e6..55022519aa 100644 --- a/src/Model/Item.php +++ b/src/Model/Item.php @@ -801,7 +801,10 @@ class Item $item['parent-uri-id'] = ItemURI::getIdByURI($item['parent-uri']); // Store conversation data - $item = Conversation::insert($item); + $source = $item['source'] ?? ''; + unset($item['conversation-uri']); + unset($item['conversation-href']); + unset($item['source']); /* * Do we already have this item? @@ -1259,6 +1262,9 @@ class Item } if ($transmit) { + if (!empty($source)) { + Post\Activity::insert($item['uri-id'], $source); + } Worker::add(['priority' => $priority, 'dont_fork' => true], 'Notifier', $notify_type, (int)$posted_item['uri-id'], (int)$posted_item['uid']); } diff --git a/src/Model/Post/Activity.php b/src/Model/Post/Activity.php new file mode 100644 index 0000000000..860e9df5d8 --- /dev/null +++ b/src/Model/Post/Activity.php @@ -0,0 +1,74 @@ +. + * + */ + +namespace Friendica\Model\Post; + +use Friendica\Database\Database; +use Friendica\Database\DBA; +use Friendica\Util\DateTimeFormat; + +class Activity +{ + /** + * Insert a new post-activity entry + * + * @param integer $uri_id + * @param array $fields + * + * @return bool success + */ + public static function insert(int $uri_id, string $source): bool + { + // Additionally assign the key fields + $fields = [ + 'uri-id' => $uri_id, + 'activity' => $source, + 'received' => DateTimeFormat::utcNow() + ]; + + return DBA::insert('post-activity', $fields, Database::INSERT_IGNORE); + } + + /** + * Retrieves activity of the given uri-id + * + * @param int $uriId + * + * @return array + */ + public static function getByURIId(int $uriId): array + { + $activity = DBA::selectFirst('post-activity', [], ['uri-id' => $uriId]); + return json_decode($activity['activity'] ?? '', true) ?? []; + } + + /** + * Checks if the given uridid has a stored activity + * + * @param integer $uriId + * + * @return boolean + */ + public static function exists(int $uriId): bool + { + return DBA::exists('post-activity', ['uri-id' => $uriId]); + } +} diff --git a/src/Model/Tag.php b/src/Model/Tag.php index 1381f68611..2371d6af82 100644 --- a/src/Model/Tag.php +++ b/src/Model/Tag.php @@ -545,7 +545,7 @@ class Tag break; default: - Logger:warning('Unknown tag type found', $tag); + Logger::warning('Unknown tag type found', $tag); } } DBA::close($taglist); diff --git a/src/Module/Admin/Item/Source.php b/src/Module/Admin/Item/Source.php index d03eb4c9e0..fe3f3674b7 100644 --- a/src/Module/Admin/Item/Source.php +++ b/src/Module/Admin/Item/Source.php @@ -35,7 +35,6 @@ class Source extends BaseAdmin $guid = basename($_REQUEST['guid'] ?? $this->parameters['guid'] ?? ''); - $source = ''; $item_uri = ''; $item_id = ''; $terms = []; @@ -43,11 +42,8 @@ class Source extends BaseAdmin $item = Model\Post::selectFirst(['id', 'uri-id', 'guid', 'uri'], ['guid' => $guid]); if ($item) { - $conversation = Model\Conversation::getByItemUri($item['uri']); - $item_id = $item['id']; $item_uri = $item['uri']; - $source = $conversation['source']; $terms = Model\Tag::getByURIId($item['uri-id'], [Model\Tag::HASHTAG, Model\Tag::MENTION, Model\Tag::IMPLICIT_MENTION]); } } @@ -56,7 +52,6 @@ class Source extends BaseAdmin $o = Renderer::replaceMacros($tpl, [ '$title' => DI::l10n()->t('Item Source'), '$guid' => ['guid', DI::l10n()->t('Item Guid'), $guid, ''], - '$source' => $source, '$item_uri' => $item_uri, '$item_id' => $item_id, '$terms' => $terms, @@ -70,7 +65,6 @@ class Source extends BaseAdmin '$urllbl' => DI::l10n()->t('URL'), '$mentionlbl' => DI::l10n()->t('Mention'), '$implicitlbl' => DI::l10n()->t('Implicit Mention'), - '$sourcelbl' => DI::l10n()->t('Source'), ]); return $o; diff --git a/src/Module/DFRN/Notify.php b/src/Module/DFRN/Notify.php index 2dae0da0ad..7db56f0b86 100644 --- a/src/Module/DFRN/Notify.php +++ b/src/Module/DFRN/Notify.php @@ -59,7 +59,7 @@ class Notify extends BaseModule } } - private static function dispatchPublic(array $postdata) + private static function dispatchPublic(string $postdata) { $msg = Diaspora::decodeRaw($postdata, '', true); if (!is_array($msg)) { diff --git a/src/Navigation/Notifications/Factory/Notification.php b/src/Navigation/Notifications/Factory/Notification.php index 5c1f74df4f..7f6988b3bf 100644 --- a/src/Navigation/Notifications/Factory/Notification.php +++ b/src/Navigation/Notifications/Factory/Notification.php @@ -163,9 +163,10 @@ class Notification extends BaseFactory implements ICanCreateFromTableRow } if (($Notification->verb == Activity::POST) || ($Notification->type === Post\UserNotification::TYPE_SHARED)) { - $item = Post::selectFirst([], ['uri-id' => $item['thr-parent-id'], 'uid' => [0, $Notification->uid]], ['order' => ['uid' => true]]); + $thrparentid = $item['thr-parent-id']; + $item = Post::selectFirst([], ['uri-id' => $thrparentid, 'uid' => [0, $Notification->uid]], ['order' => ['uid' => true]]); if (empty($item)) { - $this->logger->info('Thread parent post not found', ['uri-id' => $item['thr-parent-id']]); + $this->logger->info('Thread parent post not found', ['uri-id' => $thrparentid]); return $message; } } diff --git a/src/Protocol/ActivityPub/Delivery.php b/src/Protocol/ActivityPub/Delivery.php index 6e97b0b927..fac8536d67 100644 --- a/src/Protocol/ActivityPub/Delivery.php +++ b/src/Protocol/ActivityPub/Delivery.php @@ -110,7 +110,10 @@ class Delivery } elseif ($cmd == WorkerDelivery::PROFILEUPDATE) { $success = ActivityPub\Transmitter::sendProfileUpdate($uid, $inbox); } else { - $data = ActivityPub\Transmitter::createCachedActivityFromItem($item_id); + $data = Post\Activity::getByURIId($uri_id); + if (empty($data)) { + $data = ActivityPub\Transmitter::createCachedActivityFromItem($item_id); + } if (!empty($data)) { $timestamp = microtime(true); $response = HTTPSignature::post($data, $inbox, $uid); diff --git a/src/Protocol/ActivityPub/Processor.php b/src/Protocol/ActivityPub/Processor.php index d012498ab2..db8f28aa0c 100644 --- a/src/Protocol/ActivityPub/Processor.php +++ b/src/Protocol/ActivityPub/Processor.php @@ -361,8 +361,6 @@ class Processor if (!empty($activity['raw'])) { $item['source'] = $activity['raw']; $item['protocol'] = Conversation::PARCEL_ACTIVITYPUB; - $item['conversation-href'] = $activity['context'] ?? ''; - $item['conversation-uri'] = $activity['conversation'] ?? ''; if (isset($activity['push'])) { $item['direction'] = $activity['push'] ? Conversation::PUSH : Conversation::PULL; @@ -475,7 +473,19 @@ class Processor } // @todo To ensure that the remote system is working correctly, we can check if the "Content-Type" contains JSON - return in_array($curlResult->getReturnCode(), [404]); + if (in_array($curlResult->getReturnCode(), [404])) { + return true; + } + + $object = json_decode($curlResult->getBody(), true); + if (!empty($object)) { + $activity = JsonLD::compact($object); + if (JsonLD::fetchElement($activity, '@type') == 'as:Tombstone') { + return true; + } + } + + return false; } /** * Delete items diff --git a/src/Protocol/ActivityPub/Queue.php b/src/Protocol/ActivityPub/Queue.php index faaf0aa3bc..d47007eb13 100644 --- a/src/Protocol/ActivityPub/Queue.php +++ b/src/Protocol/ActivityPub/Queue.php @@ -25,7 +25,9 @@ use Friendica\Core\Logger; use Friendica\Database\Database; use Friendica\Database\DBA; use Friendica\DI; +use Friendica\Protocol\ActivityPub; use Friendica\Util\DateTimeFormat; +use Friendica\Util\JsonLD; /** * This class handles the processing of incoming posts @@ -214,6 +216,7 @@ class Queue Logger::debug('Process leftover entry', $entry); self::process($entry['id']); } + DBA::close($entries); } /** @@ -247,5 +250,58 @@ class Queue while ($entry = DBA::fetch($entries)) { self::process($entry['id']); } + DBA::close($entries); + } + + /** + * Prepare the queue entry. + * This is a test function that is used solely for development. + * + * @param integer $id + * @return array + */ + public static function reprepareActivityById(int $id): array + { + $entry = DBA::selectFirst('inbox-entry', [], ['id' => $id]); + if (empty($entry)) { + return []; + } + + $receiver = DBA::selectFirst('inbox-entry-receiver', ['uid'], ['queue-id' => $id]); + if (!empty($receiver)) { + $uid = $receiver['uid']; + } else { + $uid = 0; + } + + $trust_source = $entry['trust']; + + $data = json_decode($entry['activity'], true); + $activity = json_decode($data['raw'], true); + + $ldactivity = JsonLD::compact($activity); + return [ + 'data' => Receiver::prepareObjectData($ldactivity, $uid, $entry['push'], $trust_source), + 'trust' => $trust_source + ]; + } + + /** + * Set the trust for all untrusted entries. + * This is a test function that is used solely for development. + * + * @return void + */ + public static function reprepareAll() + { + $entries = DBA::select('inbox-entry', ['id'], ["NOT `trust` AND `wid` IS NULL"], ['order' => ['id' => true]]); + while ($entry = DBA::fetch($entries)) { + $data = self::reprepareActivityById($entry['id'], false); + if ($data['trust']) { + DBA::update('inbox-entry', ['trust' => true], ['id' => $entry['id']]); + } + } + DBA::close($entries); + } } diff --git a/src/Protocol/ActivityPub/Receiver.php b/src/Protocol/ActivityPub/Receiver.php index 66653579ea..9e79216996 100644 --- a/src/Protocol/ActivityPub/Receiver.php +++ b/src/Protocol/ActivityPub/Receiver.php @@ -272,26 +272,39 @@ class Receiver */ public static function prepareObjectData(array $activity, int $uid, bool $push, bool &$trust_source): array { - $id = JsonLD::fetchElement($activity, '@id'); + $id = JsonLD::fetchElement($activity, '@id'); + $type = JsonLD::fetchElement($activity, '@type'); $object_id = JsonLD::fetchElement($activity, 'as:object', '@id'); + if (!empty($object_id) && in_array($type, ['as:Create', 'as:Update'])) { + $fetch_id = $object_id; + } else { + $fetch_id = $id; + } + + if (!empty($activity['as:object'])) { + $object_type = JsonLD::fetchElement($activity['as:object'], '@type'); + } + if (!empty($id) && !$trust_source) { $fetch_uid = $uid ?: self::getBestUserForActivity($activity); - $fetched_activity = ActivityPub::fetchContent($id, $fetch_uid); + $fetched_activity = ActivityPub::fetchContent($fetch_id, $fetch_uid); if (!empty($fetched_activity)) { $object = JsonLD::compact($fetched_activity); - $fetched_id = JsonLD::fetchElement($object, '@id'); - if ($fetched_id == $id) { + + $fetched_id = JsonLD::fetchElement($object, '@id'); + $fetched_type = JsonLD::fetchElement($object, '@type'); + + if (($fetched_id == $id) && !empty($fetched_type) && ($fetched_type == $type)) { Logger::info('Activity had been fetched successfully', ['id' => $id]); $trust_source = true; - if ($id != $object_id) { - $activity = $object; - } else { - Logger::info('Fetched data is the object instead of the activity', ['id' => $id]); - unset($object['@context']); - $activity['as:object'] = $object; - } + $activity = $object; + } elseif (($fetched_id == $object_id) && !empty($fetched_type) && ($fetched_type == $object_type)) { + Logger::info('Fetched data is the object instead of the activity', ['id' => $id]); + $trust_source = true; + unset($object['@context']); + $activity['as:object'] = $object; } else { Logger::info('Activity id is not equal', ['id' => $id, 'fetched' => $fetched_id]); } @@ -371,9 +384,9 @@ class Receiver $object_data['object_object'] = JsonLD::fetchElement($activity['as:object'], 'as:object'); $object_data['object_type'] = JsonLD::fetchElement($activity['as:object'], '@type'); $object_data['push'] = $push; - if ($type == 'as:Delete') { + if (!$trust_source && ($type == 'as:Delete')) { $apcontact = APContact::getByURL($object_data['object_id'], true); - $trust_source = ($apcontact['type'] == 'Tombstone'); + $trust_source = empty($apcontact) || ($apcontact['type'] == 'Tombstone') || $apcontact['suspended']; } } elseif (in_array($type, ['as:Create', 'as:Update', 'as:Announce', 'as:Invite']) || strpos($type, '#emojiReaction')) { // Fetch the content only on activities where this matters @@ -430,8 +443,11 @@ class Receiver $object_data['object_object_type'] = self::fetchObjectType([], $object_data['object_object'], $fetch_uid); } - if (($type == 'as:Delete') && in_array($object_data['object_type'], array_merge(['as:Tombstone'], self::CONTENT_TYPES))) { + if (!$trust_source && ($type == 'as:Delete') && in_array($object_data['object_type'], array_merge(['as:Tombstone', ''], self::CONTENT_TYPES))) { $trust_source = Processor::isActivityGone($object_data['object_id']); + if (!$trust_source) { + $trust_source = !empty(APContact::getByURL($object_data['object_id'], false)); + } } } @@ -668,6 +684,9 @@ class Receiver if (!empty($object_data['raw'])) { $announce_object_data['raw'] = $object_data['raw']; } + if (!empty($object_data['raw-object'])) { + $announce_object_data['raw-object'] = $object_data['raw-object']; + } ActivityPub\Processor::createActivity($announce_object_data, Activity::ANNOUNCE); } } else { @@ -1305,7 +1324,7 @@ class Receiver $object_data = self::processObject($object); if (!empty($data)) { - $object_data['raw'] = json_encode($data); + $object_data['raw-object'] = json_encode($data); } return $object_data; } diff --git a/src/Protocol/ActivityPub/Transmitter.php b/src/Protocol/ActivityPub/Transmitter.php index 67c0c24801..9bcf312862 100644 --- a/src/Protocol/ActivityPub/Transmitter.php +++ b/src/Protocol/ActivityPub/Transmitter.php @@ -1232,10 +1232,8 @@ class Transmitter } if (!$item['deleted']) { - $condition = ['item-uri' => $item['uri'], 'protocol' => Conversation::PARCEL_ACTIVITYPUB]; - $conversation = DBA::selectFirst('conversation', ['source'], $condition); - if (!$item['origin'] && DBA::isResult($conversation)) { - $data = json_decode($conversation['source'], true); + $data = Post\Activity::getByURIId($item['uri-id']); + if (!$item['origin'] && !empty($data)) { if (!empty($data['type'])) { if (in_array($data['type'], ['Create', 'Update'])) { if ($object_mode) { diff --git a/src/Protocol/DFRN.php b/src/Protocol/DFRN.php index 625afb215a..7593711e56 100644 --- a/src/Protocol/DFRN.php +++ b/src/Protocol/DFRN.php @@ -810,27 +810,12 @@ class DFRN } // Add conversation data. This is used for OStatus - $conversation_href = DI::baseUrl()."/display/".$item["parent-guid"]; - $conversation_uri = $conversation_href; - - if (isset($parent_item)) { - $conversation = DBA::selectFirst('conversation', ['conversation-uri', 'conversation-href'], ['item-uri' => $item['thr-parent']]); - if (DBA::isResult($conversation)) { - if ($conversation['conversation-uri'] != '') { - $conversation_uri = $conversation['conversation-uri']; - } - if ($conversation['conversation-href'] != '') { - $conversation_href = $conversation['conversation-href']; - } - } - } - $attributes = [ - 'href' => $conversation_href, - 'ref' => $conversation_uri, + 'href' => $item['conversation'], + 'ref' => $item['conversation'], ]; - XML::addElement($doc, $entry, 'ostatus:conversation', $conversation_uri, $attributes); + XML::addElement($doc, $entry, 'ostatus:conversation', $item['conversation'], $attributes); XML::addElement($doc, $entry, 'id', $item['uri']); XML::addElement($doc, $entry, 'title', $item['title']); @@ -1994,16 +1979,16 @@ class DFRN self::parseLinks($links, $item); } - $item['conversation-uri'] = XML::getFirstNodeValue($xpath, 'ostatus:conversation/text()', $entry); + $item['conversation'] = XML::getFirstNodeValue($xpath, 'ostatus:conversation/text()', $entry); $conv = $xpath->query('ostatus:conversation', $entry); if (is_object($conv->item(0))) { foreach ($conv->item(0)->attributes as $attributes) { if ($attributes->name == 'ref') { - $item['conversation-uri'] = $attributes->textContent; + $item['conversation'] = $attributes->textContent; } if ($attributes->name == 'href') { - $item['conversation-href'] = $attributes->textContent; + $item['conversation'] = $attributes->textContent; } } } diff --git a/src/Protocol/OStatus.php b/src/Protocol/OStatus.php index 1efbb0bf39..64e32f2ac4 100644 --- a/src/Protocol/OStatus.php +++ b/src/Protocol/OStatus.php @@ -616,16 +616,16 @@ class OStatus $item['created'] = XML::getFirstNodeValue($xpath, 'atom:published/text()', $entry); $item['edited'] = XML::getFirstNodeValue($xpath, 'atom:updated/text()', $entry); - $item['conversation-uri'] = XML::getFirstNodeValue($xpath, 'ostatus:conversation/text()', $entry); + $item['conversation'] = XML::getFirstNodeValue($xpath, 'ostatus:conversation/text()', $entry); $conv = $xpath->query('ostatus:conversation', $entry); if (is_object($conv->item(0))) { foreach ($conv->item(0)->attributes as $attributes) { if ($attributes->name == 'ref') { - $item['conversation-uri'] = $attributes->textContent; + $item['conversation'] = $attributes->textContent; } if ($attributes->name == 'href') { - $item['conversation-href'] = $attributes->textContent; + $item['conversation'] = $attributes->textContent; } } } @@ -704,14 +704,6 @@ class OStatus } } - if (($self != '') && empty($item['protocol'])) { - self::fetchSelf($self, $item); - } - - if (!empty($item['conversation-href'])) { - self::fetchConversation($item['conversation-href'], $item['conversation-uri']); - } - if (isset($item['thr-parent'])) { if (!Post::exists(['uid' => $importer['uid'], 'uri' => $item['thr-parent']])) { if ($related != '') { @@ -725,194 +717,9 @@ class OStatus $item['gravity'] = GRAVITY_PARENT; } - if (($item['author-link'] != '') && !empty($item['protocol'])) { - $item = Conversation::insert($item); - } - self::$itemlist[] = $item; } - /** - * Fetch the conversation for posts - * - * @param string $conversation The link to the conversation - * @param string $conversation_uri The conversation in "uri" format - * @return void - * @throws \Friendica\Network\HTTPException\InternalServerErrorException - */ - private static function fetchConversation(string $conversation, string $conversation_uri) - { - // Ensure that we only store a conversation once in a process - if (isset(self::$conv_list[$conversation])) { - return; - } - - self::$conv_list[$conversation] = true; - - $curlResult = DI::httpClient()->get($conversation, HttpClientAccept::ATOM_XML); - - if (!$curlResult->isSuccess() || empty($curlResult->getBody())) { - return; - } - - $xml = ''; - - if ($curlResult->inHeader('Content-Type') && - in_array('application/atom+xml', $curlResult->getHeader('Content-Type'))) { - $xml = $curlResult->getBody(); - } - - if ($xml == '') { - $doc = new DOMDocument(); - if (!@$doc->loadHTML($curlResult->getBody())) { - return; - } - $xpath = new DOMXPath($doc); - - $links = $xpath->query('//link'); - if ($links) { - $file = ''; - foreach ($links as $link) { - $attribute = self::readAttributes($link); - if (($attribute['rel'] == 'alternate') && ($attribute['type'] == 'application/atom+xml')) { - $file = $attribute['href']; - } - } - if ($file != '') { - $conversation_atom = DI::httpClient()->get($attribute['href'], HttpClientAccept::ATOM_XML); - - if ($conversation_atom->isSuccess()) { - $xml = $conversation_atom->getBody(); - } - } - } - } - - if ($xml == '') { - return; - } - - self::storeConversation($xml, $conversation, $conversation_uri); - } - - /** - * Store a feed in several conversation entries - * - * @param string $xml The feed - * @param string $conversation conversation - * @param string $conversation_uri conversation uri - * @return void - * @throws \Exception - */ - private static function storeConversation(string $xml, string $conversation = '', string $conversation_uri = '') - { - $doc = new DOMDocument(); - @$doc->loadXML($xml); - - $xpath = new DOMXPath($doc); - $xpath->registerNamespace('atom', ActivityNamespace::ATOM1); - $xpath->registerNamespace('thr', ActivityNamespace::THREAD); - $xpath->registerNamespace('ostatus', ActivityNamespace::OSTATUS); - - $entries = $xpath->query('/atom:feed/atom:entry'); - - // Now store the entries - foreach ($entries as $entry) { - $doc2 = new DOMDocument(); - $doc2->preserveWhiteSpace = false; - $doc2->formatOutput = true; - - $conv_data = []; - - $conv_data['protocol'] = Conversation::PARCEL_SPLIT_CONVERSATION; - $conv_data['direction'] = Conversation::PULL; - $conv_data['network'] = Protocol::OSTATUS; - $conv_data['uri'] = XML::getFirstNodeValue($xpath, 'atom:id/text()', $entry); - - $inreplyto = $xpath->query('thr:in-reply-to', $entry); - if (is_object($inreplyto->item(0))) { - foreach ($inreplyto->item(0)->attributes as $attributes) { - if ($attributes->name == 'ref') { - $conv_data['reply-to-uri'] = $attributes->textContent; - } - } - } - - $conv_data['conversation-uri'] = XML::getFirstNodeValue($xpath, 'ostatus:conversation/text()', $entry); - - $conv = $xpath->query('ostatus:conversation', $entry); - if (is_object($conv->item(0))) { - foreach ($conv->item(0)->attributes as $attributes) { - if ($attributes->name == 'ref') { - $conv_data['conversation-uri'] = $attributes->textContent; - } - if ($attributes->name == 'href') { - $conv_data['conversation-href'] = $attributes->textContent; - } - } - } - - if ($conversation != '') { - $conv_data['conversation-uri'] = $conversation; - } - - if ($conversation_uri != '') { - $conv_data['conversation-uri'] = $conversation_uri; - } - - $entry = $doc2->importNode($entry, true); - - $doc2->appendChild($entry); - - $conv_data['source'] = $doc2->saveXML(); - - Logger::info('Store conversation data for uri '.$conv_data['uri']); - Conversation::insert($conv_data); - } - } - - /** - * Fetch the own post so that it can be stored later - * - * We want to store the original data for later processing. - * This function is meant for cases where we process a feed with multiple entries. - * In that case we need to fetch the single posts here. - * - * @param string $self The link to the self item - * @param array $item The item array - * @return void - * @throws \Friendica\Network\HTTPException\InternalServerErrorException - */ - private static function fetchSelf(string $self, array &$item) - { - $condition = ['item-uri' => $self, 'protocol' => [Conversation::PARCEL_DFRN, - Conversation::PARCEL_DIASPORA_DFRN, Conversation::PARCEL_LOCAL_DFRN, - Conversation::PARCEL_DIRECT, Conversation::PARCEL_SALMON]]; - if (DBA::exists('conversation', $condition)) { - Logger::info('Conversation '.$item['uri'].' is already stored.'); - return; - } - - $curlResult = DI::httpClient()->get($self, HttpClientAccept::ATOM_XML); - - if (!$curlResult->isSuccess()) { - return; - } - - // We reformat the XML to make it better readable - $doc = new DOMDocument(); - $doc->loadXML($curlResult->getBody()); - $doc->preserveWhiteSpace = false; - $doc->formatOutput = true; - $xml = $doc->saveXML(); - - $item['protocol'] = Conversation::PARCEL_SALMON; - $item['source'] = $xml; - $item['direction'] = Conversation::PULL; - - Logger::info('Conversation '.$item['uri'].' is now fetched.'); - } - /** * Fetch related posts and processes them * @@ -925,30 +732,6 @@ class OStatus */ private static function fetchRelated(string $related, string $related_uri, array $importer) { - $condition = [ - 'item-uri' => $related_uri, - 'protocol' => [ - Conversation::PARCEL_DFRN, - Conversation::PARCEL_DIASPORA_DFRN, - Conversation::PARCEL_LOCAL_DFRN, - Conversation::PARCEL_DIRECT, - Conversation::PARCEL_SALMON, - ], - ]; - $conversation = DBA::selectFirst('conversation', ['source', 'protocol'], $condition); - if (DBA::isResult($conversation)) { - $stored = true; - $xml = $conversation['source']; - if (self::process($xml, $importer, $contact, $hub, $stored, false, Conversation::PULL)) { - Logger::info('Got valid cached XML for URI '.$related_uri); - return; - } - if ($conversation['protocol'] == Conversation::PARCEL_SALMON) { - Logger::info('Delete invalid cached XML for URI '.$related_uri); - DBA::delete('conversation', ['item-uri' => $related_uri]); - } - } - $stored = false; $curlResult = DI::httpClient()->get($related, HttpClientAccept::ATOM_XML); @@ -1013,17 +796,6 @@ class OStatus } } - // Finally we take the data that we fetched from "ostatus:conversation" - if ($xml == '') { - $condition = ['item-uri' => $related_uri, 'protocol' => Conversation::PARCEL_SPLIT_CONVERSATION]; - $conversation = DBA::selectFirst('conversation', ['source'], $condition); - if (DBA::isResult($conversation)) { - $stored = true; - Logger::info('Got cached XML from conversation for URI ' . $related_uri); - $xml = $conversation['source']; - } - } - if ($xml != '') { self::process($xml, $importer, $contact, $hub, $stored, false, Conversation::PULL); } else { @@ -1130,10 +902,7 @@ class OStatus case 'ostatus:conversation': $link_data['conversation'] = $attribute['href']; - $item['conversation-href'] = $link_data['conversation']; - if (!isset($item['conversation-uri'])) { - $item['conversation-uri'] = $item['conversation-href']; - } + $item['conversation'] = $link_data['conversation']; break; case 'enclosure': @@ -1904,19 +1673,7 @@ class OStatus } if (intval($item['parent']) > 0) { - $conversation_href = $conversation_uri = str_replace('/objects/', '/context/', $item['thr-parent']); - - if (isset($parent_item)) { - $conversation = DBA::selectFirst('conversation', ['conversation-uri', 'conversation-href'], ['item-uri' => $parent_item]); - if (DBA::isResult($conversation)) { - if ($conversation['conversation-uri'] != '') { - $conversation_uri = $conversation['conversation-uri']; - } - if ($conversation['conversation-href'] != '') { - $conversation_href = $conversation['conversation-href']; - } - } - } + $conversation_href = $conversation_uri = $item['conversation']; XML::addElement($doc, $entry, 'link', '', ['rel' => 'ostatus:conversation', 'href' => $conversation_href]); diff --git a/src/Worker/Cron.php b/src/Worker/Cron.php index 2ace97d830..12ea6e60a3 100644 --- a/src/Worker/Cron.php +++ b/src/Worker/Cron.php @@ -115,7 +115,7 @@ class Cron Worker::add(PRIORITY_LOW, 'ExpirePosts'); - Worker::add(PRIORITY_LOW, 'ExpireConversations'); + Worker::add(PRIORITY_LOW, 'ExpireActivities'); Worker::add(PRIORITY_LOW, 'RemoveUnusedTags'); diff --git a/src/Worker/ExpireConversations.php b/src/Worker/ExpireActivities.php similarity index 76% rename from src/Worker/ExpireConversations.php rename to src/Worker/ExpireActivities.php index 4b5c4ad584..60bdf3f8a7 100644 --- a/src/Worker/ExpireConversations.php +++ b/src/Worker/ExpireActivities.php @@ -22,21 +22,15 @@ namespace Friendica\Worker; use Friendica\Database\DBA; -use Friendica\DI; use Friendica\Util\DateTimeFormat; -class ExpireConversations +class ExpireActivities { /** - * Delete old conversation entries + * Delete old post-activity entries */ public static function execute() { - $days = intval(DI::config()->get('system', 'dbclean_expire_conversation', 90)); - if (empty($days)) { - return; - } - - DBA::delete('conversation', ["`received` < ?", DateTimeFormat::utc('now - ' . $days . ' days')]); + DBA::delete('post-activity', ["`received` < ?", DateTimeFormat::utc('now - 7 days')]); } } diff --git a/src/Worker/Notifier.php b/src/Worker/Notifier.php index 8c0801c66d..8655d2662e 100644 --- a/src/Worker/Notifier.php +++ b/src/Worker/Notifier.php @@ -746,7 +746,7 @@ class Notifier } Logger::info('Origin item ' . $target_item['id'] . ' with URL ' . $target_item['uri'] . ' will be distributed.'); - } elseif (!DBA::exists('conversation', ['item-uri' => $target_item['uri'], 'protocol' => Conversation::PARCEL_ACTIVITYPUB])) { + } elseif (!Post\Activity::exists($target_item['uri-id'])) { Logger::info('Remote item ' . $target_item['id'] . ' with URL ' . $target_item['uri'] . ' is no AP post. It will not be distributed.'); return ['count' => 0, 'contacts' => []]; } elseif ($parent['origin']) { diff --git a/static/dbstructure.config.php b/static/dbstructure.config.php index 53ceaa7ece..cff7c1cd19 100644 --- a/static/dbstructure.config.php +++ b/static/dbstructure.config.php @@ -55,7 +55,7 @@ use Friendica\Database\DBA; if (!defined('DB_UPDATE_VERSION')) { - define('DB_UPDATE_VERSION', 1474); + define('DB_UPDATE_VERSION', 1475); } return [ @@ -400,6 +400,7 @@ return [ "featured-tags" => ["type" => "varchar(255)", "comment" => "Address for the collection of featured tags"], "manually-approve" => ["type" => "boolean", "comment" => ""], "discoverable" => ["type" => "boolean", "comment" => "Mastodon extension: true if profile is published in their directory"], + "suspended" => ["type" => "boolean", "comment" => "Mastodon extension: true if profile is suspended"], "nick" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""], "name" => ["type" => "varchar(255)", "comment" => ""], "about" => ["type" => "text", "comment" => ""], @@ -565,24 +566,6 @@ return [ "uid" => ["uid"], ] ], - "conversation" => [ - "comment" => "Raw data and structure information for messages", - "fields" => [ - "item-uri" => ["type" => "varbinary(255)", "not null" => "1", "primary" => "1", "comment" => "Original URI of the item - unrelated to the table with the same name"], - "reply-to-uri" => ["type" => "varbinary(255)", "not null" => "1", "default" => "", "comment" => "URI to which this item is a reply"], - "conversation-uri" => ["type" => "varbinary(255)", "not null" => "1", "default" => "", "comment" => "GNU Social conversation URI"], - "conversation-href" => ["type" => "varbinary(255)", "not null" => "1", "default" => "", "comment" => "GNU Social conversation link"], - "protocol" => ["type" => "tinyint unsigned", "not null" => "1", "default" => "255", "comment" => "The protocol of the item"], - "direction" => ["type" => "tinyint unsigned", "not null" => "1", "default" => "0", "comment" => "How the message arrived here: 1=push, 2=pull"], - "source" => ["type" => "mediumtext", "comment" => "Original source"], - "received" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => "Receiving date"], - ], - "indexes" => [ - "PRIMARY" => ["item-uri"], - "conversation-uri" => ["conversation-uri"], - "received" => ["received"], - ] - ], "workerqueue" => [ "comment" => "Background tasks queue entries", "fields" => [ @@ -1156,6 +1139,17 @@ return [ "vid" => ["vid"], ] ], + "post-activity" => [ + "comment" => "Original remote activity", + "fields" => [ + "uri-id" => ["type" => "int unsigned", "not null" => "1", "primary" => "1", "foreign" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the item uri"], + "activity" => ["type" => "mediumtext", "comment" => "Original activity"], + "received" => ["type" => "datetime", "comment" => ""], + ], + "indexes" => [ + "PRIMARY" => ["uri-id"], + ] + ], "post-category" => [ "comment" => "post relation to categories", "fields" => [