Improved message handling / new activity relay handling

database.sql

@@ -1,6 +1,6 @@
 -- ------------------------------------------
 -- Friendica 2022.09-dev (Giant Rhubarb)
--- DB_UPDATE_VERSION 1474
+-- DB_UPDATE_VERSION 1475
 -- ------------------------------------------
 
 
@@ -338,6 +338,7 @@ CREATE TABLE IF NOT EXISTS `apcontact` (
 	`featured-tags` varchar(255) COMMENT 'Address for the collection of featured tags',
 	`manually-approve` boolean COMMENT '',
 	`discoverable` boolean COMMENT 'Mastodon extension: true if profile is published in their directory',
+	`suspended` boolean COMMENT 'Mastodon extension: true if profile is suspended',
 	`nick` varchar(255) NOT NULL DEFAULT '' COMMENT '',
 	`name` varchar(255) COMMENT '',
 	`about` text COMMENT '',
@@ -503,23 +504,6 @@ CREATE TABLE IF NOT EXISTS `conv` (
 	FOREIGN KEY (`uid`) REFERENCES `user` (`uid`) ON UPDATE RESTRICT ON DELETE CASCADE
 ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='private messages';
 
---
--- TABLE conversation
---
-CREATE TABLE IF NOT EXISTS `conversation` (
-	`item-uri` varbinary(255) NOT NULL COMMENT 'Original URI of the item - unrelated to the table with the same name',
-	`reply-to-uri` varbinary(255) NOT NULL DEFAULT '' COMMENT 'URI to which this item is a reply',
-	`conversation-uri` varbinary(255) NOT NULL DEFAULT '' COMMENT 'GNU Social conversation URI',
-	`conversation-href` varbinary(255) NOT NULL DEFAULT '' COMMENT 'GNU Social conversation link',
-	`protocol` tinyint unsigned NOT NULL DEFAULT 255 COMMENT 'The protocol of the item',
-	`direction` tinyint unsigned NOT NULL DEFAULT 0 COMMENT 'How the message arrived here: 1=push, 2=pull',
-	`source` mediumtext COMMENT 'Original source',
-	`received` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Receiving date',
-	 PRIMARY KEY(`item-uri`),
-	 INDEX `conversation-uri` (`conversation-uri`),
-	 INDEX `received` (`received`)
-) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Raw data and structure information for messages';
-
 --
 -- TABLE workerqueue
 --
@@ -1117,6 +1101,17 @@ CREATE TABLE IF NOT EXISTS `post` (
 	FOREIGN KEY (`vid`) REFERENCES `verb` (`id`) ON UPDATE RESTRICT ON DELETE RESTRICT
 ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Structure for all posts';
 
+--
+-- TABLE post-activity
+--
+CREATE TABLE IF NOT EXISTS `post-activity` (
+	`uri-id` int unsigned NOT NULL COMMENT 'Id of the item-uri table entry that contains the item uri',
+	`activity` mediumtext COMMENT 'Original activity',
+	`received` datetime COMMENT '',
+	 PRIMARY KEY(`uri-id`),
+	FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Original remote activity';
+
 --
 -- TABLE post-category
 --

doc/database.md

@@ -19,7 +19,6 @@ Database Tables
 | [contact](help/database/db_contact) | contact table |
 | [contact-relation](help/database/db_contact-relation) | Contact relations |
 | [conv](help/database/db_conv) | private messages |
-| [conversation](help/database/db_conversation) | Raw data and structure information for messages |
 | [delayed-post](help/database/db_delayed-post) | Posts that are about to be distributed at a later time |
 | [diaspora-interaction](help/database/db_diaspora-interaction) | Signed Diaspora Interaction |
 | [endpoint](help/database/db_endpoint) | ActivityPub endpoints - used in the ActivityPub implementation |
@@ -50,6 +49,7 @@ Database Tables
 | [permissionset](help/database/db_permissionset) |  |
 | [photo](help/database/db_photo) | photo storage |
 | [post](help/database/db_post) | Structure for all posts |
+| [post-activity](help/database/db_post-activity) | Original remote activity |
 | [post-category](help/database/db_post-category) | post relation to categories |
 | [post-collection](help/database/db_post-collection) | Collection of posts |
 | [post-content](help/database/db_post-content) | Content for all posts |

doc/database/db_apcontact.md

@@ -21,6 +21,7 @@ Fields
 | featured-tags    | Address for the collection of featured tags                         | varchar(255)   | YES  |     | NULL                |       |
 | manually-approve |                                                                     | boolean        | YES  |     | NULL                |       |
 | discoverable     | Mastodon extension: true if profile is published in their directory | boolean        | YES  |     | NULL                |       |
+| suspended        | Mastodon extension: true if profile is suspended                    | boolean        | YES  |     | NULL                |       |
 | nick             |                                                                     | varchar(255)   | NO   |     |                     |       |
 | name             |                                                                     | varchar(255)   | YES  |     | NULL                |       |
 | about            |                                                                     | text           | YES  |     | NULL                |       |

doc/database/db_post-activity.md

@@ -0,0 +1,29 @@
+Table post-activity
+===========
+
+Original remote activity
+
+Fields
+------
+
+| Field    | Description                                               | Type         | Null | Key | Default | Extra |
+| -------- | --------------------------------------------------------- | ------------ | ---- | --- | ------- | ----- |
+| uri-id   | Id of the item-uri table entry that contains the item uri | int unsigned | NO   | PRI | NULL    |       |
+| activity | Original activity                                         | mediumtext   | YES  |     | NULL    |       |
+| received |                                                           | datetime     | YES  |     | NULL    |       |
+
+Indexes
+------------
+
+| Name    | Fields |
+| ------- | ------ |
+| PRIMARY | uri-id |
+
+Foreign Keys
+------------
+
+| Field | Target Table | Target Field |
+|-------|--------------|--------------|
+| uri-id | [item-uri](help/database/db_item-uri) | id |
+
+Return to [database documentation](help/database)

mod/item.php

@@ -595,16 +595,6 @@ function item_post(App $a) {
 	$datarray['protocol'] = Conversation::PARCEL_DIRECT;
 	$datarray['direction'] = Conversation::PUSH;
 
-	$conversation = DBA::selectFirst('conversation', ['conversation-uri', 'conversation-href'], ['item-uri' => $datarray['thr-parent']]);
-	if (DBA::isResult($conversation)) {
-		if ($conversation['conversation-uri'] != '') {
-			$datarray['conversation-uri'] = $conversation['conversation-uri'];
-		}
-		if ($conversation['conversation-href'] != '') {
-			$datarray['conversation-href'] = $conversation['conversation-href'];
-		}
-	}
-
 	if ($orig_post) {
 		$datarray['edit'] = true;
 	} else {

src/Database/DBStructure.php

@@ -74,7 +74,7 @@ class DBStructure
 		$old_tables = ['fserver', 'gcign', 'gcontact', 'gcontact-relation', 'gfollower' ,'glink', 'item-delivery-data',
 			'item-activity', 'item-content', 'item_id', 'participation', 'poll', 'poll_result', 'queue', 'retriever_rule',
 			'deliverq', 'dsprphotoq', 'ffinder', 'sign', 'spam', 'term', 'user-item', 'thread', 'item', 'challenge',
-			'auth_codes', 'tokens', 'clients', 'profile_check', 'host'];
+			'auth_codes', 'tokens', 'clients', 'profile_check', 'host', 'conversation'];
 
 		$tables = DBA::selectToArray('INFORMATION_SCHEMA.TABLES', ['TABLE_NAME'],
 			['TABLE_SCHEMA' => DBA::databaseName(), 'TABLE_TYPE' => 'BASE TABLE']);

src/Model/APContact.php

@@ -38,6 +38,7 @@ use Friendica\Util\DateTimeFormat;
 use Friendica\Util\HTTPSignature;
 use Friendica\Util\JsonLD;
 use Friendica\Util\Network;
+use GuzzleHttp\Psr7\Uri;
 
 class APContact
 {
@@ -310,6 +311,8 @@ class APContact
 
 		$apcontact['manually-approve'] = (int)JsonLD::fetchElement($compacted, 'as:manuallyApprovesFollowers');
 
+		$apcontact['suspended'] = (int)JsonLD::fetchElement($compacted, 'toot:suspended');
+
 		if (!empty($compacted['as:generator'])) {
 			$apcontact['baseurl'] = JsonLD::fetchElement($compacted['as:generator'], 'as:url', '@id');
 			$apcontact['generator'] = JsonLD::fetchElement($compacted['as:generator'], 'as:name', '@value');
@@ -380,11 +383,11 @@ class APContact
 		if (strlen($apcontact['photo']) > 255) {
 			$parts = parse_url($apcontact['photo']);
 			unset($parts['fragment']);
-			$apcontact['photo'] = Network::unparseURL($parts);
+			$apcontact['photo'] = Uri::fromParts($parts);
 
 			if (strlen($apcontact['photo']) > 255) {
 				unset($parts['query']);
-				$apcontact['photo'] = Network::unparseURL($parts);
+				$apcontact['photo'] = Uri::fromParts($parts);
 			}
 
 			if (strlen($apcontact['photo']) > 255) {

src/Model/Conversation.php

@@ -21,11 +21,6 @@
 
 namespace Friendica\Model;
 
-use Friendica\Core\Protocol;
-use Friendica\Database\Database;
-use Friendica\Database\DBA;
-use Friendica\Util\DateTimeFormat;
-
 class Conversation
 {
 	/*
@@ -62,61 +57,4 @@ class Conversation
 	 */
 	const RELAY   = 3;
 
-	public static function getByItemUri(string $item_uri)
-	{
-		return DBA::selectFirst('conversation', [], ['item-uri' => $item_uri]);
-	}
-
-	/**
-	 * Store the conversation data
-	 *
-	 * @param array $arr Item array with conversation data
-	 * @return array Item array with removed conversation data
-	 * @throws \Exception
-	 */
-	public static function insert(array $arr): array
-	{
-		if (in_array(($arr['network'] ?? '') ?: Protocol::PHANTOM,
-			[Protocol::ACTIVITYPUB, Protocol::DFRN, Protocol::DIASPORA, Protocol::OSTATUS, Protocol::TWITTER]) && !empty($arr['uri'])) {
-			$conversation = ['item-uri' => $arr['uri'], 'received' => DateTimeFormat::utcNow()];
-
-			if (isset($arr['parent-uri']) && ($arr['parent-uri'] != $arr['uri'])) {
-				$conversation['reply-to-uri'] = $arr['parent-uri'];
-			}
-
-			if (isset($arr['thr-parent']) && ($arr['thr-parent'] != $arr['uri'])) {
-				$conversation['reply-to-uri'] = $arr['thr-parent'];
-			}
-
-			if (isset($arr['conversation-uri'])) {
-				$conversation['conversation-uri'] = $arr['conversation-uri'];
-			}
-
-			if (isset($arr['conversation-href'])) {
-				$conversation['conversation-href'] = $arr['conversation-href'];
-			}
-
-			if (isset($arr['protocol'])) {
-				$conversation['protocol'] = $arr['protocol'];
-			}
-
-			if (isset($arr['direction'])) {
-				$conversation['direction'] = $arr['direction'];
-			}
-
-			if (isset($arr['source'])) {
-				$conversation['source'] = $arr['source'];
-			}
-
-			if (!DBA::exists('conversation', ['item-uri' => $conversation['item-uri']])) {
-				DBA::insert('conversation', $conversation, Database::INSERT_IGNORE);
-			}
-		}
-
-		unset($arr['conversation-uri']);
-		unset($arr['conversation-href']);
-		unset($arr['source']);
-
-		return $arr;
-	}
 }

src/Model/GServer.php

@@ -563,8 +563,13 @@ class GServer
 
 		$serverdata['registered-users'] = $serverdata['registered-users'] ?? 0;
 
+		// Numbers above a reasonable value (10 millions) are ignored
+		if ($serverdata['registered-users'] > 10000000) {
+			$serverdata['registered-users'] = 0;
+		}
+
 		// On an active server there has to be at least a single user
-		if (!in_array($serverdata['network'], [Protocol::PHANTOM, Protocol::FEED]) && ($serverdata['registered-users'] == 0)) {
+		if (!in_array($serverdata['network'], [Protocol::PHANTOM, Protocol::FEED]) && ($serverdata['registered-users'] <= 0)) {
 			$serverdata['registered-users'] = 1;
 		} elseif (in_array($serverdata['network'], [Protocol::PHANTOM, Protocol::FEED])) {
 			$serverdata['registered-users'] = 0;

src/Model/Item.php

@@ -801,7 +801,10 @@ class Item
 		$item['parent-uri-id'] = ItemURI::getIdByURI($item['parent-uri']);
 
 		// Store conversation data
-		$item = Conversation::insert($item);
+		$source = $item['source'] ?? '';
+		unset($item['conversation-uri']);
+		unset($item['conversation-href']);
+		unset($item['source']);
 
 		/*
 		 * Do we already have this item?
@@ -1259,6 +1262,9 @@ class Item
 		}
 
 		if ($transmit) {
+			if (!empty($source)) {
+				Post\Activity::insert($item['uri-id'], $source);
+			}
 			Worker::add(['priority' => $priority, 'dont_fork' => true], 'Notifier', $notify_type, (int)$posted_item['uri-id'], (int)$posted_item['uid']);
 		}
 

src/Model/Post/Activity.php

@@ -0,0 +1,74 @@
+<?php
+/**
+ * @copyright Copyright (C) 2010-2022, the Friendica project
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace Friendica\Model\Post;
+
+use Friendica\Database\Database;
+use Friendica\Database\DBA;
+use Friendica\Util\DateTimeFormat;
+
+class Activity
+{
+	/**
+	 * Insert a new post-activity entry
+	 *
+	 * @param integer $uri_id
+	 * @param array   $fields
+	 *
+	 * @return bool   success
+	 */
+	public static function insert(int $uri_id, string $source): bool
+	{
+		// Additionally assign the key fields
+		$fields = [
+			'uri-id'   => $uri_id,
+			'activity' => $source,
+			'received' => DateTimeFormat::utcNow()
+		];
+
+		return DBA::insert('post-activity', $fields, Database::INSERT_IGNORE);
+	}
+
+	/**
+	 * Retrieves activity of the given uri-id
+	 *
+	 * @param int   $uriId
+	 *
+	 * @return array
+	 */
+	public static function getByURIId(int $uriId): array
+	{
+		$activity = DBA::selectFirst('post-activity', [], ['uri-id' => $uriId]);
+		return json_decode($activity['activity'] ?? '', true) ?? [];
+	}
+
+	/**
+	 * Checks if the given uridid has a stored activity
+	 *
+	 * @param integer $uriId
+	 *
+	 * @return boolean
+	 */
+	public static function exists(int $uriId): bool
+	{
+		return DBA::exists('post-activity', ['uri-id' => $uriId]);
+	}
+}

src/Model/Tag.php

@@ -545,7 +545,7 @@ class Tag
 					break;
 
 				default:
-					Logger:warning('Unknown tag type found', $tag);
+					Logger::warning('Unknown tag type found', $tag);
 			}
 		}
 		DBA::close($taglist);

src/Module/Admin/Item/Source.php

@@ -35,7 +35,6 @@ class Source extends BaseAdmin
 
 		$guid = basename($_REQUEST['guid'] ?? $this->parameters['guid'] ?? '');
 
-		$source = '';
 		$item_uri = '';
 		$item_id = '';
 		$terms = [];
@@ -43,11 +42,8 @@ class Source extends BaseAdmin
 			$item = Model\Post::selectFirst(['id', 'uri-id', 'guid', 'uri'], ['guid' => $guid]);
 
 			if ($item) {
-				$conversation = Model\Conversation::getByItemUri($item['uri']);
-
 				$item_id = $item['id'];
 				$item_uri = $item['uri'];
-				$source = $conversation['source'];
 				$terms = Model\Tag::getByURIId($item['uri-id'], [Model\Tag::HASHTAG, Model\Tag::MENTION, Model\Tag::IMPLICIT_MENTION]);
 			}
 		}
@@ -56,7 +52,6 @@ class Source extends BaseAdmin
 		$o = Renderer::replaceMacros($tpl, [
 			'$title'       => DI::l10n()->t('Item Source'),
 			'$guid'        => ['guid', DI::l10n()->t('Item Guid'), $guid, ''],
-			'$source'      => $source,
 			'$item_uri'    => $item_uri,
 			'$item_id'     => $item_id,
 			'$terms'       => $terms,
@@ -70,7 +65,6 @@ class Source extends BaseAdmin
 			'$urllbl'      => DI::l10n()->t('URL'),
 			'$mentionlbl'  => DI::l10n()->t('Mention'),
 			'$implicitlbl' => DI::l10n()->t('Implicit Mention'),
-			'$sourcelbl'   => DI::l10n()->t('Source'),
 		]);
 
 		return $o;

src/Module/DFRN/Notify.php

@@ -59,7 +59,7 @@ class Notify extends BaseModule
 		}
 	}
 
-	private static function dispatchPublic(array $postdata)
+	private static function dispatchPublic(string $postdata)
 	{
 		$msg = Diaspora::decodeRaw($postdata, '', true);
 		if (!is_array($msg)) {

src/Navigation/Notifications/Factory/Notification.php

@@ -163,9 +163,10 @@ class Notification extends BaseFactory implements ICanCreateFromTableRow
 				}
 
 				if (($Notification->verb == Activity::POST) || ($Notification->type === Post\UserNotification::TYPE_SHARED)) {
-					$item = Post::selectFirst([], ['uri-id' => $item['thr-parent-id'], 'uid' => [0, $Notification->uid]], ['order' => ['uid' => true]]);
+					$thrparentid = $item['thr-parent-id'];
+					$item = Post::selectFirst([], ['uri-id' => $thrparentid, 'uid' => [0, $Notification->uid]], ['order' => ['uid' => true]]);
 					if (empty($item)) {
-						$this->logger->info('Thread parent post not found', ['uri-id' => $item['thr-parent-id']]);
+						$this->logger->info('Thread parent post not found', ['uri-id' => $thrparentid]);
 						return $message;
 					}
 				}

src/Protocol/ActivityPub/Delivery.php

@@ -110,7 +110,10 @@ class Delivery
 		} elseif ($cmd == WorkerDelivery::PROFILEUPDATE) {
 			$success = ActivityPub\Transmitter::sendProfileUpdate($uid, $inbox);
 		} else {
-			$data = ActivityPub\Transmitter::createCachedActivityFromItem($item_id);
+			$data = Post\Activity::getByURIId($uri_id);
+			if (empty($data)) {
+				$data = ActivityPub\Transmitter::createCachedActivityFromItem($item_id);
+			}
 			if (!empty($data)) {
 				$timestamp  = microtime(true);
 				$response   = HTTPSignature::post($data, $inbox, $uid);

src/Protocol/ActivityPub/Processor.php

@@ -361,8 +361,6 @@ class Processor
 		if (!empty($activity['raw'])) {
 			$item['source'] = $activity['raw'];
 			$item['protocol'] = Conversation::PARCEL_ACTIVITYPUB;
-			$item['conversation-href'] = $activity['context'] ?? '';
-			$item['conversation-uri'] = $activity['conversation'] ?? '';
 
 			if (isset($activity['push'])) {
 				$item['direction'] = $activity['push'] ? Conversation::PUSH : Conversation::PULL;
@@ -475,7 +473,19 @@ class Processor
 		}
 
 		// @todo To ensure that the remote system is working correctly, we can check if the "Content-Type" contains JSON
-		return in_array($curlResult->getReturnCode(), [404]);
+		if (in_array($curlResult->getReturnCode(), [404])) {
+			return true;
+		}
+
+		$object = json_decode($curlResult->getBody(), true);
+		if (!empty($object)) {
+			$activity = JsonLD::compact($object);
+			if (JsonLD::fetchElement($activity, '@type') == 'as:Tombstone') {
+				return true;
+			}			
+		}
+
+		return false;
 	}
 	/**
 	 * Delete items

src/Protocol/ActivityPub/Queue.php

@@ -25,7 +25,9 @@ use Friendica\Core\Logger;
 use Friendica\Database\Database;
 use Friendica\Database\DBA;
 use Friendica\DI;
+use Friendica\Protocol\ActivityPub;
 use Friendica\Util\DateTimeFormat;
+use Friendica\Util\JsonLD;
 
 /**
  * This class handles the processing of incoming posts
@@ -214,6 +216,7 @@ class Queue
 			Logger::debug('Process leftover entry', $entry);
 			self::process($entry['id']);
 		}
+		DBA::close($entries);
 	}
 
 	/**
@@ -247,5 +250,58 @@ class Queue
 		while ($entry = DBA::fetch($entries)) {
 			self::process($entry['id']);
 		}
+		DBA::close($entries);
+	}
+
+	/**
+	 * Prepare the queue entry.
+	 * This is a test function that is used solely for development.
+	 *
+	 * @param integer $id
+	 * @return array
+	 */
+	public static function reprepareActivityById(int $id): array
+	{
+		$entry = DBA::selectFirst('inbox-entry', [], ['id' => $id]);
+		if (empty($entry)) {
+			return [];
+		}
+
+		$receiver = DBA::selectFirst('inbox-entry-receiver', ['uid'], ['queue-id' => $id]);
+		if (!empty($receiver)) {
+			$uid = $receiver['uid'];
+		} else {
+			$uid = 0;
+		}
+
+		$trust_source = $entry['trust'];
+
+		$data     = json_decode($entry['activity'], true);
+		$activity = json_decode($data['raw'], true);
+
+		$ldactivity = JsonLD::compact($activity);
+		return [
+			'data'  => Receiver::prepareObjectData($ldactivity, $uid, $entry['push'], $trust_source),
+			'trust' => $trust_source
+		];
+	}
+
+	/**
+	 * Set the trust for all untrusted entries.
+	 * This is a test function that is used solely for development.
+	 *
+	 * @return void
+	 */
+	public static function reprepareAll()
+	{
+		$entries = DBA::select('inbox-entry', ['id'], ["NOT `trust` AND `wid` IS NULL"], ['order' => ['id' => true]]);
+		while ($entry = DBA::fetch($entries)) {
+			$data = self::reprepareActivityById($entry['id'], false);
+			if ($data['trust']) {
+				DBA::update('inbox-entry', ['trust' => true], ['id' => $entry['id']]);
+			}
+		}
+		DBA::close($entries);
+
 	}
 }

src/Protocol/ActivityPub/Receiver.php

@@ -272,26 +272,39 @@ class Receiver
 	 */
 	public static function prepareObjectData(array $activity, int $uid, bool $push, bool &$trust_source): array
 	{
-		$id = JsonLD::fetchElement($activity, '@id');
+		$id        = JsonLD::fetchElement($activity, '@id');
+		$type      = JsonLD::fetchElement($activity, '@type');
 		$object_id = JsonLD::fetchElement($activity, 'as:object', '@id');
 
+		if (!empty($object_id) && in_array($type, ['as:Create', 'as:Update'])) {
+			$fetch_id = $object_id;
+		} else {
+			$fetch_id = $id;
+		}
+
+		if (!empty($activity['as:object'])) {
+			$object_type = JsonLD::fetchElement($activity['as:object'], '@type');
+		}
+
 		if (!empty($id) && !$trust_source) {
 			$fetch_uid = $uid ?: self::getBestUserForActivity($activity);
 
-			$fetched_activity = ActivityPub::fetchContent($id, $fetch_uid);
+			$fetched_activity = ActivityPub::fetchContent($fetch_id, $fetch_uid);
 			if (!empty($fetched_activity)) {
 				$object = JsonLD::compact($fetched_activity);
-				$fetched_id = JsonLD::fetchElement($object, '@id');
+
-				if ($fetched_id == $id) {
+				$fetched_id   = JsonLD::fetchElement($object, '@id');
+				$fetched_type = JsonLD::fetchElement($object, '@type');
+
+				if (($fetched_id == $id) && !empty($fetched_type) && ($fetched_type == $type)) {
 					Logger::info('Activity had been fetched successfully', ['id' => $id]);
 					$trust_source = true;
-					if ($id != $object_id) {
+					$activity = $object;
-						$activity = $object;
+				} elseif (($fetched_id == $object_id) && !empty($fetched_type) && ($fetched_type == $object_type)) {
-					} else {
+					Logger::info('Fetched data is the object instead of the activity', ['id' => $id]);
-						Logger::info('Fetched data is the object instead of the activity', ['id' => $id]);
+					$trust_source = true;
-						unset($object['@context']);
+					unset($object['@context']);
-						$activity['as:object'] = $object;
+					$activity['as:object'] = $object;
-					}
 				} else {
 					Logger::info('Activity id is not equal', ['id' => $id, 'fetched' => $fetched_id]);
 				}
@@ -371,9 +384,9 @@ class Receiver
 			$object_data['object_object'] = JsonLD::fetchElement($activity['as:object'], 'as:object');
 			$object_data['object_type'] = JsonLD::fetchElement($activity['as:object'], '@type');
 			$object_data['push'] = $push;
-			if ($type == 'as:Delete') {
+			if (!$trust_source && ($type == 'as:Delete')) {
 				$apcontact = APContact::getByURL($object_data['object_id'], true);
-				$trust_source = ($apcontact['type'] == 'Tombstone');
+				$trust_source = empty($apcontact) || ($apcontact['type'] == 'Tombstone') || $apcontact['suspended'];
 			}
 		} elseif (in_array($type, ['as:Create', 'as:Update', 'as:Announce', 'as:Invite']) || strpos($type, '#emojiReaction')) {
 			// Fetch the content only on activities where this matters
@@ -430,8 +443,11 @@ class Receiver
 				$object_data['object_object_type'] = self::fetchObjectType([], $object_data['object_object'], $fetch_uid);
 			}
 
-			if (($type == 'as:Delete') && in_array($object_data['object_type'], array_merge(['as:Tombstone'], self::CONTENT_TYPES))) {
+			if (!$trust_source && ($type == 'as:Delete') && in_array($object_data['object_type'], array_merge(['as:Tombstone', ''], self::CONTENT_TYPES))) {
 				$trust_source = Processor::isActivityGone($object_data['object_id']);
+				if (!$trust_source) {
+					$trust_source = !empty(APContact::getByURL($object_data['object_id'], false));
+				}
 			}
 		}
 
@@ -668,6 +684,9 @@ class Receiver
 						if (!empty($object_data['raw'])) {
 							$announce_object_data['raw'] = $object_data['raw'];
 						}
+						if (!empty($object_data['raw-object'])) {
+							$announce_object_data['raw-object'] = $object_data['raw-object'];
+						}
 						ActivityPub\Processor::createActivity($announce_object_data, Activity::ANNOUNCE);
 					}
 				} else {
@@ -1305,7 +1324,7 @@ class Receiver
 			$object_data = self::processObject($object);
 
 			if (!empty($data)) {
-				$object_data['raw'] = json_encode($data);
+				$object_data['raw-object'] = json_encode($data);
 			}
 			return $object_data;
 		}

src/Protocol/ActivityPub/Transmitter.php

@@ -1232,10 +1232,8 @@ class Transmitter
 		}
 
 		if (!$item['deleted']) {
-			$condition = ['item-uri' => $item['uri'], 'protocol' => Conversation::PARCEL_ACTIVITYPUB];
+			$data = Post\Activity::getByURIId($item['uri-id']);
-			$conversation = DBA::selectFirst('conversation', ['source'], $condition);
+			if (!$item['origin'] && !empty($data)) {
-			if (!$item['origin'] && DBA::isResult($conversation)) {
-				$data = json_decode($conversation['source'], true);
 				if (!empty($data['type'])) {
 					if (in_array($data['type'], ['Create', 'Update'])) {
 						if ($object_mode) {

src/Protocol/DFRN.php

@@ -810,27 +810,12 @@ class DFRN
 		}
 
 		// Add conversation data. This is used for OStatus
-		$conversation_href = DI::baseUrl()."/display/".$item["parent-guid"];
-		$conversation_uri = $conversation_href;
-
-		if (isset($parent_item)) {
-			$conversation = DBA::selectFirst('conversation', ['conversation-uri', 'conversation-href'], ['item-uri' => $item['thr-parent']]);
-			if (DBA::isResult($conversation)) {
-				if ($conversation['conversation-uri'] != '') {
-					$conversation_uri = $conversation['conversation-uri'];
-				}
-				if ($conversation['conversation-href'] != '') {
-					$conversation_href = $conversation['conversation-href'];
-				}
-			}
-		}
-
 		$attributes = [
-			'href' => $conversation_href,
+			'href' => $item['conversation'],
-			'ref' => $conversation_uri,
+			'ref' => $item['conversation'],
 		];
 
-		XML::addElement($doc, $entry, 'ostatus:conversation', $conversation_uri, $attributes);
+		XML::addElement($doc, $entry, 'ostatus:conversation', $item['conversation'], $attributes);
 
 		XML::addElement($doc, $entry, 'id', $item['uri']);
 		XML::addElement($doc, $entry, 'title', $item['title']);
@@ -1994,16 +1979,16 @@ class DFRN
 			self::parseLinks($links, $item);
 		}
 
-		$item['conversation-uri'] = XML::getFirstNodeValue($xpath, 'ostatus:conversation/text()', $entry);
+		$item['conversation'] = XML::getFirstNodeValue($xpath, 'ostatus:conversation/text()', $entry);
 
 		$conv = $xpath->query('ostatus:conversation', $entry);
 		if (is_object($conv->item(0))) {
 			foreach ($conv->item(0)->attributes as $attributes) {
 				if ($attributes->name == 'ref') {
-					$item['conversation-uri'] = $attributes->textContent;
+					$item['conversation'] = $attributes->textContent;
 				}
 				if ($attributes->name == 'href') {
-					$item['conversation-href'] = $attributes->textContent;
+					$item['conversation'] = $attributes->textContent;
 				}
 			}
 		}

src/Protocol/OStatus.php

@@ -616,16 +616,16 @@ class OStatus
 
 		$item['created'] = XML::getFirstNodeValue($xpath, 'atom:published/text()', $entry);
 		$item['edited'] = XML::getFirstNodeValue($xpath, 'atom:updated/text()', $entry);
-		$item['conversation-uri'] = XML::getFirstNodeValue($xpath, 'ostatus:conversation/text()', $entry);
+		$item['conversation'] = XML::getFirstNodeValue($xpath, 'ostatus:conversation/text()', $entry);
 
 		$conv = $xpath->query('ostatus:conversation', $entry);
 		if (is_object($conv->item(0))) {
 			foreach ($conv->item(0)->attributes as $attributes) {
 				if ($attributes->name == 'ref') {
-					$item['conversation-uri'] = $attributes->textContent;
+					$item['conversation'] = $attributes->textContent;
 				}
 				if ($attributes->name == 'href') {
-					$item['conversation-href'] = $attributes->textContent;
+					$item['conversation'] = $attributes->textContent;
 				}
 			}
 		}
@@ -704,14 +704,6 @@ class OStatus
 			}
 		}
 
-		if (($self != '') && empty($item['protocol'])) {
-			self::fetchSelf($self, $item);
-		}
-
-		if (!empty($item['conversation-href'])) {
-			self::fetchConversation($item['conversation-href'], $item['conversation-uri']);
-		}
-
 		if (isset($item['thr-parent'])) {
 			if (!Post::exists(['uid' => $importer['uid'], 'uri' => $item['thr-parent']])) {
 				if ($related != '') {
@@ -725,194 +717,9 @@ class OStatus
 			$item['gravity'] = GRAVITY_PARENT;
 		}
 
-		if (($item['author-link'] != '') && !empty($item['protocol'])) {
-			$item = Conversation::insert($item);
-		}
-
 		self::$itemlist[] = $item;
 	}
 
-	/**
-	 * Fetch the conversation for posts
-	 *
-	 * @param string $conversation     The link to the conversation
-	 * @param string $conversation_uri The conversation in "uri" format
-	 * @return void
-	 * @throws \Friendica\Network\HTTPException\InternalServerErrorException
-	 */
-	private static function fetchConversation(string $conversation, string $conversation_uri)
-	{
-		// Ensure that we only store a conversation once in a process
-		if (isset(self::$conv_list[$conversation])) {
-			return;
-		}
-
-		self::$conv_list[$conversation] = true;
-
-		$curlResult = DI::httpClient()->get($conversation, HttpClientAccept::ATOM_XML);
-
-		if (!$curlResult->isSuccess() || empty($curlResult->getBody())) {
-			return;
-		}
-
-		$xml = '';
-
-		if ($curlResult->inHeader('Content-Type') &&
-			in_array('application/atom+xml', $curlResult->getHeader('Content-Type'))) {
-			$xml = $curlResult->getBody();
-		}
-
-		if ($xml == '') {
-			$doc = new DOMDocument();
-			if (!@$doc->loadHTML($curlResult->getBody())) {
-				return;
-			}
-			$xpath = new DOMXPath($doc);
-
-			$links = $xpath->query('//link');
-			if ($links) {
-				$file = '';
-				foreach ($links as $link) {
-					$attribute = self::readAttributes($link);
-					if (($attribute['rel'] == 'alternate') && ($attribute['type'] == 'application/atom+xml')) {
-						$file = $attribute['href'];
-					}
-				}
-				if ($file != '') {
-					$conversation_atom = DI::httpClient()->get($attribute['href'], HttpClientAccept::ATOM_XML);
-
-					if ($conversation_atom->isSuccess()) {
-						$xml = $conversation_atom->getBody();
-					}
-				}
-			}
-		}
-
-		if ($xml == '') {
-			return;
-		}
-
-		self::storeConversation($xml, $conversation, $conversation_uri);
-	}
-
-	/**
-	 * Store a feed in several conversation entries
-	 *
-	 * @param string $xml              The feed
-	 * @param string $conversation     conversation
-	 * @param string $conversation_uri conversation uri
-	 * @return void
-	 * @throws \Exception
-	 */
-	private static function storeConversation(string $xml, string $conversation = '', string $conversation_uri = '')
-	{
-		$doc = new DOMDocument();
-		@$doc->loadXML($xml);
-
-		$xpath = new DOMXPath($doc);
-		$xpath->registerNamespace('atom', ActivityNamespace::ATOM1);
-		$xpath->registerNamespace('thr', ActivityNamespace::THREAD);
-		$xpath->registerNamespace('ostatus', ActivityNamespace::OSTATUS);
-
-		$entries = $xpath->query('/atom:feed/atom:entry');
-
-		// Now store the entries
-		foreach ($entries as $entry) {
-			$doc2 = new DOMDocument();
-			$doc2->preserveWhiteSpace = false;
-			$doc2->formatOutput = true;
-
-			$conv_data = [];
-
-			$conv_data['protocol'] = Conversation::PARCEL_SPLIT_CONVERSATION;
-			$conv_data['direction'] = Conversation::PULL;
-			$conv_data['network'] = Protocol::OSTATUS;
-			$conv_data['uri'] = XML::getFirstNodeValue($xpath, 'atom:id/text()', $entry);
-
-			$inreplyto = $xpath->query('thr:in-reply-to', $entry);
-			if (is_object($inreplyto->item(0))) {
-				foreach ($inreplyto->item(0)->attributes as $attributes) {
-					if ($attributes->name == 'ref') {
-						$conv_data['reply-to-uri'] = $attributes->textContent;
-					}
-				}
-			}
-
-			$conv_data['conversation-uri'] = XML::getFirstNodeValue($xpath, 'ostatus:conversation/text()', $entry);
-
-			$conv = $xpath->query('ostatus:conversation', $entry);
-			if (is_object($conv->item(0))) {
-				foreach ($conv->item(0)->attributes as $attributes) {
-					if ($attributes->name == 'ref') {
-						$conv_data['conversation-uri'] = $attributes->textContent;
-					}
-					if ($attributes->name == 'href') {
-						$conv_data['conversation-href'] = $attributes->textContent;
-					}
-				}
-			}
-
-			if ($conversation != '') {
-				$conv_data['conversation-uri'] = $conversation;
-			}
-
-			if ($conversation_uri != '') {
-				$conv_data['conversation-uri'] = $conversation_uri;
-			}
-
-			$entry = $doc2->importNode($entry, true);
-
-			$doc2->appendChild($entry);
-
-			$conv_data['source'] = $doc2->saveXML();
-
-			Logger::info('Store conversation data for uri '.$conv_data['uri']);
-			Conversation::insert($conv_data);
-		}
-	}
-
-	/**
-	 * Fetch the own post so that it can be stored later
-	 *
-	 * We want to store the original data for later processing.
-	 * This function is meant for cases where we process a feed with multiple entries.
-	 * In that case we need to fetch the single posts here.
-	 *
-	 * @param string $self The link to the self item
-	 * @param array  $item The item array
-	 * @return void
-	 * @throws \Friendica\Network\HTTPException\InternalServerErrorException
-	 */
-	private static function fetchSelf(string $self, array &$item)
-	{
-		$condition = ['item-uri' => $self, 'protocol' => [Conversation::PARCEL_DFRN,
-			Conversation::PARCEL_DIASPORA_DFRN, Conversation::PARCEL_LOCAL_DFRN,
-			Conversation::PARCEL_DIRECT, Conversation::PARCEL_SALMON]];
-		if (DBA::exists('conversation', $condition)) {
-			Logger::info('Conversation '.$item['uri'].' is already stored.');
-			return;
-		}
-
-		$curlResult = DI::httpClient()->get($self, HttpClientAccept::ATOM_XML);
-
-		if (!$curlResult->isSuccess()) {
-			return;
-		}
-
-		// We reformat the XML to make it better readable
-		$doc = new DOMDocument();
-		$doc->loadXML($curlResult->getBody());
-		$doc->preserveWhiteSpace = false;
-		$doc->formatOutput = true;
-		$xml = $doc->saveXML();
-
-		$item['protocol'] = Conversation::PARCEL_SALMON;
-		$item['source'] = $xml;
-		$item['direction'] = Conversation::PULL;
-
-		Logger::info('Conversation '.$item['uri'].' is now fetched.');
-	}
-
 	/**
 	 * Fetch related posts and processes them
 	 *
@@ -925,30 +732,6 @@ class OStatus
 	 */
 	private static function fetchRelated(string $related, string $related_uri, array $importer)
 	{
-		$condition = [
-			'item-uri' => $related_uri,
-			'protocol' => [
-				Conversation::PARCEL_DFRN,
-				Conversation::PARCEL_DIASPORA_DFRN,
-				Conversation::PARCEL_LOCAL_DFRN,
-				Conversation::PARCEL_DIRECT,
-				Conversation::PARCEL_SALMON,
-			],
-		];
-		$conversation = DBA::selectFirst('conversation', ['source', 'protocol'], $condition);
-		if (DBA::isResult($conversation)) {
-			$stored = true;
-			$xml = $conversation['source'];
-			if (self::process($xml, $importer, $contact, $hub, $stored, false, Conversation::PULL)) {
-				Logger::info('Got valid cached XML for URI '.$related_uri);
-				return;
-			}
-			if ($conversation['protocol'] == Conversation::PARCEL_SALMON) {
-				Logger::info('Delete invalid cached XML for URI '.$related_uri);
-				DBA::delete('conversation', ['item-uri' => $related_uri]);
-			}
-		}
-
 		$stored = false;
 		$curlResult = DI::httpClient()->get($related, HttpClientAccept::ATOM_XML);
 
@@ -1013,17 +796,6 @@ class OStatus
 			}
 		}
 
-		// Finally we take the data that we fetched from "ostatus:conversation"
-		if ($xml == '') {
-			$condition = ['item-uri' => $related_uri, 'protocol' => Conversation::PARCEL_SPLIT_CONVERSATION];
-			$conversation = DBA::selectFirst('conversation', ['source'], $condition);
-			if (DBA::isResult($conversation)) {
-				$stored = true;
-				Logger::info('Got cached XML from conversation for URI ' . $related_uri);
-				$xml = $conversation['source'];
-			}
-		}
-
 		if ($xml != '') {
 			self::process($xml, $importer, $contact, $hub, $stored, false, Conversation::PULL);
 		} else {
@@ -1130,10 +902,7 @@ class OStatus
 
 					case 'ostatus:conversation':
 						$link_data['conversation'] = $attribute['href'];
-						$item['conversation-href'] = $link_data['conversation'];
+						$item['conversation'] = $link_data['conversation'];
-						if (!isset($item['conversation-uri'])) {
-							$item['conversation-uri'] = $item['conversation-href'];
-						}
 						break;
 
 					case 'enclosure':
@@ -1904,19 +1673,7 @@ class OStatus
 		}
 
 		if (intval($item['parent']) > 0) {
-			$conversation_href = $conversation_uri = str_replace('/objects/', '/context/', $item['thr-parent']);
+			$conversation_href = $conversation_uri = $item['conversation'];
-
-			if (isset($parent_item)) {
-				$conversation = DBA::selectFirst('conversation', ['conversation-uri', 'conversation-href'], ['item-uri' => $parent_item]);
-				if (DBA::isResult($conversation)) {
-					if ($conversation['conversation-uri'] != '') {
-						$conversation_uri = $conversation['conversation-uri'];
-					}
-					if ($conversation['conversation-href'] != '') {
-						$conversation_href = $conversation['conversation-href'];
-					}
-				}
-			}
 
 			XML::addElement($doc, $entry, 'link', '', ['rel' => 'ostatus:conversation', 'href' => $conversation_href]);
 

src/Worker/Cron.php

@@ -115,7 +115,7 @@ class Cron
 
 			Worker::add(PRIORITY_LOW, 'ExpirePosts');
 
-			Worker::add(PRIORITY_LOW, 'ExpireConversations');
+			Worker::add(PRIORITY_LOW, 'ExpireActivities');
 
 			Worker::add(PRIORITY_LOW, 'RemoveUnusedTags');
 

src/Worker/ExpireActivities.php

@@ -22,21 +22,15 @@
 namespace Friendica\Worker;
 
 use Friendica\Database\DBA;
-use Friendica\DI;
 use Friendica\Util\DateTimeFormat;
 
-class ExpireConversations
+class ExpireActivities
 {
 	/**
-	 * Delete old conversation entries
+	 * Delete old post-activity entries
 	 */
 	public static function execute()
 	{
-		$days = intval(DI::config()->get('system', 'dbclean_expire_conversation', 90));
+		DBA::delete('post-activity', ["`received` < ?", DateTimeFormat::utc('now - 7 days')]);
-		if (empty($days)) {
-			return;
-		}
-
-		DBA::delete('conversation', ["`received` < ?", DateTimeFormat::utc('now - ' . $days . ' days')]);
 	}
 }

src/Worker/Notifier.php

@@ -746,7 +746,7 @@ class Notifier
 			}
 
 			Logger::info('Origin item ' . $target_item['id'] . ' with URL ' . $target_item['uri'] . ' will be distributed.');
-		} elseif (!DBA::exists('conversation', ['item-uri' => $target_item['uri'], 'protocol' => Conversation::PARCEL_ACTIVITYPUB])) {
+		} elseif (!Post\Activity::exists($target_item['uri-id'])) {
 			Logger::info('Remote item ' . $target_item['id'] . ' with URL ' . $target_item['uri'] . ' is no AP post. It will not be distributed.');
 			return ['count' => 0, 'contacts' => []];
 		} elseif ($parent['origin']) {

static/dbstructure.config.php

@@ -55,7 +55,7 @@
 use Friendica\Database\DBA;
 
 if (!defined('DB_UPDATE_VERSION')) {
-	define('DB_UPDATE_VERSION', 1474);
+	define('DB_UPDATE_VERSION', 1475);
 }
 
 return [
@@ -400,6 +400,7 @@ return [
 			"featured-tags" => ["type" => "varchar(255)", "comment" => "Address for the collection of featured tags"],
 			"manually-approve" => ["type" => "boolean", "comment" => ""],
 			"discoverable" => ["type" => "boolean", "comment" => "Mastodon extension: true if profile is published in their directory"],
+			"suspended" => ["type" => "boolean", "comment" => "Mastodon extension: true if profile is suspended"],
 			"nick" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""],
 			"name" => ["type" => "varchar(255)", "comment" => ""],
 			"about" => ["type" => "text", "comment" => ""],
@@ -565,24 +566,6 @@ return [
 			"uid" => ["uid"],
 		]
 	],
-	"conversation" => [
-		"comment" => "Raw data and structure information for messages",
-		"fields" => [
-			"item-uri" => ["type" => "varbinary(255)", "not null" => "1", "primary" => "1", "comment" => "Original URI of the item - unrelated to the table with the same name"],
-			"reply-to-uri" => ["type" => "varbinary(255)", "not null" => "1", "default" => "", "comment" => "URI to which this item is a reply"],
-			"conversation-uri" => ["type" => "varbinary(255)", "not null" => "1", "default" => "", "comment" => "GNU Social conversation URI"],
-			"conversation-href" => ["type" => "varbinary(255)", "not null" => "1", "default" => "", "comment" => "GNU Social conversation link"],
-			"protocol" => ["type" => "tinyint unsigned", "not null" => "1", "default" => "255", "comment" => "The protocol of the item"],
-			"direction" => ["type" => "tinyint unsigned", "not null" => "1", "default" => "0", "comment" => "How the message arrived here: 1=push, 2=pull"],
-			"source" => ["type" => "mediumtext", "comment" => "Original source"],
-			"received" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => "Receiving date"],
-		],
-		"indexes" => [
-			"PRIMARY" => ["item-uri"],
-			"conversation-uri" => ["conversation-uri"],
-			"received" => ["received"],
-		]
-	],
 	"workerqueue" => [
 		"comment" => "Background tasks queue entries",
 		"fields" => [
@@ -1156,6 +1139,17 @@ return [
 			"vid" => ["vid"],
 		]
 	],
+	"post-activity" => [
+		"comment" => "Original remote activity",
+		"fields" => [
+			"uri-id" => ["type" => "int unsigned", "not null" => "1", "primary" => "1",  "foreign" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the item uri"],
+			"activity" => ["type" => "mediumtext", "comment" => "Original activity"],
+			"received" => ["type" => "datetime", "comment" => ""],
+		],
+		"indexes" => [
+			"PRIMARY" => ["uri-id"],
+		]
+	],
 	"post-category" => [
 		"comment" => "post relation to categories",
 		"fields" => [