Several performance improvements

database.sql

@@ -1,6 +1,6 @@
 -- ------------------------------------------
 -- Friendica 2022.09-dev (Giant Rhubarb)
--- DB_UPDATE_VERSION 1480
+-- DB_UPDATE_VERSION 1481
 -- ------------------------------------------
 
 
@@ -222,7 +222,7 @@ CREATE TABLE IF NOT EXISTS `contact` (
 	 INDEX `uid_contact-type` (`uid`,`contact-type`),
 	 INDEX `uid_self_contact-type` (`uid`,`self`,`contact-type`),
 	 INDEX `self_network_uid` (`self`,`network`,`uid`),
-	 INDEX `gsid` (`gsid`),
+	 INDEX `gsid_uid_failed` (`gsid`,`uid`,`failed`),
 	 INDEX `uri-id` (`uri-id`),
 	FOREIGN KEY (`uid`) REFERENCES `user` (`uid`) ON UPDATE RESTRICT ON DELETE CASCADE,
 	FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE,
@@ -920,6 +920,7 @@ CREATE TABLE IF NOT EXISTS `notification` (
 	 INDEX `target-uri-id` (`target-uri-id`),
 	 INDEX `parent-uri-id` (`parent-uri-id`),
 	 INDEX `seen_uid` (`seen`,`uid`),
+	 INDEX `uid_type_parent-uri-id_actor-id` (`uid`,`type`,`parent-uri-id`,`actor-id`),
 	FOREIGN KEY (`uid`) REFERENCES `user` (`uid`) ON UPDATE RESTRICT ON DELETE CASCADE,
 	FOREIGN KEY (`vid`) REFERENCES `verb` (`id`) ON UPDATE RESTRICT ON DELETE RESTRICT,
 	FOREIGN KEY (`actor-id`) REFERENCES `contact` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE,
@@ -1153,9 +1154,12 @@ CREATE TABLE IF NOT EXISTS `post-category` (
 CREATE TABLE IF NOT EXISTS `post-collection` (
 	`uri-id` int unsigned NOT NULL COMMENT 'Id of the item-uri table entry that contains the item uri',
 	`type` tinyint unsigned NOT NULL DEFAULT 0 COMMENT '0 - Featured',
+	`author-id` int unsigned COMMENT 'Author of the featured post',
 	 PRIMARY KEY(`uri-id`,`type`),
 	 INDEX `type` (`type`),
-	FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE
+	 INDEX `author-id` (`author-id`),
+	FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE,
+	FOREIGN KEY (`author-id`) REFERENCES `contact` (`id`) ON UPDATE RESTRICT ON DELETE RESTRICT
 ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Collection of posts';
 
 --
@@ -2423,7 +2427,7 @@ DROP VIEW IF EXISTS `collection-view`;
 CREATE VIEW `collection-view` AS SELECT 
 	`post-collection`.`uri-id` AS `uri-id`,
 	`post-collection`.`type` AS `type`,
-	`post`.`author-id` AS `cid`,
+	`post-collection`.`author-id` AS `cid`,
 	`post`.`received` AS `received`,
 	`post`.`created` AS `created`,
 	`post-thread`.`commented` AS `commented`,
@@ -2431,7 +2435,7 @@ CREATE VIEW `collection-view` AS SELECT
 	`post`.`visible` AS `visible`,
 	`post`.`deleted` AS `deleted`,
 	`post`.`thr-parent-id` AS `thr-parent-id`,
-	`post`.`author-id` AS `author-id`,
+	`post-collection`.`author-id` AS `author-id`,
 	`post`.`gravity` AS `gravity`
 	FROM `post-collection`
 			INNER JOIN `post` ON `post-collection`.`uri-id` = `post`.`uri-id`

doc/database/db_contact.md

@@ -121,7 +121,7 @@ Indexes
 | uid_contact-type            | uid, contact-type                    |
 | uid_self_contact-type       | uid, self, contact-type              |
 | self_network_uid            | self, network, uid                   |
-| gsid                        | gsid                                 |
+| gsid_uid_failed             | gsid, uid, failed                    |
 | uri-id                      | uri-id                               |
 
 Foreign Keys

doc/database/db_notification.md

@@ -31,6 +31,7 @@ Indexes
 | target-uri-id                       | target-uri-id                                   |
 | parent-uri-id                       | parent-uri-id                                   |
 | seen_uid                            | seen, uid                                       |
+| uid_type_parent-uri-id_actor-id     | uid, type, parent-uri-id, actor-id              |
 
 Foreign Keys
 ------------

doc/database/db_post-collection.md

@@ -6,18 +6,20 @@ Collection of posts
 Fields
 ------
 
-| Field  | Description                                               | Type             | Null | Key | Default | Extra |
+| Field     | Description                                               | Type             | Null | Key | Default | Extra |
-| ------ | --------------------------------------------------------- | ---------------- | ---- | --- | ------- | ----- |
+| --------- | --------------------------------------------------------- | ---------------- | ---- | --- | ------- | ----- |
-| uri-id | Id of the item-uri table entry that contains the item uri | int unsigned     | NO   | PRI | NULL    |       |
+| uri-id    | Id of the item-uri table entry that contains the item uri | int unsigned     | NO   | PRI | NULL    |       |
-| type   | 0 - Featured                                              | tinyint unsigned | NO   | PRI | 0       |       |
+| type      | 0 - Featured                                              | tinyint unsigned | NO   | PRI | 0       |       |
+| author-id | Author of the featured post                               | int unsigned     | YES  |     | NULL    |       |
 
 Indexes
 ------------
 
-| Name    | Fields       |
+| Name      | Fields       |
-| ------- | ------------ |
+| --------- | ------------ |
-| PRIMARY | uri-id, type |
+| PRIMARY   | uri-id, type |
-| type    | type         |
+| type      | type         |
+| author-id | author-id    |
 
 Foreign Keys
 ------------
@@ -25,5 +27,6 @@ Foreign Keys
 | Field | Target Table | Target Field |
 |-------|--------------|--------------|
 | uri-id | [item-uri](help/database/db_item-uri) | id |
+| author-id | [contact](help/database/db_contact) | id |
 
 Return to [database documentation](help/database)

src/Model/APContact.php

@@ -24,6 +24,7 @@ namespace Friendica\Model;
 use Friendica\Content\Text\HTML;
 use Friendica\Core\Cache\Enum\Duration;
 use Friendica\Core\Logger;
+use Friendica\Core\Protocol;
 use Friendica\Core\System;
 use Friendica\Database\DBA;
 use Friendica\DI;
@@ -352,17 +353,18 @@ class APContact
 
 		if (!empty($apcontact['outbox'])) {
 			if (!empty($local_owner)) {
-				$outbox = ActivityPub\Transmitter::getOutbox($local_owner);
+				$statuses_count = self::getStatusesCount($local_owner);
 			} else {
 				$outbox = ActivityPub::fetchContent($apcontact['outbox']);
+				$statuses_count = $outbox['totalItems'] ?? 0;
 			}
-			if (!empty($outbox['totalItems'])) {
+			if (!empty($statuses_count)) {
 				// Mastodon seriously allows for this condition?
 				// Jul 20 2021 - See https://chaos.social/@m11 for a negative posts count
-				if ($outbox['totalItems'] < 0) {
+				if ($statuses_count < 0) {
-					$outbox['totalItems'] = 0;
+					$statuses_count = 0;
 				}
-				$apcontact['statuses_count'] = $outbox['totalItems'];
+				$apcontact['statuses_count'] = $statuses_count;
 			}
 		}
 
@@ -483,6 +485,30 @@ class APContact
 		return DBA::selectFirst('apcontact', [], ['url' => $apcontact['url']]) ?: [];
 	}
 
+	/**
+	 * Fetch the number of statuses for the given owner
+	 *
+	 * @param array $owner
+	 *
+	 * @return integer
+	 */
+	private static function getStatusesCount(array $owner): int
+	{
+		$condition = [
+			'private' => [Item::PUBLIC, Item::UNLISTED],
+			'author-id'      => Contact::getIdForURL($owner['url'], 0, false),
+			'gravity'        => [GRAVITY_PARENT, GRAVITY_COMMENT],
+			'network'        => Protocol::DFRN,
+			'parent-network' => Protocol::FEDERATED,
+			'deleted'        => false,
+			'visible'        => true
+		];
+
+		$count = Post::countPosts($condition);
+
+		return $count;
+	}
+
 	/**
 	 * Mark the given AP Contact as "to archive"
 	 *

src/Model/FContact.php

@@ -100,7 +100,7 @@ class FContact
 
 			$interacted  = DBA::count('contact-relation', ["`cid` = ? AND NOT `follows` AND `last-interaction` > ?", $contact['id'], $last_interaction]);
 			$interacting = DBA::count('contact-relation', ["`relation-cid` = ? AND NOT `follows` AND `last-interaction` > ?", $contact['id'], $last_interaction]);
-			$posts       = Post::countPosts(['author-id' => $contact['id'], 'gravity' => [GRAVITY_PARENT, GRAVITY_COMMENT]]);
+			$posts       = DBA::count('post', ['author-id' => $contact['id'], 'gravity' => [GRAVITY_PARENT, GRAVITY_COMMENT]]);
 		}
 
 		$fields = [

src/Model/Post.php

@@ -400,12 +400,12 @@ class Post
 			AND (NOT `causer-blocked` OR `causer-id` = ? OR `causer-id` IS NULL) AND NOT `contact-blocked`
 			AND ((NOT `contact-readonly` AND NOT `contact-pending` AND (`contact-rel` IN (?, ?)))
 				OR `self` OR `gravity` != ? OR `contact-uid` = ?)
-			AND NOT EXISTS (SELECT `uri-id` FROM `post-user` WHERE `uid` = ? AND `uri-id` = `" . $view . "`.`uri-id` AND `hidden`)
+			AND NOT `" . $view . "`.`uri-id` IN (SELECT `uri-id` FROM `post-user` WHERE `uid` = ? AND `hidden`)
-			AND NOT EXISTS (SELECT `cid` FROM `user-contact` WHERE `uid` = ? AND `cid` = `author-id` AND `blocked`)
+			AND NOT `author-id` IN (SELECT `cid` FROM `user-contact` WHERE `uid` = ? AND `blocked`)
-			AND NOT EXISTS (SELECT `cid` FROM `user-contact` WHERE `uid` = ? AND `cid` = `owner-id` AND `blocked`)
+			AND NOT `owner-id` IN (SELECT `cid` FROM `user-contact` WHERE `uid` = ? AND `blocked`)
-			AND NOT EXISTS (SELECT `cid` FROM `user-contact` WHERE `uid` = ? AND `cid` = `author-id` AND `ignored` AND `gravity` = ?)
+			AND NOT (`gravity` = ? AND `author-id` IN (SELECT `cid` FROM `user-contact` WHERE `uid` = ? AND `ignored`))
-			AND NOT EXISTS (SELECT `cid` FROM `user-contact` WHERE `uid` = ? AND `cid` = `owner-id` AND `ignored` AND `gravity` = ?)",
+			AND NOT (`gravity` = ? AND `owner-id` IN (SELECT `cid` FROM `user-contact` WHERE `uid` = ? AND `ignored`))",
-			0, Contact::SHARING, Contact::FRIEND, GRAVITY_PARENT, 0, $uid, $uid, $uid, $uid, GRAVITY_PARENT, $uid, GRAVITY_PARENT]);
+			0, Contact::SHARING, Contact::FRIEND, GRAVITY_PARENT, 0, $uid, $uid, $uid, GRAVITY_PARENT, $uid, GRAVITY_PARENT, $uid]);
 
 		$select_string = implode(', ', array_map([DBA::class, 'quoteIdentifier'], $selected));
 

src/Model/Post/Collection.php

@@ -37,15 +37,16 @@ class Collection
 	 *
 	 * @param integer $uri_id
 	 * @param integer $type
+	 * @param integer $author_id
 	 * @param integer $cache_uid If set to a non zero value, the featured cache is cleared
 	 */
-	public static function add(int $uri_id, int $type, int $cache_uid = 0)
+	public static function add(int $uri_id, int $type, int $author_id, int $cache_uid = 0)
 	{
 		if (empty($uri_id)) {
 			throw new BadMethodCallException('Empty URI_id');
 		}
 
-		DBA::insert('post-collection', ['uri-id' => $uri_id, 'type' => $type], Database::INSERT_IGNORE);
+		DBA::insert('post-collection', ['uri-id' => $uri_id, 'type' => $type, 'author-id' => $author_id], Database::INSERT_IGNORE);
 
 		if (!empty($cache_uid) && ($type == self::FEATURED)) {
 			DI::cache()->delete(ActivityPub\Transmitter::CACHEKEY_FEATURED . $cache_uid);

src/Module/Api/Mastodon/Statuses/Pin.php

@@ -41,12 +41,12 @@ class Pin extends BaseApi
 			DI::mstdnError()->UnprocessableEntity();
 		}
 
-		$item = Post::selectFirstForUser($uid, ['id', 'gravity'], ['uri-id' => $this->parameters['id'], 'uid' => [$uid, 0]]);
+		$item = Post::selectFirstForUser($uid, ['id', 'gravity', 'author-id'], ['uri-id' => $this->parameters['id'], 'uid' => [$uid, 0]]);
 		if (!DBA::isResult($item)) {
 			DI::mstdnError()->RecordNotFound();
 		}
 
-		Post\Collection::add($this->parameters['id'], Post\Collection::FEATURED, $uid);
+		Post\Collection::add($this->parameters['id'], Post\Collection::FEATURED, $item['author-id'], $uid);
 
 		System::jsonExit(DI::mstdnStatus()->createFromUriId($this->parameters['id'], $uid)->toArray());
 	}

src/Module/Item/Pin.php

@@ -48,7 +48,7 @@ class Pin extends BaseModule
 
 		$itemId = intval($this->parameters['id']);
 
-		$item = Post::selectFirst(['uri-id', 'uid', 'featured'], ['id' => $itemId]);
+		$item = Post::selectFirst(['uri-id', 'uid', 'featured', 'author-id'], ['id' => $itemId]);
 		if (!DBA::isResult($item)) {
 			throw new HTTPException\NotFoundException();
 		}
@@ -60,7 +60,7 @@ class Pin extends BaseModule
 		$pinned = !$item['featured'];
 
 		if ($pinned) {
-			Post\Collection::add($item['uri-id'], Post\Collection::FEATURED, local_user());
+			Post\Collection::add($item['uri-id'], Post\Collection::FEATURED, $item['author-id'], local_user());
 		} else {
 			Post\Collection::remove($item['uri-id'], Post\Collection::FEATURED, local_user());
 		}

src/Protocol/ActivityPub/Processor.php

@@ -679,7 +679,7 @@ class Processor
 	 * Fetch the Uri-Id of a post for the "featured" collection
 	 *
 	 * @param array $activity
-	 * @return null|int
+	 * @return null|array
 	 */
 	private static function getUriIdForFeaturedCollection(array $activity)
 	{
@@ -697,7 +697,7 @@ class Processor
 			}
 		}
 
-		$parent = Post::selectFirst(['uri-id'], ['uri' => $activity['object_id']]);
+		$parent = Post::selectFirst(['uri-id', 'author-id'], ['uri' => $activity['object_id']]);
 		if (empty($parent['uri-id'])) {
 			if (self::fetchMissingActivity($activity['object_id'], $activity, '', Receiver::COMPLETION_AUTO)) {
 				$parent = Post::selectFirst(['uri-id'], ['uri' => $activity['object_id']]);
@@ -705,7 +705,7 @@ class Processor
 		}
 
 		if (!empty($parent['uri-id'])) {
-			return $parent['uri-id'];
+			$parent;
 		}
 
 		return null;
@@ -718,14 +718,14 @@ class Processor
 	 */
 	public static function addToFeaturedCollection(array $activity)
 	{
-		$uriid = self::getUriIdForFeaturedCollection($activity);
+		$post = self::getUriIdForFeaturedCollection($activity);
-		if (empty($uriid)) {
+		if (empty($post)) {
 			return;
 		}
 
-		Logger::debug('Add post to featured collection', ['uri-id' => $uriid]);
+		Logger::debug('Add post to featured collection', ['post' => $post]);
 
-		Post\Collection::add($uriid, Post\Collection::FEATURED);
+		Post\Collection::add($post['uri-id'], Post\Collection::FEATURED, $post['author-id']);
 		Queue::remove($activity);
 	}
 
@@ -736,14 +736,14 @@ class Processor
 	 */
 	public static function removeFromFeaturedCollection(array $activity)
 	{
-		$uriid = self::getUriIdForFeaturedCollection($activity);
+		$post = self::getUriIdForFeaturedCollection($activity);
-		if (empty($uriid)) {
+		if (empty($post)) {
 			return;
 		}
 
-		Logger::debug('Remove post from featured collection', ['uri-id' => $uriid]);
+		Logger::debug('Remove post from featured collection', ['post' => $post]);
 
-		Post\Collection::remove($uriid, Post\Collection::FEATURED);
+		Post\Collection::remove($post['uri-id'], Post\Collection::FEATURED);
 		Queue::remove($activity);
 	}
 
@@ -1335,10 +1335,10 @@ class Processor
 			}
 			$id = Item::fetchByLink($post['id']);
 			if (!empty($id)) {
-				$item = Post::selectFirst(['uri-id', 'featured'], ['id' => $id]);
+				$item = Post::selectFirst(['uri-id', 'featured', 'author-id'], ['id' => $id]);
 				if (!empty($item['uri-id'])) {
 					if (!$item['featured']) {
-						Post\Collection::add($item['uri-id'], Post\Collection::FEATURED);
+						Post\Collection::add($item['uri-id'], Post\Collection::FEATURED, $item['author-id']);
 						Logger::debug('Added featured post', ['uri-id' => $item['uri-id'], 'contact' => $url]);
 						$new++;
 					} else {

src/Protocol/ActivityPub/Transmitter.php

@@ -59,7 +59,6 @@ class Transmitter
 {
 	const CACHEKEY_FEATURED = 'transmitter:getFeatured:';
 	const CACHEKEY_CONTACTS = 'transmitter:getContacts:';
-	const CACHEKEY_OUTBOX   = 'transmitter:getOutbox:';
 
 	/**
 	 * Add relay servers to the list of inboxes
@@ -251,14 +250,6 @@ class Transmitter
 	 */
 	public static function getOutbox(array $owner, int $page = null, string $requester = '', bool $nocache = false): array
 	{
-		if (empty($page)) {
-			$cachekey = self::CACHEKEY_OUTBOX . $owner['uid'];
-			$result = DI::cache()->get($cachekey);
-			if (!$nocache && !is_null($result)) {
-				return $result;
-			}
-		}
-
 		$condition = ['private' => [Item::PUBLIC, Item::UNLISTED]];
 
 		if (!empty($requester)) {
@@ -283,12 +274,12 @@ class Transmitter
 			'visible'        => true
 		]);
 
-		$count = Post::count($condition);
+		$apcontact = APContact::getByURL($owner['url']);
 
 		$data = ['@context' => ActivityPub::CONTEXT];
 		$data['id'] = DI::baseUrl() . '/outbox/' . $owner['nickname'];
 		$data['type'] = 'OrderedCollection';
-		$data['totalItems'] = $count;
+		$data['totalItems'] = $apcontact['statuses_count'] ?? 0;
 
 		if (!empty($page)) {
 			$data['id'] .= '?' . http_build_query(['page' => $page]);
@@ -316,15 +307,17 @@ class Transmitter
 				$data['next'] = DI::baseUrl() . '/outbox/' . $owner['nickname'] . '?page=' . ($page + 1);
 			}
 
+			// Fix the cached total item count when it is lower than the real count
+			$total = (($page - 1) * 20) + $data['totalItems'];
+			if ($total > $data['totalItems']) {
+				$data['totalItems'] = $total;
+			}
+
 			$data['partOf'] = DI::baseUrl() . '/outbox/' . $owner['nickname'];
 
 			$data['orderedItems'] = $list;
 		}
 
-		if (!empty($cachekey)) {
-			DI::cache()->set($cachekey, $data, Duration::DAY);
-		}
-
 		return $data;
 	}
 

static/dbstructure.config.php

@@ -55,7 +55,7 @@
 use Friendica\Database\DBA;
 
 if (!defined('DB_UPDATE_VERSION')) {
-	define('DB_UPDATE_VERSION', 1480);
+	define('DB_UPDATE_VERSION', 1481);
 }
 
 return [
@@ -284,7 +284,7 @@ return [
 			"uid_contact-type" => ["uid", "contact-type"],
 			"uid_self_contact-type" => ["uid", "self", "contact-type"],
 			"self_network_uid" => ["self", "network", "uid"],
-			"gsid" => ["gsid"],
+			"gsid_uid_failed" => ["gsid", "uid", "failed"],
 			"uri-id" => ["uri-id"],
 		]
 	],
@@ -971,6 +971,7 @@ return [
 			"target-uri-id" => ["target-uri-id"],
 			"parent-uri-id" => ["parent-uri-id"],
 			"seen_uid" => ["seen", "uid"],
+			"uid_type_parent-uri-id_actor-id" => ["uid", "type", "parent-uri-id", "actor-id"],
 		]
 	],
 	"notify" => [
@@ -1187,10 +1188,12 @@ return [
 		"fields" => [
 			"uri-id" => ["type" => "int unsigned", "not null" => "1", "primary" => "1", "foreign" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the item uri"],
 			"type" => ["type" => "tinyint unsigned", "not null" => "1", "default" => "0", "primary" => "1", "comment" => "0 - Featured"],
+			"author-id" => ["type" => "int unsigned", "foreign" => ["contact" => "id", "on delete" => "restrict"], "comment" => "Author of the featured post"],
 		],
 		"indexes" => [
 			"PRIMARY" => ["uri-id", "type"],
 			"type" => ["type"],
+			"author-id" => ["author-id"],
 		]
 	],
 	"post-content" => [

static/dbview.config.php

@@ -702,7 +702,7 @@
 		"fields" => [
 			"uri-id" => ["post-collection", "uri-id"],
 			"type" => ["post-collection", "type"],
-			"cid" => ["post", "author-id"],
+			"cid" => ["post-collection", "author-id"],
 			"received" => ["post", "received"],
 			"created" => ["post", "created"],
 			"commented" => ["post-thread", "commented"],
@@ -710,7 +710,7 @@
 			"visible" => ["post", "visible"],
 			"deleted" => ["post", "deleted"],
 			"thr-parent-id" => ["post", "thr-parent-id"],
-			"author-id" => ["post", "author-id"],
+			"author-id" => ["post-collection", "author-id"],
 			"gravity" => ["post", "gravity"],
 		],
 		"query" => "FROM `post-collection`

update.php

@@ -1100,9 +1100,9 @@ function update_1451()
 
 function update_1457()
 {
-	$pinned = DBA::select('post-thread-user', ['uri-id'], ['pinned' => true]);
+	$pinned = DBA::select('post-thread-user', ['uri-id', 'author-id'], ['pinned' => true]);
 	while ($post = DBA::fetch($pinned)) {
-		Post\Collection::add($post['uri-id'], Post\Collection::FEATURED);
+		Post\Collection::add($post['uri-id'], Post\Collection::FEATURED, $post['author-id']);
 	}
 	DBA::close($pinned);
 
@@ -1115,3 +1115,9 @@ function update_1480()
 	DBA::update('post', ['deleted' => false], ["`uri-id` IN (SELECT `uri-id` FROM `post-user` WHERE NOT `deleted`)"]);
 	return Update::SUCCESS;
 }
+
+function update_1481()
+{
+	DBA::e("UPDATE `post-collection` INNER JOIN `post` ON `post`.`uri-id` = `post-collection`.`uri-id` SET `post-collection`.`author-id` = `post`.`author-id` WHERE `post-collection`.`author-id` IS null");
+	return Update::SUCCESS;
+}