Merge pull request #13778 from annando/post-counts

New table "post-counts" to precalculate the counts

database.sql

@@ -1,6 +1,6 @@
 -- ------------------------------------------
 -- Friendica 2024.03-dev (Yellow Archangel)
--- DB_UPDATE_VERSION 1542
+-- DB_UPDATE_VERSION 1543
 -- ------------------------------------------
 
 
@@ -1236,6 +1236,23 @@ CREATE TABLE IF NOT EXISTS `post-category` (
 	FOREIGN KEY (`tid`) REFERENCES `tag` (`id`) ON UPDATE RESTRICT ON DELETE RESTRICT
 ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='post relation to categories';
 
+--
+-- TABLE post-counts
+--
+CREATE TABLE IF NOT EXISTS `post-counts` (
+	`uri-id` int unsigned NOT NULL COMMENT 'Id of the item-uri table entry that contains the item uri',
+	`vid` smallint unsigned NOT NULL COMMENT 'Id of the verb table entry that contains the activity verbs',
+	`reaction` varchar(1) NOT NULL COMMENT 'Emoji Reaction',
+	`parent-uri-id` int unsigned COMMENT 'Id of the item-uri table that contains the parent uri',
+	`count` int unsigned DEFAULT 0 COMMENT 'Number of activities',
+	 PRIMARY KEY(`uri-id`,`vid`,`reaction`),
+	 INDEX `vid` (`vid`),
+	 INDEX `parent-uri-id` (`parent-uri-id`),
+	FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE,
+	FOREIGN KEY (`vid`) REFERENCES `verb` (`id`) ON UPDATE RESTRICT ON DELETE RESTRICT,
+	FOREIGN KEY (`parent-uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Original remote activity';
+
 --
 -- TABLE post-collection
 --

doc/database.md

@@ -61,6 +61,7 @@ Database Tables
 | [post-category](help/database/db_post-category) | post relation to categories |
 | [post-collection](help/database/db_post-collection) | Collection of posts |
 | [post-content](help/database/db_post-content) | Content for all posts |
+| [post-counts](help/database/db_post-counts) | Original remote activity |
 | [post-delivery](help/database/db_post-delivery) | Delivery data for posts for the batch processing |
 | [post-delivery-data](help/database/db_post-delivery-data) | Delivery data for items |
 | [post-engagement](help/database/db_post-engagement) | Engagement data per post |

doc/database/db_post-counts.md

@@ -0,0 +1,35 @@
+Table post-counts
+===========
+
+Original remote activity
+
+Fields
+------
+
+| Field         | Description                                                 | Type              | Null | Key | Default | Extra |
+| ------------- | ----------------------------------------------------------- | ----------------- | ---- | --- | ------- | ----- |
+| uri-id        | Id of the item-uri table entry that contains the item uri   | int unsigned      | NO   | PRI | NULL    |       |
+| vid           | Id of the verb table entry that contains the activity verbs | smallint unsigned | NO   | PRI | NULL    |       |
+| reaction      | Emoji Reaction                                              | varchar(1)        | NO   | PRI | NULL    |       |
+| parent-uri-id | Id of the item-uri table that contains the parent uri       | int unsigned      | YES  |     | NULL    |       |
+| count         | Number of activities                                        | int unsigned      | YES  |     | 0       |       |
+
+Indexes
+------------
+
+| Name          | Fields                |
+| ------------- | --------------------- |
+| PRIMARY       | uri-id, vid, reaction |
+| vid           | vid                   |
+| parent-uri-id | parent-uri-id         |
+
+Foreign Keys
+------------
+
+| Field | Target Table | Target Field |
+|-------|--------------|--------------|
+| uri-id | [item-uri](help/database/db_item-uri) | id |
+| vid | [verb](help/database/db_verb) | id |
+| parent-uri-id | [item-uri](help/database/db_item-uri) | id |
+
+Return to [database documentation](help/database)

src/Database/PostUpdate.php

@@ -32,6 +32,7 @@ use Friendica\Model\ItemURI;
 use Friendica\Model\Photo;
 use Friendica\Model\Post;
 use Friendica\Model\Post\Category;
+use Friendica\Model\Post\Counts;
 use Friendica\Model\Tag;
 use Friendica\Model\Verb;
 use Friendica\Protocol\ActivityPub\Processor;
@@ -51,7 +52,7 @@ class PostUpdate
 	// Needed for the helper function to read from the legacy term table
 	const OBJECT_TYPE_POST  = 1;
 
-	const VERSION = 1507;
+	const VERSION = 1543;
 
 	/**
 	 * Calls the post update functions
@@ -124,6 +125,9 @@ class PostUpdate
 		if (!self::update1507()) {
 			return false;
 		}
+		if (!self::update1543()) {
+			return false;
+		}
 		return true;
 	}
 
@@ -1303,4 +1307,51 @@ class PostUpdate
 
 		return false;
 	}
+
+	/**
+	 * Create "post-counts" entries for old entries.
+	 *
+	 * @return bool "true" when the job is done
+	 * @throws \Friendica\Network\HTTPException\InternalServerErrorException
+	 * @throws \ImagickException
+	 */
+	private static function update1543()
+	{
+		// Was the script completed?
+		if (DI::keyValue()->get('post_update_version') >= 1543) {
+			return true;
+		}
+
+		$id = DI::keyValue()->get('post_update_version_1543_id') ?? 0;
+
+		Logger::info('Start', ['uri-id' => $id]);
+
+		$rows = 0;
+
+		$posts = Post::selectPosts(['uri-id', 'parent-uri-id'], ["`uri-id` > ? AND `gravity` IN (?, ?)", $id, Item::GRAVITY_COMMENT, Item::GRAVITY_PARENT], ['order' => ['uri-id'], 'limit' => 1000]);
+
+		if (DBA::errorNo() != 0) {
+			Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]);
+			return false;
+		}
+
+		while ($post = Post::fetch($posts)) {
+			$id = $post['uri-id'];
+			Counts::updateForPost($post['uri-id'], $post['parent-uri-id']);
+			++$rows;
+		}
+		DBA::close($posts);
+
+		DI::keyValue()->set('post_update_version_1543_id', $id);
+
+		Logger::info('Processed', ['rows' => $rows, 'last' => $id]);
+
+		if ($rows <= 100) {
+			DI::keyValue()->set('post_update_version', 1543);
+			Logger::info('Done');
+			return true;
+		}
+
+		return false;
+	}
 }

src/Model/Item.php

@@ -1432,6 +1432,10 @@ class Item
 			self::updateDisplayCache($posted_item['uri-id']);
 		}
 
+		if (in_array($posted_item['gravity'], [self::GRAVITY_ACTIVITY, self::GRAVITY_COMMENT]) && ($posted_item['uid'] == 0)) {
+			Post\Counts::update($posted_item['thr-parent-id'], $posted_item['parent-uri-id'], $posted_item['vid'], $posted_item['verb'], $posted_item['body']);
+		}
+
 		if ($inserted) {
 			Post\Engagement::storeFromItem($posted_item);
 		}

src/Model/Post.php

@@ -72,7 +72,7 @@ class Post
 				if (array_key_exists('title', $row)) {
 					$row['title'] = '';
 				}
-				if (array_key_exists('body', $row)) {
+				if (array_key_exists('body', $row) && empty($row['body'])) {
 					$row['body'] = $row['verb'];
 				}
 				if (array_key_exists('object', $row)) {

src/Model/Post/Counts.php

@@ -0,0 +1,89 @@
+<?php
+/**
+ * @copyright Copyright (C) 2010-2023, the Friendica project
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace Friendica\Model\Post;
+
+use Friendica\Content\Smilies;
+use Friendica\Database\Database;
+use Friendica\Database\DBA;
+use Friendica\Model\Item;
+use Friendica\Model\Post;
+use Friendica\Model\Verb;
+use Friendica\Protocol\Activity;
+
+class Counts
+{
+	/**
+	 * Insert or update a post-counts entry
+	 *
+	 * @param int $uri_id
+	 */
+	public static function update(int $uri_id, int $parent_uri_id, int $vid, string $verb, string $body = null)
+	{
+		$condition = ['thr-parent-id' => $uri_id, 'vid' => $vid];
+
+		if ($body == $verb) {
+			$condition['body'] = null;
+			$body              = '';
+		} elseif (($verb != Activity::POST) && (mb_strlen($body) == 1) && Smilies::isEmojiPost($body)) {
+			$condition['body'] = $body;
+		} else {
+			$body = '';
+		}
+
+		$fields = [
+			'uri-id'        => $uri_id,
+			'vid'           => $vid,
+			'reaction'      => $body,
+			'parent-uri-id' => $parent_uri_id,
+			'count'         => Post::countPosts($condition),
+		];
+
+		if ($fields['count'] == 0) {
+			return true;
+		}
+
+		return DBA::insert('post-counts', $fields, Database::INSERT_UPDATE);
+	}
+
+	public static function updateForPost(int $uri_id, int $parent_uri_id)
+	{
+		self::update($uri_id, $parent_uri_id, Verb::getID(Activity::POST), Activity::POST);
+
+		$activities = DBA::p("SELECT `parent-uri-id`, `vid`, `verb`, `body` FROM `post-view` WHERE `thr-parent-id` = ? AND `gravity` = ? GROUP BY `parent-uri-id`, `vid`, `verb`, `body`", $uri_id, Item::GRAVITY_ACTIVITY);
+		while ($activity = DBA::fetch($activities)) {
+			self::update($uri_id, $activity['parent-uri-id'], $activity['vid'], $activity['verb'], $activity['body']);
+		}
+		DBA::close($activities);
+	}
+
+	/**
+	 * Retrieves counts of the given uri-id
+	 *
+	 * @param int $uriId
+	 *
+	 * @return array
+	 */
+	public static function getByURIId(int $uriId): array
+	{
+		return DBA::selectToArray('post-counts', [], ['uri-id' => $uriId]);
+	}
+}

src/Protocol/ActivityPub/Receiver.php

@@ -1272,6 +1272,11 @@ class Receiver
 			}
 		}
 
+		if (empty($receivers) && !empty($parent['parent-author-link'])) {
+			$uid = User::getIdForURL($parent['parent-author-link']);
+			$receivers[$uid] = ['uid' => $uid, 'type' => self::TARGET_BTO];
+		}
+
 		if (!empty($reply) && (!empty($receivers[0]) || !empty($receivers[-1]))) {
 			$parents = Post::select(['uid'], DBA::mergeConditions(['uri' => $reply], ["`uid` != ?", 0]));
 			while ($parent = Post::fetch($parents)) {

static/dbstructure.config.php

@@ -56,7 +56,7 @@ use Friendica\Database\DBA;
 
 // This file is required several times during the test in DbaDefinition which justifies this condition
 if (!defined('DB_UPDATE_VERSION')) {
-	define('DB_UPDATE_VERSION', 1542);
+	define('DB_UPDATE_VERSION', 1543);
 }
 
 return [
@@ -1264,6 +1264,21 @@ return [
 			"uid_uri-id" => ["uid", "uri-id"],
 		]
 	],
+	"post-counts" => [
+		"comment" => "Original remote activity",
+		"fields" => [
+			"uri-id" => ["type" => "int unsigned", "not null" => "1", "primary" => "1", "foreign" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the item uri"],
+			"vid" => ["type" => "smallint unsigned", "not null" => "1", "primary" => "1", "foreign" => ["verb" => "id", "on delete" => "restrict"], "comment" => "Id of the verb table entry that contains the activity verbs"],
+			"reaction" => ["type" => "varchar(1)", "not null" => "1", "primary" => "1", "comment" => "Emoji Reaction"],
+			"parent-uri-id" => ["type" => "int unsigned", "foreign" => ["item-uri" => "id"], "comment" => "Id of the item-uri table that contains the parent uri"],
+			"count" => ["type" => "int unsigned", "default" => 0, "comment" => "Number of activities"],
+		],
+		"indexes" => [
+			"PRIMARY" => ["uri-id", "vid", "reaction"],
+			"vid" => ["vid"],
+			"parent-uri-id" => ["parent-uri-id"],
+		]
+	],
 	"post-collection" => [
 		"comment" => "Collection of posts",
 		"fields" => [