From 40a126306621fe9eadb58101bd19a0be32e4c163 Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 28 Dec 2023 17:42:39 +0000 Subject: [PATCH] New table "post-counts" to precalculate the counts --- database.sql | 19 +++++- doc/database.md | 1 + doc/database/db_post-counts.md | 35 +++++++++++ src/Database/PostUpdate.php | 53 +++++++++++++++- src/Model/Item.php | 4 ++ src/Model/Post.php | 2 +- src/Model/Post/Counts.php | 90 +++++++++++++++++++++++++++ src/Protocol/ActivityPub/Receiver.php | 5 ++ static/dbstructure.config.php | 17 ++++- 9 files changed, 222 insertions(+), 4 deletions(-) create mode 100644 doc/database/db_post-counts.md create mode 100644 src/Model/Post/Counts.php diff --git a/database.sql b/database.sql index 985587e356..3c2fbb3150 100644 --- a/database.sql +++ b/database.sql @@ -1,6 +1,6 @@ -- ------------------------------------------ -- Friendica 2024.03-dev (Yellow Archangel) --- DB_UPDATE_VERSION 1542 +-- DB_UPDATE_VERSION 1543 -- ------------------------------------------ @@ -1236,6 +1236,23 @@ CREATE TABLE IF NOT EXISTS `post-category` ( FOREIGN KEY (`tid`) REFERENCES `tag` (`id`) ON UPDATE RESTRICT ON DELETE RESTRICT ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='post relation to categories'; +-- +-- TABLE post-counts +-- +CREATE TABLE IF NOT EXISTS `post-counts` ( + `uri-id` int unsigned NOT NULL COMMENT 'Id of the item-uri table entry that contains the item uri', + `vid` smallint unsigned NOT NULL COMMENT 'Id of the verb table entry that contains the activity verbs', + `reaction` varchar(1) NOT NULL COMMENT 'Emoji Reaction', + `parent-uri-id` int unsigned COMMENT 'Id of the item-uri table that contains the parent uri', + `count` int unsigned DEFAULT 0 COMMENT 'Number of activities', + PRIMARY KEY(`uri-id`,`vid`,`reaction`), + INDEX `vid` (`vid`), + INDEX `parent-uri-id` (`parent-uri-id`), + FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE, + FOREIGN KEY (`vid`) REFERENCES `verb` (`id`) ON UPDATE RESTRICT ON DELETE RESTRICT, + FOREIGN KEY (`parent-uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE +) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Original remote activity'; + -- -- TABLE post-collection -- diff --git a/doc/database.md b/doc/database.md index 26519ccfb4..f6b0406956 100644 --- a/doc/database.md +++ b/doc/database.md @@ -61,6 +61,7 @@ Database Tables | [post-category](help/database/db_post-category) | post relation to categories | | [post-collection](help/database/db_post-collection) | Collection of posts | | [post-content](help/database/db_post-content) | Content for all posts | +| [post-counts](help/database/db_post-counts) | Original remote activity | | [post-delivery](help/database/db_post-delivery) | Delivery data for posts for the batch processing | | [post-delivery-data](help/database/db_post-delivery-data) | Delivery data for items | | [post-engagement](help/database/db_post-engagement) | Engagement data per post | diff --git a/doc/database/db_post-counts.md b/doc/database/db_post-counts.md new file mode 100644 index 0000000000..86383bfc19 --- /dev/null +++ b/doc/database/db_post-counts.md @@ -0,0 +1,35 @@ +Table post-counts +=========== + +Original remote activity + +Fields +------ + +| Field | Description | Type | Null | Key | Default | Extra | +| ------------- | ----------------------------------------------------------- | ----------------- | ---- | --- | ------- | ----- | +| uri-id | Id of the item-uri table entry that contains the item uri | int unsigned | NO | PRI | NULL | | +| vid | Id of the verb table entry that contains the activity verbs | smallint unsigned | NO | PRI | NULL | | +| reaction | Emoji Reaction | varchar(1) | NO | PRI | NULL | | +| parent-uri-id | Id of the item-uri table that contains the parent uri | int unsigned | YES | | NULL | | +| count | Number of activities | int unsigned | YES | | 0 | | + +Indexes +------------ + +| Name | Fields | +| ------------- | --------------------- | +| PRIMARY | uri-id, vid, reaction | +| vid | vid | +| parent-uri-id | parent-uri-id | + +Foreign Keys +------------ + +| Field | Target Table | Target Field | +|-------|--------------|--------------| +| uri-id | [item-uri](help/database/db_item-uri) | id | +| vid | [verb](help/database/db_verb) | id | +| parent-uri-id | [item-uri](help/database/db_item-uri) | id | + +Return to [database documentation](help/database) diff --git a/src/Database/PostUpdate.php b/src/Database/PostUpdate.php index acc98c882c..ee8c060fa1 100644 --- a/src/Database/PostUpdate.php +++ b/src/Database/PostUpdate.php @@ -32,6 +32,7 @@ use Friendica\Model\ItemURI; use Friendica\Model\Photo; use Friendica\Model\Post; use Friendica\Model\Post\Category; +use Friendica\Model\Post\Counts; use Friendica\Model\Tag; use Friendica\Model\Verb; use Friendica\Protocol\ActivityPub\Processor; @@ -51,7 +52,7 @@ class PostUpdate // Needed for the helper function to read from the legacy term table const OBJECT_TYPE_POST = 1; - const VERSION = 1507; + const VERSION = 1543; /** * Calls the post update functions @@ -124,6 +125,9 @@ class PostUpdate if (!self::update1507()) { return false; } + if (!self::update1543()) { + return false; + } return true; } @@ -1303,4 +1307,51 @@ class PostUpdate return false; } + + /** + * Create "post-counts" entries for old entries. + * + * @return bool "true" when the job is done + * @throws \Friendica\Network\HTTPException\InternalServerErrorException + * @throws \ImagickException + */ + private static function update1543() + { + // Was the script completed? + if (DI::keyValue()->get('post_update_version') >= 1543) { + return true; + } + + $id = DI::keyValue()->get('post_update_version_1543_id') ?? 0; + + Logger::info('Start', ['uri-id' => $id]); + + $rows = 0; + + $posts = Post::selectPosts(['uri-id', 'parent-uri-id'], ["`uri-id` > ? AND `gravity` IN (?, ?)", $id, Item::GRAVITY_COMMENT, Item::GRAVITY_PARENT], ['order' => ['uri-id'], 'limit' => 1000]); + + if (DBA::errorNo() != 0) { + Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]); + return false; + } + + while ($post = Post::fetch($posts)) { + $id = $post['uri-id']; + Counts::updateForPost($post['uri-id'], $post['parent-uri-id']); + ++$rows; + } + DBA::close($posts); + + DI::keyValue()->set('post_update_version_1543_id', $id); + + Logger::info('Processed', ['rows' => $rows, 'last' => $id]); + + if ($rows <= 100) { + DI::keyValue()->set('post_update_version', 1543); + Logger::info('Done'); + return true; + } + + return false; + } } diff --git a/src/Model/Item.php b/src/Model/Item.php index 524c4facb4..b52f065b04 100644 --- a/src/Model/Item.php +++ b/src/Model/Item.php @@ -1432,6 +1432,10 @@ class Item self::updateDisplayCache($posted_item['uri-id']); } + if (in_array($posted_item['gravity'], [self::GRAVITY_ACTIVITY, self::GRAVITY_COMMENT]) && ($posted_item['uid'] == 0)) { + Post\Counts::update($posted_item['thr-parent-id'], $posted_item['parent-uri-id'], $posted_item['vid'], $posted_item['verb'], $posted_item['body']); + } + if ($inserted) { Post\Engagement::storeFromItem($posted_item); } diff --git a/src/Model/Post.php b/src/Model/Post.php index 2e8bbe22e3..03df5f9030 100644 --- a/src/Model/Post.php +++ b/src/Model/Post.php @@ -72,7 +72,7 @@ class Post if (array_key_exists('title', $row)) { $row['title'] = ''; } - if (array_key_exists('body', $row)) { + if (array_key_exists('body', $row) && empty($row['body'])) { $row['body'] = $row['verb']; } if (array_key_exists('object', $row)) { diff --git a/src/Model/Post/Counts.php b/src/Model/Post/Counts.php new file mode 100644 index 0000000000..bc617608ee --- /dev/null +++ b/src/Model/Post/Counts.php @@ -0,0 +1,90 @@ +. + * + */ + +namespace Friendica\Model\Post; + +use Friendica\Content\Smilies; +use Friendica\Core\Logger; +use Friendica\Database\Database; +use Friendica\Database\DBA; +use Friendica\Model\Item; +use Friendica\Model\Post; +use Friendica\Model\Verb; +use Friendica\Protocol\Activity; + +class Counts +{ + /** + * Insert or update a post-counts entry + * + * @param int $uri_id + */ + public static function update(int $uri_id, int $parent_uri_id, int $vid, string $verb, string $body = null) + { + $condition = ['thr-parent-id' => $uri_id, 'vid' => $vid]; + + if ($body == $verb) { + $condition['body'] = null; + $body = ''; + } elseif (($verb != Activity::POST) && (mb_strlen($body) == 1) && Smilies::isEmojiPost($body)) { + $condition['body'] = $body; + } else { + $body = ''; + } + + $fields = [ + 'uri-id' => $uri_id, + 'vid' => $vid, + 'reaction' => $body, + 'parent-uri-id' => $parent_uri_id, + 'count' => Post::countPosts($condition), + ]; + + if ($fields['count'] == 0) { + return true; + } + + return DBA::insert('post-counts', $fields, Database::INSERT_UPDATE); + } + + public static function updateForPost(int $uri_id, int $parent_uri_id) + { + self::update($uri_id, $parent_uri_id, Verb::getID(Activity::POST), Activity::POST); + + $activities = DBA::p("SELECT `parent-uri-id`, `vid`, `verb`, `body` FROM `post-view` WHERE `thr-parent-id` = ? AND `gravity` = ? GROUP BY `parent-uri-id`, `vid`, `verb`, `body`", $uri_id, Item::GRAVITY_ACTIVITY); + while ($activity = DBA::fetch($activities)) { + self::update($uri_id, $activity['parent-uri-id'], $activity['vid'], $activity['verb'], $activity['body']); + } + DBA::close($activities); + } + + /** + * Retrieves counts of the given uri-id + * + * @param int $uriId + * + * @return array + */ + public static function getByURIId(int $uriId): array + { + return DBA::selectToArray('post-counts', [], ['uri-id' => $uriId]); + } +} diff --git a/src/Protocol/ActivityPub/Receiver.php b/src/Protocol/ActivityPub/Receiver.php index 94233c8a89..738d1955ab 100644 --- a/src/Protocol/ActivityPub/Receiver.php +++ b/src/Protocol/ActivityPub/Receiver.php @@ -1272,6 +1272,11 @@ class Receiver } } + if (empty($receivers) && !empty($parent['parent-author-link'])) { + $uid = User::getIdForURL($parent['parent-author-link']); + $receivers[$uid] = ['uid' => $uid, 'type' => self::TARGET_BTO]; + } + if (!empty($reply) && (!empty($receivers[0]) || !empty($receivers[-1]))) { $parents = Post::select(['uid'], DBA::mergeConditions(['uri' => $reply], ["`uid` != ?", 0])); while ($parent = Post::fetch($parents)) { diff --git a/static/dbstructure.config.php b/static/dbstructure.config.php index 64c9bdff1f..d3963e205b 100644 --- a/static/dbstructure.config.php +++ b/static/dbstructure.config.php @@ -56,7 +56,7 @@ use Friendica\Database\DBA; // This file is required several times during the test in DbaDefinition which justifies this condition if (!defined('DB_UPDATE_VERSION')) { - define('DB_UPDATE_VERSION', 1542); + define('DB_UPDATE_VERSION', 1543); } return [ @@ -1264,6 +1264,21 @@ return [ "uid_uri-id" => ["uid", "uri-id"], ] ], + "post-counts" => [ + "comment" => "Original remote activity", + "fields" => [ + "uri-id" => ["type" => "int unsigned", "not null" => "1", "primary" => "1", "foreign" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the item uri"], + "vid" => ["type" => "smallint unsigned", "not null" => "1", "primary" => "1", "foreign" => ["verb" => "id", "on delete" => "restrict"], "comment" => "Id of the verb table entry that contains the activity verbs"], + "reaction" => ["type" => "varchar(1)", "not null" => "1", "primary" => "1", "comment" => "Emoji Reaction"], + "parent-uri-id" => ["type" => "int unsigned", "foreign" => ["item-uri" => "id"], "comment" => "Id of the item-uri table that contains the parent uri"], + "count" => ["type" => "int unsigned", "default" => 0, "comment" => "Number of activities"], + ], + "indexes" => [ + "PRIMARY" => ["uri-id", "vid", "reaction"], + "vid" => ["vid"], + "parent-uri-id" => ["parent-uri-id"], + ] + ], "post-collection" => [ "comment" => "Collection of posts", "fields" => [