From 919f97c9a0b3cf6c28b702d748f967d0c0b77df6 Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 17 Jan 2024 21:10:33 +0000 Subject: [PATCH] Postupdate added --- database.sql | 15 +++++++- doc/database.md | 1 + doc/database/db_post-searchindex.md | 31 +++++++++++++++ src/Database/PostUpdate.php | 56 +++++++++++++++++++++++++++- src/Model/Item.php | 13 ++----- src/Model/Post/SearchIndex.php | 58 +++++++++++++++++++++++++++++ 6 files changed, 162 insertions(+), 12 deletions(-) create mode 100644 doc/database/db_post-searchindex.md create mode 100644 src/Model/Post/SearchIndex.php diff --git a/database.sql b/database.sql index 51df6a732c..e3397a44aa 100644 --- a/database.sql +++ b/database.sql @@ -1,6 +1,6 @@ -- ------------------------------------------ -- Friendica 2024.03-dev (Yellow Archangel) --- DB_UPDATE_VERSION 1546 +-- DB_UPDATE_VERSION 1547 -- ------------------------------------------ @@ -1460,6 +1460,19 @@ CREATE TABLE IF NOT EXISTS `post-question-option` ( FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Question option'; +-- +-- TABLE post-searchindex +-- +CREATE TABLE IF NOT EXISTS `post-searchindex` ( + `uri-id` int unsigned NOT NULL COMMENT 'Id of the item-uri table entry that contains the item uri', + `network` char(4) COMMENT '', + `private` tinyint unsigned COMMENT '0=public, 1=private, 2=unlisted', + `searchtext` mediumtext COMMENT 'Simplified text for the full text search', + PRIMARY KEY(`uri-id`), + FULLTEXT INDEX `searchtext` (`searchtext`), + FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE +) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Content for all posts'; + -- -- TABLE post-tag -- diff --git a/doc/database.md b/doc/database.md index f6b0406956..5942b1144d 100644 --- a/doc/database.md +++ b/doc/database.md @@ -70,6 +70,7 @@ Database Tables | [post-media](help/database/db_post-media) | Attached media | | [post-question](help/database/db_post-question) | Question | | [post-question-option](help/database/db_post-question-option) | Question option | +| [post-searchindex](help/database/db_post-searchindex) | Content for all posts | | [post-tag](help/database/db_post-tag) | post relation to tags | | [post-thread](help/database/db_post-thread) | Thread related data | | [post-thread-user](help/database/db_post-thread-user) | Thread related data per user | diff --git a/doc/database/db_post-searchindex.md b/doc/database/db_post-searchindex.md new file mode 100644 index 0000000000..a5dec8de1b --- /dev/null +++ b/doc/database/db_post-searchindex.md @@ -0,0 +1,31 @@ +Table post-searchindex +=========== + +Content for all posts + +Fields +------ + +| Field | Description | Type | Null | Key | Default | Extra | +| ---------- | --------------------------------------------------------- | ---------------- | ---- | --- | ------- | ----- | +| uri-id | Id of the item-uri table entry that contains the item uri | int unsigned | NO | PRI | NULL | | +| network | | char(4) | YES | | NULL | | +| private | 0=public, 1=private, 2=unlisted | tinyint unsigned | YES | | NULL | | +| searchtext | Simplified text for the full text search | mediumtext | YES | | NULL | | + +Indexes +------------ + +| Name | Fields | +| ---------- | -------------------- | +| PRIMARY | uri-id | +| searchtext | FULLTEXT, searchtext | + +Foreign Keys +------------ + +| Field | Target Table | Target Field | +|-------|--------------|--------------| +| uri-id | [item-uri](help/database/db_item-uri) | id | + +Return to [database documentation](help/database) diff --git a/src/Database/PostUpdate.php b/src/Database/PostUpdate.php index 1cf8bc1ddb..0a373d88c5 100644 --- a/src/Database/PostUpdate.php +++ b/src/Database/PostUpdate.php @@ -52,7 +52,7 @@ class PostUpdate // Needed for the helper function to read from the legacy term table const OBJECT_TYPE_POST = 1; - const VERSION = 1544; + const VERSION = 1547; /** * Calls the post update functions @@ -128,6 +128,9 @@ class PostUpdate if (!self::update1544()) { return false; } + if (!self::update1547()) { + return false; + } return true; } @@ -1358,4 +1361,55 @@ class PostUpdate return false; } + + /** + * Create "post-searchindex" entries for old entries. + * + * @return bool "true" when the job is done + * @throws \Friendica\Network\HTTPException\InternalServerErrorException + * @throws \ImagickException + */ + private static function update1547() + { + // Was the script completed? + if (DI::keyValue()->get('post_update_version') >= 1547) { + return true; + } + + $id = (int)(DI::keyValue()->get('post_update_version_1547_id') ?? 0); + if ($id == 0) { + $post = Post::selectFirstPost(['uri-id'], [], ['order' => ['uri-id' => true]]); + $id = (int)($post['uri-id'] ?? 0); + } + + Logger::info('Start', ['uri-id' => $id]); + + $rows = 0; + + $posts = Post::selectPosts(['uri-id', 'network', 'private'], ["`uri-id` < ? AND `gravity` IN (?, ?)", $id, Item::GRAVITY_COMMENT, Item::GRAVITY_PARENT], ['order' => ['uri-id' => true], 'limit' => 1000]); + + if (DBA::errorNo() != 0) { + Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]); + return false; + } + + while ($post = Post::fetch($posts)) { + $id = $post['uri-id']; + Post\SearchIndex::insert($post['uri-id'], $post['network'], $post['private']); + ++$rows; + } + DBA::close($posts); + + DI::keyValue()->set('post_update_version_1547_id', $id); + + Logger::info('Processed', ['rows' => $rows, 'last' => $id]); + + if ($rows <= 100) { + DI::keyValue()->set('post_update_version', 1547); + Logger::info('Done'); + return true; + } + + return false; + } } diff --git a/src/Model/Item.php b/src/Model/Item.php index d28009a4e6..1637481554 100644 --- a/src/Model/Item.php +++ b/src/Model/Item.php @@ -247,8 +247,7 @@ class Item $searchtext = Post\Engagement::getSearchTextForUriId($item['uri-id'], true); DBA::update('post-engagement', ['searchtext' => $searchtext], ['uri-id' => $item['uri-id']]); - DBA::update('post-searchindex', ['searchtext' => $searchtext], ['uri-id' => $item['uri-id']]); - + Post\SearchIndex::update($item['uri-id']); } if (!empty($fields['file'])) { @@ -1450,14 +1449,8 @@ class Item $engagement_uri_id = Post\Engagement::storeFromItem($posted_item); - if (in_array($item['gravity'], [self::GRAVITY_PARENT, self::GRAVITY_COMMENT])) { - $search = [ - 'uri-id' => $posted_item['uri-id'], - 'network' => $posted_item['network'], - 'private' => $posted_item['private'], - 'searchtext' => Post\Engagement::getSearchTextForUriId($posted_item['uri-id']), - ]; - DBA::insert('post-searchindex', $search, Database::INSERT_IGNORE); + if (in_array($posted_item['gravity'], [self::GRAVITY_PARENT, self::GRAVITY_COMMENT])) { + Post\SearchIndex::insert($posted_item['uri-id'], $posted_item['network'], $posted_item['private']); } if (($posted_item['gravity'] == self::GRAVITY_ACTIVITY) && ($posted_item['verb'] == Activity::ANNOUNCE) && ($posted_item['parent-uri-id'] == $posted_item['thr-parent-id'])) { diff --git a/src/Model/Post/SearchIndex.php b/src/Model/Post/SearchIndex.php new file mode 100644 index 0000000000..3cc2ae2f38 --- /dev/null +++ b/src/Model/Post/SearchIndex.php @@ -0,0 +1,58 @@ +. + * + */ + +namespace Friendica\Model\Post; + +use Friendica\Database\Database; +use Friendica\Database\DBA; +use Friendica\Model\Post; + +class SearchIndex +{ + /** + * Insert a post-searchindex entry + * + * @param int $uri_id + * @param string $network + * @param int $private + */ + public static function insert(int $uri_id, string $network, int $private) + { + $search = [ + 'uri-id' => $uri_id, + 'network' => $network, + 'private' => $private, + 'searchtext' => Post\Engagement::getSearchTextForUriId($uri_id), + ]; + return DBA::insert('post-searchindex', $search, Database::INSERT_UPDATE); + } + + /** + * update a post-searchindex entry + * + * @param int $uri_id + */ + public static function update(int $uri_id) + { + $searchtext = Post\Engagement::getSearchTextForUriId($uri_id, true); + return DBA::update('post-searchindex', ['searchtext' => $searchtext], ['uri-id' => $uri_id]); + } +}