Merge pull request #14198 from annando/limited-search

Option to reduced search scope to improve the performance
This commit is contained in:
Tobias Diekershoff 2024-06-02 12:05:18 +02:00 committed by GitHub
commit 35beffc8a8
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 349 additions and 260 deletions

View file

@ -1,6 +1,6 @@
-- ------------------------------------------ -- ------------------------------------------
-- Friendica 2024.06-dev (Yellow Archangel) -- Friendica 2024.06-dev (Yellow Archangel)
-- DB_UPDATE_VERSION 1564 -- DB_UPDATE_VERSION 1565
-- ------------------------------------------ -- ------------------------------------------
@ -2099,6 +2099,38 @@ CREATE VIEW `post-counts-view` AS SELECT
FROM `post-counts` FROM `post-counts`
INNER JOIN `verb` ON `verb`.`id` = `post-counts`.`vid`; INNER JOIN `verb` ON `verb`.`id` = `post-counts`.`vid`;
--
-- VIEW post-engagement-user-view
--
DROP VIEW IF EXISTS `post-engagement-user-view`;
CREATE VIEW `post-engagement-user-view` AS SELECT
`post-thread-user`.`uid` AS `uid`,
`post-engagement`.`uri-id` AS `uri-id`,
`post-engagement`.`owner-id` AS `owner-id`,
`post-engagement`.`media-type` AS `media-type`,
`post-engagement`.`language` AS `language`,
`post-engagement`.`searchtext` AS `searchtext`,
`post-engagement`.`size` AS `size`,
`post-thread-user`.`commented` AS `commented`,
`post-thread-user`.`received` AS `received`,
`post-thread-user`.`created` AS `created`,
`post-thread-user`.`network` AS `network`,
`post-engagement`.`language` AS `restricted`,
0 AS `comments`,
0 AS `activities`
FROM `post-thread-user`
INNER JOIN `post-engagement` ON `post-engagement`.`uri-id` = `post-thread-user`.`uri-id`
INNER JOIN `post-user` ON `post-user`.`id` = `post-thread-user`.`post-user-id`
STRAIGHT_JOIN `contact` ON `contact`.`id` = `post-thread-user`.`contact-id`
STRAIGHT_JOIN `contact` AS `authorcontact` ON `authorcontact`.`id` = `post-thread-user`.`author-id`
STRAIGHT_JOIN `contact` AS `ownercontact` ON `ownercontact`.`id` = `post-thread-user`.`owner-id`
WHERE `post-user`.`visible` AND NOT `post-user`.`deleted`
AND (NOT `contact`.`readonly` AND NOT `contact`.`blocked` AND NOT `contact`.`pending`)
AND (`post-thread-user`.`hidden` IS NULL OR NOT `post-thread-user`.`hidden`)
AND NOT `authorcontact`.`blocked` AND NOT `ownercontact`.`blocked`
AND NOT EXISTS(SELECT `cid` FROM `user-contact` WHERE `uid` = `post-thread-user`.`uid` AND `cid` IN (`authorcontact`.`id`, `ownercontact`.`id`) AND (`blocked` OR `ignored`))
AND NOT EXISTS(SELECT `gsid` FROM `user-gserver` WHERE `uid` = `post-thread-user`.`uid` AND `gsid` IN (`authorcontact`.`gsid`, `ownercontact`.`gsid`) AND `ignored`);
-- --
-- VIEW post-timeline-view -- VIEW post-timeline-view
-- --

View file

@ -124,7 +124,7 @@ class Content
'limit' => [$start, $limit] 'limit' => [$start, $limit]
]; ];
$tags = DBA::select('post-searchindex', ['uri-id'], $condition, $params); $tags = DBA::select(SearchIndex::getSearchTable(), ['uri-id'], $condition, $params);
$uriids = []; $uriids = [];
while ($tag = DBA::fetch($tags)) { while ($tag = DBA::fetch($tags)) {
@ -143,6 +143,6 @@ class Content
} else { } else {
$condition = ["MATCH (`searchtext`) AGAINST (? IN BOOLEAN MODE) AND NOT `restricted", $search]; $condition = ["MATCH (`searchtext`) AGAINST (? IN BOOLEAN MODE) AND NOT `restricted", $search];
} }
return DBA::count('post-searchindex', $condition); return DBA::count(SearchIndex::getSearchTable(), $condition);
} }
} }

View file

@ -99,4 +99,14 @@ class SearchIndex
} }
return DateTimeFormat::utc('now - ' . $days . ' day'); return DateTimeFormat::utc('now - ' . $days . ' day');
} }
public static function getSearchTable(): string
{
return DI::config()->get('system', 'limited_search_scope') ? 'post-engagement' : 'post-searchindex';
}
public static function getSearchView(): string
{
return DI::config()->get('system', 'limited_search_scope') ? 'post-engagement-user-view' : 'post-searchindex-user-view';
}
} }

View file

@ -140,6 +140,7 @@ class Site extends BaseAdmin
$temppath = (!empty($_POST['temppath']) ? trim($_POST['temppath']) : ''); $temppath = (!empty($_POST['temppath']) ? trim($_POST['temppath']) : '');
$singleuser = (!empty($_POST['singleuser']) ? trim($_POST['singleuser']) : ''); $singleuser = (!empty($_POST['singleuser']) ? trim($_POST['singleuser']) : '');
$only_tag_search = !empty($_POST['only_tag_search']); $only_tag_search = !empty($_POST['only_tag_search']);
$limited_search_scope = !empty($_POST['limited_search_scope']);
$search_age_days = (!empty($_POST['search_age_days']) ? intval($_POST['search_age_days']) : 0); $search_age_days = (!empty($_POST['search_age_days']) ? intval($_POST['search_age_days']) : 0);
$compute_circle_counts = !empty($_POST['compute_circle_counts']); $compute_circle_counts = !empty($_POST['compute_circle_counts']);
$process_view = !empty($_POST['process_view']); $process_view = !empty($_POST['process_view']);
@ -317,6 +318,7 @@ class Site extends BaseAdmin
$transactionConfig->set('system', 'temppath', $temppath); $transactionConfig->set('system', 'temppath', $temppath);
$transactionConfig->set('system', 'only_tag_search', $only_tag_search); $transactionConfig->set('system', 'only_tag_search', $only_tag_search);
$transactionConfig->set('system', 'limited_search_scope', $limited_search_scope);
$transactionConfig->set('system', 'search_age_days', $search_age_days); $transactionConfig->set('system', 'search_age_days', $search_age_days);
$transactionConfig->set('system', 'compute_circle_counts', $compute_circle_counts); $transactionConfig->set('system', 'compute_circle_counts', $compute_circle_counts);
$transactionConfig->set('system', 'process_view', $process_view); $transactionConfig->set('system', 'process_view', $process_view);
@ -571,6 +573,7 @@ class Site extends BaseAdmin
'$itemspage_network_mobile' => ['itemspage_network_mobile', DI::l10n()->t('Items per page for mobile devices'), DI::config()->get('system', 'itemspage_network_mobile'), DI::l10n()->t('Number of items per page in stream pages (network, community, profile/contact statuses, search) for mobile devices.')], '$itemspage_network_mobile' => ['itemspage_network_mobile', DI::l10n()->t('Items per page for mobile devices'), DI::config()->get('system', 'itemspage_network_mobile'), DI::l10n()->t('Number of items per page in stream pages (network, community, profile/contact statuses, search) for mobile devices.')],
'$temppath' => ['temppath', DI::l10n()->t('Temp path'), DI::config()->get('system', 'temppath'), DI::l10n()->t('If you have a restricted system where the webserver can\'t access the system temp path, enter another path here.')], '$temppath' => ['temppath', DI::l10n()->t('Temp path'), DI::config()->get('system', 'temppath'), DI::l10n()->t('If you have a restricted system where the webserver can\'t access the system temp path, enter another path here.')],
'$only_tag_search' => ['only_tag_search', DI::l10n()->t('Only search in tags'), DI::config()->get('system', 'only_tag_search'), DI::l10n()->t('On large systems the text search can slow down the system extremely.')], '$only_tag_search' => ['only_tag_search', DI::l10n()->t('Only search in tags'), DI::config()->get('system', 'only_tag_search'), DI::l10n()->t('On large systems the text search can slow down the system extremely.')],
'$limited_search_scope' => ['limited_search_scope', DI::l10n()->t('Limited search scope'), DI::config()->get('system', 'limited_search_scope'), DI::l10n()->t('If enabled, searches will only be performed in the data used for the channels and not in all posts.')],
'$search_age_days' => ['search_age_days', DI::l10n()->t('Maximum age of items in the search table'), DI::config()->get('system', 'search_age_days'), DI::l10n()->t('Maximum age of items in the search table in days. Lower values will increase the performance and reduce disk usage. 0 means no age restriction.')], '$search_age_days' => ['search_age_days', DI::l10n()->t('Maximum age of items in the search table'), DI::config()->get('system', 'search_age_days'), DI::l10n()->t('Maximum age of items in the search table in days. Lower values will increase the performance and reduce disk usage. 0 means no age restriction.')],
'$compute_circle_counts' => ['compute_circle_counts', DI::l10n()->t('Generate counts per contact circle when calculating network count'), DI::config()->get('system', 'compute_circle_counts'), DI::l10n()->t('On systems with users that heavily use contact circles the query can be very expensive.')], '$compute_circle_counts' => ['compute_circle_counts', DI::l10n()->t('Generate counts per contact circle when calculating network count'), DI::config()->get('system', 'compute_circle_counts'), DI::l10n()->t('On systems with users that heavily use contact circles the query can be very expensive.')],
'$process_view' => ['process_view', DI::l10n()->t('Process "view" activities'), DI::config()->get('system', 'process_view'), DI::l10n()->t('"view" activities are mostly geberated by Peertube systems. Per default they are not processed for performance reasons. Only activate this option on performant system.')], '$process_view' => ['process_view', DI::l10n()->t('Process "view" activities'), DI::config()->get('system', 'process_view'), DI::l10n()->t('"view" activities are mostly geberated by Peertube systems. Per default they are not processed for performance reasons. Only activate this option on performant system.')],

View file

@ -28,6 +28,7 @@ use Friendica\DI;
use Friendica\Model\Contact; use Friendica\Model\Contact;
use Friendica\Model\Item; use Friendica\Model\Item;
use Friendica\Model\Post; use Friendica\Model\Post;
use Friendica\Model\Post\SearchIndex;
use Friendica\Model\Tag; use Friendica\Model\Tag;
use Friendica\Module\BaseApi; use Friendica\Module\BaseApi;
use Friendica\Util\Network; use Friendica\Util\Network;
@ -159,7 +160,7 @@ class Search extends BaseApi
} else { } else {
$q = Post\Engagement::escapeKeywords($q); $q = Post\Engagement::escapeKeywords($q);
$condition = ["MATCH (`searchtext`) AGAINST (? IN BOOLEAN MODE) AND (NOT `restricted` OR `uri-id` IN (SELECT `uri-id` FROM `post-user` WHERE `uid` = ?))", $q, $uid]; $condition = ["MATCH (`searchtext`) AGAINST (? IN BOOLEAN MODE) AND (NOT `restricted` OR `uri-id` IN (SELECT `uri-id` FROM `post-user` WHERE `uid` = ?))", $q, $uid];
$table = 'post-searchindex'; $table = SearchIndex::getSearchTable();
} }
if (!empty($account_id)) { if (!empty($account_id)) {

View file

@ -43,6 +43,7 @@ use Friendica\Database\DBA;
use Friendica\Model\Item; use Friendica\Model\Item;
use Friendica\Model\Post; use Friendica\Model\Post;
use Friendica\Model\Post\Engagement; use Friendica\Model\Post\Engagement;
use Friendica\Model\Post\SearchIndex;
use Friendica\Module\Response; use Friendica\Module\Response;
use Friendica\Protocol\Activity; use Friendica\Protocol\Activity;
use Friendica\Util\DateTimeFormat; use Friendica\Util\DateTimeFormat;
@ -381,7 +382,7 @@ class Timeline extends BaseModule
} elseif (is_numeric($this->selectedTab) && !empty($channel = $this->channelRepository->selectById($this->selectedTab, $uid))) { } elseif (is_numeric($this->selectedTab) && !empty($channel = $this->channelRepository->selectById($this->selectedTab, $uid))) {
$condition = $this->getUserChannelConditions($channel, $uid); $condition = $this->getUserChannelConditions($channel, $uid);
if (in_array($channel->circle, [-3, -4, -5])) { if (in_array($channel->circle, [-3, -4, -5])) {
$table = 'post-searchindex-user-view'; $table = SearchIndex::getSearchView();
$condition = DBA::mergeConditions($condition, ['uid' => $uid]); $condition = DBA::mergeConditions($condition, ['uid' => $uid]);
$orders = ['-3' => 'created', '-4' => 'received', '-5' => 'commented']; $orders = ['-3' => 'created', '-4' => 'received', '-5' => 'commented'];
$this->order = $orders[$channel->circle]; $this->order = $orders[$channel->circle];

View file

@ -56,7 +56,7 @@ use Friendica\Database\DBA;
// This file is required several times during the test in DbaDefinition which justifies this condition // This file is required several times during the test in DbaDefinition which justifies this condition
if (!defined('DB_UPDATE_VERSION')) { if (!defined('DB_UPDATE_VERSION')) {
define('DB_UPDATE_VERSION', 1564); define('DB_UPDATE_VERSION', 1565);
} }
return [ return [

View file

@ -100,6 +100,36 @@
"query" => "FROM `post-counts` "query" => "FROM `post-counts`
INNER JOIN `verb` ON `verb`.`id` = `post-counts`.`vid`" INNER JOIN `verb` ON `verb`.`id` = `post-counts`.`vid`"
], ],
"post-engagement-user-view" => [
"fields" => [
"uid" => ["post-thread-user", "uid"],
"uri-id" => ["post-engagement", "uri-id"],
"owner-id" => ["post-engagement", "owner-id"],
"media-type" => ["post-engagement", "media-type"],
"language" => ["post-engagement", "language"],
"searchtext" => ["post-engagement", "searchtext"],
"size" => ["post-engagement", "size"],
"commented" => ["post-thread-user", "commented"],
"received" => ["post-thread-user", "received"],
"created" => ["post-thread-user", "created"],
"network" => ["post-thread-user", "network"],
"restricted" => ["post-engagement", "language"],
"comments" => "0",
"activities" => "0",
],
"query" => "FROM `post-thread-user`
INNER JOIN `post-engagement` ON `post-engagement`.`uri-id` = `post-thread-user`.`uri-id`
INNER JOIN `post-user` ON `post-user`.`id` = `post-thread-user`.`post-user-id`
STRAIGHT_JOIN `contact` ON `contact`.`id` = `post-thread-user`.`contact-id`
STRAIGHT_JOIN `contact` AS `authorcontact` ON `authorcontact`.`id` = `post-thread-user`.`author-id`
STRAIGHT_JOIN `contact` AS `ownercontact` ON `ownercontact`.`id` = `post-thread-user`.`owner-id`
WHERE `post-user`.`visible` AND NOT `post-user`.`deleted`
AND (NOT `contact`.`readonly` AND NOT `contact`.`blocked` AND NOT `contact`.`pending`)
AND (`post-thread-user`.`hidden` IS NULL OR NOT `post-thread-user`.`hidden`)
AND NOT `authorcontact`.`blocked` AND NOT `ownercontact`.`blocked`
AND NOT EXISTS(SELECT `cid` FROM `user-contact` WHERE `uid` = `post-thread-user`.`uid` AND `cid` IN (`authorcontact`.`id`, `ownercontact`.`id`) AND (`blocked` OR `ignored`))
AND NOT EXISTS(SELECT `gsid` FROM `user-gserver` WHERE `uid` = `post-thread-user`.`uid` AND `gsid` IN (`authorcontact`.`gsid`, `ownercontact`.`gsid`) AND `ignored`)"
],
"post-timeline-view" => [ "post-timeline-view" => [
"fields" => [ "fields" => [
"uid" => ["post-user", "uid"], "uid" => ["post-user", "uid"],

File diff suppressed because it is too large Load diff

View file

@ -116,6 +116,7 @@
<h2>{{$performance}}</h2> <h2>{{$performance}}</h2>
{{include file="field_checkbox.tpl" field=$compute_circle_counts}} {{include file="field_checkbox.tpl" field=$compute_circle_counts}}
{{include file="field_checkbox.tpl" field=$only_tag_search}} {{include file="field_checkbox.tpl" field=$only_tag_search}}
{{include file="field_checkbox.tpl" field=$limited_search_scope}}
{{include file="field_input.tpl" field=$search_age_days}} {{include file="field_input.tpl" field=$search_age_days}}
{{include file="field_input.tpl" field=$max_comments}} {{include file="field_input.tpl" field=$max_comments}}
{{include file="field_input.tpl" field=$max_display_comments}} {{include file="field_input.tpl" field=$max_display_comments}}

View file

@ -250,6 +250,7 @@
<div class="panel-body"> <div class="panel-body">
{{include file="field_checkbox.tpl" field=$compute_circle_counts}} {{include file="field_checkbox.tpl" field=$compute_circle_counts}}
{{include file="field_checkbox.tpl" field=$only_tag_search}} {{include file="field_checkbox.tpl" field=$only_tag_search}}
{{include file="field_checkbox.tpl" field=$limited_search_scope}}
{{include file="field_input.tpl" field=$search_age_days}} {{include file="field_input.tpl" field=$search_age_days}}
{{include file="field_input.tpl" field=$max_comments}} {{include file="field_input.tpl" field=$max_comments}}
{{include file="field_input.tpl" field=$max_display_comments}} {{include file="field_input.tpl" field=$max_display_comments}}