mirror of
https://github.com/friendica/friendica
synced 2024-09-08 02:30:09 +00:00
Option to reduced search scope to improve the performance
This commit is contained in:
parent
d7e038a014
commit
259a676207
11 changed files with 349 additions and 260 deletions
34
database.sql
34
database.sql
|
@ -1,6 +1,6 @@
|
|||
-- ------------------------------------------
|
||||
-- Friendica 2024.06-dev (Yellow Archangel)
|
||||
-- DB_UPDATE_VERSION 1564
|
||||
-- DB_UPDATE_VERSION 1565
|
||||
-- ------------------------------------------
|
||||
|
||||
|
||||
|
@ -2099,6 +2099,38 @@ CREATE VIEW `post-counts-view` AS SELECT
|
|||
FROM `post-counts`
|
||||
INNER JOIN `verb` ON `verb`.`id` = `post-counts`.`vid`;
|
||||
|
||||
--
|
||||
-- VIEW post-engagement-user-view
|
||||
--
|
||||
DROP VIEW IF EXISTS `post-engagement-user-view`;
|
||||
CREATE VIEW `post-engagement-user-view` AS SELECT
|
||||
`post-thread-user`.`uid` AS `uid`,
|
||||
`post-engagement`.`uri-id` AS `uri-id`,
|
||||
`post-engagement`.`owner-id` AS `owner-id`,
|
||||
`post-engagement`.`media-type` AS `media-type`,
|
||||
`post-engagement`.`language` AS `language`,
|
||||
`post-engagement`.`searchtext` AS `searchtext`,
|
||||
`post-engagement`.`size` AS `size`,
|
||||
`post-thread-user`.`commented` AS `commented`,
|
||||
`post-thread-user`.`received` AS `received`,
|
||||
`post-thread-user`.`created` AS `created`,
|
||||
`post-thread-user`.`network` AS `network`,
|
||||
`post-engagement`.`language` AS `restricted`,
|
||||
0 AS `comments`,
|
||||
0 AS `activities`
|
||||
FROM `post-thread-user`
|
||||
INNER JOIN `post-engagement` ON `post-engagement`.`uri-id` = `post-thread-user`.`uri-id`
|
||||
INNER JOIN `post-user` ON `post-user`.`id` = `post-thread-user`.`post-user-id`
|
||||
STRAIGHT_JOIN `contact` ON `contact`.`id` = `post-thread-user`.`contact-id`
|
||||
STRAIGHT_JOIN `contact` AS `authorcontact` ON `authorcontact`.`id` = `post-thread-user`.`author-id`
|
||||
STRAIGHT_JOIN `contact` AS `ownercontact` ON `ownercontact`.`id` = `post-thread-user`.`owner-id`
|
||||
WHERE `post-user`.`visible` AND NOT `post-user`.`deleted`
|
||||
AND (NOT `contact`.`readonly` AND NOT `contact`.`blocked` AND NOT `contact`.`pending`)
|
||||
AND (`post-thread-user`.`hidden` IS NULL OR NOT `post-thread-user`.`hidden`)
|
||||
AND NOT `authorcontact`.`blocked` AND NOT `ownercontact`.`blocked`
|
||||
AND NOT EXISTS(SELECT `cid` FROM `user-contact` WHERE `uid` = `post-thread-user`.`uid` AND `cid` IN (`authorcontact`.`id`, `ownercontact`.`id`) AND (`blocked` OR `ignored`))
|
||||
AND NOT EXISTS(SELECT `gsid` FROM `user-gserver` WHERE `uid` = `post-thread-user`.`uid` AND `gsid` IN (`authorcontact`.`gsid`, `ownercontact`.`gsid`) AND `ignored`);
|
||||
|
||||
--
|
||||
-- VIEW post-timeline-view
|
||||
--
|
||||
|
|
|
@ -124,7 +124,7 @@ class Content
|
|||
'limit' => [$start, $limit]
|
||||
];
|
||||
|
||||
$tags = DBA::select('post-searchindex', ['uri-id'], $condition, $params);
|
||||
$tags = DBA::select(SearchIndex::getSearchTable(), ['uri-id'], $condition, $params);
|
||||
|
||||
$uriids = [];
|
||||
while ($tag = DBA::fetch($tags)) {
|
||||
|
@ -143,6 +143,6 @@ class Content
|
|||
} else {
|
||||
$condition = ["MATCH (`searchtext`) AGAINST (? IN BOOLEAN MODE) AND NOT `restricted", $search];
|
||||
}
|
||||
return DBA::count('post-searchindex', $condition);
|
||||
return DBA::count(SearchIndex::getSearchTable(), $condition);
|
||||
}
|
||||
}
|
||||
|
|
|
@ -99,4 +99,14 @@ class SearchIndex
|
|||
}
|
||||
return DateTimeFormat::utc('now - ' . $days . ' day');
|
||||
}
|
||||
|
||||
public static function getSearchTable(): string
|
||||
{
|
||||
return DI::config()->get('system', 'limited_search_scope') ? 'post-engagement' : 'post-searchindex';
|
||||
}
|
||||
|
||||
public static function getSearchView(): string
|
||||
{
|
||||
return DI::config()->get('system', 'limited_search_scope') ? 'post-engagement-user-view' : 'post-searchindex-user-view';
|
||||
}
|
||||
}
|
||||
|
|
|
@ -140,6 +140,7 @@ class Site extends BaseAdmin
|
|||
$temppath = (!empty($_POST['temppath']) ? trim($_POST['temppath']) : '');
|
||||
$singleuser = (!empty($_POST['singleuser']) ? trim($_POST['singleuser']) : '');
|
||||
$only_tag_search = !empty($_POST['only_tag_search']);
|
||||
$limited_search_scope = !empty($_POST['limited_search_scope']);
|
||||
$search_age_days = (!empty($_POST['search_age_days']) ? intval($_POST['search_age_days']) : 0);
|
||||
$compute_circle_counts = !empty($_POST['compute_circle_counts']);
|
||||
$process_view = !empty($_POST['process_view']);
|
||||
|
@ -317,6 +318,7 @@ class Site extends BaseAdmin
|
|||
$transactionConfig->set('system', 'temppath', $temppath);
|
||||
|
||||
$transactionConfig->set('system', 'only_tag_search', $only_tag_search);
|
||||
$transactionConfig->set('system', 'limited_search_scope', $limited_search_scope);
|
||||
$transactionConfig->set('system', 'search_age_days', $search_age_days);
|
||||
$transactionConfig->set('system', 'compute_circle_counts', $compute_circle_counts);
|
||||
$transactionConfig->set('system', 'process_view', $process_view);
|
||||
|
@ -571,6 +573,7 @@ class Site extends BaseAdmin
|
|||
'$itemspage_network_mobile' => ['itemspage_network_mobile', DI::l10n()->t('Items per page for mobile devices'), DI::config()->get('system', 'itemspage_network_mobile'), DI::l10n()->t('Number of items per page in stream pages (network, community, profile/contact statuses, search) for mobile devices.')],
|
||||
'$temppath' => ['temppath', DI::l10n()->t('Temp path'), DI::config()->get('system', 'temppath'), DI::l10n()->t('If you have a restricted system where the webserver can\'t access the system temp path, enter another path here.')],
|
||||
'$only_tag_search' => ['only_tag_search', DI::l10n()->t('Only search in tags'), DI::config()->get('system', 'only_tag_search'), DI::l10n()->t('On large systems the text search can slow down the system extremely.')],
|
||||
'$limited_search_scope' => ['limited_search_scope', DI::l10n()->t('Limited search scope'), DI::config()->get('system', 'limited_search_scope'), DI::l10n()->t('If enabled, searches will only be performed in the data used for the channels and not in all posts.')],
|
||||
'$search_age_days' => ['search_age_days', DI::l10n()->t('Maximum age of items in the search table'), DI::config()->get('system', 'search_age_days'), DI::l10n()->t('Maximum age of items in the search table in days. Lower values will increase the performance and reduce disk usage. 0 means no age restriction.')],
|
||||
'$compute_circle_counts' => ['compute_circle_counts', DI::l10n()->t('Generate counts per contact circle when calculating network count'), DI::config()->get('system', 'compute_circle_counts'), DI::l10n()->t('On systems with users that heavily use contact circles the query can be very expensive.')],
|
||||
'$process_view' => ['process_view', DI::l10n()->t('Process "view" activities'), DI::config()->get('system', 'process_view'), DI::l10n()->t('"view" activities are mostly geberated by Peertube systems. Per default they are not processed for performance reasons. Only activate this option on performant system.')],
|
||||
|
|
|
@ -28,6 +28,7 @@ use Friendica\DI;
|
|||
use Friendica\Model\Contact;
|
||||
use Friendica\Model\Item;
|
||||
use Friendica\Model\Post;
|
||||
use Friendica\Model\Post\SearchIndex;
|
||||
use Friendica\Model\Tag;
|
||||
use Friendica\Module\BaseApi;
|
||||
use Friendica\Util\Network;
|
||||
|
@ -159,7 +160,7 @@ class Search extends BaseApi
|
|||
} else {
|
||||
$q = Post\Engagement::escapeKeywords($q);
|
||||
$condition = ["MATCH (`searchtext`) AGAINST (? IN BOOLEAN MODE) AND (NOT `restricted` OR `uri-id` IN (SELECT `uri-id` FROM `post-user` WHERE `uid` = ?))", $q, $uid];
|
||||
$table = 'post-searchindex';
|
||||
$table = SearchIndex::getSearchTable();
|
||||
}
|
||||
|
||||
if (!empty($account_id)) {
|
||||
|
|
|
@ -43,6 +43,7 @@ use Friendica\Database\DBA;
|
|||
use Friendica\Model\Item;
|
||||
use Friendica\Model\Post;
|
||||
use Friendica\Model\Post\Engagement;
|
||||
use Friendica\Model\Post\SearchIndex;
|
||||
use Friendica\Module\Response;
|
||||
use Friendica\Protocol\Activity;
|
||||
use Friendica\Util\DateTimeFormat;
|
||||
|
@ -381,7 +382,7 @@ class Timeline extends BaseModule
|
|||
} elseif (is_numeric($this->selectedTab) && !empty($channel = $this->channelRepository->selectById($this->selectedTab, $uid))) {
|
||||
$condition = $this->getUserChannelConditions($channel, $uid);
|
||||
if (in_array($channel->circle, [-3, -4, -5])) {
|
||||
$table = 'post-searchindex-user-view';
|
||||
$table = SearchIndex::getSearchView();
|
||||
$condition = DBA::mergeConditions($condition, ['uid' => $uid]);
|
||||
$orders = ['-3' => 'created', '-4' => 'received', '-5' => 'commented'];
|
||||
$this->order = $orders[$channel->circle];
|
||||
|
|
|
@ -56,7 +56,7 @@ use Friendica\Database\DBA;
|
|||
|
||||
// This file is required several times during the test in DbaDefinition which justifies this condition
|
||||
if (!defined('DB_UPDATE_VERSION')) {
|
||||
define('DB_UPDATE_VERSION', 1564);
|
||||
define('DB_UPDATE_VERSION', 1565);
|
||||
}
|
||||
|
||||
return [
|
||||
|
|
|
@ -100,6 +100,36 @@
|
|||
"query" => "FROM `post-counts`
|
||||
INNER JOIN `verb` ON `verb`.`id` = `post-counts`.`vid`"
|
||||
],
|
||||
"post-engagement-user-view" => [
|
||||
"fields" => [
|
||||
"uid" => ["post-thread-user", "uid"],
|
||||
"uri-id" => ["post-engagement", "uri-id"],
|
||||
"owner-id" => ["post-engagement", "owner-id"],
|
||||
"media-type" => ["post-engagement", "media-type"],
|
||||
"language" => ["post-engagement", "language"],
|
||||
"searchtext" => ["post-engagement", "searchtext"],
|
||||
"size" => ["post-engagement", "size"],
|
||||
"commented" => ["post-thread-user", "commented"],
|
||||
"received" => ["post-thread-user", "received"],
|
||||
"created" => ["post-thread-user", "created"],
|
||||
"network" => ["post-thread-user", "network"],
|
||||
"restricted" => ["post-engagement", "language"],
|
||||
"comments" => "0",
|
||||
"activities" => "0",
|
||||
],
|
||||
"query" => "FROM `post-thread-user`
|
||||
INNER JOIN `post-engagement` ON `post-engagement`.`uri-id` = `post-thread-user`.`uri-id`
|
||||
INNER JOIN `post-user` ON `post-user`.`id` = `post-thread-user`.`post-user-id`
|
||||
STRAIGHT_JOIN `contact` ON `contact`.`id` = `post-thread-user`.`contact-id`
|
||||
STRAIGHT_JOIN `contact` AS `authorcontact` ON `authorcontact`.`id` = `post-thread-user`.`author-id`
|
||||
STRAIGHT_JOIN `contact` AS `ownercontact` ON `ownercontact`.`id` = `post-thread-user`.`owner-id`
|
||||
WHERE `post-user`.`visible` AND NOT `post-user`.`deleted`
|
||||
AND (NOT `contact`.`readonly` AND NOT `contact`.`blocked` AND NOT `contact`.`pending`)
|
||||
AND (`post-thread-user`.`hidden` IS NULL OR NOT `post-thread-user`.`hidden`)
|
||||
AND NOT `authorcontact`.`blocked` AND NOT `ownercontact`.`blocked`
|
||||
AND NOT EXISTS(SELECT `cid` FROM `user-contact` WHERE `uid` = `post-thread-user`.`uid` AND `cid` IN (`authorcontact`.`id`, `ownercontact`.`id`) AND (`blocked` OR `ignored`))
|
||||
AND NOT EXISTS(SELECT `gsid` FROM `user-gserver` WHERE `uid` = `post-thread-user`.`uid` AND `gsid` IN (`authorcontact`.`gsid`, `ownercontact`.`gsid`) AND `ignored`)"
|
||||
],
|
||||
"post-timeline-view" => [
|
||||
"fields" => [
|
||||
"uid" => ["post-user", "uid"],
|
||||
|
|
File diff suppressed because it is too large
Load diff
|
@ -116,6 +116,7 @@
|
|||
<h2>{{$performance}}</h2>
|
||||
{{include file="field_checkbox.tpl" field=$compute_circle_counts}}
|
||||
{{include file="field_checkbox.tpl" field=$only_tag_search}}
|
||||
{{include file="field_checkbox.tpl" field=$limited_search_scope}}
|
||||
{{include file="field_input.tpl" field=$search_age_days}}
|
||||
{{include file="field_input.tpl" field=$max_comments}}
|
||||
{{include file="field_input.tpl" field=$max_display_comments}}
|
||||
|
|
|
@ -250,6 +250,7 @@
|
|||
<div class="panel-body">
|
||||
{{include file="field_checkbox.tpl" field=$compute_circle_counts}}
|
||||
{{include file="field_checkbox.tpl" field=$only_tag_search}}
|
||||
{{include file="field_checkbox.tpl" field=$limited_search_scope}}
|
||||
{{include file="field_input.tpl" field=$search_age_days}}
|
||||
{{include file="field_input.tpl" field=$max_comments}}
|
||||
{{include file="field_input.tpl" field=$max_display_comments}}
|
||||
|
|
Loading…
Reference in a new issue