Friendica/friendica-github
1
0
Fork 0
mirror of https://github.com/friendica/friendica synced 2024-12-22 18:00:16 +00:00
Code Issues Projects Releases Packages Wiki Activity

Option to reduced search scope to improve the performance

Browse source
This commit is contained in:
Michael 2024-06-02 09:27:05 +00:00
parent d7e038a014
commit 259a676207
11 changed files with 349 additions and 260 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

34
database.sql
View file

@ -1,6 +1,6 @@
-- ------------------------------------------
-- Friendica 2024.06-dev (Yellow Archangel)
-- DB_UPDATE_VERSION 1564
-- DB_UPDATE_VERSION 1565
-- ------------------------------------------
@ -2099,6 +2099,38 @@ CREATE VIEW `post-counts-view` AS SELECT
FROM `post-counts`
INNER JOIN `verb` ON `verb`.`id` = `post-counts`.`vid`;
--
-- VIEW post-engagement-user-view
--
DROP VIEW IF EXISTS `post-engagement-user-view`;
CREATE VIEW `post-engagement-user-view` AS SELECT
`post-thread-user`.`uid` AS `uid`,
`post-engagement`.`uri-id` AS `uri-id`,
`post-engagement`.`owner-id` AS `owner-id`,
`post-engagement`.`media-type` AS `media-type`,
`post-engagement`.`language` AS `language`,
`post-engagement`.`searchtext` AS `searchtext`,
`post-engagement`.`size` AS `size`,
`post-thread-user`.`commented` AS `commented`,
`post-thread-user`.`received` AS `received`,
`post-thread-user`.`created` AS `created`,
`post-thread-user`.`network` AS `network`,
`post-engagement`.`language` AS `restricted`,
0 AS `comments`,
0 AS `activities`
FROM `post-thread-user`
INNER JOIN `post-engagement` ON `post-engagement`.`uri-id` = `post-thread-user`.`uri-id`
INNER JOIN `post-user` ON `post-user`.`id` = `post-thread-user`.`post-user-id`
STRAIGHT_JOIN `contact` ON `contact`.`id` = `post-thread-user`.`contact-id`
STRAIGHT_JOIN `contact` AS `authorcontact` ON `authorcontact`.`id` = `post-thread-user`.`author-id`
STRAIGHT_JOIN `contact` AS `ownercontact` ON `ownercontact`.`id` = `post-thread-user`.`owner-id`
WHERE `post-user`.`visible` AND NOT `post-user`.`deleted`
AND (NOT `contact`.`readonly` AND NOT `contact`.`blocked` AND NOT `contact`.`pending`)
AND (`post-thread-user`.`hidden` IS NULL OR NOT `post-thread-user`.`hidden`)
AND NOT `authorcontact`.`blocked` AND NOT `ownercontact`.`blocked`
AND NOT EXISTS(SELECT `cid` FROM `user-contact` WHERE `uid` = `post-thread-user`.`uid` AND `cid` IN (`authorcontact`.`id`, `ownercontact`.`id`) AND (`blocked` OR `ignored`))
AND NOT EXISTS(SELECT `gsid` FROM `user-gserver` WHERE `uid` = `post-thread-user`.`uid` AND `gsid` IN (`authorcontact`.`gsid`, `ownercontact`.`gsid`) AND `ignored`);
--
-- VIEW post-timeline-view
--

4
src/Model/Post/Content.php
View file

@ -124,7 +124,7 @@ class Content
'limit' => [$start, $limit]
];
$tags = DBA::select('post-searchindex', ['uri-id'], $condition, $params);
$tags = DBA::select(SearchIndex::getSearchTable(), ['uri-id'], $condition, $params);
$uriids = [];
while ($tag = DBA::fetch($tags)) {
@ -143,6 +143,6 @@ class Content
} else {
$condition = ["MATCH (`searchtext`) AGAINST (? IN BOOLEAN MODE) AND NOT `restricted", $search];
}
return DBA::count('post-searchindex', $condition);
return DBA::count(SearchIndex::getSearchTable(), $condition);
}
}

10
src/Model/Post/SearchIndex.php
View file

@ -99,4 +99,14 @@ class SearchIndex
}
return DateTimeFormat::utc('now - ' . $days . ' day');
}
public static function getSearchTable(): string
{
return DI::config()->get('system', 'limited_search_scope') ? 'post-engagement' : 'post-searchindex';
}
public static function getSearchView(): string
{
return DI::config()->get('system', 'limited_search_scope') ? 'post-engagement-user-view' : 'post-searchindex-user-view';
}
}

3
src/Module/Admin/Site.php
View file

@ -140,6 +140,7 @@ class Site extends BaseAdmin
$temppath = (!empty($_POST['temppath']) ? trim($_POST['temppath']) : '');
$singleuser = (!empty($_POST['singleuser']) ? trim($_POST['singleuser']) : '');
$only_tag_search = !empty($_POST['only_tag_search']);
$limited_search_scope = !empty($_POST['limited_search_scope']);
$search_age_days = (!empty($_POST['search_age_days']) ? intval($_POST['search_age_days']) : 0);
$compute_circle_counts = !empty($_POST['compute_circle_counts']);
$process_view = !empty($_POST['process_view']);
@ -317,6 +318,7 @@ class Site extends BaseAdmin
$transactionConfig->set('system', 'temppath', $temppath);
$transactionConfig->set('system', 'only_tag_search', $only_tag_search);
$transactionConfig->set('system', 'limited_search_scope', $limited_search_scope);
$transactionConfig->set('system', 'search_age_days', $search_age_days);
$transactionConfig->set('system', 'compute_circle_counts', $compute_circle_counts);
$transactionConfig->set('system', 'process_view', $process_view);
@ -571,6 +573,7 @@ class Site extends BaseAdmin
'$itemspage_network_mobile' => ['itemspage_network_mobile', DI::l10n()->t('Items per page for mobile devices'), DI::config()->get('system', 'itemspage_network_mobile'), DI::l10n()->t('Number of items per page in stream pages (network, community, profile/contact statuses, search) for mobile devices.')],
'$temppath' => ['temppath', DI::l10n()->t('Temp path'), DI::config()->get('system', 'temppath'), DI::l10n()->t('If you have a restricted system where the webserver can\'t access the system temp path, enter another path here.')],
'$only_tag_search' => ['only_tag_search', DI::l10n()->t('Only search in tags'), DI::config()->get('system', 'only_tag_search'), DI::l10n()->t('On large systems the text search can slow down the system extremely.')],
'$limited_search_scope' => ['limited_search_scope', DI::l10n()->t('Limited search scope'), DI::config()->get('system', 'limited_search_scope'), DI::l10n()->t('If enabled, searches will only be performed in the data used for the channels and not in all posts.')],
'$search_age_days' => ['search_age_days', DI::l10n()->t('Maximum age of items in the search table'), DI::config()->get('system', 'search_age_days'), DI::l10n()->t('Maximum age of items in the search table in days. Lower values will increase the performance and reduce disk usage. 0 means no age restriction.')],
'$compute_circle_counts' => ['compute_circle_counts', DI::l10n()->t('Generate counts per contact circle when calculating network count'), DI::config()->get('system', 'compute_circle_counts'), DI::l10n()->t('On systems with users that heavily use contact circles the query can be very expensive.')],
'$process_view' => ['process_view', DI::l10n()->t('Process "view" activities'), DI::config()->get('system', 'process_view'), DI::l10n()->t('"view" activities are mostly geberated by Peertube systems. Per default they are not processed for performance reasons. Only activate this option on performant system.')],

3
src/Module/Api/Mastodon/Search.php
View file

@ -28,6 +28,7 @@ use Friendica\DI;
use Friendica\Model\Contact;
use Friendica\Model\Item;
use Friendica\Model\Post;
use Friendica\Model\Post\SearchIndex;
use Friendica\Model\Tag;
use Friendica\Module\BaseApi;
use Friendica\Util\Network;
@ -159,7 +160,7 @@ class Search extends BaseApi
} else {
$q = Post\Engagement::escapeKeywords($q);
$condition = ["MATCH (`searchtext`) AGAINST (? IN BOOLEAN MODE) AND (NOT `restricted` OR `uri-id` IN (SELECT `uri-id` FROM `post-user` WHERE `uid` = ?))", $q, $uid];
$table = 'post-searchindex';
$table = SearchIndex::getSearchTable();
}
if (!empty($account_id)) {

3
src/Module/Conversation/Timeline.php
View file

@ -43,6 +43,7 @@ use Friendica\Database\DBA;
use Friendica\Model\Item;
use Friendica\Model\Post;
use Friendica\Model\Post\Engagement;
use Friendica\Model\Post\SearchIndex;
use Friendica\Module\Response;
use Friendica\Protocol\Activity;
use Friendica\Util\DateTimeFormat;
@ -381,7 +382,7 @@ class Timeline extends BaseModule
} elseif (is_numeric($this->selectedTab) && !empty($channel = $this->channelRepository->selectById($this->selectedTab, $uid))) {
$condition = $this->getUserChannelConditions($channel, $uid);
if (in_array($channel->circle, [-3, -4, -5])) {
$table = 'post-searchindex-user-view';
$table = SearchIndex::getSearchView();
$condition = DBA::mergeConditions($condition, ['uid' => $uid]);
$orders = ['-3' => 'created', '-4' => 'received', '-5' => 'commented'];
$this->order = $orders[$channel->circle];

2
static/dbstructure.config.php
View file

@ -56,7 +56,7 @@ use Friendica\Database\DBA;
// This file is required several times during the test in DbaDefinition which justifies this condition
if (!defined('DB_UPDATE_VERSION')) {
define('DB_UPDATE_VERSION', 1564);
define('DB_UPDATE_VERSION', 1565);
}
return [

30
static/dbview.config.php
View file

@ -100,6 +100,36 @@
"query" => "FROM `post-counts`
INNER JOIN `verb` ON `verb`.`id` = `post-counts`.`vid`"
],
"post-engagement-user-view" => [
"fields" => [
"uid" => ["post-thread-user", "uid"],
"uri-id" => ["post-engagement", "uri-id"],
"owner-id" => ["post-engagement", "owner-id"],
"media-type" => ["post-engagement", "media-type"],
"language" => ["post-engagement", "language"],
"searchtext" => ["post-engagement", "searchtext"],
"size" => ["post-engagement", "size"],
"commented" => ["post-thread-user", "commented"],
"received" => ["post-thread-user", "received"],
"created" => ["post-thread-user", "created"],
"network" => ["post-thread-user", "network"],
"restricted" => ["post-engagement", "language"],
"comments" => "0",
"activities" => "0",
],
"query" => "FROM `post-thread-user`
INNER JOIN `post-engagement` ON `post-engagement`.`uri-id` = `post-thread-user`.`uri-id`
INNER JOIN `post-user` ON `post-user`.`id` = `post-thread-user`.`post-user-id`
STRAIGHT_JOIN `contact` ON `contact`.`id` = `post-thread-user`.`contact-id`
STRAIGHT_JOIN `contact` AS `authorcontact` ON `authorcontact`.`id` = `post-thread-user`.`author-id`
STRAIGHT_JOIN `contact` AS `ownercontact` ON `ownercontact`.`id` = `post-thread-user`.`owner-id`
WHERE `post-user`.`visible` AND NOT `post-user`.`deleted`
AND (NOT `contact`.`readonly` AND NOT `contact`.`blocked` AND NOT `contact`.`pending`)
AND (`post-thread-user`.`hidden` IS NULL OR NOT `post-thread-user`.`hidden`)
AND NOT `authorcontact`.`blocked` AND NOT `ownercontact`.`blocked`
AND NOT EXISTS(SELECT `cid` FROM `user-contact` WHERE `uid` = `post-thread-user`.`uid` AND `cid` IN (`authorcontact`.`id`, `ownercontact`.`id`) AND (`blocked` OR `ignored`))
AND NOT EXISTS(SELECT `gsid` FROM `user-gserver` WHERE `uid` = `post-thread-user`.`uid` AND `gsid` IN (`authorcontact`.`gsid`, `ownercontact`.`gsid`) AND `ignored`)"
],
"post-timeline-view" => [
"fields" => [
"uid" => ["post-user", "uid"],

518
view/lang/C/messages.po
View file

File diff suppressed because it is too large Load diff

1
view/templates/admin/site.tpl
View file

@ -116,6 +116,7 @@
<h2>{{$performance}}</h2>
{{include file="field_checkbox.tpl" field=$compute_circle_counts}}
{{include file="field_checkbox.tpl" field=$only_tag_search}}
{{include file="field_checkbox.tpl" field=$limited_search_scope}}
{{include file="field_input.tpl" field=$search_age_days}}
{{include file="field_input.tpl" field=$max_comments}}
{{include file="field_input.tpl" field=$max_display_comments}}

1
view/theme/frio/templates/admin/site.tpl
View file

@ -250,6 +250,7 @@
<div class="panel-body">
{{include file="field_checkbox.tpl" field=$compute_circle_counts}}
{{include file="field_checkbox.tpl" field=$only_tag_search}}
{{include file="field_checkbox.tpl" field=$limited_search_scope}}
{{include file="field_input.tpl" field=$search_age_days}}
{{include file="field_input.tpl" field=$max_comments}}
{{include file="field_input.tpl" field=$max_display_comments}}