Merge pull request #14198 from annando/limited-search

Option to reduced search scope to improve the performance

database.sql

@@ -1,6 +1,6 @@
 -- ------------------------------------------
 -- Friendica 2024.06-dev (Yellow Archangel)
--- DB_UPDATE_VERSION 1564
+-- DB_UPDATE_VERSION 1565
 -- ------------------------------------------
 
 
@@ -2099,6 +2099,38 @@ CREATE VIEW `post-counts-view` AS SELECT
 	FROM `post-counts`
 			INNER JOIN `verb` ON `verb`.`id` = `post-counts`.`vid`;
 
+--
+-- VIEW post-engagement-user-view
+--
+DROP VIEW IF EXISTS `post-engagement-user-view`;
+CREATE VIEW `post-engagement-user-view` AS SELECT 
+	`post-thread-user`.`uid` AS `uid`,
+	`post-engagement`.`uri-id` AS `uri-id`,
+	`post-engagement`.`owner-id` AS `owner-id`,
+	`post-engagement`.`media-type` AS `media-type`,
+	`post-engagement`.`language` AS `language`,
+	`post-engagement`.`searchtext` AS `searchtext`,
+	`post-engagement`.`size` AS `size`,
+	`post-thread-user`.`commented` AS `commented`,
+	`post-thread-user`.`received` AS `received`,
+	`post-thread-user`.`created` AS `created`,
+	`post-thread-user`.`network` AS `network`,
+	`post-engagement`.`language` AS `restricted`,
+	0 AS `comments`,
+	0 AS `activities`
+	FROM `post-thread-user`
+			INNER JOIN `post-engagement` ON `post-engagement`.`uri-id` = `post-thread-user`.`uri-id`
+			INNER JOIN `post-user` ON `post-user`.`id` = `post-thread-user`.`post-user-id`
+			STRAIGHT_JOIN `contact` ON `contact`.`id` = `post-thread-user`.`contact-id`
+			STRAIGHT_JOIN `contact` AS `authorcontact` ON `authorcontact`.`id` = `post-thread-user`.`author-id`
+			STRAIGHT_JOIN `contact` AS `ownercontact` ON `ownercontact`.`id` = `post-thread-user`.`owner-id`
+			WHERE `post-user`.`visible` AND NOT `post-user`.`deleted`
+			AND (NOT `contact`.`readonly` AND NOT `contact`.`blocked` AND NOT `contact`.`pending`)
+			AND (`post-thread-user`.`hidden` IS NULL OR NOT `post-thread-user`.`hidden`)
+			AND NOT `authorcontact`.`blocked` AND NOT `ownercontact`.`blocked`
+			AND NOT EXISTS(SELECT `cid`  FROM `user-contact` WHERE `uid` = `post-thread-user`.`uid` AND `cid` IN (`authorcontact`.`id`, `ownercontact`.`id`) AND (`blocked` OR `ignored`))
+			AND NOT EXISTS(SELECT `gsid` FROM `user-gserver` WHERE `uid` = `post-thread-user`.`uid` AND `gsid` IN (`authorcontact`.`gsid`, `ownercontact`.`gsid`) AND `ignored`);
+
 --
 -- VIEW post-timeline-view
 --

src/Model/Post/Content.php

@@ -124,7 +124,7 @@ class Content
 			'limit' => [$start, $limit]
 		];
 
-		$tags = DBA::select('post-searchindex', ['uri-id'], $condition, $params);
+		$tags = DBA::select(SearchIndex::getSearchTable(), ['uri-id'], $condition, $params);
 
 		$uriids = [];
 		while ($tag = DBA::fetch($tags)) {
@@ -143,6 +143,6 @@ class Content
 		} else {
 			$condition = ["MATCH (`searchtext`) AGAINST (? IN BOOLEAN MODE) AND NOT `restricted", $search];
 		}
-		return DBA::count('post-searchindex', $condition);
+		return DBA::count(SearchIndex::getSearchTable(), $condition);
 	}
 }

src/Model/Post/SearchIndex.php

@@ -99,4 +99,14 @@ class SearchIndex
 		}
 		return DateTimeFormat::utc('now - ' . $days . ' day');
 	}
+
+	public static function getSearchTable(): string
+	{
+		return DI::config()->get('system', 'limited_search_scope') ? 'post-engagement' : 'post-searchindex';
+	}
+
+	public static function getSearchView(): string
+	{
+		return DI::config()->get('system', 'limited_search_scope') ? 'post-engagement-user-view' : 'post-searchindex-user-view';
+	}
 }

src/Module/Admin/Site.php

@@ -140,6 +140,7 @@ class Site extends BaseAdmin
 		$temppath               = (!empty($_POST['temppath'])               ? trim($_POST['temppath'])   : '');
 		$singleuser             = (!empty($_POST['singleuser'])             ? trim($_POST['singleuser']) : '');
 		$only_tag_search        = !empty($_POST['only_tag_search']);
+		$limited_search_scope   = !empty($_POST['limited_search_scope']);
 		$search_age_days        = (!empty($_POST['search_age_days'])        ? intval($_POST['search_age_days'])               : 0);
 		$compute_circle_counts  = !empty($_POST['compute_circle_counts']);
 		$process_view           = !empty($_POST['process_view']);
@@ -317,6 +318,7 @@ class Site extends BaseAdmin
 		$transactionConfig->set('system', 'temppath', $temppath);
 
 		$transactionConfig->set('system', 'only_tag_search', $only_tag_search);
+		$transactionConfig->set('system', 'limited_search_scope', $limited_search_scope);
 		$transactionConfig->set('system', 'search_age_days', $search_age_days);
 		$transactionConfig->set('system', 'compute_circle_counts', $compute_circle_counts);
 		$transactionConfig->set('system', 'process_view', $process_view);
@@ -571,6 +573,7 @@ class Site extends BaseAdmin
 			'$itemspage_network_mobile' => ['itemspage_network_mobile', DI::l10n()->t('Items per page for mobile devices'), DI::config()->get('system', 'itemspage_network_mobile'), DI::l10n()->t('Number of items per page in stream pages (network, community, profile/contact statuses, search) for mobile devices.')],
 			'$temppath'               => ['temppath', DI::l10n()->t('Temp path'), DI::config()->get('system', 'temppath'), DI::l10n()->t('If you have a restricted system where the webserver can\'t access the system temp path, enter another path here.')],
 			'$only_tag_search'        => ['only_tag_search', DI::l10n()->t('Only search in tags'), DI::config()->get('system', 'only_tag_search'), DI::l10n()->t('On large systems the text search can slow down the system extremely.')],
+			'$limited_search_scope'   => ['limited_search_scope', DI::l10n()->t('Limited search scope'), DI::config()->get('system', 'limited_search_scope'), DI::l10n()->t('If enabled, searches will only be performed in the data used for the channels and not in all posts.')],
 			'$search_age_days'        => ['search_age_days', DI::l10n()->t('Maximum age of items in the search table'), DI::config()->get('system', 'search_age_days'), DI::l10n()->t('Maximum age of items in the search table in days. Lower values will increase the performance and reduce disk usage. 0 means no age restriction.')],
 			'$compute_circle_counts'  => ['compute_circle_counts', DI::l10n()->t('Generate counts per contact circle when calculating network count'), DI::config()->get('system', 'compute_circle_counts'), DI::l10n()->t('On systems with users that heavily use contact circles the query can be very expensive.')],
 			'$process_view'           => ['process_view', DI::l10n()->t('Process "view" activities'), DI::config()->get('system', 'process_view'), DI::l10n()->t('"view" activities are mostly geberated by Peertube systems. Per default they are not processed for performance reasons. Only activate this option on performant system.')],

src/Module/Api/Mastodon/Search.php

@@ -28,6 +28,7 @@ use Friendica\DI;
 use Friendica\Model\Contact;
 use Friendica\Model\Item;
 use Friendica\Model\Post;
+use Friendica\Model\Post\SearchIndex;
 use Friendica\Model\Tag;
 use Friendica\Module\BaseApi;
 use Friendica\Util\Network;
@@ -159,7 +160,7 @@ class Search extends BaseApi
 		} else {
 			$q = Post\Engagement::escapeKeywords($q);
 			$condition = ["MATCH (`searchtext`) AGAINST (? IN BOOLEAN MODE) AND (NOT `restricted` OR `uri-id` IN (SELECT `uri-id` FROM `post-user` WHERE `uid` = ?))", $q, $uid];
-			$table = 'post-searchindex';
+			$table = SearchIndex::getSearchTable();
 		}
 
 		if (!empty($account_id)) {

src/Module/Conversation/Timeline.php

@@ -43,6 +43,7 @@ use Friendica\Database\DBA;
 use Friendica\Model\Item;
 use Friendica\Model\Post;
 use Friendica\Model\Post\Engagement;
+use Friendica\Model\Post\SearchIndex;
 use Friendica\Module\Response;
 use Friendica\Protocol\Activity;
 use Friendica\Util\DateTimeFormat;
@@ -381,7 +382,7 @@ class Timeline extends BaseModule
 		} elseif (is_numeric($this->selectedTab) && !empty($channel = $this->channelRepository->selectById($this->selectedTab, $uid))) {
 			$condition = $this->getUserChannelConditions($channel, $uid);
 			if (in_array($channel->circle, [-3, -4, -5])) {
-				$table = 'post-searchindex-user-view';
+				$table = SearchIndex::getSearchView();
 				$condition = DBA::mergeConditions($condition, ['uid' => $uid]);
 				$orders = ['-3' => 'created', '-4' => 'received', '-5' => 'commented'];
 				$this->order = $orders[$channel->circle];

static/dbstructure.config.php

@@ -56,7 +56,7 @@ use Friendica\Database\DBA;
 
 // This file is required several times during the test in DbaDefinition which justifies this condition
 if (!defined('DB_UPDATE_VERSION')) {
-	define('DB_UPDATE_VERSION', 1564);
+	define('DB_UPDATE_VERSION', 1565);
 }
 
 return [

static/dbview.config.php

@@ -100,6 +100,36 @@
 		"query" => "FROM `post-counts`
 			INNER JOIN `verb` ON `verb`.`id` = `post-counts`.`vid`"
 	],
+	"post-engagement-user-view" => [
+		"fields" => [
+			"uid" => ["post-thread-user", "uid"],
+			"uri-id" => ["post-engagement", "uri-id"],
+			"owner-id" => ["post-engagement", "owner-id"],
+			"media-type" => ["post-engagement", "media-type"],
+			"language" => ["post-engagement", "language"],
+			"searchtext" => ["post-engagement", "searchtext"],
+			"size" => ["post-engagement", "size"],
+			"commented" => ["post-thread-user", "commented"],
+			"received" => ["post-thread-user", "received"],
+			"created" => ["post-thread-user", "created"],
+			"network" => ["post-thread-user", "network"],
+			"restricted" => ["post-engagement", "language"],
+			"comments" => "0",
+			"activities" => "0",
+		],
+		"query" => "FROM `post-thread-user`
+			INNER JOIN `post-engagement` ON `post-engagement`.`uri-id` = `post-thread-user`.`uri-id`
+			INNER JOIN `post-user` ON `post-user`.`id` = `post-thread-user`.`post-user-id`
+			STRAIGHT_JOIN `contact` ON `contact`.`id` = `post-thread-user`.`contact-id`
+			STRAIGHT_JOIN `contact` AS `authorcontact` ON `authorcontact`.`id` = `post-thread-user`.`author-id`
+			STRAIGHT_JOIN `contact` AS `ownercontact` ON `ownercontact`.`id` = `post-thread-user`.`owner-id`
+			WHERE `post-user`.`visible` AND NOT `post-user`.`deleted`
+			AND (NOT `contact`.`readonly` AND NOT `contact`.`blocked` AND NOT `contact`.`pending`)
+			AND (`post-thread-user`.`hidden` IS NULL OR NOT `post-thread-user`.`hidden`)
+			AND NOT `authorcontact`.`blocked` AND NOT `ownercontact`.`blocked`
+			AND NOT EXISTS(SELECT `cid`  FROM `user-contact` WHERE `uid` = `post-thread-user`.`uid` AND `cid` IN (`authorcontact`.`id`, `ownercontact`.`id`) AND (`blocked` OR `ignored`))
+			AND NOT EXISTS(SELECT `gsid` FROM `user-gserver` WHERE `uid` = `post-thread-user`.`uid` AND `gsid` IN (`authorcontact`.`gsid`, `ownercontact`.`gsid`) AND `ignored`)"
+	],
 	"post-timeline-view" => [
 		"fields" => [
 			"uid" => ["post-user", "uid"],

view/templates/admin/site.tpl

@@ -116,6 +116,7 @@
 		<h2>{{$performance}}</h2>
 		{{include file="field_checkbox.tpl" field=$compute_circle_counts}}
 		{{include file="field_checkbox.tpl" field=$only_tag_search}}
+		{{include file="field_checkbox.tpl" field=$limited_search_scope}}
 		{{include file="field_input.tpl" field=$search_age_days}}
 		{{include file="field_input.tpl" field=$max_comments}}
 		{{include file="field_input.tpl" field=$max_display_comments}}

view/theme/frio/templates/admin/site.tpl

@@ -250,6 +250,7 @@
 					<div class="panel-body">
 						{{include file="field_checkbox.tpl" field=$compute_circle_counts}}
 						{{include file="field_checkbox.tpl" field=$only_tag_search}}
+						{{include file="field_checkbox.tpl" field=$limited_search_scope}}
 						{{include file="field_input.tpl" field=$search_age_days}}
 						{{include file="field_input.tpl" field=$max_comments}}
 						{{include file="field_input.tpl" field=$max_display_comments}}