From d2702dfe9c28cce341cd32c9761389272d0d83b2 Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 28 Jan 2024 06:32:55 +0000 Subject: [PATCH 1/3] Use centralized functions for tag string handling --- src/Model/Tag.php | 5 ++--- src/Module/Admin/Site.php | 6 ++--- src/Module/Settings/Channels.php | 23 +++++-------------- src/Module/WellKnown/XSocialRelay.php | 10 ++------- src/Protocol/Relay.php | 14 ++---------- src/Util/Strings.php | 32 +++++++++++++++++++++++++++ 6 files changed, 46 insertions(+), 44 deletions(-) diff --git a/src/Model/Tag.php b/src/Model/Tag.php index 398079e3c6..dac0401b2b 100644 --- a/src/Model/Tag.php +++ b/src/Model/Tag.php @@ -667,12 +667,11 @@ class Tag */ private static function getBlockedSQL(): string { - $blocked_txt = DI::config()->get('system', 'blocked_tags'); - if (empty($blocked_txt)) { + $blocked = Strings::getTagArrayByString(DI::config()->get('system', 'blocked_tags')); + if (empty($blocked)) { return ''; } - $blocked = explode(',', $blocked_txt); array_walk($blocked, function (&$value) { $value = "'" . DBA::escape(trim($value)) . "'"; }); diff --git a/src/Module/Admin/Site.php b/src/Module/Admin/Site.php index 22309b8eb7..6daa8dddf8 100644 --- a/src/Module/Admin/Site.php +++ b/src/Module/Admin/Site.php @@ -271,7 +271,7 @@ class Site extends BaseAdmin $transactionConfig->set('system', 'explicit_content' , $explicit_content); $transactionConfig->set('system', 'proxify_content' , $proxify_content); $transactionConfig->set('system', 'local_search' , $local_search); - $transactionConfig->set('system', 'blocked_tags' , $blocked_tags); + $transactionConfig->set('system', 'blocked_tags' , Strings::cleanTags($blocked_tags)); $transactionConfig->set('system', 'cache_contact_avatar' , $cache_contact_avatar); $transactionConfig->set('system', 'check_new_version_url' , $check_new_version_url); @@ -332,8 +332,8 @@ class Site extends BaseAdmin $transactionConfig->set('system', 'relay_directly' , $relay_directly); $transactionConfig->set('system', 'relay_scope' , $relay_scope); - $transactionConfig->set('system', 'relay_server_tags' , $relay_server_tags); - $transactionConfig->set('system', 'relay_deny_tags' , $relay_deny_tags); + $transactionConfig->set('system', 'relay_server_tags' , Strings::cleanTags($relay_server_tags)); + $transactionConfig->set('system', 'relay_deny_tags' , Strings::cleanTags($relay_deny_tags)); $transactionConfig->set('system', 'relay_max_tags' , $relay_max_tags); $transactionConfig->set('system', 'relay_user_tags' , $relay_user_tags); $transactionConfig->set('system', 'relay_deny_undetected_language', $relay_deny_undetected_language); diff --git a/src/Module/Settings/Channels.php b/src/Module/Settings/Channels.php index d846a91be7..34ba0d1d80 100644 --- a/src/Module/Settings/Channels.php +++ b/src/Module/Settings/Channels.php @@ -34,6 +34,7 @@ use Friendica\Module\BaseSettings; use Friendica\Module\Response; use Friendica\Network\HTTPException; use Friendica\Util\Profiler; +use Friendica\Util\Strings; use Psr\Log\LoggerInterface; class Channels extends BaseSettings @@ -80,8 +81,8 @@ class Channels extends BaseSettings 'access-key' => substr(mb_strtolower($request['new_access_key']), 0, 1), 'uid' => $uid, 'circle' => (int)$request['new_circle'], - 'include-tags' => $this->cleanTags($request['new_include_tags']), - 'exclude-tags' => $this->cleanTags($request['new_exclude_tags']), + 'include-tags' => Strings::cleanTags($request['new_include_tags']), + 'exclude-tags' => Strings::cleanTags($request['new_exclude_tags']), 'full-text-search' => $request['new_text_search'], 'media-type' => ($request['new_image'] ? 1 : 0) | ($request['new_video'] ? 2 : 0) | ($request['new_audio'] ? 4 : 0), 'languages' => $request['new_languages'], @@ -109,8 +110,8 @@ class Channels extends BaseSettings 'access-key' => substr(mb_strtolower($request['access_key'][$id]), 0, 1), 'uid' => $uid, 'circle' => (int)$request['circle'][$id], - 'include-tags' => $this->cleanTags($request['include_tags'][$id]), - 'exclude-tags' => $this->cleanTags($request['exclude_tags'][$id]), + 'include-tags' => Strings::cleanTags($request['include_tags'][$id]), + 'exclude-tags' => Strings::cleanTags($request['exclude_tags'][$id]), 'full-text-search' => $request['text_search'][$id], 'media-type' => ($request['image'][$id] ? 1 : 0) | ($request['video'][$id] ? 2 : 0) | ($request['audio'][$id] ? 4 : 0), 'languages' => $request['languages'][$id], @@ -222,18 +223,4 @@ class Channels extends BaseSettings '$form_security_token' => self::getFormSecurityToken('settings_channels'), ]); } - - private function cleanTags(string $tag_list): string - { - $tags = []; - - $tagitems = explode(',', mb_strtolower($tag_list)); - foreach ($tagitems as $tag) { - $tag = trim($tag, '# '); - if (!empty($tag)) { - $tags[] = preg_replace('#\s#u', '', $tag); - } - } - return implode(',', $tags); - } } diff --git a/src/Module/WellKnown/XSocialRelay.php b/src/Module/WellKnown/XSocialRelay.php index 50cfb8aa82..39b67de846 100644 --- a/src/Module/WellKnown/XSocialRelay.php +++ b/src/Module/WellKnown/XSocialRelay.php @@ -22,10 +22,10 @@ namespace Friendica\Module\WellKnown; use Friendica\BaseModule; -use Friendica\Core\System; use Friendica\DI; use Friendica\Model\Search; use Friendica\Protocol\Relay; +use Friendica\Util\Strings; /** * Node subscription preferences for social relay systems @@ -43,13 +43,7 @@ class XSocialRelay extends BaseModule $userTags = []; if ($scope == Relay::SCOPE_TAGS) { - $server_tags = $config->get('system', 'relay_server_tags'); - $tagitems = explode(',', $server_tags); - - /// @todo Check if it was better to use "strtolower" on the tags - foreach ($tagitems as $tag) { - $systemTags[] = trim($tag, '# '); - } + $systemTags = Strings::getTagArrayByString($config->get('system', 'relay_server_tags')); if ($config->get('system', 'relay_user_tags')) { $userTags = Search::getUserTags(); diff --git a/src/Protocol/Relay.php b/src/Protocol/Relay.php index db61e34791..3c655f8cb7 100644 --- a/src/Protocol/Relay.php +++ b/src/Protocol/Relay.php @@ -97,20 +97,13 @@ class Relay $body = ActivityPub\Processor::normalizeMentionLinks($body); - $denyTags = []; - if ($scope == self::SCOPE_TAGS) { $tagList = self::getSubscribedTags(); } else { $tagList = []; } - $deny_tags = $config->get('system', 'relay_deny_tags'); - $tagitems = explode(',', mb_strtolower($deny_tags)); - foreach ($tagitems as $tag) { - $tag = trim($tag, '# '); - $denyTags[] = $tag; - } + $denyTags = Strings::getTagArrayByString($config->get('system', 'relay_deny_tags')); if (!empty($tagList) || !empty($denyTags)) { $content = mb_strtolower(BBCode::toPlaintext($body, false)); @@ -168,10 +161,7 @@ class Relay */ public static function getSubscribedTags(): array { - $tags = []; - foreach (explode(',', mb_strtolower(DI::config()->get('system', 'relay_server_tags'))) as $tag) { - $tags[] = trim($tag, '# '); - } + $tags = Strings::getTagArrayByString(DI::config()->get('system', 'relay_server_tags')); if (DI::config()->get('system', 'relay_user_tags')) { $tags = array_merge($tags, Search::getUserTags()); diff --git a/src/Util/Strings.php b/src/Util/Strings.php index fff218e466..34029648ee 100644 --- a/src/Util/Strings.php +++ b/src/Util/Strings.php @@ -578,4 +578,36 @@ class Strings return $styled_url; } + + /** + * Sort a comma separated list of hashtags, convert them to lowercase and remove duplicates + * + * @param string $tag_list + * @return string + */ + public static function cleanTags(string $tag_list): string + { + $tags = []; + + $tagitems = explode(',', str_replace([' ', ';', '#'], ',', mb_strtolower($tag_list))); + foreach ($tagitems as $tag) { + if (!empty($tag)) { + $tags[] = preg_replace('#\s#u', '', $tag); + } + } + $tags = array_unique($tags); + asort($tags); + return implode(',', $tags); + } + + /** + * Get a tag array out of a comma separated list of tags + * + * @param string $tag_list + * @return array + */ + public static function getTagArrayByString(string $tag_list): array + { + return explode(',', self::cleanTags($tag_list)); + } } From b948b2c7cba0ab153d1a883bd8964e347a016281 Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 28 Jan 2024 06:56:25 +0000 Subject: [PATCH 2/3] Added tests --- tests/src/Util/StringsTest.php | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/src/Util/StringsTest.php b/tests/src/Util/StringsTest.php index 0ee7e0cdb6..c3e1beb49c 100644 --- a/tests/src/Util/StringsTest.php +++ b/tests/src/Util/StringsTest.php @@ -214,4 +214,21 @@ class StringsTest extends TestCase self::assertEquals($originalText, $text); } + + public function testCleanTags() + { + $rawTags = 'Open, #Source, Friendica Software; Federation #Fediverse'; + $cleaned = 'federation,fediverse,friendica,open,software,source'; + + self::assertEquals($cleaned, Strings::cleanTags($rawTags)); + } + + public function testgetTagArrayByString() + { + $list = 'Open, #Source, Friendica Software; Federation #Fediverse'; + $tags = ['federation', 'fediverse', 'friendica', 'open', 'software', 'source']; + + self::assertEquals($tags, Strings::getTagArrayByString($list)); + } + } From 25f45cf11621b07c4eb931c05c06f884ac6826e9 Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 28 Jan 2024 12:22:58 +0000 Subject: [PATCH 3/3] Logging improved --- src/Protocol/Relay.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Protocol/Relay.php b/src/Protocol/Relay.php index 3c655f8cb7..d57cac21bf 100644 --- a/src/Protocol/Relay.php +++ b/src/Protocol/Relay.php @@ -114,7 +114,7 @@ class Relay $content_cleaned = mb_strtolower(BBCode::toPlaintext($cleaned, false)); if (strlen($content_cleaned) < strlen($content) / 2) { - Logger::info('Possible hashtag spam detected - rejected', ['hashtags' => $tags, 'network' => $network, 'url' => $url, 'causer' => $causer, 'body' => $body]); + Logger::info('Possible hashtag spam detected - rejected', ['hashtags' => $tags, 'network' => $network, 'url' => $url, 'causer' => $causer, 'content' => $content]); return false; } }