From 15df9990daf8f608b8ea1868f711188ee245515e Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 4 Feb 2024 21:45:30 +0000 Subject: [PATCH 1/2] Issue 13845: Support "sensitive" attribute --- database.sql | 7 ++++++- doc/database/db_post-content.md | 1 + src/Factory/Api/Mastodon/Status.php | 4 ++-- src/Model/Item.php | 26 +++++++++++++++--------- src/Model/Post/SearchIndex.php | 2 +- src/Model/Tag.php | 12 +++++++++++ src/Protocol/ActivityPub/Processor.php | 1 + src/Protocol/ActivityPub/Transmitter.php | 14 +------------ static/dbstructure.config.php | 3 ++- static/dbview.config.php | 4 ++++ update.php | 7 +++++++ 11 files changed, 53 insertions(+), 28 deletions(-) diff --git a/database.sql b/database.sql index c8ab483b44..d56b45f878 100644 --- a/database.sql +++ b/database.sql @@ -1,6 +1,6 @@ -- ------------------------------------------ -- Friendica 2024.03-dev (Yellow Archangel) --- DB_UPDATE_VERSION 1550 +-- DB_UPDATE_VERSION 1552 -- ------------------------------------------ @@ -1283,6 +1283,7 @@ CREATE TABLE IF NOT EXISTS `post-content` ( `location` varchar(255) NOT NULL DEFAULT '' COMMENT 'text location where this item originated', `coord` varchar(255) NOT NULL DEFAULT '' COMMENT 'longitude/latitude pair representing location where this item originated', `language` text COMMENT 'Language information about this post', + `sensitive` boolean COMMENT 'If true, this post contains sensitive content', `app` varchar(255) NOT NULL DEFAULT '' COMMENT 'application which generated this item', `rendered-hash` varchar(32) NOT NULL DEFAULT '' COMMENT '', `rendered-html` mediumtext COMMENT 'item.body converted to html', @@ -2163,6 +2164,7 @@ CREATE VIEW `post-user-view` AS SELECT `post-content`.`plink` AS `plink`, `post-content`.`location` AS `location`, `post-content`.`coord` AS `coord`, + `post-content`.`sensitive` AS `sensitive`, `post-content`.`app` AS `app`, `post-content`.`object-type` AS `object-type`, `post-content`.`object` AS `object`, @@ -2347,6 +2349,7 @@ CREATE VIEW `post-thread-user-view` AS SELECT `post-content`.`plink` AS `plink`, `post-content`.`location` AS `location`, `post-content`.`coord` AS `coord`, + `post-content`.`sensitive` AS `sensitive`, `post-content`.`app` AS `app`, `post-content`.`object-type` AS `object-type`, `post-content`.`object` AS `object`, @@ -2517,6 +2520,7 @@ CREATE VIEW `post-view` AS SELECT `post-content`.`plink` AS `plink`, `post-content`.`location` AS `location`, `post-content`.`coord` AS `coord`, + `post-content`.`sensitive` AS `sensitive`, `post-content`.`app` AS `app`, `post-content`.`object-type` AS `object-type`, `post-content`.`object` AS `object`, @@ -2663,6 +2667,7 @@ CREATE VIEW `post-thread-view` AS SELECT `post-content`.`plink` AS `plink`, `post-content`.`location` AS `location`, `post-content`.`coord` AS `coord`, + `post-content`.`sensitive` AS `sensitive`, `post-content`.`app` AS `app`, `post-content`.`object-type` AS `object-type`, `post-content`.`object` AS `object`, diff --git a/doc/database/db_post-content.md b/doc/database/db_post-content.md index be45975a72..d1ae90f278 100644 --- a/doc/database/db_post-content.md +++ b/doc/database/db_post-content.md @@ -17,6 +17,7 @@ Fields | location | text location where this item originated | varchar(255) | NO | | | | | coord | longitude/latitude pair representing location where this item originated | varchar(255) | NO | | | | | language | Language information about this post | text | YES | | NULL | | +| sensitive | If true, this post contains sensitive content | boolean | YES | | NULL | | | app | application which generated this item | varchar(255) | NO | | | | | rendered-hash | | varchar(32) | NO | | | | | rendered-html | item.body converted to html | mediumtext | YES | | NULL | | diff --git a/src/Factory/Api/Mastodon/Status.php b/src/Factory/Api/Mastodon/Status.php index e20b457c88..e001ef80e0 100644 --- a/src/Factory/Api/Mastodon/Status.php +++ b/src/Factory/Api/Mastodon/Status.php @@ -112,7 +112,7 @@ class Status extends BaseFactory { $fields = ['uri-id', 'uid', 'author-id', 'causer-id', 'author-uri-id', 'author-link', 'causer-uri-id', 'post-reason', 'starred', 'app', 'title', 'body', 'raw-body', 'content-warning', 'question-id', 'created', 'edited', 'commented', 'received', 'changed', 'network', 'thr-parent-id', 'parent-author-id', 'language', 'uri', 'plink', 'private', 'vid', 'gravity', 'featured', 'has-media', 'quote-uri-id', - 'delivery_queue_count', 'delivery_queue_done','delivery_queue_failed', 'allow_cid', 'deny_cid', 'allow_gid', 'deny_gid']; + 'delivery_queue_count', 'delivery_queue_done','delivery_queue_failed', 'allow_cid', 'deny_cid', 'allow_gid', 'deny_gid', 'sensitive']; $item = Post::selectFirst($fields, ['uri-id' => $uriId, 'uid' => [0, $uid]], ['order' => ['uid' => true]]); if (!$item) { $mail = DBA::selectFirst('mail', ['id'], ['uri-id' => $uriId, 'uid' => $uid]); @@ -217,7 +217,7 @@ class Status extends BaseFactory $item['featured'] ); - $sensitive = $this->dba->exists('tag-view', ['uri-id' => $uriId, 'name' => 'nsfw', 'type' => TagModel::HASHTAG]); + $sensitive = (bool)$item['sensitive']; $application = new \Friendica\Object\Api\Mastodon\Application($item['app'] ?: ContactSelector::networkToName($item['network'], $item['author-link'])); $mentions = $this->mstdnMentionFactory->createFromUriId($uriId)->getArrayCopy(); diff --git a/src/Model/Item.php b/src/Model/Item.php index 3d35618396..ca63a923cc 100644 --- a/src/Model/Item.php +++ b/src/Model/Item.php @@ -99,7 +99,7 @@ class Item 'uid', 'id', 'parent', 'guid', 'network', 'gravity', 'uri-id', 'uri', 'thr-parent-id', 'thr-parent', 'parent-uri-id', 'parent-uri', 'conversation', 'commented', 'created', 'edited', 'received', 'verb', 'object-type', 'postopts', 'plink', - 'wall', 'private', 'starred', 'origin', 'parent-origin', 'title', 'body', 'language', + 'wall', 'private', 'starred', 'origin', 'parent-origin', 'title', 'body', 'language', 'sensitive', 'content-warning', 'location', 'coord', 'app', 'rendered-hash', 'rendered-html', 'object', 'quote-uri', 'quote-uri-id', 'allow_cid', 'allow_gid', 'deny_cid', 'deny_gid', 'mention', 'global', 'author-id', 'author-link', 'author-alias', 'author-name', 'author-avatar', 'author-network', 'author-updated', 'author-gsid', 'author-baseurl', 'author-addr', 'author-uri-id', @@ -1280,6 +1280,15 @@ class Item } } + // Store tags from the body if this hadn't been handled previously in the protocol classes + if (!Tag::existsForPost($item['uri-id'])) { + Tag::storeFromBody($item['uri-id'], $item['body']); + } + + if (in_array($item['gravity'], [self::GRAVITY_PARENT, self::GRAVITY_COMMENT]) && (!isset($item['sensitive']) || is_null($item['sensitive']))) { + $item['sensitive'] = Tag::existsTagForPost($item['uri-id'], 'nsfw'); + } + $item['language'] = self::getLanguage($item); $inserted = Post::insert($item['uri-id'], $item); @@ -1319,11 +1328,6 @@ class Item Post\DeliveryData::insert($item['uri-id'], $delivery_data); } - // Store tags from the body if this hadn't been handled previously in the protocol classes - if (!Tag::existsForPost($item['uri-id'])) { - Tag::storeFromBody($item['uri-id'], $item['body']); - } - $condition = ['uri-id' => $item['uri-id'], 'uid' => $item['uid']]; if (Post::exists($condition)) { Logger::notice('Item is already inserted - aborting', $condition); @@ -3467,7 +3471,7 @@ class Item } if (!empty($sharedSplitAttachments)) { - $s = self::addGallery($s, $sharedSplitAttachments['visual']); + $s = self::addGallery($s, $sharedSplitAttachments['visual'], (bool)$item['sensitive']); $s = self::addVisualAttachments($sharedSplitAttachments['visual'], $shared_item, $s, true); $s = self::addLinkAttachment($shared_uri_id ?: $item['uri-id'], $sharedSplitAttachments, $body, $s, true, $quote_shared_links); $s = self::addNonVisualAttachments($sharedSplitAttachments['additional'], $item, $s, true); @@ -3480,7 +3484,7 @@ class Item $s = substr($s, 0, $pos); } - $s = self::addGallery($s, $itemSplitAttachments['visual']); + $s = self::addGallery($s, $itemSplitAttachments['visual'], (bool)$item['sensitive']); $s = self::addVisualAttachments($itemSplitAttachments['visual'], $item, $s, false); $s = self::addLinkAttachment($item['uri-id'], $itemSplitAttachments, $body, $s, false, $shared_links); $s = self::addNonVisualAttachments($itemSplitAttachments['additional'], $item, $s, false); @@ -3516,9 +3520,10 @@ class Item * * @param string $s * @param PostMedias $PostMedias + * @param bool $sensitive * @return string */ - private static function addGallery(string $s, PostMedias $PostMedias): string + private static function addGallery(string $s, PostMedias $PostMedias, bool $sensitive): string { foreach ($PostMedias as $PostMedia) { if (!$PostMedia->preview || ($PostMedia->type !== Post\Media::IMAGE)) { @@ -3528,9 +3533,10 @@ class Item if ($PostMedia->hasDimensions()) { $pattern = '#(.*?)">#'; - $s = preg_replace_callback($pattern, function () use ($PostMedia) { + $s = preg_replace_callback($pattern, function () use ($PostMedia, $sensitive) { return Renderer::replaceMacros(Renderer::getMarkupTemplate('content/image/single_with_height_allocation.tpl'), [ '$image' => $PostMedia, + '$sensitive' => $sensitive, '$allocated_height' => $PostMedia->getAllocatedHeight(), '$allocated_max_width' => ($PostMedia->previewWidth ?? $PostMedia->width) . 'px', ]); diff --git a/src/Model/Post/SearchIndex.php b/src/Model/Post/SearchIndex.php index 98a82cae63..840f86f6ac 100644 --- a/src/Model/Post/SearchIndex.php +++ b/src/Model/Post/SearchIndex.php @@ -53,7 +53,7 @@ class SearchIndex 'uri-id' => $uri_id, 'owner-id' => $item['owner-id'], 'media-type' => Engagement::getMediaType($uri_id), - 'language' => !empty($item['language']) ? (array_key_first(json_decode($item['language'], true)) ?? L10n::UNDETERMINED_LANGUAGE) : L10n::UNDETERMINED_LANGUAGE, + 'language' => substr(!empty($item['language']) ? (array_key_first(json_decode($item['language'], true)) ?? L10n::UNDETERMINED_LANGUAGE) : L10n::UNDETERMINED_LANGUAGE, 0, 2), 'searchtext' => Post\Engagement::getSearchTextForUriId($uri_id, $refresh), 'size' => Engagement::getContentSize($item), 'created' => $item['created'], diff --git a/src/Model/Tag.php b/src/Model/Tag.php index dac0401b2b..e63a272a75 100644 --- a/src/Model/Tag.php +++ b/src/Model/Tag.php @@ -378,6 +378,18 @@ class Tag return DBA::exists('post-tag', ['uri-id' => $uriId, 'type' => [self::HASHTAG, self::MENTION, self::EXCLUSIVE_MENTION, self::IMPLICIT_MENTION]]); } + /** + * Check for a given hashtag on a given post + * + * @param integer $uriId + * @param string $tag + * @return boolean + */ + public static function existsTagForPost(int $uriId, string $tag): bool + { + return DBA::exists('tag-view', ['uri-id' => $uriId, 'type' => self::HASHTAG, 'name' => $tag]); + } + /** * Remove tag/mention * diff --git a/src/Protocol/ActivityPub/Processor.php b/src/Protocol/ActivityPub/Processor.php index 593c1e7836..9b1c84bbb1 100644 --- a/src/Protocol/ActivityPub/Processor.php +++ b/src/Protocol/ActivityPub/Processor.php @@ -466,6 +466,7 @@ class Processor } $item['uri'] = $activity['id']; + $item['sensitive'] = $activity['sensitive']; if (empty($activity['published']) || empty($activity['updated'])) { DI::logger()->notice('published or updated keys are empty for activity', ['activity' => $activity]); diff --git a/src/Protocol/ActivityPub/Transmitter.php b/src/Protocol/ActivityPub/Transmitter.php index baa46e373e..9aa25b3db5 100644 --- a/src/Protocol/ActivityPub/Transmitter.php +++ b/src/Protocol/ActivityPub/Transmitter.php @@ -1697,18 +1697,6 @@ class Transmitter }); } - /** - * Returns if the post contains sensitive content ("nsfw") - * - * @param integer $uri_id URI id - * @return boolean Whether URI id was found - * @throws \Exception - */ - private static function isSensitive(int $uri_id): bool - { - return DBA::exists('tag-view', ['uri-id' => $uri_id, 'name' => 'nsfw', 'type' => Tag::HASHTAG]); - } - /** * Creates event data * @@ -1812,7 +1800,7 @@ class Transmitter } else { $data['attributedTo'] = $item['author-link']; } - $data['sensitive'] = self::isSensitive($item['uri-id']); + $data['sensitive'] = (bool)$item['sensitive']; if (!empty($item['conversation']) && ($item['conversation'] != './')) { $data['conversation'] = $data['context'] = $item['conversation']; diff --git a/static/dbstructure.config.php b/static/dbstructure.config.php index e9082efc66..1b7f7a40de 100644 --- a/static/dbstructure.config.php +++ b/static/dbstructure.config.php @@ -56,7 +56,7 @@ use Friendica\Database\DBA; // This file is required several times during the test in DbaDefinition which justifies this condition if (!defined('DB_UPDATE_VERSION')) { - define('DB_UPDATE_VERSION', 1550); + define('DB_UPDATE_VERSION', 1552); } return [ @@ -1307,6 +1307,7 @@ return [ "location" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "text location where this item originated"], "coord" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "longitude/latitude pair representing location where this item originated"], "language" => ["type" => "text", "comment" => "Language information about this post"], + "sensitive" => ["type" => "boolean", "comment" => "If true, this post contains sensitive content"], "app" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "application which generated this item"], "rendered-hash" => ["type" => "varchar(32)", "not null" => "1", "default" => "", "comment" => ""], "rendered-html" => ["type" => "mediumtext", "comment" => "item.body converted to html"], diff --git a/static/dbview.config.php b/static/dbview.config.php index f0183db81a..45ab1121fa 100644 --- a/static/dbview.config.php +++ b/static/dbview.config.php @@ -196,6 +196,7 @@ "plink" => ["post-content", "plink"], "location" => ["post-content", "location"], "coord" => ["post-content", "coord"], + "sensitive" => ["post-content", "sensitive"], "app" => ["post-content", "app"], "object-type" => ["post-content", "object-type"], "object" => ["post-content", "object"], @@ -378,6 +379,7 @@ "plink" => ["post-content", "plink"], "location" => ["post-content", "location"], "coord" => ["post-content", "coord"], + "sensitive" => ["post-content", "sensitive"], "app" => ["post-content", "app"], "object-type" => ["post-content", "object-type"], "object" => ["post-content", "object"], @@ -546,6 +548,7 @@ "plink" => ["post-content", "plink"], "location" => ["post-content", "location"], "coord" => ["post-content", "coord"], + "sensitive" => ["post-content", "sensitive"], "app" => ["post-content", "app"], "object-type" => ["post-content", "object-type"], "object" => ["post-content", "object"], @@ -690,6 +693,7 @@ "plink" => ["post-content", "plink"], "location" => ["post-content", "location"], "coord" => ["post-content", "coord"], + "sensitive" => ["post-content", "sensitive"], "app" => ["post-content", "app"], "object-type" => ["post-content", "object-type"], "object" => ["post-content", "object"], diff --git a/update.php b/update.php index 74a8ad7015..c19bbed388 100644 --- a/update.php +++ b/update.php @@ -1422,3 +1422,10 @@ function pre_update_1550() } return Update::SUCCESS; } + +function update_1552() +{ + DBA::e("UPDATE `post-content` INNER JOIN `post-tag` ON `post-tag`.`uri-id` = `post-content`.`uri-id` INNER JOIN `tag` ON `tag`.`id` = `post-tag`.`tid` SET `sensitive` = ? WHERE `name` = ?", true, 'nsfw'); + + return Update::SUCCESS; +} From c0cd0dc74d58dc56669929caff571fde809a1fd6 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 5 Feb 2024 12:21:57 +0000 Subject: [PATCH 2/2] "sensitive" added to fierld list --- src/Model/Item.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Model/Item.php b/src/Model/Item.php index ca63a923cc..9d2bdd16b9 100644 --- a/src/Model/Item.php +++ b/src/Model/Item.php @@ -119,7 +119,7 @@ class Item const DELIVER_FIELDLIST = [ 'uid', 'id', 'parent', 'uri-id', 'uri', 'thr-parent', 'parent-uri', 'guid', 'parent-guid', 'conversation', 'received', 'created', 'edited', 'verb', 'object-type', 'object', 'target', - 'private', 'title', 'body', 'raw-body', 'language', 'location', 'coord', 'app', + 'private', 'title', 'body', 'raw-body', 'language', 'location', 'coord', 'app', 'sensitive', 'inform', 'deleted', 'extid', 'post-type', 'post-reason', 'gravity', 'allow_cid', 'allow_gid', 'deny_cid', 'deny_gid', 'author-id', 'author-addr', 'author-link', 'author-name', 'author-avatar', 'owner-id', 'owner-link', 'contact-uid',