From b6da15557b6d5c8a8eb5e20c1c9bb937440e2927 Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 31 Jul 2022 15:54:35 +0000 Subject: [PATCH 1/3] The post-reason / protocol is now filled in most cases --- database.sql | 3 +- src/Model/Conversation.php | 6 ++- src/Model/Item.php | 14 ++++-- src/Module/Api/Mastodon/PushSubscription.php | 4 +- src/Protocol/DFRN.php | 6 +-- src/Protocol/Diaspora.php | 51 +++++++++++++------- src/Protocol/Feed.php | 7 +++ src/Worker/OnePoll.php | 4 ++ static/dbstructure.config.php | 2 +- static/dbview.config.php | 1 + 10 files changed, 69 insertions(+), 29 deletions(-) diff --git a/database.sql b/database.sql index 02e21f137e..a2f09e1c38 100644 --- a/database.sql +++ b/database.sql @@ -1,6 +1,6 @@ -- ------------------------------------------ -- Friendica 2022.09-dev (Giant Rhubarb) --- DB_UPDATE_VERSION 1475 +-- DB_UPDATE_VERSION 1476 -- ------------------------------------------ @@ -1774,6 +1774,7 @@ CREATE VIEW `post-user-view` AS SELECT `post-user`.`global` AS `global`, EXISTS(SELECT `type` FROM `post-collection` WHERE `type` = 0 AND `uri-id` = `post-user`.`uri-id`) AS `featured`, `post-user`.`network` AS `network`, + `post-user`.`protocol` AS `protocol`, `post-user`.`vid` AS `vid`, `post-user`.`psid` AS `psid`, IF (`post-user`.`vid` IS NULL, '', `verb`.`name`) AS `verb`, diff --git a/src/Model/Conversation.php b/src/Model/Conversation.php index 4739b2f0a1..ef48620798 100644 --- a/src/Model/Conversation.php +++ b/src/Model/Conversation.php @@ -32,11 +32,15 @@ class Conversation const PARCEL_DIASPORA = 2; const PARCEL_SALMON = 3; const PARCEL_FEED = 4; // Deprecated - const PARCEL_SPLIT_CONVERSATION = 6; + const PARCEL_SPLIT_CONVERSATION = 6; // @deprecated since version 2022.09 const PARCEL_LEGACY_DFRN = 7; // @deprecated since version 2021.09 const PARCEL_DIASPORA_DFRN = 8; const PARCEL_LOCAL_DFRN = 9; const PARCEL_DIRECT = 10; + const PARCEL_IMAP = 11; + const PARCEL_RDF = 12; + const PARCEL_RSS = 13; + const PARCEL_ATOM = 14; const PARCEL_TWITTER = 67; const PARCEL_UNKNOWN = 255; diff --git a/src/Model/Item.php b/src/Model/Item.php index 6772a85fa2..b12eab9483 100644 --- a/src/Model/Item.php +++ b/src/Model/Item.php @@ -124,7 +124,7 @@ class Item // All fields in the item table const ITEM_FIELDLIST = ['id', 'uid', 'parent', 'uri', 'parent-uri', 'thr-parent', 'guid', 'uri-id', 'parent-uri-id', 'thr-parent-id', 'conversation', 'vid', - 'contact-id', 'wall', 'gravity', 'extid', 'psid', + 'contact-id', 'wall', 'gravity', 'extid', 'psid', 'created', 'edited', 'commented', 'received', 'changed', 'verb', 'postopts', 'plink', 'resource-id', 'event-id', 'inform', 'allow_cid', 'allow_gid', 'deny_cid', 'deny_gid', 'post-type', 'post-reason', @@ -813,6 +813,10 @@ class Item unset($item['conversation-href']); unset($item['source']); + if (in_array($item['network'], Protocol::FEDERATED) && (!isset($item['protocol']) || is_null($item['protocol']))) { + Logger::notice('Blubb', ['guid' => $item['guid'], 'uri-id' => $item['uri-id'], 'uri' => $item['uri'], 'callstack' => System::callstack(20)]); + } + /* * Do we already have this item? * We have to check several networks since Friendica posts could be repeated @@ -1391,7 +1395,7 @@ class Item $condition = ['id' => $itemid, 'uid' => 0, 'network' => array_merge(Protocol::FEDERATED ,['']), 'visible' => true, 'deleted' => false, 'private' => [self::PUBLIC, self::UNLISTED]]; - $item = Post::selectFirst(self::ITEM_FIELDLIST, $condition); + $item = Post::selectFirst(array_merge(self::ITEM_FIELDLIST, ['protocol']), $condition); if (!DBA::isResult($item)) { Logger::warning('Item not found', ['condition' => $condition]); return; @@ -1480,7 +1484,7 @@ class Item return 0; } - $item = Post::selectFirst(self::ITEM_FIELDLIST, ['uri-id' => $uri_id, 'uid' => $source_uid]); + $item = Post::selectFirst(array_merge(self::ITEM_FIELDLIST, ['protocol']), ['uri-id' => $uri_id, 'uid' => $source_uid]); if (!DBA::isResult($item)) { Logger::warning('Item could not be fetched', ['uri-id' => $uri_id, 'uid' => $source_uid]); return 0; @@ -1697,7 +1701,7 @@ class Item return; } - $item = Post::selectFirst(self::ITEM_FIELDLIST, ['id' => $itemid]); + $item = Post::selectFirst(array_merge(self::ITEM_FIELDLIST, ['protocol']), ['id' => $itemid]); if (DBA::isResult($item)) { // Preparing public shadow (removing user specific data) @@ -1733,7 +1737,7 @@ class Item */ private static function addShadowPost(int $itemid) { - $item = Post::selectFirst(self::ITEM_FIELDLIST, ['id' => $itemid]); + $item = Post::selectFirst(array_merge(self::ITEM_FIELDLIST, ['protocol']), ['id' => $itemid]); if (!DBA::isResult($item)) { return; } diff --git a/src/Module/Api/Mastodon/PushSubscription.php b/src/Module/Api/Mastodon/PushSubscription.php index dc07ab55aa..9ddcfce36c 100644 --- a/src/Module/Api/Mastodon/PushSubscription.php +++ b/src/Module/Api/Mastodon/PushSubscription.php @@ -43,7 +43,9 @@ class PushSubscription extends BaseApi 'subscription' => [], 'data' => [], ], $request); - +/* +2022-07-31T08:08:11Z index [ERROR]: DB Error {"code":1366,"error":"Incorrect integer value: 'true' for column `piratica`.`subscription`.`follow` at row 1","callstack":"DBA::replace, Subscription::replace, PushSubscription::post, BaseModule::run, BaseApi::run, App::runFrontend","params":"REPLACE `subscription` (`application-id`, `uid`, `endpoint`, `pubkey`, `secret`, `follow`, `favourite`, `reblog`, `mention`, `poll`, `follow_request`, `status`) VALUES (213, 130, 'https://ntfy.sh/upNpejYKlqt5du?up=1', 'BEiGDNV6jsPwdtP186ZLbpqewcWBJHzM0qboxp8fVGoVxVUy6xiir_2RO4gM2FnE9sVg58sQdNuyDrr1jOmMj9Y', 'En7GzwQO8xuvXka5bIF3Sg', 'true', 'true', 'true', 'true', 'true', 0, 0)"} - {"file":"Database.php","line":801,"function":"e","uid":"c35eee","process_id":1404415} +*/ $subscription = [ 'application-id' => $application['id'], 'uid' => $uid, diff --git a/src/Protocol/DFRN.php b/src/Protocol/DFRN.php index 3d4c137a5e..fd5739b62d 100644 --- a/src/Protocol/DFRN.php +++ b/src/Protocol/DFRN.php @@ -1830,8 +1830,6 @@ class DFRN $item = $header; - $item['protocol'] = $protocol; - $item['source'] = $xml; // Get the uri @@ -2260,10 +2258,12 @@ class DFRN $header = []; $header['uid'] = $importer['importer_uid']; $header['network'] = Protocol::DFRN; + $header['protocol'] = $protocol; $header['wall'] = 0; $header['origin'] = 0; $header['contact-id'] = $importer['id']; - $header['direction'] = $direction; + + $header = Diaspora::setDirection($header, $direction); if ($direction === Conversation::RELAY) { $header['post-reason'] = Item::PR_RELAY; diff --git a/src/Protocol/Diaspora.php b/src/Protocol/Diaspora.php index c5ad51215b..bec4fd333f 100644 --- a/src/Protocol/Diaspora.php +++ b/src/Protocol/Diaspora.php @@ -1535,13 +1535,7 @@ class Diaspora $datarray['owner-id'] = Contact::getIdForURL($contact['url'], 0); // Will be overwritten for sharing accounts in Item::insert - if (in_array($direction, [self::FETCHED, self::FORCED_FETCH])) { - $datarray['post-reason'] = Item::PR_FETCHED; - } elseif ($datarray['uid'] == 0) { - $datarray['post-reason'] = Item::PR_GLOBAL; - } else { - $datarray['post-reason'] = Item::PR_COMMENT; - } + $datarray = self::setDirection($datarray, $direction); $datarray['guid'] = $guid; $datarray['uri'] = self::getUriFromGuid($author, $guid); @@ -1557,7 +1551,8 @@ class Diaspora $datarray['protocol'] = Conversation::PARCEL_DIASPORA; $datarray['source'] = $xml; - $datarray['direction'] = in_array($direction, [self::FETCHED, self::FORCED_FETCH]) ? Conversation::PULL : Conversation::PUSH; + + $datarray = self::setDirection($datarray, $direction); $datarray['changed'] = $datarray['created'] = $datarray['edited'] = $created_at; @@ -1786,12 +1781,13 @@ class Diaspora $datarray = []; $datarray['protocol'] = Conversation::PARCEL_DIASPORA; - $datarray['direction'] = in_array($direction, [self::FETCHED, self::FORCED_FETCH]) ? Conversation::PULL : Conversation::PUSH; $datarray['uid'] = $importer['uid']; $datarray['contact-id'] = $author_contact['cid']; $datarray['network'] = $author_contact['network']; + $datarray = self::setDirection($datarray, $direction); + $datarray['owner-link'] = $datarray['author-link'] = $person['url']; $datarray['owner-id'] = $datarray['author-id'] = Contact::getIdForURL($person['url'], 0); @@ -1965,12 +1961,13 @@ class Diaspora $datarray = []; $datarray['protocol'] = Conversation::PARCEL_DIASPORA; - $datarray['direction'] = in_array($direction, [self::FETCHED, self::FORCED_FETCH]) ? Conversation::PULL : Conversation::PUSH; $datarray['uid'] = $importer['uid']; $datarray['contact-id'] = $author_contact['cid']; $datarray['network'] = $author_contact['network']; + $datarray = self::setDirection($datarray, $direction); + $datarray['owner-link'] = $datarray['author-link'] = $person['url']; $datarray['owner-id'] = $datarray['author-id'] = Contact::getIdForURL($person['url'], 0); @@ -2382,6 +2379,7 @@ class Diaspora $datarray['protocol'] = $item['protocol']; $datarray['source'] = $item['source']; $datarray['direction'] = $item['direction']; + $datarray['post-reason'] = $item['post-reason']; $datarray['plink'] = self::plink($author, $datarray['guid']); $datarray['private'] = $item['private']; @@ -2468,7 +2466,8 @@ class Diaspora $datarray['protocol'] = Conversation::PARCEL_DIASPORA; $datarray['source'] = $xml; - $datarray['direction'] = in_array($direction, [self::FETCHED, self::FORCED_FETCH]) ? Conversation::PULL : Conversation::PUSH; + + $datarray = self::setDirection($datarray, $direction); /// @todo Copy tag data from original post @@ -2690,6 +2689,29 @@ class Diaspora Post\Media::insert($data); } + /** + * Set direction and post reason + * + * @param array $datarray + * @param integer $direction + * + * @return array + */ + public static function setDirection(array $datarray, int $direction): array + { + $datarray['direction'] = in_array($direction, [self::FETCHED, self::FORCED_FETCH]) ? Conversation::PULL : Conversation::PUSH; + + if (in_array($direction, [self::FETCHED, self::FORCED_FETCH])) { + $datarray['post-reason'] = Item::PR_FETCHED; + } elseif ($datarray['uid'] == 0) { + $datarray['post-reason'] = Item::PR_GLOBAL; + } else { + $datarray['post-reason'] = Item::PR_PUSHED; + } + + return $datarray; + } + /** * Receives status messages * @@ -2780,13 +2802,8 @@ class Diaspora $datarray['protocol'] = Conversation::PARCEL_DIASPORA; $datarray['source'] = $xml; - $datarray['direction'] = in_array($direction, [self::FETCHED, self::FORCED_FETCH]) ? Conversation::PULL : Conversation::PUSH; - if (in_array($direction, [self::FETCHED, self::FORCED_FETCH])) { - $datarray['post-reason'] = Item::PR_FETCHED; - } elseif ($datarray['uid'] == 0) { - $datarray['post-reason'] = Item::PR_GLOBAL; - } + $datarray = self::setDirection($datarray, $direction); $datarray['body'] = self::replacePeopleGuid($body, $contact['url']); $datarray['raw-body'] = self::replacePeopleGuid($raw_body, $contact['url']); diff --git a/src/Protocol/Feed.php b/src/Protocol/Feed.php index 6601a2e8c3..7ff86c1db7 100644 --- a/src/Protocol/Feed.php +++ b/src/Protocol/Feed.php @@ -33,6 +33,7 @@ use Friendica\Core\Protocol; use Friendica\Database\DBA; use Friendica\DI; use Friendica\Model\Contact; +use Friendica\Model\Conversation; use Friendica\Model\Item; use Friendica\Model\Post; use Friendica\Model\Tag; @@ -98,9 +99,11 @@ class Feed $author = []; $entries = null; + $protocol = Conversation::PARCEL_UNKNOWN; // Is it RDF? if ($xpath->query('/rdf:RDF/rss:channel')->length > 0) { + $protocol = Conversation::PARCEL_RDF; $author['author-link'] = XML::getFirstNodeValue($xpath, '/rdf:RDF/rss:channel/rss:link/text()'); $author['author-name'] = XML::getFirstNodeValue($xpath, '/rdf:RDF/rss:channel/rss:title/text()'); @@ -112,6 +115,7 @@ class Feed // Is it Atom? if ($xpath->query('/atom:feed')->length > 0) { + $protocol = Conversation::PARCEL_ATOM; $alternate = XML::getFirstAttributes($xpath, "atom:link[@rel='alternate']"); if (is_object($alternate)) { foreach ($alternate as $attribute) { @@ -195,6 +199,7 @@ class Feed // Is it RSS? if ($xpath->query('/rss/channel')->length > 0) { + $protocol = Conversation::PARCEL_RSS; $author['author-link'] = XML::getFirstNodeValue($xpath, '/rss/channel/link/text()'); $author['author-name'] = XML::getFirstNodeValue($xpath, '/rss/channel/title/text()'); @@ -250,6 +255,8 @@ class Feed $header = []; $header['uid'] = $importer['uid'] ?? 0; $header['network'] = Protocol::FEED; + $datarray['protocol'] = $protocol; + $datarray['direction'] = Conversation::PULL; $header['wall'] = 0; $header['origin'] = 0; $header['gravity'] = GRAVITY_PARENT; diff --git a/src/Worker/OnePoll.php b/src/Worker/OnePoll.php index 65f2277b73..719bd373c5 100644 --- a/src/Worker/OnePoll.php +++ b/src/Worker/OnePoll.php @@ -27,6 +27,7 @@ use Friendica\Core\System; use Friendica\Database\DBA; use Friendica\DI; use Friendica\Model\Contact; +use Friendica\Model\Conversation; use Friendica\Model\Item; use Friendica\Model\Post; use Friendica\Model\User; @@ -247,6 +248,9 @@ class OnePoll $datarray['verb'] = Activity::POST; $datarray['object-type'] = Activity\ObjectType::NOTE; $datarray['network'] = Protocol::MAIL; + $datarray['protocol'] = Conversation::PARCEL_IMAP; + $datarray['direction'] = Conversation::PULL; + // $meta = Email::messageMeta($mbox, $msg_uid); $datarray['thr-parent'] = $datarray['uri'] = Email::msgid2iri(trim($meta->message_id, '<>')); diff --git a/static/dbstructure.config.php b/static/dbstructure.config.php index cff7c1cd19..79da57a2bb 100644 --- a/static/dbstructure.config.php +++ b/static/dbstructure.config.php @@ -55,7 +55,7 @@ use Friendica\Database\DBA; if (!defined('DB_UPDATE_VERSION')) { - define('DB_UPDATE_VERSION', 1475); + define('DB_UPDATE_VERSION', 1476); } return [ diff --git a/static/dbview.config.php b/static/dbview.config.php index 0c91016d49..875273cb02 100644 --- a/static/dbview.config.php +++ b/static/dbview.config.php @@ -96,6 +96,7 @@ "global" => ["post-user", "global"], "featured" => "EXISTS(SELECT `type` FROM `post-collection` WHERE `type` = 0 AND `uri-id` = `post-user`.`uri-id`)", "network" => ["post-user", "network"], + "protocol" => ["post-user", "protocol"], "vid" => ["post-user", "vid"], "psid" => ["post-user", "psid"], "verb" => "IF (`post-user`.`vid` IS NULL, '', `verb`.`name`)", From 0347f4fc66d8769822eb0a3bdcfff0100fed446c Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 31 Jul 2022 16:01:56 +0000 Subject: [PATCH 2/3] Remove test code --- src/Model/Item.php | 2 +- src/Module/Api/Mastodon/PushSubscription.php | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/Model/Item.php b/src/Model/Item.php index b12eab9483..91fd59d806 100644 --- a/src/Model/Item.php +++ b/src/Model/Item.php @@ -124,7 +124,7 @@ class Item // All fields in the item table const ITEM_FIELDLIST = ['id', 'uid', 'parent', 'uri', 'parent-uri', 'thr-parent', 'guid', 'uri-id', 'parent-uri-id', 'thr-parent-id', 'conversation', 'vid', - 'contact-id', 'wall', 'gravity', 'extid', 'psid', + 'contact-id', 'wall', 'gravity', 'extid', 'psid', 'created', 'edited', 'commented', 'received', 'changed', 'verb', 'postopts', 'plink', 'resource-id', 'event-id', 'inform', 'allow_cid', 'allow_gid', 'deny_cid', 'deny_gid', 'post-type', 'post-reason', diff --git a/src/Module/Api/Mastodon/PushSubscription.php b/src/Module/Api/Mastodon/PushSubscription.php index 9ddcfce36c..dc07ab55aa 100644 --- a/src/Module/Api/Mastodon/PushSubscription.php +++ b/src/Module/Api/Mastodon/PushSubscription.php @@ -43,9 +43,7 @@ class PushSubscription extends BaseApi 'subscription' => [], 'data' => [], ], $request); -/* -2022-07-31T08:08:11Z index [ERROR]: DB Error {"code":1366,"error":"Incorrect integer value: 'true' for column `piratica`.`subscription`.`follow` at row 1","callstack":"DBA::replace, Subscription::replace, PushSubscription::post, BaseModule::run, BaseApi::run, App::runFrontend","params":"REPLACE `subscription` (`application-id`, `uid`, `endpoint`, `pubkey`, `secret`, `follow`, `favourite`, `reblog`, `mention`, `poll`, `follow_request`, `status`) VALUES (213, 130, 'https://ntfy.sh/upNpejYKlqt5du?up=1', 'BEiGDNV6jsPwdtP186ZLbpqewcWBJHzM0qboxp8fVGoVxVUy6xiir_2RO4gM2FnE9sVg58sQdNuyDrr1jOmMj9Y', 'En7GzwQO8xuvXka5bIF3Sg', 'true', 'true', 'true', 'true', 'true', 0, 0)"} - {"file":"Database.php","line":801,"function":"e","uid":"c35eee","process_id":1404415} -*/ + $subscription = [ 'application-id' => $application['id'], 'uid' => $uid, From 3945de5e3b293bed0c2dcfab29cf85568407a6d4 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 1 Aug 2022 00:58:56 +0000 Subject: [PATCH 3/3] No blubb allowed --- src/Model/Item.php | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/Model/Item.php b/src/Model/Item.php index 91fd59d806..7eb36d85e0 100644 --- a/src/Model/Item.php +++ b/src/Model/Item.php @@ -813,10 +813,6 @@ class Item unset($item['conversation-href']); unset($item['source']); - if (in_array($item['network'], Protocol::FEDERATED) && (!isset($item['protocol']) || is_null($item['protocol']))) { - Logger::notice('Blubb', ['guid' => $item['guid'], 'uri-id' => $item['uri-id'], 'uri' => $item['uri'], 'callstack' => System::callstack(20)]); - } - /* * Do we already have this item? * We have to check several networks since Friendica posts could be repeated