From 2eadad49db5e6865df435dc049178d13ba44a3e6 Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 25 Jun 2024 11:39:30 +0000 Subject: [PATCH] Fetch replies to fetched posts --- database.sql | 26 ++++++- doc/database/db_post-user.md | 69 +++++++++-------- doc/database/db_post.md | 43 ++++++----- src/Model/Item.php | 8 ++ src/Protocol/ActivityPub.php | 14 +++- src/Protocol/ActivityPub/Processor.php | 101 ++++++++++++++++++++++--- src/Protocol/ActivityPub/Receiver.php | 1 + src/Worker/ExpirePosts.php | 1 + static/dbstructure.config.php | 6 +- static/dbview.config.php | 18 +++++ 10 files changed, 217 insertions(+), 70 deletions(-) diff --git a/database.sql b/database.sql index 148dcd2626..d24a4f4009 100644 --- a/database.sql +++ b/database.sql @@ -1,6 +1,6 @@ -- ------------------------------------------ -- Friendica 2024.06-rc (Yellow Archangel) --- DB_UPDATE_VERSION 1568 +-- DB_UPDATE_VERSION 1569 -- ------------------------------------------ @@ -1186,6 +1186,7 @@ CREATE TABLE IF NOT EXISTS `post` ( `parent-uri-id` int unsigned COMMENT 'Id of the item-uri table that contains the parent uri', `thr-parent-id` int unsigned COMMENT 'Id of the item-uri table that contains the thread parent uri', `external-id` int unsigned COMMENT 'Id of the item-uri table entry that contains the external uri', + `replies-id` int unsigned COMMENT 'Id of the item-uri table entry that contains the endpoint for the replies collection', `created` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Creation timestamp.', `edited` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Date of last edit (default is created)', `received` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'datetime', @@ -1204,6 +1205,7 @@ CREATE TABLE IF NOT EXISTS `post` ( INDEX `parent-uri-id` (`parent-uri-id`), INDEX `thr-parent-id` (`thr-parent-id`), INDEX `external-id` (`external-id`), + INDEX `replies-id` (`replies-id`), INDEX `owner-id` (`owner-id`), INDEX `author-id` (`author-id`), INDEX `causer-id` (`causer-id`), @@ -1212,6 +1214,7 @@ CREATE TABLE IF NOT EXISTS `post` ( FOREIGN KEY (`parent-uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE, FOREIGN KEY (`thr-parent-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE, FOREIGN KEY (`external-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE, + FOREIGN KEY (`replies-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE, FOREIGN KEY (`owner-id`) REFERENCES `contact` (`id`) ON UPDATE RESTRICT ON DELETE RESTRICT, FOREIGN KEY (`author-id`) REFERENCES `contact` (`id`) ON UPDATE RESTRICT ON DELETE RESTRICT, FOREIGN KEY (`causer-id`) REFERENCES `contact` (`id`) ON UPDATE RESTRICT ON DELETE RESTRICT, @@ -1573,6 +1576,7 @@ CREATE TABLE IF NOT EXISTS `post-user` ( `parent-uri-id` int unsigned COMMENT 'Id of the item-uri table that contains the parent uri', `thr-parent-id` int unsigned COMMENT 'Id of the item-uri table that contains the thread parent uri', `external-id` int unsigned COMMENT 'Id of the item-uri table entry that contains the external uri', + `replies-id` int unsigned COMMENT 'Id of the item-uri table entry that contains the endpoint for the replies collection', `created` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Creation timestamp.', `edited` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Date of last edit (default is created)', `received` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'datetime', @@ -1605,6 +1609,7 @@ CREATE TABLE IF NOT EXISTS `post-user` ( INDEX `parent-uri-id` (`parent-uri-id`), INDEX `thr-parent-id` (`thr-parent-id`), INDEX `external-id` (`external-id`), + INDEX `replies-id` (`replies-id`), INDEX `owner-id` (`owner-id`), INDEX `author-id` (`author-id`), INDEX `causer-id` (`causer-id`), @@ -1625,6 +1630,7 @@ CREATE TABLE IF NOT EXISTS `post-user` ( FOREIGN KEY (`parent-uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE, FOREIGN KEY (`thr-parent-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE, FOREIGN KEY (`external-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE, + FOREIGN KEY (`replies-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE, FOREIGN KEY (`owner-id`) REFERENCES `contact` (`id`) ON UPDATE RESTRICT ON DELETE RESTRICT, FOREIGN KEY (`author-id`) REFERENCES `contact` (`id`) ON UPDATE RESTRICT ON DELETE RESTRICT, FOREIGN KEY (`causer-id`) REFERENCES `contact` (`id`) ON UPDATE RESTRICT ON DELETE RESTRICT, @@ -2281,6 +2287,8 @@ CREATE VIEW `post-origin-view` AS SELECT `post-origin`.`gravity` AS `gravity`, `external-item-uri`.`uri` AS `extid`, `post-user`.`external-id` AS `external-id`, + `replies-item-uri`.`uri` AS `replies`, + `post-user`.`replies-id` AS `replies-id`, `post-origin`.`created` AS `created`, `post-user`.`edited` AS `edited`, `post-thread-user`.`commented` AS `commented`, @@ -2435,6 +2443,7 @@ CREATE VIEW `post-origin-view` AS SELECT LEFT JOIN `item-uri` AS `parent-item-uri` ON `parent-item-uri`.`id` = `post-origin`.`parent-uri-id` LEFT JOIN `item-uri` AS `conversation-item-uri` ON `conversation-item-uri`.`id` = `post-thread-user`.`conversation-id` LEFT JOIN `item-uri` AS `external-item-uri` ON `external-item-uri`.`id` = `post-user`.`external-id` + LEFT JOIN `item-uri` AS `replies-item-uri` ON `replies-item-uri`.`id` = `post-user`.`replies-id` LEFT JOIN `verb` ON `verb`.`id` = `post-origin`.`vid` LEFT JOIN `event` ON `event`.`id` = `post-user`.`event-id` LEFT JOIN `diaspora-interaction` ON `diaspora-interaction`.`uri-id` = `post-origin`.`uri-id` @@ -2469,6 +2478,8 @@ CREATE VIEW `post-thread-origin-view` AS SELECT `post-origin`.`gravity` AS `gravity`, `external-item-uri`.`uri` AS `extid`, `post-user`.`external-id` AS `external-id`, + `replies-item-uri`.`uri` AS `replies`, + `post-user`.`replies-id` AS `replies-id`, `post-origin`.`created` AS `created`, `post-user`.`edited` AS `edited`, `post-thread-user`.`commented` AS `commented`, @@ -2622,6 +2633,7 @@ CREATE VIEW `post-thread-origin-view` AS SELECT LEFT JOIN `item-uri` AS `parent-item-uri` ON `parent-item-uri`.`id` = `post-origin`.`parent-uri-id` LEFT JOIN `item-uri` AS `conversation-item-uri` ON `conversation-item-uri`.`id` = `post-thread-user`.`conversation-id` LEFT JOIN `item-uri` AS `external-item-uri` ON `external-item-uri`.`id` = `post-user`.`external-id` + LEFT JOIN `item-uri` AS `replies-item-uri` ON `replies-item-uri`.`id` = `post-user`.`replies-id` LEFT JOIN `verb` ON `verb`.`id` = `post-origin`.`vid` LEFT JOIN `event` ON `event`.`id` = `post-user`.`event-id` LEFT JOIN `diaspora-interaction` ON `diaspora-interaction`.`uri-id` = `post-origin`.`uri-id` @@ -2655,6 +2667,8 @@ CREATE VIEW `post-user-view` AS SELECT `post-user`.`gravity` AS `gravity`, `external-item-uri`.`uri` AS `extid`, `post-user`.`external-id` AS `external-id`, + `replies-item-uri`.`uri` AS `replies`, + `post-user`.`replies-id` AS `replies-id`, `post-user`.`created` AS `created`, `post-user`.`edited` AS `edited`, `post-thread-user`.`commented` AS `commented`, @@ -2808,6 +2822,7 @@ CREATE VIEW `post-user-view` AS SELECT LEFT JOIN `item-uri` AS `parent-item-uri` ON `parent-item-uri`.`id` = `post-user`.`parent-uri-id` LEFT JOIN `item-uri` AS `conversation-item-uri` ON `conversation-item-uri`.`id` = `post-thread-user`.`conversation-id` LEFT JOIN `item-uri` AS `external-item-uri` ON `external-item-uri`.`id` = `post-user`.`external-id` + LEFT JOIN `item-uri` AS `replies-item-uri` ON `replies-item-uri`.`id` = `post-user`.`replies-id` LEFT JOIN `verb` ON `verb`.`id` = `post-user`.`vid` LEFT JOIN `event` ON `event`.`id` = `post-user`.`event-id` LEFT JOIN `diaspora-interaction` ON `diaspora-interaction`.`uri-id` = `post-user`.`uri-id` @@ -2842,6 +2857,8 @@ CREATE VIEW `post-thread-user-view` AS SELECT `post-user`.`gravity` AS `gravity`, `external-item-uri`.`uri` AS `extid`, `post-user`.`external-id` AS `external-id`, + `replies-item-uri`.`uri` AS `replies`, + `post-user`.`replies-id` AS `replies-id`, `post-thread-user`.`created` AS `created`, `post-user`.`edited` AS `edited`, `post-thread-user`.`commented` AS `commented`, @@ -2994,6 +3011,7 @@ CREATE VIEW `post-thread-user-view` AS SELECT LEFT JOIN `item-uri` AS `parent-item-uri` ON `parent-item-uri`.`id` = `post-user`.`parent-uri-id` LEFT JOIN `item-uri` AS `conversation-item-uri` ON `conversation-item-uri`.`id` = `post-thread-user`.`conversation-id` LEFT JOIN `item-uri` AS `external-item-uri` ON `external-item-uri`.`id` = `post-user`.`external-id` + LEFT JOIN `item-uri` AS `replies-item-uri` ON `replies-item-uri`.`id` = `post-user`.`replies-id` LEFT JOIN `verb` ON `verb`.`id` = `post-user`.`vid` LEFT JOIN `event` ON `event`.`id` = `post-user`.`event-id` LEFT JOIN `diaspora-interaction` ON `diaspora-interaction`.`uri-id` = `post-thread-user`.`uri-id` @@ -3022,6 +3040,8 @@ CREATE VIEW `post-view` AS SELECT `post`.`gravity` AS `gravity`, `external-item-uri`.`uri` AS `extid`, `post`.`external-id` AS `external-id`, + `replies-item-uri`.`uri` AS `replies`, + `post`.`replies-id` AS `replies-id`, `post`.`created` AS `created`, `post`.`edited` AS `edited`, `post-thread`.`commented` AS `commented`, @@ -3143,6 +3163,7 @@ CREATE VIEW `post-view` AS SELECT LEFT JOIN `item-uri` AS `parent-item-uri` ON `parent-item-uri`.`id` = `post`.`parent-uri-id` LEFT JOIN `item-uri` AS `conversation-item-uri` ON `conversation-item-uri`.`id` = `post-thread`.`conversation-id` LEFT JOIN `item-uri` AS `external-item-uri` ON `external-item-uri`.`id` = `post`.`external-id` + LEFT JOIN `item-uri` AS `replies-item-uri` ON `replies-item-uri`.`id` = `post`.`replies-id` LEFT JOIN `verb` ON `verb`.`id` = `post`.`vid` LEFT JOIN `diaspora-interaction` ON `diaspora-interaction`.`uri-id` = `post`.`uri-id` LEFT JOIN `post-content` ON `post-content`.`uri-id` = `post`.`uri-id` @@ -3169,6 +3190,8 @@ CREATE VIEW `post-thread-view` AS SELECT `post`.`gravity` AS `gravity`, `external-item-uri`.`uri` AS `extid`, `post`.`external-id` AS `external-id`, + `replies-item-uri`.`uri` AS `replies`, + `post`.`replies-id` AS `replies-id`, `post-thread`.`created` AS `created`, `post`.`edited` AS `edited`, `post-thread`.`commented` AS `commented`, @@ -3292,6 +3315,7 @@ CREATE VIEW `post-thread-view` AS SELECT LEFT JOIN `item-uri` AS `parent-item-uri` ON `parent-item-uri`.`id` = `post`.`parent-uri-id` LEFT JOIN `item-uri` AS `conversation-item-uri` ON `conversation-item-uri`.`id` = `post-thread`.`conversation-id` LEFT JOIN `item-uri` AS `external-item-uri` ON `external-item-uri`.`id` = `post`.`external-id` + LEFT JOIN `item-uri` AS `replies-item-uri` ON `replies-item-uri`.`id` = `post`.`replies-id` LEFT JOIN `verb` ON `verb`.`id` = `post`.`vid` LEFT JOIN `diaspora-interaction` ON `diaspora-interaction`.`uri-id` = `post-thread`.`uri-id` LEFT JOIN `post-content` ON `post-content`.`uri-id` = `post-thread`.`uri-id` diff --git a/doc/database/db_post-user.md b/doc/database/db_post-user.md index c20d8bcb47..f702502bf2 100644 --- a/doc/database/db_post-user.md +++ b/doc/database/db_post-user.md @@ -6,39 +6,40 @@ User specific post data Fields ------ -| Field | Description | Type | Null | Key | Default | Extra | -| ----------------- | --------------------------------------------------------------------------------- | ------------------ | ---- | --- | ------------------- | -------------- | -| id | | int unsigned | NO | PRI | NULL | auto_increment | -| uri-id | Id of the item-uri table entry that contains the item uri | int unsigned | NO | | NULL | | -| parent-uri-id | Id of the item-uri table that contains the parent uri | int unsigned | YES | | NULL | | -| thr-parent-id | Id of the item-uri table that contains the thread parent uri | int unsigned | YES | | NULL | | -| external-id | Id of the item-uri table entry that contains the external uri | int unsigned | YES | | NULL | | -| created | Creation timestamp. | datetime | NO | | 0001-01-01 00:00:00 | | -| edited | Date of last edit (default is created) | datetime | NO | | 0001-01-01 00:00:00 | | -| received | datetime | datetime | NO | | 0001-01-01 00:00:00 | | -| gravity | | tinyint unsigned | NO | | 0 | | -| network | Network from where the item comes from | char(4) | NO | | | | -| owner-id | Link to the contact table with uid=0 of the owner of this item | int unsigned | NO | | 0 | | -| author-id | Link to the contact table with uid=0 of the author of this item | int unsigned | NO | | 0 | | -| causer-id | Link to the contact table with uid=0 of the contact that caused the item creation | int unsigned | YES | | NULL | | -| post-type | Post type (personal note, image, article, ...) | tinyint unsigned | NO | | 0 | | -| post-reason | Reason why the post arrived at the user | tinyint unsigned | NO | | 0 | | -| vid | Id of the verb table entry that contains the activity verbs | smallint unsigned | YES | | NULL | | -| private | 0=public, 1=private, 2=unlisted | tinyint unsigned | NO | | 0 | | -| restrictions | Bit array of post restrictions (1 = Reply, 2 = Like, 4 = Announce) | tinyint unsigned | YES | | NULL | | -| global | | boolean | NO | | 0 | | -| visible | | boolean | NO | | 0 | | -| deleted | item has been marked for deletion | boolean | NO | | 0 | | -| uid | Owner id which owns this copy of the item | mediumint unsigned | NO | | NULL | | -| protocol | Protocol used to deliver the item for this user | tinyint unsigned | YES | | NULL | | -| contact-id | contact.id | int unsigned | NO | | 0 | | -| event-id | Used to link to the event.id | int unsigned | YES | | NULL | | -| unseen | post has not been seen | boolean | NO | | 1 | | -| hidden | Marker to hide the post from the user | boolean | NO | | 0 | | -| notification-type | | smallint unsigned | NO | | 0 | | -| wall | This item was posted to the wall of uid | boolean | NO | | 0 | | -| origin | item originated at this site | boolean | NO | | 0 | | -| psid | ID of the permission set of this post | int unsigned | YES | | NULL | | +| Field | Description | Type | Null | Key | Default | Extra | +| ----------------- | ------------------------------------------------------------------------------------ | ------------------ | ---- | --- | ------------------- | -------------- | +| id | | int unsigned | NO | PRI | NULL | auto_increment | +| uri-id | Id of the item-uri table entry that contains the item uri | int unsigned | NO | | NULL | | +| parent-uri-id | Id of the item-uri table that contains the parent uri | int unsigned | YES | | NULL | | +| thr-parent-id | Id of the item-uri table that contains the thread parent uri | int unsigned | YES | | NULL | | +| external-id | Id of the item-uri table entry that contains the external uri | int unsigned | YES | | NULL | | +| replies-id | Id of the item-uri table entry that contains the endpoint for the replies collection | int unsigned | YES | | NULL | | +| created | Creation timestamp. | datetime | NO | | 0001-01-01 00:00:00 | | +| edited | Date of last edit (default is created) | datetime | NO | | 0001-01-01 00:00:00 | | +| received | datetime | datetime | NO | | 0001-01-01 00:00:00 | | +| gravity | | tinyint unsigned | NO | | 0 | | +| network | Network from where the item comes from | char(4) | NO | | | | +| owner-id | Link to the contact table with uid=0 of the owner of this item | int unsigned | NO | | 0 | | +| author-id | Link to the contact table with uid=0 of the author of this item | int unsigned | NO | | 0 | | +| causer-id | Link to the contact table with uid=0 of the contact that caused the item creation | int unsigned | YES | | NULL | | +| post-type | Post type (personal note, image, article, ...) | tinyint unsigned | NO | | 0 | | +| post-reason | Reason why the post arrived at the user | tinyint unsigned | NO | | 0 | | +| vid | Id of the verb table entry that contains the activity verbs | smallint unsigned | YES | | NULL | | +| private | 0=public, 1=private, 2=unlisted | tinyint unsigned | NO | | 0 | | +| restrictions | Bit array of post restrictions (1 = Reply, 2 = Like, 4 = Announce) | tinyint unsigned | YES | | NULL | | +| global | | boolean | NO | | 0 | | +| visible | | boolean | NO | | 0 | | +| deleted | item has been marked for deletion | boolean | NO | | 0 | | +| uid | Owner id which owns this copy of the item | mediumint unsigned | NO | | NULL | | +| protocol | Protocol used to deliver the item for this user | tinyint unsigned | YES | | NULL | | +| contact-id | contact.id | int unsigned | NO | | 0 | | +| event-id | Used to link to the event.id | int unsigned | YES | | NULL | | +| unseen | post has not been seen | boolean | NO | | 1 | | +| hidden | Marker to hide the post from the user | boolean | NO | | 0 | | +| notification-type | | smallint unsigned | NO | | 0 | | +| wall | This item was posted to the wall of uid | boolean | NO | | 0 | | +| origin | item originated at this site | boolean | NO | | 0 | | +| psid | ID of the permission set of this post | int unsigned | YES | | NULL | | Indexes ------------ @@ -51,6 +52,7 @@ Indexes | parent-uri-id | parent-uri-id | | thr-parent-id | thr-parent-id | | external-id | external-id | +| replies-id | replies-id | | owner-id | owner-id | | author-id | author-id | | causer-id | causer-id | @@ -77,6 +79,7 @@ Foreign Keys | parent-uri-id | [item-uri](help/database/db_item-uri) | id | | thr-parent-id | [item-uri](help/database/db_item-uri) | id | | external-id | [item-uri](help/database/db_item-uri) | id | +| replies-id | [item-uri](help/database/db_item-uri) | id | | owner-id | [contact](help/database/db_contact) | id | | author-id | [contact](help/database/db_contact) | id | | causer-id | [contact](help/database/db_contact) | id | diff --git a/doc/database/db_post.md b/doc/database/db_post.md index 303269b1c6..8f6690be06 100644 --- a/doc/database/db_post.md +++ b/doc/database/db_post.md @@ -6,26 +6,27 @@ Structure for all posts Fields ------ -| Field | Description | Type | Null | Key | Default | Extra | -| ------------- | --------------------------------------------------------------------------------- | ----------------- | ---- | --- | ------------------- | ----- | -| uri-id | Id of the item-uri table entry that contains the item uri | int unsigned | NO | PRI | NULL | | -| parent-uri-id | Id of the item-uri table that contains the parent uri | int unsigned | YES | | NULL | | -| thr-parent-id | Id of the item-uri table that contains the thread parent uri | int unsigned | YES | | NULL | | -| external-id | Id of the item-uri table entry that contains the external uri | int unsigned | YES | | NULL | | -| created | Creation timestamp. | datetime | NO | | 0001-01-01 00:00:00 | | -| edited | Date of last edit (default is created) | datetime | NO | | 0001-01-01 00:00:00 | | -| received | datetime | datetime | NO | | 0001-01-01 00:00:00 | | -| gravity | | tinyint unsigned | NO | | 0 | | -| network | Network from where the item comes from | char(4) | NO | | | | -| owner-id | Link to the contact table with uid=0 of the owner of this item | int unsigned | NO | | 0 | | -| author-id | Link to the contact table with uid=0 of the author of this item | int unsigned | NO | | 0 | | -| causer-id | Link to the contact table with uid=0 of the contact that caused the item creation | int unsigned | YES | | NULL | | -| post-type | Post type (personal note, image, article, ...) | tinyint unsigned | NO | | 0 | | -| vid | Id of the verb table entry that contains the activity verbs | smallint unsigned | YES | | NULL | | -| private | 0=public, 1=private, 2=unlisted | tinyint unsigned | NO | | 0 | | -| global | | boolean | NO | | 0 | | -| visible | | boolean | NO | | 0 | | -| deleted | item has been marked for deletion | boolean | NO | | 0 | | +| Field | Description | Type | Null | Key | Default | Extra | +| ------------- | ------------------------------------------------------------------------------------ | ----------------- | ---- | --- | ------------------- | ----- | +| uri-id | Id of the item-uri table entry that contains the item uri | int unsigned | NO | PRI | NULL | | +| parent-uri-id | Id of the item-uri table that contains the parent uri | int unsigned | YES | | NULL | | +| thr-parent-id | Id of the item-uri table that contains the thread parent uri | int unsigned | YES | | NULL | | +| external-id | Id of the item-uri table entry that contains the external uri | int unsigned | YES | | NULL | | +| replies-id | Id of the item-uri table entry that contains the endpoint for the replies collection | int unsigned | YES | | NULL | | +| created | Creation timestamp. | datetime | NO | | 0001-01-01 00:00:00 | | +| edited | Date of last edit (default is created) | datetime | NO | | 0001-01-01 00:00:00 | | +| received | datetime | datetime | NO | | 0001-01-01 00:00:00 | | +| gravity | | tinyint unsigned | NO | | 0 | | +| network | Network from where the item comes from | char(4) | NO | | | | +| owner-id | Link to the contact table with uid=0 of the owner of this item | int unsigned | NO | | 0 | | +| author-id | Link to the contact table with uid=0 of the author of this item | int unsigned | NO | | 0 | | +| causer-id | Link to the contact table with uid=0 of the contact that caused the item creation | int unsigned | YES | | NULL | | +| post-type | Post type (personal note, image, article, ...) | tinyint unsigned | NO | | 0 | | +| vid | Id of the verb table entry that contains the activity verbs | smallint unsigned | YES | | NULL | | +| private | 0=public, 1=private, 2=unlisted | tinyint unsigned | NO | | 0 | | +| global | | boolean | NO | | 0 | | +| visible | | boolean | NO | | 0 | | +| deleted | item has been marked for deletion | boolean | NO | | 0 | | Indexes ------------ @@ -36,6 +37,7 @@ Indexes | parent-uri-id | parent-uri-id | | thr-parent-id | thr-parent-id | | external-id | external-id | +| replies-id | replies-id | | owner-id | owner-id | | author-id | author-id | | causer-id | causer-id | @@ -50,6 +52,7 @@ Foreign Keys | parent-uri-id | [item-uri](help/database/db_item-uri) | id | | thr-parent-id | [item-uri](help/database/db_item-uri) | id | | external-id | [item-uri](help/database/db_item-uri) | id | +| replies-id | [item-uri](help/database/db_item-uri) | id | | owner-id | [contact](help/database/db_contact) | id | | author-id | [contact](help/database/db_contact) | id | | causer-id | [contact](help/database/db_contact) | id | diff --git a/src/Model/Item.php b/src/Model/Item.php index c9607839ea..14e2225d6a 100644 --- a/src/Model/Item.php +++ b/src/Model/Item.php @@ -198,6 +198,10 @@ class Item $fields['external-id'] = ItemURI::getIdByURI($fields['extid']); } + if (!empty($fields['replies'])) { + $fields['replies-id'] = ItemURI::getIdByURI($fields['replies']); + } + if (!empty($fields['verb'])) { $fields['vid'] = Verb::getID($fields['verb']); } @@ -1165,6 +1169,10 @@ class Item $item['external-id'] = ItemURI::getIdByURI($item['extid']); } + if (!empty($item['replies'])) { + $item['replies-id'] = ItemURI::getIdByURI($item['replies']); + } + if ($item['verb'] == Activity::ANNOUNCE) { self::setOwnerforResharedItem($item); } diff --git a/src/Protocol/ActivityPub.php b/src/Protocol/ActivityPub.php index 2608756b19..bf95011c32 100644 --- a/src/Protocol/ActivityPub.php +++ b/src/Protocol/ActivityPub.php @@ -88,9 +88,9 @@ class ActivityPub ]; const ACCOUNT_TYPES = ['Person', 'Organization', 'Service', 'Group', 'Application', 'Tombstone']; - CONST ARTICLE_DEFAULT = 0; - CONST ARTICLE_USE_SUMMARY = 1; - CONST ARTICLE_EMBED_TITLE = 2; + const ARTICLE_DEFAULT = 0; + const ARTICLE_USE_SUMMARY = 1; + const ARTICLE_EMBED_TITLE = 2; /** * Checks if the web request is done for the AP protocol @@ -259,13 +259,19 @@ class ActivityPub $items = $data['orderedItems']; } elseif (!empty($data['first']['orderedItems'])) { $items = $data['first']['orderedItems']; + } elseif (!empty($data['items'])) { + $items = $data['items']; + } elseif (!empty($data['first']['items'])) { + $items = $data['first']['items']; } elseif (!empty($data['first']) && is_string($data['first']) && ($data['first'] != $url)) { return self::fetchItems($data['first'], $uid, $start_timestamp); } else { return []; } - if (!empty($data['next']) && is_string($data['next'])) { + if (!empty($data['first']['next']) && is_string($data['first']['next'])) { + $items = array_merge($items, self::fetchItems($data['first']['next'], $uid, $start_timestamp)); + } elseif (!empty($data['next']) && is_string($data['next'])) { $items = array_merge($items, self::fetchItems($data['next'], $uid, $start_timestamp)); } diff --git a/src/Protocol/ActivityPub/Processor.php b/src/Protocol/ActivityPub/Processor.php index 2dd7c05d70..7a9ad827cb 100644 --- a/src/Protocol/ActivityPub/Processor.php +++ b/src/Protocol/ActivityPub/Processor.php @@ -130,7 +130,8 @@ class Processor */ private static function replaceEmojis(int $uri_id, string $body, array $emojis): string { - $body = strtr($body, + $body = strtr( + $body, array_combine( array_column($emojis, 'name'), array_map(function ($emoji) { @@ -494,6 +495,10 @@ class Processor $item['plink'] = $activity['alternate-url'] ?? $item['uri']; + if (!empty($activity['replies'])) { + $item['replies'] = $activity['replies']; + } + self::storeAttachments($activity, $item); self::storeQuestion($activity, $item); @@ -510,6 +515,27 @@ class Processor } } + if (DI::config()->get('system', 'decoupled_receiver')) { + $replies = [$item['thr-parent']]; + if (!empty($item['parent-uri'])) { + $replies[] = $item['parent-uri']; + } + $condition = DBA::mergeConditions(['uri' => $replies], ["`replies-id` IS NOT NULL"]); + $posts = Post::select(['replies', 'replies-id'], $condition); + while ($post = Post::fetch($posts)) { + $cachekey = 'Processor-CreateItem-Replies-' . $post['replies-id']; + if (!DI::cache()->get($cachekey)) { + self::fetchReplies($post['replies'], $activity); + DI::cache()->set($cachekey, true); + } + } + DBA::close($replies); + + if (!empty($activity['completion-mode']) && !empty($item['replies'])) { + self::fetchReplies($item['replies'], $activity); + } + } + return $item; } @@ -870,14 +896,15 @@ class Processor if ($id) { $shared_item = Post::selectFirst(['uri-id'], ['id' => $id]); $item['quote-uri-id'] = $shared_item['uri-id']; + Logger::debug('Quote is found', ['guid' => $item['guid'], 'uri-id' => $item['uri-id'], 'quote' => $activity['quote-url'], 'quote-uri-id' => $item['quote-uri-id']]); } elseif ($uri_id = ItemURI::getIdByURI($activity['quote-url'], false)) { - Logger::info('Quote was not fetched but the uri-id existed', ['guid' => $item['guid'], 'uri-id' => $item['uri-id'], 'quote' => $activity['quote-url'], 'uri-id' => $uri_id]); + Logger::info('Quote was not fetched but the uri-id existed', ['guid' => $item['guid'], 'uri-id' => $item['uri-id'], 'quote' => $activity['quote-url'], 'quote-uri-id' => $uri_id]); $item['quote-uri-id'] = $uri_id; } elseif (Queue::exists($activity['quote-url'], 'as:Create')) { - Logger::info('Quote is queued but not processed yet', ['guid' => $item['guid'], 'uri-id' => $item['uri-id'], 'quote' => $activity['quote-url'], 'uri-id' => $uri_id]); $item['quote-uri-id'] = ItemURI::getIdByURI($activity['quote-url']); + Logger::info('Quote is queued but not processed yet', ['guid' => $item['guid'], 'uri-id' => $item['uri-id'], 'quote' => $activity['quote-url'], 'quote-uri-id' => $item['quote-uri-id']]); } else { - Logger::info('Quote was not fetched', ['guid' => $item['guid'], 'uri-id' => $item['uri-id'], 'quote' => $activity['quote-url']]); + Logger::notice('Quote was not fetched', ['guid' => $item['guid'], 'uri-id' => $item['uri-id'], 'quote' => $activity['quote-url']]); } } @@ -931,14 +958,14 @@ class Processor } $item['restrictions'] = null; - foreach ($restrictions as $restriction) { + foreach ($restrictions as $restriction) { if ($restriction == Tag::CAN_REPLY) { $item['restrictions'] = $item['restrictions'] | Item::CANT_REPLY; } elseif ($restriction == Tag::CAN_LIKE) { $item['restrictions'] = $item['restrictions'] | Item::CANT_LIKE; } elseif ($restriction == Tag::CAN_ANNOUNCE) { $item['restrictions'] = $item['restrictions'] | Item::CANT_ANNOUNCE; - } + } } $item['location'] = $activity['location']; @@ -982,7 +1009,7 @@ class Processor $path = implode("/", $parsed); - return $host_hash . '-'. hash('fnv164', $path) . '-'. hash('joaat', $path); + return $host_hash . '-' . hash('fnv164', $path) . '-' . hash('joaat', $path); } /** @@ -1077,7 +1104,7 @@ class Processor $item['uid'] = $receiver; $type = $activity['reception_type'][$receiver] ?? Receiver::TARGET_UNKNOWN; - switch($type) { + switch ($type) { case Receiver::TARGET_TO: $item['post-reason'] = Item::PR_TO; break; @@ -1380,7 +1407,7 @@ class Processor $name = $host; } else { Logger::warning('Unable to coerce name from capability', ['element' => $element, 'type' => $type, 'capability' => $capability]); - $name = ''; + $name = ''; } $restricted = false; Tag::store($uriid, $type, $name, $capability); @@ -1634,6 +1661,11 @@ class Processor return null; } + return self::processActivity($object, $url, $child, $relay_actor, $completion, $uid); + } + + private static function processActivity(array $object, string $url, array $child, string $relay_actor, int $completion, int $uid = 0): ?string + { $ldobject = JsonLD::compact($object); $signer = []; @@ -1723,6 +1755,53 @@ class Processor return $activity['id']; } + private static function fetchReplies(string $url, array $child) + { + $replies = ActivityPub::fetchItems($url); + if (empty($replies)) { + Logger::notice('No replies', ['replies' => $url]); + return; + } + Logger::notice('Fetch replies - start', ['replies' => $url]); + $fetched = 0; + foreach ($replies as $reply) { + if (is_array($reply)) { + $ldobject = JsonLD::compact($reply); + $id = JsonLD::fetchElement($ldobject, '@id'); + if ($id == $child['id']) { + Logger::debug('Incluced activity is currently processed', ['replies' => $url, 'id' => $id]); + continue; + } elseif (Item::searchByLink($id)) { + Logger::debug('Incluced activity already exists', ['replies' => $url, 'id' => $id]); + continue; + } elseif (Queue::exists($id, 'as:Create')) { + Logger::debug('Incluced activity is already queued', ['replies' => $url, 'id' => $id]); + continue; + } + if (parse_url($id, PHP_URL_HOST) == parse_url($url, PHP_URL_HOST)) { + Logger::debug('Incluced activity will be processed', ['replies' => $url, 'id' => $id]); + self::processActivity($reply, $id, $child, '', Receiver::COMPLETION_AUTO); + ++$fetched; + continue; + } + } elseif (is_string($reply)) { + $id = $reply; + } + if ($id == $child['id']) { + Logger::debug('Activity is currently processed', ['replies' => $url, 'id' => $id]); + } elseif (Item::searchByLink($id)) { + Logger::debug('Activity already exists', ['replies' => $url, 'id' => $id]); + } elseif (Queue::exists($id, 'as:Create')) { + Logger::debug('Activity is already queued', ['replies' => $url, 'id' => $id]); + } else { + Logger::debug('Missing Activity will be fetched and processed', ['replies' => $url, 'id' => $id]); + self::fetchMissingActivity($id, $child, '', Receiver::COMPLETION_AUTO); + ++$fetched; + } + } + Logger::notice('Fetch replies - done', ['fetched' => $fetched, 'total' => count($replies), 'replies' => $url]); + } + private static function refetchObjectOnHostDifference(array $object, string $url): array { $ldobject = JsonLD::compact($object); @@ -1914,7 +1993,7 @@ class Processor foreach ($languages as $language) { if ($language == $content) { continue; - } + } $language = DI::l10n()->toISO6391($language); if (!in_array($language, array_column($iso639->allLanguages(), 0))) { continue; @@ -2407,7 +2486,7 @@ class Processor $kept_mentions = []; // Extract one prepended mention at a time from the body - while(preg_match('#^(@\[url=([^\]]+)].*?\[\/url]\s)(.*)#is', $body, $matches)) { + while (preg_match('#^(@\[url=([^\]]+)].*?\[\/url]\s)(.*)#is', $body, $matches)) { if (!in_array($matches[2], $potential_mentions)) { $kept_mentions[] = $matches[1]; } diff --git a/src/Protocol/ActivityPub/Receiver.php b/src/Protocol/ActivityPub/Receiver.php index f23bcc4e73..c81c569b6f 100644 --- a/src/Protocol/ActivityPub/Receiver.php +++ b/src/Protocol/ActivityPub/Receiver.php @@ -2070,6 +2070,7 @@ class Receiver $object_data['generator'] = JsonLD::fetchElement($object, 'as:generator', 'as:name', '@type', 'as:Application'); $object_data['generator'] = JsonLD::fetchElement($object_data, 'generator', '@value'); $object_data['alternate-url'] = JsonLD::fetchElement($object, 'as:url', '@id'); + $object_data['replies'] = JsonLD::fetchElement($object, 'as:replies', '@id'); // Special treatment for Hubzilla links if (is_array($object_data['alternate-url'])) { diff --git a/src/Worker/ExpirePosts.php b/src/Worker/ExpirePosts.php index 5e750cb835..30da0a6cff 100644 --- a/src/Worker/ExpirePosts.php +++ b/src/Worker/ExpirePosts.php @@ -203,6 +203,7 @@ class ExpirePosts AND NOT EXISTS(SELECT `parent-uri-id` FROM `post-user` WHERE `parent-uri-id` = `item-uri`.`id`) AND NOT EXISTS(SELECT `thr-parent-id` FROM `post-user` WHERE `thr-parent-id` = `item-uri`.`id`) AND NOT EXISTS(SELECT `external-id` FROM `post-user` WHERE `external-id` = `item-uri`.`id`) + AND NOT EXISTS(SELECT `replies-id` FROM `post-user` WHERE `replies-id` = `item-uri`.`id`) AND NOT EXISTS(SELECT `conversation-id` FROM `post-thread` WHERE `conversation-id` = `item-uri`.`id`) AND NOT EXISTS(SELECT `uri-id` FROM `mail` WHERE `uri-id` = `item-uri`.`id`) AND NOT EXISTS(SELECT `uri-id` FROM `event` WHERE `uri-id` = `item-uri`.`id`) diff --git a/static/dbstructure.config.php b/static/dbstructure.config.php index 2451270c53..a020e441de 100644 --- a/static/dbstructure.config.php +++ b/static/dbstructure.config.php @@ -56,7 +56,7 @@ use Friendica\Database\DBA; // This file is required several times during the test in DbaDefinition which justifies this condition if (!defined('DB_UPDATE_VERSION')) { - define('DB_UPDATE_VERSION', 1568); + define('DB_UPDATE_VERSION', 1569); } return [ @@ -1222,6 +1222,7 @@ return [ "parent-uri-id" => ["type" => "int unsigned", "foreign" => ["item-uri" => "id"], "comment" => "Id of the item-uri table that contains the parent uri"], "thr-parent-id" => ["type" => "int unsigned", "foreign" => ["item-uri" => "id"], "comment" => "Id of the item-uri table that contains the thread parent uri"], "external-id" => ["type" => "int unsigned", "foreign" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the external uri"], + "replies-id" => ["type" => "int unsigned", "foreign" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the endpoint for the replies collection"], "created" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => "Creation timestamp."], "edited" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => "Date of last edit (default is created)"], "received" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => "datetime"], @@ -1242,6 +1243,7 @@ return [ "parent-uri-id" => ["parent-uri-id"], "thr-parent-id" => ["thr-parent-id"], "external-id" => ["external-id"], + "replies-id" => ["replies-id"], "owner-id" => ["owner-id"], "author-id" => ["author-id"], "causer-id" => ["causer-id"], @@ -1581,6 +1583,7 @@ return [ "parent-uri-id" => ["type" => "int unsigned", "foreign" => ["item-uri" => "id"], "comment" => "Id of the item-uri table that contains the parent uri"], "thr-parent-id" => ["type" => "int unsigned", "foreign" => ["item-uri" => "id"], "comment" => "Id of the item-uri table that contains the thread parent uri"], "external-id" => ["type" => "int unsigned", "foreign" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the external uri"], + "replies-id" => ["type" => "int unsigned", "foreign" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the endpoint for the replies collection"], "created" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => "Creation timestamp."], "edited" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => "Date of last edit (default is created)"], "received" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => "datetime"], @@ -1615,6 +1618,7 @@ return [ "parent-uri-id" => ["parent-uri-id"], "thr-parent-id" => ["thr-parent-id"], "external-id" => ["external-id"], + "replies-id" => ["replies-id"], "owner-id" => ["owner-id"], "author-id" => ["author-id"], "causer-id" => ["causer-id"], diff --git a/static/dbview.config.php b/static/dbview.config.php index 8f2c6bc8c9..29756b6af4 100644 --- a/static/dbview.config.php +++ b/static/dbview.config.php @@ -268,6 +268,8 @@ "gravity" => ["post-origin", "gravity"], "extid" => ["external-item-uri", "uri"], "external-id" => ["post-user", "external-id"], + "replies" => ["replies-item-uri", "uri"], + "replies-id" => ["post-user", "replies-id"], "created" => ["post-origin", "created"], "edited" => ["post-user", "edited"], "commented" => ["post-thread-user", "commented"], @@ -423,6 +425,7 @@ LEFT JOIN `item-uri` AS `parent-item-uri` ON `parent-item-uri`.`id` = `post-origin`.`parent-uri-id` LEFT JOIN `item-uri` AS `conversation-item-uri` ON `conversation-item-uri`.`id` = `post-thread-user`.`conversation-id` LEFT JOIN `item-uri` AS `external-item-uri` ON `external-item-uri`.`id` = `post-user`.`external-id` + LEFT JOIN `item-uri` AS `replies-item-uri` ON `replies-item-uri`.`id` = `post-user`.`replies-id` LEFT JOIN `verb` ON `verb`.`id` = `post-origin`.`vid` LEFT JOIN `event` ON `event`.`id` = `post-user`.`event-id` LEFT JOIN `diaspora-interaction` ON `diaspora-interaction`.`uri-id` = `post-origin`.`uri-id` @@ -454,6 +457,8 @@ "gravity" => ["post-origin", "gravity"], "extid" => ["external-item-uri", "uri"], "external-id" => ["post-user", "external-id"], + "replies" => ["replies-item-uri", "uri"], + "replies-id" => ["post-user", "replies-id"], "created" => ["post-origin", "created"], "edited" => ["post-user", "edited"], "commented" => ["post-thread-user", "commented"], @@ -608,6 +613,7 @@ LEFT JOIN `item-uri` AS `parent-item-uri` ON `parent-item-uri`.`id` = `post-origin`.`parent-uri-id` LEFT JOIN `item-uri` AS `conversation-item-uri` ON `conversation-item-uri`.`id` = `post-thread-user`.`conversation-id` LEFT JOIN `item-uri` AS `external-item-uri` ON `external-item-uri`.`id` = `post-user`.`external-id` + LEFT JOIN `item-uri` AS `replies-item-uri` ON `replies-item-uri`.`id` = `post-user`.`replies-id` LEFT JOIN `verb` ON `verb`.`id` = `post-origin`.`vid` LEFT JOIN `event` ON `event`.`id` = `post-user`.`event-id` LEFT JOIN `diaspora-interaction` ON `diaspora-interaction`.`uri-id` = `post-origin`.`uri-id` @@ -638,6 +644,8 @@ "gravity" => ["post-user", "gravity"], "extid" => ["external-item-uri", "uri"], "external-id" => ["post-user", "external-id"], + "replies" => ["replies-item-uri", "uri"], + "replies-id" => ["post-user", "replies-id"], "created" => ["post-user", "created"], "edited" => ["post-user", "edited"], "commented" => ["post-thread-user", "commented"], @@ -792,6 +800,7 @@ LEFT JOIN `item-uri` AS `parent-item-uri` ON `parent-item-uri`.`id` = `post-user`.`parent-uri-id` LEFT JOIN `item-uri` AS `conversation-item-uri` ON `conversation-item-uri`.`id` = `post-thread-user`.`conversation-id` LEFT JOIN `item-uri` AS `external-item-uri` ON `external-item-uri`.`id` = `post-user`.`external-id` + LEFT JOIN `item-uri` AS `replies-item-uri` ON `replies-item-uri`.`id` = `post-user`.`replies-id` LEFT JOIN `verb` ON `verb`.`id` = `post-user`.`vid` LEFT JOIN `event` ON `event`.`id` = `post-user`.`event-id` LEFT JOIN `diaspora-interaction` ON `diaspora-interaction`.`uri-id` = `post-user`.`uri-id` @@ -823,6 +832,8 @@ "gravity" => ["post-user", "gravity"], "extid" => ["external-item-uri", "uri"], "external-id" => ["post-user", "external-id"], + "replies" => ["replies-item-uri", "uri"], + "replies-id" => ["post-user", "replies-id"], "created" => ["post-thread-user", "created"], "edited" => ["post-user", "edited"], "commented" => ["post-thread-user", "commented"], @@ -976,6 +987,7 @@ LEFT JOIN `item-uri` AS `parent-item-uri` ON `parent-item-uri`.`id` = `post-user`.`parent-uri-id` LEFT JOIN `item-uri` AS `conversation-item-uri` ON `conversation-item-uri`.`id` = `post-thread-user`.`conversation-id` LEFT JOIN `item-uri` AS `external-item-uri` ON `external-item-uri`.`id` = `post-user`.`external-id` + LEFT JOIN `item-uri` AS `replies-item-uri` ON `replies-item-uri`.`id` = `post-user`.`replies-id` LEFT JOIN `verb` ON `verb`.`id` = `post-user`.`vid` LEFT JOIN `event` ON `event`.`id` = `post-user`.`event-id` LEFT JOIN `diaspora-interaction` ON `diaspora-interaction`.`uri-id` = `post-thread-user`.`uri-id` @@ -1001,6 +1013,8 @@ "gravity" => ["post", "gravity"], "extid" => ["external-item-uri", "uri"], "external-id" => ["post", "external-id"], + "replies" => ["replies-item-uri", "uri"], + "replies-id" => ["post", "replies-id"], "created" => ["post", "created"], "edited" => ["post", "edited"], "commented" => ["post-thread", "commented"], @@ -1123,6 +1137,7 @@ LEFT JOIN `item-uri` AS `parent-item-uri` ON `parent-item-uri`.`id` = `post`.`parent-uri-id` LEFT JOIN `item-uri` AS `conversation-item-uri` ON `conversation-item-uri`.`id` = `post-thread`.`conversation-id` LEFT JOIN `item-uri` AS `external-item-uri` ON `external-item-uri`.`id` = `post`.`external-id` + LEFT JOIN `item-uri` AS `replies-item-uri` ON `replies-item-uri`.`id` = `post`.`replies-id` LEFT JOIN `verb` ON `verb`.`id` = `post`.`vid` LEFT JOIN `diaspora-interaction` ON `diaspora-interaction`.`uri-id` = `post`.`uri-id` LEFT JOIN `post-content` ON `post-content`.`uri-id` = `post`.`uri-id` @@ -1146,6 +1161,8 @@ "gravity" => ["post", "gravity"], "extid" => ["external-item-uri", "uri"], "external-id" => ["post", "external-id"], + "replies" => ["replies-item-uri", "uri"], + "replies-id" => ["post", "replies-id"], "created" => ["post-thread", "created"], "edited" => ["post", "edited"], "commented" => ["post-thread", "commented"], @@ -1270,6 +1287,7 @@ LEFT JOIN `item-uri` AS `parent-item-uri` ON `parent-item-uri`.`id` = `post`.`parent-uri-id` LEFT JOIN `item-uri` AS `conversation-item-uri` ON `conversation-item-uri`.`id` = `post-thread`.`conversation-id` LEFT JOIN `item-uri` AS `external-item-uri` ON `external-item-uri`.`id` = `post`.`external-id` + LEFT JOIN `item-uri` AS `replies-item-uri` ON `replies-item-uri`.`id` = `post`.`replies-id` LEFT JOIN `verb` ON `verb`.`id` = `post`.`vid` LEFT JOIN `diaspora-interaction` ON `diaspora-interaction`.`uri-id` = `post-thread`.`uri-id` LEFT JOIN `post-content` ON `post-content`.`uri-id` = `post-thread`.`uri-id`