Merge pull request #11457 from annando/performance

Improve page performance
This commit is contained in:
Hypolite Petovan 2022-05-06 11:11:05 -04:00 committed by GitHub
commit 4729fca5d9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
28 changed files with 550 additions and 232 deletions

View file

@ -1,6 +1,6 @@
-- ------------------------------------------
-- Friendica 2022.05-rc (Siberian Iris)
-- DB_UPDATE_VERSION 1460
-- DB_UPDATE_VERSION 1461
-- ------------------------------------------
@ -712,13 +712,16 @@ CREATE TABLE IF NOT EXISTS `hook` (
--
CREATE TABLE IF NOT EXISTS `inbox-status` (
`url` varbinary(255) NOT NULL COMMENT 'URL of the inbox',
`uri-id` int unsigned COMMENT 'Item-uri id of inbox url',
`created` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Creation date of this entry',
`success` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Date of the last successful delivery',
`failure` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Date of the last failed delivery',
`previous` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Previous delivery date',
`archive` boolean NOT NULL DEFAULT '0' COMMENT 'Is the inbox archived?',
`shared` boolean NOT NULL DEFAULT '0' COMMENT 'Is it a shared inbox?',
PRIMARY KEY(`url`)
PRIMARY KEY(`url`),
INDEX `uri-id` (`uri-id`),
FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Status of ActivityPub inboxes';
--
@ -1068,8 +1071,8 @@ CREATE TABLE IF NOT EXISTS `post-category` (
`type` tinyint unsigned NOT NULL DEFAULT 0 COMMENT '',
`tid` int unsigned NOT NULL DEFAULT 0 COMMENT '',
PRIMARY KEY(`uri-id`,`uid`,`type`,`tid`),
INDEX `uri-id` (`tid`),
INDEX `uid` (`uid`),
INDEX `tid` (`tid`),
INDEX `uid_uri-id` (`uid`,`uri-id`),
FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE,
FOREIGN KEY (`uid`) REFERENCES `user` (`uid`) ON UPDATE RESTRICT ON DELETE CASCADE,
FOREIGN KEY (`tid`) REFERENCES `tag` (`id`) ON UPDATE RESTRICT ON DELETE RESTRICT
@ -1114,6 +1117,23 @@ CREATE TABLE IF NOT EXISTS `post-content` (
FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Content for all posts';
--
-- TABLE post-delivery
--
CREATE TABLE IF NOT EXISTS `post-delivery` (
`uri-id` int unsigned NOT NULL COMMENT 'Id of the item-uri table entry that contains the item uri',
`inbox-id` int unsigned NOT NULL COMMENT 'Item-uri id of inbox url',
`uid` mediumint unsigned COMMENT 'Delivering user',
`created` datetime DEFAULT '0001-01-01 00:00:00' COMMENT '',
`command` varbinary(32) COMMENT '',
PRIMARY KEY(`uri-id`,`inbox-id`),
INDEX `inbox-id_created` (`inbox-id`,`created`),
INDEX `uid` (`uid`),
FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE,
FOREIGN KEY (`inbox-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE,
FOREIGN KEY (`uid`) REFERENCES `user` (`uid`) ON UPDATE RESTRICT ON DELETE CASCADE
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Delivery data for posts for the batch processing';
--
-- TABLE post-delivery-data
--
@ -1171,6 +1191,7 @@ CREATE TABLE IF NOT EXISTS `post-media` (
`publisher-image` varbinary(255) COMMENT 'Image of the publisher of the media',
PRIMARY KEY(`id`),
UNIQUE INDEX `uri-id-url` (`uri-id`,`url`),
INDEX `uri-id-id` (`uri-id`,`id`),
FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Attached media';
@ -1719,6 +1740,9 @@ CREATE VIEW `post-user-view` AS SELECT
`author`.`network` AS `author-network`,
`author`.`blocked` AS `author-blocked`,
`author`.`hidden` AS `author-hidden`,
`author`.`updated` AS `author-updated`,
`author`.`gsid` AS `author-gsid`,
`author`.`uri-id` AS `author-uri-id`,
`post-user`.`owner-id` AS `owner-id`,
`owner`.`url` AS `owner-link`,
`owner`.`addr` AS `owner-addr`,
@ -1728,6 +1752,7 @@ CREATE VIEW `post-user-view` AS SELECT
`owner`.`network` AS `owner-network`,
`owner`.`blocked` AS `owner-blocked`,
`owner`.`hidden` AS `owner-hidden`,
`owner`.`updated` AS `owner-updated`,
`owner`.`contact-type` AS `owner-contact-type`,
`post-user`.`causer-id` AS `causer-id`,
`causer`.`url` AS `causer-link`,
@ -1884,6 +1909,9 @@ CREATE VIEW `post-thread-user-view` AS SELECT
`author`.`network` AS `author-network`,
`author`.`blocked` AS `author-blocked`,
`author`.`hidden` AS `author-hidden`,
`author`.`updated` AS `author-updated`,
`author`.`gsid` AS `author-gsid`,
`author`.`uri-id` AS `author-uri-id`,
`post-thread-user`.`owner-id` AS `owner-id`,
`owner`.`url` AS `owner-link`,
`owner`.`addr` AS `owner-addr`,
@ -1893,6 +1921,7 @@ CREATE VIEW `post-thread-user-view` AS SELECT
`owner`.`network` AS `owner-network`,
`owner`.`blocked` AS `owner-blocked`,
`owner`.`hidden` AS `owner-hidden`,
`owner`.`updated` AS `owner-updated`,
`owner`.`contact-type` AS `owner-contact-type`,
`post-thread-user`.`causer-id` AS `causer-id`,
`causer`.`url` AS `causer-link`,
@ -2035,6 +2064,9 @@ CREATE VIEW `post-view` AS SELECT
`author`.`network` AS `author-network`,
`author`.`blocked` AS `author-blocked`,
`author`.`hidden` AS `author-hidden`,
`author`.`updated` AS `author-updated`,
`author`.`gsid` AS `author-gsid`,
`author`.`uri-id` AS `author-uri-id`,
`post`.`owner-id` AS `owner-id`,
`owner`.`url` AS `owner-link`,
`owner`.`addr` AS `owner-addr`,
@ -2044,6 +2076,7 @@ CREATE VIEW `post-view` AS SELECT
`owner`.`network` AS `owner-network`,
`owner`.`blocked` AS `owner-blocked`,
`owner`.`hidden` AS `owner-hidden`,
`owner`.`updated` AS `owner-updated`,
`owner`.`contact-type` AS `owner-contact-type`,
`post`.`causer-id` AS `causer-id`,
`causer`.`url` AS `causer-link`,
@ -2162,6 +2195,9 @@ CREATE VIEW `post-thread-view` AS SELECT
`author`.`network` AS `author-network`,
`author`.`blocked` AS `author-blocked`,
`author`.`hidden` AS `author-hidden`,
`author`.`updated` AS `author-updated`,
`author`.`gsid` AS `author-gsid`,
`author`.`uri-id` AS `author-uri-id`,
`post-thread`.`owner-id` AS `owner-id`,
`owner`.`url` AS `owner-link`,
`owner`.`addr` AS `owner-addr`,
@ -2171,6 +2207,7 @@ CREATE VIEW `post-thread-view` AS SELECT
`owner`.`network` AS `owner-network`,
`owner`.`blocked` AS `owner-blocked`,
`owner`.`hidden` AS `owner-hidden`,
`owner`.`updated` AS `owner-updated`,
`owner`.`contact-type` AS `owner-contact-type`,
`post-thread`.`causer-id` AS `causer-id`,
`causer`.`url` AS `causer-link`,
@ -2234,9 +2271,14 @@ CREATE VIEW `collection-view` AS SELECT
`post-collection`.`type` AS `type`,
`post`.`author-id` AS `cid`,
`post`.`received` AS `received`,
`post`.`created` AS `created`
`post`.`created` AS `created`,
`post-thread`.`commented` AS `commented`,
`post`.`thr-parent-id` AS `thr-parent-id`,
`post`.`author-id` AS `author-id`,
`post`.`gravity` AS `gravity`
FROM `post-collection`
INNER JOIN `post` ON `post-collection`.`uri-id` = `post`.`uri-id`;
INNER JOIN `post` ON `post-collection`.`uri-id` = `post`.`uri-id`
INNER JOIN `post-thread` ON `post-thread`.`uri-id` = `post`.`parent-uri-id`;
--
-- VIEW tag-view

View file

@ -50,6 +50,7 @@ Database Tables
| [post-category](help/database/db_post-category) | post relation to categories |
| [post-collection](help/database/db_post-collection) | Collection of posts |
| [post-content](help/database/db_post-content) | Content for all posts |
| [post-delivery](help/database/db_post-delivery) | Delivery data for posts for the batch processing |
| [post-delivery-data](help/database/db_post-delivery-data) | Delivery data for items |
| [post-link](help/database/db_post-link) | Post related external links |
| [post-media](help/database/db_post-media) | Attached media |

View file

@ -9,6 +9,7 @@ Fields
| Field | Description | Type | Null | Key | Default | Extra |
| -------- | ------------------------------------ | -------------- | ---- | --- | ------------------- | ----- |
| url | URL of the inbox | varbinary(255) | NO | PRI | NULL | |
| uri-id | Item-uri id of inbox url | int unsigned | YES | | NULL | |
| created | Creation date of this entry | datetime | NO | | 0001-01-01 00:00:00 | |
| success | Date of the last successful delivery | datetime | NO | | 0001-01-01 00:00:00 | |
| failure | Date of the last failed delivery | datetime | NO | | 0001-01-01 00:00:00 | |
@ -22,6 +23,13 @@ Indexes
| Name | Fields |
| ------- | ------ |
| PRIMARY | url |
| uri-id | uri-id |
Foreign Keys
------------
| Field | Target Table | Target Field |
|-------|--------------|--------------|
| uri-id | [item-uri](help/database/db_item-uri) | id |
Return to [database documentation](help/database)

View file

@ -17,10 +17,10 @@ Indexes
------------
| Name | Fields |
| ------- | ---------------------- |
| ---------- | ---------------------- |
| PRIMARY | uri-id, uid, type, tid |
| uri-id | tid |
| uid | uid |
| tid | tid |
| uid_uri-id | uid, uri-id |
Foreign Keys
------------

View file

@ -0,0 +1,35 @@
Table post-delivery
===========
Delivery data for posts for the batch processing
Fields
------
| Field | Description | Type | Null | Key | Default | Extra |
| -------- | --------------------------------------------------------- | ------------------ | ---- | --- | ------------------- | ----- |
| uri-id | Id of the item-uri table entry that contains the item uri | int unsigned | NO | PRI | NULL | |
| inbox-id | Item-uri id of inbox url | int unsigned | NO | PRI | NULL | |
| uid | Delivering user | mediumint unsigned | YES | | NULL | |
| created | | datetime | YES | | 0001-01-01 00:00:00 | |
| command | | varbinary(32) | YES | | NULL | |
Indexes
------------
| Name | Fields |
| ---------------- | ----------------- |
| PRIMARY | uri-id, inbox-id |
| inbox-id_created | inbox-id, created |
| uid | uid |
Foreign Keys
------------
| Field | Target Table | Target Field |
|-------|--------------|--------------|
| uri-id | [item-uri](help/database/db_item-uri) | id |
| inbox-id | [item-uri](help/database/db_item-uri) | id |
| uid | [user](help/database/db_user) | uid |
Return to [database documentation](help/database)

View file

@ -35,6 +35,7 @@ Indexes
| ---------- | ------------------- |
| PRIMARY | id |
| uri-id-url | UNIQUE, uri-id, url |
| uri-id-id | uri-id, id |
Foreign Keys
------------

View file

@ -621,6 +621,9 @@ function item_post(App $a) {
$datarray["id"] = -1;
$datarray["uri-id"] = -1;
$datarray["author-network"] = Protocol::DFRN;
$datarray["author-updated"] = '';
$datarray["author-gsid"] = 0;
$datarray["owner-updated"] = '';
$o = DI::conversation()->create([array_merge($contact_record, $datarray)], 'search', false, true);
@ -663,21 +666,17 @@ function item_post(App $a) {
$datarray['uri-id'] = ItemURI::getIdByURI($datarray['uri']);
if ($orig_post) {
// Fill the cache field
// This could be done in Item::update as well - but we have to check for the existance of some fields.
Item::putInCache($datarray);
$fields = [
'title' => $datarray['title'],
'body' => $datarray['body'],
'attach' => $datarray['attach'],
'file' => $datarray['file'],
'rendered-html' => $datarray['rendered-html'],
'rendered-hash' => $datarray['rendered-hash'],
'edited' => DateTimeFormat::utcNow(),
'changed' => DateTimeFormat::utcNow()];
'changed' => DateTimeFormat::utcNow()
];
Item::update($fields, ['id' => $post_id]);
Item::updateDisplayCache($datarray['uri-id']);
if ($return_path) {
DI::baseUrl()->redirect($return_path);

View file

@ -184,7 +184,7 @@ class ContactSelector
* @return string
* @throws \Exception
*/
public static function networkToIcon($network, $profile = "")
public static function networkToIcon($network, $profile = "", $gsid = 0)
{
$nets = [
Protocol::DFRN => 'friendica',
@ -218,7 +218,14 @@ class ContactSelector
$network_icon = str_replace($search, $replace, $network);
if ((in_array($network, Protocol::FEDERATED)) && ($profile != "")) {
if (!empty($gsid) && !empty(self::$serverdata[$gsid])) {
$gserver = self::$serverdata[$gsid];
} elseif (!empty($gsid)) {
$gserver = DBA::selectFirst('gserver', ['platform', 'network'], ['id' => $gsid]);
self::$serverdata[$gsid] = $gserver;
} else {
$gserver = self::getServerForProfile($profile);
}
if (!empty($gserver['platform'])) {
$network_icon = $platform_icons[strtolower($gserver['platform'])] ?? $network_icon;
}

View file

@ -108,6 +108,8 @@ class Conversation
*/
public function builtinActivityPuller(array $activity, array &$conv_responses)
{
$thread_parent = $activity['thr-parent-row'] ?? [];
foreach ($conv_responses as $mode => $v) {
$sparkle = '';
@ -152,9 +154,8 @@ class Conversation
$activity['thr-parent-id'] = $activity['parent-uri-id'];
}
// Skip when the causer of the parent is the same than the author of the announce
if (($verb == Activity::ANNOUNCE) && Post::exists(['uri-id' => $activity['thr-parent-id'],
'uid' => $activity['uid'], 'causer-id' => $activity['author-id'], 'gravity' => [GRAVITY_PARENT, GRAVITY_COMMENT]])) {
// Skip when the causer of the parent is the same as the author of the announce
if (($verb == Activity::ANNOUNCE) && !empty($thread_parent['causer-id'] && ($thread_parent['causer-id'] == $activity['author-id']))) {
continue;
}
@ -658,6 +659,16 @@ class Conversation
$pinned = '';
}
if (in_array($item['network'], [Protocol::FEED, Protocol::MAIL])) {
$owner_avatar = $author_avatar = $item['contact-id'];
$owner_updated = $author_updated = '';
} else {
$owner_avatar = $item['owner-id'];
$owner_updated = $item['owner-updated'];
$author_avatar = $item['author-id'];
$author_updated = $item['author-updated'];
}
$tmp_item = [
'template' => $tpl,
'id' => ($preview ? 'P0' : $item['id']),
@ -667,15 +678,15 @@ class Conversation
'created_date' => $item['created'],
'uriid' => $item['uri-id'],
'network' => $item['network'],
'network_name' => ContactSelector::networkToName($item['author-network'], $item['author-link'], $item['network']),
'network_icon' => ContactSelector::networkToIcon($item['network'], $item['author-link']),
'network_name' => ContactSelector::networkToName($item['author-network'], $item['author-link'], $item['network'], $item['author-gsid']),
'network_icon' => ContactSelector::networkToIcon($item['network'], $item['author-link'], $item['author-gsid']),
'linktitle' => $this->l10n->t('View %s\'s profile @ %s', $profile_name, $item['author-link']),
'profile_url' => $profile_link,
'item_photo_menu_html' => $this->item->photoMenu($item, $formSecurityToken),
'name' => $profile_name,
'sparkle' => $sparkle,
'lock' => false,
'thumb' => $this->baseURL->remove(Contact::getAvatarUrlForUrl($item['author-link'], $item['uid'], Proxy::SIZE_THUMB)),
'thumb' => $this->baseURL->remove(Contact::getAvatarUrlForId($author_avatar, Proxy::SIZE_THUMB, $author_updated)),
'title' => $title,
'body_html' => $body_html,
'tags' => $tags['tags'],
@ -696,7 +707,7 @@ class Conversation
'indent' => '',
'owner_name' => '',
'owner_url' => '',
'owner_photo' => $this->baseURL->remove(Contact::getAvatarUrlForUrl($item['owner-link'], $item['uid'], Proxy::SIZE_THUMB)),
'owner_photo' => $this->baseURL->remove(Contact::getAvatarUrlForId($owner_avatar, Proxy::SIZE_THUMB, $owner_updated)),
'plink' => ItemModel::getPlink($item),
'edpost' => false,
'pinned' => $pinned,
@ -811,10 +822,11 @@ class Conversation
*
* @param array $row Post row
* @param array $activity Contact data of the resharer
* @param array $thr_parent Thread parent row
*
* @return array items with parents and comments
*/
private function addRowInformation(array $row, array $activity)
private function addRowInformation(array $row, array $activity, array $thr_parent)
{
$this->profiler->startRecording('rendering');
@ -889,6 +901,8 @@ class Conversation
break;
}
$row['thr-parent-row'] = $thr_parent;
$this->profiler->stopRecording();
return $row;
}
@ -916,8 +930,6 @@ class Conversation
$max_comments = $this->config->get('system', 'max_display_comments', 1000);
}
$params = ['order' => ['uri-id' => true, 'uid' => true]];
$activities = [];
$uriids = [];
$commentcounter = [];
@ -951,6 +963,17 @@ class Conversation
$condition = DBA::mergeConditions($condition,
["`uid` IN (0, ?) AND (`vid` != ? OR `vid` IS NULL)", $uid, Verb::getID(Activity::FOLLOW)]);
$thread_parents = Post::select(['uri-id', 'causer-id'], $condition, ['order' => ['uri-id' => false, 'uid' => true]]);
$thr_parent = [];
while ($row = Post::fetch($thread_parents)) {
$thr_parent[$row['uri-id']] = $row;
}
DBA::close($thread_parents);
$params = ['order' => ['uri-id' => true, 'uid' => true]];
$thread_items = Post::selectForUser($uid, array_merge(ItemModel::DISPLAY_FIELDLIST, ['featured', 'contact-uid', 'gravity', 'post-type', 'post-reason']), $condition, $params);
$items = [];
@ -968,7 +991,8 @@ class Conversation
continue;
}
}
$items[$row['uri-id']] = $this->addRowInformation($row, $activities[$row['uri-id']] ?? []);
$items[$row['uri-id']] = $this->addRowInformation($row, $activities[$row['uri-id']] ?? [], $thr_parent[$row['thr-parent-id']] ?? []);
}
DBA::close($thread_items);

View file

@ -35,7 +35,6 @@ use Friendica\Model\Tag;
use Friendica\Model\Post;
use Friendica\Protocol\Activity;
use Friendica\Util\Profiler;
use Friendica\Util\Strings;
use Friendica\Util\XML;
/**
@ -93,6 +92,10 @@ class Item
$uid = $item['uid'] ?: $uid;
if (!Post\Category::existsForURIId($item['uri-id'], $uid)) {
return [$categories, $folders];
}
foreach (Post\Category::getArrayByURIId($item['uri-id'], $uid, Post\Category::CATEGORY) as $savedFolderName) {
if (!empty($item['author-link'])) {
$url = $item['author-link'] . "?category=" . rawurlencode($savedFolderName);
@ -353,22 +356,6 @@ class Item
}
}
$matches = null;
if (preg_match_all('/@\[url=(.*?)\]/is', $item['body'], $matches, PREG_SET_ORDER)) {
foreach ($matches as $mtch) {
if (!strpos($mtch[1], 'zrl=')) {
$item['body'] = str_replace($mtch[0], '@[url=' . Contact::magicLink($mtch[1]) . ']', $item['body']);
}
}
}
// add sparkle links to appropriate permalinks
// Only create a redirection to a magic link when logged in
if (!empty($item['plink']) && Session::isAuthenticated() && $item['private'] == ModelItem::PRIVATE) {
$author = ['uid' => 0, 'id' => $item['author-id'],
'network' => $item['author-network'], 'url' => $item['author-link']];
$item['plink'] = Contact::magicLinkByContact($author, $item['plink']);
}
$this->profiler->stopRecording();
}
@ -398,7 +385,7 @@ class Item
$pcid = $item['author-id'];
$network = '';
$rel = 0;
$condition = ['uid' => local_user(), 'nurl' => Strings::normaliseLink($item['author-link'])];
$condition = ['uid' => local_user(), 'uri-id' => $item['author-uri-id']];
$contact = DBA::selectFirst('contact', ['id', 'network', 'rel'], $condition);
if (DBA::isResult($contact)) {
$cid = $contact['id'];

View file

@ -22,7 +22,6 @@
namespace Friendica\Core;
use Friendica\App\Mode;
use Friendica\Core;
use Friendica\Core\Worker\Entity\Process;
use Friendica\Database\DBA;
use Friendica\DI;
@ -105,8 +104,7 @@ class Worker
// Don't refetch when a worker fetches tasks for multiple workers
$refetched = DI::config()->get('system', 'worker_multiple_fetch');
foreach ($r as $entry) {
// Assure that the priority is an integer value
$entry['priority'] = (int)$entry['priority'];
$entry = self::checkPriority($entry);
// The work will be done
if (!self::execute($entry)) {
@ -168,6 +166,24 @@ class Worker
Logger::info("Couldn't select a workerqueue entry, quitting process", ['pid' => getmypid()]);
}
/**
* Check and fix the priority of a worker task
* @param array $entry
* @return array
*/
private static function checkPriority(array $entry)
{
$entry['priority'] = (int)$entry['priority'];
if (!in_array($entry['priority'], PRIORITIES)) {
Logger::warning('Invalid priority', ['entry' => $entry, 'callstack' => System::callstack(20)]);
DBA::update('workerqueue', ['priority' => PRIORITY_MEDIUM], ['id' => $entry['id']]);
$entry['priority'] = PRIORITY_MEDIUM;
}
return $entry;
}
/**
* Checks if the system is ready.
*
@ -261,7 +277,7 @@ class Worker
$workerqueue = DBA::selectFirst('workerqueue', ['priority'], $condition, ['order' => ['priority']]);
self::$db_duration += (microtime(true) - $stamp);
if (DBA::isResult($workerqueue)) {
return $workerqueue["priority"];
return $workerqueue['priority'];
} else {
return 0;
}
@ -466,13 +482,13 @@ class Worker
$cooldown = DI::config()->get("system", "worker_cooldown", 0);
if ($cooldown > 0) {
Logger::info('Pre execution cooldown.', ['priority' => $queue["priority"], 'id' => $queue["id"], 'cooldown' => $cooldown]);
Logger::info('Pre execution cooldown.', ['priority' => $queue['priority'], 'id' => $queue["id"], 'cooldown' => $cooldown]);
sleep($cooldown);
}
Logger::enableWorker($funcname);
Logger::info("Process start.", ['priority' => $queue["priority"], 'id' => $queue["id"]]);
Logger::info("Process start.", ['priority' => $queue['priority'], 'id' => $queue["id"]]);
$stamp = (float)microtime(true);
@ -480,11 +496,6 @@ class Worker
// For this reason the variables have to be initialized.
DI::profiler()->reset();
if (!in_array($queue['priority'], PRIORITIES)) {
Logger::warning('Invalid priority', ['queue' => $queue, 'callstack' => System::callstack(20)]);
$queue['priority'] = PRIORITY_MEDIUM;
}
$a->setQueue($queue);
$up_duration = microtime(true) - self::$up_start;
@ -529,21 +540,21 @@ class Worker
self::$lock_duration = 0;
if ($duration > 3600) {
Logger::info('Longer than 1 hour.', ['priority' => $queue["priority"], 'id' => $queue["id"], 'duration' => round($duration/60, 3)]);
Logger::info('Longer than 1 hour.', ['priority' => $queue['priority'], 'id' => $queue["id"], 'duration' => round($duration/60, 3)]);
} elseif ($duration > 600) {
Logger::info('Longer than 10 minutes.', ['priority' => $queue["priority"], 'id' => $queue["id"], 'duration' => round($duration/60, 3)]);
Logger::info('Longer than 10 minutes.', ['priority' => $queue['priority'], 'id' => $queue["id"], 'duration' => round($duration/60, 3)]);
} elseif ($duration > 300) {
Logger::info('Longer than 5 minutes.', ['priority' => $queue["priority"], 'id' => $queue["id"], 'duration' => round($duration/60, 3)]);
Logger::info('Longer than 5 minutes.', ['priority' => $queue['priority'], 'id' => $queue["id"], 'duration' => round($duration/60, 3)]);
} elseif ($duration > 120) {
Logger::info('Longer than 2 minutes.', ['priority' => $queue["priority"], 'id' => $queue["id"], 'duration' => round($duration/60, 3)]);
Logger::info('Longer than 2 minutes.', ['priority' => $queue['priority'], 'id' => $queue["id"], 'duration' => round($duration/60, 3)]);
}
Logger::info('Process done.', ['priority' => $queue["priority"], 'id' => $queue["id"], 'duration' => round($duration, 3)]);
Logger::info('Process done.', ['priority' => $queue['priority'], 'id' => $queue["id"], 'duration' => round($duration, 3)]);
DI::profiler()->saveLog(DI::logger(), "ID " . $queue["id"] . ": " . $funcname);
if ($cooldown > 0) {
Logger::info('Post execution cooldown.', ['priority' => $queue["priority"], 'id' => $queue["id"], 'cooldown' => $cooldown]);
Logger::info('Post execution cooldown.', ['priority' => $queue['priority'], 'id' => $queue["id"], 'cooldown' => $cooldown]);
sleep($cooldown);
}
}
@ -649,6 +660,8 @@ class Worker
self::$db_duration += (microtime(true) - $stamp);
while ($entry = DBA::fetch($entries)) {
$entry = self::checkPriority($entry);
if (!posix_kill($entry["pid"], 0)) {
$stamp = (float)microtime(true);
DBA::update(
@ -660,14 +673,10 @@ class Worker
self::$db_duration_write += (microtime(true) - $stamp);
} else {
// Kill long running processes
// Check if the priority is in a valid range
if (!in_array($entry["priority"], [PRIORITY_CRITICAL, PRIORITY_HIGH, PRIORITY_MEDIUM, PRIORITY_LOW, PRIORITY_NEGLIGIBLE])) {
$entry["priority"] = PRIORITY_MEDIUM;
}
// Define the maximum durations
$max_duration_defaults = [PRIORITY_CRITICAL => 720, PRIORITY_HIGH => 10, PRIORITY_MEDIUM => 60, PRIORITY_LOW => 180, PRIORITY_NEGLIGIBLE => 720];
$max_duration = $max_duration_defaults[$entry["priority"]];
$max_duration = $max_duration_defaults[$entry['priority']];
$argv = json_decode($entry['parameter'], true);
if (!empty($entry['command'])) {
@ -689,12 +698,12 @@ class Worker
// We killed the stale process.
// To avoid a blocking situation we reschedule the process at the beginning of the queue.
// Additionally we are lowering the priority. (But not PRIORITY_CRITICAL)
$new_priority = $entry["priority"];
if ($entry["priority"] == PRIORITY_HIGH) {
$new_priority = $entry['priority'];
if ($entry['priority'] == PRIORITY_HIGH) {
$new_priority = PRIORITY_MEDIUM;
} elseif ($entry["priority"] == PRIORITY_MEDIUM) {
} elseif ($entry['priority'] == PRIORITY_MEDIUM) {
$new_priority = PRIORITY_LOW;
} elseif ($entry["priority"] != PRIORITY_CRITICAL) {
} elseif ($entry['priority'] != PRIORITY_CRITICAL) {
$new_priority = PRIORITY_NEGLIGIBLE;
}
$stamp = (float)microtime(true);
@ -778,12 +787,12 @@ class Worker
self::$db_duration_stat += (microtime(true) - $stamp);
while ($entry = DBA::fetch($jobs)) {
$stamp = (float)microtime(true);
$running = DBA::count('workerqueue-view', ['priority' => $entry["priority"]]);
$running = DBA::count('workerqueue-view', ['priority' => $entry['priority']]);
self::$db_duration += (microtime(true) - $stamp);
self::$db_duration_stat += (microtime(true) - $stamp);
$idle_workers -= $running;
$waiting_processes += $entry["entries"];
$listitem[$entry["priority"]] = $entry["priority"] . ":" . $running . "/" . $entry["entries"];
$listitem[$entry['priority']] = $entry['priority'] . ":" . $running . "/" . $entry["entries"];
}
DBA::close($jobs);
} else {
@ -795,7 +804,7 @@ class Worker
while ($entry = DBA::fetch($jobs)) {
$idle_workers -= $entry["running"];
$listitem[$entry["priority"]] = $entry["priority"].":".$entry["running"];
$listitem[$entry['priority']] = $entry['priority'].":".$entry["running"];
}
DBA::close($jobs);
}
@ -1140,6 +1149,32 @@ class Worker
// Cleaning dead processes
self::killStaleWorkers();
// Remove old entries from the workerqueue
self::cleanWorkerQueue();
}
/**
* Remove old entries from the workerqueue
*
* @return void
*/
private static function cleanWorkerQueue()
{
DBA::delete('workerqueue', ["`done` AND `executed` < ?", DateTimeFormat::utc('now - 1 hour')]);
// Optimizing this table only last seconds
if (DI::config()->get('system', 'optimize_tables')) {
// We are acquiring the two locks from the worker to avoid locking problems
if (DI::lock()->acquire(Worker::LOCK_PROCESS, 10)) {
if (DI::lock()->acquire(Worker::LOCK_WORKER, 10)) {
DBA::e("OPTIMIZE TABLE `workerqueue`");
DBA::e("OPTIMIZE TABLE `process`");
DI::lock()->release(Worker::LOCK_WORKER);
}
DI::lock()->release(Worker::LOCK_PROCESS);
}
}
}
/**
@ -1388,6 +1423,8 @@ class Worker
return false;
}
$queue = self::checkPriority($queue);
$id = $queue['id'];
$priority = $queue['priority'];

View file

@ -828,8 +828,10 @@ class Contact
if (in_array($contact['rel'], [self::SHARING, self::FRIEND])) {
$cdata = Contact::getPublicAndUserContactID($contact['id'], $contact['uid']);
if (!empty($cdata['public'])) {
Worker::add(PRIORITY_HIGH, 'Contact\Unfollow', $cdata['public'], $contact['uid']);
}
}
self::removeSharer($contact);
}
@ -855,8 +857,10 @@ class Contact
if (in_array($contact['rel'], [self::FOLLOWER, self::FRIEND])) {
$cdata = Contact::getPublicAndUserContactID($contact['id'], $contact['uid']);
if (!empty($cdata['public'])) {
Worker::add(PRIORITY_HIGH, 'Contact\RevokeFollow', $cdata['public'], $contact['uid']);
}
}
self::removeFollower($contact);
}
@ -880,11 +884,11 @@ class Contact
$cdata = Contact::getPublicAndUserContactID($contact['id'], $contact['uid']);
if (in_array($contact['rel'], [self::SHARING, self::FRIEND])) {
if (in_array($contact['rel'], [self::SHARING, self::FRIEND]) && !empty($cdata['public'])) {
Worker::add(PRIORITY_HIGH, 'Contact\Unfollow', $cdata['public'], $contact['uid']);
}
if (in_array($contact['rel'], [self::FOLLOWER, self::FRIEND])) {
if (in_array($contact['rel'], [self::FOLLOWER, self::FRIEND]) && !empty($cdata['public'])) {
Worker::add(PRIORITY_HIGH, 'Contact\RevokeFollow', $cdata['public'], $contact['uid']);
}
@ -1799,7 +1803,7 @@ class Contact
{
// We have to fetch the "updated" variable when it wasn't provided
// The parameter can be provided to improve performance
if (empty($updated) || empty($guid)) {
if (empty($updated)) {
$account = DBA::selectFirst('account-user-view', ['updated', 'guid'], ['id' => $cid]);
$updated = $account['updated'] ?? '';
$guid = $account['guid'] ?? '';

View file

@ -87,8 +87,8 @@ class Item
'wall', 'private', 'starred', 'origin', 'parent-origin', 'title', 'body', 'language',
'content-warning', 'location', 'coord', 'app', 'rendered-hash', 'rendered-html', 'object',
'allow_cid', 'allow_gid', 'deny_cid', 'deny_gid', 'mention', 'global',
'author-id', 'author-link', 'author-name', 'author-avatar', 'author-network',
'owner-id', 'owner-link', 'owner-name', 'owner-avatar', 'owner-network', 'owner-contact-type',
'author-id', 'author-link', 'author-name', 'author-avatar', 'author-network', 'author-updated', 'author-gsid', 'author-addr', 'author-uri-id',
'owner-id', 'owner-link', 'owner-name', 'owner-avatar', 'owner-network', 'owner-contact-type', 'owner-updated',
'causer-id', 'causer-link', 'causer-name', 'causer-avatar', 'causer-contact-type', 'causer-network',
'contact-id', 'contact-uid', 'contact-link', 'contact-name', 'contact-avatar',
'writable', 'self', 'cid', 'alias',
@ -1215,9 +1215,26 @@ class Item
Worker::add(['priority' => $priority, 'dont_fork' => true], 'Notifier', $notify_type, (int)$posted_item['uri-id'], (int)$posted_item['uid']);
}
// Fill the cache with the rendered content.
if (in_array($posted_item['gravity'], [GRAVITY_PARENT, GRAVITY_COMMENT]) && ($posted_item['uid'] == 0)) {
self::updateDisplayCache($posted_item['uri-id']);
}
return $post_user_id;
}
/**
* Update the display cache
*
* @param integer $uri_id
* @return void
*/
public static function updateDisplayCache(int $uri_id)
{
$item = Post::selectFirst(self::DISPLAY_FIELDLIST, ['uri-id' => $uri_id]);
self::prepareBody($item, false, false, true);
}
/**
* Change the owner of a parent item if it had been shared by a forum
*
@ -2692,7 +2709,7 @@ class Item
* @throws \Friendica\Network\HTTPException\InternalServerErrorException
* @todo Remove reference, simply return "rendered-html" and "rendered-hash"
*/
public static function putInCache(&$item)
private static function putInCache(&$item)
{
// Save original body to prevent addons to modify it
$body = $item['body'];
@ -2705,8 +2722,6 @@ class Item
|| $rendered_hash != hash('md5', BBCode::VERSION . '::' . $body)
|| DI::config()->get('system', 'ignore_cache')
) {
self::addRedirToImageTags($item);
$item['rendered-html'] = BBCode::convertForUriId($item['uri-id'], $item['body']);
$item['rendered-hash'] = hash('md5', BBCode::VERSION . '::' . $body);
@ -2731,31 +2746,6 @@ class Item
$item['body'] = $body;
}
/**
* Find any non-embedded images in private items and add redir links to them
*
* @param array &$item The field array of an item row
*/
private static function addRedirToImageTags(array &$item)
{
$app = DI::app();
$matches = [];
$cnt = preg_match_all('|\[img\](http[^\[]*?/photo/[a-fA-F0-9]+?(-[0-9]\.[\w]+?)?)\[\/img\]|', $item['body'], $matches, PREG_SET_ORDER);
if ($cnt) {
foreach ($matches as $mtch) {
if (strpos($mtch[1], '/redir') !== false) {
continue;
}
if ((local_user() == $item['uid']) && ($item['private'] == self::PRIVATE) && ($item['contact-id'] != $app->getContactId()) && ($item['network'] == Protocol::DFRN)) {
$img_url = 'redir/' . $item['contact-id'] . '?url=' . urlencode($mtch[1]);
$item['body'] = str_replace($mtch[0], '[img]' . $img_url . '[/img]', $item['body']);
}
}
}
}
/**
* Given an item array, convert the body element from bbcode to html and add smilie icons.
* If attach is true, also add icons for item attachments.
@ -2763,6 +2753,7 @@ class Item
* @param array $item
* @param boolean $attach
* @param boolean $is_preview
* @param boolean $only_cache
* @return string item body html
* @throws \Friendica\Network\HTTPException\InternalServerErrorException
* @throws \ImagickException
@ -2771,7 +2762,7 @@ class Item
* @hook prepare_body ('item'=>item array, 'html'=>body string, 'is_preview'=>boolean, 'filter_reasons'=>string array) after first bbcode to html
* @hook prepare_body_final ('item'=>item array, 'html'=>body string) after attach icons and blockquote special case handling (spoiler, author)
*/
public static function prepareBody(array &$item, $attach = false, $is_preview = false)
public static function prepareBody(array &$item, $attach = false, $is_preview = false, $only_cache = false)
{
$a = DI::app();
Hook::callAll('prepare_body_init', $item);
@ -2812,6 +2803,10 @@ class Item
$item['body'] = $body;
$s = $item["rendered-html"];
if ($only_cache) {
return;
}
// Compile eventual content filter reasons
$filter_reasons = [];
if (!$is_preview && public_contact() != $item['author-id']) {
@ -3205,6 +3200,12 @@ class Item
'orig_title' => DI::l10n()->t('View on separate page'),
];
if (!empty($plink) && ($item['private'] == self::PRIVATE)) {
$author = ['uid' => 0, 'id' => $item['author-id'],
'network' => $item['author-network'], 'url' => $item['author-link']];
$plink = Contact::magicLinkByContact($author, $plink);
}
if (!empty($plink)) {
$ret['href'] = DI::baseUrl()->remove($plink);
$ret['title'] = DI::l10n()->t('Link to source');

View file

@ -111,6 +111,11 @@ class Category
return array_column($tags, 'name');
}
public static function existsForURIId(int $uri_id, int $uid)
{
return DBA::exists('post-category', ['uri-id' => $uri_id, 'uid' => $uid]);
}
/**
* Generates an array of files or categories of a given uri-id
*

View file

@ -0,0 +1,64 @@
<?php
/**
* @copyright Copyright (C) 2010-2022, the Friendica project
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
namespace Friendica\Model\Post;
use Friendica\Database\DBA;
use BadMethodCallException;
use Friendica\Database\Database;
use Friendica\Model\ItemURI;
class Delivery
{
/**
* Add a post to an inbox
*
* @param integer $uri_id
* @param string $inbox
* @param string $created
*/
public static function add(int $uri_id, int $uid, string $inbox, string $created, string $command)
{
if (empty($uri_id)) {
throw new BadMethodCallException('Empty URI_id');
}
$fields = ['uri-id' => $uri_id, 'uid' => $uid, 'inbox-id' => ItemURI::getIdByURI($inbox), 'created' => $created, 'command' => $command];
DBA::insert('post-delivery', $fields, Database::INSERT_IGNORE);
}
/**
* Remove post from an inbox after delivery
*
* @param integer $uri_id
* @param string $inbox
*/
public static function remove(int $uri_id, string $inbox)
{
DBA::delete('post-delivery', ['uri-id' => $uri_id, 'inbox-id' => ItemURI::getIdByURI($inbox)]);
}
public static function selectForInbox(string $inbox)
{
return DBA::selectToArray('post-delivery', [], ['inbox-id' => ItemURI::getIdByURI($inbox)], ['order' => ['created']]);
}
}

View file

@ -25,6 +25,7 @@ use Friendica\BaseModule;
use Friendica\Core\System;
use Friendica\DI;
use Friendica\Protocol\Feed as ProtocolFeed;
use Friendica\Network\HTTPException;
/**
* Provides public Atom feeds
@ -42,7 +43,7 @@ use Friendica\Protocol\Feed as ProtocolFeed;
*/
class Feed extends BaseModule
{
protected function content(array $request = []): string
protected function rawContent(array $request = [])
{
$last_update = $request['last_update'] ?? '';
$nocache = !empty($request['nocache']) && local_user();
@ -66,6 +67,11 @@ class Feed extends BaseModule
$type = 'posts';
}
System::httpExit(ProtocolFeed::atom($this->parameters['nickname'], $last_update, 10, $type, $nocache, true), Response::TYPE_ATOM);
$feed = ProtocolFeed::atom($this->parameters['nickname'], $last_update, 10, $type, $nocache, true);
if (empty($feed)) {
throw new HTTPException\NotFoundException(DI::l10n()->t('User not found.'));
}
System::httpExit($feed, Response::TYPE_ATOM);
}
}

View file

@ -140,7 +140,7 @@ class HttpClient implements ICanSendHttpRequests
}
};
if (empty($conf[HttpClientOptions::HEADERS]['Accept'])) {
if (empty($conf[HttpClientOptions::HEADERS]['Accept']) && in_array($method, ['get', 'head'])) {
$this->logger->info('Accept header was missing, using default.', ['url' => $url, 'callstack' => System::callstack()]);
$conf[HttpClientOptions::HEADERS]['Accept'] = HttpClientAccept::DEFAULT;
}

View file

@ -29,7 +29,6 @@ use Friendica\Core\Logger;
use Friendica\Core\Protocol;
use Friendica\Core\Renderer;
use Friendica\Core\Session;
use Friendica\Database\DBA;
use Friendica\DI;
use Friendica\Model\Contact;
use Friendica\Model\Item;
@ -451,6 +450,16 @@ class Post
$browsershare = null;
}
if (in_array($item['network'], [Protocol::FEED, Protocol::MAIL])) {
$owner_avatar = $author_avatar = $item['contact-id'];
$owner_updated = $author_updated = '';
} else {
$owner_avatar = $item['owner-id'];
$owner_updated = $item['owner-updated'];
$author_avatar = $item['author-id'];
$author_updated = $item['author-updated'];
}
$tmp_item = [
'template' => $this->getTemplate(),
'type' => implode("", array_slice(explode("/", $item['verb']), -1)),
@ -482,7 +491,7 @@ class Post
'profile_url' => $profile_link,
'name' => $profile_name,
'item_photo_menu_html' => DI::contentItem()->photoMenu($item, $formSecurityToken),
'thumb' => DI::baseUrl()->remove(Contact::getAvatarUrlForUrl($item['author-link'], $item['uid'], Proxy::SIZE_THUMB)),
'thumb' => DI::baseUrl()->remove(Contact::getAvatarUrlForId($author_avatar, Proxy::SIZE_THUMB, $author_updated)),
'osparkle' => $osparkle,
'sparkle' => $sparkle,
'title' => $title,
@ -499,7 +508,7 @@ class Post
'shiny' => $shiny,
'owner_self' => $item['author-link'] == Session::get('my_url'),
'owner_url' => $this->getOwnerUrl(),
'owner_photo' => DI::baseUrl()->remove(Contact::getAvatarUrlForUrl($item['owner-link'], $item['uid'], Proxy::SIZE_THUMB)),
'owner_photo' => DI::baseUrl()->remove(Contact::getAvatarUrlForId($owner_avatar, Proxy::SIZE_THUMB, $owner_updated)),
'owner_name' => $this->getOwnerName(),
'plink' => Item::getPlink($item),
'browsershare' => $browsershare,
@ -529,8 +538,8 @@ class Post
'thread_level' => $thread_level,
'edited' => $edited,
'network' => $item["network"],
'network_name' => ContactSelector::networkToName($item['author-network'], $item['author-link'], $item['network']),
'network_icon' => ContactSelector::networkToIcon($item['network'], $item['author-link']),
'network_name' => ContactSelector::networkToName($item['author-network'], $item['author-link'], $item['network'], $item['author-gsid']),
'network_icon' => ContactSelector::networkToIcon($item['network'], $item['author-link'], $item['author-gsid']),
'received' => $item['received'],
'commented' => $item['commented'],
'created_date' => $item['created'],
@ -898,12 +907,7 @@ class Post
}
$owner = User::getOwnerDataById($a->getLoggedInUserId());
$item = PostModel::selectFirst(['author-addr', 'uri-id', 'network', 'gravity', 'content-warning'], ['id' => $this->getId()]);
if (!DBA::isResult($item) || empty($item['author-addr'])) {
// Should not happen
return '';
}
$item = $this->getData();
if (!empty($item['content-warning']) && Feature::isEnabled(local_user(), 'add_abstract')) {
$text = '[abstract=' . Protocol::ACTIVITYPUB . ']' . $item['content-warning'] . "[/abstract]\n";
@ -967,7 +971,7 @@ class Post
$uid = $conv->getProfileOwner();
$parent_uid = $this->getDataValue('uid');
$contact = Contact::getById($a->getContactId());
$owner = User::getOwnerDataById($a->getLoggedInUserId());
$default_text = $this->getDefaultText();
@ -986,9 +990,9 @@ class Post
'$qcomment' => $qcomment,
'$default' => $default_text,
'$profile_uid' => $uid,
'$mylink' => DI::baseUrl()->remove($contact['url'] ?? ''),
'$mylink' => DI::baseUrl()->remove($owner['url'] ?? ''),
'$mytitle' => DI::l10n()->t('This is you'),
'$myphoto' => DI::baseUrl()->remove($contact['thumb'] ?? ''),
'$myphoto' => DI::baseUrl()->remove($owner['thumb'] ?? ''),
'$comment' => DI::l10n()->t('Comment'),
'$submit' => DI::l10n()->t('Submit'),
'$loading' => DI::l10n()->t('Loading...'),

View file

@ -938,7 +938,7 @@ class Receiver
// Fetch the receivers for the public and the followers collection
if ((($receiver == $followers) || (($receiver == self::PUBLIC_COLLECTION) && !$is_forum)) && !empty($actor)) {
$receivers = self::getReceiverForActor($actor, $tags, $receivers, $follower_target);
$receivers = self::getReceiverForActor($actor, $tags, $receivers, $follower_target, $profile);
continue;
}
@ -1000,15 +1000,27 @@ class Receiver
* @param array $tags
* @param array $receivers
* @param integer $target_type
* @param array $profile
*
* @return array with receivers (user id)
* @throws \Exception
*/
private static function getReceiverForActor($actor, $tags, $receivers, $target_type)
private static function getReceiverForActor($actor, $tags, $receivers, $target_type, $profile)
{
$basecondition = ['rel' => [Contact::SHARING, Contact::FRIEND, Contact::FOLLOWER],
'network' => Protocol::FEDERATED, 'archive' => false, 'pending' => false];
if (!empty($profile['uri-id'])) {
$condition = DBA::mergeConditions($basecondition, ["`uri-id` = ? AND `uid` != ?", $profile['uri-id'], 0]);
$contacts = DBA::select('contact', ['uid', 'rel'], $condition);
while ($contact = DBA::fetch($contacts)) {
if (empty($receivers[$contact['uid']]) && self::isValidReceiverForActor($contact, $tags)) {
$receivers[$contact['uid']] = ['uid' => $contact['uid'], 'type' => $target_type];
}
}
DBA::close($contacts);
} else {
// This part will only be called while post update 1426 wasn't finished
$condition = DBA::mergeConditions($basecondition, ["`nurl` = ? AND `uid` != ?", Strings::normaliseLink($actor), 0]);
$contacts = DBA::select('contact', ['uid', 'rel'], $condition);
while ($contact = DBA::fetch($contacts)) {
@ -1027,6 +1039,7 @@ class Receiver
}
}
DBA::close($contacts);
}
return $receivers;
}

View file

@ -27,6 +27,7 @@ use Friendica\Database\DBA;
use Friendica\DI;
use Friendica\Model\APContact;
use Friendica\Model\Contact;
use Friendica\Model\ItemURI;
use Friendica\Model\User;
use Friendica\Network\HTTPClient\Client\HttpClientAccept;
use Friendica\Network\HTTPClient\Client\HttpClientOptions;
@ -329,7 +330,7 @@ class HTTPSignature
$status = DBA::selectFirst('inbox-status', [], ['url' => $url]);
if (!DBA::isResult($status)) {
DBA::insert('inbox-status', ['url' => $url, 'created' => $now, 'shared' => $shared], Database::INSERT_IGNORE);
DBA::insert('inbox-status', ['url' => $url, 'uri-id' => ItemURI::getIdByURI($url), 'created' => $now, 'shared' => $shared], Database::INSERT_IGNORE);
$status = DBA::selectFirst('inbox-status', [], ['url' => $url]);
}
@ -369,6 +370,10 @@ class HTTPSignature
$fields['archive'] = false;
}
if (empty($status['uri-id'])) {
$fields['uri-id'] = ItemURI::getIdByURI($url);
}
DBA::update('inbox-status', $fields, ['url' => $url]);
}

View file

@ -25,6 +25,7 @@ use Friendica\Core\Logger;
use Friendica\Core\Worker;
use Friendica\Model\Contact;
use Friendica\Model\GServer;
use Friendica\Model\Item;
use Friendica\Model\Post;
use Friendica\Protocol\ActivityPub;
use Friendica\Util\HTTPSignature;
@ -56,6 +57,72 @@ class APDelivery
Logger::info('Invoked', ['cmd' => $cmd, 'inbox' => $inbox, 'id' => $item_id, 'uri-id' => $uri_id, 'uid' => $uid]);
if (empty($uri_id)) {
$result = self::deliver($inbox);
$success = $result['success'];
$uri_ids = $result['uri_ids'];
}
if (empty($uri_ids)) {
$success = self::deliverToInbox($cmd, $item_id, $inbox, $uid, $receivers, $uri_id);
}
if (!$success && !Worker::defer() && in_array($cmd, [Delivery::POST])) {
if (!empty($uri_id)) {
Post\Delivery::remove($uri_id, $inbox);
Post\DeliveryData::incrementQueueFailed($uri_id);
} elseif (!empty($uri_ids)) {
foreach ($uri_ids as $uri_id) {
Post\Delivery::remove($uri_id, $inbox);
Post\DeliveryData::incrementQueueFailed($uri_id);
}
}
} elseif ($success && in_array($cmd, [Delivery::POST])) {
if (!empty($uri_id)) {
Post\DeliveryData::incrementQueueDone($uri_id, Post\DeliveryData::ACTIVITYPUB);
} elseif (!empty($uri_ids)) {
foreach ($uri_ids as $uri_id) {
Post\DeliveryData::incrementQueueDone($uri_id, Post\DeliveryData::ACTIVITYPUB);
}
}
}
}
private static function deliver(string $inbox)
{
$uri_ids = [];
$success = true;
$posts = Post\Delivery::selectForInbox($inbox);
foreach ($posts as $post) {
$uri_ids[] = $post['uri-id'];
if ($success) {
$success = self::deliverToInbox($post['command'], 0, $inbox, $post['uid'], [], $post['uri-id']);
}
}
return ['success' => $success, 'uri_ids' => $uri_ids];
}
private static function deliverToInbox(string $cmd, int $item_id, string $inbox, int $uid, array $receivers, int $uri_id)
{
if (empty($item_id) && !empty($uri_id) && !empty($uid)) {
$item = Post::selectFirst(['id', 'parent', 'origin'], ['uri-id' => $uri_id, 'uid' => $uid]);
$item_id = $item['id'] ?? 0;
if (empty($receivers) && !empty($item)) {
$parent = Post::selectFirst(Item::DELIVER_FIELDLIST, ['id' => $item['parent']]);
$inboxes = ActivityPub\Transmitter::fetchTargetInboxes($parent, $uid);
$receivers = $inboxes[$inbox] ?? [];
// When we haven't fetched the receiver list, it can be a personal inbox
if (empty($receivers)) {
$inboxes = ActivityPub\Transmitter::fetchTargetInboxes($parent, $uid, true);
$receivers = $inboxes[$inbox] ?? [];
}
}
}
$success = true;
if ($cmd == Delivery::MAIL) {
@ -77,9 +144,21 @@ class APDelivery
$data = ActivityPub\Transmitter::createCachedActivityFromItem($item_id);
if (!empty($data)) {
$success = HTTPSignature::transmit($data, $inbox, $uid);
if ($success && $uri_id) {
Post\Delivery::remove($uri_id, $inbox);
}
}
}
self::setSuccess($receivers, $success);
Logger::info('Delivered', ['cmd' => $cmd, 'inbox' => $inbox, 'id' => $item_id, 'uri-id' => $uri_id, 'uid' => $uid, 'success' => $success]);
return $success;
}
private static function setSuccess(array $receivers, bool $success)
{
$gsid = null;
foreach ($receivers as $receiver) {
@ -100,11 +179,5 @@ class APDelivery
if (!empty($gsid)) {
GServer::setProtocol($gsid, Post\DeliveryData::ACTIVITYPUB);
}
if (!$success && !Worker::defer() && in_array($cmd, [Delivery::POST])) {
Post\DeliveryData::incrementQueueFailed($uri_id);
} elseif ($success && in_array($cmd, [Delivery::POST])) {
Post\DeliveryData::incrementQueueDone($uri_id, Post\DeliveryData::ACTIVITYPUB);
}
}
}

View file

@ -1,51 +0,0 @@
<?php
/**
* @copyright Copyright (C) 2010-2022, the Friendica project
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
namespace Friendica\Worker;
use Friendica\Core\Worker;
use Friendica\Database\DBA;
use Friendica\DI;
use Friendica\Util\DateTimeFormat;
/**
* Delete all done workerqueue entries
*/
class CleanWorkerQueue
{
public static function execute()
{
DBA::delete('workerqueue', ["`done` AND `executed` < ?", DateTimeFormat::utc('now - 1 hour')]);
// Optimizing this table only last seconds
if (DI::config()->get('system', 'optimize_tables')) {
// We are acquiring the two locks from the worker to avoid locking problems
if (DI::lock()->acquire(Worker::LOCK_PROCESS, 10)) {
if (DI::lock()->acquire(Worker::LOCK_WORKER, 10)) {
DBA::e("OPTIMIZE TABLE `workerqueue`");
DBA::e("OPTIMIZE TABLE `process`");
DI::lock()->release(Worker::LOCK_WORKER);
}
DI::lock()->release(Worker::LOCK_PROCESS);
}
}
}
}

View file

@ -92,9 +92,6 @@ class Cron
Worker::add(PRIORITY_LOW, 'PullDirectory');
}
// Delete all done workerqueue entries
Worker::add(PRIORITY_LOW, 'CleanWorkerQueue');
// Clear cache entries
Worker::add(PRIORITY_LOW, 'ClearCache');

View file

@ -189,6 +189,9 @@ class ExpirePosts
AND NOT EXISTS(SELECT `uri-id` FROM `contact` WHERE `uri-id` = `item-uri`.`id`)
AND NOT EXISTS(SELECT `uri-id` FROM `apcontact` WHERE `uri-id` = `item-uri`.`id`)
AND NOT EXISTS(SELECT `uri-id` FROM `fcontact` WHERE `uri-id` = `item-uri`.`id`)
AND NOT EXISTS(SELECT `uri-id` FROM `inbox-status` WHERE `uri-id` = `item-uri`.`id`)
AND NOT EXISTS(SELECT `uri-id` FROM `post-delivery` WHERE `uri-id` = `item-uri`.`id`)
AND NOT EXISTS(SELECT `uri-id` FROM `post-delivery` WHERE `inbox-id` = `item-uri`.`id`)
AND NOT EXISTS(SELECT `parent-uri-id` FROM `mail` WHERE `parent-uri-id` = `item-uri`.`id`)
AND NOT EXISTS(SELECT `thr-parent-id` FROM `mail` WHERE `thr-parent-id` = `item-uri`.`id`)", $item['uri-id']]);

View file

@ -677,7 +677,7 @@ class Notifier
}
DBA::close($contacts_stmt);
$inboxes = ActivityPub\Transmitter::fetchTargetInboxesforUser(0);
$inboxes = ActivityPub\Transmitter::fetchTargetInboxesforUser($self_user_id);
foreach ($inboxes as $inbox => $receivers) {
Logger::info('Account removal via ActivityPub', ['uid' => $self_user_id, 'inbox' => $inbox]);
Worker::add(['priority' => PRIORITY_NEGLIGIBLE, 'created' => $created, 'dont_fork' => true],
@ -750,9 +750,7 @@ class Notifier
Logger::info('Remote item ' . $target_item['id'] . ' with URL ' . $target_item['uri'] . ' is no AP post. It will not be distributed.');
return ['count' => 0, 'contacts' => []];
} elseif ($parent['origin']) {
// Remote items are transmitted via the personal inboxes.
// Doing so ensures that the dedicated receiver will get the message.
$inboxes = ActivityPub\Transmitter::fetchTargetInboxes($parent, $uid, true, $target_item['id']);
$inboxes = ActivityPub\Transmitter::fetchTargetInboxes($parent, $uid, false, $target_item['id']);
if (in_array($target_item['private'], [Item::PUBLIC])) {
$inboxes = ActivityPub\Transmitter::addRelayServerInboxesForItem($parent['id'], $inboxes);
@ -788,20 +786,32 @@ class Notifier
Logger::info('Delivery via ActivityPub', ['cmd' => $cmd, 'id' => $target_item['id'], 'inbox' => $inbox]);
if (DI::config()->get('system', 'bulk_delivery')) {
$delivery_queue_count++;
Post\Delivery::add($target_item['uri-id'], $uid, $inbox, $target_item['created'], $cmd);
Worker::add(['priority' => $priority, 'dont_fork' => true], 'APDelivery', $cmd, 0, $inbox, $uid);
} else {
if (Worker::add(['priority' => $priority, 'created' => $created, 'dont_fork' => true],
'APDelivery', $cmd, $target_item['id'], $inbox, $uid, $receivers, $target_item['uri-id'])) {
$delivery_queue_count++;
}
}
}
// We deliver posts to relay servers slightly delayed to priorize the direct delivery
foreach ($relay_inboxes as $inbox) {
Logger::info('Delivery to relay servers via ActivityPub', ['cmd' => $cmd, 'id' => $target_item['id'], 'inbox' => $inbox]);
if (DI::config()->get('system', 'bulk_delivery')) {
$delivery_queue_count++;
Post\Delivery::add($target_item['uri-id'], $uid, $inbox, $target_item['created'], $cmd);
Worker::add(['priority' => $priority, 'dont_fork' => true], 'APDelivery', $cmd, 0, $inbox, $uid);
} else {
if (Worker::add(['priority' => $priority, 'dont_fork' => true], 'APDelivery', $cmd, $target_item['id'], $inbox, $uid, [], $target_item['uri-id'])) {
$delivery_queue_count++;
}
}
}
return ['count' => $delivery_queue_count, 'contacts' => $contacts];
}

View file

@ -55,7 +55,7 @@
use Friendica\Database\DBA;
if (!defined('DB_UPDATE_VERSION')) {
define('DB_UPDATE_VERSION', 1460);
define('DB_UPDATE_VERSION', 1461);
}
return [
@ -772,6 +772,7 @@ return [
"comment" => "Status of ActivityPub inboxes",
"fields" => [
"url" => ["type" => "varbinary(255)", "not null" => "1", "primary" => "1", "comment" => "URL of the inbox"],
"uri-id" => ["type" => "int unsigned", "foreign" => ["item-uri" => "id"], "comment" => "Item-uri id of inbox url"],
"created" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => "Creation date of this entry"],
"success" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => "Date of the last successful delivery"],
"failure" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => "Date of the last failed delivery"],
@ -780,7 +781,8 @@ return [
"shared" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => "Is it a shared inbox?"]
],
"indexes" => [
"PRIMARY" => ["url"]
"PRIMARY" => ["url"],
"uri-id" => ["uri-id"],
]
],
"intro" => [
@ -1112,8 +1114,8 @@ return [
],
"indexes" => [
"PRIMARY" => ["uri-id", "uid", "type", "tid"],
"uri-id" => ["tid"],
"uid" => ["uid"],
"tid" => ["tid"],
"uid_uri-id" => ["uid", "uri-id"],
]
],
"post-collection" => [
@ -1155,6 +1157,21 @@ return [
"title-content-warning-body" => ["FULLTEXT", "title", "content-warning", "body"],
]
],
"post-delivery" => [
"comment" => "Delivery data for posts for the batch processing",
"fields" => [
"uri-id" => ["type" => "int unsigned", "not null" => "1", "primary" => "1", "foreign" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the item uri"],
"inbox-id" => ["type" => "int unsigned", "not null" => "1", "primary" => "1", "foreign" => ["item-uri" => "id"], "comment" => "Item-uri id of inbox url"],
"uid" => ["type" => "mediumint unsigned", "foreign" => ["user" => "uid"], "comment" => "Delivering user"],
"created" => ["type" => "datetime", "default" => DBA::NULL_DATETIME, "comment" => ""],
"command" => ["type" => "varbinary(32)", "comment" => ""],
],
"indexes" => [
"PRIMARY" => ["uri-id", "inbox-id"],
"inbox-id_created" => ["inbox-id", "created"],
"uid" => ["uid"],
]
],
"post-delivery-data" => [
"comment" => "Delivery data for items",
"fields" => [
@ -1213,6 +1230,7 @@ return [
"indexes" => [
"PRIMARY" => ["id"],
"uri-id-url" => ["UNIQUE", "uri-id", "url"],
"uri-id-id" => ["uri-id", "id"],
]
],
"post-question" => [

View file

@ -146,6 +146,9 @@
"author-network" => ["author", "network"],
"author-blocked" => ["author", "blocked"],
"author-hidden" => ["author", "hidden"],
"author-updated" => ["author", "updated"],
"author-gsid" => ["author", "gsid"],
"author-uri-id" => ["author", "uri-id"],
"owner-id" => ["post-user", "owner-id"],
"owner-link" => ["owner", "url"],
"owner-addr" => ["owner", "addr"],
@ -155,6 +158,7 @@
"owner-network" => ["owner", "network"],
"owner-blocked" => ["owner", "blocked"],
"owner-hidden" => ["owner", "hidden"],
"owner-updated" => ["owner", "updated"],
"owner-contact-type" => ["owner", "contact-type"],
"causer-id" => ["post-user", "causer-id"],
"causer-link" => ["causer", "url"],
@ -309,6 +313,9 @@
"author-network" => ["author", "network"],
"author-blocked" => ["author", "blocked"],
"author-hidden" => ["author", "hidden"],
"author-updated" => ["author", "updated"],
"author-gsid" => ["author", "gsid"],
"author-uri-id" => ["author", "uri-id"],
"owner-id" => ["post-thread-user", "owner-id"],
"owner-link" => ["owner", "url"],
"owner-addr" => ["owner", "addr"],
@ -318,6 +325,7 @@
"owner-network" => ["owner", "network"],
"owner-blocked" => ["owner", "blocked"],
"owner-hidden" => ["owner", "hidden"],
"owner-updated" => ["owner", "updated"],
"owner-contact-type" => ["owner", "contact-type"],
"causer-id" => ["post-thread-user", "causer-id"],
"causer-link" => ["causer", "url"],
@ -458,6 +466,9 @@
"author-network" => ["author", "network"],
"author-blocked" => ["author", "blocked"],
"author-hidden" => ["author", "hidden"],
"author-updated" => ["author", "updated"],
"author-gsid" => ["author", "gsid"],
"author-uri-id" => ["author", "uri-id"],
"owner-id" => ["post", "owner-id"],
"owner-link" => ["owner", "url"],
"owner-addr" => ["owner", "addr"],
@ -467,6 +478,7 @@
"owner-network" => ["owner", "network"],
"owner-blocked" => ["owner", "blocked"],
"owner-hidden" => ["owner", "hidden"],
"owner-updated" => ["owner", "updated"],
"owner-contact-type" => ["owner", "contact-type"],
"causer-id" => ["post", "causer-id"],
"causer-link" => ["causer", "url"],
@ -583,6 +595,9 @@
"author-network" => ["author", "network"],
"author-blocked" => ["author", "blocked"],
"author-hidden" => ["author", "hidden"],
"author-updated" => ["author", "updated"],
"author-gsid" => ["author", "gsid"],
"author-uri-id" => ["author", "uri-id"],
"owner-id" => ["post-thread", "owner-id"],
"owner-link" => ["owner", "url"],
"owner-addr" => ["owner", "addr"],
@ -592,6 +607,7 @@
"owner-network" => ["owner", "network"],
"owner-blocked" => ["owner", "blocked"],
"owner-hidden" => ["owner", "hidden"],
"owner-updated" => ["owner", "updated"],
"owner-contact-type" => ["owner", "contact-type"],
"causer-id" => ["post-thread", "causer-id"],
"causer-link" => ["causer", "url"],
@ -652,9 +668,14 @@
"cid" => ["post", "author-id"],
"received" => ["post", "received"],
"created" => ["post", "created"],
"commented" => ["post-thread", "commented"],
"thr-parent-id" => ["post", "thr-parent-id"],
"author-id" => ["post", "author-id"],
"gravity" => ["post", "gravity"],
],
"query" => "FROM `post-collection`
INNER JOIN `post` ON `post-collection`.`uri-id` = `post`.`uri-id`"
INNER JOIN `post` ON `post-collection`.`uri-id` = `post`.`uri-id`
INNER JOIN `post-thread` ON `post-thread`.`uri-id` = `post`.`parent-uri-id`"
],
"tag-view" => [
"fields" => [

View file

@ -122,6 +122,10 @@ return [
// Display "Emoji Only" posts in big.
'big_emojis' => false,
// bulk_delivery (Boolean)
// Delivers AP messages in a bulk (experimental)
'bulk_delivery' => false,
// block_local_dir (Boolean)
// Deny public access to the local user directory.
'block_local_dir' => false,