mirror of
https://github.com/friendica/friendica
synced 2024-12-23 04:00:15 +00:00
Merge pull request #14594 from annando/media-types
Reduce the number of HTTP requests in the media handling
This commit is contained in:
commit
59f7f87226
11 changed files with 207 additions and 89 deletions
|
@ -2110,6 +2110,7 @@ CREATE VIEW `post-engagement-user-view` AS SELECT
|
||||||
`post-thread-user`.`received` AS `received`,
|
`post-thread-user`.`received` AS `received`,
|
||||||
`post-thread-user`.`created` AS `created`,
|
`post-thread-user`.`created` AS `created`,
|
||||||
`post-thread-user`.`network` AS `network`,
|
`post-thread-user`.`network` AS `network`,
|
||||||
|
`post-user`.`protocol` AS `protocol`,
|
||||||
`post-engagement`.`language` AS `restricted`,
|
`post-engagement`.`language` AS `restricted`,
|
||||||
0 AS `comments`,
|
0 AS `comments`,
|
||||||
0 AS `activities`
|
0 AS `activities`
|
||||||
|
@ -2236,6 +2237,7 @@ CREATE VIEW `post-searchindex-user-view` AS SELECT
|
||||||
`post-thread-user`.`received` AS `received`,
|
`post-thread-user`.`received` AS `received`,
|
||||||
`post-thread-user`.`created` AS `created`,
|
`post-thread-user`.`created` AS `created`,
|
||||||
`post-thread-user`.`network` AS `network`,
|
`post-thread-user`.`network` AS `network`,
|
||||||
|
`post-user`.`protocol` AS `protocol`,
|
||||||
`post-searchindex`.`language` AS `restricted`,
|
`post-searchindex`.`language` AS `restricted`,
|
||||||
0 AS `comments`,
|
0 AS `comments`,
|
||||||
0 AS `activities`
|
0 AS `activities`
|
||||||
|
@ -2494,6 +2496,7 @@ CREATE VIEW `post-thread-origin-view` AS SELECT
|
||||||
`post-user`.`global` AS `global`,
|
`post-user`.`global` AS `global`,
|
||||||
EXISTS(SELECT `type` FROM `post-collection` WHERE `type` = 0 AND `uri-id` = `post-thread-user`.`uri-id`) AS `featured`,
|
EXISTS(SELECT `type` FROM `post-collection` WHERE `type` = 0 AND `uri-id` = `post-thread-user`.`uri-id`) AS `featured`,
|
||||||
`post-thread-user`.`network` AS `network`,
|
`post-thread-user`.`network` AS `network`,
|
||||||
|
`post-user`.`protocol` AS `protocol`,
|
||||||
`post-origin`.`vid` AS `vid`,
|
`post-origin`.`vid` AS `vid`,
|
||||||
`post-thread-user`.`psid` AS `psid`,
|
`post-thread-user`.`psid` AS `psid`,
|
||||||
IF (`post-origin`.`vid` IS NULL, '', `verb`.`name`) AS `verb`,
|
IF (`post-origin`.`vid` IS NULL, '', `verb`.`name`) AS `verb`,
|
||||||
|
@ -2881,6 +2884,7 @@ CREATE VIEW `post-thread-user-view` AS SELECT
|
||||||
`post-user`.`global` AS `global`,
|
`post-user`.`global` AS `global`,
|
||||||
EXISTS(SELECT `type` FROM `post-collection` WHERE `type` = 0 AND `uri-id` = `post-thread-user`.`uri-id`) AS `featured`,
|
EXISTS(SELECT `type` FROM `post-collection` WHERE `type` = 0 AND `uri-id` = `post-thread-user`.`uri-id`) AS `featured`,
|
||||||
`post-thread-user`.`network` AS `network`,
|
`post-thread-user`.`network` AS `network`,
|
||||||
|
`post-user`.`protocol` AS `protocol`,
|
||||||
`post-user`.`vid` AS `vid`,
|
`post-user`.`vid` AS `vid`,
|
||||||
`post-thread-user`.`psid` AS `psid`,
|
`post-thread-user`.`psid` AS `psid`,
|
||||||
IF (`post-user`.`vid` IS NULL, '', `verb`.`name`) AS `verb`,
|
IF (`post-user`.`vid` IS NULL, '', `verb`.`name`) AS `verb`,
|
||||||
|
@ -3061,6 +3065,7 @@ CREATE VIEW `post-view` AS SELECT
|
||||||
`post`.`global` AS `global`,
|
`post`.`global` AS `global`,
|
||||||
EXISTS(SELECT `type` FROM `post-collection` WHERE `type` = 0 AND `uri-id` = `post`.`uri-id`) AS `featured`,
|
EXISTS(SELECT `type` FROM `post-collection` WHERE `type` = 0 AND `uri-id` = `post`.`uri-id`) AS `featured`,
|
||||||
`post`.`network` AS `network`,
|
`post`.`network` AS `network`,
|
||||||
|
255 AS `protocol`,
|
||||||
`post`.`vid` AS `vid`,
|
`post`.`vid` AS `vid`,
|
||||||
IF (`post`.`vid` IS NULL, '', `verb`.`name`) AS `verb`,
|
IF (`post`.`vid` IS NULL, '', `verb`.`name`) AS `verb`,
|
||||||
`post-content`.`title` AS `title`,
|
`post-content`.`title` AS `title`,
|
||||||
|
@ -3215,6 +3220,7 @@ CREATE VIEW `post-thread-view` AS SELECT
|
||||||
`post`.`global` AS `global`,
|
`post`.`global` AS `global`,
|
||||||
EXISTS(SELECT `type` FROM `post-collection` WHERE `type` = 0 AND `uri-id` = `post-thread`.`uri-id`) AS `featured`,
|
EXISTS(SELECT `type` FROM `post-collection` WHERE `type` = 0 AND `uri-id` = `post-thread`.`uri-id`) AS `featured`,
|
||||||
`post-thread`.`network` AS `network`,
|
`post-thread`.`network` AS `network`,
|
||||||
|
255 AS `protocol`,
|
||||||
`post`.`vid` AS `vid`,
|
`post`.`vid` AS `vid`,
|
||||||
IF (`post`.`vid` IS NULL, '', `verb`.`name`) AS `verb`,
|
IF (`post`.`vid` IS NULL, '', `verb`.`name`) AS `verb`,
|
||||||
`post-content`.`title` AS `title`,
|
`post-content`.`title` AS `title`,
|
||||||
|
@ -3418,6 +3424,7 @@ CREATE VIEW `network-thread-view` AS SELECT
|
||||||
`post-thread-user`.`starred` AS `starred`,
|
`post-thread-user`.`starred` AS `starred`,
|
||||||
`post-thread-user`.`mention` AS `mention`,
|
`post-thread-user`.`mention` AS `mention`,
|
||||||
`post-thread-user`.`network` AS `network`,
|
`post-thread-user`.`network` AS `network`,
|
||||||
|
`post-user`.`protocol` AS `protocol`,
|
||||||
`post-thread-user`.`contact-id` AS `contact-id`,
|
`post-thread-user`.`contact-id` AS `contact-id`,
|
||||||
`ownercontact`.`contact-type` AS `contact-type`
|
`ownercontact`.`contact-type` AS `contact-type`
|
||||||
FROM `post-thread-user`
|
FROM `post-thread-user`
|
||||||
|
@ -3446,6 +3453,7 @@ CREATE VIEW `network-thread-circle-view` AS SELECT
|
||||||
`post-thread-user`.`starred` AS `starred`,
|
`post-thread-user`.`starred` AS `starred`,
|
||||||
`post-thread-user`.`mention` AS `mention`,
|
`post-thread-user`.`mention` AS `mention`,
|
||||||
`post-thread-user`.`network` AS `network`,
|
`post-thread-user`.`network` AS `network`,
|
||||||
|
`post-user`.`protocol` AS `protocol`,
|
||||||
`post-thread-user`.`contact-id` AS `contact-id`,
|
`post-thread-user`.`contact-id` AS `contact-id`,
|
||||||
`ownercontact`.`contact-type` AS `contact-type`
|
`ownercontact`.`contact-type` AS `contact-type`
|
||||||
FROM `post-thread-user`
|
FROM `post-thread-user`
|
||||||
|
@ -3818,6 +3826,7 @@ CREATE VIEW `tag-search-view` AS SELECT
|
||||||
`post-user`.`gravity` AS `gravity`,
|
`post-user`.`gravity` AS `gravity`,
|
||||||
`post-user`.`received` AS `received`,
|
`post-user`.`received` AS `received`,
|
||||||
`post-user`.`network` AS `network`,
|
`post-user`.`network` AS `network`,
|
||||||
|
`post-user`.`protocol` AS `protocol`,
|
||||||
`post-user`.`author-id` AS `author-id`,
|
`post-user`.`author-id` AS `author-id`,
|
||||||
`tag`.`name` AS `name`
|
`tag`.`name` AS `name`
|
||||||
FROM `post-tag`
|
FROM `post-tag`
|
||||||
|
|
|
@ -86,7 +86,7 @@ class Item
|
||||||
|
|
||||||
// Field list that is used to display the items
|
// Field list that is used to display the items
|
||||||
const DISPLAY_FIELDLIST = [
|
const DISPLAY_FIELDLIST = [
|
||||||
'uid', 'id', 'parent', 'guid', 'network', 'gravity',
|
'uid', 'id', 'parent', 'guid', 'network', 'protocol', 'gravity',
|
||||||
'uri-id', 'uri', 'thr-parent-id', 'thr-parent', 'parent-uri-id', 'parent-uri', 'conversation',
|
'uri-id', 'uri', 'thr-parent-id', 'thr-parent', 'parent-uri-id', 'parent-uri', 'conversation',
|
||||||
'commented', 'created', 'edited', 'received', 'verb', 'object-type', 'postopts', 'plink',
|
'commented', 'created', 'edited', 'received', 'verb', 'object-type', 'postopts', 'plink',
|
||||||
'wall', 'private', 'starred', 'origin', 'parent-origin', 'title', 'body', 'language', 'sensitive',
|
'wall', 'private', 'starred', 'origin', 'parent-origin', 'title', 'body', 'language', 'sensitive',
|
||||||
|
@ -4169,10 +4169,11 @@ class Item
|
||||||
* @param string $uri
|
* @param string $uri
|
||||||
* @param int $uid
|
* @param int $uid
|
||||||
* @param int $completion
|
* @param int $completion
|
||||||
|
* @param string $mimetype
|
||||||
*
|
*
|
||||||
* @return integer item id
|
* @return integer item id
|
||||||
*/
|
*/
|
||||||
public static function fetchByLink(string $uri, int $uid = 0, int $completion = ActivityPub\Receiver::COMPLETION_MANUAL): int
|
public static function fetchByLink(string $uri, int $uid = 0, int $completion = ActivityPub\Receiver::COMPLETION_MANUAL, string $mimetype = ''): int
|
||||||
{
|
{
|
||||||
Logger::info('Trying to fetch link', ['uid' => $uid, 'uri' => $uri]);
|
Logger::info('Trying to fetch link', ['uid' => $uid, 'uri' => $uri]);
|
||||||
$item_id = self::searchByLink($uri, $uid);
|
$item_id = self::searchByLink($uri, $uid);
|
||||||
|
@ -4194,35 +4195,49 @@ class Item
|
||||||
Hook::callAll('item_by_link', $hookData);
|
Hook::callAll('item_by_link', $hookData);
|
||||||
|
|
||||||
if (isset($hookData['item_id'])) {
|
if (isset($hookData['item_id'])) {
|
||||||
|
Logger::info('Hook link fetched', ['uid' => $uid, 'uri' => $uri, 'id' => $hookData['item_id']]);
|
||||||
return is_numeric($hookData['item_id']) ? $hookData['item_id'] : 0;
|
return is_numeric($hookData['item_id']) ? $hookData['item_id'] : 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (!$mimetype) {
|
||||||
try {
|
try {
|
||||||
$curlResult = DI::httpClient()->head($uri, [HttpClientOptions::ACCEPT_CONTENT => HttpClientAccept::JSON_AS, HttpClientOptions::REQUEST => HttpClientRequest::ACTIVITYPUB]);
|
$curlResult = DI::httpClient()->head($uri, [HttpClientOptions::ACCEPT_CONTENT => HttpClientAccept::JSON_AS, HttpClientOptions::REQUEST => HttpClientRequest::ACTIVITYPUB]);
|
||||||
if (!HTTPSignature::isValidContentType($curlResult->getContentType(), $uri) && (current(explode(';', $curlResult->getContentType())) == 'application/json')) {
|
$mimetype = empty($curlResult) ? '' : $curlResult->getContentType();
|
||||||
// Issue 14126: Workaround for Mastodon servers that return "application/json" on a "head" request.
|
|
||||||
$curlResult = HTTPSignature::fetchRaw($uri, $uid);
|
|
||||||
}
|
|
||||||
if (HTTPSignature::isValidContentType($curlResult->getContentType(), $uri)) {
|
|
||||||
$fetched_uri = ActivityPub\Processor::fetchMissingActivity($uri, [], '', $completion, $uid);
|
|
||||||
}
|
|
||||||
} catch (\Throwable $th) {
|
} catch (\Throwable $th) {
|
||||||
Logger::info('Invalid link', ['uid' => $uid, 'uri' => $uri, 'code' => $th->getCode(), 'message' => $th->getMessage()]);
|
Logger::info('Error while fetching HTTP link via HEAD', ['uid' => $uid, 'uri' => $uri, 'code' => $th->getCode(), 'message' => $th->getMessage()]);
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!empty($fetched_uri)) {
|
|
||||||
$item_id = self::searchByLink($fetched_uri, $uid);
|
|
||||||
} else {
|
|
||||||
$item_id = Diaspora::fetchByURL($uri);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!empty($item_id)) {
|
if (!HTTPSignature::isValidContentType($mimetype, $uri) && (current(explode(';', $mimetype)) == 'application/json')) {
|
||||||
Logger::info('Link fetched', ['uid' => $uid, 'uri' => $uri, 'id' => $item_id]);
|
try {
|
||||||
|
// Issue 14126: Workaround for Mastodon servers that return "application/json" on a "head" request.
|
||||||
|
$curlResult = HTTPSignature::fetchRaw($uri, $uid);
|
||||||
|
$mimetype = empty($curlResult) ? '' : $curlResult->getContentType();
|
||||||
|
} catch (\Throwable $th) {
|
||||||
|
Logger::info('Error while fetching HTTP link via signed GET', ['uid' => $uid, 'uri' => $uri, 'code' => $th->getCode(), 'message' => $th->getMessage()]);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (HTTPSignature::isValidContentType($mimetype, $uri)) {
|
||||||
|
$fetched_uri = ActivityPub\Processor::fetchMissingActivity($uri, [], '', $completion, $uid);
|
||||||
|
if (!empty($fetched_uri)) {
|
||||||
|
$item_id = self::searchByLink($fetched_uri, $uid);
|
||||||
|
if ($item_id) {
|
||||||
|
Logger::info('ActivityPub link fetched', ['uid' => $uid, 'uri' => $uri, 'id' => $item_id]);
|
||||||
|
return $item_id;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
$item_id = Diaspora::fetchByURL($uri);
|
||||||
|
if ($item_id) {
|
||||||
|
Logger::info('Diaspora link fetched', ['uid' => $uid, 'uri' => $uri, 'id' => $item_id]);
|
||||||
return $item_id;
|
return $item_id;
|
||||||
}
|
}
|
||||||
|
|
||||||
Logger::info('Link not found', ['uid' => $uid, 'uri' => $uri]);
|
Logger::info('This is not an item link', ['uid' => $uid, 'uri' => $uri]);
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -51,6 +51,8 @@ class Media
|
||||||
const ACTIVITY = 20;
|
const ACTIVITY = 20;
|
||||||
const ACCOUNT = 21;
|
const ACCOUNT = 21;
|
||||||
const HLS = 22;
|
const HLS = 22;
|
||||||
|
const JSON = 23;
|
||||||
|
const LD = 24;
|
||||||
const DOCUMENT = 128;
|
const DOCUMENT = 128;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -180,14 +182,14 @@ class Media
|
||||||
}
|
}
|
||||||
|
|
||||||
// Fetch the mimetype or size if missing.
|
// Fetch the mimetype or size if missing.
|
||||||
if (Network::isValidHttpUrl($media['url']) && (empty($media['mimetype']) || empty($media['size']))) {
|
if (Network::isValidHttpUrl($media['url']) && empty($media['mimetype']) && !in_array($media['type'], [self::IMAGE, self::HLS])) {
|
||||||
$timeout = DI::config()->get('system', 'xrd_timeout');
|
$timeout = DI::config()->get('system', 'xrd_timeout');
|
||||||
try {
|
try {
|
||||||
$curlResult = DI::httpClient()->head($media['url'], [HttpClientOptions::TIMEOUT => $timeout, HttpClientOptions::REQUEST => HttpClientRequest::CONTENTTYPE]);
|
$curlResult = DI::httpClient()->head($media['url'], [HttpClientOptions::ACCEPT_CONTENT => HttpClientAccept::AS_DEFAULT, HttpClientOptions::TIMEOUT => $timeout, HttpClientOptions::REQUEST => HttpClientRequest::CONTENTTYPE]);
|
||||||
|
|
||||||
// Workaround for systems that can't handle a HEAD request
|
// Workaround for systems that can't handle a HEAD request
|
||||||
if (!$curlResult->isSuccess() && ($curlResult->getReturnCode() == 405)) {
|
if (!$curlResult->isSuccess() && in_array($curlResult->getReturnCode(), [400, 403, 405])) {
|
||||||
$curlResult = DI::httpClient()->get($media['url'], HttpClientAccept::DEFAULT, [HttpClientOptions::TIMEOUT => $timeout]);
|
$curlResult = DI::httpClient()->get($media['url'], HttpClientAccept::AS_DEFAULT, [HttpClientOptions::TIMEOUT => $timeout]);
|
||||||
}
|
}
|
||||||
if ($curlResult->isSuccess()) {
|
if ($curlResult->isSuccess()) {
|
||||||
if (empty($media['mimetype'])) {
|
if (empty($media['mimetype'])) {
|
||||||
|
@ -197,16 +199,20 @@ class Media
|
||||||
$media['size'] = (int)($curlResult->getHeader('Content-Length')[0] ?? strlen($curlResult->getBodyString() ?? ''));
|
$media['size'] = (int)($curlResult->getHeader('Content-Length')[0] ?? strlen($curlResult->getBodyString() ?? ''));
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
Logger::notice('Could not fetch head', ['media' => $media]);
|
Logger::notice('Could not fetch head', ['media' => $media, 'code' => $curlResult->getReturnCode()]);
|
||||||
}
|
}
|
||||||
} catch (\Throwable $th) {
|
} catch (\Throwable $th) {
|
||||||
Logger::notice('Got exception', ['code' => $th->getCode(), 'message' => $th->getMessage()]);
|
Logger::notice('Got exception', ['code' => $th->getCode(), 'message' => $th->getMessage()]);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
$filetype = !empty($media['mimetype']) ? strtolower(current(explode('/', $media['mimetype']))) : '';
|
if (($media['type'] != self::DOCUMENT) && !empty($media['mimetype'])) {
|
||||||
|
$media = self::addType($media);
|
||||||
|
}
|
||||||
|
|
||||||
if (($media['type'] == self::IMAGE) || ($filetype == 'image')) {
|
Logger::debug('Got type for url', ['type' => $media['type'], 'mimetype' => $media['mimetype'] ?? '', 'url' => $media['url']]);
|
||||||
|
|
||||||
|
if ($media['type'] == self::IMAGE) {
|
||||||
$imagedata = Images::getInfoFromURLCached($media['url'], empty($media['description']));
|
$imagedata = Images::getInfoFromURLCached($media['url'], empty($media['description']));
|
||||||
if ($imagedata) {
|
if ($imagedata) {
|
||||||
$media['mimetype'] = $imagedata['mime'];
|
$media['mimetype'] = $imagedata['mime'];
|
||||||
|
@ -223,23 +229,19 @@ class Media
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if ($media['type'] != self::DOCUMENT) {
|
|
||||||
$media = self::addType($media);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!empty($media['preview'])) {
|
if (!empty($media['preview'])) {
|
||||||
$media = self::addPreviewData($media);
|
$media = self::addPreviewData($media);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (in_array($media['type'], [self::TEXT, self::APPLICATION, self::HTML, self::XML, self::PLAIN])) {
|
if (in_array($media['type'], [self::TEXT, self::ACTIVITY, self::LD, self::JSON, self::HTML, self::XML, self::PLAIN])) {
|
||||||
$media = self::addActivity($media);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (in_array($media['type'], [self::TEXT, self::APPLICATION, self::HTML, self::XML, self::PLAIN])) {
|
|
||||||
$media = self::addAccount($media);
|
$media = self::addAccount($media);
|
||||||
}
|
}
|
||||||
|
|
||||||
if ($media['type'] == self::HTML) {
|
if (in_array($media['type'], [self::ACTIVITY, self::LD, self::JSON])) {
|
||||||
|
$media = self::addActivity($media);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (in_array($media['type'], [self::HTML, self::LD, self::JSON])) {
|
||||||
$media = self::addPage($media);
|
$media = self::addPage($media);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -254,6 +256,16 @@ class Media
|
||||||
|
|
||||||
$imagedata = Images::getInfoFromURLCached($media['preview']);
|
$imagedata = Images::getInfoFromURLCached($media['preview']);
|
||||||
if ($imagedata) {
|
if ($imagedata) {
|
||||||
|
$media['blurhash'] = $imagedata['blurhash'] ?? null;
|
||||||
|
|
||||||
|
// When the original picture is potentially animated but the preview isn't, we override the preview
|
||||||
|
if (in_array($media['mimetype'] ?? '', ['image/gif', 'image/png']) && !in_array($imagedata['mime'], ['image/gif', 'image/png'])) {
|
||||||
|
$media['preview'] = $media['url'];
|
||||||
|
$media['preview-width'] = $media['width'];
|
||||||
|
$media['preview-height'] = $media['height'];
|
||||||
|
return $media;
|
||||||
|
}
|
||||||
|
|
||||||
$media['preview-width'] = $imagedata[0];
|
$media['preview-width'] = $imagedata[0];
|
||||||
$media['preview-height'] = $imagedata[1];
|
$media['preview-height'] = $imagedata[1];
|
||||||
}
|
}
|
||||||
|
@ -269,19 +281,22 @@ class Media
|
||||||
*/
|
*/
|
||||||
private static function addActivity(array $media): array
|
private static function addActivity(array $media): array
|
||||||
{
|
{
|
||||||
$id = Item::fetchByLink($media['url'], 0, ActivityPub\Receiver::COMPLETION_ASYNC);
|
$id = Item::fetchByLink($media['url'], 0, ActivityPub\Receiver::COMPLETION_ASYNC, $media['mimetype'] ?? '');
|
||||||
if (empty($id)) {
|
if (empty($id)) {
|
||||||
|
$media['type'] = $media['type'] == self::ACTIVITY ? self::JSON : $media['type'];
|
||||||
return $media;
|
return $media;
|
||||||
}
|
}
|
||||||
|
|
||||||
$item = Post::selectFirst([], ['id' => $id, 'network' => Protocol::FEDERATED]);
|
$item = Post::selectFirst([], ['id' => $id, 'network' => Protocol::FEDERATED]);
|
||||||
if (empty($item['id'])) {
|
if (empty($item['id'])) {
|
||||||
Logger::debug('Not a federated activity', ['id' => $id, 'uri-id' => $media['uri-id'], 'url' => $media['url']]);
|
Logger::debug('Not a federated activity', ['id' => $id, 'uri-id' => $media['uri-id'], 'url' => $media['url']]);
|
||||||
|
$media['type'] = $media['type'] == self::ACTIVITY ? self::JSON : $media['type'];
|
||||||
return $media;
|
return $media;
|
||||||
}
|
}
|
||||||
|
|
||||||
if ($item['uri-id'] == $media['uri-id']) {
|
if ($item['uri-id'] == $media['uri-id']) {
|
||||||
Logger::info('Media-Uri-Id is identical to Uri-Id', ['uri-id' => $media['uri-id']]);
|
Logger::info('Media-Uri-Id is identical to Uri-Id', ['uri-id' => $media['uri-id']]);
|
||||||
|
$media['type'] = $media['type'] == self::ACTIVITY ? self::JSON : $media['type'];
|
||||||
return $media;
|
return $media;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -290,6 +305,7 @@ class Media
|
||||||
parse_url($item['plink'], PHP_URL_HOST) != parse_url($item['uri'], PHP_URL_HOST)
|
parse_url($item['plink'], PHP_URL_HOST) != parse_url($item['uri'], PHP_URL_HOST)
|
||||||
) {
|
) {
|
||||||
Logger::debug('Not a link to an activity', ['uri-id' => $media['uri-id'], 'url' => $media['url'], 'plink' => $item['plink'], 'uri' => $item['uri']]);
|
Logger::debug('Not a link to an activity', ['uri-id' => $media['uri-id'], 'url' => $media['url'], 'plink' => $item['plink'], 'uri' => $item['uri']]);
|
||||||
|
$media['type'] = $media['type'] == self::ACTIVITY ? self::JSON : $media['type'];
|
||||||
return $media;
|
return $media;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -375,14 +391,23 @@ class Media
|
||||||
*/
|
*/
|
||||||
private static function addPage(array $media): array
|
private static function addPage(array $media): array
|
||||||
{
|
{
|
||||||
$data = ParseUrl::getSiteinfoCached($media['url']);
|
$data = ParseUrl::getSiteinfoCached($media['url'], $media['mimetype'] ?? '');
|
||||||
$media['size'] = $data['size'] ?? null;
|
if (empty($data['images'][0]['src']) && empty($data['text']) && empty($data['title'])) {
|
||||||
|
if (!empty($media['preview'])) {
|
||||||
|
$media = self::addPreviewData($media);
|
||||||
|
Logger::debug('Detected site data is empty, use suggested media data instead', ['uri-id' => $media['uri-id'], 'url' => $media['url'], 'type' => $data['type']]);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
$media['preview'] = $data['images'][0]['src'] ?? null;
|
$media['preview'] = $data['images'][0]['src'] ?? null;
|
||||||
$media['preview-height'] = $data['images'][0]['height'] ?? null;
|
$media['preview-height'] = $data['images'][0]['height'] ?? null;
|
||||||
$media['preview-width'] = $data['images'][0]['width'] ?? null;
|
$media['preview-width'] = $data['images'][0]['width'] ?? null;
|
||||||
$media['blurhash'] = $data['images'][0]['blurhash'] ?? null;
|
$media['blurhash'] = $data['images'][0]['blurhash'] ?? null;
|
||||||
$media['description'] = $data['text'] ?? null;
|
$media['description'] = $data['text'] ?? null;
|
||||||
$media['name'] = $data['title'] ?? null;
|
$media['name'] = $data['title'] ?? null;
|
||||||
|
}
|
||||||
|
|
||||||
|
$media['type'] = self::HTML;
|
||||||
|
$media['size'] = $data['size'] ?? null;
|
||||||
$media['author-url'] = $data['author_url'] ?? null;
|
$media['author-url'] = $data['author_url'] ?? null;
|
||||||
$media['author-name'] = $data['author_name'] ?? null;
|
$media['author-name'] = $data['author_name'] ?? null;
|
||||||
$media['author-image'] = $data['author_img'] ?? null;
|
$media['author-image'] = $data['author_img'] ?? null;
|
||||||
|
@ -481,6 +506,12 @@ class Media
|
||||||
$type = self::TORRENT;
|
$type = self::TORRENT;
|
||||||
} elseif (($filetype == 'application') && ($subtype == 'vnd.apple.mpegurl')) {
|
} elseif (($filetype == 'application') && ($subtype == 'vnd.apple.mpegurl')) {
|
||||||
$type = self::HLS;
|
$type = self::HLS;
|
||||||
|
} elseif (($filetype == 'application') && ($subtype == 'activity+json')) {
|
||||||
|
$type = self::ACTIVITY;
|
||||||
|
} elseif (($filetype == 'application') && ($subtype == 'ld+json')) {
|
||||||
|
$type = self::LD;
|
||||||
|
} elseif (($filetype == 'application') && ($subtype == 'json')) {
|
||||||
|
$type = self::JSON;
|
||||||
} elseif ($filetype == 'application') {
|
} elseif ($filetype == 'application') {
|
||||||
$type = self::APPLICATION;
|
$type = self::APPLICATION;
|
||||||
} else {
|
} else {
|
||||||
|
|
|
@ -94,6 +94,13 @@ class Stats extends BaseModule
|
||||||
'deferred' => [],
|
'deferred' => [],
|
||||||
'total' => [],
|
'total' => [],
|
||||||
],
|
],
|
||||||
|
'jetstream' => [
|
||||||
|
'drift' => intval($this->keyValue->get('jetstream_drift')),
|
||||||
|
'did_count' => intval($this->keyValue->get('jetstream_did_count')),
|
||||||
|
'did_limit' => intval($this->keyValue->get('jetstream_did_limit')),
|
||||||
|
'messages' => intval($this->keyValue->get('jetstream_messages')),
|
||||||
|
'timestamp' => intval($this->keyValue->get('jetstream_timestamp')),
|
||||||
|
],
|
||||||
'users' => [
|
'users' => [
|
||||||
'total' => intval($this->keyValue->get('nodeinfo_total_users')),
|
'total' => intval($this->keyValue->get('nodeinfo_total_users')),
|
||||||
'activeWeek' => intval($this->keyValue->get('nodeinfo_active_users_weekly')),
|
'activeWeek' => intval($this->keyValue->get('nodeinfo_active_users_weekly')),
|
||||||
|
|
|
@ -15,6 +15,9 @@ class HttpClientAccept
|
||||||
/** @var string Default value for "Accept" header */
|
/** @var string Default value for "Accept" header */
|
||||||
public const DEFAULT = '*/*';
|
public const DEFAULT = '*/*';
|
||||||
|
|
||||||
|
/** @var string Accept all types with a preferences of ActivityStream content */
|
||||||
|
public const AS_DEFAULT = 'application/activity+json,application/ld+json; profile="https://www.w3.org/ns/activitystreams",*/*;q=0.9';
|
||||||
|
|
||||||
public const ATOM_XML = 'application/atom+xml,text/xml;q=0.9,*/*;q=0.8';
|
public const ATOM_XML = 'application/atom+xml,text/xml;q=0.9,*/*;q=0.8';
|
||||||
public const FEED_XML = 'application/atom+xml,application/rss+xml;q=0.9,application/rdf+xml;q=0.8,text/xml;q=0.7,*/*;q=0.6';
|
public const FEED_XML = 'application/atom+xml,application/rss+xml;q=0.9,application/rdf+xml;q=0.8,text/xml;q=0.7,*/*;q=0.6';
|
||||||
public const HTML = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8';
|
public const HTML = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8';
|
||||||
|
|
|
@ -16,6 +16,7 @@ use Friendica\Core\Protocol;
|
||||||
use Friendica\Core\Renderer;
|
use Friendica\Core\Renderer;
|
||||||
use Friendica\DI;
|
use Friendica\DI;
|
||||||
use Friendica\Model\Contact;
|
use Friendica\Model\Contact;
|
||||||
|
use Friendica\Model\Conversation;
|
||||||
use Friendica\Model\Item;
|
use Friendica\Model\Item;
|
||||||
use Friendica\Model\Post as PostModel;
|
use Friendica\Model\Post as PostModel;
|
||||||
use Friendica\Model\Tag;
|
use Friendica\Model\Tag;
|
||||||
|
@ -193,7 +194,7 @@ class Post
|
||||||
|
|
||||||
$privacy = $this->fetchPrivacy($item);
|
$privacy = $this->fetchPrivacy($item);
|
||||||
$lock = ($item['private'] == Item::PRIVATE) ? $privacy : false;
|
$lock = ($item['private'] == Item::PRIVATE) ? $privacy : false;
|
||||||
$connector = !in_array($item['network'], Protocol::NATIVE_SUPPORT) ? DI::l10n()->t('Connector Message') : false;
|
$connector = !in_array($item['network'], Protocol::NATIVE_SUPPORT) && ($item['protocol'] != Conversation::PARCEL_JETSTREAM) ? DI::l10n()->t('Connector Message') : false;
|
||||||
|
|
||||||
$shareable = in_array($conv->getProfileOwner(), [0, DI::userSession()->getLocalUserId()]) && $item['private'] != Item::PRIVATE;
|
$shareable = in_array($conv->getProfileOwner(), [0, DI::userSession()->getLocalUserId()]) && $item['private'] != Item::PRIVATE;
|
||||||
$announceable = $shareable && in_array($item['network'], [Protocol::ACTIVITYPUB, Protocol::DFRN, Protocol::DIASPORA, Protocol::TWITTER, Protocol::TUMBLR, Protocol::BLUESKY]);
|
$announceable = $shareable && in_array($item['network'], [Protocol::ACTIVITYPUB, Protocol::DFRN, Protocol::DIASPORA, Protocol::TWITTER, Protocol::TUMBLR, Protocol::BLUESKY]);
|
||||||
|
|
|
@ -147,7 +147,7 @@ final class ATProtocol
|
||||||
return $data;
|
return $data;
|
||||||
}
|
}
|
||||||
|
|
||||||
private function post(int $uid, string $url, string $params, array $headers): ?stdClass
|
public function post(int $uid, string $url, string $params, array $headers): ?stdClass
|
||||||
{
|
{
|
||||||
$pds = $this->getUserPds($uid);
|
$pds = $this->getUserPds($uid);
|
||||||
if (empty($pds)) {
|
if (empty($pds)) {
|
||||||
|
@ -172,8 +172,11 @@ final class ATProtocol
|
||||||
$data->code = $curlResult->getReturnCode();
|
$data->code = $curlResult->getReturnCode();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (!empty($data->code) && ($data->code >= 200) && ($data->code < 400)) {
|
||||||
$this->pConfig->set($uid, 'bluesky', 'status', self::STATUS_SUCCESS);
|
$this->pConfig->set($uid, 'bluesky', 'status', self::STATUS_SUCCESS);
|
||||||
Item::incrementOutbound(Protocol::BLUESKY);
|
} else {
|
||||||
|
$this->pConfig->set($uid, 'bluesky', 'status', self::STATUS_API_FAIL);
|
||||||
|
}
|
||||||
return $data;
|
return $data;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -230,7 +233,7 @@ final class ATProtocol
|
||||||
return $did;
|
return $did;
|
||||||
}
|
}
|
||||||
|
|
||||||
private function getDid(string $handle): string
|
public function getDid(string $handle): string
|
||||||
{
|
{
|
||||||
if ($handle == '') {
|
if ($handle == '') {
|
||||||
return '';
|
return '';
|
||||||
|
@ -373,7 +376,7 @@ final class ATProtocol
|
||||||
return in_array('at://' . $handle, $data->alsoKnownAs);
|
return in_array('at://' . $handle, $data->alsoKnownAs);
|
||||||
}
|
}
|
||||||
|
|
||||||
private function getUserToken(int $uid): string
|
public function getUserToken(int $uid): string
|
||||||
{
|
{
|
||||||
$token = $this->pConfig->get($uid, 'bluesky', 'access_token');
|
$token = $this->pConfig->get($uid, 'bluesky', 'access_token');
|
||||||
$created = $this->pConfig->get($uid, 'bluesky', 'token_created');
|
$created = $this->pConfig->get($uid, 'bluesky', 'token_created');
|
||||||
|
@ -393,6 +396,11 @@ final class ATProtocol
|
||||||
|
|
||||||
$data = $this->post($uid, '/xrpc/com.atproto.server.refreshSession', '', ['Authorization' => ['Bearer ' . $token]]);
|
$data = $this->post($uid, '/xrpc/com.atproto.server.refreshSession', '', ['Authorization' => ['Bearer ' . $token]]);
|
||||||
if (empty($data) || empty($data->accessJwt)) {
|
if (empty($data) || empty($data->accessJwt)) {
|
||||||
|
$this->logger->debug('Refresh failed', ['return' => $data]);
|
||||||
|
$password = $this->pConfig->get($uid, 'bluesky', 'password');
|
||||||
|
if (!empty($password)) {
|
||||||
|
return $this->createUserToken($uid, $password);
|
||||||
|
}
|
||||||
$this->pConfig->set($uid, 'bluesky', 'status', self::STATUS_TOKEN_FAIL);
|
$this->pConfig->set($uid, 'bluesky', 'status', self::STATUS_TOKEN_FAIL);
|
||||||
return '';
|
return '';
|
||||||
}
|
}
|
||||||
|
|
|
@ -41,6 +41,7 @@ class Jetstream
|
||||||
private $uids = [];
|
private $uids = [];
|
||||||
private $self = [];
|
private $self = [];
|
||||||
private $capped = false;
|
private $capped = false;
|
||||||
|
private $next_stat = 0;
|
||||||
|
|
||||||
/** @var LoggerInterface */
|
/** @var LoggerInterface */
|
||||||
private $logger;
|
private $logger;
|
||||||
|
@ -108,6 +109,7 @@ class Jetstream
|
||||||
$timestamp = $data->time_us;
|
$timestamp = $data->time_us;
|
||||||
$this->route($data);
|
$this->route($data);
|
||||||
$this->keyValue->set('jetstream_timestamp', $timestamp);
|
$this->keyValue->set('jetstream_timestamp', $timestamp);
|
||||||
|
$this->incrementMessages();
|
||||||
} else {
|
} else {
|
||||||
$this->logger->warning('Unexpected return value', ['data' => $data]);
|
$this->logger->warning('Unexpected return value', ['data' => $data]);
|
||||||
break;
|
break;
|
||||||
|
@ -135,6 +137,15 @@ class Jetstream
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private function incrementMessages()
|
||||||
|
{
|
||||||
|
$packets = (int)($this->keyValue->get('jetstream_messages') ?? 0);
|
||||||
|
if ($packets >= PHP_INT_MAX) {
|
||||||
|
$packets = 0;
|
||||||
|
}
|
||||||
|
$this->keyValue->set('jetstream_messages', $packets + 1);
|
||||||
|
}
|
||||||
|
|
||||||
private function syncContacts()
|
private function syncContacts()
|
||||||
{
|
{
|
||||||
$active_uids = $this->atprotocol->getUids();
|
$active_uids = $this->atprotocol->getUids();
|
||||||
|
@ -184,10 +195,14 @@ class Jetstream
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!$this->capped && count($dids) < $did_limit) {
|
if (!$this->capped && count($dids) < $did_limit) {
|
||||||
$contacts = Contact::selectToArray(['url'], ['uid' => 0, 'network' => Protocol::BLUESKY], ['order' => ['last-item' => true], 'limit' => $did_limit]);
|
$condition = ["`uid` = ? AND `network` = ? AND EXISTS(SELECT `author-id` FROM `post-user` WHERE `author-id` = `contact`.`id` AND `post-user`.`uid` != ?)", 0, Protocol::BLUESKY, 0];
|
||||||
|
$contacts = Contact::selectToArray(['url'], $condition, ['order' => ['last-item' => true], 'limit' => $did_limit]);
|
||||||
$dids = $this->addDids($contacts, $uids, $did_limit, $dids);
|
$dids = $this->addDids($contacts, $uids, $did_limit, $dids);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
$this->keyValue->set('jetstream_did_count', count($dids));
|
||||||
|
$this->keyValue->set('jetstream_did_limit', $did_limit);
|
||||||
|
|
||||||
$this->logger->debug('Selected DIDs', ['uids' => $active_uids, 'count' => count($dids), 'capped' => $this->capped]);
|
$this->logger->debug('Selected DIDs', ['uids' => $active_uids, 'count' => count($dids), 'capped' => $this->capped]);
|
||||||
$update = [
|
$update = [
|
||||||
'type' => 'options_update',
|
'type' => 'options_update',
|
||||||
|
@ -241,17 +256,7 @@ class Jetstream
|
||||||
|
|
||||||
private function routeCommits(stdClass $data)
|
private function routeCommits(stdClass $data)
|
||||||
{
|
{
|
||||||
$drift = max(0, round(time() - $data->time_us / 1000000));
|
$drift = $this->getDrift($data);
|
||||||
if ($drift > 60 && !$this->capped) {
|
|
||||||
$this->capped = true;
|
|
||||||
$this->setOptions();
|
|
||||||
$this->logger->notice('Drift is too high, dids will be capped');
|
|
||||||
} elseif ($drift == 0 && $this->capped) {
|
|
||||||
$this->capped = false;
|
|
||||||
$this->setOptions();
|
|
||||||
$this->logger->notice('Drift is low enough, dids will be uncapped');
|
|
||||||
}
|
|
||||||
|
|
||||||
$this->logger->notice('Received commit', ['time' => date(DateTimeFormat::ATOM, $data->time_us / 1000000), 'drift' => $drift, 'capped' => $this->capped, 'did' => $data->did, 'operation' => $data->commit->operation, 'collection' => $data->commit->collection, 'timestamp' => $data->time_us]);
|
$this->logger->notice('Received commit', ['time' => date(DateTimeFormat::ATOM, $data->time_us / 1000000), 'drift' => $drift, 'capped' => $this->capped, 'did' => $data->did, 'operation' => $data->commit->operation, 'collection' => $data->commit->collection, 'timestamp' => $data->time_us]);
|
||||||
$timestamp = microtime(true);
|
$timestamp = microtime(true);
|
||||||
|
|
||||||
|
@ -299,6 +304,23 @@ class Jetstream
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private function getDrift(stdClass $data): int
|
||||||
|
{
|
||||||
|
$drift = max(0, round(time() - $data->time_us / 1000000));
|
||||||
|
$this->keyValue->set('jetstream_drift', $drift);
|
||||||
|
|
||||||
|
if ($drift > 60 && !$this->capped) {
|
||||||
|
$this->capped = true;
|
||||||
|
$this->setOptions();
|
||||||
|
$this->logger->notice('Drift is too high, dids will be capped');
|
||||||
|
} elseif ($drift == 0 && $this->capped) {
|
||||||
|
$this->capped = false;
|
||||||
|
$this->setOptions();
|
||||||
|
$this->logger->notice('Drift is low enough, dids will be uncapped');
|
||||||
|
}
|
||||||
|
return $drift;
|
||||||
|
}
|
||||||
|
|
||||||
private function routePost(stdClass $data, int $drift)
|
private function routePost(stdClass $data, int $drift)
|
||||||
{
|
{
|
||||||
switch ($data->commit->operation) {
|
switch ($data->commit->operation) {
|
||||||
|
|
|
@ -13,6 +13,7 @@ namespace Friendica\Protocol\ATProtocol;
|
||||||
use Friendica\App\BaseURL;
|
use Friendica\App\BaseURL;
|
||||||
use Friendica\Core\Protocol;
|
use Friendica\Core\Protocol;
|
||||||
use Friendica\Database\Database;
|
use Friendica\Database\Database;
|
||||||
|
use Friendica\Database\DBA;
|
||||||
use Friendica\Model\Contact;
|
use Friendica\Model\Contact;
|
||||||
use Friendica\Model\Conversation;
|
use Friendica\Model\Conversation;
|
||||||
use Friendica\Model\Item;
|
use Friendica\Model\Item;
|
||||||
|
@ -128,12 +129,12 @@ class Processor
|
||||||
if (!empty($data->commit->record->reply)) {
|
if (!empty($data->commit->record->reply)) {
|
||||||
$root = $this->getUri($data->commit->record->reply->root);
|
$root = $this->getUri($data->commit->record->reply->root);
|
||||||
$parent = $this->getUri($data->commit->record->reply->parent);
|
$parent = $this->getUri($data->commit->record->reply->parent);
|
||||||
$uids = $this->getPostUids($root);
|
$uids = $this->getPostUids($root, true);
|
||||||
if (!$uids) {
|
if (!$uids) {
|
||||||
$this->logger->debug('Comment is not imported since the root post is not found.', ['root' => $root, 'parent' => $parent]);
|
$this->logger->debug('Comment is not imported since the root post is not found.', ['root' => $root, 'parent' => $parent]);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if ($dont_fetch && !$this->getPostUids($parent)) {
|
if ($dont_fetch && !$this->getPostUids($parent, false)) {
|
||||||
$this->logger->debug('Comment is not imported since the parent post is not found.', ['root' => $root, 'parent' => $parent]);
|
$this->logger->debug('Comment is not imported since the parent post is not found.', ['root' => $root, 'parent' => $parent]);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
@ -166,7 +167,8 @@ class Processor
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
$item = $this->addMedia($post->thread->post->embed, $item, 0, 0, 0);
|
$item['source'] = json_encode($post);
|
||||||
|
$item = $this->addMedia($post->thread->post->embed, $item, 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
$id = Item::insert($item);
|
$id = Item::insert($item);
|
||||||
|
@ -183,7 +185,7 @@ class Processor
|
||||||
|
|
||||||
public function createRepost(stdClass $data, array $uids, bool $dont_fetch)
|
public function createRepost(stdClass $data, array $uids, bool $dont_fetch)
|
||||||
{
|
{
|
||||||
if ($dont_fetch && !$this->getPostUids($this->getUri($data->commit->record->subject))) {
|
if ($dont_fetch && !$this->getPostUids($this->getUri($data->commit->record->subject), true)) {
|
||||||
$this->logger->debug('Repost is not imported since the subject is not found.', ['subject' => $this->getUri($data->commit->record->subject)]);
|
$this->logger->debug('Repost is not imported since the subject is not found.', ['subject' => $this->getUri($data->commit->record->subject)]);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
@ -213,7 +215,7 @@ class Processor
|
||||||
|
|
||||||
public function createLike(stdClass $data)
|
public function createLike(stdClass $data)
|
||||||
{
|
{
|
||||||
$uids = $this->getPostUids($this->getUri($data->commit->record->subject));
|
$uids = $this->getPostUids($this->getUri($data->commit->record->subject), false);
|
||||||
if (!$uids) {
|
if (!$uids) {
|
||||||
$this->logger->debug('Like is not imported since the subject is not found.', ['subject' => $this->getUri($data->commit->record->subject)]);
|
$this->logger->debug('Like is not imported since the subject is not found.', ['subject' => $this->getUri($data->commit->record->subject)]);
|
||||||
return;
|
return;
|
||||||
|
@ -270,7 +272,7 @@ class Processor
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
private function processPost(stdClass $post, int $uid, int $post_reason, int $causer, int $level, int $protocol): int
|
public function processPost(stdClass $post, int $uid, int $post_reason, int $causer, int $level, int $protocol): int
|
||||||
{
|
{
|
||||||
$uri = $this->getUri($post);
|
$uri = $this->getUri($post);
|
||||||
|
|
||||||
|
@ -295,7 +297,7 @@ class Processor
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!empty($post->embed)) {
|
if (!empty($post->embed)) {
|
||||||
$item = $this->addMedia($post->embed, $item, $uid, $level);
|
$item = $this->addMedia($post->embed, $item, $level);
|
||||||
}
|
}
|
||||||
|
|
||||||
$item['restrictions'] = $this->getRestrictionsForUser($post, $item, $post_reason);
|
$item['restrictions'] = $this->getRestrictionsForUser($post, $item, $post_reason);
|
||||||
|
@ -378,7 +380,7 @@ class Processor
|
||||||
return $item;
|
return $item;
|
||||||
}
|
}
|
||||||
|
|
||||||
private function getHeaderFromPost(stdClass $post, string $uri, int $uid, int $protocol): array
|
public function getHeaderFromPost(stdClass $post, string $uri, int $uid, int $protocol): array
|
||||||
{
|
{
|
||||||
$parts = $this->getUriParts($uri);
|
$parts = $this->getUriParts($uri);
|
||||||
if (empty($post->author) || empty($post->cid) || empty($parts->rkey)) {
|
if (empty($post->author) || empty($post->cid) || empty($parts->rkey)) {
|
||||||
|
@ -538,6 +540,8 @@ class Processor
|
||||||
'url' => $image->fullsize,
|
'url' => $image->fullsize,
|
||||||
'preview' => $image->thumb,
|
'preview' => $image->thumb,
|
||||||
'description' => $image->alt,
|
'description' => $image->alt,
|
||||||
|
'height' => $image->aspectRatio->height ?? null,
|
||||||
|
'width' => $image->aspectRatio->width ?? null,
|
||||||
];
|
];
|
||||||
Post\Media::insert($media);
|
Post\Media::insert($media);
|
||||||
}
|
}
|
||||||
|
@ -561,6 +565,7 @@ class Processor
|
||||||
'uri-id' => $item['uri-id'],
|
'uri-id' => $item['uri-id'],
|
||||||
'type' => Post\Media::HTML,
|
'type' => Post\Media::HTML,
|
||||||
'url' => $embed->external->uri,
|
'url' => $embed->external->uri,
|
||||||
|
'preview' => $embed->external->thumb ?? null,
|
||||||
'name' => $embed->external->title,
|
'name' => $embed->external->title,
|
||||||
'description' => $embed->external->description,
|
'description' => $embed->external->description,
|
||||||
];
|
];
|
||||||
|
@ -686,7 +691,7 @@ class Processor
|
||||||
return $restrict ? Item::CANT_REPLY : null;
|
return $restrict ? Item::CANT_REPLY : null;
|
||||||
}
|
}
|
||||||
|
|
||||||
private function fetchMissingPost(string $uri, int $uid, int $post_reason, int $causer, int $level, string $fallback = '', bool $always_fetch = false, int $Protocol = Conversation::PARCEL_JETSTREAM): string
|
public function fetchMissingPost(string $uri, int $uid, int $post_reason, int $causer, int $level, string $fallback = '', bool $always_fetch = false, int $Protocol = Conversation::PARCEL_JETSTREAM): string
|
||||||
{
|
{
|
||||||
$timestamp = microtime(true);
|
$timestamp = microtime(true);
|
||||||
$stamp = Strings::getRandomHex(30);
|
$stamp = Strings::getRandomHex(30);
|
||||||
|
@ -794,7 +799,7 @@ class Processor
|
||||||
return $uri;
|
return $uri;
|
||||||
}
|
}
|
||||||
|
|
||||||
private function getUriParts(string $uri): ?stdClass
|
public function getUriParts(string $uri): ?stdClass
|
||||||
{
|
{
|
||||||
$class = $this->getUriClass($uri);
|
$class = $this->getUriClass($uri);
|
||||||
if (empty($class)) {
|
if (empty($class)) {
|
||||||
|
@ -812,7 +817,7 @@ class Processor
|
||||||
return $class;
|
return $class;
|
||||||
}
|
}
|
||||||
|
|
||||||
private function getUriClass(string $uri): ?stdClass
|
public function getUriClass(string $uri): ?stdClass
|
||||||
{
|
{
|
||||||
if (empty($uri)) {
|
if (empty($uri)) {
|
||||||
return null;
|
return null;
|
||||||
|
@ -837,7 +842,7 @@ class Processor
|
||||||
return $class;
|
return $class;
|
||||||
}
|
}
|
||||||
|
|
||||||
private function fetchUriId(string $uri, int $uid): string
|
public function fetchUriId(string $uri, int $uid): string
|
||||||
{
|
{
|
||||||
$reply = Post::selectFirst(['uri-id'], ['uri' => $uri, 'uid' => [$uid, 0]]);
|
$reply = Post::selectFirst(['uri-id'], ['uri' => $uri, 'uid' => [$uid, 0]]);
|
||||||
if (!empty($reply['uri-id'])) {
|
if (!empty($reply['uri-id'])) {
|
||||||
|
@ -852,16 +857,18 @@ class Processor
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
private function getPostUids(string $uri): array
|
private function getPostUids(string $uri, bool $with_public_user): array
|
||||||
{
|
{
|
||||||
|
$condition = $with_public_user ? [] : ["`uid` != ?", 0];
|
||||||
|
|
||||||
$uids = [];
|
$uids = [];
|
||||||
$posts = Post::select(['uid'], ['uri' => $uri]);
|
$posts = Post::select(['uid'], DBA::mergeConditions(['uri' => $uri], $condition));
|
||||||
while ($post = Post::fetch($posts)) {
|
while ($post = Post::fetch($posts)) {
|
||||||
$uids[] = $post['uid'];
|
$uids[] = $post['uid'];
|
||||||
}
|
}
|
||||||
$this->db->close($posts);
|
$this->db->close($posts);
|
||||||
|
|
||||||
$posts = Post::select(['uid'], ['extid' => $uri]);
|
$posts = Post::select(['uid'], DBA::mergeConditions(['extid' => $uri], $condition));
|
||||||
while ($post = Post::fetch($posts)) {
|
while ($post = Post::fetch($posts)) {
|
||||||
$uids[] = $post['uid'];
|
$uids[] = $post['uid'];
|
||||||
}
|
}
|
||||||
|
@ -878,7 +885,7 @@ class Processor
|
||||||
return Post::exists(['extid' => $uri, 'uid' => $uids]);
|
return Post::exists(['extid' => $uri, 'uid' => $uids]);
|
||||||
}
|
}
|
||||||
|
|
||||||
private function getUri(stdClass $post): string
|
public function getUri(stdClass $post): string
|
||||||
{
|
{
|
||||||
if (empty($post->cid)) {
|
if (empty($post->cid)) {
|
||||||
$this->logger->info('Invalid URI', ['post' => $post]);
|
$this->logger->info('Invalid URI', ['post' => $post]);
|
||||||
|
@ -887,7 +894,7 @@ class Processor
|
||||||
return $post->uri . ':' . $post->cid;
|
return $post->uri . ':' . $post->cid;
|
||||||
}
|
}
|
||||||
|
|
||||||
private function getPostUri(string $uri, int $uid): string
|
public function getPostUri(string $uri, int $uid): string
|
||||||
{
|
{
|
||||||
if (Post::exists(['uri' => $uri, 'uid' => [$uid, 0]])) {
|
if (Post::exists(['uri' => $uri, 'uid' => [$uid, 0]])) {
|
||||||
$this->logger->debug('Post exists', ['uri' => $uri]);
|
$this->logger->debug('Post exists', ['uri' => $uri]);
|
||||||
|
|
|
@ -90,6 +90,7 @@ class ParseUrl
|
||||||
* Search for cached embeddable data of an url otherwise fetch it
|
* Search for cached embeddable data of an url otherwise fetch it
|
||||||
*
|
*
|
||||||
* @param string $url The url of the page which should be scraped
|
* @param string $url The url of the page which should be scraped
|
||||||
|
* @param string $mimetype Optional mimetype that had already been detected for this page
|
||||||
*
|
*
|
||||||
* @return array which contains needed data for embedding
|
* @return array which contains needed data for embedding
|
||||||
* string 'url' => The url of the parsed page
|
* string 'url' => The url of the parsed page
|
||||||
|
@ -104,7 +105,7 @@ class ParseUrl
|
||||||
* @see ParseUrl::getSiteinfo() for more information about scraping
|
* @see ParseUrl::getSiteinfo() for more information about scraping
|
||||||
* embeddable content
|
* embeddable content
|
||||||
*/
|
*/
|
||||||
public static function getSiteinfoCached(string $url): array
|
public static function getSiteinfoCached(string $url, string $mimetype = ''): array
|
||||||
{
|
{
|
||||||
if (empty($url)) {
|
if (empty($url)) {
|
||||||
return [
|
return [
|
||||||
|
@ -123,7 +124,7 @@ class ParseUrl
|
||||||
return $data;
|
return $data;
|
||||||
}
|
}
|
||||||
|
|
||||||
$data = self::getSiteinfo($url);
|
$data = self::getSiteinfo($url, $mimetype);
|
||||||
|
|
||||||
$expires = $data['expires'];
|
$expires = $data['expires'];
|
||||||
|
|
||||||
|
@ -156,6 +157,7 @@ class ParseUrl
|
||||||
* \<meta name="description" content="An awesome description"\>
|
* \<meta name="description" content="An awesome description"\>
|
||||||
*
|
*
|
||||||
* @param string $url The url of the page which should be scraped
|
* @param string $url The url of the page which should be scraped
|
||||||
|
* @param string $mimetype Optional mimetype that had already been detected for this page
|
||||||
* @param int $count Internal counter to avoid endless loops
|
* @param int $count Internal counter to avoid endless loops
|
||||||
*
|
*
|
||||||
* @return array which contains needed data for embedding
|
* @return array which contains needed data for embedding
|
||||||
|
@ -181,7 +183,7 @@ class ParseUrl
|
||||||
* </body>
|
* </body>
|
||||||
* @endverbatim
|
* @endverbatim
|
||||||
*/
|
*/
|
||||||
public static function getSiteinfo(string $url, int $count = 1): array
|
public static function getSiteinfo(string $url, string $mimetype = '', int $count = 1): array
|
||||||
{
|
{
|
||||||
if (empty($url)) {
|
if (empty($url)) {
|
||||||
return [
|
return [
|
||||||
|
@ -212,7 +214,11 @@ class ParseUrl
|
||||||
return $siteinfo;
|
return $siteinfo;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (!empty($mimetype)) {
|
||||||
|
$type = explode('/', current(explode(';', $mimetype)));
|
||||||
|
} else {
|
||||||
$type = self::getContentType($url);
|
$type = self::getContentType($url);
|
||||||
|
}
|
||||||
Logger::info('Got content-type', ['content-type' => $type, 'url' => $url]);
|
Logger::info('Got content-type', ['content-type' => $type, 'url' => $url]);
|
||||||
if (!empty($type) && in_array($type[0], ['image', 'video', 'audio'])) {
|
if (!empty($type) && in_array($type[0], ['image', 'video', 'audio'])) {
|
||||||
$siteinfo['type'] = $type[0];
|
$siteinfo['type'] = $type[0];
|
||||||
|
@ -309,7 +315,7 @@ class ParseUrl
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if ($content != '') {
|
if ($content != '') {
|
||||||
$siteinfo = self::getSiteinfo($content, ++$count);
|
$siteinfo = self::getSiteinfo($content, $mimetype, ++$count);
|
||||||
return $siteinfo;
|
return $siteinfo;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -101,6 +101,7 @@ return [
|
||||||
"received" => ["post-thread-user", "received"],
|
"received" => ["post-thread-user", "received"],
|
||||||
"created" => ["post-thread-user", "created"],
|
"created" => ["post-thread-user", "created"],
|
||||||
"network" => ["post-thread-user", "network"],
|
"network" => ["post-thread-user", "network"],
|
||||||
|
"protocol" => ["post-user", "protocol"],
|
||||||
"restricted" => ["post-engagement", "language"],
|
"restricted" => ["post-engagement", "language"],
|
||||||
"comments" => "0",
|
"comments" => "0",
|
||||||
"activities" => "0",
|
"activities" => "0",
|
||||||
|
@ -221,6 +222,7 @@ return [
|
||||||
"received" => ["post-thread-user", "received"],
|
"received" => ["post-thread-user", "received"],
|
||||||
"created" => ["post-thread-user", "created"],
|
"created" => ["post-thread-user", "created"],
|
||||||
"network" => ["post-thread-user", "network"],
|
"network" => ["post-thread-user", "network"],
|
||||||
|
"protocol" => ["post-user", "protocol"],
|
||||||
"restricted" => ["post-searchindex", "language"],
|
"restricted" => ["post-searchindex", "language"],
|
||||||
"comments" => "0",
|
"comments" => "0",
|
||||||
"activities" => "0",
|
"activities" => "0",
|
||||||
|
@ -475,6 +477,7 @@ return [
|
||||||
"global" => ["post-user", "global"],
|
"global" => ["post-user", "global"],
|
||||||
"featured" => "EXISTS(SELECT `type` FROM `post-collection` WHERE `type` = 0 AND `uri-id` = `post-thread-user`.`uri-id`)",
|
"featured" => "EXISTS(SELECT `type` FROM `post-collection` WHERE `type` = 0 AND `uri-id` = `post-thread-user`.`uri-id`)",
|
||||||
"network" => ["post-thread-user", "network"],
|
"network" => ["post-thread-user", "network"],
|
||||||
|
"protocol" => ["post-user", "protocol"],
|
||||||
"vid" => ["post-origin", "vid"],
|
"vid" => ["post-origin", "vid"],
|
||||||
"psid" => ["post-thread-user", "psid"],
|
"psid" => ["post-thread-user", "psid"],
|
||||||
"verb" => "IF (`post-origin`.`vid` IS NULL, '', `verb`.`name`)",
|
"verb" => "IF (`post-origin`.`vid` IS NULL, '', `verb`.`name`)",
|
||||||
|
@ -858,6 +861,7 @@ return [
|
||||||
"global" => ["post-user", "global"],
|
"global" => ["post-user", "global"],
|
||||||
"featured" => "EXISTS(SELECT `type` FROM `post-collection` WHERE `type` = 0 AND `uri-id` = `post-thread-user`.`uri-id`)",
|
"featured" => "EXISTS(SELECT `type` FROM `post-collection` WHERE `type` = 0 AND `uri-id` = `post-thread-user`.`uri-id`)",
|
||||||
"network" => ["post-thread-user", "network"],
|
"network" => ["post-thread-user", "network"],
|
||||||
|
"protocol" => ["post-user", "protocol"],
|
||||||
"vid" => ["post-user", "vid"],
|
"vid" => ["post-user", "vid"],
|
||||||
"psid" => ["post-thread-user", "psid"],
|
"psid" => ["post-thread-user", "psid"],
|
||||||
"verb" => "IF (`post-user`.`vid` IS NULL, '', `verb`.`name`)",
|
"verb" => "IF (`post-user`.`vid` IS NULL, '', `verb`.`name`)",
|
||||||
|
@ -1036,6 +1040,7 @@ return [
|
||||||
"global" => ["post", "global"],
|
"global" => ["post", "global"],
|
||||||
"featured" => "EXISTS(SELECT `type` FROM `post-collection` WHERE `type` = 0 AND `uri-id` = `post`.`uri-id`)",
|
"featured" => "EXISTS(SELECT `type` FROM `post-collection` WHERE `type` = 0 AND `uri-id` = `post`.`uri-id`)",
|
||||||
"network" => ["post", "network"],
|
"network" => ["post", "network"],
|
||||||
|
"protocol" => "255",
|
||||||
"vid" => ["post", "vid"],
|
"vid" => ["post", "vid"],
|
||||||
"verb" => "IF (`post`.`vid` IS NULL, '', `verb`.`name`)",
|
"verb" => "IF (`post`.`vid` IS NULL, '', `verb`.`name`)",
|
||||||
"title" => ["post-content", "title"],
|
"title" => ["post-content", "title"],
|
||||||
|
@ -1188,6 +1193,7 @@ return [
|
||||||
"global" => ["post", "global"],
|
"global" => ["post", "global"],
|
||||||
"featured" => "EXISTS(SELECT `type` FROM `post-collection` WHERE `type` = 0 AND `uri-id` = `post-thread`.`uri-id`)",
|
"featured" => "EXISTS(SELECT `type` FROM `post-collection` WHERE `type` = 0 AND `uri-id` = `post-thread`.`uri-id`)",
|
||||||
"network" => ["post-thread", "network"],
|
"network" => ["post-thread", "network"],
|
||||||
|
"protocol" => "255",
|
||||||
"vid" => ["post", "vid"],
|
"vid" => ["post", "vid"],
|
||||||
"verb" => "IF (`post`.`vid` IS NULL, '', `verb`.`name`)",
|
"verb" => "IF (`post`.`vid` IS NULL, '', `verb`.`name`)",
|
||||||
"title" => ["post-content", "title"],
|
"title" => ["post-content", "title"],
|
||||||
|
@ -1381,6 +1387,7 @@ return [
|
||||||
"starred" => ["post-thread-user", "starred"],
|
"starred" => ["post-thread-user", "starred"],
|
||||||
"mention" => ["post-thread-user", "mention"],
|
"mention" => ["post-thread-user", "mention"],
|
||||||
"network" => ["post-thread-user", "network"],
|
"network" => ["post-thread-user", "network"],
|
||||||
|
"protocol" => ["post-user", "protocol"],
|
||||||
"contact-id" => ["post-thread-user", "contact-id"],
|
"contact-id" => ["post-thread-user", "contact-id"],
|
||||||
"contact-type" => ["ownercontact", "contact-type"],
|
"contact-type" => ["ownercontact", "contact-type"],
|
||||||
],
|
],
|
||||||
|
@ -1407,6 +1414,7 @@ return [
|
||||||
"starred" => ["post-thread-user", "starred"],
|
"starred" => ["post-thread-user", "starred"],
|
||||||
"mention" => ["post-thread-user", "mention"],
|
"mention" => ["post-thread-user", "mention"],
|
||||||
"network" => ["post-thread-user", "network"],
|
"network" => ["post-thread-user", "network"],
|
||||||
|
"protocol" => ["post-user", "protocol"],
|
||||||
"contact-id" => ["post-thread-user", "contact-id"],
|
"contact-id" => ["post-thread-user", "contact-id"],
|
||||||
"contact-type" => ["ownercontact", "contact-type"],
|
"contact-type" => ["ownercontact", "contact-type"],
|
||||||
],
|
],
|
||||||
|
@ -1769,6 +1777,7 @@ return [
|
||||||
"gravity" => ["post-user", "gravity"],
|
"gravity" => ["post-user", "gravity"],
|
||||||
"received" => ["post-user", "received"],
|
"received" => ["post-user", "received"],
|
||||||
"network" => ["post-user", "network"],
|
"network" => ["post-user", "network"],
|
||||||
|
"protocol" => ["post-user", "protocol"],
|
||||||
"author-id" => ["post-user", "author-id"],
|
"author-id" => ["post-user", "author-id"],
|
||||||
"name" => ["tag", "name"],
|
"name" => ["tag", "name"],
|
||||||
],
|
],
|
||||||
|
|
Loading…
Reference in a new issue