Merge pull request #14598 from Art4/merge-2024.09-rc-into-develop

Merge 2024.09-rc into develop

database.sql

@@ -2110,6 +2110,7 @@ CREATE VIEW `post-engagement-user-view` AS SELECT
 	`post-thread-user`.`received` AS `received`,
 	`post-thread-user`.`created` AS `created`,
 	`post-thread-user`.`network` AS `network`,
+	`post-user`.`protocol` AS `protocol`,
 	`post-engagement`.`language` AS `restricted`,
 	0 AS `comments`,
 	0 AS `activities`
@@ -2236,6 +2237,7 @@ CREATE VIEW `post-searchindex-user-view` AS SELECT
 	`post-thread-user`.`received` AS `received`,
 	`post-thread-user`.`created` AS `created`,
 	`post-thread-user`.`network` AS `network`,
+	`post-user`.`protocol` AS `protocol`,
 	`post-searchindex`.`language` AS `restricted`,
 	0 AS `comments`,
 	0 AS `activities`
@@ -2494,6 +2496,7 @@ CREATE VIEW `post-thread-origin-view` AS SELECT
 	`post-user`.`global` AS `global`,
 	EXISTS(SELECT `type` FROM `post-collection` WHERE `type` = 0 AND `uri-id` = `post-thread-user`.`uri-id`) AS `featured`,
 	`post-thread-user`.`network` AS `network`,
+	`post-user`.`protocol` AS `protocol`,
 	`post-origin`.`vid` AS `vid`,
 	`post-thread-user`.`psid` AS `psid`,
 	IF (`post-origin`.`vid` IS NULL, '', `verb`.`name`) AS `verb`,
@@ -2881,6 +2884,7 @@ CREATE VIEW `post-thread-user-view` AS SELECT
 	`post-user`.`global` AS `global`,
 	EXISTS(SELECT `type` FROM `post-collection` WHERE `type` = 0 AND `uri-id` = `post-thread-user`.`uri-id`) AS `featured`,
 	`post-thread-user`.`network` AS `network`,
+	`post-user`.`protocol` AS `protocol`,
 	`post-user`.`vid` AS `vid`,
 	`post-thread-user`.`psid` AS `psid`,
 	IF (`post-user`.`vid` IS NULL, '', `verb`.`name`) AS `verb`,
@@ -3061,6 +3065,7 @@ CREATE VIEW `post-view` AS SELECT
 	`post`.`global` AS `global`,
 	EXISTS(SELECT `type` FROM `post-collection` WHERE `type` = 0 AND `uri-id` = `post`.`uri-id`) AS `featured`,
 	`post`.`network` AS `network`,
+	255 AS `protocol`,
 	`post`.`vid` AS `vid`,
 	IF (`post`.`vid` IS NULL, '', `verb`.`name`) AS `verb`,
 	`post-content`.`title` AS `title`,
@@ -3215,6 +3220,7 @@ CREATE VIEW `post-thread-view` AS SELECT
 	`post`.`global` AS `global`,
 	EXISTS(SELECT `type` FROM `post-collection` WHERE `type` = 0 AND `uri-id` = `post-thread`.`uri-id`) AS `featured`,
 	`post-thread`.`network` AS `network`,
+	255 AS `protocol`,
 	`post`.`vid` AS `vid`,
 	IF (`post`.`vid` IS NULL, '', `verb`.`name`) AS `verb`,
 	`post-content`.`title` AS `title`,
@@ -3418,6 +3424,7 @@ CREATE VIEW `network-thread-view` AS SELECT
 	`post-thread-user`.`starred` AS `starred`,
 	`post-thread-user`.`mention` AS `mention`,
 	`post-thread-user`.`network` AS `network`,
+	`post-user`.`protocol` AS `protocol`,
 	`post-thread-user`.`contact-id` AS `contact-id`,
 	`ownercontact`.`contact-type` AS `contact-type`
 	FROM `post-thread-user`
@@ -3446,6 +3453,7 @@ CREATE VIEW `network-thread-circle-view` AS SELECT
 	`post-thread-user`.`starred` AS `starred`,
 	`post-thread-user`.`mention` AS `mention`,
 	`post-thread-user`.`network` AS `network`,
+	`post-user`.`protocol` AS `protocol`,
 	`post-thread-user`.`contact-id` AS `contact-id`,
 	`ownercontact`.`contact-type` AS `contact-type`
 	FROM `post-thread-user`
@@ -3818,6 +3826,7 @@ CREATE VIEW `tag-search-view` AS SELECT
 	`post-user`.`gravity` AS `gravity`,
 	`post-user`.`received` AS `received`,
 	`post-user`.`network` AS `network`,
+	`post-user`.`protocol` AS `protocol`,
 	`post-user`.`author-id` AS `author-id`,
 	`tag`.`name` AS `name`
 	FROM `post-tag`

src/Contact/Avatar.php

@@ -83,7 +83,7 @@ class Avatar
 		$filename  = self::getFilename($contact['url']);
 		$timestamp = time();
 
-		$fields['blurhash'] = $image->getBlurHash();
+		$fields['blurhash'] = $image->getBlurHash($img_str);
 
 		$fields['photo'] = self::storeAvatarCache($image, $filename, Proxy::PIXEL_SMALL, $timestamp);
 		$fields['thumb'] = self::storeAvatarCache($image, $filename, Proxy::PIXEL_THUMB, $timestamp);

src/Core/Protocol.php

@@ -302,7 +302,7 @@ class Protocol
 			return false;
 		}
 
-		if (in_array($protocol, array_merge(self::NATIVE_SUPPORT, [self::ZOT, self::PHANTOM]))) {
+		if (in_array($protocol, array_merge(self::NATIVE_SUPPORT, [self::ZOT, self::BLUESKY, self::PHANTOM]))) {
 			return true;
 		}
 

src/Model/Contact.php

@@ -2321,7 +2321,7 @@ class Contact
 						if ($fetchResult->isSuccess() && !empty($img_str)) {
 							$image = new Image($img_str, $fetchResult->getContentType(), $avatar);
 							if ($image->isValid()) {
-								$update_fields['blurhash'] = $image->getBlurHash();
+								$update_fields['blurhash'] = $image->getBlurHash($img_str);
 							} else {
 								return;
 							}

src/Model/Item.php

@@ -86,7 +86,7 @@ class Item
 
 	// Field list that is used to display the items
 	const DISPLAY_FIELDLIST = [
-		'uid', 'id', 'parent', 'guid', 'network', 'gravity',
+		'uid', 'id', 'parent', 'guid', 'network', 'protocol', 'gravity',
 		'uri-id', 'uri', 'thr-parent-id', 'thr-parent', 'parent-uri-id', 'parent-uri', 'conversation',
 		'commented', 'created', 'edited', 'received', 'verb', 'object-type', 'postopts', 'plink',
 		'wall', 'private', 'starred', 'origin', 'parent-origin', 'title', 'body', 'language', 'sensitive',
@@ -4174,10 +4174,11 @@ class Item
 	 * @param string $uri
 	 * @param int    $uid
 	 * @param int    $completion
+	 * @param string $mimetype
 	 *
 	 * @return integer item id
 	 */
-	public static function fetchByLink(string $uri, int $uid = 0, int $completion = ActivityPub\Receiver::COMPLETION_MANUAL): int
+	public static function fetchByLink(string $uri, int $uid = 0, int $completion = ActivityPub\Receiver::COMPLETION_MANUAL, string $mimetype = ''): int
 	{
 		Logger::info('Trying to fetch link', ['uid' => $uid, 'uri' => $uri]);
 		$item_id = self::searchByLink($uri, $uid);
@@ -4199,35 +4200,49 @@ class Item
 		Hook::callAll('item_by_link', $hookData);
 
 		if (isset($hookData['item_id'])) {
+			Logger::info('Hook link fetched', ['uid' => $uid, 'uri' => $uri, 'id' => $hookData['item_id']]);
 			return is_numeric($hookData['item_id']) ? $hookData['item_id'] : 0;
 		}
 
+		if (!$mimetype) {
 			try {
 				$curlResult = DI::httpClient()->head($uri, [HttpClientOptions::ACCEPT_CONTENT => HttpClientAccept::JSON_AS, HttpClientOptions::REQUEST => HttpClientRequest::ACTIVITYPUB]);
-			if (!HTTPSignature::isValidContentType($curlResult->getContentType(), $uri) && (current(explode(';', $curlResult->getContentType())) == 'application/json')) {
+				$mimetype = empty($curlResult) ? '' : $curlResult->getContentType();
-				// Issue 14126: Workaround for Mastodon servers that return "application/json" on a "head" request.
-				$curlResult = HTTPSignature::fetchRaw($uri, $uid);
-			}
-			if (HTTPSignature::isValidContentType($curlResult->getContentType(), $uri)) {
-				$fetched_uri = ActivityPub\Processor::fetchMissingActivity($uri, [], '', $completion, $uid);
-			}
 			} catch (\Throwable $th) {
-			Logger::info('Invalid link', ['uid' => $uid, 'uri' => $uri, 'code' => $th->getCode(), 'message' => $th->getMessage()]);
+				Logger::info('Error while fetching HTTP link via HEAD', ['uid' => $uid, 'uri' => $uri, 'code' => $th->getCode(), 'message' => $th->getMessage()]);
 				return 0;
 			}
-
-		if (!empty($fetched_uri)) {
-			$item_id = self::searchByLink($fetched_uri, $uid);
-		} else {
-			$item_id = Diaspora::fetchByURL($uri);
 		}
 
-		if (!empty($item_id)) {
+		if (!HTTPSignature::isValidContentType($mimetype, $uri) && (current(explode(';', $mimetype)) == 'application/json')) {
-			Logger::info('Link fetched', ['uid' => $uid, 'uri' => $uri, 'id' => $item_id]);
+			try {
+				// Issue 14126: Workaround for Mastodon servers that return "application/json" on a "head" request.
+				$curlResult = HTTPSignature::fetchRaw($uri, $uid);
+				$mimetype = empty($curlResult) ? '' : $curlResult->getContentType();
+			} catch (\Throwable $th) {
+				Logger::info('Error while fetching HTTP link via signed GET', ['uid' => $uid, 'uri' => $uri, 'code' => $th->getCode(), 'message' => $th->getMessage()]);
+				return 0;
+			}
+		}
+
+		if (HTTPSignature::isValidContentType($mimetype, $uri)) {
+			$fetched_uri = ActivityPub\Processor::fetchMissingActivity($uri, [], '', $completion, $uid);
+			if (!empty($fetched_uri)) {
+				$item_id = self::searchByLink($fetched_uri, $uid);
+				if ($item_id) {
+					Logger::info('ActivityPub link fetched', ['uid' => $uid, 'uri' => $uri, 'id' => $item_id]);
+					return $item_id;
+				}
+			}
+		}
+
+		$item_id = Diaspora::fetchByURL($uri);
+		if ($item_id) {
+			Logger::info('Diaspora link fetched', ['uid' => $uid, 'uri' => $uri, 'id' => $item_id]);
 			return $item_id;
 		}
 
-		Logger::info('Link not found', ['uid' => $uid, 'uri' => $uri]);
+		Logger::info('This is not an item link', ['uid' => $uid, 'uri' => $uri]);
 		return 0;
 	}
 

src/Model/Photo.php

@@ -434,6 +434,7 @@ class Photo
 		$data        = '';
 		$backend_ref = '';
 		$storage     = '';
+		$img_str     = $image->asString();
 
 		try {
 			if (DBA::isResult($existing_photo)) {
@@ -442,9 +443,9 @@ class Photo
 			} else {
 				$storage = DI::storage();
 			}
-			$backend_ref = $storage->put($image->asString(), $backend_ref);
+			$backend_ref = $storage->put($img_str, $backend_ref);
 		} catch (InvalidClassStorageException $storageException) {
-			$data = $image->asString();
+			$data = $img_str;
 		}
 
 		$fields = [
@@ -452,7 +453,7 @@ class Photo
 			'contact-id' => $cid,
 			'guid' => $guid,
 			'resource-id' => $rid,
-			'hash' => md5($image->asString()),
+			'hash' => md5($img_str),
 			'created' => $created,
 			'edited' => DateTimeFormat::utcNow(),
 			'filename' => basename($filename),
@@ -460,8 +461,8 @@ class Photo
 			'album' => $album,
 			'height' => $image->getHeight(),
 			'width' => $image->getWidth(),
-			'datasize' => strlen($image->asString()),
+			'datasize' => strlen($img_str),
-			'blurhash' => $image->getBlurHash(),
+			'blurhash' => $image->getBlurHash($img_str),
 			'data' => $data,
 			'scale' => $scale,
 			'photo-type' => $type,

src/Model/Post/Link.php

@@ -127,12 +127,12 @@ class Link
 
 		if (Images::isSupportedMimeType($fields['mimetype'])) {
 			$img_str = $curlResult->getBodyString();
-			$image = new Image($img_str, $fields['mimetype'], $url);
+			$image = new Image($img_str, $fields['mimetype'], $url, false);
 			if ($image->isValid()) {
 				$fields['mimetype'] = $image->getType();
 				$fields['width']    = $image->getWidth();
 				$fields['height']   = $image->getHeight();
-				$fields['blurhash'] = $image->getBlurHash();
+				$fields['blurhash'] = $image->getBlurHash($img_str);
 			}
 		}
 

src/Model/Post/Media.php

@@ -51,6 +51,8 @@ class Media
 	const ACTIVITY    = 20;
 	const ACCOUNT     = 21;
 	const HLS         = 22;
+	const JSON        = 23;
+	const LD          = 24;
 	const DOCUMENT    = 128;
 
 	/**
@@ -180,14 +182,14 @@ class Media
 		}
 
 		// Fetch the mimetype or size if missing.
-		if (Network::isValidHttpUrl($media['url']) && (empty($media['mimetype']) || empty($media['size']))) {
+		if (Network::isValidHttpUrl($media['url']) && empty($media['mimetype']) && !in_array($media['type'], [self::IMAGE, self::HLS])) {
 			$timeout = DI::config()->get('system', 'xrd_timeout');
 			try {
-				$curlResult = DI::httpClient()->head($media['url'], [HttpClientOptions::TIMEOUT => $timeout, HttpClientOptions::REQUEST => HttpClientRequest::CONTENTTYPE]);
+				$curlResult = DI::httpClient()->head($media['url'], [HttpClientOptions::ACCEPT_CONTENT => HttpClientAccept::AS_DEFAULT, HttpClientOptions::TIMEOUT => $timeout, HttpClientOptions::REQUEST => HttpClientRequest::CONTENTTYPE]);
 
 				// Workaround for systems that can't handle a HEAD request
-				if (!$curlResult->isSuccess() && ($curlResult->getReturnCode() == 405)) {
+				if (!$curlResult->isSuccess() && in_array($curlResult->getReturnCode(), [400, 403, 405])) {
-					$curlResult = DI::httpClient()->get($media['url'], HttpClientAccept::DEFAULT, [HttpClientOptions::TIMEOUT => $timeout]);
+					$curlResult = DI::httpClient()->get($media['url'], HttpClientAccept::AS_DEFAULT, [HttpClientOptions::TIMEOUT => $timeout]);
 				}
 				if ($curlResult->isSuccess()) {
 					if (empty($media['mimetype'])) {
@@ -197,16 +199,20 @@ class Media
 						$media['size'] = (int)($curlResult->getHeader('Content-Length')[0] ?? strlen($curlResult->getBodyString() ?? ''));
 					}
 				} else {
-					Logger::notice('Could not fetch head', ['media' => $media]);
+					Logger::notice('Could not fetch head', ['media' => $media, 'code' => $curlResult->getReturnCode()]);
 				}
 			} catch (\Throwable $th) {
 				Logger::notice('Got exception', ['code' => $th->getCode(), 'message' => $th->getMessage()]);
 			}
 		}
 
-		$filetype = !empty($media['mimetype']) ? strtolower(current(explode('/', $media['mimetype']))) : '';
+		if (($media['type'] != self::DOCUMENT) && !empty($media['mimetype'])) {
+			$media = self::addType($media);
+		}
 
-		if (($media['type'] == self::IMAGE) || ($filetype == 'image')) {
+		Logger::debug('Got type for url', ['type' => $media['type'], 'mimetype' => $media['mimetype'] ?? '', 'url' => $media['url']]);
+
+		if ($media['type'] == self::IMAGE) {
 			$imagedata = Images::getInfoFromURLCached($media['url'], empty($media['description']));
 			if ($imagedata) {
 				$media['mimetype'] = $imagedata['mime'];
@@ -223,23 +229,19 @@ class Media
 			}
 		}
 
-		if ($media['type'] != self::DOCUMENT) {
-			$media = self::addType($media);
-		}
-
 		if (!empty($media['preview'])) {
 			$media = self::addPreviewData($media);
 		}
 
-		if (in_array($media['type'], [self::TEXT, self::APPLICATION, self::HTML, self::XML, self::PLAIN])) {
+		if (in_array($media['type'], [self::TEXT, self::ACTIVITY, self::LD, self::JSON, self::HTML, self::XML, self::PLAIN])) {
-			$media = self::addActivity($media);
-		}
-
-		if (in_array($media['type'], [self::TEXT, self::APPLICATION, self::HTML, self::XML, self::PLAIN])) {
 			$media = self::addAccount($media);
 		}
 
-		if ($media['type'] == self::HTML) {
+		if (in_array($media['type'], [self::ACTIVITY, self::LD, self::JSON])) {
+			$media = self::addActivity($media);
+		}
+
+		if (in_array($media['type'], [self::HTML, self::LD, self::JSON])) {
 			$media = self::addPage($media);
 		}
 
@@ -254,6 +256,16 @@ class Media
 
 		$imagedata = Images::getInfoFromURLCached($media['preview']);
 		if ($imagedata) {
+			$media['blurhash'] = $imagedata['blurhash'] ?? null;
+
+			// When the original picture is potentially animated but the preview isn't, we override the preview
+			if (in_array($media['mimetype'] ?? '', ['image/gif', 'image/png']) && !in_array($imagedata['mime'], ['image/gif', 'image/png'])) {
+				$media['preview'] = $media['url'];
+				$media['preview-width'] = $media['width'];
+				$media['preview-height'] = $media['height'];
+				return $media;
+			}
+
 			$media['preview-width'] = $imagedata[0];
 			$media['preview-height'] = $imagedata[1];
 		}
@@ -269,19 +281,22 @@ class Media
 	 */
 	private static function addActivity(array $media): array
 	{
-		$id = Item::fetchByLink($media['url'], 0, ActivityPub\Receiver::COMPLETION_ASYNC);
+		$id = Item::fetchByLink($media['url'], 0, ActivityPub\Receiver::COMPLETION_ASYNC, $media['mimetype'] ?? '');
 		if (empty($id)) {
+			$media['type'] = $media['type'] == self::ACTIVITY ? self::JSON : $media['type'];
 			return $media;
 		}
 
 		$item = Post::selectFirst([], ['id' => $id, 'network' => Protocol::FEDERATED]);
 		if (empty($item['id'])) {
 			Logger::debug('Not a federated activity', ['id' => $id, 'uri-id' => $media['uri-id'], 'url' => $media['url']]);
+			$media['type'] = $media['type'] == self::ACTIVITY ? self::JSON : $media['type'];
 			return $media;
 		}
 
 		if ($item['uri-id'] == $media['uri-id']) {
 			Logger::info('Media-Uri-Id is identical to Uri-Id', ['uri-id' => $media['uri-id']]);
+			$media['type'] = $media['type'] == self::ACTIVITY ? self::JSON : $media['type'];
 			return $media;
 		}
 
@@ -290,6 +305,7 @@ class Media
 			parse_url($item['plink'], PHP_URL_HOST) != parse_url($item['uri'], PHP_URL_HOST)
 		) {
 			Logger::debug('Not a link to an activity', ['uri-id' => $media['uri-id'], 'url' => $media['url'], 'plink' => $item['plink'], 'uri' => $item['uri']]);
+			$media['type'] = $media['type'] == self::ACTIVITY ? self::JSON : $media['type'];
 			return $media;
 		}
 
@@ -375,14 +391,23 @@ class Media
 	 */
 	private static function addPage(array $media): array
 	{
-		$data = ParseUrl::getSiteinfoCached($media['url']);
+		$data = ParseUrl::getSiteinfoCached($media['url'], $media['mimetype'] ?? '');
-		$media['size'] = $data['size'] ?? null;
+		if (empty($data['images'][0]['src']) && empty($data['text']) && empty($data['title'])) {
+			if (!empty($media['preview'])) {
+				$media = self::addPreviewData($media);
+				Logger::debug('Detected site data is empty, use suggested media data instead', ['uri-id' => $media['uri-id'], 'url' => $media['url'], 'type' => $data['type']]);
+			}
+		} else {
 			$media['preview'] = $data['images'][0]['src'] ?? null;
 			$media['preview-height'] = $data['images'][0]['height'] ?? null;
 			$media['preview-width'] = $data['images'][0]['width'] ?? null;
 			$media['blurhash'] = $data['images'][0]['blurhash'] ?? null;
 			$media['description'] = $data['text'] ?? null;
 			$media['name'] = $data['title'] ?? null;	
+		}
+		
+		$media['type'] = self::HTML;
+		$media['size'] = $data['size'] ?? null;
 		$media['author-url'] = $data['author_url'] ?? null;
 		$media['author-name'] = $data['author_name'] ?? null;
 		$media['author-image'] = $data['author_img'] ?? null;
@@ -481,6 +506,12 @@ class Media
 			$type = self::TORRENT;
 		} elseif (($filetype == 'application') && ($subtype == 'vnd.apple.mpegurl')) {
 			$type = self::HLS;
+		} elseif (($filetype == 'application') && ($subtype == 'activity+json')) {
+			$type = self::ACTIVITY;
+		} elseif (($filetype == 'application') && ($subtype == 'ld+json')) {
+			$type = self::LD;
+		} elseif (($filetype == 'application') && ($subtype == 'json')) {
+			$type = self::JSON;
 		} elseif ($filetype == 'application') {
 			$type = self::APPLICATION;
 		} else {

src/Module/Stats.php

@@ -94,6 +94,13 @@ class Stats extends BaseModule
 				'deferred'      => [],
 				'total'         => [],
 			],
+			'jetstream' => [
+				'drift'     => intval($this->keyValue->get('jetstream_drift')),
+				'did_count' => intval($this->keyValue->get('jetstream_did_count')),
+				'did_limit' => intval($this->keyValue->get('jetstream_did_limit')),
+				'messages'  => intval($this->keyValue->get('jetstream_messages')),
+				'timestamp' => intval($this->keyValue->get('jetstream_timestamp')),
+			],
 			'users' => [
 				'total'          => intval($this->keyValue->get('nodeinfo_total_users')),
 				'activeWeek'     => intval($this->keyValue->get('nodeinfo_active_users_weekly')),

src/Network/HTTPClient/Client/HttpClientAccept.php

@@ -15,6 +15,9 @@ class HttpClientAccept
 	/** @var string Default value for "Accept" header */
 	public const DEFAULT = '*/*';
 
+	/** @var string Accept all types with a preferences of ActivityStream content */
+	public const AS_DEFAULT = 'application/activity+json,application/ld+json; profile="https://www.w3.org/ns/activitystreams",*/*;q=0.9';
+
 	public const ATOM_XML  = 'application/atom+xml,text/xml;q=0.9,*/*;q=0.8';
 	public const FEED_XML  = 'application/atom+xml,application/rss+xml;q=0.9,application/rdf+xml;q=0.8,text/xml;q=0.7,*/*;q=0.6';
 	public const HTML      = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8';

src/Network/Probe.php

@@ -10,6 +10,7 @@ namespace Friendica\Network;
 use DOMDocument;
 use DomXPath;
 use Exception;
+use Friendica\Content\Text\HTML;
 use Friendica\Core\Hook;
 use Friendica\Core\Logger;
 use Friendica\Core\Protocol;
@@ -24,6 +25,7 @@ use Friendica\Network\HTTPClient\Client\HttpClientOptions;
 use Friendica\Network\HTTPClient\Client\HttpClientRequest;
 use Friendica\Protocol\ActivityNamespace;
 use Friendica\Protocol\ActivityPub;
+use Friendica\Protocol\ATProtocol;
 use Friendica\Protocol\Diaspora;
 use Friendica\Protocol\Email;
 use Friendica\Protocol\Feed;
@@ -732,8 +734,8 @@ class Probe
 
 		$parts = parse_url($uri);
 		if (empty($parts['scheme']) && empty($parts['host']) && (empty($parts['path']) || strpos($parts['path'], '@') === false)) {
-			Logger::info('URI was not detectable', ['uri' => $uri]);
+			Logger::info('URI was not detectable, probe for AT Protocol now', ['uri' => $uri]);
-			return [];
+			return self::atProtocol($uri);
 		}
 
 		// If the URI starts with "mailto:" then jump directly to the mail detection
@@ -757,6 +759,10 @@ class Probe
 		}
 
 		if (empty($data)) {
+			$data = self::atProtocol($uri);
+			if (!empty($data)) {
+				return $data;
+			}
 			if (!empty($parts['scheme'])) {
 				return self::feed($uri);
 			} elseif (!empty($uid)) {
@@ -1687,6 +1693,75 @@ class Probe
 		return (string)Uri::fromParts((array)(array)$baseParts);
 	}
 
+	/**
+	 * Check for AT Protocol (Bluesky)
+	 *
+	 * @param string $uri Profile link
+	 * @return array Profile data or empty array
+	 */
+	private static function atProtocol(string $uri): array
+	{
+		if (parse_url($uri, PHP_URL_SCHEME) == 'did') {
+			$did = $uri;
+		} elseif (parse_url($uri, PHP_URL_PATH) == $uri && strpos($uri, '@') === false) {
+			$did = DI::atProtocol()->getDid($uri);
+			if (empty($did)) {
+				return [];
+			}
+		} elseif (Network::isValidHttpUrl($uri)) {
+			$did = DI::atProtocol()->getDidByProfile($uri);
+			if (empty($did)) {
+				return [];
+			}
+		} else {
+			return [];
+		}
+	
+		$profile = DI::atProtocol()->XRPCGet('app.bsky.actor.getProfile', ['actor' => $did]);
+		if (empty($profile) || empty($profile->did)) {
+			return [];
+		}
+
+		$nick = $profile->handle ?? $profile->did;
+		$name = $profile->displayName ?? $nick;
+
+		$data = [
+			'network'  => Protocol::BLUESKY,
+			'url'      => $profile->did,
+			'alias'    => ATProtocol::WEB . '/profile/' . $nick,
+			'name'     => $name ?: $nick,
+			'nick'     => $nick,
+			'addr'     => $nick,
+			'poll'     => ATProtocol::WEB . '/profile/' . $profile->did . '/rss',
+			'photo'    => $profile->avatar ?? '',
+		];
+	
+		if (!empty($profile->description)) {
+			$data['about'] = HTML::toBBCode($profile->description);
+		}
+	
+		if (!empty($profile->banner)) {
+			$data['header'] = $profile->banner;
+		}
+
+		$directory = DI::atProtocol()->get(ATProtocol::DIRECTORY . '/' . $profile->did);
+		if (!empty($directory)) {
+			foreach ($directory->service as $service) {
+				if (($service->id == '#atproto_pds') && ($service->type == 'AtprotoPersonalDataServer') && !empty($service->serviceEndpoint)) {
+					$data['baseurl'] = $service->serviceEndpoint;
+				}
+			}
+
+			foreach ($directory->verificationMethod as $method) {
+				if (!empty($method->publicKeyMultibase)) {
+					$data['pubkey'] = $method->publicKeyMultibase;
+				}
+			}
+		}
+
+		return $data;
+	}
+
 	/**
 	 * Check for feed contact
 	 *

src/Object/Image.php

@@ -105,7 +105,8 @@ class Image
 	 * @param string $data
 	 * @return boolean
 	 */
-	private function isAnimatedWebP(string $data) {
+	private function isAnimatedWebP(string $data)
+	{
 		$header_format = 'A4Riff/I1Filesize/A4Webp/A4Vp/A74Chunk';
 		$header = @unpack($header_format, $data);
 
@@ -356,7 +357,6 @@ class Image
 		} else {
 			return false;
 		}
-
 	}
 
 	/**
@@ -767,9 +767,9 @@ class Image
 	 *
 	 * @return string
 	 */
-	public function getBlurHash(): string
+	public function getBlurHash(string $img_str = ''): string
 	{
-		$image = New Image($this->asString(), $this->getType(), $this->filename, false);
+		$image = new Image($img_str ?: $this->asString(), $this->getType(), $this->filename, false);
 		if (!$this->isValid()) {
 			return '';
 		}

src/Object/Post.php

@@ -16,6 +16,7 @@ use Friendica\Core\Protocol;
 use Friendica\Core\Renderer;
 use Friendica\DI;
 use Friendica\Model\Contact;
+use Friendica\Model\Conversation;
 use Friendica\Model\Item;
 use Friendica\Model\Post as PostModel;
 use Friendica\Model\Tag;
@@ -193,7 +194,7 @@ class Post
 
 		$privacy   = $this->fetchPrivacy($item);
 		$lock      = ($item['private'] == Item::PRIVATE) ? $privacy : false;
-		$connector = !in_array($item['network'], Protocol::NATIVE_SUPPORT) ? DI::l10n()->t('Connector Message') : false;
+		$connector = !in_array($item['network'], Protocol::NATIVE_SUPPORT) && ($item['protocol'] != Conversation::PARCEL_JETSTREAM) ? DI::l10n()->t('Connector Message') : false;
 
 		$shareable    = in_array($conv->getProfileOwner(), [0, DI::userSession()->getLocalUserId()]) && $item['private'] != Item::PRIVATE;
 		$announceable = $shareable && in_array($item['network'], [Protocol::ACTIVITYPUB, Protocol::DFRN, Protocol::DIASPORA, Protocol::TWITTER, Protocol::TUMBLR, Protocol::BLUESKY]);

src/Protocol/ATProtocol.php

@@ -66,6 +66,11 @@ final class ATProtocol
 		$this->httpClient = $httpClient;
 	}
 
+	/**
+	 * Returns an array of user ids who want to import the Bluesky timeline
+	 *
+	 * @return array user ids
+	 */
 	public function getUids(): array
 	{
 		$uids         = [];
@@ -92,6 +97,15 @@ final class ATProtocol
 		return $uids;
 	}
 
+	/**
+	 * Fetches XRPC data
+	 * @see https://atproto.com/specs/xrpc#lexicon-http-endpoints
+	 *
+	 * @param string  $url        for example "app.bsky.feed.getTimeline"
+	 * @param array   $parameters Array with parameters
+	 * @param integer $uid        User ID
+	 * @return stdClass|null Fetched data
+	 */
 	public function XRPCGet(string $url, array $parameters = [], int $uid = 0): ?stdClass
 	{
 		if (!empty($parameters)) {
@@ -119,6 +133,13 @@ final class ATProtocol
 		return $data;
 	}
 
+	/**
+	 * Fetch data from the given URL via GET and return it as a JSON class
+	 *
+	 * @param string $url HTTP URL
+	 * @param array $opts HTTP options
+	 * @return stdClass|null Fetched data
+	 */
 	public function get(string $url, array $opts = []): ?stdClass
 	{
 		try {
@@ -141,13 +162,31 @@ final class ATProtocol
 		return $data;
 	}
 
+	/**
+	 * Perform an XRPC post for a given user
+	 * @see https://atproto.com/specs/xrpc#lexicon-http-endpoints
+	 *
+	 * @param integer $uid       User ID
+	 * @param string $url        Endpoints like "com.atproto.repo.createRecord"
+	 * @param [type] $parameters array or StdClass with parameters
+	 * @return stdClass|null
+	 */
 	public function XRPCPost(int $uid, string $url, $parameters): ?stdClass
 	{
 		$data = $this->post($uid, '/xrpc/' . $url, json_encode($parameters), ['Content-type' => 'application/json', 'Authorization' => ['Bearer ' . $this->getUserToken($uid)]]);
 		return $data;
 	}
 
-	private function post(int $uid, string $url, string $params, array $headers): ?stdClass
+	/**
+	 * Post data to the user PDS
+	 *
+	 * @param integer $uid   User ID
+	 * @param string $url    HTTP URL without the hostname
+	 * @param string $params Parameter string
+	 * @param array $headers HTTP header information
+	 * @return stdClass|null
+	 */
+	public function post(int $uid, string $url, string $params, array $headers): ?stdClass
 	{
 		$pds = $this->getUserPds($uid);
 		if (empty($pds)) {
@@ -172,11 +211,21 @@ final class ATProtocol
 			$data->code = $curlResult->getReturnCode();
 		}
 
+		if (!empty($data->code) && ($data->code >= 200) && ($data->code < 400)) {
 			$this->pConfig->set($uid, 'bluesky', 'status', self::STATUS_SUCCESS);
-		Item::incrementOutbound(Protocol::BLUESKY);
+		} else {
+			$this->pConfig->set($uid, 'bluesky', 'status', self::STATUS_API_FAIL);
+		}
 		return $data;
 	}
 
+	/**
+	 * Fetches the PDS for a given user
+	 * @see https://atproto.com/guides/glossary#pds-personal-data-server
+	 *
+	 * @param integer $uid User ID or 0
+	 * @return string|null PDS or null if the user has got no PDS assigned. If UID set to 0, the public api URL is used
+	 */
 	private function getUserPds(int $uid): ?string
 	{
 		if ($uid == 0) {
@@ -202,6 +251,14 @@ final class ATProtocol
 		return $pds;
 	}
 
+	/**
+	 * Fetch the DID for a given user
+	 * @see https://atproto.com/guides/glossary#did-decentralized-id
+	 *
+	 * @param integer $uid     User ID
+	 * @param boolean $refresh Default "false". If set to true, the DID is detected from the handle again.
+	 * @return string|null     DID or null if no DID has been found.
+	 */
 	public function getUserDid(int $uid, bool $refresh = false): ?string
 	{
 		if (!$this->pConfig->get($uid, 'bluesky', 'post')) {
@@ -230,7 +287,13 @@ final class ATProtocol
 		return $did;
 	}
 
-	private function getDid(string $handle): string
+	/**
+	 * Fetches the DID for a given handle
+	 *
+	 * @param string $handle The user handle
+	 * @return string DID (did:plc:...)
+	 */
+	public function getDid(string $handle): string
 	{
 		if ($handle == '') {
 			return '';
@@ -265,6 +328,12 @@ final class ATProtocol
 		return '';
 	}
 
+	/**
+	 * Fetches a DID for a given profile URL
+	 *
+	 * @param string $url HTTP path to the profile in the format https://bsky.app/profile/username
+	 * @return string DID (did:plc:...)
+	 */
 	public function getDidByProfile(string $url): string
 	{
 		if (preg_match('#^' . self::WEB . '/profile/(.+)#', $url, $matches)) {
@@ -314,6 +383,13 @@ final class ATProtocol
 		return $ids['bsky_did'];
 	}
 
+	/**
+	 * Fetches the DID of a given handle via a HTTP request to the .well-known URL.
+	 * This is one of the ways, custom handles can be authorized.
+	 *
+	 * @param string $handle The user handle
+	 * @return string DID (did:plc:...)
+	 */
 	private function getDidByWellknown(string $handle): string
 	{
 		$curlResult = $this->httpClient->get('http://' . $handle . '/.well-known/atproto-did');
@@ -328,6 +404,13 @@ final class ATProtocol
 		return '';
 	}
 
+	/**
+	 * Fetches the DID of a given handle via a DND request.
+	 * This is one of the ways, custom handles can be authorized.
+	 *
+	 * @param string $handle The user handle
+	 * @return string DID (did:plc:...)
+	 */
 	private function getDidByDns(string $handle): string
 	{
 		$records = @dns_get_record('_atproto.' . $handle . '.', DNS_TXT);
@@ -347,7 +430,13 @@ final class ATProtocol
 		return '';
 	}
 
-	private function getPdsOfDid(string $did): ?string
+	/**
+	 * Fetch the PDS of a given DID
+	 *
+	 * @param string $did DID (did:plc:...)
+	 * @return string|null URL of the PDS, e.g. https://enoki.us-east.host.bsky.network
+	 */
+	public function getPdsOfDid(string $did): ?string
 	{
 		$data = $this->get(self::DIRECTORY . '/' . $did);
 		if (empty($data) || empty($data->service)) {
@@ -363,6 +452,13 @@ final class ATProtocol
 		return null;
 	}
 
+	/**
+	 * Checks if the provided DID matches the handle
+	 *
+	 * @param string $did DID (did:plc:...)
+	 * @param string $handle The user handle
+	 * @return boolean
+	 */
 	private function isValidDid(string $did, string $handle): bool
 	{
 		$data = $this->get(self::DIRECTORY . '/' . $did);
@@ -373,7 +469,13 @@ final class ATProtocol
 		return in_array('at://' . $handle, $data->alsoKnownAs);
 	}
 
-	private function getUserToken(int $uid): string
+	/**
+	 * Fetches the user token for a given user
+	 *
+	 * @param integer $uid User ID
+	 * @return string user token
+	 */
+	public function getUserToken(int $uid): string
 	{
 		$token   = $this->pConfig->get($uid, 'bluesky', 'access_token');
 		$created = $this->pConfig->get($uid, 'bluesky', 'token_created');
@@ -387,12 +489,23 @@ final class ATProtocol
 		return $token;
 	}
 
+	/**
+	 * Refresh and returns the user token for a given user.
+	 *
+	 * @param integer $uid User ID
+	 * @return string user token
+	 */
 	private function refreshUserToken(int $uid): string
 	{
 		$token = $this->pConfig->get($uid, 'bluesky', 'refresh_token');
 
 		$data = $this->post($uid, '/xrpc/com.atproto.server.refreshSession', '', ['Authorization' => ['Bearer ' . $token]]);
 		if (empty($data) || empty($data->accessJwt)) {
+			$this->logger->debug('Refresh failed', ['return' => $data]);
+			$password = $this->pConfig->get($uid, 'bluesky', 'password');
+			if (!empty($password)) {
+				return $this->createUserToken($uid, $password);
+			}
 			$this->pConfig->set($uid, 'bluesky', 'status', self::STATUS_TOKEN_FAIL);
 			return '';
 		}
@@ -404,6 +517,13 @@ final class ATProtocol
 		return $data->accessJwt;
 	}
 
+	/**
+	 * Create a user token for the given user
+	 *
+	 * @param integer $uid      User ID
+	 * @param string  $password Application password
+	 * @return string user token
+	 */
 	public function createUserToken(int $uid, string $password): string
 	{
 		$did = $this->getUserDid($uid);

src/Protocol/ATProtocol/Actor.php

@@ -35,7 +35,13 @@ class Actor
 		$this->atprotocol = $atprotocol;
 	}
 
-	public function syncContacts(int $uid)
+	/**
+	 * Syncronize the contacts (followers, sharers) for the given user
+	 *
+	 * @param integer $uid User ID
+	 * @return void
+	 */
+	public function syncContacts(int $uid): void
 	{
 		$this->logger->info('Sync contacts for user - start', ['uid' => $uid]);
 		$contacts = Contact::selectToArray(['id', 'url', 'rel'], ['uid' => $uid, 'network' => Protocol::BLUESKY, 'rel' => [Contact::FRIEND, Contact::SHARING, Contact::FOLLOWER]]);
@@ -93,9 +99,16 @@ class Actor
 		$this->logger->info('Sync contacts for user - done', ['uid' => $uid]);
 	}
 
-	public function updateContactByDID(string $did)
+	/**
+	 * Update a contact for a given DID and user id
+	 *
+	 * @param string  $did         DID (did:plc:...)
+	 * @param integer $contact_uid User id of the contact to be updated
+	 * @return void
+	 */
+	public function updateContactByDID(string $did, int $contact_uid): void
 	{
-		$profile = $this->atprotocol->XRPCGet('app.bsky.actor.getProfile', ['actor' => $did]);
+		$profile = $this->atprotocol->XRPCGet('app.bsky.actor.getProfile', ['actor' => $did], $contact_uid);
 		if (empty($profile) || empty($profile->did)) {
 			return;
 		}
@@ -139,20 +152,7 @@ class Actor
 			}
 		}
 
-		/*
+		Contact::update($fields, ['nurl' => $profile->did, 'network' => Protocol::BLUESKY]);
-		@todo Add this part when the function will be callable with a uid
-		if (!empty($profile->viewer)) {
-			if (!empty($profile->viewer->following) && !empty($profile->viewer->followedBy)) {
-				$fields['rel'] = Contact::FRIEND;
-			} elseif (!empty($profile->viewer->following) && empty($profile->viewer->followedBy)) {
-				$fields['rel'] = Contact::SHARING;
-			} elseif (empty($profile->viewer->following) && !empty($profile->viewer->followedBy)) {
-				$fields['rel'] = Contact::FOLLOWER;
-			} else {
-				$fields['rel'] = Contact::NOTHING;
-			}
-		}
-		*/
 
 		if (!empty($profile->avatar)) {
 			$contact = Contact::selectFirst(['id', 'avatar'], ['network' => Protocol::BLUESKY, 'nurl' => $did, 'uid' => 0]);
@@ -161,16 +161,37 @@ class Actor
 			}
 		}
 
-		$this->logger->notice('Update profile', ['did' => $profile->did, 'fields' => $fields]);
+		$this->logger->notice('Update global profile', ['did' => $profile->did, 'fields' => $fields]);
 
-		Contact::update($fields, ['nurl' => $profile->did, 'network' => Protocol::BLUESKY]);
+		if (!empty($profile->viewer) && ($contact_uid != 0)) {
+			if (!empty($profile->viewer->following) && !empty($profile->viewer->followedBy)) {
+				$user_fields = ['rel' => Contact::FRIEND];
+			} elseif (!empty($profile->viewer->following) && empty($profile->viewer->followedBy)) {
+				$user_fields = ['rel' => Contact::SHARING];
+			} elseif (empty($profile->viewer->following) && !empty($profile->viewer->followedBy)) {
+				$user_fields = ['rel' => Contact::FOLLOWER];
+			} else {
+				$user_fields = ['rel' => Contact::NOTHING];
+			}
+			Contact::update($user_fields, ['nurl' => $profile->did, 'network' => Protocol::BLUESKY, 'uid' => $contact_uid]);
+			$this->logger->notice('Update user profile', ['uid' => $contact_uid, 'did' => $profile->did, 'fields' => $user_fields]);
+		}
 	}
 
-	public function getContactByDID(string $did, int $uid, int $contact_uid): array
+	/**
+	 * Fetch and possibly create a contact array for a given DID
+	 *
+	 * @param string  $did         The contact DID
+	 * @param integer $uid         "0" when either the public contact or the user contact is desired
+	 * @param integer $contact_uid If not found, the contact will be created for this user id
+	 * @param boolean $auto_update Default "false". If activated, the contact will be updated every 24 hours
+	 * @return array Contact array
+	 */
+	public function getContactByDID(string $did, int $uid, int $contact_uid, bool $auto_update = false): array
 	{
 		$contact = Contact::selectFirst([], ['network' => Protocol::BLUESKY, 'nurl' => $did, 'uid' => [$contact_uid, $uid]], ['order' => ['uid' => true]]);
 
-		if (!empty($contact)) {
+		if (!empty($contact) && (!$auto_update || ($contact['updated'] > DateTimeFormat::utc('now -24 hours')))) {
 			return $contact;
 		}
 
@@ -193,7 +214,7 @@ class Actor
 
 		$cid = Contact::insert($fields);
 
-		$this->updateContactByDID($did);
+		$this->updateContactByDID($did, $contact_uid);
 
 		return Contact::getById($cid);
 	}

src/Protocol/ATProtocol/Jetstream.php

@@ -73,10 +73,12 @@ class Jetstream
 		$this->processor  = $processor;
 	}
 
-	// *****************************************
+	/**
-	// * Listener
+	 * Listen to incoming webstream messages from Jetstream
-	// *****************************************
+	 *
-	public function listen()
+	 * @return void
+	 */
+	public function listen(): void
 	{
 		$timeout       = 300;
 		$timeout_limit = 10;
@@ -108,6 +110,7 @@ class Jetstream
 						$timestamp = $data->time_us;
 						$this->route($data);
 						$this->keyValue->set('jetstream_timestamp', $timestamp);
+						$this->incrementMessages();
 					} else {
 						$this->logger->warning('Unexpected return value', ['data' => $data]);
 						break;
@@ -135,6 +138,25 @@ class Jetstream
 		}
 	}
 
+	/**
+	 * Increment the message counter for the statistics page
+	 *
+	 * @return void
+	 */
+	private function incrementMessages(): void
+	{
+		$packets = (int)($this->keyValue->get('jetstream_messages') ?? 0);
+		if ($packets >= PHP_INT_MAX) {
+			$packets = 0;
+		}
+		$this->keyValue->set('jetstream_messages', $packets + 1);
+	}
+
+	/**
+	 * Synchronize contacts for all active users
+	 *
+	 * @return void
+	 */
 	private function syncContacts()
 	{
 		$active_uids = $this->atprotocol->getUids();
@@ -147,6 +169,11 @@ class Jetstream
 		}
 	}
 
+	/**
+	 * Set options like the followed DIDs
+	 *
+	 * @return void
+	 */
 	private function setOptions()
 	{
 		$active_uids = $this->atprotocol->getUids();
@@ -184,10 +211,14 @@ class Jetstream
 		}
 
 		if (!$this->capped && count($dids) < $did_limit) {
-			$contacts = Contact::selectToArray(['url'], ['uid' => 0, 'network' => Protocol::BLUESKY], ['order' => ['last-item' => true], 'limit' => $did_limit]);
+			$condition = ["`uid` = ? AND `network` = ? AND EXISTS(SELECT `author-id` FROM `post-user` WHERE `author-id` = `contact`.`id` AND `post-user`.`uid` != ?)", 0, Protocol::BLUESKY, 0];
+			$contacts = Contact::selectToArray(['url'], $condition, ['order' => ['last-item' => true], 'limit' => $did_limit]);
 			$dids     = $this->addDids($contacts, $uids, $did_limit, $dids);
 		}
 
+		$this->keyValue->set('jetstream_did_count', count($dids));
+		$this->keyValue->set('jetstream_did_limit', $did_limit);
+
 		$this->logger->debug('Selected DIDs', ['uids' => $active_uids, 'count' => count($dids), 'capped' => $this->capped]);
 		$update = [
 			'type'    => 'options_update',
@@ -204,6 +235,15 @@ class Jetstream
 		}
 	}
 
+	/**
+	 * Returns an array of DIDs provided by an array of contacts
+	 *
+	 * @param array   $contacts  Array of contact records
+	 * @param array   $uids      Array with the user ids with enabled bluesky timeline import
+	 * @param integer $did_limit Maximum limit of entries
+	 * @param array   $dids      Array of DIDs that are added to the output list
+	 * @return array DIDs
+	 */
 	private function addDids(array $contacts, array $uids, int $did_limit, array $dids): array
 	{
 		foreach ($contacts as $contact) {
@@ -218,7 +258,13 @@ class Jetstream
 		return $dids;
 	}
 
-	private function route(stdClass $data)
+	/**
+	 * Route incoming messages
+	 *
+	 * @param stdClass $data message object
+	 * @return void
+	 */
+	private function route(stdClass $data): void
 	{
 		Item::incrementInbound(Protocol::BLUESKY);
 
@@ -239,19 +285,15 @@ class Jetstream
 		}
 	}
 
-	private function routeCommits(stdClass $data)
+	/**
+	 * Route incoming commit messages
+	 *
+	 * @param stdClass $data message object
+	 * @return void
+	 */
+	private function routeCommits(stdClass $data): void
 	{
-		$drift = max(0, round(time() - $data->time_us / 1000000));
+		$drift = $this->getDrift($data);
-		if ($drift > 60 && !$this->capped) {
-			$this->capped = true;
-			$this->setOptions();
-			$this->logger->notice('Drift is too high, dids will be capped');
-		} elseif ($drift == 0 && $this->capped) {
-			$this->capped = false;
-			$this->setOptions();
-			$this->logger->notice('Drift is low enough, dids will be uncapped');
-		}
-
 		$this->logger->notice('Received commit', ['time' => date(DateTimeFormat::ATOM, $data->time_us / 1000000), 'drift' => $drift, 'capped' => $this->capped, 'did' => $data->did, 'operation' => $data->commit->operation, 'collection' => $data->commit->collection, 'timestamp' => $data->time_us]);
 		$timestamp = microtime(true);
 
@@ -295,11 +337,41 @@ class Jetstream
 				break;
 		}
 		if (microtime(true) - $timestamp > 2) {
-			$this->logger->notice('Commit processed', ['duration' => round(microtime(true) - $timestamp, 3), 'time' => date(DateTimeFormat::ATOM, $data->time_us / 1000000), 'did' => $data->did, 'operation' => $data->commit->operation, 'collection' => $data->commit->collection]);
+			$this->logger->notice('Commit processed', ['duration' => round(microtime(true) - $timestamp, 3), 'drift' => $drift, 'capped' => $this->capped, 'time' => date(DateTimeFormat::ATOM, $data->time_us / 1000000), 'did' => $data->did, 'operation' => $data->commit->operation, 'collection' => $data->commit->collection]);
 		}
 	}
 
-	private function routePost(stdClass $data, int $drift)
+	/**
+	 * Calculate the drift between the server timestamp and the current time.
+	 *
+	 * @param stdClass $data message object
+	 * @return integer The calculated drift
+	 */
+	private function getDrift(stdClass $data): int
+	{
+		$drift = max(0, round(time() - $data->time_us / 1000000));
+		$this->keyValue->set('jetstream_drift', $drift);
+
+		if ($drift > 60 && !$this->capped) {
+			$this->capped = true;
+			$this->setOptions();
+			$this->logger->notice('Drift is too high, dids will be capped');
+		} elseif ($drift == 0 && $this->capped) {
+			$this->capped = false;
+			$this->setOptions();
+			$this->logger->notice('Drift is low enough, dids will be uncapped');
+		}
+		return $drift;
+	}
+
+	/**
+	 * Route app.bsky.feed.post commits 
+	 *
+	 * @param stdClass $data message object
+	 * @param integer $drift
+	 * @return void
+	 */
+	private function routePost(stdClass $data, int $drift): void
 	{
 		switch ($data->commit->operation) {
 			case 'delete':
@@ -316,7 +388,14 @@ class Jetstream
 		}
 	}
 
-	private function routeRepost(stdClass $data, int $drift)
+	/**
+	 * Route app.bsky.feed.repost commits 
+	 *
+	 * @param stdClass $data message object
+	 * @param integer $drift
+	 * @return void
+	 */
+	private function routeRepost(stdClass $data, int $drift): void
 	{
 		switch ($data->commit->operation) {
 			case 'delete':
@@ -333,7 +412,13 @@ class Jetstream
 		}
 	}
 
-	private function routeLike(stdClass $data)
+	/**
+	 * Route app.bsky.feed.like commits 
+	 *
+	 * @param stdClass $data message object
+	 * @return void
+	 */
+	private function routeLike(stdClass $data): void
 	{
 		switch ($data->commit->operation) {
 			case 'delete':
@@ -350,7 +435,13 @@ class Jetstream
 		}
 	}
 
-	private function routeProfile(stdClass $data)
+	/**
+	 * Route app.bsky.actor.profile commits 
+	 *
+	 * @param stdClass $data message object
+	 * @return void
+	 */
+	private function routeProfile(stdClass $data): void
 	{
 		switch ($data->commit->operation) {
 			case 'delete':
@@ -358,11 +449,11 @@ class Jetstream
 				break;
 
 			case 'create':
-				$this->actor->updateContactByDID($data->did);
+				$this->actor->updateContactByDID($data->did, 0);
 				break;
 
 			case 'update':
-				$this->actor->updateContactByDID($data->did);
+				$this->actor->updateContactByDID($data->did, 0);
 				break;
 
 			default:
@@ -371,7 +462,13 @@ class Jetstream
 		}
 	}
 
-	private function routeFollow(stdClass $data)
+	/**
+	 * Route app.bsky.graph.follow commits 
+	 *
+	 * @param stdClass $data message object
+	 * @return void
+	 */
+	private function routeFollow(stdClass $data): void
 	{
 		switch ($data->commit->operation) {
 			case 'delete':
@@ -394,7 +491,13 @@ class Jetstream
 		}
 	}
 
-	private function storeCommitMessage(stdClass $data)
+	/**
+	 * Store commit messages for debugging purposes
+	 *
+	 * @param stdClass $data message object
+	 * @return void
+	 */
+	private function storeCommitMessage(stdClass $data): void
 	{
 		if ($this->config->get('debug', 'jetstream_log')) {
 			$tempfile = tempnam(System::getTempPath(), 'at-proto.commit.' . $data->commit->collection . '.' . $data->commit->operation . '-');

src/Protocol/ATProtocol/Processor.php

@@ -13,6 +13,7 @@ namespace Friendica\Protocol\ATProtocol;
 use Friendica\App\BaseURL;
 use Friendica\Core\Protocol;
 use Friendica\Database\Database;
+use Friendica\Database\DBA;
 use Friendica\Model\Contact;
 use Friendica\Model\Conversation;
 use Friendica\Model\Item;
@@ -130,12 +131,12 @@ class Processor
 		if (!empty($data->commit->record->reply)) {
 			$root   = $this->getUri($data->commit->record->reply->root);
 			$parent = $this->getUri($data->commit->record->reply->parent);
-			$uids   = $this->getPostUids($root);
+			$uids   = $this->getPostUids($root, true);
 			if (!$uids) {
 				$this->logger->debug('Comment is not imported since the root post is not found.', ['root' => $root, 'parent' => $parent]);
 				return;
 			}
-			if ($dont_fetch && !$this->getPostUids($parent)) {
+			if ($dont_fetch && !$this->getPostUids($parent, false)) {
 				$this->logger->debug('Comment is not imported since the parent post is not found.', ['root' => $root, 'parent' => $parent]);
 				return;
 			}
@@ -168,6 +169,7 @@ class Processor
 						return;
 					}
 				}
+				$item['source'] = json_encode($post);
 				$item = $this->addMedia($post->thread->post->embed, $item, 0);
 			}
 
@@ -185,7 +187,7 @@ class Processor
 
 	public function createRepost(stdClass $data, array $uids, bool $dont_fetch)
 	{
-		if ($dont_fetch && !$this->getPostUids($this->getUri($data->commit->record->subject))) {
+		if ($dont_fetch && !$this->getPostUids($this->getUri($data->commit->record->subject), true)) {
 			$this->logger->debug('Repost is not imported since the subject is not found.', ['subject' => $this->getUri($data->commit->record->subject)]);
 			return;
 		}
@@ -215,7 +217,7 @@ class Processor
 
 	public function createLike(stdClass $data)
 	{
-		$uids = $this->getPostUids($this->getUri($data->commit->record->subject));
+		$uids = $this->getPostUids($this->getUri($data->commit->record->subject), false);
 		if (!$uids) {
 			$this->logger->debug('Like is not imported since the subject is not found.', ['subject' => $this->getUri($data->commit->record->subject)]);
 			return;
@@ -272,7 +274,7 @@ class Processor
 		return true;
 	}
 
-	private function processPost(stdClass $post, int $uid, int $post_reason, int $causer, int $level, int $protocol): int
+	public function processPost(stdClass $post, int $uid, int $post_reason, int $causer, int $level, int $protocol): int
 	{
 		$uri = $this->getUri($post);
 
@@ -380,7 +382,7 @@ class Processor
 		return $item;
 	}
 
-	private function getHeaderFromPost(stdClass $post, string $uri, int $uid, int $protocol): array
+	public function getHeaderFromPost(stdClass $post, string $uri, int $uid, int $protocol): array
 	{
 		$parts = $this->getUriParts($uri);
 		if (empty($post->author) || empty($post->cid) || empty($parts->rkey)) {
@@ -540,6 +542,8 @@ class Processor
 						'url'         => $image->fullsize,
 						'preview'     => $image->thumb,
 						'description' => $image->alt,
+						'height'      => $image->aspectRatio->height ?? null,
+						'width'       => $image->aspectRatio->width ?? null,
 					];
 					Post\Media::insert($media);
 				}
@@ -563,6 +567,7 @@ class Processor
 					'uri-id'      => $item['uri-id'],
 					'type'        => Post\Media::HTML,
 					'url'         => $embed->external->uri,
+					'preview'     => $embed->external->thumb ?? null,
 					'name'        => $embed->external->title,
 					'description' => $embed->external->description,
 				];
@@ -688,7 +693,7 @@ class Processor
 		return $restrict ? Item::CANT_REPLY : null;
 	}
 
-	private function fetchMissingPost(string $uri, int $uid, int $post_reason, int $causer, int $level, string $fallback = '', bool $always_fetch = false, int $Protocol = Conversation::PARCEL_JETSTREAM): string
+	public function fetchMissingPost(string $uri, int $uid, int $post_reason, int $causer, int $level, string $fallback = '', bool $always_fetch = false, int $Protocol = Conversation::PARCEL_JETSTREAM): string
 	{
 		$timestamp = microtime(true);
 		$stamp     = Strings::getRandomHex(30);
@@ -796,7 +801,7 @@ class Processor
 		return $uri;
 	}
 
-	private function getUriParts(string $uri): ?stdClass
+	public function getUriParts(string $uri): ?stdClass
 	{
 		$class = $this->getUriClass($uri);
 		if (empty($class)) {
@@ -814,7 +819,7 @@ class Processor
 		return $class;
 	}
 
-	private function getUriClass(string $uri): ?stdClass
+	public function getUriClass(string $uri): ?stdClass
 	{
 		if (empty($uri)) {
 			return null;
@@ -839,7 +844,7 @@ class Processor
 		return $class;
 	}
 
-	private function fetchUriId(string $uri, int $uid): string
+	public function fetchUriId(string $uri, int $uid): string
 	{
 		$reply = Post::selectFirst(['uri-id'], ['uri' => $uri, 'uid' => [$uid, 0]]);
 		if (!empty($reply['uri-id'])) {
@@ -854,16 +859,18 @@ class Processor
 		return 0;
 	}
 
-	private function getPostUids(string $uri): array
+	private function getPostUids(string $uri, bool $with_public_user): array
 	{
+		$condition = $with_public_user ? [] : ["`uid` != ?", 0];
+
 		$uids  = [];
-		$posts = Post::select(['uid'], ['uri' => $uri]);
+		$posts = Post::select(['uid'], DBA::mergeConditions(['uri' => $uri], $condition));
 		while ($post = Post::fetch($posts)) {
 			$uids[] = $post['uid'];
 		}
 		$this->db->close($posts);
 
-		$posts = Post::select(['uid'], ['extid' => $uri]);
+		$posts = Post::select(['uid'], DBA::mergeConditions(['extid' => $uri], $condition));
 		while ($post = Post::fetch($posts)) {
 			$uids[] = $post['uid'];
 		}
@@ -880,7 +887,7 @@ class Processor
 		return Post::exists(['extid' => $uri, 'uid' => $uids]);
 	}
 
-	private function getUri(stdClass $post): string
+	public function getUri(stdClass $post): string
 	{
 		if (empty($post->cid)) {
 			$this->logger->info('Invalid URI', ['post' => $post]);
@@ -889,7 +896,7 @@ class Processor
 		return $post->uri . ':' . $post->cid;
 	}
 
-	private function getPostUri(string $uri, int $uid): string
+	public function getPostUri(string $uri, int $uid): string
 	{
 		if (Post::exists(['uri' => $uri, 'uid' => [$uid, 0]])) {
 			$this->logger->debug('Post exists', ['uri' => $uri]);

src/Util/Images.php

@@ -366,10 +366,10 @@ class Images
 			return [];
 		}
 
-		$image = new Image($img_str, '', $url);
+		$image = new Image($img_str, '', $url, false);
 
 		if ($image->isValid()) {
-			$data['blurhash'] = $image->getBlurHash();
+			$data['blurhash'] = $image->getBlurHash($img_str);
 
 			if ($ocr) {
 				$media = ['img_str' => $img_str];

src/Util/ParseUrl.php

@@ -90,6 +90,7 @@ class ParseUrl
 	 * Search for cached embeddable data of an url otherwise fetch it
 	 *
 	 * @param string $url      The url of the page which should be scraped
+	 * @param string $mimetype Optional mimetype that had already been detected for this page
 	 *
 	 * @return array which contains needed data for embedding
 	 *    string 'url'      => The url of the parsed page
@@ -104,7 +105,7 @@ class ParseUrl
 	 * @see   ParseUrl::getSiteinfo() for more information about scraping
 	 * embeddable content
 	 */
-	public static function getSiteinfoCached(string $url): array
+	public static function getSiteinfoCached(string $url, string $mimetype = ''): array
 	{
 		if (empty($url)) {
 			return [
@@ -123,7 +124,7 @@ class ParseUrl
 			return $data;
 		}
 
-		$data = self::getSiteinfo($url);
+		$data = self::getSiteinfo($url, $mimetype);
 
 		$expires = $data['expires'];
 
@@ -156,6 +157,7 @@ class ParseUrl
 	 * \<meta name="description" content="An awesome description"\>
 	 *
 	 * @param string $url      The url of the page which should be scraped
+	 * @param string $mimetype Optional mimetype that had already been detected for this page
 	 * @param int    $count    Internal counter to avoid endless loops
 	 *
 	 * @return array which contains needed data for embedding
@@ -181,7 +183,7 @@ class ParseUrl
 	 * </body>
 	 * @endverbatim
 	 */
-	public static function getSiteinfo(string $url, int $count = 1): array
+	public static function getSiteinfo(string $url, string $mimetype = '', int $count = 1): array
 	{
 		if (empty($url)) {
 			return [
@@ -212,7 +214,11 @@ class ParseUrl
 			return $siteinfo;
 		}
 
+		if (!empty($mimetype)) {
+			$type = explode('/', current(explode(';', $mimetype)));
+		} else {
 			$type = self::getContentType($url);
+		}
 		Logger::info('Got content-type', ['content-type' => $type, 'url' => $url]);
 		if (!empty($type) && in_array($type[0], ['image', 'video', 'audio'])) {
 			$siteinfo['type'] = $type[0];
@@ -309,7 +315,7 @@ class ParseUrl
 					}
 				}
 				if ($content != '') {
-					$siteinfo = self::getSiteinfo($content, ++$count);
+					$siteinfo = self::getSiteinfo($content, $mimetype, ++$count);
 					return $siteinfo;
 				}
 			}

static/dbview.config.php

@@ -101,6 +101,7 @@ return [
 			"received" => ["post-thread-user", "received"],
 			"created" => ["post-thread-user", "created"],
 			"network" => ["post-thread-user", "network"],
+			"protocol" => ["post-user", "protocol"],
 			"restricted" => ["post-engagement", "language"],
 			"comments" => "0",
 			"activities" => "0",
@@ -221,6 +222,7 @@ return [
 			"received" => ["post-thread-user", "received"],
 			"created" => ["post-thread-user", "created"],
 			"network" => ["post-thread-user", "network"],
+			"protocol" => ["post-user", "protocol"],
 			"restricted" => ["post-searchindex", "language"],
 			"comments" => "0",
 			"activities" => "0",
@@ -475,6 +477,7 @@ return [
 			"global" => ["post-user", "global"],
 			"featured" => "EXISTS(SELECT `type` FROM `post-collection` WHERE `type` = 0 AND `uri-id` = `post-thread-user`.`uri-id`)",
 			"network" => ["post-thread-user", "network"],
+			"protocol" => ["post-user", "protocol"],
 			"vid" => ["post-origin", "vid"],
 			"psid" => ["post-thread-user", "psid"],
 			"verb" => "IF (`post-origin`.`vid` IS NULL, '', `verb`.`name`)",
@@ -858,6 +861,7 @@ return [
 			"global" => ["post-user", "global"],
 			"featured" => "EXISTS(SELECT `type` FROM `post-collection` WHERE `type` = 0 AND `uri-id` = `post-thread-user`.`uri-id`)",
 			"network" => ["post-thread-user", "network"],
+			"protocol" => ["post-user", "protocol"],
 			"vid" => ["post-user", "vid"],
 			"psid" => ["post-thread-user", "psid"],
 			"verb" => "IF (`post-user`.`vid` IS NULL, '', `verb`.`name`)",
@@ -1036,6 +1040,7 @@ return [
 			"global" => ["post", "global"],
 			"featured" => "EXISTS(SELECT `type` FROM `post-collection` WHERE `type` = 0 AND `uri-id` = `post`.`uri-id`)",
 			"network" => ["post", "network"],
+			"protocol" => "255",
 			"vid" => ["post", "vid"],
 			"verb" => "IF (`post`.`vid` IS NULL, '', `verb`.`name`)",
 			"title" => ["post-content", "title"],
@@ -1188,6 +1193,7 @@ return [
 			"global" => ["post", "global"],
 			"featured" => "EXISTS(SELECT `type` FROM `post-collection` WHERE `type` = 0 AND `uri-id` = `post-thread`.`uri-id`)",
 			"network" => ["post-thread", "network"],
+			"protocol" => "255",
 			"vid" => ["post", "vid"],
 			"verb" => "IF (`post`.`vid` IS NULL, '', `verb`.`name`)",
 			"title" => ["post-content", "title"],
@@ -1381,6 +1387,7 @@ return [
 			"starred" => ["post-thread-user", "starred"],
 			"mention" => ["post-thread-user", "mention"],
 			"network" => ["post-thread-user", "network"],
+			"protocol" => ["post-user", "protocol"],
 			"contact-id" => ["post-thread-user", "contact-id"],
 			"contact-type" => ["ownercontact", "contact-type"],
 		],
@@ -1407,6 +1414,7 @@ return [
 			"starred" => ["post-thread-user", "starred"],
 			"mention" => ["post-thread-user", "mention"],
 			"network" => ["post-thread-user", "network"],
+			"protocol" => ["post-user", "protocol"],
 			"contact-id" => ["post-thread-user", "contact-id"],
 			"contact-type" => ["ownercontact", "contact-type"],
 		],
@@ -1769,6 +1777,7 @@ return [
 			"gravity" => ["post-user", "gravity"],
 			"received" => ["post-user", "received"],
 			"network" => ["post-user", "network"],
+			"protocol" => ["post-user", "protocol"],
 			"author-id" => ["post-user", "author-id"],
 			"name" => ["tag", "name"],
 		],