Merge remote-tracking branch 'upstream/develop' into failed

mod/ping.php

@@ -136,13 +136,9 @@ function ping_init(App $a)
 
 		$notifs = ping_get_notifications(local_user());
 
-		$condition = ["`unseen` AND `uid` = ? AND `contact-id` != ? AND (`vid` != ? OR `vid` IS NULL)",
+		$condition = ["`unseen` AND `uid` = ? AND NOT `origin` AND (`vid` != ? OR `vid` IS NULL)",
-			local_user(), local_user(), Verb::getID(Activity::FOLLOW)];
+			local_user(), Verb::getID(Activity::FOLLOW)];
-		$fields = ['id', 'parent', 'verb', 'author-name', 'unseen', 'author-link', 'author-avatar', 'contact-avatar',
+		$items = Item::selectForUser(local_user(), ['wall', 'uid', 'uri-id'], $condition);
-			'network', 'created', 'object', 'parent-author-name', 'parent-author-link', 'parent-guid', 'wall', 'activity'];
-		$params = ['order' => ['received' => true]];
-		$items = Item::selectForUser(local_user(), $fields, $condition, $params);
-
 		if (DBA::isResult($items)) {
 			$items_unseen = Item::inArray($items);
 			$arr = ['items' => $items_unseen];
@@ -156,6 +152,7 @@ function ping_init(App $a)
 				}
 			}
 		}
+		DBA::close($items);
 
 		if ($network_count) {
 			// Find out how unseen network posts are spread across groups

src/Content/PageInfo.php

@@ -40,7 +40,7 @@ class PageInfo
 	 * @return string
 	 * @throws HTTPException\InternalServerErrorException
 	 */
-	public static function appendToBody(string $body, bool $searchNakedUrls = false, bool $no_photos = false)
+	public static function searchAndAppendToBody(string $body, bool $searchNakedUrls = false, bool $no_photos = false)
 	{
 		Logger::info('add_page_info_to_body: fetch page info for body', ['body' => $body]);
 
@@ -49,14 +49,34 @@ class PageInfo
 			return $body;
 		}
 
-		$footer = self::getFooterFromUrl($url, $no_photos);
+		$data = self::queryUrl($url);
-		if (!$footer) {
+		if (!$data) {
 			return $body;
 		}
 
-		$body = self::stripTrailingUrlFromBody($body, $url);
+		return self::appendDataToBody($body, $data, $no_photos);
+	}
 
+	/**
+	 * @param string $body
+	 * @param array  $data
+	 * @param bool   $no_photos
+	 * @return string
+	 * @throws HTTPException\InternalServerErrorException
+	 */
+	public static function appendDataToBody(string $body, array $data, bool $no_photos = false)
+	{
+		// Only one [attachment] tag per body is allowed
+		$existingAttachmentPos = strpos($body, '[attachment');
+		if ($existingAttachmentPos !== false) {
+			$linkTitle = $data['title'] ?: $data['url'];
+			// Additional link attachments are prepended before the existing [attachment] tag
+			$body = substr_replace($body, "\n[bookmark=" . $data['url'] . ']' . $linkTitle . "[/bookmark]\n", $existingAttachmentPos, 0);
+		} else {
+			$footer = PageInfo::getFooterFromData($data, $no_photos);
+			$body = self::stripTrailingUrlFromBody($body, $data['url']);
 			$body .= "\n" . $footer;
+		}
 
 		return $body;
 	}
@@ -252,22 +272,35 @@ class PageInfo
 
 	/**
 	 * Remove the provided URL from the body if it is at the end of it.
-	 * Keep the link label if it isn't the full URL.
+	 * Keep the link label if it isn't the full URL or a shortened version of it.
 	 *
 	 * @param string $body
 	 * @param string $url
-	 * @return string|string[]|null
+	 * @return string
 	 */
 	protected static function stripTrailingUrlFromBody(string $body, string $url)
 	{
 		$quotedUrl = preg_quote($url, '#');
-		$body = preg_replace("#(?:
+		$body = preg_replace_callback("#(?:
 			\[url]$quotedUrl\[/url]|
 			\[url=$quotedUrl]$quotedUrl\[/url]|
 			\[url=$quotedUrl]([^[]*?)\[/url]|
 			$quotedUrl
-		)$#isx", '$1', $body);
+		)$#isx", function ($match) use ($url) {
+			// Stripping URLs with no label
+			if (!isset($match[1])) {
+				return '';
+			}
 
-		return $body;
+			// Stripping link labels that include a shortened version of the URL
+			if (strpos($url, trim($match[1], '.…')) !== false) {
+				return '';
+			}
+
+			// Keep all other labels
+			return $match[1];
+		}, $body);
+
+		return rtrim($body);
 	}
 }

src/Database/Database.php

@@ -343,7 +343,7 @@ class Database
 				                                                                      $row['key'] . "\t" . $row['rows'] . "\t" . $row['Extra'] . "\t" .
 				                                                                      basename($backtrace[1]["file"]) . "\t" .
 				                                                                      $backtrace[1]["line"] . "\t" . $backtrace[2]["function"] . "\t" .
-				                                                                      substr($query, 0, 2000) . "\n", FILE_APPEND);
+				                                                                      substr($query, 0, 4000) . "\n", FILE_APPEND);
 			}
 		}
 	}
@@ -712,7 +712,7 @@ class Database
 				@file_put_contents($this->configCache->get('system', 'db_log'), DateTimeFormat::utcNow() . "\t" . $duration . "\t" .
 				                                                                basename($backtrace[1]["file"]) . "\t" .
 				                                                                $backtrace[1]["line"] . "\t" . $backtrace[2]["function"] . "\t" .
-				                                                                substr($this->replaceParameters($sql, $args), 0, 2000) . "\n", FILE_APPEND);
+				                                                                substr($this->replaceParameters($sql, $args), 0, 4000) . "\n", FILE_APPEND);
 			}
 		}
 		return $retval;

src/Model/Item.php

@@ -2093,7 +2093,7 @@ class Item
 		DBA::close($contacts);
 
 		if (!empty($owner['alias'])) {
-			$condition = ['url' => $owner['alias'], 'rel' => [Contact::SHARING, Contact::FRIEND]];
+			$condition = ['nurl' => Strings::normaliseLink($owner['alias']), 'rel' => [Contact::SHARING, Contact::FRIEND]];
 			$contacts = DBA::select('contact', ['uid'], $condition);
 			while ($contact = DBA::fetch($contacts)) {
 				if ($contact['uid'] == 0) {

src/Module/Debug/Babel.php

@@ -24,9 +24,12 @@ namespace Friendica\Module\Debug;
 use Friendica\BaseModule;
 use Friendica\Content\PageInfo;
 use Friendica\Content\Text;
+use Friendica\Core\Protocol;
 use Friendica\Core\Renderer;
 use Friendica\DI;
+use Friendica\Model\Conversation;
 use Friendica\Model\Item;
+use Friendica\Protocol\Activity;
 use Friendica\Model\Tag;
 use Friendica\Util\XML;
 
@@ -115,7 +118,7 @@ class Babel extends BaseModule
 						'content' => visible_whitespace(var_export($tags, true)),
 					];
 
-					$body2 = PageInfo::appendToBody($bbcode, true);
+					$body2 = PageInfo::searchAndAppendToBody($bbcode, true);
 					$results[] = [
 						'title'   => DI::l10n()->t('PageInfo::appendToBody'),
 						'content' => visible_whitespace($body2)
@@ -215,6 +218,60 @@ class Babel extends BaseModule
 						'title'   => DI::l10n()->t('HTML::toPlaintext (compact)'),
 						'content' => visible_whitespace($text),
 					];
+					break;
+				case 'twitter':
+					$json = trim($_REQUEST['text']);
+
+					$status = json_decode($json);
+
+					$results[] = [
+						'title'   => DI::l10n()->t('Decoded post'),
+						'content' => visible_whitespace(var_export($status, true)),
+					];
+
+					$postarray = [];
+					$postarray['object-type'] = Activity\ObjectType::NOTE;
+
+					if (!empty($status->full_text)) {
+						$postarray['body'] = $status->full_text;
+					} else {
+						$postarray['body'] = $status->text;
+					}
+
+					// When the post contains links then use the correct object type
+					if (count($status->entities->urls) > 0) {
+						$postarray['object-type'] = Activity\ObjectType::BOOKMARK;
+					}
+
+					if (file_exists('addon/twitter/twitter.php')) {
+						require_once 'addon/twitter/twitter.php';
+
+						$picture = \twitter_media_entities($status, $postarray);
+
+						$results[] = [
+							'title'   => DI::l10n()->t('Post array before expand entities'),
+							'content' => visible_whitespace(var_export($postarray, true)),
+						];
+
+						$converted = \twitter_expand_entities($postarray['body'], $status, $picture);
+
+						$results[] = [
+							'title'   => DI::l10n()->t('Post converted'),
+							'content' => visible_whitespace(var_export($converted, true)),
+						];
+
+						$results[] = [
+							'title'   => DI::l10n()->t('Converted body'),
+							'content' => visible_whitespace($converted['body']),
+						];
+					} else {
+						$results[] = [
+							'title'   => DI::l10n()->t('Error'),
+							'content' => DI::l10n()->t('Twitter addon is absent from the addon/ folder.'),
+						];
+					}
+
+					break;
 			}
 		}
 
@@ -225,6 +282,8 @@ class Babel extends BaseModule
 			'$type_diaspora' => ['type', DI::l10n()->t('Diaspora'), 'diaspora', '', (($_REQUEST['type'] ?? '') ?: 'bbcode') == 'diaspora'],
 			'$type_markdown' => ['type', DI::l10n()->t('Markdown'), 'markdown', '', (($_REQUEST['type'] ?? '') ?: 'bbcode') == 'markdown'],
 			'$type_html'     => ['type', DI::l10n()->t('HTML'), 'html', '', (($_REQUEST['type'] ?? '') ?: 'bbcode') == 'html'],
+			'$flag_twitter'  => file_exists('addon/twitter/twitter.php'),
+			'$type_twitter'  => ['type', DI::l10n()->t('Twitter Source'), 'twitter', '', (($_REQUEST['type'] ?? '') ?: 'bbcode') == 'twitter'],
 			'$results'       => $results
 		]);
 

src/Protocol/ActivityPub/Processor.php

@@ -21,6 +21,7 @@
 
 namespace Friendica\Protocol\ActivityPub;
 
+use Friendica\Content\PageInfo;
 use Friendica\Content\Text\BBCode;
 use Friendica\Content\Text\HTML;
 use Friendica\Core\Logger;
@@ -96,18 +97,16 @@ class Processor
 		foreach ($activity['attachments'] as $attach) {
 			switch ($attach['type']) {
 				case 'link':
-					// Only one [attachment] tag is allowed
+					$data = [
-					$existingAttachmentPos = strpos($item['body'], '[attachment');
+						'url'      => $attach['url'],
-					if ($existingAttachmentPos !== false) {
+						'type'     => $attach['type'],
-						$linkTitle = $attach['title'] ?: $attach['url'];
+						'title'    => $attach['title'] ?? '',
-						// Additional link attachments are prepended before the existing [attachment] tag
+						'text'     => $attach['desc']  ?? '',
-						$item['body'] = substr_replace($item['body'], "\n[bookmark=" . $attach['url'] . ']' . $linkTitle . "[/bookmark]\n", $existingAttachmentPos, 0);
+						'image'    => $attach['image'] ?? '',
-					} else {
+						'images'   => [],
-						// Strip the link preview URL from the end of the body if any
+						'keywords' => [],
-						$quotedUrl = preg_quote($attach['url'], '#');
+					];
-						$item['body'] = preg_replace("#\s*(?:\[bookmark={$quotedUrl}].+?\[/bookmark]|\[url={$quotedUrl}].+?\[/url]|\[url]{$quotedUrl}\[/url]|{$quotedUrl})\s*$#", '', $item['body']);
+					$item['body'] = PageInfo::appendDataToBody($item['body'], $data);
-						$item['body'] .= "\n[attachment type='link' url='" . $attach['url'] . "' title='" . htmlspecialchars($attach['title'] ?? '', ENT_QUOTES) . "' image='" . ($attach['image'] ?? '') . "']" . ($attach['desc'] ?? '') . '[/attachment]';
-					}
 					break;
 				default:
 					$filetype = strtolower(substr($attach['mediaType'], 0, strpos($attach['mediaType'], '/')));

src/Protocol/Diaspora.php

@@ -2622,7 +2622,7 @@ class Diaspora
 				$item["body"] = self::replacePeopleGuid($item["body"], $item["author-link"]);
 
 				// Add OEmbed and other information to the body
-				$item["body"] = PageInfo::appendToBody($item["body"], false, true);
+				$item["body"] = PageInfo::searchAndAppendToBody($item["body"], false, true);
 
 				return $item;
 			} else {
@@ -2986,7 +2986,7 @@ class Diaspora
 
 			// Add OEmbed and other information to the body
 			if (!self::isHubzilla($contact["url"])) {
-				$body = PageInfo::appendToBody($body, false, true);
+				$body = PageInfo::searchAndAppendToBody($body, false, true);
 			}
 		}
 

src/Protocol/OStatus.php

@@ -698,7 +698,7 @@ class OStatus
 
 		// Only add additional data when there is no picture in the post
 		if (!strstr($item["body"], '[/img]')) {
-			$item["body"] = PageInfo::appendToBody($item["body"]);
+			$item["body"] = PageInfo::searchAndAppendToBody($item["body"]);
 		}
 
 		Tag::storeFromBody($item['uri-id'], $item['body']);

src/Util/ParseUrl.php

@@ -57,12 +57,11 @@ class ParseUrl
 	 * @return array which contains needed data for embedding
 	 *    string 'url'      => The url of the parsed page
 	 *    string 'type'     => Content type
-	 *    string 'title' => The title of the content
+	 *    string 'title'    => (optional) The title of the content
-	 *    string 'text' => The description for the content
+	 *    string 'text'     => (optional) The description for the content
-	 *    string 'image' => A preview image of the content (only available
+	 *    string 'image'    => (optional) A preview image of the content (only available if $no_geuessing = false)
-	 *                if $no_geuessing = false
+	 *    array  'images'   => (optional) Array of preview pictures
-	 *    array'images' = Array of preview pictures
+	 *    string 'keywords' => (optional) The tags which belong to the content
-	 *    string 'keywords' => The tags which belong to the content
 	 *
 	 * @throws \Friendica\Network\HTTPException\InternalServerErrorException
 	 * @see   ParseUrl::getSiteinfo() for more information about scraping
@@ -117,12 +116,11 @@ class ParseUrl
 	 * @return array which contains needed data for embedding
 	 *    string 'url'      => The url of the parsed page
 	 *    string 'type'     => Content type
-	 *    string 'title' => The title of the content
+	 *    string 'title'    => (optional) The title of the content
-	 *    string 'text' => The description for the content
+	 *    string 'text'     => (optional) The description for the content
-	 *    string 'image' => A preview image of the content (only available
+	 *    string 'image'    => (optional) A preview image of the content (only available if $no_guessing = false)
-	 *                if $no_geuessing = false
+	 *    array  'images'   => (optional) Array of preview pictures
-	 *    array'images' = Array of preview pictures
+	 *    string 'keywords' => (optional) The tags which belong to the content
-	 *    string 'keywords' => The tags which belong to the content
 	 *
 	 * @throws \Friendica\Network\HTTPException\InternalServerErrorException
 	 * @todo  https://developers.google.com/+/plugins/snippet/
@@ -140,28 +138,27 @@ class ParseUrl
 	 */
 	public static function getSiteinfo($url, $no_guessing = false, $do_oembed = true, $count = 1)
 	{
-		$siteinfo = [];
-
 		// Check if the URL does contain a scheme
 		$scheme = parse_url($url, PHP_URL_SCHEME);
 
 		if ($scheme == '') {
-			$url = 'http://' . trim($url, '/');
+			$url = 'http://' . ltrim($url, '/');
 		}
 
+		$url = trim($url, "'\"");
+
+		$url = Network::stripTrackingQueryParams($url);
+
+		$siteinfo = [
+			'url' => $url,
+			'type' => 'link',
+		];
+
 		if ($count > 10) {
 			Logger::log('Endless loop detected for ' . $url, Logger::DEBUG);
 			return $siteinfo;
 		}
 
-		$url = trim($url, "'");
-		$url = trim($url, '"');
-
-		$url = Network::stripTrackingQueryParams($url);
-
-		$siteinfo['url'] = $url;
-		$siteinfo['type'] = 'link';
-
 		$curlResult = Network::curl($url);
 		if (!$curlResult->isSuccess()) {
 			return $siteinfo;

src/Worker/Cron.php

@@ -160,7 +160,6 @@ class Cron
 		$condition = ["`network` IN (?, ?, ?, ?) AND `uid` = ? AND NOT `self` AND `last-update` < ?",
 			Protocol::ACTIVITYPUB, Protocol::DFRN, Protocol::DIASPORA, Protocol::OSTATUS, 0, $last_updated];
 
-		$total = DBA::count('contact', $condition);
 		$oldest_date = '';
 		$oldest_id = '';
 		$contacts = DBA::select('contact', ['id', 'last-update'], $condition, ['limit' => 100, 'order' => ['last-update']]);
@@ -172,7 +171,7 @@ class Cron
 			Worker::add(PRIORITY_LOW, "UpdateContact", $contact['id'], 'force');
 			++$count;
 		}
-		Logger::info('Initiated update for public contacts', ['interval' => $count, 'total' => $total, 'id' => $oldest_id, 'oldest' => $oldest_date]);
+		Logger::info('Initiated update for public contacts', ['interval' => $count, 'id' => $oldest_id, 'oldest' => $oldest_date]);
 		DBA::close($contacts);
 	}
 

static/dbstructure.config.php

@@ -54,7 +54,7 @@
 use Friendica\Database\DBA;
 
 if (!defined('DB_UPDATE_VERSION')) {
-	define('DB_UPDATE_VERSION', 1355);
+	define('DB_UPDATE_VERSION', 1356);
 }
 
 return [
@@ -200,6 +200,7 @@ return [
 			"attag_uid" => ["attag(32)", "uid"],
 			"dfrn-id" => ["dfrn-id(64)"],
 			"issued-id" => ["issued-id(64)"],
+			"network_uid_lastupdate" => ["network", "uid", "last-update"],
 			"gsid" => ["gsid"]
 		]
 	],
@@ -321,6 +322,7 @@ return [
 			"addr" => ["addr(32)"],
 			"alias" => ["alias(190)"],
 			"followers" => ["followers(190)"],
+			"baseurl" => ["baseurl(190)"],
 			"gsid" => ["gsid"]
 		]
 	],

tests/src/Content/PageInfoTest.php

@@ -108,6 +108,21 @@ class PageInfoTest extends MockedTest
 				'body' => '[url=https://example.com]link label[/url]',
 				'url' => 'https://example.com',
 			],
+			'task-8797-shortened-link-label' => [
+				'expected' => 'content',
+				'body' => 'content [url=https://example.com/page]example.com/[/url]',
+				'url' => 'https://example.com/page',
+			],
+			'task-8797-shortened-link-label-ellipsis' => [
+				'expected' => 'content',
+				'body' => 'content [url=https://example.com/page]example.com…[/url]',
+				'url' => 'https://example.com/page',
+			],
+			'task-8797-shortened-link-label-dots' => [
+				'expected' => 'content',
+				'body' => 'content [url=https://example.com/page]example.com...[/url]',
+				'url' => 'https://example.com/page',
+			],
 		];
 	}
 

view/templates/babel.tpl

@@ -9,6 +9,9 @@
 			{{include file="field_radio.tpl" field=$type_diaspora}}
 			{{include file="field_radio.tpl" field=$type_markdown}}
 			{{include file="field_radio.tpl" field=$type_html}}
+		{{if $flag_twitter}}
+			{{include file="field_radio.tpl" field=$type_twitter}}
+		{{/if}}
 		</div>
 		<p><button type="submit" class="btn btn-primary">Submit</button></p>
 	</div>