New struture for post related links

database.sql

@@ -1,6 +1,6 @@
 -- ------------------------------------------
--- Friendica 2021.06-rc (Siberian Iris)
--- DB_UPDATE_VERSION 1424
+-- Friendica 2021.09-dev (Siberian Iris)
+-- DB_UPDATE_VERSION 1425
 -- ------------------------------------------
 
 
@@ -1103,6 +1103,19 @@ CREATE TABLE IF NOT EXISTS `post-delivery-data` (
 	FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE
 ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Delivery data for items';
 
+--
+-- TABLE post-link
+--
+CREATE TABLE IF NOT EXISTS `post-link` (
+	`id` int unsigned NOT NULL auto_increment COMMENT 'sequential ID',
+	`uri-id` int unsigned NOT NULL COMMENT 'Id of the item-uri table entry that contains the item uri',
+	`url` varbinary(511) NOT NULL COMMENT 'External URL',
+	`mimetype` varchar(60) COMMENT '',
+	 PRIMARY KEY(`id`),
+	 UNIQUE INDEX `uri-id-url` (`uri-id`,`url`),
+	FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Post related external links';
+
 --
 -- TABLE post-media
 --

doc/database.md

@@ -53,6 +53,7 @@ Database Tables
 | [post-category](help/database/db_post-category) | post relation to categories |
 | [post-content](help/database/db_post-content) | Content for all posts |
 | [post-delivery-data](help/database/db_post-delivery-data) | Delivery data for items |
+| [post-link](help/database/db_post-link) | Post related external links |
 | [post-media](help/database/db_post-media) | Attached media |
 | [post-tag](help/database/db_post-tag) | post relation to tags |
 | [post-thread](help/database/db_post-thread) | Thread related data |

doc/database/db_post-link.md

@@ -0,0 +1,31 @@
+Table post-link
+===========
+
+Post related external links
+
+Fields
+------
+
+| Field    | Description                                               | Type           | Null | Key | Default | Extra          |
+| -------- | --------------------------------------------------------- | -------------- | ---- | --- | ------- | -------------- |
+| id       | sequential ID                                             | int unsigned   | NO   | PRI | NULL    | auto_increment |
+| uri-id   | Id of the item-uri table entry that contains the item uri | int unsigned   | NO   |     | NULL    |                |
+| url      | External URL                                              | varbinary(511) | NO   |     | NULL    |                |
+| mimetype |                                                           | varchar(60)    | YES  |     | NULL    |                |
+
+Indexes
+------------
+
+| Name       | Fields              |
+| ---------- | ------------------- |
+| PRIMARY    | id                  |
+| uri-id-url | UNIQUE, uri-id, url |
+
+Foreign Keys
+------------
+
+| Field | Target Table | Target Field |
+|-------|--------------|--------------|
+| uri-id | [item-uri](help/database/db_item-uri) | id |
+
+Return to [database documentation](help/database)

src/Model/Item.php

@@ -2755,6 +2755,7 @@ class Item
 		}
 		$attachments = Post\Media::splitAttachments($item['uri-id'], $item['guid'] ?? '', $shared_links);
 		$item['body'] = self::replaceVisualAttachments($attachments, $item['body'] ?? '');
+		$item['body'] = Post\Link::insertFromBody($item['uri-id'], $item['body']);
 
 		$item['body'] = preg_replace("/\s*\[attachment .*?\].*?\[\/attachment\]\s*/ism", "\n", $item['body']);
 		self::putInCache($item);
@@ -2780,13 +2781,13 @@ class Item
 
 		if (!empty($shared_attachments)) {
 			$s = self::addVisualAttachments($shared_attachments, $item, $s, true);
-			$s = self::addLinkAttachment($shared_attachments, $body, $s, true, []);
+			$s = self::addLinkAttachment($shared_uri_id, $shared_attachments, $body, $s, true, []);
 			$s = self::addNonVisualAttachments($shared_attachments, $item, $s, true);
 			$body = preg_replace("/\s*\[share .*?\].*?\[\/share\]\s*/ism", '', $body);
 		}
 
 		$s = self::addVisualAttachments($attachments, $item, $s, false);
-		$s = self::addLinkAttachment($attachments, $body, $s, false, $shared_links);
+		$s = self::addLinkAttachment($item['uri-id'], $attachments, $body, $s, false, $shared_links);
 		$s = self::addNonVisualAttachments($attachments, $item, $s, false);
 
 		// Map.
@@ -2967,7 +2968,7 @@ class Item
 	 * @param array  $ignore_links A list of URLs to ignore
 	 * @return string modified content
 	 */
-	private static function addLinkAttachment(array $attachments, string $body, string $content, bool $shared, array $ignore_links)
+	private static function addLinkAttachment(int $uriid, array $attachments, string $body, string $content, bool $shared, array $ignore_links)
 	{
 		$stamp1 = microtime(true);
 		// @ToDo Check only for audio and video
@@ -3052,6 +3053,14 @@ class Item
 					}
 				}
 
+				if (!empty($data['image'])) {
+					$data['image']   = Post\Link::getByLink($uriid, $data['image']);
+				}
+
+				if (!empty($data['preview'])) {
+					$data['preview'] = Post\Link::getByLink($uriid, $data['preview']);
+				}
+
 				// @todo Use a template
 				$rendered = BBCode::convertAttachment('', BBCode::INTERNAL, false, $data);
 			} elseif (!self::containsLink($content, $data['url'], Post\Media::HTML)) {

src/Model/Post/Link.php

@@ -0,0 +1,122 @@
+<?php
+/**
+ * @copyright Copyright (C) 2010-2021, the Friendica project
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace Friendica\Model\Post;
+
+use Friendica\Core\Logger;
+use Friendica\Database\DBA;
+use Friendica\DI;
+use Friendica\Util\Proxy;
+
+/**
+ * Class Link
+ *
+ * This Model class handles post related external links
+ */
+class Link
+{
+	public static function getByLink(int $uri_id, string $url, $size = '')
+	{
+		if (empty($url) || Proxy::isLocalImage($url)) {
+			return $url;
+		}
+
+		$link = DBA::selectFirst('post-link', ['id'], ['uri-id' => $uri_id, 'url' => $url]);
+		if (!empty($link['id'])) {
+			$id = $link['id'];
+			Logger::info('Found', ['id' => $id, 'url' => $url]);
+		} else {
+			$mime = self::fetchMimeType($url);
+		
+			DBA::insert('post-link', ['uri-id' => $uri_id, 'url' => $url, 'mimetype' => $mime]);
+			$id = DBA::lastInsertId();
+			Logger::info('Inserted', ['id' => $id, 'url' => $url]);
+		}
+
+		if (empty($id)) {
+			return $url;
+		}
+
+		$url = DI::baseUrl() . '/photo/link/';
+		switch ($size) {
+			case Proxy::SIZE_MICRO:
+				$url .= Proxy::PIXEL_MICRO . '/';
+				break;
+			case Proxy::SIZE_THUMB:
+				$url .= Proxy::PIXEL_THUMB . '/';
+				break;
+			case Proxy::SIZE_SMALL:
+				$url .= Proxy::PIXEL_SMALL . '/';
+				break;
+			case Proxy::SIZE_MEDIUM:
+				$url .= Proxy::PIXEL_MEDIUM . '/';
+				break;
+			case Proxy::SIZE_LARGE:
+				$url .= Proxy::PIXEL_LARGE . '/';
+				break;
+		}
+		return $url . $id;
+	}
+
+	private static function fetchMimeType(string $url)
+	{
+		$timeout = DI::config()->get('system', 'xrd_timeout');
+		$curlResult = DI::httpRequest()->head($url, ['timeout' => $timeout]);
+		if ($curlResult->isSuccess()) {
+			if (empty($media['mimetype'])) {
+				return $curlResult->getHeader('Content-Type');
+			}
+		}
+		return '';
+	}
+
+	/**
+	 * Add external links and replace them in the body
+	 *
+	 * @param integer $uriid
+	 * @param string $body
+	 * @return string Body with replaced links
+	 */
+	public static function insertFromBody(int $uriid, string $body)
+	{
+		if (preg_match_all("/\[img\=([0-9]*)x([0-9]*)\](.*?)\[\/img\]/ism", $body, $pictures, PREG_SET_ORDER)) {
+			foreach ($pictures as $picture) {
+				$body = str_replace($picture[3], self::getByLink($uriid, $picture[3]), $body);
+			}
+		}
+
+		if (preg_match_all("/\[img=([^\[\]]*)\]([^\[\]]*)\[\/img\]/Usi", $body, $pictures, PREG_SET_ORDER)) {
+			foreach ($pictures as $picture) {
+				if (parse_url($picture[1], PHP_URL_SCHEME)) {
+					$body = str_replace($picture[1], self::getByLink($uriid, $picture[1]), $body);
+				}
+			}
+		}
+
+		if (preg_match_all("/\[img\]([^\[\]]*)\[\/img\]/ism", $body, $pictures, PREG_SET_ORDER)) {
+			foreach ($pictures as $picture) {
+				$body = str_replace($picture[1], self::getByLink($uriid, $picture[1]), $body);
+			}
+		}
+
+		return trim($body);
+	}
+}

src/Module/Photo.php

@@ -176,7 +176,7 @@ class Photo extends BaseModule
 	{
 		switch($type) {
 			case "preview":
-				$media = DBA::selectFirst('post-media', ['preview', 'url', 'type', 'uri-id'], ['id' => $uid]);
+				$media = DBA::selectFirst('post-media', ['preview', 'url', 'mimetype', 'type', 'uri-id'], ['id' => $uid]);
 				if (empty($media)) {
 					return false;
 				}
@@ -194,9 +194,9 @@ class Photo extends BaseModule
 					return MPhoto::getPhoto($matches[1], $matches[2]);
 				}
 		
-				return MPhoto::createPhotoForExternalResource($url, (int)local_user());
+				return MPhoto::createPhotoForExternalResource($url, (int)local_user(), $media['mimetype']);
 			case "media":
-				$media = DBA::selectFirst('post-media', ['url', 'uri-id'], ['id' => $uid, 'type' => Post\Media::IMAGE]);
+				$media = DBA::selectFirst('post-media', ['url', 'mimetype', 'uri-id'], ['id' => $uid, 'type' => Post\Media::IMAGE]);
 				if (empty($media)) {
 					return false;
 				}
@@ -205,7 +205,14 @@ class Photo extends BaseModule
 					return MPhoto::getPhoto($matches[1], $matches[2]);
 				}
 
-				return MPhoto::createPhotoForExternalResource($media['url'], (int)local_user());
+				return MPhoto::createPhotoForExternalResource($media['url'], (int)local_user(), $media['mimetype']);
+			case "link":
+				$link = DBA::selectFirst('post-link', ['url', 'mimetype'], ['id' => $uid]);
+				if (empty($link)) {
+					return false;
+				}
+
+				return MPhoto::createPhotoForExternalResource($link['url'], (int)local_user(), $link['mimetype']);
 			case "contact":
 				$contact = Contact::getById($uid, ['uid', 'url', 'avatar', 'photo', 'xmpp', 'addr']);
 				if (empty($contact)) {

static/dbstructure.config.php

@@ -55,7 +55,7 @@
 use Friendica\Database\DBA;
 
 if (!defined('DB_UPDATE_VERSION')) {
-	define('DB_UPDATE_VERSION', 1424);
+	define('DB_UPDATE_VERSION', 1425);
 }
 
 return [
@@ -1151,6 +1151,19 @@ return [
 			"PRIMARY" => ["uri-id"],
 		]
 	],
+	"post-link" => [
+		"comment" => "Post related external links",
+		"fields" => [
+			"id" => ["type" => "int unsigned", "not null" => "1", "extra" => "auto_increment", "primary" => "1", "comment" => "sequential ID"],
+			"uri-id" => ["type" => "int unsigned", "not null" => "1", "foreign" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the item uri"],
+			"url" => ["type" => "varbinary(511)", "not null" => "1", "comment" => "External URL"],
+			"mimetype" => ["type" => "varchar(60)", "comment" => ""],
+		],
+		"indexes" => [
+			"PRIMARY" => ["id"],
+			"uri-id-url" => ["UNIQUE", "uri-id", "url"],
+		]
+	],
 	"post-media" => [
 		"comment" => "Attached media",
 		"fields" => [