Friendica/friendica-github
1
0
Fork 0
mirror of https://github.com/friendica/friendica synced 2024-11-17 16:23:46 +00:00
Code Issues Projects Releases Packages Wiki Activity

Delete unused media attachments

Browse source
This commit is contained in:
Michael 2024-08-24 08:37:56 +00:00
parent af28b82858
commit 425f23bbdc
17 changed files with 459 additions and 372 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
database.sql
View file

@ -1,6 +1,6 @@
-- ------------------------------------------
-- Friendica 2024.09-dev (Yellow Archangel)
-- DB_UPDATE_VERSION 1572
-- DB_UPDATE_VERSION 1573
-- ------------------------------------------
@ -1424,6 +1424,7 @@ CREATE TABLE IF NOT EXISTS `post-media` (
`uri-id` int unsigned NOT NULL COMMENT 'Id of the item-uri table entry that contains the item uri',
`url` varbinary(1024) NOT NULL COMMENT 'Media URL',
`media-uri-id` int unsigned COMMENT 'Id of the item-uri table entry that contains the activities uri-id',
`attach-id` int unsigned COMMENT 'In case of a local attachment, this field is filled with the id in the attach table',
`type` tinyint unsigned NOT NULL DEFAULT 0 COMMENT 'Media type',
`mimetype` varchar(60) COMMENT '',
`height` smallint unsigned COMMENT 'Height of the media',
@ -1445,8 +1446,10 @@ CREATE TABLE IF NOT EXISTS `post-media` (
UNIQUE INDEX `uri-id-url` (`uri-id`,`url`(512)),
INDEX `uri-id-id` (`uri-id`,`id`),
INDEX `media-uri-id` (`media-uri-id`),
INDEX `attach-id` (`attach-id`),
FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE,
FOREIGN KEY (`media-uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE
FOREIGN KEY (`media-uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE,
FOREIGN KEY (`attach-id`) REFERENCES `attach` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Attached media';
--

49
doc/database/db_post-media.md
View file

@ -6,29 +6,30 @@ Attached media
Fields
------
| Field | Description | Type | Null | Key | Default | Extra |
| --------------- | ------------------------------------------------------------------ | ----------------- | ---- | --- | ------- | -------------- |
| id | sequential ID | int unsigned | NO | PRI | NULL | auto_increment |
| uri-id | Id of the item-uri table entry that contains the item uri | int unsigned | NO | | NULL | |
| url | Media URL | varbinary(1024) | NO | | NULL | |
| media-uri-id | Id of the item-uri table entry that contains the activities uri-id | int unsigned | YES | | NULL | |
| type | Media type | tinyint unsigned | NO | | 0 | |
| mimetype | | varchar(60) | YES | | NULL | |
| height | Height of the media | smallint unsigned | YES | | NULL | |
| width | Width of the media | smallint unsigned | YES | | NULL | |
| size | Media size | bigint unsigned | YES | | NULL | |
| blurhash | BlurHash representation of the image | varbinary(255) | YES | | NULL | |
| preview | Preview URL | varbinary(512) | YES | | NULL | |
| preview-height | Height of the preview picture | smallint unsigned | YES | | NULL | |
| preview-width | Width of the preview picture | smallint unsigned | YES | | NULL | |
| description | | text | YES | | NULL | |
| name | Name of the media | varchar(255) | YES | | NULL | |
| author-url | URL of the author of the media | varbinary(383) | YES | | NULL | |
| author-name | Name of the author of the media | varchar(255) | YES | | NULL | |
| author-image | Image of the author of the media | varbinary(383) | YES | | NULL | |
| publisher-url | URL of the publisher of the media | varbinary(383) | YES | | NULL | |
| publisher-name | Name of the publisher of the media | varchar(255) | YES | | NULL | |
| publisher-image | Image of the publisher of the media | varbinary(383) | YES | | NULL | |
| Field | Description | Type | Null | Key | Default | Extra |
| --------------- | ----------------------------------------------------------------------------------- | ----------------- | ---- | --- | ------- | -------------- |
| id | sequential ID | int unsigned | NO | PRI | NULL | auto_increment |
| uri-id | Id of the item-uri table entry that contains the item uri | int unsigned | NO | | NULL | |
| url | Media URL | varbinary(1024) | NO | | NULL | |
| media-uri-id | Id of the item-uri table entry that contains the activities uri-id | int unsigned | YES | | NULL | |
| attach-id | In case of a local attachment, this field is filled with the id in the attach table | int unsigned | YES | | NULL | |
| type | Media type | tinyint unsigned | NO | | 0 | |
| mimetype | | varchar(60) | YES | | NULL | |
| height | Height of the media | smallint unsigned | YES | | NULL | |
| width | Width of the media | smallint unsigned | YES | | NULL | |
| size | Media size | bigint unsigned | YES | | NULL | |
| blurhash | BlurHash representation of the image | varbinary(255) | YES | | NULL | |
| preview | Preview URL | varbinary(512) | YES | | NULL | |
| preview-height | Height of the preview picture | smallint unsigned | YES | | NULL | |
| preview-width | Width of the preview picture | smallint unsigned | YES | | NULL | |
| description | | text | YES | | NULL | |
| name | Name of the media | varchar(255) | YES | | NULL | |
| author-url | URL of the author of the media | varbinary(383) | YES | | NULL | |
| author-name | Name of the author of the media | varchar(255) | YES | | NULL | |
| author-image | Image of the author of the media | varbinary(383) | YES | | NULL | |
| publisher-url | URL of the publisher of the media | varbinary(383) | YES | | NULL | |
| publisher-name | Name of the publisher of the media | varchar(255) | YES | | NULL | |
| publisher-image | Image of the publisher of the media | varbinary(383) | YES | | NULL | |
Indexes
------------
@ -39,6 +40,7 @@ Indexes
| uri-id-url | UNIQUE, uri-id, url(512) |
| uri-id-id | uri-id, id |
| media-uri-id | media-uri-id |
| attach-id | attach-id |
Foreign Keys
------------
@ -47,5 +49,6 @@ Foreign Keys
|-------|--------------|--------------|
| uri-id | [item-uri](help/database/db_item-uri) | id |
| media-uri-id | [item-uri](help/database/db_item-uri) | id |
| attach-id | [attach](help/database/db_attach) | id |
Return to [database documentation](help/database)

8
src/Model/Item.php
View file

@ -380,14 +380,6 @@ class Item
Event::delete($item['event-id']);
}
// If item has attachments, drop them
$attachments = Post\Media::getByURIId($item['uri-id'], [Post\Media::DOCUMENT]);
foreach ($attachments as $attachment) {
if (preg_match('|attach/(\d+)|', $attachment['url'], $matches)) {
Attach::delete(['id' => $matches[1], 'uid' => $item['uid']]);
}
}
// Set the item to "deleted"
$item_fields = ['deleted' => true, 'edited' => DateTimeFormat::utcNow(), 'changed' => DateTimeFormat::utcNow()];
Post::update($item_fields, ['id' => $item['id']]);

13
src/Model/Post/Media.php
View file

@ -384,11 +384,12 @@ class Media
private static function fetchLocalData(array $media): array
{
if (preg_match('|.*?/attach/(\d+)|', $media['url'], $matches)) {
$attachment = Attach::selectFirst(['filename', 'filetype', 'filesize'], ['id' => $matches[1]]);
$attachment = Attach::selectFirst(['id', 'filename', 'filetype', 'filesize'], ['id' => $matches[1]]);
if (!empty($attachment)) {
$media['name'] = $attachment['filename'];
$media['mimetype'] = $attachment['filetype'];
$media['size'] = $attachment['filesize'];
$media['attach-id'] = $attachment['id'];
$media['name'] = $attachment['filename'];
$media['mimetype'] = $attachment['filetype'];
$media['size'] = $attachment['filesize'];
}
return $media;
}
@ -396,7 +397,7 @@ class Media
if (!preg_match('|.*?/photo/(.*[a-fA-F0-9])\-(.*[0-9])\..*[\w]|', $media['url'], $matches)) {
return $media;
}
$photo = Photo::selectFirst([], ['resource-id' => $matches[1], 'scale' => $matches[2]]);
$photo = Photo::selectFirst(['type', 'datasize', 'width', 'height', 'blurhash'], ['resource-id' => $matches[1], 'scale' => $matches[2]]);
if (!empty($photo)) {
$media['mimetype'] = $photo['type'];
$media['size'] = $photo['datasize'];
@ -408,7 +409,7 @@ class Media
if (!preg_match('|.*?/photo/(.*[a-fA-F0-9])\-(.*[0-9])\..*[\w]|', $media['preview'] ?? '', $matches)) {
return $media;
}
$photo = Photo::selectFirst([], ['resource-id' => $matches[1], 'scale' => $matches[2]]);
$photo = Photo::selectFirst(['width', 'height'], ['resource-id' => $matches[1], 'scale' => $matches[2]]);
if (!empty($photo)) {
$media['preview-width'] = $photo['width'];
$media['preview-height'] = $photo['height'];

12
src/Module/Admin/Site.php
View file

@ -53,7 +53,8 @@ class Site extends BaseAdmin
$language = (!empty($_POST['language']) ? trim($_POST['language']) : '');
$theme = (!empty($_POST['theme']) ? trim($_POST['theme']) : '');
$theme_mobile = (!empty($_POST['theme_mobile']) ? trim($_POST['theme_mobile']) : '');
$maximagesize = (!empty($_POST['maximagesize']) ? trim($_POST['maximagesize']) : 0);
$maxfilesize = (!empty($_POST['maxfilesize']) ? trim($_POST['maxfilesize']) : 0);
$maximagesize = (!empty($_POST['maximagesize']) ? trim($_POST['maximagesize']) : 0);
$maximagelength = (!empty($_POST['maximagelength']) ? intval(trim($_POST['maximagelength'])) : -1);
$jpegimagequality = (!empty($_POST['jpegimagequality']) ? intval(trim($_POST['jpegimagequality'])) : 100);
@ -224,6 +225,11 @@ class Site extends BaseAdmin
} else {
$transactionConfig->set('system', 'singleuser', $singleuser);
}
if (preg_match('/\d+(?:\s*[kmg])?/i', $maxfilesize)) {
$transactionConfig->set('system', 'maxfilesize', $maxfilesize);
} else {
DI::sysmsg()->addNotice(DI::l10n()->t('%s is no valid input for maximum media size', $maxfilesize));
}
if (preg_match('/\d+(?:\s*[kmg])?/i', $maximagesize)) {
$transactionConfig->set('system', 'maximagesize', $maximagesize);
} else {
@ -484,6 +490,10 @@ class Site extends BaseAdmin
'', 'pattern="\d+(?:\s*[kmg])?"'],
'$maximagelength' => ['maximagelength', DI::l10n()->t('Maximum image length'), DI::config()->get('system', 'max_image_length'), DI::l10n()->t('Maximum length in pixels of the longest side of uploaded images. Default is -1, which means no limits.')],
'$jpegimagequality' => ['jpegimagequality', DI::l10n()->t('JPEG image quality'), DI::config()->get('system', 'jpeg_quality'), DI::l10n()->t('Uploaded JPEGS will be saved at this quality setting [0-100]. Default is 100, which is full quality.')],
'$maxfilesize' => ['maxfilesize', DI::l10n()->t('Maximum media file size'), DI::config()->get('system', 'maxfilesize'), DI::l10n()->t('Maximum size in bytes of uploaded media files. Default is 0, which means no limits. You can put k, m, or g behind the desired value for KiB, MiB, GiB, respectively.
The value of <code>upload_max_filesize</code> in your <code>PHP.ini</code> needs be set to at least the desired limit.
Currently <code>upload_max_filesize</code> is set to %s (%s byte)', Strings::formatBytes(Strings::getBytesFromShorthand(ini_get('upload_max_filesize'))), Strings::getBytesFromShorthand(ini_get('upload_max_filesize'))),
'', 'pattern="\d+(?:\s*[kmg])?"'],
'$register_policy' => ['register_policy', DI::l10n()->t('Register policy'), DI::config()->get('config', 'register_policy'), '', $register_choices],
'$max_registered_users' => ['max_registered_users', DI::l10n()->t('Maximum Users'), DI::config()->get('config', 'max_registered_users'), DI::l10n()->t('If defined, the register policy is automatically closed when the given number of users is reached and reopens the registry when the number drops below the limit. It only works when the policy is set to open or close, but not when the policy is set to approval.')],

7
src/Module/Api/Mastodon/Instance.php
View file

@ -79,9 +79,14 @@ class Instance extends BaseApi
$image_matrix_limit = 33177600; // 5760^2
}
$media_size_limit = Strings::getBytesFromShorthand($this->config->get('system', 'maxfilesize'));
if (empty($media_size_limit)) {
$media_size_limit = Strings::getBytesFromShorthand(ini_get('upload_max_filesize'));
}
return new InstanceV2Entity\Configuration(
$statuses_config,
new InstanceV2Entity\MediaAttachmentsConfig(Images::supportedMimeTypes(), $image_size_limit, $image_matrix_limit),
new InstanceV2Entity\MediaAttachmentsConfig(Images::supportedMimeTypes(), $image_size_limit, $image_matrix_limit, $media_size_limit),
new InstanceV2Entity\Polls(),
new InstanceV2Entity\Accounts(),
);

7
src/Module/Api/Mastodon/InstanceV2.php
View file

@ -115,9 +115,14 @@ class InstanceV2 extends BaseApi
$image_matrix_limit = 33177600; // 5760^2
}
$media_size_limit = Strings::getBytesFromShorthand($this->config->get('system', 'maxfilesize'));
if (empty($media_size_limit)) {
$media_size_limit = Strings::getBytesFromShorthand(ini_get('upload_max_filesize'));
}
return new InstanceEntity\Configuration(
$statuses_config,
new InstanceEntity\MediaAttachmentsConfig($this->supportedMimeTypes(), $image_size_limit, $image_matrix_limit),
new InstanceEntity\MediaAttachmentsConfig($this->supportedMimeTypes(), $image_size_limit, $image_matrix_limit, $media_size_limit),
new InstanceEntity\Polls(),
new InstanceEntity\Accounts(),
);

22
src/Module/Api/Mastodon/Media.php
View file

@ -14,6 +14,7 @@ use Friendica\Model\Contact;
use Friendica\Model\Photo;
use Friendica\Model\Post;
use Friendica\Module\BaseApi;
use Friendica\Util\Strings;
/**
* @see https://docs.joinmastodon.org/methods/statuses/media/
@ -26,7 +27,7 @@ class Media extends BaseApi
$uid = self::getCurrentUserID();
$request = $this->getRequest([
'file' => [], // The file to be attached, using multipart form data.
'file' => $_FILES['file'] ?? [], // The file to be attached, using multipart form data.
'thumbnail' => [], // The custom thumbnail of the media to be attached, using multipart form data.
'description' => '', // A plain-text description of the media, for accessibility purposes.
'focus' => '', // Two floating points (x,y), comma-delimited ranging from -1.0 to 1.0
@ -34,14 +35,15 @@ class Media extends BaseApi
Logger::info('Photo post', ['request' => $request, 'files' => $_FILES]);
if (empty($_FILES['file'])) {
if (empty($request['file'])) {
Logger::notice('Upload is invalid', ['request' => $request]);
$this->logAndJsonError(422, $this->errorFactory->UnprocessableEntity());
}
$type = Post\Media::getType($_FILES['file']['type']);
$type = Post\Media::getType($request['file']['type']);
if (in_array($type, [Post\Media::IMAGE, Post\Media::UNKNOWN])) {
$media = Photo::upload($uid, $_FILES['file'], '', null, null, '', '', $request['description']);
$media = Photo::upload($uid, $request['file'], '', null, null, '', '', $request['description']);
if (empty($media)) {
$this->logAndJsonError(422, $this->errorFactory->UnprocessableEntity());
}
@ -50,22 +52,24 @@ class Media extends BaseApi
$this->jsonExit(DI::mstdnAttachment()->createFromPhoto($media['id']));
} else {
$tempFileName = $_FILES['file']['tmp_name'];
$fileName = basename($_FILES['file']['name']);
$fileSize = intval($_FILES['file']['size']);
$maxFileSize = DI::config()->get('system', 'maxfilesize');
$tempFileName = $request['file']['tmp_name'];
$fileName = basename($request['file']['name']);
$fileSize = intval($request['file']['size']);
$maxFileSize = Strings::getBytesFromShorthand(DI::config()->get('system', 'maxfilesize'));
if ($fileSize <= 0) {
Logger::notice('Filesize is invalid', ['size' => $fileSize, 'request' => $request]);
@unlink($tempFileName);
$this->logAndJsonError(422, $this->errorFactory->UnprocessableEntity());
}
if ($maxFileSize && $fileSize > $maxFileSize) {
Logger::notice('Filesize is too large', ['size' => $fileSize, 'max' => $maxFileSize, 'request' => $request]);
@unlink($tempFileName);
$this->logAndJsonError(422, $this->errorFactory->UnprocessableEntity());
}
$id = Attach::storeFile($tempFileName, self::getCurrentUserID(), $fileName, $_FILES['file']['type'], '<' . Contact::getPublicIdByUserId(self::getCurrentUserID()) . '>');
$id = Attach::storeFile($tempFileName, self::getCurrentUserID(), $fileName, $request['file']['type'], '<' . Contact::getPublicIdByUserId(self::getCurrentUserID()) . '>');
@unlink($tempFileName);
Logger::info('Uploaded media', ['id' => $id]);
$this->jsonExit(DI::mstdnAttachment()->createFromAttach($id));

2
src/Module/Media/Attachment/Upload.php
View file

@ -71,7 +71,7 @@ class Upload extends \Friendica\BaseModule
$tempFileName = $_FILES['userfile']['tmp_name'];
$fileName = basename($_FILES['userfile']['name']);
$fileSize = intval($_FILES['userfile']['size']);
$maxFileSize = $this->config->get('system', 'maxfilesize');
$maxFileSize = Strings::getBytesFromShorthand($this->config->get('system', 'maxfilesize'));
/*
* Found html code written in text field of form, when trying to upload a

4
src/Object/Api/Mastodon/InstanceV2/MediaAttachmentsConfig.php
View file

@ -32,12 +32,12 @@ class MediaAttachmentsConfig extends BaseDataTransferObject
/**
* @param array $supported_mime_types
*/
public function __construct(array $supported_mime_types, int $image_size_limit, int $image_matrix_limit)
public function __construct(array $supported_mime_types, int $image_size_limit, int $image_matrix_limit, int $media_size_limit)
{
$this->supported_mime_types = $supported_mime_types;
$this->image_size_limit = $image_size_limit;
$this->image_matrix_limit = $image_matrix_limit;
$this->video_size_limit = $image_size_limit;
$this->video_size_limit = $media_size_limit;
$this->video_matrix_limit = $image_matrix_limit;
}
}

8
src/Protocol/ActivityPub/Processor.php
View file

@ -979,9 +979,11 @@ class Processor
}
}
$author = Contact::selectFirstAccount(['ap-posting-restricted'], ['id' => $item['author-id']]);
if (!empty($author['ap-posting-restricted'])) {
$item['restrictions'] = $item['restrictions'] | Item::CANT_REPLY;
if (!empty($item['author-id'])) {
$author = Contact::selectFirstAccount(['ap-posting-restricted'], ['id' => $item['author-id']]);
if (!empty($author['ap-posting-restricted'])) {
$item['restrictions'] = $item['restrictions'] | Item::CANT_REPLY;
}
}
$item['location'] = $activity['location'];

16
src/Worker/ExpirePosts.php
View file

@ -13,6 +13,7 @@ use Friendica\Database\Database;
use Friendica\Database\DBA;
use Friendica\Database\DBStructure;
use Friendica\DI;
use Friendica\Model\Attach;
use Friendica\Model\Item;
use Friendica\Model\Post;
use Friendica\Util\DateTimeFormat;
@ -43,6 +44,8 @@ class ExpirePosts
self::addMissingEntries();
}
self::deleteUnusedAttachments();
DBA::releaseOptimizeLock();
// Set the expiry for origin posts
@ -304,4 +307,17 @@ class ExpirePosts
} while ($affected_count);
}
}
/**
* Delete media attachments (excluding photos) that aren't linked to any post
*
* @return void
*/
private static function deleteUnusedAttachments()
{
$postmedia = DBA::select('attach', ['id'], ["`id` NOT IN (SELECT `attach-id` FROM `post-media`)"]);
while ($media = DBA::fetch($postmedia)) {
Attach::delete(['id' => $media['id']]);
}
}
}

4
static/dbstructure.config.php
View file

@ -44,7 +44,7 @@ use Friendica\Database\DBA;
// This file is required several times during the test in DbaDefinition which justifies this condition
if (!defined('DB_UPDATE_VERSION')) {
define('DB_UPDATE_VERSION', 1572);
define('DB_UPDATE_VERSION', 1573);
}
return [
@ -1431,6 +1431,7 @@ return [
"uri-id" => ["type" => "int unsigned", "not null" => "1", "foreign" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the item uri"],
"url" => ["type" => "varbinary(1024)", "not null" => "1", "comment" => "Media URL"],
"media-uri-id" => ["type" => "int unsigned", "foreign" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the activities uri-id"],
"attach-id" => ["type" => "int unsigned", "foreign" => ["attach" => "id"], "comment" => "In case of a local attachment, this field is filled with the id in the attach table"],
"type" => ["type" => "tinyint unsigned", "not null" => "1", "default" => "0", "comment" => "Media type"],
"mimetype" => ["type" => "varchar(60)", "comment" => ""],
"height" => ["type" => "smallint unsigned", "comment" => "Height of the media"],
@ -1454,6 +1455,7 @@ return [
"uri-id-url" => ["UNIQUE", "uri-id", "url(512)"],
"uri-id-id" => ["uri-id", "id"],
"media-uri-id" => ["media-uri-id"],
"attach-id" => ["attach-id"],
]
],
"post-origin" => [

25
update.php
View file

@ -40,6 +40,7 @@ use Friendica\Database\Database;
use Friendica\Database\DBA;
use Friendica\Database\DBStructure;
use Friendica\DI;
use Friendica\Model\Attach;
use Friendica\Model\Contact;
use Friendica\Model\Item;
use Friendica\Model\ItemURI;
@ -1502,3 +1503,27 @@ function update_1571()
return Update::SUCCESS;
}
function update_1573()
{
$postmedia = DBA::select('post-media', ['id', 'url'], ["`url` LIKE ?", '%/attach/%']);
while ($media = DBA::fetch($postmedia)) {
if (!DI::baseUrl()->isLocalUrl($media['url'])) {
continue;
}
if (preg_match('|.*?/attach/(\d+)|', $media['url'], $matches)) {
$attachment = Attach::selectFirst(['id', 'filename', 'filetype', 'filesize'], ['id' => $matches[1]]);
if (!empty($attachment)) {
$fields = [
'attach-id' => $attachment['id'],
'name' => $attachment['filename'],
'mimetype' => $attachment['filetype'],
'size' => $attachment['filesize'],
];
DBA::update('post-media', $fields, ['id' => $media['id']]);
}
}
}
DBA::close($media);
return Update::SUCCESS;
}

645
view/lang/C/messages.po
View file

File diff suppressed because it is too large Load diff

1
view/templates/admin/site.tpl
View file

@ -47,6 +47,7 @@
{{include file="field_input.tpl" field=$maximagesize}}
{{include file="field_input.tpl" field=$maximagelength}}
{{include file="field_input.tpl" field=$jpegimagequality}}
{{include file="field_input.tpl" field=$maxfilesize}}
<div class="submit"><input type="submit" name="page_site" value="{{$submit}}"/></div>
<h2>{{$corporate}}</h2>

1
view/theme/frio/templates/admin/site.tpl
View file

@ -106,6 +106,7 @@
{{include file="field_input.tpl" field=$maximagesize}}
{{include file="field_input.tpl" field=$maximagelength}}
{{include file="field_input.tpl" field=$jpegimagequality}}
{{include file="field_input.tpl" field=$maxfilesize}}
</div>
<div class="panel-footer">
<input type="submit" name="page_site" class="btn btn-primary" value="{{$submit}}"/>