mirror of
https://github.com/element-hq/synapse
synced 2024-09-06 01:50:08 +00:00
Store URL cache preview downloads seperately
This makes it easier to clear old media out at a later date
This commit is contained in:
parent
e204062310
commit
7fe8ed1787
7 changed files with 94 additions and 20 deletions
|
@ -73,6 +73,11 @@ class DownloadResource(Resource):
|
||||||
media_type = media_info["media_type"]
|
media_type = media_info["media_type"]
|
||||||
media_length = media_info["media_length"]
|
media_length = media_info["media_length"]
|
||||||
upload_name = name if name else media_info["upload_name"]
|
upload_name = name if name else media_info["upload_name"]
|
||||||
|
if media_info["url_cache"]:
|
||||||
|
# TODO: Check the file still exists, if it doesn't we can redownload
|
||||||
|
# it from the url `media_info["url_cache"]`
|
||||||
|
file_path = self.filepaths.url_cache_filepath(media_id)
|
||||||
|
else:
|
||||||
file_path = self.filepaths.local_media_filepath(media_id)
|
file_path = self.filepaths.local_media_filepath(media_id)
|
||||||
|
|
||||||
yield respond_with_file(
|
yield respond_with_file(
|
||||||
|
|
|
@ -71,3 +71,21 @@ class MediaFilePaths(object):
|
||||||
self.base_path, "remote_thumbnail", server_name,
|
self.base_path, "remote_thumbnail", server_name,
|
||||||
file_id[0:2], file_id[2:4], file_id[4:],
|
file_id[0:2], file_id[2:4], file_id[4:],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def url_cache_filepath(self, media_id):
|
||||||
|
return os.path.join(
|
||||||
|
self.base_path, "url_cache",
|
||||||
|
media_id[0:2], media_id[2:4], media_id[4:]
|
||||||
|
)
|
||||||
|
|
||||||
|
def url_cache_thumbnail(self, media_id, width, height, content_type,
|
||||||
|
method):
|
||||||
|
top_level_type, sub_type = content_type.split("/")
|
||||||
|
file_name = "%i-%i-%s-%s-%s" % (
|
||||||
|
width, height, top_level_type, sub_type, method
|
||||||
|
)
|
||||||
|
return os.path.join(
|
||||||
|
self.base_path, "url_cache_thumbnails",
|
||||||
|
media_id[0:2], media_id[2:4], media_id[4:],
|
||||||
|
file_name
|
||||||
|
)
|
||||||
|
|
|
@ -326,13 +326,17 @@ class MediaRepository(object):
|
||||||
defer.returnValue(t_path)
|
defer.returnValue(t_path)
|
||||||
|
|
||||||
@defer.inlineCallbacks
|
@defer.inlineCallbacks
|
||||||
def _generate_local_thumbnails(self, media_id, media_info):
|
def _generate_local_thumbnails(self, media_id, media_info, url_cache=False):
|
||||||
media_type = media_info["media_type"]
|
media_type = media_info["media_type"]
|
||||||
requirements = self._get_thumbnail_requirements(media_type)
|
requirements = self._get_thumbnail_requirements(media_type)
|
||||||
if not requirements:
|
if not requirements:
|
||||||
return
|
return
|
||||||
|
|
||||||
|
if url_cache:
|
||||||
|
input_path = self.filepaths.url_cache_filepath(media_id)
|
||||||
|
else:
|
||||||
input_path = self.filepaths.local_media_filepath(media_id)
|
input_path = self.filepaths.local_media_filepath(media_id)
|
||||||
|
|
||||||
thumbnailer = Thumbnailer(input_path)
|
thumbnailer = Thumbnailer(input_path)
|
||||||
m_width = thumbnailer.width
|
m_width = thumbnailer.width
|
||||||
m_height = thumbnailer.height
|
m_height = thumbnailer.height
|
||||||
|
@ -360,6 +364,11 @@ class MediaRepository(object):
|
||||||
|
|
||||||
for t_width, t_height, t_type in scales:
|
for t_width, t_height, t_type in scales:
|
||||||
t_method = "scale"
|
t_method = "scale"
|
||||||
|
if url_cache:
|
||||||
|
t_path = self.filepaths.url_cache_thumbnail(
|
||||||
|
media_id, t_width, t_height, t_type, t_method
|
||||||
|
)
|
||||||
|
else:
|
||||||
t_path = self.filepaths.local_media_thumbnail(
|
t_path = self.filepaths.local_media_thumbnail(
|
||||||
media_id, t_width, t_height, t_type, t_method
|
media_id, t_width, t_height, t_type, t_method
|
||||||
)
|
)
|
||||||
|
@ -377,6 +386,11 @@ class MediaRepository(object):
|
||||||
# thumbnail.
|
# thumbnail.
|
||||||
continue
|
continue
|
||||||
t_method = "crop"
|
t_method = "crop"
|
||||||
|
if url_cache:
|
||||||
|
t_path = self.filepaths.url_cache_thumbnail(
|
||||||
|
media_id, t_width, t_height, t_type, t_method
|
||||||
|
)
|
||||||
|
else:
|
||||||
t_path = self.filepaths.local_media_thumbnail(
|
t_path = self.filepaths.local_media_thumbnail(
|
||||||
media_id, t_width, t_height, t_type, t_method
|
media_id, t_width, t_height, t_type, t_method
|
||||||
)
|
)
|
||||||
|
|
|
@ -164,7 +164,7 @@ class PreviewUrlResource(Resource):
|
||||||
|
|
||||||
if _is_media(media_info['media_type']):
|
if _is_media(media_info['media_type']):
|
||||||
dims = yield self.media_repo._generate_local_thumbnails(
|
dims = yield self.media_repo._generate_local_thumbnails(
|
||||||
media_info['filesystem_id'], media_info
|
media_info['filesystem_id'], media_info, url_cache=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
og = {
|
og = {
|
||||||
|
@ -210,7 +210,7 @@ class PreviewUrlResource(Resource):
|
||||||
if _is_media(image_info['media_type']):
|
if _is_media(image_info['media_type']):
|
||||||
# TODO: make sure we don't choke on white-on-transparent images
|
# TODO: make sure we don't choke on white-on-transparent images
|
||||||
dims = yield self.media_repo._generate_local_thumbnails(
|
dims = yield self.media_repo._generate_local_thumbnails(
|
||||||
image_info['filesystem_id'], image_info
|
image_info['filesystem_id'], image_info, url_cache=True,
|
||||||
)
|
)
|
||||||
if dims:
|
if dims:
|
||||||
og["og:image:width"] = dims['width']
|
og["og:image:width"] = dims['width']
|
||||||
|
@ -256,7 +256,7 @@ class PreviewUrlResource(Resource):
|
||||||
# XXX: horrible duplication with base_resource's _download_remote_file()
|
# XXX: horrible duplication with base_resource's _download_remote_file()
|
||||||
file_id = random_string(24)
|
file_id = random_string(24)
|
||||||
|
|
||||||
fname = self.filepaths.local_media_filepath(file_id)
|
fname = self.filepaths.url_cache_filepath(file_id)
|
||||||
self.media_repo._makedirs(fname)
|
self.media_repo._makedirs(fname)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
@ -303,6 +303,7 @@ class PreviewUrlResource(Resource):
|
||||||
upload_name=download_name,
|
upload_name=download_name,
|
||||||
media_length=length,
|
media_length=length,
|
||||||
user_id=user,
|
user_id=user,
|
||||||
|
url_cache=url,
|
||||||
)
|
)
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
|
|
|
@ -101,6 +101,13 @@ class ThumbnailResource(Resource):
|
||||||
t_type = thumbnail_info["thumbnail_type"]
|
t_type = thumbnail_info["thumbnail_type"]
|
||||||
t_method = thumbnail_info["thumbnail_method"]
|
t_method = thumbnail_info["thumbnail_method"]
|
||||||
|
|
||||||
|
if media_info["url_cache"]:
|
||||||
|
# TODO: Check the file still exists, if it doesn't we can redownload
|
||||||
|
# it from the url `media_info["url_cache"]`
|
||||||
|
file_path = self.filepaths.url_cache_thumbnail(
|
||||||
|
media_id, t_width, t_height, t_type, t_method,
|
||||||
|
)
|
||||||
|
else:
|
||||||
file_path = self.filepaths.local_media_thumbnail(
|
file_path = self.filepaths.local_media_thumbnail(
|
||||||
media_id, t_width, t_height, t_type, t_method,
|
media_id, t_width, t_height, t_type, t_method,
|
||||||
)
|
)
|
||||||
|
@ -134,8 +141,17 @@ class ThumbnailResource(Resource):
|
||||||
t_type = info["thumbnail_type"] == desired_type
|
t_type = info["thumbnail_type"] == desired_type
|
||||||
|
|
||||||
if t_w and t_h and t_method and t_type:
|
if t_w and t_h and t_method and t_type:
|
||||||
|
if media_info["url_cache"]:
|
||||||
|
# TODO: Check the file still exists, if it doesn't we can redownload
|
||||||
|
# it from the url `media_info["url_cache"]`
|
||||||
|
file_path = self.filepaths.url_cache_thumbnail(
|
||||||
|
media_id, desired_width, desired_height, desired_type,
|
||||||
|
desired_method,
|
||||||
|
)
|
||||||
|
else:
|
||||||
file_path = self.filepaths.local_media_thumbnail(
|
file_path = self.filepaths.local_media_thumbnail(
|
||||||
media_id, desired_width, desired_height, desired_type, desired_method,
|
media_id, desired_width, desired_height, desired_type,
|
||||||
|
desired_method,
|
||||||
)
|
)
|
||||||
yield respond_with_file(request, desired_type, file_path)
|
yield respond_with_file(request, desired_type, file_path)
|
||||||
return
|
return
|
||||||
|
|
|
@ -30,13 +30,16 @@ class MediaRepositoryStore(SQLBaseStore):
|
||||||
return self._simple_select_one(
|
return self._simple_select_one(
|
||||||
"local_media_repository",
|
"local_media_repository",
|
||||||
{"media_id": media_id},
|
{"media_id": media_id},
|
||||||
("media_type", "media_length", "upload_name", "created_ts", "quarantined_by"),
|
(
|
||||||
|
"media_type", "media_length", "upload_name", "created_ts",
|
||||||
|
"quarantined_by", "url_cache",
|
||||||
|
),
|
||||||
allow_none=True,
|
allow_none=True,
|
||||||
desc="get_local_media",
|
desc="get_local_media",
|
||||||
)
|
)
|
||||||
|
|
||||||
def store_local_media(self, media_id, media_type, time_now_ms, upload_name,
|
def store_local_media(self, media_id, media_type, time_now_ms, upload_name,
|
||||||
media_length, user_id):
|
media_length, user_id, url_cache=None):
|
||||||
return self._simple_insert(
|
return self._simple_insert(
|
||||||
"local_media_repository",
|
"local_media_repository",
|
||||||
{
|
{
|
||||||
|
@ -46,6 +49,7 @@ class MediaRepositoryStore(SQLBaseStore):
|
||||||
"upload_name": upload_name,
|
"upload_name": upload_name,
|
||||||
"media_length": media_length,
|
"media_length": media_length,
|
||||||
"user_id": user_id.to_string(),
|
"user_id": user_id.to_string(),
|
||||||
|
"url_cache": url_cache,
|
||||||
},
|
},
|
||||||
desc="store_local_media",
|
desc="store_local_media",
|
||||||
)
|
)
|
||||||
|
|
16
synapse/storage/schema/delta/43/url_cache.sql
Normal file
16
synapse/storage/schema/delta/43/url_cache.sql
Normal file
|
@ -0,0 +1,16 @@
|
||||||
|
/* Copyright 2017 Vector Creations Ltd
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
ALTER TABLE local_media_repository ADD COLUMN url_cache TEXT;
|
Loading…
Reference in a new issue