mirror of
https://github.com/element-hq/synapse
synced 2024-09-01 16:50:20 +00:00
Merge remote-tracking branch 'origin/develop' into rav/track_db_scheduling
This commit is contained in:
commit
e8f7541d3f
5 changed files with 78 additions and 34 deletions
|
@ -96,7 +96,7 @@ class TlsConfig(Config):
|
||||||
# certificates returned by this server match one of the fingerprints.
|
# certificates returned by this server match one of the fingerprints.
|
||||||
#
|
#
|
||||||
# Synapse automatically adds the fingerprint of its own certificate
|
# Synapse automatically adds the fingerprint of its own certificate
|
||||||
# to the list. So if federation traffic is handle directly by synapse
|
# to the list. So if federation traffic is handled directly by synapse
|
||||||
# then no modification to the list is required.
|
# then no modification to the list is required.
|
||||||
#
|
#
|
||||||
# If synapse is run behind a load balancer that handles the TLS then it
|
# If synapse is run behind a load balancer that handles the TLS then it
|
||||||
|
|
|
@ -322,15 +322,6 @@ class JsonResource(HttpServer, resource.Resource):
|
||||||
|
|
||||||
def _send_response(self, request, code, response_json_object,
|
def _send_response(self, request, code, response_json_object,
|
||||||
response_code_message=None):
|
response_code_message=None):
|
||||||
# could alternatively use request.notifyFinish() and flip a flag when
|
|
||||||
# the Deferred fires, but since the flag is RIGHT THERE it seems like
|
|
||||||
# a waste.
|
|
||||||
if request._disconnected:
|
|
||||||
logger.warn(
|
|
||||||
"Not sending response to request %s, already disconnected.",
|
|
||||||
request)
|
|
||||||
return
|
|
||||||
|
|
||||||
outgoing_responses_counter.inc(request.method, str(code))
|
outgoing_responses_counter.inc(request.method, str(code))
|
||||||
|
|
||||||
# TODO: Only enable CORS for the requests that need it.
|
# TODO: Only enable CORS for the requests that need it.
|
||||||
|
@ -409,6 +400,15 @@ class RootRedirect(resource.Resource):
|
||||||
def respond_with_json(request, code, json_object, send_cors=False,
|
def respond_with_json(request, code, json_object, send_cors=False,
|
||||||
response_code_message=None, pretty_print=False,
|
response_code_message=None, pretty_print=False,
|
||||||
version_string="", canonical_json=True):
|
version_string="", canonical_json=True):
|
||||||
|
# could alternatively use request.notifyFinish() and flip a flag when
|
||||||
|
# the Deferred fires, but since the flag is RIGHT THERE it seems like
|
||||||
|
# a waste.
|
||||||
|
if request._disconnected:
|
||||||
|
logger.warn(
|
||||||
|
"Not sending response to request %s, already disconnected.",
|
||||||
|
request)
|
||||||
|
return
|
||||||
|
|
||||||
if pretty_print:
|
if pretty_print:
|
||||||
json_bytes = encode_pretty_printed_json(json_object) + "\n"
|
json_bytes = encode_pretty_printed_json(json_object) + "\n"
|
||||||
else:
|
else:
|
||||||
|
|
|
@ -146,12 +146,10 @@ class MediaRepository(object):
|
||||||
media_length=content_length,
|
media_length=content_length,
|
||||||
user_id=auth_user,
|
user_id=auth_user,
|
||||||
)
|
)
|
||||||
media_info = {
|
|
||||||
"media_type": media_type,
|
|
||||||
"media_length": content_length,
|
|
||||||
}
|
|
||||||
|
|
||||||
yield self._generate_thumbnails(None, media_id, media_info)
|
yield self._generate_thumbnails(
|
||||||
|
None, media_id, media_id, media_type,
|
||||||
|
)
|
||||||
|
|
||||||
defer.returnValue("mxc://%s/%s" % (self.server_name, media_id))
|
defer.returnValue("mxc://%s/%s" % (self.server_name, media_id))
|
||||||
|
|
||||||
|
@ -227,6 +225,34 @@ class MediaRepository(object):
|
||||||
else:
|
else:
|
||||||
respond_404(request)
|
respond_404(request)
|
||||||
|
|
||||||
|
@defer.inlineCallbacks
|
||||||
|
def get_remote_media_info(self, server_name, media_id):
|
||||||
|
"""Gets the media info associated with the remote file, downloading
|
||||||
|
if necessary.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
server_name (str): Remote server_name where the media originated.
|
||||||
|
media_id (str): The media ID of the content (as defined by the
|
||||||
|
remote server).
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Deferred[dict]: The media_info of the file
|
||||||
|
"""
|
||||||
|
# We linearize here to ensure that we don't try and download remote
|
||||||
|
# media multiple times concurrently
|
||||||
|
key = (server_name, media_id)
|
||||||
|
with (yield self.remote_media_linearizer.queue(key)):
|
||||||
|
responder, media_info = yield self._get_remote_media_impl(
|
||||||
|
server_name, media_id,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Ensure we actually use the responder so that it releases resources
|
||||||
|
if responder:
|
||||||
|
with responder:
|
||||||
|
pass
|
||||||
|
|
||||||
|
defer.returnValue(media_info)
|
||||||
|
|
||||||
@defer.inlineCallbacks
|
@defer.inlineCallbacks
|
||||||
def _get_remote_media_impl(self, server_name, media_id):
|
def _get_remote_media_impl(self, server_name, media_id):
|
||||||
"""Looks for media in local cache, if not there then attempt to
|
"""Looks for media in local cache, if not there then attempt to
|
||||||
|
@ -257,6 +283,7 @@ class MediaRepository(object):
|
||||||
# If we have an entry in the DB, try and look for it
|
# If we have an entry in the DB, try and look for it
|
||||||
if media_info:
|
if media_info:
|
||||||
if media_info["quarantined_by"]:
|
if media_info["quarantined_by"]:
|
||||||
|
logger.info("Media is quarantined")
|
||||||
raise NotFoundError()
|
raise NotFoundError()
|
||||||
|
|
||||||
responder = yield self.media_storage.fetch_media(file_info)
|
responder = yield self.media_storage.fetch_media(file_info)
|
||||||
|
@ -384,7 +411,7 @@ class MediaRepository(object):
|
||||||
}
|
}
|
||||||
|
|
||||||
yield self._generate_thumbnails(
|
yield self._generate_thumbnails(
|
||||||
server_name, media_id, media_info
|
server_name, media_id, file_id, media_type,
|
||||||
)
|
)
|
||||||
|
|
||||||
defer.returnValue(media_info)
|
defer.returnValue(media_info)
|
||||||
|
@ -496,21 +523,22 @@ class MediaRepository(object):
|
||||||
defer.returnValue(output_path)
|
defer.returnValue(output_path)
|
||||||
|
|
||||||
@defer.inlineCallbacks
|
@defer.inlineCallbacks
|
||||||
def _generate_thumbnails(self, server_name, media_id, media_info, url_cache=False):
|
def _generate_thumbnails(self, server_name, media_id, file_id, media_type,
|
||||||
|
url_cache=False):
|
||||||
"""Generate and store thumbnails for an image.
|
"""Generate and store thumbnails for an image.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
server_name(str|None): The server name if remote media, else None if local
|
server_name (str|None): The server name if remote media, else None if local
|
||||||
media_id(str)
|
media_id (str): The media ID of the content. (This is the same as
|
||||||
media_info(dict)
|
the file_id for local content)
|
||||||
url_cache(bool): If we are thumbnailing images downloaded for the URL cache,
|
file_id (str): Local file ID
|
||||||
|
media_type (str): The content type of the file
|
||||||
|
url_cache (bool): If we are thumbnailing images downloaded for the URL cache,
|
||||||
used exclusively by the url previewer
|
used exclusively by the url previewer
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Deferred[dict]: Dict with "width" and "height" keys of original image
|
Deferred[dict]: Dict with "width" and "height" keys of original image
|
||||||
"""
|
"""
|
||||||
media_type = media_info["media_type"]
|
|
||||||
file_id = media_info.get("filesystem_id")
|
|
||||||
requirements = self._get_thumbnail_requirements(media_type)
|
requirements = self._get_thumbnail_requirements(media_type)
|
||||||
if not requirements:
|
if not requirements:
|
||||||
return
|
return
|
||||||
|
@ -568,7 +596,7 @@ class MediaRepository(object):
|
||||||
try:
|
try:
|
||||||
file_info = FileInfo(
|
file_info = FileInfo(
|
||||||
server_name=server_name,
|
server_name=server_name,
|
||||||
file_id=media_id,
|
file_id=file_id,
|
||||||
thumbnail=True,
|
thumbnail=True,
|
||||||
thumbnail_width=t_width,
|
thumbnail_width=t_width,
|
||||||
thumbnail_height=t_height,
|
thumbnail_height=t_height,
|
||||||
|
|
|
@ -185,8 +185,10 @@ class PreviewUrlResource(Resource):
|
||||||
logger.debug("got media_info of '%s'" % media_info)
|
logger.debug("got media_info of '%s'" % media_info)
|
||||||
|
|
||||||
if _is_media(media_info['media_type']):
|
if _is_media(media_info['media_type']):
|
||||||
|
file_id = media_info['filesystem_id']
|
||||||
dims = yield self.media_repo._generate_thumbnails(
|
dims = yield self.media_repo._generate_thumbnails(
|
||||||
None, media_info['filesystem_id'], media_info, url_cache=True,
|
None, file_id, file_id, media_info["media_type"],
|
||||||
|
url_cache=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
og = {
|
og = {
|
||||||
|
@ -231,8 +233,10 @@ class PreviewUrlResource(Resource):
|
||||||
|
|
||||||
if _is_media(image_info['media_type']):
|
if _is_media(image_info['media_type']):
|
||||||
# TODO: make sure we don't choke on white-on-transparent images
|
# TODO: make sure we don't choke on white-on-transparent images
|
||||||
|
file_id = image_info['filesystem_id']
|
||||||
dims = yield self.media_repo._generate_thumbnails(
|
dims = yield self.media_repo._generate_thumbnails(
|
||||||
None, image_info['filesystem_id'], image_info, url_cache=True,
|
None, file_id, file_id, image_info["media_type"],
|
||||||
|
url_cache=True,
|
||||||
)
|
)
|
||||||
if dims:
|
if dims:
|
||||||
og["og:image:width"] = dims['width']
|
og["og:image:width"] = dims['width']
|
||||||
|
|
|
@ -84,7 +84,11 @@ class ThumbnailResource(Resource):
|
||||||
method, m_type):
|
method, m_type):
|
||||||
media_info = yield self.store.get_local_media(media_id)
|
media_info = yield self.store.get_local_media(media_id)
|
||||||
|
|
||||||
if not media_info or media_info["quarantined_by"]:
|
if not media_info:
|
||||||
|
respond_404(request)
|
||||||
|
return
|
||||||
|
if media_info["quarantined_by"]:
|
||||||
|
logger.info("Media is quarantined")
|
||||||
respond_404(request)
|
respond_404(request)
|
||||||
return
|
return
|
||||||
|
|
||||||
|
@ -111,6 +115,7 @@ class ThumbnailResource(Resource):
|
||||||
responder = yield self.media_storage.fetch_media(file_info)
|
responder = yield self.media_storage.fetch_media(file_info)
|
||||||
yield respond_with_responder(request, responder, t_type, t_length)
|
yield respond_with_responder(request, responder, t_type, t_length)
|
||||||
else:
|
else:
|
||||||
|
logger.info("Couldn't find any generated thumbnails")
|
||||||
respond_404(request)
|
respond_404(request)
|
||||||
|
|
||||||
@defer.inlineCallbacks
|
@defer.inlineCallbacks
|
||||||
|
@ -119,7 +124,11 @@ class ThumbnailResource(Resource):
|
||||||
desired_type):
|
desired_type):
|
||||||
media_info = yield self.store.get_local_media(media_id)
|
media_info = yield self.store.get_local_media(media_id)
|
||||||
|
|
||||||
if not media_info or media_info["quarantined_by"]:
|
if not media_info:
|
||||||
|
respond_404(request)
|
||||||
|
return
|
||||||
|
if media_info["quarantined_by"]:
|
||||||
|
logger.info("Media is quarantined")
|
||||||
respond_404(request)
|
respond_404(request)
|
||||||
return
|
return
|
||||||
|
|
||||||
|
@ -149,7 +158,7 @@ class ThumbnailResource(Resource):
|
||||||
yield respond_with_responder(request, responder, t_type, t_length)
|
yield respond_with_responder(request, responder, t_type, t_length)
|
||||||
return
|
return
|
||||||
|
|
||||||
logger.debug("We don't have a local thumbnail of that size. Generating")
|
logger.debug("We don't have a thumbnail of that size. Generating")
|
||||||
|
|
||||||
# Okay, so we generate one.
|
# Okay, so we generate one.
|
||||||
file_path = yield self.media_repo.generate_local_exact_thumbnail(
|
file_path = yield self.media_repo.generate_local_exact_thumbnail(
|
||||||
|
@ -159,13 +168,14 @@ class ThumbnailResource(Resource):
|
||||||
if file_path:
|
if file_path:
|
||||||
yield respond_with_file(request, desired_type, file_path)
|
yield respond_with_file(request, desired_type, file_path)
|
||||||
else:
|
else:
|
||||||
|
logger.warn("Failed to generate thumbnail")
|
||||||
respond_404(request)
|
respond_404(request)
|
||||||
|
|
||||||
@defer.inlineCallbacks
|
@defer.inlineCallbacks
|
||||||
def _select_or_generate_remote_thumbnail(self, request, server_name, media_id,
|
def _select_or_generate_remote_thumbnail(self, request, server_name, media_id,
|
||||||
desired_width, desired_height,
|
desired_width, desired_height,
|
||||||
desired_method, desired_type):
|
desired_method, desired_type):
|
||||||
media_info = yield self.media_repo.get_remote_media(server_name, media_id)
|
media_info = yield self.media_repo.get_remote_media_info(server_name, media_id)
|
||||||
|
|
||||||
thumbnail_infos = yield self.store.get_remote_media_thumbnails(
|
thumbnail_infos = yield self.store.get_remote_media_thumbnails(
|
||||||
server_name, media_id,
|
server_name, media_id,
|
||||||
|
@ -181,7 +191,7 @@ class ThumbnailResource(Resource):
|
||||||
|
|
||||||
if t_w and t_h and t_method and t_type:
|
if t_w and t_h and t_method and t_type:
|
||||||
file_info = FileInfo(
|
file_info = FileInfo(
|
||||||
server_name=None, file_id=media_id,
|
server_name=server_name, file_id=media_info["filesystem_id"],
|
||||||
thumbnail=True,
|
thumbnail=True,
|
||||||
thumbnail_width=info["thumbnail_width"],
|
thumbnail_width=info["thumbnail_width"],
|
||||||
thumbnail_height=info["thumbnail_height"],
|
thumbnail_height=info["thumbnail_height"],
|
||||||
|
@ -197,7 +207,7 @@ class ThumbnailResource(Resource):
|
||||||
yield respond_with_responder(request, responder, t_type, t_length)
|
yield respond_with_responder(request, responder, t_type, t_length)
|
||||||
return
|
return
|
||||||
|
|
||||||
logger.debug("We don't have a local thumbnail of that size. Generating")
|
logger.debug("We don't have a thumbnail of that size. Generating")
|
||||||
|
|
||||||
# Okay, so we generate one.
|
# Okay, so we generate one.
|
||||||
file_path = yield self.media_repo.generate_remote_exact_thumbnail(
|
file_path = yield self.media_repo.generate_remote_exact_thumbnail(
|
||||||
|
@ -208,6 +218,7 @@ class ThumbnailResource(Resource):
|
||||||
if file_path:
|
if file_path:
|
||||||
yield respond_with_file(request, desired_type, file_path)
|
yield respond_with_file(request, desired_type, file_path)
|
||||||
else:
|
else:
|
||||||
|
logger.warn("Failed to generate thumbnail")
|
||||||
respond_404(request)
|
respond_404(request)
|
||||||
|
|
||||||
@defer.inlineCallbacks
|
@defer.inlineCallbacks
|
||||||
|
@ -216,7 +227,7 @@ class ThumbnailResource(Resource):
|
||||||
# TODO: Don't download the whole remote file
|
# TODO: Don't download the whole remote file
|
||||||
# We should proxy the thumbnail from the remote server instead of
|
# We should proxy the thumbnail from the remote server instead of
|
||||||
# downloading the remote file and generating our own thumbnails.
|
# downloading the remote file and generating our own thumbnails.
|
||||||
yield self.media_repo.get_remote_media(server_name, media_id)
|
media_info = yield self.media_repo.get_remote_media_info(server_name, media_id)
|
||||||
|
|
||||||
thumbnail_infos = yield self.store.get_remote_media_thumbnails(
|
thumbnail_infos = yield self.store.get_remote_media_thumbnails(
|
||||||
server_name, media_id,
|
server_name, media_id,
|
||||||
|
@ -227,7 +238,7 @@ class ThumbnailResource(Resource):
|
||||||
width, height, method, m_type, thumbnail_infos
|
width, height, method, m_type, thumbnail_infos
|
||||||
)
|
)
|
||||||
file_info = FileInfo(
|
file_info = FileInfo(
|
||||||
server_name=None, file_id=media_id,
|
server_name=server_name, file_id=media_info["filesystem_id"],
|
||||||
thumbnail=True,
|
thumbnail=True,
|
||||||
thumbnail_width=thumbnail_info["thumbnail_width"],
|
thumbnail_width=thumbnail_info["thumbnail_width"],
|
||||||
thumbnail_height=thumbnail_info["thumbnail_height"],
|
thumbnail_height=thumbnail_info["thumbnail_height"],
|
||||||
|
@ -241,6 +252,7 @@ class ThumbnailResource(Resource):
|
||||||
responder = yield self.media_storage.fetch_media(file_info)
|
responder = yield self.media_storage.fetch_media(file_info)
|
||||||
yield respond_with_responder(request, responder, t_type, t_length)
|
yield respond_with_responder(request, responder, t_type, t_length)
|
||||||
else:
|
else:
|
||||||
|
logger.info("Failed to find any generated thumbnails")
|
||||||
respond_404(request)
|
respond_404(request)
|
||||||
|
|
||||||
def _select_thumbnail(self, desired_width, desired_height, desired_method,
|
def _select_thumbnail(self, desired_width, desired_height, desired_method,
|
||||||
|
|
Loading…
Reference in a new issue