mirror of
https://github.com/element-hq/synapse
synced 2024-09-01 20:20:20 +00:00
Merge branch 'develop' into rav/fix_closed_connection_errors
This commit is contained in:
commit
11ab2f56f5
7 changed files with 102 additions and 47 deletions
|
@ -96,7 +96,7 @@ class TlsConfig(Config):
|
|||
# certificates returned by this server match one of the fingerprints.
|
||||
#
|
||||
# Synapse automatically adds the fingerprint of its own certificate
|
||||
# to the list. So if federation traffic is handle directly by synapse
|
||||
# to the list. So if federation traffic is handled directly by synapse
|
||||
# then no modification to the list is required.
|
||||
#
|
||||
# If synapse is run behind a load balancer that handles the TLS then it
|
||||
|
|
|
@ -146,12 +146,10 @@ class MediaRepository(object):
|
|||
media_length=content_length,
|
||||
user_id=auth_user,
|
||||
)
|
||||
media_info = {
|
||||
"media_type": media_type,
|
||||
"media_length": content_length,
|
||||
}
|
||||
|
||||
yield self._generate_thumbnails(None, media_id, media_info)
|
||||
yield self._generate_thumbnails(
|
||||
None, media_id, media_id, media_type,
|
||||
)
|
||||
|
||||
defer.returnValue("mxc://%s/%s" % (self.server_name, media_id))
|
||||
|
||||
|
@ -227,6 +225,34 @@ class MediaRepository(object):
|
|||
else:
|
||||
respond_404(request)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def get_remote_media_info(self, server_name, media_id):
|
||||
"""Gets the media info associated with the remote file, downloading
|
||||
if necessary.
|
||||
|
||||
Args:
|
||||
server_name (str): Remote server_name where the media originated.
|
||||
media_id (str): The media ID of the content (as defined by the
|
||||
remote server).
|
||||
|
||||
Returns:
|
||||
Deferred[dict]: The media_info of the file
|
||||
"""
|
||||
# We linearize here to ensure that we don't try and download remote
|
||||
# media multiple times concurrently
|
||||
key = (server_name, media_id)
|
||||
with (yield self.remote_media_linearizer.queue(key)):
|
||||
responder, media_info = yield self._get_remote_media_impl(
|
||||
server_name, media_id,
|
||||
)
|
||||
|
||||
# Ensure we actually use the responder so that it releases resources
|
||||
if responder:
|
||||
with responder:
|
||||
pass
|
||||
|
||||
defer.returnValue(media_info)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _get_remote_media_impl(self, server_name, media_id):
|
||||
"""Looks for media in local cache, if not there then attempt to
|
||||
|
@ -257,6 +283,7 @@ class MediaRepository(object):
|
|||
# If we have an entry in the DB, try and look for it
|
||||
if media_info:
|
||||
if media_info["quarantined_by"]:
|
||||
logger.info("Media is quarantined")
|
||||
raise NotFoundError()
|
||||
|
||||
responder = yield self.media_storage.fetch_media(file_info)
|
||||
|
@ -384,7 +411,7 @@ class MediaRepository(object):
|
|||
}
|
||||
|
||||
yield self._generate_thumbnails(
|
||||
server_name, media_id, media_info
|
||||
server_name, media_id, file_id, media_type,
|
||||
)
|
||||
|
||||
defer.returnValue(media_info)
|
||||
|
@ -496,21 +523,22 @@ class MediaRepository(object):
|
|||
defer.returnValue(output_path)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _generate_thumbnails(self, server_name, media_id, media_info, url_cache=False):
|
||||
def _generate_thumbnails(self, server_name, media_id, file_id, media_type,
|
||||
url_cache=False):
|
||||
"""Generate and store thumbnails for an image.
|
||||
|
||||
Args:
|
||||
server_name (str|None): The server name if remote media, else None if local
|
||||
media_id(str)
|
||||
media_info(dict)
|
||||
media_id (str): The media ID of the content. (This is the same as
|
||||
the file_id for local content)
|
||||
file_id (str): Local file ID
|
||||
media_type (str): The content type of the file
|
||||
url_cache (bool): If we are thumbnailing images downloaded for the URL cache,
|
||||
used exclusively by the url previewer
|
||||
|
||||
Returns:
|
||||
Deferred[dict]: Dict with "width" and "height" keys of original image
|
||||
"""
|
||||
media_type = media_info["media_type"]
|
||||
file_id = media_info.get("filesystem_id")
|
||||
requirements = self._get_thumbnail_requirements(media_type)
|
||||
if not requirements:
|
||||
return
|
||||
|
@ -568,7 +596,7 @@ class MediaRepository(object):
|
|||
try:
|
||||
file_info = FileInfo(
|
||||
server_name=server_name,
|
||||
file_id=media_id,
|
||||
file_id=file_id,
|
||||
thumbnail=True,
|
||||
thumbnail_width=t_width,
|
||||
thumbnail_height=t_height,
|
||||
|
|
|
@ -185,8 +185,10 @@ class PreviewUrlResource(Resource):
|
|||
logger.debug("got media_info of '%s'" % media_info)
|
||||
|
||||
if _is_media(media_info['media_type']):
|
||||
file_id = media_info['filesystem_id']
|
||||
dims = yield self.media_repo._generate_thumbnails(
|
||||
None, media_info['filesystem_id'], media_info, url_cache=True,
|
||||
None, file_id, file_id, media_info["media_type"],
|
||||
url_cache=True,
|
||||
)
|
||||
|
||||
og = {
|
||||
|
@ -231,8 +233,10 @@ class PreviewUrlResource(Resource):
|
|||
|
||||
if _is_media(image_info['media_type']):
|
||||
# TODO: make sure we don't choke on white-on-transparent images
|
||||
file_id = image_info['filesystem_id']
|
||||
dims = yield self.media_repo._generate_thumbnails(
|
||||
None, image_info['filesystem_id'], image_info, url_cache=True,
|
||||
None, file_id, file_id, image_info["media_type"],
|
||||
url_cache=True,
|
||||
)
|
||||
if dims:
|
||||
og["og:image:width"] = dims['width']
|
||||
|
|
|
@ -84,7 +84,11 @@ class ThumbnailResource(Resource):
|
|||
method, m_type):
|
||||
media_info = yield self.store.get_local_media(media_id)
|
||||
|
||||
if not media_info or media_info["quarantined_by"]:
|
||||
if not media_info:
|
||||
respond_404(request)
|
||||
return
|
||||
if media_info["quarantined_by"]:
|
||||
logger.info("Media is quarantined")
|
||||
respond_404(request)
|
||||
return
|
||||
|
||||
|
@ -111,6 +115,7 @@ class ThumbnailResource(Resource):
|
|||
responder = yield self.media_storage.fetch_media(file_info)
|
||||
yield respond_with_responder(request, responder, t_type, t_length)
|
||||
else:
|
||||
logger.info("Couldn't find any generated thumbnails")
|
||||
respond_404(request)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
|
@ -119,7 +124,11 @@ class ThumbnailResource(Resource):
|
|||
desired_type):
|
||||
media_info = yield self.store.get_local_media(media_id)
|
||||
|
||||
if not media_info or media_info["quarantined_by"]:
|
||||
if not media_info:
|
||||
respond_404(request)
|
||||
return
|
||||
if media_info["quarantined_by"]:
|
||||
logger.info("Media is quarantined")
|
||||
respond_404(request)
|
||||
return
|
||||
|
||||
|
@ -149,7 +158,7 @@ class ThumbnailResource(Resource):
|
|||
yield respond_with_responder(request, responder, t_type, t_length)
|
||||
return
|
||||
|
||||
logger.debug("We don't have a local thumbnail of that size. Generating")
|
||||
logger.debug("We don't have a thumbnail of that size. Generating")
|
||||
|
||||
# Okay, so we generate one.
|
||||
file_path = yield self.media_repo.generate_local_exact_thumbnail(
|
||||
|
@ -159,13 +168,14 @@ class ThumbnailResource(Resource):
|
|||
if file_path:
|
||||
yield respond_with_file(request, desired_type, file_path)
|
||||
else:
|
||||
logger.warn("Failed to generate thumbnail")
|
||||
respond_404(request)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _select_or_generate_remote_thumbnail(self, request, server_name, media_id,
|
||||
desired_width, desired_height,
|
||||
desired_method, desired_type):
|
||||
media_info = yield self.media_repo.get_remote_media(server_name, media_id)
|
||||
media_info = yield self.media_repo.get_remote_media_info(server_name, media_id)
|
||||
|
||||
thumbnail_infos = yield self.store.get_remote_media_thumbnails(
|
||||
server_name, media_id,
|
||||
|
@ -181,7 +191,7 @@ class ThumbnailResource(Resource):
|
|||
|
||||
if t_w and t_h and t_method and t_type:
|
||||
file_info = FileInfo(
|
||||
server_name=None, file_id=media_id,
|
||||
server_name=server_name, file_id=media_info["filesystem_id"],
|
||||
thumbnail=True,
|
||||
thumbnail_width=info["thumbnail_width"],
|
||||
thumbnail_height=info["thumbnail_height"],
|
||||
|
@ -197,7 +207,7 @@ class ThumbnailResource(Resource):
|
|||
yield respond_with_responder(request, responder, t_type, t_length)
|
||||
return
|
||||
|
||||
logger.debug("We don't have a local thumbnail of that size. Generating")
|
||||
logger.debug("We don't have a thumbnail of that size. Generating")
|
||||
|
||||
# Okay, so we generate one.
|
||||
file_path = yield self.media_repo.generate_remote_exact_thumbnail(
|
||||
|
@ -208,6 +218,7 @@ class ThumbnailResource(Resource):
|
|||
if file_path:
|
||||
yield respond_with_file(request, desired_type, file_path)
|
||||
else:
|
||||
logger.warn("Failed to generate thumbnail")
|
||||
respond_404(request)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
|
@ -216,7 +227,7 @@ class ThumbnailResource(Resource):
|
|||
# TODO: Don't download the whole remote file
|
||||
# We should proxy the thumbnail from the remote server instead of
|
||||
# downloading the remote file and generating our own thumbnails.
|
||||
yield self.media_repo.get_remote_media(server_name, media_id)
|
||||
media_info = yield self.media_repo.get_remote_media_info(server_name, media_id)
|
||||
|
||||
thumbnail_infos = yield self.store.get_remote_media_thumbnails(
|
||||
server_name, media_id,
|
||||
|
@ -227,7 +238,7 @@ class ThumbnailResource(Resource):
|
|||
width, height, method, m_type, thumbnail_infos
|
||||
)
|
||||
file_info = FileInfo(
|
||||
server_name=None, file_id=media_id,
|
||||
server_name=server_name, file_id=media_info["filesystem_id"],
|
||||
thumbnail=True,
|
||||
thumbnail_width=thumbnail_info["thumbnail_width"],
|
||||
thumbnail_height=thumbnail_info["thumbnail_height"],
|
||||
|
@ -241,6 +252,7 @@ class ThumbnailResource(Resource):
|
|||
responder = yield self.media_storage.fetch_media(file_info)
|
||||
yield respond_with_responder(request, responder, t_type, t_length)
|
||||
else:
|
||||
logger.info("Failed to find any generated thumbnails")
|
||||
respond_404(request)
|
||||
|
||||
def _select_thumbnail(self, desired_width, desired_height, desired_method,
|
||||
|
|
|
@ -52,13 +52,16 @@ except Exception:
|
|||
class LoggingContext(object):
|
||||
"""Additional context for log formatting. Contexts are scoped within a
|
||||
"with" block.
|
||||
|
||||
Args:
|
||||
name (str): Name for the context for debugging.
|
||||
"""
|
||||
|
||||
__slots__ = [
|
||||
"previous_context", "name", "usage_start", "usage_end", "main_thread",
|
||||
"__dict__", "tag", "alive",
|
||||
"previous_context", "name", "ru_stime", "ru_utime",
|
||||
"db_txn_count", "db_txn_duration", "usage_start", "usage_end",
|
||||
"main_thread", "alive",
|
||||
"request", "tag",
|
||||
]
|
||||
|
||||
thread_local = threading.local()
|
||||
|
@ -96,7 +99,9 @@ class LoggingContext(object):
|
|||
self.db_txn_count = 0
|
||||
self.db_txn_duration = 0.
|
||||
self.usage_start = None
|
||||
self.usage_end = None
|
||||
self.main_thread = threading.current_thread()
|
||||
self.request = None
|
||||
self.tag = ""
|
||||
self.alive = True
|
||||
|
||||
|
@ -105,7 +110,11 @@ class LoggingContext(object):
|
|||
|
||||
@classmethod
|
||||
def current_context(cls):
|
||||
"""Get the current logging context from thread local storage"""
|
||||
"""Get the current logging context from thread local storage
|
||||
|
||||
Returns:
|
||||
LoggingContext: the current logging context
|
||||
"""
|
||||
return getattr(cls.thread_local, "current_context", cls.sentinel)
|
||||
|
||||
@classmethod
|
||||
|
@ -155,11 +164,13 @@ class LoggingContext(object):
|
|||
self.alive = False
|
||||
|
||||
def copy_to(self, record):
|
||||
"""Copy fields from this context to the record"""
|
||||
for key, value in self.__dict__.items():
|
||||
setattr(record, key, value)
|
||||
"""Copy logging fields from this context to a log record or
|
||||
another LoggingContext
|
||||
"""
|
||||
|
||||
record.ru_utime, record.ru_stime = self.get_resource_usage()
|
||||
# 'request' is the only field we currently use in the logger, so that's
|
||||
# all we need to copy
|
||||
record.request = self.request
|
||||
|
||||
def start(self):
|
||||
if threading.current_thread() is not self.main_thread:
|
||||
|
|
|
@ -68,7 +68,7 @@ class KeyringTestCase(unittest.TestCase):
|
|||
|
||||
def check_context(self, _, expected):
|
||||
self.assertEquals(
|
||||
getattr(LoggingContext.current_context(), "test_key", None),
|
||||
getattr(LoggingContext.current_context(), "request", None),
|
||||
expected
|
||||
)
|
||||
|
||||
|
@ -82,7 +82,7 @@ class KeyringTestCase(unittest.TestCase):
|
|||
lookup_2_deferred = defer.Deferred()
|
||||
|
||||
with LoggingContext("one") as context_one:
|
||||
context_one.test_key = "one"
|
||||
context_one.request = "one"
|
||||
|
||||
wait_1_deferred = kr.wait_for_previous_lookups(
|
||||
["server1"],
|
||||
|
@ -96,7 +96,7 @@ class KeyringTestCase(unittest.TestCase):
|
|||
wait_1_deferred.addBoth(self.check_context, "one")
|
||||
|
||||
with LoggingContext("two") as context_two:
|
||||
context_two.test_key = "two"
|
||||
context_two.request = "two"
|
||||
|
||||
# set off another wait. It should block because the first lookup
|
||||
# hasn't yet completed.
|
||||
|
@ -137,7 +137,7 @@ class KeyringTestCase(unittest.TestCase):
|
|||
@defer.inlineCallbacks
|
||||
def get_perspectives(**kwargs):
|
||||
self.assertEquals(
|
||||
LoggingContext.current_context().test_key, "11",
|
||||
LoggingContext.current_context().request, "11",
|
||||
)
|
||||
with logcontext.PreserveLoggingContext():
|
||||
yield persp_deferred
|
||||
|
@ -145,7 +145,7 @@ class KeyringTestCase(unittest.TestCase):
|
|||
self.http_client.post_json.side_effect = get_perspectives
|
||||
|
||||
with LoggingContext("11") as context_11:
|
||||
context_11.test_key = "11"
|
||||
context_11.request = "11"
|
||||
|
||||
# start off a first set of lookups
|
||||
res_deferreds = kr.verify_json_objects_for_server(
|
||||
|
@ -173,7 +173,7 @@ class KeyringTestCase(unittest.TestCase):
|
|||
self.assertIs(LoggingContext.current_context(), context_11)
|
||||
|
||||
context_12 = LoggingContext("12")
|
||||
context_12.test_key = "12"
|
||||
context_12.request = "12"
|
||||
with logcontext.PreserveLoggingContext(context_12):
|
||||
# a second request for a server with outstanding requests
|
||||
# should block rather than start a second call
|
||||
|
@ -211,7 +211,7 @@ class KeyringTestCase(unittest.TestCase):
|
|||
sentinel_context = LoggingContext.current_context()
|
||||
|
||||
with LoggingContext("one") as context_one:
|
||||
context_one.test_key = "one"
|
||||
context_one.request = "one"
|
||||
|
||||
defer = kr.verify_json_for_server("server9", {})
|
||||
try:
|
||||
|
|
|
@ -12,12 +12,12 @@ class LoggingContextTestCase(unittest.TestCase):
|
|||
|
||||
def _check_test_key(self, value):
|
||||
self.assertEquals(
|
||||
LoggingContext.current_context().test_key, value
|
||||
LoggingContext.current_context().request, value
|
||||
)
|
||||
|
||||
def test_with_context(self):
|
||||
with LoggingContext() as context_one:
|
||||
context_one.test_key = "test"
|
||||
context_one.request = "test"
|
||||
self._check_test_key("test")
|
||||
|
||||
@defer.inlineCallbacks
|
||||
|
@ -25,14 +25,14 @@ class LoggingContextTestCase(unittest.TestCase):
|
|||
@defer.inlineCallbacks
|
||||
def competing_callback():
|
||||
with LoggingContext() as competing_context:
|
||||
competing_context.test_key = "competing"
|
||||
competing_context.request = "competing"
|
||||
yield sleep(0)
|
||||
self._check_test_key("competing")
|
||||
|
||||
reactor.callLater(0, competing_callback)
|
||||
|
||||
with LoggingContext() as context_one:
|
||||
context_one.test_key = "one"
|
||||
context_one.request = "one"
|
||||
yield sleep(0)
|
||||
self._check_test_key("one")
|
||||
|
||||
|
@ -43,14 +43,14 @@ class LoggingContextTestCase(unittest.TestCase):
|
|||
|
||||
@defer.inlineCallbacks
|
||||
def cb():
|
||||
context_one.test_key = "one"
|
||||
context_one.request = "one"
|
||||
yield function()
|
||||
self._check_test_key("one")
|
||||
|
||||
callback_completed[0] = True
|
||||
|
||||
with LoggingContext() as context_one:
|
||||
context_one.test_key = "one"
|
||||
context_one.request = "one"
|
||||
|
||||
# fire off function, but don't wait on it.
|
||||
logcontext.preserve_fn(cb)()
|
||||
|
@ -107,7 +107,7 @@ class LoggingContextTestCase(unittest.TestCase):
|
|||
sentinel_context = LoggingContext.current_context()
|
||||
|
||||
with LoggingContext() as context_one:
|
||||
context_one.test_key = "one"
|
||||
context_one.request = "one"
|
||||
|
||||
d1 = logcontext.make_deferred_yieldable(blocking_function())
|
||||
# make sure that the context was reset by make_deferred_yieldable
|
||||
|
@ -124,7 +124,7 @@ class LoggingContextTestCase(unittest.TestCase):
|
|||
argument isn't actually a deferred"""
|
||||
|
||||
with LoggingContext() as context_one:
|
||||
context_one.test_key = "one"
|
||||
context_one.request = "one"
|
||||
|
||||
d1 = logcontext.make_deferred_yieldable("bum")
|
||||
self._check_test_key("one")
|
||||
|
|
Loading…
Reference in a new issue