mirror of
https://github.com/element-hq/synapse
synced 2024-08-20 02:40:22 +00:00
merge branch 'erikj/ultrajson' of github.com:matrix-org/synapse into erikj/profile_rununtilconcurrent
This commit is contained in:
commit
38a84884da
8 changed files with 47 additions and 20 deletions
|
@ -54,6 +54,8 @@ from synapse.rest.client.v1 import ClientV1RestResource
|
||||||
from synapse.rest.client.v2_alpha import ClientV2AlphaRestResource
|
from synapse.rest.client.v2_alpha import ClientV2AlphaRestResource
|
||||||
from synapse.metrics.resource import MetricsResource, METRICS_PREFIX
|
from synapse.metrics.resource import MetricsResource, METRICS_PREFIX
|
||||||
|
|
||||||
|
from synapse import events
|
||||||
|
|
||||||
from daemonize import Daemonize
|
from daemonize import Daemonize
|
||||||
import twisted.manhole.telnet
|
import twisted.manhole.telnet
|
||||||
|
|
||||||
|
@ -415,6 +417,8 @@ def setup(config_options):
|
||||||
logger.info("Server hostname: %s", config.server_name)
|
logger.info("Server hostname: %s", config.server_name)
|
||||||
logger.info("Server version: %s", version_string)
|
logger.info("Server version: %s", version_string)
|
||||||
|
|
||||||
|
events.USE_FROZEN_DICTS = config.use_frozen_dicts
|
||||||
|
|
||||||
if re.search(":[0-9]+$", config.server_name):
|
if re.search(":[0-9]+$", config.server_name):
|
||||||
domain_with_port = config.server_name
|
domain_with_port = config.server_name
|
||||||
else:
|
else:
|
||||||
|
|
|
@ -28,6 +28,7 @@ class ServerConfig(Config):
|
||||||
self.web_client = config["web_client"]
|
self.web_client = config["web_client"]
|
||||||
self.soft_file_limit = config["soft_file_limit"]
|
self.soft_file_limit = config["soft_file_limit"]
|
||||||
self.daemonize = config.get("daemonize")
|
self.daemonize = config.get("daemonize")
|
||||||
|
self.use_frozen_dicts = config.get("use_frozen_dicts", True)
|
||||||
|
|
||||||
# Attempt to guess the content_addr for the v0 content repostitory
|
# Attempt to guess the content_addr for the v0 content repostitory
|
||||||
content_addr = config.get("content_addr")
|
content_addr = config.get("content_addr")
|
||||||
|
|
|
@ -16,6 +16,12 @@
|
||||||
from synapse.util.frozenutils import freeze
|
from synapse.util.frozenutils import freeze
|
||||||
|
|
||||||
|
|
||||||
|
# Whether we should use frozen_dict in FrozenEvent. Using frozen_dicts prevents
|
||||||
|
# bugs where we accidentally share e.g. signature dicts. However, converting
|
||||||
|
# a dict to frozen_dicts is expensive.
|
||||||
|
USE_FROZEN_DICTS = True
|
||||||
|
|
||||||
|
|
||||||
class _EventInternalMetadata(object):
|
class _EventInternalMetadata(object):
|
||||||
def __init__(self, internal_metadata_dict):
|
def __init__(self, internal_metadata_dict):
|
||||||
self.__dict__ = dict(internal_metadata_dict)
|
self.__dict__ = dict(internal_metadata_dict)
|
||||||
|
@ -122,7 +128,10 @@ class FrozenEvent(EventBase):
|
||||||
|
|
||||||
unsigned = dict(event_dict.pop("unsigned", {}))
|
unsigned = dict(event_dict.pop("unsigned", {}))
|
||||||
|
|
||||||
frozen_dict = freeze(event_dict)
|
if USE_FROZEN_DICTS:
|
||||||
|
frozen_dict = freeze(event_dict)
|
||||||
|
else:
|
||||||
|
frozen_dict = dict(event_dict)
|
||||||
|
|
||||||
super(FrozenEvent, self).__init__(
|
super(FrozenEvent, self).__init__(
|
||||||
frozen_dict,
|
frozen_dict,
|
||||||
|
|
|
@ -19,9 +19,10 @@ from synapse.api.errors import (
|
||||||
)
|
)
|
||||||
from synapse.util.logcontext import LoggingContext, PreserveLoggingContext
|
from synapse.util.logcontext import LoggingContext, PreserveLoggingContext
|
||||||
import synapse.metrics
|
import synapse.metrics
|
||||||
|
import synapse.events
|
||||||
|
|
||||||
from syutil.jsonutil import (
|
from syutil.jsonutil import (
|
||||||
encode_canonical_json, encode_pretty_printed_json
|
encode_canonical_json, encode_pretty_printed_json, encode_json
|
||||||
)
|
)
|
||||||
|
|
||||||
from twisted.internet import defer
|
from twisted.internet import defer
|
||||||
|
@ -168,9 +169,10 @@ class JsonResource(HttpServer, resource.Resource):
|
||||||
|
|
||||||
_PathEntry = collections.namedtuple("_PathEntry", ["pattern", "callback"])
|
_PathEntry = collections.namedtuple("_PathEntry", ["pattern", "callback"])
|
||||||
|
|
||||||
def __init__(self, hs):
|
def __init__(self, hs, canonical_json=True):
|
||||||
resource.Resource.__init__(self)
|
resource.Resource.__init__(self)
|
||||||
|
|
||||||
|
self.canonical_json = canonical_json
|
||||||
self.clock = hs.get_clock()
|
self.clock = hs.get_clock()
|
||||||
self.path_regexs = {}
|
self.path_regexs = {}
|
||||||
self.version_string = hs.version_string
|
self.version_string = hs.version_string
|
||||||
|
@ -256,6 +258,7 @@ class JsonResource(HttpServer, resource.Resource):
|
||||||
response_code_message=response_code_message,
|
response_code_message=response_code_message,
|
||||||
pretty_print=_request_user_agent_is_curl(request),
|
pretty_print=_request_user_agent_is_curl(request),
|
||||||
version_string=self.version_string,
|
version_string=self.version_string,
|
||||||
|
canonical_json=self.canonical_json,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@ -277,11 +280,17 @@ class RootRedirect(resource.Resource):
|
||||||
|
|
||||||
def respond_with_json(request, code, json_object, send_cors=False,
|
def respond_with_json(request, code, json_object, send_cors=False,
|
||||||
response_code_message=None, pretty_print=False,
|
response_code_message=None, pretty_print=False,
|
||||||
version_string=""):
|
version_string="", canonical_json=True):
|
||||||
if pretty_print:
|
if pretty_print:
|
||||||
json_bytes = encode_pretty_printed_json(json_object) + "\n"
|
json_bytes = encode_pretty_printed_json(json_object) + "\n"
|
||||||
else:
|
else:
|
||||||
json_bytes = encode_canonical_json(json_object)
|
if canonical_json:
|
||||||
|
json_bytes = encode_canonical_json(json_object)
|
||||||
|
else:
|
||||||
|
json_bytes = encode_json(
|
||||||
|
json_object, using_frozen_dicts=synapse.events.USE_FROZEN_DICTS
|
||||||
|
)
|
||||||
|
logger.debug("json_bytes: %r %r", json_object, json_bytes)
|
||||||
|
|
||||||
return respond_with_json_bytes(
|
return respond_with_json_bytes(
|
||||||
request, code, json_bytes,
|
request, code, json_bytes,
|
||||||
|
|
|
@ -18,7 +18,7 @@ from distutils.version import LooseVersion
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
REQUIREMENTS = {
|
REQUIREMENTS = {
|
||||||
"syutil>=0.0.6": ["syutil>=0.0.6"],
|
"syutil>=0.0.7": ["syutil>=0.0.7"],
|
||||||
"Twisted==14.0.2": ["twisted==14.0.2"],
|
"Twisted==14.0.2": ["twisted==14.0.2"],
|
||||||
"service_identity>=1.0.0": ["service_identity>=1.0.0"],
|
"service_identity>=1.0.0": ["service_identity>=1.0.0"],
|
||||||
"pyopenssl>=0.14": ["OpenSSL>=0.14"],
|
"pyopenssl>=0.14": ["OpenSSL>=0.14"],
|
||||||
|
@ -30,6 +30,7 @@ REQUIREMENTS = {
|
||||||
"frozendict>=0.4": ["frozendict"],
|
"frozendict>=0.4": ["frozendict"],
|
||||||
"pillow": ["PIL"],
|
"pillow": ["PIL"],
|
||||||
"pydenticon": ["pydenticon"],
|
"pydenticon": ["pydenticon"],
|
||||||
|
"ujson": ["ujson"],
|
||||||
}
|
}
|
||||||
CONDITIONAL_REQUIREMENTS = {
|
CONDITIONAL_REQUIREMENTS = {
|
||||||
"web_client": {
|
"web_client": {
|
||||||
|
@ -52,8 +53,8 @@ def github_link(project, version, egg):
|
||||||
DEPENDENCY_LINKS = [
|
DEPENDENCY_LINKS = [
|
||||||
github_link(
|
github_link(
|
||||||
project="matrix-org/syutil",
|
project="matrix-org/syutil",
|
||||||
version="v0.0.6",
|
version="v0.0.7",
|
||||||
egg="syutil-0.0.6",
|
egg="syutil-0.0.7",
|
||||||
),
|
),
|
||||||
github_link(
|
github_link(
|
||||||
project="matrix-org/matrix-angular-sdk",
|
project="matrix-org/matrix-angular-sdk",
|
||||||
|
|
|
@ -25,7 +25,7 @@ class ClientV1RestResource(JsonResource):
|
||||||
"""A resource for version 1 of the matrix client API."""
|
"""A resource for version 1 of the matrix client API."""
|
||||||
|
|
||||||
def __init__(self, hs):
|
def __init__(self, hs):
|
||||||
JsonResource.__init__(self, hs)
|
JsonResource.__init__(self, hs, canonical_json=False)
|
||||||
self.register_servlets(self, hs)
|
self.register_servlets(self, hs)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
|
|
|
@ -28,7 +28,7 @@ class ClientV2AlphaRestResource(JsonResource):
|
||||||
"""A resource for version 2 alpha of the matrix client API."""
|
"""A resource for version 2 alpha of the matrix client API."""
|
||||||
|
|
||||||
def __init__(self, hs):
|
def __init__(self, hs):
|
||||||
JsonResource.__init__(self, hs)
|
JsonResource.__init__(self, hs, canonical_json=False)
|
||||||
self.register_servlets(self, hs)
|
self.register_servlets(self, hs)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
|
|
|
@ -17,7 +17,7 @@ from _base import SQLBaseStore, _RollbackButIsFineException
|
||||||
|
|
||||||
from twisted.internet import defer, reactor
|
from twisted.internet import defer, reactor
|
||||||
|
|
||||||
from synapse.events import FrozenEvent
|
from synapse.events import FrozenEvent, USE_FROZEN_DICTS
|
||||||
from synapse.events.utils import prune_event
|
from synapse.events.utils import prune_event
|
||||||
|
|
||||||
from synapse.util.logcontext import preserve_context_over_deferred
|
from synapse.util.logcontext import preserve_context_over_deferred
|
||||||
|
@ -26,11 +26,11 @@ from synapse.api.constants import EventTypes
|
||||||
from synapse.crypto.event_signing import compute_event_reference_hash
|
from synapse.crypto.event_signing import compute_event_reference_hash
|
||||||
|
|
||||||
from syutil.base64util import decode_base64
|
from syutil.base64util import decode_base64
|
||||||
from syutil.jsonutil import encode_canonical_json
|
from syutil.jsonutil import encode_json
|
||||||
from contextlib import contextmanager
|
from contextlib import contextmanager
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
import simplejson as json
|
import ujson as json
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
@ -166,8 +166,9 @@ class EventsStore(SQLBaseStore):
|
||||||
allow_none=True,
|
allow_none=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
metadata_json = encode_canonical_json(
|
metadata_json = encode_json(
|
||||||
event.internal_metadata.get_dict()
|
event.internal_metadata.get_dict(),
|
||||||
|
using_frozen_dicts=USE_FROZEN_DICTS
|
||||||
).decode("UTF-8")
|
).decode("UTF-8")
|
||||||
|
|
||||||
# If we have already persisted this event, we don't need to do any
|
# If we have already persisted this event, we don't need to do any
|
||||||
|
@ -235,12 +236,14 @@ class EventsStore(SQLBaseStore):
|
||||||
"event_id": event.event_id,
|
"event_id": event.event_id,
|
||||||
"room_id": event.room_id,
|
"room_id": event.room_id,
|
||||||
"internal_metadata": metadata_json,
|
"internal_metadata": metadata_json,
|
||||||
"json": encode_canonical_json(event_dict).decode("UTF-8"),
|
"json": encode_json(
|
||||||
|
event_dict, using_frozen_dicts=USE_FROZEN_DICTS
|
||||||
|
).decode("UTF-8"),
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
content = encode_canonical_json(
|
content = encode_json(
|
||||||
event.content
|
event.content, using_frozen_dicts=USE_FROZEN_DICTS
|
||||||
).decode("UTF-8")
|
).decode("UTF-8")
|
||||||
|
|
||||||
vals = {
|
vals = {
|
||||||
|
@ -266,8 +269,8 @@ class EventsStore(SQLBaseStore):
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|
||||||
vals["unrecognized_keys"] = encode_canonical_json(
|
vals["unrecognized_keys"] = encode_json(
|
||||||
unrec
|
unrec, using_frozen_dicts=USE_FROZEN_DICTS
|
||||||
).decode("UTF-8")
|
).decode("UTF-8")
|
||||||
|
|
||||||
sql = (
|
sql = (
|
||||||
|
|
Loading…
Reference in a new issue