Merge pull request #243 from matrix-org/markjh/remove_syutil

Replace syutil dependency with smaller, single-purpose libraries

scripts-dev/check_event_hash.py

@@ -1,5 +1,5 @@
 from synapse.crypto.event_signing import *
-from syutil.base64util import encode_base64
+from unpaddedbase64 import encode_base64
 
 import argparse
 import hashlib

scripts-dev/check_signature.py

@@ -1,9 +1,7 @@
 
-from syutil.crypto.jsonsign import verify_signed_json
+from signedjson.sign import verify_signed_json
-from syutil.crypto.signing_key import (
+from signedjson.key import decode_verify_key_bytes, write_signing_keys
-    decode_verify_key_bytes, write_signing_keys
+from unpaddedbase64 import decode_base64
-)
-from syutil.base64util import decode_base64
 
 import urllib2
 import json

scripts-dev/convert_server_keys.py

@@ -4,10 +4,10 @@ import sys
 import json
 import time
 import hashlib
-from syutil.base64util import encode_base64
+from unpaddedbase64 import encode_base64
-from syutil.crypto.signing_key import read_signing_keys
+from signedjson.key import read_signing_keys
-from syutil.crypto.jsonsign import sign_json
+from signedjson.sign import sign_json
-from syutil.jsonutil import encode_canonical_json
+from canonicaljson import encode_canonical_json
 
 
 def select_v1_keys(connection):

scripts-dev/hash_history.py

@@ -6,8 +6,8 @@ from synapse.crypto.event_signing import (
     add_event_pdu_content_hash, compute_pdu_event_reference_hash
 )
 from synapse.api.events.utils import prune_pdu
-from syutil.base64util import encode_base64, decode_base64
+from unpaddedbase64 import encode_base64, decode_base64
-from syutil.jsonutil import encode_canonical_json
+from canonicaljson import encode_canonical_json
 import sqlite3
 import sys
 

scripts/upgrade_db_to_v0.6.0.py

@@ -4,7 +4,7 @@ from synapse.storage._base import SQLBaseStore
 from synapse.storage.signatures import SignatureStore
 from synapse.storage.event_federation import EventFederationStore
 
-from syutil.base64util import encode_base64, decode_base64
+from unpaddedbase64 import encode_base64, decode_base64
 
 from synapse.crypto.event_signing import compute_event_signature
 
@@ -13,12 +13,10 @@ from synapse.events.utils import prune_event
 
 from synapse.crypto.event_signing import check_event_content_hash
 
-from syutil.crypto.jsonsign import (
+from signedjson.sign import verify_signed_json, SignatureVerifyException
-    verify_signed_json, SignatureVerifyException,
+from signedjson.key import decode_verify_key_bytes
-)
-from syutil.crypto.signing_key import decode_verify_key_bytes
 
-from syutil.jsonutil import encode_canonical_json
+from canonicaljson import encode_canonical_json
 
 import argparse
 # import dns.resolver
@@ -26,7 +24,6 @@ import hashlib
 import httplib
 import json
 import sqlite3
-import syutil
 import urllib2
 
 
@@ -324,8 +321,6 @@ if __name__ == "__main__":
     )
     args = parser.parse_args()
 
-    signing_key = syutil.crypto.signing_key.read_signing_keys(
+    signing_key = signedjson.key.read_signing_keys(args.signing_key)
-        args.signing_key
-    )
 
     main(args.database, args.server_name, signing_key[0])

synapse/config/key.py

@@ -13,14 +13,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import os
 from ._base import Config, ConfigError
-import syutil.crypto.signing_key
+
-from syutil.crypto.signing_key import (
-    is_signing_algorithm_supported, decode_verify_key_bytes
-)
-from syutil.base64util import decode_base64
 from synapse.util.stringutils import random_string
+from signedjson.key import (
+    generate_signing_key, is_signing_algorithm_supported,
+    decode_signing_key_base64, decode_verify_key_bytes,
+    read_signing_keys, write_signing_keys, NACL_ED25519
+)
+from unpaddedbase64 import decode_base64
+
+import os
 
 
 class KeyConfig(Config):
@@ -83,9 +86,7 @@ class KeyConfig(Config):
     def read_signing_key(self, signing_key_path):
         signing_keys = self.read_file(signing_key_path, "signing_key")
         try:
-            return syutil.crypto.signing_key.read_signing_keys(
+            return read_signing_keys(signing_keys.splitlines(True))
-                signing_keys.splitlines(True)
-            )
         except Exception:
             raise ConfigError(
                 "Error reading signing_key."
@@ -112,22 +113,18 @@ class KeyConfig(Config):
         if not os.path.exists(signing_key_path):
             with open(signing_key_path, "w") as signing_key_file:
                 key_id = "a_" + random_string(4)
-                syutil.crypto.signing_key.write_signing_keys(
+                write_signing_keys(
-                    signing_key_file,
+                    signing_key_file, (generate_signing_key(key_id),),
-                    (syutil.crypto.signing_key.generate_signing_key(key_id),),
                 )
         else:
             signing_keys = self.read_file(signing_key_path, "signing_key")
             if len(signing_keys.split("\n")[0].split()) == 1:
                 # handle keys in the old format.
                 key_id = "a_" + random_string(4)
-                key = syutil.crypto.signing_key.decode_signing_key_base64(
+                key = decode_signing_key_base64(
-                    syutil.crypto.signing_key.NACL_ED25519,
+                    NACL_ED25519, key_id, signing_keys.split("\n")[0]
-                    key_id,
-                    signing_keys.split("\n")[0]
                 )
                 with open(signing_key_path, "w") as signing_key_file:
-                    syutil.crypto.signing_key.write_signing_keys(
+                    write_signing_keys(
-                        signing_key_file,
+                        signing_key_file, (key,),
-                        (key,),
                     )

synapse/crypto/event_signing.py

@@ -15,11 +15,12 @@
 # limitations under the License.
 
 
-from synapse.events.utils import prune_event
-from syutil.jsonutil import encode_canonical_json
-from syutil.base64util import encode_base64, decode_base64
-from syutil.crypto.jsonsign import sign_json
 from synapse.api.errors import SynapseError, Codes
+from synapse.events.utils import prune_event
+
+from canonicaljson import encode_canonical_json
+from unpaddedbase64 import encode_base64, decode_base64
+from signedjson.sign import sign_json
 
 import hashlib
 import logging

synapse/crypto/keyring.py

@@ -14,21 +14,21 @@
 # limitations under the License.
 
 from synapse.crypto.keyclient import fetch_server_key
-from twisted.internet import defer
-from syutil.crypto.jsonsign import (
-    verify_signed_json, signature_ids, sign_json, encode_canonical_json
-)
-from syutil.crypto.signing_key import (
-    is_signing_algorithm_supported, decode_verify_key_bytes
-)
-from syutil.base64util import decode_base64, encode_base64
 from synapse.api.errors import SynapseError, Codes
-
 from synapse.util.retryutils import get_retry_limiter
 from synapse.util import unwrapFirstError
-
 from synapse.util.async import ObservableDeferred
 
+from twisted.internet import defer
+
+from signedjson.sign import (
+    verify_signed_json, signature_ids, sign_json, encode_canonical_json
+)
+from signedjson.key import (
+    is_signing_algorithm_supported, decode_verify_key_bytes
+)
+from unpaddedbase64 import decode_base64, encode_base64
+
 from OpenSSL import crypto
 
 from collections import namedtuple

synapse/http/client.py

@@ -15,9 +15,10 @@
 
 from synapse.api.errors import CodeMessageException
 from synapse.util.logcontext import preserve_context_over_fn
-from syutil.jsonutil import encode_canonical_json
 import synapse.metrics
 
+from canonicaljson import encode_canonical_json
+
 from twisted.internet import defer, reactor
 from twisted.web.client import (
     Agent, readBody, FileBodyProducer, PartialDownloadError,

synapse/http/matrixfederationclient.py

@@ -25,13 +25,13 @@ from synapse.util.async import sleep
 from synapse.util.logcontext import preserve_context_over_fn
 import synapse.metrics
 
-from syutil.jsonutil import encode_canonical_json
+from canonicaljson import encode_canonical_json
 
 from synapse.api.errors import (
     SynapseError, Codes, HttpResponseException,
 )
 
-from syutil.crypto.jsonsign import sign_json
+from signedjson.sign import sign_json
 
 import simplejson as json
 import logging

synapse/http/server.py

@@ -21,8 +21,8 @@ from synapse.util.logcontext import LoggingContext, PreserveLoggingContext
 import synapse.metrics
 import synapse.events
 
-from syutil.jsonutil import (
+from canonicaljson import (
-    encode_canonical_json, encode_pretty_printed_json, encode_json
+    encode_canonical_json, encode_pretty_printed_json
 )
 
 from twisted.internet import defer
@@ -33,6 +33,7 @@ from twisted.web.util import redirectTo
 import collections
 import logging
 import urllib
+import ujson
 
 logger = logging.getLogger(__name__)
 
@@ -270,12 +271,11 @@ def respond_with_json(request, code, json_object, send_cors=False,
     if pretty_print:
         json_bytes = encode_pretty_printed_json(json_object) + "\n"
     else:
-        if canonical_json:
+        if canonical_json or synapse.events.USE_FROZEN_DICTS:
             json_bytes = encode_canonical_json(json_object)
         else:
-            json_bytes = encode_json(
+            # ujson doesn't like frozen_dicts.
-                json_object, using_frozen_dicts=synapse.events.USE_FROZEN_DICTS
+            json_bytes = ujson.dumps(json_object, ensure_ascii=False)
-            )
 
     return respond_with_json_bytes(
         request, code, json_bytes,

synapse/python_dependencies.py

@@ -18,7 +18,9 @@ from distutils.version import LooseVersion
 logger = logging.getLogger(__name__)
 
 REQUIREMENTS = {
-    "syutil>=0.0.7": ["syutil>=0.0.7"],
+    "unpaddedbase64>=1.0.1": ["unpaddedbase64>=1.0.1"],
+    "canonicaljson>=1.0.0": ["canonicaljson>=1.0.0"],
+    "signedjson>=1.0.0": ["signedjson>=1.0.0"],
     "Twisted>=15.1.0": ["twisted>=15.1.0"],
     "service_identity>=1.0.0": ["service_identity>=1.0.0"],
     "pyopenssl>=0.14": ["OpenSSL>=0.14"],
@@ -54,11 +56,6 @@ def github_link(project, version, egg):
     return "https://github.com/%s/tarball/%s/#egg=%s" % (project, version, egg)
 
 DEPENDENCY_LINKS = [
-    github_link(
-        project="matrix-org/syutil",
-        version="v0.0.7",
-        egg="syutil-0.0.7",
-    ),
     github_link(
         project="matrix-org/matrix-angular-sdk",
         version="v0.6.6",

synapse/rest/client/v1/voip.py

@@ -40,7 +40,7 @@ class VoipRestServlet(ClientV1RestServlet):
         username = "%d:%s" % (expiry, auth_user.to_string())
 
         mac = hmac.new(turnSecret, msg=username, digestmod=hashlib.sha1)
-        # We need to use standard base64 encoding here, *not* syutil's
+        # We need to use standard padded base64 encoding here
         # encode_base64 because we need to add the standard padding to get the
         # same result as the TURN server.
         password = base64.b64encode(mac.digest())

synapse/rest/client/v2_alpha/keys.py

@@ -18,7 +18,8 @@ from twisted.internet import defer
 from synapse.api.errors import SynapseError
 from synapse.http.servlet import RestServlet
 from synapse.types import UserID
-from syutil.jsonutil import encode_canonical_json
+
+from canonicaljson import encode_canonical_json
 
 from ._base import client_v2_pattern
 

synapse/rest/key/v1/server_key_resource.py

@@ -16,9 +16,9 @@
 
 from twisted.web.resource import Resource
 from synapse.http.server import respond_with_json_bytes
-from syutil.crypto.jsonsign import sign_json
+from signedjson.sign import sign_json
-from syutil.base64util import encode_base64
+from unpaddedbase64 import encode_base64
-from syutil.jsonutil import encode_canonical_json
+from canonicaljson import encode_canonical_json
 from OpenSSL import crypto
 import logging
 

synapse/rest/key/v2/local_key_resource.py

@@ -16,9 +16,9 @@
 
 from twisted.web.resource import Resource
 from synapse.http.server import respond_with_json_bytes
-from syutil.crypto.jsonsign import sign_json
+from signedjson.sign import sign_json
-from syutil.base64util import encode_base64
+from unpaddedbase64 import encode_base64
-from syutil.jsonutil import encode_canonical_json
+from canonicaljson import encode_canonical_json
 from hashlib import sha256
 from OpenSSL import crypto
 import logging

synapse/storage/event_federation.py

@@ -17,7 +17,7 @@ from twisted.internet import defer
 
 from ._base import SQLBaseStore
 from synapse.util.caches.descriptors import cached
-from syutil.base64util import encode_base64
+from unpaddedbase64 import encode_base64
 
 import logging
 from Queue import PriorityQueue, Empty

synapse/storage/events.py

@@ -24,7 +24,7 @@ from synapse.util.logcontext import preserve_context_over_deferred
 from synapse.util.logutils import log_function
 from synapse.api.constants import EventTypes
 
-from syutil.jsonutil import encode_json
+from canonicaljson import encode_canonical_json
 from contextlib import contextmanager
 
 import logging
@@ -33,6 +33,13 @@ import ujson as json
 logger = logging.getLogger(__name__)
 
 
+def encode_json(json_object):
+    if USE_FROZEN_DICTS:
+        # ujson doesn't like frozen_dicts
+        return encode_canonical_json(json_object)
+    else:
+        return json.dumps(json_object, ensure_ascii=False)
+
 # These values are used in the `enqueus_event` and `_do_fetch` methods to
 # control how we batch/bulk fetch events from the database.
 # The values are plucked out of thing air to make initial sync run faster
@@ -253,8 +260,7 @@ class EventsStore(SQLBaseStore):
                 )
 
                 metadata_json = encode_json(
-                    event.internal_metadata.get_dict(),
+                    event.internal_metadata.get_dict()
-                    using_frozen_dicts=USE_FROZEN_DICTS
                 ).decode("UTF-8")
 
                 sql = (
@@ -329,12 +335,9 @@ class EventsStore(SQLBaseStore):
                     "event_id": event.event_id,
                     "room_id": event.room_id,
                     "internal_metadata": encode_json(
-                        event.internal_metadata.get_dict(),
+                        event.internal_metadata.get_dict()
-                        using_frozen_dicts=USE_FROZEN_DICTS
-                    ).decode("UTF-8"),
-                    "json": encode_json(
-                        event_dict(event), using_frozen_dicts=USE_FROZEN_DICTS
                     ).decode("UTF-8"),
+                    "json": encode_json(event_dict(event)).decode("UTF-8"),
                 }
                 for event, _ in events_and_contexts
             ],
@@ -353,9 +356,7 @@ class EventsStore(SQLBaseStore):
                     "type": event.type,
                     "processed": True,
                     "outlier": event.internal_metadata.is_outlier(),
-                    "content": encode_json(
+                    "content": encode_json(event.content).decode("UTF-8"),
-                        event.content, using_frozen_dicts=USE_FROZEN_DICTS
-                    ).decode("UTF-8"),
                 }
                 for event, _ in events_and_contexts
             ],

synapse/storage/keys.py

@@ -19,7 +19,7 @@ from synapse.util.caches.descriptors import cachedInlineCallbacks
 from twisted.internet import defer
 
 import OpenSSL
-from syutil.crypto.signing_key import decode_verify_key_bytes
+from signedjson.key import decode_verify_key_bytes
 import hashlib
 
 

synapse/storage/pusher.py

@@ -18,7 +18,7 @@ from twisted.internet import defer
 
 from synapse.api.errors import StoreError
 
-from syutil.jsonutil import encode_canonical_json
+from canonicaljson import encode_canonical_json
 
 import logging
 import simplejson as json

synapse/storage/signatures.py

@@ -17,7 +17,7 @@ from twisted.internet import defer
 
 from _base import SQLBaseStore
 
-from syutil.base64util import encode_base64
+from unpaddedbase64 import encode_base64
 from synapse.crypto.event_signing import compute_event_reference_hash
 
 

synapse/storage/transactions.py

@@ -18,7 +18,7 @@ from synapse.util.caches.descriptors import cached
 
 from collections import namedtuple
 
-from syutil.jsonutil import encode_canonical_json
+from canonicaljson import encode_canonical_json
 import logging
 
 logger = logging.getLogger(__name__)