Merge pull request #243 from matrix-org/markjh/remove_syutil

Replace syutil dependency with smaller, single-purpose libraries
This commit is contained in:
Mark Haines 2015-08-25 10:52:16 +01:00
commit 1c847af28a
22 changed files with 85 additions and 94 deletions

View file

@ -1,5 +1,5 @@
from synapse.crypto.event_signing import *
from syutil.base64util import encode_base64
from unpaddedbase64 import encode_base64
import argparse
import hashlib

View file

@ -1,9 +1,7 @@
from syutil.crypto.jsonsign import verify_signed_json
from syutil.crypto.signing_key import (
decode_verify_key_bytes, write_signing_keys
)
from syutil.base64util import decode_base64
from signedjson.sign import verify_signed_json
from signedjson.key import decode_verify_key_bytes, write_signing_keys
from unpaddedbase64 import decode_base64
import urllib2
import json

View file

@ -4,10 +4,10 @@ import sys
import json
import time
import hashlib
from syutil.base64util import encode_base64
from syutil.crypto.signing_key import read_signing_keys
from syutil.crypto.jsonsign import sign_json
from syutil.jsonutil import encode_canonical_json
from unpaddedbase64 import encode_base64
from signedjson.key import read_signing_keys
from signedjson.sign import sign_json
from canonicaljson import encode_canonical_json
def select_v1_keys(connection):

View file

@ -6,8 +6,8 @@ from synapse.crypto.event_signing import (
add_event_pdu_content_hash, compute_pdu_event_reference_hash
)
from synapse.api.events.utils import prune_pdu
from syutil.base64util import encode_base64, decode_base64
from syutil.jsonutil import encode_canonical_json
from unpaddedbase64 import encode_base64, decode_base64
from canonicaljson import encode_canonical_json
import sqlite3
import sys

View file

@ -4,7 +4,7 @@ from synapse.storage._base import SQLBaseStore
from synapse.storage.signatures import SignatureStore
from synapse.storage.event_federation import EventFederationStore
from syutil.base64util import encode_base64, decode_base64
from unpaddedbase64 import encode_base64, decode_base64
from synapse.crypto.event_signing import compute_event_signature
@ -13,12 +13,10 @@ from synapse.events.utils import prune_event
from synapse.crypto.event_signing import check_event_content_hash
from syutil.crypto.jsonsign import (
verify_signed_json, SignatureVerifyException,
)
from syutil.crypto.signing_key import decode_verify_key_bytes
from signedjson.sign import verify_signed_json, SignatureVerifyException
from signedjson.key import decode_verify_key_bytes
from syutil.jsonutil import encode_canonical_json
from canonicaljson import encode_canonical_json
import argparse
# import dns.resolver
@ -26,7 +24,6 @@ import hashlib
import httplib
import json
import sqlite3
import syutil
import urllib2
@ -324,8 +321,6 @@ if __name__ == "__main__":
)
args = parser.parse_args()
signing_key = syutil.crypto.signing_key.read_signing_keys(
args.signing_key
)
signing_key = signedjson.key.read_signing_keys(args.signing_key)
main(args.database, args.server_name, signing_key[0])

View file

@ -13,14 +13,17 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import os
from ._base import Config, ConfigError
import syutil.crypto.signing_key
from syutil.crypto.signing_key import (
is_signing_algorithm_supported, decode_verify_key_bytes
)
from syutil.base64util import decode_base64
from synapse.util.stringutils import random_string
from signedjson.key import (
generate_signing_key, is_signing_algorithm_supported,
decode_signing_key_base64, decode_verify_key_bytes,
read_signing_keys, write_signing_keys, NACL_ED25519
)
from unpaddedbase64 import decode_base64
import os
class KeyConfig(Config):
@ -83,9 +86,7 @@ class KeyConfig(Config):
def read_signing_key(self, signing_key_path):
signing_keys = self.read_file(signing_key_path, "signing_key")
try:
return syutil.crypto.signing_key.read_signing_keys(
signing_keys.splitlines(True)
)
return read_signing_keys(signing_keys.splitlines(True))
except Exception:
raise ConfigError(
"Error reading signing_key."
@ -112,22 +113,18 @@ class KeyConfig(Config):
if not os.path.exists(signing_key_path):
with open(signing_key_path, "w") as signing_key_file:
key_id = "a_" + random_string(4)
syutil.crypto.signing_key.write_signing_keys(
signing_key_file,
(syutil.crypto.signing_key.generate_signing_key(key_id),),
write_signing_keys(
signing_key_file, (generate_signing_key(key_id),),
)
else:
signing_keys = self.read_file(signing_key_path, "signing_key")
if len(signing_keys.split("\n")[0].split()) == 1:
# handle keys in the old format.
key_id = "a_" + random_string(4)
key = syutil.crypto.signing_key.decode_signing_key_base64(
syutil.crypto.signing_key.NACL_ED25519,
key_id,
signing_keys.split("\n")[0]
key = decode_signing_key_base64(
NACL_ED25519, key_id, signing_keys.split("\n")[0]
)
with open(signing_key_path, "w") as signing_key_file:
syutil.crypto.signing_key.write_signing_keys(
signing_key_file,
(key,),
write_signing_keys(
signing_key_file, (key,),
)

View file

@ -15,11 +15,12 @@
# limitations under the License.
from synapse.events.utils import prune_event
from syutil.jsonutil import encode_canonical_json
from syutil.base64util import encode_base64, decode_base64
from syutil.crypto.jsonsign import sign_json
from synapse.api.errors import SynapseError, Codes
from synapse.events.utils import prune_event
from canonicaljson import encode_canonical_json
from unpaddedbase64 import encode_base64, decode_base64
from signedjson.sign import sign_json
import hashlib
import logging

View file

@ -14,21 +14,21 @@
# limitations under the License.
from synapse.crypto.keyclient import fetch_server_key
from twisted.internet import defer
from syutil.crypto.jsonsign import (
verify_signed_json, signature_ids, sign_json, encode_canonical_json
)
from syutil.crypto.signing_key import (
is_signing_algorithm_supported, decode_verify_key_bytes
)
from syutil.base64util import decode_base64, encode_base64
from synapse.api.errors import SynapseError, Codes
from synapse.util.retryutils import get_retry_limiter
from synapse.util import unwrapFirstError
from synapse.util.async import ObservableDeferred
from twisted.internet import defer
from signedjson.sign import (
verify_signed_json, signature_ids, sign_json, encode_canonical_json
)
from signedjson.key import (
is_signing_algorithm_supported, decode_verify_key_bytes
)
from unpaddedbase64 import decode_base64, encode_base64
from OpenSSL import crypto
from collections import namedtuple

View file

@ -15,9 +15,10 @@
from synapse.api.errors import CodeMessageException
from synapse.util.logcontext import preserve_context_over_fn
from syutil.jsonutil import encode_canonical_json
import synapse.metrics
from canonicaljson import encode_canonical_json
from twisted.internet import defer, reactor
from twisted.web.client import (
Agent, readBody, FileBodyProducer, PartialDownloadError,

View file

@ -25,13 +25,13 @@ from synapse.util.async import sleep
from synapse.util.logcontext import preserve_context_over_fn
import synapse.metrics
from syutil.jsonutil import encode_canonical_json
from canonicaljson import encode_canonical_json
from synapse.api.errors import (
SynapseError, Codes, HttpResponseException,
)
from syutil.crypto.jsonsign import sign_json
from signedjson.sign import sign_json
import simplejson as json
import logging

View file

@ -21,8 +21,8 @@ from synapse.util.logcontext import LoggingContext, PreserveLoggingContext
import synapse.metrics
import synapse.events
from syutil.jsonutil import (
encode_canonical_json, encode_pretty_printed_json, encode_json
from canonicaljson import (
encode_canonical_json, encode_pretty_printed_json
)
from twisted.internet import defer
@ -33,6 +33,7 @@ from twisted.web.util import redirectTo
import collections
import logging
import urllib
import ujson
logger = logging.getLogger(__name__)
@ -270,12 +271,11 @@ def respond_with_json(request, code, json_object, send_cors=False,
if pretty_print:
json_bytes = encode_pretty_printed_json(json_object) + "\n"
else:
if canonical_json:
if canonical_json or synapse.events.USE_FROZEN_DICTS:
json_bytes = encode_canonical_json(json_object)
else:
json_bytes = encode_json(
json_object, using_frozen_dicts=synapse.events.USE_FROZEN_DICTS
)
# ujson doesn't like frozen_dicts.
json_bytes = ujson.dumps(json_object, ensure_ascii=False)
return respond_with_json_bytes(
request, code, json_bytes,

View file

@ -18,7 +18,9 @@ from distutils.version import LooseVersion
logger = logging.getLogger(__name__)
REQUIREMENTS = {
"syutil>=0.0.7": ["syutil>=0.0.7"],
"unpaddedbase64>=1.0.1": ["unpaddedbase64>=1.0.1"],
"canonicaljson>=1.0.0": ["canonicaljson>=1.0.0"],
"signedjson>=1.0.0": ["signedjson>=1.0.0"],
"Twisted>=15.1.0": ["twisted>=15.1.0"],
"service_identity>=1.0.0": ["service_identity>=1.0.0"],
"pyopenssl>=0.14": ["OpenSSL>=0.14"],
@ -54,11 +56,6 @@ def github_link(project, version, egg):
return "https://github.com/%s/tarball/%s/#egg=%s" % (project, version, egg)
DEPENDENCY_LINKS = [
github_link(
project="matrix-org/syutil",
version="v0.0.7",
egg="syutil-0.0.7",
),
github_link(
project="matrix-org/matrix-angular-sdk",
version="v0.6.6",

View file

@ -40,7 +40,7 @@ class VoipRestServlet(ClientV1RestServlet):
username = "%d:%s" % (expiry, auth_user.to_string())
mac = hmac.new(turnSecret, msg=username, digestmod=hashlib.sha1)
# We need to use standard base64 encoding here, *not* syutil's
# We need to use standard padded base64 encoding here
# encode_base64 because we need to add the standard padding to get the
# same result as the TURN server.
password = base64.b64encode(mac.digest())

View file

@ -18,7 +18,8 @@ from twisted.internet import defer
from synapse.api.errors import SynapseError
from synapse.http.servlet import RestServlet
from synapse.types import UserID
from syutil.jsonutil import encode_canonical_json
from canonicaljson import encode_canonical_json
from ._base import client_v2_pattern

View file

@ -16,9 +16,9 @@
from twisted.web.resource import Resource
from synapse.http.server import respond_with_json_bytes
from syutil.crypto.jsonsign import sign_json
from syutil.base64util import encode_base64
from syutil.jsonutil import encode_canonical_json
from signedjson.sign import sign_json
from unpaddedbase64 import encode_base64
from canonicaljson import encode_canonical_json
from OpenSSL import crypto
import logging

View file

@ -16,9 +16,9 @@
from twisted.web.resource import Resource
from synapse.http.server import respond_with_json_bytes
from syutil.crypto.jsonsign import sign_json
from syutil.base64util import encode_base64
from syutil.jsonutil import encode_canonical_json
from signedjson.sign import sign_json
from unpaddedbase64 import encode_base64
from canonicaljson import encode_canonical_json
from hashlib import sha256
from OpenSSL import crypto
import logging

View file

@ -17,7 +17,7 @@ from twisted.internet import defer
from ._base import SQLBaseStore
from synapse.util.caches.descriptors import cached
from syutil.base64util import encode_base64
from unpaddedbase64 import encode_base64
import logging
from Queue import PriorityQueue, Empty

View file

@ -24,7 +24,7 @@ from synapse.util.logcontext import preserve_context_over_deferred
from synapse.util.logutils import log_function
from synapse.api.constants import EventTypes
from syutil.jsonutil import encode_json
from canonicaljson import encode_canonical_json
from contextlib import contextmanager
import logging
@ -33,6 +33,13 @@ import ujson as json
logger = logging.getLogger(__name__)
def encode_json(json_object):
if USE_FROZEN_DICTS:
# ujson doesn't like frozen_dicts
return encode_canonical_json(json_object)
else:
return json.dumps(json_object, ensure_ascii=False)
# These values are used in the `enqueus_event` and `_do_fetch` methods to
# control how we batch/bulk fetch events from the database.
# The values are plucked out of thing air to make initial sync run faster
@ -253,8 +260,7 @@ class EventsStore(SQLBaseStore):
)
metadata_json = encode_json(
event.internal_metadata.get_dict(),
using_frozen_dicts=USE_FROZEN_DICTS
event.internal_metadata.get_dict()
).decode("UTF-8")
sql = (
@ -329,12 +335,9 @@ class EventsStore(SQLBaseStore):
"event_id": event.event_id,
"room_id": event.room_id,
"internal_metadata": encode_json(
event.internal_metadata.get_dict(),
using_frozen_dicts=USE_FROZEN_DICTS
).decode("UTF-8"),
"json": encode_json(
event_dict(event), using_frozen_dicts=USE_FROZEN_DICTS
event.internal_metadata.get_dict()
).decode("UTF-8"),
"json": encode_json(event_dict(event)).decode("UTF-8"),
}
for event, _ in events_and_contexts
],
@ -353,9 +356,7 @@ class EventsStore(SQLBaseStore):
"type": event.type,
"processed": True,
"outlier": event.internal_metadata.is_outlier(),
"content": encode_json(
event.content, using_frozen_dicts=USE_FROZEN_DICTS
).decode("UTF-8"),
"content": encode_json(event.content).decode("UTF-8"),
}
for event, _ in events_and_contexts
],

View file

@ -19,7 +19,7 @@ from synapse.util.caches.descriptors import cachedInlineCallbacks
from twisted.internet import defer
import OpenSSL
from syutil.crypto.signing_key import decode_verify_key_bytes
from signedjson.key import decode_verify_key_bytes
import hashlib

View file

@ -18,7 +18,7 @@ from twisted.internet import defer
from synapse.api.errors import StoreError
from syutil.jsonutil import encode_canonical_json
from canonicaljson import encode_canonical_json
import logging
import simplejson as json

View file

@ -17,7 +17,7 @@ from twisted.internet import defer
from _base import SQLBaseStore
from syutil.base64util import encode_base64
from unpaddedbase64 import encode_base64
from synapse.crypto.event_signing import compute_event_reference_hash

View file

@ -18,7 +18,7 @@ from synapse.util.caches.descriptors import cached
from collections import namedtuple
from syutil.jsonutil import encode_canonical_json
from canonicaljson import encode_canonical_json
import logging
logger = logging.getLogger(__name__)