Get uploads working with new media repo

This commit is contained in:
Mark Haines 2014-12-02 19:51:47 +00:00
parent 279c48c8b4
commit 5da65085d1
14 changed files with 56 additions and 29 deletions

View file

@ -20,3 +20,4 @@ FEDERATION_PREFIX = "/_matrix/federation/v1"
WEB_CLIENT_PREFIX = "/_matrix/client"
CONTENT_REPO_PREFIX = "/_matrix/content"
SERVER_KEY_PREFIX = "/_matrix/key/v1"
MEDIA_PREFIX = "/_matrix/media/v1"

View file

@ -24,12 +24,13 @@ from twisted.web.resource import Resource
from twisted.web.static import File
from twisted.web.server import Site
from synapse.http.server import JsonResource, RootRedirect
from synapse.http.content_repository import ContentRepoResource
from synapse.media.v0.content_repository import ContentRepoResource
from synapse.media.v1.media_repository import MediaRepositoryResource
from synapse.http.server_key_resource import LocalKey
from synapse.http.matrixfederationclient import MatrixFederationHttpClient
from synapse.api.urls import (
CLIENT_PREFIX, FEDERATION_PREFIX, WEB_CLIENT_PREFIX, CONTENT_REPO_PREFIX,
SERVER_KEY_PREFIX,
SERVER_KEY_PREFIX, MEDIA_PREFIX
)
from synapse.config.homeserver import HomeServerConfig
from synapse.crypto import context_factory
@ -69,6 +70,9 @@ class SynapseHomeServer(HomeServer):
self, self.upload_dir, self.auth, self.content_addr
)
def build_resource_for_media_repository(self):
return MediaRepositoryResource(self)
def build_resource_for_server_key(self):
return LocalKey(self)
@ -99,6 +103,7 @@ class SynapseHomeServer(HomeServer):
(FEDERATION_PREFIX, self.get_resource_for_federation()),
(CONTENT_REPO_PREFIX, self.get_resource_for_content_repo()),
(SERVER_KEY_PREFIX, self.get_resource_for_server_key()),
(MEDIA_PREFIX, self.get_resource_for_media_repository()),
]
if web_client:
logger.info("Adding the web client.")

View file

@ -50,12 +50,26 @@ class Config(object):
)
return cls.abspath(file_path)
@staticmethod
def ensure_directory(dir_path):
if not os.path.exists(dir_path):
os.makedirs(dir_path)
if not os.path.isdir(dir_path):
raise ConfigError(
"%s is not a directory" % (dir_path,)
)
return dir_path
@classmethod
def read_file(cls, file_path, config_name):
cls.check_file(file_path, config_name)
with open(file_path) as file_stream:
return file_stream.read()
@staticmethod
def default_path(name):
return os.path.abspath(os.path.join(os.path.curdir, name))
@staticmethod
def read_config_file(file_path):
with open(file_path) as file_stream:

View file

@ -20,6 +20,7 @@ class ContentRepositoryConfig(Config):
def __init__(self, args):
super(ContentRepositoryConfig, self).__init__(args)
self.max_upload_size = self.parse_size(args.max_upload_size)
self.media_store_path = self.ensure_directory(args.media_store_path)
def parse_size(self, string):
sizes = {"K": 1024, "M": 1024 * 1024}
@ -37,3 +38,6 @@ class ContentRepositoryConfig(Config):
db_group.add_argument(
"--max-upload-size", default="1M"
)
db_group.add_argument(
"--media-store-path", default=cls.default_path("media_store")
)

View file

@ -201,9 +201,9 @@ class RootRedirect(resource.Resource):
def respond_with_json(request, code, json_object, send_cors=False,
response_code_message=None, pretty_print=False):
if not pretty_print:
json_bytes = encode_pretty_printed_json(response_json_object)
json_bytes = encode_pretty_printed_json(json_object)
else:
json_bytes = encode_canonical_json(response_json_object)
json_bytes = encode_canonical_json(json_object)
return respond_with_json_bytes(request, code, json_bytes, send_cors,
response_code_message=response_code_message)

View file

View file

View file

@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from .server import respond_with_json_bytes
from synapse.http.server import respond_with_json_bytes
from synapse.util.stringutils import random_string
from synapse.api.errors import (

View file

View file

@ -13,27 +13,17 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from synapse.http.server import respond_with_json_bytes
from .upload_resource import UploadResource
from .filepath import MediaFilePaths
from synapse.util.stringutils import random_string
from synapse.api.errors import (
cs_exception, SynapseError, CodeMessageException, Codes, cs_error
)
from twisted.web.resource import Resource
from twisted.protocols.basic import FileSender
from twisted.web import server, resource
from twisted.internet import defer
import base64
import json
import logging
import os
import re
logger = logging.getLogger(__name__)
class MediaRepository():
class MediaRepositoryResource(Resource):
"""Profiles file uploading and downloading.
Uploads are POSTed to a resource which returns a token which is used to GET
@ -68,5 +58,6 @@ class MediaRepository():
"""
def __init__(self, hs):
filepaths = MediaFilePaths
Resource.__init__(self)
filepaths = MediaFilePaths(hs.config.media_store_path)
self.putChild("upload", UploadResource(hs, filepaths))

View file

@ -23,6 +23,8 @@ from synapse.api.errors import (
from twisted.web import server, resource
from twisted.internet import defer
import os
import logging
logger = logging.getLogger(__name__)
@ -31,8 +33,9 @@ class UploadResource(resource.Resource):
def __init__(self, hs, filepaths):
self.auth = hs.get_auth()
self.clock = hs.get_clock()
self.store = hs.get_datastore()
self.max_upload_size = hs.config.max_upload_size()
self.max_upload_size = hs.config.max_upload_size
self.filepaths = filepaths
def render_POST(self, request):
@ -45,10 +48,8 @@ class UploadResource(resource.Resource):
@defer.inlineCallbacks
def _async_render_POST(self, request):
auth_user = yield self.auth.get_user_by_req(request)
try:
auth_user = yield self.auth.get_user_by_req(request)
# TODO: The checks here are a bit late. The content will have
# already been uploaded to a tmp file at this point
content_length = request.getHeader("Content-Length")
@ -62,7 +63,7 @@ class UploadResource(resource.Resource):
code=413,
)
headers = request.requestHeaders()
headers = request.requestHeaders
if headers.hasHeader("Content-Type"):
media_type = headers.getRawHeaders("Content-Type")[0]
@ -78,7 +79,8 @@ class UploadResource(resource.Resource):
media_id = random_string(24)
fname = self.filepaths.local_media_file_path(media_id)
fname = self.filepaths.local_media_filepath(media_id)
os.makedirs(os.path.dirname(fname))
# This shouldn't block for very long because the content will have
# already been uploaded at this point.

View file

@ -78,6 +78,7 @@ class BaseHomeServer(object):
'resource_for_web_client',
'resource_for_content_repo',
'resource_for_server_key',
'resource_for_media_repository',
'event_sources',
'ratelimiter',
'keyring',

View file

@ -33,6 +33,7 @@ from .stream import StreamStore
from .transactions import TransactionStore
from .keys import KeyStore
from .event_federation import EventFederationStore
from .media_repository import MediaRepositoryStore
from .state import StateStore
from .signatures import SignatureStore
@ -62,6 +63,7 @@ SCHEMAS = [
"state",
"event_edges",
"event_signatures",
"media_repository",
]
@ -81,7 +83,9 @@ class DataStore(RoomMemberStore, RoomStore,
RegistrationStore, StreamStore, ProfileStore, FeedbackStore,
PresenceStore, TransactionStore,
DirectoryStore, KeyStore, StateStore, SignatureStore,
EventFederationStore, ):
EventFederationStore,
MediaRepositoryStore,
):
def __init__(self, hs):
super(DataStore, self).__init__(hs)

View file

@ -20,10 +20,15 @@ class MediaRepositoryStore(SQLBaseStore):
"""Persistence for attachments and avatars"""
def get_local_media(self, media_id):
"""Get the metadata for a local piece of media
Returns:
None if the media_id doesn't exist.
"""
return self._simple_select_one(
"local_media_repository",
{"media_id": media_id},
("media_type", "media_length", "upload_name", "created_ts"),
True,
)
def store_local_media(self, media_id, media_type, time_now_ms, upload_name,
@ -36,7 +41,7 @@ class MediaRepositoryStore(SQLBaseStore):
"created_ts": time_now_ms,
"upload_name": upload_name,
"media_length": media_length,
"user_id": user_id,
"user_id": user_id.to_string(),
}
)