Merge pull request #2812 from matrix-org/erikj/media_storage_provider_config

Make storage providers configurable
This commit is contained in:
Erik Johnston 2018-01-18 18:33:57 +00:00 committed by GitHub
commit 8ff6726c0d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 101 additions and 33 deletions

View file

@ -16,6 +16,8 @@
from ._base import Config, ConfigError from ._base import Config, ConfigError
from collections import namedtuple from collections import namedtuple
from synapse.util.module_loader import load_module
MISSING_NETADDR = ( MISSING_NETADDR = (
"Missing netaddr library. This is required for URL preview API." "Missing netaddr library. This is required for URL preview API."
@ -36,6 +38,14 @@ ThumbnailRequirement = namedtuple(
"ThumbnailRequirement", ["width", "height", "method", "media_type"] "ThumbnailRequirement", ["width", "height", "method", "media_type"]
) )
MediaStorageProviderConfig = namedtuple(
"MediaStorageProviderConfig", (
"store_local", # Whether to store newly uploaded local files
"store_remote", # Whether to store newly downloaded remote files
"store_synchronous", # Whether to wait for successful storage for local uploads
),
)
def parse_thumbnail_requirements(thumbnail_sizes): def parse_thumbnail_requirements(thumbnail_sizes):
""" Takes a list of dictionaries with "width", "height", and "method" keys """ Takes a list of dictionaries with "width", "height", and "method" keys
@ -73,14 +83,59 @@ class ContentRepositoryConfig(Config):
self.media_store_path = self.ensure_directory(config["media_store_path"]) self.media_store_path = self.ensure_directory(config["media_store_path"])
self.backup_media_store_path = config.get("backup_media_store_path") backup_media_store_path = config.get("backup_media_store_path")
if self.backup_media_store_path:
self.backup_media_store_path = self.ensure_directory( synchronous_backup_media_store = config.get(
self.backup_media_store_path "synchronous_backup_media_store", False
) )
self.synchronous_backup_media_store = config.get( storage_providers = config.get("media_storage_providers", [])
"synchronous_backup_media_store", False
if backup_media_store_path:
if storage_providers:
raise ConfigError(
"Cannot use both 'backup_media_store_path' and 'storage_providers'"
)
storage_providers = [{
"module": "file_system",
"store_local": True,
"store_synchronous": synchronous_backup_media_store,
"store_remote": True,
"config": {
"directory": backup_media_store_path,
}
}]
# This is a list of config that can be used to create the storage
# providers. The entries are tuples of (Class, class_config,
# MediaStorageProviderConfig), where Class is the class of the provider,
# the class_config the config to pass to it, and
# MediaStorageProviderConfig are options for StorageProviderWrapper.
#
# We don't create the storage providers here as not all workers need
# them to be started.
self.media_storage_providers = []
for provider_config in storage_providers:
# We special case the module "file_system" so as not to need to
# expose FileStorageProviderBackend
if provider_config["module"] == "file_system":
provider_config["module"] = (
"synapse.rest.media.v1.storage_provider"
".FileStorageProviderBackend"
)
provider_class, parsed_config = load_module(provider_config)
wrapper_config = MediaStorageProviderConfig(
provider_config.get("store_local", False),
provider_config.get("store_remote", False),
provider_config.get("store_synchronous", False),
)
self.media_storage_providers.append(
(provider_class, parsed_config, wrapper_config,)
) )
self.uploads_path = self.ensure_directory(config["uploads_path"]) self.uploads_path = self.ensure_directory(config["uploads_path"])
@ -127,13 +182,19 @@ class ContentRepositoryConfig(Config):
# Directory where uploaded images and attachments are stored. # Directory where uploaded images and attachments are stored.
media_store_path: "%(media_store)s" media_store_path: "%(media_store)s"
# A secondary directory where uploaded images and attachments are # Media storage providers allow media to be stored in different
# stored as a backup. # locations.
# backup_media_store_path: "%(media_store)s" # media_storage_providers:
# - module: file_system
# Whether to wait for successful write to backup media store before # # Whether to write new local files.
# returning successfully. # store_local: false
# synchronous_backup_media_store: false # # Whether to write new remote media
# store_remote: false
# # Whether to block upload requests waiting for write to this
# # provider to complete
# store_synchronous: false
# config:
# directory: /mnt/some/other/directory
# Directory where in-progress uploads are stored. # Directory where in-progress uploads are stored.
uploads_path: "%(uploads_path)s" uploads_path: "%(uploads_path)s"

View file

@ -27,9 +27,7 @@ from .identicon_resource import IdenticonResource
from .preview_url_resource import PreviewUrlResource from .preview_url_resource import PreviewUrlResource
from .filepath import MediaFilePaths from .filepath import MediaFilePaths
from .thumbnailer import Thumbnailer from .thumbnailer import Thumbnailer
from .storage_provider import ( from .storage_provider import StorageProviderWrapper
StorageProviderWrapper, FileStorageProviderBackend,
)
from .media_storage import MediaStorage from .media_storage import MediaStorage
from synapse.http.matrixfederationclient import MatrixFederationHttpClient from synapse.http.matrixfederationclient import MatrixFederationHttpClient
@ -81,17 +79,13 @@ class MediaRepository(object):
# potentially upload to. # potentially upload to.
storage_providers = [] storage_providers = []
# TODO: Move this into config and allow other storage providers to be for clz, provider_config, wrapper_config in hs.config.media_storage_providers:
# defined. backend = clz(hs, provider_config)
if hs.config.backup_media_store_path:
backend = FileStorageProviderBackend(
self.primary_base_path, hs.config.backup_media_store_path,
)
provider = StorageProviderWrapper( provider = StorageProviderWrapper(
backend, backend,
store=True, store_local=wrapper_config.store_local,
store_synchronous=hs.config.synchronous_backup_media_store, store_remote=wrapper_config.store_remote,
store_remote=True, store_synchronous=wrapper_config.store_synchronous,
) )
storage_providers.append(provider) storage_providers.append(provider)

View file

@ -17,6 +17,7 @@ from twisted.internet import defer, threads
from .media_storage import FileResponder from .media_storage import FileResponder
from synapse.config._base import Config
from synapse.util.logcontext import preserve_fn from synapse.util.logcontext import preserve_fn
import logging import logging
@ -64,14 +65,14 @@ class StorageProviderWrapper(StorageProvider):
Args: Args:
backend (StorageProvider) backend (StorageProvider)
store (bool): Whether to store new files or not. store_local (bool): Whether to store new local files or not.
store_synchronous (bool): Whether to wait for file to be successfully store_synchronous (bool): Whether to wait for file to be successfully
uploaded, or todo the upload in the backgroud. uploaded, or todo the upload in the backgroud.
store_remote (bool): Whether remote media should be uploaded store_remote (bool): Whether remote media should be uploaded
""" """
def __init__(self, backend, store, store_synchronous, store_remote): def __init__(self, backend, store_local, store_synchronous, store_remote):
self.backend = backend self.backend = backend
self.store = store self.store_local = store_local
self.store_synchronous = store_synchronous self.store_synchronous = store_synchronous
self.store_remote = store_remote self.store_remote = store_remote
@ -97,13 +98,13 @@ class FileStorageProviderBackend(StorageProvider):
"""A storage provider that stores files in a directory on a filesystem. """A storage provider that stores files in a directory on a filesystem.
Args: Args:
cache_directory (str): Base path of the local media repository hs (HomeServer)
base_directory (str): Base path to store new files config: The config returned by `parse_config`.
""" """
def __init__(self, cache_directory, base_directory): def __init__(self, hs, config):
self.cache_directory = cache_directory self.cache_directory = hs.config.media_store_path
self.base_directory = base_directory self.base_directory = config
def store_file(self, path, file_info): def store_file(self, path, file_info):
"""See StorageProvider.store_file""" """See StorageProvider.store_file"""
@ -125,3 +126,15 @@ class FileStorageProviderBackend(StorageProvider):
backup_fname = os.path.join(self.base_directory, path) backup_fname = os.path.join(self.base_directory, path)
if os.path.isfile(backup_fname): if os.path.isfile(backup_fname):
return FileResponder(open(backup_fname, "rb")) return FileResponder(open(backup_fname, "rb"))
@staticmethod
def parse_config(config):
"""Called on startup to parse config supplied. This should parse
the config and raise if there is a problem.
The returned value is passed into the constructor.
In this case we only care about a single param, the directory, so let's
just pull that out.
"""
return Config.ensure_directory(config["directory"])