mirror of
https://github.com/element-hq/synapse
synced 2024-09-01 07:30:21 +00:00
Merge pull request #2812 from matrix-org/erikj/media_storage_provider_config
Make storage providers configurable
This commit is contained in:
commit
8ff6726c0d
3 changed files with 101 additions and 33 deletions
|
@ -16,6 +16,8 @@
|
||||||
from ._base import Config, ConfigError
|
from ._base import Config, ConfigError
|
||||||
from collections import namedtuple
|
from collections import namedtuple
|
||||||
|
|
||||||
|
from synapse.util.module_loader import load_module
|
||||||
|
|
||||||
|
|
||||||
MISSING_NETADDR = (
|
MISSING_NETADDR = (
|
||||||
"Missing netaddr library. This is required for URL preview API."
|
"Missing netaddr library. This is required for URL preview API."
|
||||||
|
@ -36,6 +38,14 @@ ThumbnailRequirement = namedtuple(
|
||||||
"ThumbnailRequirement", ["width", "height", "method", "media_type"]
|
"ThumbnailRequirement", ["width", "height", "method", "media_type"]
|
||||||
)
|
)
|
||||||
|
|
||||||
|
MediaStorageProviderConfig = namedtuple(
|
||||||
|
"MediaStorageProviderConfig", (
|
||||||
|
"store_local", # Whether to store newly uploaded local files
|
||||||
|
"store_remote", # Whether to store newly downloaded remote files
|
||||||
|
"store_synchronous", # Whether to wait for successful storage for local uploads
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def parse_thumbnail_requirements(thumbnail_sizes):
|
def parse_thumbnail_requirements(thumbnail_sizes):
|
||||||
""" Takes a list of dictionaries with "width", "height", and "method" keys
|
""" Takes a list of dictionaries with "width", "height", and "method" keys
|
||||||
|
@ -73,16 +83,61 @@ class ContentRepositoryConfig(Config):
|
||||||
|
|
||||||
self.media_store_path = self.ensure_directory(config["media_store_path"])
|
self.media_store_path = self.ensure_directory(config["media_store_path"])
|
||||||
|
|
||||||
self.backup_media_store_path = config.get("backup_media_store_path")
|
backup_media_store_path = config.get("backup_media_store_path")
|
||||||
if self.backup_media_store_path:
|
|
||||||
self.backup_media_store_path = self.ensure_directory(
|
|
||||||
self.backup_media_store_path
|
|
||||||
)
|
|
||||||
|
|
||||||
self.synchronous_backup_media_store = config.get(
|
synchronous_backup_media_store = config.get(
|
||||||
"synchronous_backup_media_store", False
|
"synchronous_backup_media_store", False
|
||||||
)
|
)
|
||||||
|
|
||||||
|
storage_providers = config.get("media_storage_providers", [])
|
||||||
|
|
||||||
|
if backup_media_store_path:
|
||||||
|
if storage_providers:
|
||||||
|
raise ConfigError(
|
||||||
|
"Cannot use both 'backup_media_store_path' and 'storage_providers'"
|
||||||
|
)
|
||||||
|
|
||||||
|
storage_providers = [{
|
||||||
|
"module": "file_system",
|
||||||
|
"store_local": True,
|
||||||
|
"store_synchronous": synchronous_backup_media_store,
|
||||||
|
"store_remote": True,
|
||||||
|
"config": {
|
||||||
|
"directory": backup_media_store_path,
|
||||||
|
}
|
||||||
|
}]
|
||||||
|
|
||||||
|
# This is a list of config that can be used to create the storage
|
||||||
|
# providers. The entries are tuples of (Class, class_config,
|
||||||
|
# MediaStorageProviderConfig), where Class is the class of the provider,
|
||||||
|
# the class_config the config to pass to it, and
|
||||||
|
# MediaStorageProviderConfig are options for StorageProviderWrapper.
|
||||||
|
#
|
||||||
|
# We don't create the storage providers here as not all workers need
|
||||||
|
# them to be started.
|
||||||
|
self.media_storage_providers = []
|
||||||
|
|
||||||
|
for provider_config in storage_providers:
|
||||||
|
# We special case the module "file_system" so as not to need to
|
||||||
|
# expose FileStorageProviderBackend
|
||||||
|
if provider_config["module"] == "file_system":
|
||||||
|
provider_config["module"] = (
|
||||||
|
"synapse.rest.media.v1.storage_provider"
|
||||||
|
".FileStorageProviderBackend"
|
||||||
|
)
|
||||||
|
|
||||||
|
provider_class, parsed_config = load_module(provider_config)
|
||||||
|
|
||||||
|
wrapper_config = MediaStorageProviderConfig(
|
||||||
|
provider_config.get("store_local", False),
|
||||||
|
provider_config.get("store_remote", False),
|
||||||
|
provider_config.get("store_synchronous", False),
|
||||||
|
)
|
||||||
|
|
||||||
|
self.media_storage_providers.append(
|
||||||
|
(provider_class, parsed_config, wrapper_config,)
|
||||||
|
)
|
||||||
|
|
||||||
self.uploads_path = self.ensure_directory(config["uploads_path"])
|
self.uploads_path = self.ensure_directory(config["uploads_path"])
|
||||||
self.dynamic_thumbnails = config["dynamic_thumbnails"]
|
self.dynamic_thumbnails = config["dynamic_thumbnails"]
|
||||||
self.thumbnail_requirements = parse_thumbnail_requirements(
|
self.thumbnail_requirements = parse_thumbnail_requirements(
|
||||||
|
@ -127,13 +182,19 @@ class ContentRepositoryConfig(Config):
|
||||||
# Directory where uploaded images and attachments are stored.
|
# Directory where uploaded images and attachments are stored.
|
||||||
media_store_path: "%(media_store)s"
|
media_store_path: "%(media_store)s"
|
||||||
|
|
||||||
# A secondary directory where uploaded images and attachments are
|
# Media storage providers allow media to be stored in different
|
||||||
# stored as a backup.
|
# locations.
|
||||||
# backup_media_store_path: "%(media_store)s"
|
# media_storage_providers:
|
||||||
|
# - module: file_system
|
||||||
# Whether to wait for successful write to backup media store before
|
# # Whether to write new local files.
|
||||||
# returning successfully.
|
# store_local: false
|
||||||
# synchronous_backup_media_store: false
|
# # Whether to write new remote media
|
||||||
|
# store_remote: false
|
||||||
|
# # Whether to block upload requests waiting for write to this
|
||||||
|
# # provider to complete
|
||||||
|
# store_synchronous: false
|
||||||
|
# config:
|
||||||
|
# directory: /mnt/some/other/directory
|
||||||
|
|
||||||
# Directory where in-progress uploads are stored.
|
# Directory where in-progress uploads are stored.
|
||||||
uploads_path: "%(uploads_path)s"
|
uploads_path: "%(uploads_path)s"
|
||||||
|
|
|
@ -27,9 +27,7 @@ from .identicon_resource import IdenticonResource
|
||||||
from .preview_url_resource import PreviewUrlResource
|
from .preview_url_resource import PreviewUrlResource
|
||||||
from .filepath import MediaFilePaths
|
from .filepath import MediaFilePaths
|
||||||
from .thumbnailer import Thumbnailer
|
from .thumbnailer import Thumbnailer
|
||||||
from .storage_provider import (
|
from .storage_provider import StorageProviderWrapper
|
||||||
StorageProviderWrapper, FileStorageProviderBackend,
|
|
||||||
)
|
|
||||||
from .media_storage import MediaStorage
|
from .media_storage import MediaStorage
|
||||||
|
|
||||||
from synapse.http.matrixfederationclient import MatrixFederationHttpClient
|
from synapse.http.matrixfederationclient import MatrixFederationHttpClient
|
||||||
|
@ -81,17 +79,13 @@ class MediaRepository(object):
|
||||||
# potentially upload to.
|
# potentially upload to.
|
||||||
storage_providers = []
|
storage_providers = []
|
||||||
|
|
||||||
# TODO: Move this into config and allow other storage providers to be
|
for clz, provider_config, wrapper_config in hs.config.media_storage_providers:
|
||||||
# defined.
|
backend = clz(hs, provider_config)
|
||||||
if hs.config.backup_media_store_path:
|
|
||||||
backend = FileStorageProviderBackend(
|
|
||||||
self.primary_base_path, hs.config.backup_media_store_path,
|
|
||||||
)
|
|
||||||
provider = StorageProviderWrapper(
|
provider = StorageProviderWrapper(
|
||||||
backend,
|
backend,
|
||||||
store=True,
|
store_local=wrapper_config.store_local,
|
||||||
store_synchronous=hs.config.synchronous_backup_media_store,
|
store_remote=wrapper_config.store_remote,
|
||||||
store_remote=True,
|
store_synchronous=wrapper_config.store_synchronous,
|
||||||
)
|
)
|
||||||
storage_providers.append(provider)
|
storage_providers.append(provider)
|
||||||
|
|
||||||
|
|
|
@ -17,6 +17,7 @@ from twisted.internet import defer, threads
|
||||||
|
|
||||||
from .media_storage import FileResponder
|
from .media_storage import FileResponder
|
||||||
|
|
||||||
|
from synapse.config._base import Config
|
||||||
from synapse.util.logcontext import preserve_fn
|
from synapse.util.logcontext import preserve_fn
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
|
@ -64,14 +65,14 @@ class StorageProviderWrapper(StorageProvider):
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
backend (StorageProvider)
|
backend (StorageProvider)
|
||||||
store (bool): Whether to store new files or not.
|
store_local (bool): Whether to store new local files or not.
|
||||||
store_synchronous (bool): Whether to wait for file to be successfully
|
store_synchronous (bool): Whether to wait for file to be successfully
|
||||||
uploaded, or todo the upload in the backgroud.
|
uploaded, or todo the upload in the backgroud.
|
||||||
store_remote (bool): Whether remote media should be uploaded
|
store_remote (bool): Whether remote media should be uploaded
|
||||||
"""
|
"""
|
||||||
def __init__(self, backend, store, store_synchronous, store_remote):
|
def __init__(self, backend, store_local, store_synchronous, store_remote):
|
||||||
self.backend = backend
|
self.backend = backend
|
||||||
self.store = store
|
self.store_local = store_local
|
||||||
self.store_synchronous = store_synchronous
|
self.store_synchronous = store_synchronous
|
||||||
self.store_remote = store_remote
|
self.store_remote = store_remote
|
||||||
|
|
||||||
|
@ -97,13 +98,13 @@ class FileStorageProviderBackend(StorageProvider):
|
||||||
"""A storage provider that stores files in a directory on a filesystem.
|
"""A storage provider that stores files in a directory on a filesystem.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
cache_directory (str): Base path of the local media repository
|
hs (HomeServer)
|
||||||
base_directory (str): Base path to store new files
|
config: The config returned by `parse_config`.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, cache_directory, base_directory):
|
def __init__(self, hs, config):
|
||||||
self.cache_directory = cache_directory
|
self.cache_directory = hs.config.media_store_path
|
||||||
self.base_directory = base_directory
|
self.base_directory = config
|
||||||
|
|
||||||
def store_file(self, path, file_info):
|
def store_file(self, path, file_info):
|
||||||
"""See StorageProvider.store_file"""
|
"""See StorageProvider.store_file"""
|
||||||
|
@ -125,3 +126,15 @@ class FileStorageProviderBackend(StorageProvider):
|
||||||
backup_fname = os.path.join(self.base_directory, path)
|
backup_fname = os.path.join(self.base_directory, path)
|
||||||
if os.path.isfile(backup_fname):
|
if os.path.isfile(backup_fname):
|
||||||
return FileResponder(open(backup_fname, "rb"))
|
return FileResponder(open(backup_fname, "rb"))
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def parse_config(config):
|
||||||
|
"""Called on startup to parse config supplied. This should parse
|
||||||
|
the config and raise if there is a problem.
|
||||||
|
|
||||||
|
The returned value is passed into the constructor.
|
||||||
|
|
||||||
|
In this case we only care about a single param, the directory, so let's
|
||||||
|
just pull that out.
|
||||||
|
"""
|
||||||
|
return Config.ensure_directory(config["directory"])
|
||||||
|
|
Loading…
Reference in a new issue