From 162e2c1ce5171c5794f81f1668345bfe7caa6108 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Nov 2015 15:54:50 +0000 Subject: [PATCH 1/5] Fix database port script to work with new event_search table --- scripts/synapse_port_db | 55 ++++++++++++++++++++++++++++++++--------- 1 file changed, 44 insertions(+), 11 deletions(-) diff --git a/scripts/synapse_port_db b/scripts/synapse_port_db index 62515997b1..c0da2af230 100755 --- a/scripts/synapse_port_db +++ b/scripts/synapse_port_db @@ -68,6 +68,7 @@ APPEND_ONLY_TABLES = [ "state_groups_state", "event_to_state_groups", "rejections", + "event_search", ] @@ -229,19 +230,51 @@ class Porter(object): if rows: next_chunk = rows[-1][0] + 1 - self._convert_rows(table, headers, rows) + if table == "event_search": + # We have to treat event_search differnetly since it has a + # different structure in the two different databases. + def insert(txn): + sql = ( + "INSERT INTO event_search (event_id, room_id, key, sender, vector)" + " VALUES (?,?,?,?,to_tsvector('english', ?))" + ) - def insert(txn): - self.postgres_store.insert_many_txn( - txn, table, headers[1:], rows - ) + rows_dict = [ + dict(zip(headers, row)) + for row in rows + ] - self.postgres_store._simple_update_one_txn( - txn, - table="port_from_sqlite3", - keyvalues={"table_name": table}, - updatevalues={"rowid": next_chunk}, - ) + txn.executemany(sql, [ + ( + row["event_id"], + row["room_id"], + row["key"], + row["sender"], + row["value"], + ) + for row in rows_dict + ]) + + self.postgres_store._simple_update_one_txn( + txn, + table="port_from_sqlite3", + keyvalues={"table_name": table}, + updatevalues={"rowid": next_chunk}, + ) + else: + self._convert_rows(table, headers, rows) + + def insert(txn): + self.postgres_store.insert_many_txn( + txn, table, headers[1:], rows + ) + + self.postgres_store._simple_update_one_txn( + txn, + table="port_from_sqlite3", + keyvalues={"table_name": table}, + updatevalues={"rowid": next_chunk}, + ) yield self.postgres_store.execute(insert) From 037ce4c68fbe9f3d869da31cb28fb8d910d398f7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Nov 2015 18:37:03 +0000 Subject: [PATCH 2/5] Split out text for missing config options. This allows packages to more easily override the default messages to include package specific options. --- synapse/config/_base.py | 44 +++++++++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 17 deletions(-) diff --git a/synapse/config/_base.py b/synapse/config/_base.py index c18e0bdbb8..dca3157dfc 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -25,18 +25,27 @@ class ConfigError(Exception): pass +MISSING_REPORT_STATS_CONFIG_INSTRUCTIONS = """\ +Please opt in or out of reporting anonymized homeserver usage statistics, by +setting the `report_stats` key in your config file to either True or False. +""" + +MISSING_REPORT_STATS_SPIEL = """\ +We would really appreciate it if you could help our project out by reporting +anonymized usage statistics from your homeserver. Only very basic aggregate +data (e.g. number of users) will be reported, but it helps us to track the +growth of the Matrix community, and helps us to make Matrix a success, as well +as to convince other networks that they should peer with us. + +Thank you. +""" + +MISSING_SERVER_NAME = """\ +Missing mandatory `server_name` config option. +""" + + class Config(object): - - stats_reporting_begging_spiel = ( - "We would really appreciate it if you could help our project out by" - " reporting anonymized usage statistics from your homeserver. Only very" - " basic aggregate data (e.g. number of users) will be reported, but it" - " helps us to track the growth of the Matrix community, and helps us to" - " make Matrix a success, as well as to convince other networks that they" - " should peer with us." - "\nThank you." - ) - @staticmethod def parse_size(value): if isinstance(value, int) or isinstance(value, long): @@ -215,7 +224,7 @@ class Config(object): if config_args.report_stats is None: config_parser.error( "Please specify either --report-stats=yes or --report-stats=no\n\n" + - cls.stats_reporting_begging_spiel + MISSING_REPORT_STATS_SPIEL ) if not config_files: config_parser.error( @@ -290,6 +299,10 @@ class Config(object): yaml_config = cls.read_config_file(config_file) specified_config.update(yaml_config) + if "server_name" not in specified_config: + sys.stderr.write("\n" + MISSING_SERVER_NAME + "\n") + sys.exit(1) + server_name = specified_config["server_name"] _, config = obj.generate_config( config_dir_path=config_dir_path, @@ -299,11 +312,8 @@ class Config(object): config.update(specified_config) if "report_stats" not in config: sys.stderr.write( - "Please opt in or out of reporting anonymized homeserver usage " - "statistics, by setting the report_stats key in your config file " - " ( " + config_path + " ) " + - "to either True or False.\n\n" + - Config.stats_reporting_begging_spiel + "\n") + "\n" + MISSING_REPORT_STATS_CONFIG_INSTRUCTIONS + "\n" + + MISSING_REPORT_STATS_SPIEL + "\n") sys.exit(1) if generate_keys: From b36144073864b20e465e4db4e261476b0fe021f4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 19 Nov 2015 09:11:42 +0000 Subject: [PATCH 3/5] Spelling --- scripts/synapse_port_db | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/synapse_port_db b/scripts/synapse_port_db index c0da2af230..d4772fcf6e 100755 --- a/scripts/synapse_port_db +++ b/scripts/synapse_port_db @@ -231,7 +231,7 @@ class Porter(object): next_chunk = rows[-1][0] + 1 if table == "event_search": - # We have to treat event_search differnetly since it has a + # We have to treat event_search differently since it has a # different structure in the two different databases. def insert(txn): sql = ( From 06f74068f463fe5a7dde566eadf7c430a30745f6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 19 Nov 2015 13:05:51 +0000 Subject: [PATCH 4/5] Comment --- synapse/config/_base.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/synapse/config/_base.py b/synapse/config/_base.py index dca3157dfc..d0c9972445 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -25,6 +25,8 @@ class ConfigError(Exception): pass +# We split these messages out to allow packages to override with package +# specific instructions. MISSING_REPORT_STATS_CONFIG_INSTRUCTIONS = """\ Please opt in or out of reporting anonymized homeserver usage statistics, by setting the `report_stats` key in your config file to either True or False. From 5fcef78c6a03ab2973e905c5b7e974ac64b9fdbf Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 19 Nov 2015 13:09:48 +0000 Subject: [PATCH 5/5] Bump changes and version --- CHANGES.rst | 5 +++++ synapse/__init__.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 5bea8e82d5..d01eca4004 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,8 @@ +Changes in synapse v0.11.0-r2 (2015-11-19) +========================================== + +* Fix bug in database port script (PR #387) + Changes in synapse v0.11.0-r1 (2015-11-18) ========================================== diff --git a/synapse/__init__.py b/synapse/__init__.py index 5a83f5f3ec..7ff37edf2c 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.11.0-r1" +__version__ = "0.11.0-r2"