mirror of
https://github.com/element-hq/synapse
synced 2024-08-17 17:30:52 +00:00
Merge pull request #2611 from matrix-org/dbkr/port_script_drop_nuls
Make the port script drop NUL values in all tables
This commit is contained in:
commit
e025aec028
1 changed files with 19 additions and 7 deletions
|
@ -320,7 +320,7 @@ class Porter(object):
|
||||||
backward_chunk = min(row[0] for row in brows) - 1
|
backward_chunk = min(row[0] for row in brows) - 1
|
||||||
|
|
||||||
rows = frows + brows
|
rows = frows + brows
|
||||||
self._convert_rows(table, headers, rows)
|
rows = self._convert_rows(table, headers, rows)
|
||||||
|
|
||||||
def insert(txn):
|
def insert(txn):
|
||||||
self.postgres_store.insert_many_txn(
|
self.postgres_store.insert_many_txn(
|
||||||
|
@ -556,17 +556,29 @@ class Porter(object):
|
||||||
i for i, h in enumerate(headers) if h in bool_col_names
|
i for i, h in enumerate(headers) if h in bool_col_names
|
||||||
]
|
]
|
||||||
|
|
||||||
|
class BadValueException(Exception):
|
||||||
|
pass
|
||||||
|
|
||||||
def conv(j, col):
|
def conv(j, col):
|
||||||
if j in bool_cols:
|
if j in bool_cols:
|
||||||
return bool(col)
|
return bool(col)
|
||||||
|
elif isinstance(col, basestring) and "\0" in col:
|
||||||
|
logger.warn("DROPPING ROW: NUL value in table %s col %s: %r", table, headers[j], col)
|
||||||
|
raise BadValueException();
|
||||||
return col
|
return col
|
||||||
|
|
||||||
|
outrows = []
|
||||||
for i, row in enumerate(rows):
|
for i, row in enumerate(rows):
|
||||||
rows[i] = tuple(
|
try:
|
||||||
|
outrows.append(tuple(
|
||||||
conv(j, col)
|
conv(j, col)
|
||||||
for j, col in enumerate(row)
|
for j, col in enumerate(row)
|
||||||
if j > 0
|
if j > 0
|
||||||
)
|
))
|
||||||
|
except BadValueException:
|
||||||
|
pass
|
||||||
|
|
||||||
|
return outrows
|
||||||
|
|
||||||
@defer.inlineCallbacks
|
@defer.inlineCallbacks
|
||||||
def _setup_sent_transactions(self):
|
def _setup_sent_transactions(self):
|
||||||
|
@ -594,7 +606,7 @@ class Porter(object):
|
||||||
"select", r,
|
"select", r,
|
||||||
)
|
)
|
||||||
|
|
||||||
self._convert_rows("sent_transactions", headers, rows)
|
rows = self._convert_rows("sent_transactions", headers, rows)
|
||||||
|
|
||||||
inserted_rows = len(rows)
|
inserted_rows = len(rows)
|
||||||
if inserted_rows:
|
if inserted_rows:
|
||||||
|
|
Loading…
Reference in a new issue