mirror of
https://github.com/element-hq/synapse
synced 2024-08-20 03:50:20 +00:00
Merge pull request #3245 from NotAFile/batch-iter
Add batch_iter to utils
This commit is contained in:
commit
2aff6eab6d
1 changed files with 18 additions and 0 deletions
|
@ -20,6 +20,8 @@ from twisted.internet import defer, reactor, task
|
||||||
import time
|
import time
|
||||||
import logging
|
import logging
|
||||||
|
|
||||||
|
from itertools import islice
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
@ -79,3 +81,19 @@ class Clock(object):
|
||||||
except Exception:
|
except Exception:
|
||||||
if not ignore_errs:
|
if not ignore_errs:
|
||||||
raise
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def batch_iter(iterable, size):
|
||||||
|
"""batch an iterable up into tuples with a maximum size
|
||||||
|
|
||||||
|
Args:
|
||||||
|
iterable (iterable): the iterable to slice
|
||||||
|
size (int): the maximum batch size
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
an iterator over the chunks
|
||||||
|
"""
|
||||||
|
# make sure we can deal with iterables like lists too
|
||||||
|
sourceiter = iter(iterable)
|
||||||
|
# call islice until it returns an empty tuple
|
||||||
|
return iter(lambda: tuple(islice(sourceiter, size)), ())
|
||||||
|
|
Loading…
Reference in a new issue