datafun 0.5.1__tar.gz → 0.5.2__tar.gz

{datafun-0.5.1 → datafun-0.5.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datafun
-Version: 0.5.1
+Version: 0.5.2
 Summary: datafun brings the fun back to data pipelines
 Author-email: "Diego Giorgini, Luigi Di Sotto, Saeed Choobani" <diego.giorgini@aitechnologies.it>
 Requires-Python: >=3.8

{datafun-0.5.1 → datafun-0.5.2}/datafun/dataset.py

@@ -312,7 +312,7 @@ class DatasetSource(Dataset):
 
     def __call__(self, stream: Stream, **kwargs) -> Generator[Stream, None, None]:
         if not isinstance(stream, StartOfStream):
-            raise TypeError(f"Source datasets only process StartOfStream requests.")
+            raise TypeError("Source datasets only process StartOfStream requests.")
 
         # propagate StartOfStream
         _ = self.forward(stream)
@@ -683,7 +683,7 @@ class CacheDatasetSource(DatasetSource):
 
     def _generate_examples(self) -> Generator[tuple, None, None]:
         # Restart called after a keyboard interrupt in the middle of a generation
-        if self.cached_data != [] and self.finished == False:
+        if self.cached_data != [] and self.finished is False:
             self.cached_data = []
 
         if self.cached_data == []:
@@ -1014,11 +1014,11 @@ class Aggregate(DatasetNode):
         **kwargs,
     ):
         if not isinstance(init, Callable):
-            raise TypeError(f"Aggregate: Argument init is not a function")
+            raise TypeError("Aggregate: Argument init is not a function")
         if not isinstance(agg, Callable):
-            raise TypeError(f"Aggregate: Argument agg is not a function")
+            raise TypeError("Aggregate: Argument agg is not a function")
         if not isinstance(agg, Callable):
-            raise TypeError(f"Aggregate: Argument reduce is not a function")
+            raise TypeError("Aggregate: Argument reduce is not a function")
 
         super().__init__(
             function=agg,

{datafun-0.5.1 → datafun-0.5.2}/datafun/sources/elk.py

@@ -57,7 +57,7 @@ class ELKDataset(DatasetSource):
             hits = dl.get(res, 'hits.hits', default=[])
 
             old_scroll_id = dl.get(res, '_scroll_id')
-            while (scroll_sz := len(hits)) > 0:
+            while len(hits) > 0:
                 # iterate over the document hits for each 'scroll'
                 for doc in hits:  # type: ignore
                     yield doc
@@ -74,6 +74,8 @@ class ELKDataset(DatasetSource):
                     # _scroll_id is None, because the response is not "paginated"
                     hits = []
 
+            self.es.clear_scroll(scroll_id=old_scroll_id)
+
     def set_time_interval(self, query: dict) -> dict:
         if not isinstance(query, dict):
             raise ValueError(f"query must be a dict, not {type(query)}.")
@@ -88,7 +90,7 @@ class ELKDataset(DatasetSource):
 
         xs = dl.get(query, 'query.bool.filter')
         if xs is None:
-            raise ValueError(f'Field query.bool.filter has not been found.')
+            raise ValueError('Field query.bool.filter has not been found.')
         if not isinstance(xs, List):
             raise TypeError(f'Field query.bool.filter must be of type List, but found of type {type(xs)}')
 

{datafun-0.5.1 → datafun-0.5.2}/datafun.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datafun
-Version: 0.5.1
+Version: 0.5.2
 Summary: datafun brings the fun back to data pipelines
 Author-email: "Diego Giorgini, Luigi Di Sotto, Saeed Choobani" <diego.giorgini@aitechnologies.it>
 Requires-Python: >=3.8

{datafun-0.5.1 → datafun-0.5.2}/pyproject.toml

@@ -8,7 +8,7 @@ authors = [
     { name = "Diego Giorgini, Luigi Di Sotto, Saeed Choobani", email = "diego.giorgini@aitechnologies.it" }
 ]
 description = "datafun brings the fun back to data pipelines"
-version = "0.5.1"
+version = "0.5.2"
 requires-python = ">=3.8"
 dependencies = [
     "backoff",