flow.record 3.19.dev2__tar.gz → 3.19.dev4__tar.gz

{flow_record-3.19.dev2/flow.record.egg-info → flow_record-3.19.dev4}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: flow.record
-Version: 3.19.dev2
+Version: 3.19.dev4
 Summary: A library for defining and creating structured data (called records) that can be streamed to disk or piped to other tools that use flow.record
 Author-email: Dissect Team <dissect@fox-it.com>
 License: Affero General Public License v3

{flow_record-3.19.dev2 → flow_record-3.19.dev4}/flow/record/adapter/elastic.py

@@ -25,6 +25,7 @@ Read usage: rdump elastic+[PROTOCOL]://[IP]:[PORT]?index=[INDEX]
 
 Optional arguments:
   [API_KEY]: base64 encoded api key to authenticate with (default: False)
+  [QUEUE_SIZE]: maximum queue size for writing records; limits memory usage (default: 100000)
   [INDEX]: name of the index to use (default: records)
   [VERIFY_CERTS]: verify certs of Elasticsearch instance (default: True)
   [HASH_RECORD]: make record unique by hashing record [slow] (default: False)
@@ -43,6 +44,7 @@ class ElasticWriter(AbstractWriter):
         http_compress: str | bool = True,
         hash_record: str | bool = False,
         api_key: str | None = None,
+        queue_size: int = 100000,
         **kwargs,
     ) -> None:
         self.index = index
@@ -50,11 +52,12 @@ class ElasticWriter(AbstractWriter):
         verify_certs = str(verify_certs).lower() in ("1", "true")
         http_compress = str(http_compress).lower() in ("1", "true")
         self.hash_record = str(hash_record).lower() in ("1", "true")
+        queue_size = int(queue_size)
 
         if not uri.lower().startswith(("http://", "https://")):
             uri = "http://" + uri
 
-        self.queue: queue.Queue[Record | StopIteration] = queue.Queue()
+        self.queue: queue.Queue[Record | StopIteration] = queue.Queue(maxsize=queue_size)
         self.event = threading.Event()
 
         self.es = elasticsearch.Elasticsearch(
@@ -128,6 +131,8 @@ class ElasticWriter(AbstractWriter):
             record = self.queue.get()
             if record is StopIteration:
                 break
+            if not record:
+                continue
             yield self.record_to_document(record, index=self.index)
 
     def streaming_bulk_thread(self) -> None:
@@ -145,7 +150,7 @@ class ElasticWriter(AbstractWriter):
         self.event.set()
 
     def write(self, record: Record) -> None:
-        self.queue.put_nowait(record)
+        self.queue.put(record)
 
     def flush(self) -> None:
         pass

{flow_record-3.19.dev2 → flow_record-3.19.dev4}/flow/record/version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '3.19.dev2'
-__version_tuple__ = version_tuple = (3, 19, 'dev2')
+__version__ = version = '3.19.dev4'
+__version_tuple__ = version_tuple = (3, 19, 'dev4')

{flow_record-3.19.dev2 → flow_record-3.19.dev4/flow.record.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: flow.record
-Version: 3.19.dev2
+Version: 3.19.dev4
 Summary: A library for defining and creating structured data (called records) that can be streamed to disk or piped to other tools that use flow.record
 Author-email: Dissect Team <dissect@fox-it.com>
 License: Affero General Public License v3