nv-ingest-client 2025.9.30.dev20250930__tar.gz → 2025.10.1.dev20251001__tar.gz

{nv_ingest_client-2025.9.30.dev20250930/src/nv_ingest_client.egg-info → nv_ingest_client-2025.10.1.dev20251001}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nv-ingest-client
-Version: 2025.9.30.dev20250930
+Version: 2025.10.1.dev20251001
 Summary: Python client for the nv-ingest service
 Author-email: Jeremy Dyer <jdyer@nvidia.com>
 License:                                  Apache License

{nv_ingest_client-2025.9.30.dev20250930 → nv_ingest_client-2025.10.1.dev20251001}/src/nv_ingest_client/client/interface.py

@@ -6,6 +6,7 @@
 
 import collections
 import glob
+import gzip
 import json
 import logging
 import os
@@ -93,17 +94,20 @@ def ensure_job_specs(func):
 
 
 class LazyLoadedList(collections.abc.Sequence):
-    def __init__(self, filepath: str, expected_len: Optional[int] = None):
+    def __init__(self, filepath: str, expected_len: Optional[int] = None, compression: Optional[str] = None):
         self.filepath = filepath
         self._len: Optional[int] = expected_len  # Store pre-calculated length
         self._offsets: Optional[List[int]] = None
+        self.compression = compression
 
         if self._len == 0:
             self._offsets = []
 
+        self._open = gzip.open if self.compression == "gzip" else open
+
     def __iter__(self) -> Iterator[Any]:
         try:
-            with open(self.filepath, "r", encoding="utf-8") as f:
+            with self._open(self.filepath, "rt", encoding="utf-8") as f:
                 for line in f:
                     yield json.loads(line)
         except FileNotFoundError:
@@ -120,7 +124,7 @@ class LazyLoadedList(collections.abc.Sequence):
         self._offsets = []
         line_count = 0
         try:
-            with open(self.filepath, "rb") as f:
+            with self._open(self.filepath, "rb") as f:
                 while True:
                     current_pos = f.tell()
                     line = f.readline()
@@ -144,10 +148,12 @@ class LazyLoadedList(collections.abc.Sequence):
     def __len__(self) -> int:
         if self._len is not None:
             return self._len
+
         if self._offsets is not None:
             self._len = len(self._offsets)
             return self._len
         self._build_index()
+
         return self._len if self._len is not None else 0
 
     def __getitem__(self, idx: int) -> Any:
@@ -170,7 +176,7 @@ class LazyLoadedList(collections.abc.Sequence):
             raise IndexError(f"Index {idx} out of range for {self.filepath} (len: {len(self._offsets)})")
 
         try:
-            with open(self.filepath, "rb") as f:
+            with self._open(self.filepath, "rb") as f:
                 f.seek(self._offsets[idx])
                 line_bytes = f.readline()
                 return json.loads(line_bytes.decode("utf-8"))
@@ -455,6 +461,8 @@ class Ingestor:
                 clean_source_basename = get_valid_filename(os.path.basename(source_name))
                 file_name, file_ext = os.path.splitext(clean_source_basename)
                 file_suffix = f".{file_ext.strip('.')}.results.jsonl"
+                if self._output_config["compression"] == "gzip":
+                    file_suffix += ".gz"
                 jsonl_filepath = os.path.join(output_dir, safe_filename(output_dir, file_name, file_suffix))
 
                 num_items_saved = save_document_results_to_jsonl(
@@ -462,10 +470,13 @@ class Ingestor:
                     jsonl_filepath,
                     source_name,
                     ensure_parent_dir_exists=False,
+                    compression=self._output_config["compression"],
                 )
 
                 if num_items_saved > 0:
-                    results = LazyLoadedList(jsonl_filepath, expected_len=num_items_saved)
+                    results = LazyLoadedList(
+                        jsonl_filepath, expected_len=num_items_saved, compression=self._output_config["compression"]
+                    )
                     if results_lock:
                         with results_lock:
                             final_results_payload_list.append(results)
@@ -1068,6 +1079,7 @@ class Ingestor:
         self,
         output_directory: Optional[str] = None,
         cleanup: bool = True,
+        compression: Optional[str] = "gzip",
     ) -> "Ingestor":
         """Configures the Ingestor to save results to disk instead of memory.
 
@@ -1092,6 +1104,12 @@ class Ingestor:
             when the Ingestor's context is exited (i.e., when used in a `with`
             statement).
             Defaults to True.
+        compression : str, optional
+            The compression algorithm to use for the saved result files.
+            Currently, the only supported value is `'gzip'`. To disable
+            compression, set this parameter to `None`. Defaults to `'gzip'`,
+            which significantly reduces the disk space required for results.
+            When enabled, files are saved with a `.gz` suffix (e.g., `results.jsonl.gz`).
 
         Returns
         -------
@@ -1107,6 +1125,7 @@ class Ingestor:
         self._output_config = {
             "output_directory": output_directory,
             "cleanup": cleanup,
+            "compression": compression,
         }
         ensure_directory_with_permissions(output_directory)
 

{nv_ingest_client-2025.9.30.dev20250930 → nv_ingest_client-2025.10.1.dev20251001}/src/nv_ingest_client/client/util/processing.py

@@ -1,3 +1,4 @@
+import gzip
 import io
 import json
 import logging
@@ -6,6 +7,7 @@ import re
 from typing import Any
 from typing import Dict
 from typing import List
+from typing import Optional
 from typing import Tuple
 
 try:
@@ -33,6 +35,7 @@ def save_document_results_to_jsonl(
     jsonl_output_filepath: str,
     original_source_name_for_log: str,
     ensure_parent_dir_exists: bool = True,
+    compression: Optional[str] = None,
 ) -> Tuple[int, Dict[str, str]]:
     """
     Saves a list of extraction items (for a single source document) to a JSON Lines file.
@@ -50,6 +53,13 @@ def save_document_results_to_jsonl(
             if parent_dir:
                 os.makedirs(parent_dir, exist_ok=True)
 
+        if compression == "gzip":
+            open_func = gzip.open
+        elif compression is None:
+            open_func = open
+        else:
+            raise ValueError(f"Unsupported compression type: {compression}")
+
         with io.BytesIO() as buffer:
             for extraction_item in doc_response_data:
                 if USING_ORJSON:
@@ -60,7 +70,7 @@ def save_document_results_to_jsonl(
 
         count_items_written = len(doc_response_data)
 
-        with open(jsonl_output_filepath, "wb") as f_jsonl:
+        with open_func(jsonl_output_filepath, "wb") as f_jsonl:
             f_jsonl.write(full_byte_content)
 
         logger.info(

{nv_ingest_client-2025.9.30.dev20250930 → nv_ingest_client-2025.10.1.dev20251001/src/nv_ingest_client.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nv-ingest-client
-Version: 2025.9.30.dev20250930
+Version: 2025.10.1.dev20251001
 Summary: Python client for the nv-ingest service
 Author-email: Jeremy Dyer <jdyer@nvidia.com>
 License:                                  Apache License