corva-worker-python 1.2.3__tar.gz → 1.2.4__tar.gz

{corva-worker-python-1.2.3 → corva-worker-python-1.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: corva-worker-python
-Version: 1.2.3
+Version: 1.2.4
 Summary: SDK for interacting with Corva
 Home-page: https://github.com/corva-ai/corva-worker-python
 Author: Jordan Ambra <jordan.ambra@corva.ai>, Mohammadreza Kamyab <m.kamyab@corva.ai>

{corva-worker-python-1.2.3 → corva-worker-python-1.2.4}/corva_worker_python.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: corva-worker-python
-Version: 1.2.3
+Version: 1.2.4
 Summary: SDK for interacting with Corva
 Home-page: https://github.com/corva-ai/corva-worker-python
 Author: Jordan Ambra <jordan.ambra@corva.ai>, Mohammadreza Kamyab <m.kamyab@corva.ai>

{corva-worker-python-1.2.3 → corva-worker-python-1.2.4}/setup.py

@@ -17,7 +17,7 @@ setup(
     name="corva-worker-python",
     author="Jordan Ambra <jordan.ambra@corva.ai>, Mohammadreza Kamyab <m.kamyab@corva.ai>",
     url="https://github.com/corva-ai/corva-worker-python",
-    version="1.2.3",
+    version="1.2.4",
     classifiers=classifiers,
     python_requires=">=3.8",
     description="SDK for interacting with Corva",

{corva-worker-python-1.2.3 → corva-worker-python-1.2.4}/worker/partial_rerun_merge/merge.py

@@ -181,6 +181,7 @@ class PartialRerunMerge:
         self,
         collection_name: str,
         downsample_count: Optional[int] = None,
+        downsample_ratio: Optional[float] = None,
         skip_progress: Optional[bool] = False,
     ) -> bool:
         """
@@ -191,6 +192,7 @@ class PartialRerunMerge:
         Args:
             collection_name (str): the collection name
             downsample_count (Optional[int]): the downsample count. Defaults to None.
+            downsample_ratio (Optional[float]): the downsample ratio. Defaults to None.
             skip_progress (Optional[bool]): whether to skip the progress. Defaults to False.
         """
         dataset_progress = self.merging_progress.get_dataset_progress(collection_name)
@@ -216,6 +218,7 @@ class PartialRerunMerge:
                 break
 
             updated_data = self._get_data(collection_name, self.rerun_asset_id, start_time, self.end_timestamp, "once")
+            original_records_count = len(updated_data)
             if not updated_data:
                 dataset_progress.mark_completed_at(self.end_timestamp)
                 is_completed = True
@@ -223,14 +226,14 @@ class PartialRerunMerge:
 
             last_timestamp = updated_data[-1]["timestamp"]
 
-            if downsample_count:
-                updated_data = choose_items(updated_data, self.MAX_RECORDS_COUNT)
+            if downsample_count is not None or downsample_ratio is not None:
+                updated_data = choose_items(updated_data, downsample_count, downsample_ratio)
 
             self._move_records(collection_name, updated_data)
 
             dataset_progress.processed_timestamp = last_timestamp
 
-            if len(updated_data) < self.MAX_API_GET_LIMIT:
+            if original_records_count < self.MAX_API_GET_LIMIT:
                 dataset_progress.mark_completed_at(self.end_timestamp)
                 is_completed = True
                 break
@@ -454,7 +457,11 @@ class PartialRerunMerge:
         time.sleep(seconds)
 
 
-def choose_items(records: List, max_records_count: Optional[int] = None) -> List:
+def choose_items(
+    records: List,
+    max_records_count: Optional[int] = None,
+    max_records_ratio: Optional[float] = None,
+) -> List:
     """
     Choose a subset of records from a list of records; the first and last
     records an inclusive.
@@ -463,11 +470,23 @@ def choose_items(records: List, max_records_count: Optional[int] = None) -> List
         records (List): A list of records.
         max_records_count (Optional[int]): The maximum number of records
             to choose. If None, all records are chosen.
+        max_records_ratio (Optional[float]): The maximum ratio of records
+            to choose. If None, all records are chosen.
 
     Returns:
         List: A list of chosen records.
     """
-    if max_records_count is None or max_records_count <= 0 or len(records) <= max_records_count or len(records) < 3:
+    if max_records_count is None and max_records_ratio is None:
+        return records
+
+    # only one of max_records_count and max_records_ratio can be provided
+    if max_records_count is not None and max_records_ratio is not None:
+        raise ValueError("Only one of max_records_count and max_records_ratio can be provided.")
+
+    if max_records_ratio is not None:
+        max_records_count = int(len(records) * max_records_ratio)
+
+    if max_records_count <= 0 or len(records) <= max_records_count or len(records) < 3:
         return records
 
     step = pymath.ceil((len(records) - 2) / max_records_count)