corva-worker-python 1.2.2__tar.gz → 1.2.4__tar.gz

{corva-worker-python-1.2.2 → corva-worker-python-1.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: corva-worker-python
-Version: 1.2.2
+Version: 1.2.4
 Summary: SDK for interacting with Corva
 Home-page: https://github.com/corva-ai/corva-worker-python
 Author: Jordan Ambra <jordan.ambra@corva.ai>, Mohammadreza Kamyab <m.kamyab@corva.ai>

{corva-worker-python-1.2.2 → corva-worker-python-1.2.4}/corva_worker_python.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: corva-worker-python
-Version: 1.2.2
+Version: 1.2.4
 Summary: SDK for interacting with Corva
 Home-page: https://github.com/corva-ai/corva-worker-python
 Author: Jordan Ambra <jordan.ambra@corva.ai>, Mohammadreza Kamyab <m.kamyab@corva.ai>

corva-worker-python-1.2.4/setup.py

@@ -0,0 +1,29 @@
+from setuptools import find_packages, setup
+
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Intended Audience :: Developers",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Topic :: Software Development :: Libraries",
+]
+
+setup(
+    name="corva-worker-python",
+    author="Jordan Ambra <jordan.ambra@corva.ai>, Mohammadreza Kamyab <m.kamyab@corva.ai>",
+    url="https://github.com/corva-ai/corva-worker-python",
+    version="1.2.4",
+    classifiers=classifiers,
+    python_requires=">=3.8",
+    description="SDK for interacting with Corva",
+    keywords="corva, worker",
+    packages=find_packages(exclude=["testcase"]),
+    install_requires=["numpy", "redis", "requests", "simplejson"],
+    include_package_data=True,
+    license="Unlicensed",
+)

{corva-worker-python-1.2.2 → corva-worker-python-1.2.4}/worker/partial_rerun_merge/merge.py

@@ -181,6 +181,7 @@ class PartialRerunMerge:
         self,
         collection_name: str,
         downsample_count: Optional[int] = None,
+        downsample_ratio: Optional[float] = None,
         skip_progress: Optional[bool] = False,
     ) -> bool:
         """
@@ -191,6 +192,7 @@ class PartialRerunMerge:
         Args:
             collection_name (str): the collection name
             downsample_count (Optional[int]): the downsample count. Defaults to None.
+            downsample_ratio (Optional[float]): the downsample ratio. Defaults to None.
             skip_progress (Optional[bool]): whether to skip the progress. Defaults to False.
         """
         dataset_progress = self.merging_progress.get_dataset_progress(collection_name)
@@ -216,6 +218,7 @@ class PartialRerunMerge:
                 break
 
             updated_data = self._get_data(collection_name, self.rerun_asset_id, start_time, self.end_timestamp, "once")
+            original_records_count = len(updated_data)
             if not updated_data:
                 dataset_progress.mark_completed_at(self.end_timestamp)
                 is_completed = True
@@ -223,14 +226,14 @@ class PartialRerunMerge:
 
             last_timestamp = updated_data[-1]["timestamp"]
 
-            if downsample_count:
-                updated_data = choose_items(updated_data, self.MAX_RECORDS_COUNT)
+            if downsample_count is not None or downsample_ratio is not None:
+                updated_data = choose_items(updated_data, downsample_count, downsample_ratio)
 
             self._move_records(collection_name, updated_data)
 
             dataset_progress.processed_timestamp = last_timestamp
 
-            if len(updated_data) < self.MAX_API_GET_LIMIT:
+            if original_records_count < self.MAX_API_GET_LIMIT:
                 dataset_progress.mark_completed_at(self.end_timestamp)
                 is_completed = True
                 break
@@ -454,7 +457,11 @@ class PartialRerunMerge:
         time.sleep(seconds)
 
 
-def choose_items(records: List, max_records_count: Optional[int] = None) -> List:
+def choose_items(
+    records: List,
+    max_records_count: Optional[int] = None,
+    max_records_ratio: Optional[float] = None,
+) -> List:
     """
     Choose a subset of records from a list of records; the first and last
     records an inclusive.
@@ -463,11 +470,23 @@ def choose_items(records: List, max_records_count: Optional[int] = None) -> List
         records (List): A list of records.
         max_records_count (Optional[int]): The maximum number of records
             to choose. If None, all records are chosen.
+        max_records_ratio (Optional[float]): The maximum ratio of records
+            to choose. If None, all records are chosen.
 
     Returns:
         List: A list of chosen records.
     """
-    if max_records_count is None or max_records_count <= 0 or len(records) <= max_records_count or len(records) < 3:
+    if max_records_count is None and max_records_ratio is None:
+        return records
+
+    # only one of max_records_count and max_records_ratio can be provided
+    if max_records_count is not None and max_records_ratio is not None:
+        raise ValueError("Only one of max_records_count and max_records_ratio can be provided.")
+
+    if max_records_ratio is not None:
+        max_records_count = int(len(records) * max_records_ratio)
+
+    if max_records_count <= 0 or len(records) <= max_records_count or len(records) < 3:
         return records
 
     step = pymath.ceil((len(records) - 2) / max_records_count)

corva-worker-python-1.2.2/setup.py

@@ -1,30 +0,0 @@
-from setuptools import find_packages, setup
-
-
-classifiers = [
-    'Development Status :: 5 - Production/Stable',
-    'Intended Audience :: Developers',
-    'Operating System :: OS Independent',
-    'Programming Language :: Python',
-    'Programming Language :: Python :: 3',
-    'Programming Language :: Python :: 3.8',
-    'Programming Language :: Python :: 3.9',
-    'Programming Language :: Python :: 3.10',
-    'Programming Language :: Python :: 3.11',
-    'Topic :: Software Development :: Libraries',
-]
-
-setup(
-    name='corva-worker-python',
-    author='Jordan Ambra <jordan.ambra@corva.ai>, Mohammadreza Kamyab <m.kamyab@corva.ai>',
-    url='https://github.com/corva-ai/corva-worker-python',
-    version='1.2.2',
-    classifiers=classifiers,
-    python_requires='>=3.8',
-    description='SDK for interacting with Corva',
-    keywords='corva, worker',
-    packages=find_packages(exclude=["testcase"]),
-    install_requires=["numpy", "redis", "requests", "simplejson"],
-    include_package_data=True,
-    license='Unlicensed',
-)

{corva-worker-python-1.2.2 → corva-worker-python-1.2.4}/worker/data/operations.py

@@ -391,12 +391,12 @@ def get_cleaned_event_and_type(event) -> Tuple[Union[list, dict], EventType]:
         if event.get("event_type") == "partial-well-rerun-merge":
             return event, EventType.PARTIAL_RERUN
 
-        if "asset_id" in event:
-            return event, EventType.GENERIC
-
         if "task_id" in event:
             return event, EventType.TASK
 
+        if "asset_id" in event:
+            return event, EventType.GENERIC
+
         raise TypeError("Missing task_id or asset_id keys in event")
 
     if not isinstance(event, list):