rapidata 2.11.0__tar.gz → 2.12.0__tar.gz

{rapidata-2.11.0 → rapidata-2.12.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: rapidata
-Version: 2.11.0
+Version: 2.12.0
 Summary: Rapidata package containing the Rapidata Python Client to interact with the Rapidata Web API in an easy way.
 License: Apache-2.0
 Author: Rapidata AG

{rapidata-2.11.0 → rapidata-2.12.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "rapidata"
-version = "2.11.0"
+version = "2.12.0"
 description = "Rapidata package containing the Rapidata Python Client to interact with the Rapidata Web API in an easy way."
 authors = ["Rapidata AG <info@rapidata.ai>"]
 license = "Apache-2.0"

{rapidata-2.11.0 → rapidata-2.12.0}/src/rapidata/rapidata_client/assets/_media_asset.py

@@ -78,7 +78,7 @@ class MediaAsset(BaseAsset):
         
         self._url = None
         self._content = None
-        self._session: None | requests.Session  = None
+        self.session: None | requests.Session  = None
         
         if re.match(r'^https?://', path):
             self._url = path
@@ -238,11 +238,11 @@ class MediaAsset(BaseAsset):
             requests.exceptions.RequestException: If download fails after all retries
         """
         # Use existing session or throw error if not set
-        if self._session is None:
+        if self.session is None:
             raise RuntimeError("HTTP session not configured")
 
         try:
-            response = self._session.get(
+            response = self.session.get(
                 url, 
                 stream=False,
                 timeout=(5, 30)  # (connect timeout, read timeout)

{rapidata-2.11.0 → rapidata-2.12.0}/src/rapidata/rapidata_client/order/_rapidata_dataset.py

@@ -136,14 +136,14 @@ class RapidataDataset:
             try:
                 # Get identifier for this upload (URL or file path)
                 if isinstance(media_asset, MediaAsset):
-                    media_asset._session = session
+                    media_asset.session = session
                     assets = [media_asset]
                     identifier = media_asset._url if media_asset._url else media_asset.path
                     identifiers_to_track = [identifier] if identifier else []
                 elif isinstance(media_asset, MultiAsset):
                     assets = cast(list[MediaAsset], media_asset.assets)
                     for asset in assets:
-                        asset._session = session
+                        asset.session = session
                     identifiers_to_track: list[str] = [
                         (asset._url if asset._url else cast(str, asset.path)) 
                         for asset in assets

{rapidata-2.11.0 → rapidata-2.12.0}/src/rapidata/rapidata_client/order/_rapidata_order_builder.py

@@ -134,23 +134,22 @@ class RapidataOrderBuilder:
 
         order = RapidataOrder(
             order_id=self.order_id,
-            dataset=self.__dataset,
             openapi_service=self.__openapi_service,
             name=self._name,
         )
 
-        if all(isinstance(item, MediaAsset) for item in self.__assets) and order.dataset:
+        if all(isinstance(item, MediaAsset) for item in self.__assets) and self.__dataset:
             assets = cast(list[MediaAsset], self.__assets)
-            order.dataset._add_media_from_paths(assets, self.__metadata, max_upload_workers)
+            self.__dataset._add_media_from_paths(assets, self.__metadata, max_upload_workers)
 
         elif (
-            all(isinstance(item, TextAsset) for item in self.__assets) and order.dataset
+            all(isinstance(item, TextAsset) for item in self.__assets) and self.__dataset
         ):
             assets = cast(list[TextAsset], self.__assets)
-            order.dataset._add_texts(assets)
+            self.__dataset._add_texts(assets)
 
         elif (
-            all(isinstance(item, MultiAsset) for item in self.__assets) and order.dataset
+            all(isinstance(item, MultiAsset) for item in self.__assets) and self.__dataset
         ):
             multi_assets = cast(list[MultiAsset], self.__assets)
 
@@ -167,19 +166,19 @@ class RapidataOrderBuilder:
 
             # Process based on the asset type
             if issubclass(first_asset_type, MediaAsset):
-                order.dataset._add_media_from_paths(
+                self.__dataset._add_media_from_paths(
                     multi_assets, self.__metadata, max_upload_workers
                 )
 
             elif issubclass(first_asset_type, TextAsset):
-                order.dataset._add_texts(multi_assets)
+                self.__dataset._add_texts(multi_assets)
 
             else:
                 raise ValueError(
                     "MultiAsset must contain MediaAssets or TextAssets objects."
                 )
 
-        elif order.dataset:
+        elif self.__dataset:
             raise ValueError(
                 "Media paths must all be of the same type: MediaAsset, TextAsset, or MultiAsset."
             )

{rapidata-2.11.0 → rapidata-2.12.0}/src/rapidata/rapidata_client/order/rapidata_order.py

@@ -1,27 +1,35 @@
-from time import sleep
-from rapidata.rapidata_client.order._rapidata_dataset import RapidataDataset
-from rapidata.service.openapi_service import OpenAPIService
+# Standard library imports
 import json
+import urllib.parse
+import webbrowser
+from time import sleep
+from typing import cast
+from colorama import Fore
+
+# Third-party imports
+from tqdm import tqdm
+
+# Local/application imports
 from rapidata.api_client.exceptions import ApiException
+from rapidata.api_client.models.campaign_artifact_model import CampaignArtifactModel
 from rapidata.api_client.models.order_state import OrderState
-from typing import Optional, cast, Any
-from rapidata.api_client.models.workflow_artifact_model import WorkflowArtifactModel
 from rapidata.api_client.models.preliminary_download_model import PreliminaryDownloadModel
-from tqdm import tqdm
+from rapidata.api_client.models.workflow_artifact_model import WorkflowArtifactModel
 from rapidata.rapidata_client.order.rapidata_results import RapidataResults
+from rapidata.service.openapi_service import OpenAPIService
+
 
 class RapidataOrder:
     """
     An instance of a Rapidata order.
 
-    Used to interact with a specific order in the Rapidata system. 
-    Such as starting, pausing, and getting the results of the order.
+    Used to interact with a specific order in the Rapidata system, such as starting,
+    pausing, and retrieving results.
 
     Args:
         name: The name of the order.
         order_id: The ID of the order.
-        openapi_service: The OpenAPIService instance used to interact with the Rapidata API.
-        dataset: The optional Dataset associated with the order.
+        openapi_service: The OpenAPIService instance for API interaction.
     """
 
     def __init__(
@@ -29,37 +37,30 @@ class RapidataOrder:
         name: str,
         order_id: str,
         openapi_service: OpenAPIService,
-        dataset: Optional[RapidataDataset]=None,
     ):
         self.order_id = order_id
         self.name = name
         self.__openapi_service = openapi_service
-        self.__dataset = dataset
-        self.__workflow_id = None
-        self.__pipeline_id = ""
-
-    def run(self, print_link: bool=True):
-        """
-        Runs the order for to start collecting votes.
-        """
+        self.__workflow_id: str = ""
+        self.__campaign_id: str = ""
+        self.__pipeline_id: str = ""
+        self._max_retries = 10  
+        self._retry_delay = 2   
+
+    def run(self, print_link: bool = True) -> "RapidataOrder":
+        """Runs the order to start collecting responses."""
         self.__openapi_service.order_api.order_submit_post(self.order_id)
-
         if print_link:
             print(f"Order '{self.name}' is now viewable under: https://app.{self.__openapi_service.enviroment}/order/detail/{self.order_id}")
-        
         return self
-    
-    def pause(self):
-        """
-        Pauses the order.
-        """
+
+    def pause(self) -> None:
+        """Pauses the order."""
         self.__openapi_service.order_api.order_pause_post(self.order_id)
         print(f"Order '{self}' has been paused.")
 
-    def unpause(self):
-        """
-        Unpauses/resume the order.
-        """
+    def unpause(self) -> None:
+        """Unpauses/resumes the order."""
         self.__openapi_service.order_api.order_resume_post(self.order_id)
         print(f"Order '{self}' has been unpaused.")
 
@@ -67,7 +68,7 @@ class RapidataOrder:
         """
         Gets the status of the order.
 
-        Different states are:
+        States:
             Created: The order has been created but not started yet.\n
             Submitted: The order has been submitted and is being reviewed.\n
             ManualReview: The order is in manual review - something went wrong with the automatic approval.\n
@@ -75,13 +76,10 @@ class RapidataOrder:
             Paused: The order has been paused.\n
             Completed: The order has been completed.\n
             Failed: The order has failed.
-        
-        Returns: 
-            The status of the order.
         """
         return self.__openapi_service.order_api.order_get_by_id_get(self.order_id).state
 
-    def display_progress_bar(self, refresh_rate: int=5):
+    def display_progress_bar(self, refresh_rate: int=5) -> None:
         """
         Displays a progress bar for the order processing using tqdm.
         
@@ -95,17 +93,20 @@ class RapidataOrder:
             raise Exception("Order has not been started yet. Please start it first.")
         
         while self.get_status() == OrderState.SUBMITTED:
-            print(f"Order '{self.name}' is submitted and being reviewed. standby...", end="\r")
+            print(f"Order '{self.name}' is submitted and being reviewed. Standby...", end="\r")
             sleep(1)
 
         if self.get_status() == OrderState.MANUALREVIEW:
-            raise Exception(f"Order '{self.name}' is in manual review. It might take some time to start. To speed up the process, please contact support (info@rapidata.ai).\
-                            \nOnce the order has started, you can run this method again to display the progress bar.")
+            raise Exception(
+                f"Order '{self.name}' is in manual review. It might take some time to start. "
+                "To speed up the process, contact support (info@rapidata.ai).\n"
+                "Once started, run this method again to display the progress bar."
+            )
 
         with tqdm(total=100, desc="Processing order", unit="%", bar_format="{desc}: {percentage:3.0f}%|{bar}| completed [{elapsed}<{remaining}, {rate_fmt}]") as pbar:
             last_percentage = 0
             while True:
-                current_percentage = self.__get_workflow_progress().completion_percentage
+                current_percentage = self._workflow_progress.completion_percentage
                 if current_percentage > last_percentage:
                     pbar.update(current_percentage - last_percentage)
                     last_percentage = current_percentage
@@ -115,62 +116,24 @@ class RapidataOrder:
 
                 sleep(refresh_rate)
 
-    def __get_pipeline_id(self):
-        if self.__pipeline_id:
-            return self.__pipeline_id
-
-        self.__pipeline_id = self.__openapi_service.order_api.order_get_by_id_get(self.order_id).pipeline_id
-        return self.__pipeline_id
-
-    def __get_workflow_id(self):
-        if self.__workflow_id:
-            return self.__workflow_id
-
-        for _ in range(10): # Try for 20 seconds to get the workflow id if workflow has not started by then, raise an exception
-            try:
-                self.__get_pipeline_id()
-                pipeline = self.__openapi_service.pipeline_api.pipeline_id_get(self.__pipeline_id)
-                self.__workflow_id = cast(WorkflowArtifactModel, pipeline.artifacts["workflow-artifact"].actual_instance).workflow_id
-                break
-            except Exception:
-                sleep(2)
-
-        if not self.__workflow_id:
-            raise Exception("Something went wrong when trying to get the order progress.")
-        
-        return self.__workflow_id
-    
-    def __get_workflow_progress(self):
-        workflow_id = self.__get_workflow_id()
+    @property
+    def _workflow_progress(self):
+        """
+        Gets the workflow progress.
+        """
         progress = None
-        for _ in range(2):
+        for _ in range(self._max_retries // 2):
             try:
+                workflow_id = self.__get_workflow_id()
                 progress = self.__openapi_service.workflow_api.workflow_get_progress_get(workflow_id)
                 break
             except Exception:
-                sleep(5)
-
+                sleep(self._retry_delay * 2)
         if not progress:
-            raise Exception(f"Failed to get progress. Please try again in a few seconds.")
-        
+            raise Exception("Failed to get progress. Please try again later.")
         return progress
-    
-    def __get_preliminary_results(self) -> RapidataResults:
-        pipeline_id = self.__get_pipeline_id()
-        try: 
-            download_id = self.__openapi_service.pipeline_api.pipeline_pipeline_id_preliminary_download_post(pipeline_id, PreliminaryDownloadModel(sendEmail=False)).download_id
-            while not (preliminary_results := self.__openapi_service.pipeline_api.pipeline_preliminary_download_preliminary_download_id_get(preliminary_download_id=download_id)):
-                sleep(1)
-            return RapidataResults(json.loads(preliminary_results.decode()))
-        
-        except ApiException as e:
-            # Handle API exceptions
-            raise Exception(f"Failed to get preliminary order results: {str(e)}") from e
-        except json.JSONDecodeError as e:
-            # Handle JSON parsing errors
-            raise Exception(f"Failed to parse preliminary order results: {str(e)}") from e
-
-    def get_results(self, preliminary_results: bool=False) -> RapidataResults:
+
+    def get_results(self, preliminary_results: bool = False) -> RapidataResults:
         """
         Gets the results of the order. 
         If the order is still processing, this method will block until the order is completed and then return the results.
@@ -179,9 +142,6 @@ class RapidataOrder:
             preliminary_results: If True, returns the preliminary results of the order. Defaults to False. 
                 Note that preliminary results are not final and may not contain all the datapoints & responses. Only the onese that are already available.
                 This will throw an exception if there are no responses available yet.
-
-        Returns: 
-            The results of the order.
         """
 
         if preliminary_results and self.get_status() not in [OrderState.COMPLETED]:
@@ -194,27 +154,91 @@ class RapidataOrder:
             sleep(5)
 
         try:
-            # Get the raw result string
             return RapidataResults(self.__openapi_service.order_api.order_get_order_results_get(id=self.order_id)) # type: ignore
-        
-        except ApiException as e:
-            # Handle API exceptions
+        except (ApiException, json.JSONDecodeError) as e:
             raise Exception(f"Failed to get order results: {str(e)}") from e
-        except json.JSONDecodeError as e:
-            # Handle JSON parsing errors
-            raise Exception(f"Failed to parse order results: {str(e)}") from e
-
-    @property
-    def dataset(self) -> RapidataDataset | None:
+        
+    def preview(self) -> None:
         """
-        The dataset associated with the order.
-        Returns: 
-            The RapidataDataset instance.
+        Opens a preview of the order in the browser.
+        
+        Raises:
+            Exception: If the order is not in processing state.
         """
-        return self.__dataset
-    
+        if self.get_status() != OrderState.PROCESSING:
+            raise Exception("Order is not processing. Preview unavailable.")
+        
+        campaign_id = self.__get_campaign_id()
+        auth_url = f"https://rapids.{self.__openapi_service.enviroment}/preview/campaign?id={campaign_id}"
+        could_open_browser = webbrowser.open(auth_url)
+        if not could_open_browser:
+            encoded_url = urllib.parse.quote(auth_url, safe="%/:=&?~#+!$,;'@()*[]")
+            print(Fore.RED + f'Please open this URL in your browser: "{encoded_url}"' + Fore.RESET)
+
+    def __get_pipeline_id(self) -> str:
+        """Internal method to fetch and cache the pipeline ID."""
+        if not self.__pipeline_id:
+            for _ in range(self._max_retries):
+                try:
+                    self.__pipeline_id = self.__openapi_service.order_api.order_get_by_id_get(self.order_id).pipeline_id
+                    break
+                except Exception:
+                    sleep(self._retry_delay)
+            else:
+                raise Exception("Failed to fetch pipeline ID after retries.")
+        return self.__pipeline_id
+
+    def __get_workflow_id(self) -> str:
+        """Internal method to fetch and cache the workflow ID."""
+        if not self.__workflow_id:
+            self.__fetch_workflow_and_campaign_ids()
+        return self.__workflow_id
+
+    def __get_campaign_id(self) -> str:
+        """Internal method to fetch and cache the campaign ID."""
+        if not self.__campaign_id:
+            self.__fetch_workflow_and_campaign_ids()
+        return self.__campaign_id
+
+    def __fetch_workflow_and_campaign_ids(self) -> None:
+        """Fetches workflow and campaign IDs from the pipeline."""
+        if self.__workflow_id and self.__campaign_id:
+            return
+        pipeline_id = self.__get_pipeline_id()
+        for _ in range(self._max_retries):
+            try:
+                pipeline = self.__openapi_service.pipeline_api.pipeline_id_get(pipeline_id)
+                self.__workflow_id = cast(WorkflowArtifactModel, pipeline.artifacts["workflow-artifact"].actual_instance).workflow_id
+                self.__campaign_id = cast(CampaignArtifactModel, pipeline.artifacts["campaign-artifact"].actual_instance).campaign_id
+                return
+            except Exception:
+                sleep(self._retry_delay)
+        raise Exception("Failed to fetch workflow and campaign IDs after retries.")
+
+    def __get_preliminary_results(self) -> RapidataResults:
+        """Internal method to fetch preliminary results."""
+        try:
+            pipeline_id = self.__get_pipeline_id()
+            download_id = self.__openapi_service.pipeline_api.pipeline_pipeline_id_preliminary_download_post(
+                pipeline_id, PreliminaryDownloadModel(sendEmail=False)
+            ).download_id
+            
+            elapsed = 0
+            timeout = 60
+            while elapsed < timeout:
+                preliminary_results = self.__openapi_service.pipeline_api.pipeline_preliminary_download_preliminary_download_id_get(
+                    preliminary_download_id=download_id
+                )
+                if preliminary_results:
+                    return RapidataResults(json.loads(preliminary_results.decode()))
+                sleep(1)
+                elapsed += 1
+            raise Exception("Timeout waiting for preliminary results.")
+        except (ApiException, json.JSONDecodeError) as e:
+            raise Exception(f"Failed to get preliminary results: {str(e)}") from e
+
     def __str__(self) -> str:
         return f"name: '{self.name}' order id: {self.order_id}"
-    
+
     def __repr__(self) -> str:
-        return f"name: '{self.name}' order id: {self.order_id}"
+        return f"RapidataOrder(name='{self.name}', order_id='{self.order_id}')"

{rapidata-2.11.0 → rapidata-2.12.0}/src/rapidata/rapidata_client/validation/rapidata_validation_set.py

@@ -1,5 +1,7 @@
 from rapidata.rapidata_client.validation.rapids.rapids import Rapid
 from rapidata.service.openapi_service import OpenAPIService
+from requests.adapters import HTTPAdapter, Retry
+import requests
 
 class RapidataValidationSet:
     """A class for interacting with a Rapidata validation set.
@@ -17,6 +19,7 @@ class RapidataValidationSet:
         self.id = validation_set_id
         self.name = name
         self.__openapi_service = openapi_service
+        self.__session = self._get_session()
     
     def add_rapid(self, rapid: Rapid):
         """Add a Rapid to the validation set.
@@ -24,7 +27,37 @@ class RapidataValidationSet:
         Args:
             rapid (Rapid): The Rapid to add to the validation set.
         """
-        rapid._add_to_validation_set(self.id, self.__openapi_service)
+        rapid._add_to_validation_set(self.id, self.__openapi_service, self.__session)
+
+    def _get_session(self, max_retries: int = 5, max_workers: int = 10) -> requests.Session:   
+        """Get a requests session with retry logic.
+
+
+        Args:
+            max_retries (int): The maximum number of retries.
+            max_workers (int): The maximum number of workers.
+
+        Returns:
+            requests.Session: A requests session with retry logic.
+        """
+        session = requests.Session()
+        retries = Retry(
+            total=max_retries,
+            backoff_factor=1,
+            status_forcelist=[500, 502, 503, 504],
+            allowed_methods=["GET"],
+            respect_retry_after_header=True
+        )
+
+        adapter = HTTPAdapter(
+            pool_connections=max_workers * 2,
+            pool_maxsize=max_workers * 4,
+            max_retries=retries
+        )
+        session.mount('http://', adapter)
+        session.mount('https://', adapter)
+
+        return session
 
     def __str__(self):
         return f"name: '{self.name}' id: {self.id}"

{rapidata-2.11.0 → rapidata-2.12.0}/src/rapidata/rapidata_client/validation/rapids/rapids.py

@@ -22,6 +22,8 @@ from rapidata.api_client.models.datapoint_metadata_model_metadata_inner import (
 
 from rapidata.service.openapi_service import OpenAPIService
 
+import requests
+
 class Rapid():
     def __init__(self, asset: MediaAsset | TextAsset | MultiAsset, metadata: Sequence[Metadata], payload: Any, truth: Any, randomCorrectProbability: float, explanation: str | None):
         self.asset = asset
@@ -31,14 +33,14 @@ class Rapid():
         self.randomCorrectProbability = randomCorrectProbability
         self.explanation = explanation 
 
-    def _add_to_validation_set(self, validationSetId: str, openapi_service: OpenAPIService):
+    def _add_to_validation_set(self, validationSetId: str, openapi_service: OpenAPIService, session: requests.Session) -> None:
         if isinstance(self.asset, TextAsset) or (isinstance(self.asset, MultiAsset) and isinstance(self.asset.assets[0], TextAsset)):
             openapi_service.validation_api.validation_add_validation_text_rapid_post(
                 add_validation_text_rapid_model=self.__to_text_model(validationSetId)
             )
 
         elif isinstance(self.asset, MediaAsset) or (isinstance(self.asset, MultiAsset) and isinstance(self.asset.assets[0], MediaAsset)):
-            model = self.__to_media_model(validationSetId)
+            model = self.__to_media_model(validationSetId, session=session)
             openapi_service.validation_api.validation_add_validation_rapid_post(
                 model=model[0], files=model[1]
             )
@@ -46,7 +48,7 @@ class Rapid():
         else:
             raise TypeError("The asset must be a MediaAsset, TextAsset, or MultiAsset")
 
-    def __to_media_model(self, validationSetId: str) -> tuple[AddValidationRapidModel, list[StrictStr | tuple[StrictStr, StrictBytes] | StrictBytes]]:
+    def __to_media_model(self, validationSetId: str, session: requests.Session) -> tuple[AddValidationRapidModel, list[StrictStr | tuple[StrictStr, StrictBytes] | StrictBytes]]:
         assets: list[MediaAsset] = [] 
         if isinstance(self.asset, MultiAsset):
             for asset in self.asset.assets:
@@ -61,6 +63,9 @@ class Rapid():
         if isinstance(self.asset, MediaAsset):
             assets = [self.asset]
 
+        for asset in assets:
+            asset.session = session
+
         return (AddValidationRapidModel(
             validationSetId=validationSetId,
             payload=AddValidationRapidModelPayload(self.payload),