rapidata 2.23.2__py3-none-any.whl → 3.0.1__py3-none-any.whl

rapidata/rapidata_client/metadata/_private_text_metadata.py

@@ -1,17 +0,0 @@
-from rapidata.api_client.models.private_text_metadata_input import (
-    PrivateTextMetadataInput,
-)
-from rapidata.rapidata_client.metadata._base_metadata import Metadata
-
-
-class PrivateTextMetadata(Metadata):
-
-    def __init__(self, text: str, identifier: str = "private_text"):
-        super().__init__()
-        self._identifier = identifier
-        self._text = text
-
-    def to_model(self):
-        return PrivateTextMetadataInput(
-            _t="PrivateTextMetadataInput", identifier=self._identifier, text=self._text
-        )

rapidata/rapidata_client/metadata/_prompt_metadata.py

@@ -1,18 +0,0 @@
-from rapidata.rapidata_client.metadata._base_metadata import Metadata
-from rapidata.api_client.models.prompt_metadata_input import PromptMetadataInput
-
-
-class PromptMetadata(Metadata):
-    """The PromptMetadata class is used to display a prompt to the user."""
-
-    def __init__(self, prompt: str):
-        super().__init__()
-
-        if not isinstance(prompt, str):
-            raise ValueError("Prompt must be a string")
-        
-        self._prompt = prompt
-    
-
-    def to_model(self):
-        return PromptMetadataInput(_t="PromptMetadataInput", prompt=self._prompt)

rapidata/rapidata_client/metadata/_public_text_metadata.py

@@ -1,17 +0,0 @@
-from rapidata.api_client.models.public_text_metadata_input import (
-    PublicTextMetadataInput,
-)
-from rapidata.rapidata_client.metadata._base_metadata import Metadata
-
-
-class PublicTextMetadata(Metadata):
-
-    def __init__(self, text: str, identifier: str = "public_text"):
-        super().__init__()
-        self._identifier = identifier
-        self._text = text
-
-    def to_model(self):
-        return PublicTextMetadataInput(
-            _t="PublicTextMetadataInput", identifier=self._identifier, text=self._text
-        )

rapidata/rapidata_client/order/_rapidata_dataset.py

@@ -1,443 +0,0 @@
-from itertools import zip_longest
-
-from rapidata.api_client.models.datapoint_metadata_model import DatapointMetadataModel
-from rapidata.api_client.models.create_datapoint_from_urls_model import (
-    CreateDatapointFromUrlsModel,
-)
-from rapidata.api_client.models.create_datapoint_from_files_model import CreateDatapointFromFilesModel
-from rapidata.api_client.models.create_datapoint_from_urls_model import CreateDatapointFromUrlsModel
-from rapidata.api_client.models.create_datapoint_from_text_sources_model import CreateDatapointFromTextSourcesModel
-from rapidata.api_client.models.create_datapoint_from_files_model_metadata_inner import CreateDatapointFromFilesModelMetadataInner
-from rapidata.rapidata_client.metadata._base_metadata import Metadata
-from rapidata.rapidata_client.assets import TextAsset, MediaAsset, MultiAsset
-from rapidata.service import LocalFileService
-from rapidata.service.openapi_service import OpenAPIService
-from concurrent.futures import ThreadPoolExecutor, as_completed
-from tqdm import tqdm
-
-from pydantic import StrictStr
-from typing import cast, Sequence, Generator
-from rapidata.rapidata_client.logging import logger
-import time
-import threading
-
-
-def chunk_list(lst: list, chunk_size: int) -> Generator:
-    """Split list into chunks to prevent resource exhaustion"""
-    for i in range(0, len(lst), chunk_size):
-        yield lst[i:i + chunk_size]
-
-class RapidataDataset:
-
-    def __init__(self, dataset_id: str, openapi_service: OpenAPIService):
-        self.dataset_id = dataset_id
-        self.openapi_service = openapi_service
-        self.local_file_service = LocalFileService()
-
-    def _add_texts(
-        self,
-        text_assets: list[TextAsset] | list[MultiAsset],
-        metadata_list: Sequence[Sequence[Metadata]] | None = None,
-        max_workers: int = 10,
-    ):
-        for text_asset in text_assets:
-            if isinstance(text_asset, MultiAsset):
-                assert all(
-                    isinstance(asset, TextAsset) for asset in text_asset.assets
-                ), "All assets in a MultiAsset must be of type TextAsset."
-
-        def upload_text_datapoint(text_asset: TextAsset | MultiAsset, metadata_per_datapoint: Sequence[Metadata] | None, index: int) -> None:
-            if isinstance(text_asset, TextAsset):
-                texts = [text_asset.text]
-            elif isinstance(text_asset, MultiAsset):
-                texts = [asset.text for asset in text_asset.assets if isinstance(asset, TextAsset)]
-            else:
-                raise ValueError(f"Unsupported asset type: {type(text_asset)}")
-            
-            metadata = []
-            if metadata_per_datapoint:
-                for meta in metadata_per_datapoint:
-                    meta_model = meta.to_model() if meta else None
-                    if meta_model:
-                        metadata.append(CreateDatapointFromFilesModelMetadataInner(meta_model))
-
-            model = CreateDatapointFromTextSourcesModel(
-                textSources=texts,
-                sortIndex=index,
-                metadata=metadata,
-            )
-
-            upload_response = self.openapi_service.dataset_api.dataset_dataset_id_datapoints_texts_post(dataset_id=self.dataset_id, create_datapoint_from_text_sources_model=model)
-
-            if upload_response.errors:
-                raise ValueError(f"Error uploading text datapoint: {upload_response.errors}")
-
-        total_uploads = len(text_assets)
-        with ThreadPoolExecutor(max_workers=max_workers) as executor:
-            futures = [
-                executor.submit(upload_text_datapoint, text_asset, metadata, index=i)
-                for i, (text_asset, metadata) in enumerate(zip_longest(text_assets, metadata_list or []))
-            ]
-
-            with tqdm(total=total_uploads, desc="Uploading text datapoints") as pbar:
-                for future in as_completed(futures):
-                    future.result()  # This will raise any exceptions that occurred during execution
-                    pbar.update(1)
-
-    def _process_single_upload(
-        self,
-        media_asset: MediaAsset | MultiAsset, 
-        meta_list: Sequence[Metadata] | None, 
-        index: int,
-    ) -> tuple[list[str], list[str]]:
-        """
-        Process single upload with error tracking.
-        
-        Args:
-            media_asset: MediaAsset or MultiAsset to upload
-            meta_list: Optional sequence of metadata for the asset
-            index: Sort index for the upload
-            
-        Returns:
-            tuple[list[str], list[str]]: Lists of successful and failed identifiers
-        """
-        local_successful: list[str] = []
-        local_failed: list[str] = []
-        identifiers_to_track: list[str] = []
-        
-        try:
-            # Get identifier for this upload (URL or file path)
-            if isinstance(media_asset, MediaAsset):
-                assets = [media_asset]
-                identifier = media_asset._url if media_asset._url else media_asset.path
-                identifiers_to_track = [identifier] if identifier else []
-            elif isinstance(media_asset, MultiAsset):
-                assets = cast(list[MediaAsset], media_asset.assets)
-                identifiers_to_track = [
-                    (asset._url if asset._url else cast(str, asset.path)) 
-                    for asset in assets
-                ]
-            else:
-                raise ValueError(f"Unsupported asset type: {type(media_asset)}")
-
-            # Convert multiple metadata to models
-            metadata = []
-            if meta_list:
-                for meta in meta_list:
-                    meta_model = meta.to_model() if meta else None
-                    if meta_model:
-                        metadata.append(CreateDatapointFromFilesModelMetadataInner(meta_model))
-
-            local_paths: bool = assets[0].is_local()
-            files: list[StrictStr] = []
-            for asset in assets:
-                if isinstance(asset, MediaAsset):
-                    files.append(asset.path)
-
-            if local_paths:
-                model = CreateDatapointFromFilesModel(
-                    metadata=metadata,
-                    sortIndex=index,
-                )
-                upload_response = self.openapi_service.dataset_api.dataset_dataset_id_datapoints_files_post(
-                    dataset_id=self.dataset_id,
-                    model=model,
-                    files=files # type: ignore
-                )
-            else:
-                upload_response = self.openapi_service.dataset_api.dataset_dataset_id_datapoints_urls_post(
-                    dataset_id=self.dataset_id,
-                    create_datapoint_from_urls_model=CreateDatapointFromUrlsModel(
-                        urls=files,
-                        metadata=metadata,
-                        sortIndex=index
-                    ),
-                )
-                
-            if upload_response.errors:
-                error_msg = f"Error uploading datapoint: {upload_response.errors}"
-                logger.error(error_msg)
-                local_failed.extend(identifiers_to_track)
-                raise ValueError(error_msg)
-
-            local_successful.extend(identifiers_to_track)
-
-        except Exception as e:
-            logger.error(f"\nUpload failed for {identifiers_to_track}: {str(e)}") # \n to avoid same line as tqdm
-            local_failed.extend(identifiers_to_track)
-
-        return local_successful, local_failed
-
-    def _get_progress_tracker(
-        self, 
-        total_uploads: int, 
-        stop_event: threading.Event, 
-        progress_error_event: threading.Event,
-        progress_poll_interval: float,
-    ) -> threading.Thread:
-        """
-        Create and return a progress tracking thread that shows actual API progress.
-        
-        Args:
-            total_uploads: Total number of uploads to track
-            initial_ready: Initial number of ready items
-            initial_progress: Initial progress state
-            stop_event: Event to signal thread to stop
-            progress_error_event: Event to signal an error in progress tracking
-            progress_poll_interval: Time between progress checks
-            
-        Returns:
-            threading.Thread: The progress tracking thread
-        """
-        def progress_tracking_thread():
-            try:
-                # Initialize progress bar with 0 completions
-                with tqdm(total=total_uploads, desc="Uploading datapoints") as pbar:
-                    prev_ready = 0
-                    prev_failed = 0
-                    stall_count = 0
-                    last_progress_time = time.time()
-                    
-                    # We'll wait for all uploads to finish + some extra time
-                    # for the backend to fully process everything
-                    all_uploads_complete = threading.Event()
-                    
-                    while not stop_event.is_set() or not all_uploads_complete.is_set():
-                        try:
-                            current_progress = self.openapi_service.dataset_api.dataset_dataset_id_progress_get(self.dataset_id)
-                            
-                            # Calculate items completed since our initialization
-                            completed_ready = current_progress.ready
-                            completed_failed = current_progress.failed
-                            total_completed = completed_ready + completed_failed
-                            
-                            # Calculate newly completed items since our last check
-                            new_ready = current_progress.ready - prev_ready
-                            new_failed = current_progress.failed - prev_failed
-                            
-                            # Update progress bar position to show actual completed items
-                            # First reset to match the actual completed count
-                            pbar.n = total_completed
-                            pbar.refresh()
-                            
-                            if new_ready > 0 or new_failed > 0:
-                                # We saw progress
-                                stall_count = 0
-                                last_progress_time = time.time()
-                            else:
-                                stall_count += 1
-                            
-                            # Update our tracking variables
-                            prev_ready = current_progress.ready
-                            prev_failed = current_progress.failed or 0
-                            
-                            # Check if stop_event was set (all uploads submitted)
-                            if stop_event.is_set():
-                                elapsed_since_last_progress = time.time() - last_progress_time
-                                
-                                # If we haven't seen progress for a while after all uploads were submitted
-                                if elapsed_since_last_progress > 5.0:
-                                    # If we're at 100%, we're done
-                                    if total_completed >= total_uploads:
-                                        all_uploads_complete.set()
-                                        break
-                                        
-                                    # If we're not at 100% but it's been a while with no progress
-                                    if stall_count > 5:
-                                        # We've polled several times with no progress, assume we're done
-                                        logger.warning(f"\nProgress seems stalled at {total_completed}/{total_uploads}. Please try again.")
-                                        break
-                                
-                        except Exception as e:
-                            logger.error(f"\nError checking progress: {str(e)}")
-                            stall_count += 1
-                            
-                            if stall_count > 10:  # Too many consecutive errors
-                                progress_error_event.set()
-                                break
-                        
-                        # Sleep before next poll
-                        time.sleep(progress_poll_interval)
-                
-            except Exception as e:
-                logger.error(f"Progress tracking thread error: {str(e)}")
-                progress_error_event.set()
-                
-        # Create and return the thread
-        progress_thread = threading.Thread(target=progress_tracking_thread)
-        progress_thread.daemon = True
-        return progress_thread
-
-    def _process_uploads_in_chunks(
-        self,
-        media_paths: list[MediaAsset] | list[MultiAsset],
-        multi_metadata: Sequence[Sequence[Metadata]] | None,
-        max_workers: int,
-        chunk_size: int,
-        stop_progress_tracking: threading.Event,
-        progress_tracking_error: threading.Event
-    ) -> tuple[list[str], list[str]]:
-        """
-        Process uploads in chunks with a ThreadPoolExecutor.
-        
-        Args:
-            media_paths: List of assets to upload
-            multi_metadata: Optional sequence of sequences of metadata
-            max_workers: Maximum number of concurrent workers
-            chunk_size: Number of items to process in each batch
-            stop_progress_tracking: Event to signal progress tracking to stop
-            progress_tracking_error: Event to detect progress tracking errors
-            
-        Returns:
-            tuple[list[str], list[str]]: Lists of successful and failed uploads
-        """
-        successful_uploads: list[str] = []
-        failed_uploads: list[str] = []
-        
-        try:
-            with ThreadPoolExecutor(max_workers=max_workers) as executor:
-                # Process uploads in chunks to avoid overwhelming the system
-                for chunk_idx, chunk in enumerate(chunk_list(media_paths, chunk_size)):
-                    chunk_multi_metadata = multi_metadata[chunk_idx * chunk_size:(chunk_idx + 1) * chunk_size] if multi_metadata else None
-                    
-                    futures = [
-                        executor.submit(
-                            self._process_single_upload, 
-                            media_asset, 
-                            meta_list, 
-                            index=(chunk_idx * chunk_size + i)
-                        )
-                        for i, (media_asset, meta_list) in enumerate(zip_longest(chunk, chunk_multi_metadata or []))
-                    ]
-                    
-                    # Wait for this chunk to complete before starting the next one
-                    for future in as_completed(futures):
-                        if progress_tracking_error.is_set():
-                            raise RuntimeError("Progress tracking failed, aborting uploads")
-                            
-                        try:
-                            chunk_successful, chunk_failed = future.result()
-                            successful_uploads.extend(chunk_successful)
-                            failed_uploads.extend(chunk_failed)
-                        except Exception as e:
-                            logger.error(f"Future execution failed: {str(e)}")
-        finally:
-            # Signal to the progress tracking thread that all uploads have been submitted
-            stop_progress_tracking.set()
-                
-        return successful_uploads, failed_uploads
-
-    def _log_final_progress(
-        self, 
-        total_uploads: int, 
-        progress_poll_interval: float,
-        successful_uploads: list[str],
-        failed_uploads: list[str]
-    ) -> None:
-        """
-        Log the final progress of the upload operation.
-        
-        Args:
-            total_uploads: Total number of uploads
-            initial_ready: Initial number of ready items
-            initial_progress: Initial progress state
-            progress_poll_interval: Time between progress checks
-            successful_uploads: List of successful uploads for fallback reporting
-            failed_uploads: List of failed uploads for fallback reporting
-        """
-        try:            
-            # Get final progress
-            final_progress = self.openapi_service.dataset_api.dataset_dataset_id_progress_get(self.dataset_id)
-            total_ready = final_progress.ready
-            total_failed = final_progress.failed
-            
-            # Make sure we account for all uploads
-            if total_ready + total_failed < total_uploads:
-                # Try one more time after a longer wait
-                time.sleep(5 * progress_poll_interval)
-                final_progress = self.openapi_service.dataset_api.dataset_dataset_id_progress_get(self.dataset_id)
-                total_ready = final_progress.ready
-                total_failed = final_progress.failed
-            
-            success_rate = (total_ready / total_uploads * 100) if total_uploads > 0 else 0
-            
-            logger.info(f"Upload complete: {total_ready} ready, {total_uploads-total_ready} failed ({success_rate:.1f}% success rate)")
-        except Exception as e:
-            logger.error(f"Error getting final progress: {str(e)}")
-            logger.info(f"Upload summary from local tracking: {len(successful_uploads)} succeeded, {len(failed_uploads)} failed")
-
-        if failed_uploads:
-            logger.error(f"Failed uploads: {failed_uploads}")
-
-    def _add_media_from_paths(
-        self,
-        media_paths: list[MediaAsset] | list[MultiAsset],
-        multi_metadata: Sequence[Sequence[Metadata]] | None = None,
-        max_workers: int = 5,
-        chunk_size: int = 50,
-        progress_poll_interval: float = 0.5,
-    ) -> tuple[list[str], list[str]]:
-        """
-        Upload media paths in chunks with managed resources.
-        
-        Args:
-            media_paths: List of MediaAsset or MultiAsset objects to upload
-            multi_metadata: Optional sequence of sequences of metadata matching media_paths length
-            max_workers: Maximum number of concurrent upload workers
-            chunk_size: Number of items to process in each batch
-            progress_poll_interval: Time in seconds between progress checks
-            
-        Returns:
-            tuple[list[str], list[str]]: Lists of successful and failed URLs
-            
-        Raises:
-            ValueError: If multi_metadata lengths don't match media_paths length
-        """
-
-        if multi_metadata and not len(multi_metadata) == len(media_paths):
-            raise ValueError("The number of assets must match the number of metadatas.")
-        
-        if multi_metadata and not all(len(data) == len(multi_metadata[0]) for data in multi_metadata):
-            raise ValueError("All metadatas must have the same length.")
-        
-        # Setup tracking variables
-        total_uploads = len(media_paths)
-        
-        # Create thread control events
-        stop_progress_tracking = threading.Event()
-        progress_tracking_error = threading.Event()
-        
-        # Create and start progress tracking thread
-        progress_thread = self._get_progress_tracker(
-            total_uploads, 
-            stop_progress_tracking, 
-            progress_tracking_error,
-            progress_poll_interval
-        )
-        progress_thread.start()
-        
-        # Process uploads in chunks
-        try:
-            successful_uploads, failed_uploads = self._process_uploads_in_chunks(
-                media_paths,
-                multi_metadata,
-                max_workers,
-                chunk_size,
-                stop_progress_tracking,
-                progress_tracking_error
-            )
-        finally:
-            progress_thread.join(10)  # Add margin to the timeout for tqdm
-        
-        # Log final progress
-        self._log_final_progress(
-            total_uploads, 
-            progress_poll_interval,
-            successful_uploads,
-            failed_uploads
-        )
-
-        if failed_uploads:
-            raise RuntimeError(f"Upload failed for {failed_uploads}")
-
-        return successful_uploads, failed_uploads

rapidata/rapidata_client/workflow/_ranking_workflow.py

@@ -1,51 +0,0 @@
-from rapidata.api_client import CompareWorkflowModelPairMakerConfig, OnlinePairMakerConfigModel, EloConfigModel
-from rapidata.api_client.models.compare_workflow_model import CompareWorkflowModel
-from rapidata.rapidata_client.workflow._base_workflow import Workflow
-from rapidata.rapidata_client.metadata import PromptMetadata
-from rapidata.api_client.models.create_datapoint_from_urls_model import (
-    CreateDatapointFromFilesModelMetadataInner,
-)
-
-
-class RankingWorkflow(Workflow):
-
-
-    def __init__(self,
-                 criteria: str,
-                 total_comparison_budget: int,
-                 random_comparisons_ratio,
-                 elo_start: int = 1200,
-                 elo_k_factor: int = 40,
-                 elo_scaling_factor: int = 400,
-                 context: str | None = None,
-                 ):
-        super().__init__(type="CompareWorkflowConfig")
-
-        self.context = [CreateDatapointFromFilesModelMetadataInner(
-            PromptMetadata(context).to_model())
-            ] if context else None
-
-        self.criteria = criteria
-        self.pair_maker_config = CompareWorkflowModelPairMakerConfig(
-            OnlinePairMakerConfigModel(
-                _t='OnlinePairMaker',
-                totalComparisonBudget=total_comparison_budget,
-                randomMatchesRatio=random_comparisons_ratio,
-            )
-        )
-
-        self.elo_config = EloConfigModel(
-            startingElo=elo_start,
-            kFactor=elo_k_factor,
-            scalingFactor=elo_scaling_factor,
-        )
-
-    def _to_model(self) -> CompareWorkflowModel:
-
-        return CompareWorkflowModel(
-            _t="CompareWorkflow",
-            criteria=self.criteria,
-            eloConfig=self.elo_config,
-            pairMakerConfig=self.pair_maker_config,
-            metadata=self.context,
-        )

rapidata/service/local_file_service.py

@@ -1,25 +0,0 @@
-import os
-from PIL import Image
-
-class LocalFileService:
-    
-    def load_image(self, image_path: str) -> Image.Image:
-        self.check_file_exists(image_path)
-        return Image.open(image_path)
-    
-    def load_images(self, image_paths: list[str]) -> list[Image.Image]:
-        return [self.load_image(image_path) for image_path in image_paths]
-    
-    def load_video(self, video_path: str):
-        self.check_file_exists(video_path)
-        return open(video_path, 'rb')
-    
-    def load_videos(self, video_paths: list[str]):
-        return [self.load_video(video_path) for video_path in video_paths]
-    
-    def _file_exists(self, file_path: str) -> bool:
-        return os.path.exists(file_path)
-    
-    def check_file_exists(self, file_path: str):
-        if not self._file_exists(file_path):
-            raise FileNotFoundError(f"File {file_path} not found.")

rapidata-2.23.2.dist-info/METADATA

@@ -1,34 +0,0 @@
-Metadata-Version: 2.3
-Name: rapidata
-Version: 2.23.2
-Summary: Rapidata package containing the Rapidata Python Client to interact with the Rapidata Web API in an easy way.
-License: Apache-2.0
-Author: Rapidata AG
-Author-email: info@rapidata.ai
-Requires-Python: >=3.10,<4.0
-Classifier: License :: OSI Approved :: Apache Software License
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
-Classifier: Programming Language :: Python :: 3.13
-Requires-Dist: authlib (>=1.5.1,<2.0.0)
-Requires-Dist: colorama (==0.4.6)
-Requires-Dist: deprecated (>=1.2.14,<2.0.0)
-Requires-Dist: httpx (>=0.28.1,<0.29.0)
-Requires-Dist: pandas (>=2.2.3,<3.0.0)
-Requires-Dist: pillow (>=10.4.0,<11.0.0)
-Requires-Dist: pydantic (>=2.8.2,<3.0.0)
-Requires-Dist: pyjwt (>=2.9.0,<3.0.0)
-Requires-Dist: python-dateutil (>=2.9.0.post0,<3.0.0)
-Requires-Dist: requests (>=2.32.3,<3.0.0)
-Requires-Dist: tinytag (>=2.0.0,<3.0.0)
-Requires-Dist: tqdm (>=4.66.5,<5.0.0)
-Description-Content-Type: text/markdown
-
-# Rapidata Python Client
-
-Python client to interface with the Rapidata API.
-
-Docs: https://docs.rapidata.ai/
-