rapidata 1.8.3__py3-none-any.whl → 2.44.4__py3-none-any.whl

rapidata/rapidata_client/dataset/rapid_builders/transcription_rapid_builders.py

@@ -1,132 +0,0 @@
-from rapidata.rapidata_client.assets import MediaAsset
-from rapidata.rapidata_client.dataset.rapid_builders.rapids import TranscriptionRapid
-
-class TranscriptionRapidBuilder:
-    """Final builder class for transcription rapid.
-    
-    This class handles the final construction of a transcription rapid with all required parameters.
-    """
-    def __init__(self, instruction: str, truths: list[int], asset: MediaAsset, transcription_text: str):
-        self._instruction = instruction
-        self._truths = truths
-        self._asset = asset
-        self._transcription_text = transcription_text
-        self._strict_grading = True
-
-    def strict_grading(self, strict_grading: bool = True):
-        """Set whether to use strict grading for the transcription.
-        Strict grading true: In order to be correct, you must select all of the right words
-        Strict grading false: In order to be correct, you must select at least one right word
-        In both cases it will be incorrect if you select any wrong words
-        
-        Args:
-            strict_grading (bool): Whether to use strict grading. Defaults to True.
-            
-        Returns:
-            TranscriptionRapidBuilder: The builder instance for method chaining
-        """
-        self._strict_grading = strict_grading
-        return self
-
-    def build(self):
-        """Constructs and returns the final transcription rapid.
-        
-        Returns:
-            TranscriptionRapid: The constructed transcription rapid
-        """
-        return TranscriptionRapid(
-            instruction=self._instruction,
-            truths=self._truths,
-            asset=self._asset,
-            transcription=self._transcription_text,
-            strict_grading=self._strict_grading
-        )
-
-class TranscriptionRapidTruthsBuilder:
-    """Builder class for the truths of the transcription rapid.
-
-    This adds the truths to the transcription rapid.
-    """
-    def __init__(self, instruction: str, media: MediaAsset, transcription_text: str):
-        self._instruction = instruction
-        self._media = media
-        self._transcription_text = transcription_text
-        self._truths = None
-
-    def truths(self, truths: list[int]):
-        """Set the truths for the transcription rapid.
-
-        Args:
-            truths (list[int]): The correct answers for the transcription task. \
-                Each integer represents the index of the correct word in the transcription text."""
-        
-        if not isinstance(truths, list) or not all(isinstance(truth, int) for truth in truths):
-            raise ValueError("Truths must be a list of integers")
-        
-        self._truths = truths
-        return self._build()
-    
-    def _build(self):
-        if self._truths is None:
-            raise ValueError("Truths are required")
-        
-        return TranscriptionRapidBuilder(
-            instruction=self._instruction,
-            truths=self._truths,
-            asset=self._media,
-            transcription_text=self._transcription_text
-        )
-
-class TranscriptionRapidAssetBuilder:
-    """Builder class for the asset of the transcription rapid.
-
-    This adds the asset to the transcription rapid.
-    """
-    def __init__(self, instruction: str):
-        self._instruction = instruction
-
-    def media(self, media: str, transcription_text: str):
-        """Set the media asset for the transcription rapid.
-
-        Args:
-            media (str): The local file path of the audio or video file to be transcribed 
-            transcription_text (str): The text to be transcribed from the media asset""" # is video file okay?
-        
-        self._asset = MediaAsset(media)
-        self._transcription_text = transcription_text
-
-        return self._build()
-
-    def _build(self):
-        if not self._asset:
-            raise ValueError("Media is required")
-        
-        return TranscriptionRapidTruthsBuilder(
-            instruction=self._instruction,
-            media=self._asset,
-            transcription_text=self._transcription_text
-        )
-
-class TranscriptionRapidInstructionBuilder:
-    def __init__(self):
-        self._instruction = None
-
-    def instruction(self, instruction: str):
-        """Set the instruction for the transcription rapid.
-
-        Args:
-            instruction (str): The instruction for the transcription task"""
-        
-        if not isinstance(instruction, str):
-            raise ValueError("Instruction must be a string")
-        
-        self._instruction = instruction
-        return self._build()
-    
-    def _build(self):
-        if self._instruction is None:
-            raise ValueError("Instruction is required")
-        
-        return TranscriptionRapidAssetBuilder(
-            instruction=self._instruction,
-        )

rapidata/rapidata_client/dataset/rapidata_dataset.py

@@ -1,126 +0,0 @@
-from itertools import zip_longest
-import os
-
-from rapidata.api_client.models.datapoint_metadata_model import DatapointMetadataModel
-from rapidata.api_client.models.datapoint_metadata_model_metadata_inner import (
-    DatapointMetadataModelMetadataInner,
-)
-from rapidata.api_client.models.upload_text_sources_to_dataset_model import (
-    UploadTextSourcesToDatasetModel,
-)
-from rapidata.rapidata_client.metadata.base_metadata import Metadata
-from rapidata.rapidata_client.assets import TextAsset, MediaAsset, MultiAsset
-from rapidata.service import LocalFileService
-from rapidata.service.openapi_service import OpenAPIService
-from concurrent.futures import ThreadPoolExecutor, as_completed
-from tqdm import tqdm
-
-from typing import cast
-
-
-class RapidataDataset:
-
-    def __init__(self, dataset_id: str, openapi_service: OpenAPIService):
-        self.dataset_id = dataset_id
-        self.openapi_service = openapi_service
-        self.local_file_service = LocalFileService()
-
-    def add_texts(
-        self,
-        text_assets: list[TextAsset] | list[MultiAsset],
-        max_workers: int = 10,
-    ):
-        for text_asset in text_assets:
-            if isinstance(text_asset, MultiAsset):
-                assert all(
-                    isinstance(asset, TextAsset) for asset in text_asset.assets
-                ), "All assets in a MultiAsset must be of type TextAsset."
-
-        def upload_text_datapoint(text_asset: TextAsset | MultiAsset) -> None:
-            if isinstance(text_asset, TextAsset):
-                texts = [text_asset.text]
-            elif isinstance(text_asset, MultiAsset):
-                texts = [asset.text for asset in text_asset.assets if isinstance(asset, TextAsset)]
-            else:
-                raise ValueError(f"Unsupported asset type: {type(text_asset)}")
-
-            model = UploadTextSourcesToDatasetModel(
-                datasetId=self.dataset_id,
-                textSources=texts
-            )
-
-            self.openapi_service.dataset_api.dataset_creat_text_datapoint_post(model)
-
-        total_uploads = len(text_assets)
-        with ThreadPoolExecutor(max_workers=max_workers) as executor:
-            futures = [
-                executor.submit(upload_text_datapoint, text_asset)
-                for text_asset in text_assets
-            ]
-
-            with tqdm(total=total_uploads, desc="Uploading text datapoints") as pbar:
-                for future in as_completed(futures):
-                    future.result()  # This will raise any exceptions that occurred during execution
-                    pbar.update(1)
-
-    def add_media_from_paths(
-        self,
-        media_paths: list[MediaAsset] |  list[MultiAsset], # where multiasset is a list of media assets
-        metadata: list[Metadata] | None = None,
-        max_workers: int = 10,
-    ):
-        if metadata is not None and len(metadata) != len(media_paths):
-            raise ValueError(
-                "metadata must be None or have the same length as media_paths"
-            )
-
-        for media_path in media_paths:
-            if isinstance(media_path, MultiAsset):
-                assert all(
-                    isinstance(asset, MediaAsset) for asset in media_path.assets
-                ), "All assets in a MultiAsset must be of type MediaAsset."
-
-        def upload_datapoint(media_asset: MediaAsset | MultiAsset, meta: Metadata | None) -> None:
-            if isinstance(media_asset, MediaAsset):
-                assets = [media_asset]
-            elif isinstance(media_asset, MultiAsset):
-                assets = media_asset.assets
-            else:
-                raise ValueError(f"Unsupported asset type: {type(media_asset)}")
-
-            meta_model = meta.to_model() if meta else None
-            model = DatapointMetadataModel(
-                datasetId=self.dataset_id,
-                metadata=(
-                    [DatapointMetadataModelMetadataInner(meta_model)]
-                    if meta_model
-                    else []
-                ),
-            )
-
-            files: list[tuple[str, bytes] | str] = []
-            for asset in assets:
-                if isinstance(asset, MediaAsset):
-                    if isinstance(asset.path, bytes):
-                        files.append((asset.name, asset.path))
-                    else:
-                        files.append(cast(str, asset.path))
-
-            upload_response = self.openapi_service.dataset_api.dataset_create_datapoint_post(
-                model=model,
-                files=files # type: ignore
-            )
-            if upload_response.errors:
-                raise ValueError(f"Error uploading datapoint: {upload_response.errors}")
-
-        total_uploads = len(media_paths)
-        with ThreadPoolExecutor(max_workers=max_workers) as executor:
-            futures = [
-                executor.submit(upload_datapoint, media_asset, meta)
-                for media_asset, meta in zip_longest(media_paths, metadata or [])
-            ]
-
-            with tqdm(total=total_uploads, desc="Uploading datapoints") as pbar:
-                for future in as_completed(futures):
-                    future.result()  # This will raise any exceptions that occurred during execution
-                    pbar.update(1)

rapidata/rapidata_client/dataset/rapidata_validation_set.py

@@ -1,300 +0,0 @@
-import os
-from typing import Any
-from rapidata.api_client.models.add_validation_rapid_model import (
-    AddValidationRapidModel,
-)
-from rapidata.api_client.models.add_validation_text_rapid_model import (
-    AddValidationTextRapidModel,
-)
-from rapidata.api_client.models.add_validation_rapid_model_payload import (
-    AddValidationRapidModelPayload,
-)
-from rapidata.api_client.models.add_validation_rapid_model_truth import (
-    AddValidationRapidModelTruth,
-)
-from rapidata.api_client.models.attach_category_truth import AttachCategoryTruth
-from rapidata.api_client.models.bounding_box_payload import BoundingBoxPayload
-from rapidata.api_client.models.bounding_box_truth import BoundingBoxTruth
-from rapidata.api_client.models.classify_payload import ClassifyPayload
-from rapidata.api_client.models.compare_payload import ComparePayload
-from rapidata.api_client.models.compare_truth import CompareTruth
-from rapidata.api_client.models.datapoint_metadata_model_metadata_inner import (
-    DatapointMetadataModelMetadataInner,
-)
-from rapidata.api_client.models.empty_validation_truth import EmptyValidationTruth
-from rapidata.api_client.models.free_text_payload import FreeTextPayload
-from rapidata.api_client.models.line_payload import LinePayload
-from rapidata.api_client.models.line_truth import LineTruth
-from rapidata.api_client.models.locate_box_truth import LocateBoxTruth
-from rapidata.api_client.models.locate_payload import LocatePayload
-from rapidata.api_client.models.named_entity_payload import NamedEntityPayload
-from rapidata.api_client.models.named_entity_truth import NamedEntityTruth
-from rapidata.api_client.models.polygon_payload import PolygonPayload
-from rapidata.api_client.models.polygon_truth import PolygonTruth
-from rapidata.api_client.models.transcription_payload import TranscriptionPayload
-from rapidata.api_client.models.transcription_truth import TranscriptionTruth
-from rapidata.api_client.models.transcription_word import TranscriptionWord
-from rapidata.rapidata_client.assets.media_asset import MediaAsset
-from rapidata.rapidata_client.assets.multi_asset import MultiAsset
-from rapidata.rapidata_client.assets.text_asset import TextAsset
-from rapidata.rapidata_client.metadata.base_metadata import Metadata
-from rapidata.service.openapi_service import OpenAPIService
-
-
-class RapidataValidationSet:
-    """A class for interacting with a Rapidata validation set.
-
-    Get a `ValidationSet` either by using `rapi.get_validation_set(id)` to get an existing validation set or by using `rapi.new_validation_set(name)` to create a new validation set.
-    """
-
-    def __init__(self, validation_set_id, openapi_service: OpenAPIService, name: str):
-        self.id = validation_set_id
-        self.openapi_service = openapi_service
-        self.name = name
-
-    def upload_files(self, model: AddValidationRapidModel, assets: list[MediaAsset]):
-        """Upload a file to the validation set.
-
-        Args:
-            asset list[(MediaAsset)]: The asset to upload.
-
-        Returns:
-            str: The path to the uploaded file.
-        """
-        files = []
-        for asset in assets:
-            if isinstance(asset.path, str):
-                files.append(asset.path)
-            elif isinstance(asset.path, bytes):
-                files.append((asset.name, asset.path))
-            else:
-                raise ValueError("upload file failed")
-        self.openapi_service.validation_api.validation_add_validation_rapid_post(
-            model=model, files=files
-        )        
-
-    def add_general_validation_rapid(
-        self,
-        payload: (
-            BoundingBoxPayload
-            | ClassifyPayload
-            | ComparePayload
-            | FreeTextPayload
-            | LinePayload
-            | LocatePayload
-            | NamedEntityPayload
-            | PolygonPayload
-            | TranscriptionPayload
-        ),
-        truths: (
-            AttachCategoryTruth
-            | BoundingBoxTruth
-            | CompareTruth
-            | EmptyValidationTruth
-            | LineTruth
-            | LocateBoxTruth
-            | NamedEntityTruth
-            | PolygonTruth
-            | TranscriptionTruth
-        ),
-        metadata: list[Metadata],
-        asset: MediaAsset | TextAsset | MultiAsset,
-        randomCorrectProbability: float,
-    ) -> None:
-        """Add a validation rapid to the validation set.
-
-        Args:
-            payload: The payload for the rapid.
-            truths: The truths for the rapid.
-            metadata (list[Metadata]): The metadata for the rapid.
-            asset: The asset(s) for the rapid.
-            randomCorrectProbability (float): The random correct probability for the rapid.
-
-        Returns:
-            None
-
-        Raises:
-            ValueError: If an invalid asset type is provided.
-        """
-
-        model = AddValidationRapidModel(
-            validationSetId=self.id,
-            payload=AddValidationRapidModelPayload(payload),
-            truth=AddValidationRapidModelTruth(truths),
-            metadata=[
-                DatapointMetadataModelMetadataInner(meta.to_model())
-                for meta in metadata
-            ],
-            randomCorrectProbability=randomCorrectProbability,
-        )
-        if isinstance(asset, MediaAsset):
-            self.upload_files(model=model, assets=[asset])
-
-        elif isinstance(asset, TextAsset):
-            model = AddValidationTextRapidModel(
-                validationSetId=self.id,
-                payload=AddValidationRapidModelPayload(payload),
-                truth=AddValidationRapidModelTruth(truths),
-                metadata=[
-                    DatapointMetadataModelMetadataInner(meta.to_model())
-                    for meta in metadata
-                ],
-                randomCorrectProbability=randomCorrectProbability,
-                texts=[asset.text],
-            )
-            self.openapi_service.validation_api.validation_add_validation_text_rapid_post(
-                add_validation_text_rapid_model=model
-            )
-
-        elif isinstance(asset, MultiAsset):
-            files = [a for a in asset if isinstance(a, MediaAsset)]
-            texts = [a.text for a in asset if isinstance(a, TextAsset)]
-            if files:
-                self.upload_files(model=model, assets=files)
-            if texts:
-                model = AddValidationTextRapidModel(
-                    validationSetId=self.id,
-                    payload=AddValidationRapidModelPayload(payload),
-                    truth=AddValidationRapidModelTruth(truths),
-                    metadata=[
-                        DatapointMetadataModelMetadataInner(meta.to_model())
-                        for meta in metadata
-                    ],
-                    randomCorrectProbability=randomCorrectProbability,
-                    texts=texts,
-                )
-                self.openapi_service.validation_api.validation_add_validation_text_rapid_post(
-                    add_validation_text_rapid_model=model
-                )
-
-
-        else:
-            raise ValueError("Invalid asset type")
-
-    def add_classify_rapid(
-        self,
-        asset: MediaAsset | TextAsset,
-        question: str,
-        categories: list[str],
-        truths: list[str],
-        metadata: list[Metadata] = [],
-    ) -> None:
-        """Add a classify rapid to the validation set.
-
-        Args:
-            asset (MediaAsset | TextAsset): The asset for the rapid.
-            question (str): The question for the rapid.
-            categories (list[str]): The list of categories for the rapid.
-            truths (list[str]): The list of truths for the rapid.
-            metadata (list[Metadata], optional): The metadata for the rapid. Defaults to an empty list.
-
-        Returns:
-            None
-        """
-        payload = ClassifyPayload(
-            _t="ClassifyPayload", possibleCategories=categories, title=question
-        )
-        model_truth = AttachCategoryTruth(
-            correctCategories=truths, _t="AttachCategoryTruth"
-        )
-
-        self.add_general_validation_rapid(
-            payload=payload,
-            truths=model_truth,
-            metadata=metadata,
-            asset=asset,
-            randomCorrectProbability=len(truths) / len(categories),
-        )
-
-    def add_compare_rapid(
-        self,
-        asset: MultiAsset,
-        question: str,
-        truth: str,
-        metadata: list[Metadata] = [],
-    ) -> None:
-        """Add a compare rapid to the validation set.
-
-        Args:
-            asset (MultiAsset): The assets for the rapid.
-            question (str): The question for the rapid.
-            truth (str): The path to the truth file.
-            metadata (list[Metadata], optional): The metadata for the rapid. Defaults to an empty list.
-
-        Returns:
-            None
-
-        Raises:
-            ValueError: If the number of assets is not exactly two.
-        """
-        payload = ComparePayload(_t="ComparePayload", criteria=question)
-        # take only last part of truth path
-        truth = os.path.basename(truth)
-        model_truth = CompareTruth(_t="CompareTruth", winnerId=truth)
-
-        if len(asset) != 2:
-            raise ValueError("Compare rapid requires exactly two media paths")
-
-        self.add_general_validation_rapid(
-            payload=payload,
-            truths=model_truth,
-            metadata=metadata,
-            asset=asset,
-            randomCorrectProbability=1 / len(asset),
-        )
-
-    def add_transcription_rapid(
-        self,
-        asset: MediaAsset | TextAsset,
-        question: str,
-        transcription: list[str],
-        correct_words: list[str],
-        strict_grading: bool | None = None,
-        metadata: list[Metadata] = [],
-    ) -> None:
-        """Add a transcription rapid to the validation set.
-
-        Args:
-            asset (MediaAsset | TextAsset): The asset for the rapid.
-            question (str): The question for the rapid.
-            transcription (list[str]): The transcription for the rapid.
-            correct_words (list[str]): The list of correct words for the rapid.
-            strict_grading (bool | None, optional): The strict grading for the rapid. Defaults to None.
-            metadata (list[Metadata], optional): The metadata for the rapid. Defaults to an empty list.
-
-        Returns:
-            None
-
-        Raises:
-            ValueError: If a correct word is not found in the transcription.
-        """
-        transcription_words = [
-            TranscriptionWord(word=word, wordIndex=i)
-            for i, word in enumerate(transcription)
-        ]
-
-        correct_transcription_words = []
-        for word in correct_words:
-            if word not in transcription:
-                raise ValueError(f"Correct word '{word}' not found in transcription")
-            correct_transcription_words.append(
-                TranscriptionWord(word=word, wordIndex=transcription.index(word))
-            )
-
-        payload = TranscriptionPayload(
-            _t="TranscriptionPayload", title=question, transcription=transcription_words
-        )
-
-        model_truth = TranscriptionTruth(
-            _t="TranscriptionTruth",
-            correctWords=correct_transcription_words,
-            strictGrading=strict_grading,
-        )
-
-        self.add_general_validation_rapid(
-            payload=payload,
-            truths=model_truth,
-            metadata=metadata,
-            asset=asset,
-            randomCorrectProbability=len(correct_words) / len(transcription),
-        )

rapidata/rapidata_client/dataset/validation_rapid_parts.py

@@ -1,55 +0,0 @@
-from dataclasses import dataclass
-from typing import Any
-
-from rapidata.api_client.models.attach_category_truth import AttachCategoryTruth
-from rapidata.api_client.models.bounding_box_payload import BoundingBoxPayload
-from rapidata.api_client.models.bounding_box_truth import BoundingBoxTruth
-from rapidata.api_client.models.classify_payload import ClassifyPayload
-from rapidata.api_client.models.compare_payload import ComparePayload
-from rapidata.api_client.models.compare_truth import CompareTruth
-from rapidata.api_client.models.empty_validation_truth import EmptyValidationTruth
-from rapidata.api_client.models.free_text_payload import FreeTextPayload
-from rapidata.api_client.models.line_payload import LinePayload
-from rapidata.api_client.models.line_truth import LineTruth
-from rapidata.api_client.models.locate_box_truth import LocateBoxTruth
-from rapidata.api_client.models.locate_payload import LocatePayload
-from rapidata.api_client.models.named_entity_payload import NamedEntityPayload
-from rapidata.api_client.models.named_entity_truth import NamedEntityTruth
-from rapidata.api_client.models.polygon_payload import PolygonPayload
-from rapidata.api_client.models.polygon_truth import PolygonTruth
-from rapidata.api_client.models.transcription_payload import TranscriptionPayload
-from rapidata.api_client.models.transcription_truth import TranscriptionTruth
-from rapidata.rapidata_client.assets.media_asset import MediaAsset
-from rapidata.rapidata_client.assets.multi_asset import MultiAsset
-from rapidata.rapidata_client.assets.text_asset import TextAsset
-from rapidata.rapidata_client.metadata.base_metadata import Metadata
-
-
-@dataclass
-class ValidatioRapidParts:
-    question: str
-    asset: MediaAsset | TextAsset | MultiAsset
-    payload: (
-        BoundingBoxPayload
-        | ClassifyPayload
-        | ComparePayload
-        | FreeTextPayload
-        | LinePayload
-        | LocatePayload
-        | NamedEntityPayload
-        | PolygonPayload
-        | TranscriptionPayload
-    )
-    truths: (
-        AttachCategoryTruth
-        | BoundingBoxTruth
-        | CompareTruth
-        | EmptyValidationTruth
-        | LineTruth
-        | LocateBoxTruth
-        | NamedEntityTruth
-        | PolygonTruth
-        | TranscriptionTruth
-    )
-    metadata: list[Metadata]
-    randomCorrectProbability: float