rapidata 2.37.0__py3-none-any.whl → 3.0.0a8__py3-none-any.whl

rapidata/rapidata_client/datapoints/_datapoint.py

@@ -1,196 +1,92 @@
-from typing import Sequence, cast
-from rapidata.rapidata_client.datapoints.assets import (
-    MediaAsset,
-    TextAsset,
-    MultiAsset,
-    BaseAsset,
-)
+from __future__ import annotations
+
+from typing import Literal, TYPE_CHECKING
+
+from pydantic import BaseModel, model_validator, field_validator
+from typing_extensions import Self
 from rapidata.rapidata_client.datapoints.assets.constants import (
     ALLOWED_VIDEO_EXTENSIONS,
     ALLOWED_IMAGE_EXTENSIONS,
     ALLOWED_AUDIO_EXTENSIONS,
 )
-from rapidata.rapidata_client.datapoints.metadata import Metadata
-from rapidata.api_client.models.dataset_dataset_id_datapoints_post_request_metadata_inner import (
-    DatasetDatasetIdDatapointsPostRequestMetadataInner,
-)
-from rapidata.api_client.models.create_datapoint_from_text_sources_model import (
-    CreateDatapointFromTextSourcesModel,
-)
-from pydantic import StrictStr, StrictBytes
-from rapidata.api_client.models.asset_type import AssetType
-from rapidata.api_client.models.prompt_type import PromptType
-from rapidata.rapidata_client.datapoints.metadata._media_asset_metadata import (
-    MediaAssetMetadata,
-)
-from rapidata.rapidata_client.datapoints.metadata._prompt_metadata import PromptMetadata
-from rapidata.rapidata_client.logging import logger
-
-
-class Datapoint:
-    def __init__(
-        self,
-        asset: MediaAsset | TextAsset | MultiAsset,
-        metadata: Sequence[Metadata] | None = None,
-    ):
-        if not isinstance(asset, (MediaAsset, TextAsset, MultiAsset)):
-            raise TypeError(
-                "Asset must be of type MediaAsset, TextAsset, or MultiAsset."
+from rapidata.rapidata_client.config import logger
+
+if TYPE_CHECKING:
+    from rapidata.api_client.models.asset_type import AssetType
+    from rapidata.api_client.models.prompt_type import PromptType
+
+
+class Datapoint(BaseModel):
+    asset: str | list[str]
+    data_type: Literal["text", "media"]
+    context: str | None = None
+    media_context: str | None = None
+    sentence: str | None = None
+    private_note: str | None = None
+    group: str | None = None
+
+    @field_validator("context")
+    @classmethod
+    def context_not_empty(cls, v: str | None) -> str | None:
+        if v is not None and v == "":
+            raise ValueError(
+                "context cannot be an empty string. If not needed, set to None."
             )
+        return v
 
-        if metadata and not isinstance(metadata, Sequence):
-            raise TypeError("Metadata must be a list of Metadata objects.")
-
-        if metadata and not all(isinstance(m, Metadata) for m in metadata):
-            raise TypeError("All metadata objects must be of type Metadata.")
-
-        self.asset = asset
-        self.metadata = metadata
-
-    def _get_effective_asset_type(self) -> type:
-        """Get the effective asset type, handling MultiAsset by looking at its first asset."""
-        if isinstance(self.asset, MultiAsset):
-            return type(self.asset.assets[0])
-        return type(self.asset)
-
-    def is_media_asset(self) -> bool:
-        """Check if this datapoint contains media assets."""
-        effective_type = self._get_effective_asset_type()
-        return issubclass(effective_type, MediaAsset)
-
-    def is_text_asset(self) -> bool:
-        """Check if this datapoint contains text assets."""
-        effective_type = self._get_effective_asset_type()
-        return issubclass(effective_type, TextAsset)
+    @field_validator("media_context")
+    @classmethod
+    def media_context_not_empty(cls, v: str | None) -> str | None:
+        if v is not None and v == "":
+            raise ValueError(
+                "media_context cannot be an empty string. If not needed, set to None."
+            )
+        return v
+
+    @field_validator("sentence")
+    @classmethod
+    def sentence_has_space(cls, v: str | None) -> str | None:
+        if v is not None and len(v.split()) <= 1:
+            raise ValueError("sentence must contain at least two words.")
+        return v
+
+    @model_validator(mode="after")
+    def check_sentence_and_context(self) -> Self:
+        if isinstance(self.sentence, str) and isinstance(self.context, str):
+            raise ValueError(
+                "Both 'sentence' and 'context' cannot be strings at the same time."
+            )
+        return self
 
     def get_asset_type(self) -> AssetType:
-        """Get the asset type of the datapoint."""
-        if self.is_text_asset():
+        from rapidata.api_client.models.asset_type import AssetType
+
+        if self.data_type == "text":
             return AssetType.TEXT
-        elif self.is_media_asset():
-            if isinstance(self.asset, MultiAsset):
-                asset = self.asset.assets[0]
-            else:
-                asset = self.asset
-            assert isinstance(asset, MediaAsset)
-            if any(asset.path.endswith(ext) for ext in ALLOWED_IMAGE_EXTENSIONS):
-                return AssetType.IMAGE
-            elif any(asset.path.endswith(ext) for ext in ALLOWED_VIDEO_EXTENSIONS):
-                return AssetType.VIDEO
-            elif any(asset.path.endswith(ext) for ext in ALLOWED_AUDIO_EXTENSIONS):
-                return AssetType.AUDIO
-            else:
-                logger.debug(
-                    f"Cannot get asset type for asset type: {type(self.asset)}"
-                )
-                return AssetType.NONE
+
+        evaluation_asset = self.asset[0] if isinstance(self.asset, list) else self.asset
+        if any(evaluation_asset.endswith(ext) for ext in ALLOWED_IMAGE_EXTENSIONS):
+            return AssetType.IMAGE
+        elif any(evaluation_asset.endswith(ext) for ext in ALLOWED_VIDEO_EXTENSIONS):
+            return AssetType.VIDEO
+        elif any(evaluation_asset.endswith(ext) for ext in ALLOWED_AUDIO_EXTENSIONS):
+            return AssetType.AUDIO
         else:
-            logger.debug(f"Cannot get asset type for asset type: {type(self.asset)}")
+            logger.debug(
+                f"Cannot get asset type for asset type: {type(evaluation_asset)}"
+            )
             return AssetType.NONE
 
     def get_prompt_type(self) -> list[PromptType]:
-        """Get the prompt type of the datapoint."""
+        from rapidata.api_client.models.prompt_type import PromptType
+
         prompt_types = []
-        for metadata in self.metadata or []:
-            if isinstance(metadata, MediaAssetMetadata):
-                prompt_types.append(PromptType.ASSET)
-            elif isinstance(metadata, PromptMetadata):
-                prompt_types.append(PromptType.TEXT)
+        if self.context:
+            prompt_types.append(PromptType.TEXT)
+        if self.media_context:
+            prompt_types.append(PromptType.ASSET)
 
         if len(prompt_types) == 0:
             return [PromptType.NONE]
 
         return prompt_types
-
-    def get_texts(self) -> list[str]:
-        """Extract text content from the asset(s)."""
-        if isinstance(self.asset, TextAsset):
-            return [self.asset.text]
-        elif isinstance(self.asset, MultiAsset):
-            texts = []
-            for asset in self.asset.assets:
-                if isinstance(asset, TextAsset):
-                    texts.append(asset.text)
-            return texts
-        else:
-            raise ValueError(f"Cannot extract text from asset type: {type(self.asset)}")
-
-    def get_media_assets(self) -> list[MediaAsset]:
-        """Extract media assets from the datapoint."""
-        if isinstance(self.asset, MediaAsset):
-            return [self.asset]
-        elif isinstance(self.asset, MultiAsset):
-            media_assets = []
-            for asset in self.asset.assets:
-                if isinstance(asset, MediaAsset):
-                    media_assets.append(asset)
-            return media_assets
-        else:
-            raise ValueError(
-                f"Cannot extract media assets from asset type: {type(self.asset)}"
-            )
-
-    def get_local_file_paths(
-        self,
-    ) -> list[StrictStr | tuple[StrictStr, StrictBytes] | StrictBytes]:
-        """Get local file paths for media assets that are stored locally."""
-        if not self.is_media_asset():
-            return []
-
-        media_assets = self.get_media_assets()
-        return [asset.to_file() for asset in media_assets if asset.is_local()]
-
-    def get_urls(self) -> list[str]:
-        """Get URLs for media assets that are remote."""
-        if not self.is_media_asset():
-            return []
-
-        media_assets = self.get_media_assets()
-        return [asset.path for asset in media_assets if not asset.is_local()]
-
-    def get_prepared_metadata(
-        self,
-    ) -> list[DatasetDatasetIdDatapointsPostRequestMetadataInner]:
-        """Prepare metadata for API upload."""
-        metadata: list[DatasetDatasetIdDatapointsPostRequestMetadataInner] = []
-        if self.metadata:
-            for meta in self.metadata:
-                meta_model = meta.to_model() if meta else None
-                if meta_model:
-                    metadata.append(
-                        DatasetDatasetIdDatapointsPostRequestMetadataInner(meta_model)
-                    )
-        return metadata
-
-    def create_text_upload_model(
-        self, index: int
-    ) -> CreateDatapointFromTextSourcesModel:
-        """Create the model for uploading text datapoints."""
-        if not self.is_text_asset():
-            raise ValueError("Cannot create text upload model for non-text asset")
-
-        texts = self.get_texts()
-        metadata = self.get_prepared_metadata()
-
-        return CreateDatapointFromTextSourcesModel(
-            textSources=texts,
-            sortIndex=index,
-            metadata=metadata,
-        )
-
-    def get_datapoint_string(self) -> str:
-        """Get the datapoint string for the datapoint."""
-        if isinstance(self.asset, MediaAsset):
-            return self.asset.path
-        elif isinstance(self.asset, TextAsset):
-            return self.asset.text
-        else:
-            raise ValueError(
-                f"Cannot get datapoint string for asset type: {type(self.asset)}"
-            )
-
-    def __str__(self):
-        return f"Datapoint(asset={self.asset})"
-
-    def __repr__(self):
-        return self.__str__()

rapidata/rapidata_client/datapoints/_datapoint_uploader.py

@@ -0,0 +1,47 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from rapidata.rapidata_client.datapoints._datapoint import Datapoint
+from rapidata.service.openapi_service import OpenAPIService
+from rapidata.rapidata_client.datapoints._asset_uploader import AssetUploader
+
+if TYPE_CHECKING:
+    from rapidata.api_client.models.create_datapoint_result import CreateDatapointResult
+
+
+class DatapointUploader:
+    def __init__(self, openapi_service: OpenAPIService):
+        self.openapi_service = openapi_service
+        self.asset_uploader = AssetUploader(openapi_service)
+
+    def upload_datapoint(
+        self, datapoint: Datapoint, dataset_id: str, index: int
+    ) -> CreateDatapointResult:
+        from rapidata.api_client.models.create_datapoint_model import (
+            CreateDatapointModel,
+        )
+        from rapidata.api_client.models.i_asset_input import IAssetInput
+
+        uploaded_asset: IAssetInput = (
+            self.asset_uploader.get_uploaded_asset_input(datapoint.asset)
+            if datapoint.data_type == "media"
+            else self.asset_uploader.get_uploaded_text_input(datapoint.asset)
+        )
+        return self.openapi_service.dataset_api.dataset_dataset_id_datapoint_post(
+            dataset_id=dataset_id,
+            create_datapoint_model=CreateDatapointModel(
+                asset=uploaded_asset,
+                context=datapoint.context,
+                contextAsset=(
+                    self.asset_uploader.get_uploaded_asset_input(
+                        datapoint.media_context
+                    )
+                    if datapoint.media_context
+                    else None
+                ),
+                transcription=datapoint.sentence,
+                sortIndex=index,
+                group=datapoint.group,
+            ),
+        )

rapidata/rapidata_client/datapoints/_datapoints_validator.py

@@ -0,0 +1,70 @@
+from itertools import zip_longest
+from typing import Literal, cast, Iterable
+from rapidata.rapidata_client.datapoints._datapoint import Datapoint
+
+
+class DatapointsValidator:
+    @staticmethod
+    def validate_datapoints(
+        datapoints: list[str] | list[list[str]],
+        contexts: list[str] | None = None,
+        media_contexts: list[str] | None = None,
+        sentences: list[str] | None = None,
+        private_notes: list[str] | None = None,
+        groups: list[str] | None = None,
+    ) -> None:
+        if contexts and len(contexts) != len(datapoints):
+            raise ValueError("Number of contexts must match number of datapoints")
+        if media_contexts and len(media_contexts) != len(datapoints):
+            raise ValueError("Number of media contexts must match number of datapoints")
+        if sentences and len(sentences) != len(datapoints):
+            raise ValueError("Number of sentences must match number of datapoints")
+        if private_notes and len(private_notes) != len(datapoints):
+            raise ValueError("Number of private notes must match number of datapoints")
+        if groups and (
+            len(groups) != len(datapoints) or len(groups) != len(set(groups))
+        ):
+            raise ValueError(
+                "Number of groups must match number of datapoints and must be unique."
+            )
+
+    @staticmethod
+    def map_datapoints(
+        datapoints: list[str] | list[list[str]],
+        contexts: list[str] | None = None,
+        media_contexts: list[str] | None = None,
+        sentences: list[str] | None = None,
+        private_notes: list[str] | None = None,
+        groups: list[str] | None = None,
+        data_type: Literal["text", "media"] = "media",
+    ) -> list[Datapoint]:
+        DatapointsValidator.validate_datapoints(
+            datapoints=datapoints,
+            contexts=contexts,
+            media_contexts=media_contexts,
+            sentences=sentences,
+            private_notes=private_notes,
+            groups=groups,
+        )
+        return [
+            Datapoint(
+                asset=asset,
+                data_type=data_type,
+                context=context,
+                media_context=media_context,
+                sentence=sentence,
+                private_note=private_note,
+                group=group,
+            )
+            for asset, context, media_context, sentence, private_note, group in cast(
+                "Iterable[tuple[str | list[str], str | None, str | None, str | None, str | None, str | None]]",  # because iterator only supports 5 arguments with specific type casting
+                zip_longest(
+                    datapoints,
+                    contexts or [],
+                    media_contexts or [],
+                    sentences or [],
+                    private_notes or [],
+                    groups or [],
+                ),
+            )
+        ]

rapidata/rapidata_client/datapoints/assets/__init__.py

@@ -1,10 +0,0 @@
-"""Assets Package
-
-This package provides classes for different types of assets, including MediaAsset, TextAsset, and MultiAsset.
-"""
-from ._base_asset import BaseAsset
-from ._media_asset import MediaAsset
-from ._text_asset import TextAsset
-from ._multi_asset import MultiAsset
-from .data_type_enum import RapidataDataTypes
-from ._sessions import SessionManager

rapidata/rapidata_client/datapoints/metadata/__init__.py

@@ -1,7 +1,5 @@
 from ._base_metadata import Metadata
 from ._private_text_metadata import PrivateTextMetadata
 from ._public_text_metadata import PublicTextMetadata
-from ._prompt_metadata import PromptMetadata
 from ._select_words_metadata import SelectWordsMetadata
-from ._media_asset_metadata import MediaAssetMetadata
 from ._prompt_identifier_metadata import PromptIdentifierMetadata

rapidata/rapidata_client/datapoints/metadata/_private_text_metadata.py

@@ -2,16 +2,15 @@ from rapidata.api_client.models.private_text_metadata_input import (
     PrivateTextMetadataInput,
 )
 from rapidata.rapidata_client.datapoints.metadata._base_metadata import Metadata
+from pydantic import BaseModel
 
 
-class PrivateTextMetadata(Metadata):
+class PrivateTextMetadata(Metadata, BaseModel):
 
-    def __init__(self, text: str, identifier: str = "private_text"):
-        super().__init__()
-        self._identifier = identifier
-        self._text = text
+    text: str
+    identifier: str = "private_text"
 
     def to_model(self):
         return PrivateTextMetadataInput(
-            _t="PrivateTextMetadataInput", identifier=self._identifier, text=self._text
+            _t="PrivateTextMetadataInput", identifier=self.identifier, text=self.text
         )

rapidata/rapidata_client/datapoints/metadata/_prompt_identifier_metadata.py

@@ -2,14 +2,13 @@ from rapidata.rapidata_client.datapoints.metadata._base_metadata import Metadata
 from rapidata.api_client.models.private_text_metadata_input import (
     PrivateTextMetadataInput,
 )
+from pydantic import BaseModel
 
 
-class PromptIdentifierMetadata(Metadata):
-    def __init__(self, identifier: str):
-        super().__init__()
-        self._identifier = identifier
+class PromptIdentifierMetadata(Metadata, BaseModel):
+    identifier: str
 
     def to_model(self):
         return PrivateTextMetadataInput(
-            _t="PrivateTextMetadataInput", identifier="prompt-id", text=self._identifier
+            _t="PrivateTextMetadataInput", identifier="prompt-id", text=self.identifier
         )

rapidata/rapidata_client/datapoints/metadata/_public_text_metadata.py

@@ -2,16 +2,15 @@ from rapidata.api_client.models.public_text_metadata_input import (
     PublicTextMetadataInput,
 )
 from rapidata.rapidata_client.datapoints.metadata._base_metadata import Metadata
+from pydantic import BaseModel
 
 
-class PublicTextMetadata(Metadata):
+class PublicTextMetadata(Metadata, BaseModel):
 
-    def __init__(self, text: str, identifier: str = "public_text"):
-        super().__init__()
-        self._identifier = identifier
-        self._text = text
+    text: str
+    identifier: str = "public_text"
 
     def to_model(self):
         return PublicTextMetadataInput(
-            _t="PublicTextMetadataInput", identifier=self._identifier, text=self._text
+            _t="PublicTextMetadataInput", identifier=self.identifier, text=self.text
         )

rapidata/rapidata_client/datapoints/metadata/_select_words_metadata.py

@@ -2,14 +2,13 @@ from rapidata.api_client.models.transcription_metadata_input import (
     TranscriptionMetadataInput,
 )
 from rapidata.rapidata_client.datapoints.metadata._base_metadata import Metadata
+from pydantic import BaseModel
 
 
-class SelectWordsMetadata(Metadata):
+class SelectWordsMetadata(Metadata, BaseModel):
     """SelectWordsMetadata Class is used to define the Sentence that will be display to the user."""
 
-    def __init__(self, select_words: str):
-        super().__init__()
-        self.select_words = select_words
+    select_words: str
 
     def to_model(self):
         return TranscriptionMetadataInput(

rapidata/rapidata_client/dataset/__init__.py

@@ -0,0 +1 @@
+from ._rapidata_dataset import RapidataDataset

rapidata/rapidata_client/dataset/_rapidata_dataset.py

@@ -0,0 +1,55 @@
+from rapidata.rapidata_client.datapoints._datapoint import Datapoint
+from rapidata.service.openapi_service import OpenAPIService
+from rapidata.rapidata_client.config import logger
+from rapidata.rapidata_client.datapoints._datapoint_uploader import DatapointUploader
+from rapidata.rapidata_client.utils.threaded_uploader import ThreadedUploader
+
+
+class RapidataDataset:
+    def __init__(self, dataset_id: str, openapi_service: OpenAPIService):
+        self.id = dataset_id
+        self.openapi_service = openapi_service
+        self.datapoint_uploader = DatapointUploader(openapi_service)
+
+    def add_datapoints(
+        self,
+        datapoints: list[Datapoint],
+    ) -> tuple[list[Datapoint], list[Datapoint]]:
+        """
+        Process uploads in chunks with a ThreadPoolExecutor.
+
+        Args:
+            datapoints: List of datapoints to upload
+
+        Returns:
+            tuple[list[Datapoint], list[Datapoint]]: Lists of successful and failed uploads
+        """
+
+        def upload_single_datapoint(datapoint: Datapoint, index: int) -> None:
+            self.datapoint_uploader.upload_datapoint(
+                dataset_id=self.id,
+                datapoint=datapoint,
+                index=index,
+            )
+
+        uploader: ThreadedUploader[Datapoint] = ThreadedUploader(
+            upload_fn=upload_single_datapoint,
+            description="Uploading datapoints",
+        )
+
+        successful_uploads, failed_uploads = uploader.upload(datapoints)
+
+        if failed_uploads:
+            logger.error(
+                "Upload failed for %s datapoints: %s",
+                len(failed_uploads),
+                failed_uploads,
+            )
+
+        return successful_uploads, failed_uploads
+
+    def __str__(self) -> str:
+        return f"RapidataDataset(id={self.id})"
+
+    def __repr__(self) -> str:
+        return self.__str__()

rapidata/rapidata_client/demographic/demographic_manager.py

@@ -1,29 +1,54 @@
 from rapidata.service.openapi_service import OpenAPIService
-from rapidata.rapidata_client.datapoints.assets import MediaAsset
-from rapidata.api_client.models.create_demographic_rapid_model import CreateDemographicRapidModel
-from rapidata.api_client.models.classify_payload import ClassifyPayload
-from rapidata.rapidata_client.logging import logger
+from rapidata.rapidata_client.config import logger
+from rapidata.rapidata_client.datapoints._asset_uploader import AssetUploader
+
 
 class DemographicManager:
     def __init__(self, openapi_service: OpenAPIService):
         self._openapi_service = openapi_service
+        self._asset_uploader = AssetUploader(openapi_service)
         logger.debug("DemographicManager initialized")
-    
-    def create_demographic_rapid(self, 
-                                 instruction: str,
-                                 answer_options: list[str],
-                                 datapoint: str,
-                                 key: str):
-        
-        media = MediaAsset(path=datapoint)
+
+    def create_demographic_rapid(
+        self, instruction: str, answer_options: list[str], datapoint: str, key: str
+    ):
+        from rapidata.api_client.models.classify_payload import ClassifyPayload
+        from rapidata.api_client.models.classify_payload_category import (
+            ClassifyPayloadCategory,
+        )
+        from rapidata.api_client.models.create_demographic_rapid_model import (
+            CreateDemographicRapidModel,
+        )
+        from rapidata.api_client.models.i_asset_input import IAssetInput
+        from rapidata.api_client.models.i_asset_input_existing_asset_input import (
+            IAssetInputExistingAssetInput,
+        )
+
         model = CreateDemographicRapidModel(
             key=key,
             payload=ClassifyPayload(
-                _t="ClassifyPayload",
-                possibleCategories=answer_options,
-                title=instruction
-            )
+                categories=[
+                    ClassifyPayloadCategory(label=option, value=option)
+                    for option in answer_options
+                ],
+                title=instruction,
+            ),
+            asset=IAssetInput(
+                actual_instance=IAssetInputExistingAssetInput(
+                    _t="ExistingAssetInput",
+                    name=self._asset_uploader.upload_asset(datapoint),
+                ),
+            ),
+        )
+
+        result = self._openapi_service.rapid_api.rapid_demographic_post(
+            create_demographic_rapid_model=model
         )
-        
-        self._openapi_service.rapid_api.rapid_demographic_post(model=model, file=[media.to_file()])
-        
+        logger.info(f"Demographic Rapid created: {result.rapid_id}")
+        return result.rapid_id
+
+    def __str__(self) -> str:
+        return "DemographicManager"
+
+    def __repr__(self) -> str:
+        return self.__str__()