rapidata 2.4.0__py3-none-any.whl → 2.5.0__py3-none-any.whl

rapidata/rapidata_client/assets/_media_asset.py

@@ -12,6 +12,8 @@ from PIL import Image
 from tinytag import TinyTag
 import tempfile
 from pydantic import StrictStr, StrictBytes
+from typing import Optional
+import logging
 
 
 class MediaAsset(BaseAsset):
@@ -26,6 +28,7 @@ class MediaAsset(BaseAsset):
     Raises:
         FileNotFoundError: If the provided file path does not exist.
     """
+    _logger = logging.getLogger(__name__ + '.MediaAsset')
 
     ALLOWED_TYPES = [
         'image/', 
@@ -33,6 +36,28 @@ class MediaAsset(BaseAsset):
         'video/mp4',       # MP4
     ]
 
+    MIME_TYPES = {
+        'jpg': 'image/jpeg',
+        'jpeg': 'image/jpeg',
+        'png': 'image/png',
+        'gif': 'image/gif',
+        'webp': 'image/webp',
+        'mp3': 'audio/mp3',
+        'mp4': 'video/mp4'
+    }
+
+    FILE_SIGNATURES = {
+        b'\xFF\xD8\xFF': 'image/jpeg',
+        b'\x89PNG\r\n\x1a\n': 'image/png',
+        b'GIF87a': 'image/gif',
+        b'GIF89a': 'image/gif',
+        b'RIFF': 'image/webp',
+        b'ID3': 'audio/mp3',
+        b'\xFF\xFB': 'audio/mp3',
+        b'\xFF\xF3': 'audio/mp3',
+        b'ftyp': 'video/mp4',
+    }
+
     def __init__(self, path: str):
         """
         Initialize a MediaAsset instance.
@@ -134,35 +159,111 @@ class MediaAsset(BaseAsset):
             name = name + '.jpg'
         return name
 
+    def __get_media_type_from_extension(self, url: str) -> Optional[str]:
+        """
+        Determine media type from URL file extension.
+        
+        Args:
+            url: The URL to check
+            
+        Returns:
+            Optional[str]: MIME type if valid extension found, None otherwise
+        """
+        try:
+            ext = url.lower().split('?')[0].split('.')[-1]
+            return self.MIME_TYPES.get(ext)
+        except IndexError:
+            return None
+        
+    def __validate_image_content(self, content: bytes) -> bool:
+        """
+        Validate image content using PIL.
+        
+        Args:
+            content: Image bytes to validate
+            
+        Returns:
+            bool: True if valid image, False otherwise
+        """
+        try:
+            img = Image.open(BytesIO(content))
+            img.verify()
+            return True
+        except Exception as e:
+            self._logger.debug(f"Image validation failed: {str(e)}")
+            return False
+        
+    def __get_media_type_from_signature(self, content: bytes) -> Optional[str]:
+        """
+        Determine media type from file signature.
+        
+        Args:
+            content: File content bytes
+            
+        Returns:
+            Optional[str]: MIME type if valid signature found, None otherwise
+        """
+        file_start = content[:32]
+        for signature, mime_type in self.FILE_SIGNATURES.items():
+            if file_start.startswith(signature) or (signature in file_start[:10]):
+                return mime_type
+        return None
+
     def __get_media_bytes(self, url: str) -> bytes:
         """
-        Downloads media files from URL and validates type and duration.
+        Downloads and validates media files from URL.
         
         Args:
             url: URL of the media file
-                
+            
         Returns:
-            bytes: Media data
+            bytes: Validated media content
             
         Raises:
-            ValueError: If media type is unsupported or duration exceeds limit
+            ValueError: If media type is unsupported or content validation fails
             requests.exceptions.RequestException: If download fails
         """
-        response = requests.get(url, stream=False)  # Don't stream, we need full file
-        response.raise_for_status()
+        try:
+            response = requests.get(url, stream=False)
+            response.raise_for_status()
+        except requests.exceptions.RequestException as e:
+            self._logger.error(f"Failed to download media from {url}: {str(e)}")
+            raise
 
+        content = response.content
         content_type = response.headers.get('content-type', '').lower()
-        
-        # Validate content type
-        if not any(content_type.startswith(t) for t in self.ALLOWED_TYPES):
-            raise ValueError(
-                f'URL does not point to an allowed media type.\n'
-                f'Content-Type: {content_type}\n'
-                f'Allowed types: {self.ALLOWED_TYPES}'
-            )
-
-        content = BytesIO(response.content)
-        return content.getvalue()
+
+        # Case 1: Content-type is already allowed
+        if any(content_type.startswith(t) for t in self.ALLOWED_TYPES):
+            self._logger.debug(f"Content-type {content_type} is allowed")
+            return content
+
+        # Case 2: Try to validate based on extension
+        mime_type = self.__get_media_type_from_extension(url)
+        if mime_type and mime_type.startswith(tuple(self.ALLOWED_TYPES)):
+            self._logger.debug(f"Found valid mime type from extension: {mime_type}")
+            return content
+
+        # Case 3: Try to validate based on file signature
+        mime_type = self.__get_media_type_from_signature(content)
+        if mime_type and mime_type.startswith(tuple(self.ALLOWED_TYPES)):
+            self._logger.debug(f"Found valid mime type from signature: {mime_type}")
+            return content
+
+        # Case 4: Last resort - try direct image validation
+        if self.__validate_image_content(content):
+            self._logger.debug("Content validated as image through direct validation")
+            return content
+
+        # If we get here, validation failed
+        error_msg = (
+            f'Could not validate media type from content.\n'
+            f'Content-Type: {content_type}\n'
+            f'URL extension: {url.split("?")[0].split(".")[-1]}\n'
+            f'Allowed types: {self.ALLOWED_TYPES}'
+        )
+        self._logger.error(error_msg)
+        raise ValueError(error_msg)
     
     def to_file(self) -> StrictStr | tuple[StrictStr, StrictBytes] | StrictBytes: # types for autogenerated models
         if isinstance(self.path, str):

rapidata/rapidata_client/validation/rapidata_validation_set.py

@@ -1,3 +1,6 @@
+from rapidata.rapidata_client.validation.rapids.rapids import Rapid
+from rapidata.service.openapi_service import OpenAPIService
+
 class RapidataValidationSet:
     """A class for interacting with a Rapidata validation set.
 
@@ -10,9 +13,18 @@ class RapidataValidationSet:
         name (str): The name of the validation set.
     """
 
-    def __init__(self, validation_set_id, name: str):
+    def __init__(self, validation_set_id, name: str, openapi_service: OpenAPIService):
         self.id = validation_set_id
         self.name = name
+        self.__openapi_service = openapi_service
+    
+    def add_rapid(self, rapid: Rapid):
+        """Add a Rapid to the validation set.
+
+        Args:
+            rapid (Rapid): The Rapid to add to the validation set.
+        """
+        rapid._add_to_validation_set(self.id, self.__openapi_service)
 
     def __str__(self):
         return f"name: '{self.name}' id: {self.id}"

rapidata/rapidata_client/validation/rapids/rapids.py

@@ -20,6 +20,7 @@ from rapidata.api_client.models.datapoint_metadata_model_metadata_inner import (
     DatapointMetadataModelMetadataInner,
 )
 
+from rapidata.service.openapi_service import OpenAPIService
 
 class Rapid():
     def __init__(self, asset: MediaAsset | TextAsset | MultiAsset, metadata: Sequence[Metadata], payload: Any, truth: Any, randomCorrectProbability: float, explanation: str | None):
@@ -30,7 +31,22 @@ class Rapid():
         self.randomCorrectProbability = randomCorrectProbability
         self.explanation = explanation 
 
-    def to_media_model(self, validationSetId: str) -> tuple[AddValidationRapidModel, list[StrictStr | tuple[StrictStr, StrictBytes] | StrictBytes]]:
+    def _add_to_validation_set(self, validationSetId: str, openapi_service: OpenAPIService):
+        if isinstance(self.asset, TextAsset) or (isinstance(self.asset, MultiAsset) and isinstance(self.asset.assets[0], TextAsset)):
+            openapi_service.validation_api.validation_add_validation_text_rapid_post(
+                add_validation_text_rapid_model=self.__to_text_model(validationSetId)
+            )
+
+        elif isinstance(self.asset, MediaAsset) or (isinstance(self.asset, MultiAsset) and isinstance(self.asset.assets[0], MediaAsset)):
+            model = self.__to_media_model(validationSetId)
+            openapi_service.validation_api.validation_add_validation_rapid_post(
+                model=model[0], files=model[1]
+            )
+            
+        else:
+            raise TypeError("The asset must be a MediaAsset, TextAsset, or MultiAsset")
+
+    def __to_media_model(self, validationSetId: str) -> tuple[AddValidationRapidModel, list[StrictStr | tuple[StrictStr, StrictBytes] | StrictBytes]]:
         assets: list[MediaAsset] = [] 
         if isinstance(self.asset, MultiAsset):
             for asset in self.asset.assets:
@@ -57,7 +73,7 @@ class Rapid():
             explanation=self.explanation
         ), [asset.to_file() for asset in assets])
 
-    def to_text_model(self, validationSetId: str) -> AddValidationTextRapidModel:
+    def __to_text_model(self, validationSetId: str) -> AddValidationTextRapidModel:
         texts: list[str] = []
         if isinstance(self.asset, MultiAsset):
             for asset in self.asset.assets:

rapidata/rapidata_client/validation/rapids/rapids_manager.py

@@ -132,7 +132,7 @@ class RapidsManager:
         asset = MediaAsset(datapoint)
         transcription_words = [
             TranscriptionWord(word=word, wordIndex=i)
-            for i, word in enumerate(sentence)
+            for i, word in enumerate(sentence.split(" "))
         ]
 
         correct_transcription_words: list[TranscriptionWord] = []

rapidata/rapidata_client/validation/validation_set_manager.py

@@ -85,7 +85,7 @@ class ValidationSetManager:
                 )
             )
 
-        return self._submit(name=name, data_type=data_type, rapids=rapids, print_confirmation=print_confirmation)
+        return self._submit(name=name, rapids=rapids, print_confirmation=print_confirmation)
 
     def create_compare_set(self,
         name: str,
@@ -142,7 +142,7 @@ class ValidationSetManager:
                 )
             )
  
-        return self._submit(name=name, data_type=data_type, rapids=rapids, print_confirmation=print_confirmation)
+        return self._submit(name=name, rapids=rapids, print_confirmation=print_confirmation)
   
     def create_select_words_set(self,
         name: str,
@@ -197,7 +197,7 @@ class ValidationSetManager:
                 )
             )
 
-        return self._submit(name=name, data_type=RapidataDataTypes.MEDIA, rapids=rapids, print_confirmation=print_confirmation)
+        return self._submit(name=name, rapids=rapids, print_confirmation=print_confirmation)
 
     def create_locate_set(self,
         name: str,
@@ -249,7 +249,7 @@ class ValidationSetManager:
                 )
             )
         
-        return self._submit(name=name, data_type=RapidataDataTypes.MEDIA, rapids=rapids, print_confirmation=print_confirmation)
+        return self._submit(name=name, rapids=rapids, print_confirmation=print_confirmation)
     
     def create_draw_set(self,
         name: str,
@@ -300,7 +300,7 @@ class ValidationSetManager:
                 )
             )
 
-        return self._submit(name=name, data_type=RapidataDataTypes.MEDIA, rapids=rapids, print_confirmation=print_confirmation)
+        return self._submit(name=name, rapids=rapids, print_confirmation=print_confirmation)
 
     def create_timestamp_set(self,
         name: str,
@@ -352,11 +352,10 @@ class ValidationSetManager:
                 )
             )
 
-        return self._submit(name=name, data_type=RapidataDataTypes.MEDIA, rapids=rapids, print_confirmation=print_confirmation)
+        return self._submit(name=name, rapids=rapids, print_confirmation=print_confirmation)
     
     def create_mixed_set(self,
         name: str,
-        data_type: str,
         rapids: list[Rapid],
         print_confirmation: bool = True
     ) -> RapidataValidationSet:
@@ -368,7 +367,7 @@ class ValidationSetManager:
             print_confirmation (bool, optional): Whether to print a confirmation message that validation set has been created. Defaults to True.
         """
 
-        return self._submit(name, data_type, rapids, print_confirmation)
+        return self._submit(name, rapids, print_confirmation)
     
     def get_validation_set_by_id(self, validation_set_id: str) -> RapidataValidationSet:
         """Get a validation set by ID.
@@ -384,9 +383,9 @@ class ValidationSetManager:
         except Exception:
             raise ValueError(f"ValidationSet with ID {validation_set_id} not found.")
         
-        return RapidataValidationSet(validation_set_id, validation_set.name)
+        return RapidataValidationSet(validation_set_id, validation_set.name, self.__openapi_service)
 
-    def _submit(self, name: str, data_type: str, rapids: list[Rapid], print_confirmation: bool) -> RapidataValidationSet:
+    def _submit(self, name: str, rapids: list[Rapid], print_confirmation: bool) -> RapidataValidationSet:
         validation_set_id = (
             self.__openapi_service.validation_api.validation_create_validation_set_post(
                 name=name
@@ -399,22 +398,17 @@ class ValidationSetManager:
         if print_confirmation:
             print(f"Validation set '{name}' created with ID {validation_set_id}")
 
-        for rapid in rapids:
-            if data_type == RapidataDataTypes.TEXT:
-                self.__openapi_service.validation_api.validation_add_validation_text_rapid_post(
-                    add_validation_text_rapid_model=rapid.to_text_model(validation_set_id)
-                )
-            else:
-                model = rapid.to_media_model(validation_set_id)
-                self.__openapi_service.validation_api.validation_add_validation_rapid_post(
-                   model=model[0], files=model[1]
-                )
-
-        return RapidataValidationSet(
+        validation_set = RapidataValidationSet(
             name=name,
             validation_set_id=validation_set_id,
+            openapi_service=self.__openapi_service
         )
 
+        for rapid in rapids:
+            validation_set.add_rapid(rapid)
+        
+        return validation_set
+
 
     def find_validation_sets(self, name: str = "", amount: int = 1) -> list[RapidataValidationSet]:
         """Find validation sets by name.

{rapidata-2.4.0.dist-info → rapidata-2.5.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: rapidata
-Version: 2.4.0
+Version: 2.5.0
 Summary: Rapidata package containing the Rapidata Python Client to interact with the Rapidata Web API in an easy way.
 License: Apache-2.0
 Author: Rapidata AG

{rapidata-2.4.0.dist-info → rapidata-2.5.0.dist-info}/RECORD

@@ -349,7 +349,7 @@ rapidata/api_client_README.md,sha256=TwdMWXKwwnzD5CyLjXOqAwKVyMxTJcKewudONv31p40
 rapidata/rapidata_client/__init__.py,sha256=kkT6FMU4P8rTdYKTJgZNcyptr-Rq1iZmsyLIyRCwpYA,896
 rapidata/rapidata_client/assets/__init__.py,sha256=hKgrOSn8gJcBSULaf4auYhH1S1N5AfcwIhBSq1BOKwQ,323
 rapidata/rapidata_client/assets/_base_asset.py,sha256=B2YWH1NgaeYUYHDW3OPpHM_bqawHbH4EjnRCE2BYwiM,298
-rapidata/rapidata_client/assets/_media_asset.py,sha256=dji8rM2W6TtzIPDiVVyaHfEDNTmu8uo3rsgrPUcbqDE,5949
+rapidata/rapidata_client/assets/_media_asset.py,sha256=s7_bfhvYEFLIvpZvECsjtVSLnvZmi19xLJoOsDShzgk,9391
 rapidata/rapidata_client/assets/_multi_asset.py,sha256=o4_-OvovADSVpl0tr6fPzRR_gHCcIQKfeZKcoFDFBLo,1667
 rapidata/rapidata_client/assets/_text_asset.py,sha256=itoe3vicn41LbdJ7UeydImORUo9iDL0SZu-ptOlbMRM,618
 rapidata/rapidata_client/assets/data_type_enum.py,sha256=ELC-ymeKnQlfNAzfqsI7MmUuRiGYamCHVcTc0qR6Fm4,185
@@ -406,12 +406,12 @@ rapidata/rapidata_client/settings/play_video_until_the_end.py,sha256=LLHx2_72k5Z
 rapidata/rapidata_client/settings/rapidata_settings.py,sha256=Kjxm4GStrpfLKylx84BiKdEQOjvNfO74DiSChmH63fg,1165
 rapidata/rapidata_client/settings/translation_behaviour.py,sha256=i9n_H0eKJyKW6m3MKH_Cm1XEKWVEWsAV_79xGmGIC-4,742
 rapidata/rapidata_client/validation/__init__.py,sha256=s5wHVtcJkncXSFuL9I0zNwccNOKpWAqxqUjkeohzi2E,24
-rapidata/rapidata_client/validation/rapidata_validation_set.py,sha256=c4ywPef1yGwx0D9wTzj7465p-VQa1V6WnHJUU5ReK7Q,646
+rapidata/rapidata_client/validation/rapidata_validation_set.py,sha256=IkHOdQX3ihBBHDvG2EE6wu1ZSG7evOWMYyOx9Lva0tw,1111
 rapidata/rapidata_client/validation/rapids/__init__.py,sha256=WU5PPwtTJlte6U90MDakzx4I8Y0laj7siw9teeXj5R0,21
 rapidata/rapidata_client/validation/rapids/box.py,sha256=t3_Kn6doKXdnJdtbwefXnYKPiTKHneJl9E2inkDSqL8,589
-rapidata/rapidata_client/validation/rapids/rapids.py,sha256=sZEzKj4FdVChm_rUwcmG_v7X-yO5Eh4q1g54T4c4puc,3507
-rapidata/rapidata_client/validation/rapids/rapids_manager.py,sha256=rDmxYkv067O-S95TNc789FktfCPBCudyzBRLSArmfus,14236
-rapidata/rapidata_client/validation/validation_set_manager.py,sha256=I2ICynEOjgmp5KDUceXCKGyKGSt21N_GTn1Mi4-6DTc,23295
+rapidata/rapidata_client/validation/rapids/rapids.py,sha256=aWkcjWR6Pr2BFwz8a0MfqEkXnEJPkAbKMeQtGXnsags,4440
+rapidata/rapidata_client/validation/rapids/rapids_manager.py,sha256=4HOX6c42sCOfpGR2aDoiMgrfjontM8z2KEJwQp4ir6A,14247
+rapidata/rapidata_client/validation/validation_set_manager.py,sha256=RR3lP3VtOVCtmbKjEAFYVbQxZyUCXvOuWMqw8dGz-WA,22730
 rapidata/rapidata_client/workflow/__init__.py,sha256=eFRx0fm280alXpds6hYcnxN_yERlabF9B5sTdPFsL1g,430
 rapidata/rapidata_client/workflow/_base_workflow.py,sha256=XyIZFKS_RxAuwIHS848S3AyLEHqd07oTD_5jm2oUbsw,762
 rapidata/rapidata_client/workflow/_classify_workflow.py,sha256=9bT54wxVJgxC-zLk6MVNbseFpzYrvFPjt7DHvxqYfnk,1736
@@ -427,7 +427,7 @@ rapidata/service/credential_manager.py,sha256=Of0BQs_V1T7rkrWX9groLX790nOknaARwn
 rapidata/service/local_file_service.py,sha256=pgorvlWcx52Uh3cEG6VrdMK_t__7dacQ_5AnfY14BW8,877
 rapidata/service/openapi_service.py,sha256=Z4NrAuilLlIWBdGOv6otz36tHS_vvU36w5jmvOUTmqo,3198
 rapidata/service/token_manager.py,sha256=JZ5YbR5Di8dO3H4kK11d0kzWlrXxjgCmeNkHA4AapCM,6425
-rapidata-2.4.0.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-rapidata-2.4.0.dist-info/METADATA,sha256=ZbXMkT2eRglrqdHTuUgOJlRhKOg8BneVJ2rPRs2-U6k,1107
-rapidata-2.4.0.dist-info/WHEEL,sha256=IYZQI976HJqqOpQU6PHkJ8fb3tMNBFjg-Cn-pwAbaFM,88
-rapidata-2.4.0.dist-info/RECORD,,
+rapidata-2.5.0.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+rapidata-2.5.0.dist-info/METADATA,sha256=Bi3-kOaNpfnjqHS9QwC0ua5dnnlhluiMZdopp0fx08U,1107
+rapidata-2.5.0.dist-info/WHEEL,sha256=IYZQI976HJqqOpQU6PHkJ8fb3tMNBFjg-Cn-pwAbaFM,88
+rapidata-2.5.0.dist-info/RECORD,,