rapidata 2.41.3__py3-none-any.whl → 2.42.0__py3-none-any.whl

{rapidata-2.41.3.dist-info → rapidata-2.42.0.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 2.2.0
+Generator: poetry-core 2.2.1
 Root-Is-Purelib: true
 Tag: py3-none-any

rapidata/rapidata_client/datapoints/assets/_base_asset.py

@@ -1,13 +0,0 @@
-"""Base Asset Module
-
-Defines the BaseAsset class, which serves as the abstract base class for all asset types.
-"""
-
-
-class BaseAsset:
-    """BaseAsset Class
-
-    An abstract base class for different types of assets. This class is intended to be subclassed by specific asset types.
-    """
-
-    pass

rapidata/rapidata_client/datapoints/assets/_media_asset.py

@@ -1,318 +0,0 @@
-"""Media Asset Module with Lazy Loading
-
-Defines the MediaAsset class for handling media file paths within assets.
-Implements lazy loading for URL-based media to prevent unnecessary downloads.
-"""
-
-from typing import Optional, cast
-import os
-from io import BytesIO
-from rapidata.rapidata_client.datapoints.assets._base_asset import BaseAsset
-import requests
-import re
-from PIL import Image
-from tinytag import TinyTag
-import tempfile
-from pydantic import StrictStr, StrictBytes
-import logging
-from functools import cached_property
-from rapidata.rapidata_client.datapoints.assets._sessions import SessionManager
-from rapidata.rapidata_client.config import logger
-from rapidata.rapidata_client.datapoints.assets.constants import (
-    ALLOWED_IMAGE_EXTENSIONS,
-    ALLOWED_MEDIA_EXTENSIONS,
-)
-
-
-class MediaAsset(BaseAsset):
-    """MediaAsset Class with Lazy Loading
-
-    Represents a media asset by storing the file path or URL.
-    Only downloads URL content when needed.
-    Supports local files and URLs for images, MP3, and MP4.
-
-    Args:
-        path (str): The file system path to the media asset or URL.
-
-    Raises:
-        FileNotFoundError: If the provided file path does not exist.
-    """
-
-    _logger = logging.getLogger(__name__ + ".MediaAsset")
-
-    ALLOWED_TYPES = [
-        "image/",
-        "audio/mp3",  # MP3
-        "video/mp4",  # MP4
-    ]
-
-    MIME_TYPES = {
-        "jpg": "image/jpeg",
-        "jpeg": "image/jpeg",
-        "png": "image/png",
-        "gif": "image/gif",
-        "webp": "image/webp",
-        "mp3": "audio/mp3",
-        "mp4": "video/mp4",
-    }
-
-    FILE_SIGNATURES = {
-        b"\xFF\xD8\xFF": "image/jpeg",
-        b"\x89PNG\r\n\x1a\n": "image/png",
-        b"GIF87a": "image/gif",
-        b"GIF89a": "image/gif",
-        b"RIFF": "image/webp",
-        b"ID3": "audio/mp3",
-        b"\xFF\xFB": "audio/mp3",
-        b"\xFF\xF3": "audio/mp3",
-        b"ftyp": "video/mp4",
-    }
-
-    def __init__(self, path: str):
-        """
-        Initialize a MediaAsset instance.
-
-        Args:
-            path (str): The file system path to the media asset or a URL.
-
-        Raises:
-            FileNotFoundError: If the provided file path does not exist.
-            ValueError: If path is not a string.
-        """
-        if not isinstance(path, str):
-            raise ValueError(
-                f"Media must be a string, either a local file path or a URL, got {type(path)}"
-            )
-
-        self._url = None
-        self._content = None
-        self.session: requests.Session = SessionManager.get_session()
-
-        if re.match(r"^https?://", path):
-            self._url = path
-            self.name = path.split("/")[-1]
-            self.name = self.__check_name_ending(self.name)
-            self.path = path
-            return
-
-        if not os.path.exists(path):
-            raise FileNotFoundError(f"File not found: {path}")
-
-        self.path = path
-        self.name = path
-
-    @cached_property
-    def content(self) -> bytes:
-        """
-        Lazy loader for URL content. Only downloads when first accessed.
-        Uses cached_property to store the result after first download.
-        """
-        if self._url is None:
-            self.path = cast(str, self.path)
-            with open(self.path, "rb") as f:
-                return f.read()
-
-        return self.__get_media_bytes(self._url)
-
-    def get_duration(self) -> int:
-        """
-        Get the duration of audio/video files in milliseconds.
-        Returns 0 for static images.
-
-        Returns:
-            int: Duration in milliseconds for audio/video, 0 for static images
-
-        Raises:
-            ValueError: If the duration cannot be determined
-        """
-        path_to_check = self.name.lower()
-
-        # Return 0 for static images
-        if any(path_to_check.endswith(ext) for ext in ALLOWED_IMAGE_EXTENSIONS):
-            return 0
-
-        try:
-            # Create temporary file from content
-            with tempfile.NamedTemporaryFile(
-                suffix=os.path.splitext(self.name)[1], delete=False
-            ) as tmp:
-                tmp.write(self.content)
-                tmp.flush()
-                tmp_path = tmp.name
-
-            try:
-                tag = TinyTag.get(tmp_path)
-            finally:
-                # Clean up the temporary file
-                os.unlink(tmp_path)
-
-            if tag.duration is None:
-                raise ValueError("Could not read duration from file")
-
-            return int(tag.duration * 1000)  # Convert to milliseconds
-
-        except Exception as e:
-            raise ValueError(f"Could not determine media duration: {str(e)}")
-
-    def get_image_dimension(self) -> tuple[int, int] | None:
-        """
-        Get the dimensions (width, height) of an image file.
-        Returns None for non-image files or if dimensions can't be determined.
-        """
-        if not any(self.name.lower().endswith(ext) for ext in ALLOWED_IMAGE_EXTENSIONS):
-            return None
-
-        try:
-            img = Image.open(BytesIO(self.content))
-            return img.size
-        except Exception:
-            return None
-
-    def set_custom_name(self, name: str) -> "MediaAsset":
-        """Set a custom name for the media asset (only works with URLs)."""
-        if self._url is not None:
-            self.name = self.__check_name_ending(name)
-        else:
-            raise ValueError("Custom name can only be set for URLs.")
-        return self
-
-    def __check_name_ending(self, name: str) -> str:
-        """Check if the media path is valid."""
-        if not any(name.endswith(ext) for ext in ALLOWED_MEDIA_EXTENSIONS):
-            logger.warning(
-                f"Warning: Supported file types: {ALLOWED_MEDIA_EXTENSIONS}. Image might not be displayed correctly."
-            )
-            name = name + ".jpg"
-        return name
-
-    def __get_media_type_from_extension(self, url: str) -> Optional[str]:
-        """
-        Determine media type from URL file extension.
-
-        Args:
-            url: The URL to check
-
-        Returns:
-            Optional[str]: MIME type if valid extension found, None otherwise
-        """
-        try:
-            ext = url.lower().split("?")[0].split(".")[-1]
-            return self.MIME_TYPES.get(ext)
-        except IndexError:
-            return None
-
-    def __validate_image_content(self, content: bytes) -> bool:
-        """
-        Validate image content using PIL.
-
-        Args:
-            content: Image bytes to validate
-
-        Returns:
-            bool: True if valid image, False otherwise
-        """
-        try:
-            img = Image.open(BytesIO(content))
-            img.verify()
-            return True
-        except Exception as e:
-            self._logger.debug(f"Image validation failed: {str(e)}")
-            return False
-
-    def __get_media_type_from_signature(self, content: bytes) -> Optional[str]:
-        """
-        Determine media type from file signature.
-
-        Args:
-            content: File content bytes
-
-        Returns:
-            Optional[str]: MIME type if valid signature found, None otherwise
-        """
-        file_start = content[:32]
-        for signature, mime_type in self.FILE_SIGNATURES.items():
-            if file_start.startswith(signature) or (signature in file_start[:10]):
-                return mime_type
-        return None
-
-    def __get_media_bytes(self, url: str) -> bytes:
-        """
-        Downloads and validates media files from URL with retry logic and session reuse.
-
-        Args:
-            url: URL of the media file
-
-        Returns:
-            bytes: Validated media content
-
-        Raises:
-            ValueError: If media type is unsupported or content validation fails
-            requests.exceptions.RequestException: If download fails after all retries
-        """
-        # Use existing session or throw error if not set
-        if self.session is None:
-            raise RuntimeError("HTTP session not configured")
-
-        try:
-            response = self.session.get(
-                url, stream=False, timeout=(5, 30)  # (connect timeout, read timeout)
-            )
-            response.raise_for_status()
-        except requests.exceptions.RequestException as e:
-            self._logger.error(
-                f"Failed to download media from {url} after retries: {str(e)}"
-            )
-            raise
-
-        content = response.content
-        content_type = response.headers.get("content-type", "").lower()
-
-        # Case 1: Content-type is already allowed
-        if any(content_type.startswith(t) for t in self.ALLOWED_TYPES):
-            self._logger.debug(f"Content-type {content_type} is allowed")
-            return content
-
-        # Case 2: Try to validate based on extension
-        mime_type = self.__get_media_type_from_extension(url)
-        if mime_type and mime_type.startswith(tuple(self.ALLOWED_TYPES)):
-            self._logger.debug(f"Found valid mime type from extension: {mime_type}")
-            return content
-
-        # Case 3: Try to validate based on file signature
-        mime_type = self.__get_media_type_from_signature(content)
-        if mime_type and mime_type.startswith(tuple(self.ALLOWED_TYPES)):
-            self._logger.debug(f"Found valid mime type from signature: {mime_type}")
-            return content
-
-        # Case 4: Last resort - try direct image validation
-        if self.__validate_image_content(content):
-            self._logger.debug("Content validated as image through direct validation")
-            return content
-
-        # If we get here, validation failed
-        error_msg = (
-            f"Could not validate media type from content.\n"
-            f"Content-Type: {content_type}\n"
-            f'URL extension: {url.split("?")[0].split(".")[-1]}\n'
-            f"Allowed types: {self.ALLOWED_TYPES}"
-        )
-        self._logger.error(error_msg)
-        raise ValueError(error_msg)
-
-    def is_local(self) -> bool:
-        """Check if the media asset is a local file."""
-        return self._url is None
-
-    def to_file(self) -> StrictStr | tuple[StrictStr, StrictBytes] | StrictBytes:
-        """Convert the media asset to a file representation."""
-        if self._url is None:
-            self.path = cast(str, self.path)
-            return self.path
-        else:
-            return (self.name, self.content)
-
-    def __str__(self) -> str:
-        return f"MediaAsset(path={self.path})"
-
-    def __repr__(self) -> str:
-        return f"MediaAsset(path={self.path})"

rapidata/rapidata_client/datapoints/assets/_multi_asset.py

@@ -1,61 +0,0 @@
-"""Multi Asset Module
-
-Defines the MultiAsset class for handling multiple BaseAsset instances.
-"""
-
-from rapidata.rapidata_client.datapoints.assets._base_asset import BaseAsset
-from rapidata.rapidata_client.datapoints.assets import MediaAsset, TextAsset
-from typing import Iterator, Sequence
-
-
-class MultiAsset(BaseAsset):
-    """MultiAsset Class
-
-    Represents a collection of multiple BaseAsset instances.
-
-    Args:
-        assets (List[BaseAsset]): A list of BaseAsset instances to be managed together.
-    """
-
-    def __init__(self, assets: Sequence[BaseAsset]) -> None:
-        """
-        Initialize a MultiAsset instance.
-
-        Args:
-            assets (List[BaseAsset]): A list of BaseAsset instances to be managed together.
-        """
-        if len(assets) != 2:
-            raise ValueError("Assets must come in pairs for comparison tasks.")
-
-        for asset in assets:
-            if not isinstance(asset, (TextAsset, MediaAsset)):
-                raise TypeError("All assets must be a TextAsset or MediaAsset.")
-
-        if not all(isinstance(asset, type(assets[0])) for asset in assets):
-            raise ValueError("All assets must be of the same type.")
-
-        self.assets = assets
-
-    def __len__(self) -> int:
-        """
-        Get the number of assets in the MultiAsset.
-
-        Returns:
-            int: The number of assets.
-        """
-        return len(self.assets)
-
-    def __iter__(self) -> Iterator[BaseAsset]:
-        """
-        Return an iterator over the assets in the MultiAsset.
-
-        Returns:
-            Iterator[BaseAsset]: An iterator over the assets.
-        """
-        return iter(self.assets)
-
-    def __str__(self) -> str:
-        return f"MultiAsset(assets={self.assets})"
-
-    def __repr__(self) -> str:
-        return f"MultiAsset(assets={self.assets})"

rapidata/rapidata_client/datapoints/assets/_sessions.py

@@ -1,40 +0,0 @@
-import requests
-from requests.adapters import HTTPAdapter
-from urllib3.util.retry import Retry
-
-from rapidata.rapidata_client.config.rapidata_config import rapidata_config
-
-
-class SessionManager:
-    _session = None
-
-    @classmethod
-    def get_session(
-        cls,
-    ) -> requests.Session:
-        """Get a singleton requests session with retry logic.
-
-        Returns:
-            requests.Session: A singleton requests session with retry logic.
-        """
-        if cls._session is None:
-            max_retries: int = rapidata_config.upload.maxRetries
-            max_workers: int = rapidata_config.upload.maxWorkers
-            cls._session = requests.Session()
-            retries = Retry(
-                total=max_retries,
-                backoff_factor=1,
-                status_forcelist=[500, 502, 503, 504],
-                allowed_methods=["GET"],
-                respect_retry_after_header=True,
-            )
-
-            adapter = HTTPAdapter(
-                pool_connections=max_workers * 2,
-                pool_maxsize=max_workers * 4,
-                max_retries=retries,
-            )
-            cls._session.mount("http://", adapter)
-            cls._session.mount("https://", adapter)
-
-        return cls._session

rapidata/rapidata_client/datapoints/assets/_text_asset.py

@@ -1,34 +0,0 @@
-"""Text Asset Module
-
-Defines the TextAsset class for handling textual data within assets.
-"""
-
-from rapidata.rapidata_client.datapoints.assets._base_asset import BaseAsset
-
-
-class TextAsset(BaseAsset):
-    """TextAsset Class
-
-    Represents a textual asset.
-
-    Args:
-        text (str): The text content of the asset.
-    """
-
-    def __init__(self, text: str):
-        """
-        Initialize a TextAsset instance.
-
-        Args:
-            text (str): The textual content of the asset.
-        """
-        if not isinstance(text, str):
-            raise ValueError(f"Text must be a string, got {type(text)}")
-
-        self.text = text
-
-    def __str__(self) -> str:
-        return f"TextAsset(text={self.text})"
-
-    def __repr__(self) -> str:
-        return f"TextAsset(text={self.text})"

rapidata/rapidata_client/datapoints/assets/data_type_enum.py

@@ -1,8 +0,0 @@
-class RapidataDataTypes:
-    # deprecated use Literal["media", "text"] instead
-    MEDIA = "media"  # any form of image, video or audio
-    TEXT = "text"
-
-    @classmethod
-    def _possible_values(cls):
-        return [cls.MEDIA, cls.TEXT]

rapidata/rapidata_client/order/dataset/_progress_tracker.py

@@ -1,100 +0,0 @@
-import threading
-import time
-from tqdm import tqdm
-
-from rapidata.service.openapi_service import OpenAPIService
-from rapidata.rapidata_client.config import logger, rapidata_config
-
-
-class ProgressTracker:
-    """
-    Track dataset upload progress in a background thread with shallow indentation.
-
-    This class encapsulates the progress polling loop to keep methods in
-    `RapidataDataset` simpler and below the maximum indentation depth.
-    """
-
-    def __init__(
-        self,
-        dataset_id: str,
-        openapi_service: OpenAPIService,
-        total_uploads: int,
-        progress_poll_interval: float,
-    ) -> None:
-        self.dataset_id = dataset_id
-        self.openapi_service = openapi_service
-        self.total_uploads = total_uploads
-        self.progress_poll_interval = progress_poll_interval
-        self.upload_complete = False
-
-    def _get_progress_or_none(self):
-        try:
-            return self.openapi_service.dataset_api.dataset_dataset_id_progress_get(
-                self.dataset_id
-            )
-        except Exception:
-            return None
-
-    def complete(self) -> None:
-        logger.debug("Upload complete, setting upload_complete to True")
-        self.upload_complete = True
-
-    def run(self) -> None:
-        try:
-            with tqdm(
-                total=self.total_uploads,
-                desc="Uploading datapoints",
-                disable=rapidata_config.logging.silent_mode,
-            ) as pbar:
-                final_pass = False
-                while True:
-                    current_progress = self._get_progress_or_none()
-                    if current_progress is None:
-                        time.sleep(self.progress_poll_interval)
-                        logger.debug(
-                            "No progress yet, sleeping for %s seconds",
-                            self.progress_poll_interval,
-                        )
-                        continue
-
-                    total_completed = current_progress.ready + current_progress.failed
-
-                    pbar.n = total_completed
-                    pbar.refresh()
-
-                    time.sleep(self.progress_poll_interval)
-                    if total_completed >= self.total_uploads:
-                        break
-
-                    if self.upload_complete and current_progress.pending == 0:
-                        if not final_pass:
-                            logger.debug("Final pass")
-                            time.sleep(self.progress_poll_interval)
-                            final_pass = True
-                            continue
-                        logger.debug("Final pass done, breaking out of loop")
-                        break
-
-                pbar.close()
-
-                success_rate = (
-                    round((current_progress.ready / self.total_uploads * 100), 2)
-                    if self.total_uploads > 0
-                    else 0
-                )
-
-                logger.info(
-                    "Upload complete: %s ready, %s failed, %s pending (%s%% success rate)",
-                    current_progress.ready,
-                    current_progress.failed,
-                    current_progress.pending,
-                    success_rate,
-                )
-        except Exception as e:
-            logger.error("Progress tracking thread error: %s", str(e))
-            raise RuntimeError("Progress tracking failed, aborting uploads")
-
-    def create_thread(self) -> threading.Thread:
-        thread = threading.Thread(target=self.run)
-        thread.daemon = True
-        return thread