rapidata 2.37.0__py3-none-any.whl → 3.0.0a8__py3-none-any.whl

rapidata/rapidata_client/benchmark/rapidata_benchmark_manager.py

@@ -1,4 +1,4 @@
-from typing import Optional
+from typing import Optional, cast
 from rapidata.rapidata_client.benchmark.rapidata_benchmark import RapidataBenchmark
 from rapidata.api_client.models.create_benchmark_model import CreateBenchmarkModel
 from rapidata.service.openapi_service import OpenAPIService
@@ -9,6 +9,7 @@ from rapidata.api_client.models.filter import Filter
 from rapidata.api_client.models.sort_criterion import SortCriterion
 from rapidata.api_client.models.sort_direction import SortDirection
 from rapidata.api_client.models.filter_operator import FilterOperator
+from rapidata.rapidata_client.config import logger, tracer
 
 
 class RapidataBenchmarkManager:
@@ -18,9 +19,6 @@ class RapidataBenchmarkManager:
     Used to create and retrieve benchmarks.
 
     A benchmark is a collection of leaderboards.
-
-    Args:
-        openapi_service: The OpenAPIService instance for API interaction.
     """
 
     def __init__(self, openapi_service: OpenAPIService):
@@ -29,20 +27,20 @@ class RapidataBenchmarkManager:
     def create_new_benchmark(
         self,
         name: str,
-        identifiers: list[str],
-        prompts: Optional[list[str | None]] = None,
-        prompt_assets: Optional[list[str | None]] = None,
-        tags: Optional[list[list[str] | None]] = None,
+        identifiers: Optional[list[str]] = None,
+        prompts: Optional[list[str | None] | list[str]] = None,
+        prompt_assets: Optional[list[str | None] | list[str]] = None,
+        tags: Optional[list[list[str] | None] | list[list[str]]] = None,
     ) -> RapidataBenchmark:
         """
         Creates a new benchmark with the given name, identifiers, prompts, and media assets.
         Everything is matched up by the indexes of the lists.
 
-        prompts or prompt_assets must be provided.
+        prompts or identifiers must be provided, as well as prompts or prompt_assets.
 
         Args:
             name: The name of the benchmark.
-            identifiers: The identifiers of the prompts/assets/tags that will be used to match up the media
+            identifiers: The identifiers of the prompts/assets/tags that will be used to match up the media. If not provided, it will use the prompts as the identifiers.
             prompts: The prompts that will be registered for the benchmark.
             prompt_assets: The prompt assets that will be registered for the benchmark.
             tags: The tags that will be associated with the prompts to use for filtering the leaderboard results. They will NOT be shown to the users.
@@ -58,78 +56,133 @@ class RapidataBenchmarkManager:
             benchmark = create_new_benchmark(name=name, identifiers=identifiers, prompts=prompts, prompt_assets=prompt_assets, tags=tags)
             ```
         """
-        if not isinstance(name, str):
-            raise ValueError("Name must be a string.")
-
-        if prompts and (
-            not isinstance(prompts, list)
-            or not all(isinstance(prompt, str) or prompt is None for prompt in prompts)
+        with tracer.start_as_current_span(
+            "RapidataBenchmarkManager.create_new_benchmark"
         ):
-            raise ValueError("Prompts must be a list of strings or None.")
+            if not isinstance(name, str):
+                raise ValueError("Name must be a string.")
+
+            if prompts and (
+                not isinstance(prompts, list)
+                or not all(
+                    isinstance(prompt, str) or prompt is None for prompt in prompts
+                )
+            ):
+                raise ValueError("Prompts must be a list of strings or None.")
+
+            if prompt_assets and (
+                not isinstance(prompt_assets, list)
+                or not all(
+                    isinstance(asset, str) or asset is None for asset in prompt_assets
+                )
+            ):
+                raise ValueError("Media assets must be a list of strings or None.")
+
+            if identifiers and (
+                not isinstance(identifiers, list)
+                or not all(isinstance(identifier, str) for identifier in identifiers)
+            ):
+                raise ValueError("Identifiers must be a list of strings.")
+
+            if identifiers:
+                if not len(set(identifiers)) == len(identifiers):
+                    raise ValueError("Identifiers must be unique.")
+
+            if tags is not None:
+                if not isinstance(tags, list):
+                    raise ValueError("Tags must be a list of lists of strings or None.")
+
+                for tag in tags:
+                    if tag is not None and (
+                        not isinstance(tag, list)
+                        or not all(isinstance(item, str) for item in tag)
+                    ):
+                        raise ValueError(
+                            "Tags must be a list of lists of strings or None."
+                        )
 
-        if prompt_assets and (
-            not isinstance(prompt_assets, list)
-            or not all(
-                isinstance(asset, str) or asset is None for asset in prompt_assets
-            )
-        ):
-            raise ValueError("Media assets must be a list of strings or None.")
+            if not identifiers and not prompts:
+                raise ValueError(
+                    "At least one of identifiers or prompts must be provided."
+                )
+
+            if not prompts and not prompt_assets:
+                raise ValueError(
+                    "At least one of prompts or media assets must be provided."
+                )
+
+            if not identifiers:
+                assert prompts is not None
+                if not len(set(prompts)) == len(prompts):
+                    raise ValueError(
+                        "Prompts must be unique. Otherwise use identifiers."
+                    )
+                if any(prompt is None for prompt in prompts):
+                    raise ValueError(
+                        "Prompts must not be None. Otherwise use identifiers."
+                    )
 
-        if not isinstance(identifiers, list) or not all(
-            isinstance(identifier, str) for identifier in identifiers
-        ):
-            raise ValueError("Identifiers must be a list of strings.")
+                identifiers = cast(list[str], prompts)
 
-        if prompts and len(identifiers) != len(prompts):
-            raise ValueError("Identifiers and prompts must have the same length.")
+            assert identifiers is not None
 
-        if prompt_assets and len(identifiers) != len(prompt_assets):
-            raise ValueError("Identifiers and media assets must have the same length.")
+            expected_length = len(identifiers)
 
-        if not prompts and not prompt_assets:
-            raise ValueError(
-                "At least one of prompts or media assets must be provided."
-            )
+            if not prompts:
+                prompts = cast(list[str | None], [None] * expected_length)
 
-        if len(set(identifiers)) != len(identifiers):
-            raise ValueError("Identifiers must be unique.")
+            if not prompt_assets:
+                prompt_assets = cast(list[str | None], [None] * expected_length)
 
-        if tags and len(identifiers) != len(tags):
-            raise ValueError("Identifiers and tags must have the same length.")
+            if not tags:
+                tags = cast(list[list[str] | None], [None] * expected_length)
 
-        benchmark_result = self.__openapi_service.benchmark_api.benchmark_post(
-            create_benchmark_model=CreateBenchmarkModel(
-                name=name,
+            # At this point, all variables are guaranteed to be lists, not None
+            assert prompts is not None
+            assert prompt_assets is not None
+            assert tags is not None
+
+            if not (expected_length == len(prompts) == len(prompt_assets) == len(tags)):
+                raise ValueError(
+                    "Identifiers, prompts, media assets, and tags must have the same length or set to None."
+                )
+
+            logger.info("Creating new benchmark %s", name)
+
+            benchmark_result = self.__openapi_service.benchmark_api.benchmark_post(
+                create_benchmark_model=CreateBenchmarkModel(
+                    name=name,
+                )
             )
-        )
 
-        benchmark = RapidataBenchmark(name, benchmark_result.id, self.__openapi_service)
+            logger.info("Benchmark created with id %s", benchmark_result.id)
 
-        prompts_list = prompts if prompts is not None else [None] * len(identifiers)
-        media_assets_list = (
-            prompt_assets if prompt_assets is not None else [None] * len(identifiers)
-        )
-        tags_list = tags if tags is not None else [None] * len(identifiers)
+            benchmark = RapidataBenchmark(
+                name, benchmark_result.id, self.__openapi_service
+            )
 
-        for identifier, prompt, asset, tag in zip(
-            identifiers, prompts_list, media_assets_list, tags_list
-        ):
-            benchmark.add_prompt(identifier, prompt, asset, tag)
+            for identifier, prompt, asset, tag in zip(
+                identifiers, prompts, prompt_assets, tags
+            ):
+                benchmark.add_prompt(identifier, prompt, asset, tag)
 
-        return benchmark
+            return benchmark
 
     def get_benchmark_by_id(self, id: str) -> RapidataBenchmark:
         """
         Returns a benchmark by its ID.
         """
-        benchmark_result = (
-            self.__openapi_service.benchmark_api.benchmark_benchmark_id_get(
-                benchmark_id=id
+        with tracer.start_as_current_span(
+            "RapidataBenchmarkManager.get_benchmark_by_id"
+        ):
+            benchmark_result = (
+                self.__openapi_service.benchmark_api.benchmark_benchmark_id_get(
+                    benchmark_id=id
+                )
+            )
+            return RapidataBenchmark(
+                benchmark_result.name, benchmark_result.id, self.__openapi_service
             )
-        )
-        return RapidataBenchmark(
-            benchmark_result.name, benchmark_result.id, self.__openapi_service
-        )
 
     def find_benchmarks(
         self, name: str = "", amount: int = 10
@@ -137,24 +190,27 @@ class RapidataBenchmarkManager:
         """
         Returns a list of benchmarks by their name.
         """
-        benchmark_result = self.__openapi_service.benchmark_api.benchmarks_get(
-            QueryModel(
-                page=PageInfo(index=1, size=amount),
-                filter=RootFilter(
-                    filters=[
-                        Filter(
-                            field="Name", operator=FilterOperator.CONTAINS, value=name
+        with tracer.start_as_current_span("RapidataBenchmarkManager.find_benchmarks"):
+            benchmark_result = self.__openapi_service.benchmark_api.benchmarks_get(
+                QueryModel(
+                    page=PageInfo(index=1, size=amount),
+                    filter=RootFilter(
+                        filters=[
+                            Filter(
+                                field="Name",
+                                operator=FilterOperator.CONTAINS,
+                                value=name,
+                            )
+                        ]
+                    ),
+                    sortCriteria=[
+                        SortCriterion(
+                            direction=SortDirection.DESC, propertyName="CreatedAt"
                         )
-                    ]
-                ),
-                sortCriteria=[
-                    SortCriterion(
-                        direction=SortDirection.DESC, propertyName="CreatedAt"
-                    )
-                ],
+                    ],
+                )
             )
-        )
-        return [
-            RapidataBenchmark(benchmark.name, benchmark.id, self.__openapi_service)
-            for benchmark in benchmark_result.items
-        ]
+            return [
+                RapidataBenchmark(benchmark.name, benchmark.id, self.__openapi_service)
+                for benchmark in benchmark_result.items
+            ]

rapidata/rapidata_client/config/__init__.py

@@ -1 +1,4 @@
+from .logger import logger  # has to be instantiated before rapidata_config
+from .tracer import tracer  # has to be instantiated before rapidata_config
 from .rapidata_config import rapidata_config
+from .managed_print import managed_print

rapidata/rapidata_client/config/logger.py

@@ -0,0 +1,135 @@
+import logging
+from typing import Protocol, runtime_checkable
+from opentelemetry._logs import set_logger_provider
+from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
+from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
+from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
+from opentelemetry.sdk.resources import Resource
+from rapidata import __version__
+from .logging_config import LoggingConfig, register_config_handler
+
+
+@runtime_checkable
+class LoggerProtocol(Protocol):
+    """Protocol that defines the logger interface for type checking."""
+
+    def debug(self, msg: object, *args, **kwargs) -> None: ...
+    def info(self, msg: object, *args, **kwargs) -> None: ...
+    def warning(self, msg: object, *args, **kwargs) -> None: ...
+    def warn(self, msg: object, *args, **kwargs) -> None: ...
+    def error(self, msg: object, *args, **kwargs) -> None: ...
+    def exception(self, msg: object, *args, exc_info=True, **kwargs) -> None: ...
+    def critical(self, msg: object, *args, **kwargs) -> None: ...
+    def fatal(self, msg: object, *args, **kwargs) -> None: ...
+    def log(self, level: int, msg: object, *args, **kwargs) -> None: ...
+    def isEnabledFor(self, level: int) -> bool: ...
+    def getEffectiveLevel(self) -> int: ...
+    def setLevel(self, level: int | str) -> None: ...
+    def addHandler(self, handler: logging.Handler) -> None: ...
+    def removeHandler(self, handler: logging.Handler) -> None: ...
+    @property
+    def handlers(self) -> list[logging.Handler]: ...
+    @property
+    def level(self) -> int: ...
+    @property
+    def name(self) -> str: ...
+
+
+class RapidataLogger:
+    """Logger implementation that updates when the configuration changes."""
+
+    def __init__(self, name: str = "rapidata"):
+        self._logger = logging.getLogger(name)
+        self._otlp_initialized = False
+        self._otlp_handler = None
+
+        # Register this logger to receive configuration updates
+        register_config_handler(self._handle_config_update)
+
+    def _handle_config_update(self, config: LoggingConfig) -> None:
+        """Handle configuration updates."""
+        self._update_logger(config)
+
+    def _update_logger(self, config: LoggingConfig) -> None:
+        """Update the logger based on the new configuration."""
+        # Initialize OTLP logging only once and only if not disabled
+        if not self._otlp_initialized and config.enable_otlp:
+            try:
+                logger_provider = LoggerProvider(
+                    resource=Resource.create(
+                        {
+                            "service.name": "Rapidata.Python.SDK",
+                            "service.version": __version__,
+                        }
+                    ),
+                )
+                set_logger_provider(logger_provider)
+
+                exporter = OTLPLogExporter(
+                    endpoint="https://otlp-sdk.rapidata.ai/v1/logs",
+                    timeout=30,
+                )
+
+                processor = BatchLogRecordProcessor(
+                    exporter,
+                    max_queue_size=2048,
+                    export_timeout_millis=30000,
+                    max_export_batch_size=512,
+                )
+
+                logger_provider.add_log_record_processor(processor)
+
+                # OTLP handler - captures DEBUG and above
+                self._otlp_handler = LoggingHandler(logger_provider=logger_provider)
+                self._otlp_handler.setLevel(logging.DEBUG)  # OTLP gets everything
+
+                self._otlp_initialized = True
+
+            except Exception as e:
+                self._logger.warning(f"Failed to initialize OTLP logging: {e}")
+                import traceback
+
+                traceback.print_exc()
+
+        # Console handler with configurable level
+        console_handler = logging.StreamHandler()
+        console_level = getattr(logging, config.level.upper())
+        console_handler.setLevel(console_level)
+        console_formatter = logging.Formatter(config.format)
+        console_handler.setFormatter(console_formatter)
+
+        # Configure the logger
+        self._logger.setLevel(logging.DEBUG)  # Logger must allow DEBUG for OTLP
+
+        # Remove any existing handlers (except OTLP when appropriate)
+        for handler in self._logger.handlers[:]:
+            if handler != self._otlp_handler:
+                self._logger.removeHandler(handler)
+            elif handler == self._otlp_handler and not config.enable_otlp:
+                self._logger.removeHandler(handler)
+
+        # Add OTLP handler if initialized and not disabled
+        if (
+            self._otlp_handler
+            and self._otlp_handler not in self._logger.handlers
+            and config.enable_otlp
+        ):
+            self._logger.addHandler(self._otlp_handler)
+
+        # Add console handler
+        self._logger.addHandler(console_handler)
+
+        # Add file handler if log_file is provided
+        if config.log_file:
+            file_handler = logging.FileHandler(config.log_file)
+            file_handler.setLevel(console_level)  # Use same level as console
+            file_formatter = logging.Formatter(config.format)
+            file_handler.setFormatter(file_formatter)
+            self._logger.addHandler(file_handler)
+
+    def __getattr__(self, name: str) -> object:
+        """Delegate attribute access to the underlying logger."""
+        return getattr(self._logger, name)
+
+
+logger: LoggerProtocol = RapidataLogger()  # type: ignore[assignment]

rapidata/rapidata_client/config/logging_config.py

@@ -0,0 +1,55 @@
+from typing import Callable
+from pydantic import BaseModel, Field
+from rapidata.rapidata_client.config import logger
+
+# Type alias for config update handlers
+ConfigUpdateHandler = Callable[["LoggingConfig"], None]
+
+# Global list to store registered handlers
+_config_handlers: list[ConfigUpdateHandler] = []
+
+
+def register_config_handler(handler: ConfigUpdateHandler) -> None:
+    """Register a handler to be called when the logging configuration updates."""
+    _config_handlers.append(handler)
+
+
+def unregister_config_handler(handler: ConfigUpdateHandler) -> None:
+    """Unregister a previously registered handler."""
+    if handler in _config_handlers:
+        _config_handlers.remove(handler)
+
+
+class LoggingConfig(BaseModel):
+    """
+    Holds the configuration for the logging process.
+
+    Attributes:
+        level (str): The logging level. Defaults to "WARNING".
+        log_file (str | None): The logging file. Defaults to None.
+        format (str): The logging format. Defaults to "%(asctime)s - %(name)s - %(levelname)s - %(message)s".
+        silent_mode (bool): Whether to disable the prints and progress bars. Does NOT affect the logging. Defaults to False.
+        enable_otlp (bool): Whether to enable OpenTelemetry trace logs. Defaults to True.
+    """
+
+    level: str = Field(default="WARNING")
+    log_file: str | None = Field(default=None)
+    format: str = Field(default="%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+    silent_mode: bool = Field(default=False)
+    enable_otlp: bool = Field(default=True)
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self._notify_handlers()
+
+    def __setattr__(self, name: str, value) -> None:
+        super().__setattr__(name, value)
+        self._notify_handlers()
+
+    def _notify_handlers(self) -> None:
+        """Notify all registered handlers that the configuration has updated."""
+        for handler in _config_handlers:
+            try:
+                handler(self)
+            except Exception as e:
+                logger.warning(f"Warning: Config handler failed: {e}")

rapidata/rapidata_client/config/managed_print.py

@@ -0,0 +1,6 @@
+from rapidata.rapidata_client.config import rapidata_config
+
+
+def managed_print(*args, **kwargs) -> None:
+    if not rapidata_config.logging.silent_mode:
+        print(*args, **kwargs)

rapidata/rapidata_client/config/order_config.py

@@ -0,0 +1,14 @@
+from pydantic import BaseModel, Field
+
+
+class OrderConfig(BaseModel):
+    """
+    Holds the configuration for the order process.
+
+    Attributes:
+        minOrderDatapointsForValidation (int): The minimum number of datapoints required so that an automatic validationset gets created if no recommended was found. Defaults to 50.
+        autoValidationSetSize (int): The maximum size of the auto-generated validation set. Defaults to 20.
+    """
+
+    autoValidationSetCreation: bool = Field(default=True)
+    minOrderDatapointsForValidation: int = Field(default=20)

rapidata/rapidata_client/config/rapidata_config.py

@@ -1,5 +1,9 @@
 from pydantic import BaseModel, Field
 
+from rapidata.rapidata_client.config.logging_config import LoggingConfig
+from rapidata.rapidata_client.config.order_config import OrderConfig
+from rapidata.rapidata_client.config.upload_config import UploadConfig
+
 
 class RapidataConfig(BaseModel):
     """
@@ -7,25 +11,26 @@ class RapidataConfig(BaseModel):
 
     To adjust the configurations used, you can modify the `rapidata_config` object.
 
-    Args:
-        maxUploadWorkers (int): The maximum number of worker threads for processing media paths. Defaults to 10.
-        uploadMaxRetries (int): The maximum number of retries for failed uploads. Defaults to 3.
+    Attributes:
         enableBetaFeatures (bool): Whether to enable beta features. Defaults to False.
-        minOrderDatapointsForValidation (int): The minimum number of datapoints required so that an automatic validationset gets created if no recommended was found. Defaults to 50.
-        autoValidationSetSize (int): The maximum size of the auto-generated validation set. Defaults to 20.
+        upload (UploadConfig): The configuration for the upload process.
+            Such as the maximum number of worker threads for processing media paths and the maximum number of retries for failed uploads.
+        order (OrderConfig): The configuration for the order process.
+            Such as the minimum number of datapoints required so that an automatic validationset gets created if no recommended was found.
+        logging (LoggingConfig): The configuration for the logging process.
+            Such as the logging level and the logging file.
 
     Example:
         ```python
         from rapidata import rapidata_config
-        rapidata_config.maxUploadWorkers = 20
+        rapidata_config.upload.maxUploadWorkers = 20
         ```
     """
 
-    maxUploadWorkers: int = Field(default=10)
-    uploadMaxRetries: int = Field(default=3)
     enableBetaFeatures: bool = False
-    minOrderDatapointsForValidation: int = Field(default=50)
-    autoValidationSetSize: int = Field(default=20)
+    upload: UploadConfig = Field(default_factory=UploadConfig)
+    order: OrderConfig = Field(default_factory=OrderConfig)
+    logging: LoggingConfig = Field(default_factory=LoggingConfig)
 
 
 rapidata_config = RapidataConfig()