scorable 1.6.4__py3-none-any.whl

root/__about__.py

@@ -0,0 +1,4 @@
+# SPDX-FileCopyrightText: 2024-present juho.ylikyla <juho.ylikyla@rootsignals.ai>
+#
+# SPDX-License-Identifier: MIT
+__version__ = "1.6.4"

root/__init__.py

@@ -0,0 +1,17 @@
+"""Root Signals SDK.
+
+The use should start by creating a :class:`root.client.RootSignals` instance.
+
+Example::
+
+  from root import RootSignals
+  client = RootSignals()
+
+"""
+
+from .__about__ import __version__
+from .client import RootSignals
+
+# Note: PEP-396 was rejected but we provide __version__ anyway
+# ( https://peps.python.org/pep-0396/ )
+__all__ = ["__version__", "RootSignals"]

root/client.py

@@ -0,0 +1,207 @@
+from __future__ import annotations
+
+import os
+import re
+import textwrap
+from contextlib import asynccontextmanager, contextmanager
+from functools import cached_property
+from typing import (
+    TYPE_CHECKING,
+    AsyncContextManager,
+    AsyncGenerator,
+    Callable,
+    ContextManager,
+    Generator,
+    Optional,
+    Type,
+    Union,
+)
+
+from .__about__ import __version__
+from .generated import openapi_aclient, openapi_client
+from .generated.openapi_aclient.configuration import Configuration as _AConfiguration
+from .generated.openapi_client.configuration import Configuration as _Configuration
+
+if TYPE_CHECKING:
+    from .datasets import DataSets
+    from .execution_logs import ExecutionLogs
+    from .judges import Judges
+    from .models import Models
+    from .objectives import Objectives
+    from .skills import Evaluators
+
+
+def _get_api_key(*, dot_env: str = ".env") -> str:
+    var = "ROOTSIGNALS_API_KEY"
+    api_key = os.environ.get(var)
+    if api_key is not None:
+        return api_key
+    if os.path.exists(dot_env):
+        for line in open(dot_env):
+            m = re.fullmatch(line, rf"^\s*{var}\s*=\s*(\S+)\s*$")
+            if m is not None:
+                return m.group(1)
+    raise ValueError(
+        textwrap.dedent("""
+    Root Signals API key cannot be found.
+    It can be provided in client invocation, using ROOTSIGNALS_API_KEY environment variable or .env file line
+    """)
+    )
+
+
+class Beta:
+    """Beta API features namespace"""
+
+    def __init__(
+        self,
+        get_client_context: Union[
+            Callable[[], AsyncContextManager[openapi_aclient.ApiClient]],
+            Callable[[], ContextManager[openapi_client.ApiClient]],
+        ],
+    ) -> None:
+        self._get_client_context = get_client_context
+
+    @cached_property
+    def judges(self) -> Judges:
+        """Get Judges API (Beta)"""
+        from .judges import Judges
+
+        return Judges(self._get_client_context)
+
+
+class Scorable:
+    """Root Signals API Python client.
+
+    The API key must be provided via one of the following methods - the code uses the first one that is found:
+
+    1. as an argument to RootSignals constructor,
+    2. environment variable `ROOTSIGNALS_API_KEY`, or
+    3. .env file containing `ROOTSIGNALS_API_KEY=`
+
+    Args:
+        api_key: Root Signals API Key (if not provided from environment)
+        run_async: Whether to run the API client asynchronously
+    """
+
+    def __init__(
+        self,
+        api_key: Optional[str] = None,
+        *,
+        run_async: bool = False,
+        _api_client: Union[Optional[openapi_aclient.ApiClient], Optional[openapi_client.ApiClient]] = None,
+        base_url: Optional[str] = None,
+    ):
+        self.run_async = run_async
+        if api_key is None:
+            api_key = _get_api_key()
+        if base_url is None:
+            base_url = os.environ.get("ROOTSIGNALS_API_URL", "https://api.app.rootsignals.ai")
+        self.base_url = base_url
+        self.api_key = api_key
+        self._api_client_arg = _api_client
+
+    @cached_property
+    def get_client_context(
+        self,
+    ) -> Union[
+        Callable[[], AsyncContextManager[openapi_aclient.ApiClient]],
+        Callable[[], ContextManager[openapi_client.ApiClient]],
+    ]:
+        if self._api_client_arg is not None:
+            if isinstance(self._api_client_arg, openapi_aclient.ApiClient):
+
+                @asynccontextmanager
+                async def async_client_context() -> AsyncGenerator[openapi_aclient.ApiClient, None]:
+                    assert isinstance(self._api_client_arg, openapi_aclient.ApiClient)
+                    yield self._api_client_arg
+
+                return async_client_context
+            else:
+
+                @contextmanager
+                def sync_client_context() -> Generator[openapi_client.ApiClient, None, None]:
+                    assert isinstance(self._api_client_arg, openapi_client.ApiClient)
+                    yield self._api_client_arg
+
+                return sync_client_context
+
+        if self.run_async:
+            return self._configure_client_context(openapi_aclient.ApiClient, _AConfiguration)
+        return self._configure_client_context(openapi_client.ApiClient, _Configuration)
+
+    def _configure_client_context(
+        self,
+        client_cls: Union[Type[openapi_client.ApiClient], Type[openapi_aclient.ApiClient]],
+        config_cls: Union[Type[_Configuration], Type[_AConfiguration]],
+    ) -> Union[
+        Callable[[], ContextManager[openapi_client.ApiClient]],
+        Callable[[], AsyncContextManager[openapi_aclient.ApiClient]],
+    ]:
+        config = config_cls(host=self.base_url)
+        config.api_key["publicApiKey"] = f"Api-Key {self.api_key}"
+
+        if issubclass(client_cls, openapi_aclient.ApiClient):
+
+            @asynccontextmanager
+            async def async_client_context() -> AsyncGenerator[openapi_aclient.ApiClient, None]:
+                async with client_cls(config) as client:
+                    client.user_agent = f"rs-python-sdk/{__version__}"
+                    yield client
+
+            return async_client_context
+        else:
+
+            @contextmanager
+            def sync_client_context() -> Generator[openapi_client.ApiClient, None, None]:
+                with client_cls(config) as client:
+                    client.user_agent = f"rs-python-sdk/{__version__}"
+                    yield client
+
+            return sync_client_context
+
+    @cached_property
+    def datasets(self) -> DataSets:
+        """Get DataSets API"""
+        from .datasets import DataSets
+
+        return DataSets(self.get_client_context, self.base_url, self.api_key)
+
+    @cached_property
+    def evaluators(self) -> Evaluators:
+        """Get Evaluators API"""
+        from .skills import Evaluators
+
+        return Evaluators(self.get_client_context)
+
+    @cached_property
+    def execution_logs(self) -> ExecutionLogs:
+        """Get Execution Logs API"""
+        from .execution_logs import ExecutionLogs
+
+        return ExecutionLogs(self.get_client_context)
+
+    @cached_property
+    def models(self) -> Models:
+        """Get Models API"""
+        from .models import Models
+
+        return Models(self.get_client_context)
+
+    @cached_property
+    def objectives(self) -> Objectives:
+        """Get Objectives API"""
+        from .objectives import Objectives
+
+        return Objectives(self.get_client_context)
+
+    @cached_property
+    def judges(self) -> Judges:
+        """Get Judges API"""
+        from .judges import Judges
+
+        return Judges(self.get_client_context)
+
+    @cached_property
+    def beta(self) -> Beta:
+        """Get Beta API features"""
+        return Beta(self.get_client_context)

root/datasets.py

@@ -0,0 +1,231 @@
+from contextlib import AbstractAsyncContextManager
+from functools import partial
+from typing import Any, AsyncIterator, Dict, Iterator, Optional
+
+import aiohttp
+import requests
+from pydantic import StrictStr
+
+from root.generated.openapi_client.api_client import ApiClient
+
+from .generated.openapi_aclient import ApiClient as AApiClient
+from .generated.openapi_aclient.api.datasets_api import DatasetsApi as ADatasetsApi
+from .generated.openapi_aclient.models.data_set_create import DataSetCreate as ADataSetCreate
+from .generated.openapi_aclient.models.data_set_list import DataSetList as ADataSetList
+from .generated.openapi_aclient.models.paginated_data_set_list_list import (
+    PaginatedDataSetListList as APaginatedDataSetListList,
+)
+from .generated.openapi_client.api.datasets_api import DatasetsApi as DatasetsApi
+from .generated.openapi_client.models.data_set_create import DataSetCreate
+from .generated.openapi_client.models.data_set_list import DataSetList
+from .utils import ClientContextCallable, iterate_cursor_list, with_async_client, with_sync_client
+
+
+class DataSets:
+    """
+    DataSets (sub) API
+
+    Note:
+
+      The construction of the API instance should be handled by
+      accesing an attribute of a :class:`root.client.RootSignals` instance.
+    """
+
+    def __init__(self, client_context: ClientContextCallable, base_url: str, api_key: str):
+        self.client_context = client_context
+        self.base_url = base_url
+        self.api_key = api_key
+
+    def create(
+        self,
+        *,
+        name: Optional[str] = None,
+        path: Optional[str] = None,
+        type: str = "reference",
+        _request_timeout: Optional[int] = None,
+    ) -> Optional[DataSetCreate]:
+        """
+        Create a dataset object with the given parameters to the registry.
+        If the dataset has a path, it will be uploaded to the registry.
+        """
+
+        payload: Dict[str, Any] = {"name": name, "type": type, "tags": []}
+        file = None
+        try:
+            if path:
+                file = open(path, "rb")
+                files = {"file": file}
+            else:
+                files = None
+
+            response = requests.post(
+                f"{self.base_url}/datasets/",
+                headers={"Authorization": f"Api-Key {self.api_key}"},
+                data=payload,
+                files=files,
+                timeout=_request_timeout or 120,
+            )
+            if not response.ok:
+                raise Exception(f"create failed with status code {response.status_code} and message\n{response.text}")
+
+            return DataSetCreate.from_dict(response.json())
+        finally:
+            if file and not file.closed:
+                file.close()
+
+    async def acreate(
+        self,
+        *,
+        name: Optional[str] = None,
+        path: Optional[str] = None,
+        type: str = "reference",
+        _request_timeout: Optional[int] = None,
+    ) -> Optional[ADataSetCreate]:
+        """
+        Asynchronously create a dataset object with the given parameters to the registry.
+        If the dataset has a path, it will be uploaded to the registry.
+
+        """
+
+        payload = aiohttp.FormData()
+        payload.add_field("name", name)
+        payload.add_field("type", type)
+
+        file = None
+        try:
+            if path:
+                file = open(path, "rb")
+                payload.add_field("file", file)
+
+            async with aiohttp.ClientSession() as session:
+                async with session.post(
+                    f"{self.base_url}/datasets/",
+                    data=payload,
+                    headers={"Authorization": f"Api-Key {self.api_key}"},
+                    timeout=aiohttp.ClientTimeout(_request_timeout) or aiohttp.ClientTimeout(120),
+                ) as response:
+                    if not response.ok:
+                        raise Exception(
+                            f"create failed with status code {response.status} and message\n{response.text}"
+                        )
+                    return ADataSetCreate.from_dict(await response.json())
+        finally:
+            if file and not file.closed:
+                file.close()
+
+    @with_sync_client
+    def get(
+        self,
+        dataset_id: str,
+        *,
+        _request_timeout: Optional[int] = None,
+        _client: ApiClient,
+    ) -> DataSetList:
+        """
+        Get a dataset object from the registry.
+        """
+
+        api_instance = DatasetsApi(_client)
+        return api_instance.datasets_retrieve(dataset_id, _request_timeout=_request_timeout)
+
+    @with_async_client
+    async def aget(
+        self,
+        dataset_id: str,
+        *,
+        _request_timeout: Optional[int] = None,
+        _client: AApiClient,
+    ) -> ADataSetList:
+        """
+        Asynchronously get a dataset object from the registry.
+        """
+
+        api_instance = ADatasetsApi(_client)
+        return await api_instance.datasets_retrieve(dataset_id, _request_timeout=_request_timeout)
+
+    @with_sync_client
+    def list(
+        self,
+        search_term: Optional[str] = None,
+        *,
+        limit: int = 100,
+        _request_timeout: Optional[int] = None,
+        _client: ApiClient,
+    ) -> Iterator[DataSetList]:
+        """
+        Iterate through the datasets.
+
+        Args:
+          limit: Number of entries to iterate through at most.
+          search_term: Can be used to limit returned datasets.
+        """
+
+        api_instance = DatasetsApi(_client)
+        yield from iterate_cursor_list(
+            partial(api_instance.datasets_list, search=search_term, _request_timeout=_request_timeout), limit=limit
+        )
+
+    async def alist(
+        self,
+        search_term: Optional[str] = None,
+        *,
+        limit: int = 100,
+        _request_timeout: Optional[int] = None,
+    ) -> AsyncIterator[ADataSetList]:
+        """
+        Asynchronously iterate through the datasets.
+
+        Args:
+          limit: Number of entries to iterate through at most.
+          search_term: Can be used to limit returned datasets.
+        """
+
+        context = self.client_context()
+        assert isinstance(context, AbstractAsyncContextManager), "This method is not available in synchronous mode"
+        async with context as client:
+            api_instance = ADatasetsApi(client)
+            partial_list = partial(api_instance.datasets_list, search=search_term, _request_timeout=_request_timeout)
+            cursor: Optional[StrictStr] = None
+            while limit > 0:
+                result: APaginatedDataSetListList = await partial_list(page_size=limit, cursor=cursor)
+                if not result.results:
+                    return
+
+                used_results = result.results[:limit]
+                limit -= len(used_results)
+
+                for used_result in used_results:
+                    yield used_result
+
+                if not (cursor := result.next):
+                    return
+
+    @with_sync_client
+    def delete(
+        self,
+        dataset_id: str,
+        *,
+        _request_timeout: Optional[int] = None,
+        _client: ApiClient,
+    ) -> None:
+        """
+        Delete a dataset object from the registry.
+        """
+
+        api_instance = DatasetsApi(_client)
+        return api_instance.datasets_destroy(dataset_id, _request_timeout=_request_timeout)
+
+    @with_async_client
+    async def adelete(
+        self,
+        dataset_id: str,
+        *,
+        _request_timeout: Optional[int] = None,
+        _client: AApiClient,
+    ) -> None:
+        """
+        Asynchronously delete a dataset object from the registry.
+        """
+
+        api_instance = ADatasetsApi(_client)
+        return await api_instance.datasets_destroy(dataset_id, _request_timeout=_request_timeout)

root/execution_logs.py

@@ -0,0 +1,162 @@
+from __future__ import annotations
+
+from contextlib import AbstractAsyncContextManager
+from functools import partial
+from typing import TYPE_CHECKING, AsyncIterator, Iterator, List, Optional, Protocol
+
+from pydantic import StrictStr
+
+from .generated.openapi_aclient import ApiClient as AApiClient
+from .generated.openapi_aclient.api.execution_logs_api import ExecutionLogsApi as AExecutionLogsApi
+from .generated.openapi_aclient.models.execution_log_details import ExecutionLogDetails as AExecutionLogDetails
+from .generated.openapi_aclient.models.execution_log_list import ExecutionLogList as AExecutionLogList
+from .generated.openapi_aclient.models.paginated_execution_log_list_list import (
+    PaginatedExecutionLogListList as APaginatedExecutionLogListList,
+)
+from .generated.openapi_client import ApiClient
+from .generated.openapi_client.api.execution_logs_api import ExecutionLogsApi as ExecutionLogsApi
+from .generated.openapi_client.models.execution_log_details import ExecutionLogDetails
+from .generated.openapi_client.models.execution_log_list import ExecutionLogList
+from .utils import ClientContextCallable, iterate_cursor_list, with_async_client, with_sync_client
+
+if TYPE_CHECKING:
+
+    class ExecutionResult(Protocol):
+        execution_log_id: str
+
+
+class ExecutionLogs:
+    """Execution logs API"""
+
+    def __init__(self, client_context: ClientContextCallable):
+        self.client_context = client_context
+
+    @with_sync_client
+    def list(
+        self,
+        *,
+        limit: int = 100,
+        search_term: Optional[str] = None,
+        tags: Optional[List[str]] = None,
+        include: Optional[List[str]] = None,
+        _request_timeout: Optional[int] = None,
+        _client: ApiClient,
+    ) -> Iterator[ExecutionLogList]:
+        """
+        List execution logs
+
+        Args:
+          limit: Number of entries to iterate through at most.
+          search_term: Can be used to limit returned logs. For example, a evaluator id or name.
+          tags: Optional tags to filter the logs by.
+          include: Optional fields to include in the response.
+        """
+        api_instance = ExecutionLogsApi(_client)
+        yield from iterate_cursor_list(
+            partial(
+                api_instance.execution_logs_list,
+                search=search_term,
+                tags=",".join(tags) if tags else None,
+                include=",".join(include) if include else None,
+                _request_timeout=_request_timeout,
+            ),
+            limit=limit,
+        )
+
+    async def alist(
+        self,
+        *,
+        limit: int = 100,
+        search_term: Optional[str] = None,
+        tags: Optional[List[str]] = None,
+        include: Optional[List[str]] = None,
+        _request_timeout: Optional[int] = None,
+    ) -> AsyncIterator[AExecutionLogList]:
+        """
+        Asynchronously list execution logs
+
+        Args:
+          limit: Number of entries to iterate through at most.
+          search_term: Can be used to limit returned logs. For example, a evaluator id or name.
+          tags: Optional tags to filter the logs by.
+          include: Optional fields to include in the response.
+        """
+
+        context = self.client_context()
+        assert isinstance(context, AbstractAsyncContextManager), "This method is not available in synchronous mode"
+        async with context as client:
+            api_instance = AExecutionLogsApi(client)
+            partial_list = partial(
+                api_instance.execution_logs_list,
+                search=search_term,
+                tags=",".join(tags) if tags else None,
+                include=",".join(include) if include else None,
+                _request_timeout=_request_timeout,
+            )
+
+            cursor: Optional[StrictStr] = None
+            while limit > 0:
+                result: APaginatedExecutionLogListList = await partial_list(page_size=limit, cursor=cursor)
+                if not result.results:
+                    return
+
+                used_results = result.results[:limit]
+                limit -= len(used_results)
+
+                for used_result in used_results:
+                    yield used_result
+
+                if not (cursor := result.next):
+                    return
+
+    @with_sync_client
+    def get(
+        self,
+        *,
+        log_id: Optional[str] = None,
+        execution_result: Optional[ExecutionResult] = None,
+        _request_timeout: Optional[int] = None,
+        _client: ApiClient,
+    ) -> ExecutionLogDetails:
+        """
+        Get a specific execution log details
+
+        Args:
+          log_id: The log to be fetched
+          execution_result: The execution result containing the log ID.
+
+        Raises:
+          ValueError: If both log_id and execution_result are None.
+        """
+
+        api_instance = ExecutionLogsApi(_client)
+        _log_id = execution_result.execution_log_id if execution_result else log_id
+        if _log_id is None:
+            raise ValueError("Either log_id or execution_result must be provided")
+        return api_instance.execution_logs_retrieve(_log_id, _request_timeout=_request_timeout)
+
+    @with_async_client
+    async def aget(
+        self,
+        *,
+        log_id: Optional[str] = None,
+        execution_result: Optional[ExecutionResult] = None,
+        _request_timeout: Optional[int] = None,
+        _client: AApiClient,
+    ) -> AExecutionLogDetails:
+        """
+        Asynchronously get a specific execution log details
+
+        Args:
+          log_id: The log to be fetched
+          execution_result: The execution result containing the log ID.
+
+        Raises:
+          ValueError: If both log_id and execution_result are None.
+        """
+
+        api_instance = AExecutionLogsApi(_client)
+        _log_id = execution_result.execution_log_id if execution_result else log_id
+        if _log_id is None:
+            raise ValueError("Either log_id or execution_result must be provided")
+        return await api_instance.execution_logs_retrieve(_log_id, _request_timeout=_request_timeout)

root/generated/openapi_aclient/__init__.py

@@ -0,0 +1 @@
+from .api_client import ApiClient

root/generated/openapi_aclient/api/__init__.py

@@ -0,0 +1 @@
+