calibrate-python-sdk 0.0.1__tar.gz

calibrate_python_sdk-0.0.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Artpark.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

calibrate_python_sdk-0.0.1/PKG-INFO

@@ -0,0 +1,55 @@
+Metadata-Version: 2.1
+Name: calibrate-python-sdk
+Version: 0.0.1
+Summary: 
+License: MIT
+Requires-Python: >=3.10,<4.0
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: MacOS
+Classifier: Operating System :: Microsoft :: Windows
+Classifier: Operating System :: OS Independent
+Classifier: Operating System :: POSIX
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Programming Language :: Python :: 3.15
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Typing :: Typed
+Provides-Extra: aiohttp
+Requires-Dist: aiohttp (>=3.14.1,<4) ; (python_version >= "3.10") and (extra == "aiohttp")
+Requires-Dist: httpx (>=0.21.2)
+Requires-Dist: httpx-aiohttp (==0.1.8) ; (python_version >= "3.10") and (extra == "aiohttp")
+Requires-Dist: pydantic (>=1.9.2)
+Requires-Dist: pydantic-core (>=2.18.2,<3.0.0)
+Requires-Dist: typing_extensions (>=4.0.0)
+Project-URL: Repository, https://github.com/dalmia/calibrate-python-sdk
+Description-Content-Type: text/markdown
+
+# Calibrate Python SDK
+
+Python client for the [Calibrate](https://pense-backend.artpark.ai) API.
+
+## Installation
+
+```sh
+pip install calibrate-python-sdk
+```
+
+## Usage
+
+```python
+from calibrate_python_sdk import Calibrate
+
+client = Calibrate(token="sk_your_api_key")
+
+agents = client.agents.list()
+```
+
+This SDK is generated from Calibrate's public API with [Fern](https://buildwithfern.com).
+

calibrate_python_sdk-0.0.1/README.md

@@ -0,0 +1,21 @@
+# Calibrate Python SDK
+
+Python client for the [Calibrate](https://pense-backend.artpark.ai) API.
+
+## Installation
+
+```sh
+pip install calibrate-python-sdk
+```
+
+## Usage
+
+```python
+from calibrate_python_sdk import Calibrate
+
+client = Calibrate(token="sk_your_api_key")
+
+agents = client.agents.list()
+```
+
+This SDK is generated from Calibrate's public API with [Fern](https://buildwithfern.com).

calibrate_python_sdk-0.0.1/pyproject.toml

@@ -0,0 +1,98 @@
+[project]
+name = "calibrate-python-sdk"
+dynamic = ["version"]
+
+[tool.poetry]
+name = "calibrate-python-sdk"
+version = "0.0.1"
+description = ""
+readme = "README.md"
+authors = []
+keywords = []
+license = "MIT"
+classifiers = [
+    "Intended Audience :: Developers",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
+    "Programming Language :: Python :: 3.15",
+    "Operating System :: OS Independent",
+    "Operating System :: POSIX",
+    "Operating System :: MacOS",
+    "Operating System :: POSIX :: Linux",
+    "Operating System :: Microsoft :: Windows",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+    "Typing :: Typed",
+    "License :: OSI Approved :: MIT License"
+]
+packages = [
+    { include = "artpark", from = "src"}
+]
+
+[tool.poetry.urls]
+Repository = 'https://github.com/dalmia/calibrate-python-sdk'
+
+[tool.poetry.dependencies]
+python = "^3.10"
+aiohttp = { version = ">=3.14.1,<4", optional = true, python = ">=3.10"}
+httpx = ">=0.21.2"
+httpx-aiohttp = { version = "0.1.8", optional = true, python = ">=3.10"}
+pydantic = ">= 1.9.2"
+pydantic-core = ">=2.18.2,<3.0.0"
+typing_extensions = ">= 4.0.0"
+
+[tool.poetry.group.dev.dependencies]
+mypy = "==1.13.0"
+pytest = "^9.0.3"
+pytest-asyncio = "^1.0.0"
+pytest-xdist = "^3.6.1"
+python-dateutil = "^2.9.0"
+types-python-dateutil = "^2.9.0.20240316"
+urllib3 = ">=2.6.3,<3.0.0"
+ruff = "==0.11.5"
+
+[tool.pytest.ini_options]
+testpaths = [ "tests" ]
+asyncio_mode = "auto"
+norecursedirs = [ "src" ]
+markers = [
+    "aiohttp: tests that require httpx_aiohttp to be installed",
+]
+
+[tool.mypy]
+plugins = ["pydantic.mypy"]
+
+[tool.ruff]
+line-length = 120
+
+[tool.ruff.lint]
+select = [
+    "E",  # pycodestyle errors
+    "F",  # pyflakes
+    "I",  # isort
+]
+ignore = [
+    "E402",  # Module level import not at top of file
+    "E501",  # Line too long
+    "E711",  # Comparison to `None` should be `cond is not None`
+    "E712",  # Avoid equality comparisons to `True`; use `if ...:` checks
+    "E721",  # Use `is` and `is not` for type comparisons, or `isinstance()` for insinstance checks
+    "E722",  # Do not use bare `except`
+    "E731",  # Do not assign a `lambda` expression, use a `def`
+    "F821",  # Undefined name
+    "F841"   # Local variable ... is assigned to but never used
+]
+
+[tool.ruff.lint.isort]
+section-order = ["future", "standard-library", "third-party", "first-party"]
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"
+
+[tool.poetry.extras]
+aiohttp=["aiohttp", "httpx-aiohttp"]

calibrate_python_sdk-0.0.1/src/artpark/__init__.py

@@ -0,0 +1,109 @@
+# This file was auto-generated by Fern from our API Definition.
+
+# isort: skip_file
+
+import typing
+from importlib import import_module
+
+if typing.TYPE_CHECKING:
+    from .types import (
+        BatchRunRequest,
+        BatchTestRun,
+        BatchTestRunResponse,
+        BatchTestSkip,
+        HttpValidationError,
+        JudgeResult,
+        ResolveAgentNamesResponse,
+        RoutersAgentTestsAgentResponse,
+        RoutersAgentTestsAgentResponseType,
+        TaskCreateResponse,
+        TestCaseResult,
+        TestOutput,
+        TestRunStatusResponse,
+        ToolCallOutput,
+        ValidationError,
+        ValidationErrorLocItem,
+    )
+    from .errors import UnprocessableEntityError
+    from . import agent_tests, agents
+    from ._default_clients import DefaultAioHttpClient, DefaultAsyncHttpxClient
+    from .client import AsyncCalibrate, Calibrate
+    from .environment import CalibrateEnvironment
+    from .version import __version__
+_dynamic_imports: typing.Dict[str, str] = {
+    "AsyncCalibrate": ".client",
+    "BatchRunRequest": ".types",
+    "BatchTestRun": ".types",
+    "BatchTestRunResponse": ".types",
+    "BatchTestSkip": ".types",
+    "Calibrate": ".client",
+    "CalibrateEnvironment": ".environment",
+    "DefaultAioHttpClient": "._default_clients",
+    "DefaultAsyncHttpxClient": "._default_clients",
+    "HttpValidationError": ".types",
+    "JudgeResult": ".types",
+    "ResolveAgentNamesResponse": ".types",
+    "RoutersAgentTestsAgentResponse": ".types",
+    "RoutersAgentTestsAgentResponseType": ".types",
+    "TaskCreateResponse": ".types",
+    "TestCaseResult": ".types",
+    "TestOutput": ".types",
+    "TestRunStatusResponse": ".types",
+    "ToolCallOutput": ".types",
+    "UnprocessableEntityError": ".errors",
+    "ValidationError": ".types",
+    "ValidationErrorLocItem": ".types",
+    "__version__": ".version",
+    "agent_tests": ".agent_tests",
+    "agents": ".agents",
+}
+
+
+def __getattr__(attr_name: str) -> typing.Any:
+    module_name = _dynamic_imports.get(attr_name)
+    if module_name is None:
+        raise AttributeError(f"No {attr_name} found in _dynamic_imports for module name -> {__name__}")
+    try:
+        module = import_module(module_name, __package__)
+        if module_name == f".{attr_name}":
+            return module
+        else:
+            return getattr(module, attr_name)
+    except ImportError as e:
+        raise ImportError(f"Failed to import {attr_name} from {module_name}: {e}") from e
+    except AttributeError as e:
+        raise AttributeError(f"Failed to get {attr_name} from {module_name}: {e}") from e
+
+
+def __dir__():
+    lazy_attrs = list(_dynamic_imports.keys())
+    return sorted(lazy_attrs)
+
+
+__all__ = [
+    "AsyncCalibrate",
+    "BatchRunRequest",
+    "BatchTestRun",
+    "BatchTestRunResponse",
+    "BatchTestSkip",
+    "Calibrate",
+    "CalibrateEnvironment",
+    "DefaultAioHttpClient",
+    "DefaultAsyncHttpxClient",
+    "HttpValidationError",
+    "JudgeResult",
+    "ResolveAgentNamesResponse",
+    "RoutersAgentTestsAgentResponse",
+    "RoutersAgentTestsAgentResponseType",
+    "TaskCreateResponse",
+    "TestCaseResult",
+    "TestOutput",
+    "TestRunStatusResponse",
+    "ToolCallOutput",
+    "UnprocessableEntityError",
+    "ValidationError",
+    "ValidationErrorLocItem",
+    "__version__",
+    "agent_tests",
+    "agents",
+]

calibrate_python_sdk-0.0.1/src/artpark/_default_clients.py

@@ -0,0 +1,32 @@
+# This file was auto-generated by Fern from our API Definition.
+
+import typing
+
+import httpx
+
+SDK_DEFAULT_TIMEOUT = 60
+
+try:
+    import httpx_aiohttp  # type: ignore[import-not-found]
+except ImportError:
+
+    class DefaultAioHttpClient(httpx.AsyncClient):  # type: ignore
+        def __init__(self, **kwargs: typing.Any) -> None:
+            raise RuntimeError(
+                "To use the aiohttp client, install the aiohttp extra: pip install calibrate-python-sdk[aiohttp]"
+            )
+
+else:
+
+    class DefaultAioHttpClient(httpx_aiohttp.HttpxAiohttpClient):  # type: ignore
+        def __init__(self, **kwargs: typing.Any) -> None:
+            kwargs.setdefault("timeout", SDK_DEFAULT_TIMEOUT)
+            kwargs.setdefault("follow_redirects", True)
+            super().__init__(**kwargs)
+
+
+class DefaultAsyncHttpxClient(httpx.AsyncClient):
+    def __init__(self, **kwargs: typing.Any) -> None:
+        kwargs.setdefault("timeout", SDK_DEFAULT_TIMEOUT)
+        kwargs.setdefault("follow_redirects", True)
+        super().__init__(**kwargs)

calibrate_python_sdk-0.0.1/src/artpark/agent_tests/__init__.py

@@ -0,0 +1,4 @@
+# This file was auto-generated by Fern from our API Definition.
+
+# isort: skip_file
+

calibrate_python_sdk-0.0.1/src/artpark/agent_tests/client.py

@@ -0,0 +1,356 @@
+# This file was auto-generated by Fern from our API Definition.
+
+import typing
+
+from ..core.client_wrapper import AsyncClientWrapper, SyncClientWrapper
+from ..core.request_options import RequestOptions
+from ..types.batch_run_request import BatchRunRequest
+from ..types.batch_test_run_response import BatchTestRunResponse
+from ..types.task_create_response import TaskCreateResponse
+from ..types.test_run_status_response import TestRunStatusResponse
+from .raw_client import AsyncRawAgentTestsClient, RawAgentTestsClient
+
+# this is used as the default value for optional parameters
+OMIT = typing.cast(typing.Any, ...)
+
+
+class AgentTestsClient:
+    def __init__(self, *, client_wrapper: SyncClientWrapper):
+        self._raw_client = RawAgentTestsClient(client_wrapper=client_wrapper)
+
+    @property
+    def with_raw_response(self) -> RawAgentTestsClient:
+        """
+        Retrieves a raw implementation of this client that returns raw responses.
+
+        Returns
+        -------
+        RawAgentTestsClient
+        """
+        return self._raw_client
+
+    def run(
+        self,
+        agent_uuid: str,
+        *,
+        test_uuids: typing.Optional[typing.Sequence[str]] = OMIT,
+        request_options: typing.Optional[RequestOptions] = None,
+    ) -> TaskCreateResponse:
+        """
+        Run one or more tests for an agent.
+
+        This starts a background task that runs the calibrate LLM tests command
+        with the agent's config and the combined test cases from all specified tests.
+
+        Returns a task ID that can be used to poll for status and results.
+
+        Auth: requires either a JWT (frontend) or an `sk_` API key. The agent
+        must belong to the caller's org or this 404s.
+
+        Parameters
+        ----------
+        agent_uuid : str
+
+        test_uuids : typing.Optional[typing.Sequence[str]]
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        TaskCreateResponse
+            Successful Response
+
+        Examples
+        --------
+        from artpark import Calibrate
+
+        client = Calibrate(
+            api_key="YOUR_API_KEY",
+            org_uuid="YOUR_ORG_UUID",
+            token="YOUR_TOKEN",
+        )
+        client.agent_tests.run(
+            agent_uuid="agent_uuid",
+        )
+        """
+        _response = self._raw_client.run(agent_uuid, test_uuids=test_uuids, request_options=request_options)
+        return _response.data
+
+    def run_batch(
+        self,
+        *,
+        request: typing.Optional[BatchRunRequest] = None,
+        request_options: typing.Optional[RequestOptions] = None,
+    ) -> BatchTestRunResponse:
+        """
+        Run every linked test for a set of agents, one ``llm-unit-test`` job per agent.
+
+        Scope is driven by the optional ``agent_names`` payload:
+
+        - **Provided (non-empty)** — run only those agents. Names are unique per org
+          and **all are validated up front**: if any doesn't resolve to a
+          (non-deleted) agent in the caller's org, the call 404s with the offending
+          names and NO jobs are created.
+        - **Omitted / null / empty** — run every agent in the caller's org.
+
+        For each selected agent, its linked tests are launched as one job. Agents
+        with no linked tests or an unverified connection are reported under
+        ``skipped`` instead of failing the batch. Subject to the normal per-org
+        concurrency queue, so over-limit jobs come back ``queued``.
+
+        Auth accepts a JWT (frontend) or an `sk_` API key (programmatic clients).
+        Returns one ``runs`` entry per launched agent with ``agent_name``,
+        ``agent_uuid``, ``task_id``, and ``status``.
+
+        Parameters
+        ----------
+        request : typing.Optional[BatchRunRequest]
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        BatchTestRunResponse
+            Successful Response
+
+        Examples
+        --------
+        from artpark import BatchRunRequest, Calibrate
+
+        client = Calibrate(
+            api_key="YOUR_API_KEY",
+            org_uuid="YOUR_ORG_UUID",
+            token="YOUR_TOKEN",
+        )
+        client.agent_tests.run_batch(
+            request=BatchRunRequest(),
+        )
+        """
+        _response = self._raw_client.run_batch(request=request, request_options=request_options)
+        return _response.data
+
+    def get_run(
+        self, task_id: str, *, request_options: typing.Optional[RequestOptions] = None
+    ) -> TestRunStatusResponse:
+        """
+        Get the status of an agent test run.
+
+        Requires either a JWT (frontend) or an `sk_` API key, plus org
+        ownership of the run. Unauthenticated access to a completed run is only
+        possible once it is made public, via the share-token endpoint in the public
+        router.
+
+        Returns the current status and, if done, the test results.
+
+        Parameters
+        ----------
+        task_id : str
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        TestRunStatusResponse
+            Successful Response
+
+        Examples
+        --------
+        from artpark import Calibrate
+
+        client = Calibrate(
+            api_key="YOUR_API_KEY",
+            org_uuid="YOUR_ORG_UUID",
+            token="YOUR_TOKEN",
+        )
+        client.agent_tests.get_run(
+            task_id="task_id",
+        )
+        """
+        _response = self._raw_client.get_run(task_id, request_options=request_options)
+        return _response.data
+
+
+class AsyncAgentTestsClient:
+    def __init__(self, *, client_wrapper: AsyncClientWrapper):
+        self._raw_client = AsyncRawAgentTestsClient(client_wrapper=client_wrapper)
+
+    @property
+    def with_raw_response(self) -> AsyncRawAgentTestsClient:
+        """
+        Retrieves a raw implementation of this client that returns raw responses.
+
+        Returns
+        -------
+        AsyncRawAgentTestsClient
+        """
+        return self._raw_client
+
+    async def run(
+        self,
+        agent_uuid: str,
+        *,
+        test_uuids: typing.Optional[typing.Sequence[str]] = OMIT,
+        request_options: typing.Optional[RequestOptions] = None,
+    ) -> TaskCreateResponse:
+        """
+        Run one or more tests for an agent.
+
+        This starts a background task that runs the calibrate LLM tests command
+        with the agent's config and the combined test cases from all specified tests.
+
+        Returns a task ID that can be used to poll for status and results.
+
+        Auth: requires either a JWT (frontend) or an `sk_` API key. The agent
+        must belong to the caller's org or this 404s.
+
+        Parameters
+        ----------
+        agent_uuid : str
+
+        test_uuids : typing.Optional[typing.Sequence[str]]
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        TaskCreateResponse
+            Successful Response
+
+        Examples
+        --------
+        import asyncio
+
+        from artpark import AsyncCalibrate
+
+        client = AsyncCalibrate(
+            api_key="YOUR_API_KEY",
+            org_uuid="YOUR_ORG_UUID",
+            token="YOUR_TOKEN",
+        )
+
+
+        async def main() -> None:
+            await client.agent_tests.run(
+                agent_uuid="agent_uuid",
+            )
+
+
+        asyncio.run(main())
+        """
+        _response = await self._raw_client.run(agent_uuid, test_uuids=test_uuids, request_options=request_options)
+        return _response.data
+
+    async def run_batch(
+        self,
+        *,
+        request: typing.Optional[BatchRunRequest] = None,
+        request_options: typing.Optional[RequestOptions] = None,
+    ) -> BatchTestRunResponse:
+        """
+        Run every linked test for a set of agents, one ``llm-unit-test`` job per agent.
+
+        Scope is driven by the optional ``agent_names`` payload:
+
+        - **Provided (non-empty)** — run only those agents. Names are unique per org
+          and **all are validated up front**: if any doesn't resolve to a
+          (non-deleted) agent in the caller's org, the call 404s with the offending
+          names and NO jobs are created.
+        - **Omitted / null / empty** — run every agent in the caller's org.
+
+        For each selected agent, its linked tests are launched as one job. Agents
+        with no linked tests or an unverified connection are reported under
+        ``skipped`` instead of failing the batch. Subject to the normal per-org
+        concurrency queue, so over-limit jobs come back ``queued``.
+
+        Auth accepts a JWT (frontend) or an `sk_` API key (programmatic clients).
+        Returns one ``runs`` entry per launched agent with ``agent_name``,
+        ``agent_uuid``, ``task_id``, and ``status``.
+
+        Parameters
+        ----------
+        request : typing.Optional[BatchRunRequest]
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        BatchTestRunResponse
+            Successful Response
+
+        Examples
+        --------
+        import asyncio
+
+        from artpark import AsyncCalibrate, BatchRunRequest
+
+        client = AsyncCalibrate(
+            api_key="YOUR_API_KEY",
+            org_uuid="YOUR_ORG_UUID",
+            token="YOUR_TOKEN",
+        )
+
+
+        async def main() -> None:
+            await client.agent_tests.run_batch(
+                request=BatchRunRequest(),
+            )
+
+
+        asyncio.run(main())
+        """
+        _response = await self._raw_client.run_batch(request=request, request_options=request_options)
+        return _response.data
+
+    async def get_run(
+        self, task_id: str, *, request_options: typing.Optional[RequestOptions] = None
+    ) -> TestRunStatusResponse:
+        """
+        Get the status of an agent test run.
+
+        Requires either a JWT (frontend) or an `sk_` API key, plus org
+        ownership of the run. Unauthenticated access to a completed run is only
+        possible once it is made public, via the share-token endpoint in the public
+        router.
+
+        Returns the current status and, if done, the test results.
+
+        Parameters
+        ----------
+        task_id : str
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        TestRunStatusResponse
+            Successful Response
+
+        Examples
+        --------
+        import asyncio
+
+        from artpark import AsyncCalibrate
+
+        client = AsyncCalibrate(
+            api_key="YOUR_API_KEY",
+            org_uuid="YOUR_ORG_UUID",
+            token="YOUR_TOKEN",
+        )
+
+
+        async def main() -> None:
+            await client.agent_tests.get_run(
+                task_id="task_id",
+            )
+
+
+        asyncio.run(main())
+        """
+        _response = await self._raw_client.get_run(task_id, request_options=request_options)
+        return _response.data