calibrate-python-sdk 0.0.1__py3-none-any.whl

artpark/core/serialization.py

@@ -0,0 +1,347 @@
+# This file was auto-generated by Fern from our API Definition.
+
+import collections
+import inspect
+import typing
+
+import pydantic
+import typing_extensions
+
+
+class FieldMetadata:
+    """
+    Metadata class used to annotate fields to provide additional information.
+
+    Example:
+    class MyDict(TypedDict):
+        field: typing.Annotated[str, FieldMetadata(alias="field_name")]
+
+    Will serialize: `{"field": "value"}`
+    To: `{"field_name": "value"}`
+    """
+
+    alias: str
+
+    def __init__(self, *, alias: str) -> None:
+        self.alias = alias
+
+
+# Resolving type hints (typing.get_type_hints) is expensive because it eval/compiles
+# forward-reference annotations. The result is constant for a given type, so we cache it.
+# This is critical for hot paths like SSE event parsing, where the same (often large
+# discriminated-union) type is converted on every single event.
+_type_hints_cache: typing.Dict[typing.Any, typing.Dict[str, typing.Any]] = {}
+
+
+def _get_cached_type_hints(expected_type: typing.Any) -> typing.Dict[str, typing.Any]:
+    try:
+        cached = _type_hints_cache.get(expected_type)
+    except TypeError:
+        # Unhashable type; resolve without caching.
+        return _resolve_type_hints(expected_type)
+    if cached is None:
+        cached = _resolve_type_hints(expected_type)
+        _type_hints_cache[expected_type] = cached
+    return cached
+
+
+def _resolve_type_hints(expected_type: typing.Any) -> typing.Dict[str, typing.Any]:
+    try:
+        return typing_extensions.get_type_hints(expected_type, include_extras=True)
+    except NameError:
+        # The type contains a circular reference, so we use the __annotations__ attribute directly.
+        return getattr(expected_type, "__annotations__", {})
+
+
+# Whether convert_and_respect_annotation_metadata can possibly rewrite anything for a given
+# annotation, i.e. whether any reachable model/TypedDict field carries a FieldMetadata alias.
+# This is constant per type, so we cache it and use it to short-circuit the recursive walk.
+_requires_conversion_cache: typing.Dict[typing.Any, bool] = {}
+
+
+def _requires_conversion(type_: typing.Any) -> bool:
+    try:
+        cached = _requires_conversion_cache.get(type_)
+    except TypeError:
+        # Unhashable annotation; compute without caching.
+        return _compute_requires_conversion(type_, set())
+    if cached is None:
+        cached = _compute_requires_conversion(type_, set())
+        _requires_conversion_cache[type_] = cached
+    return cached
+
+
+def _compute_requires_conversion(type_: typing.Any, seen: typing.Set[typing.Any]) -> bool:
+    clean_type = _remove_annotations(type_)
+
+    try:
+        if clean_type in seen:
+            return False
+        seen = seen | {clean_type}
+    except TypeError:
+        # Unhashable type; skip cycle tracking (the type graph is finite in practice).
+        pass
+
+    # Models / TypedDicts: a field alias here means we must dealias; otherwise recurse into fields.
+    if (inspect.isclass(clean_type) and issubclass(clean_type, pydantic.BaseModel)) or typing_extensions.is_typeddict(
+        clean_type
+    ):
+        annotations = _get_cached_type_hints(clean_type)
+        if _get_alias_to_field_name(annotations):
+            return True
+        return any(_compute_requires_conversion(hint, seen) for hint in annotations.values())
+
+    # Containers / unions: recurse into the type arguments (List/Set/Sequence/Dict/Union/etc.).
+    return any(_compute_requires_conversion(arg, seen) for arg in typing_extensions.get_args(clean_type))
+
+
+def convert_and_respect_annotation_metadata(
+    *,
+    object_: typing.Any,
+    annotation: typing.Any,
+    inner_type: typing.Optional[typing.Any] = None,
+    direction: typing.Literal["read", "write"],
+) -> typing.Any:
+    """
+    Respect the metadata annotations on a field, such as aliasing. This function effectively
+    manipulates the dict-form of an object to respect the metadata annotations. This is primarily used for
+    TypedDicts, which cannot support aliasing out of the box, and can be extended for additional
+    utilities, such as defaults.
+
+    Parameters
+    ----------
+    object_ : typing.Any
+
+    annotation : type
+        The type we're looking to apply typing annotations from
+
+    inner_type : typing.Optional[type]
+
+    Returns
+    -------
+    typing.Any
+    """
+
+    if object_ is None:
+        return None
+    if inner_type is None:
+        inner_type = annotation
+        # The only thing this function ever rewrites is keys that carry a FieldMetadata
+        # alias. If nothing in the (cached) type graph has such an alias, the conversion is
+        # a content-identity transform, so we can skip the entire recursive walk. This is
+        # the hot path for SSE streaming, where a large discriminated union would otherwise
+        # be traversed on every single event.
+        if not _requires_conversion(annotation):
+            return object_
+
+    clean_type = _remove_annotations(inner_type)
+    # Pydantic models
+    if (
+        inspect.isclass(clean_type)
+        and issubclass(clean_type, pydantic.BaseModel)
+        and isinstance(object_, typing.Mapping)
+    ):
+        return _convert_mapping(object_, clean_type, direction)
+    # TypedDicts
+    if typing_extensions.is_typeddict(clean_type) and isinstance(object_, typing.Mapping):
+        return _convert_mapping(object_, clean_type, direction)
+
+    if (
+        typing_extensions.get_origin(clean_type) == typing.Dict
+        or typing_extensions.get_origin(clean_type) == dict
+        or clean_type == typing.Dict
+    ) and isinstance(object_, typing.Dict):
+        key_type = typing_extensions.get_args(clean_type)[0]
+        value_type = typing_extensions.get_args(clean_type)[1]
+
+        return {
+            key: convert_and_respect_annotation_metadata(
+                object_=value,
+                annotation=annotation,
+                inner_type=value_type,
+                direction=direction,
+            )
+            for key, value in object_.items()
+        }
+
+    # If you're iterating on a string, do not bother to coerce it to a sequence.
+    if not isinstance(object_, str):
+        if (
+            typing_extensions.get_origin(clean_type) == typing.Set
+            or typing_extensions.get_origin(clean_type) == set
+            or clean_type == typing.Set
+        ) and isinstance(object_, typing.Set):
+            inner_type = typing_extensions.get_args(clean_type)[0]
+            return {
+                convert_and_respect_annotation_metadata(
+                    object_=item,
+                    annotation=annotation,
+                    inner_type=inner_type,
+                    direction=direction,
+                )
+                for item in object_
+            }
+        elif (
+            (
+                typing_extensions.get_origin(clean_type) == typing.List
+                or typing_extensions.get_origin(clean_type) == list
+                or clean_type == typing.List
+            )
+            and isinstance(object_, typing.List)
+        ) or (
+            (
+                typing_extensions.get_origin(clean_type) == typing.Sequence
+                or typing_extensions.get_origin(clean_type) == collections.abc.Sequence
+                or clean_type == typing.Sequence
+            )
+            and isinstance(object_, typing.Sequence)
+        ):
+            inner_type = typing_extensions.get_args(clean_type)[0]
+            return [
+                convert_and_respect_annotation_metadata(
+                    object_=item,
+                    annotation=annotation,
+                    inner_type=inner_type,
+                    direction=direction,
+                )
+                for item in object_
+            ]
+
+    if typing_extensions.get_origin(clean_type) == typing.Union:
+        # We should be able to ~relatively~ safely try to convert keys against all
+        # member types in the union, the edge case here is if one member aliases a field
+        # of the same name to a different name from another member
+        # Or if another member aliases a field of the same name that another member does not.
+        for member in typing_extensions.get_args(clean_type):
+            object_ = convert_and_respect_annotation_metadata(
+                object_=object_,
+                annotation=annotation,
+                inner_type=member,
+                direction=direction,
+            )
+        return object_
+
+    annotated_type = _get_annotation(annotation)
+    if annotated_type is None:
+        return object_
+
+    # If the object is not a TypedDict, a Union, or other container (list, set, sequence, etc.)
+    # Then we can safely call it on the recursive conversion.
+    return object_
+
+
+def _convert_mapping(
+    object_: typing.Mapping[str, object],
+    expected_type: typing.Any,
+    direction: typing.Literal["read", "write"],
+) -> typing.Mapping[str, object]:
+    converted_object: typing.Dict[str, object] = {}
+    annotations = _get_cached_type_hints(expected_type)
+    aliases_to_field_names = _get_alias_to_field_name(annotations)
+    for key, value in object_.items():
+        if direction == "read" and key in aliases_to_field_names:
+            dealiased_key = aliases_to_field_names.get(key)
+            if dealiased_key is not None:
+                type_ = annotations.get(dealiased_key)
+        else:
+            type_ = annotations.get(key)
+        # Note you can't get the annotation by the field name if you're in read mode, so you must check the aliases map
+        #
+        # So this is effectively saying if we're in write mode, and we don't have a type, or if we're in read mode and we don't have an alias
+        # then we can just pass the value through as is
+        if type_ is None:
+            converted_object[key] = value
+        elif direction == "read" and key not in aliases_to_field_names:
+            converted_object[key] = convert_and_respect_annotation_metadata(
+                object_=value, annotation=type_, direction=direction
+            )
+        else:
+            converted_object[_alias_key(key, type_, direction, aliases_to_field_names)] = (
+                convert_and_respect_annotation_metadata(object_=value, annotation=type_, direction=direction)
+            )
+    return converted_object
+
+
+def _get_annotation(type_: typing.Any) -> typing.Optional[typing.Any]:
+    maybe_annotated_type = typing_extensions.get_origin(type_)
+    if maybe_annotated_type is None:
+        return None
+
+    if maybe_annotated_type == typing_extensions.NotRequired:
+        type_ = typing_extensions.get_args(type_)[0]
+        maybe_annotated_type = typing_extensions.get_origin(type_)
+
+    if maybe_annotated_type == typing_extensions.Annotated:
+        return type_
+
+    return None
+
+
+def _remove_annotations(type_: typing.Any) -> typing.Any:
+    maybe_annotated_type = typing_extensions.get_origin(type_)
+    if maybe_annotated_type is None:
+        return type_
+
+    if maybe_annotated_type == typing_extensions.NotRequired:
+        return _remove_annotations(typing_extensions.get_args(type_)[0])
+
+    if maybe_annotated_type == typing_extensions.Annotated:
+        return _remove_annotations(typing_extensions.get_args(type_)[0])
+
+    return type_
+
+
+def get_alias_to_field_mapping(type_: typing.Any) -> typing.Dict[str, str]:
+    annotations = _get_cached_type_hints(type_)
+    return _get_alias_to_field_name(annotations)
+
+
+def get_field_to_alias_mapping(type_: typing.Any) -> typing.Dict[str, str]:
+    annotations = _get_cached_type_hints(type_)
+    return _get_field_to_alias_name(annotations)
+
+
+def _get_alias_to_field_name(
+    field_to_hint: typing.Dict[str, typing.Any],
+) -> typing.Dict[str, str]:
+    aliases = {}
+    for field, hint in field_to_hint.items():
+        maybe_alias = _get_alias_from_type(hint)
+        if maybe_alias is not None:
+            aliases[maybe_alias] = field
+    return aliases
+
+
+def _get_field_to_alias_name(
+    field_to_hint: typing.Dict[str, typing.Any],
+) -> typing.Dict[str, str]:
+    aliases = {}
+    for field, hint in field_to_hint.items():
+        maybe_alias = _get_alias_from_type(hint)
+        if maybe_alias is not None:
+            aliases[field] = maybe_alias
+    return aliases
+
+
+def _get_alias_from_type(type_: typing.Any) -> typing.Optional[str]:
+    maybe_annotated_type = _get_annotation(type_)
+
+    if maybe_annotated_type is not None:
+        # The actual annotations are 1 onward, the first is the annotated type
+        annotations = typing_extensions.get_args(maybe_annotated_type)[1:]
+
+        for annotation in annotations:
+            if isinstance(annotation, FieldMetadata) and annotation.alias is not None:
+                return annotation.alias
+    return None
+
+
+def _alias_key(
+    key: str,
+    type_: typing.Any,
+    direction: typing.Literal["read", "write"],
+    aliases_to_field_names: typing.Dict[str, str],
+) -> str:
+    if direction == "read":
+        return aliases_to_field_names.get(key, key)
+    return _get_alias_from_type(type_=type_) or key

artpark/environment.py

@@ -0,0 +1,7 @@
+# This file was auto-generated by Fern from our API Definition.
+
+import enum
+
+
+class CalibrateEnvironment(enum.Enum):
+    PRODUCTION = "https://pense-backend.artpark.ai"

artpark/errors/__init__.py

@@ -0,0 +1,34 @@
+# This file was auto-generated by Fern from our API Definition.
+
+# isort: skip_file
+
+import typing
+from importlib import import_module
+
+if typing.TYPE_CHECKING:
+    from .unprocessable_entity_error import UnprocessableEntityError
+_dynamic_imports: typing.Dict[str, str] = {"UnprocessableEntityError": ".unprocessable_entity_error"}
+
+
+def __getattr__(attr_name: str) -> typing.Any:
+    module_name = _dynamic_imports.get(attr_name)
+    if module_name is None:
+        raise AttributeError(f"No {attr_name} found in _dynamic_imports for module name -> {__name__}")
+    try:
+        module = import_module(module_name, __package__)
+        if module_name == f".{attr_name}":
+            return module
+        else:
+            return getattr(module, attr_name)
+    except ImportError as e:
+        raise ImportError(f"Failed to import {attr_name} from {module_name}: {e}") from e
+    except AttributeError as e:
+        raise AttributeError(f"Failed to get {attr_name} from {module_name}: {e}") from e
+
+
+def __dir__():
+    lazy_attrs = list(_dynamic_imports.keys())
+    return sorted(lazy_attrs)
+
+
+__all__ = ["UnprocessableEntityError"]

artpark/errors/unprocessable_entity_error.py

@@ -0,0 +1,11 @@
+# This file was auto-generated by Fern from our API Definition.
+
+import typing
+
+from ..core.api_error import ApiError
+from ..types.http_validation_error import HttpValidationError
+
+
+class UnprocessableEntityError(ApiError):
+    def __init__(self, body: HttpValidationError, headers: typing.Optional[typing.Dict[str, str]] = None):
+        super().__init__(status_code=422, headers=headers, body=body)

artpark/types/__init__.py

@@ -0,0 +1,83 @@
+# This file was auto-generated by Fern from our API Definition.
+
+# isort: skip_file
+
+import typing
+from importlib import import_module
+
+if typing.TYPE_CHECKING:
+    from .batch_run_request import BatchRunRequest
+    from .batch_test_run import BatchTestRun
+    from .batch_test_run_response import BatchTestRunResponse
+    from .batch_test_skip import BatchTestSkip
+    from .http_validation_error import HttpValidationError
+    from .judge_result import JudgeResult
+    from .resolve_agent_names_response import ResolveAgentNamesResponse
+    from .routers_agent_tests_agent_response import RoutersAgentTestsAgentResponse
+    from .routers_agent_tests_agent_response_type import RoutersAgentTestsAgentResponseType
+    from .task_create_response import TaskCreateResponse
+    from .test_case_result import TestCaseResult
+    from .test_output import TestOutput
+    from .test_run_status_response import TestRunStatusResponse
+    from .tool_call_output import ToolCallOutput
+    from .validation_error import ValidationError
+    from .validation_error_loc_item import ValidationErrorLocItem
+_dynamic_imports: typing.Dict[str, str] = {
+    "BatchRunRequest": ".batch_run_request",
+    "BatchTestRun": ".batch_test_run",
+    "BatchTestRunResponse": ".batch_test_run_response",
+    "BatchTestSkip": ".batch_test_skip",
+    "HttpValidationError": ".http_validation_error",
+    "JudgeResult": ".judge_result",
+    "ResolveAgentNamesResponse": ".resolve_agent_names_response",
+    "RoutersAgentTestsAgentResponse": ".routers_agent_tests_agent_response",
+    "RoutersAgentTestsAgentResponseType": ".routers_agent_tests_agent_response_type",
+    "TaskCreateResponse": ".task_create_response",
+    "TestCaseResult": ".test_case_result",
+    "TestOutput": ".test_output",
+    "TestRunStatusResponse": ".test_run_status_response",
+    "ToolCallOutput": ".tool_call_output",
+    "ValidationError": ".validation_error",
+    "ValidationErrorLocItem": ".validation_error_loc_item",
+}
+
+
+def __getattr__(attr_name: str) -> typing.Any:
+    module_name = _dynamic_imports.get(attr_name)
+    if module_name is None:
+        raise AttributeError(f"No {attr_name} found in _dynamic_imports for module name -> {__name__}")
+    try:
+        module = import_module(module_name, __package__)
+        if module_name == f".{attr_name}":
+            return module
+        else:
+            return getattr(module, attr_name)
+    except ImportError as e:
+        raise ImportError(f"Failed to import {attr_name} from {module_name}: {e}") from e
+    except AttributeError as e:
+        raise AttributeError(f"Failed to get {attr_name} from {module_name}: {e}") from e
+
+
+def __dir__():
+    lazy_attrs = list(_dynamic_imports.keys())
+    return sorted(lazy_attrs)
+
+
+__all__ = [
+    "BatchRunRequest",
+    "BatchTestRun",
+    "BatchTestRunResponse",
+    "BatchTestSkip",
+    "HttpValidationError",
+    "JudgeResult",
+    "ResolveAgentNamesResponse",
+    "RoutersAgentTestsAgentResponse",
+    "RoutersAgentTestsAgentResponseType",
+    "TaskCreateResponse",
+    "TestCaseResult",
+    "TestOutput",
+    "TestRunStatusResponse",
+    "ToolCallOutput",
+    "ValidationError",
+    "ValidationErrorLocItem",
+]

artpark/types/batch_run_request.py

@@ -0,0 +1,19 @@
+# This file was auto-generated by Fern from our API Definition.
+
+import typing
+
+import pydantic
+from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel
+
+
+class BatchRunRequest(UniversalBaseModel):
+    agent_names: typing.Optional[typing.List[str]] = None
+
+    if IS_PYDANTIC_V2:
+        model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True)  # type: ignore # Pydantic v2
+    else:
+
+        class Config:
+            frozen = True
+            smart_union = True
+            extra = pydantic.Extra.allow

artpark/types/batch_test_run.py

@@ -0,0 +1,22 @@
+# This file was auto-generated by Fern from our API Definition.
+
+import typing
+
+import pydantic
+from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel
+
+
+class BatchTestRun(UniversalBaseModel):
+    agent_name: str
+    agent_uuid: str
+    task_id: str
+    status: str
+
+    if IS_PYDANTIC_V2:
+        model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True)  # type: ignore # Pydantic v2
+    else:
+
+        class Config:
+            frozen = True
+            smart_union = True
+            extra = pydantic.Extra.allow

artpark/types/batch_test_run_response.py

@@ -0,0 +1,22 @@
+# This file was auto-generated by Fern from our API Definition.
+
+import typing
+
+import pydantic
+from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel
+from .batch_test_run import BatchTestRun
+from .batch_test_skip import BatchTestSkip
+
+
+class BatchTestRunResponse(UniversalBaseModel):
+    runs: typing.List[BatchTestRun]
+    skipped: typing.Optional[typing.List[BatchTestSkip]] = None
+
+    if IS_PYDANTIC_V2:
+        model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True)  # type: ignore # Pydantic v2
+    else:
+
+        class Config:
+            frozen = True
+            smart_union = True
+            extra = pydantic.Extra.allow

artpark/types/batch_test_skip.py

@@ -0,0 +1,21 @@
+# This file was auto-generated by Fern from our API Definition.
+
+import typing
+
+import pydantic
+from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel
+
+
+class BatchTestSkip(UniversalBaseModel):
+    agent_name: str
+    agent_uuid: str
+    reason: str
+
+    if IS_PYDANTIC_V2:
+        model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True)  # type: ignore # Pydantic v2
+    else:
+
+        class Config:
+            frozen = True
+            smart_union = True
+            extra = pydantic.Extra.allow

artpark/types/http_validation_error.py

@@ -0,0 +1,20 @@
+# This file was auto-generated by Fern from our API Definition.
+
+import typing
+
+import pydantic
+from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel
+from .validation_error import ValidationError
+
+
+class HttpValidationError(UniversalBaseModel):
+    detail: typing.Optional[typing.List[ValidationError]] = None
+
+    if IS_PYDANTIC_V2:
+        model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True)  # type: ignore # Pydantic v2
+    else:
+
+        class Config:
+            frozen = True
+            smart_union = True
+            extra = pydantic.Extra.allow

artpark/types/judge_result.py

@@ -0,0 +1,51 @@
+# This file was auto-generated by Fern from our API Definition.
+
+import typing
+
+import pydantic
+from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel
+
+
+class JudgeResult(UniversalBaseModel):
+    """
+    One evaluator's verdict for a response-type test case.
+
+    Per-row data only — anything constant across rows for the same evaluator
+    (name, description, output_type, output_config, scale_min/max) lives on
+    the response-level `evaluators[]` block; look it up by `evaluator_uuid`.
+
+    `evaluator_uuid` is None for legacy runs that pre-date the evaluator-
+    snapshot capture or when the evaluator can't be resolved from the
+    snapshot.
+
+    Exactly one of `match` (binary) / `score` (rating) is set per entry;
+    both are None for tool-call tests, but tool-call tests don't carry
+    `judge_results`.
+
+    `variable_values` are the {{var}} substitutions used for this evaluator
+    on this test case, frozen from `test_evaluators.variable_values` at
+    submission time — stays on the row because it varies per test case.
+
+    `value_name` is the human-readable label for `match`/`score` resolved
+    against the rubric the run actually used (snapshot's
+    `output_config.scale.name`). Falls back to `Correct`/`Wrong` for binary
+    or the stringified score for rating when the snapshot lacks named
+    scale entries (e.g. legacy runs captured before the rubric was
+    snapshotted).
+    """
+
+    evaluator_uuid: typing.Optional[str] = None
+    reasoning: typing.Optional[str] = None
+    match: typing.Optional[bool] = None
+    score: typing.Optional[float] = None
+    value_name: typing.Optional[str] = None
+    variable_values: typing.Optional[typing.Dict[str, typing.Any]] = None
+
+    if IS_PYDANTIC_V2:
+        model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True)  # type: ignore # Pydantic v2
+    else:
+
+        class Config:
+            frozen = True
+            smart_union = True
+            extra = pydantic.Extra.allow

artpark/types/resolve_agent_names_response.py

@@ -0,0 +1,20 @@
+# This file was auto-generated by Fern from our API Definition.
+
+import typing
+
+import pydantic
+from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel
+
+
+class ResolveAgentNamesResponse(UniversalBaseModel):
+    resolved: typing.Dict[str, str]
+    not_found: typing.List[str]
+
+    if IS_PYDANTIC_V2:
+        model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True)  # type: ignore # Pydantic v2
+    else:
+
+        class Config:
+            frozen = True
+            smart_union = True
+            extra = pydantic.Extra.allow