agenta 0.52.6__py3-none-any.whl → 0.63.2__py3-none-any.whl

agenta/sdk/models/evaluations.py

@@ -0,0 +1,119 @@
+from typing import Dict, List, Optional, Union, Literal, Callable, Any
+from enum import Enum
+from uuid import UUID
+from datetime import datetime
+
+from pydantic import BaseModel
+
+from agenta.sdk.models.shared import (
+    TraceID,
+    SpanID,
+    Link,
+    Identifier,
+    Slug,
+    Version,
+    Reference,
+    Lifecycle,
+    Header,
+    Flags,
+    Tags,
+    Meta,
+    Metadata,
+    Data,
+    Commit,
+    AliasConfig,
+    sync_alias,
+)
+
+
+# ------------------------------------------------------------------------------
+
+
+Origin = Literal["custom", "human", "auto"]
+# Target = Union[List[UUID], Dict[UUID, Origin], List[Callable]]
+Target = Union[
+    List[List[Dict[str, Any]]],  # testcases_data
+    List[Callable],  # workflow_handlers
+    List[UUID],  # entity_ids
+    Dict[UUID, Origin],  # entity_ids with origins
+]
+
+
+# oss.src.core.evaluations.types
+
+
+class EvaluationStatus(str, Enum):
+    PENDING = "pending"
+    QUEUED = "queued"
+    RUNNING = "running"
+    SUCCESS = "success"
+    FAILURE = "failure"
+    ERRORS = "errors"
+    CANCELLED = "cancelled"
+
+
+class EvaluationRunFlags(BaseModel):
+    is_closed: Optional[bool] = None  # Indicates if the run is immutable
+    is_live: Optional[bool] = None  # Indicates if the run is updated periodically
+    is_active: Optional[bool] = None  # Indicates if the run is currently active
+
+
+class SimpleEvaluationFlags(EvaluationRunFlags):
+    pass
+
+
+SimpleEvaluationStatus = EvaluationStatus
+
+
+class SimpleEvaluationData(BaseModel):
+    status: Optional[SimpleEvaluationStatus] = None
+
+    query_steps: Optional[Target] = None
+    testset_steps: Optional[Target] = None
+    application_steps: Optional[Target] = None
+    evaluator_steps: Optional[Target] = None
+
+    repeats: Optional[int] = None
+
+
+class EvaluationRun(BaseModel):
+    id: UUID
+
+
+class EvaluationScenario(BaseModel):
+    id: UUID
+
+    run_id: UUID
+
+
+class EvaluationResult(BaseModel):
+    id: UUID
+
+    run_id: UUID
+    scenario_id: UUID
+    step_key: str
+
+    testcase_id: Optional[UUID] = None
+    trace_id: Optional[UUID] = None
+    error: Optional[dict] = None
+
+    flags: Optional[Dict[str, Any]] = None
+    tags: Optional[Dict[str, Any]] = None
+    meta: Optional[Dict[str, Any]] = None
+
+
+class EvaluationMetrics(Identifier, Lifecycle):
+    flags: Optional[Dict[str, Any]] = None
+    tags: Optional[Dict[str, Any]] = None
+    meta: Optional[Dict[str, Any]] = None
+
+    status: Optional[EvaluationStatus] = None
+
+    timestamp: Optional[datetime] = None
+    interval: Optional[int] = None
+
+    data: Optional[Data] = None
+
+    scenario_id: Optional[UUID] = None
+
+    run_id: UUID

agenta/sdk/models/git.py

@@ -0,0 +1,126 @@
+from typing import List, Optional
+from uuid import UUID
+
+from pydantic import BaseModel, Field
+
+from agenta.sdk.models.shared import (
+    TraceID,
+    SpanID,
+    Link,
+    Identifier,
+    Slug,
+    Version,
+    Reference,
+    Lifecycle,
+    Header,
+    Flags,
+    Tags,
+    Meta,
+    Metadata,
+    Data,
+    Commit,
+    AliasConfig,
+    sync_alias,
+)
+
+
+from typing import Optional, List
+from uuid import UUID
+
+from pydantic import BaseModel
+
+
+# artifacts --------------------------------------------------------------------
+
+
+class Artifact(Identifier, Slug, Lifecycle, Header, Metadata):
+    pass
+
+
+class ArtifactCreate(Slug, Header, Metadata):
+    pass
+
+
+class ArtifactEdit(Identifier, Header, Metadata):
+    pass
+
+
+class ArtifactQuery(Metadata):
+    pass
+
+
+# variants ---------------------------------------------------------------------
+
+
+class Variant(Identifier, Slug, Lifecycle, Header, Metadata):
+    artifact_id: Optional[UUID] = None
+
+
+class VariantCreate(Slug, Header, Metadata):
+    artifact_id: Optional[UUID] = None
+
+
+class VariantEdit(Identifier, Header, Metadata):
+    pass
+
+
+class VariantQuery(Metadata):
+    pass
+
+
+# revisions --------------------------------------------------------------------
+
+
+class Revision(Identifier, Slug, Version, Lifecycle, Header, Metadata, Commit):
+    data: Optional[Data] = None
+
+    artifact_id: Optional[UUID] = None
+    variant_id: Optional[UUID] = None
+
+
+class RevisionCreate(Slug, Header, Metadata):
+    artifact_id: Optional[UUID] = None
+    variant_id: Optional[UUID] = None
+
+
+class RevisionEdit(Identifier, Header, Metadata):
+    pass
+
+
+class RevisionQuery(Metadata):
+    authors: Optional[List[UUID]] = None
+
+
+class RevisionCommit(Slug, Header, Metadata):
+    data: Optional[Data] = None
+
+    message: Optional[str] = None
+
+    artifact_id: Optional[UUID] = None
+    variant_id: Optional[UUID] = None
+
+
+class RevisionsLog(BaseModel):
+    artifact_id: Optional[UUID] = None
+    variant_id: Optional[UUID] = None
+    revision_id: Optional[UUID] = None
+
+    depth: Optional[int] = None
+
+
+# forks ------------------------------------------------------------------------
+
+
+class RevisionFork(Slug, Header, Metadata):
+    data: Optional[Data] = None
+
+    message: Optional[str] = None
+
+
+class VariantFork(Slug, Header, Metadata):
+    pass
+
+
+class ArtifactFork(RevisionsLog):
+    variant: Optional[VariantFork] = None
+    revision: Optional[RevisionFork] = None

agenta/sdk/models/shared.py

@@ -0,0 +1,167 @@
+from typing import Optional, Dict, List, Union, Literal
+from typing_extensions import TypeAliasType
+from datetime import datetime
+from uuid import UUID
+from re import match
+
+from pydantic import BaseModel, field_validator
+
+BoolJson = TypeAliasType(  # type: ignore
+    "BoolJson",
+    Union[bool, Dict[str, "BoolJson"]],  # type: ignore
+)
+
+StringJson = TypeAliasType(  # type: ignore
+    "StringJson",
+    Union[str, Dict[str, "StringJson"]],  # type: ignore
+)
+
+FullJson = TypeAliasType(  # type: ignore
+    "FullJson",
+    Union[str, int, float, bool, None, Dict[str, "FullJson"], List["FullJson"]],  # type: ignore
+)
+
+NumericJson = TypeAliasType(  # type: ignore
+    "NumericJson",
+    Union[int, float, Dict[str, "NumericJson"]],  # type: ignore
+)
+
+NoListJson = TypeAliasType(  # type: ignore
+    "NoListJson",
+    Union[str, int, float, bool, None, Dict[str, "NoListJson"]],  # type: ignore
+)
+
+LabelJson = TypeAliasType(  # type: ignore
+    "LabelJson",
+    Union[bool, str, Dict[str, "LabelJson"]],  # type: ignore
+)
+
+Json = Dict[str, FullJson]  # type: ignore
+
+Data = Dict[str, FullJson]  # type: ignore
+
+Flags = Dict[str, LabelJson]  # type: ignore
+
+Tags = Dict[str, LabelJson]  # type: ignore
+
+Meta = Dict[str, FullJson]  # type: ignore
+
+Hashes = Dict[str, StringJson]  # type: ignore
+
+Metrics = Dict[str, NumericJson]  # type: ignore
+
+Schema = Dict[str, FullJson]  # type: ignore
+
+Mappings = Dict[str, str]
+
+
+class Lifecycle(BaseModel):
+    created_at: Optional[datetime] = None
+    updated_at: Optional[datetime] = None
+    deleted_at: Optional[datetime] = None
+
+    created_by_id: Optional[UUID] = None
+    updated_by_id: Optional[UUID] = None
+    deleted_by_id: Optional[UUID] = None
+
+
+class TraceID(BaseModel):
+    trace_id: Optional[str] = None
+
+
+class SpanID(BaseModel):
+    span_id: Optional[str] = None
+
+
+class Link(TraceID, SpanID):
+    pass
+
+
+class Identifier(BaseModel):
+    id: Optional[UUID] = None
+
+
+class Slug(BaseModel):
+    slug: Optional[str] = None
+
+    @field_validator("slug")
+    def check_url_safety(cls, v):
+        if v is not None:
+            if not match(r"^[a-zA-Z0-9_-]+$", v):
+                raise ValueError("slug must be URL-safe.")
+        return v
+
+
+class Version(BaseModel):
+    version: Optional[str] = None
+
+
+class Reference(Identifier, Slug, Version):
+    pass
+
+
+class Header(BaseModel):
+    name: Optional[str] = None
+    description: Optional[str] = None
+
+
+class Metadata(BaseModel):
+    flags: Optional[Flags] = None
+    tags: Optional[Tags] = None
+    meta: Optional[Meta] = None
+
+
+class Commit(BaseModel):
+    author: Optional[UUID] = None
+    date: Optional[datetime] = None
+    message: Optional[str] = None
+
+
+class Status(BaseModel):
+    code: Optional[int] = 200
+    message: Optional[str] = "Success"
+
+
+class AliasConfig(BaseModel):
+    model_config = {
+        "populate_by_name": True,
+        "from_attributes": True,
+    }
+
+
+def sync_alias(primary: str, alias: str, instance: BaseModel) -> None:
+    primary_val = getattr(instance, primary)
+    alias_val = getattr(instance, alias)
+
+    if primary_val and alias_val is None:
+        object.__setattr__(instance, alias, primary_val)
+    elif alias_val and primary_val is None:
+        object.__setattr__(instance, primary, alias_val)
+
+
+class Windowing(BaseModel):
+    # RANGE
+    newest: Optional[datetime] = None
+    oldest: Optional[datetime] = None
+    # TOKEN
+    next: Optional[UUID] = None
+    # LIMIT
+    limit: Optional[int] = None
+    # ORDER
+    order: Optional[Literal["ascending", "descending"]] = None
+    # BUCKETS
+    interval: Optional[int] = None
+    # SAMPLES
+    rate: Optional[float] = None
+
+    @field_validator("rate")
+    def check_rate(cls, v):
+        if v is not None and (v < 0.0 or v > 1.0):
+            raise ValueError("Sampling rate must be between 0.0 and 1.0.")
+        return v
+
+    @field_validator("interval")
+    def check_interval(cls, v):
+        if v is not None and v <= 0:
+            raise ValueError("Bucket interval must be a positive integer.")
+        return v

agenta/sdk/models/testsets.py

@@ -0,0 +1,163 @@
+from typing import List, Optional, Dict, Any
+from uuid import UUID
+
+from pydantic import BaseModel, Field
+
+from agenta.sdk.models.shared import (
+    TraceID,
+    SpanID,
+    Link,
+    Identifier,
+    Slug,
+    Version,
+    Reference,
+    Lifecycle,
+    Header,
+    Flags,
+    Tags,
+    Meta,
+    Metadata,
+    Data,
+    Commit,
+    AliasConfig,
+    sync_alias,
+)
+
+from agenta.sdk.models.git import (
+    Artifact,
+    ArtifactCreate,
+    ArtifactEdit,
+    ArtifactQuery,
+    ArtifactFork,
+    Variant,
+    VariantCreate,
+    VariantEdit,
+    VariantQuery,
+    VariantFork,
+    Revision,
+    RevisionCreate,
+    RevisionEdit,
+    RevisionQuery,
+    RevisionCommit,
+    RevisionsLog,
+    RevisionFork,
+)
+
+from agenta.sdk.models.blobs import (
+    Blob,
+)
+
+
+class TestsetIdAlias(AliasConfig):
+    testset_id: Optional[UUID] = None
+    set_id: Optional[UUID] = Field(
+        default=None,
+        exclude=True,
+        alias="testset_id",
+    )
+
+
+class TestsetVariantIdAlias(AliasConfig):
+    testset_variant_id: Optional[UUID] = None
+    variant_id: Optional[UUID] = Field(
+        default=None,
+        exclude=True,
+        alias="testset_variant_id",
+    )
+
+
+class Testcase(Blob, TestsetIdAlias):
+    def model_post_init(self, __context) -> None:
+        sync_alias("testset_id", "set_id", self)
+
+
+class TestsetFlags(BaseModel):
+    has_testcases: Optional[bool] = None
+    has_traces: Optional[bool] = None
+
+
+class TestsetRevisionData(BaseModel):
+    testcase_ids: Optional[List[UUID]] = None
+    testcases: Optional[List[Testcase]] = None
+
+
+class SimpleTestset(
+    Identifier,
+    Slug,
+    Lifecycle,
+    Header,
+):
+    flags: Optional[TestsetFlags] = None
+    tags: Optional[Tags] = None  # type: ignore
+    meta: Optional[Meta] = None  # type: ignore
+
+    data: Optional[TestsetRevisionData] = None
+
+
+class Testset(Artifact):
+    flags: Optional[TestsetFlags] = None  # type: ignore
+
+
+class TestsetRevision(
+    Revision,
+    TestsetIdAlias,
+    TestsetVariantIdAlias,
+):
+    flags: Optional[TestsetFlags] = None  # type: ignore
+
+    data: Optional[TestsetRevisionData] = None  # type: ignore
+
+    def model_post_init(self, __context) -> None:
+        sync_alias("testset_id", "artifact_id", self)
+        sync_alias("testset_variant_id", "variant_id", self)
+
+
+class SimpleTestsetCreate(Slug, Header):
+    tags: Optional[Tags] = None  # type: ignore
+    meta: Optional[Meta] = None  # type: ignore
+    data: Optional[TestsetRevisionData] = None
+
+
+class SimpleTestsetEdit(
+    Identifier,
+    Header,
+):
+    # flags: Optional[TestsetFlags] = None
+    tags: Optional[Tags] = None  # type: ignore
+    meta: Optional[Meta] = None  # type: ignore
+
+    data: Optional[TestsetRevisionData] = None
+
+
+class TestsetResponse(BaseModel):
+    count: int = 0
+    testset: Optional[Testset] = None
+
+
+class TestsetRevisionResponse(BaseModel):
+    count: int = 0
+    testset_revision: Optional[TestsetRevision] = None
+
+
+class SimpleTestsetResponse(BaseModel):
+    count: int = 0
+    testset: Optional[SimpleTestset] = None
+
+
+class TestsetsResponse(BaseModel):
+    count: int = 0
+    testsets: List[Testset] = []
+
+
+class SimpleTestsetsResponse(BaseModel):
+    count: int = 0
+    testsets: List[SimpleTestset] = []
+
+
+# LEGACY TESTSETS --------------------------------------------------------------
+
+
+class LegacyTestset(BaseModel):
+    id: str
+    name: Optional[str] = None
+    csvdata: Optional[List[Dict[str, Any]]] = None