deeprails 0.3.2__py3-none-any.whl → 1.0.0__py3-none-any.whl

deeprails/types/defend/workflow_event_response.py

@@ -0,0 +1,33 @@
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from typing import Optional
+
+from ..._models import BaseModel
+
+__all__ = ["WorkflowEventResponse"]
+
+
+class WorkflowEventResponse(BaseModel):
+    event_id: str
+    """A unique workflow event ID."""
+
+    workflow_id: str
+    """Workflow ID associated with the event."""
+
+    attempt_number: Optional[int] = None
+    """Count of improvement attempts for the event.
+
+    If greater than one then all previous improvement attempts failed.
+    """
+
+    evaluation_id: Optional[str] = None
+    """A unique evaluation ID associated with this event.
+
+    Every event has one or more evaluation attempts.
+    """
+
+    filtered: Optional[bool] = None
+    """
+    `False` if evaluation passed all of the guardrail metrics, `True` if evaluation
+    failed any of the guardrail metrics.
+    """

deeprails/types/defend_create_workflow_params.py

@@ -0,0 +1,56 @@
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from __future__ import annotations
+
+from typing import Dict, Optional
+from typing_extensions import Literal, Required, TypedDict
+
+__all__ = ["DefendCreateWorkflowParams"]
+
+
+class DefendCreateWorkflowParams(TypedDict, total=False):
+    improvement_action: Required[Optional[Literal["regenerate", "fixit"]]]
+    """
+    The action used to improve outputs that fail one or guardrail metrics for the
+    workflow events. May be `regenerate`, `fixit`, or null which represents “do
+    nothing”. ReGen runs the user's exact input prompt with minor induced variance.
+    Fixit attempts to directly address the shortcomings of the output using the
+    guardrail failure rationale. Do nothing does not attempt any improvement.
+    """
+
+    metrics: Required[Dict[str, float]]
+    """Mapping of guardrail metrics to floating point threshold values.
+
+    If the workflow type is automatic, only the metric names are used
+    (`automatic_tolerance` determines thresholds). Possible metrics are
+    `correctness`, `completeness`, `instruction_adherence`, `context_adherence`,
+    `ground_truth_adherence`, or `comprehensive_safety`.
+    """
+
+    name: Required[str]
+    """Name of the workflow."""
+
+    type: Required[Literal["automatic", "custom"]]
+    """Type of thresholds to use for the workflow, either `automatic` or `custom`.
+
+    Automatic thresholds are assigned internally after the user specifies a
+    qualitative tolerance for the metrics, whereas custom metrics allow the user to
+    set the threshold for each metric as a floating point number between 0.0 and
+    1.0.
+    """
+
+    automatic_tolerance: Literal["low", "medium", "high"]
+    """
+    Hallucination tolerance for automatic workflows; may be `low`, `medium`, or
+    `high`. Ignored if `type` is `custom`.
+    """
+
+    description: str
+    """Description for the workflow."""
+
+    max_retries: int
+    """Max.
+
+    number of improvement action retries until a given event passes the guardrails.
+    Defaults to 10.
+    """

deeprails/types/defend_response.py

@@ -0,0 +1,50 @@
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from typing import Optional
+from datetime import datetime
+from typing_extensions import Literal
+
+from .._models import BaseModel
+
+__all__ = ["DefendResponse"]
+
+
+class DefendResponse(BaseModel):
+    name: str
+    """Name of the workflow."""
+
+    workflow_id: str
+    """A unique workflow ID."""
+
+    created_at: Optional[datetime] = None
+    """The time the workflow was created in UTC."""
+
+    description: Optional[str] = None
+    """Description for the workflow."""
+
+    improvement_action: Optional[Literal["regenerate", "fixit"]] = None
+    """
+    The action used to improve outputs that fail one or more guardrail metrics for
+    the workflow events. May be `regenerate`, `fixit`, or null which represents “do
+    nothing”. ReGen runs the user's exact input prompt with minor induced variance.
+    Fixit attempts to directly address the shortcomings of the output using the
+    guardrail failure rationale. Do nothing does not attempt any improvement.
+    """
+
+    max_retries: Optional[int] = None
+    """Max.
+
+    number of improvement action retries until a given event passes the guardrails.
+    """
+
+    modified_at: Optional[datetime] = None
+    """The most recent time the workflow was modified in UTC."""
+
+    status: Optional[Literal["archived", "active"]] = None
+    """Status of the selected workflow.
+
+    May be `archived` or `active`. Archived workflows will not accept events.
+    """
+
+    success_rate: Optional[float] = None
+    """Rate of events associated with this workflow that passed evaluation."""

deeprails/types/defend_update_workflow_params.py

@@ -0,0 +1,18 @@
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from __future__ import annotations
+
+from typing_extensions import Literal, TypedDict
+
+__all__ = ["DefendUpdateWorkflowParams"]
+
+
+class DefendUpdateWorkflowParams(TypedDict, total=False):
+    description: str
+    """Description for the workflow."""
+
+    name: str
+    """Name of the workflow."""
+
+    type: Literal["automatic", "custom"]
+    """Type of thresholds to use for the workflow, either `automatic` or `custom`."""

deeprails/types/evaluate_create_params.py

@@ -0,0 +1,60 @@
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from __future__ import annotations
+
+from typing import Dict, List, Union
+from typing_extensions import Literal, Required, TypeAlias, TypedDict
+
+__all__ = ["EvaluateCreateParams", "ModelInput"]
+
+
+class EvaluateCreateParams(TypedDict, total=False):
+    model_input: Required[ModelInput]
+    """A dictionary of inputs sent to the LLM to generate output.
+
+    This must contain a `user_prompt` field and an optional `context` field.
+    Additional properties are allowed.
+    """
+
+    model_output: Required[str]
+    """Output generated by the LLM to be evaluated."""
+
+    run_mode: Required[Literal["precision_plus", "precision", "smart", "economy"]]
+    """Run mode for the evaluation.
+
+    The run mode allows the user to optimize for speed, accuracy, and cost by
+    determining which models are used to evaluate the event. Available run modes
+    include `precision_plus`, `precision`, `smart`, and `economy`. Defaults to
+    `smart`.
+    """
+
+    guardrail_metrics: List[
+        Literal[
+            "correctness",
+            "completeness",
+            "instruction_adherence",
+            "context_adherence",
+            "ground_truth_adherence",
+            "comprehensive_safety",
+        ]
+    ]
+    """
+    An array of guardrail metrics that the model input and output pair will be
+    evaluated on. For non-enterprise users, these will be limited to the allowed
+    guardrail metrics.
+    """
+
+    model_used: str
+    """Model ID used to generate the output, like `gpt-4o` or `o3`."""
+
+    nametag: str
+    """An optional, user-defined tag for the evaluation."""
+
+
+class ModelInputTyped(TypedDict, total=False):
+    user_prompt: Required[str]
+
+    context: str
+
+
+ModelInput: TypeAlias = Union[ModelInputTyped, Dict[str, object]]

deeprails/types/evaluation.py

@@ -0,0 +1,113 @@
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from typing import TYPE_CHECKING, Dict, List, Optional
+from datetime import datetime
+from typing_extensions import Literal
+
+from pydantic import Field as FieldInfo
+
+from .._models import BaseModel
+
+__all__ = ["Evaluation", "ModelInput"]
+
+
+class ModelInput(BaseModel):
+    user_prompt: str
+    """The user prompt used to generate the output."""
+
+    context: Optional[str] = None
+    """Optional context supplied to the LLM when generating the output."""
+
+    if TYPE_CHECKING:
+        # Some versions of Pydantic <2.8.0 have a bug and don’t allow assigning a
+        # value to this field, so for compatibility we avoid doing it at runtime.
+        __pydantic_extra__: Dict[str, object] = FieldInfo(init=False)  # pyright: ignore[reportIncompatibleVariableOverride]
+
+        # Stub to indicate that arbitrary properties are accepted.
+        # To access properties that are not valid identifiers you can use `getattr`, e.g.
+        # `getattr(obj, '$type')`
+        def __getattr__(self, attr: str) -> object: ...
+    else:
+        __pydantic_extra__: Dict[str, object]
+
+
+class Evaluation(BaseModel):
+    eval_id: str
+    """A unique evaluation ID."""
+
+    evaluation_status: Literal["in_progress", "completed", "canceled", "queued", "failed"]
+    """Status of the evaluation."""
+
+    api_model_input: ModelInput = FieldInfo(alias="model_input")
+    """A dictionary of inputs sent to the LLM to generate output.
+
+    The dictionary must contain a `user_prompt` field and an optional `context`
+    field. Additional properties are allowed.
+    """
+
+    api_model_output: str = FieldInfo(alias="model_output")
+    """Output generated by the LLM to be evaluated."""
+
+    run_mode: Literal["precision_plus", "precision", "smart", "economy"]
+    """Run mode for the evaluation.
+
+    The run mode allows the user to optimize for speed, accuracy, and cost by
+    determining which models are used to evaluate the event.
+    """
+
+    created_at: Optional[datetime] = None
+    """The time the evaluation was created in UTC."""
+
+    end_timestamp: Optional[datetime] = None
+    """The time the evaluation completed in UTC."""
+
+    error_message: Optional[str] = None
+    """Description of the error causing the evaluation to fail, if any."""
+
+    error_timestamp: Optional[datetime] = None
+    """The time the error causing the evaluation to fail was recorded."""
+
+    evaluation_result: Optional[Dict[str, object]] = None
+    """
+    Evaluation result consisting of average scores and rationales for each of the
+    evaluated guardrail metrics.
+    """
+
+    evaluation_total_cost: Optional[float] = None
+    """Total cost of the evaluation."""
+
+    guardrail_metrics: Optional[
+        List[
+            Literal[
+                "correctness",
+                "completeness",
+                "instruction_adherence",
+                "context_adherence",
+                "ground_truth_adherence",
+                "comprehensive_safety",
+            ]
+        ]
+    ] = None
+    """
+    An array of guardrail metrics that the model input and output pair will be
+    evaluated on.
+    """
+
+    api_model_used: Optional[str] = FieldInfo(alias="model_used", default=None)
+    """Model ID used to generate the output, like `gpt-4o` or `o3`."""
+
+    modified_at: Optional[datetime] = None
+    """The most recent time the evaluation was modified in UTC."""
+
+    nametag: Optional[str] = None
+    """An optional, user-defined tag for the evaluation."""
+
+    progress: Optional[int] = None
+    """Evaluation progress.
+
+    Values range between 0 and 100; 100 corresponds to a completed
+    `evaluation_status`.
+    """
+
+    start_timestamp: Optional[datetime] = None
+    """The time the evaluation started in UTC."""

deeprails/types/monitor_create_params.py

@@ -0,0 +1,15 @@
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from __future__ import annotations
+
+from typing_extensions import Required, TypedDict
+
+__all__ = ["MonitorCreateParams"]
+
+
+class MonitorCreateParams(TypedDict, total=False):
+    name: Required[str]
+    """Name of the new monitor."""
+
+    description: str
+    """Description of the new monitor."""

deeprails/types/monitor_retrieve_params.py

@@ -0,0 +1,12 @@
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from __future__ import annotations
+
+from typing_extensions import TypedDict
+
+__all__ = ["MonitorRetrieveParams"]
+
+
+class MonitorRetrieveParams(TypedDict, total=False):
+    limit: int
+    """Limit the returned events associated with this monitor. Defaults to 10."""

deeprails/types/monitor_retrieve_response.py

@@ -0,0 +1,81 @@
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from typing import List, Optional
+from datetime import datetime
+from typing_extensions import Literal
+
+from .._models import BaseModel
+from .evaluation import Evaluation
+
+__all__ = ["MonitorRetrieveResponse", "Data", "DataStats"]
+
+
+class DataStats(BaseModel):
+    completed_evaluations: Optional[int] = None
+    """Number of evaluations that completed successfully."""
+
+    failed_evaluations: Optional[int] = None
+    """Number of evaluations that failed."""
+
+    in_progress_evaluations: Optional[int] = None
+    """Number of evaluations currently in progress."""
+
+    queued_evaluations: Optional[int] = None
+    """Number of evaluations currently queued."""
+
+    total_evaluations: Optional[int] = None
+    """Total number of evaluations performed by this monitor."""
+
+
+class Data(BaseModel):
+    monitor_id: str
+    """A unique monitor ID."""
+
+    monitor_status: Literal["active", "inactive"]
+    """Status of the monitor.
+
+    Can be `active` or `inactive`. Inactive monitors no longer record and evaluate
+    events.
+    """
+
+    name: str
+    """Name of this monitor."""
+
+    created_at: Optional[datetime] = None
+    """The time the monitor was created in UTC."""
+
+    description: Optional[str] = None
+    """Description of this monitor."""
+
+    evaluations: Optional[List[Evaluation]] = None
+    """An array of all evaluations performed by this monitor.
+
+    Each one corresponds to a separate monitor event.
+    """
+
+    stats: Optional[DataStats] = None
+    """
+    Contains five fields used for stats of this monitor: total evaluations,
+    completed evaluations, failed evaluations, queued evaluations, and in progress
+    evaluations.
+    """
+
+    updated_at: Optional[datetime] = None
+    """The most recent time the monitor was modified in UTC."""
+
+    user_id: Optional[str] = None
+    """User ID of the user who created the monitor."""
+
+
+class MonitorRetrieveResponse(BaseModel):
+    success: bool
+    """Represents whether the request was completed successfully."""
+
+    data: Optional[Data] = None
+    """Detailed response payload for retrieving a monitor and its evaluations."""
+
+    message: Optional[str] = None
+    """The accompanying message for the request.
+
+    Includes error details when applicable.
+    """

deeprails/types/monitor_submit_event_params.py

@@ -0,0 +1,63 @@
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from __future__ import annotations
+
+from typing import Dict, List, Union
+from typing_extensions import Literal, Required, TypeAlias, TypedDict
+
+__all__ = ["MonitorSubmitEventParams", "ModelInput"]
+
+
+class MonitorSubmitEventParams(TypedDict, total=False):
+    guardrail_metrics: Required[
+        List[
+            Literal[
+                "correctness",
+                "completeness",
+                "instruction_adherence",
+                "context_adherence",
+                "ground_truth_adherence",
+                "comprehensive_safety",
+            ]
+        ]
+    ]
+    """
+    An array of guardrail metrics that the model input and output pair will be
+    evaluated on. For non-enterprise users, these will be limited to `correctness`,
+    `completeness`, `instruction_adherence`, `context_adherence`,
+    `ground_truth_adherence`, and/or `comprehensive_safety`.
+    """
+
+    model_input: Required[ModelInput]
+    """A dictionary of inputs sent to the LLM to generate output.
+
+    This must contain a `user_prompt` field and an optional `context` field.
+    Additional properties are allowed.
+    """
+
+    model_output: Required[str]
+    """Output generated by the LLM to be evaluated."""
+
+    model_used: str
+    """Model ID used to generate the output, like `gpt-4o` or `o3`."""
+
+    nametag: str
+    """An optional, user-defined tag for the event."""
+
+    run_mode: Literal["precision_plus", "precision", "smart", "economy"]
+    """Run mode for the monitor event.
+
+    The run mode allows the user to optimize for speed, accuracy, and cost by
+    determining which models are used to evaluate the event. Available run modes
+    include `precision_plus`, `precision`, `smart`, and `economy`. Defaults to
+    `smart`.
+    """
+
+
+class ModelInputTyped(TypedDict, total=False):
+    user_prompt: Required[str]
+
+    context: str
+
+
+ModelInput: TypeAlias = Union[ModelInputTyped, Dict[str, object]]

deeprails/types/monitor_submit_event_response.py

@@ -0,0 +1,36 @@
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from typing import Optional
+from datetime import datetime
+
+from .._models import BaseModel
+
+__all__ = ["MonitorSubmitEventResponse", "Data"]
+
+
+class Data(BaseModel):
+    evaluation_id: str
+    """A unique evaluation ID associated with this event."""
+
+    event_id: str
+    """A unique monitor event ID."""
+
+    monitor_id: str
+    """Monitor ID associated with this event."""
+
+    created_at: Optional[datetime] = None
+    """The time the monitor event was created in UTC."""
+
+
+class MonitorSubmitEventResponse(BaseModel):
+    success: bool
+    """Represents whether the request was completed successfully."""
+
+    data: Optional[Data] = None
+    """Response payload for monitor event operations."""
+
+    message: Optional[str] = None
+    """The accompanying message for the request.
+
+    Includes error details when applicable.
+    """

deeprails/types/monitor_update_params.py

@@ -0,0 +1,22 @@
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from __future__ import annotations
+
+from typing_extensions import Literal, TypedDict
+
+__all__ = ["MonitorUpdateParams"]
+
+
+class MonitorUpdateParams(TypedDict, total=False):
+    description: str
+    """Description of the monitor."""
+
+    monitor_status: Literal["active", "inactive"]
+    """Status of the monitor.
+
+    Can be `active` or `inactive`. Inactive monitors no longer record and evaluate
+    events.
+    """
+
+    name: str
+    """Name of the monitor."""