agenta 0.57.0__py3-none-any.whl → 0.65.0__py3-none-any.whl

agenta/sdk/managers/evaluators.py

@@ -0,0 +1,303 @@
+from typing import Dict, Any, Callable, Optional
+from uuid import uuid4, UUID
+from traceback import print_exc
+
+from agenta.sdk.utils.client import authed_api
+from agenta.sdk.decorators.running import auto_workflow, is_workflow
+from agenta.sdk.models.workflows import (
+    EvaluatorRevision,
+    #
+    EvaluatorRevisionResponse,
+    #
+    SimpleEvaluatorFlags,
+    SimpleEvaluatorData,
+    SimpleEvaluatorCreate,
+    SimpleEvaluatorEdit,
+    #
+    SimpleEvaluatorResponse,
+    #
+    Reference,
+)
+
+from agenta.sdk.utils.references import get_slug_from_name_and_id
+
+
+async def _retrieve_evaluator(
+    evaluator_id: Optional[UUID] = None,
+    evaluator_slug: Optional[str] = None,
+    evaluator_revision_id: Optional[UUID] = None,
+    evaluator_revision_slug: Optional[str] = None,
+) -> Optional[EvaluatorRevision]:
+    payload = {
+        "evaluator_ref": (
+            {
+                "id": str(evaluator_id) if evaluator_id else None,
+                "slug": str(evaluator_slug),
+            }
+            if evaluator_id or evaluator_slug
+            else None
+        ),
+        "evaluator_revision_ref": (
+            {
+                "id": str(evaluator_revision_id) if evaluator_revision_id else None,
+                "slug": evaluator_revision_slug,
+            }
+            if evaluator_revision_id or evaluator_revision_slug
+            else None
+        ),
+    }
+
+    # print(" --- payload:", payload)
+
+    response = authed_api()(
+        method="POST",
+        endpoint=f"/preview/evaluators/revisions/retrieve",
+        json=payload,
+    )
+
+    response.raise_for_status()
+
+    evaluator_revision_response = EvaluatorRevisionResponse(**response.json())
+
+    evaluator_revision = evaluator_revision_response.evaluator_revision
+
+    # print(" --- evaluator_revision:", evaluator_revision)
+
+    return evaluator_revision
+
+
+async def aretrieve(
+    evaluator_revision_id: Optional[UUID] = None,
+) -> Optional[EvaluatorRevision]:
+    # print("\n--------- RETRIEVE EVALUATOR")
+    response = await _retrieve_evaluator(
+        evaluator_revision_id=evaluator_revision_id,
+    )
+
+    return response
+
+
+async def aupsert(
+    *,
+    evaluator_id: Optional[UUID] = None,
+    evaluator_slug: Optional[str] = None,
+    evaluator_revision_id: Optional[UUID] = None,
+    evaluator_revision_slug: Optional[str] = None,
+    #
+    handler: Callable,
+    script: Optional[str] = None,
+    parameters: Optional[Dict[str, Any]] = None,
+    #
+    name: Optional[str] = None,
+    description: Optional[str] = None,
+) -> Optional[UUID]:
+    # print("\n---------   UPSERT EVALUATOR")
+    try:
+        if not is_workflow(handler):
+            evaluator_workflow = auto_workflow(
+                handler,
+                #
+                script=script,
+                parameters=parameters,
+                #
+                name=name,
+                description=description,
+            )
+        else:
+            evaluator_workflow = handler
+
+        req = await evaluator_workflow.inspect()
+
+        legacy_application_flags = SimpleEvaluatorFlags(**req.flags)
+
+        simple_evaluator_data = SimpleEvaluatorData(
+            **(
+                req.interface.model_dump(mode="json", exclude_none=True)
+                if req and req.interface
+                else {}
+            ),
+            **(
+                req.configuration.model_dump(mode="json", exclude_none=True)
+                if req and req.configuration
+                else {}
+            ),
+        )
+        # print(" ---:", simple_evaluator_data.model_dump(mode="json", exclude_none=True))
+
+        retrieve_response = None
+
+        if req.references is not None:
+            _evaluator_revision_ref = req.references.get("evaluator_revision", {})
+            if isinstance(_evaluator_revision_ref, Reference):
+                _evaluator_revision_ref = _evaluator_revision_ref.model_dump(
+                    mode="json",
+                    exclude_none=True,
+                )
+            if not isinstance(_evaluator_revision_ref, dict):
+                _evaluator_revision_ref = {}
+
+            _evaluator_revision_id = _evaluator_revision_ref.get("id")
+            _evaluator_revision_slug = _evaluator_revision_ref.get("slug")
+
+            evaluator_revision_id = evaluator_revision_id or _evaluator_revision_id
+            evaluator_revision_slug = (
+                evaluator_revision_slug or _evaluator_revision_slug
+            )
+
+            _evaluator_ref = req.references.get("evaluator", {})
+            if isinstance(_evaluator_ref, Reference):
+                _evaluator_ref = _evaluator_ref.model_dump(
+                    mode="json",
+                    exclude_none=True,
+                )
+            if not isinstance(_evaluator_ref, dict):
+                _evaluator_ref = {}
+
+            _evaluator_id = _evaluator_ref.get("id")
+            _evaluator_slug = _evaluator_ref.get("slug")
+
+            evaluator_id = evaluator_id or _evaluator_id
+            evaluator_slug = evaluator_slug or _evaluator_slug
+
+            revision = req.data.revision if req and req.data else None
+            if revision:
+                name = name or revision.get("name")
+                description = description or revision.get("description")
+
+        name = (
+            name or req.data.revision.get("name")
+            if req and req.data and req.data.revision
+            else None
+        )
+
+        description = (
+            description or req.data.revision.get("description")
+            if req and req.data and req.data.revision
+            else None
+        )
+
+        evaluator_slug = (
+            evaluator_slug
+            or get_slug_from_name_and_id(
+                name=name,
+                id=evaluator_id or uuid4(),
+            )
+            if name
+            else uuid4().hex[-12:]
+        )
+
+        # print(
+        #     evaluator_id,
+        #     evaluator_slug,
+        #     evaluator_revision_id,
+        #     evaluator_revision_slug,
+        # )
+
+        if evaluator_revision_id or evaluator_revision_slug:
+            retrieve_response = await _retrieve_evaluator(
+                evaluator_revision_id=evaluator_revision_id,
+                evaluator_revision_slug=evaluator_revision_slug,
+            )
+        elif evaluator_id or evaluator_slug:
+            retrieve_response = await _retrieve_evaluator(
+                evaluator_id=evaluator_id,
+                evaluator_slug=evaluator_slug,
+            )
+
+    except Exception as e:
+        print("[ERROR]: Failed to prepare evaluator:")
+        print_exc()
+        return None
+
+    # print("Retrieve response:", retrieve_response)
+
+    if retrieve_response and retrieve_response.id and retrieve_response.evaluator_id:
+        evaluator_id = retrieve_response.evaluator_id
+        # print(" --- Updating evaluator...", evaluator_id)
+        evaluator_edit_request = SimpleEvaluatorEdit(
+            id=evaluator_id,
+            #
+            name=name,
+            description=description,
+            #
+            flags=legacy_application_flags,
+            #
+            data=simple_evaluator_data,
+        )
+
+        # print(" --- evaluator_edit_request:", evaluator_edit_request)
+
+        response = authed_api()(
+            method="PUT",
+            endpoint=f"/preview/simple/evaluators/{evaluator_id}",
+            json={
+                "evaluator": evaluator_edit_request.model_dump(
+                    mode="json",
+                    exclude_none=True,
+                )
+            },
+        )
+
+        # print(" --- response:", response.status_code, response.text)
+
+        try:
+            response.raise_for_status()
+        except Exception as e:
+            print("[ERROR]: Failed to update evaluator:", e)
+            print_exc()
+            return None
+
+    else:
+        # print(" --- Creating evaluator...")
+        evaluator_create_request = SimpleEvaluatorCreate(
+            slug=evaluator_slug or uuid4().hex[-12:],
+            #
+            name=name,
+            description=description,
+            #
+            flags=legacy_application_flags,
+            #
+            data=simple_evaluator_data,
+        )
+
+        # print(" --- evaluator_create_request:", evaluator_create_request)
+
+        response = authed_api()(
+            method="POST",
+            endpoint="/preview/simple/evaluators/",
+            json={
+                "evaluator": evaluator_create_request.model_dump(
+                    mode="json",
+                    exclude_none=True,
+                )
+            },
+        )
+
+        # print(" --- response:", response.status_code, response.text)
+
+        try:
+            response.raise_for_status()
+        except Exception as e:
+            print("[ERROR]: Failed to create evaluator:", e)
+            print_exc()
+            return None
+
+    evaluator_response = SimpleEvaluatorResponse(**response.json())
+
+    evaluator = evaluator_response.evaluator
+
+    if not evaluator or not evaluator.id:
+        return None
+
+    # print(" --- evaluator:", evaluator)
+
+    evaluator_revision = await _retrieve_evaluator(
+        evaluator_id=evaluator.id,
+    )
+
+    if not evaluator_revision or not evaluator_revision.id:
+        return None
+
+    # print(evaluator_revision, "----------")
+
+    return evaluator_revision.id

agenta/sdk/managers/secrets.py

@@ -1,19 +1,27 @@
 import re
 from typing import Optional, Dict, Any, List
 
-from agenta.sdk.context.serving import serving_context
+from agenta.sdk.utils.logging import get_module_logger
+from agenta.sdk.contexts.routing import RoutingContext
+from agenta.sdk.contexts.running import RunningContext
 from agenta.sdk.assets import model_to_provider_mapping as _standard_providers
 
+from agenta.sdk.middlewares.running.vault import get_secrets
+
+import agenta as ag
+
+log = get_module_logger(__name__)
+
 
 class SecretsManager:
     @staticmethod
     def get_from_route() -> Optional[List[Dict[str, Any]]]:
-        context = serving_context.get()
+        context = RoutingContext.get()
 
         secrets = context.secrets
 
         if not secrets:
-            return None
+            return []
 
         return secrets
 
@@ -119,11 +127,17 @@ class SecretsManager:
         # The reason is that custom providers are in fact openai compatible providers
         # They need to be passed in litellm as openai/modelname
 
-        if "custom" in model:
-            modified_model = model.replace(f"{provider_slug}/custom/", "openai/")
-            return modified_model.replace(f"{provider_slug}/", "")
+        modified_model = model
+
+        if "custom" in modified_model:
+            modified_model = modified_model.replace(
+                f"{provider_slug}/custom/", "openai/"
+            )
+
+        if provider_slug:
+            modified_model = modified_model.replace(f"{provider_slug}/", "")
 
-        return model.replace(f"{provider_slug}/", "")
+        return modified_model
 
     @staticmethod
     def get_provider_settings(model: str) -> Optional[Dict]:
@@ -137,6 +151,8 @@ class SecretsManager:
             Dict: A dictionary containing all parameters needed for litellm.completion
         """
 
+        request_provider_model = model
+
         # STEP 1: get vault secrets from route context and transform it
         secrets = SecretsManager.get_from_route()
         if not secrets:
@@ -146,11 +162,11 @@ class SecretsManager:
         secrets = SecretsManager._parse_secrets(secrets=secrets)
 
         # STEP 2: check model exists in supported standard models
-        provider = _standard_providers.get(model)
+        provider = _standard_providers.get(request_provider_model)
         if not provider:
             # check and get provider kind if model exists in custom provider models
             provider = SecretsManager._custom_providers_get(
-                model=model,
+                model=request_provider_model,
                 secrets=secrets,
             )
 
@@ -159,16 +175,19 @@ class SecretsManager:
             return None
 
         # STEP 2c: get litellm compatible model
-        provider_slug = SecretsManager._custom_provider_slug_get(
-            model=model, secrets=secrets
+        request_provider_slug = (
+            SecretsManager._custom_provider_slug_get(
+                model=request_provider_model, secrets=secrets
+            )
+            or ""
         )
-        model = SecretsManager._get_compatible_model(
-            model=model, provider_slug=provider_slug
+        compatible_provider_model = SecretsManager._get_compatible_model(
+            model=request_provider_model, provider_slug=request_provider_slug
         )
 
         # STEP 3: initialize provider settings and simplify provider name
-        provider_settings = {"model": model}
-        provider_name = re.sub(
+        provider_settings = dict(model=compatible_provider_model)
+        request_provider_kind = re.sub(
             r"[\s-]+", "", provider.lower()
         )  # normalizing other special characters too (azure-openai)
 
@@ -180,23 +199,141 @@ class SecretsManager:
             # i). Extract API key if present
             # (for standard models -- openai/anthropic/gemini, etc)
             if secret.get("kind") == "provider_key":
-                provider_kind = secret_data.get("kind", "")
+                secret_provider_kind = secret_data.get("kind", "")
 
-                if provider_kind == provider_name:
+                if request_provider_kind == secret_provider_kind:
                     if "key" in provider_info:
                         provider_settings["api_key"] = provider_info["key"]
                 continue
 
             # ii). Extract Credentials if present
-            # (for custom providers -- aws bedrock/sagemaker, vertexai, etc)
+            # (for custom providers -- aws bedrock/sagemaker, vertex_ai, etc)
             elif secret.get("kind") == "custom_provider":
-                provider_kind = provider_info.get("kind", "").lower().replace(" ", "")
-                provider_slug = secret_data.get("provider_slug", "")
-                provider_extras = provider_info.get("extras", {})
+                secret_provider_kind = (
+                    provider_info.get("kind", "").lower().replace(" ", "")
+                )
+                secret_provider_slug = secret_data.get("provider_slug", "")
+                secret_provider_models = secret_data.get("models", "")
+                secret_provider_extras = provider_info.get("extras", {})
+
+                if (
+                    request_provider_kind == secret_provider_kind
+                    and request_provider_slug == secret_provider_slug
+                    and request_provider_model in secret_provider_models
+                ):
+                    if secret_provider_extras:
+                        provider_settings.update(secret_provider_extras)
+                continue
+
+        if len(provider_settings.keys()) <= 1:
+            return None
+
+        return provider_settings
+
+    @staticmethod
+    async def retrieve_secrets():
+        return await get_secrets(
+            f"{ag.DEFAULT_AGENTA_SINGLETON_INSTANCE.host}/api",
+            RunningContext.get().credentials,
+        )
+
+    @staticmethod
+    async def ensure_secrets_in_workflow():
+        ctx = RunningContext.get()
+
+        ctx.secrets = await SecretsManager.retrieve_secrets()
+
+        RunningContext.set(ctx)
+
+        return ctx.secrets
+
+    @staticmethod
+    def get_provider_settings_from_workflow(model: str) -> Optional[Dict]:
+        """
+        Builds the LLM request with appropriate kwargs based on the custom provider/model
+
+        Args:
+            model (str): The name of the model
+
+        Returns:
+            Dict: A dictionary containing all parameters needed for litellm.completion
+        """
+
+        request_provider_model = model
+
+        # STEP 1: get vault secrets from route context and transform it
+        secrets = RunningContext.get().secrets
+        if not secrets:
+            return None
+
+        # STEP 1b: Parse secrets into usable format
+        secrets = SecretsManager._parse_secrets(secrets=secrets)
+
+        # STEP 2: check model exists in supported standard models
+        provider = _standard_providers.get(request_provider_model)
+        if not provider:
+            # check and get provider kind if model exists in custom provider models
+            provider = SecretsManager._custom_providers_get(
+                model=request_provider_model,
+                secrets=secrets,
+            )
+
+        # STEP 2b: return None in the case provider is None
+        if not provider:
+            return None
+
+        # STEP 2c: get litellm compatible model
+        request_provider_slug = (
+            SecretsManager._custom_provider_slug_get(
+                model=request_provider_model, secrets=secrets
+            )
+            or ""
+        )
+        compatible_provider_model = SecretsManager._get_compatible_model(
+            model=request_provider_model, provider_slug=request_provider_slug
+        )
 
-                if provider_kind == provider_name or provider_slug == provider_name:
-                    if provider_extras:
-                        provider_settings.update(provider_extras)
+        # STEP 3: initialize provider settings and simplify provider name
+        provider_settings = dict(model=compatible_provider_model)
+        request_provider_kind = re.sub(
+            r"[\s-]+", "", provider.lower()
+        )  # normalizing other special characters too (azure-openai)
+
+        # STEP 4: get credentials for model
+        for secret in secrets:
+            secret_data = secret.get("data", {})
+            provider_info = secret_data.get("provider", {})
+
+            # i). Extract API key if present
+            # (for standard models -- openai/anthropic/gemini, etc)
+            if secret.get("kind") == "provider_key":
+                secret_provider_kind = secret_data.get("kind", "")
+
+                if request_provider_kind == secret_provider_kind:
+                    if "key" in provider_info:
+                        provider_settings["api_key"] = provider_info["key"]
                 continue
 
+            # ii). Extract Credentials if present
+            # (for custom providers -- aws bedrock/sagemaker, vertex_ai, etc)
+            elif secret.get("kind") == "custom_provider":
+                secret_provider_kind = (
+                    provider_info.get("kind", "").lower().replace(" ", "")
+                )
+                secret_provider_slug = secret_data.get("provider_slug", "")
+                secret_provider_models = secret_data.get("models", "")
+                secret_provider_extras = provider_info.get("extras", {})
+
+                if (
+                    request_provider_kind == secret_provider_kind
+                    and request_provider_slug == secret_provider_slug
+                    and request_provider_model in secret_provider_models
+                ):
+                    if secret_provider_extras:
+                        provider_settings.update(secret_provider_extras)
+                continue
+
+        if len(provider_settings.keys()) <= 1:
+            return None
+
         return provider_settings

agenta/sdk/managers/shared.py

@@ -478,7 +478,9 @@ class SharedManager:
             config=ConfigRequest(
                 params=parameters,
                 variant_ref=variant_ref.model_dump() if variant_ref else None,  # type: ignore
-                application_ref=application_ref.model_dump() if application_ref else None,  # type: ignore
+                application_ref=application_ref.model_dump()
+                if application_ref
+                else None,  # type: ignore
             )
         )