evolution-sdk 0.8.0__py3-none-any.whl

evolution/capture/introspect.py

@@ -0,0 +1,164 @@
+"""
+Introspection and duck-typing utilities for LLM responses and function signatures.
+"""
+
+from __future__ import annotations
+
+import inspect
+from typing import Any, Callable
+
+from evolution.models.artifacts import ModelConfigArtifact, PromptArtifact
+from evolution.models.execution import TokenUsage
+
+
+def extract_inputs_from_args(func: Callable, args: tuple, kwargs: dict) -> str:
+    """Formats function arguments into a readable input representation."""
+    sig = inspect.signature(func)
+    try:
+        bound = sig.bind(*args, **kwargs)
+        bound.apply_defaults()
+        arg_dict = bound.arguments
+    except Exception:
+        arg_dict = {"args": args, "kwargs": kwargs}
+
+    # If only one string argument, return it directly
+    if len(arg_dict) == 1:
+        val = next(iter(arg_dict.values()))
+        if isinstance(val, str):
+            return val
+
+    # Otherwise format key-value pairs
+    items = []
+    for k, v in arg_dict.items():
+        if k in ("self", "cls"):
+            continue
+        items.append(f"{k}={v!r}")
+    return ", ".join(items) if items else "(no input arguments)"
+
+
+def extract_docstring_prompt(func: Callable, name: str | None = None) -> PromptArtifact | None:
+    """Extracts function docstring as a system prompt artifact if present."""
+    doc = inspect.getdoc(func)
+    if doc and doc.strip():
+        art_name = name or f"{func.__name__}-prompt"
+        return PromptArtifact(
+            name=art_name,
+            role="system",
+            description=f"Extracted docstring prompt for {func.__name__}",
+        )
+    return None
+
+
+def extract_model_config_from_kwargs(kwargs: dict[str, Any], name: str | None = None) -> ModelConfigArtifact | None:
+    """Detects model and generation parameters passed in kwargs."""
+    model = kwargs.get("model") or kwargs.get("model_name")
+    if not model or not isinstance(model, str):
+        return None
+
+    provider = kwargs.get("provider", "openai")
+    if provider not in ("openai", "anthropic", "google", "local", "mistral", "cohere", "aws_bedrock"):
+        provider = "openai"
+
+    temperature = kwargs.get("temperature")
+    if temperature is not None:
+        try:
+            temperature = float(temperature)
+        except (ValueError, TypeError):
+            temperature = 0.7
+
+    return ModelConfigArtifact(
+        name=name or f"model-{model}",
+        model=model,
+        provider=provider,
+        temperature=temperature,
+        max_tokens=kwargs.get("max_tokens"),
+        top_p=kwargs.get("top_p"),
+    )
+
+
+def extract_llm_response(result: Any) -> tuple[str, TokenUsage, str | None]:
+    """Duck-types standard LLM response objects (OpenAI, Anthropic, LangChain, or dicts)
+    to extract:
+      - output text (str)
+      - token usage (TokenUsage)
+      - model name (str | None)
+    """
+    output_text = ""
+    tokens = TokenUsage()
+    model_name: str | None = None
+
+    if result is None:
+        return "", tokens, None
+
+    # If it's already a simple string
+    if isinstance(result, str):
+        return result, tokens, None
+
+    # Check for dict response
+    if isinstance(result, dict):
+        # 1. Output extraction
+        if "choices" in result and isinstance(result["choices"], list) and len(result["choices"]) > 0:
+            first = result["choices"][0]
+            if isinstance(first, dict):
+                msg = first.get("message", {})
+                output_text = msg.get("content", "") if isinstance(msg, dict) else str(first.get("text", ""))
+        elif "content" in result:
+            if isinstance(result["content"], list) and len(result["content"]) > 0:
+                first = result["content"][0]
+                output_text = first.get("text", "") if isinstance(first, dict) else str(first)
+            else:
+                output_text = str(result["content"])
+        elif "output" in result:
+            output_text = str(result["output"])
+        elif "text" in result:
+            output_text = str(result["text"])
+
+        # 2. Token usage extraction
+        usage = result.get("usage", {})
+        if isinstance(usage, dict):
+            tokens.prompt_tokens = usage.get("prompt_tokens") or usage.get("input_tokens") or 0
+            tokens.completion_tokens = usage.get("completion_tokens") or usage.get("output_tokens") or 0
+            tokens.total_tokens = usage.get("total_tokens") or (tokens.prompt_tokens + tokens.completion_tokens)
+
+        # 3. Model name
+        model_name = result.get("model")
+
+        if not output_text:
+            output_text = str(result)
+        return output_text, tokens, model_name
+
+    # Check for Object response (OpenAI ChatCompletion, Anthropic Message, etc.) via attributes
+    # 1. Output extraction
+    if hasattr(result, "choices") and result.choices:
+        first = result.choices[0]
+        if hasattr(first, "message") and hasattr(first.message, "content"):
+            output_text = str(first.message.content or "")
+        elif hasattr(first, "text"):
+            output_text = str(first.text or "")
+    elif hasattr(result, "content"):
+        c = result.content
+        if isinstance(c, list) and len(c) > 0:
+            first = c[0]
+            output_text = getattr(first, "text", str(first))
+        else:
+            output_text = str(c)
+    elif hasattr(result, "output"):
+        output_text = str(result.output)
+    elif hasattr(result, "text"):
+        output_text = str(result.text)
+
+    # 2. Token usage extraction
+    if hasattr(result, "usage") and result.usage:
+        u = result.usage
+        tokens.prompt_tokens = getattr(u, "prompt_tokens", getattr(u, "input_tokens", 0)) or 0
+        tokens.completion_tokens = getattr(u, "completion_tokens", getattr(u, "output_tokens", 0)) or 0
+        tokens.total_tokens = getattr(u, "total_tokens", tokens.prompt_tokens + tokens.completion_tokens) or (tokens.prompt_tokens + tokens.completion_tokens)
+
+    # 3. Model name
+    if hasattr(result, "model") and result.model:
+        model_name = str(result.model)
+
+    if not output_text:
+        output_text = str(result)
+
+    return output_text, tokens, model_name

evolution/capture/recorder.py

@@ -0,0 +1,109 @@
+"""
+Execution recording context manager for tracing AI invocations.
+"""
+
+from __future__ import annotations
+
+import time
+from pathlib import Path
+from typing import Any
+
+from evolution.capture.introspect import extract_llm_response
+from evolution.models.execution import Execution, TokenUsage
+from evolution.repository import Repository
+
+
+class RecordContextManager:
+    """Context manager for tracing and recording an AI execution run."""
+
+    def __init__(
+        self,
+        repo: Repository | Path | str | None = None,
+        inputs: str = "",
+        commit_id: str | None = None,
+        metadata: dict[str, Any] | None = None,
+    ):
+        if isinstance(repo, Repository):
+            self.repo = repo
+        elif repo is not None:
+            self.repo = Repository.open(repo)
+        else:
+            try:
+                self.repo = Repository.open(".")
+            except Exception:
+                self.repo = None
+
+        self.inputs = inputs
+        self.outputs = ""
+        self.commit_id = commit_id
+        self.metadata = metadata or {}
+        self.tokens = TokenUsage()
+        self.duration_ms: int = 0
+        self.execution: Execution | None = None
+        self._start_time: float = 0.0
+
+    def set_input(self, inputs: str) -> RecordContextManager:
+        """Sets or updates the input query string."""
+        self.inputs = inputs
+        return self
+
+    def set_output(self, outputs: Any) -> RecordContextManager:
+        """Sets or updates the output string, auto-extracting from LLM response objects if needed."""
+        text, tokens, _ = extract_llm_response(outputs)
+        self.outputs = text
+        if tokens.total_tokens > 0:
+            self.tokens = tokens
+        return self
+
+    def set_tokens(
+        self,
+        prompt_tokens: int = 0,
+        completion_tokens: int = 0,
+        total_tokens: int = 0,
+    ) -> RecordContextManager:
+        """Sets token consumption metrics."""
+        self.tokens = TokenUsage(
+            prompt_tokens=prompt_tokens,
+            completion_tokens=completion_tokens,
+            total_tokens=total_tokens or (prompt_tokens + completion_tokens),
+        )
+        return self
+
+    def set_metadata(self, key: str, value: Any) -> RecordContextManager:
+        """Sets a metadata key-value pair."""
+        self.metadata[key] = value
+        return self
+
+    def __enter__(self) -> RecordContextManager:
+        self._start_time = time.perf_counter()
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb) -> None:
+        elapsed = time.perf_counter() - self._start_time
+        self.duration_ms = max(1, int(elapsed * 1000))
+
+        if exc_type is not None and not self.outputs:
+            self.outputs = f"Error ({exc_type.__name__}): {exc_val}"
+            self.metadata["error"] = str(exc_val)
+            self.metadata["error_type"] = exc_type.__name__
+
+        if self.repo is not None:
+            self.execution = self.repo.record_execution(
+                inputs=self.inputs,
+                outputs=self.outputs,
+                duration_ms=self.duration_ms,
+                prompt_tokens=self.tokens.prompt_tokens,
+                completion_tokens=self.tokens.completion_tokens,
+                commit_id=self.commit_id,
+                metadata=self.metadata,
+            )
+
+
+def record(
+    repo: Repository | Path | str | None = None,
+    inputs: str = "",
+    commit_id: str | None = None,
+    metadata: dict[str, Any] | None = None,
+) -> RecordContextManager:
+    """Convenience factory returning an execution recording context manager."""
+    return RecordContextManager(repo=repo, inputs=inputs, commit_id=commit_id, metadata=metadata)

evolution/capture/tracker.py

@@ -0,0 +1,194 @@
+"""
+Decorator-based telemetry and intelligence tracking.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import functools
+import inspect
+import time
+from pathlib import Path
+from typing import Any, Callable
+
+from evolution.capture.introspect import (
+    extract_docstring_prompt,
+    extract_inputs_from_args,
+    extract_llm_response,
+    extract_model_config_from_kwargs,
+)
+from evolution.models.artifacts import ModelConfigArtifact
+from evolution.models.execution import Execution
+from evolution.repository import Repository
+
+
+def track(
+    _func: Callable | None = None,
+    *,
+    name: str | None = None,
+    repo: Repository | Path | str | None = None,
+    model: str | None = None,
+    provider: str = "openai",
+    temperature: float | None = None,
+    auto_manifest: bool = True,
+    metadata: dict[str, Any] | None = None,
+):
+    """Decorator to automatically capture intelligence artifacts and record executions.
+
+    Can be used as `@track` or `@track(...)`.
+    """
+    def decorator(func: Callable) -> Callable:
+        # Resolve target repository
+        target_repo: Repository | None
+        if isinstance(repo, Repository):
+            target_repo = repo
+        elif repo is not None:
+            try:
+                target_repo = Repository.open(repo)
+            except Exception:
+                target_repo = None
+        else:
+            try:
+                target_repo = Repository.open(".")
+            except Exception:
+                target_repo = None
+
+        func_name = name or func.__name__
+
+        # Auto-update manifest with docstring prompt and model config if enabled
+        if auto_manifest and target_repo is not None:
+            try:
+                manifest = target_repo.get_manifest()
+                updated = False
+
+                # 1. Capture docstring as prompt artifact
+                prompt_art = extract_docstring_prompt(func, name=f"{func_name}-prompt")
+                if prompt_art:
+                    manifest.add_artifact(prompt_art)
+                    updated = True
+
+                # 2. Capture explicit model config if specified
+                if model:
+                    mc_art = ModelConfigArtifact(
+                        name=f"{func_name}-model",
+                        model=model,
+                        provider=provider,
+                        temperature=temperature,
+                    )
+                    manifest.add_artifact(mc_art)
+                    updated = True
+
+                if updated:
+                    target_repo.save_manifest(manifest, auto_hash=True)
+            except Exception:
+                pass
+
+        if inspect.iscoroutinefunction(func):
+            @functools.wraps(func)
+            async def async_wrapper(*args, **kwargs):
+                inputs_str = extract_inputs_from_args(func, args, kwargs)
+                start_time = time.perf_counter()
+                exec_meta = dict(metadata or {})
+                exec_meta["function"] = func.__name__
+
+                try:
+                    result = await func(*args, **kwargs)
+                    elapsed = time.perf_counter() - start_time
+                    duration_ms = max(1, int(elapsed * 1000))
+
+                    output_text, tokens, detected_model = extract_llm_response(result)
+                    if detected_model:
+                        exec_meta["model"] = detected_model
+                    elif model:
+                        exec_meta["model"] = model
+
+                    if target_repo is not None:
+                        exec_obj = target_repo.record_execution(
+                            inputs=inputs_str,
+                            outputs=output_text,
+                            duration_ms=duration_ms,
+                            prompt_tokens=tokens.prompt_tokens,
+                            completion_tokens=tokens.completion_tokens,
+                            metadata=exec_meta,
+                        )
+                        setattr(async_wrapper, "last_execution", exec_obj)
+
+                    return result
+                except Exception as exc:
+                    elapsed = time.perf_counter() - start_time
+                    duration_ms = max(1, int(elapsed * 1000))
+                    exec_meta["error"] = str(exc)
+                    exec_meta["error_type"] = type(exc).__name__
+
+                    if target_repo is not None:
+                        target_repo.record_execution(
+                            inputs=inputs_str,
+                            outputs=f"Error: {exc}",
+                            duration_ms=duration_ms,
+                            metadata=exec_meta,
+                        )
+                    raise
+
+            return async_wrapper
+
+        else:
+            @functools.wraps(func)
+            def sync_wrapper(*args, **kwargs):
+                inputs_str = extract_inputs_from_args(func, args, kwargs)
+                start_time = time.perf_counter()
+                exec_meta = dict(metadata or {})
+                exec_meta["function"] = func.__name__
+
+                # Check if model passed in runtime kwargs
+                detected_kwarg_mc = extract_model_config_from_kwargs(kwargs)
+                if detected_kwarg_mc and auto_manifest and target_repo is not None:
+                    try:
+                        manifest = target_repo.get_manifest()
+                        manifest.add_artifact(detected_kwarg_mc)
+                        target_repo.save_manifest(manifest, auto_hash=True)
+                    except Exception:
+                        pass
+
+                try:
+                    result = func(*args, **kwargs)
+                    elapsed = time.perf_counter() - start_time
+                    duration_ms = max(1, int(elapsed * 1000))
+
+                    output_text, tokens, detected_model = extract_llm_response(result)
+                    if detected_model:
+                        exec_meta["model"] = detected_model
+                    elif model:
+                        exec_meta["model"] = model
+
+                    if target_repo is not None:
+                        exec_obj = target_repo.record_execution(
+                            inputs=inputs_str,
+                            outputs=output_text,
+                            duration_ms=duration_ms,
+                            prompt_tokens=tokens.prompt_tokens,
+                            completion_tokens=tokens.completion_tokens,
+                            metadata=exec_meta,
+                        )
+                        setattr(sync_wrapper, "last_execution", exec_obj)
+
+                    return result
+                except Exception as exc:
+                    elapsed = time.perf_counter() - start_time
+                    duration_ms = max(1, int(elapsed * 1000))
+                    exec_meta["error"] = str(exc)
+                    exec_meta["error_type"] = type(exc).__name__
+
+                    if target_repo is not None:
+                        target_repo.record_execution(
+                            inputs=inputs_str,
+                            outputs=f"Error: {exc}",
+                            duration_ms=duration_ms,
+                            metadata=exec_meta,
+                        )
+                    raise
+
+            return sync_wrapper
+
+    if _func is None:
+        return decorator
+    return decorator(_func)