autobots-devtools-shared-lib 0.6.1__tar.gz → 0.7.0__tar.gz

{autobots_devtools_shared_lib-0.6.1 → autobots_devtools_shared_lib-0.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: autobots-devtools-shared-lib
-Version: 0.6.1
+Version: 0.7.0
 Summary: Shared library functions to be used for all autobots projects
 License: MIT
 Author: Pralhad

{autobots_devtools_shared_lib-0.6.1 → autobots_devtools_shared_lib-0.7.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "autobots-devtools-shared-lib"
-version = "0.6.1"
+version = "0.7.0"
 description = "Shared library functions to be used for all autobots projects"
 readme = "README.md"
 authors = [

{autobots_devtools_shared_lib-0.6.1 → autobots_devtools_shared_lib-0.7.0}/src/autobots_devtools_shared_lib/eval/assertions/golden.py

@@ -31,21 +31,38 @@ class JsonDiff:
         return "\n".join(lines)
 
 
-def _diff_json(reference: Any, actual: Any, path: str = "") -> JsonDiff:
-    """Recursive deep diff between two JSON-like structures."""
+def _diff_json(
+    reference: Any,
+    actual: Any,
+    path: str = "",
+    ignore_fields: list[str] | None = None,
+) -> JsonDiff:
+    """Recursive deep diff between two JSON-like structures.
+
+    Args:
+        reference: Expected JSON value.
+        actual: Actual JSON value from agent output.
+        path: Dot-path prefix used in diff messages (internal).
+        ignore_fields: Key names to skip at any level of the dict tree.
+    """
+    ignore = set(ignore_fields or [])
     diff = JsonDiff()
 
     if isinstance(reference, dict) and isinstance(actual, dict):
         for key in reference:
+            if key in ignore:
+                continue
             child_path = f"{path}.{key}" if path else key
             if key not in actual:
                 diff.missing.append(f"{child_path}: {json.dumps(reference[key])}")
             else:
-                child = _diff_json(reference[key], actual[key], child_path)
+                child = _diff_json(reference[key], actual[key], child_path, ignore_fields)
                 diff.missing.extend(child.missing)
                 diff.unexpected.extend(child.unexpected)
                 diff.changed.extend(child.changed)
         for key in actual:
+            if key in ignore:
+                continue
             child_path = f"{path}.{key}" if path else key
             if key not in reference:
                 diff.unexpected.append(f"{child_path}: {json.dumps(actual[key])}")
@@ -58,7 +75,7 @@ def _diff_json(reference: Any, actual: Any, path: str = "") -> JsonDiff:
             elif i >= len(reference):
                 diff.unexpected.append(f"{child_path}: {json.dumps(actual[i])}")
             else:
-                child = _diff_json(reference[i], actual[i], child_path)
+                child = _diff_json(reference[i], actual[i], child_path, ignore_fields)
                 diff.missing.extend(child.missing)
                 diff.unexpected.extend(child.unexpected)
                 diff.changed.extend(child.changed)
@@ -135,7 +152,7 @@ def golden_match(output: AgentOutput, config: Any) -> AssertionResult:
     actual = output.structured_response
 
     if mode == "exact":
-        diff = _diff_json(reference, actual)
+        diff = _diff_json(reference, actual, ignore_fields=ignore_fields)
         if diff.has_differences:
             return AssertionResult(
                 passed=False,

{autobots_devtools_shared_lib-0.6.1 → autobots_devtools_shared_lib-0.7.0}/src/autobots_devtools_shared_lib/eval/assertions/registry.py

@@ -39,6 +39,7 @@ def _register_builtins() -> None:
         llm_judge,
         trajectory_quality,
     )
+    from autobots_devtools_shared_lib.eval.assertions.written_file import written_file_matches
 
     _REGISTRY.update(
         cast(
@@ -56,6 +57,7 @@ def _register_builtins() -> None:
                 "llm_judge": llm_judge,
                 "trajectory_quality": trajectory_quality,
                 "golden_match": golden_match,
+                "written_file_matches": written_file_matches,
             },
         )
     )

autobots_devtools_shared_lib-0.7.0/src/autobots_devtools_shared_lib/eval/assertions/written_file.py

@@ -0,0 +1,233 @@
+# ABOUTME: Assertions for files written to the file server workspace by agents.
+# ABOUTME: Complements golden_match (structured_response) for agents that output via file tools.
+"""written_file_matches assertion evaluator."""
+
+from __future__ import annotations
+
+import json
+import re
+from pathlib import Path
+from typing import Any
+
+import jsonschema as js
+
+from autobots_devtools_shared_lib.common.utils.fserver_client_utils import read_file as _read_file
+from autobots_devtools_shared_lib.eval.assertions.golden import _deep_structural_compare, _diff_json
+from autobots_devtools_shared_lib.eval.core.workspace import resolve_workspace_context
+from autobots_devtools_shared_lib.eval.models.result import AgentOutput, AssertionResult
+
+# ---------------------------------------------------------------------------
+# Mode handlers
+# ---------------------------------------------------------------------------
+
+FileModeHandler = Any  # (content: str, actual: Any, config: dict, name: str) -> AssertionResult
+
+_MODE_REGISTRY: dict[str, FileModeHandler] = {}
+
+
+def _register_mode(name: str, fn: FileModeHandler) -> None:
+    _MODE_REGISTRY[name] = fn
+
+
+def _resolve_mode(mode: str, assertion_name: str) -> FileModeHandler | AssertionResult:
+    if mode not in _MODE_REGISTRY:
+        available = ", ".join(sorted(_MODE_REGISTRY.keys()))
+        return AssertionResult(
+            passed=False,
+            name=assertion_name,
+            detail=f"Unknown mode: '{mode}'. Available: {available}",
+        )
+    return _MODE_REGISTRY[mode]
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+
+def _strip_code_fences(text: str) -> str:
+    match = re.search(r"```(?:\w+)?\s*\n?(.*?)\n?```", text, re.DOTALL)
+    return match.group(1).strip() if match else text.strip()
+
+
+def _read_workspace_file(file_name: str, raw_state: dict[str, Any]) -> str:
+    workspace_context = resolve_workspace_context(raw_state)
+    content = _read_file(file_name, workspace_context)
+    if content.startswith("Error"):
+        raise RuntimeError(f"File server read failed for '{file_name}': {content}")
+    return content
+
+
+def _load_json(content: str, assertion_name: str) -> tuple[Any, AssertionResult | None]:
+    try:
+        return json.loads(_strip_code_fences(content)), None
+    except json.JSONDecodeError as e:
+        return None, AssertionResult(
+            passed=False, name=assertion_name, detail=f"JSON parse error: {e}"
+        )
+
+
+def _load_reference(
+    config: dict[str, Any], assertion_name: str
+) -> tuple[Any, AssertionResult | None]:
+    ref_path_str = config.get("reference")
+    if not ref_path_str:
+        return None, AssertionResult(
+            passed=False,
+            name=assertion_name,
+            detail=f"Mode '{config.get('mode')}' requires 'reference'",
+        )
+    ref_path = Path(ref_path_str)
+    if not ref_path.exists():
+        return None, AssertionResult(
+            passed=False, name=assertion_name, detail=f"Reference not found: {ref_path}"
+        )
+    return json.loads(ref_path.read_text()), None
+
+
+# ---------------------------------------------------------------------------
+# Built-in mode implementations
+# ---------------------------------------------------------------------------
+
+
+def _mode_contains(
+    content: str, _actual: Any, config: dict[str, Any], name: str
+) -> AssertionResult:
+    value = str(config.get("value", ""))
+    found = value.lower() in content.lower()
+    return AssertionResult(
+        passed=found,
+        name=name,
+        detail=f"{'Found' if found else 'Not found'}: {value!r}",
+    )
+
+
+def _mode_schema(_content: str, actual: Any, config: dict[str, Any], name: str) -> AssertionResult:
+    schema_source = config.get("schema")
+    if schema_source is None:
+        return AssertionResult(
+            passed=False, name=name, detail="Mode 'schema' requires 'schema' key"
+        )
+    try:
+        schema: dict[str, Any] = (
+            json.loads(Path(str(schema_source)).read_text())
+            if isinstance(schema_source, str)
+            else schema_source
+        )
+        js.validate(instance=actual, schema=schema)
+        return AssertionResult(passed=True, name=name, detail="Schema valid")
+    except js.ValidationError as e:
+        return AssertionResult(passed=False, name=name, detail=f"Schema invalid: {e.message}")
+    except (FileNotFoundError, OSError) as e:
+        return AssertionResult(passed=False, name=name, detail=f"Schema load error: {e}")
+
+
+def _mode_exact(_content: str, actual: Any, config: dict[str, Any], name: str) -> AssertionResult:
+    reference, err = _load_reference(config, name)
+    if err:
+        return err
+    ignore_fields: list[str] = config.get("ignore_fields", [])
+    diff = _diff_json(reference, actual, ignore_fields=ignore_fields)
+    if diff.has_differences:
+        return AssertionResult(passed=False, name=name, detail=diff.to_detail())
+    return AssertionResult(passed=True, name=name, detail="Exact match")
+
+
+def _mode_structural(
+    _content: str, actual: Any, config: dict[str, Any], name: str
+) -> AssertionResult:
+    reference, err = _load_reference(config, name)
+    if err:
+        return err
+    ignore_fields: list[str] = config.get("ignore_fields", [])
+    issues = _deep_structural_compare(reference, actual, ignore_fields=ignore_fields)
+    if issues:
+        return AssertionResult(
+            passed=False,
+            name=name,
+            detail="Structural mismatch:\n" + "\n".join(f"  {i}" for i in issues),
+        )
+    return AssertionResult(passed=True, name=name, detail="Structural match")
+
+
+_register_mode("contains", _mode_contains)
+_register_mode("schema", _mode_schema)
+_register_mode("exact", _mode_exact)
+_register_mode("structural", _mode_structural)
+
+# ---------------------------------------------------------------------------
+# Core dispatch
+# ---------------------------------------------------------------------------
+
+_JSON_MODES = {"schema", "exact", "structural"}
+
+
+def _single_file_match(
+    path: str, config: dict[str, Any], agent_output: AgentOutput
+) -> AssertionResult:
+    assertion_name = f"written_file_matches:{path}"
+    mode = config.get("mode", "schema")
+
+    handler = _resolve_mode(mode, assertion_name)
+    if isinstance(handler, AssertionResult):
+        return handler
+
+    try:
+        content = _read_workspace_file(path, agent_output.raw_state)
+    except RuntimeError as e:
+        return AssertionResult(passed=False, name=assertion_name, detail=str(e))
+
+    # Modes that need parsed JSON get it up front
+    actual: Any = None
+    if mode in _JSON_MODES:
+        actual, err = _load_json(content, assertion_name)
+        if err:
+            return err
+
+    return handler(content, actual, config, assertion_name)
+
+
+# ---------------------------------------------------------------------------
+# Public entry point
+# ---------------------------------------------------------------------------
+
+
+def written_file_matches(agent_output: AgentOutput, config: Any) -> AssertionResult:
+    """Assert one or more workspace files match expected content/structure.
+
+    Config can be a single dict or a list of dicts. All entries must pass.
+
+    YAML config keys (per entry):
+        path (str): Workspace-relative file path (required).
+        mode (str): schema | exact | structural | contains (default: schema).
+        schema (str): Path to JSON schema file (mode=schema).
+        reference (str): Path to golden reference file (mode=exact|structural).
+        ignore_fields (list[str]): Keys to skip at any dict level (mode=exact|structural).
+        value (str): Substring to search for (mode=contains).
+    """
+    if isinstance(config, list):
+        entries = config
+    elif isinstance(config, dict):
+        entries = [config]
+    else:
+        return AssertionResult(
+            passed=False, name="written_file_matches", detail="Config must be a dict or list"
+        )
+
+    results = [_single_file_match(entry.get("path", ""), entry, agent_output) for entry in entries]
+    failures = [r for r in results if not r.passed]
+    if failures:
+        if len(entries) == 1:
+            return failures[0]
+        return AssertionResult(
+            passed=False,
+            name="written_file_matches",
+            detail="\n".join(f"{r.name}: {r.detail}" for r in failures),
+        )
+    if len(entries) == 1:
+        return results[0]
+    return AssertionResult(
+        passed=True,
+        name="written_file_matches",
+        detail=f"All {len(results)} files matched",
+    )

{autobots_devtools_shared_lib-0.6.1 → autobots_devtools_shared_lib-0.7.0}/src/autobots_devtools_shared_lib/eval/core/runner.py

@@ -8,6 +8,7 @@ from typing import TYPE_CHECKING, Any
 
 from autobots_devtools_shared_lib.dynagent.agents.invocation_utils import ainvoke_agent
 from autobots_devtools_shared_lib.eval.assertions.registry import resolve_assertion
+from autobots_devtools_shared_lib.eval.core.workspace import resolve_eval_state_schema
 from autobots_devtools_shared_lib.eval.models.result import (
     AgentOutput,
     AssertionResult,
@@ -111,6 +112,7 @@ async def run_linear_eval(
                 config=config,
                 enable_tracing=trace_metadata is not None,
                 trace_metadata=trace_metadata,
+                state_schema=resolve_eval_state_schema(),
             )
 
             agent_output = _build_agent_output(result)

autobots_devtools_shared_lib-0.7.0/src/autobots_devtools_shared_lib/eval/core/workspace.py

@@ -0,0 +1,129 @@
+# ABOUTME: Workspace staging for eval runs + WorkspaceContextProvider interface.
+# ABOUTME: Consumers register a provider so shared-lib never hard-codes workspace path formation.
+"""Workspace file staging and pluggable workspace context provider."""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Protocol
+
+from autobots_devtools_shared_lib.common.utils.fserver_client_utils import write_file
+from autobots_devtools_shared_lib.dynagent.models.state import Dynagent
+
+if TYPE_CHECKING:
+    from langchain.agents import AgentState
+
+    from autobots_devtools_shared_lib.eval.models.eval_case import SetupConfig
+
+_state_schema: type[Any] = Dynagent
+
+
+def register_eval_state_schema(schema: type[Any]) -> None:
+    """Register the LangGraph state schema for eval agent invocations.
+
+    Call once in conftest.py when your agents use a custom state class (e.g. MerState)
+    that extends Dynagent with domain-specific fields. Without this, ainvoke_agent
+    defaults to Dynagent and drops extra state fields (e.g. jira_number, repo_name).
+
+    Example (MER consumer)::
+
+        from autobots_devtools_shared_lib.eval.core.workspace import register_eval_state_schema
+        from autobots_agents_mer.common.models.state import MerState
+
+        register_eval_state_schema(MerState)
+    """
+    global _state_schema
+    _state_schema = schema
+
+
+def resolve_eval_state_schema() -> type[AgentState]:
+    """Return the registered state schema for eval agent invocations."""
+    return _state_schema
+
+
+class WorkspaceContextProvider(Protocol):
+    """Protocol for building file-server workspace context from agent state.
+
+    Implement this in your consumer conftest.py and register via
+    register_workspace_context_provider(). Path formation is intentionally
+    kept out of shared-lib — each consumer app may have a different convention.
+
+    Example (MER consumer)::
+
+        class MerWorkspaceContextProvider:
+            def get_workspace_context(self, state: dict) -> str:
+                ws = get_workspace_context(state)  # MER util
+                return json.dumps(ws)
+
+        register_workspace_context_provider(MerWorkspaceContextProvider())
+    """
+
+    def get_workspace_context(self, state: dict[str, Any]) -> str:
+        """Return workspace_context JSON string for fserver_client_utils calls.
+
+        Args:
+            state: Agent state dict (e.g. user_name, repo_name, jira_number).
+
+        Returns:
+            JSON string, e.g. '{"workspace_base_path": "alice/fbp-core-MER-99999"}'.
+        """
+        ...
+
+
+_provider: WorkspaceContextProvider | None = None
+
+
+def register_workspace_context_provider(provider: WorkspaceContextProvider) -> None:
+    """Register the workspace context provider.
+
+    Call once at eval startup, typically from conftest.py, before any evals run.
+    """
+    global _provider
+    _provider = provider
+
+
+def resolve_workspace_context(state: dict[str, Any]) -> str:
+    """Return workspace_context JSON via the registered provider.
+
+    Raises:
+        RuntimeError: If no provider has been registered.
+    """
+    if _provider is None:
+        raise RuntimeError(
+            "No WorkspaceContextProvider registered. "
+            "Call register_workspace_context_provider() in your conftest.py before running evals."
+        )
+    return _provider.get_workspace_context(state)
+
+
+def setup_workspace(config: SetupConfig, state: dict[str, Any] | None = None) -> None:
+    """Stage fixture files into the file server workspace before agent invocation.
+
+    Args:
+        config: Setup configuration with workspace_files to stage.
+        state: EvalCase state dict used to resolve workspace context via the provider.
+
+    Raises:
+        FileNotFoundError: If a source fixture file does not exist.
+        RuntimeError: If the file server returns an error or no provider is registered.
+    """
+    app_root_path = os.getenv("APP_ROOT_PATH", "")
+    workspace_context = resolve_workspace_context(state or {})
+
+    for wf in config.workspace_files:
+        src = Path(app_root_path, wf.src)
+        if not src.exists():
+            raise FileNotFoundError(
+                f"Fixture file not found: {src}. "
+                f"Ensure the file exists in the eval fixtures directory."
+            )
+        content = src.read_text(encoding="utf-8")
+        result = write_file(wf.dest, content, workspace_context)
+        if result.startswith("Error"):
+            raise RuntimeError(f"File server failed to stage '{wf.src}' → '{wf.dest}': {result}")
+
+
+def teardown_workspace(_workspace_path: str) -> None:
+    """No-op: file server manages its own storage. Kept for interface compatibility."""
+    pass

{autobots_devtools_shared_lib-0.6.1 → autobots_devtools_shared_lib-0.7.0}/src/autobots_devtools_shared_lib/eval/pytest_plugin/fixtures.py

@@ -51,7 +51,6 @@ def make_dynagent_eval(
 
     async def _eval(eval_case: EvalCase) -> EvalResult:
         session_id = str(uuid.uuid4())
-        workspace_path = "/Users/shruthi/Projects/workspace/khushboo-2802394_infosys/fbp-core-genai-sanity-MER-9999"
 
         config: RunnableConfig = {
             "configurable": {
@@ -67,7 +66,7 @@ def make_dynagent_eval(
 
         try:
             # Stage workspace files
-            setup_workspace(eval_case.setup, workspace_path)
+            setup_workspace(eval_case.setup, state=eval_case.state or None)
 
             # Run the eval
             if eval_case.mode == "linear":
@@ -104,7 +103,7 @@ def make_dynagent_eval(
                 post_scores(session_id, result)
 
         finally:
-            teardown_workspace(workspace_path)
+            teardown_workspace("")
 
         return result
 

autobots_devtools_shared_lib-0.6.1/src/autobots_devtools_shared_lib/eval/core/workspace.py

@@ -1,51 +0,0 @@
-# ABOUTME: Workspace file staging for eval runs.
-# ABOUTME: Copies fixture files into workspace directory before agent invocation.
-"""Workspace file staging for eval runs."""
-
-from __future__ import annotations
-
-import shutil
-from pathlib import Path
-from typing import TYPE_CHECKING
-
-if TYPE_CHECKING:
-    from autobots_devtools_shared_lib.eval.models.eval_case import SetupConfig
-
-
-def setup_workspace(config: SetupConfig, workspace_path: str) -> None:
-    """Create workspace directory and stage fixture files.
-
-    Args:
-        config: Setup configuration with workspace_files to stage.
-        workspace_path: Target workspace directory path.
-
-    Raises:
-        FileNotFoundError: If a source fixture file does not exist.
-    """
-    import os
-
-    app_root_path = os.getenv("APP_ROOT_PATH", "")
-    workspace = Path(workspace_path)
-    workspace.mkdir(parents=True, exist_ok=True)
-
-    for wf in config.workspace_files:
-        src = Path(app_root_path, wf.src)
-        if not src.exists():
-            raise FileNotFoundError(
-                f"Fixture file not found: {src}. "
-                f"Ensure the file exists in the eval fixtures directory."
-            )
-        dest = workspace / wf.dest
-        dest.parent.mkdir(parents=True, exist_ok=True)
-        shutil.copy2(src, dest)
-
-
-def teardown_workspace(workspace_path: str) -> None:
-    """Remove workspace directory and all contents.
-
-    Args:
-        workspace_path: Workspace directory to remove.
-    """
-    workspace = Path(workspace_path)
-    if workspace.exists():
-        shutil.rmtree(workspace)