tactus 0.31.0__py3-none-any.whl

tactus/testing/events.py

@@ -0,0 +1,94 @@
+"""
+Structured log events for IDE integration.
+
+Provides Pydantic models for test and evaluation events
+that can be emitted as structured logs for IDE display.
+"""
+
+from datetime import datetime
+from typing import List
+from pydantic import BaseModel, Field
+
+from .models import TestResult, EvaluationResult
+
+
+class TestStartedEvent(BaseModel):
+    """Event emitted when tests start."""
+
+    event_type: str = "test_started"
+    procedure_file: str
+    total_scenarios: int
+    timestamp: datetime = Field(default_factory=datetime.now)
+
+
+class TestCompletedEvent(BaseModel):
+    """Event emitted when tests complete."""
+
+    event_type: str = "test_completed"
+    result: TestResult
+    timestamp: datetime = Field(default_factory=datetime.now)
+
+
+class TestScenarioStartedEvent(BaseModel):
+    """Event emitted when a scenario starts."""
+
+    event_type: str = "test_scenario_started"
+    scenario_name: str
+    timestamp: datetime = Field(default_factory=datetime.now)
+
+
+class TestScenarioCompletedEvent(BaseModel):
+    """Event emitted when a scenario completes."""
+
+    event_type: str = "test_scenario_completed"
+    scenario_name: str
+    status: str  # passed, failed, skipped
+    duration: float
+    total_cost: float = 0.0  # Total LLM cost for this scenario
+    total_tokens: int = 0  # Total tokens used in this scenario
+    timestamp: datetime = Field(default_factory=datetime.now)
+
+
+class EvaluationStartedEvent(BaseModel):
+    """Event emitted when evaluation starts."""
+
+    event_type: str = "evaluation_started"
+    procedure_file: str
+    total_scenarios: int
+    runs_per_scenario: int
+    timestamp: datetime = Field(default_factory=datetime.now)
+
+
+class EvaluationCompletedEvent(BaseModel):
+    """Event emitted when evaluation completes."""
+
+    event_type: str = "evaluation_completed"
+    results: List[EvaluationResult]
+    timestamp: datetime = Field(default_factory=datetime.now)
+
+
+class EvaluationScenarioStartedEvent(BaseModel):
+    """Event emitted when scenario evaluation starts."""
+
+    event_type: str = "evaluation_scenario_started"
+    scenario_name: str
+    runs: int
+    timestamp: datetime = Field(default_factory=datetime.now)
+
+
+class EvaluationScenarioCompletedEvent(BaseModel):
+    """Event emitted when scenario evaluation completes."""
+
+    event_type: str = "evaluation_scenario_completed"
+    result: EvaluationResult
+    timestamp: datetime = Field(default_factory=datetime.now)
+
+
+class EvaluationProgressEvent(BaseModel):
+    """Event emitted during evaluation progress."""
+
+    event_type: str = "evaluation_progress"
+    scenario_name: str
+    completed_runs: int
+    total_runs: int
+    timestamp: datetime = Field(default_factory=datetime.now)

tactus/testing/gherkin_parser.py

@@ -0,0 +1,134 @@
+"""
+Gherkin parser integration using gherkin-official library.
+"""
+
+import logging
+from typing import Optional
+
+try:
+    from gherkin.parser import Parser
+    from gherkin.token_scanner import TokenScanner
+
+    GHERKIN_AVAILABLE = True
+except ImportError:
+    GHERKIN_AVAILABLE = False
+
+from .models import ParsedStep, ParsedScenario, ParsedFeature
+
+
+logger = logging.getLogger(__name__)
+
+
+class GherkinParser:
+    """
+    Parses Gherkin text into structured Pydantic models.
+
+    Uses the official Gherkin parser library for accurate parsing.
+    """
+
+    def __init__(self):
+        if not GHERKIN_AVAILABLE:
+            raise ImportError(
+                "gherkin-official library not installed. Install with: pip install gherkin-official"
+            )
+        self.parser = Parser()
+
+    def parse(self, gherkin_text: str) -> ParsedFeature:
+        """
+        Parse Gherkin text into a ParsedFeature model.
+
+        Args:
+            gherkin_text: Raw Gherkin feature text
+
+        Returns:
+            ParsedFeature with all scenarios and steps
+
+        Raises:
+            ValueError: If Gherkin syntax is invalid
+        """
+        try:
+            scanner = TokenScanner(gherkin_text)
+            gherkin_document = self.parser.parse(scanner)
+
+            if not gherkin_document or not gherkin_document.get("feature"):
+                raise ValueError("No feature found in Gherkin text")
+
+            return self._convert_to_pydantic(gherkin_document)
+
+        except Exception as e:
+            logger.error(f"Failed to parse Gherkin: {e}")
+            raise ValueError(f"Invalid Gherkin syntax: {e}")
+
+    def _convert_to_pydantic(self, gherkin_document: dict) -> ParsedFeature:
+        """Convert Gherkin parser output to Pydantic models."""
+        feature_data = gherkin_document["feature"]
+
+        # Extract feature metadata
+        feature_name = feature_data.get("name", "Unnamed Feature")
+        feature_description = feature_data.get("description", "")
+        feature_tags = [tag["name"] for tag in feature_data.get("tags", [])]
+        feature_line = feature_data.get("location", {}).get("line")
+
+        # Parse scenarios
+        scenarios = []
+        for child in feature_data.get("children", []):
+            if child.get("scenario"):
+                scenario = self._parse_scenario(child["scenario"])
+                scenarios.append(scenario)
+
+        return ParsedFeature(
+            name=feature_name,
+            description=feature_description,
+            scenarios=scenarios,
+            tags=feature_tags,
+            line=feature_line,
+        )
+
+    def _parse_scenario(self, scenario_data: dict) -> ParsedScenario:
+        """Parse a scenario from Gherkin parser output."""
+        scenario_name = scenario_data.get("name", "Unnamed Scenario")
+        scenario_tags = [tag["name"] for tag in scenario_data.get("tags", [])]
+        scenario_line = scenario_data.get("location", {}).get("line")
+
+        # Parse steps
+        steps = []
+        for step_data in scenario_data.get("steps", []):
+            step = self._parse_step(step_data)
+            steps.append(step)
+
+        return ParsedScenario(
+            name=scenario_name,
+            tags=scenario_tags,
+            steps=steps,
+            line=scenario_line,
+        )
+
+    def _parse_step(self, step_data: dict) -> ParsedStep:
+        """Parse a step from Gherkin parser output."""
+        keyword = step_data.get("keyword", "").strip()
+        text = step_data.get("text", "")
+        line = step_data.get("location", {}).get("line")
+
+        return ParsedStep(
+            keyword=keyword,
+            message=text,
+            line=line,
+        )
+
+
+def parse_gherkin(gherkin_text: str) -> Optional[ParsedFeature]:
+    """
+    Convenience function to parse Gherkin text.
+
+    Args:
+        gherkin_text: Raw Gherkin feature text
+
+    Returns:
+        ParsedFeature or None if parsing fails
+    """
+    try:
+        parser = GherkinParser()
+        return parser.parse(gherkin_text)
+    except Exception as e:
+        logger.error(f"Failed to parse Gherkin: {e}")
+        return None

tactus/testing/mock_agent.py

@@ -0,0 +1,315 @@
+"""
+Mock agent primitive for BDD testing.
+
+Provides mock agent that simulates turns without LLM calls.
+Uses agent mock configurations from Mocks {} in .tac files.
+"""
+
+import logging
+from typing import Any, Dict, List, Optional
+
+logger = logging.getLogger(__name__)
+
+
+class MockAgentResult:
+    """Result from a mock agent turn."""
+
+    def __init__(
+        self,
+        message: str = "",
+        tool_calls: Optional[List[Dict]] = None,
+        data: Optional[Dict[str, Any]] = None,
+        usage: Optional[Dict[str, Any]] = None,
+        new_messages: Optional[List[Dict[str, Any]]] = None,
+        lua_table_from: Optional[Any] = None,
+    ):
+        self.message = message
+        self.response = message
+        self.tool_calls = tool_calls or []
+        self.data = data or {}
+        self.usage = usage or {}
+        self.cost = 0.0
+        try:
+            self.tokens = int(self.usage.get("total_tokens", 0) or 0)
+        except Exception:
+            self.tokens = 0
+
+        self._new_messages = new_messages or []
+        self._lua_table_from = lua_table_from
+
+    def __repr__(self) -> str:
+        return (
+            f"MockAgentResult(message={self.message!r}, tool_calls={len(self.tool_calls)}, "
+            f"data_keys={len(self.data) if hasattr(self.data, '__len__') else 'n/a'})"
+        )
+
+    def new_messages(self):
+        """
+        Return messages generated in this turn.
+
+        In Lua, callers expect a table (for `#msgs` and 1-based indexing).
+        """
+        if self._lua_table_from is not None:
+            try:
+                return self._lua_table_from(self._new_messages)
+            except Exception:
+                # Fall back to raw Python list if conversion fails.
+                pass
+        return self._new_messages
+
+
+class MockAgentPrimitive:
+    """
+    Mock agent that simulates turns without making LLM calls.
+
+    Uses agent mock configurations from Mocks {} in .tac files.
+    The mock config specifies exactly which tool calls to simulate,
+    allowing tests to pass in CI without real LLM calls.
+
+    Example Mocks {} configuration:
+        Mocks {
+            my_agent = {
+                tool_calls = {
+                    {tool = "search", args = {query = "test"}},
+                    {tool = "done", args = {reason = "completed"}}
+                },
+                message = "I found the results."
+            }
+        }
+    """
+
+    def __init__(
+        self,
+        name: str,
+        tool_primitive: Any,
+        registry: Any = None,
+        mock_manager: Any = None,
+        lua_runtime: Any = None,
+        lua_table_from: Any = None,
+    ):
+        """
+        Initialize mock agent.
+
+        Args:
+            name: Agent name
+            tool_primitive: ToolPrimitive for recording tool calls
+            registry: Registry containing agent_mocks configuration
+            mock_manager: Optional MockManager (for tool response mocking)
+        """
+        self.name = name
+        self.tool_primitive = tool_primitive
+        self.registry = registry
+        self.mock_manager = mock_manager
+        self.turn_count = 0
+        if lua_table_from is not None:
+            self._lua_table_from = lua_table_from
+        elif lua_runtime is not None and hasattr(lua_runtime, "table_from"):
+            self._lua_table_from = lua_runtime.table_from
+        else:
+            self._lua_table_from = None
+
+    def turn(self, opts: Optional[Dict[str, Any]] = None) -> MockAgentResult:
+        """
+        Simulate an agent turn by executing configured tool calls.
+
+        Looks up agent mock config in registry.agent_mocks and executes
+        the specified tool calls, then returns the configured message.
+
+        Args:
+            opts: Optional turn options (for compatibility)
+
+        Returns:
+            MockAgentResult with message and tool call info
+
+        Raises:
+            ValueError: If no mock config is found for this agent
+        """
+        opts = opts or {}
+        self.turn_count += 1
+        logger.info(f"Mock agent turn: {self.name} (turn {self.turn_count})")
+
+        # Get agent mock config
+        mock_config = self._get_agent_mock_config()
+
+        if mock_config is None:
+            raise ValueError(
+                f"Agent '{self.name}' requires mock config in Mocks {{}}. "
+                f"Add a mock configuration like:\n"
+                f"Mocks {{\n"
+                f"    {self.name} = {{\n"
+                f"        tool_calls = {{\n"
+                f'            {{tool = "done", args = {{reason = "completed"}}}}\n'
+                f"        }},\n"
+                f'        message = "Task completed."\n'
+                f"    }}\n"
+                f"}}"
+            )
+
+        temporal_turns = getattr(mock_config, "temporal", None) or []
+        if temporal_turns:
+            injected = opts.get("message")
+
+            selected_turn = None
+            if injected is not None:
+                for turn in temporal_turns:
+                    if isinstance(turn, dict) and turn.get("when_message") == injected:
+                        selected_turn = turn
+                        break
+
+            if selected_turn is None:
+                idx = self.turn_count - 1  # 1-indexed turns
+                if idx < 0:
+                    idx = 0
+                if idx >= len(temporal_turns):
+                    idx = len(temporal_turns) - 1
+                selected_turn = temporal_turns[idx]
+
+            turn = selected_turn
+            if isinstance(turn, dict):
+                message = turn.get("message", mock_config.message)
+                tool_calls = turn.get("tool_calls", mock_config.tool_calls)
+                data = turn.get("data", mock_config.data)
+                raw_usage = turn.get("usage", mock_config.usage)
+            else:
+                message = mock_config.message
+                tool_calls = mock_config.tool_calls
+                data = mock_config.data
+                raw_usage = mock_config.usage
+        else:
+            message = mock_config.message
+            tool_calls = mock_config.tool_calls
+            data = mock_config.data
+            raw_usage = mock_config.usage
+
+        # Execute the configured tool calls
+        tool_calls_executed = self._execute_tool_calls(tool_calls)
+
+        # Structured payload (optional) for result.data
+        data = data or {}
+        if not data:
+            data = {"response": message}
+
+        # Token usage payload (optional) for result.usage
+        usage = dict(raw_usage) if isinstance(raw_usage, dict) else {}
+        prompt_tokens = int(usage.get("prompt_tokens", 0) or 0)
+        completion_tokens = int(usage.get("completion_tokens", 0) or 0)
+        total_tokens = usage.get("total_tokens")
+        if total_tokens is None:
+            total_tokens = prompt_tokens + completion_tokens
+        total_tokens = int(total_tokens or 0)
+        usage.setdefault("prompt_tokens", prompt_tokens)
+        usage.setdefault("completion_tokens", completion_tokens)
+        usage.setdefault("total_tokens", total_tokens)
+
+        # Messages generated in this turn
+        user_message = opts.get("message")
+        new_messages = []
+        if user_message:
+            new_messages.append({"role": "user", "content": user_message})
+        if message:
+            new_messages.append({"role": "assistant", "content": message})
+
+        # Return the configured message
+        return MockAgentResult(
+            message=message,
+            tool_calls=tool_calls_executed,
+            data=data,
+            usage=usage,
+            new_messages=new_messages,
+            lua_table_from=self._lua_table_from,
+        )
+
+    def _get_agent_mock_config(self) -> Optional[Any]:
+        """
+        Get agent mock config from registry.agent_mocks.
+
+        Returns:
+            AgentMockConfig if found, None otherwise
+        """
+        if not self.registry:
+            return None
+
+        # Check for agent mock in registry.agent_mocks
+        if hasattr(self.registry, "agent_mocks"):
+            return self.registry.agent_mocks.get(self.name)
+
+        return None
+
+    def _execute_tool_calls(self, tool_calls: List[Dict[str, Any]]) -> List[Dict]:
+        """
+        Execute the configured tool calls.
+
+        Records each tool call via the tool_primitive, which will
+        use mock responses from the MockManager if configured.
+
+        Args:
+            tool_calls: List of tool call configs [{tool: "name", args: {...}}, ...]
+
+        Returns:
+            List of executed tool calls with results
+        """
+        executed = []
+
+        for tool_call in tool_calls:
+            tool_name = tool_call.get("tool")
+            args = tool_call.get("args", {})
+
+            if not tool_name:
+                logger.warning(f"Skipping invalid tool call config: {tool_call}")
+                continue
+
+            logger.debug(f"Mock agent {self.name} executing tool call: {tool_name}({args})")
+
+            # Record the tool call via tool primitive
+            # MockedToolPrimitive.record_call(tool_name, args) returns the mock response
+            result = None
+            if self.tool_primitive:
+                try:
+                    # record_call returns the mock response and records the call
+                    result = self.tool_primitive.record_call(tool_name, args)
+                except Exception as e:
+                    logger.warning(f"Error recording tool call {tool_name}: {e}")
+                    result = {"status": "ok", "tool": tool_name}
+
+            executed.append(
+                {
+                    "tool": tool_name,
+                    "args": args,
+                    "result": result,
+                }
+            )
+
+        return executed
+
+    def __call__(self, inputs: Optional[Dict[str, Any]] = None) -> MockAgentResult:
+        """
+        Execute an agent turn using the callable interface.
+
+        This makes the mock agent callable like real agents:
+            result = worker({message = "Hello"})
+
+        Args:
+            inputs: Input dict (ignored in mock mode, tool calls are from config)
+
+        Returns:
+            MockAgentResult with response and tool call info
+        """
+        inputs = inputs or {}
+
+        # Convert Lua table to dict if needed
+        if hasattr(inputs, "items"):
+            try:
+                inputs = dict(inputs.items())
+            except (AttributeError, TypeError):
+                pass
+
+        # Extract message field for logging
+        message = inputs.get("message", "")
+        if message:
+            logger.debug(f"Mock agent {self.name} received message: {message}")
+
+        # Execute the turn (tool calls come from config, not inputs)
+        return self.turn(inputs)
+
+    def __repr__(self) -> str:
+        return f"MockAgentPrimitive({self.name}, turns={self.turn_count})"