tactus 0.30.0__py3-none-any.whl → 0.31.1__py3-none-any.whl

tactus/testing/steps/builtin.py

@@ -3,7 +3,6 @@ Built-in step definitions for Tactus primitives.
 
 Provides a comprehensive library of steps for testing:
 - Tool calls
-- Stage transitions
 - State management
 - Procedure completion
 - Iterations and timing
@@ -14,6 +13,7 @@ Provides a comprehensive library of steps for testing:
 
 import logging
 import re
+import ast
 from typing import Any
 
 from .registry import StepRegistry
@@ -22,6 +22,30 @@ from .registry import StepRegistry
 logger = logging.getLogger(__name__)
 
 
+def _parse_step_string_literal(value: str) -> tuple[str, bool]:
+    """
+    Parse an optional quoted string literal from a step capture group.
+
+    Supports single-quoted or double-quoted Python-style escapes, e.g.:
+      "Hello! I'm World"
+      'He said: "hi"'
+      "Line 1\\nLine 2"
+
+    Returns:
+      (parsed_value, was_quoted)
+    """
+    stripped = value.strip()
+    if len(stripped) >= 2 and stripped[0] in {"'", '"'} and stripped[-1] == stripped[0]:
+        try:
+            parsed = ast.literal_eval(stripped)
+            if isinstance(parsed, str):
+                return parsed, True
+        except Exception:
+            # Fall back to raw string if the literal is malformed.
+            return stripped, True
+    return value, False
+
+
 def register_builtin_steps(registry: StepRegistry) -> None:
     """
     Register all built-in step definitions.
@@ -32,9 +56,6 @@ def register_builtin_steps(registry: StepRegistry) -> None:
     # Tool-related steps
     register_tool_steps(registry)
 
-    # Stage-related steps
-    register_stage_steps(registry)
-
     # State-related steps
     register_state_steps(registry)
 
@@ -69,34 +90,40 @@ def register_builtin_steps(registry: StepRegistry) -> None:
 def register_tool_steps(registry: StepRegistry) -> None:
     """Register tool-related step definitions."""
 
-    registry.register(r"the (?P<tool>\w+) tool should be called", step_tool_called)
+    registry.register(r"the (?P<tool>[-\w]+) tool should be called", step_tool_called)
 
-    registry.register(r"the (?P<tool>\w+) tool should not be called", step_tool_not_called)
+    registry.register(r"the (?P<tool>[-\w]+) tool should not be called", step_tool_not_called)
 
     registry.register(
-        r"the (?P<tool>\w+) tool should be called at least (?P<n>\d+) time",
+        r"the (?P<tool>[-\w]+) tool should be called at least (?P<n>\d+) time",
         step_tool_called_at_least,
     )
 
     registry.register(
-        r"the (?P<tool>\w+) tool should be called at least (?P<n>\d+) times",
+        r"the (?P<tool>[-\w]+) tool should be called at least (?P<n>\d+) times",
         step_tool_called_at_least,
     )
 
     registry.register(
-        r"the (?P<tool>\w+) tool should be called exactly (?P<n>\d+) time", step_tool_called_exactly
+        r"the (?P<tool>[-\w]+) tool should be called exactly (?P<n>\d+) time",
+        step_tool_called_exactly,
     )
 
     registry.register(
-        r"the (?P<tool>\w+) tool should be called exactly (?P<n>\d+) times",
+        r"the (?P<tool>[-\w]+) tool should be called exactly (?P<n>\d+) times",
         step_tool_called_exactly,
     )
 
     registry.register(
-        r"the (?P<tool>\w+) tool should be called with (?P<param>\w+)=(?P<value>.+)",
+        r"the (?P<tool>[-\w]+) tool should be called with (?P<param>\w+)=(?P<value>.+)",
         step_tool_called_with_param,
     )
 
+    registry.register(
+        r'the tool "(?P<tool>[-\w]+)" returns (?P<value>.+)',
+        step_mock_tool_returns,
+    )
+
 
 def step_tool_called(context: Any, tool: str) -> None:
     """Check if a tool was called."""
@@ -132,24 +159,20 @@ def step_tool_called_with_param(context: Any, tool: str, param: str, value: str)
     assert found, f"Tool '{tool}' was not called with {param}={value}"
 
 
-# Stage-related steps
-
-
-def register_stage_steps(registry: StepRegistry) -> None:
-    """Register stage-related step definitions."""
-
-    registry.register(r"the procedure has started", step_procedure_started)
-
-    registry.register(r"the stage is (?P<stage>\w+)", step_stage_is)
-
-    registry.register(r"the stage should be (?P<stage>\w+)", step_stage_is)
+def step_mock_tool_returns(context: Any, tool: str, value: str) -> None:
+    """Configure a runtime tool mock response for this scenario."""
+    parsed_value, was_quoted = _parse_step_string_literal(value)
+    if not was_quoted:
+        try:
+            parsed_value = ast.literal_eval(parsed_value)
+        except Exception:
+            # Treat unquoted values as plain strings (e.g., positive/neutral)
+            pass
 
-    registry.register(
-        r"the stage should transition from (?P<from_stage>\w+) to (?P<to_stage>\w+)",
-        step_stage_transition,
-    )
+    if not hasattr(context, "mock_tool_returns"):
+        raise AssertionError("Context does not support tool mocking")
 
-    registry.register(r"we are in stage (?P<stage>\w+)", step_in_stage)
+    context.mock_tool_returns(tool, parsed_value)
 
 
 def step_procedure_started(context: Any) -> None:
@@ -159,32 +182,6 @@ def step_procedure_started(context: Any) -> None:
     assert context is not None, "Test context not initialized"
 
 
-def step_stage_is(context: Any, stage: str) -> None:
-    """Check if current stage matches expected."""
-    current = context.current_stage()
-    assert current == stage, f"Expected stage '{stage}', but current stage is '{current}'"
-
-
-def step_stage_transition(context: Any, from_stage: str, to_stage: str) -> None:
-    """Check if stage transition occurred."""
-    history = context.stage_history()
-
-    # Build list of transitions
-    transitions = [(history[i], history[i + 1]) for i in range(len(history) - 1)]
-
-    expected_transition = (from_stage, to_stage)
-    assert expected_transition in transitions, (
-        f"Stage transition from '{from_stage}' to '{to_stage}' did not occur. "
-        f"Actual transitions: {transitions}"
-    )
-
-
-def step_in_stage(context: Any, stage: str) -> None:
-    """Check if currently in specified stage."""
-    current = context.current_stage()
-    assert current == stage, f"Not in stage '{stage}', current stage is '{current}'"
-
-
 # State-related steps
 
 
@@ -201,8 +198,12 @@ def register_state_steps(registry: StepRegistry) -> None:
 def step_state_equals(context: Any, key: str, value: str) -> None:
     """Check if state value equals expected."""
     actual = context.state_get(key)
+    value, was_quoted = _parse_step_string_literal(value)
     # Convert to string for comparison
     actual_str = str(actual) if actual is not None else "None"
+    if was_quoted:
+        assert actual_str == value, f"State '{key}' is '{actual_str}', expected '{value}'"
+        return
     assert actual_str == value, f"State '{key}' is '{actual_str}', expected '{value}'"
 
 
@@ -224,6 +225,14 @@ def step_state_contains(context: Any, key: str) -> None:
 def register_output_steps(registry: StepRegistry) -> None:
     """Register output-related step definitions."""
 
+    registry.register(r"the output should exist", step_output_value_exists)
+    registry.register(r"the output should be (?P<value>.+)", step_output_value_equals)
+    registry.register(
+        r"the output should fuzzy match (?P<value>.+) with threshold (?P<threshold>[0-9]*\.?[0-9]+)",
+        step_output_value_fuzzy_match,
+    )
+    registry.register(r"the output should fuzzy match (?P<value>.+)", step_output_value_fuzzy_match)
+
     registry.register(r"the output (?P<key>\w+) should be (?P<value>.+)", step_output_equals)
 
     registry.register(
@@ -238,6 +247,11 @@ def register_output_steps(registry: StepRegistry) -> None:
 def step_output_equals(context: Any, key: str, value: str) -> None:
     """Check if output value equals expected."""
     actual = context.output_get(key)
+    value, was_quoted = _parse_step_string_literal(value)
+    if was_quoted:
+        actual_str = str(actual) if actual is not None else "None"
+        assert actual_str == value, f"Output '{key}' is '{actual_str}', expected '{value}'"
+        return
 
     # Handle boolean comparison specially
     if value.lower() in ("true", "false"):
@@ -266,9 +280,132 @@ def step_output_equals(context: Any, key: str, value: str) -> None:
             assert actual_str == value, f"Output '{key}' is '{actual_str}', expected '{value}'"
 
 
+def step_output_value_exists(context: Any) -> None:
+    """Check if scalar output exists (non-None)."""
+    actual = context.output_value()
+    assert actual is not None, "Output is missing"
+
+
+def step_output_value_equals(context: Any, value: str) -> None:
+    """Check if scalar output equals expected."""
+    actual = context.output_value()
+    value, was_quoted = _parse_step_string_literal(value)
+    if was_quoted:
+        actual_str = str(actual) if actual is not None else "None"
+        assert actual_str == value, f"Output is '{actual_str}', expected '{value}'"
+        return
+
+    # Handle boolean comparison specially
+    if value.lower() in ("true", "false"):
+        expected_bool = value.lower() == "true"
+        if isinstance(actual, bool):
+            assert actual == expected_bool, f"Output is {actual}, expected {expected_bool}"
+        else:
+            actual_str = str(actual).lower() if actual is not None else "none"
+            assert actual_str == value.lower(), f"Output is '{actual}', expected '{value}'"
+        return
+
+    # Try numeric comparison first
+    try:
+        expected_num = float(value)
+        if isinstance(actual, (int, float)):
+            assert actual == expected_num, f"Output is {actual}, expected {expected_num}"
+        else:
+            actual_num = float(actual)
+            assert actual_num == expected_num, f"Output is '{actual}', expected {expected_num}"
+        return
+    except (ValueError, TypeError):
+        pass
+
+    actual_str = str(actual) if actual is not None else "None"
+    assert actual_str == value, f"Output is '{actual_str}', expected '{value}'"
+
+
+def step_output_value_fuzzy_match(context: Any, value: str, threshold: str = "0.8") -> None:
+    """Check if scalar output is similar to expected value above a threshold.
+
+    This is a deterministic, non-LLM fuzzy match based on string similarity.
+
+    Default behavior:
+    - Case-insensitive (compares lowercased text)
+    - Punctuation-insensitive (strips punctuation)
+
+    Multi-match syntax (best-effort):
+      Then the output should fuzzy match any of ["Hello", "Hi", "Hey"] with threshold 0.9
+    """
+    import difflib
+
+    def _normalize_text(text: str) -> str:
+        # Lowercase + strip punctuation + collapse whitespace.
+        normalized = re.sub(r"[^\w\s]", "", text.lower())
+        normalized = re.sub(r"\s+", " ", normalized).strip()
+        return normalized
+
+    actual = context.output_value()
+    assert actual is not None, "Output is missing"
+
+    try:
+        threshold_f = float(threshold)
+    except ValueError:
+        raise AssertionError(f"Invalid threshold: {threshold}")
+
+    expected_raw, was_quoted = _parse_step_string_literal(value)
+    expected_raw = expected_raw.strip() if not was_quoted else expected_raw
+
+    expected_values: list[str]
+
+    if expected_raw.lower().startswith("any of "):
+        values_str = expected_raw[7:].strip()
+        try:
+            parsed = ast.literal_eval(values_str)
+        except Exception:
+            parsed = None
+
+        expected_values = []
+        if isinstance(parsed, (list, tuple)):
+            for item in parsed:
+                expected_values.append(item if isinstance(item, str) else str(item))
+        else:
+            parts = [p.strip() for p in values_str.split(",") if p.strip()]
+            for part in parts:
+                parsed_part, _ = _parse_step_string_literal(part)
+                expected_values.append(parsed_part)
+
+        if not expected_values:
+            raise AssertionError(f"No expected values provided: {value}")
+    else:
+        expected_values = [expected_raw]
+
+    actual_norm = _normalize_text(str(actual))
+    best_ratio = -1.0
+    best_expected = None
+
+    for expected in expected_values:
+        expected_norm = _normalize_text(expected)
+        if expected_norm and (expected_norm in actual_norm or actual_norm in expected_norm):
+            ratio = 1.0
+        else:
+            ratio = difflib.SequenceMatcher(None, actual_norm, expected_norm).ratio()
+
+        if ratio > best_ratio:
+            best_ratio = ratio
+            best_expected = expected
+
+    assert best_ratio >= threshold_f, (
+        f"Output similarity is {best_ratio:.3f} (threshold {threshold_f:.3f}). "
+        f"Output is '{actual}', best match was '{best_expected}'. "
+        f"Expected: {expected_values}"
+    )
+
+
 def step_output_not_equals(context: Any, key: str, value: str) -> None:
     """Check if output value does not equal the specified value."""
     actual = context.output_get(key)
+    value, was_quoted = _parse_step_string_literal(value)
+    if was_quoted:
+        actual_str = str(actual) if actual is not None else "None"
+        assert actual_str != value, f"Output '{key}' is '{actual_str}', should not be '{value}'"
+        return
 
     # Handle boolean comparison specially
     if value.lower() in ("true", "false"):
@@ -319,6 +456,7 @@ def step_output_contains(context: Any, key: str) -> None:
 def register_completion_steps(registry: StepRegistry) -> None:
     """Register completion-related step definitions."""
 
+    registry.register(r"the procedure has started", step_procedure_started)
     registry.register(r"the procedure should complete successfully", step_procedure_completes)
 
     registry.register(r"the procedure should fail", step_procedure_fails)
@@ -478,6 +616,23 @@ def register_agent_steps(registry: StepRegistry) -> None:
 
     registry.register(r"the (?P<agent>\w+) agent takes turns", step_agent_takes_turn)
 
+    registry.register(
+        r'the agent "(?P<agent>[^"]+)" responds with (?P<message>.+)',
+        step_mock_agent_responds_with,
+    )
+
+    registry.register(
+        r'the agent "(?P<agent>[^"]+)" calls tool "(?P<tool>[^"]+)" with args (?P<args>.+)',
+        step_mock_agent_calls_tool_with_args,
+    )
+
+    registry.register(
+        r'the agent "(?P<agent>[^"]+)" returns data (?P<data>.+)',
+        step_mock_agent_returns_data,
+    )
+
+    registry.register(r"the message is (?P<message>.+)", step_set_scenario_message)
+
     registry.register(r"the procedure run", step_procedure_runs)
 
     registry.register(r"the procedure runs", step_procedure_runs)
@@ -490,6 +645,61 @@ def step_agent_takes_turn(context: Any, agent: str) -> None:
     context.run_procedure()
 
 
+def step_mock_agent_responds_with(
+    context: Any, agent: str, message: str, when_message: str | None = None
+) -> None:
+    """Configure a per-scenario mock agent response (temporal)."""
+    message, _ = _parse_step_string_literal(message)
+    when_message_parsed = None
+    if when_message is not None:
+        when_message_parsed, _ = _parse_step_string_literal(when_message)
+    if not hasattr(context, "mock_agent_response"):
+        raise AssertionError("Context does not support agent mocking")
+    context.mock_agent_response(agent, message, when_message=when_message_parsed)
+
+
+def step_set_scenario_message(context: Any, message: str) -> None:
+    """Set the scenario's primary message for coordinating mocks with expectations."""
+    message, _ = _parse_step_string_literal(message)
+    if not hasattr(context, "set_scenario_message"):
+        raise AssertionError("Context does not support scenario message")
+    context.set_scenario_message(message)
+
+
+def step_mock_agent_calls_tool_with_args(context: Any, agent: str, tool: str, args: str) -> None:
+    """Configure a per-scenario mocked agent tool call (recorded into Tool primitive)."""
+    args_str, _ = _parse_step_string_literal(args)
+    try:
+        parsed_args = ast.literal_eval(args_str)
+    except Exception:
+        raise AssertionError(f"Invalid tool args literal: {args}")
+
+    if not isinstance(parsed_args, dict):
+        raise AssertionError(f"Tool args must be an object/dict, got {type(parsed_args).__name__}")
+
+    if not hasattr(context, "mock_agent_tool_call"):
+        raise AssertionError("Context does not support agent tool call mocking")
+
+    context.mock_agent_tool_call(agent, tool, parsed_args)
+
+
+def step_mock_agent_returns_data(context: Any, agent: str, data: str) -> None:
+    """Configure structured output mock data for an agent's next mocked turn."""
+    data_str, _ = _parse_step_string_literal(data)
+    try:
+        parsed = ast.literal_eval(data_str)
+    except Exception:
+        raise AssertionError(f"Invalid data literal: {data}")
+
+    if not isinstance(parsed, dict):
+        raise AssertionError(f"Data must be an object/dict, got {type(parsed).__name__}")
+
+    if not hasattr(context, "mock_agent_data"):
+        raise AssertionError("Context does not support agent data mocking")
+
+    context.mock_agent_data(agent, parsed)
+
+
 def step_procedure_runs(context: Any) -> None:
     """Execute the procedure.
 
@@ -532,7 +742,7 @@ def register_regex_steps(registry: StepRegistry) -> None:
 
     # Tool argument regex matching
     registry.register(
-        r'the (?P<tool>\w+) tool should be called with (?P<param>\w+) matching pattern "(?P<pattern>.+)"',
+        r'the (?P<tool>[-\w]+) tool should be called with (?P<param>\w+) matching pattern "(?P<pattern>.+)"',
         step_tool_arg_matches_pattern,
     )
 

tactus/testing/test_runner.py

@@ -46,6 +46,8 @@ class TactusTestRunner:
         procedure_file: Path,
         mock_tools: Optional[Dict] = None,
         params: Optional[Dict] = None,
+        mcp_servers: Optional[Dict] = None,
+        tool_paths: Optional[List[str]] = None,
         mocked: bool = False,
     ):
         if not BEHAVE_AVAILABLE:
@@ -54,6 +56,8 @@ class TactusTestRunner:
         self.procedure_file = procedure_file
         self.mock_tools = mock_tools or {}
         self.params = params or {}
+        self.mcp_servers = mcp_servers or {}
+        self.tool_paths = tool_paths or []
         self.mocked = mocked  # Whether to use mocked dependencies
         self.work_dir: Optional[Path] = None
         self.parsed_feature: Optional[ParsedFeature] = None
@@ -83,6 +87,8 @@ class TactusTestRunner:
             self.procedure_file,
             mock_tools=self.mock_tools,
             params=self.params,
+            mcp_servers=self.mcp_servers,
+            tool_paths=self.tool_paths,
             mocked=self.mocked,
         )
 

tactus/validation/semantic_visitor.py

@@ -31,7 +31,6 @@ class TactusDSLVisitor(LuaParserVisitor):
         "Procedure",  # CamelCase
         "Prompt",  # CamelCase
         "Hitl",  # CamelCase
-        "Stages",  # CamelCase
         "Specification",  # CamelCase
         "Specifications",  # CamelCase - Gherkin BDD specs
         "Step",  # CamelCase - Custom step definitions
@@ -467,17 +466,18 @@ class TactusDSLVisitor(LuaParserVisitor):
         elif func_name == "Hitl":  # CamelCase
             if args and len(args) >= 2:
                 self.builder.register_hitl(args[0], args[1] if isinstance(args[1], dict) else {})
-        elif func_name == "Stages":  # CamelCase
-            if args:
-                # Stages() can take multiple string arguments
-                self.builder.set_stages(args)
         elif func_name == "Specification":  # CamelCase
-            if args and len(args) >= 2:
+            # Either:
+            # - Specification([[ Gherkin text ]]) (alias for Specifications)
+            # - Specification("name", { ... })   (structured form)
+            if args and len(args) == 1:
+                self.builder.register_specifications(args[0])
+            elif args and len(args) >= 2:
                 self.builder.register_specification(
                     args[0], args[1] if isinstance(args[1], list) else []
                 )
         elif func_name == "Specifications":  # CamelCase
-            # Specifications([[ Gherkin text ]])
+            # Specifications([[ Gherkin text ]]) (plural form; singular is Specification([[...]]))
             if args and len(args) >= 1:
                 self.builder.register_specifications(args[0])
         elif func_name == "Step":  # CamelCase
@@ -485,11 +485,19 @@ class TactusDSLVisitor(LuaParserVisitor):
             if args and len(args) >= 2:
                 self.builder.register_custom_step(args[0], args[1])
         elif func_name == "Evaluation":  # CamelCase
-            # Evaluation({ runs = 10, parallel = true })
-            if args and len(args) >= 1:
-                self.builder.set_evaluation_config(args[0] if isinstance(args[0], dict) else {})
+            # Either:
+            # - Evaluation({ runs = 10, parallel = true })               (simple config)
+            # - Evaluation({ dataset = {...}, evaluators = {...}, ... }) (alias for Evaluations)
+            if args and len(args) >= 1 and isinstance(args[0], dict):
+                cfg = args[0]
+                if any(k in cfg for k in ("dataset", "dataset_file", "evaluators", "thresholds")):
+                    self.builder.register_evaluations(cfg)
+                else:
+                    self.builder.set_evaluation_config(cfg)
+            elif args and len(args) >= 1:
+                self.builder.set_evaluation_config({})
         elif func_name == "Evaluations":  # CamelCase
-            # Evaluations({ dataset = {...}, evaluators = {...} })
+            # Evaluation(s)({ dataset = {...}, evaluators = {...} })
             if args and len(args) >= 1:
                 self.builder.register_evaluations(args[0] if isinstance(args[0], dict) else {})
         elif func_name == "default_provider":

{tactus-0.30.0.dist-info → tactus-0.31.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tactus
-Version: 0.30.0
+Version: 0.31.1
 Summary: Tactus: Lua-based DSL for agentic workflows
 Project-URL: Homepage, https://github.com/AnthusAI/Tactus
 Project-URL: Documentation, https://github.com/AnthusAI/Tactus/tree/main/docs
@@ -170,7 +170,7 @@ Procedure {
   end
 }
 
-Specifications([[
+Specification([[
 Feature: Research
   Scenario: Completes research
     When the researcher agent takes turns
@@ -421,12 +421,12 @@ calculator = Agent {
 done = tactus.done
 
 text_processor = Agent {
-  tools = {
-    done,
+  inline_tools = {
     {name = "uppercase", input = {...}, handler = function(args)
       return string.upper(args.text)
     end}
-  }
+  },
+  tools = {done}
 }
 ```
 
@@ -533,7 +533,7 @@ Procedure {
   end
 }
 
-Specifications([[
+Specification([[
 Feature: Greeting
   Scenario: Agent greets and completes
     When the greeter agent takes turns
@@ -616,7 +616,7 @@ This creates a rhythm: **tool call → summarization → tool call → summariza
 
 **Why this matters:**
 
-Without per-call control, an agent might call another tool when you just want it to explain the previous result. By temporarily restricting tools to an empty set (`tools = {}`), you ensure the agent focuses on summarization.
+Without per-call control, an agent might call another tool when you just want it to explain the previous result. By temporarily restricting toolsets to an empty set (`tools = {}`), you ensure the agent focuses on summarization.
 
 **Other per-call overrides:**
 
@@ -731,10 +731,9 @@ Then the search tool should be called exactly 2 times
 Then the search tool should be called with query=test
 ```
 
-**State & Stage Steps:**
+**State Steps:**
 ```gherkin
 Given the procedure has started
-Then the stage should be processing
 Then the state count should be 5
 Then the state error should exist
 ```
@@ -866,10 +865,9 @@ Then the search tool should be called exactly 2 times
 Then the search tool should be called with query=test
 ```
 
-**State & Stage Steps:**
+**State Steps:**
 ```gherkin
 Given the procedure has started
-Then the stage should be processing
 Then the state count should be 5
 Then the state error should exist
 ```