janus-labs 0.2.0__py3-none-any.whl

governance/bridge.py

@@ -0,0 +1,124 @@
+"""Bridge between Janus v3.6 governance and Janus Labs Gauge layer."""
+
+from dataclasses import dataclass
+from enum import Enum
+from pathlib import Path
+import sys
+from typing import List, Optional
+
+# Try to import from mcp-janus (full installation)
+# Fall back to stubs for standalone operation
+try:
+    MCP_JANUS_PATH = Path(__file__).resolve().parents[2] / "mcp-janus"
+    if MCP_JANUS_PATH.exists() and str(MCP_JANUS_PATH) not in sys.path:
+        sys.path.insert(0, str(MCP_JANUS_PATH))
+    from tools import foundation_check, handle_escalation, infer_confidence  # noqa: E402
+except ImportError:
+    # Standalone mode - use stubs
+    from janus_types import foundation_check, handle_escalation, infer_confidence  # noqa: E402
+
+
+class GovernanceDecision(Enum):
+    """Governance decision outcomes."""
+    PASS = "pass"
+    WARN = "warn"
+    HALT = "halt"
+
+
+@dataclass
+class GovernanceContext:
+    """Context for governance evaluation during rollout."""
+    rollout_index: int
+    behavior_id: str
+    current_approach: Optional[str] = None
+    approach_history: Optional[List[str]] = None
+    reasoning_text: Optional[str] = None
+    target_dir: str = "."
+
+
+@dataclass
+class GovernanceResult:
+    """Result of governance check."""
+    decision: GovernanceDecision
+    trigger: Optional[str]
+    signals: dict
+    recommendation: str
+    escalation_id: Optional[str] = None
+    should_halt: bool = False
+
+
+def _to_decision(result_value: str) -> GovernanceDecision:
+    normalized = result_value.upper()
+    if normalized == "HALT":
+        return GovernanceDecision.HALT
+    if normalized == "WARN":
+        return GovernanceDecision.WARN
+    return GovernanceDecision.PASS
+
+
+def check_governance(context: GovernanceContext) -> GovernanceResult:
+    """
+    Evaluate governance signals for a rollout iteration.
+
+    Integrates Janus v3.6 foundation_check with rollout-specific context.
+
+    Args:
+        context: GovernanceContext with rollout state
+
+    Returns:
+        GovernanceResult with decision and metadata
+    """
+    confidence = None
+    if context.reasoning_text:
+        confidence, _ = infer_confidence(context.reasoning_text)
+
+    confidence_history = None
+    if context.approach_history and len(context.approach_history) > 1:
+        count = len(context.approach_history)
+        confidence_history = [max(0.1, 0.9 - (i * 0.1)) for i in range(count)]
+
+    same_pattern = bool(context.approach_history and len(context.approach_history) > 1)
+
+    result = foundation_check(
+        iteration_count=context.rollout_index + 1,
+        same_pattern=same_pattern,
+        merge_ready=False,
+        current_approach=context.current_approach,
+        approach_history=context.approach_history,
+        confidence=confidence,
+        confidence_history=confidence_history,
+    )
+
+    if isinstance(result, str):
+        if result.startswith("HALT"):
+            decision = GovernanceDecision.HALT
+            trigger = "iteration"
+        elif result.startswith("WARN"):
+            decision = GovernanceDecision.WARN
+            trigger = "iteration"
+        else:
+            decision = GovernanceDecision.PASS
+            trigger = "none"
+        signals = {"iteration": context.rollout_index + 1}
+        recommendation = result
+        escalation_id = None
+    else:
+        decision = _to_decision(str(result.get("result", "PASS")))
+        trigger = result.get("trigger", "none")
+        signals = result.get("signals", {}) if isinstance(result.get("signals"), dict) else {}
+        recommendation = result.get("recommendation", "")
+        escalation_id = None
+
+        if decision == GovernanceDecision.HALT:
+            esc_result = handle_escalation(result, {"target_dir": context.target_dir})
+            if isinstance(esc_result, dict):
+                escalation_id = esc_result.get("escalation_id")
+
+    return GovernanceResult(
+        decision=decision,
+        trigger=trigger,
+        signals=signals,
+        recommendation=recommendation,
+        escalation_id=escalation_id,
+        should_halt=(decision == GovernanceDecision.HALT),
+    )

governance/memory.py

@@ -0,0 +1,116 @@
+"""Persistence of governance decisions to Janus memory tiers."""
+
+from datetime import datetime, timezone
+from pathlib import Path
+import sys
+from typing import List, Optional
+
+
+# Try mcp-janus memory module, fallback to local stubs for standalone operation
+MCP_JANUS_PATH = Path(__file__).resolve().parents[2] / "mcp-janus"
+if MCP_JANUS_PATH.exists() and str(MCP_JANUS_PATH) not in sys.path:
+    sys.path.insert(0, str(MCP_JANUS_PATH))
+
+try:
+    from memory import read_tier, write_tier  # noqa: E402
+except ImportError:
+    from janus_types import read_tier, write_tier  # noqa: E402
+
+from governance.bridge import GovernanceResult  # noqa: E402
+
+
+def persist_governance_decision(
+    result: GovernanceResult,
+    behavior_id: str,
+    rollout_index: int,
+    target_dir: str = ".",
+) -> bool:
+    """
+    Persist a governance decision to the governance memory tier.
+
+    Args:
+        result: GovernanceResult from check_governance()
+        behavior_id: ID of the behavior being evaluated
+        rollout_index: Index of the rollout
+        target_dir: Directory containing .janus/
+
+    Returns:
+        True if persisted successfully
+    """
+    current = read_tier("governance", target_dir)
+    if not isinstance(current, dict):
+        current = {"schema_version": "1.0.0"}
+
+    decisions = current.get("governance_decisions", [])
+    if not isinstance(decisions, list):
+        decisions = []
+
+    entry = {
+        "timestamp": datetime.now(timezone.utc).isoformat(),
+        "behavior_id": behavior_id,
+        "rollout_index": rollout_index,
+        "decision": result.decision.value,
+        "trigger": result.trigger,
+        "signals": result.signals,
+        "escalation_id": result.escalation_id,
+    }
+
+    decisions.append(entry)
+
+    if len(decisions) > 100:
+        decisions = decisions[-100:]
+
+    current["governance_decisions"] = decisions
+    current["last_decision"] = entry
+
+    success, _errors = write_tier("governance", current, target_dir)
+    return success
+
+
+def get_governance_history(
+    behavior_id: Optional[str] = None,
+    limit: int = 20,
+    target_dir: str = ".",
+) -> List[dict]:
+    """
+    Retrieve governance decision history.
+
+    Args:
+        behavior_id: Optional filter by behavior
+        limit: Maximum entries to return
+        target_dir: Directory containing .janus/
+
+    Returns:
+        List of governance decision entries (most recent first)
+    """
+    current = read_tier("governance", target_dir)
+    if not isinstance(current, dict):
+        return []
+
+    decisions = current.get("governance_decisions", [])
+    if not isinstance(decisions, list):
+        return []
+
+    if behavior_id:
+        decisions = [d for d in decisions if d.get("behavior_id") == behavior_id]
+
+    return list(reversed(decisions[-limit:]))
+
+
+def get_halt_count(behavior_id: Optional[str] = None, target_dir: str = ".") -> int:
+    """
+    Count HALT decisions for governance statistics.
+
+    Args:
+        behavior_id: Optional filter by behavior
+        target_dir: Directory containing .janus/
+
+    Returns:
+        Count of HALT decisions
+    """
+    history = get_governance_history(
+        behavior_id=behavior_id,
+        limit=100,
+        target_dir=target_dir,
+    )
+    return sum(1 for entry in history if entry.get("decision") == "halt")

harness/__init__.py

@@ -0,0 +1 @@
+"""Harness components for Janus Labs."""

harness/artifacts.py

@@ -0,0 +1,195 @@
+"""Artifact collection for Janus Labs harness."""
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from pathlib import Path
+import re
+import subprocess
+import time
+from typing import Any, Callable
+
+from .types import RunArtifactBundle, Message, ToolInvocation, GitDiff, TestReport, Timings
+
+
+def _run_git(args: list[str], cwd: Path) -> str | None:
+    try:
+        result = subprocess.run(
+            ["git", *args],
+            cwd=str(cwd),
+            capture_output=True,
+            text=True,
+            check=True,
+        )
+    except (FileNotFoundError, subprocess.CalledProcessError):
+        return None
+    return result.stdout
+
+
+class ArtifactCollector:
+    """
+    Collects all components of a RunArtifactBundle during agent execution.
+    """
+
+    def __init__(self):
+        self.messages: list[Message] = []
+        self.tool_traces: list[ToolInvocation] = []
+        self.start_time: float = time.perf_counter()
+        self.tool_time_ms: int = 0
+        self.repo_diff: GitDiff | None = None
+        self.test_results: TestReport | None = None
+
+    def _timestamp(self) -> str:
+        return datetime.now(timezone.utc).isoformat()
+
+    def record_message(self, role: str, content: str) -> None:
+        """Record a conversation message."""
+        message: Message = {
+            "role": role,
+            "content": content,
+            "timestamp": self._timestamp(),
+        }
+        self.messages.append(message)
+
+    def record_tool_call(self, tool_name: str, args: dict, result: Any, duration_ms: int) -> None:
+        """Record a tool invocation with timing."""
+        if not isinstance(result, (str, dict)):
+            result = str(result)
+        trace: ToolInvocation = {
+            "tool_name": tool_name,
+            "arguments": args,
+            "result": result,
+            "duration_ms": max(int(duration_ms), 0),
+            "timestamp": self._timestamp(),
+        }
+        self.tool_traces.append(trace)
+        self.tool_time_ms += trace["duration_ms"]
+
+    def capture_git_diff(self, repo_path: str) -> GitDiff:
+        """Capture git diff from repo."""
+        repo = Path(repo_path).resolve()
+        files_changed: list[str] = []
+        insertions = 0
+        deletions = 0
+        patch = ""
+
+        try:
+            numstat = _run_git(["diff", "--numstat"], repo)
+            if numstat:
+                for line in numstat.splitlines():
+                    parts = line.split("\t")
+                    if len(parts) >= 3:
+                        ins, dels, file_path = parts[0], parts[1], parts[2]
+                        if ins.isdigit():
+                            insertions += int(ins)
+                        if dels.isdigit():
+                            deletions += int(dels)
+                        files_changed.append(file_path)
+
+            name_only = _run_git(["diff", "--name-only"], repo)
+            if name_only is not None:
+                files_changed = [line for line in name_only.splitlines() if line.strip()]
+
+            patch = _run_git(["diff"], repo) or ""
+        except Exception:
+            files_changed = []
+            insertions = 0
+            deletions = 0
+            patch = ""
+
+        diff: GitDiff = {
+            "files_changed": files_changed,
+            "insertions": insertions,
+            "deletions": deletions,
+            "patch": patch,
+        }
+        self.repo_diff = diff
+        return diff
+
+    def capture_test_results(self, test_output: str, framework: str = "pytest") -> TestReport:
+        """Parse test output into TestReport."""
+        passed = 0
+        failed = 0
+        skipped = 0
+
+        if framework == "pytest":
+            # Search the full output for the summary line
+            # The summary is at the end: "7 passed in 0.02s" or "3 passed, 1 failed"
+            passed_match = re.search(r"(\d+)\s+passed", test_output)
+            failed_match = re.search(r"(\d+)\s+failed", test_output)
+            skipped_match = re.search(r"(\d+)\s+skipped", test_output)
+            if passed_match:
+                passed = int(passed_match.group(1))
+            if failed_match:
+                failed = int(failed_match.group(1))
+            if skipped_match:
+                skipped = int(skipped_match.group(1))
+
+        report: TestReport = {
+            "framework": framework if framework in {"pytest", "jest", "other"} else "other",
+            "passed": passed,
+            "failed": failed,
+            "skipped": skipped,
+            "output": test_output,
+        }
+        self.test_results = report
+        return report
+
+    def create_tool_wrapper(self, original_tool: Callable) -> Callable:
+        """
+        Wrap a tool function to automatically record invocations.
+        """
+        def wrapped(*args, **kwargs):
+            start = time.perf_counter()
+            result = original_tool(*args, **kwargs)
+            duration_ms = int((time.perf_counter() - start) * 1000)
+            call_args = {"args": list(args), "kwargs": kwargs}
+            self.record_tool_call(original_tool.__name__, call_args, result, duration_ms)
+            return result
+
+        return wrapped
+
+    def finalize(self, exit_code: str) -> RunArtifactBundle:
+        """
+        Finalize and return complete bundle.
+
+        Guarantees:
+        - All 5 components present
+        - No None values in required fields
+        """
+        tool_time_ms = max(int(self.tool_time_ms), 1)
+        elapsed_ms = int((time.perf_counter() - self.start_time) * 1000)
+        total_ms = max(elapsed_ms, tool_time_ms, 1)
+        model_time_ms = max(total_ms - tool_time_ms, 1)
+
+        timings: Timings = {
+            "total_ms": total_ms,
+            "tool_time_ms": tool_time_ms,
+            "model_time_ms": model_time_ms,
+        }
+
+        repo_diff = self.repo_diff or {
+            "files_changed": [],
+            "insertions": 0,
+            "deletions": 0,
+            "patch": "",
+        }
+        test_results = self.test_results or {
+            "framework": "pytest",
+            "passed": 0,
+            "failed": 0,
+            "skipped": 0,
+            "output": "",
+        }
+
+        if exit_code not in {"success", "timeout", "crash", "halt"}:
+            exit_code = "crash"
+
+        bundle: RunArtifactBundle = {
+            "transcript": list(self.messages),
+            "tool_traces": list(self.tool_traces),
+            "repo_diff": repo_diff,
+            "test_results": test_results,
+            "timings": timings,
+            "exit_code": exit_code,
+        }
+        return bundle

harness/executor.py

@@ -0,0 +1,51 @@
+"""Execution helpers for Janus Labs harness."""
+from __future__ import annotations
+
+from pathlib import Path
+import subprocess
+from typing import Sequence
+
+
+def _run_git(args: Sequence[str], cwd: Path) -> bool:
+    try:
+        subprocess.run(
+            ["git", *args],
+            cwd=str(cwd),
+            capture_output=True,
+            text=True,
+            check=True,
+        )
+    except (FileNotFoundError, subprocess.CalledProcessError):
+        return False
+    return True
+
+
+def init_fixture(fixture_path: str) -> bool:
+    """
+    Initialize fixture repo to clean state.
+
+    Guarantees:
+    - git reset --hard HEAD
+    - git clean -fd (remove untracked files)
+    - Returns True if successful
+
+    Args:
+        fixture_path: Absolute path to fixture repo
+
+    Returns:
+        bool: True if initialization succeeded
+    """
+    repo_path = Path(fixture_path).resolve()
+    if not repo_path.exists():
+        return False
+
+    if not (repo_path / ".git").exists():
+        return False
+
+    if not _run_git(["reset", "--hard", "HEAD"], repo_path):
+        return False
+
+    if not _run_git(["clean", "-fd"], repo_path):
+        return False
+
+    return True

harness/sandbox.py

@@ -0,0 +1,40 @@
+"""Filesystem sandbox enforcement for agent writes."""
+from pathlib import Path
+from typing import Set
+
+
+class Sandbox:
+    """
+    Filesystem sandbox that tracks and validates all write operations.
+    """
+
+    def __init__(self, allowed_paths: list[str]):
+        """
+        Args:
+            allowed_paths: List of absolute paths agent can write to
+        """
+        self.allowed_paths: Set[Path] = {Path(p).resolve() for p in allowed_paths}
+        self.write_log: list[Path] = []
+
+    def validate_write(self, path: str) -> bool:
+        """
+        Check if path is within allowed sandbox.
+
+        Returns:
+            bool: True if write is allowed
+        """
+        candidate = Path(path).resolve()
+        for allowed in self.allowed_paths:
+            if candidate == allowed or allowed in candidate.parents:
+                return True
+
+        self.write_log.append(candidate)
+        return False
+
+    def get_violations(self) -> list[Path]:
+        """Return list of paths written outside sandbox."""
+        return list(self.write_log)
+
+    def is_clean(self) -> bool:
+        """Return True if no violations occurred."""
+        return len(self.write_log) == 0

harness/types.py

@@ -0,0 +1,46 @@
+"""Typed structures for RunArtifactBundle capture."""
+from typing import TypedDict, Literal
+
+
+class Message(TypedDict):
+    role: Literal["user", "assistant", "system"]
+    content: str
+    timestamp: str  # ISO8601
+
+
+class ToolInvocation(TypedDict):
+    tool_name: str
+    arguments: dict
+    result: str | dict
+    duration_ms: int
+    timestamp: str  # ISO8601
+
+
+class GitDiff(TypedDict):
+    files_changed: list[str]
+    insertions: int
+    deletions: int
+    patch: str  # Full diff output
+
+
+class TestReport(TypedDict):
+    framework: Literal["pytest", "jest", "other"]
+    passed: int
+    failed: int
+    skipped: int
+    output: str  # Full test output
+
+
+class Timings(TypedDict):
+    total_ms: int
+    tool_time_ms: int
+    model_time_ms: int
+
+
+class RunArtifactBundle(TypedDict):
+    transcript: list[Message]
+    tool_traces: list[ToolInvocation]
+    repo_diff: GitDiff
+    test_results: TestReport
+    timings: Timings
+    exit_code: Literal["success", "timeout", "crash", "halt"]

janus_labs/__init__.py

@@ -0,0 +1,16 @@
+"""
+Janus Labs - 3DMark for AI Agents.
+
+Benchmark and measure AI coding agent reliability with standardized,
+reproducible tests.
+
+This module provides the Python API. For CLI usage:
+    python -m janus_labs run --suite refactor-storm
+
+Or if janus-labs is in your PATH:
+    janus-labs run --suite refactor-storm
+"""
+
+__version__ = "0.2.0"
+
+__all__ = ["__version__"]

janus_labs/__main__.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python3
+"""
+Module entry point for janus-labs CLI.
+
+Enables running the CLI via:
+    python -m janus_labs <command> [args]
+
+This is the recommended fallback if 'janus-labs' is not in your PATH.
+
+Examples:
+    python -m janus_labs run --suite refactor-storm
+    python -m janus_labs bench --submit
+    python -m janus_labs submit result.json --github myhandle
+"""
+
+import sys
+
+
+def main():
+    """Entry point that delegates to CLI main."""
+    try:
+        from cli.main import main as cli_main
+        return cli_main()
+    except ImportError as e:
+        # Provide helpful error if dependencies are missing
+        print(f"Error: {e}", file=sys.stderr)
+        print(file=sys.stderr)
+        print("Janus Labs requires additional dependencies.", file=sys.stderr)
+        print("Install with: pip install janus-labs", file=sys.stderr)
+        print(file=sys.stderr)
+        print("If you've already installed, ensure you're using the correct Python:", file=sys.stderr)
+        print(f"  Current: {sys.executable}", file=sys.stderr)
+        return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())