agent-cli-dispatcher 0.1.0__py3-none-any.whl

agent_cli_dispatcher-0.1.0.dist-info/METADATA

@@ -0,0 +1,5 @@
+Metadata-Version: 2.4
+Name: agent-cli-dispatcher
+Version: 0.1.0
+Summary: Structured LLM task runner with outcome routing via file signals
+Requires-Python: >=3.11

agent_cli_dispatcher-0.1.0.dist-info/RECORD

@@ -0,0 +1,12 @@
+llm_eval/__init__.py,sha256=ntKJRXSfXzS2pJTRn6NzfZM2wCzeesWO7rbSwGjAMLQ,3918
+llm_eval/job.py,sha256=iz-KIojEV5_7emGjLHXv1-KnCMFhXGq9R7alm9Qf4HY,543
+llm_eval/llm_svc.py,sha256=ZS_FMQPdWtcJHTOSgjy6DOm1kPyO3shRz9pGwr5TUM4,9048
+llm_eval/llm_target.py,sha256=E_tCZHpHmZ-tpTW__bAESVFZOu80Ucy8owI9el1NYkg,368
+llm_eval/preflight.py,sha256=kJX_ozQiwZl7qp4P3hdp8Vol8Ybz8jjtQWVRZsyTRHw,3614
+llm_eval/prompt_builder.py,sha256=BZrgcAhpWqTsJSW_EsMB-I_vQu3Q82fDMQpgv185Cuc,1166
+llm_eval/status_resolver.py,sha256=BcADbc-0RfrCdwcGLq3xALt7f5spUEvk9IeVLdGQqmQ,2000
+llm_eval/workspace.py,sha256=M03nffvpHqUDMGlAHZRSCdmo0GJvPEWYGohJwiNvMhY,426
+agent_cli_dispatcher-0.1.0.dist-info/METADATA,sha256=7XU_mh8v_dq3O0uCXxWQmkeOb9aLzCWUHL6TOZYo4wA,162
+agent_cli_dispatcher-0.1.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+agent_cli_dispatcher-0.1.0.dist-info/top_level.txt,sha256=qHnZEKloXHXIzVDe-vmFJ8gMIp8Kxy_PftF1f3HMCxs,9
+agent_cli_dispatcher-0.1.0.dist-info/RECORD,,

agent_cli_dispatcher-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

agent_cli_dispatcher-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+llm_eval

llm_eval/__init__.py

@@ -0,0 +1,108 @@
+import logging
+import time
+from pathlib import Path
+from typing import Callable, Union
+
+from llm_eval import llm_svc
+from llm_eval.job import JobResult, Outcome
+from llm_eval.llm_svc import LLMEvaluationError
+from llm_eval.llm_target import LLMTarget, parse_targets
+from llm_eval.preflight import TargetStatus, check_all, check_target
+from llm_eval.prompt_builder import build_prompt
+from llm_eval.status_resolver import resolve
+from llm_eval.workspace import cleanup_workspace, create_workspace
+
+__all__ = ["evaluate", "Outcome", "JobResult", "LLMTarget", "LLMEvaluationError",
+           "check_target", "check_all", "TargetStatus", "parse_targets"]
+
+logger = logging.getLogger(__name__)
+
+
+def evaluate(
+    target: LLMTarget | None,
+    purpose: Union[str, Callable[[Path], str]],
+    outcomes: list[Outcome],
+    *,
+    targets: list[LLMTarget] | None = None,
+    on_exception: Callable[[Exception], None] | None = None,
+    model: str | None = None,
+    timeout: float = 1800,
+    cwd: str | None = None,
+) -> None:
+    """Run an LLM evaluation against one or more targets.
+
+    Pass ``targets`` (list) to enable ordered fallback: each target is tried in
+    sequence and the next is used only when the previous raises LLMEvaluationError.
+    Pass ``target`` (single) for the original single-target behaviour.
+    Providing both raises ValueError; providing neither raises ValueError.
+
+    ``purpose`` may be a plain string or a callable that receives the workspace
+    Path and returns a string. Use the callable form when the prompt must embed
+    the workspace path as the LLM output directory so that resolve() can find
+    the files written by the LLM.
+    """
+    if targets is not None and target is not None:
+        raise ValueError("Provide either 'target' or 'targets', not both.")
+    _targets: list[LLMTarget] = targets if targets is not None else ([target] if target is not None else [])
+    if not _targets:
+        raise ValueError("evaluate() requires 'target' or 'targets'.")
+
+    job_id, workspace = create_workspace(cwd)
+    purpose_str = purpose(workspace) if callable(purpose) else purpose
+    prompt = build_prompt(purpose_str, outcomes)
+    start = time.monotonic()
+
+    try:
+        stdout, winning_target = _run_with_fallback(
+            _targets, prompt, model=model, cwd=str(workspace), timeout=timeout
+        )
+    except Exception as exc:
+        cleanup_workspace(workspace)
+        if on_exception is not None:
+            on_exception(exc)
+            return
+        raise
+
+    duration = time.monotonic() - start
+
+    try:
+        matched_outcome, result = resolve(
+            workspace=workspace,
+            outcomes=outcomes,
+            job_id=job_id,
+            target=winning_target.value,
+            duration_seconds=duration,
+            stdout=stdout,
+        )
+    except Exception as exc:
+        cleanup_workspace(workspace)
+        if on_exception is not None:
+            on_exception(exc)
+            return
+        raise
+
+    # Callback exceptions are intentionally NOT caught here — they originate from
+    # business logic, not the LLM layer, and must propagate directly to the caller.
+    try:
+        matched_outcome.callback(result)
+    finally:
+        cleanup_workspace(workspace)
+
+
+def _run_with_fallback(
+    targets: list[LLMTarget],
+    prompt: str,
+    *,
+    model: str | None = None,
+    cwd: str | None = None,
+    timeout: float = 1800,
+) -> tuple[str, LLMTarget]:
+    """Try each target in order; return (stdout, winning_target) on first success."""
+    last_exc: llm_svc.LLMEvaluationError | None = None
+    for t in targets:
+        try:
+            return llm_svc.run(t, prompt, model=model, cwd=cwd, timeout=timeout), t
+        except llm_svc.LLMEvaluationError as exc:
+            logger.warning("evaluate fallback: %s failed, trying next. error: %s", t.value, exc)
+            last_exc = exc
+    raise last_exc  # type: ignore[misc]

llm_eval/job.py

@@ -0,0 +1,24 @@
+from dataclasses import dataclass, field
+from typing import Callable
+
+
+@dataclass(frozen=True)
+class Outcome:
+    description: str
+    callback: Callable[["JobResult"], None]
+    status: str | None = None
+    output_files: list[str] | None = None
+
+
+def effective_status(outcome: Outcome, index: int) -> str:
+    return outcome.status if outcome.status is not None else str(index)
+
+
+@dataclass(frozen=True)
+class JobResult:
+    job_id: str
+    status: str
+    target: str
+    duration_seconds: float
+    files: dict[str, bytes]
+    stdout: str

llm_eval/llm_svc.py

@@ -0,0 +1,233 @@
+import json
+import logging
+import os
+import re
+import shutil
+import subprocess
+import time
+import uuid
+from datetime import datetime, timezone
+from pathlib import Path
+
+from llm_eval.llm_target import LLMTarget
+
+logger = logging.getLogger(__name__)
+
+
+class LLMEvaluationError(RuntimeError):
+    """LLM subprocess failure: non-zero exit, quota exhaustion, timeout, or execution error."""
+
+_QUOTA_ERROR_PATTERNS: list[re.Pattern] = [
+    re.compile(p, re.IGNORECASE)
+    for p in [
+        r"exceeded your monthly token limit",
+        r"exceeded your current quota",
+        r"insufficient.quota",
+        r"quota.exceeded",
+        r"billing hard limit",
+        r"credit balance is too low",
+        r"out of credits",
+        r"rate.limit.exceeded",
+        r"429",
+        r"payment required",
+    ]
+]
+
+_QUOTA_RETRY_INTERVAL_SECONDS: int = int(os.getenv("LLM_QUOTA_RETRY_INTERVAL", "300"))
+_QUOTA_MAX_RETRIES: int = int(os.getenv("LLM_QUOTA_MAX_RETRIES", "288"))
+
+_ALLOW_ALL_OPENCODE_PERMISSION = {
+    "bash": "allow", "read": "allow", "edit": "allow", "task": "allow",
+    "glob": "allow", "grep": "allow", "list": "allow",
+    "external_directory": "allow", "todowrite": "allow", "todoread": "allow",
+    "question": "allow", "webfetch": "allow", "websearch": "allow",
+    "codesearch": "allow", "lsp": "allow", "doom_loop": "allow", "skill": "allow",
+}
+
+
+def _is_quota_error(text: str) -> bool:
+    for pattern in _QUOTA_ERROR_PATTERNS:
+        if pattern.search(text):
+            return True
+    return False
+
+
+def _resolve_cli(command_name: str) -> str:
+    if os.name == "nt":
+        cmd_candidate = shutil.which(f"{command_name}.cmd")
+        if cmd_candidate:
+            return cmd_candidate
+    resolved = shutil.which(command_name)
+    return resolved if resolved else command_name
+
+
+def run(
+    target: LLMTarget,
+    prompt: str,
+    *,
+    model: str | None = None,
+    cwd: str | None = None,
+    timeout: float | None = 1800,
+    encoding: str = "utf-8",
+    quota_retry_interval: int | None = None,
+    quota_max_retries: int | None = None,
+) -> str:
+    if not prompt.strip():
+        raise ValueError("prompt must not be empty.")
+
+    _retry_interval = quota_retry_interval if quota_retry_interval is not None else _QUOTA_RETRY_INTERVAL_SECONDS
+    _max_retries    = quota_max_retries    if quota_max_retries    is not None else _QUOTA_MAX_RETRIES
+
+    work_dir = str(Path(cwd).resolve()) if cwd else None
+    effective_dir = work_dir or str(Path.cwd())
+
+    run_id = uuid.uuid4().hex[:8]
+    io_dir = Path(effective_dir) / ".llm_io"
+    io_dir.mkdir(parents=True, exist_ok=True)
+    prompt_file = io_dir / f"prompt_{run_id}.txt"
+    output_file = io_dir / f"output_{run_id}.txt"
+    prompt_file.write_text(prompt, encoding=encoding)
+
+    stdin_input: str = ""
+    env = dict(os.environ)
+
+    try:
+        if target == LLMTarget.CLAUDE:
+            command = [_resolve_cli("claude"), "--print", "--dangerously-skip-permissions"]
+            if model:
+                command.extend(["--model", model])
+            stdin_input = prompt_file.read_text(encoding=encoding)
+
+        elif target == LLMTarget.GEMINI:
+            command = [_resolve_cli("gemini"), "--approval-mode", "auto_edit",
+                       "--prompt", prompt_file.read_text(encoding=encoding)]
+
+        elif target == LLMTarget.CODEX:
+            # Pass prompt via stdin to avoid Windows cmd line length limits (~8191 chars).
+            # codex reads stdin when no PROMPT arg is given (or arg is "-").
+            command = [_resolve_cli("codex"), "exec", "--dangerously-bypass-approvals-and-sandbox",
+                       "--skip-git-repo-check"]
+            stdin_input = prompt_file.read_text(encoding=encoding)
+
+        elif target == LLMTarget.OPENCODE:
+            env.setdefault("OPENCODE_PERMISSION", json.dumps(_ALLOW_ALL_OPENCODE_PERMISSION))
+            runtime_root = Path(effective_dir).resolve() / "data" / "tool-runtime" / "opencode"
+            for subdir in ("config", "data", "state"):
+                (runtime_root / subdir).mkdir(parents=True, exist_ok=True)
+            env.setdefault("XDG_CONFIG_HOME", str(runtime_root / "config"))
+            env.setdefault("XDG_DATA_HOME",   str(runtime_root / "data"))
+            env.setdefault("XDG_STATE_HOME",  str(runtime_root / "state"))
+            command = [_resolve_cli("opencode"), "run",
+                       "--dir", effective_dir, "--format", "json", "-"]
+            stdin_input = prompt_file.read_text(encoding=encoding)
+
+        elif target == LLMTarget.COPILOT:
+            command = [_resolve_cli("copilot"), "-p", prompt_file.read_text(encoding=encoding),
+                       "--allow-all", "--no-ask-user", "--output-format", "text", "--silent",
+                       "--add-dir", effective_dir]
+            if model:
+                command.extend(["--model", model])
+
+        else:
+            raise ValueError(f"Unsupported LLM target: {target}")
+
+        logger.info("run [%s] cwd=%s", target.value, work_dir or "(inherit)")
+        logger.debug("run [%s] command=%s", target.value, command)
+        logger.debug("run [%s] prompt_file=%s\n%s", target.value, prompt_file, prompt)
+
+        completed = None
+        for quota_attempt in range(_max_retries + 1):
+            try:
+                completed = subprocess.run(
+                    command,
+                    input=stdin_input,
+                    capture_output=True,
+                    text=True,
+                    encoding=encoding,
+                    errors="replace",
+                    cwd=work_dir,
+                    env=env,
+                    timeout=timeout,
+                )
+            except LLMEvaluationError:
+                raise
+            except Exception as e:
+                logger.error("execute cmd exception: %s", e)
+                raise LLMEvaluationError(f"{target.value} subprocess error: {e}") from e
+
+            if completed.returncode != 0:
+                stderr = (completed.stderr or "").strip()
+                stdout = (completed.stdout or "").strip()
+                parts = [s for s in [stderr, stdout] if s]
+                detail = "\n".join(parts) if parts else "(no output)"
+
+                if _is_quota_error(detail):
+                    ts = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC")
+                    logger.warning(
+                        "[QUOTA EXHAUSTED] %s | attempt=%d/%d | Retrying in %ds. Detail: %s",
+                        target.value, quota_attempt + 1, _max_retries, _retry_interval, detail[:300],
+                    )
+                    if quota_attempt < _max_retries:
+                        time.sleep(_retry_interval)
+                        continue
+                    raise LLMEvaluationError(
+                        f"{target.value} quota exhausted after {_max_retries} retries. "
+                        f"Last error: {detail[:300]}"
+                    )
+
+                raise LLMEvaluationError(
+                    f"{target.value} CLI failed (exit {completed.returncode}): {detail[:500]}"
+                )
+
+            break
+
+        raw_stdout = (completed.stdout or "").strip()
+
+        if target == LLMTarget.OPENCODE and raw_stdout:
+            try:
+                chunks = []
+                for line in raw_stdout.splitlines():
+                    if not line.strip():
+                        continue
+                    event = json.loads(line)
+                    if event.get("type") == "error":
+                        msg = (event.get("error") or {}).get("data", {}).get("message", "")
+                        raise RuntimeError(str(msg))
+                    message = event.get("message")
+                    if isinstance(message, dict):
+                        for item in (message.get("content") or []):
+                            if isinstance(item, dict) and item.get("type") == "text":
+                                chunks.append(str(item["text"]))
+                if chunks:
+                    raw_stdout = "\n".join(chunks).strip()
+            except json.JSONDecodeError:
+                pass
+
+        logger.info("run [%s] done. stdout_len=%d", target.value, len(raw_stdout))
+        output_file.write_text(raw_stdout, encoding=encoding)
+        return output_file.read_text(encoding=encoding)
+
+    finally:
+        prompt_file.unlink(missing_ok=True)
+        output_file.unlink(missing_ok=True)
+
+
+def run_with_fallback(
+    targets: list[LLMTarget],
+    prompt: str,
+    *,
+    model: str | None = None,
+    cwd: str | None = None,
+    timeout: float | None = 1800,
+    encoding: str = "utf-8",
+) -> str:
+    if not targets:
+        raise ValueError("targets must not be empty")
+    last_exc: LLMEvaluationError | None = None
+    for target in targets:
+        try:
+            return run(target, prompt, model=model, cwd=cwd, timeout=timeout, encoding=encoding)
+        except LLMEvaluationError as exc:
+            logger.warning("run_with_fallback: %s failed, trying next. error: %s", target.value, exc)
+            last_exc = exc
+    raise last_exc

llm_eval/llm_target.py

@@ -0,0 +1,14 @@
+from enum import Enum
+
+
+class LLMTarget(Enum):
+    CLAUDE   = "claude"
+    GEMINI   = "gemini"
+    CODEX    = "codex"
+    OPENCODE = "opencode"
+    COPILOT  = "copilot"
+
+
+def parse_targets(value: str) -> list[LLMTarget]:
+    """Parse 'claude,gemini' → [LLMTarget.CLAUDE, LLMTarget.GEMINI]."""
+    return [LLMTarget(v.strip()) for v in value.split(",") if v.strip()]

llm_eval/preflight.py

@@ -0,0 +1,101 @@
+import os
+import shutil
+import subprocess
+from dataclasses import dataclass
+from typing import Callable
+
+from llm_eval.llm_target import LLMTarget
+
+
+@dataclass(frozen=True)
+class TargetStatus:
+    ok: bool
+    reason: str | None = None
+
+
+def _resolve_cli(command_name: str) -> str:
+    if os.name == "nt":
+        cmd_candidate = shutil.which(f"{command_name}.cmd")
+        if cmd_candidate:
+            return cmd_candidate
+    resolved = shutil.which(command_name)
+    return resolved if resolved else command_name
+
+
+def _check_claude() -> TargetStatus:
+    binary = _resolve_cli("claude")
+    try:
+        result = subprocess.run(
+            [binary, "auth", "status", "--json"],
+            capture_output=True, text=True, timeout=15, input="",
+        )
+        if result.returncode == 0 and '"loggedIn": true' in result.stdout:
+            return TargetStatus(ok=True)
+        reason = result.stderr.strip() or result.stdout.strip() or "loggedIn not true"
+        return TargetStatus(ok=False, reason=reason[:200])
+    except FileNotFoundError:
+        return TargetStatus(ok=False, reason="claude CLI not found on PATH")
+    except subprocess.TimeoutExpired:
+        return TargetStatus(ok=False, reason="claude auth status timed out")
+    except Exception as e:
+        return TargetStatus(ok=False, reason=str(e)[:200])
+
+
+def _check_codex() -> TargetStatus:
+    binary = _resolve_cli("codex")
+    try:
+        result = subprocess.run(
+            [binary, "login", "status"],
+            capture_output=True, text=True, timeout=15, input="",
+        )
+        if result.returncode == 0 and "Logged in" in result.stderr:
+            return TargetStatus(ok=True)
+        reason = result.stderr.strip() or result.stdout.strip() or "not logged in"
+        return TargetStatus(ok=False, reason=reason[:200])
+    except FileNotFoundError:
+        return TargetStatus(ok=False, reason="codex CLI not found on PATH")
+    except subprocess.TimeoutExpired:
+        return TargetStatus(ok=False, reason="codex login status timed out")
+    except Exception as e:
+        return TargetStatus(ok=False, reason=str(e)[:200])
+
+
+def _check_via_version(tool: str) -> TargetStatus:
+    binary = _resolve_cli(tool)
+    try:
+        result = subprocess.run(
+            [binary, "--version"],
+            capture_output=True, text=True, timeout=15, input="",
+        )
+        if result.returncode == 0:
+            return TargetStatus(ok=True)
+        reason = (result.stderr or result.stdout or "non-zero exit").strip()
+        return TargetStatus(ok=False, reason=reason[:200])
+    except FileNotFoundError:
+        return TargetStatus(ok=False, reason=f"{tool} CLI not found on PATH")
+    except subprocess.TimeoutExpired:
+        return TargetStatus(ok=False, reason=f"{tool} --version timed out")
+    except Exception as e:
+        return TargetStatus(ok=False, reason=str(e)[:200])
+
+
+_CHECKERS: dict[LLMTarget, Callable[[], TargetStatus]] = {
+    LLMTarget.CLAUDE:   _check_claude,
+    LLMTarget.GEMINI:   lambda: _check_via_version("gemini"),
+    LLMTarget.CODEX:    _check_codex,
+    LLMTarget.OPENCODE: lambda: _check_via_version("opencode"),
+    LLMTarget.COPILOT:  lambda: _check_via_version("copilot"),
+}
+
+
+def check_target(target: LLMTarget) -> TargetStatus:
+    """Check whether a single LLM target's CLI is installed and authenticated."""
+    return _CHECKERS[target]()
+
+
+def check_all() -> dict[LLMTarget, TargetStatus]:
+    """Check all known LLM targets. Always runs live; no caching."""
+    return {target: checker() for target, checker in _CHECKERS.items()}
+
+if __name__ == '__main__':
+    _check_codex()

llm_eval/prompt_builder.py

@@ -0,0 +1,34 @@
+from llm_eval.job import Outcome, effective_status
+
+
+def build_prompt(purpose: str, outcomes: list[Outcome]) -> str:
+    lines = [
+        purpose,
+        "",
+        "---",
+        "",
+        "After completing your analysis, you MUST create exactly one of the following "
+        "empty files in your current working directory to signal your conclusion. "
+        "This must be the last action you take.",
+        "",
+        "Status files (create exactly one, leave it empty):",
+    ]
+
+    for i, outcome in enumerate(outcomes):
+        status = effective_status(outcome, i)
+        lines.append(f"  status_{status:<20} — {outcome.description}")
+
+    lines.append("")
+
+    for i, outcome in enumerate(outcomes):
+        if outcome.output_files:
+            status = effective_status(outcome, i)
+            lines.append(f'If the outcome is "{status}", also write these files:')
+            for filename in outcome.output_files:
+                lines.append(f"  {filename}")
+            lines.append("")
+
+    lines.append("Do not create more than one status file.")
+    lines.append("Do not write anything inside the status file.")
+
+    return "\n".join(lines)

llm_eval/status_resolver.py

@@ -0,0 +1,67 @@
+import logging
+from pathlib import Path
+
+from llm_eval.job import JobResult, Outcome, effective_status
+
+logger = logging.getLogger(__name__)
+
+
+def resolve(
+    workspace: Path,
+    outcomes: list[Outcome],
+    job_id: str,
+    target: str,
+    duration_seconds: float,
+    stdout: str,
+) -> tuple[Outcome, JobResult]:
+    status_files = sorted(workspace.glob("status_*"))
+
+    if len(status_files) > 1:
+        logger.warning(
+            "Multiple status files found: %s. Using %s.",
+            [p.name for p in status_files],
+            status_files[0].name,
+        )
+
+    if not status_files:
+        error_outcome = next(
+            (o for i, o in enumerate(outcomes) if effective_status(o, i) == "error"),
+            None,
+        )
+        if error_outcome is None:
+            raise RuntimeError(
+                "No status file created by LLM and no 'error' outcome defined."
+            )
+        matched = error_outcome
+        status_name = "error"
+    else:
+        status_name = status_files[0].stem[len("status_"):]
+        matched = next(
+            (o for i, o in enumerate(outcomes) if effective_status(o, i) == status_name),
+            None,
+        )
+        if matched is None:
+            raise RuntimeError(
+                f"Status file 'status_{status_name}' does not match any defined outcome."
+            )
+
+    if matched.output_files is not None:
+        missing = [f for f in matched.output_files if not (workspace / f).exists()]
+        if missing:
+            raise RuntimeError(
+                f"Outcome '{status_name}' declared output_files {missing} "
+                "but LLM did not create them."
+            )
+
+    declared = matched.output_files or []
+    files: dict[str, bytes] = {f: (workspace / f).read_bytes() for f in declared}
+
+    result = JobResult(
+        job_id=job_id,
+        status=status_name,
+        target=target,
+        duration_seconds=duration_seconds,
+        files=files,
+        stdout=stdout,
+    )
+    return matched, result

llm_eval/workspace.py

@@ -0,0 +1,15 @@
+import shutil
+import uuid
+from pathlib import Path
+
+
+def create_workspace(base_dir: str | None) -> tuple[str, Path]:
+    job_id = uuid.uuid4().hex[:8]
+    base = Path(base_dir) if base_dir else Path.cwd()
+    workspace = base / ".llm_eval" / job_id
+    workspace.mkdir(parents=True, exist_ok=True)
+    return job_id, workspace
+
+
+def cleanup_workspace(workspace: Path) -> None:
+    shutil.rmtree(workspace, ignore_errors=True)