devflow-engine 1.0.0__py3-none-any.whl

devflow_engine/llm/cli_stream.py

@@ -0,0 +1,371 @@
+from __future__ import annotations
+
+import json
+import os
+import shlex
+import sqlite3
+import subprocess
+import threading
+import time
+from dataclasses import dataclass
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+from typing import Any, Literal
+
+from ..api_keys import bootstrap_provider_api_keys
+from .execution_context import get_execution_context
+
+StreamName = Literal["stdout", "stderr"]
+
+
+def utc_now_iso() -> str:
+    return datetime.now(timezone.utc).replace(microsecond=0).isoformat()
+
+
+def _split_base(cmd: str) -> list[str]:
+    parts = shlex.split(cmd)
+    return [p for p in parts if p]
+
+
+def _random_token(n: int = 6) -> str:
+    import secrets
+    import string
+
+    alphabet = string.ascii_lowercase + string.digits
+    return "".join(secrets.choice(alphabet) for _ in range(n))
+
+
+def _global_devflow_dir() -> Path:
+    base = os.environ.get("DEVFLOW_HOME") or os.environ.get("HOME")
+    if base:
+        return Path(base).expanduser().resolve() / ".devflow"
+    return Path.home().resolve() / ".devflow"
+
+
+def llm_logs_dir() -> Path:
+    return _global_devflow_dir() / "llm_logs"
+
+
+def llm_sessions_db() -> Path:
+    return _global_devflow_dir() / "llm_sessions.sqlite"
+
+
+def llm_journal_dir() -> Path:
+    return _global_devflow_dir() / "journal"
+
+
+def _ensure_sessions_schema(conn: sqlite3.Connection) -> None:
+    conn.executescript(
+        """
+        create table if not exists llm_sessions (
+          session_id text primary key,
+          provider text not null,
+          base_cmd text not null,
+          delivery text not null,
+          cwd text not null,
+          started_at text not null,
+          ended_at text,
+          returncode integer,
+          status text not null,
+          log_path text not null,
+          summary text
+        );
+
+        create table if not exists dev_journal_entries (
+          journal_entry_id text primary key,
+          session_id text not null,
+          provider text not null,
+          base_cmd text not null,
+          delivery text not null,
+          cwd text not null,
+          repo_root text,
+          run_id text,
+          node_exec_id text,
+          story_id text,
+          story_uuid text,
+          dag_id text,
+          started_at text not null,
+          ended_at text,
+          returncode integer,
+          status text not null,
+          summary text,
+          log_path text not null,
+          metadata_json text not null default '{}',
+          foreign key(session_id) references llm_sessions(session_id)
+        );
+
+        create index if not exists idx_dev_journal_entries_run_id_started_at on dev_journal_entries(run_id, started_at);
+        create index if not exists idx_dev_journal_entries_story_id_started_at on dev_journal_entries(story_id, started_at);
+        create index if not exists idx_dev_journal_entries_story_uuid_started_at on dev_journal_entries(story_uuid, started_at);
+        create index if not exists idx_dev_journal_entries_node_exec_id_started_at on dev_journal_entries(node_exec_id, started_at);
+        """
+    )
+
+
+@dataclass(frozen=True)
+class StreamResult:
+    ok: bool
+    stdout: str
+    stderr: str
+    returncode: int
+    session_id: str
+    log_path: Path
+
+
+def purge_llm_logs(*, days: int = 7) -> int:
+    """Purge log files older than N days."""
+
+    cutoff = datetime.now(timezone.utc) - timedelta(days=days)
+    root = llm_logs_dir()
+    if not root.exists():
+        return 0
+
+    removed = 0
+    for p in root.glob("*.jsonl"):
+        try:
+            mtime = datetime.fromtimestamp(p.stat().st_mtime, tz=timezone.utc)
+            if mtime < cutoff:
+                p.unlink(missing_ok=True)
+                removed += 1
+        except Exception:
+            continue
+
+    return removed
+
+
+def _resolve_journal_context(*, cwd: Path) -> dict[str, Any]:
+    ctx = get_execution_context() or {}
+    env_map = {
+        "run_id": os.environ.get("DEVFLOW_RUN_ID"),
+        "node_exec_id": os.environ.get("DEVFLOW_NODE_EXEC_ID"),
+        "story_id": os.environ.get("DEVFLOW_STORY_ID"),
+        "story_uuid": os.environ.get("DEVFLOW_STORY_UUID"),
+        "dag_id": os.environ.get("DEVFLOW_DAG_ID"),
+        "repo_root": os.environ.get("DEVFLOW_REPO_ROOT"),
+        "story_title": os.environ.get("DEVFLOW_STORY_TITLE"),
+    }
+    for key, value in env_map.items():
+        if key not in ctx and value:
+            ctx[key] = value
+    if "repo_root" not in ctx:
+        ctx["repo_root"] = str(cwd)
+    return ctx
+
+
+
+def _append_markdown_journal(*, session_id: str, provider: str, cwd: Path, base_cmd: str, status: str, summary: str, log_path: Path) -> Path:
+    """Append a short markdown entry to the daily journal."""
+
+    d = llm_journal_dir()
+    d.mkdir(parents=True, exist_ok=True)
+    day = datetime.now(timezone.utc).strftime("%Y-%m-%d")
+    md = d / f"{day}.md"
+
+    lines = [
+        f"## LLM session {session_id}",
+        f"- time (UTC): {utc_now_iso()}",
+        f"- provider: `{provider}`",
+        f"- status: `{status}`",
+        f"- cwd: `{cwd}`",
+        f"- cmd: `{base_cmd}`",
+        f"- log: `{log_path}`",
+        "",
+        "Summary:",
+        "```",
+        summary,
+        "```",
+        "",
+    ]
+
+    if md.exists():
+        existing = md.read_text(encoding="utf-8")
+        if not existing.endswith("\n"):
+            existing += "\n"
+        md.write_text(existing + "\n".join(lines), encoding="utf-8")
+    else:
+        md.write_text("# LLM Journal\n\n" + "\n".join(lines), encoding="utf-8")
+
+    return md
+
+
+def run_streaming(
+    *,
+    provider: str,
+    base_cmd: str,
+    delivery: str,
+    prompt: str,
+    cwd: Path,
+    purge_days: int = 7,
+    timeout_s: int | None = None,
+) -> StreamResult:
+    """Run a CLI prompt in the background while streaming stdout/stderr to JSONL.
+
+    - Captures full stdout/stderr in-memory (bounded by process output).
+    - Writes JSONL log lines to $DEVFLOW_HOME/.devflow/llm_logs/<session_id>.jsonl
+    - Records a session row into $DEVFLOW_HOME/.devflow/llm_sessions.sqlite
+
+    delivery:
+      - argument: append prompt as final argv element
+      - stdin: pass prompt via stdin
+    """
+
+    purge_llm_logs(days=purge_days)
+
+    argv = _split_base(base_cmd)
+    if not argv:
+        raise RuntimeError("llm cli base command is empty")
+
+    session_id = f"llm_{int(time.time())}_{_random_token()}"
+    log_dir = llm_logs_dir()
+    log_dir.mkdir(parents=True, exist_ok=True)
+    log_path = log_dir / f"{session_id}.jsonl"
+
+    started_at = utc_now_iso()
+    journal_ctx = _resolve_journal_context(cwd=cwd)
+
+    child_env = os.environ.copy()
+    if provider:
+        bootstrap_provider_api_keys(env=child_env)
+
+    # Prepare process
+    if delivery == "argument":
+        argv = argv + [prompt]
+        p = subprocess.Popen(argv, cwd=str(cwd), text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=child_env)
+        stdin_text = None
+    elif delivery == "stdin":
+        p = subprocess.Popen(argv, cwd=str(cwd), text=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=child_env)
+        stdin_text = prompt
+    else:
+        raise RuntimeError(f"unsupported delivery: {delivery}")
+
+    stdout_buf: list[str] = []
+    stderr_buf: list[str] = []
+    lock = threading.Lock()
+
+    def log_line(stream: StreamName, line: str) -> None:
+        rec = {"ts": utc_now_iso(), "stream": stream, "line": line.rstrip("\n")}
+        with log_path.open("a", encoding="utf-8") as f:
+            f.write(json.dumps(rec, sort_keys=True) + "\n")
+
+    def reader(stream: StreamName, fh) -> None:
+        try:
+            for line in iter(fh.readline, ""):
+                with lock:
+                    if stream == "stdout":
+                        stdout_buf.append(line)
+                    else:
+                        stderr_buf.append(line)
+                log_line(stream, line)
+        finally:
+            try:
+                fh.close()
+            except Exception:
+                pass
+
+    t_out = threading.Thread(target=reader, args=("stdout", p.stdout), daemon=True)  # type: ignore[arg-type]
+    t_err = threading.Thread(target=reader, args=("stderr", p.stderr), daemon=True)  # type: ignore[arg-type]
+    t_out.start()
+    t_err.start()
+
+    # Write initial session row + deterministic journal row at stream start.
+    db_path = llm_sessions_db()
+    db_path.parent.mkdir(parents=True, exist_ok=True)
+    journal_entry_id = f"journal_{session_id}"
+    conn = sqlite3.connect(str(db_path))
+    try:
+        _ensure_sessions_schema(conn)
+        conn.execute(
+            "insert or replace into llm_sessions(session_id,provider,base_cmd,delivery,cwd,started_at,status,log_path) values (?,?,?,?,?,?,?,?)",
+            (session_id, provider, base_cmd, delivery, str(cwd), started_at, "running", str(log_path)),
+        )
+        conn.execute(
+            "insert or replace into dev_journal_entries(journal_entry_id,session_id,provider,base_cmd,delivery,cwd,repo_root,run_id,node_exec_id,story_id,story_uuid,dag_id,started_at,status,log_path,metadata_json) values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
+            (
+                journal_entry_id,
+                session_id,
+                provider,
+                base_cmd,
+                delivery,
+                str(cwd),
+                str(journal_ctx.get("repo_root") or str(cwd)),
+                journal_ctx.get("run_id"),
+                journal_ctx.get("node_exec_id"),
+                journal_ctx.get("story_id"),
+                journal_ctx.get("story_uuid"),
+                journal_ctx.get("dag_id"),
+                started_at,
+                "running",
+                str(log_path),
+                json.dumps(journal_ctx, sort_keys=True),
+            ),
+        )
+        conn.commit()
+    finally:
+        conn.close()
+
+    if stdin_text is not None and p.stdin is not None:
+        try:
+            p.stdin.write(stdin_text)
+            p.stdin.close()
+        except Exception:
+            pass
+
+    # Wait for completion (with optional timeout)
+    try:
+        rc = p.wait(timeout=timeout_s)
+    except subprocess.TimeoutExpired:
+        try:
+            p.kill()
+        except Exception:
+            pass
+        rc = -1
+
+    # Ensure threads drained
+    t_out.join(timeout=1)
+    t_err.join(timeout=1)
+
+    ended_at = utc_now_iso()
+
+    stdout_text = "".join(stdout_buf)
+    stderr_text = "".join(stderr_buf)
+
+    ok = rc == 0
+    status = "ok" if ok else "error"
+
+    # Small deterministic summary (no LLM)
+    summary = f"returncode={rc}; stdout_lines={stdout_text.count(chr(10))}; stderr_lines={stderr_text.count(chr(10))}"
+
+    conn = sqlite3.connect(str(db_path))
+    try:
+        _ensure_sessions_schema(conn)
+        conn.execute(
+            "update llm_sessions set ended_at=?, returncode=?, status=?, summary=? where session_id=?",
+            (ended_at, int(rc), status, summary, session_id),
+        )
+        conn.execute(
+            "update dev_journal_entries set ended_at=?, returncode=?, status=?, summary=? where journal_entry_id=?",
+            (ended_at, int(rc), status, summary, journal_entry_id),
+        )
+        conn.commit()
+    finally:
+        conn.close()
+
+    # Append to markdown developer journal
+    _append_markdown_journal(
+        session_id=session_id,
+        provider=provider,
+        cwd=cwd,
+        base_cmd=base_cmd,
+        status=status,
+        summary=summary,
+        log_path=log_path,
+    )
+
+    return StreamResult(
+        ok=ok,
+        stdout=stdout_text,
+        stderr=stderr_text,
+        returncode=int(rc),
+        session_id=session_id,
+        log_path=log_path,
+    )

devflow_engine/llm/execution_context.py

@@ -0,0 +1,26 @@
+from __future__ import annotations
+
+from contextlib import contextmanager
+from contextvars import ContextVar
+from pathlib import Path
+from typing import Any, Iterator
+
+
+_CURRENT_EXECUTION_CONTEXT: ContextVar[dict[str, Any] | None] = ContextVar("devflow_execution_context", default=None)
+
+
+def get_execution_context() -> dict[str, Any] | None:
+    ctx = _CURRENT_EXECUTION_CONTEXT.get()
+    return dict(ctx) if isinstance(ctx, dict) else None
+
+
+@contextmanager
+def execution_context(**kwargs: Any) -> Iterator[None]:
+    ctx = {k: v for k, v in kwargs.items() if v is not None}
+    if "repo_root" in ctx and isinstance(ctx["repo_root"], Path):
+        ctx["repo_root"] = str(ctx["repo_root"])
+    token = _CURRENT_EXECUTION_CONTEXT.set(ctx)
+    try:
+        yield
+    finally:
+        _CURRENT_EXECUTION_CONTEXT.reset(token)