agentdir-cli 0.7.5__py3-none-any.whl

agentdir/daemon.py

@@ -0,0 +1,253 @@
+from __future__ import annotations
+
+import json
+import os
+import signal
+import subprocess
+import sys
+import time
+from datetime import UTC, datetime
+from pathlib import Path
+from threading import Event
+from typing import Any
+
+from .federation import rebuild_registered_roots
+from .index import rebuild_index
+from .store import AgentDirError, init_root, paths_for, require_root
+
+DAEMON_STATE_FILE = "memory-daemon.json"
+DAEMON_STOP_FILE = "memory-daemon.stop"
+DAEMON_LOG_FILE = "memory-daemon.log"
+
+
+def start_memory_daemon(
+    root: str | Path,
+    *,
+    interval: float = 2.0,
+    group: str | None = None,
+    force: bool = False,
+) -> dict[str, Any]:
+    paths = init_root(root)
+    current = memory_daemon_status(paths.root)
+    if current["running"] and not force:
+        return {**current, "started": False}
+    _stop_path(paths.root).unlink(missing_ok=True)
+    log_path = paths.state / DAEMON_LOG_FILE
+    command = [
+        sys.executable,
+        "-m",
+        "agentdir",
+        "memory",
+        "daemon",
+        "run",
+        "--root",
+        str(paths.root),
+        "--interval",
+        str(interval),
+    ]
+    if group:
+        command.extend(["--group", group])
+    log = log_path.open("ab")
+    process = subprocess.Popen(
+        command,
+        stdout=log,
+        stderr=subprocess.STDOUT,
+        cwd=Path.cwd(),
+        env=os.environ.copy(),
+        start_new_session=True,
+    )
+    state = _daemon_state(paths.root, pid=process.pid, interval=interval, group=group)
+    state.update({"started": True, "log_path": str(log_path)})
+    _write_state(paths.root, state)
+    return memory_daemon_status(paths.root) | {"started": True}
+
+
+def run_memory_daemon(
+    root: str | Path,
+    *,
+    interval: float = 2.0,
+    group: str | None = None,
+    once: bool = False,
+) -> dict[str, Any]:
+    paths = init_root(root)
+    state = _daemon_state(paths.root, pid=os.getpid(), interval=interval, group=group)
+    _write_state(paths.root, state)
+    _refresh(paths.root, group=group)
+    if once:
+        return memory_daemon_status(paths.root)
+    if _watchfiles_available():
+        _watch_loop(paths.root, group=group, interval=interval)
+    else:
+        _poll_loop(paths.root, group=group, interval=interval)
+    return memory_daemon_status(paths.root)
+
+
+def stop_memory_daemon(root: str | Path, *, timeout: float = 5.0) -> dict[str, Any]:
+    paths = require_root(root)
+    _stop_path(paths.root).write_text(now_iso() + "\n", encoding="utf-8")
+    status = memory_daemon_status(paths.root)
+    pid = status.get("pid")
+    if pid and status["running"]:
+        try:
+            os.kill(int(pid), signal.SIGTERM)
+        except OSError:
+            pass
+        deadline = time.monotonic() + timeout
+        while time.monotonic() < deadline:
+            if not _pid_running(int(pid)):
+                break
+            time.sleep(0.05)
+    return memory_daemon_status(paths.root)
+
+
+def memory_daemon_status(root: str | Path) -> dict[str, Any]:
+    paths = require_root(root)
+    state = _read_state(paths.root)
+    pid = state.get("pid")
+    running = _pid_running(int(pid)) if pid else False
+    return {
+        **state,
+        "running": running,
+        "stop_requested": _stop_path(paths.root).is_file(),
+        "watch_backend": "watchfiles" if _watchfiles_available() else "poll",
+        "state_path": str(_state_path(paths.root)),
+        "log_path": str(paths.state / DAEMON_LOG_FILE),
+    }
+
+
+def format_memory_daemon_status(status: dict[str, Any]) -> str:
+    lines = [
+        f"running={str(status['running']).lower()}",
+        f"pid={status.get('pid') or ''}",
+        f"backend={status['watch_backend']}",
+        f"interval={status.get('interval') or ''}",
+        f"group={status.get('group') or ''}",
+        f"last_refresh_at={status.get('last_refresh_at') or ''}",
+        f"last_refresh_ok={str(status.get('last_refresh_ok', False)).lower()}",
+    ]
+    if status.get("last_error"):
+        lines.append(f"last_error={status['last_error']}")
+    return "\n".join(lines) + "\n"
+
+
+def now_iso() -> str:
+    return datetime.now(UTC).isoformat()
+
+
+def _poll_loop(root: Path, *, group: str | None, interval: float) -> None:
+    while not _stop_path(root).is_file():
+        time.sleep(max(interval, 0.1))
+        if _stop_path(root).is_file():
+            break
+        _refresh(root, group=group)
+
+
+def _watch_loop(root: Path, *, group: str | None, interval: float) -> None:
+    try:
+        from watchfiles import watch
+    except ImportError:
+        _poll_loop(root, group=group, interval=interval)
+        return
+    stop = Event()
+    watch_paths = _watch_paths(root)
+    timeout_ms = max(int(interval * 1000), 100)
+    for changes in watch(
+        *watch_paths,
+        stop_event=stop,
+        yield_on_timeout=True,
+        rust_timeout=timeout_ms,
+        recursive=True,
+    ):
+        if _stop_path(root).is_file():
+            stop.set()
+            break
+        if changes:
+            _refresh(root, group=group)
+
+
+def _refresh(root: Path, *, group: str | None) -> None:
+    state = _read_state(root)
+    try:
+        result = rebuild_index(root)
+        roots = rebuild_registered_roots(root, group=group)
+    except Exception as exc:
+        state.update(
+            {
+                "last_refresh_at": now_iso(),
+                "last_refresh_ok": False,
+                "last_error": str(exc),
+            }
+        )
+    else:
+        state.update(
+            {
+                "last_refresh_at": now_iso(),
+                "last_refresh_ok": True,
+                "last_error": None,
+                "indexed": result.indexed,
+                "malformed": result.malformed,
+                "registered_roots": len(roots),
+            }
+        )
+    _write_state(root, state)
+
+
+def _watch_paths(root: Path) -> list[str]:
+    paths = paths_for(root)
+    candidates = [paths.sessions, paths.actors, paths.queues, paths.state]
+    return [str(path) for path in candidates if path.exists()]
+
+
+def _daemon_state(root: Path, *, pid: int, interval: float, group: str | None) -> dict[str, Any]:
+    return {
+        "pid": pid,
+        "root": str(root),
+        "interval": interval,
+        "group": group,
+        "started_at": now_iso(),
+        "last_refresh_at": None,
+        "last_refresh_ok": False,
+        "last_error": None,
+    }
+
+
+def _read_state(root: Path) -> dict[str, Any]:
+    path = _state_path(root)
+    if not path.is_file():
+        return {}
+    try:
+        return json.loads(path.read_text(encoding="utf-8"))
+    except json.JSONDecodeError as exc:
+        raise AgentDirError(f"Invalid daemon state file: {exc}") from exc
+
+
+def _write_state(root: Path, state: dict[str, Any]) -> None:
+    path = _state_path(root)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(state, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+
+
+def _state_path(root: Path) -> Path:
+    return paths_for(root).state / DAEMON_STATE_FILE
+
+
+def _stop_path(root: Path) -> Path:
+    return paths_for(root).state / DAEMON_STOP_FILE
+
+
+def _pid_running(pid: int) -> bool:
+    if pid <= 0:
+        return False
+    try:
+        os.kill(pid, 0)
+    except OSError:
+        return False
+    return True
+
+
+def _watchfiles_available() -> bool:
+    try:
+        import watchfiles  # noqa: F401
+    except ImportError:
+        return False
+    return True

agentdir/doctor.py

@@ -0,0 +1,115 @@
+from __future__ import annotations
+
+import json
+import sqlite3
+from collections import defaultdict
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from .artifacts import artifact_path
+from .envelope import parse_envelope, validate_required
+from .mailbox import iter_records
+from .redaction import looks_secret_bearing
+from .secrets import scan_secret_records
+from .store import discover_mailboxes, require_root
+
+
+@dataclass
+class DoctorReport:
+    ok: bool = True
+    errors: list[str] = field(default_factory=list)
+    warnings: list[str] = field(default_factory=list)
+
+    def add_error(self, message: str) -> None:
+        self.ok = False
+        self.errors.append(message)
+
+    def add_warning(self, message: str) -> None:
+        self.warnings.append(message)
+
+    def as_dict(self) -> dict[str, object]:
+        return {"ok": self.ok, "errors": self.errors, "warnings": self.warnings}
+
+
+def run_doctor(root: str | Path) -> DoctorReport:
+    report = DoctorReport()
+    try:
+        paths = require_root(root)
+    except Exception as exc:
+        report.add_error(str(exc))
+        return report
+
+    for required in (paths.meta, paths.sessions, paths.actors, paths.artifacts, paths.indexes):
+        if not required.exists():
+            report.add_error(f"missing required path: {required}")
+
+    seen: dict[str, list[tuple[str, str]]] = defaultdict(list)
+    for _kind, mailbox in discover_mailboxes(root):
+        for record in iter_records(mailbox):
+            try:
+                parsed = parse_envelope(record.path)
+                missing = validate_required(parsed)
+                if missing:
+                    report.add_error(f"{record.path}: missing headers {', '.join(missing)}")
+                relative_path = str(record.path.relative_to(paths.root))
+                if parsed.message_id:
+                    seen[parsed.message_id].append((relative_path, parsed.body_sha256))
+                for sha in parsed.message.get_all("X-AgentDir-Blob-SHA256", []):
+                    if not artifact_path(root, sha).exists():
+                        report.add_error(f"{record.path}: missing artifact blob {sha}")
+            except Exception as exc:
+                report.add_error(f"{record.path}: parse error: {exc}")
+
+    for message_id, file_hashes in seen.items():
+        if len(file_hashes) > 1:
+            files = [path for path, _hash in file_hashes]
+            body_hashes = {_hash for _path, _hash in file_hashes}
+            if len(body_hashes) == 1:
+                report.add_warning(f"replicated Message-ID {message_id}: {json.dumps(files)}")
+            else:
+                report.add_error(f"conflicting duplicate Message-ID {message_id}: {json.dumps(files)}")
+
+    for tmp in paths.root.glob("**/Maildir/tmp/*"):
+        if tmp.is_file():
+            report.add_warning(f"incomplete tmp record ignored: {tmp.relative_to(paths.root)}")
+    secret_findings = scan_secret_records(paths.root)
+    for finding in secret_findings:
+        labels = ",".join(finding.labels)
+        report.add_error(
+            f"{paths.root / finding.path}: body contains secret-like text "
+            f"labels={labels}; run 'agentdir secrets redact --apply'"
+        )
+    if not secret_findings:
+        for finding in _index_secret_findings(paths.index_path):
+            report.add_error(
+                f"{paths.index_path}: {finding} contains secret-like indexed text; "
+                "run 'agentdir index rebuild'"
+            )
+    return report
+
+
+def _index_secret_findings(index_path: Path) -> list[str]:
+    if not index_path.is_file():
+        return []
+    findings: list[str] = []
+    tables = (
+        ("messages", "id", "body_text"),
+        ("memory_documents", "id", "body_text"),
+    )
+    try:
+        with sqlite3.connect(index_path) as conn:
+            for table, id_column, body_column in tables:
+                if not _table_exists(conn, table):
+                    continue
+                for row_id, body_text in conn.execute(f"select {id_column}, {body_column} from {table}"):
+                    if looks_secret_bearing(body_text or ""):
+                        findings.append(f"{table}.{id_column}={row_id}")
+                        break
+    except sqlite3.DatabaseError as exc:
+        findings.append(f"index scan failed: {exc}")
+    return findings
+
+
+def _table_exists(conn: sqlite3.Connection, table: str) -> bool:
+    row = conn.execute("select 1 from sqlite_master where type = 'table' and name = ?", (table,)).fetchone()
+    return row is not None

agentdir/envelope.py

@@ -0,0 +1,147 @@
+from __future__ import annotations
+
+import hashlib
+import time
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from email import policy
+from email.message import EmailMessage, Message
+from email.parser import BytesParser
+from email.utils import format_datetime, make_msgid
+from pathlib import Path
+from typing import Iterable
+
+from .store import AgentDirError
+
+AGENTDIR_VERSION = "0.1"
+
+
+@dataclass(frozen=True)
+class ParsedEnvelope:
+    message: Message
+    raw: bytes
+    path: Path
+
+    @property
+    def message_id(self) -> str | None:
+        return self.message.get("Message-ID")
+
+    @property
+    def body_text(self) -> str:
+        if self.message.is_multipart():
+            parts: list[str] = []
+            for part in self.message.walk():
+                if part.get_content_maintype() == "text":
+                    try:
+                        parts.append(part.get_content())
+                    except Exception:
+                        continue
+            return "\n".join(parts)
+        try:
+            content = self.message.get_content()
+        except Exception:
+            payload = self.message.get_payload(decode=True)
+            if payload is None:
+                return ""
+            return payload.decode("utf-8", errors="replace")
+        return content if isinstance(content, str) else str(content)
+
+    @property
+    def body_sha256(self) -> str:
+        return hashlib.sha256(self.body_text.encode("utf-8", errors="replace")).hexdigest()
+
+
+def actor_address(actor: str | None) -> str:
+    local = (actor or "agentdir").strip() or "agentdir"
+    if "@" in local:
+        return local
+    safe = "".join(ch if ch.isalnum() or ch in "._+-" else "-" for ch in local)
+    return f"{safe}@agentdir.local"
+
+
+def build_envelope(
+    *,
+    event_type: str,
+    body: str,
+    subject: str | None = None,
+    from_actor: str | None = None,
+    to_actor: str | None = None,
+    session_id: str | None = None,
+    task_id: str | None = None,
+    workspace: str | None = None,
+    git_head: str | None = None,
+    tool: str | None = None,
+    tool_exit_code: int | None = None,
+    parent_message_id: str | None = None,
+    references: Iterable[str] | None = None,
+    artifact_headers: dict[str, str | Iterable[str]] | None = None,
+    extra_headers: dict[str, str | Iterable[str]] | None = None,
+    message_id: str | None = None,
+) -> EmailMessage:
+    if not event_type:
+        raise AgentDirError("event_type is required")
+
+    msg = EmailMessage(policy=policy.default)
+    msg["Message-ID"] = message_id or make_msgid(domain="agentdir.local")
+    msg["Date"] = format_datetime(datetime.now(UTC))
+    msg["From"] = actor_address(from_actor)
+    msg["To"] = actor_address(to_actor or session_id or "agentdir")
+    msg["Subject"] = subject or event_type
+    msg["X-AgentDir-Version"] = AGENTDIR_VERSION
+    msg["X-AgentDir-Event-Type"] = event_type
+    msg["X-AgentDir-Created-Ns"] = str(time.time_ns())
+
+    optional = {
+        "X-AgentDir-Session": session_id,
+        "X-AgentDir-Task": task_id,
+        "X-AgentDir-Workspace": workspace,
+        "X-AgentDir-Git-Head": git_head,
+        "X-AgentDir-Tool": tool,
+        "X-AgentDir-Tool-Exit-Code": str(tool_exit_code) if tool_exit_code is not None else None,
+    }
+    for name, value in optional.items():
+        if value:
+            msg[name] = value
+
+    if parent_message_id:
+        msg["In-Reply-To"] = parent_message_id
+    refs = list(references or [])
+    if parent_message_id and parent_message_id not in refs:
+        refs.append(parent_message_id)
+    if refs:
+        msg["References"] = " ".join(refs)
+
+    for source in (artifact_headers or {}, extra_headers or {}):
+        for name, value in source.items():
+            if value is None:
+                continue
+            if isinstance(value, str):
+                msg[name] = value
+                continue
+            for item in value:
+                if item is not None:
+                    msg[name] = str(item)
+
+    msg.set_content(body, subtype="plain", charset="utf-8")
+    return msg
+
+
+def envelope_bytes(message: EmailMessage) -> bytes:
+    return message.as_bytes(policy=policy.default.clone(linesep="\n"))
+
+
+def parse_envelope(path: Path) -> ParsedEnvelope:
+    raw = path.read_bytes()
+    return ParsedEnvelope(
+        message=BytesParser(policy=policy.default).parsebytes(raw),
+        raw=raw,
+        path=path,
+    )
+
+
+def validate_required(parsed: ParsedEnvelope) -> list[str]:
+    return [
+        name
+        for name in ("Message-ID", "Date", "From", "To", "X-AgentDir-Version", "X-AgentDir-Event-Type")
+        if not parsed.message.get(name)
+    ]

agentdir/events.py

@@ -0,0 +1,72 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Iterable
+
+from .artifacts import add_artifact, artifact_headers
+from .envelope import build_envelope, envelope_bytes
+from .mailbox import atomic_deliver
+from .redaction import redact_text
+from .store import init_root, session_mailbox
+
+
+@dataclass(frozen=True)
+class EmittedEvent:
+    path: Path
+    session_id: str
+    event_type: str
+
+
+def emit_event(
+    root: str | Path,
+    *,
+    session_id: str,
+    event_type: str,
+    body: str,
+    subject: str | None = None,
+    from_actor: str | None = None,
+    to_actor: str | None = None,
+    task_id: str | None = None,
+    workspace: str | None = None,
+    git_head: str | None = None,
+    tool: str | None = None,
+    tool_exit_code: int | None = None,
+    parent_message_id: str | None = None,
+    artifact: str | Path | None = None,
+    extra_headers: dict[str, str | Iterable[str]] | None = None,
+    message_id: str | None = None,
+) -> EmittedEvent:
+    init_root(root)
+    stored_artifact = add_artifact(root, artifact) if artifact else None
+    redacted = redact_text(body)
+    headers = dict(extra_headers or {})
+    if redacted.replacements:
+        headers["X-AgentDir-Redactions"] = str(_redaction_count(headers) + redacted.replacements)
+        headers["X-AgentDir-Redaction-Labels"] = ",".join(redacted.labels)
+    message = build_envelope(
+        event_type=event_type,
+        body=redacted.text,
+        subject=subject,
+        from_actor=from_actor,
+        to_actor=to_actor,
+        session_id=session_id,
+        task_id=task_id,
+        workspace=workspace,
+        git_head=git_head,
+        tool=tool,
+        tool_exit_code=tool_exit_code,
+        parent_message_id=parent_message_id,
+        artifact_headers=artifact_headers(stored_artifact) if stored_artifact else {},
+        extra_headers=headers,
+        message_id=message_id,
+    )
+    delivered = atomic_deliver(session_mailbox(root, session_id), envelope_bytes(message))
+    return EmittedEvent(path=delivered, session_id=session_id, event_type=event_type)
+
+
+def _redaction_count(headers: dict[str, str | Iterable[str]]) -> int:
+    try:
+        return int(str(headers.get("X-AgentDir-Redactions") or "0"))
+    except ValueError:
+        return 0