netcodex-agent-exporter 0.3.0__py3-none-any.whl

netcodex/__init__.py

@@ -0,0 +1,6 @@
+"""NetCodex agent conversation exporter."""
+
+__all__ = ["__version__"]
+
+__version__ = "0.3.0"
+

netcodex/cli.py

@@ -0,0 +1,309 @@
+from __future__ import annotations
+
+import json
+import time
+from datetime import datetime
+from pathlib import Path
+
+import typer
+
+from netcodex.renderers import DEFAULT_TEMPLATE_ID, MarkdownOptions
+from netcodex.sources.paths import current_platform, path_matrix
+from netcodex.sources.registry import create_source, registered
+from netcodex.services import (
+    analyze_sources,
+    export_run,
+    SessionFilter,
+    format_names,
+    parse_when,
+    read_metadata_files,
+    scan_sources,
+    source_names,
+)
+
+app = typer.Typer(help="Export local agent conversations to Markdown, Quarkdown and PDF.")
+
+
+@app.command()
+def scan(
+    sources: str = typer.Option("all", help="Comma-separated sources, or all."),
+    json_output: bool = typer.Option(False, "--json", help="Print machine-readable JSON."),
+) -> None:
+    """List supported local conversation sessions without printing transcript bodies."""
+
+    payload = []
+    try:
+        payload = scan_sources(source_names(sources))
+    except ValueError as exc:
+        raise typer.BadParameter(str(exc)) from exc
+    if json_output:
+        typer.echo(json.dumps(payload, indent=2))
+        return
+    for item in payload:
+        typer.echo(f"{item['source']}: {item['count']} sessions")
+        for latest in item["latest"]:
+            typer.echo(f"  - {latest['id']} | {latest['title']} | {latest['updated_at']}")
+
+
+@app.command()
+def analyze(
+    source: str = typer.Option("auto", help="Source to analyze: auto, claude or codex."),
+    json_output: bool = typer.Option(False, "--json", help="Print machine-readable JSON."),
+) -> None:
+    """Analyze local conversation sources without printing transcript bodies."""
+
+    try:
+        payload = analyze_sources(source)
+    except ValueError as exc:
+        raise typer.BadParameter(str(exc)) from exc
+    if json_output:
+        typer.echo(json.dumps(payload, indent=2))
+        return
+
+    detected = payload["detected_source"] or "none"
+    typer.echo(f"detected source: {detected}")
+    for item in payload["sources"]:
+        typer.echo(f"{item['source']}: {item['count']} sessions")
+        for latest in item["latest"]:
+            typer.echo(f"  - {latest['id']} | {latest['title']} | {latest['updated_at']}")
+    for warning in payload["warnings"]:
+        typer.echo(f"warning: {warning}")
+
+
+@app.command("sources")
+def list_sources(
+    json_output: bool = typer.Option(False, "--json", help="Print machine-readable JSON."),
+) -> None:
+    """List registered sources, whether their local store exists, and where they are read from."""
+
+    matrix = path_matrix()
+    payload = []
+    for info in registered():
+        source = create_source(info.name)
+        available = source.available()
+        diagnostics = getattr(source, "diagnostics", None)
+        payload.append(
+            {
+                "source": info.name,
+                "label": info.label,
+                "available": available,
+                "paths": [str(path) for path in matrix.get(info.name, [])],
+                "notes": diagnostics() if available and callable(diagnostics) else [],
+            }
+        )
+    if json_output:
+        typer.echo(json.dumps({"platform": current_platform(), "sources": payload}, indent=2))
+        return
+    for item in payload:
+        status = "found" if item["available"] else "not found"
+        typer.echo(f"{item['source']}: {status} ({item['label']})")
+        for note in item["notes"]:
+            typer.echo(f"  note: {note}")
+
+
+@app.command()
+def export(
+    source: str = typer.Option(
+        "auto", help="Source(s) to export: auto, all, or a comma-separated list (see `netcodex sources`)."
+    ),
+    export_all: bool = typer.Option(False, "--all", help="Export every available source (same as --source all)."),
+    since: str | None = typer.Option(None, help="Only sessions active since a date (2026-06-01) or age (7d, 12h)."),
+    until: str | None = typer.Option(None, help="Only sessions last active before this date or age."),
+    workspace: str | None = typer.Option(None, help="Only sessions whose workspace path contains this text."),
+    thread: str | None = typer.Option(None, help="Conversation/session ID to export."),
+    formats: str = typer.Option("md", help="Comma-separated formats: md,qd,pdf."),
+    out: Path = typer.Option(Path("exports"), help="Output directory."),
+    limit: int = typer.Option(0, min=0, help="Latest sessions per source to export (0 = all)."),
+    include_paths: bool = typer.Option(False, help="Include full local source paths in metadata."),
+    template: str = typer.Option(DEFAULT_TEMPLATE_ID, help="Quarkdown template id for qd/pdf exports."),
+    json_output: bool = typer.Option(False, "--json", help="Print machine-readable JSON."),
+    local: bool = typer.Option(False, "--local", help="Confirm this export stays in the local runtime."),
+    include_reasoning: bool = typer.Option(
+        True, "--include-reasoning/--no-reasoning", help="Include reasoning/thinking as collapsible blocks."
+    ),
+    include_tool_output: bool = typer.Option(
+        True, "--include-tool-output/--no-tool-output", help="Include tool results under each tool call."
+    ),
+    include_system_context: bool = typer.Option(
+        False,
+        "--include-system-context/--no-system-context",
+        help="Include injected environment/instructions/hook context (hidden by default).",
+    ),
+    include_subagents: bool = typer.Option(
+        True, "--include-subagents/--no-subagents", help="Append subagent transcripts to their parent."
+    ),
+    max_tool_output_lines: int = typer.Option(
+        200, min=0, help="Truncate each tool output after N lines (0 = no limit)."
+    ),
+    max_tool_output_bytes: int = typer.Option(
+        32_000, min=0, help="Truncate each tool input/output after N bytes (0 = no limit)."
+    ),
+    jobs: int = typer.Option(0, min=0, help="Worker processes for large exports (0 = auto, 1 = none)."),
+    incremental: bool = typer.Option(
+        True,
+        "--incremental/--no-incremental",
+        help="Skip sessions unchanged since the last export into --out (state in .netcodex-state.json).",
+    ),
+    force: bool = typer.Option(False, "--force", help="Re-export every selected session."),
+) -> None:
+    """Export selected conversations (one folder per conversation plus an index.md)."""
+
+    try:
+        run = export_run(
+            "all" if export_all else source,
+            thread,
+            format_names(formats),
+            out,
+            limit,
+            include_paths,
+            template_id=template,
+            markdown_options=MarkdownOptions(
+                include_reasoning=include_reasoning,
+                include_tool_output=include_tool_output,
+                include_system_context=include_system_context,
+                include_subagents=include_subagents,
+                max_tool_output_lines=max_tool_output_lines,
+                max_tool_output_bytes=max_tool_output_bytes,
+                max_tool_input_bytes=max_tool_output_bytes,
+            ),
+            session_filter=SessionFilter(since=parse_when(since), until=parse_when(until), workspace=workspace),
+            incremental=incremental,
+            force=force,
+            jobs=jobs,
+        )
+    except ValueError as exc:
+        typer.echo(str(exc), err=True)
+        raise typer.Exit(1) from exc
+    results = run.results
+    if json_output:
+        typer.echo(
+            json.dumps(
+                {
+                    "local": local,
+                    "selected": run.selected,
+                    "skipped_unchanged": run.skipped,
+                    "results": [
+                        {
+                            "source": result.conversation.source,
+                            "conversation_id": result.conversation.id,
+                            "target": str(result.target),
+                            "steps": [step.model_dump(mode="json") for step in result.package.steps],
+                            "manifest": result.package.manifest.model_dump(mode="json"),
+                        }
+                        for result in results
+                    ],
+                },
+                indent=2,
+            )
+        )
+        return
+    for result in results:
+        typer.echo(
+            f"exported {result.conversation.source}:{result.conversation.id} -> {result.target}"
+        )
+        for step in result.package.steps:
+            typer.echo(f"  {step.name}: {step.status} | {step.message}")
+    if run.skipped:
+        typer.echo(f"skipped {run.skipped} unchanged session(s); use --force to re-export")
+
+
+@app.command()
+def watch(
+    source: str = typer.Option("all", help="Source(s) to watch: all, auto or a comma-separated list."),
+    out: Path = typer.Option(Path("exports"), help="Output directory."),
+    interval: float = typer.Option(60.0, min=0.0, help="Seconds between polls."),
+    iterations: int = typer.Option(0, min=0, help="Stop after N polls (0 = run until interrupted)."),
+    since: str | None = typer.Option(None, help="Only sessions active since a date or age (7d)."),
+    workspace: str | None = typer.Option(None, help="Only sessions whose workspace contains this text."),
+    formats: str = typer.Option("md", help="Comma-separated formats: md,qd,pdf."),
+) -> None:
+    """Poll the local stores and incrementally export new or changed conversations."""
+
+    try:
+        format_list = format_names(formats)
+        session_filter = SessionFilter(since=parse_when(since), workspace=workspace)
+    except ValueError as exc:
+        raise typer.BadParameter(str(exc)) from exc
+    poll = 0
+    typer.echo(f"watching {source} -> {out} every {interval:g}s (Ctrl+C to stop)")
+    try:
+        while True:
+            poll += 1
+            try:
+                run = export_run(source, None, format_list, out, 0, False, session_filter=session_filter, jobs=0)
+                typer.echo(
+                    f"[{datetime.now().strftime('%H:%M:%S')}] poll {poll}: "
+                    f"{len(run.results)} exported, {run.skipped} unchanged"
+                )
+            except ValueError as exc:
+                typer.echo(f"[{datetime.now().strftime('%H:%M:%S')}] poll {poll}: {exc}")
+            if iterations and poll >= iterations:
+                break
+            time.sleep(interval)
+    except KeyboardInterrupt:
+        typer.echo("stopped")
+
+
+@app.command()
+def validate(path: Path = typer.Argument(Path("exports"))) -> None:
+    """Validate generated export metadata files."""
+
+    if not path.exists():
+        raise typer.Exit(f"Path does not exist: {path}")
+    typer.echo(f"validated {read_metadata_files(path)} export metadata file(s)")
+
+
+LOOPBACK_BIND_HOSTS = {"127.0.0.1", "localhost", "::1"}
+
+
+@app.command()
+def ui(
+    host: str = typer.Option("127.0.0.1", help="Address to bind. Loopback only unless --allow-remote."),
+    port: int = typer.Option(0, min=0, max=65535, help="Port to listen on (0 = pick a free port)."),
+    out: Path = typer.Option(Path("exports"), help="Default export folder."),
+    open_browser: bool = typer.Option(True, "--open/--no-open", help="Open the UI in the default browser."),
+    allow_remote: bool = typer.Option(
+        False,
+        "--allow-remote",
+        help="Allow binding to a non-loopback address. Anyone who can reach it and has the URL token "
+        "can read your conversations.",
+    ),
+) -> None:
+    """Start the local web UI (reads local stores directly; nothing is uploaded)."""
+
+    if host not in LOOPBACK_BIND_HOSTS and not allow_remote:
+        raise typer.BadParameter(
+            f"Refusing to bind to {host}: the UI exposes your local conversations. "
+            "Use 127.0.0.1, or pass --allow-remote if you really mean it.",
+            param_hint="--host",
+        )
+    try:
+        import uvicorn
+
+        from netcodex.web.local_ui import create_local_app
+    except ImportError as exc:
+        typer.echo(
+            "The local UI needs the optional 'ui' extra: "
+            "pipx install 'netcodex-agent-exporter[ui]' (or uv tool install 'netcodex-agent-exporter[ui]').",
+            err=True,
+        )
+        raise typer.Exit(1) from exc
+
+    import secrets
+    import socket
+    import threading
+    import webbrowser
+
+    if port == 0:
+        with socket.socket(socket.AF_INET6 if ":" in host else socket.AF_INET) as probe:
+            probe.bind((host, 0))
+            port = probe.getsockname()[1]
+    token = secrets.token_urlsafe(24)
+    application = create_local_app(out.resolve(), token=token, allow_remote=allow_remote)
+    shown_host = f"[{host}]" if ":" in host else host
+    url = f"http://{shown_host}:{port}/#token={token}"
+    typer.echo(f"NetCodex UI: {url}")
+    typer.echo("Press Ctrl+C to stop.")
+    if open_browser:
+        threading.Timer(1.0, webbrowser.open, args=(url,)).start()
+    uvicorn.run(application, host=host, port=port, log_level="warning")

netcodex/incremental.py

@@ -0,0 +1,106 @@
+"""Incremental export state.
+
+``<out>/.netcodex-state.json`` remembers, per ``<source>:<id>``, a fingerprint of the source
+files (size + mtime of the session file and any subagent files) together with a hash of the
+export options, and the folder the conversation was written to. Unchanged sessions are
+skipped on the next run; ``--force`` re-exports everything.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import shutil
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+from netcodex.models import SourceSession
+
+STATE_FILE = ".netcodex-state.json"
+STATE_VERSION = 1
+
+
+def session_fingerprint(session: SourceSession) -> str:
+    """Cheap change detector: size and mtime of every file the session is parsed from."""
+
+    paths = [session.path]
+    subagents = session.path.parent / session.path.stem / "subagents"
+    if subagents.is_dir():
+        paths.extend(sorted(subagents.iterdir()))
+    extra = session.metadata.get("fingerprint_paths")
+    if isinstance(extra, str):
+        paths.extend(Path(item) for item in extra.split("|") if item)
+    digest = hashlib.sha256(session.title.encode("utf-8", errors="replace"))
+    for path in paths:
+        try:
+            stat = path.stat()
+        except OSError:
+            digest.update(f"{path.name}:missing".encode())
+            continue
+        digest.update(f"{path.name}:{stat.st_size}:{stat.st_mtime_ns}".encode())
+    return digest.hexdigest()[:32]
+
+
+def options_fingerprint(options: dict[str, Any]) -> str:
+    return hashlib.sha256(json.dumps(options, sort_keys=True, default=str).encode()).hexdigest()[:16]
+
+
+@dataclass
+class ExportState:
+    out: Path
+    entries: dict[str, dict[str, Any]] = field(default_factory=dict)
+
+    @classmethod
+    def load(cls, out: Path) -> "ExportState":
+        path = out / STATE_FILE
+        try:
+            data = json.loads(path.read_text(encoding="utf-8"))
+        except (OSError, ValueError):
+            return cls(out=out)
+        if not isinstance(data, dict) or data.get("version") != STATE_VERSION:
+            return cls(out=out)
+        entries = data.get("entries")
+        return cls(out=out, entries=entries if isinstance(entries, dict) else {})
+
+    @staticmethod
+    def key(session: SourceSession) -> str:
+        return f"{session.source}:{session.id}"
+
+    def is_current(self, session: SourceSession, fingerprint: str, options: str) -> bool:
+        entry = self.entries.get(self.key(session))
+        if not entry or entry.get("fingerprint") != fingerprint or entry.get("options") != options:
+            return False
+        folder = entry.get("folder")
+        return bool(folder) and (self.out / str(folder)).is_dir()
+
+    def record(self, session: SourceSession, fingerprint: str, options: str, target: Path) -> None:
+        key = self.key(session)
+        folder = target.relative_to(self.out).as_posix()
+        previous = self.entries.get(key, {}).get("folder")
+        if previous and previous != folder:
+            self._remove_stale_folder(str(previous))
+        self.entries[key] = {
+            "fingerprint": fingerprint,
+            "options": options,
+            "folder": folder,
+            "exported_at": datetime.now(timezone.utc).isoformat(),
+        }
+
+    def save(self) -> Path:
+        self.out.mkdir(parents=True, exist_ok=True)
+        path = self.out / STATE_FILE
+        payload = {"version": STATE_VERSION, "entries": dict(sorted(self.entries.items()))}
+        tmp = path.with_suffix(".tmp")
+        tmp.write_text(json.dumps(payload, indent=2), encoding="utf-8")
+        tmp.replace(path)
+        return path
+
+    def _remove_stale_folder(self, folder: str) -> None:
+        # A renamed conversation (new title -> new folder) must not leave a duplicate behind.
+        # Only folders recorded by this state file and inside the export root are removed.
+        stale = (self.out / folder).resolve()
+        root = self.out.resolve()
+        if stale != root and root in stale.parents and (stale / "metadata.json").exists():
+            shutil.rmtree(stale, ignore_errors=True)

netcodex/models.py

@@ -0,0 +1,145 @@
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Literal
+
+from pydantic import BaseModel, Field
+
+SourceName = str  # registry key, e.g. "codex", "claude", "copilot-chat"
+Role = Literal["user", "assistant", "system", "tool", "summary"]
+PartKind = Literal[
+    "text",
+    "code",
+    "tool_call",
+    "tool_result",
+    "reasoning",
+    "image",
+    "attachment",
+    "system_context",
+    "summary",
+    "tool_event",
+]
+MetadataValue = str | int | float | bool | None
+PipelineStepName = Literal["analyze", "parse", "render", "package"]
+PipelineStepStatus = Literal["pending", "running", "success", "warning", "error"]
+
+
+class MessagePart(BaseModel):
+    """One block of a turn.
+
+    ``tool_call`` parts carry the tool ``name`` and ``call_id``; their ``content`` is the
+    arguments/input. ``tool_result`` parts carry the matching ``call_id`` and the output as
+    ``content``. ``image``/``attachment`` parts are placeholders: ``content`` is a short label
+    and ``metadata`` describes the original payload (media type, size, file name...).
+    """
+
+    kind: PartKind = "text"
+    content: str = ""
+    language: str | None = None
+    name: str | None = None
+    call_id: str | None = None
+    is_error: bool = False
+    metadata: dict[str, MetadataValue] = Field(default_factory=dict)
+
+
+class Turn(BaseModel):
+    id: str
+    role: Role
+    timestamp: datetime | None = None
+    parts: list[MessagePart] = Field(default_factory=list)
+
+    @property
+    def text(self) -> str:
+        return "\n\n".join(part.content for part in self.parts if part.content)
+
+
+class Conversation(BaseModel):
+    id: str
+    source: SourceName
+    title: str
+    created_at: datetime | None = None
+    updated_at: datetime | None = None
+    ended_at: datetime | None = None
+    workspace: str | None = None
+    tool: str | None = None
+    originator: str | None = None
+    model: str | None = None
+    git_branch: str | None = None
+    parent_id: str | None = None
+    agent_name: str | None = None
+    turns: list[Turn] = Field(default_factory=list)
+    subagents: list["Conversation"] = Field(default_factory=list)
+    metadata: dict[str, MetadataValue] = Field(default_factory=dict)
+
+    @property
+    def started_at(self) -> datetime | None:
+        if self.created_at:
+            return self.created_at
+        stamps = [turn.timestamp for turn in self.turns if turn.timestamp]
+        return min(stamps) if stamps else None
+
+    @property
+    def last_activity_at(self) -> datetime | None:
+        if self.ended_at:
+            return self.ended_at
+        stamps = [turn.timestamp for turn in self.turns if turn.timestamp]
+        return max(stamps) if stamps else self.updated_at
+
+
+class SourceSession(BaseModel):
+    id: str
+    source: SourceName
+    title: str
+    path: Path
+    updated_at: datetime | None = None
+    metadata: dict[str, MetadataValue] = Field(default_factory=dict)
+
+
+class ExportMetadata(BaseModel):
+    exported_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
+    source: SourceName
+    conversation_id: str
+    formats: list[str]
+    template: str | None = None
+    include_paths: bool = False
+    title: str | None = None
+    tool: str | None = None
+    model: str | None = None
+    started_at: datetime | None = None
+    ended_at: datetime | None = None
+    turn_count: int | None = None
+    parent_id: str | None = None
+    subagent_count: int = 0
+    markdown: str | None = None
+
+
+class PipelineStep(BaseModel):
+    name: PipelineStepName
+    status: PipelineStepStatus
+    label: str
+    message: str
+    warnings: list[str] = Field(default_factory=list)
+
+
+class ExportArtifact(BaseModel):
+    path: str
+    bytes: int
+    sha256: str
+
+
+class ExportManifest(BaseModel):
+    schema_version: str = "0.1"
+    job_id: str
+    created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
+    source: SourceName
+    session_count: int
+    conversation_id: str
+    conversation_title: str
+    formats: list[str]
+    template: str
+    template_label: str
+    privacy: dict[str, bool]
+    steps: list[PipelineStep]
+    artifacts: list[ExportArtifact]
+    warnings: list[str] = Field(default_factory=list)

netcodex/renderers/__init__.py

@@ -0,0 +1,20 @@
+from netcodex.renderers.markdown import MarkdownOptions, render_index, render_markdown, visible_turns
+from netcodex.renderers.quarkdown import (
+    DEFAULT_TEMPLATE_ID,
+    render_quarkdown,
+    resolve_template,
+    template_choices,
+    template_label,
+)
+
+__all__ = [
+    "DEFAULT_TEMPLATE_ID",
+    "MarkdownOptions",
+    "render_index",
+    "visible_turns",
+    "render_markdown",
+    "render_quarkdown",
+    "resolve_template",
+    "template_choices",
+    "template_label",
+]