promty-collector 0.1.0

package/src/response_capture.py

@@ -0,0 +1,199 @@
+from __future__ import annotations
+
+import json
+import os
+from collections.abc import Mapping
+from pathlib import Path
+from typing import Any
+
+from payloads import first_string, first_value, payload_views
+
+RESPONSE_CAPTURE_MAX_CHARS = int(
+    os.environ.get("PROMPTHUB_RESPONSE_CAPTURE_MAX_CHARS", "50000")
+)
+TRANSCRIPT_CAPTURE_MAX_BYTES = int(
+    os.environ.get("PROMPTHUB_RESPONSE_TRANSCRIPT_MAX_BYTES", "1048576")
+)
+RESPONSE_KEYS = (
+    "response",
+    "assistant_response",
+    "assistant_message",
+    "last_assistant_message",
+    "answer",
+    "completion",
+    "output",
+    "output_text",
+    "result",
+    "text",
+    "message",
+    "content",
+)
+TRANSCRIPT_PATH_KEYS = ("transcript_path", "transcript")
+ASSISTANT_ROLES = {"assistant", "ai", "model"}
+SKIPPED_CONTENT_TYPES = {"tool_call", "tool_result", "tool_use", "function_call"}
+
+
+def response_payload_fields(raw_payload: Mapping[str, Any]) -> dict[str, Any]:
+    payloads = payload_views(raw_payload)
+    response_source: str | None = None
+    response_text = _direct_response_text(payloads)
+    if response_text:
+        response_source = "hook_payload"
+
+    transcript_path = first_string(payloads, TRANSCRIPT_PATH_KEYS)
+    if not response_text and transcript_path:
+        response_text = _response_text_from_transcript(transcript_path)
+        if response_text:
+            response_source = "transcript"
+
+    fields: dict[str, Any] = {}
+    if transcript_path:
+        fields["transcript_path"] = transcript_path
+    if not response_text:
+        return fields
+
+    original_length = len(response_text)
+    limit = max(RESPONSE_CAPTURE_MAX_CHARS, 1)
+    fields.update(
+        {
+            "response": response_text[:limit],
+            "response_original_length": original_length,
+            "response_source": response_source,
+            "response_storage_limit": limit,
+            "response_truncated": original_length > limit,
+        }
+    )
+    return fields
+
+
+def _direct_response_text(payloads: tuple[Mapping[str, Any], ...]) -> str | None:
+    for payload in payloads:
+        for key in RESPONSE_KEYS:
+            value = first_value((payload,), (key,))
+            text = _extract_text(value)
+            if text:
+                return text
+    return None
+
+
+def _response_text_from_transcript(path_value: str) -> str | None:
+    transcript_path = Path(path_value).expanduser()
+    if not transcript_path.is_file():
+        return None
+
+    try:
+        content = _read_transcript_tail(transcript_path)
+    except OSError:
+        return None
+
+    if not content.strip():
+        return None
+
+    parsed = _parse_json_document(content)
+    if parsed is not None:
+        text = _extract_assistant_text(parsed)
+        if text:
+            return text
+
+    records = []
+    for line in content.splitlines():
+        if not line.strip():
+            continue
+        try:
+            record = json.loads(line)
+        except json.JSONDecodeError:
+            continue
+        records.append(record)
+
+    for record in reversed(records):
+        text = _extract_assistant_text(record)
+        if text:
+            return text
+    return None
+
+
+def _read_transcript_tail(path: Path) -> str:
+    size = path.stat().st_size
+    max_bytes = max(TRANSCRIPT_CAPTURE_MAX_BYTES, 1)
+    with path.open("rb") as file:
+        if size > max_bytes:
+            file.seek(size - max_bytes)
+            file.readline()
+        return file.read(max_bytes).decode("utf-8", errors="ignore")
+
+
+def _parse_json_document(content: str) -> Any:
+    try:
+        return json.loads(content)
+    except json.JSONDecodeError:
+        return None
+
+
+def _extract_assistant_text(value: Any) -> str | None:
+    if isinstance(value, list):
+        for item in reversed(value):
+            text = _extract_assistant_text(item)
+            if text:
+                return text
+        return None
+
+    if not isinstance(value, dict):
+        return None
+
+    role = _role(value)
+    if role in ASSISTANT_ROLES:
+        return _extract_text(value)
+
+    value_type = value.get("type")
+    if isinstance(value_type, str) and "assistant" in value_type.lower():
+        text = _extract_text(value)
+        if text:
+            return text
+
+    for key in ("message", "payload", "item", "data", "response", "event"):
+        text = _extract_assistant_text(value.get(key))
+        if text:
+            return text
+
+    for key in ("messages", "items", "events", "responses", "output"):
+        text = _extract_assistant_text(value.get(key))
+        if text:
+            return text
+
+    return None
+
+
+def _role(value: Mapping[str, Any]) -> str | None:
+    role = value.get("role") or value.get("author")
+    if isinstance(role, str):
+        return role.lower()
+    if isinstance(role, Mapping):
+        nested_role = role.get("role") or role.get("name")
+        if isinstance(nested_role, str):
+            return nested_role.lower()
+    return None
+
+
+def _extract_text(value: Any) -> str | None:
+    if isinstance(value, str):
+        text = value.strip()
+        return text or None
+
+    if isinstance(value, list):
+        parts = [_extract_text(item) for item in value]
+        text = "\n".join(part for part in parts if part)
+        return text.strip() or None
+
+    if not isinstance(value, dict):
+        return None
+
+    value_type = value.get("type")
+    if isinstance(value_type, str) and value_type in SKIPPED_CONTENT_TYPES:
+        return None
+
+    for key in ("text", "output_text", "content", "message", "body", "response"):
+        text = _extract_text(value.get(key))
+        if text:
+            return text
+
+    return None

package/src/runtime_install.py

@@ -0,0 +1,162 @@
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+from pathlib import Path
+import secrets
+import shlex
+import shutil
+import subprocess
+import sys
+
+from file_lock import locked_file
+
+
+RUNTIME_MODULES: tuple[str, ...] = (
+    "change_tracking.py",
+    "cli.py",
+    "config.py",
+    "events.py",
+    "file_lock.py",
+    "git_context.py",
+    "payloads.py",
+    "response_capture.py",
+    "runtime_install.py",
+    "sequence.py",
+    "session_index.py",
+)
+RUNTIME_PACKAGES: tuple[str, ...] = ("adapters", "uploader")
+
+
+def runtime_home() -> Path:
+    return Path(os.environ.get("PROMTY_HOME", "~/.promty")).expanduser().resolve()
+
+
+def _runtime_python_command() -> str:
+    configured = os.environ.get("PROMPTHUB_HOOK_PYTHON")
+    if configured:
+        return configured
+    promty_python = os.environ.get("PROMTY_PYTHON")
+    if promty_python:
+        if os.name == "nt":
+            return subprocess.list2cmdline([promty_python])
+        return shlex.quote(promty_python)
+    if os.name == "nt":
+        return subprocess.list2cmdline([sys.executable])
+    return shlex.quote(sys.executable)
+
+
+def _source_files() -> tuple[tuple[Path, Path], ...]:
+    source_root = Path(__file__).resolve().parent
+    files: list[tuple[Path, Path]] = []
+    for module_name in RUNTIME_MODULES:
+        source_path = source_root / module_name
+        if not source_path.is_file():
+            raise FileNotFoundError(f"Promty runtime source is missing: {source_path}")
+        files.append((Path(module_name), source_path))
+
+    for package_name in RUNTIME_PACKAGES:
+        package_root = source_root / package_name
+        package_files = sorted(package_root.rglob("*.py"))
+        if not package_files:
+            raise FileNotFoundError(f"Promty runtime package is missing: {package_root}")
+        files.extend(
+            (source_path.relative_to(source_root), source_path) for source_path in package_files
+        )
+    return tuple(files)
+
+
+def _digest(files: tuple[tuple[Path, Path], ...]) -> str:
+    digest = hashlib.sha256()
+    for relative_path, source_path in files:
+        digest.update(relative_path.as_posix().encode("utf-8"))
+        digest.update(b"\0")
+        digest.update(source_path.read_bytes())
+        digest.update(b"\0")
+    return digest.hexdigest()
+
+
+def _is_complete(
+    runtime_path: Path,
+    files: tuple[tuple[Path, Path], ...],
+    expected_digest: str,
+) -> bool:
+    runtime_files = tuple(
+        (relative_path, runtime_path / relative_path) for relative_path, _ in files
+    )
+    if not all(path.is_file() for _, path in runtime_files):
+        return False
+    return _digest(runtime_files) == expected_digest
+
+
+def _install_files() -> Path:
+    files = _source_files()
+    digest = _digest(files)
+    runtime_root = runtime_home() / "runtime"
+    runtime_path = runtime_root / digest
+    runtime_root.mkdir(parents=True, exist_ok=True)
+    with locked_file(runtime_root / ".install.lock"):
+        if _is_complete(runtime_path, files, digest):
+            return runtime_path
+        if runtime_path.exists():
+            shutil.rmtree(runtime_path)
+
+        staging_path = runtime_root / f".{digest}.{secrets.token_hex(4)}.tmp"
+        try:
+            for relative_path, source_path in files:
+                destination = staging_path / relative_path
+                destination.parent.mkdir(parents=True, exist_ok=True)
+                shutil.copy2(source_path, destination)
+            (staging_path / "runtime.json").write_text(
+                json.dumps(
+                    {
+                        "digest": digest,
+                        "files": [path.as_posix() for path, _ in files],
+                    },
+                    indent=2,
+                    sort_keys=True,
+                )
+                + "\n",
+                encoding="utf-8",
+            )
+            staging_path.replace(runtime_path)
+        finally:
+            if staging_path.exists():
+                shutil.rmtree(staging_path)
+    return runtime_path
+
+
+def launcher_path() -> Path:
+    name = "promty.cmd" if os.name == "nt" else "promty"
+    return runtime_home() / "bin" / name
+
+
+def _write_launcher(runtime_path: Path) -> Path:
+    target = launcher_path()
+    cli_path = runtime_path / "cli.py"
+    if os.name == "nt":
+        content = f'@echo off\r\n{_runtime_python_command()} "{cli_path}" %*\r\n'
+    else:
+        content = f'#!/bin/sh\nexec {_runtime_python_command()} {shlex.quote(str(cli_path))} "$@"\n'
+
+    target.parent.mkdir(parents=True, exist_ok=True)
+    temporary_path = target.with_name(f".{target.name}.{secrets.token_hex(4)}.tmp")
+    try:
+        temporary_path.write_text(content, encoding="utf-8", newline="")
+        if os.name != "nt":
+            temporary_path.chmod(0o755)
+        temporary_path.replace(target)
+    finally:
+        temporary_path.unlink(missing_ok=True)
+    return target
+
+
+def install_runtime() -> Path:
+    return _write_launcher(_install_files())
+
+
+def quote_command_path(path: Path) -> str:
+    if os.name == "nt":
+        return subprocess.list2cmdline([str(path)])
+    return shlex.quote(str(path))

package/src/sequence.py

@@ -0,0 +1,68 @@
+from __future__ import annotations
+
+from contextlib import contextmanager
+import json
+import os
+from pathlib import Path
+from uuid import uuid4
+
+from events import BaseEvent
+from file_lock import locked_file
+
+DEFAULT_SEQUENCE_PATH = Path(
+    os.environ.get("PROMPTHUB_SEQUENCE_PATH", "~/.prompthub/sequences.json")
+).expanduser()
+
+
+class SequenceStore:
+    def __init__(self, path: str | Path | None = None) -> None:
+        self.path = Path(path).expanduser() if path else DEFAULT_SEQUENCE_PATH
+        self.lock_path = self.path.with_suffix(f"{self.path.suffix}.lock")
+
+    def assign(self, event: BaseEvent) -> BaseEvent:
+        with self._locked():
+            if event.sequence > 0:
+                self._observe(event.session_id, event.sequence)
+                return event
+
+            sequences = self._read()
+            next_sequence = sequences.get(event.session_id, 0) + 1
+            sequences[event.session_id] = next_sequence
+            self._write(sequences)
+            event.sequence = next_sequence
+            return event
+
+    @contextmanager
+    def _locked(self):
+        with locked_file(self.lock_path):
+            yield
+
+    def _observe(self, session_id: str, sequence: int) -> None:
+        sequences = self._read()
+        if sequences.get(session_id, 0) >= sequence:
+            return
+        sequences[session_id] = sequence
+        self._write(sequences)
+
+    def _read(self) -> dict[str, int]:
+        if not self.path.exists():
+            return {}
+
+        with self.path.open("r", encoding="utf-8") as file:
+            payload = json.load(file)
+
+        if not isinstance(payload, dict):
+            return {}
+
+        return {
+            key: value
+            for key, value in payload.items()
+            if isinstance(key, str) and isinstance(value, int)
+        }
+
+    def _write(self, sequences: dict[str, int]) -> None:
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        tmp_path = self.path.with_suffix(f"{self.path.suffix}.{uuid4()}.tmp")
+        with tmp_path.open("w", encoding="utf-8") as file:
+            json.dump(sequences, file, ensure_ascii=False)
+        tmp_path.replace(self.path)

package/src/session_index.py

@@ -0,0 +1,75 @@
+from __future__ import annotations
+
+from contextlib import contextmanager
+import json
+import os
+from pathlib import Path
+from typing import Any
+from uuid import uuid4
+
+from events import BaseEvent, SupportedTool, utc_now_iso
+from file_lock import locked_file
+
+DEFAULT_SESSION_INDEX_PATH = Path(
+    os.environ.get("PROMPTHUB_SESSION_INDEX_PATH", "~/.prompthub/session-index.json")
+).expanduser()
+
+
+class SessionIndex:
+    def __init__(self, path: str | Path | None = None) -> None:
+        self.path = Path(path).expanduser() if path else DEFAULT_SESSION_INDEX_PATH
+        self.lock_path = self.path.with_suffix(f"{self.path.suffix}.lock")
+
+    def lookup(self, tool: SupportedTool, external_session_id: str) -> dict[str, Any] | None:
+        with self._locked():
+            record = self._read().get(self._key(tool, external_session_id))
+        return record if isinstance(record, dict) else None
+
+    def observe(
+        self,
+        tool: SupportedTool,
+        external_session_id: str,
+        event: BaseEvent,
+        cwd: str | None = None,
+    ) -> None:
+        with self._locked():
+            records = self._read()
+            key = self._key(tool, external_session_id)
+            existing = records.get(key)
+            existing_cwd = existing.get("cwd") if isinstance(existing, dict) else None
+            records[key] = {
+                "tool": tool,
+                "external_session_id": external_session_id,
+                "project_id": event.project_id,
+                "session_id": event.session_id,
+                "cwd": cwd or existing_cwd,
+                "updated_at": utc_now_iso(),
+            }
+            self._write(records)
+
+    def _key(self, tool: SupportedTool, external_session_id: str) -> str:
+        return f"{tool}:{external_session_id}"
+
+    @contextmanager
+    def _locked(self):
+        with locked_file(self.lock_path):
+            yield
+
+    def _read(self) -> dict[str, Any]:
+        if not self.path.exists():
+            return {}
+
+        try:
+            with self.path.open("r", encoding="utf-8") as file:
+                payload = json.load(file)
+        except (OSError, json.JSONDecodeError):
+            return {}
+
+        return payload if isinstance(payload, dict) else {}
+
+    def _write(self, records: dict[str, Any]) -> None:
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        tmp_path = self.path.with_suffix(f"{self.path.suffix}.{uuid4()}.tmp")
+        with tmp_path.open("w", encoding="utf-8") as file:
+            json.dump(records, file, ensure_ascii=False)
+        tmp_path.replace(self.path)

package/src/uploader/__init__.py

@@ -0,0 +1 @@
+

package/src/uploader/client.py

@@ -0,0 +1,35 @@
+from __future__ import annotations
+
+import json
+from typing import Any
+from urllib import request
+
+
+class PromptHubUploader:
+    def __init__(self, api_url: str, token: str | None = None, timeout: float = 10) -> None:
+        self.api_url = api_url.rstrip("/")
+        self.token = token
+        self.timeout = timeout
+
+    def upload_events(self, events: list[dict[str, Any]]) -> list[str]:
+        if not events:
+            return []
+
+        body = json.dumps({"events": events}).encode("utf-8")
+        headers = {"Content-Type": "application/json"}
+        if self.token:
+            headers["Authorization"] = f"Bearer {self.token}"
+
+        req = request.Request(
+            f"{self.api_url}/api/events/batch",
+            data=body,
+            headers=headers,
+            method="POST",
+        )
+        with request.urlopen(req, timeout=self.timeout) as response:
+            payload = json.loads(response.read().decode("utf-8"))
+
+        event_ids = payload.get("event_ids")
+        if isinstance(event_ids, list):
+            return [event_id for event_id in event_ids if isinstance(event_id, str)]
+        return [event["id"] for event in events if isinstance(event.get("id"), str)]

package/src/uploader/queue.py

@@ -0,0 +1,130 @@
+from __future__ import annotations
+
+from contextlib import contextmanager
+import json
+import os
+from pathlib import Path
+from typing import Any
+from uuid import uuid4
+
+from events import BaseEvent
+from file_lock import locked_file
+
+DEFAULT_QUEUE_ROOT = Path(
+    os.environ.get("PROMPTHUB_QUEUE_DIR", "~/.prompthub/events")
+).expanduser()
+LEGACY_QUEUE_PATH = Path("~/.prompthub/events.jsonl").expanduser()
+
+
+def _safe_path_part(value: Any) -> str:
+    safe_value = "".join(
+        char for char in str(value) if char.isalnum() or char in ("-", "_")
+    )
+    return safe_value or "unknown"
+
+
+class JSONLQueue:
+    def __init__(self, path: str | Path | None = None) -> None:
+        configured_path = path or os.environ.get("PROMPTHUB_QUEUE_PATH")
+        self.root = None if configured_path else DEFAULT_QUEUE_ROOT
+        self.path = Path(configured_path).expanduser() if configured_path else DEFAULT_QUEUE_ROOT
+        self.lock_path = (
+            self.path.with_suffix(f"{self.path.suffix}.lock")
+            if self.root is None
+            else self.path / ".lock"
+        )
+
+    def push(self, event: BaseEvent) -> None:
+        with self._locked():
+            queue_file = self._event_path(event)
+            queue_file.parent.mkdir(parents=True, exist_ok=True)
+            with queue_file.open("a", encoding="utf-8") as file:
+                file.write(json.dumps(event.to_dict(), ensure_ascii=False))
+                file.write("\n")
+
+    def read_batch(self, limit: int = 100) -> list[dict[str, Any]]:
+        with self._locked():
+            events: list[dict[str, Any]] = []
+            for queue_file in self._queue_files():
+                with queue_file.open("r", encoding="utf-8") as file:
+                    for line in file:
+                        if len(events) >= limit:
+                            return events
+                        if not line.strip():
+                            continue
+                        try:
+                            event = json.loads(line)
+                        except json.JSONDecodeError:
+                            continue
+                        if isinstance(event, dict):
+                            events.append(event)
+            return events
+
+    def ack(self, event_ids: set[str]) -> None:
+        if not event_ids:
+            return
+
+        with self._locked():
+            for queue_file in self._queue_files():
+                self._ack_file(queue_file, event_ids)
+
+    def _event_path(self, event: BaseEvent) -> Path:
+        if self.root is None:
+            return self.path
+        return (
+            self.root
+            / _safe_path_part(event.project_id)
+            / _safe_path_part(event.session_id)
+            / "events.jsonl"
+        )
+
+    def _queue_files(self) -> list[Path]:
+        if self.root is None:
+            return [self.path] if self.path.exists() else []
+
+        queue_files: list[Path] = []
+        if LEGACY_QUEUE_PATH.exists():
+            queue_files.append(LEGACY_QUEUE_PATH)
+        if self.root.exists():
+            queue_files.extend(sorted(self.root.glob("*/*/events.jsonl")))
+        return queue_files
+
+    def _ack_file(self, queue_file: Path, event_ids: set[str]) -> None:
+        if not queue_file.exists():
+            return
+
+        remaining: list[str] = []
+        with queue_file.open("r", encoding="utf-8") as file:
+            for line in file:
+                if not line.strip():
+                    continue
+                try:
+                    event = json.loads(line)
+                except json.JSONDecodeError:
+                    continue
+                if not isinstance(event, dict):
+                    continue
+                if event.get("id") not in event_ids:
+                    remaining.append(line)
+
+        tmp_path = queue_file.with_suffix(f"{queue_file.suffix}.{uuid4()}.tmp")
+        if remaining:
+            with tmp_path.open("w", encoding="utf-8") as file:
+                file.writelines(remaining)
+            tmp_path.replace(queue_file)
+            return
+
+        try:
+            queue_file.unlink()
+        except OSError:
+            pass
+        if tmp_path.exists():
+            try:
+                tmp_path.unlink()
+            except OSError:
+                pass
+
+    @contextmanager
+    def _locked(self):
+        with locked_file(self.lock_path):
+            yield