ghost-reader 0.0.1__py3-none-any.whl

ghost_reader/sync.py

@@ -0,0 +1,112 @@
+from __future__ import annotations
+
+import json
+import time
+import urllib.error
+import urllib.request
+from pathlib import Path
+
+from ghost_reader.errors import GhostReaderError
+from ghost_reader.io import write_yaml
+from ghost_reader.paths import sync_state_path
+from ghost_reader.project import read_config
+from ghost_reader.telemetry import pending_events, read_sync_state
+from ghost_reader.time import now_iso, timestamp
+
+_MAX_RETRIES = 3
+_RETRY_BASE_DELAY = 2.0  # seconds, doubles each retry
+
+
+def _post_bundle(endpoint: str, bundle_path: Path) -> bool:
+    """POST the NDJSON bundle to the sync endpoint with retry + backoff."""
+    payload = bundle_path.read_bytes()
+    last_error: str | None = None
+    for attempt in range(_MAX_RETRIES):
+        try:
+            req = urllib.request.Request(
+                endpoint,
+                data=payload,
+                headers={"Content-Type": "application/x-ndjson"},
+                method="POST",
+            )
+            resp = urllib.request.urlopen(req, timeout=30)
+            if 200 <= resp.status < 300:
+                return True
+            last_error = f"HTTP {resp.status}"
+        except urllib.error.HTTPError as exc:
+            last_error = f"HTTP {exc.code}"
+        except Exception as exc:
+            last_error = str(exc)
+        if attempt < _MAX_RETRIES - 1:
+            time.sleep(_RETRY_BASE_DELAY * (2 ** attempt))
+    raise GhostReaderError(f"Sync failed after {_MAX_RETRIES} attempts: {last_error}")
+
+
+def create_sync_bundle(home: Path) -> dict[str, object]:
+    events = pending_events(home)
+    if not events:
+        return {
+            "sync_status": "no_pending_events",
+            "event_count": 0,
+            "retained_local_telemetry": True,
+        }
+
+    created_at = now_iso()
+    bundle_id = timestamp()
+    bundle = home / "telemetry" / "synced" / f"{bundle_id}-bundle.ndjson"
+    bundle.write_text(
+        "\n".join(
+            json.dumps(event, ensure_ascii=False, sort_keys=True) for event in events
+        )
+        + "\n",
+        encoding="utf-8",
+    )
+
+    # Attempt remote sync if configured
+    sync_status = "local_bundle_created"
+    last_error: str | None = None
+    try:
+        config = read_config(home)
+        endpoint = (config.get("telemetry") or {}).get("sync_endpoint")
+        if endpoint and isinstance(endpoint, str) and endpoint.strip():
+            _post_bundle(endpoint.strip(), bundle)
+            sync_status = "remote_sync_succeeded"
+    except Exception as exc:
+        last_error = str(exc)
+        sync_status = "remote_sync_failed"
+
+    receipt = {
+        "schema_version": 1,
+        "created_at": created_at,
+        "status": sync_status,
+        "bundle": str(bundle),
+        "event_count": len(events),
+        "event_ids": [event["event_id"] for event in events],
+    }
+    if last_error:
+        receipt["last_error"] = last_error
+    write_yaml(home / "telemetry" / "receipts" / f"{bundle_id}.yaml", receipt)
+    state = read_sync_state(home)
+    state.update(
+        {
+            "schema_version": 1,
+            "synced_event_ids": list(
+                dict.fromkeys(
+                    [*state.get("synced_event_ids", []), *receipt["event_ids"]]
+                )
+            ),
+            "last_sync_at": receipt["created_at"],
+            "last_sync_status": receipt["status"],
+            "last_error": last_error,
+        }
+    )
+    write_yaml(sync_state_path(home), state)
+    result: dict[str, object] = {
+        "sync_status": sync_status,
+        "bundle": str(bundle),
+        "event_count": len(events),
+        "retained_local_telemetry": True,
+    }
+    if last_error:
+        result["last_error"] = last_error
+    return result

ghost_reader/telemetry.py

@@ -0,0 +1,111 @@
+from __future__ import annotations
+
+import json
+import uuid
+from pathlib import Path
+from typing import Any
+
+from ghost_reader import __version__
+from ghost_reader.constants import CONTENT_SENSITIVE_TELEMETRY_KEYS, TELEMETRY_EVENTS
+from ghost_reader.errors import GhostReaderError
+from ghost_reader.io import load_yaml_file
+from ghost_reader.paths import outbox_path, session_dir, sync_state_path
+from ghost_reader.project import read_config
+from ghost_reader.time import now_iso
+
+
+def append_ndjson(path: Path, payload: dict[str, Any]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("a", encoding="utf-8") as handle:
+        handle.write(json.dumps(payload, ensure_ascii=False, sort_keys=True) + "\n")
+
+
+def read_events(path: Path) -> list[dict[str, Any]]:
+    if not path.exists():
+        return []
+    return [
+        json.loads(line)
+        for line in path.read_text(encoding="utf-8").splitlines()
+        if line.strip()
+    ]
+
+
+def sanitize_meta(meta: dict[str, Any]) -> dict[str, Any]:
+    forbidden = sorted(CONTENT_SENSITIVE_TELEMETRY_KEYS.intersection(meta.keys()))
+    if forbidden:
+        raise GhostReaderError(
+            f"Telemetry metadata includes forbidden content-sensitive keys: {', '.join(forbidden)}"
+        )
+    return meta
+
+
+def append_event(
+    home: Path,
+    project_root: Path | None,
+    event_name: str,
+    session_id: str | None = None,
+    surface: str = "cli",
+    meta: dict[str, Any] | None = None,
+    correlation_id: str | None = None,
+) -> dict[str, Any]:
+    if event_name not in TELEMETRY_EVENTS:
+        raise GhostReaderError(f"Unsupported telemetry event `{event_name}`.")
+    config = read_config(home)
+    payload = {
+        "schema_version": 1,
+        "event_id": str(uuid.uuid4()),
+        "occurred_at": now_iso(),
+        "event": event_name,
+        "tool_version": __version__,
+        "surface": surface,
+        "install_id": config.get("install_id"),
+        "session_id": session_id,
+        "meta": sanitize_meta(meta or {}),
+    }
+    if correlation_id:
+        payload["correlation_id"] = correlation_id
+    append_ndjson(outbox_path(home), payload)
+    if project_root and session_id:
+        append_ndjson(
+            session_dir(project_root, session_id) / "telemetry" / "events.ndjson",
+            payload,
+        )
+    return payload
+
+
+def read_sync_state(home: Path) -> dict[str, Any]:
+    path = sync_state_path(home)
+    if not path.exists():
+        return {
+            "schema_version": 1,
+            "synced_event_ids": [],
+            "last_sync_at": None,
+            "last_sync_status": None,
+            "last_error": None,
+        }
+    state = load_yaml_file(path)
+    state.setdefault("synced_event_ids", [])
+    return state
+
+
+def pending_events(home: Path) -> list[dict[str, Any]]:
+    synced = set(read_sync_state(home).get("synced_event_ids", []))
+    return [
+        event
+        for event in read_events(outbox_path(home))
+        if event.get("event_id") not in synced
+    ]
+
+
+def telemetry_status(home: Path) -> dict[str, Any]:
+    state = read_sync_state(home)
+    events = pending_events(home)
+    return {
+        "pending_events": len(events),
+        "pending_sessions": len(
+            {event.get("session_id") for event in events if event.get("session_id")}
+        ),
+        "last_sync_at": state.get("last_sync_at"),
+        "last_sync_status": state.get("last_sync_status"),
+        "last_error": state.get("last_error"),
+    }

ghost_reader/time.py

@@ -0,0 +1,20 @@
+from __future__ import annotations
+
+from datetime import datetime, timezone
+
+
+def now_iso() -> str:
+    return datetime.now(timezone.utc).astimezone().isoformat(timespec="seconds")
+
+
+def timestamp() -> str:
+    return datetime.now().strftime("%Y%m%d-%H%M%S")
+
+
+def new_session_id(story_unit: str | None = None) -> str:
+    stamp = datetime.now().strftime("%Y-%m-%d-%H%M%S")
+    if not story_unit:
+        return stamp
+    slug = "".join(ch.lower() if ch.isalnum() else "-" for ch in story_unit).strip("-")
+    slug = "-".join(part for part in slug.split("-") if part)
+    return f"{stamp}-{slug[:48]}" if slug else stamp

ghost_reader/validators.py

@@ -0,0 +1,66 @@
+"""Validator functions — thin wrappers around Pydantic models.
+
+Preserved for backward compatibility. All functions delegate to Pydantic
+model_validate() and wrap ValidationError as GhostReaderError.
+
+New code should use ghost_reader.models.{Persona,Review,Feedback} directly.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from ghost_reader.errors import GhostReaderError
+from ghost_reader.models import Feedback, Persona, Review
+
+
+def _wrap(cls: type, data: dict[str, Any]) -> Any:
+    """Validate dict against Pydantic model, wrapping as GhostReaderError."""
+    from pydantic import ValidationError
+
+    try:
+        return cls.model_validate(data)
+    except ValidationError as e:
+        msgs = [f"{err['loc']}: {err['msg']}" for err in e.errors()]
+        raise GhostReaderError(f"Validation failed: {'; '.join(msgs)}") from e
+
+
+def validate_persona(persona: dict[str, Any]) -> None:
+    """Validate a persona dict against the Persona schema.
+
+    Raises GhostReaderError on validation failure (backward compatible).
+    """
+    _wrap(Persona, persona)
+
+
+def validate_review(
+    review: dict[str, Any],
+    session_id: str | None = None,
+    persona_id: str | None = None,
+) -> None:
+    """Validate a review dict against the Review schema.
+
+    Raises GhostReaderError on validation failure (backward compatible).
+    Optionally validates session_id and persona_id match if provided.
+    """
+    model = _wrap(Review, review)
+    if session_id and model.session_id != session_id:
+        raise GhostReaderError(
+            f"Review session_id `{model.session_id}` does not match `{session_id}`."
+        )
+    if persona_id and model.persona_id != persona_id:
+        raise GhostReaderError(
+            f"Review persona_id `{model.persona_id}` does not match `{persona_id}`."
+        )
+
+
+def validate_feedback(feedback: dict[str, Any], session_id: str) -> None:
+    """Validate a feedback dict against the Feedback schema.
+
+    Raises GhostReaderError on validation failure (backward compatible).
+    """
+    model = _wrap(Feedback, feedback)
+    if model.session_id != session_id:
+        raise GhostReaderError(
+            f"Feedback session_id `{model.session_id}` does not match `{session_id}`."
+        )

ghost_reader/verify.py

@@ -0,0 +1,255 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+from ghost_reader.constants import SCHEMA_VERSIONS
+from ghost_reader.errors import GhostReaderError
+from ghost_reader.io import load_yaml_file
+from ghost_reader.paths import artifact_paths
+from ghost_reader.project import load_manifest, save_manifest, save_round_manifest
+from ghost_reader.reviews import feedback_summary, load_reviews
+from ghost_reader.telemetry import read_events
+
+
+def verify_session(project_root: Path, home: Path, session_id: str) -> dict[str, Any]:
+    paths = artifact_paths(project_root, session_id)
+    session_path = paths["session"]
+    manifest = load_manifest(project_root, session_id)
+    personas = manifest.get("personas", [])
+    current_round = manifest.get("current_round", 1)
+    issues: list[str] = []
+    warnings: list[str] = []
+
+    if not personas:
+        issues.append("manifest has no selected personas")
+
+    reviews = load_reviews(project_root, home, session_id)
+    review_personas = {review["persona_id"] for review in reviews}
+    missing_reviews = [
+        persona_id for persona_id in personas if persona_id not in review_personas
+    ]
+    if missing_reviews:
+        issues.append(f"missing review artifacts for: {', '.join(missing_reviews)}")
+
+    round_manifest_path = paths["active_round"] / "manifest.yaml"
+    refinement_count = 0
+    round_manifest = None
+    if round_manifest_path.exists():
+        round_manifest = load_yaml_file(round_manifest_path)
+        refinement_count = round_manifest.get("refinement_count", 0)
+
+    report_summary = verify_report(paths, session_id, personas, issues)
+    telemetry_summary = verify_telemetry(session_path, session_id, personas, issues)
+    feedback = feedback_summary(project_root, session_id)
+    prompt_summary = verify_prompt(
+        paths, session_id, feedback["feedback_found"], issues
+    )
+    sync_summary = verify_sync_scaffold(home, issues)
+    verify_no_mixed_artifacts(session_path, issues)
+
+    dialogue_dir = paths["dialogue"]
+    dialogue_files = (
+        sorted(dialogue_dir.glob("*.md"))
+        if dialogue_dir.exists()
+        else []
+    )
+    dialogue_index_found = (dialogue_dir / "index.yaml").exists() if dialogue_dir.exists() else False
+
+    if issues:
+        raise GhostReaderError("Verification failed: " + "; ".join(issues))
+
+    revision_prompt_found = prompt_summary.get("revision_prompt_found", False)
+    if feedback["feedback_found"] and revision_prompt_found:
+        status = "phase1_complete"
+    elif refinement_count > 0 and not revision_prompt_found:
+        status = "refinement_complete"
+    else:
+        status = "review_ready_feedback_pending"
+
+    if round_manifest_path.exists() and round_manifest.get("status") != status:
+        round_manifest["status"] = status
+        save_round_manifest(project_root, session_id, current_round, round_manifest)
+        for entry in manifest.get("rounds", []):
+            if entry.get("id") == current_round:
+                entry["status"] = status
+                break
+        save_manifest(project_root, session_id, manifest)
+
+    return {
+        "ok": True,
+        "status": status,
+        "session_id": session_id,
+        "current_round": current_round,
+        "story_unit": manifest.get("story_unit"),
+        "personas": personas,
+        "reviews": {
+            "expected": personas,
+            "found": [review["persona_id"] for review in reviews],
+            "paths": {
+                review["persona_id"]: str(
+                    paths["reviews"] / f"{review['persona_id']}.review.yaml"
+                )
+                for review in reviews
+            },
+        },
+        "report": report_summary,
+        "feedback": {
+            "feedback_found": feedback["feedback_found"],
+            "selected_item_count": len(feedback["selected_items"]),
+        },
+        "revision_prompt": prompt_summary,
+        "telemetry": telemetry_summary,
+        "dialogue": {
+            "personas_with_dialogue": [
+                path.stem for path in dialogue_files
+            ],
+            "file_count": len(dialogue_files),
+            "index_found": dialogue_index_found,
+        },
+        "round": {
+            "current": current_round,
+            "refinement_count": refinement_count,
+        },
+        "sync_scaffold": sync_summary,
+        "warnings": warnings,
+    }
+
+
+def verify_report(
+    paths: dict[str, Path], session_id: str, personas: list[str], issues: list[str]
+) -> dict[str, Any]:
+    payload_path = paths["report"] / "payload.json"
+    html_path = paths["report"] / "index.html"
+    if not payload_path.exists():
+        issues.append("missing report/payload.json")
+        return {"payload_found": False, "html_found": html_path.exists()}
+
+    payload = json.loads(payload_path.read_text(encoding="utf-8"))
+    template_name = payload.get("template_name") or "report"
+    output_name = "index.html" if template_name == "report" else f"{template_name}.html"
+    html_path = paths["report"] / output_name
+    if not html_path.exists():
+        issues.append(f"missing report/{output_name}")
+
+    if payload.get("session_id") != session_id:
+        issues.append("report payload session_id does not match session")
+    payload_personas = [
+        review.get("persona_id") for review in payload.get("reviews", [])
+    ]
+    missing_payload_reviews = [
+        persona_id for persona_id in personas if persona_id not in payload_personas
+    ]
+    if missing_payload_reviews:
+        issues.append(
+            f"report payload missing reviews for: {', '.join(missing_payload_reviews)}"
+        )
+    if html_path.exists() and "__GHOST_READER_PAYLOAD__" in html_path.read_text(
+        encoding="utf-8"
+    ):
+        issues.append("report HTML still contains the payload placeholder")
+
+    return {
+        "payload_found": True,
+        "html_found": html_path.exists(),
+        "template_name": template_name,
+        "payload_reviews": payload_personas,
+        "html": str(html_path),
+        "payload": str(payload_path),
+    }
+
+
+def verify_telemetry(
+    session_path: Path, session_id: str, personas: list[str], issues: list[str]
+) -> dict[str, Any]:
+    events = read_events(session_path / "telemetry" / "events.ndjson")
+    event_names = [event.get("event") for event in events]
+    if "session_created" not in event_names:
+        issues.append("telemetry missing session_created")
+    if "html_rendered" not in event_names:
+        issues.append("telemetry missing html_rendered")
+    completed_personas = [
+        event.get("meta", {}).get("persona_id")
+        for event in events
+        if event.get("event") == "review_completed"
+    ]
+    missing_review_events = [
+        persona_id for persona_id in personas if persona_id not in completed_personas
+    ]
+    if missing_review_events:
+        issues.append(
+            f"telemetry missing review_completed for: {', '.join(missing_review_events)}"
+        )
+    if any(event.get("session_id") != session_id for event in events):
+        issues.append("telemetry contains event with mismatched session_id")
+
+    return {
+        "event_count": len(events),
+        "events": event_names,
+        "review_completed_personas": completed_personas,
+    }
+
+
+def verify_prompt(
+    paths: dict[str, Path], session_id: str, feedback_found: bool, issues: list[str]
+) -> dict[str, Any]:
+    yaml_path = paths["prompts"] / "revision-prompt.yaml"
+    markdown_path = paths["prompts"] / "revision-prompt.md"
+    if not feedback_found:
+        return {
+            "revision_prompt_found": False,
+            "required_now": False,
+        }
+
+    if not yaml_path.exists():
+        issues.append("feedback exists but prompts/revision-prompt.yaml is missing")
+        return {
+            "revision_prompt_found": False,
+            "required_now": True,
+        }
+    if not markdown_path.exists():
+        issues.append("feedback exists but prompts/revision-prompt.md is missing")
+
+    prompt = load_yaml_file(yaml_path)
+    if prompt.get("schema_version") != SCHEMA_VERSIONS["revision_prompt"]:
+        issues.append(
+            f"revision prompt schema_version must be {SCHEMA_VERSIONS['revision_prompt']}"
+        )
+    if prompt.get("session_id") != session_id:
+        issues.append("revision prompt session_id does not match session")
+
+    return {
+        "revision_prompt_found": True,
+        "required_now": True,
+        "yaml": str(yaml_path),
+        "markdown": str(markdown_path),
+    }
+
+
+def verify_no_mixed_artifacts(session_path: Path, issues: list[str]) -> None:
+    flat_dirs = ["reviews", "feedback", "prompts", "report", "context"]
+    has_round = (session_path / "round-1").exists()
+    has_flat = any((session_path / d).exists() for d in flat_dirs)
+    if has_round and has_flat:
+        issues.append(
+            "session mixes round-1/ with legacy flat artifact directories; "
+            "remove flat directories or start a new session"
+        )
+
+
+def verify_sync_scaffold(home: Path, issues: list[str]) -> dict[str, Any]:
+    telemetry_root = home / "telemetry"
+    required_dirs = ["outbox", "synced", "receipts"]
+    missing = [
+        str(telemetry_root / directory)
+        for directory in required_dirs
+        if not (telemetry_root / directory).is_dir()
+    ]
+    if missing:
+        issues.append("missing telemetry sync scaffold: " + ", ".join(missing))
+    return {
+        "outbox_found": (telemetry_root / "outbox").is_dir(),
+        "synced_found": (telemetry_root / "synced").is_dir(),
+        "receipts_found": (telemetry_root / "receipts").is_dir(),
+    }