ghost-api-cli 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. ghost_api/__init__.py +3 -0
  2. ghost_api/__main__.py +5 -0
  3. ghost_api/adopt.py +173 -0
  4. ghost_api/agent/__init__.py +0 -0
  5. ghost_api/agent/orchestrator.py +413 -0
  6. ghost_api/agent/prompts.py +149 -0
  7. ghost_api/agent/tools.py +84 -0
  8. ghost_api/analysis/__init__.py +0 -0
  9. ghost_api/analysis/analyzer.py +130 -0
  10. ghost_api/analysis/detectors.py +188 -0
  11. ghost_api/analysis/exposure.py +80 -0
  12. ghost_api/analysis/graphql.py +67 -0
  13. ghost_api/analysis/models.py +64 -0
  14. ghost_api/analysis/schema.py +109 -0
  15. ghost_api/analysis/url_grouping.py +217 -0
  16. ghost_api/api.py +264 -0
  17. ghost_api/audit.py +78 -0
  18. ghost_api/browser/__init__.py +0 -0
  19. ghost_api/browser/auth_state.py +172 -0
  20. ghost_api/browser/driver.py +273 -0
  21. ghost_api/browser/egress.py +183 -0
  22. ghost_api/browser/gate.py +127 -0
  23. ghost_api/browser/install.py +28 -0
  24. ghost_api/browser/snapshot.py +221 -0
  25. ghost_api/capture/__init__.py +0 -0
  26. ghost_api/capture/filters.py +255 -0
  27. ghost_api/capture/har.py +148 -0
  28. ghost_api/capture/models.py +41 -0
  29. ghost_api/capture/personal.py +188 -0
  30. ghost_api/capture/recorder.py +181 -0
  31. ghost_api/cli/__init__.py +44 -0
  32. ghost_api/cli/_common.py +219 -0
  33. ghost_api/cli/client.py +487 -0
  34. ghost_api/cli/data.py +492 -0
  35. ghost_api/cli/explore.py +379 -0
  36. ghost_api/cli/manage.py +437 -0
  37. ghost_api/config.py +132 -0
  38. ghost_api/drift.py +344 -0
  39. ghost_api/drive.py +211 -0
  40. ghost_api/errors.py +106 -0
  41. ghost_api/events.py +213 -0
  42. ghost_api/exitcodes.py +21 -0
  43. ghost_api/export/__init__.py +71 -0
  44. ghost_api/export/confidence.py +45 -0
  45. ghost_api/export/html.py +41 -0
  46. ghost_api/export/openapi.py +201 -0
  47. ghost_api/export/report.py +227 -0
  48. ghost_api/export/site_data.py +196 -0
  49. ghost_api/export/tabular.py +88 -0
  50. ghost_api/export/viewer/report.html +286 -0
  51. ghost_api/generator/__init__.py +221 -0
  52. ghost_api/generator/cassettes.py +80 -0
  53. ghost_api/generator/models_gen.py +144 -0
  54. ghost_api/generator/naming.py +174 -0
  55. ghost_api/generator/package.py +197 -0
  56. ghost_api/generator/python_gen.py +172 -0
  57. ghost_api/generator/spec.py +486 -0
  58. ghost_api/generator/templates/client.py.j2 +518 -0
  59. ghost_api/generator/templates/client.test.ts.j2 +106 -0
  60. ghost_api/generator/templates/client.ts.j2 +339 -0
  61. ghost_api/generator/templates/models.py.j2 +45 -0
  62. ghost_api/generator/templates/package_readme.md.j2 +111 -0
  63. ghost_api/generator/templates/pyproject.toml.j2 +22 -0
  64. ghost_api/generator/templates/test_client.py.j2 +105 -0
  65. ghost_api/generator/ts_gen.py +232 -0
  66. ghost_api/integrations/__init__.py +0 -0
  67. ghost_api/integrations/claude_code/commands/ghost-data.md +10 -0
  68. ghost_api/integrations/claude_code/commands/ghost-diff.md +10 -0
  69. ghost_api/integrations/claude_code/commands/ghost-explore.md +10 -0
  70. ghost_api/integrations/claude_code/skills/ghost-api/SKILL.md +46 -0
  71. ghost_api/integrations/setup.py +84 -0
  72. ghost_api/llm/__init__.py +0 -0
  73. ghost_api/llm/doctor.py +60 -0
  74. ghost_api/llm/gateway.py +476 -0
  75. ghost_api/llm/providers.py +124 -0
  76. ghost_api/memory/__init__.py +0 -0
  77. ghost_api/memory/knowledge.py +157 -0
  78. ghost_api/memory/store.py +883 -0
  79. ghost_api/net.py +235 -0
  80. ghost_api/netguard.py +203 -0
  81. ghost_api/pipeline.py +745 -0
  82. ghost_api/probe/__init__.py +0 -0
  83. ghost_api/probe/prober.py +311 -0
  84. ghost_api/probe/replay.py +313 -0
  85. ghost_api/project.py +186 -0
  86. ghost_api/py.typed +0 -0
  87. ghost_api/runtime.py +397 -0
  88. ghost_api/serve/__init__.py +101 -0
  89. ghost_api/serve/explorer_mcp.py +305 -0
  90. ghost_api/serve/mcp.py +121 -0
  91. ghost_api/serve/mock.py +130 -0
  92. ghost_api/serve/rest.py +132 -0
  93. ghost_api/site.py +84 -0
  94. ghost_api/specdiff.py +122 -0
  95. ghost_api/ui/__init__.py +0 -0
  96. ghost_api/ui/keys.py +134 -0
  97. ghost_api/ui/repl.py +453 -0
  98. ghost_api/ui/terminal.py +573 -0
  99. ghost_api/verify/__init__.py +0 -0
  100. ghost_api/verify/classify.py +84 -0
  101. ghost_api/verify/repair.py +40 -0
  102. ghost_api/verify/runner.py +110 -0
  103. ghost_api/verify/verifier.py +291 -0
  104. ghost_api/watch.py +313 -0
  105. ghost_api/web/__init__.py +1 -0
  106. ghost_api/web/server.py +690 -0
  107. ghost_api/web/static/assets/Endpoints-zPnjlcM8.js +46 -0
  108. ghost_api/web/static/assets/Run-D85zWHey.js +91 -0
  109. ghost_api/web/static/assets/Settings-D0aWHFAt.js +11 -0
  110. ghost_api/web/static/assets/Site-a88shEmP.js +71 -0
  111. ghost_api/web/static/assets/Watches-CqZWIa9G.js +16 -0
  112. ghost_api/web/static/assets/circle-x-BcTE_1tr.js +11 -0
  113. ghost_api/web/static/assets/index-Ds2e5B9f.js +141 -0
  114. ghost_api/web/static/assets/index-xQpyIBbG.css +1 -0
  115. ghost_api/web/static/assets/minus-C2xqP1eQ.js +6 -0
  116. ghost_api/web/static/assets/refresh-cw-CxdR_8Oi.js +6 -0
  117. ghost_api/web/static/favicon.svg +1 -0
  118. ghost_api/web/static/index.html +26 -0
  119. ghost_api/workflows.py +203 -0
  120. ghost_api_cli-0.2.0.dist-info/METADATA +1019 -0
  121. ghost_api_cli-0.2.0.dist-info/RECORD +125 -0
  122. ghost_api_cli-0.2.0.dist-info/WHEEL +5 -0
  123. ghost_api_cli-0.2.0.dist-info/entry_points.txt +2 -0
  124. ghost_api_cli-0.2.0.dist-info/licenses/LICENSE +21 -0
  125. ghost_api_cli-0.2.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,690 @@
1
+ """The local web dashboard: `ghost-api ui`.
2
+
3
+ A FastAPI app on 127.0.0.1 that serves the React frontend (built into `static/`) and a
4
+ JSON API over the same database and output folder the CLI uses.
5
+
6
+ Security, since this runs on your machine:
7
+ - It listens on 127.0.0.1 only.
8
+ - Every /api request needs the random token made at launch. The token is embedded in
9
+ the page the server sends; other websites can't read that page, so they can't send
10
+ requests here even though your browser can reach the port.
11
+ - The Host header must be this server (blocks DNS-rebinding tricks), and requests that
12
+ change something must come from this origin.
13
+ - API keys and unredacted requests never leave the server.
14
+ """
15
+
16
+ from __future__ import annotations
17
+
18
+ import asyncio
19
+ import io
20
+ import json
21
+ import os
22
+ import re
23
+ import secrets
24
+ import threading
25
+ import time
26
+ import webbrowser
27
+ import zipfile
28
+ from collections.abc import AsyncIterator, Awaitable, Callable
29
+ from pathlib import Path
30
+ from typing import Any
31
+ from urllib.parse import urlsplit
32
+
33
+ from fastapi import FastAPI, HTTPException, Request, Response
34
+ from fastapi.responses import (
35
+ FileResponse,
36
+ HTMLResponse,
37
+ JSONResponse,
38
+ PlainTextResponse,
39
+ StreamingResponse,
40
+ )
41
+ from pydantic import BaseModel, Field
42
+ from rich.console import Console
43
+
44
+ from ghost_api import __version__, watch
45
+ from ghost_api.config import SUPPORTED_LANGUAGES, state_dir
46
+ from ghost_api.errors import friendly_error
47
+ from ghost_api.events import EventBus, Notice
48
+ from ghost_api.export import tabular
49
+ from ghost_api.export.site_data import SAFE_NAME, endpoint_details, run_dict, site_files
50
+ from ghost_api.generator import NothingToGenerate, generate_client
51
+ from ghost_api.generator.python_gen import GenerationError
52
+ from ghost_api.llm.doctor import check_providers
53
+ from ghost_api.llm.gateway import LLMGateway
54
+ from ghost_api.llm.providers import default_providers
55
+ from ghost_api.memory.knowledge import load_memory, memory_prompt
56
+ from ghost_api.memory.store import RunRow, Store
57
+ from ghost_api.net import RateLimiter
58
+ from ghost_api.pipeline import (
59
+ RunOptions,
60
+ client_event,
61
+ explore_site,
62
+ finish,
63
+ open_session,
64
+ robots_policy,
65
+ screenshots_dir,
66
+ )
67
+ from ghost_api.probe.replay import Replayer, ReplayRefused
68
+ from ghost_api.runtime import OperationError, SiteRuntime
69
+ from ghost_api.site import from_url_id, is_site, site_host, to_url_id
70
+
71
+ STATIC = Path(__file__).parent / "static"
72
+ TOKEN_PLACEHOLDER = "__GHOST_TOKEN__"
73
+ SCREEN_NAME = re.compile(r"^[A-Za-z0-9_-]+\.jpg$")
74
+ MAX_TRY_BODY = 200_000
75
+
76
+ GatewayFactory = Callable[[int], LLMGateway | None]
77
+
78
+
79
+ def default_gateway(max_calls: int) -> LLMGateway | None:
80
+ gateway = LLMGateway(
81
+ default_providers(), max_calls=max_calls, cooldown_file=state_dir() / "cooldowns.json"
82
+ )
83
+ return gateway if gateway.providers else None
84
+
85
+
86
+ class StartRun(BaseModel):
87
+ url: str = Field(min_length=3, max_length=2000)
88
+ goal: str = Field(min_length=1, max_length=500)
89
+ lang: str = "python"
90
+ agent: bool = True
91
+ max_steps: int = Field(20, ge=1, le=60)
92
+ max_llm_calls: int = Field(30, ge=1, le=100)
93
+ ignore_robots: bool = False
94
+
95
+
96
+ class TryCall(BaseModel):
97
+ endpoint: str # Endpoint.key
98
+ params: dict[str, str] = Field(default_factory=dict)
99
+
100
+
101
+ class FetchCall(TryCall):
102
+ pages: int = Field(5, ge=1, le=50)
103
+ format: str = "csv"
104
+
105
+
106
+ class AllowCall(BaseModel):
107
+ endpoint: str # Endpoint.key
108
+ allowed: bool
109
+
110
+
111
+ MAX_FETCH_ROWS = 5000
112
+
113
+
114
+ class NewWatch(BaseModel):
115
+ site: str
116
+ endpoint: str # Endpoint.key or operation name
117
+ params: dict[str, str] = Field(default_factory=dict)
118
+ every: str = "6h"
119
+ notify: list[str] = Field(default_factory=list)
120
+ track: list[str] = Field(default_factory=list)
121
+ pages: int = Field(5, ge=1, le=50)
122
+
123
+
124
+ class Regenerate(BaseModel):
125
+ lang: str = "python"
126
+
127
+
128
+ # --- running one exploration at a time ------------------------------------------------------
129
+
130
+
131
+ class RunManager:
132
+ def __init__(self, db: Path, out: Path, gateway_factory: GatewayFactory) -> None:
133
+ self.db = db
134
+ self.out = out
135
+ self.gateway_factory = gateway_factory
136
+ self.active_run: int | None = None
137
+ self._stop = threading.Event()
138
+ self._lock = threading.Lock()
139
+ self._thread: threading.Thread | None = None
140
+
141
+ @property
142
+ def busy(self) -> bool:
143
+ return self._thread is not None and self._thread.is_alive()
144
+
145
+ def start(self, req: StartRun, timeout: float = 60) -> int:
146
+ with self._lock:
147
+ if self.busy:
148
+ raise HTTPException(409, "A run is already in progress. Stop it or wait.")
149
+ if req.lang not in SUPPORTED_LANGUAGES:
150
+ raise HTTPException(400, f"Unsupported language: {req.lang}")
151
+ ready = threading.Event()
152
+ result: dict[str, Any] = {}
153
+ self._stop.clear()
154
+ self._thread = threading.Thread(
155
+ target=self._run, args=(req, ready, result), daemon=True, name="ghost-run"
156
+ )
157
+ self._thread.start()
158
+ if not ready.wait(timeout):
159
+ raise HTTPException(504, "The run did not start in time.")
160
+ if "error" in result:
161
+ raise HTTPException(400, result["error"])
162
+ return int(result["run_id"])
163
+
164
+ def stop(self, run_id: int) -> bool:
165
+ if self.active_run != run_id:
166
+ return False
167
+ self._stop.set()
168
+ return True
169
+
170
+ def wait(self, timeout: float | None = None) -> None:
171
+ if self._thread is not None:
172
+ self._thread.join(timeout)
173
+
174
+ def _run(self, req: StartRun, ready: threading.Event, result: dict[str, Any]) -> None:
175
+ bus = EventBus()
176
+ notices: list[Notice] = []
177
+ bus.subscribe(lambda e: notices.append(e) if isinstance(e, Notice) else None)
178
+ try:
179
+ url = req.url if "://" in req.url else f"https://{req.url}"
180
+ host = (urlsplit(url).hostname or "").lower()
181
+ if not host or urlsplit(url).scheme not in ("http", "https"):
182
+ result["error"] = f"Not a valid http(s) URL: {req.url}"
183
+ return
184
+ limiter = RateLimiter(1.0)
185
+ robots, allowed = robots_policy(url, req.ignore_robots, limiter, bus)
186
+ if not allowed:
187
+ result["error"] = notices[-1].text if notices else "Disallowed by robots.txt"
188
+ return
189
+ gateway = self.gateway_factory(req.max_llm_calls) if req.agent else None
190
+ if req.agent and gateway is None:
191
+ result["error"] = (
192
+ "No LLM API key found. Add one to .env, or turn off the AI agent to only "
193
+ "load and scroll the page."
194
+ )
195
+ return
196
+ with Store(self.db) as store:
197
+ s = open_session(
198
+ store,
199
+ bus,
200
+ host,
201
+ url,
202
+ req.goal,
203
+ False,
204
+ limiter,
205
+ robots,
206
+ gateway,
207
+ 60,
208
+ screens_root=self.db,
209
+ lang=req.lang,
210
+ )
211
+ s.stop = self._stop.is_set
212
+ for notice in notices: # the robots.txt result, now that the run is recorded
213
+ store.add_event(s.run_id, notice.to_dict())
214
+ known = load_memory(store, host)
215
+ if known is not None and known.run_id != s.run_id:
216
+ s.memory = memory_prompt(known.endpoints, known.lessons)
217
+ bus.emit(
218
+ Notice(
219
+ f"Using memory: {len(known.endpoints)} known endpoints, "
220
+ f"{len(known.lessons)} lessons"
221
+ )
222
+ )
223
+ self.active_run = s.run_id
224
+ result["run_id"] = s.run_id
225
+ ready.set()
226
+ opts = RunOptions(
227
+ lang=req.lang, max_steps=req.max_steps, max_minutes=10, out=self.out
228
+ )
229
+ started = time.monotonic()
230
+ status, reason = explore_site(s, opts, use_agent=gateway is not None)
231
+ finish(s, status, reason, started, opts)
232
+ except Exception as exc: # noqa: BLE001 - report it to the page, keep serving
233
+ result.setdefault("error", friendly_error(exc))
234
+ finally:
235
+ self.active_run = None
236
+ ready.set()
237
+
238
+
239
+ # --- reading what is stored -----------------------------------------------------------------
240
+
241
+
242
+ def _site_name(url_id: str) -> str:
243
+ """The site from its URL form ('www.google.com~maps' -> 'www.google.com/maps')."""
244
+ site = from_url_id(url_id)
245
+ if not is_site(site):
246
+ raise HTTPException(400, "Invalid site name")
247
+ return site
248
+
249
+
250
+ def _site_folder(out: Path, site: str) -> Path:
251
+ if not is_site(site):
252
+ raise HTTPException(400, "Invalid site name")
253
+ return out / site
254
+
255
+
256
+ def _site_files(out: Path, domain: str) -> list[dict[str, Any]]:
257
+ return site_files(_site_folder(out, domain))
258
+
259
+
260
+ # --- the app --------------------------------------------------------------------------------
261
+
262
+
263
+ def create_app(
264
+ db: Path,
265
+ out: Path,
266
+ token: str,
267
+ manager: RunManager | None = None,
268
+ allowed_hosts: set[str] | None = None,
269
+ ) -> FastAPI:
270
+ manager = manager or RunManager(db, out, default_gateway)
271
+ app = FastAPI(title="Ghost API", docs_url=None, redoc_url=None, openapi_url=None)
272
+ app.state.manager = manager
273
+
274
+ @app.middleware("http")
275
+ async def guard(
276
+ request: Request, call_next: Callable[[Request], Awaitable[Response]]
277
+ ) -> Response:
278
+ host = request.headers.get("host", "")
279
+ if allowed_hosts is not None and host not in allowed_hosts:
280
+ return JSONResponse({"detail": "Unknown host"}, status_code=403)
281
+ if request.url.path.startswith("/api/"):
282
+ given = request.headers.get("x-ghost-token") or request.query_params.get("token", "")
283
+ if not secrets.compare_digest(given, token):
284
+ return JSONResponse({"detail": "Missing or wrong token"}, status_code=401)
285
+ origin = request.headers.get("origin")
286
+ if request.method != "GET" and origin and urlsplit(origin).netloc != host:
287
+ return JSONResponse({"detail": "Cross-origin request refused"}, status_code=403)
288
+ return await call_next(request)
289
+
290
+ def store() -> Store:
291
+ return Store(db)
292
+
293
+ @app.get("/api/state")
294
+ def state() -> dict[str, Any]:
295
+ return {
296
+ "version": __version__,
297
+ "running": manager.active_run,
298
+ "has_llm": any(p.api_key for p in default_providers()),
299
+ "languages": list(SUPPORTED_LANGUAGES),
300
+ "db": str(db),
301
+ "out": str(out),
302
+ }
303
+
304
+ @app.get("/api/sites")
305
+ def sites() -> list[dict[str, Any]]:
306
+ if not db.exists():
307
+ return []
308
+ with store() as s:
309
+ runs = s.list_runs()
310
+ by_domain: dict[str, list[RunRow]] = {}
311
+ for r in runs:
312
+ by_domain.setdefault(r.domain, []).append(r)
313
+ result = []
314
+ for domain, domain_runs in by_domain.items():
315
+ endpoints = endpoint_details(s, domain)
316
+ counts = {
317
+ k: sum(e["status"] == k for e in endpoints)
318
+ for k in ("verified", "unverified", "failed", "discovered")
319
+ }
320
+ result.append(
321
+ {
322
+ "domain": domain,
323
+ "latest": run_dict(domain_runs[0]),
324
+ "runs": len(domain_runs),
325
+ "endpoints": len(endpoints),
326
+ "counts": counts,
327
+ "methods": sorted({e["method"] for e in endpoints}),
328
+ "has_client": any(f["kind"] == "client" for f in _site_files(out, domain)),
329
+ }
330
+ )
331
+ return result
332
+
333
+ @app.get("/api/sites/{domain}")
334
+ def site(domain: str) -> dict[str, Any]:
335
+ domain = _site_name(domain)
336
+ _site_folder(out, domain)
337
+ with store() as s:
338
+ runs = s.list_runs(domain)
339
+ if not runs:
340
+ raise HTTPException(404, f"Nothing stored for {domain}")
341
+ return {
342
+ "domain": domain,
343
+ "runs": [run_dict(r) for r in runs],
344
+ "endpoints": endpoint_details(s, domain),
345
+ "lessons": s.lessons_for(domain, limit=20),
346
+ "files": _site_files(out, domain),
347
+ }
348
+
349
+ @app.get("/api/runs/{run_id}")
350
+ def run(run_id: int) -> dict[str, Any]:
351
+ with store() as s:
352
+ r = s.get_run(run_id)
353
+ if r is None:
354
+ raise HTTPException(404, "Run not found")
355
+ return {
356
+ **run_dict(r),
357
+ "active": manager.active_run == run_id,
358
+ "steps": s.steps_for_run(run_id),
359
+ "hypotheses": s.hypotheses_for_run(run_id),
360
+ }
361
+
362
+ @app.get("/api/runs/{run_id}/events")
363
+ async def events(run_id: int, after: int = 0) -> StreamingResponse:
364
+ async def stream() -> AsyncIterator[str]:
365
+ last = after
366
+ idle = 0
367
+ while True:
368
+ with store() as s:
369
+ batch = s.events_for_run(run_id, after=last)
370
+ finished = False
371
+ for event in batch:
372
+ last = int(event["seq"])
373
+ finished = finished or event["type"] == "run_finished"
374
+ yield f"id: {last}\ndata: {json.dumps(event)}\n\n"
375
+ idle = 0 if batch else idle + 1
376
+ if finished or (manager.active_run != run_id and not batch and idle > 2):
377
+ yield "event: end\ndata: {}\n\n"
378
+ return
379
+ await asyncio.sleep(0.4)
380
+
381
+ headers = {"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}
382
+ return StreamingResponse(stream(), media_type="text/event-stream", headers=headers)
383
+
384
+ @app.post("/api/runs")
385
+ async def start_run(req: StartRun) -> dict[str, int]:
386
+ run_id = await asyncio.to_thread(manager.start, req)
387
+ return {"run_id": run_id}
388
+
389
+ @app.post("/api/runs/{run_id}/stop")
390
+ def stop_run(run_id: int) -> dict[str, bool]:
391
+ return {"stopping": manager.stop(run_id)}
392
+
393
+ @app.get("/api/runs/{run_id}/screens/{name}")
394
+ def screen(run_id: int, name: str) -> FileResponse:
395
+ if not SCREEN_NAME.match(name):
396
+ raise HTTPException(400, "Invalid screenshot name")
397
+ path = screenshots_dir(db, run_id) / name
398
+ if not path.is_file():
399
+ raise HTTPException(404, "No such screenshot")
400
+ return FileResponse(path, media_type="image/jpeg")
401
+
402
+ @app.get("/api/sites/{domain}/files")
403
+ def files(domain: str) -> list[dict[str, Any]]:
404
+ domain = _site_name(domain)
405
+ return _site_files(out, domain)
406
+
407
+ @app.get("/api/sites/{domain}/files/{name}")
408
+ def file(domain: str, name: str) -> PlainTextResponse:
409
+ domain = _site_name(domain)
410
+ folder = _site_folder(out, domain)
411
+ if not SAFE_NAME.match(name) or name.startswith("."):
412
+ raise HTTPException(400, "Invalid file name")
413
+ path = (folder / name).resolve()
414
+ if path.parent != folder.resolve() or not path.is_file():
415
+ raise HTTPException(404, "No such file")
416
+ return PlainTextResponse(path.read_text(encoding="utf-8", errors="replace"))
417
+
418
+ @app.get("/api/sites/{domain}/download.zip")
419
+ def download(domain: str) -> Response:
420
+ domain = _site_name(domain)
421
+ folder = _site_folder(out, domain)
422
+ listing = _site_files(out, domain)
423
+ if not listing:
424
+ raise HTTPException(404, "No files for this site yet")
425
+ buffer = io.BytesIO()
426
+ with zipfile.ZipFile(buffer, "w", zipfile.ZIP_DEFLATED) as zf:
427
+ for f in listing:
428
+ zf.write(folder / f["name"], f"{domain}/{f['name']}")
429
+ filename = to_url_id(domain).replace("~", "-")
430
+ headers = {"Content-Disposition": f'attachment; filename="{filename}.zip"'}
431
+ return Response(buffer.getvalue(), media_type="application/zip", headers=headers)
432
+
433
+ @app.post("/api/sites/{domain}/generate")
434
+ def regenerate(domain: str, body: Regenerate) -> dict[str, Any]:
435
+ domain = _site_name(domain)
436
+ _site_folder(out, domain)
437
+ if body.lang not in SUPPORTED_LANGUAGES:
438
+ raise HTTPException(400, f"Unsupported language: {body.lang}")
439
+ try:
440
+ with store() as s:
441
+ result = generate_client(s, domain, out, llm=None, lang=body.lang)
442
+ except (NothingToGenerate, GenerationError) as exc:
443
+ raise HTTPException(400, str(exc)) from exc
444
+ return client_event(result).to_dict()
445
+
446
+ @app.post("/api/sites/{domain}/try")
447
+ def try_endpoint(domain: str, call: TryCall) -> dict[str, Any]:
448
+ """Call an endpoint once, read-only, with the given parameters."""
449
+ domain = _site_name(domain)
450
+ _site_folder(out, domain)
451
+ replayer = Replayer(site_host(domain), RateLimiter(1.0), max_requests=1)
452
+ try:
453
+ with store() as s:
454
+ runtime = SiteRuntime(s, domain, replayer=replayer)
455
+ if not any(e.key == call.endpoint for e in runtime.endpoints):
456
+ raise HTTPException(404, "Unknown endpoint")
457
+ op = next((o for o in runtime.operations() if o.endpoint.key == call.endpoint), None)
458
+ if op is None:
459
+ raise HTTPException(
460
+ 409,
461
+ "This endpoint changes data or can't be called outside the browser, so "
462
+ "Ghost API never calls it. If you checked it only reads, allow it first.",
463
+ )
464
+ started = time.monotonic()
465
+ result = runtime.call(op, call.params)
466
+ except OperationError as exc:
467
+ raise HTTPException(409, str(exc)) from exc
468
+ except ReplayRefused as exc:
469
+ raise HTTPException(400, str(exc)) from exc
470
+ finally:
471
+ replayer.close()
472
+ text = json.dumps(result.body, indent=2) if result.body is not None else ""
473
+ return {
474
+ "status": result.status,
475
+ "ok": result.ok,
476
+ "error": result.error,
477
+ "url": result.captured.url if result.captured else "",
478
+ "ms": round((time.monotonic() - started) * 1000),
479
+ "body": text[:MAX_TRY_BODY],
480
+ "truncated": len(text) > MAX_TRY_BODY,
481
+ }
482
+
483
+ @app.post("/api/sites/{domain}/fetch")
484
+ def fetch_data(domain: str, call: FetchCall) -> Response:
485
+ """All pages of an endpoint's data as a CSV / Excel / JSON download."""
486
+ domain = _site_name(domain)
487
+ if call.format not in tabular.FORMATS:
488
+ raise HTTPException(400, f"Unsupported format: {call.format}")
489
+ try:
490
+ with store() as s:
491
+ runtime = SiteRuntime(s, domain)
492
+ with runtime:
493
+ op = next(
494
+ (o for o in runtime.operations() if o.endpoint.key == call.endpoint), None
495
+ )
496
+ if op is None:
497
+ raise HTTPException(409, "Ghost API doesn't call this endpoint by itself.")
498
+ records: list[Any] = []
499
+ for page in runtime.iterate(op, call.params, call.pages, MAX_FETCH_ROWS):
500
+ if not page.result.ok:
501
+ if not records:
502
+ detail = page.result.error or f"HTTP {page.result.status}"
503
+ raise HTTPException(502, f"The site answered: {detail}")
504
+ break
505
+ records += page.records
506
+ except OperationError as exc:
507
+ raise HTTPException(409, str(exc)) from exc
508
+ except ReplayRefused as exc:
509
+ raise HTTPException(400, str(exc)) from exc
510
+ if not records:
511
+ raise HTTPException(404, "No rows came back. Try other inputs.")
512
+ name = f"{to_url_id(domain).replace('~', '-')}-{op.name}.{call.format}"
513
+ return Response(
514
+ tabular.to_bytes(records, call.format),
515
+ media_type=tabular.MEDIA_TYPES[call.format],
516
+ headers={
517
+ "Content-Disposition": f'attachment; filename="{name}"',
518
+ "X-Ghost-Rows": str(len(records)),
519
+ },
520
+ )
521
+
522
+ @app.post("/api/sites/{domain}/allow")
523
+ def allow_endpoint(domain: str, call: AllowCall) -> dict[str, bool]:
524
+ """The user checked this endpoint only reads (e.g. a POST search): allow calling it."""
525
+ domain = _site_name(domain)
526
+ with store() as s:
527
+ keys = {e.key for e in s.current_endpoints(domain)}
528
+ if call.endpoint not in keys:
529
+ raise HTTPException(404, "Unknown endpoint")
530
+ if call.allowed:
531
+ s.allow_endpoints(domain, [call.endpoint])
532
+ else:
533
+ s.disallow_endpoint(domain, call.endpoint)
534
+ return {"allowed": call.allowed}
535
+
536
+ # --- watches ---
537
+
538
+ def _watch_json(w: dict[str, Any]) -> dict[str, Any]:
539
+ return {
540
+ **w,
541
+ "every": watch.every_label(w["every_minutes"]),
542
+ "due": watch.is_due(w),
543
+ "describe": watch.describe(w),
544
+ "site_id": to_url_id(w["site"]),
545
+ }
546
+
547
+ @app.get("/api/watches")
548
+ def list_watches() -> dict[str, Any]:
549
+ if not db.exists():
550
+ return {"watches": [], "changes": []}
551
+ with store() as s:
552
+ return {
553
+ "watches": [_watch_json(w) for w in s.watches()],
554
+ "changes": s.watch_changes(limit=100),
555
+ }
556
+
557
+ @app.post("/api/watches")
558
+ async def add_watch(req: NewWatch) -> dict[str, Any]:
559
+ site = _site_name(to_url_id(req.site))
560
+ try:
561
+ minutes = watch.parse_every(req.every)
562
+ except ValueError as exc:
563
+ raise HTTPException(400, str(exc)) from exc
564
+ bad = [n for n in req.notify if not n.startswith(("ntfy:", "webhook:", "desktop"))]
565
+ if bad:
566
+ raise HTTPException(400, f"Unknown notification target: {bad[0]}")
567
+
568
+ def create() -> dict[str, Any]:
569
+ with store() as s:
570
+ with SiteRuntime(s, site) as rt:
571
+ op = rt.operation(req.endpoint)
572
+ watch_id = s.add_watch(
573
+ {
574
+ "site": site,
575
+ "operation": op.name,
576
+ "params": req.params,
577
+ "every_minutes": minutes,
578
+ "notify": req.notify,
579
+ "track": req.track,
580
+ "max_pages": req.pages,
581
+ } # fmt: skip
582
+ )
583
+ w = s.get_watch(watch_id)
584
+ assert w is not None
585
+ return {"watch": _watch_json(w), "check": watch.to_json(watch.check(s, w))}
586
+
587
+ try:
588
+ return await asyncio.to_thread(create)
589
+ except OperationError as exc:
590
+ raise HTTPException(400, str(exc)) from exc
591
+
592
+ @app.post("/api/watches/{watch_id}/check")
593
+ async def check_watch(watch_id: int) -> dict[str, Any]:
594
+ def run() -> dict[str, Any]:
595
+ with store() as s:
596
+ w = s.get_watch(watch_id)
597
+ if w is None:
598
+ raise HTTPException(404, "No such watch")
599
+ return watch.to_json(watch.check(s, w))
600
+
601
+ return await asyncio.to_thread(run)
602
+
603
+ @app.delete("/api/watches/{watch_id}")
604
+ def remove_watch(watch_id: int) -> dict[str, bool]:
605
+ with store() as s:
606
+ if not s.remove_watch(watch_id):
607
+ raise HTTPException(404, "No such watch")
608
+ return {"removed": True}
609
+
610
+ @app.post("/api/doctor")
611
+ async def doctor() -> list[dict[str, Any]]:
612
+ return await asyncio.to_thread(check_providers)
613
+
614
+ # --- the frontend ---
615
+
616
+ def index() -> HTMLResponse:
617
+ page = STATIC / "index.html"
618
+ if not page.is_file():
619
+ return HTMLResponse(
620
+ "<h1>Ghost API</h1><p>The dashboard frontend is not built. Run "
621
+ "<code>npm run build</code> in the <code>web/</code> folder.</p>",
622
+ status_code=503,
623
+ )
624
+ html = page.read_text(encoding="utf-8").replace(TOKEN_PLACEHOLDER, token)
625
+ return HTMLResponse(html, headers={"Cache-Control": "no-store"})
626
+
627
+ @app.get("/assets/{name}")
628
+ def asset(name: str) -> FileResponse:
629
+ if not SAFE_NAME.match(name):
630
+ raise HTTPException(404)
631
+ path = STATIC / "assets" / name
632
+ if not path.is_file():
633
+ raise HTTPException(404)
634
+ return FileResponse(path)
635
+
636
+ @app.get("/{path:path}")
637
+ def spa(path: str) -> Response:
638
+ if path.startswith("api/"):
639
+ raise HTTPException(404)
640
+ static_file = STATIC / path
641
+ if path and SAFE_NAME.match(path) and static_file.is_file():
642
+ return FileResponse(static_file) # favicon, logo...
643
+ return index() # client-side routes
644
+
645
+ return app
646
+
647
+
648
+ def launch_token() -> str:
649
+ """A new random token per launch, or GHOST_API_UI_TOKEN for frontend development
650
+ (`npm run dev` sends the same value as VITE_GHOST_TOKEN)."""
651
+ return os.getenv("GHOST_API_UI_TOKEN") or secrets.token_urlsafe(24)
652
+
653
+
654
+ def _watch_loop(db: Path, console: Console) -> None:
655
+ """While the dashboard is open, check due watches every minute."""
656
+ while True:
657
+ try:
658
+ if db.exists():
659
+ with Store(db) as s:
660
+ for result in watch.run_due(s):
661
+ console.print(f"[dim]watch #{result.watch['id']}: {result.headline()}[/]")
662
+ except Exception as exc: # noqa: BLE001 - keep the dashboard running
663
+ console.print(f"[dim]watch check failed: {exc}[/]")
664
+ time.sleep(60)
665
+
666
+
667
+ def serve(
668
+ db: Path,
669
+ out: Path,
670
+ port: int,
671
+ open_browser: bool,
672
+ console: Console | None = None,
673
+ run_watches: bool = True,
674
+ ) -> None:
675
+ import uvicorn
676
+
677
+ console = console or Console()
678
+ token = launch_token()
679
+ hosts = {f"127.0.0.1:{port}", f"localhost:{port}"}
680
+ app = create_app(db, out, token, allowed_hosts=hosts)
681
+ if run_watches:
682
+ threading.Thread(target=_watch_loop, args=(db, console), daemon=True).start()
683
+ url = f"http://127.0.0.1:{port}/"
684
+ from ghost_api.ui.terminal import ACCENT
685
+
686
+ console.print(f"[bold {ACCENT}]◆ Ghost API dashboard[/] {url}")
687
+ console.print("[dim]Local only. Press Ctrl+C to stop.[/]")
688
+ if open_browser:
689
+ threading.Timer(1.0, lambda: webbrowser.open(url)).start()
690
+ uvicorn.run(app, host="127.0.0.1", port=port, log_level="warning")