ghost-api-cli 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ghost_api/__init__.py +3 -0
- ghost_api/__main__.py +5 -0
- ghost_api/adopt.py +173 -0
- ghost_api/agent/__init__.py +0 -0
- ghost_api/agent/orchestrator.py +413 -0
- ghost_api/agent/prompts.py +149 -0
- ghost_api/agent/tools.py +84 -0
- ghost_api/analysis/__init__.py +0 -0
- ghost_api/analysis/analyzer.py +130 -0
- ghost_api/analysis/detectors.py +188 -0
- ghost_api/analysis/exposure.py +80 -0
- ghost_api/analysis/graphql.py +67 -0
- ghost_api/analysis/models.py +64 -0
- ghost_api/analysis/schema.py +109 -0
- ghost_api/analysis/url_grouping.py +217 -0
- ghost_api/api.py +264 -0
- ghost_api/audit.py +78 -0
- ghost_api/browser/__init__.py +0 -0
- ghost_api/browser/auth_state.py +172 -0
- ghost_api/browser/driver.py +273 -0
- ghost_api/browser/egress.py +183 -0
- ghost_api/browser/gate.py +127 -0
- ghost_api/browser/install.py +28 -0
- ghost_api/browser/snapshot.py +221 -0
- ghost_api/capture/__init__.py +0 -0
- ghost_api/capture/filters.py +255 -0
- ghost_api/capture/har.py +148 -0
- ghost_api/capture/models.py +41 -0
- ghost_api/capture/personal.py +188 -0
- ghost_api/capture/recorder.py +181 -0
- ghost_api/cli/__init__.py +44 -0
- ghost_api/cli/_common.py +219 -0
- ghost_api/cli/client.py +487 -0
- ghost_api/cli/data.py +492 -0
- ghost_api/cli/explore.py +379 -0
- ghost_api/cli/manage.py +437 -0
- ghost_api/config.py +132 -0
- ghost_api/drift.py +344 -0
- ghost_api/drive.py +211 -0
- ghost_api/errors.py +106 -0
- ghost_api/events.py +213 -0
- ghost_api/exitcodes.py +21 -0
- ghost_api/export/__init__.py +71 -0
- ghost_api/export/confidence.py +45 -0
- ghost_api/export/html.py +41 -0
- ghost_api/export/openapi.py +201 -0
- ghost_api/export/report.py +227 -0
- ghost_api/export/site_data.py +196 -0
- ghost_api/export/tabular.py +88 -0
- ghost_api/export/viewer/report.html +286 -0
- ghost_api/generator/__init__.py +221 -0
- ghost_api/generator/cassettes.py +80 -0
- ghost_api/generator/models_gen.py +144 -0
- ghost_api/generator/naming.py +174 -0
- ghost_api/generator/package.py +197 -0
- ghost_api/generator/python_gen.py +172 -0
- ghost_api/generator/spec.py +486 -0
- ghost_api/generator/templates/client.py.j2 +518 -0
- ghost_api/generator/templates/client.test.ts.j2 +106 -0
- ghost_api/generator/templates/client.ts.j2 +339 -0
- ghost_api/generator/templates/models.py.j2 +45 -0
- ghost_api/generator/templates/package_readme.md.j2 +111 -0
- ghost_api/generator/templates/pyproject.toml.j2 +22 -0
- ghost_api/generator/templates/test_client.py.j2 +105 -0
- ghost_api/generator/ts_gen.py +232 -0
- ghost_api/integrations/__init__.py +0 -0
- ghost_api/integrations/claude_code/commands/ghost-data.md +10 -0
- ghost_api/integrations/claude_code/commands/ghost-diff.md +10 -0
- ghost_api/integrations/claude_code/commands/ghost-explore.md +10 -0
- ghost_api/integrations/claude_code/skills/ghost-api/SKILL.md +46 -0
- ghost_api/integrations/setup.py +84 -0
- ghost_api/llm/__init__.py +0 -0
- ghost_api/llm/doctor.py +60 -0
- ghost_api/llm/gateway.py +476 -0
- ghost_api/llm/providers.py +124 -0
- ghost_api/memory/__init__.py +0 -0
- ghost_api/memory/knowledge.py +157 -0
- ghost_api/memory/store.py +883 -0
- ghost_api/net.py +235 -0
- ghost_api/netguard.py +203 -0
- ghost_api/pipeline.py +745 -0
- ghost_api/probe/__init__.py +0 -0
- ghost_api/probe/prober.py +311 -0
- ghost_api/probe/replay.py +313 -0
- ghost_api/project.py +186 -0
- ghost_api/py.typed +0 -0
- ghost_api/runtime.py +397 -0
- ghost_api/serve/__init__.py +101 -0
- ghost_api/serve/explorer_mcp.py +305 -0
- ghost_api/serve/mcp.py +121 -0
- ghost_api/serve/mock.py +130 -0
- ghost_api/serve/rest.py +132 -0
- ghost_api/site.py +84 -0
- ghost_api/specdiff.py +122 -0
- ghost_api/ui/__init__.py +0 -0
- ghost_api/ui/keys.py +134 -0
- ghost_api/ui/repl.py +453 -0
- ghost_api/ui/terminal.py +573 -0
- ghost_api/verify/__init__.py +0 -0
- ghost_api/verify/classify.py +84 -0
- ghost_api/verify/repair.py +40 -0
- ghost_api/verify/runner.py +110 -0
- ghost_api/verify/verifier.py +291 -0
- ghost_api/watch.py +313 -0
- ghost_api/web/__init__.py +1 -0
- ghost_api/web/server.py +690 -0
- ghost_api/web/static/assets/Endpoints-zPnjlcM8.js +46 -0
- ghost_api/web/static/assets/Run-D85zWHey.js +91 -0
- ghost_api/web/static/assets/Settings-D0aWHFAt.js +11 -0
- ghost_api/web/static/assets/Site-a88shEmP.js +71 -0
- ghost_api/web/static/assets/Watches-CqZWIa9G.js +16 -0
- ghost_api/web/static/assets/circle-x-BcTE_1tr.js +11 -0
- ghost_api/web/static/assets/index-Ds2e5B9f.js +141 -0
- ghost_api/web/static/assets/index-xQpyIBbG.css +1 -0
- ghost_api/web/static/assets/minus-C2xqP1eQ.js +6 -0
- ghost_api/web/static/assets/refresh-cw-CxdR_8Oi.js +6 -0
- ghost_api/web/static/favicon.svg +1 -0
- ghost_api/web/static/index.html +26 -0
- ghost_api/workflows.py +203 -0
- ghost_api_cli-0.2.0.dist-info/METADATA +1019 -0
- ghost_api_cli-0.2.0.dist-info/RECORD +125 -0
- ghost_api_cli-0.2.0.dist-info/WHEEL +5 -0
- ghost_api_cli-0.2.0.dist-info/entry_points.txt +2 -0
- ghost_api_cli-0.2.0.dist-info/licenses/LICENSE +21 -0
- ghost_api_cli-0.2.0.dist-info/top_level.txt +1 -0
ghost_api/web/server.py
ADDED
|
@@ -0,0 +1,690 @@
|
|
|
1
|
+
"""The local web dashboard: `ghost-api ui`.
|
|
2
|
+
|
|
3
|
+
A FastAPI app on 127.0.0.1 that serves the React frontend (built into `static/`) and a
|
|
4
|
+
JSON API over the same database and output folder the CLI uses.
|
|
5
|
+
|
|
6
|
+
Security, since this runs on your machine:
|
|
7
|
+
- It listens on 127.0.0.1 only.
|
|
8
|
+
- Every /api request needs the random token made at launch. The token is embedded in
|
|
9
|
+
the page the server sends; other websites can't read that page, so they can't send
|
|
10
|
+
requests here even though your browser can reach the port.
|
|
11
|
+
- The Host header must be this server (blocks DNS-rebinding tricks), and requests that
|
|
12
|
+
change something must come from this origin.
|
|
13
|
+
- API keys and unredacted requests never leave the server.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import asyncio
|
|
19
|
+
import io
|
|
20
|
+
import json
|
|
21
|
+
import os
|
|
22
|
+
import re
|
|
23
|
+
import secrets
|
|
24
|
+
import threading
|
|
25
|
+
import time
|
|
26
|
+
import webbrowser
|
|
27
|
+
import zipfile
|
|
28
|
+
from collections.abc import AsyncIterator, Awaitable, Callable
|
|
29
|
+
from pathlib import Path
|
|
30
|
+
from typing import Any
|
|
31
|
+
from urllib.parse import urlsplit
|
|
32
|
+
|
|
33
|
+
from fastapi import FastAPI, HTTPException, Request, Response
|
|
34
|
+
from fastapi.responses import (
|
|
35
|
+
FileResponse,
|
|
36
|
+
HTMLResponse,
|
|
37
|
+
JSONResponse,
|
|
38
|
+
PlainTextResponse,
|
|
39
|
+
StreamingResponse,
|
|
40
|
+
)
|
|
41
|
+
from pydantic import BaseModel, Field
|
|
42
|
+
from rich.console import Console
|
|
43
|
+
|
|
44
|
+
from ghost_api import __version__, watch
|
|
45
|
+
from ghost_api.config import SUPPORTED_LANGUAGES, state_dir
|
|
46
|
+
from ghost_api.errors import friendly_error
|
|
47
|
+
from ghost_api.events import EventBus, Notice
|
|
48
|
+
from ghost_api.export import tabular
|
|
49
|
+
from ghost_api.export.site_data import SAFE_NAME, endpoint_details, run_dict, site_files
|
|
50
|
+
from ghost_api.generator import NothingToGenerate, generate_client
|
|
51
|
+
from ghost_api.generator.python_gen import GenerationError
|
|
52
|
+
from ghost_api.llm.doctor import check_providers
|
|
53
|
+
from ghost_api.llm.gateway import LLMGateway
|
|
54
|
+
from ghost_api.llm.providers import default_providers
|
|
55
|
+
from ghost_api.memory.knowledge import load_memory, memory_prompt
|
|
56
|
+
from ghost_api.memory.store import RunRow, Store
|
|
57
|
+
from ghost_api.net import RateLimiter
|
|
58
|
+
from ghost_api.pipeline import (
|
|
59
|
+
RunOptions,
|
|
60
|
+
client_event,
|
|
61
|
+
explore_site,
|
|
62
|
+
finish,
|
|
63
|
+
open_session,
|
|
64
|
+
robots_policy,
|
|
65
|
+
screenshots_dir,
|
|
66
|
+
)
|
|
67
|
+
from ghost_api.probe.replay import Replayer, ReplayRefused
|
|
68
|
+
from ghost_api.runtime import OperationError, SiteRuntime
|
|
69
|
+
from ghost_api.site import from_url_id, is_site, site_host, to_url_id
|
|
70
|
+
|
|
71
|
+
STATIC = Path(__file__).parent / "static"
|
|
72
|
+
TOKEN_PLACEHOLDER = "__GHOST_TOKEN__"
|
|
73
|
+
SCREEN_NAME = re.compile(r"^[A-Za-z0-9_-]+\.jpg$")
|
|
74
|
+
MAX_TRY_BODY = 200_000
|
|
75
|
+
|
|
76
|
+
GatewayFactory = Callable[[int], LLMGateway | None]
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def default_gateway(max_calls: int) -> LLMGateway | None:
|
|
80
|
+
gateway = LLMGateway(
|
|
81
|
+
default_providers(), max_calls=max_calls, cooldown_file=state_dir() / "cooldowns.json"
|
|
82
|
+
)
|
|
83
|
+
return gateway if gateway.providers else None
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
class StartRun(BaseModel):
|
|
87
|
+
url: str = Field(min_length=3, max_length=2000)
|
|
88
|
+
goal: str = Field(min_length=1, max_length=500)
|
|
89
|
+
lang: str = "python"
|
|
90
|
+
agent: bool = True
|
|
91
|
+
max_steps: int = Field(20, ge=1, le=60)
|
|
92
|
+
max_llm_calls: int = Field(30, ge=1, le=100)
|
|
93
|
+
ignore_robots: bool = False
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
class TryCall(BaseModel):
|
|
97
|
+
endpoint: str # Endpoint.key
|
|
98
|
+
params: dict[str, str] = Field(default_factory=dict)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
class FetchCall(TryCall):
|
|
102
|
+
pages: int = Field(5, ge=1, le=50)
|
|
103
|
+
format: str = "csv"
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
class AllowCall(BaseModel):
|
|
107
|
+
endpoint: str # Endpoint.key
|
|
108
|
+
allowed: bool
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
MAX_FETCH_ROWS = 5000
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
class NewWatch(BaseModel):
|
|
115
|
+
site: str
|
|
116
|
+
endpoint: str # Endpoint.key or operation name
|
|
117
|
+
params: dict[str, str] = Field(default_factory=dict)
|
|
118
|
+
every: str = "6h"
|
|
119
|
+
notify: list[str] = Field(default_factory=list)
|
|
120
|
+
track: list[str] = Field(default_factory=list)
|
|
121
|
+
pages: int = Field(5, ge=1, le=50)
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
class Regenerate(BaseModel):
|
|
125
|
+
lang: str = "python"
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
# --- running one exploration at a time ------------------------------------------------------
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
class RunManager:
|
|
132
|
+
def __init__(self, db: Path, out: Path, gateway_factory: GatewayFactory) -> None:
|
|
133
|
+
self.db = db
|
|
134
|
+
self.out = out
|
|
135
|
+
self.gateway_factory = gateway_factory
|
|
136
|
+
self.active_run: int | None = None
|
|
137
|
+
self._stop = threading.Event()
|
|
138
|
+
self._lock = threading.Lock()
|
|
139
|
+
self._thread: threading.Thread | None = None
|
|
140
|
+
|
|
141
|
+
@property
|
|
142
|
+
def busy(self) -> bool:
|
|
143
|
+
return self._thread is not None and self._thread.is_alive()
|
|
144
|
+
|
|
145
|
+
def start(self, req: StartRun, timeout: float = 60) -> int:
|
|
146
|
+
with self._lock:
|
|
147
|
+
if self.busy:
|
|
148
|
+
raise HTTPException(409, "A run is already in progress. Stop it or wait.")
|
|
149
|
+
if req.lang not in SUPPORTED_LANGUAGES:
|
|
150
|
+
raise HTTPException(400, f"Unsupported language: {req.lang}")
|
|
151
|
+
ready = threading.Event()
|
|
152
|
+
result: dict[str, Any] = {}
|
|
153
|
+
self._stop.clear()
|
|
154
|
+
self._thread = threading.Thread(
|
|
155
|
+
target=self._run, args=(req, ready, result), daemon=True, name="ghost-run"
|
|
156
|
+
)
|
|
157
|
+
self._thread.start()
|
|
158
|
+
if not ready.wait(timeout):
|
|
159
|
+
raise HTTPException(504, "The run did not start in time.")
|
|
160
|
+
if "error" in result:
|
|
161
|
+
raise HTTPException(400, result["error"])
|
|
162
|
+
return int(result["run_id"])
|
|
163
|
+
|
|
164
|
+
def stop(self, run_id: int) -> bool:
|
|
165
|
+
if self.active_run != run_id:
|
|
166
|
+
return False
|
|
167
|
+
self._stop.set()
|
|
168
|
+
return True
|
|
169
|
+
|
|
170
|
+
def wait(self, timeout: float | None = None) -> None:
|
|
171
|
+
if self._thread is not None:
|
|
172
|
+
self._thread.join(timeout)
|
|
173
|
+
|
|
174
|
+
def _run(self, req: StartRun, ready: threading.Event, result: dict[str, Any]) -> None:
|
|
175
|
+
bus = EventBus()
|
|
176
|
+
notices: list[Notice] = []
|
|
177
|
+
bus.subscribe(lambda e: notices.append(e) if isinstance(e, Notice) else None)
|
|
178
|
+
try:
|
|
179
|
+
url = req.url if "://" in req.url else f"https://{req.url}"
|
|
180
|
+
host = (urlsplit(url).hostname or "").lower()
|
|
181
|
+
if not host or urlsplit(url).scheme not in ("http", "https"):
|
|
182
|
+
result["error"] = f"Not a valid http(s) URL: {req.url}"
|
|
183
|
+
return
|
|
184
|
+
limiter = RateLimiter(1.0)
|
|
185
|
+
robots, allowed = robots_policy(url, req.ignore_robots, limiter, bus)
|
|
186
|
+
if not allowed:
|
|
187
|
+
result["error"] = notices[-1].text if notices else "Disallowed by robots.txt"
|
|
188
|
+
return
|
|
189
|
+
gateway = self.gateway_factory(req.max_llm_calls) if req.agent else None
|
|
190
|
+
if req.agent and gateway is None:
|
|
191
|
+
result["error"] = (
|
|
192
|
+
"No LLM API key found. Add one to .env, or turn off the AI agent to only "
|
|
193
|
+
"load and scroll the page."
|
|
194
|
+
)
|
|
195
|
+
return
|
|
196
|
+
with Store(self.db) as store:
|
|
197
|
+
s = open_session(
|
|
198
|
+
store,
|
|
199
|
+
bus,
|
|
200
|
+
host,
|
|
201
|
+
url,
|
|
202
|
+
req.goal,
|
|
203
|
+
False,
|
|
204
|
+
limiter,
|
|
205
|
+
robots,
|
|
206
|
+
gateway,
|
|
207
|
+
60,
|
|
208
|
+
screens_root=self.db,
|
|
209
|
+
lang=req.lang,
|
|
210
|
+
)
|
|
211
|
+
s.stop = self._stop.is_set
|
|
212
|
+
for notice in notices: # the robots.txt result, now that the run is recorded
|
|
213
|
+
store.add_event(s.run_id, notice.to_dict())
|
|
214
|
+
known = load_memory(store, host)
|
|
215
|
+
if known is not None and known.run_id != s.run_id:
|
|
216
|
+
s.memory = memory_prompt(known.endpoints, known.lessons)
|
|
217
|
+
bus.emit(
|
|
218
|
+
Notice(
|
|
219
|
+
f"Using memory: {len(known.endpoints)} known endpoints, "
|
|
220
|
+
f"{len(known.lessons)} lessons"
|
|
221
|
+
)
|
|
222
|
+
)
|
|
223
|
+
self.active_run = s.run_id
|
|
224
|
+
result["run_id"] = s.run_id
|
|
225
|
+
ready.set()
|
|
226
|
+
opts = RunOptions(
|
|
227
|
+
lang=req.lang, max_steps=req.max_steps, max_minutes=10, out=self.out
|
|
228
|
+
)
|
|
229
|
+
started = time.monotonic()
|
|
230
|
+
status, reason = explore_site(s, opts, use_agent=gateway is not None)
|
|
231
|
+
finish(s, status, reason, started, opts)
|
|
232
|
+
except Exception as exc: # noqa: BLE001 - report it to the page, keep serving
|
|
233
|
+
result.setdefault("error", friendly_error(exc))
|
|
234
|
+
finally:
|
|
235
|
+
self.active_run = None
|
|
236
|
+
ready.set()
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
# --- reading what is stored -----------------------------------------------------------------
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def _site_name(url_id: str) -> str:
|
|
243
|
+
"""The site from its URL form ('www.google.com~maps' -> 'www.google.com/maps')."""
|
|
244
|
+
site = from_url_id(url_id)
|
|
245
|
+
if not is_site(site):
|
|
246
|
+
raise HTTPException(400, "Invalid site name")
|
|
247
|
+
return site
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def _site_folder(out: Path, site: str) -> Path:
|
|
251
|
+
if not is_site(site):
|
|
252
|
+
raise HTTPException(400, "Invalid site name")
|
|
253
|
+
return out / site
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def _site_files(out: Path, domain: str) -> list[dict[str, Any]]:
|
|
257
|
+
return site_files(_site_folder(out, domain))
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
# --- the app --------------------------------------------------------------------------------
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def create_app(
|
|
264
|
+
db: Path,
|
|
265
|
+
out: Path,
|
|
266
|
+
token: str,
|
|
267
|
+
manager: RunManager | None = None,
|
|
268
|
+
allowed_hosts: set[str] | None = None,
|
|
269
|
+
) -> FastAPI:
|
|
270
|
+
manager = manager or RunManager(db, out, default_gateway)
|
|
271
|
+
app = FastAPI(title="Ghost API", docs_url=None, redoc_url=None, openapi_url=None)
|
|
272
|
+
app.state.manager = manager
|
|
273
|
+
|
|
274
|
+
@app.middleware("http")
|
|
275
|
+
async def guard(
|
|
276
|
+
request: Request, call_next: Callable[[Request], Awaitable[Response]]
|
|
277
|
+
) -> Response:
|
|
278
|
+
host = request.headers.get("host", "")
|
|
279
|
+
if allowed_hosts is not None and host not in allowed_hosts:
|
|
280
|
+
return JSONResponse({"detail": "Unknown host"}, status_code=403)
|
|
281
|
+
if request.url.path.startswith("/api/"):
|
|
282
|
+
given = request.headers.get("x-ghost-token") or request.query_params.get("token", "")
|
|
283
|
+
if not secrets.compare_digest(given, token):
|
|
284
|
+
return JSONResponse({"detail": "Missing or wrong token"}, status_code=401)
|
|
285
|
+
origin = request.headers.get("origin")
|
|
286
|
+
if request.method != "GET" and origin and urlsplit(origin).netloc != host:
|
|
287
|
+
return JSONResponse({"detail": "Cross-origin request refused"}, status_code=403)
|
|
288
|
+
return await call_next(request)
|
|
289
|
+
|
|
290
|
+
def store() -> Store:
|
|
291
|
+
return Store(db)
|
|
292
|
+
|
|
293
|
+
@app.get("/api/state")
|
|
294
|
+
def state() -> dict[str, Any]:
|
|
295
|
+
return {
|
|
296
|
+
"version": __version__,
|
|
297
|
+
"running": manager.active_run,
|
|
298
|
+
"has_llm": any(p.api_key for p in default_providers()),
|
|
299
|
+
"languages": list(SUPPORTED_LANGUAGES),
|
|
300
|
+
"db": str(db),
|
|
301
|
+
"out": str(out),
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
@app.get("/api/sites")
|
|
305
|
+
def sites() -> list[dict[str, Any]]:
|
|
306
|
+
if not db.exists():
|
|
307
|
+
return []
|
|
308
|
+
with store() as s:
|
|
309
|
+
runs = s.list_runs()
|
|
310
|
+
by_domain: dict[str, list[RunRow]] = {}
|
|
311
|
+
for r in runs:
|
|
312
|
+
by_domain.setdefault(r.domain, []).append(r)
|
|
313
|
+
result = []
|
|
314
|
+
for domain, domain_runs in by_domain.items():
|
|
315
|
+
endpoints = endpoint_details(s, domain)
|
|
316
|
+
counts = {
|
|
317
|
+
k: sum(e["status"] == k for e in endpoints)
|
|
318
|
+
for k in ("verified", "unverified", "failed", "discovered")
|
|
319
|
+
}
|
|
320
|
+
result.append(
|
|
321
|
+
{
|
|
322
|
+
"domain": domain,
|
|
323
|
+
"latest": run_dict(domain_runs[0]),
|
|
324
|
+
"runs": len(domain_runs),
|
|
325
|
+
"endpoints": len(endpoints),
|
|
326
|
+
"counts": counts,
|
|
327
|
+
"methods": sorted({e["method"] for e in endpoints}),
|
|
328
|
+
"has_client": any(f["kind"] == "client" for f in _site_files(out, domain)),
|
|
329
|
+
}
|
|
330
|
+
)
|
|
331
|
+
return result
|
|
332
|
+
|
|
333
|
+
@app.get("/api/sites/{domain}")
|
|
334
|
+
def site(domain: str) -> dict[str, Any]:
|
|
335
|
+
domain = _site_name(domain)
|
|
336
|
+
_site_folder(out, domain)
|
|
337
|
+
with store() as s:
|
|
338
|
+
runs = s.list_runs(domain)
|
|
339
|
+
if not runs:
|
|
340
|
+
raise HTTPException(404, f"Nothing stored for {domain}")
|
|
341
|
+
return {
|
|
342
|
+
"domain": domain,
|
|
343
|
+
"runs": [run_dict(r) for r in runs],
|
|
344
|
+
"endpoints": endpoint_details(s, domain),
|
|
345
|
+
"lessons": s.lessons_for(domain, limit=20),
|
|
346
|
+
"files": _site_files(out, domain),
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
@app.get("/api/runs/{run_id}")
|
|
350
|
+
def run(run_id: int) -> dict[str, Any]:
|
|
351
|
+
with store() as s:
|
|
352
|
+
r = s.get_run(run_id)
|
|
353
|
+
if r is None:
|
|
354
|
+
raise HTTPException(404, "Run not found")
|
|
355
|
+
return {
|
|
356
|
+
**run_dict(r),
|
|
357
|
+
"active": manager.active_run == run_id,
|
|
358
|
+
"steps": s.steps_for_run(run_id),
|
|
359
|
+
"hypotheses": s.hypotheses_for_run(run_id),
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
@app.get("/api/runs/{run_id}/events")
|
|
363
|
+
async def events(run_id: int, after: int = 0) -> StreamingResponse:
|
|
364
|
+
async def stream() -> AsyncIterator[str]:
|
|
365
|
+
last = after
|
|
366
|
+
idle = 0
|
|
367
|
+
while True:
|
|
368
|
+
with store() as s:
|
|
369
|
+
batch = s.events_for_run(run_id, after=last)
|
|
370
|
+
finished = False
|
|
371
|
+
for event in batch:
|
|
372
|
+
last = int(event["seq"])
|
|
373
|
+
finished = finished or event["type"] == "run_finished"
|
|
374
|
+
yield f"id: {last}\ndata: {json.dumps(event)}\n\n"
|
|
375
|
+
idle = 0 if batch else idle + 1
|
|
376
|
+
if finished or (manager.active_run != run_id and not batch and idle > 2):
|
|
377
|
+
yield "event: end\ndata: {}\n\n"
|
|
378
|
+
return
|
|
379
|
+
await asyncio.sleep(0.4)
|
|
380
|
+
|
|
381
|
+
headers = {"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}
|
|
382
|
+
return StreamingResponse(stream(), media_type="text/event-stream", headers=headers)
|
|
383
|
+
|
|
384
|
+
@app.post("/api/runs")
|
|
385
|
+
async def start_run(req: StartRun) -> dict[str, int]:
|
|
386
|
+
run_id = await asyncio.to_thread(manager.start, req)
|
|
387
|
+
return {"run_id": run_id}
|
|
388
|
+
|
|
389
|
+
@app.post("/api/runs/{run_id}/stop")
|
|
390
|
+
def stop_run(run_id: int) -> dict[str, bool]:
|
|
391
|
+
return {"stopping": manager.stop(run_id)}
|
|
392
|
+
|
|
393
|
+
@app.get("/api/runs/{run_id}/screens/{name}")
|
|
394
|
+
def screen(run_id: int, name: str) -> FileResponse:
|
|
395
|
+
if not SCREEN_NAME.match(name):
|
|
396
|
+
raise HTTPException(400, "Invalid screenshot name")
|
|
397
|
+
path = screenshots_dir(db, run_id) / name
|
|
398
|
+
if not path.is_file():
|
|
399
|
+
raise HTTPException(404, "No such screenshot")
|
|
400
|
+
return FileResponse(path, media_type="image/jpeg")
|
|
401
|
+
|
|
402
|
+
@app.get("/api/sites/{domain}/files")
|
|
403
|
+
def files(domain: str) -> list[dict[str, Any]]:
|
|
404
|
+
domain = _site_name(domain)
|
|
405
|
+
return _site_files(out, domain)
|
|
406
|
+
|
|
407
|
+
@app.get("/api/sites/{domain}/files/{name}")
|
|
408
|
+
def file(domain: str, name: str) -> PlainTextResponse:
|
|
409
|
+
domain = _site_name(domain)
|
|
410
|
+
folder = _site_folder(out, domain)
|
|
411
|
+
if not SAFE_NAME.match(name) or name.startswith("."):
|
|
412
|
+
raise HTTPException(400, "Invalid file name")
|
|
413
|
+
path = (folder / name).resolve()
|
|
414
|
+
if path.parent != folder.resolve() or not path.is_file():
|
|
415
|
+
raise HTTPException(404, "No such file")
|
|
416
|
+
return PlainTextResponse(path.read_text(encoding="utf-8", errors="replace"))
|
|
417
|
+
|
|
418
|
+
@app.get("/api/sites/{domain}/download.zip")
|
|
419
|
+
def download(domain: str) -> Response:
|
|
420
|
+
domain = _site_name(domain)
|
|
421
|
+
folder = _site_folder(out, domain)
|
|
422
|
+
listing = _site_files(out, domain)
|
|
423
|
+
if not listing:
|
|
424
|
+
raise HTTPException(404, "No files for this site yet")
|
|
425
|
+
buffer = io.BytesIO()
|
|
426
|
+
with zipfile.ZipFile(buffer, "w", zipfile.ZIP_DEFLATED) as zf:
|
|
427
|
+
for f in listing:
|
|
428
|
+
zf.write(folder / f["name"], f"{domain}/{f['name']}")
|
|
429
|
+
filename = to_url_id(domain).replace("~", "-")
|
|
430
|
+
headers = {"Content-Disposition": f'attachment; filename="{filename}.zip"'}
|
|
431
|
+
return Response(buffer.getvalue(), media_type="application/zip", headers=headers)
|
|
432
|
+
|
|
433
|
+
@app.post("/api/sites/{domain}/generate")
|
|
434
|
+
def regenerate(domain: str, body: Regenerate) -> dict[str, Any]:
|
|
435
|
+
domain = _site_name(domain)
|
|
436
|
+
_site_folder(out, domain)
|
|
437
|
+
if body.lang not in SUPPORTED_LANGUAGES:
|
|
438
|
+
raise HTTPException(400, f"Unsupported language: {body.lang}")
|
|
439
|
+
try:
|
|
440
|
+
with store() as s:
|
|
441
|
+
result = generate_client(s, domain, out, llm=None, lang=body.lang)
|
|
442
|
+
except (NothingToGenerate, GenerationError) as exc:
|
|
443
|
+
raise HTTPException(400, str(exc)) from exc
|
|
444
|
+
return client_event(result).to_dict()
|
|
445
|
+
|
|
446
|
+
@app.post("/api/sites/{domain}/try")
|
|
447
|
+
def try_endpoint(domain: str, call: TryCall) -> dict[str, Any]:
|
|
448
|
+
"""Call an endpoint once, read-only, with the given parameters."""
|
|
449
|
+
domain = _site_name(domain)
|
|
450
|
+
_site_folder(out, domain)
|
|
451
|
+
replayer = Replayer(site_host(domain), RateLimiter(1.0), max_requests=1)
|
|
452
|
+
try:
|
|
453
|
+
with store() as s:
|
|
454
|
+
runtime = SiteRuntime(s, domain, replayer=replayer)
|
|
455
|
+
if not any(e.key == call.endpoint for e in runtime.endpoints):
|
|
456
|
+
raise HTTPException(404, "Unknown endpoint")
|
|
457
|
+
op = next((o for o in runtime.operations() if o.endpoint.key == call.endpoint), None)
|
|
458
|
+
if op is None:
|
|
459
|
+
raise HTTPException(
|
|
460
|
+
409,
|
|
461
|
+
"This endpoint changes data or can't be called outside the browser, so "
|
|
462
|
+
"Ghost API never calls it. If you checked it only reads, allow it first.",
|
|
463
|
+
)
|
|
464
|
+
started = time.monotonic()
|
|
465
|
+
result = runtime.call(op, call.params)
|
|
466
|
+
except OperationError as exc:
|
|
467
|
+
raise HTTPException(409, str(exc)) from exc
|
|
468
|
+
except ReplayRefused as exc:
|
|
469
|
+
raise HTTPException(400, str(exc)) from exc
|
|
470
|
+
finally:
|
|
471
|
+
replayer.close()
|
|
472
|
+
text = json.dumps(result.body, indent=2) if result.body is not None else ""
|
|
473
|
+
return {
|
|
474
|
+
"status": result.status,
|
|
475
|
+
"ok": result.ok,
|
|
476
|
+
"error": result.error,
|
|
477
|
+
"url": result.captured.url if result.captured else "",
|
|
478
|
+
"ms": round((time.monotonic() - started) * 1000),
|
|
479
|
+
"body": text[:MAX_TRY_BODY],
|
|
480
|
+
"truncated": len(text) > MAX_TRY_BODY,
|
|
481
|
+
}
|
|
482
|
+
|
|
483
|
+
@app.post("/api/sites/{domain}/fetch")
|
|
484
|
+
def fetch_data(domain: str, call: FetchCall) -> Response:
|
|
485
|
+
"""All pages of an endpoint's data as a CSV / Excel / JSON download."""
|
|
486
|
+
domain = _site_name(domain)
|
|
487
|
+
if call.format not in tabular.FORMATS:
|
|
488
|
+
raise HTTPException(400, f"Unsupported format: {call.format}")
|
|
489
|
+
try:
|
|
490
|
+
with store() as s:
|
|
491
|
+
runtime = SiteRuntime(s, domain)
|
|
492
|
+
with runtime:
|
|
493
|
+
op = next(
|
|
494
|
+
(o for o in runtime.operations() if o.endpoint.key == call.endpoint), None
|
|
495
|
+
)
|
|
496
|
+
if op is None:
|
|
497
|
+
raise HTTPException(409, "Ghost API doesn't call this endpoint by itself.")
|
|
498
|
+
records: list[Any] = []
|
|
499
|
+
for page in runtime.iterate(op, call.params, call.pages, MAX_FETCH_ROWS):
|
|
500
|
+
if not page.result.ok:
|
|
501
|
+
if not records:
|
|
502
|
+
detail = page.result.error or f"HTTP {page.result.status}"
|
|
503
|
+
raise HTTPException(502, f"The site answered: {detail}")
|
|
504
|
+
break
|
|
505
|
+
records += page.records
|
|
506
|
+
except OperationError as exc:
|
|
507
|
+
raise HTTPException(409, str(exc)) from exc
|
|
508
|
+
except ReplayRefused as exc:
|
|
509
|
+
raise HTTPException(400, str(exc)) from exc
|
|
510
|
+
if not records:
|
|
511
|
+
raise HTTPException(404, "No rows came back. Try other inputs.")
|
|
512
|
+
name = f"{to_url_id(domain).replace('~', '-')}-{op.name}.{call.format}"
|
|
513
|
+
return Response(
|
|
514
|
+
tabular.to_bytes(records, call.format),
|
|
515
|
+
media_type=tabular.MEDIA_TYPES[call.format],
|
|
516
|
+
headers={
|
|
517
|
+
"Content-Disposition": f'attachment; filename="{name}"',
|
|
518
|
+
"X-Ghost-Rows": str(len(records)),
|
|
519
|
+
},
|
|
520
|
+
)
|
|
521
|
+
|
|
522
|
+
@app.post("/api/sites/{domain}/allow")
|
|
523
|
+
def allow_endpoint(domain: str, call: AllowCall) -> dict[str, bool]:
|
|
524
|
+
"""The user checked this endpoint only reads (e.g. a POST search): allow calling it."""
|
|
525
|
+
domain = _site_name(domain)
|
|
526
|
+
with store() as s:
|
|
527
|
+
keys = {e.key for e in s.current_endpoints(domain)}
|
|
528
|
+
if call.endpoint not in keys:
|
|
529
|
+
raise HTTPException(404, "Unknown endpoint")
|
|
530
|
+
if call.allowed:
|
|
531
|
+
s.allow_endpoints(domain, [call.endpoint])
|
|
532
|
+
else:
|
|
533
|
+
s.disallow_endpoint(domain, call.endpoint)
|
|
534
|
+
return {"allowed": call.allowed}
|
|
535
|
+
|
|
536
|
+
# --- watches ---
|
|
537
|
+
|
|
538
|
+
def _watch_json(w: dict[str, Any]) -> dict[str, Any]:
|
|
539
|
+
return {
|
|
540
|
+
**w,
|
|
541
|
+
"every": watch.every_label(w["every_minutes"]),
|
|
542
|
+
"due": watch.is_due(w),
|
|
543
|
+
"describe": watch.describe(w),
|
|
544
|
+
"site_id": to_url_id(w["site"]),
|
|
545
|
+
}
|
|
546
|
+
|
|
547
|
+
@app.get("/api/watches")
|
|
548
|
+
def list_watches() -> dict[str, Any]:
|
|
549
|
+
if not db.exists():
|
|
550
|
+
return {"watches": [], "changes": []}
|
|
551
|
+
with store() as s:
|
|
552
|
+
return {
|
|
553
|
+
"watches": [_watch_json(w) for w in s.watches()],
|
|
554
|
+
"changes": s.watch_changes(limit=100),
|
|
555
|
+
}
|
|
556
|
+
|
|
557
|
+
@app.post("/api/watches")
|
|
558
|
+
async def add_watch(req: NewWatch) -> dict[str, Any]:
|
|
559
|
+
site = _site_name(to_url_id(req.site))
|
|
560
|
+
try:
|
|
561
|
+
minutes = watch.parse_every(req.every)
|
|
562
|
+
except ValueError as exc:
|
|
563
|
+
raise HTTPException(400, str(exc)) from exc
|
|
564
|
+
bad = [n for n in req.notify if not n.startswith(("ntfy:", "webhook:", "desktop"))]
|
|
565
|
+
if bad:
|
|
566
|
+
raise HTTPException(400, f"Unknown notification target: {bad[0]}")
|
|
567
|
+
|
|
568
|
+
def create() -> dict[str, Any]:
|
|
569
|
+
with store() as s:
|
|
570
|
+
with SiteRuntime(s, site) as rt:
|
|
571
|
+
op = rt.operation(req.endpoint)
|
|
572
|
+
watch_id = s.add_watch(
|
|
573
|
+
{
|
|
574
|
+
"site": site,
|
|
575
|
+
"operation": op.name,
|
|
576
|
+
"params": req.params,
|
|
577
|
+
"every_minutes": minutes,
|
|
578
|
+
"notify": req.notify,
|
|
579
|
+
"track": req.track,
|
|
580
|
+
"max_pages": req.pages,
|
|
581
|
+
} # fmt: skip
|
|
582
|
+
)
|
|
583
|
+
w = s.get_watch(watch_id)
|
|
584
|
+
assert w is not None
|
|
585
|
+
return {"watch": _watch_json(w), "check": watch.to_json(watch.check(s, w))}
|
|
586
|
+
|
|
587
|
+
try:
|
|
588
|
+
return await asyncio.to_thread(create)
|
|
589
|
+
except OperationError as exc:
|
|
590
|
+
raise HTTPException(400, str(exc)) from exc
|
|
591
|
+
|
|
592
|
+
@app.post("/api/watches/{watch_id}/check")
|
|
593
|
+
async def check_watch(watch_id: int) -> dict[str, Any]:
|
|
594
|
+
def run() -> dict[str, Any]:
|
|
595
|
+
with store() as s:
|
|
596
|
+
w = s.get_watch(watch_id)
|
|
597
|
+
if w is None:
|
|
598
|
+
raise HTTPException(404, "No such watch")
|
|
599
|
+
return watch.to_json(watch.check(s, w))
|
|
600
|
+
|
|
601
|
+
return await asyncio.to_thread(run)
|
|
602
|
+
|
|
603
|
+
@app.delete("/api/watches/{watch_id}")
|
|
604
|
+
def remove_watch(watch_id: int) -> dict[str, bool]:
|
|
605
|
+
with store() as s:
|
|
606
|
+
if not s.remove_watch(watch_id):
|
|
607
|
+
raise HTTPException(404, "No such watch")
|
|
608
|
+
return {"removed": True}
|
|
609
|
+
|
|
610
|
+
@app.post("/api/doctor")
|
|
611
|
+
async def doctor() -> list[dict[str, Any]]:
|
|
612
|
+
return await asyncio.to_thread(check_providers)
|
|
613
|
+
|
|
614
|
+
# --- the frontend ---
|
|
615
|
+
|
|
616
|
+
def index() -> HTMLResponse:
|
|
617
|
+
page = STATIC / "index.html"
|
|
618
|
+
if not page.is_file():
|
|
619
|
+
return HTMLResponse(
|
|
620
|
+
"<h1>Ghost API</h1><p>The dashboard frontend is not built. Run "
|
|
621
|
+
"<code>npm run build</code> in the <code>web/</code> folder.</p>",
|
|
622
|
+
status_code=503,
|
|
623
|
+
)
|
|
624
|
+
html = page.read_text(encoding="utf-8").replace(TOKEN_PLACEHOLDER, token)
|
|
625
|
+
return HTMLResponse(html, headers={"Cache-Control": "no-store"})
|
|
626
|
+
|
|
627
|
+
@app.get("/assets/{name}")
|
|
628
|
+
def asset(name: str) -> FileResponse:
|
|
629
|
+
if not SAFE_NAME.match(name):
|
|
630
|
+
raise HTTPException(404)
|
|
631
|
+
path = STATIC / "assets" / name
|
|
632
|
+
if not path.is_file():
|
|
633
|
+
raise HTTPException(404)
|
|
634
|
+
return FileResponse(path)
|
|
635
|
+
|
|
636
|
+
@app.get("/{path:path}")
|
|
637
|
+
def spa(path: str) -> Response:
|
|
638
|
+
if path.startswith("api/"):
|
|
639
|
+
raise HTTPException(404)
|
|
640
|
+
static_file = STATIC / path
|
|
641
|
+
if path and SAFE_NAME.match(path) and static_file.is_file():
|
|
642
|
+
return FileResponse(static_file) # favicon, logo...
|
|
643
|
+
return index() # client-side routes
|
|
644
|
+
|
|
645
|
+
return app
|
|
646
|
+
|
|
647
|
+
|
|
648
|
+
def launch_token() -> str:
|
|
649
|
+
"""A new random token per launch, or GHOST_API_UI_TOKEN for frontend development
|
|
650
|
+
(`npm run dev` sends the same value as VITE_GHOST_TOKEN)."""
|
|
651
|
+
return os.getenv("GHOST_API_UI_TOKEN") or secrets.token_urlsafe(24)
|
|
652
|
+
|
|
653
|
+
|
|
654
|
+
def _watch_loop(db: Path, console: Console) -> None:
|
|
655
|
+
"""While the dashboard is open, check due watches every minute."""
|
|
656
|
+
while True:
|
|
657
|
+
try:
|
|
658
|
+
if db.exists():
|
|
659
|
+
with Store(db) as s:
|
|
660
|
+
for result in watch.run_due(s):
|
|
661
|
+
console.print(f"[dim]watch #{result.watch['id']}: {result.headline()}[/]")
|
|
662
|
+
except Exception as exc: # noqa: BLE001 - keep the dashboard running
|
|
663
|
+
console.print(f"[dim]watch check failed: {exc}[/]")
|
|
664
|
+
time.sleep(60)
|
|
665
|
+
|
|
666
|
+
|
|
667
|
+
def serve(
|
|
668
|
+
db: Path,
|
|
669
|
+
out: Path,
|
|
670
|
+
port: int,
|
|
671
|
+
open_browser: bool,
|
|
672
|
+
console: Console | None = None,
|
|
673
|
+
run_watches: bool = True,
|
|
674
|
+
) -> None:
|
|
675
|
+
import uvicorn
|
|
676
|
+
|
|
677
|
+
console = console or Console()
|
|
678
|
+
token = launch_token()
|
|
679
|
+
hosts = {f"127.0.0.1:{port}", f"localhost:{port}"}
|
|
680
|
+
app = create_app(db, out, token, allowed_hosts=hosts)
|
|
681
|
+
if run_watches:
|
|
682
|
+
threading.Thread(target=_watch_loop, args=(db, console), daemon=True).start()
|
|
683
|
+
url = f"http://127.0.0.1:{port}/"
|
|
684
|
+
from ghost_api.ui.terminal import ACCENT
|
|
685
|
+
|
|
686
|
+
console.print(f"[bold {ACCENT}]◆ Ghost API dashboard[/] {url}")
|
|
687
|
+
console.print("[dim]Local only. Press Ctrl+C to stop.[/]")
|
|
688
|
+
if open_browser:
|
|
689
|
+
threading.Timer(1.0, lambda: webbrowser.open(url)).start()
|
|
690
|
+
uvicorn.run(app, host="127.0.0.1", port=port, log_level="warning")
|