ghost-api-cli 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ghost_api/__init__.py +3 -0
- ghost_api/__main__.py +5 -0
- ghost_api/adopt.py +173 -0
- ghost_api/agent/__init__.py +0 -0
- ghost_api/agent/orchestrator.py +413 -0
- ghost_api/agent/prompts.py +149 -0
- ghost_api/agent/tools.py +84 -0
- ghost_api/analysis/__init__.py +0 -0
- ghost_api/analysis/analyzer.py +130 -0
- ghost_api/analysis/detectors.py +188 -0
- ghost_api/analysis/exposure.py +80 -0
- ghost_api/analysis/graphql.py +67 -0
- ghost_api/analysis/models.py +64 -0
- ghost_api/analysis/schema.py +109 -0
- ghost_api/analysis/url_grouping.py +217 -0
- ghost_api/api.py +264 -0
- ghost_api/audit.py +78 -0
- ghost_api/browser/__init__.py +0 -0
- ghost_api/browser/auth_state.py +172 -0
- ghost_api/browser/driver.py +273 -0
- ghost_api/browser/egress.py +183 -0
- ghost_api/browser/gate.py +127 -0
- ghost_api/browser/install.py +28 -0
- ghost_api/browser/snapshot.py +221 -0
- ghost_api/capture/__init__.py +0 -0
- ghost_api/capture/filters.py +255 -0
- ghost_api/capture/har.py +148 -0
- ghost_api/capture/models.py +41 -0
- ghost_api/capture/personal.py +188 -0
- ghost_api/capture/recorder.py +181 -0
- ghost_api/cli/__init__.py +44 -0
- ghost_api/cli/_common.py +219 -0
- ghost_api/cli/client.py +487 -0
- ghost_api/cli/data.py +492 -0
- ghost_api/cli/explore.py +379 -0
- ghost_api/cli/manage.py +437 -0
- ghost_api/config.py +132 -0
- ghost_api/drift.py +344 -0
- ghost_api/drive.py +211 -0
- ghost_api/errors.py +106 -0
- ghost_api/events.py +213 -0
- ghost_api/exitcodes.py +21 -0
- ghost_api/export/__init__.py +71 -0
- ghost_api/export/confidence.py +45 -0
- ghost_api/export/html.py +41 -0
- ghost_api/export/openapi.py +201 -0
- ghost_api/export/report.py +227 -0
- ghost_api/export/site_data.py +196 -0
- ghost_api/export/tabular.py +88 -0
- ghost_api/export/viewer/report.html +286 -0
- ghost_api/generator/__init__.py +221 -0
- ghost_api/generator/cassettes.py +80 -0
- ghost_api/generator/models_gen.py +144 -0
- ghost_api/generator/naming.py +174 -0
- ghost_api/generator/package.py +197 -0
- ghost_api/generator/python_gen.py +172 -0
- ghost_api/generator/spec.py +486 -0
- ghost_api/generator/templates/client.py.j2 +518 -0
- ghost_api/generator/templates/client.test.ts.j2 +106 -0
- ghost_api/generator/templates/client.ts.j2 +339 -0
- ghost_api/generator/templates/models.py.j2 +45 -0
- ghost_api/generator/templates/package_readme.md.j2 +111 -0
- ghost_api/generator/templates/pyproject.toml.j2 +22 -0
- ghost_api/generator/templates/test_client.py.j2 +105 -0
- ghost_api/generator/ts_gen.py +232 -0
- ghost_api/integrations/__init__.py +0 -0
- ghost_api/integrations/claude_code/commands/ghost-data.md +10 -0
- ghost_api/integrations/claude_code/commands/ghost-diff.md +10 -0
- ghost_api/integrations/claude_code/commands/ghost-explore.md +10 -0
- ghost_api/integrations/claude_code/skills/ghost-api/SKILL.md +46 -0
- ghost_api/integrations/setup.py +84 -0
- ghost_api/llm/__init__.py +0 -0
- ghost_api/llm/doctor.py +60 -0
- ghost_api/llm/gateway.py +476 -0
- ghost_api/llm/providers.py +124 -0
- ghost_api/memory/__init__.py +0 -0
- ghost_api/memory/knowledge.py +157 -0
- ghost_api/memory/store.py +883 -0
- ghost_api/net.py +235 -0
- ghost_api/netguard.py +203 -0
- ghost_api/pipeline.py +745 -0
- ghost_api/probe/__init__.py +0 -0
- ghost_api/probe/prober.py +311 -0
- ghost_api/probe/replay.py +313 -0
- ghost_api/project.py +186 -0
- ghost_api/py.typed +0 -0
- ghost_api/runtime.py +397 -0
- ghost_api/serve/__init__.py +101 -0
- ghost_api/serve/explorer_mcp.py +305 -0
- ghost_api/serve/mcp.py +121 -0
- ghost_api/serve/mock.py +130 -0
- ghost_api/serve/rest.py +132 -0
- ghost_api/site.py +84 -0
- ghost_api/specdiff.py +122 -0
- ghost_api/ui/__init__.py +0 -0
- ghost_api/ui/keys.py +134 -0
- ghost_api/ui/repl.py +453 -0
- ghost_api/ui/terminal.py +573 -0
- ghost_api/verify/__init__.py +0 -0
- ghost_api/verify/classify.py +84 -0
- ghost_api/verify/repair.py +40 -0
- ghost_api/verify/runner.py +110 -0
- ghost_api/verify/verifier.py +291 -0
- ghost_api/watch.py +313 -0
- ghost_api/web/__init__.py +1 -0
- ghost_api/web/server.py +690 -0
- ghost_api/web/static/assets/Endpoints-zPnjlcM8.js +46 -0
- ghost_api/web/static/assets/Run-D85zWHey.js +91 -0
- ghost_api/web/static/assets/Settings-D0aWHFAt.js +11 -0
- ghost_api/web/static/assets/Site-a88shEmP.js +71 -0
- ghost_api/web/static/assets/Watches-CqZWIa9G.js +16 -0
- ghost_api/web/static/assets/circle-x-BcTE_1tr.js +11 -0
- ghost_api/web/static/assets/index-Ds2e5B9f.js +141 -0
- ghost_api/web/static/assets/index-xQpyIBbG.css +1 -0
- ghost_api/web/static/assets/minus-C2xqP1eQ.js +6 -0
- ghost_api/web/static/assets/refresh-cw-CxdR_8Oi.js +6 -0
- ghost_api/web/static/favicon.svg +1 -0
- ghost_api/web/static/index.html +26 -0
- ghost_api/workflows.py +203 -0
- ghost_api_cli-0.2.0.dist-info/METADATA +1019 -0
- ghost_api_cli-0.2.0.dist-info/RECORD +125 -0
- ghost_api_cli-0.2.0.dist-info/WHEEL +5 -0
- ghost_api_cli-0.2.0.dist-info/entry_points.txt +2 -0
- ghost_api_cli-0.2.0.dist-info/licenses/LICENSE +21 -0
- ghost_api_cli-0.2.0.dist-info/top_level.txt +1 -0
ghost_api/project.py
ADDED
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
"""A Ghost API project: a `ghost-api.toml` file and the folder it sits in.
|
|
2
|
+
|
|
3
|
+
`ghost-api init` writes one. Commands run anywhere inside that folder (or below it) then use
|
|
4
|
+
the project's database, output folder and defaults, wherever they are started from:
|
|
5
|
+
|
|
6
|
+
[project]
|
|
7
|
+
database = ".ghost-api/ghost_api.db" # relative to this file
|
|
8
|
+
output = "output"
|
|
9
|
+
|
|
10
|
+
[defaults] # any command option, by its name
|
|
11
|
+
lang = "typescript"
|
|
12
|
+
rate_limit = 0.5
|
|
13
|
+
|
|
14
|
+
Per-site rules (a team can write down what is acceptable on each site):
|
|
15
|
+
|
|
16
|
+
[project]
|
|
17
|
+
require_terms = true # every site needs terms_ok = true before the first run
|
|
18
|
+
|
|
19
|
+
[sites."example.com"]
|
|
20
|
+
terms_ok = true # someone checked the site's terms allow this
|
|
21
|
+
deny_paths = ["/admin", "/api/private"] # never requested, by the browser or directly
|
|
22
|
+
max_direct_calls = 30 # endpoint tests and fetches per run
|
|
23
|
+
agent = false # no AI explores this site (Ghost API's or an assistant)
|
|
24
|
+
|
|
25
|
+
Without a project file, Ghost API behaves as before: `ghost_api.db` and `output/` in the
|
|
26
|
+
current folder. `GHOST_API_DB` and command-line options always win.
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
from __future__ import annotations
|
|
30
|
+
|
|
31
|
+
import os
|
|
32
|
+
import tomllib
|
|
33
|
+
from dataclasses import dataclass, field
|
|
34
|
+
from pathlib import Path
|
|
35
|
+
from typing import Any
|
|
36
|
+
|
|
37
|
+
PROJECT_FILE = "ghost-api.toml"
|
|
38
|
+
STATE_FOLDER = ".ghost-api"
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class ProjectError(Exception):
|
|
42
|
+
"""The project file can't be read."""
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
@dataclass(frozen=True)
|
|
46
|
+
class SitePolicy:
|
|
47
|
+
terms_ok: bool = False
|
|
48
|
+
deny_paths: tuple[str, ...] = ()
|
|
49
|
+
max_direct_calls: int | None = None
|
|
50
|
+
agent: bool = True
|
|
51
|
+
|
|
52
|
+
def denies(self, path: str) -> bool:
|
|
53
|
+
return any(path == p or path.startswith(p.rstrip("/") + "/") for p in self.deny_paths)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
@dataclass(frozen=True)
|
|
57
|
+
class Project:
|
|
58
|
+
root: Path
|
|
59
|
+
database: Path
|
|
60
|
+
output: Path
|
|
61
|
+
defaults: dict[str, Any] = field(default_factory=dict)
|
|
62
|
+
sites: dict[str, SitePolicy] = field(default_factory=dict)
|
|
63
|
+
require_terms: bool = False
|
|
64
|
+
|
|
65
|
+
def policy(self, site: str) -> SitePolicy:
|
|
66
|
+
"""The rules for a site: its own entry, else its host's, else none."""
|
|
67
|
+
return self.sites.get(site) or self.sites.get(site.split("/")[0]) or SitePolicy()
|
|
68
|
+
|
|
69
|
+
@property
|
|
70
|
+
def file(self) -> Path:
|
|
71
|
+
return self.root / PROJECT_FILE
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def find_project(start: Path | None = None) -> Project | None:
|
|
75
|
+
"""The nearest `ghost-api.toml` in `start` or a folder above it."""
|
|
76
|
+
if os.getenv("GHOST_API_NO_PROJECT"):
|
|
77
|
+
return None
|
|
78
|
+
here = (start or Path.cwd()).resolve()
|
|
79
|
+
for folder in (here, *here.parents):
|
|
80
|
+
candidate = folder / PROJECT_FILE
|
|
81
|
+
if candidate.is_file():
|
|
82
|
+
return load_project(candidate)
|
|
83
|
+
return None
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def load_project(path: Path) -> Project:
|
|
87
|
+
try:
|
|
88
|
+
data = tomllib.loads(path.read_text(encoding="utf-8"))
|
|
89
|
+
except (OSError, tomllib.TOMLDecodeError) as exc:
|
|
90
|
+
raise ProjectError(f"{path} can't be read: {exc}") from exc
|
|
91
|
+
root = path.parent
|
|
92
|
+
settings = data.get("project", {})
|
|
93
|
+
defaults = data.get("defaults", {})
|
|
94
|
+
if not isinstance(settings, dict) or not isinstance(defaults, dict):
|
|
95
|
+
raise ProjectError(f"{path}: [project] and [defaults] must be tables")
|
|
96
|
+
sites = data.get("sites", {})
|
|
97
|
+
if not isinstance(sites, dict):
|
|
98
|
+
raise ProjectError(f"{path}: [sites] must be a table of sites")
|
|
99
|
+
return Project(
|
|
100
|
+
root=root,
|
|
101
|
+
database=root / str(settings.get("database", f"{STATE_FOLDER}/ghost_api.db")),
|
|
102
|
+
output=root / str(settings.get("output", "output")),
|
|
103
|
+
defaults=dict(defaults),
|
|
104
|
+
sites={name: _policy(path, name, rules) for name, rules in sites.items()},
|
|
105
|
+
require_terms=bool(settings.get("require_terms", False)),
|
|
106
|
+
)
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _policy(path: Path, name: str, rules: Any) -> SitePolicy:
|
|
110
|
+
if not isinstance(rules, dict):
|
|
111
|
+
raise ProjectError(f'{path}: [sites."{name}"] must be a table')
|
|
112
|
+
deny = rules.get("deny_paths", [])
|
|
113
|
+
if not isinstance(deny, list) or not all(isinstance(p, str) and p.startswith("/")
|
|
114
|
+
for p in deny): # fmt: skip
|
|
115
|
+
raise ProjectError(f'{path}: deny_paths for "{name}" must be paths like "/admin"')
|
|
116
|
+
limit = rules.get("max_direct_calls")
|
|
117
|
+
return SitePolicy(
|
|
118
|
+
terms_ok=bool(rules.get("terms_ok", False)),
|
|
119
|
+
deny_paths=tuple(deny),
|
|
120
|
+
max_direct_calls=int(limit) if limit is not None else None,
|
|
121
|
+
agent=bool(rules.get("agent", True)),
|
|
122
|
+
)
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def site_policy(site: str) -> tuple[SitePolicy, bool]:
|
|
126
|
+
"""(the site's rules, whether the project requires terms_ok), from ghost-api.toml."""
|
|
127
|
+
project = find_project()
|
|
128
|
+
if project is None:
|
|
129
|
+
return SitePolicy(), False
|
|
130
|
+
return project.policy(site), project.require_terms
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
TEMPLATE = """\
|
|
134
|
+
# Ghost API project settings. Commands run in this folder (or below it) use them.
|
|
135
|
+
# Docs: https://github.com/Danielraj5605/Ghost-api#project-settings
|
|
136
|
+
|
|
137
|
+
[project]
|
|
138
|
+
database = ".ghost-api/ghost_api.db" # runs, captured traffic, endpoints (keep private)
|
|
139
|
+
output = "output" # reports, clients, downloads
|
|
140
|
+
|
|
141
|
+
[defaults]
|
|
142
|
+
# Any command option, by its name (dashes or underscores). Command-line options win.
|
|
143
|
+
lang = "{lang}"
|
|
144
|
+
rate_limit = 1.0 # requests per second Ghost API sends to a site
|
|
145
|
+
# max_steps = 40 # agent actions per run
|
|
146
|
+
# max_llm_calls = 50
|
|
147
|
+
"""
|
|
148
|
+
|
|
149
|
+
GITIGNORE_LINES = (".env", ".ghost-api/", "output/")
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def write_project(root: Path, lang: str = "python") -> Path:
|
|
153
|
+
path = root / PROJECT_FILE
|
|
154
|
+
path.write_text(TEMPLATE.format(lang=lang), encoding="utf-8")
|
|
155
|
+
(root / STATE_FOLDER).mkdir(exist_ok=True)
|
|
156
|
+
return path
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def ensure_gitignore(root: Path) -> list[str]:
|
|
160
|
+
"""Keep keys and collected data out of git. Returns the lines that were added."""
|
|
161
|
+
path = root / ".gitignore"
|
|
162
|
+
existing = path.read_text(encoding="utf-8").splitlines() if path.exists() else []
|
|
163
|
+
missing = [line for line in GITIGNORE_LINES if line not in existing]
|
|
164
|
+
if missing:
|
|
165
|
+
text = "\n".join(existing).rstrip()
|
|
166
|
+
block = "\n".join(["# Ghost API: keys and collected data", *missing])
|
|
167
|
+
path.write_text(f"{text}\n\n{block}\n" if text else f"{block}\n", encoding="utf-8")
|
|
168
|
+
return missing
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def default_map(project: Project, command: Any) -> dict[str, Any]:
|
|
172
|
+
"""Click `default_map` for every (sub)command: the project's output folder and
|
|
173
|
+
[defaults] for each command that has an option of that name."""
|
|
174
|
+
values = {k.replace("-", "_"): v for k, v in project.defaults.items()}
|
|
175
|
+
values.setdefault("out", str(project.output))
|
|
176
|
+
|
|
177
|
+
def build(cmd: Any) -> dict[str, Any]:
|
|
178
|
+
names = {p.name for p in getattr(cmd, "params", [])}
|
|
179
|
+
result: dict[str, Any] = {k: v for k, v in values.items() if k in names}
|
|
180
|
+
for name, sub in getattr(cmd, "commands", {}).items():
|
|
181
|
+
nested = build(sub)
|
|
182
|
+
if nested:
|
|
183
|
+
result[name] = nested
|
|
184
|
+
return result
|
|
185
|
+
|
|
186
|
+
return build(command)
|
ghost_api/py.typed
ADDED
|
File without changes
|
ghost_api/runtime.py
ADDED
|
@@ -0,0 +1,397 @@
|
|
|
1
|
+
"""Run a site's endpoints by name, from stored data: the base for fetch, watch and serve.
|
|
2
|
+
|
|
3
|
+
A `SiteRuntime` needs no generated code. Each operation replays the endpoint's latest
|
|
4
|
+
stored sample with the caller's parameters, through the same `Replayer` the prober uses
|
|
5
|
+
(robots.txt, rate limit, read-only rule), follows pages, and turns replies into rows.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
from collections.abc import Callable, Iterator
|
|
12
|
+
from dataclasses import dataclass, field, replace
|
|
13
|
+
from typing import Any
|
|
14
|
+
from urllib.parse import parse_qsl, urlsplit
|
|
15
|
+
|
|
16
|
+
from ghost_api.analysis.analyzer import endpoint_keys
|
|
17
|
+
from ghost_api.analysis.detectors import (
|
|
18
|
+
find_items,
|
|
19
|
+
guess_pagination,
|
|
20
|
+
item_fingerprints,
|
|
21
|
+
next_cursor,
|
|
22
|
+
)
|
|
23
|
+
from ghost_api.analysis.models import Endpoint
|
|
24
|
+
from ghost_api.audit import AuditLog
|
|
25
|
+
from ghost_api.browser import auth_state
|
|
26
|
+
from ghost_api.capture import personal
|
|
27
|
+
from ghost_api.capture.filters import REDACTED, is_sensitive_name
|
|
28
|
+
from ghost_api.capture.models import CapturedRequest, RawRequest
|
|
29
|
+
from ghost_api.generator.naming import NamingPlan, apply_plan
|
|
30
|
+
from ghost_api.memory.knowledge import rebuild_raw
|
|
31
|
+
from ghost_api.memory.store import Store
|
|
32
|
+
from ghost_api.net import DenyPaths, RateLimiter, RobotsPolicy, fetch_robots
|
|
33
|
+
from ghost_api.probe.replay import Replayer, ReplayResult, apply_params, is_replayable
|
|
34
|
+
from ghost_api.project import site_policy
|
|
35
|
+
from ghost_api.site import site_host
|
|
36
|
+
|
|
37
|
+
MAX_PAGES = 50
|
|
38
|
+
MAX_ROWS = 10_000
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class OperationError(Exception):
|
|
42
|
+
"""A problem the user can act on (unknown operation, needs a fresh sample...)."""
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
@dataclass
|
|
46
|
+
class Input:
|
|
47
|
+
name: str
|
|
48
|
+
location: str # "path", "query" or "body"
|
|
49
|
+
type: str
|
|
50
|
+
example: str
|
|
51
|
+
required: bool
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
@dataclass
|
|
55
|
+
class Operation:
|
|
56
|
+
name: str
|
|
57
|
+
summary: str
|
|
58
|
+
endpoint: Endpoint
|
|
59
|
+
inputs: list[Input]
|
|
60
|
+
allowed: bool # the user checked it only reads (a POST search, `generate --allow`)
|
|
61
|
+
pagination: dict[str, Any] | None = None
|
|
62
|
+
|
|
63
|
+
@property
|
|
64
|
+
def label(self) -> str:
|
|
65
|
+
return self.endpoint.label
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
@dataclass
|
|
69
|
+
class Page:
|
|
70
|
+
number: int
|
|
71
|
+
result: ReplayResult
|
|
72
|
+
records: list[Any] = field(default_factory=list)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
class SiteRuntime:
|
|
76
|
+
def __init__(
|
|
77
|
+
self,
|
|
78
|
+
store: Store,
|
|
79
|
+
site: str,
|
|
80
|
+
replayer: Replayer | None = None,
|
|
81
|
+
rate: float = 1.0,
|
|
82
|
+
include_sensitive: bool = False,
|
|
83
|
+
) -> None:
|
|
84
|
+
"""`include_sensitive`: return personal data (emails, phones, PAN...) as the site
|
|
85
|
+
sends it. Only for auditing your own site; by default such values are hidden."""
|
|
86
|
+
self.site = site
|
|
87
|
+
self.include_sensitive = include_sensitive
|
|
88
|
+
run_id = store.latest_endpoint_run(site)
|
|
89
|
+
if run_id is None:
|
|
90
|
+
raise OperationError(f"Nothing stored for {site}. Run `ghost-api run` first.")
|
|
91
|
+
self.endpoints = store.current_endpoints(site)
|
|
92
|
+
self.allowed = store.allowed_endpoints(site)
|
|
93
|
+
plan = load_plan(store.load_naming_plan(site))
|
|
94
|
+
_, self.names = apply_plan(plan, self.endpoints, site)
|
|
95
|
+
stored = store.requests_for_domain(site)
|
|
96
|
+
self.samples: dict[str, list[CapturedRequest]] = {}
|
|
97
|
+
for key, req in zip(endpoint_keys(stored), stored, strict=False):
|
|
98
|
+
self.samples.setdefault(key, []).append(req)
|
|
99
|
+
policy, _ = site_policy(site)
|
|
100
|
+
self.replayer = replayer or Replayer(
|
|
101
|
+
site_host(site),
|
|
102
|
+
RateLimiter(rate),
|
|
103
|
+
max_requests=policy.max_direct_calls or 10**6,
|
|
104
|
+
robots_fetcher=_with_rules(policy.deny_paths, site_host(site)),
|
|
105
|
+
)
|
|
106
|
+
if replayer is None: # fetch, watch, serve, MCP: audited like a run's direct calls
|
|
107
|
+
log = AuditLog(store.path, site)
|
|
108
|
+
self.replayer.on_sent = lambda method, url, status, decision: log.record(
|
|
109
|
+
"direct", method, url, status, decision
|
|
110
|
+
)
|
|
111
|
+
try: # ghost-api login: its cookies are added to direct calls
|
|
112
|
+
found = auth_state.load(store.path, site)
|
|
113
|
+
except auth_state.LoginError:
|
|
114
|
+
found = None
|
|
115
|
+
self.login = found[0] if found else None
|
|
116
|
+
|
|
117
|
+
def close(self) -> None:
|
|
118
|
+
self.replayer.close()
|
|
119
|
+
|
|
120
|
+
def __enter__(self) -> SiteRuntime:
|
|
121
|
+
return self
|
|
122
|
+
|
|
123
|
+
def __exit__(self, *exc: object) -> None:
|
|
124
|
+
self.close()
|
|
125
|
+
|
|
126
|
+
# --- what can be called ---
|
|
127
|
+
|
|
128
|
+
def operations(self) -> list[Operation]:
|
|
129
|
+
ops = []
|
|
130
|
+
for ep in self.endpoints:
|
|
131
|
+
allowed = ep.key in self.allowed
|
|
132
|
+
if not is_callable(ep, allowed):
|
|
133
|
+
continue # may change data, or can't be called outside the browser
|
|
134
|
+
names = self.names[ep.key]
|
|
135
|
+
ops.append(
|
|
136
|
+
Operation(
|
|
137
|
+
names.method_name,
|
|
138
|
+
names.summary,
|
|
139
|
+
ep,
|
|
140
|
+
inputs_for(ep, self.samples.get(ep.key, [])),
|
|
141
|
+
allowed,
|
|
142
|
+
ep.pagination or self._guess_pagination(ep),
|
|
143
|
+
)
|
|
144
|
+
)
|
|
145
|
+
return ops
|
|
146
|
+
|
|
147
|
+
def operation(self, name: str) -> Operation:
|
|
148
|
+
ops = self.operations()
|
|
149
|
+
for op in ops:
|
|
150
|
+
if name in (op.name, op.endpoint.key, op.label):
|
|
151
|
+
return op
|
|
152
|
+
choices = ", ".join(o.name for o in ops) or "none"
|
|
153
|
+
raise OperationError(f"No operation {name!r} for {self.site}. Choose one of: {choices}")
|
|
154
|
+
|
|
155
|
+
# --- calling ---
|
|
156
|
+
|
|
157
|
+
def call(self, op: Operation, params: dict[str, str] | None = None) -> ReplayResult:
|
|
158
|
+
sample = self._sample(op)
|
|
159
|
+
values = self._coerce(op, sample, params or {})
|
|
160
|
+
request = apply_params(
|
|
161
|
+
sample, op.endpoint.path_template, {p.name for p in op.endpoint.path_params}, values
|
|
162
|
+
)
|
|
163
|
+
result = self.replayer.send(request, trusted_read=op.allowed)
|
|
164
|
+
if self.include_sensitive or result.body is None:
|
|
165
|
+
return result
|
|
166
|
+
body, hidden = personal.hide(result.body)
|
|
167
|
+
return replace(result, body=body, hidden=personal.summarize(hidden))
|
|
168
|
+
|
|
169
|
+
def iterate(
|
|
170
|
+
self,
|
|
171
|
+
op: Operation,
|
|
172
|
+
params: dict[str, str] | None = None,
|
|
173
|
+
max_pages: int = 5,
|
|
174
|
+
max_rows: int = 1000,
|
|
175
|
+
) -> Iterator[Page]:
|
|
176
|
+
"""Pages of records, following the endpoint's pagination; stops when a page is
|
|
177
|
+
empty, repeats what was seen, or a limit is reached."""
|
|
178
|
+
params = dict(params or {})
|
|
179
|
+
pag = op.pagination
|
|
180
|
+
if pag and pag["style"] != "cursor" and pag["param"] not in params:
|
|
181
|
+
params[pag["param"]] = self._first_position(op, pag) # start at the beginning
|
|
182
|
+
seen: set[str] = set()
|
|
183
|
+
rows = 0
|
|
184
|
+
for number in range(1, min(max_pages, MAX_PAGES) + 1):
|
|
185
|
+
result = self.call(op, params)
|
|
186
|
+
records = records_of(result.body, is_single(op)) if result.ok else []
|
|
187
|
+
fresh = [k for k in item_fingerprints(records) if k not in seen]
|
|
188
|
+
if result.ok and records and not fresh:
|
|
189
|
+
return # the "next page" repeated the last one: pagination ignored
|
|
190
|
+
if result.ok and not records and number > 1:
|
|
191
|
+
return # ran past the last page
|
|
192
|
+
seen.update(fresh)
|
|
193
|
+
records = records[: max(0, min(max_rows, MAX_ROWS) - rows)]
|
|
194
|
+
rows += len(records)
|
|
195
|
+
yield Page(number, result, records)
|
|
196
|
+
if not result.ok or not records or not pag or rows >= min(max_rows, MAX_ROWS):
|
|
197
|
+
return
|
|
198
|
+
if _says_last_page(result.body):
|
|
199
|
+
return
|
|
200
|
+
nxt = _next_position(pag, params, result.body, len(records))
|
|
201
|
+
if nxt is None:
|
|
202
|
+
return
|
|
203
|
+
params[pag["param"]] = nxt
|
|
204
|
+
|
|
205
|
+
# --- helpers ---
|
|
206
|
+
|
|
207
|
+
def _sample(self, op: Operation) -> RawRequest:
|
|
208
|
+
for captured in reversed(self.samples.get(op.endpoint.key, [])):
|
|
209
|
+
raw = rebuild_raw(self._with_login(captured), op.endpoint.required_headers)
|
|
210
|
+
if raw is not None:
|
|
211
|
+
return raw
|
|
212
|
+
raise OperationError(
|
|
213
|
+
f"{op.label} needs a fresh key or session. Run `ghost-api refresh {self.site}` first."
|
|
214
|
+
)
|
|
215
|
+
|
|
216
|
+
def _with_login(self, captured: CapturedRequest) -> CapturedRequest:
|
|
217
|
+
"""The stored sample's redacted cookie, filled in from the saved login (if any)."""
|
|
218
|
+
if self.login is None:
|
|
219
|
+
return captured
|
|
220
|
+
cookies = auth_state.cookie_header(self.login, captured.url)
|
|
221
|
+
headers = dict(captured.req_headers)
|
|
222
|
+
names = [k for k in headers if k.lower() == "cookie"]
|
|
223
|
+
if not cookies or (names and headers[names[0]] != REDACTED):
|
|
224
|
+
return captured
|
|
225
|
+
for name in names:
|
|
226
|
+
del headers[name]
|
|
227
|
+
headers["cookie"] = cookies
|
|
228
|
+
return replace(captured, req_headers=headers)
|
|
229
|
+
|
|
230
|
+
def _guess_pagination(self, ep: Endpoint) -> dict[str, Any] | None:
|
|
231
|
+
samples = self.samples.get(ep.key, [])
|
|
232
|
+
if not samples:
|
|
233
|
+
return None
|
|
234
|
+
last = samples[-1]
|
|
235
|
+
query = dict(parse_qsl(urlsplit(last.url).query, keep_blank_values=True))
|
|
236
|
+
guess = guess_pagination(query, _json_body([last]), _json(last.resp_body))
|
|
237
|
+
if guess is None or (guess.location == "query" and guess.param not in query):
|
|
238
|
+
return None # only follow pages the request really has a parameter for
|
|
239
|
+
return {"style": guess.style, "param": guess.param, "location": guess.location,
|
|
240
|
+
"size_param": guess.size_param} # fmt: skip
|
|
241
|
+
|
|
242
|
+
def _first_position(self, op: Operation, pag: dict[str, Any]) -> str:
|
|
243
|
+
"""The first page (or offset 0): the smallest value seen, so 0-based APIs work."""
|
|
244
|
+
values = []
|
|
245
|
+
for req in self.samples.get(op.endpoint.key, []):
|
|
246
|
+
if pag["location"] == "query":
|
|
247
|
+
value = dict(parse_qsl(urlsplit(req.url).query)).get(pag["param"])
|
|
248
|
+
else:
|
|
249
|
+
value = (_json_body([req]) or {}).get(pag["param"])
|
|
250
|
+
try:
|
|
251
|
+
values.append(int(float(str(value))))
|
|
252
|
+
except ValueError:
|
|
253
|
+
continue
|
|
254
|
+
default = 1 if pag["style"] == "page" else 0
|
|
255
|
+
return str(min(values) if values else default)
|
|
256
|
+
|
|
257
|
+
def _coerce(self, op: Operation, sample: RawRequest, params: dict[str, str]) -> dict[str, str]:
|
|
258
|
+
"""Friendlier inputs: 'football' or 'football,cricket' for a list body field."""
|
|
259
|
+
body = _json(sample.body) if sample.body else None
|
|
260
|
+
out = {}
|
|
261
|
+
for name, value in params.items():
|
|
262
|
+
current = body.get(name) if isinstance(body, dict) else None
|
|
263
|
+
if isinstance(current, list) and not str(value).lstrip().startswith("["):
|
|
264
|
+
value = json.dumps([v.strip() for v in str(value).split(",") if v.strip()])
|
|
265
|
+
out[name] = str(value)
|
|
266
|
+
return out
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
def is_callable(ep: Endpoint, allowed: bool) -> bool:
|
|
270
|
+
"""Ghost API calls an endpoint only if it reads data: by its URL, or because the user
|
|
271
|
+
checked it (`allowed`), and only if it worked outside the browser."""
|
|
272
|
+
if allowed:
|
|
273
|
+
return True
|
|
274
|
+
probe = RawRequest(ep.method, f"https://{ep.host}{ep.path_template}", {}, None)
|
|
275
|
+
return ep.replayable is not False and is_replayable(probe)
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
def inputs_for(ep: Endpoint, samples: list[CapturedRequest]) -> list[Input]:
|
|
279
|
+
"""What a caller can set: path and query parameters, and top-level JSON body fields."""
|
|
280
|
+
inputs = [Input(p.name, "path", p.type, (p.examples or [""])[0], True) for p in ep.path_params]
|
|
281
|
+
inputs += [
|
|
282
|
+
Input(p.name, "query", p.type, (p.examples or [""])[0], p.kind == "required")
|
|
283
|
+
for p in ep.query_params
|
|
284
|
+
if not p.sensitive
|
|
285
|
+
]
|
|
286
|
+
body = _json_body(samples)
|
|
287
|
+
for key, value in (body or {}).items():
|
|
288
|
+
if is_sensitive_name(key):
|
|
289
|
+
continue
|
|
290
|
+
kind = "list" if isinstance(value, list) else type(value).__name__
|
|
291
|
+
example = json.dumps(value) if isinstance(value, list | dict) else str(value)
|
|
292
|
+
inputs.append(Input(key, "body", kind, example, False))
|
|
293
|
+
return inputs
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
_WRAPPERS = ("data", "result", "item", "record", "payload")
|
|
297
|
+
|
|
298
|
+
|
|
299
|
+
def records_of(body: Any, single: bool = False) -> list[Any]:
|
|
300
|
+
"""The rows in a reply: its main list, or the reply itself as one row.
|
|
301
|
+
|
|
302
|
+
`single`: the endpoint returns one thing (its path ends in an ID, like
|
|
303
|
+
/venue/details/{id}), so lists inside it (reviews, photos) are not the rows."""
|
|
304
|
+
if single and isinstance(body, dict):
|
|
305
|
+
inner = next((body[k] for k in _WRAPPERS if isinstance(body.get(k), dict)), None)
|
|
306
|
+
return [inner if inner is not None else body]
|
|
307
|
+
items = find_items(body)
|
|
308
|
+
if items is not None:
|
|
309
|
+
return items
|
|
310
|
+
return [body] if isinstance(body, dict) else []
|
|
311
|
+
|
|
312
|
+
|
|
313
|
+
def is_single(op: Operation) -> bool:
|
|
314
|
+
"""One record per reply: /books/{id}, not /books or /books/{id}/reviews."""
|
|
315
|
+
return op.pagination is None and op.endpoint.path_template.rstrip("/").endswith("}")
|
|
316
|
+
|
|
317
|
+
|
|
318
|
+
def flatten(record: Any, prefix: str = "") -> dict[str, Any]:
|
|
319
|
+
"""{'venue': {'name': 'X'}, 'tags': ['a', 'b']} -> {'venue.name': 'X', 'tags': 'a; b'}."""
|
|
320
|
+
if not isinstance(record, dict):
|
|
321
|
+
return {prefix or "value": record}
|
|
322
|
+
out: dict[str, Any] = {}
|
|
323
|
+
for key, value in record.items():
|
|
324
|
+
name = f"{prefix}.{key}" if prefix else str(key)
|
|
325
|
+
if isinstance(value, dict):
|
|
326
|
+
out.update(flatten(value, name))
|
|
327
|
+
elif isinstance(value, list):
|
|
328
|
+
scalars = all(not isinstance(v, dict | list) for v in value)
|
|
329
|
+
out[name] = "; ".join(str(v) for v in value) if scalars else json.dumps(value)
|
|
330
|
+
else:
|
|
331
|
+
out[name] = value
|
|
332
|
+
return out
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
def _next_position(
|
|
336
|
+
pag: dict[str, Any], params: dict[str, str], body: Any, count: int
|
|
337
|
+
) -> str | None:
|
|
338
|
+
"""The value to send for the next page, counted from what was just sent."""
|
|
339
|
+
if pag["style"] == "cursor":
|
|
340
|
+
return next_cursor(body)
|
|
341
|
+
try:
|
|
342
|
+
position = int(float(params[pag["param"]]))
|
|
343
|
+
except (KeyError, ValueError):
|
|
344
|
+
return None
|
|
345
|
+
if pag["style"] == "page":
|
|
346
|
+
return str(position + 1)
|
|
347
|
+
return str(position + count) # offset: skip what this page returned
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
# Reply fields that say whether another page exists.
|
|
351
|
+
_MORE_FIELDS = ("has_more", "hasMore", "has_next", "hasNext", "more", "has_next_page")
|
|
352
|
+
|
|
353
|
+
|
|
354
|
+
def _says_last_page(body: Any) -> bool:
|
|
355
|
+
if not isinstance(body, dict):
|
|
356
|
+
return False
|
|
357
|
+
for obj in (body, body.get("meta"), body.get("pagination"), body.get("data")):
|
|
358
|
+
if isinstance(obj, dict):
|
|
359
|
+
for key in _MORE_FIELDS:
|
|
360
|
+
if obj.get(key) is False:
|
|
361
|
+
return True
|
|
362
|
+
return False
|
|
363
|
+
|
|
364
|
+
|
|
365
|
+
def _json(text: str | None) -> Any:
|
|
366
|
+
try:
|
|
367
|
+
return json.loads(text or "")
|
|
368
|
+
except ValueError:
|
|
369
|
+
return None
|
|
370
|
+
|
|
371
|
+
|
|
372
|
+
def _json_body(samples: list[CapturedRequest]) -> dict[str, Any] | None:
|
|
373
|
+
for req in reversed(samples):
|
|
374
|
+
data = _json(req.req_body)
|
|
375
|
+
if isinstance(data, dict):
|
|
376
|
+
return data
|
|
377
|
+
return None
|
|
378
|
+
|
|
379
|
+
|
|
380
|
+
def load_plan(stored: str | None) -> NamingPlan | None:
|
|
381
|
+
if not stored:
|
|
382
|
+
return None
|
|
383
|
+
try:
|
|
384
|
+
return NamingPlan.model_validate_json(stored)
|
|
385
|
+
except ValueError:
|
|
386
|
+
return None
|
|
387
|
+
|
|
388
|
+
|
|
389
|
+
def _with_rules(deny: tuple[str, ...], host: str) -> Callable[[str], RobotsPolicy]:
|
|
390
|
+
"""robots.txt for direct calls, plus the project's denied paths on the site's host."""
|
|
391
|
+
|
|
392
|
+
def fetch(url: str) -> RobotsPolicy:
|
|
393
|
+
rules = fetch_robots(url)
|
|
394
|
+
same_host = (urlsplit(url).hostname or "").lower() == host.lower()
|
|
395
|
+
return DenyPaths(rules, deny) if deny and same_host else rules
|
|
396
|
+
|
|
397
|
+
return fetch
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
"""Use sites from other tools: a local REST API (`rest.py`) and an MCP server (`mcp.py`).
|
|
2
|
+
|
|
3
|
+
Both serve the operations of `SiteRuntime`: read-only, rate-limited, robots.txt respected,
|
|
4
|
+
personal data hidden. One runtime per site, used one call at a time, so a busy client
|
|
5
|
+
can't make Ghost API hammer a site.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import re
|
|
11
|
+
import threading
|
|
12
|
+
from dataclasses import dataclass, field
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from typing import Any
|
|
15
|
+
|
|
16
|
+
from ghost_api.memory.store import Store
|
|
17
|
+
from ghost_api.runtime import Operation, OperationError, SiteRuntime
|
|
18
|
+
|
|
19
|
+
__all__ = [
|
|
20
|
+
"MAX_PAGES",
|
|
21
|
+
"ServedSite",
|
|
22
|
+
"coerce",
|
|
23
|
+
"load_sites",
|
|
24
|
+
"tool_name",
|
|
25
|
+
]
|
|
26
|
+
|
|
27
|
+
MAX_PAGES = 20
|
|
28
|
+
_COMMON_LABELS = {"www", "com", "in", "org", "net", "io", "co"}
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@dataclass
|
|
32
|
+
class ServedSite:
|
|
33
|
+
site: str
|
|
34
|
+
runtime: SiteRuntime
|
|
35
|
+
operations: list[Operation]
|
|
36
|
+
lock: threading.Lock = field(default_factory=threading.Lock)
|
|
37
|
+
|
|
38
|
+
def fetch(
|
|
39
|
+
self, op: Operation, params: dict[str, str], pages: int, max_rows: int
|
|
40
|
+
) -> dict[str, Any]:
|
|
41
|
+
"""All rows for these inputs (one call at a time per site)."""
|
|
42
|
+
records: list[Any] = []
|
|
43
|
+
hidden = 0
|
|
44
|
+
read = 0
|
|
45
|
+
error = ""
|
|
46
|
+
with self.lock:
|
|
47
|
+
for page in self.runtime.iterate(op, params, min(pages, MAX_PAGES), max_rows):
|
|
48
|
+
read += 1
|
|
49
|
+
if not page.result.ok:
|
|
50
|
+
error = page.result.error or f"HTTP {page.result.status}"
|
|
51
|
+
break
|
|
52
|
+
records += page.records
|
|
53
|
+
hidden += sum(int(h.get("count", 1)) for h in page.result.hidden)
|
|
54
|
+
return {
|
|
55
|
+
"site": self.site,
|
|
56
|
+
"operation": op.name,
|
|
57
|
+
"rows": len(records),
|
|
58
|
+
"pages": read,
|
|
59
|
+
"personal_values_hidden": hidden,
|
|
60
|
+
"error": error,
|
|
61
|
+
"data": records,
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def load_sites(db: Path, names: list[str] | None = None) -> dict[str, ServedSite]:
|
|
66
|
+
"""The sites to serve: the ones named, or every stored site with something callable."""
|
|
67
|
+
with Store(db) as store:
|
|
68
|
+
sites = names or sorted({r.domain for r in store.list_runs()})
|
|
69
|
+
served = {}
|
|
70
|
+
for site in sites:
|
|
71
|
+
try:
|
|
72
|
+
rt = SiteRuntime(store, site)
|
|
73
|
+
except OperationError:
|
|
74
|
+
if names:
|
|
75
|
+
raise
|
|
76
|
+
continue
|
|
77
|
+
ops = rt.operations()
|
|
78
|
+
if ops:
|
|
79
|
+
served[site] = ServedSite(site, rt, ops)
|
|
80
|
+
else:
|
|
81
|
+
rt.close()
|
|
82
|
+
return served
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def tool_name(site: str, op: Operation) -> str:
|
|
86
|
+
"""'turftown.in/chennai' + get_nearby_venues -> 'turftown_chennai_get_nearby_venues'."""
|
|
87
|
+
host, _, section = site.partition("/")
|
|
88
|
+
labels = [p for p in host.lower().split(".") if p not in _COMMON_LABELS]
|
|
89
|
+
slug = "_".join([*labels[:2], section] if section else labels[:2]) or "site"
|
|
90
|
+
return re.sub(r"[^a-zA-Z0-9_-]", "_", f"{slug}_{op.name}")[:64]
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def coerce(value: Any) -> str:
|
|
94
|
+
"""Inputs arrive as JSON values (MCP) or text (REST); runtime takes text."""
|
|
95
|
+
if isinstance(value, list | dict):
|
|
96
|
+
import json
|
|
97
|
+
|
|
98
|
+
return json.dumps(value)
|
|
99
|
+
if isinstance(value, bool):
|
|
100
|
+
return "true" if value else "false"
|
|
101
|
+
return str(value)
|