ghost-api-cli 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. ghost_api/__init__.py +3 -0
  2. ghost_api/__main__.py +5 -0
  3. ghost_api/adopt.py +173 -0
  4. ghost_api/agent/__init__.py +0 -0
  5. ghost_api/agent/orchestrator.py +413 -0
  6. ghost_api/agent/prompts.py +149 -0
  7. ghost_api/agent/tools.py +84 -0
  8. ghost_api/analysis/__init__.py +0 -0
  9. ghost_api/analysis/analyzer.py +130 -0
  10. ghost_api/analysis/detectors.py +188 -0
  11. ghost_api/analysis/exposure.py +80 -0
  12. ghost_api/analysis/graphql.py +67 -0
  13. ghost_api/analysis/models.py +64 -0
  14. ghost_api/analysis/schema.py +109 -0
  15. ghost_api/analysis/url_grouping.py +217 -0
  16. ghost_api/api.py +264 -0
  17. ghost_api/audit.py +78 -0
  18. ghost_api/browser/__init__.py +0 -0
  19. ghost_api/browser/auth_state.py +172 -0
  20. ghost_api/browser/driver.py +273 -0
  21. ghost_api/browser/egress.py +183 -0
  22. ghost_api/browser/gate.py +127 -0
  23. ghost_api/browser/install.py +28 -0
  24. ghost_api/browser/snapshot.py +221 -0
  25. ghost_api/capture/__init__.py +0 -0
  26. ghost_api/capture/filters.py +255 -0
  27. ghost_api/capture/har.py +148 -0
  28. ghost_api/capture/models.py +41 -0
  29. ghost_api/capture/personal.py +188 -0
  30. ghost_api/capture/recorder.py +181 -0
  31. ghost_api/cli/__init__.py +44 -0
  32. ghost_api/cli/_common.py +219 -0
  33. ghost_api/cli/client.py +487 -0
  34. ghost_api/cli/data.py +492 -0
  35. ghost_api/cli/explore.py +379 -0
  36. ghost_api/cli/manage.py +437 -0
  37. ghost_api/config.py +132 -0
  38. ghost_api/drift.py +344 -0
  39. ghost_api/drive.py +211 -0
  40. ghost_api/errors.py +106 -0
  41. ghost_api/events.py +213 -0
  42. ghost_api/exitcodes.py +21 -0
  43. ghost_api/export/__init__.py +71 -0
  44. ghost_api/export/confidence.py +45 -0
  45. ghost_api/export/html.py +41 -0
  46. ghost_api/export/openapi.py +201 -0
  47. ghost_api/export/report.py +227 -0
  48. ghost_api/export/site_data.py +196 -0
  49. ghost_api/export/tabular.py +88 -0
  50. ghost_api/export/viewer/report.html +286 -0
  51. ghost_api/generator/__init__.py +221 -0
  52. ghost_api/generator/cassettes.py +80 -0
  53. ghost_api/generator/models_gen.py +144 -0
  54. ghost_api/generator/naming.py +174 -0
  55. ghost_api/generator/package.py +197 -0
  56. ghost_api/generator/python_gen.py +172 -0
  57. ghost_api/generator/spec.py +486 -0
  58. ghost_api/generator/templates/client.py.j2 +518 -0
  59. ghost_api/generator/templates/client.test.ts.j2 +106 -0
  60. ghost_api/generator/templates/client.ts.j2 +339 -0
  61. ghost_api/generator/templates/models.py.j2 +45 -0
  62. ghost_api/generator/templates/package_readme.md.j2 +111 -0
  63. ghost_api/generator/templates/pyproject.toml.j2 +22 -0
  64. ghost_api/generator/templates/test_client.py.j2 +105 -0
  65. ghost_api/generator/ts_gen.py +232 -0
  66. ghost_api/integrations/__init__.py +0 -0
  67. ghost_api/integrations/claude_code/commands/ghost-data.md +10 -0
  68. ghost_api/integrations/claude_code/commands/ghost-diff.md +10 -0
  69. ghost_api/integrations/claude_code/commands/ghost-explore.md +10 -0
  70. ghost_api/integrations/claude_code/skills/ghost-api/SKILL.md +46 -0
  71. ghost_api/integrations/setup.py +84 -0
  72. ghost_api/llm/__init__.py +0 -0
  73. ghost_api/llm/doctor.py +60 -0
  74. ghost_api/llm/gateway.py +476 -0
  75. ghost_api/llm/providers.py +124 -0
  76. ghost_api/memory/__init__.py +0 -0
  77. ghost_api/memory/knowledge.py +157 -0
  78. ghost_api/memory/store.py +883 -0
  79. ghost_api/net.py +235 -0
  80. ghost_api/netguard.py +203 -0
  81. ghost_api/pipeline.py +745 -0
  82. ghost_api/probe/__init__.py +0 -0
  83. ghost_api/probe/prober.py +311 -0
  84. ghost_api/probe/replay.py +313 -0
  85. ghost_api/project.py +186 -0
  86. ghost_api/py.typed +0 -0
  87. ghost_api/runtime.py +397 -0
  88. ghost_api/serve/__init__.py +101 -0
  89. ghost_api/serve/explorer_mcp.py +305 -0
  90. ghost_api/serve/mcp.py +121 -0
  91. ghost_api/serve/mock.py +130 -0
  92. ghost_api/serve/rest.py +132 -0
  93. ghost_api/site.py +84 -0
  94. ghost_api/specdiff.py +122 -0
  95. ghost_api/ui/__init__.py +0 -0
  96. ghost_api/ui/keys.py +134 -0
  97. ghost_api/ui/repl.py +453 -0
  98. ghost_api/ui/terminal.py +573 -0
  99. ghost_api/verify/__init__.py +0 -0
  100. ghost_api/verify/classify.py +84 -0
  101. ghost_api/verify/repair.py +40 -0
  102. ghost_api/verify/runner.py +110 -0
  103. ghost_api/verify/verifier.py +291 -0
  104. ghost_api/watch.py +313 -0
  105. ghost_api/web/__init__.py +1 -0
  106. ghost_api/web/server.py +690 -0
  107. ghost_api/web/static/assets/Endpoints-zPnjlcM8.js +46 -0
  108. ghost_api/web/static/assets/Run-D85zWHey.js +91 -0
  109. ghost_api/web/static/assets/Settings-D0aWHFAt.js +11 -0
  110. ghost_api/web/static/assets/Site-a88shEmP.js +71 -0
  111. ghost_api/web/static/assets/Watches-CqZWIa9G.js +16 -0
  112. ghost_api/web/static/assets/circle-x-BcTE_1tr.js +11 -0
  113. ghost_api/web/static/assets/index-Ds2e5B9f.js +141 -0
  114. ghost_api/web/static/assets/index-xQpyIBbG.css +1 -0
  115. ghost_api/web/static/assets/minus-C2xqP1eQ.js +6 -0
  116. ghost_api/web/static/assets/refresh-cw-CxdR_8Oi.js +6 -0
  117. ghost_api/web/static/favicon.svg +1 -0
  118. ghost_api/web/static/index.html +26 -0
  119. ghost_api/workflows.py +203 -0
  120. ghost_api_cli-0.2.0.dist-info/METADATA +1019 -0
  121. ghost_api_cli-0.2.0.dist-info/RECORD +125 -0
  122. ghost_api_cli-0.2.0.dist-info/WHEEL +5 -0
  123. ghost_api_cli-0.2.0.dist-info/entry_points.txt +2 -0
  124. ghost_api_cli-0.2.0.dist-info/licenses/LICENSE +21 -0
  125. ghost_api_cli-0.2.0.dist-info/top_level.txt +1 -0
ghost_api/project.py ADDED
@@ -0,0 +1,186 @@
1
+ """A Ghost API project: a `ghost-api.toml` file and the folder it sits in.
2
+
3
+ `ghost-api init` writes one. Commands run anywhere inside that folder (or below it) then use
4
+ the project's database, output folder and defaults, wherever they are started from:
5
+
6
+ [project]
7
+ database = ".ghost-api/ghost_api.db" # relative to this file
8
+ output = "output"
9
+
10
+ [defaults] # any command option, by its name
11
+ lang = "typescript"
12
+ rate_limit = 0.5
13
+
14
+ Per-site rules (a team can write down what is acceptable on each site):
15
+
16
+ [project]
17
+ require_terms = true # every site needs terms_ok = true before the first run
18
+
19
+ [sites."example.com"]
20
+ terms_ok = true # someone checked the site's terms allow this
21
+ deny_paths = ["/admin", "/api/private"] # never requested, by the browser or directly
22
+ max_direct_calls = 30 # endpoint tests and fetches per run
23
+ agent = false # no AI explores this site (Ghost API's or an assistant)
24
+
25
+ Without a project file, Ghost API behaves as before: `ghost_api.db` and `output/` in the
26
+ current folder. `GHOST_API_DB` and command-line options always win.
27
+ """
28
+
29
+ from __future__ import annotations
30
+
31
+ import os
32
+ import tomllib
33
+ from dataclasses import dataclass, field
34
+ from pathlib import Path
35
+ from typing import Any
36
+
37
+ PROJECT_FILE = "ghost-api.toml"
38
+ STATE_FOLDER = ".ghost-api"
39
+
40
+
41
+ class ProjectError(Exception):
42
+ """The project file can't be read."""
43
+
44
+
45
+ @dataclass(frozen=True)
46
+ class SitePolicy:
47
+ terms_ok: bool = False
48
+ deny_paths: tuple[str, ...] = ()
49
+ max_direct_calls: int | None = None
50
+ agent: bool = True
51
+
52
+ def denies(self, path: str) -> bool:
53
+ return any(path == p or path.startswith(p.rstrip("/") + "/") for p in self.deny_paths)
54
+
55
+
56
+ @dataclass(frozen=True)
57
+ class Project:
58
+ root: Path
59
+ database: Path
60
+ output: Path
61
+ defaults: dict[str, Any] = field(default_factory=dict)
62
+ sites: dict[str, SitePolicy] = field(default_factory=dict)
63
+ require_terms: bool = False
64
+
65
+ def policy(self, site: str) -> SitePolicy:
66
+ """The rules for a site: its own entry, else its host's, else none."""
67
+ return self.sites.get(site) or self.sites.get(site.split("/")[0]) or SitePolicy()
68
+
69
+ @property
70
+ def file(self) -> Path:
71
+ return self.root / PROJECT_FILE
72
+
73
+
74
+ def find_project(start: Path | None = None) -> Project | None:
75
+ """The nearest `ghost-api.toml` in `start` or a folder above it."""
76
+ if os.getenv("GHOST_API_NO_PROJECT"):
77
+ return None
78
+ here = (start or Path.cwd()).resolve()
79
+ for folder in (here, *here.parents):
80
+ candidate = folder / PROJECT_FILE
81
+ if candidate.is_file():
82
+ return load_project(candidate)
83
+ return None
84
+
85
+
86
+ def load_project(path: Path) -> Project:
87
+ try:
88
+ data = tomllib.loads(path.read_text(encoding="utf-8"))
89
+ except (OSError, tomllib.TOMLDecodeError) as exc:
90
+ raise ProjectError(f"{path} can't be read: {exc}") from exc
91
+ root = path.parent
92
+ settings = data.get("project", {})
93
+ defaults = data.get("defaults", {})
94
+ if not isinstance(settings, dict) or not isinstance(defaults, dict):
95
+ raise ProjectError(f"{path}: [project] and [defaults] must be tables")
96
+ sites = data.get("sites", {})
97
+ if not isinstance(sites, dict):
98
+ raise ProjectError(f"{path}: [sites] must be a table of sites")
99
+ return Project(
100
+ root=root,
101
+ database=root / str(settings.get("database", f"{STATE_FOLDER}/ghost_api.db")),
102
+ output=root / str(settings.get("output", "output")),
103
+ defaults=dict(defaults),
104
+ sites={name: _policy(path, name, rules) for name, rules in sites.items()},
105
+ require_terms=bool(settings.get("require_terms", False)),
106
+ )
107
+
108
+
109
+ def _policy(path: Path, name: str, rules: Any) -> SitePolicy:
110
+ if not isinstance(rules, dict):
111
+ raise ProjectError(f'{path}: [sites."{name}"] must be a table')
112
+ deny = rules.get("deny_paths", [])
113
+ if not isinstance(deny, list) or not all(isinstance(p, str) and p.startswith("/")
114
+ for p in deny): # fmt: skip
115
+ raise ProjectError(f'{path}: deny_paths for "{name}" must be paths like "/admin"')
116
+ limit = rules.get("max_direct_calls")
117
+ return SitePolicy(
118
+ terms_ok=bool(rules.get("terms_ok", False)),
119
+ deny_paths=tuple(deny),
120
+ max_direct_calls=int(limit) if limit is not None else None,
121
+ agent=bool(rules.get("agent", True)),
122
+ )
123
+
124
+
125
+ def site_policy(site: str) -> tuple[SitePolicy, bool]:
126
+ """(the site's rules, whether the project requires terms_ok), from ghost-api.toml."""
127
+ project = find_project()
128
+ if project is None:
129
+ return SitePolicy(), False
130
+ return project.policy(site), project.require_terms
131
+
132
+
133
+ TEMPLATE = """\
134
+ # Ghost API project settings. Commands run in this folder (or below it) use them.
135
+ # Docs: https://github.com/Danielraj5605/Ghost-api#project-settings
136
+
137
+ [project]
138
+ database = ".ghost-api/ghost_api.db" # runs, captured traffic, endpoints (keep private)
139
+ output = "output" # reports, clients, downloads
140
+
141
+ [defaults]
142
+ # Any command option, by its name (dashes or underscores). Command-line options win.
143
+ lang = "{lang}"
144
+ rate_limit = 1.0 # requests per second Ghost API sends to a site
145
+ # max_steps = 40 # agent actions per run
146
+ # max_llm_calls = 50
147
+ """
148
+
149
+ GITIGNORE_LINES = (".env", ".ghost-api/", "output/")
150
+
151
+
152
+ def write_project(root: Path, lang: str = "python") -> Path:
153
+ path = root / PROJECT_FILE
154
+ path.write_text(TEMPLATE.format(lang=lang), encoding="utf-8")
155
+ (root / STATE_FOLDER).mkdir(exist_ok=True)
156
+ return path
157
+
158
+
159
+ def ensure_gitignore(root: Path) -> list[str]:
160
+ """Keep keys and collected data out of git. Returns the lines that were added."""
161
+ path = root / ".gitignore"
162
+ existing = path.read_text(encoding="utf-8").splitlines() if path.exists() else []
163
+ missing = [line for line in GITIGNORE_LINES if line not in existing]
164
+ if missing:
165
+ text = "\n".join(existing).rstrip()
166
+ block = "\n".join(["# Ghost API: keys and collected data", *missing])
167
+ path.write_text(f"{text}\n\n{block}\n" if text else f"{block}\n", encoding="utf-8")
168
+ return missing
169
+
170
+
171
+ def default_map(project: Project, command: Any) -> dict[str, Any]:
172
+ """Click `default_map` for every (sub)command: the project's output folder and
173
+ [defaults] for each command that has an option of that name."""
174
+ values = {k.replace("-", "_"): v for k, v in project.defaults.items()}
175
+ values.setdefault("out", str(project.output))
176
+
177
+ def build(cmd: Any) -> dict[str, Any]:
178
+ names = {p.name for p in getattr(cmd, "params", [])}
179
+ result: dict[str, Any] = {k: v for k, v in values.items() if k in names}
180
+ for name, sub in getattr(cmd, "commands", {}).items():
181
+ nested = build(sub)
182
+ if nested:
183
+ result[name] = nested
184
+ return result
185
+
186
+ return build(command)
ghost_api/py.typed ADDED
File without changes
ghost_api/runtime.py ADDED
@@ -0,0 +1,397 @@
1
+ """Run a site's endpoints by name, from stored data: the base for fetch, watch and serve.
2
+
3
+ A `SiteRuntime` needs no generated code. Each operation replays the endpoint's latest
4
+ stored sample with the caller's parameters, through the same `Replayer` the prober uses
5
+ (robots.txt, rate limit, read-only rule), follows pages, and turns replies into rows.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import json
11
+ from collections.abc import Callable, Iterator
12
+ from dataclasses import dataclass, field, replace
13
+ from typing import Any
14
+ from urllib.parse import parse_qsl, urlsplit
15
+
16
+ from ghost_api.analysis.analyzer import endpoint_keys
17
+ from ghost_api.analysis.detectors import (
18
+ find_items,
19
+ guess_pagination,
20
+ item_fingerprints,
21
+ next_cursor,
22
+ )
23
+ from ghost_api.analysis.models import Endpoint
24
+ from ghost_api.audit import AuditLog
25
+ from ghost_api.browser import auth_state
26
+ from ghost_api.capture import personal
27
+ from ghost_api.capture.filters import REDACTED, is_sensitive_name
28
+ from ghost_api.capture.models import CapturedRequest, RawRequest
29
+ from ghost_api.generator.naming import NamingPlan, apply_plan
30
+ from ghost_api.memory.knowledge import rebuild_raw
31
+ from ghost_api.memory.store import Store
32
+ from ghost_api.net import DenyPaths, RateLimiter, RobotsPolicy, fetch_robots
33
+ from ghost_api.probe.replay import Replayer, ReplayResult, apply_params, is_replayable
34
+ from ghost_api.project import site_policy
35
+ from ghost_api.site import site_host
36
+
37
+ MAX_PAGES = 50
38
+ MAX_ROWS = 10_000
39
+
40
+
41
+ class OperationError(Exception):
42
+ """A problem the user can act on (unknown operation, needs a fresh sample...)."""
43
+
44
+
45
+ @dataclass
46
+ class Input:
47
+ name: str
48
+ location: str # "path", "query" or "body"
49
+ type: str
50
+ example: str
51
+ required: bool
52
+
53
+
54
+ @dataclass
55
+ class Operation:
56
+ name: str
57
+ summary: str
58
+ endpoint: Endpoint
59
+ inputs: list[Input]
60
+ allowed: bool # the user checked it only reads (a POST search, `generate --allow`)
61
+ pagination: dict[str, Any] | None = None
62
+
63
+ @property
64
+ def label(self) -> str:
65
+ return self.endpoint.label
66
+
67
+
68
+ @dataclass
69
+ class Page:
70
+ number: int
71
+ result: ReplayResult
72
+ records: list[Any] = field(default_factory=list)
73
+
74
+
75
+ class SiteRuntime:
76
+ def __init__(
77
+ self,
78
+ store: Store,
79
+ site: str,
80
+ replayer: Replayer | None = None,
81
+ rate: float = 1.0,
82
+ include_sensitive: bool = False,
83
+ ) -> None:
84
+ """`include_sensitive`: return personal data (emails, phones, PAN...) as the site
85
+ sends it. Only for auditing your own site; by default such values are hidden."""
86
+ self.site = site
87
+ self.include_sensitive = include_sensitive
88
+ run_id = store.latest_endpoint_run(site)
89
+ if run_id is None:
90
+ raise OperationError(f"Nothing stored for {site}. Run `ghost-api run` first.")
91
+ self.endpoints = store.current_endpoints(site)
92
+ self.allowed = store.allowed_endpoints(site)
93
+ plan = load_plan(store.load_naming_plan(site))
94
+ _, self.names = apply_plan(plan, self.endpoints, site)
95
+ stored = store.requests_for_domain(site)
96
+ self.samples: dict[str, list[CapturedRequest]] = {}
97
+ for key, req in zip(endpoint_keys(stored), stored, strict=False):
98
+ self.samples.setdefault(key, []).append(req)
99
+ policy, _ = site_policy(site)
100
+ self.replayer = replayer or Replayer(
101
+ site_host(site),
102
+ RateLimiter(rate),
103
+ max_requests=policy.max_direct_calls or 10**6,
104
+ robots_fetcher=_with_rules(policy.deny_paths, site_host(site)),
105
+ )
106
+ if replayer is None: # fetch, watch, serve, MCP: audited like a run's direct calls
107
+ log = AuditLog(store.path, site)
108
+ self.replayer.on_sent = lambda method, url, status, decision: log.record(
109
+ "direct", method, url, status, decision
110
+ )
111
+ try: # ghost-api login: its cookies are added to direct calls
112
+ found = auth_state.load(store.path, site)
113
+ except auth_state.LoginError:
114
+ found = None
115
+ self.login = found[0] if found else None
116
+
117
+ def close(self) -> None:
118
+ self.replayer.close()
119
+
120
+ def __enter__(self) -> SiteRuntime:
121
+ return self
122
+
123
+ def __exit__(self, *exc: object) -> None:
124
+ self.close()
125
+
126
+ # --- what can be called ---
127
+
128
+ def operations(self) -> list[Operation]:
129
+ ops = []
130
+ for ep in self.endpoints:
131
+ allowed = ep.key in self.allowed
132
+ if not is_callable(ep, allowed):
133
+ continue # may change data, or can't be called outside the browser
134
+ names = self.names[ep.key]
135
+ ops.append(
136
+ Operation(
137
+ names.method_name,
138
+ names.summary,
139
+ ep,
140
+ inputs_for(ep, self.samples.get(ep.key, [])),
141
+ allowed,
142
+ ep.pagination or self._guess_pagination(ep),
143
+ )
144
+ )
145
+ return ops
146
+
147
+ def operation(self, name: str) -> Operation:
148
+ ops = self.operations()
149
+ for op in ops:
150
+ if name in (op.name, op.endpoint.key, op.label):
151
+ return op
152
+ choices = ", ".join(o.name for o in ops) or "none"
153
+ raise OperationError(f"No operation {name!r} for {self.site}. Choose one of: {choices}")
154
+
155
+ # --- calling ---
156
+
157
+ def call(self, op: Operation, params: dict[str, str] | None = None) -> ReplayResult:
158
+ sample = self._sample(op)
159
+ values = self._coerce(op, sample, params or {})
160
+ request = apply_params(
161
+ sample, op.endpoint.path_template, {p.name for p in op.endpoint.path_params}, values
162
+ )
163
+ result = self.replayer.send(request, trusted_read=op.allowed)
164
+ if self.include_sensitive or result.body is None:
165
+ return result
166
+ body, hidden = personal.hide(result.body)
167
+ return replace(result, body=body, hidden=personal.summarize(hidden))
168
+
169
+ def iterate(
170
+ self,
171
+ op: Operation,
172
+ params: dict[str, str] | None = None,
173
+ max_pages: int = 5,
174
+ max_rows: int = 1000,
175
+ ) -> Iterator[Page]:
176
+ """Pages of records, following the endpoint's pagination; stops when a page is
177
+ empty, repeats what was seen, or a limit is reached."""
178
+ params = dict(params or {})
179
+ pag = op.pagination
180
+ if pag and pag["style"] != "cursor" and pag["param"] not in params:
181
+ params[pag["param"]] = self._first_position(op, pag) # start at the beginning
182
+ seen: set[str] = set()
183
+ rows = 0
184
+ for number in range(1, min(max_pages, MAX_PAGES) + 1):
185
+ result = self.call(op, params)
186
+ records = records_of(result.body, is_single(op)) if result.ok else []
187
+ fresh = [k for k in item_fingerprints(records) if k not in seen]
188
+ if result.ok and records and not fresh:
189
+ return # the "next page" repeated the last one: pagination ignored
190
+ if result.ok and not records and number > 1:
191
+ return # ran past the last page
192
+ seen.update(fresh)
193
+ records = records[: max(0, min(max_rows, MAX_ROWS) - rows)]
194
+ rows += len(records)
195
+ yield Page(number, result, records)
196
+ if not result.ok or not records or not pag or rows >= min(max_rows, MAX_ROWS):
197
+ return
198
+ if _says_last_page(result.body):
199
+ return
200
+ nxt = _next_position(pag, params, result.body, len(records))
201
+ if nxt is None:
202
+ return
203
+ params[pag["param"]] = nxt
204
+
205
+ # --- helpers ---
206
+
207
+ def _sample(self, op: Operation) -> RawRequest:
208
+ for captured in reversed(self.samples.get(op.endpoint.key, [])):
209
+ raw = rebuild_raw(self._with_login(captured), op.endpoint.required_headers)
210
+ if raw is not None:
211
+ return raw
212
+ raise OperationError(
213
+ f"{op.label} needs a fresh key or session. Run `ghost-api refresh {self.site}` first."
214
+ )
215
+
216
+ def _with_login(self, captured: CapturedRequest) -> CapturedRequest:
217
+ """The stored sample's redacted cookie, filled in from the saved login (if any)."""
218
+ if self.login is None:
219
+ return captured
220
+ cookies = auth_state.cookie_header(self.login, captured.url)
221
+ headers = dict(captured.req_headers)
222
+ names = [k for k in headers if k.lower() == "cookie"]
223
+ if not cookies or (names and headers[names[0]] != REDACTED):
224
+ return captured
225
+ for name in names:
226
+ del headers[name]
227
+ headers["cookie"] = cookies
228
+ return replace(captured, req_headers=headers)
229
+
230
+ def _guess_pagination(self, ep: Endpoint) -> dict[str, Any] | None:
231
+ samples = self.samples.get(ep.key, [])
232
+ if not samples:
233
+ return None
234
+ last = samples[-1]
235
+ query = dict(parse_qsl(urlsplit(last.url).query, keep_blank_values=True))
236
+ guess = guess_pagination(query, _json_body([last]), _json(last.resp_body))
237
+ if guess is None or (guess.location == "query" and guess.param not in query):
238
+ return None # only follow pages the request really has a parameter for
239
+ return {"style": guess.style, "param": guess.param, "location": guess.location,
240
+ "size_param": guess.size_param} # fmt: skip
241
+
242
+ def _first_position(self, op: Operation, pag: dict[str, Any]) -> str:
243
+ """The first page (or offset 0): the smallest value seen, so 0-based APIs work."""
244
+ values = []
245
+ for req in self.samples.get(op.endpoint.key, []):
246
+ if pag["location"] == "query":
247
+ value = dict(parse_qsl(urlsplit(req.url).query)).get(pag["param"])
248
+ else:
249
+ value = (_json_body([req]) or {}).get(pag["param"])
250
+ try:
251
+ values.append(int(float(str(value))))
252
+ except ValueError:
253
+ continue
254
+ default = 1 if pag["style"] == "page" else 0
255
+ return str(min(values) if values else default)
256
+
257
+ def _coerce(self, op: Operation, sample: RawRequest, params: dict[str, str]) -> dict[str, str]:
258
+ """Friendlier inputs: 'football' or 'football,cricket' for a list body field."""
259
+ body = _json(sample.body) if sample.body else None
260
+ out = {}
261
+ for name, value in params.items():
262
+ current = body.get(name) if isinstance(body, dict) else None
263
+ if isinstance(current, list) and not str(value).lstrip().startswith("["):
264
+ value = json.dumps([v.strip() for v in str(value).split(",") if v.strip()])
265
+ out[name] = str(value)
266
+ return out
267
+
268
+
269
+ def is_callable(ep: Endpoint, allowed: bool) -> bool:
270
+ """Ghost API calls an endpoint only if it reads data: by its URL, or because the user
271
+ checked it (`allowed`), and only if it worked outside the browser."""
272
+ if allowed:
273
+ return True
274
+ probe = RawRequest(ep.method, f"https://{ep.host}{ep.path_template}", {}, None)
275
+ return ep.replayable is not False and is_replayable(probe)
276
+
277
+
278
+ def inputs_for(ep: Endpoint, samples: list[CapturedRequest]) -> list[Input]:
279
+ """What a caller can set: path and query parameters, and top-level JSON body fields."""
280
+ inputs = [Input(p.name, "path", p.type, (p.examples or [""])[0], True) for p in ep.path_params]
281
+ inputs += [
282
+ Input(p.name, "query", p.type, (p.examples or [""])[0], p.kind == "required")
283
+ for p in ep.query_params
284
+ if not p.sensitive
285
+ ]
286
+ body = _json_body(samples)
287
+ for key, value in (body or {}).items():
288
+ if is_sensitive_name(key):
289
+ continue
290
+ kind = "list" if isinstance(value, list) else type(value).__name__
291
+ example = json.dumps(value) if isinstance(value, list | dict) else str(value)
292
+ inputs.append(Input(key, "body", kind, example, False))
293
+ return inputs
294
+
295
+
296
+ _WRAPPERS = ("data", "result", "item", "record", "payload")
297
+
298
+
299
+ def records_of(body: Any, single: bool = False) -> list[Any]:
300
+ """The rows in a reply: its main list, or the reply itself as one row.
301
+
302
+ `single`: the endpoint returns one thing (its path ends in an ID, like
303
+ /venue/details/{id}), so lists inside it (reviews, photos) are not the rows."""
304
+ if single and isinstance(body, dict):
305
+ inner = next((body[k] for k in _WRAPPERS if isinstance(body.get(k), dict)), None)
306
+ return [inner if inner is not None else body]
307
+ items = find_items(body)
308
+ if items is not None:
309
+ return items
310
+ return [body] if isinstance(body, dict) else []
311
+
312
+
313
+ def is_single(op: Operation) -> bool:
314
+ """One record per reply: /books/{id}, not /books or /books/{id}/reviews."""
315
+ return op.pagination is None and op.endpoint.path_template.rstrip("/").endswith("}")
316
+
317
+
318
+ def flatten(record: Any, prefix: str = "") -> dict[str, Any]:
319
+ """{'venue': {'name': 'X'}, 'tags': ['a', 'b']} -> {'venue.name': 'X', 'tags': 'a; b'}."""
320
+ if not isinstance(record, dict):
321
+ return {prefix or "value": record}
322
+ out: dict[str, Any] = {}
323
+ for key, value in record.items():
324
+ name = f"{prefix}.{key}" if prefix else str(key)
325
+ if isinstance(value, dict):
326
+ out.update(flatten(value, name))
327
+ elif isinstance(value, list):
328
+ scalars = all(not isinstance(v, dict | list) for v in value)
329
+ out[name] = "; ".join(str(v) for v in value) if scalars else json.dumps(value)
330
+ else:
331
+ out[name] = value
332
+ return out
333
+
334
+
335
+ def _next_position(
336
+ pag: dict[str, Any], params: dict[str, str], body: Any, count: int
337
+ ) -> str | None:
338
+ """The value to send for the next page, counted from what was just sent."""
339
+ if pag["style"] == "cursor":
340
+ return next_cursor(body)
341
+ try:
342
+ position = int(float(params[pag["param"]]))
343
+ except (KeyError, ValueError):
344
+ return None
345
+ if pag["style"] == "page":
346
+ return str(position + 1)
347
+ return str(position + count) # offset: skip what this page returned
348
+
349
+
350
+ # Reply fields that say whether another page exists.
351
+ _MORE_FIELDS = ("has_more", "hasMore", "has_next", "hasNext", "more", "has_next_page")
352
+
353
+
354
+ def _says_last_page(body: Any) -> bool:
355
+ if not isinstance(body, dict):
356
+ return False
357
+ for obj in (body, body.get("meta"), body.get("pagination"), body.get("data")):
358
+ if isinstance(obj, dict):
359
+ for key in _MORE_FIELDS:
360
+ if obj.get(key) is False:
361
+ return True
362
+ return False
363
+
364
+
365
+ def _json(text: str | None) -> Any:
366
+ try:
367
+ return json.loads(text or "")
368
+ except ValueError:
369
+ return None
370
+
371
+
372
+ def _json_body(samples: list[CapturedRequest]) -> dict[str, Any] | None:
373
+ for req in reversed(samples):
374
+ data = _json(req.req_body)
375
+ if isinstance(data, dict):
376
+ return data
377
+ return None
378
+
379
+
380
+ def load_plan(stored: str | None) -> NamingPlan | None:
381
+ if not stored:
382
+ return None
383
+ try:
384
+ return NamingPlan.model_validate_json(stored)
385
+ except ValueError:
386
+ return None
387
+
388
+
389
+ def _with_rules(deny: tuple[str, ...], host: str) -> Callable[[str], RobotsPolicy]:
390
+ """robots.txt for direct calls, plus the project's denied paths on the site's host."""
391
+
392
+ def fetch(url: str) -> RobotsPolicy:
393
+ rules = fetch_robots(url)
394
+ same_host = (urlsplit(url).hostname or "").lower() == host.lower()
395
+ return DenyPaths(rules, deny) if deny and same_host else rules
396
+
397
+ return fetch
@@ -0,0 +1,101 @@
1
+ """Use sites from other tools: a local REST API (`rest.py`) and an MCP server (`mcp.py`).
2
+
3
+ Both serve the operations of `SiteRuntime`: read-only, rate-limited, robots.txt respected,
4
+ personal data hidden. One runtime per site, used one call at a time, so a busy client
5
+ can't make Ghost API hammer a site.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import re
11
+ import threading
12
+ from dataclasses import dataclass, field
13
+ from pathlib import Path
14
+ from typing import Any
15
+
16
+ from ghost_api.memory.store import Store
17
+ from ghost_api.runtime import Operation, OperationError, SiteRuntime
18
+
19
+ __all__ = [
20
+ "MAX_PAGES",
21
+ "ServedSite",
22
+ "coerce",
23
+ "load_sites",
24
+ "tool_name",
25
+ ]
26
+
27
+ MAX_PAGES = 20
28
+ _COMMON_LABELS = {"www", "com", "in", "org", "net", "io", "co"}
29
+
30
+
31
+ @dataclass
32
+ class ServedSite:
33
+ site: str
34
+ runtime: SiteRuntime
35
+ operations: list[Operation]
36
+ lock: threading.Lock = field(default_factory=threading.Lock)
37
+
38
+ def fetch(
39
+ self, op: Operation, params: dict[str, str], pages: int, max_rows: int
40
+ ) -> dict[str, Any]:
41
+ """All rows for these inputs (one call at a time per site)."""
42
+ records: list[Any] = []
43
+ hidden = 0
44
+ read = 0
45
+ error = ""
46
+ with self.lock:
47
+ for page in self.runtime.iterate(op, params, min(pages, MAX_PAGES), max_rows):
48
+ read += 1
49
+ if not page.result.ok:
50
+ error = page.result.error or f"HTTP {page.result.status}"
51
+ break
52
+ records += page.records
53
+ hidden += sum(int(h.get("count", 1)) for h in page.result.hidden)
54
+ return {
55
+ "site": self.site,
56
+ "operation": op.name,
57
+ "rows": len(records),
58
+ "pages": read,
59
+ "personal_values_hidden": hidden,
60
+ "error": error,
61
+ "data": records,
62
+ }
63
+
64
+
65
+ def load_sites(db: Path, names: list[str] | None = None) -> dict[str, ServedSite]:
66
+ """The sites to serve: the ones named, or every stored site with something callable."""
67
+ with Store(db) as store:
68
+ sites = names or sorted({r.domain for r in store.list_runs()})
69
+ served = {}
70
+ for site in sites:
71
+ try:
72
+ rt = SiteRuntime(store, site)
73
+ except OperationError:
74
+ if names:
75
+ raise
76
+ continue
77
+ ops = rt.operations()
78
+ if ops:
79
+ served[site] = ServedSite(site, rt, ops)
80
+ else:
81
+ rt.close()
82
+ return served
83
+
84
+
85
+ def tool_name(site: str, op: Operation) -> str:
86
+ """'turftown.in/chennai' + get_nearby_venues -> 'turftown_chennai_get_nearby_venues'."""
87
+ host, _, section = site.partition("/")
88
+ labels = [p for p in host.lower().split(".") if p not in _COMMON_LABELS]
89
+ slug = "_".join([*labels[:2], section] if section else labels[:2]) or "site"
90
+ return re.sub(r"[^a-zA-Z0-9_-]", "_", f"{slug}_{op.name}")[:64]
91
+
92
+
93
+ def coerce(value: Any) -> str:
94
+ """Inputs arrive as JSON values (MCP) or text (REST); runtime takes text."""
95
+ if isinstance(value, list | dict):
96
+ import json
97
+
98
+ return json.dumps(value)
99
+ if isinstance(value, bool):
100
+ return "true" if value else "false"
101
+ return str(value)