openconstruction-tools 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- openconstruction_mcp/__init__.py +5 -0
- openconstruction_mcp/acquisition.py +221 -0
- openconstruction_mcp/catalog.py +327 -0
- openconstruction_mcp/cli.py +242 -0
- openconstruction_mcp/config.py +82 -0
- openconstruction_mcp/downloads.py +1179 -0
- openconstruction_mcp/provider_auth.py +206 -0
- openconstruction_mcp/provider_downloads.py +89 -0
- openconstruction_mcp/providers.py +180 -0
- openconstruction_mcp/research.py +156 -0
- openconstruction_mcp/security.py +114 -0
- openconstruction_mcp/server.py +377 -0
- openconstruction_mcp/service.py +344 -0
- openconstruction_mcp/skills.py +49 -0
- openconstruction_mcp/store.py +345 -0
- openconstruction_mcp/web.py +452 -0
- openconstruction_mcp/workflows.py +319 -0
- openconstruction_tools-0.1.0.data/data/share/openconstruction/README.md +61 -0
- openconstruction_tools-0.1.0.data/data/share/openconstruction/docs/catalog.md +27 -0
- openconstruction_tools-0.1.0.data/data/share/openconstruction/docs/development.md +42 -0
- openconstruction_tools-0.1.0.data/data/share/openconstruction/docs/hosting.md +76 -0
- openconstruction_tools-0.1.0.data/data/share/openconstruction/docs/mcp.md +58 -0
- openconstruction_tools-0.1.0.data/data/share/openconstruction/docs/releasing.md +51 -0
- openconstruction_tools-0.1.0.data/data/share/openconstruction/docs/usage.md +192 -0
- openconstruction_tools-0.1.0.data/data/share/openconstruction/skills/dataset-discovery/SKILL.md +40 -0
- openconstruction_tools-0.1.0.data/data/share/openconstruction/skills/dataset-discovery/examples/hardhat-detection.json +16 -0
- openconstruction_tools-0.1.0.data/data/share/openconstruction/skills/dataset-discovery/metadata.json +129 -0
- openconstruction_tools-0.1.0.data/data/share/openconstruction/skills/dataset-discovery/tests/dataset-discovery-smoke.json +15 -0
- openconstruction_tools-0.1.0.data/data/share/openconstruction/skills/index.json +137 -0
- openconstruction_tools-0.1.0.dist-info/METADATA +85 -0
- openconstruction_tools-0.1.0.dist-info/RECORD +36 -0
- openconstruction_tools-0.1.0.dist-info/WHEEL +5 -0
- openconstruction_tools-0.1.0.dist-info/entry_points.txt +4 -0
- openconstruction_tools-0.1.0.dist-info/licenses/LICENSE +201 -0
- openconstruction_tools-0.1.0.dist-info/licenses/NOTICE +8 -0
- openconstruction_tools-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,242 @@
|
|
|
1
|
+
"""Command-line adapter for the same operations exposed through MCP."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import argparse
|
|
5
|
+
import json
|
|
6
|
+
import sys
|
|
7
|
+
import time
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
from .catalog import CATALOG_SOURCES, CatalogClient
|
|
12
|
+
from .downloads import DownloadManager
|
|
13
|
+
from .service import OpenConstructionService
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
RESOURCE_TYPES = [kind for kind, _ in CATALOG_SOURCES]
|
|
17
|
+
ACTIVE_STATUSES = {"queued", "resolving", "downloading", "verifying", "preparing_bundle"}
|
|
18
|
+
ACTION_REQUIRED = {"license_acceptance_required", "auth_required", "instructions_required",
|
|
19
|
+
"source_changed", "already_running", "interrupted"}
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def positive_limit(value: str) -> int:
|
|
23
|
+
number = int(value)
|
|
24
|
+
if not 1 <= number <= 100:
|
|
25
|
+
raise argparse.ArgumentTypeError("limit must be between 1 and 100")
|
|
26
|
+
return number
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def parser() -> argparse.ArgumentParser:
|
|
30
|
+
common = argparse.ArgumentParser(add_help=False)
|
|
31
|
+
common.add_argument("--json", action="store_true", default=argparse.SUPPRESS,
|
|
32
|
+
help="Emit one JSON result to stdout; progress goes to stderr")
|
|
33
|
+
common.add_argument("--data-base-url", default=argparse.SUPPRESS,
|
|
34
|
+
help="Catalog directory URL (overrides OPENCONSTRUCTION_DATA_BASE_URL)")
|
|
35
|
+
common.add_argument("--download-root", default=argparse.SUPPRESS,
|
|
36
|
+
help="Local dataset directory (overrides OC_DOWNLOAD_ROOT)")
|
|
37
|
+
root = argparse.ArgumentParser(prog="openconstruction", parents=[common])
|
|
38
|
+
commands = root.add_subparsers(dest="command", required=True)
|
|
39
|
+
search = commands.add_parser("search", parents=[common], help="Search catalog records")
|
|
40
|
+
search.add_argument("query")
|
|
41
|
+
search.add_argument("--type", choices=RESOURCE_TYPES, action="append", dest="types")
|
|
42
|
+
search.add_argument("--limit", type=positive_limit, default=20)
|
|
43
|
+
show = commands.add_parser("show", aliases=["get"], parents=[common], help="Retrieve an exact resource ID")
|
|
44
|
+
show.add_argument("type", choices=RESOURCE_TYPES)
|
|
45
|
+
show.add_argument("id")
|
|
46
|
+
compare = commands.add_parser("compare", parents=[common], help="Compare reported metadata")
|
|
47
|
+
compare.add_argument("ids", nargs="+")
|
|
48
|
+
compare.add_argument("--type", choices=RESOURCE_TYPES, default="dataset")
|
|
49
|
+
compare.add_argument("--goal", default="")
|
|
50
|
+
discover = commands.add_parser("discover", parents=[common], help="Rank datasets with explicit constraint checks")
|
|
51
|
+
for field in ("task", "modality", "object_class", "annotation", "license", "access_requirement"):
|
|
52
|
+
discover.add_argument("--" + field.replace("_", "-"), default="")
|
|
53
|
+
discover.add_argument("--limit", type=positive_limit, default=8)
|
|
54
|
+
for name in ("plan", "download", "prepare"):
|
|
55
|
+
sub = commands.add_parser(name, parents=[common], help={
|
|
56
|
+
"plan": "Inspect the dataset download plan without writing files",
|
|
57
|
+
"download": "Download a dataset and wait for completion",
|
|
58
|
+
"prepare": "Prepare a dataset research bundle and wait for completion",
|
|
59
|
+
}[name])
|
|
60
|
+
sub.add_argument("dataset_id")
|
|
61
|
+
if name != "plan":
|
|
62
|
+
sub.add_argument("--accept-license", action="store_true",
|
|
63
|
+
help="Confirm that you have reviewed and accepted the dataset terms")
|
|
64
|
+
if name == "download":
|
|
65
|
+
sub.add_argument("--destination", help="One directory name within the download root")
|
|
66
|
+
status = commands.add_parser("status", parents=[common], help="Read a saved download or research job")
|
|
67
|
+
status.add_argument("job_id")
|
|
68
|
+
commands.add_parser("stats", parents=[common], help="Show catalog counts and provenance")
|
|
69
|
+
export = commands.add_parser("export", parents=[common], help="Export selected records and snapshot provenance")
|
|
70
|
+
export.add_argument("ids", nargs="+")
|
|
71
|
+
export.add_argument("--type", choices=RESOURCE_TYPES, default="dataset")
|
|
72
|
+
export.add_argument("--output", type=Path, required=True)
|
|
73
|
+
return root
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def request(args: argparse.Namespace) -> tuple[str, dict[str, Any]]:
|
|
77
|
+
if args.command == "search":
|
|
78
|
+
return "search_resources", {"query": args.query, "types": args.types, "limit": args.limit}
|
|
79
|
+
if args.command in {"show", "get"}:
|
|
80
|
+
return "get_resource", {"type": args.type, "id": args.id}
|
|
81
|
+
if args.command == "compare":
|
|
82
|
+
if len(args.ids) < 2:
|
|
83
|
+
raise ValueError("compare requires at least two resource IDs")
|
|
84
|
+
return "compare_resources", {"resources": [{"type": args.type, "id": value} for value in args.ids], "goal": args.goal}
|
|
85
|
+
if args.command == "discover":
|
|
86
|
+
return "run_dataset_discovery", {key: getattr(args, key) for key in (
|
|
87
|
+
"task", "modality", "object_class", "annotation", "license", "access_requirement", "limit")}
|
|
88
|
+
if args.command == "stats":
|
|
89
|
+
return "get_catalog_stats", {}
|
|
90
|
+
if args.command == "status":
|
|
91
|
+
if args.job_id.startswith("rp_"):
|
|
92
|
+
return "get_research_preparation_status", {"preparation_id": args.job_id}
|
|
93
|
+
return "get_download_status", {"download_id": args.job_id}
|
|
94
|
+
if args.command == "prepare":
|
|
95
|
+
return "prepare_dataset_for_research", {"dataset_id": args.dataset_id, "accept_license": args.accept_license}
|
|
96
|
+
if args.command in {"plan", "download"}:
|
|
97
|
+
payload = {"dataset_id": args.dataset_id}
|
|
98
|
+
if args.command == "download":
|
|
99
|
+
payload.update(accept_license=args.accept_license, destination=args.destination)
|
|
100
|
+
return "get_dataset_download_plan" if args.command == "plan" else "download_dataset", payload
|
|
101
|
+
raise ValueError(f"Unknown command: {args.command}")
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def wait_for_job(service: OpenConstructionService, result: dict[str, Any]) -> dict[str, Any]:
|
|
105
|
+
job_id = result.get("preparation_id") or result.get("download_id")
|
|
106
|
+
if not job_id:
|
|
107
|
+
return result
|
|
108
|
+
research = "preparation_id" in result
|
|
109
|
+
operation = "get_research_preparation_status" if research else "get_download_status"
|
|
110
|
+
key = "preparation_id" if research else "download_id"
|
|
111
|
+
last_progress = None
|
|
112
|
+
try:
|
|
113
|
+
while result.get("status") in ACTIVE_STATUSES:
|
|
114
|
+
progress = result.get("progress_text") or result.get("user_message") or result["status"]
|
|
115
|
+
if progress != last_progress:
|
|
116
|
+
print(progress, file=sys.stderr)
|
|
117
|
+
last_progress = progress
|
|
118
|
+
time.sleep(0.2)
|
|
119
|
+
result = service.call(operation, {key: job_id})
|
|
120
|
+
service.download_manager.wait(job_id)
|
|
121
|
+
except KeyboardInterrupt:
|
|
122
|
+
if not result.get("already_running"):
|
|
123
|
+
service.call("cancel_download", {"download_id": job_id})
|
|
124
|
+
# Give the worker time to close streams and persist the checkpoint.
|
|
125
|
+
deadline = time.monotonic() + 3
|
|
126
|
+
while time.monotonic() < deadline:
|
|
127
|
+
if service.call("get_download_status", {"download_id": job_id}).get("status") not in ACTIVE_STATUSES:
|
|
128
|
+
break
|
|
129
|
+
time.sleep(0.1)
|
|
130
|
+
service.download_manager.wait(job_id, timeout=3)
|
|
131
|
+
raise
|
|
132
|
+
return result
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def exit_code(result: dict[str, Any]) -> int:
|
|
136
|
+
if result.get("status") in ACTION_REQUIRED:
|
|
137
|
+
return 3
|
|
138
|
+
if result.get("error") or result.get("status") in {"failed", "cancelled"}:
|
|
139
|
+
return 1
|
|
140
|
+
if "resource" in result and result["resource"] is None:
|
|
141
|
+
return 1
|
|
142
|
+
if result.get("missing_resources"):
|
|
143
|
+
return 1
|
|
144
|
+
if result.get("catalog_provenance", {}).get("complete") is False:
|
|
145
|
+
return 1
|
|
146
|
+
return 0
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def emit(result: dict[str, Any], as_json: bool) -> None:
|
|
150
|
+
if as_json:
|
|
151
|
+
print(json.dumps(result, ensure_ascii=True))
|
|
152
|
+
return
|
|
153
|
+
provenance = result.get("catalog_provenance")
|
|
154
|
+
if provenance:
|
|
155
|
+
print(f"Catalog: {provenance['source']}\nSnapshot: {provenance['snapshot_sha256']}")
|
|
156
|
+
if not provenance["complete"]:
|
|
157
|
+
print("Catalog is incomplete; inspect load_errors in --json output.", file=sys.stderr)
|
|
158
|
+
if "resources" in result and "comparison" not in result:
|
|
159
|
+
for item in result["resources"]:
|
|
160
|
+
print(f"{item['type']}\t{item['id']}\t{item['title']}")
|
|
161
|
+
if not result["resources"]:
|
|
162
|
+
print("No matching resources.")
|
|
163
|
+
else:
|
|
164
|
+
print(json.dumps({key: value for key, value in result.items() if key != "catalog_provenance"},
|
|
165
|
+
ensure_ascii=True, indent=2))
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def confirm_download_license(result: dict[str, Any]) -> bool:
|
|
169
|
+
"""A terminal equivalent of the website review; EOF and anything but yes decline."""
|
|
170
|
+
plan = result.get("plan") or {}
|
|
171
|
+
review = result.get("license_review") or {}
|
|
172
|
+
# Strip terminal control characters from catalog text before displaying it.
|
|
173
|
+
def text(value: Any) -> str:
|
|
174
|
+
return "".join(char for char in str(value or "Not specified") if char.isprintable())
|
|
175
|
+
print("\nReview dataset license", file=sys.stderr)
|
|
176
|
+
print("Dataset: " + text(plan.get("dataset_name") or plan.get("dataset_id")), file=sys.stderr)
|
|
177
|
+
print("License: " + text(review.get("dataset_license") or plan.get("license")), file=sys.stderr)
|
|
178
|
+
print("Source terms: " + text(review.get("dataset_source") or plan.get("url")), file=sys.stderr)
|
|
179
|
+
if review.get("license_url"):
|
|
180
|
+
print("License terms: " + text(review["license_url"]), file=sys.stderr)
|
|
181
|
+
if review.get("review_url"):
|
|
182
|
+
print("Review details: " + text(review["review_url"]), file=sys.stderr)
|
|
183
|
+
print(text(review.get("notice") or "Review the source license before proceeding. A missing license does not grant permission."), file=sys.stderr)
|
|
184
|
+
print("Have you reviewed the terms and confirmed permission to download? [y/N] ", end="", file=sys.stderr, flush=True)
|
|
185
|
+
try:
|
|
186
|
+
answer = input()
|
|
187
|
+
except EOFError:
|
|
188
|
+
return False
|
|
189
|
+
return answer.strip().lower() in {"y", "yes"}
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def main(argv: list[str] | None = None, *, service: OpenConstructionService | None = None) -> int:
|
|
193
|
+
args = parser().parse_args(argv)
|
|
194
|
+
as_json = getattr(args, "json", False)
|
|
195
|
+
try:
|
|
196
|
+
if service is None:
|
|
197
|
+
catalog = CatalogClient()
|
|
198
|
+
if hasattr(args, "data_base_url"):
|
|
199
|
+
catalog.data_base_url = args.data_base_url.rstrip("/")
|
|
200
|
+
service = OpenConstructionService(catalog=catalog,
|
|
201
|
+
download_manager=DownloadManager(root=getattr(args, "download_root", None)))
|
|
202
|
+
if args.command == "export":
|
|
203
|
+
records = [service.catalog.get(args.type, value) for value in args.ids]
|
|
204
|
+
missing = [value for value, record in zip(args.ids, records) if record is None]
|
|
205
|
+
result = {"schema_version": "1.0", "resources": [record for record in records if record],
|
|
206
|
+
"missing_resources": missing, "catalog_provenance": service.catalog.provenance()}
|
|
207
|
+
if missing or not result["catalog_provenance"]["complete"]:
|
|
208
|
+
emit(result, as_json)
|
|
209
|
+
return 1
|
|
210
|
+
args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
211
|
+
result["output"] = str(args.output.resolve())
|
|
212
|
+
else:
|
|
213
|
+
operation, payload = request(args)
|
|
214
|
+
if args.command in {"download", "prepare"} and not args.accept_license and not as_json and sys.stdin.isatty():
|
|
215
|
+
review = service.call("get_dataset_download_plan", {"dataset_id": args.dataset_id})
|
|
216
|
+
if not confirm_download_license(review):
|
|
217
|
+
emit({"status": "license_acceptance_required", "message": "No download started. License confirmation was not given."}, as_json)
|
|
218
|
+
return 3
|
|
219
|
+
args.accept_license = True
|
|
220
|
+
payload["accept_license"] = True
|
|
221
|
+
if args.command == "download" and not args.accept_license:
|
|
222
|
+
# Display a useful read-only plan instead of starting a worker.
|
|
223
|
+
result = service.call("get_dataset_download_plan", {
|
|
224
|
+
"dataset_id": args.dataset_id})
|
|
225
|
+
result.update(status="license_acceptance_required",
|
|
226
|
+
next_step="Review the plan, then repeat with --accept-license.")
|
|
227
|
+
else:
|
|
228
|
+
result = service.call(operation, payload)
|
|
229
|
+
if args.command in {"download", "prepare"}:
|
|
230
|
+
result = wait_for_job(service, result)
|
|
231
|
+
emit(result, as_json)
|
|
232
|
+
return exit_code(result)
|
|
233
|
+
except KeyboardInterrupt:
|
|
234
|
+
emit({"status": "interrupted", "message": "Stopped waiting; repeat the command to resume saved progress."}, as_json)
|
|
235
|
+
return 130
|
|
236
|
+
except Exception as exc:
|
|
237
|
+
emit({"error": str(exc)}, as_json)
|
|
238
|
+
return 1
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
if __name__ == "__main__":
|
|
242
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from urllib.parse import urlparse
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def _clean_url(value: str) -> str:
|
|
10
|
+
return value.strip().rstrip("/")
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@dataclass(frozen=True)
|
|
14
|
+
class Settings:
|
|
15
|
+
public_url: str = "http://127.0.0.1:8000"
|
|
16
|
+
site_url: str = "http://127.0.0.1:8000"
|
|
17
|
+
database_path: str = ".openconstruction/state.db"
|
|
18
|
+
supabase_url: str = ""
|
|
19
|
+
supabase_anon_key: str = ""
|
|
20
|
+
token_encryption_key: str = ""
|
|
21
|
+
environment: str = "development"
|
|
22
|
+
dev_user_token: str = ""
|
|
23
|
+
dev_user_id: str = ""
|
|
24
|
+
static_dir: str = ""
|
|
25
|
+
cors_origins: tuple[str, ...] = ()
|
|
26
|
+
connected_accounts_enabled: bool = False
|
|
27
|
+
|
|
28
|
+
@property
|
|
29
|
+
def issuer(self) -> str:
|
|
30
|
+
return self.public_url
|
|
31
|
+
|
|
32
|
+
@property
|
|
33
|
+
def mcp_resource(self) -> str:
|
|
34
|
+
return f"{self.public_url}/mcp"
|
|
35
|
+
|
|
36
|
+
@classmethod
|
|
37
|
+
def from_env(cls) -> "Settings":
|
|
38
|
+
public_url = _clean_url(os.getenv("OC_PUBLIC_URL", "http://127.0.0.1:8000"))
|
|
39
|
+
site_url = _clean_url(os.getenv("OC_SITE_URL", public_url))
|
|
40
|
+
origins = tuple(
|
|
41
|
+
origin.strip().rstrip("/")
|
|
42
|
+
for origin in os.getenv("OC_CORS_ORIGINS", site_url).split(",")
|
|
43
|
+
if origin.strip()
|
|
44
|
+
)
|
|
45
|
+
return cls(
|
|
46
|
+
public_url=public_url,
|
|
47
|
+
site_url=site_url,
|
|
48
|
+
database_path=os.getenv("OC_AUTH_DB", ".openconstruction/state.db"),
|
|
49
|
+
supabase_url=_clean_url(os.getenv("SUPABASE_URL", "")),
|
|
50
|
+
supabase_anon_key=os.getenv("SUPABASE_ANON_KEY", "").strip(),
|
|
51
|
+
token_encryption_key=os.getenv("OC_TOKEN_ENCRYPTION_KEY", "").strip(),
|
|
52
|
+
environment=os.getenv("OC_ENV", "development").strip().lower(),
|
|
53
|
+
dev_user_token=os.getenv("OC_DEV_USER_TOKEN", "").strip(),
|
|
54
|
+
dev_user_id=os.getenv("OC_DEV_USER_ID", "").strip(),
|
|
55
|
+
static_dir=os.getenv("OC_STATIC_DIR", "").strip(),
|
|
56
|
+
cors_origins=origins,
|
|
57
|
+
connected_accounts_enabled=os.getenv("OC_CONNECTED_ACCOUNTS_ENABLED", "false").strip().lower()
|
|
58
|
+
in {"1", "true", "yes", "on"},
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
def ensure_state_parent(self) -> None:
|
|
62
|
+
if self.database_path == ":memory:":
|
|
63
|
+
return
|
|
64
|
+
Path(self.database_path).expanduser().resolve().parent.mkdir(parents=True, exist_ok=True)
|
|
65
|
+
|
|
66
|
+
def validate(self) -> None:
|
|
67
|
+
if self.environment != "production":
|
|
68
|
+
return
|
|
69
|
+
for name, value in (("OC_PUBLIC_URL", self.public_url), ("OC_SITE_URL", self.site_url),
|
|
70
|
+
("SUPABASE_URL", self.supabase_url)):
|
|
71
|
+
parsed = urlparse(value)
|
|
72
|
+
if (parsed.scheme != "https" or not parsed.hostname or parsed.username or parsed.password
|
|
73
|
+
or parsed.query or parsed.fragment or parsed.path not in {"", "/"}):
|
|
74
|
+
raise ValueError(f"{name} must be an HTTPS origin in production")
|
|
75
|
+
if not self.supabase_anon_key:
|
|
76
|
+
raise ValueError("SUPABASE_ANON_KEY is required in production")
|
|
77
|
+
if self.dev_user_token or self.dev_user_id:
|
|
78
|
+
raise ValueError("Development login settings must be unset in production")
|
|
79
|
+
if not self.database_path or self.database_path == ":memory:":
|
|
80
|
+
raise ValueError("OC_AUTH_DB must point to persistent storage in production")
|
|
81
|
+
if "*" in self.cors_origins:
|
|
82
|
+
raise ValueError("OC_CORS_ORIGINS must list explicit origins in production")
|