openconstruction-tools 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- openconstruction_mcp/__init__.py +5 -0
- openconstruction_mcp/acquisition.py +221 -0
- openconstruction_mcp/catalog.py +327 -0
- openconstruction_mcp/cli.py +242 -0
- openconstruction_mcp/config.py +82 -0
- openconstruction_mcp/downloads.py +1179 -0
- openconstruction_mcp/provider_auth.py +206 -0
- openconstruction_mcp/provider_downloads.py +89 -0
- openconstruction_mcp/providers.py +180 -0
- openconstruction_mcp/research.py +156 -0
- openconstruction_mcp/security.py +114 -0
- openconstruction_mcp/server.py +377 -0
- openconstruction_mcp/service.py +344 -0
- openconstruction_mcp/skills.py +49 -0
- openconstruction_mcp/store.py +345 -0
- openconstruction_mcp/web.py +452 -0
- openconstruction_mcp/workflows.py +319 -0
- openconstruction_tools-0.1.0.data/data/share/openconstruction/README.md +61 -0
- openconstruction_tools-0.1.0.data/data/share/openconstruction/docs/catalog.md +27 -0
- openconstruction_tools-0.1.0.data/data/share/openconstruction/docs/development.md +42 -0
- openconstruction_tools-0.1.0.data/data/share/openconstruction/docs/hosting.md +76 -0
- openconstruction_tools-0.1.0.data/data/share/openconstruction/docs/mcp.md +58 -0
- openconstruction_tools-0.1.0.data/data/share/openconstruction/docs/releasing.md +51 -0
- openconstruction_tools-0.1.0.data/data/share/openconstruction/docs/usage.md +192 -0
- openconstruction_tools-0.1.0.data/data/share/openconstruction/skills/dataset-discovery/SKILL.md +40 -0
- openconstruction_tools-0.1.0.data/data/share/openconstruction/skills/dataset-discovery/examples/hardhat-detection.json +16 -0
- openconstruction_tools-0.1.0.data/data/share/openconstruction/skills/dataset-discovery/metadata.json +129 -0
- openconstruction_tools-0.1.0.data/data/share/openconstruction/skills/dataset-discovery/tests/dataset-discovery-smoke.json +15 -0
- openconstruction_tools-0.1.0.data/data/share/openconstruction/skills/index.json +137 -0
- openconstruction_tools-0.1.0.dist-info/METADATA +85 -0
- openconstruction_tools-0.1.0.dist-info/RECORD +36 -0
- openconstruction_tools-0.1.0.dist-info/WHEEL +5 -0
- openconstruction_tools-0.1.0.dist-info/entry_points.txt +4 -0
- openconstruction_tools-0.1.0.dist-info/licenses/LICENSE +201 -0
- openconstruction_tools-0.1.0.dist-info/licenses/NOTICE +8 -0
- openconstruction_tools-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
import shlex
|
|
5
|
+
import urllib.parse
|
|
6
|
+
from dataclasses import asdict, dataclass, field
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
from .catalog import CatalogClient, clean_text
|
|
10
|
+
from .provider_auth import provider_auth_status
|
|
11
|
+
from .provider_downloads import discover_public_files
|
|
12
|
+
import httpx
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
AUTOMATED_METHODS = {
|
|
16
|
+
"github_clone",
|
|
17
|
+
"huggingface_snapshot",
|
|
18
|
+
"figshare_files",
|
|
19
|
+
"http_files",
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _http_url(value: Any) -> str | None:
|
|
24
|
+
text = clean_text(value, 1000)
|
|
25
|
+
if not text:
|
|
26
|
+
return None
|
|
27
|
+
try:
|
|
28
|
+
parsed = urllib.parse.urlparse(text)
|
|
29
|
+
except ValueError:
|
|
30
|
+
return None
|
|
31
|
+
if parsed.scheme not in {"http", "https"} or not parsed.hostname:
|
|
32
|
+
return None
|
|
33
|
+
if parsed.username or parsed.password:
|
|
34
|
+
return None
|
|
35
|
+
return text
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _objects(value: Any) -> list[dict[str, Any]]:
|
|
39
|
+
if not isinstance(value, list):
|
|
40
|
+
return []
|
|
41
|
+
return [item for item in value if isinstance(item, dict)]
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _safe_dirname(value: str) -> str:
|
|
45
|
+
safe = re.sub(r"[^A-Za-z0-9._-]+", "-", value.strip()).strip("-.")
|
|
46
|
+
return safe or "dataset"
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
@dataclass(frozen=True)
|
|
50
|
+
class AcquisitionPlan:
|
|
51
|
+
dataset_id: str
|
|
52
|
+
dataset_name: str
|
|
53
|
+
kind: str
|
|
54
|
+
provider: str | None = None
|
|
55
|
+
method: str | None = None
|
|
56
|
+
url: str | None = None
|
|
57
|
+
filename: str | None = None
|
|
58
|
+
license: str | None = None
|
|
59
|
+
requires_auth: bool = False
|
|
60
|
+
executable_locally: bool = False
|
|
61
|
+
estimated_size: int | None = None
|
|
62
|
+
distributions: list[dict[str, Any]] = field(default_factory=list)
|
|
63
|
+
programmatic_access: dict[str, Any] | None = None
|
|
64
|
+
instructions: dict[str, Any] | None = None
|
|
65
|
+
auth: dict[str, Any] | None = None
|
|
66
|
+
warnings: list[str] = field(default_factory=list)
|
|
67
|
+
|
|
68
|
+
def to_dict(self) -> dict[str, Any]:
|
|
69
|
+
return asdict(self)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def get_exact_dataset(catalog: CatalogClient, dataset_id: str) -> dict[str, Any]:
|
|
73
|
+
requested = clean_text(dataset_id, 180).casefold()
|
|
74
|
+
if not requested:
|
|
75
|
+
raise ValueError("dataset_id is required")
|
|
76
|
+
for resource in catalog.load():
|
|
77
|
+
if resource.get("type") != "dataset":
|
|
78
|
+
continue
|
|
79
|
+
if clean_text(resource.get("id"), 180).casefold() == requested:
|
|
80
|
+
return resource
|
|
81
|
+
raise ValueError(f"Unknown dataset id: {dataset_id}")
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _instructions(dataset_id: str, method: str, metadata: dict[str, Any]) -> dict[str, Any]:
|
|
85
|
+
destination = f"./{_safe_dirname(dataset_id)}"
|
|
86
|
+
if method == "github_clone":
|
|
87
|
+
repo_id = clean_text(metadata.get("repo_id"), 240)
|
|
88
|
+
if re.fullmatch(r"[A-Za-z0-9._-]+/[A-Za-z0-9._-]+", repo_id):
|
|
89
|
+
return {
|
|
90
|
+
"summary": "Clone the versioned dataset repository with Git.",
|
|
91
|
+
"command": f"git clone {shlex.quote(f'https://github.com/{repo_id}.git')} {shlex.quote(destination)}",
|
|
92
|
+
}
|
|
93
|
+
if method == "huggingface_snapshot":
|
|
94
|
+
repo_id = clean_text(metadata.get("repo_id"), 240)
|
|
95
|
+
if re.fullmatch(r"[A-Za-z0-9._-]+/[A-Za-z0-9._-]+", repo_id):
|
|
96
|
+
revision = clean_text(metadata.get("revision"), 160) or "main"
|
|
97
|
+
return {
|
|
98
|
+
"summary": "Download the pinned Hugging Face dataset snapshot.",
|
|
99
|
+
"command": (
|
|
100
|
+
"python -m pip install -U huggingface_hub\n"
|
|
101
|
+
f"hf download {shlex.quote(repo_id)} --repo-type dataset "
|
|
102
|
+
f"--revision {shlex.quote(revision)} --local-dir {shlex.quote(destination)}"
|
|
103
|
+
),
|
|
104
|
+
}
|
|
105
|
+
if method in AUTOMATED_METHODS:
|
|
106
|
+
return {
|
|
107
|
+
"summary": f"The local OC MCP can execute the {method} adapter.",
|
|
108
|
+
"command": None,
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
labels = {
|
|
112
|
+
"google_drive_folder": "Use the Google Drive API with the user's own credentials.",
|
|
113
|
+
"dataverse_collection": "Enumerate the Dataverse collection and download its published dataset bundles.",
|
|
114
|
+
"designsafe_globus": "Authenticate with Globus and select a user-controlled destination collection.",
|
|
115
|
+
"dreamhouse_setup": "Install the official DreamHouse package and run its artifact setup command.",
|
|
116
|
+
"roboflow_version": "Authenticate with Roboflow and request the declared versioned export.",
|
|
117
|
+
"kaggle_competition": "Accept the competition rules and authenticate with the Kaggle CLI.",
|
|
118
|
+
"baidu_share_transfer": "Authenticate locally with BaiduPCS-Go and transfer the shared dataset.",
|
|
119
|
+
}
|
|
120
|
+
return {
|
|
121
|
+
"summary": labels.get(method, "Follow the provider-specific access instructions."),
|
|
122
|
+
"command": None,
|
|
123
|
+
"documentation_url": _http_url(metadata.get("documentation_url")),
|
|
124
|
+
"notice": clean_text(metadata.get("notice"), 1000) or None,
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def resolve_dataset_download_plan(catalog: CatalogClient, dataset_id: str) -> AcquisitionPlan:
|
|
129
|
+
resource = get_exact_dataset(catalog, dataset_id)
|
|
130
|
+
source = resource.get("source") if isinstance(resource.get("source"), dict) else {}
|
|
131
|
+
canonical_id = clean_text(resource.get("id"), 180)
|
|
132
|
+
dataset_name = clean_text(resource.get("title"), 240) or canonical_id
|
|
133
|
+
license_name = clean_text(source.get("license") or resource.get("license"), 160) or None
|
|
134
|
+
distributions = _objects(source.get("distribution"))
|
|
135
|
+
|
|
136
|
+
if len(distributions) == 1:
|
|
137
|
+
distribution = distributions[0]
|
|
138
|
+
url = _http_url(distribution.get("content_url") or distribution.get("contentUrl") or distribution.get("url"))
|
|
139
|
+
if distribution.get("browser_download") is not False and url:
|
|
140
|
+
size = distribution.get("content_size")
|
|
141
|
+
return AcquisitionPlan(
|
|
142
|
+
dataset_id=canonical_id,
|
|
143
|
+
dataset_name=dataset_name,
|
|
144
|
+
kind="direct",
|
|
145
|
+
provider=clean_text(distribution.get("provider"), 120) or None,
|
|
146
|
+
method=clean_text(distribution.get("download_method"), 80) or "navigate",
|
|
147
|
+
url=url,
|
|
148
|
+
filename=clean_text(distribution.get("filename"), 240) or None,
|
|
149
|
+
license=license_name,
|
|
150
|
+
executable_locally=True,
|
|
151
|
+
estimated_size=size if isinstance(size, int) and size >= 0 else None,
|
|
152
|
+
distributions=distributions,
|
|
153
|
+
)
|
|
154
|
+
|
|
155
|
+
methods = _objects(source.get("programmatic_access"))
|
|
156
|
+
if methods:
|
|
157
|
+
method_metadata = methods[0]
|
|
158
|
+
method = clean_text(method_metadata.get("method"), 120)
|
|
159
|
+
provider = clean_text(method_metadata.get("provider"), 120) or None
|
|
160
|
+
requires_auth = method_metadata.get("requires_auth") is True
|
|
161
|
+
sizes = [item.get("content_size") for item in distributions]
|
|
162
|
+
estimated_size = sum(size for size in sizes if isinstance(size, int) and size >= 0) if sizes else None
|
|
163
|
+
warnings: list[str] = []
|
|
164
|
+
if method not in AUTOMATED_METHODS:
|
|
165
|
+
warnings.append("This provider still requires an assisted or provider-specific workflow.")
|
|
166
|
+
if requires_auth:
|
|
167
|
+
warnings.append("Provider authentication is required and must be completed locally by the user.")
|
|
168
|
+
if estimated_size is None and method in AUTOMATED_METHODS:
|
|
169
|
+
warnings.append("The source does not declare a total size, so disk usage cannot be fully checked before execution.")
|
|
170
|
+
return AcquisitionPlan(
|
|
171
|
+
dataset_id=canonical_id,
|
|
172
|
+
dataset_name=dataset_name,
|
|
173
|
+
kind="programmatic",
|
|
174
|
+
provider=provider,
|
|
175
|
+
method=method or None,
|
|
176
|
+
url=_http_url(method_metadata.get("documentation_url")) or _http_url(source.get("access")),
|
|
177
|
+
license=license_name,
|
|
178
|
+
requires_auth=requires_auth,
|
|
179
|
+
executable_locally=method in AUTOMATED_METHODS,
|
|
180
|
+
estimated_size=estimated_size,
|
|
181
|
+
distributions=distributions,
|
|
182
|
+
programmatic_access=method_metadata,
|
|
183
|
+
instructions=_instructions(canonical_id, method, method_metadata),
|
|
184
|
+
auth=(
|
|
185
|
+
provider_auth_status(provider, method, required=True, detect_credentials=False)
|
|
186
|
+
if requires_auth
|
|
187
|
+
else None
|
|
188
|
+
),
|
|
189
|
+
warnings=warnings,
|
|
190
|
+
)
|
|
191
|
+
|
|
192
|
+
fallback = _http_url(source.get("access")) or _http_url(source.get("url")) or _http_url(resource.get("url"))
|
|
193
|
+
warnings = ["No machine-readable download route is currently available for this dataset."]
|
|
194
|
+
if fallback:
|
|
195
|
+
try:
|
|
196
|
+
discovered = discover_public_files(fallback)
|
|
197
|
+
if discovered and discovered["files"]:
|
|
198
|
+
files = discovered["files"]
|
|
199
|
+
sizes = [item["content_size"] for item in files]
|
|
200
|
+
return AcquisitionPlan(
|
|
201
|
+
dataset_id=canonical_id, dataset_name=dataset_name,
|
|
202
|
+
kind="programmatic", method="http_files", provider=discovered["provider"],
|
|
203
|
+
url=fallback, license=discovered.get("license") or license_name,
|
|
204
|
+
executable_locally=True, distributions=files,
|
|
205
|
+
estimated_size=sum(sizes) if all(type(size) is int for size in sizes) else None,
|
|
206
|
+
instructions={"summary": "Download the public files supplied by the provider after license review.",
|
|
207
|
+
"license_url": discovered.get("license_url"), "catalog_license": license_name},
|
|
208
|
+
)
|
|
209
|
+
if discovered:
|
|
210
|
+
warnings = [discovered["notice"]]
|
|
211
|
+
except (httpx.HTTPError, ValueError, TypeError, KeyError):
|
|
212
|
+
warnings = ["The provider's file metadata could not be retrieved. Retry later or open the original source."]
|
|
213
|
+
return AcquisitionPlan(
|
|
214
|
+
dataset_id=canonical_id,
|
|
215
|
+
dataset_name=dataset_name,
|
|
216
|
+
kind="site",
|
|
217
|
+
url=fallback,
|
|
218
|
+
license=license_name,
|
|
219
|
+
instructions={"summary": "Open the source page and complete the provider's manual access workflow.", "command": None},
|
|
220
|
+
warnings=warnings,
|
|
221
|
+
)
|
|
@@ -0,0 +1,327 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import hashlib
|
|
4
|
+
import json
|
|
5
|
+
import os
|
|
6
|
+
import time
|
|
7
|
+
import urllib.parse
|
|
8
|
+
import urllib.request
|
|
9
|
+
from dataclasses import dataclass, field
|
|
10
|
+
from typing import Any, Callable
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
DEFAULT_DATA_BASE_URL = "https://www.openconstruction.org/data"
|
|
14
|
+
DEFAULT_SITE_BASE_URL = "https://www.openconstruction.org"
|
|
15
|
+
DEFAULT_CACHE_TTL_SECONDS = 600
|
|
16
|
+
|
|
17
|
+
CATALOG_SOURCES: tuple[tuple[str, str], ...] = (
|
|
18
|
+
("dataset", "datasets.json"),
|
|
19
|
+
("model", "models.json"),
|
|
20
|
+
("workflow", "use-cases.json"),
|
|
21
|
+
("oer", "oer.json"),
|
|
22
|
+
("tool", "tools.json"),
|
|
23
|
+
("guide", "guides.json"),
|
|
24
|
+
("contributor", "contributors.json"),
|
|
25
|
+
("benchmark", "benchmark-results.json"),
|
|
26
|
+
("vocabulary", "task-vocabulary.json"),
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def clean_text(value: Any, max_length: int = 500) -> str:
|
|
31
|
+
if value is None:
|
|
32
|
+
return ""
|
|
33
|
+
if isinstance(value, (list, tuple, set)):
|
|
34
|
+
value = " ".join(clean_text(item, max_length) for item in value)
|
|
35
|
+
elif isinstance(value, dict):
|
|
36
|
+
value = json.dumps(value, ensure_ascii=False, sort_keys=True)
|
|
37
|
+
return " ".join(str(value).split())[:max_length]
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def to_list(value: Any) -> list[str]:
|
|
41
|
+
if isinstance(value, list):
|
|
42
|
+
return [clean_text(item, 120) for item in value if clean_text(item, 120)]
|
|
43
|
+
text = clean_text(value, 120)
|
|
44
|
+
return [text] if text else []
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def join_url(base: str, path: str) -> str:
|
|
48
|
+
return f"{base.rstrip('/')}/{path.lstrip('/')}"
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def resource_href(resource_type: str, resource_id: str) -> str:
|
|
52
|
+
encoded = urllib.parse.quote(resource_id)
|
|
53
|
+
if resource_type == "dataset":
|
|
54
|
+
return f"{DEFAULT_SITE_BASE_URL}/datasets/detail.html?id={encoded}"
|
|
55
|
+
if resource_type == "model":
|
|
56
|
+
return f"{DEFAULT_SITE_BASE_URL}/models/details.html?id={encoded}"
|
|
57
|
+
if resource_type == "workflow":
|
|
58
|
+
return f"{DEFAULT_SITE_BASE_URL}/workflows/details.html?id={encoded}"
|
|
59
|
+
if resource_type == "oer":
|
|
60
|
+
return f"{DEFAULT_SITE_BASE_URL}/oers/details.html?id={encoded}"
|
|
61
|
+
if resource_type == "guide":
|
|
62
|
+
return f"{DEFAULT_SITE_BASE_URL}/guides.html#{encoded}"
|
|
63
|
+
if resource_type == "contributor":
|
|
64
|
+
return f"{DEFAULT_SITE_BASE_URL}/contributors.html#{encoded}"
|
|
65
|
+
if resource_type == "benchmark":
|
|
66
|
+
return f"{DEFAULT_SITE_BASE_URL}/benchmark_results.html#{encoded}"
|
|
67
|
+
if resource_type == "vocabulary":
|
|
68
|
+
return f"{DEFAULT_SITE_BASE_URL}/taxonomy.html#{encoded}"
|
|
69
|
+
return f"{DEFAULT_SITE_BASE_URL}/tools.html"
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def looks_like_record(row: dict[str, Any]) -> bool:
|
|
73
|
+
return any(row.get(key) for key in ("id", "name", "title", "summary", "abstract", "url", "source", "link_url"))
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def extract_rows(raw: Any, parent: dict[str, Any] | None = None) -> list[dict[str, Any]]:
|
|
77
|
+
parent = parent or {}
|
|
78
|
+
if isinstance(raw, list):
|
|
79
|
+
rows: list[dict[str, Any]] = []
|
|
80
|
+
for item in raw:
|
|
81
|
+
rows.extend(extract_rows(item, parent))
|
|
82
|
+
return rows
|
|
83
|
+
|
|
84
|
+
if not isinstance(raw, dict):
|
|
85
|
+
return []
|
|
86
|
+
|
|
87
|
+
# Recurse only through catalog containers, never through a record's metadata.
|
|
88
|
+
child_keys = ("sections", "items", "resources", "guides", "benchmarks", "entries", "use_cases", "useCases",
|
|
89
|
+
"datasets", "models", "tools", "oer", "contributors", "results", "tasks")
|
|
90
|
+
# A titled section is a container; a named/id-bearing record may itself have
|
|
91
|
+
# list-valued fields (e.g. tasks), which must remain part of that record.
|
|
92
|
+
is_section = any(isinstance(raw.get(key), list) for key in ("sections", "items", "resources", "entries"))
|
|
93
|
+
if not raw.get("id") and not raw.get("name"):
|
|
94
|
+
is_section = is_section or any(isinstance(raw.get(key), (list, dict)) for key in (
|
|
95
|
+
"datasets", "models", "tools", "oer", "guides", "benchmarks", "use_cases", "useCases"))
|
|
96
|
+
is_section = is_section or (bool(raw.get("schema_version")) and isinstance(raw.get("tasks"), list))
|
|
97
|
+
if looks_like_record(raw) and not is_section:
|
|
98
|
+
return [{**parent, **raw}]
|
|
99
|
+
rows: list[dict[str, Any]] = []
|
|
100
|
+
child_parent = {
|
|
101
|
+
"section": raw.get("title") or raw.get("name") or parent.get("section"),
|
|
102
|
+
"kicker": raw.get("kicker") or parent.get("kicker"),
|
|
103
|
+
}
|
|
104
|
+
for key in child_keys:
|
|
105
|
+
if isinstance(raw.get(key), (list, dict)):
|
|
106
|
+
rows.extend(extract_rows(raw[key], child_parent))
|
|
107
|
+
|
|
108
|
+
skip_keys = {"schema", "schema_version", "about", "metadata", "sections", *child_keys}
|
|
109
|
+
for key, value in raw.items():
|
|
110
|
+
if key in skip_keys or not isinstance(value, dict):
|
|
111
|
+
continue
|
|
112
|
+
rows.extend(extract_rows({"id": key, **value}, parent))
|
|
113
|
+
|
|
114
|
+
return rows
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def normalize_resource(row: dict[str, Any], resource_type: str, site_base_url: str) -> dict[str, Any] | None:
|
|
118
|
+
resource_id = clean_text(row.get("id") or row.get("resource_id") or row.get("slug") or row.get("name") or row.get("title"), 160)
|
|
119
|
+
title = clean_text(row.get("title") or row.get("name") or resource_id, 220)
|
|
120
|
+
if not resource_id or not title:
|
|
121
|
+
return None
|
|
122
|
+
|
|
123
|
+
raw_href = clean_text(
|
|
124
|
+
row.get("href") or row.get("url") or row.get("page_url") or row.get("access") or row.get("source") or row.get("link_url") or row.get("data_url") or row.get("code_url"),
|
|
125
|
+
300,
|
|
126
|
+
)
|
|
127
|
+
href = raw_href or resource_href(resource_type, resource_id)
|
|
128
|
+
href = href.replace(DEFAULT_SITE_BASE_URL, site_base_url)
|
|
129
|
+
|
|
130
|
+
tags = [
|
|
131
|
+
*to_list(row.get("tags")),
|
|
132
|
+
*to_list(row.get("tasks")),
|
|
133
|
+
*to_list(row.get("potential_tasks")),
|
|
134
|
+
*to_list(row.get("modalities")),
|
|
135
|
+
*to_list(row.get("data_modality")),
|
|
136
|
+
*to_list(row.get("data_modalities")),
|
|
137
|
+
*to_list(row.get("annotation_types")),
|
|
138
|
+
*to_list(row.get("topics")),
|
|
139
|
+
*to_list(row.get("classes")),
|
|
140
|
+
*to_list(row.get("objects")),
|
|
141
|
+
][:18]
|
|
142
|
+
|
|
143
|
+
evidence = clean_text(
|
|
144
|
+
" ".join(
|
|
145
|
+
[
|
|
146
|
+
clean_text(row.get("summary"), 400),
|
|
147
|
+
clean_text(row.get("description"), 400),
|
|
148
|
+
clean_text(row.get("abstract"), 500),
|
|
149
|
+
clean_text(row.get("subtitle"), 300),
|
|
150
|
+
clean_text(row.get("role"), 120),
|
|
151
|
+
clean_text(row.get("affiliation"), 180),
|
|
152
|
+
clean_text(row.get("provider"), 120),
|
|
153
|
+
clean_text(row.get("publisher"), 120),
|
|
154
|
+
" ".join(tags),
|
|
155
|
+
]
|
|
156
|
+
),
|
|
157
|
+
1200,
|
|
158
|
+
)
|
|
159
|
+
|
|
160
|
+
return {
|
|
161
|
+
"id": resource_id,
|
|
162
|
+
"type": resource_type,
|
|
163
|
+
"title": title,
|
|
164
|
+
"summary": clean_text(row.get("summary") or row.get("description") or row.get("abstract") or row.get("subtitle") or row.get("role"), 600),
|
|
165
|
+
"href": href,
|
|
166
|
+
"url": clean_text(row.get("url") or row.get("access") or row.get("source") or row.get("data_url") or row.get("code_url"), 300) or None,
|
|
167
|
+
"year": row.get("year"),
|
|
168
|
+
"license": clean_text(row.get("license"), 120) or None,
|
|
169
|
+
"tags": tags,
|
|
170
|
+
"source": row,
|
|
171
|
+
"evidence": evidence,
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def score_resource(resource: dict[str, Any], query: str) -> int:
|
|
176
|
+
terms = [term for term in clean_text(query, 1200).lower().replace("_", " ").split() if len(term) > 1]
|
|
177
|
+
if not terms:
|
|
178
|
+
return 0
|
|
179
|
+
|
|
180
|
+
title = clean_text(resource.get("title"), 500).lower()
|
|
181
|
+
summary = clean_text(resource.get("summary"), 800).lower()
|
|
182
|
+
evidence = clean_text(resource.get("evidence") or resource.get("source"), 1800).lower()
|
|
183
|
+
|
|
184
|
+
score = 0
|
|
185
|
+
for term in terms:
|
|
186
|
+
if term in title:
|
|
187
|
+
score += 8
|
|
188
|
+
if term in summary:
|
|
189
|
+
score += 4
|
|
190
|
+
if term in evidence:
|
|
191
|
+
score += 1
|
|
192
|
+
return score
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
@dataclass
|
|
196
|
+
class CatalogClient:
|
|
197
|
+
data_base_url: str = field(default_factory=lambda: os.environ.get("OPENCONSTRUCTION_DATA_BASE_URL", DEFAULT_DATA_BASE_URL))
|
|
198
|
+
site_base_url: str = field(default_factory=lambda: os.environ.get("OPENCONSTRUCTION_SITE_BASE_URL", DEFAULT_SITE_BASE_URL))
|
|
199
|
+
cache_ttl_seconds: int = field(default_factory=lambda: int(os.environ.get("CATALOG_CACHE_TTL_SECONDS", DEFAULT_CACHE_TTL_SECONDS)))
|
|
200
|
+
fetch_json: Callable[[str], Any] | None = None
|
|
201
|
+
|
|
202
|
+
def __post_init__(self) -> None:
|
|
203
|
+
self._loaded_at = 0.0
|
|
204
|
+
self._resources: list[dict[str, Any]] = []
|
|
205
|
+
self._errors: list[dict[str, str]] = []
|
|
206
|
+
self._source_digests: dict[str, str] = {}
|
|
207
|
+
self._has_loaded = False
|
|
208
|
+
|
|
209
|
+
def load(self, force: bool = False) -> list[dict[str, Any]]:
|
|
210
|
+
now = time.time()
|
|
211
|
+
if self._has_loaded and not force and now - self._loaded_at < self.cache_ttl_seconds:
|
|
212
|
+
return self._resources
|
|
213
|
+
|
|
214
|
+
resources: list[dict[str, Any]] = []
|
|
215
|
+
errors: list[dict[str, str]] = []
|
|
216
|
+
digests: dict[str, str] = {}
|
|
217
|
+
for resource_type, path in CATALOG_SOURCES:
|
|
218
|
+
try:
|
|
219
|
+
raw = self._fetch(path)
|
|
220
|
+
digests[path] = hashlib.sha256(json.dumps(raw, sort_keys=True, ensure_ascii=False, separators=(",", ":")).encode("utf-8")).hexdigest()
|
|
221
|
+
except Exception as exc:
|
|
222
|
+
errors.append({"type": resource_type, "path": path, "error": str(exc)})
|
|
223
|
+
continue
|
|
224
|
+
for row in extract_rows(raw):
|
|
225
|
+
normalized = normalize_resource(row, resource_type, self.site_base_url)
|
|
226
|
+
if normalized:
|
|
227
|
+
resources.append(normalized)
|
|
228
|
+
|
|
229
|
+
self._loaded_at = now
|
|
230
|
+
self._resources = resources
|
|
231
|
+
self._errors = errors
|
|
232
|
+
self._source_digests = digests
|
|
233
|
+
self._has_loaded = True
|
|
234
|
+
return resources
|
|
235
|
+
|
|
236
|
+
def provenance(self) -> dict[str, Any]:
|
|
237
|
+
self.load()
|
|
238
|
+
return {
|
|
239
|
+
"source": self.data_base_url,
|
|
240
|
+
"loaded_at": int(self._loaded_at),
|
|
241
|
+
"snapshot_sha256": hashlib.sha256(json.dumps(self._source_digests, sort_keys=True).encode("utf-8")).hexdigest(),
|
|
242
|
+
"source_sha256": dict(self._source_digests),
|
|
243
|
+
"complete": not self._errors,
|
|
244
|
+
"load_errors": list(self._errors),
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
def search(self, query: str, types: list[str] | None = None, limit: int = 20) -> list[dict[str, Any]]:
|
|
248
|
+
requested = set(types or [])
|
|
249
|
+
scored = []
|
|
250
|
+
for resource in self.load():
|
|
251
|
+
if requested and resource["type"] not in requested:
|
|
252
|
+
continue
|
|
253
|
+
score = score_resource(resource, query)
|
|
254
|
+
if score > 0:
|
|
255
|
+
scored.append({**resource, "score": score})
|
|
256
|
+
scored.sort(key=lambda item: (-item["score"], item["title"]))
|
|
257
|
+
return scored[: max(1, min(limit, 100))]
|
|
258
|
+
|
|
259
|
+
def get(self, resource_type: str, resource_id: str) -> dict[str, Any] | None:
|
|
260
|
+
key = clean_text(resource_id, 180).lower()
|
|
261
|
+
if not key:
|
|
262
|
+
return None
|
|
263
|
+
for resource in self.load():
|
|
264
|
+
if resource["type"] != resource_type:
|
|
265
|
+
continue
|
|
266
|
+
if key == resource["id"].lower():
|
|
267
|
+
return resource
|
|
268
|
+
return None
|
|
269
|
+
|
|
270
|
+
def compare(self, resources: list[dict[str, str]], goal: str = "") -> dict[str, Any]:
|
|
271
|
+
found = [self.get(item.get("type", ""), item.get("id", "")) for item in resources]
|
|
272
|
+
matches = [item for item in found if item]
|
|
273
|
+
summary = (
|
|
274
|
+
f"Compared {len(matches)} matching OpenConstruction resources for: {goal}."
|
|
275
|
+
if len(matches) >= 2 and goal
|
|
276
|
+
else f"Compared {len(matches)} matching OpenConstruction resources."
|
|
277
|
+
if len(matches) >= 2
|
|
278
|
+
else "At least two matching resources are needed for comparison."
|
|
279
|
+
)
|
|
280
|
+
fields = ("title", "summary", "year", "license", "tags", "url")
|
|
281
|
+
return {
|
|
282
|
+
"summary": summary, "resources": matches,
|
|
283
|
+
"missing_resources": [requested for requested, result in zip(resources, found) if result is None],
|
|
284
|
+
"comparison": [{"field": field, "values": [
|
|
285
|
+
{"type": item["type"], "id": item["id"], "value": item.get(field) or None,
|
|
286
|
+
"status": "reported" if item.get(field) else "unknown"} for item in matches
|
|
287
|
+
]} for field in fields],
|
|
288
|
+
"limitations": ["Compares reported catalog metadata; does not establish task suitability or executable compatibility."],
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
def stats(self) -> dict[str, Any]:
|
|
292
|
+
by_type: dict[str, int] = {}
|
|
293
|
+
for resource in self.load():
|
|
294
|
+
by_type[resource["type"]] = by_type.get(resource["type"], 0) + 1
|
|
295
|
+
return {
|
|
296
|
+
"source": self.data_base_url,
|
|
297
|
+
"loaded_at": int(self._loaded_at),
|
|
298
|
+
"resources": len(self._resources),
|
|
299
|
+
"by_type": by_type,
|
|
300
|
+
"load_errors": self._errors,
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
def answer(self, query: str) -> dict[str, Any]:
|
|
304
|
+
resources = self.search(query, limit=8)
|
|
305
|
+
answer = (
|
|
306
|
+
"Top OpenConstruction matches: "
|
|
307
|
+
+ ", ".join(item["title"] for item in resources[:3])
|
|
308
|
+
if resources
|
|
309
|
+
else "No strong OpenConstruction catalog matches found. Try a broader task, modality, resource type, or object class."
|
|
310
|
+
)
|
|
311
|
+
return {
|
|
312
|
+
"answer": answer,
|
|
313
|
+
"resources": resources,
|
|
314
|
+
"followups": [
|
|
315
|
+
"Compare the top matches",
|
|
316
|
+
"Filter by license or modality",
|
|
317
|
+
"Show datasets only",
|
|
318
|
+
],
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
def _fetch(self, path: str) -> Any:
|
|
322
|
+
if self.fetch_json:
|
|
323
|
+
return self.fetch_json(path)
|
|
324
|
+
url = join_url(self.data_base_url, path)
|
|
325
|
+
request = urllib.request.Request(url, headers={"User-Agent": "OpenConstruction/0.1 (+https://www.openconstruction.org)"})
|
|
326
|
+
with urllib.request.urlopen(request, timeout=20) as response:
|
|
327
|
+
return json.loads(response.read().decode("utf-8"))
|