fd-open-data-protocol 0.3.0__tar.gz → 0.3.2__tar.gz

{fd_open_data_protocol-0.3.0 → fd_open_data_protocol-0.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fd-open-data-protocol
-Version: 0.3.0
+Version: 0.3.2
 Summary: The open-data datasource protocol: a manifest contract a datasource exposes to be ingested by fd-open-data-mcp (or any consumer).
 Author: FindDataOfficial
 License: MIT
@@ -10,9 +10,13 @@ Requires-Dist: pydantic>=2.0
 Requires-Dist: pyyaml>=6.0
 Provides-Extra: dev
 Requires-Dist: pytest>=8.0; extra == "dev"
+Provides-Extra: registry
+Requires-Dist: psycopg[binary]>=3.1; extra == "registry"
 
 # fd-open-data-protocol
 
+> **Wire (柏讯) product line** · the open-data supply line of [FindData](https://www.finddatatech.cloud/products/wire) — the datasource manifest contract
+
 **English** | [中文](README.zh-CN.md)
 
 The **open-data datasource protocol**: a manifest contract a datasource exposes

{fd_open_data_protocol-0.3.0 → fd_open_data_protocol-0.3.2}/README.md

@@ -1,5 +1,7 @@
 # fd-open-data-protocol
 
+> **Wire (柏讯) product line** · the open-data supply line of [FindData](https://www.finddatatech.cloud/products/wire) — the datasource manifest contract
+
 **English** | [中文](README.zh-CN.md)
 
 The **open-data datasource protocol**: a manifest contract a datasource exposes

{fd_open_data_protocol-0.3.0 → fd_open_data_protocol-0.3.2}/fd_open_data_protocol/__init__.py

@@ -5,4 +5,4 @@ concept hints + fetch reference) so that fd-open-data-mcp - or any consumer -
 can ingest it via ``register_datasource``. Ship one manifest file -> the
 datasource is added; no consumer-side wiring.
 """
-__version__ = "0.3.0"
+__version__ = "0.3.2"

fd_open_data_protocol-0.3.2/fd_open_data_protocol/openalex_provider.py

@@ -0,0 +1,213 @@
+"""OpenAlex 学术图谱数据源 provider（CC0，免费）。
+
+契约要点（对应 academic-graph-connector spec）：
+  - 礼貌调用：请求带含 mail 的 User-Agent（免费档 1000 credits/次级）
+  - 失败显式：配额/限流/不可达分别报错，**禁止返回空结果冒充无数据**
+  - 收录偏差：未收录如实标注，不断言「不存在引用」
+  - 引用图是增益，不替代既有 papers 检索
+"""
+from __future__ import annotations
+
+import json
+import os
+import urllib.error
+import urllib.parse
+import urllib.request
+from typing import Any
+
+from .provider import BaseDataProvider
+
+API = "https://api.openalex.org"
+DEFAULT_MAILTO = os.environ.get("OPENALEX_MAILTO", "ops@finddata.dev")
+TIMEOUT = float(os.environ.get("OPENALEX_TIMEOUT", "25"))
+
+# 礼貌调用：OpenAlex 要求可识别的 User-Agent（含 mail），否则可能被限流
+UA = f"finddata-academic-connector/1.0 (mailto:{DEFAULT_MAILTO})"
+
+
+class OpenAlexUnavailable(RuntimeError):
+    """上游不可用/不可达——显式失败，不静默返回空。"""
+
+
+class OpenAlexQuotaExceeded(RuntimeError):
+    """配额/限流——显式失败，不冒充「无数据」。"""
+
+
+class OpenAlexProvider(BaseDataProvider):
+    name = "academic-openalex"
+
+    def registry(self) -> dict:
+        return {"manifest": "academic_openalex.yaml"}
+
+    def run(self, command: str, params: dict) -> Any:
+        handlers = {
+            "get_work": self.get_work,
+            "search_works": self.search_works,
+            "get_references": self.get_references,
+            "get_citations": self.get_citations,
+        }
+        if command not in handlers:
+            raise ValueError(f"unknown command {command!r}")
+        return handlers[command](params or {})
+
+    # ------------------------------------------------------------------ HTTP
+
+    def _get(self, path: str, params: dict[str, Any] | None = None) -> tuple[dict, dict]:
+        """GET 并返回 (payload, quota_headers)。失败显式抛错。"""
+        url = f"{API}/{path}"
+        if params:
+            url += "?" + urllib.parse.urlencode(params)
+        req = urllib.request.Request(
+            url, headers={"User-Agent": UA, "Accept": "application/json"}
+        )
+        try:
+            with urllib.request.urlopen(req, timeout=TIMEOUT) as resp:
+                raw = json.loads(resp.read().decode("utf-8", "replace"))
+                headers = dict(resp.headers.items())
+        except urllib.error.HTTPError as exc:
+            # 429 或 5xx 都要区分清楚：前者是配额，后者是上游故障
+            if exc.code == 429:
+                raise OpenAlexQuotaExceeded(
+                    f"OpenAlex rate limit / quota exceeded (HTTP 429): {exc.read()[:200]!r}"
+                ) from exc
+            if 500 <= exc.code < 600:
+                raise OpenAlexUnavailable(
+                    f"OpenAlex upstream unavailable (HTTP {exc.code})"
+                ) from exc
+            raise OpenAlexUnavailable(
+                f"OpenAlex request failed (HTTP {exc.code}): {exc.read()[:200]!r}"
+            ) from exc
+        except (urllib.error.URLError, TimeoutError, OSError) as exc:
+            raise OpenAlexUnavailable(f"OpenAlex unreachable: {exc}") from exc
+        quota = {
+            k.lower(): v
+            for k, v in headers.items()
+            if k.lower().startswith("x-ratelimit") or k.lower() == "retry-after"
+        }
+        return raw, quota
+
+    def _resolve_work_id(self, params: dict) -> str:
+        """按 work_id 或 doi 定位作品。两者皆缺则报错（不猜）。"""
+        work_id = (params.get("work_id") or "").strip()
+        doi = (params.get("doi") or "").strip()
+        if not work_id and not doi:
+            raise ValueError("get_references/get_citations requires `work_id` or `doi`")
+        if work_id:
+            # 允许传完整 URL，统一取末段
+            return work_id.rstrip("/").split("/")[-1]
+        payload, _ = self._get(
+            "works",
+            {"filter": f"doi:{doi}", "select": "id", "per-page": "1"},
+        )
+        results = payload.get("results") or []
+        if not results:
+            raise ValueError(
+                f"work not found in OpenAlex for doi={doi!r} "
+                f"（未收录，不等于该作品不存在）"
+            )
+        return str(results[0]["id"]).rstrip("/").split("/")[-1]
+
+    # ------------------------------------------------------------- commands
+
+    def get_work(self, params: dict) -> dict:
+        doi = (params.get("doi") or "").strip()
+        if not doi:
+            raise ValueError("get_work requires `doi`")
+        payload, quota = self._get(
+            "works", {"filter": f"doi:{doi}", "per-page": "1"}
+        )
+        results = payload.get("results") or []
+        if not results:
+            # 未收录 ≠ 不存在：如实说明
+            raise ValueError(
+                f"work not found in OpenAlex for doi={doi!r} （未收录，不等于该作品不存在）"
+            )
+        w = results[0]
+        return {
+            "work_id": str(w.get("id", "")).rstrip("/").split("/")[-1],
+            "title": w.get("title"),
+            "year": w.get("publication_year"),
+            "venue": (w.get("primary_location") or {}).get("source", {}).get("display_name")
+            if isinstance(w.get("primary_location"), dict)
+            else None,
+            "cited_by_count": w.get("cited_by_count"),
+            "doi": w.get("doi"),
+            "source": "openalex",
+            "license": "CC0",
+            "quota": quota,
+        }
+
+    def search_works(self, params: dict) -> dict:
+        query = (params.get("query") or "").strip()
+        if not query:
+            raise ValueError("search_works requires `query`")
+        limit = int(params.get("limit") or 10)
+        payload, quota = self._get(
+            "works",
+            {
+                "search": query,
+                "per-page": str(min(limit, 200)),
+                "select": "id,title,publication_year,cited_by_count",
+            },
+        )
+        rows = []
+        for w in payload.get("results") or []:
+            rows.append(
+                {
+                    "work_id": str(w.get("id", "")).rstrip("/").split("/")[-1],
+                    "title": w.get("title"),
+                    "year": w.get("publication_year"),
+                    "cited_by_count": w.get("cited_by_count"),
+                }
+            )
+        return {
+            "total": (payload.get("meta") or {}).get("count"),
+            "results": rows,
+            "source": "openalex",
+            "license": "CC0",
+            "quota": quota,
+        }
+
+    def get_references(self, params: dict) -> dict:
+        work_id = self._resolve_work_id(params)
+        payload, quota = self._get(
+            f"works/{work_id}", {"select": "id,referenced_works_count,referenced_works"}
+        )
+        refs = payload.get("referenced_works") or []
+        return {
+            "work_id": work_id,
+            "reference_count": payload.get("referenced_works_count"),
+            "reference_ids": [str(r).rstrip("/").split("/")[-1] for r in refs],
+            "source": "openalex",
+            "license": "CC0",
+            "quota": quota,
+        }
+
+    def get_citations(self, params: dict) -> dict:
+        work_id = self._resolve_work_id(params)
+        limit = int(params.get("limit") or 10)
+        payload, quota = self._get(
+            "works",
+            {
+                "filter": f"cites:{work_id}",
+                "per-page": str(min(limit, 200)),
+                "select": "id,title,publication_year",
+            },
+        )
+        rows = []
+        for w in payload.get("results") or []:
+            rows.append(
+                {
+                    "citing_work_id": str(w.get("id", "")).rstrip("/").split("/")[-1],
+                    "title": w.get("title"),
+                    "year": w.get("publication_year"),
+                }
+            )
+        return {
+            "work_id": work_id,
+            "total_count": (payload.get("meta") or {}).get("count"),
+            "results": rows,
+            "source": "openalex",
+            "license": "CC0",
+            "quota": quota,
+        }

fd_open_data_protocol-0.3.2/fd_open_data_protocol/registry.py

@@ -0,0 +1,274 @@
+"""Declarative registry adapters and coverage computation.
+
+An adapter is a small YAML file mapping a source table's rows onto
+:class:`~fd_open_data_protocol.schema.RegistryEntry` fields - no core code
+changes when a new source database is onboarded. The recipe (see REGISTRY.md):
+
+    写适配器 → 批量登记（register）→ 概念绑定（bind）→ 命名门核验（verify）
+
+Row sources are independent of mapping: rows can come from a Postgres source
+DSN or a CSV file (tests, dry runs, the shipped template example), so the
+whole pipeline is exercisable without a database.
+
+Coverage is asset-denominated: per source "registered / actual in source",
+with the denominator taken from the source database (or supplied counts) -
+never from an MCP concept dictionary.
+"""
+from __future__ import annotations
+
+import csv
+import json
+from pathlib import Path
+from typing import Any, Iterable, Optional, Sequence
+
+import yaml
+from pydantic import BaseModel, Field, field_validator
+
+from fd_open_data_protocol.schema import RegistryEntry
+
+#: registry fields an adapter may map; anchors come from dedicated keys.
+MAPPABLE_FIELDS = ("semantic_code", "name_zh", "name_en", "unit", "frequency", "domain")
+
+#: sources known to the fleet but NOT in the first registration batch; the
+#: coverage view must show them explicitly instead of silently omitting them.
+DEFAULT_SCOPE: dict[str, dict[str, Any]] = {
+    "world_bank": {"phase": 1, "label": "世界银行 WDI（含宏观数据链路）"},
+    "yearbook_catalog": {"phase": 1, "label": "国家统计年鉴"},
+    "gta_panel": {"phase": 1, "label": "GTA 治理面板"},
+    "china_city_panel": {"phase": 1, "label": "中国城市面板"},
+    "law_db": {
+        "phase": 1,
+        "label": "法律库·文档域（法条/切片/指导案例为文档资产，非指标口径；登记 0 属预期，经 business-mcp 可查）",
+    },
+    "fd_open_data": {"phase": 1, "label": "fd_open_data 镜像概念字典"},
+    "ia35": {"phase": 2, "label": "工商注册 161G（实体为主，二期）"},
+    "wenshu_db": {"phase": 2, "label": "裁判文书（增量暂停，二期）"},
+}
+
+
+class FieldSource(BaseModel):
+    """Where one registry field's value comes from.
+
+    ``from: column``  - a column of the source row (or CSV header)
+    ``from: literal`` - a constant (e.g. ``domain: macro``)
+    ``from: map_file``- a JSON ``{native_code: value}`` lookup, typically a
+                        curated/LLM-drafted naming map (name translations,
+                        semantic codes); misses fall back to ``missing``
+    """
+
+    from_: str = Field(alias="from")
+    value: str
+    missing: Optional[str] = None  # value to store on a miss; default None = absent
+    uppercase: bool = False
+
+    @field_validator("from_")
+    @classmethod
+    def _known_kind(cls, v: str) -> str:
+        if v not in ("column", "literal", "map_file"):
+            raise ValueError(f"unknown field source kind: {v!r} (column|literal|map_file)")
+        return v
+
+    model_config = {"populate_by_name": True}
+
+
+class AdapterConfig(BaseModel):
+    """Declarative source-table → registry mapping."""
+
+    source_db: str
+    source_table: str
+    source_column: str = ""  # anchor label: the column the native code was read from
+    native_code_column: str
+    fields: dict[str, FieldSource]
+    provenance_columns: list[str] = Field(default_factory=list)
+    source_query: Optional[str] = None  # default: SELECT * FROM <source_table>
+    semantic_code_dedupe_suffix: str = "-%d"  # in-batch collision suffix (marked needs_review)
+
+    @field_validator("fields")
+    @classmethod
+    def _fields_known(cls, v: dict[str, FieldSource]) -> dict[str, FieldSource]:
+        unknown = set(v) - set(MAPPABLE_FIELDS)
+        if unknown:
+            raise ValueError(f"unknown mappable field(s): {sorted(unknown)}")
+        return v
+
+
+def load_adapter(source: str | Path | dict) -> AdapterConfig:
+    """Load an adapter from a YAML/JSON file path or an already-parsed dict."""
+    if isinstance(source, dict):
+        return AdapterConfig(**source)
+    p = Path(source)
+    data = yaml.safe_load(p.read_text(encoding="utf-8"))
+    if not isinstance(data, dict):
+        raise ValueError(f"adapter file did not parse to a mapping: {p}")
+    adapter = AdapterConfig(**data)
+    adapter._base_dir = p.parent  # type: ignore[attr-defined]  # resolve relative map_files
+    return adapter
+
+
+def _lookup(
+    spec: FieldSource, row: dict, native_code: str, base_dir: Path
+) -> Optional[str]:
+    value: Any
+    if spec.from_ == "literal":
+        value = spec.value
+    elif spec.from_ == "column":
+        value = row.get(spec.value)
+    else:  # map_file
+        map_path = (base_dir / spec.value) if not Path(spec.value).is_absolute() else Path(spec.value)
+        try:
+            mapping = json.loads(map_path.read_text(encoding="utf-8"))
+        except FileNotFoundError:
+            value = spec.missing
+            mapping = None
+        else:
+            value = mapping.get(native_code, spec.missing)
+    if value is None:
+        return None
+    text = str(value).strip()
+    if not text:
+        return None
+    return text.upper() if spec.uppercase else text
+
+
+def map_rows(
+    adapter: AdapterConfig, rows: Iterable[dict[str, Any]]
+) -> list[RegistryEntry]:
+    """Map source rows to registry entries (pure; no store access).
+
+    In-batch semantic_code collisions get a numeric suffix and a
+    ``needs_review: true`` provenance flag - never a silent merge.
+    """
+    base_dir = getattr(adapter, "_base_dir", Path("."))
+    seen_codes: dict[str, int] = {}
+    entries: list[RegistryEntry] = []
+    for row in rows:
+        native_code = str(row.get(adapter.native_code_column, "") or "").strip()
+        if not native_code:
+            raise ValueError(
+                f"row has empty {adapter.native_code_column!r}; adapter={adapter.source_db}.{adapter.source_table}"
+            )
+        fields: dict[str, Optional[str]] = {}
+        needs_review = False
+        for name, spec in adapter.fields.items():
+            value = _lookup(spec, row, native_code, base_dir)
+            if name == "semantic_code" and value is not None:
+                base = value
+                n = seen_codes.get(base, 0)
+                if n:
+                    value = base + (adapter.semantic_code_dedupe_suffix % (n + 1))
+                    needs_review = True
+                seen_codes[base] = n + 1
+            fields[name] = value
+        provenance = {c: row.get(c) for c in adapter.provenance_columns if c in row}
+        if needs_review:
+            provenance["needs_review"] = True
+        entries.append(
+            RegistryEntry(
+                source_db=adapter.source_db,
+                source_table=adapter.source_table,
+                source_column=adapter.source_column or adapter.native_code_column,
+                native_code=native_code,
+                provenance=provenance or None,
+                **fields,
+            )
+        )
+    return entries
+
+
+def fetch_rows(adapter: AdapterConfig, source_dsn: str) -> list[dict[str, Any]]:
+    """Read all source rows via Postgres (psycopg3, dict rows)."""
+    import psycopg
+
+    sql = adapter.source_query or f"SELECT * FROM {adapter.source_table}"
+    with psycopg.connect(source_dsn) as conn:
+        with conn.cursor() as cur:
+            cur.execute(sql)
+            cols = [d.name for d in cur.description]
+            return [dict(zip(cols, r)) for r in cur.fetchall()]
+
+
+def read_rows_csv(path: str | Path) -> list[dict[str, Any]]:
+    """Read source rows from a CSV (headers = column names). DB-free path."""
+    with open(path, encoding="utf-8-sig", newline="") as fh:
+        return [{k: v for k, v in row.items() if k is not None} for row in csv.DictReader(fh)]
+
+
+# ---------------------------------------------------------------------------
+# Coverage (asset-denominated)
+
+
+def count_source_rows(adapter: AdapterConfig, source_dsn: str) -> int:
+    """Actual indicator rows in the source table - the coverage denominator."""
+    import psycopg
+
+    table = adapter.source_table
+    with psycopg.connect(source_dsn) as conn:
+        with conn.cursor() as cur:
+            cur.execute(f"SELECT count(*) FROM {table}")
+            return cur.fetchone()[0]
+
+
+def coverage_report(
+    store,
+    actual_counts: Optional[dict[str, int]] = None,
+    scope: Optional[dict[str, dict[str, Any]]] = None,
+    module_map: Optional[dict[str, dict[str, list[str]]]] = None,
+) -> dict[str, Any]:
+    """Per-source registered/actual coverage + phase-2 visibility + unmapped audit.
+
+    ``actual_counts``: {source_db: actual rows} - from the source databases
+    themselves (``count_source_rows``) or supplied by the caller.
+    ``module_map``: {"module": {"source_dbs": [...]}} - registered entries not
+    matched by any module rule are listed as unmapped.
+    """
+    actual_counts = actual_counts or {}
+    scope = scope or DEFAULT_SCOPE
+    registered_by_db: dict[str, dict[str, int]] = {}
+    for (source_db, _source_table), counts in store.count_by_source().items():
+        bucket = registered_by_db.setdefault(source_db, {"registered": 0, "verified": 0})
+        bucket["registered"] += counts["registered"]
+        bucket["verified"] += counts["verified"]
+
+    sources: list[dict[str, Any]] = []
+    visible = (
+        set(registered_by_db)
+        | set(actual_counts)
+        | {s for s, v in scope.items() if v.get("phase") == 2}
+    )
+    for source_db in sorted(visible):
+        reg = registered_by_db.get(source_db, {"registered": 0, "verified": 0})
+        actual = actual_counts.get(source_db)
+        info = scope.get(source_db, {})
+        sources.append(
+            {
+                "source_db": source_db,
+                "label": info.get("label", ""),
+                "phase": info.get("phase", 1),
+                "registered": reg["registered"],
+                "verified": reg["verified"],
+                "unverified": reg["registered"] - reg["verified"],
+                "actual_in_source": actual,
+                "coverage": (
+                    round(reg["registered"] / actual, 4) if actual else None
+                ),
+            }
+        )
+
+    report: dict[str, Any] = {"sources": sources}
+    if module_map:
+        unmapped = []
+        covered_dbs = {db for m in module_map.values() for db in m.get("source_dbs", [])}
+        for entry in store.iter_entries():
+            if entry.source_db not in covered_dbs:
+                unmapped.append(
+                    {
+                        "source_db": entry.source_db,
+                        "native_code": entry.native_code,
+                        "semantic_code": entry.semantic_code,
+                        "name_zh": entry.name_zh,
+                    }
+                )
+        report["module_map"] = module_map
+        report["unmapped_count"] = len(unmapped)
+        report["unmapped_sample"] = unmapped[:200]
+    return report