fd-open-data-protocol 0.3.0__tar.gz → 0.3.1__tar.gz

{fd_open_data_protocol-0.3.0 → fd_open_data_protocol-0.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fd-open-data-protocol
-Version: 0.3.0
+Version: 0.3.1
 Summary: The open-data datasource protocol: a manifest contract a datasource exposes to be ingested by fd-open-data-mcp (or any consumer).
 Author: FindDataOfficial
 License: MIT
@@ -10,9 +10,13 @@ Requires-Dist: pydantic>=2.0
 Requires-Dist: pyyaml>=6.0
 Provides-Extra: dev
 Requires-Dist: pytest>=8.0; extra == "dev"
+Provides-Extra: registry
+Requires-Dist: psycopg[binary]>=3.1; extra == "registry"
 
 # fd-open-data-protocol
 
+> **Wire (柏讯) product line** · the open-data supply line of [FindData](https://www.finddatatech.cloud/products/wire) — the datasource manifest contract
+
 **English** | [中文](README.zh-CN.md)
 
 The **open-data datasource protocol**: a manifest contract a datasource exposes

{fd_open_data_protocol-0.3.0 → fd_open_data_protocol-0.3.1}/README.md

@@ -1,5 +1,7 @@
 # fd-open-data-protocol
 
+> **Wire (柏讯) product line** · the open-data supply line of [FindData](https://www.finddatatech.cloud/products/wire) — the datasource manifest contract
+
 **English** | [中文](README.zh-CN.md)
 
 The **open-data datasource protocol**: a manifest contract a datasource exposes

{fd_open_data_protocol-0.3.0 → fd_open_data_protocol-0.3.1}/fd_open_data_protocol/__init__.py

@@ -5,4 +5,4 @@ concept hints + fetch reference) so that fd-open-data-mcp - or any consumer -
 can ingest it via ``register_datasource``. Ship one manifest file -> the
 datasource is added; no consumer-side wiring.
 """
-__version__ = "0.3.0"
+__version__ = "0.3.1"

fd_open_data_protocol-0.3.1/fd_open_data_protocol/registry.py

@@ -0,0 +1,274 @@
+"""Declarative registry adapters and coverage computation.
+
+An adapter is a small YAML file mapping a source table's rows onto
+:class:`~fd_open_data_protocol.schema.RegistryEntry` fields - no core code
+changes when a new source database is onboarded. The recipe (see REGISTRY.md):
+
+    写适配器 → 批量登记（register）→ 概念绑定（bind）→ 命名门核验（verify）
+
+Row sources are independent of mapping: rows can come from a Postgres source
+DSN or a CSV file (tests, dry runs, the shipped template example), so the
+whole pipeline is exercisable without a database.
+
+Coverage is asset-denominated: per source "registered / actual in source",
+with the denominator taken from the source database (or supplied counts) -
+never from an MCP concept dictionary.
+"""
+from __future__ import annotations
+
+import csv
+import json
+from pathlib import Path
+from typing import Any, Iterable, Optional, Sequence
+
+import yaml
+from pydantic import BaseModel, Field, field_validator
+
+from fd_open_data_protocol.schema import RegistryEntry
+
+#: registry fields an adapter may map; anchors come from dedicated keys.
+MAPPABLE_FIELDS = ("semantic_code", "name_zh", "name_en", "unit", "frequency", "domain")
+
+#: sources known to the fleet but NOT in the first registration batch; the
+#: coverage view must show them explicitly instead of silently omitting them.
+DEFAULT_SCOPE: dict[str, dict[str, Any]] = {
+    "world_bank": {"phase": 1, "label": "世界银行 WDI（含宏观数据链路）"},
+    "yearbook_catalog": {"phase": 1, "label": "国家统计年鉴"},
+    "gta_panel": {"phase": 1, "label": "GTA 治理面板"},
+    "china_city_panel": {"phase": 1, "label": "中国城市面板"},
+    "law_db": {
+        "phase": 1,
+        "label": "法律库·文档域（法条/切片/指导案例为文档资产，非指标口径；登记 0 属预期，经 business-mcp 可查）",
+    },
+    "fd_open_data": {"phase": 1, "label": "fd_open_data 镜像概念字典"},
+    "ia35": {"phase": 2, "label": "工商注册 161G（实体为主，二期）"},
+    "wenshu_db": {"phase": 2, "label": "裁判文书（增量暂停，二期）"},
+}
+
+
+class FieldSource(BaseModel):
+    """Where one registry field's value comes from.
+
+    ``from: column``  - a column of the source row (or CSV header)
+    ``from: literal`` - a constant (e.g. ``domain: macro``)
+    ``from: map_file``- a JSON ``{native_code: value}`` lookup, typically a
+                        curated/LLM-drafted naming map (name translations,
+                        semantic codes); misses fall back to ``missing``
+    """
+
+    from_: str = Field(alias="from")
+    value: str
+    missing: Optional[str] = None  # value to store on a miss; default None = absent
+    uppercase: bool = False
+
+    @field_validator("from_")
+    @classmethod
+    def _known_kind(cls, v: str) -> str:
+        if v not in ("column", "literal", "map_file"):
+            raise ValueError(f"unknown field source kind: {v!r} (column|literal|map_file)")
+        return v
+
+    model_config = {"populate_by_name": True}
+
+
+class AdapterConfig(BaseModel):
+    """Declarative source-table → registry mapping."""
+
+    source_db: str
+    source_table: str
+    source_column: str = ""  # anchor label: the column the native code was read from
+    native_code_column: str
+    fields: dict[str, FieldSource]
+    provenance_columns: list[str] = Field(default_factory=list)
+    source_query: Optional[str] = None  # default: SELECT * FROM <source_table>
+    semantic_code_dedupe_suffix: str = "-%d"  # in-batch collision suffix (marked needs_review)
+
+    @field_validator("fields")
+    @classmethod
+    def _fields_known(cls, v: dict[str, FieldSource]) -> dict[str, FieldSource]:
+        unknown = set(v) - set(MAPPABLE_FIELDS)
+        if unknown:
+            raise ValueError(f"unknown mappable field(s): {sorted(unknown)}")
+        return v
+
+
+def load_adapter(source: str | Path | dict) -> AdapterConfig:
+    """Load an adapter from a YAML/JSON file path or an already-parsed dict."""
+    if isinstance(source, dict):
+        return AdapterConfig(**source)
+    p = Path(source)
+    data = yaml.safe_load(p.read_text(encoding="utf-8"))
+    if not isinstance(data, dict):
+        raise ValueError(f"adapter file did not parse to a mapping: {p}")
+    adapter = AdapterConfig(**data)
+    adapter._base_dir = p.parent  # type: ignore[attr-defined]  # resolve relative map_files
+    return adapter
+
+
+def _lookup(
+    spec: FieldSource, row: dict, native_code: str, base_dir: Path
+) -> Optional[str]:
+    value: Any
+    if spec.from_ == "literal":
+        value = spec.value
+    elif spec.from_ == "column":
+        value = row.get(spec.value)
+    else:  # map_file
+        map_path = (base_dir / spec.value) if not Path(spec.value).is_absolute() else Path(spec.value)
+        try:
+            mapping = json.loads(map_path.read_text(encoding="utf-8"))
+        except FileNotFoundError:
+            value = spec.missing
+            mapping = None
+        else:
+            value = mapping.get(native_code, spec.missing)
+    if value is None:
+        return None
+    text = str(value).strip()
+    if not text:
+        return None
+    return text.upper() if spec.uppercase else text
+
+
+def map_rows(
+    adapter: AdapterConfig, rows: Iterable[dict[str, Any]]
+) -> list[RegistryEntry]:
+    """Map source rows to registry entries (pure; no store access).
+
+    In-batch semantic_code collisions get a numeric suffix and a
+    ``needs_review: true`` provenance flag - never a silent merge.
+    """
+    base_dir = getattr(adapter, "_base_dir", Path("."))
+    seen_codes: dict[str, int] = {}
+    entries: list[RegistryEntry] = []
+    for row in rows:
+        native_code = str(row.get(adapter.native_code_column, "") or "").strip()
+        if not native_code:
+            raise ValueError(
+                f"row has empty {adapter.native_code_column!r}; adapter={adapter.source_db}.{adapter.source_table}"
+            )
+        fields: dict[str, Optional[str]] = {}
+        needs_review = False
+        for name, spec in adapter.fields.items():
+            value = _lookup(spec, row, native_code, base_dir)
+            if name == "semantic_code" and value is not None:
+                base = value
+                n = seen_codes.get(base, 0)
+                if n:
+                    value = base + (adapter.semantic_code_dedupe_suffix % (n + 1))
+                    needs_review = True
+                seen_codes[base] = n + 1
+            fields[name] = value
+        provenance = {c: row.get(c) for c in adapter.provenance_columns if c in row}
+        if needs_review:
+            provenance["needs_review"] = True
+        entries.append(
+            RegistryEntry(
+                source_db=adapter.source_db,
+                source_table=adapter.source_table,
+                source_column=adapter.source_column or adapter.native_code_column,
+                native_code=native_code,
+                provenance=provenance or None,
+                **fields,
+            )
+        )
+    return entries
+
+
+def fetch_rows(adapter: AdapterConfig, source_dsn: str) -> list[dict[str, Any]]:
+    """Read all source rows via Postgres (psycopg3, dict rows)."""
+    import psycopg
+
+    sql = adapter.source_query or f"SELECT * FROM {adapter.source_table}"
+    with psycopg.connect(source_dsn) as conn:
+        with conn.cursor() as cur:
+            cur.execute(sql)
+            cols = [d.name for d in cur.description]
+            return [dict(zip(cols, r)) for r in cur.fetchall()]
+
+
+def read_rows_csv(path: str | Path) -> list[dict[str, Any]]:
+    """Read source rows from a CSV (headers = column names). DB-free path."""
+    with open(path, encoding="utf-8-sig", newline="") as fh:
+        return [{k: v for k, v in row.items() if k is not None} for row in csv.DictReader(fh)]
+
+
+# ---------------------------------------------------------------------------
+# Coverage (asset-denominated)
+
+
+def count_source_rows(adapter: AdapterConfig, source_dsn: str) -> int:
+    """Actual indicator rows in the source table - the coverage denominator."""
+    import psycopg
+
+    table = adapter.source_table
+    with psycopg.connect(source_dsn) as conn:
+        with conn.cursor() as cur:
+            cur.execute(f"SELECT count(*) FROM {table}")
+            return cur.fetchone()[0]
+
+
+def coverage_report(
+    store,
+    actual_counts: Optional[dict[str, int]] = None,
+    scope: Optional[dict[str, dict[str, Any]]] = None,
+    module_map: Optional[dict[str, dict[str, list[str]]]] = None,
+) -> dict[str, Any]:
+    """Per-source registered/actual coverage + phase-2 visibility + unmapped audit.
+
+    ``actual_counts``: {source_db: actual rows} - from the source databases
+    themselves (``count_source_rows``) or supplied by the caller.
+    ``module_map``: {"module": {"source_dbs": [...]}} - registered entries not
+    matched by any module rule are listed as unmapped.
+    """
+    actual_counts = actual_counts or {}
+    scope = scope or DEFAULT_SCOPE
+    registered_by_db: dict[str, dict[str, int]] = {}
+    for (source_db, _source_table), counts in store.count_by_source().items():
+        bucket = registered_by_db.setdefault(source_db, {"registered": 0, "verified": 0})
+        bucket["registered"] += counts["registered"]
+        bucket["verified"] += counts["verified"]
+
+    sources: list[dict[str, Any]] = []
+    visible = (
+        set(registered_by_db)
+        | set(actual_counts)
+        | {s for s, v in scope.items() if v.get("phase") == 2}
+    )
+    for source_db in sorted(visible):
+        reg = registered_by_db.get(source_db, {"registered": 0, "verified": 0})
+        actual = actual_counts.get(source_db)
+        info = scope.get(source_db, {})
+        sources.append(
+            {
+                "source_db": source_db,
+                "label": info.get("label", ""),
+                "phase": info.get("phase", 1),
+                "registered": reg["registered"],
+                "verified": reg["verified"],
+                "unverified": reg["registered"] - reg["verified"],
+                "actual_in_source": actual,
+                "coverage": (
+                    round(reg["registered"] / actual, 4) if actual else None
+                ),
+            }
+        )
+
+    report: dict[str, Any] = {"sources": sources}
+    if module_map:
+        unmapped = []
+        covered_dbs = {db for m in module_map.values() for db in m.get("source_dbs", [])}
+        for entry in store.iter_entries():
+            if entry.source_db not in covered_dbs:
+                unmapped.append(
+                    {
+                        "source_db": entry.source_db,
+                        "native_code": entry.native_code,
+                        "semantic_code": entry.semantic_code,
+                        "name_zh": entry.name_zh,
+                    }
+                )
+        report["module_map"] = module_map
+        report["unmapped_count"] = len(unmapped)
+        report["unmapped_sample"] = unmapped[:200]
+    return report

fd_open_data_protocol-0.3.1/fd_open_data_protocol/registry_cli.py

@@ -0,0 +1,239 @@
+"""CLI for the unified indicator registry.
+
+    python -m fd_open_data_protocol.registry_cli <command> ...
+
+Commands:
+  init-db     create the registry_entries table on the registry database
+  register    map source rows through an adapter and batch-upsert (idempotent
+              by native-code anchor; re-runs report "unchanged", new rows are
+              counted as increments)
+  verify      mark gate-passing entries verified (rule + printed sample)
+  coverage    per-source registered/actual report (+ phase-2 visibility,
+              + unmapped-by-module audit)
+  export      dump entries to JSON (verified-only by default) for consumers
+              (fd-open-data-mcp catalog, official-web db_stats)
+
+DSN defaults come from $FD_REGISTRY_DSN (fd_open_data on the data machine)
+and $FD_SOURCE_DSN (the source database being registered). ``--rows CSV``
+replaces the source database entirely - tests and the shipped template
+example run the full pipeline without any DB.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import random
+import sys
+from datetime import datetime
+from pathlib import Path
+from typing import Optional
+
+from fd_open_data_protocol.registry import (
+    count_source_rows,
+    coverage_report,
+    fetch_rows,
+    load_adapter,
+    map_rows,
+    read_rows_csv,
+)
+from fd_open_data_protocol.registry_store import (
+    BaseRegistryStore,
+    MemoryRegistryStore,
+    PostgresRegistryStore,
+)
+
+
+def _registry_store(args: argparse.Namespace) -> BaseRegistryStore:
+    if getattr(args, "dry_run", False):
+        return MemoryRegistryStore()
+    dsn = args.registry_dsn or os.environ.get("FD_REGISTRY_DSN")
+    if not dsn:
+        sys.exit("error: --registry-dsn or $FD_REGISTRY_DSN is required")
+    return PostgresRegistryStore(dsn)
+
+
+def _load_adapters(paths: list[str]):
+    adapters = [load_adapter(p) for p in paths]
+    for a in adapters:
+        print(f"adapter: {a.source_db}.{a.source_table} (code column: {a.native_code_column})")
+    return adapters
+
+
+def _print_sample(store: BaseRegistryStore, source_db: str, sample: int) -> None:
+    """The sampling half of the "rule + sample" verification record."""
+    entries = [e for e in store.iter_entries(verified_only=True, source_db=source_db)]
+    if not entries:
+        print(f"verify sample [{source_db}]: no verified entries to sample")
+        return
+    picked = random.sample(entries, min(sample, len(entries)))
+    print(f"verify sample [{source_db}] ({len(picked)}/{len(entries)} verified):")
+    for e in picked:
+        print(
+            f"  {e.semantic_code}  |  {e.native_code}  |  zh={e.name_zh}  |  en={e.name_en}"
+        )
+
+
+def cmd_init_db(args: argparse.Namespace) -> int:
+    store = _registry_store(args)
+    store.init_schema()
+    print("registry_entries table ready (CREATE TABLE IF NOT EXISTS applied)")
+    return 0
+
+
+def cmd_register(args: argparse.Namespace) -> int:
+    store = _registry_store(args)
+    adapters = _load_adapters(args.adapter)
+    grand = {"new": 0, "updated": 0, "unchanged": 0, "demoted": 0}
+    for adapter in adapters:
+        if args.rows:
+            rows = read_rows_csv(args.rows)
+        else:
+            dsn = args.source_dsn or os.environ.get("FD_SOURCE_DSN")
+            if not dsn:
+                sys.exit(
+                    f"error: adapter {adapter.source_db}.{adapter.source_table} needs "
+                    "--source-dsn or $FD_SOURCE_DSN (or --rows CSV)"
+                )
+            rows = fetch_rows(adapter, dsn)
+        entries = map_rows(adapter, rows)
+        stats = store.upsert_entries(entries)
+        gate_passing = sum(
+            1
+            for e in entries
+            if e.semantic_code and e.name_zh and e.name_en
+        )
+        verified_now = 0
+        if args.auto_verify and not args.dry_run:
+            verified_now = store.set_verified(
+                source_db=adapter.source_db, verified_by=f"rule:naming-gate"
+            )
+        print(
+            f"{adapter.source_db}.{adapter.source_table}: source_rows={len(rows)} "
+            f"registered={stats.new} updated={stats.updated} unchanged={stats.unchanged} "
+            f"demoted={stats.demoted} | gate_passing={gate_passing} "
+            f"verified_now={verified_now}"
+        )
+        for key in grand:
+            grand[key] += getattr(stats, key)
+        if args.auto_verify and not args.dry_run:
+            _print_sample(store, adapter.source_db, args.sample)
+    print(json.dumps(grand, ensure_ascii=False))
+    return 0
+
+
+def cmd_verify(args: argparse.Namespace) -> int:
+    store = _registry_store(args)
+    count = store.set_verified(
+        source_db=args.source_db, verified_by=args.by or "rule:naming-gate"
+    )
+    print(f"newly verified: {count}" + (f" (source_db={args.source_db})" if args.source_db else ""))
+    if args.source_db:
+        _print_sample(store, args.source_db, args.sample)
+    return 0
+
+
+def cmd_coverage(args: argparse.Namespace) -> int:
+    store = _registry_store(args)
+    actual_counts: dict[str, int] = {}
+    if args.actual_json:
+        actual_counts = json.loads(Path(args.actual_json).read_text(encoding="utf-8"))
+    elif args.adapter:
+        for adapter in _load_adapters(args.adapter):
+            dsn = args.source_dsn or os.environ.get("FD_SOURCE_DSN")
+            if not dsn:
+                sys.exit("error: coverage with --adapter needs --source-dsn or $FD_SOURCE_DSN")
+            actual_counts[adapter.source_db] = count_source_rows(adapter, dsn)
+    module_map = (
+        json.loads(Path(args.module_map).read_text(encoding="utf-8"))
+        if args.module_map
+        else None
+    )
+    report = coverage_report(store, actual_counts, module_map=module_map)
+    text = json.dumps(report, ensure_ascii=False, indent=2)
+    if args.out:
+        Path(args.out).write_text(text, encoding="utf-8")
+        print(f"coverage report written: {args.out}")
+    print(text)
+    return 0
+
+
+def cmd_export(args: argparse.Namespace) -> int:
+    store = _registry_store(args)
+    entries = [
+        e.model_dump()
+        for e in store.iter_entries(verified_only=not args.all, source_db=args.source_db)
+    ]
+    payload = {
+        "as_of": datetime.now().astimezone().isoformat(timespec="seconds"),
+        "verified_only": not args.all,
+        "count": len(entries),
+        "entries": entries,
+    }
+    text = json.dumps(payload, ensure_ascii=False, indent=1)
+    if args.out:
+        Path(args.out).write_text(text, encoding="utf-8")
+        print(f"exported {len(entries)} entries -> {args.out}")
+    else:
+        print(text)
+    return 0
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="fd-registry", description="Unified indicator registry CLI"
+    )
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    def common(p: argparse.ArgumentParser, registry: bool = True) -> None:
+        if registry:
+            p.add_argument("--registry-dsn", help="registry DB DSN (default $FD_REGISTRY_DSN)")
+        p.add_argument("--dry-run", action="store_true", help="use in-memory store, write nothing")
+
+    p = sub.add_parser("init-db", help="create registry_entries")
+    common(p)
+    p.set_defaults(func=cmd_init_db)
+
+    p = sub.add_parser("register", help="batch-register one or more adapters")
+    common(p)
+    p.add_argument("--adapter", action="append", required=True, help="adapter YAML (repeatable)")
+    p.add_argument("--source-dsn", help="source DB DSN (default $FD_SOURCE_DSN)")
+    p.add_argument("--rows", help="CSV rows file instead of a live source DB")
+    p.add_argument("--auto-verify", action="store_true",
+                   help="after upsert, verify all gate-passing entries (rule:naming-gate)")
+    p.add_argument("--sample", type=int, default=5, help="sample size printed as verify record")
+    p.set_defaults(func=cmd_register)
+
+    p = sub.add_parser("verify", help="mark gate-passing entries verified")
+    common(p)
+    p.add_argument("--source-db", help="restrict to one source_db")
+    p.add_argument("--by", help="verification record (default rule:naming-gate)")
+    p.add_argument("--sample", type=int, default=5)
+    p.set_defaults(func=cmd_verify)
+
+    p = sub.add_parser("coverage", help="registered/actual coverage report")
+    common(p)
+    p.add_argument("--adapter", action="append", help="adapter(s) to count actual source rows")
+    p.add_argument("--source-dsn", help="source DB DSN for actual counts")
+    p.add_argument("--actual-json", help="JSON {source_db: count} instead of live counts")
+    p.add_argument("--module-map", help="JSON module map for unmapped audit")
+    p.add_argument("--out", help="also write the report to this file")
+    p.set_defaults(func=cmd_coverage)
+
+    p = sub.add_parser("export", help="dump entries (verified-only by default)")
+    common(p)
+    p.add_argument("--out", help="output JSON file (default stdout)")
+    p.add_argument("--all", action="store_true", help="include unverified entries")
+    p.add_argument("--source-db", help="restrict to one source_db")
+    p.set_defaults(func=cmd_export)
+
+    return parser
+
+
+def main(argv: Optional[list] = None) -> int:
+    args = build_parser().parse_args(argv)
+    return args.func(args)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())