govio 0.2.11__py3-none-any.whl

govio/__init__.py

@@ -0,0 +1,10 @@
+
+from .graph.falkordb_graph import FalkorDBGraph
+from .graph.networkx_graph import NetworkXGraph
+from .cli import main
+
+__all__ = [
+    "FalkorDBGraph",
+    "NetworkXGraph",
+    "main",
+]

govio/cli/__init__.py

@@ -0,0 +1,4 @@
+from .config import ConfigManager
+from .main import main
+
+__all__ = ["ConfigManager", "main"]

govio/cli/config.py

@@ -0,0 +1,136 @@
+import shutil
+import yaml
+from pathlib import Path
+from typing import Any
+
+# 旧格式中属于 metadata section 的字段
+_METADATA_KEYS = {"kundb", "workspace_uuid", "app_list", "app_map", "relationship", "metric", "csv_dir"}
+# 旧格式中属于 graph section 的字段
+_GRAPH_KEYS = {"backend", "networkx", "falkordb"}
+
+
+class ConfigManager:
+    """管理 govio 配置文件"""
+
+    def __init__(self, config_path: Path | None = None) -> None:
+        if config_path is None:
+            self.config_path = Path.home() / ".govio" / "config.yaml"
+        else:
+            self.config_path = config_path
+
+        self.config_path.parent.mkdir(parents=True, exist_ok=True)
+
+    def exists(self) -> bool:
+        """检查配置文件是否存在"""
+        return self.config_path.exists()
+
+    def load(self) -> dict[str, Any]:
+        """加载配置文件，自动迁移旧格式"""
+        if not self.exists():
+            raise FileNotFoundError(f"配置文件不存在: {self.config_path}")
+
+        with open(self.config_path, "r", encoding="utf-8") as f:
+            config = yaml.safe_load(f) or {}
+
+        if self._is_old_format(config):
+            config = self._migrate(config)
+
+        return config
+
+    def save(self, config: dict[str, Any]) -> None:
+        """保存配置文件"""
+        with open(self.config_path, "w", encoding="utf-8") as f:
+            yaml.dump(config, f, allow_unicode=True, default_flow_style=False)
+
+    def _is_old_format(self, config: dict[str, Any]) -> bool:
+        """检测是否为旧的扁平格式"""
+        return "kundb" in config or ("backend" in config and "graph" not in config)
+
+    def _migrate(self, config: dict[str, Any]) -> dict[str, Any]:
+        """将旧扁平格式迁移为新的嵌套格式"""
+        backup_path = self.config_path.with_suffix(".yaml.bak")
+        shutil.copy2(self.config_path, backup_path)
+
+        new_config: dict[str, Any] = {}
+        known_keys = _METADATA_KEYS | _GRAPH_KEYS | {"datasources"}
+
+        metadata = {}
+        for key in _METADATA_KEYS:
+            if key in config:
+                metadata[key] = config[key]
+        if metadata:
+            new_config["metadata"] = metadata
+
+        graph = {}
+        for key in _GRAPH_KEYS:
+            if key in config:
+                graph[key] = config[key]
+        if graph:
+            new_config["graph"] = graph
+
+        if "datasources" in config:
+            new_config["datasources"] = config["datasources"]
+
+        # 保留未知字段
+        for key, value in config.items():
+            if key not in known_keys and key not in new_config:
+                new_config[key] = value
+
+        self.save(new_config)
+
+        return new_config
+
+    @staticmethod
+    def _validate_backend(scope: dict[str, Any]) -> None:
+        """验证 backend 相关配置（networkx/falkordb）"""
+        if "backend" not in scope:
+            raise ValueError("配置缺少 'backend' 字段")
+        backend = scope["backend"]
+        if backend not in ["networkx", "falkordb"]:
+            raise ValueError(f"不支持的 backend: {backend}")
+        if backend == "networkx":
+            if "networkx" not in scope:
+                raise ValueError("NetworkX backend 需要 'networkx' 配置")
+            if "gml_path" not in scope["networkx"]:
+                raise ValueError("NetworkX 配置缺少 'gml_path' 字段")
+        elif backend == "falkordb":
+            if "falkordb" not in scope:
+                raise ValueError("FalkorDB backend 需要 'falkordb' 配置")
+            for field in ["host", "port", "graph"]:
+                if field not in scope["falkordb"]:
+                    raise ValueError(f"FalkorDB 配置缺少 '{field}' 字段")
+
+    def validate(self, config: dict[str, Any]) -> bool:
+        """验证配置的有效性
+
+        支持新格式（嵌套）和旧格式（扁平）的验证。
+        """
+        if "graph" in config:
+            self._validate_backend(config["graph"])
+        elif "backend" in config:
+            self._validate_backend(config)
+        else:
+            raise ValueError("配置缺少 'backend' 字段")
+
+        csv_dir = config.get("metadata", {}).get("csv_dir") or config.get("csv_dir")
+        if csv_dir:
+            csv_path = Path(csv_dir)
+            if not csv_path.exists():
+                raise ValueError(f"csv_dir 不存在: {csv_path}")
+
+        if "graph_dir" in config:
+            graph_path = Path(config["graph_dir"])
+            if not graph_path.exists():
+                raise ValueError(f"graph_dir 不存在: {graph_path}")
+
+        datasources = config.get("datasources")
+        if datasources:
+            if not isinstance(datasources, dict):
+                raise ValueError("datasources 必须为字典类型")
+            for name, ds_data in datasources.items():
+                if not isinstance(ds_data, dict):
+                    raise ValueError(f"数据源 '{name}' 配置必须为字典类型")
+                if "url" not in ds_data:
+                    raise ValueError(f"数据源 '{name}' 缺少 'url' 字段")
+
+        return True

govio/cli/main.py

@@ -0,0 +1,106 @@
+import argparse
+from pathlib import Path
+import sys
+
+from govio.cli.config import ConfigManager
+
+from .meta_export import meta_export
+from .onboard import onboard
+from .std_recommend import std_recommend
+from .observe import observe
+from .query import query
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        description="数据治理知识图谱项目，提供元数据查询、表字段比较、SQL 生成、数据标准推荐等数据治理支持功能。",
+    )
+    sub = parser.add_subparsers(dest="action")
+
+    p_onboard = sub.add_parser("onboard", help="初始化配置向导")
+    p_onboard.add_argument(
+        "--new-falkordb",
+        type=Path,
+        metavar="CSV_DIR",
+        help="跳过 CSV 生成，直接将指定目录的 CSV 导入 FalkorDB",
+    )
+    p_onboard.add_argument(
+        "--new-networkx",
+        type=Path,
+        metavar="CSV_DIR",
+        help="跳过 CSV 生成，直接从指定目录的 CSV 生成 GML 文件",
+    )
+    sub.add_parser("backend", help="显示当前后端类型")
+
+    p_std = sub.add_parser("std-recommend", help="数据标准推荐")
+    p_std.add_argument("--output-dir", type=Path, help="推荐数据标准的输出目录")
+
+    # query 子命令
+    p_query = sub.add_parser("query", help="知识图谱查询")
+    code_type = "NetworkX 用 Python 代码，FalkorDB 用 Cypher"
+    config_manager = ConfigManager()
+    if config_manager.exists():
+        config = config_manager.load()
+        backend = (config.get("graph") or {}).get("backend")
+        if backend == "falkordb":
+            code_type = "Cypher"
+        elif backend == "networkx":
+            code_type = "Python 代码"
+
+    p_query.add_argument(
+        "-c",
+        "--code",
+        help=f"查询语句（{code_type}）",
+    )
+
+    # meta-export 子命令：从 DuckDB 导出元数据 CSV
+    p_meta = sub.add_parser("meta-export", help="从 DuckDB + TDS 合并导出全量元数据 CSV")
+    p_meta.add_argument("--db", type=str, required=True, help="DuckDB 数据库文件路径")
+    p_meta.add_argument(
+        "--schemas",
+        type=str,
+        required=True,
+        help="要导出的 schema 列表，逗号分隔（如 dm,dwd,dws）",
+    )
+    p_meta.add_argument("--output", type=Path, required=True, help="CSV 输出目录")
+    p_meta.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="仅生成 CSV 并输出状态，不更新图数据和生成 assets",
+    )
+
+    # observe 子命令：保留未知参数传递给 observe()
+    p_observe = sub.add_parser("observe", help="数据表探查")
+    p_observe.add_argument(
+        "observe_args", nargs=argparse.REMAINDER, help="observe 子命令参数"
+    )
+
+    args = parser.parse_args()
+
+    if args.action == "onboard":
+        onboard(new_falkordb=args.new_falkordb, new_networkx=args.new_networkx)
+    elif args.action == "backend":
+        config_manager = ConfigManager()
+        if not config_manager.exists():
+            print("错误: 未找到配置文件，请先运行 govio-cli onboard", file=sys.stderr)
+            sys.exit(1)
+        config = config_manager.load()
+        backend = (config.get("graph") or {}).get("backend")
+        if not backend:
+            print("错误: 配置文件中未设置后端类型", file=sys.stderr)
+            sys.exit(1)
+        print(backend)
+    elif args.action == "std-recommend":
+        std_recommend(args.output_dir)
+    elif args.action == "query":
+        query(args.code)
+    elif args.action == "meta-export":
+        meta_export(args.db, args.schemas.split(","), args.output, dry_run=args.dry_run)
+    elif args.action == "observe":
+        # 将 observe 子命令参数设为 sys.argv 供 observe() 解析
+        sys.argv = ["govio-cli"] + args.observe_args
+        observe()
+    else:
+        parser.print_help()
+        sys.exit(1)

govio/cli/meta_export.py

@@ -0,0 +1,263 @@
+import sys
+from pathlib import Path
+
+import pandas as pd
+
+from govio.cli.config import ConfigManager
+from govio.core.graph_factory import GraphFactory
+from govio.core.assets_generator import AssetsGenerator
+from govio.metadata.database import TDSLoader
+from govio.metadata.application import AppInfoLoader
+from govio.metadata.standard import StandardLoader
+from govio.metadata.duckdb_loader import DuckDBLoader
+from govio.metadata.utility import reorder_index
+from govio.metadata.relationship import load_relationships
+from govio.metadata.metric import MetricLoader
+
+SKILLS_ASSETS_DIR = Path("skills/govio/assets")
+
+
+def merge_metadata(
+    df_tds: pd.DataFrame, df_duck: pd.DataFrame, key: str
+) -> pd.DataFrame:
+    """TDS full + DuckDB incremental. DuckDB wins on conflict."""
+    combined = pd.concat([df_tds, df_duck], ignore_index=True)
+    return combined.drop_duplicates(subset=[key], keep="last").reset_index(drop=True)
+
+
+def meta_export(db_path: str, schemas: list[str], output: Path, dry_run: bool = True):
+    output.mkdir(parents=True, exist_ok=True)
+
+    # --- Load config for TDS ---
+    config = ConfigManager().load()
+    metadata = config.get("metadata") or {}
+    kundb = metadata.get("kundb", "")
+    workspace_uuid = metadata.get("workspace_uuid", "82ee37374b314a938bf28170ab4db7cf")
+    app_list_file = metadata.get("app_list", "")
+    app_map_file = metadata.get("app_map", "")
+    relationship_file = metadata.get("relationship")
+    metric_file = metadata.get("metric")
+
+    if not all([kundb, app_list_file, app_map_file]):
+        print("❌ 配置缺少必要字段，请检查 metadata 中的 kundb, app_list, app_map")
+        sys.exit(1)
+
+    df_app_db_map = pd.read_json(app_map_file, orient="records")
+
+    # --- Load TDS metadata ---
+    tds_loader = TDSLoader(kundb, workspace_uuid, df_app_db_map["schema"].to_list())
+    tds_tables = tds_loader.PhysicalTable
+    tds_columns = tds_loader.Col
+
+    # --- Load DuckDB metadata ---
+    duck_loader = DuckDBLoader(db_path, schemas)
+    duck_tables = duck_loader.PhysicalTable
+    duck_columns = duck_loader.Col
+
+    # --- Merge ---
+    df_tables = merge_metadata(tds_tables, duck_tables, "full_table_name")
+    df_columns = merge_metadata(tds_columns, duck_columns, "column")
+
+    # --- Load apps and standards ---
+    app_loader = AppInfoLoader(app_list_file, df_app_db_map["name"].to_list())
+    df_apps = app_loader.Application
+    std_loader = StandardLoader(kundb, workspace_uuid)
+    df_stds = std_loader.Standard
+
+    # --- Assign IDs ---
+    reorder_index([df_tables, df_columns, df_apps, df_stds], start=1)
+
+    files = []
+
+    # --- Node CSVs ---
+    df_tables.to_csv(output / "PhysicalTable.csv", index_label=":ID(PhysicalTable)")
+    files.append("-n " + str(output / "PhysicalTable.csv"))
+
+    df_columns.to_csv(output / "Col.csv", index_label=":ID(Col)")
+    files.append("-n " + str(output / "Col.csv"))
+
+    df_apps.to_csv(output / "Application.csv", index_label=":ID(Application)")
+    files.append("-n " + str(output / "Application.csv"))
+
+    df_stds.to_csv(output / "Standard.csv", index_label=":ID(Standard)")
+    files.append("-n " + str(output / "Standard.csv"))
+
+    # --- HAS_COLUMN edge ---
+    df_has_column = pd.merge(
+        df_tables[["full_table_name"]]
+        .reset_index()
+        .rename(columns={"index": ":START_ID(PhysicalTable)"}),
+        df_columns[["full_table_name"]]
+        .reset_index()
+        .rename(columns={"index": ":END_ID(Col)"}),
+        on="full_table_name",
+        how="inner",
+    )[[":START_ID(PhysicalTable)", ":END_ID(Col)"]]
+    df_has_column.to_csv(output / "HAS_COLUMN.csv", index=False)
+    files.append("-r " + str(output / "HAS_COLUMN.csv"))
+
+    # --- USE edge ---
+    df_app_table = pd.merge(
+        df_app_db_map,
+        df_tables[["schema"]]
+        .reset_index()
+        .rename(columns={"index": ":END_ID(PhysicalTable)"}),
+        on="schema",
+        how="inner",
+    )
+    df_use = pd.merge(
+        df_apps[["name"]]
+        .reset_index()
+        .rename(columns={"index": ":START_ID(Application)"}),
+        df_app_table,
+        on="name",
+        how="inner",
+    )[[":START_ID(Application)", ":END_ID(PhysicalTable)"]]
+    df_use.to_csv(output / "USE.csv", index=False)
+    files.append("-r " + str(output / "USE.csv"))
+
+    # --- Optional: RELATES_TO ---
+    relations_count = 0
+    if relationship_file:
+        try:
+            df_relates_to = load_relationships(relationship_file, df_tables, df_columns)
+            relations_count = len(df_relates_to)
+            df_relates_to.to_csv(
+                output / "RELATES_TO.csv",
+                index=False,
+                header=[
+                    ":START_ID(PhysicalTable)",
+                    ":END_ID(PhysicalTable)",
+                    "relationship_type",
+                    "description",
+                    "source_columns",
+                    "target_columns",
+                ],
+            )
+            files.append("-r " + str(output / "RELATES_TO.csv"))
+            print(f"成功生成 RELATES_TO.csv，包含 {len(df_relates_to)} 个关系 来自[{relationship_file}]")
+        except Exception as e:
+            print(f"警告: 无法加载关系文件: {e}")
+
+    # --- Optional: metrics ---
+    metric_count = 0
+    if metric_file:
+        try:
+            metric_loader = MetricLoader(metric_file, df_tables, df_columns)
+            df_metrics = metric_loader.Metric
+            df_dimensions = metric_loader.Dimension
+
+            # 计算 Metric/Dimension 的 ID 起始偏移（接续已有节点）
+            metric_offset = (
+                len(df_tables) + len(df_columns) + len(df_apps) + len(df_stds) + 1
+            )
+            dim_offset = metric_offset + len(df_metrics)
+            reorder_index([df_metrics, df_dimensions], start=metric_offset)
+
+            df_metrics.to_csv(output / "Metric.csv", index_label=":ID(Metric)")
+            files.append("-n " + str(output / "Metric.csv"))
+
+            df_dimensions.to_csv(
+                output / "Dimension.csv", index_label=":ID(Dimension)"
+            )
+            files.append("-n " + str(output / "Dimension.csv"))
+
+            # USES_TABLE 边
+            uses_table = metric_loader.uses_table_edges.copy()
+            if not uses_table.empty:
+                uses_table[":START_ID(Metric)"] += metric_offset
+                uses_table.to_csv(output / "USES_TABLE.csv", index=False)
+                files.append("-r " + str(output / "USES_TABLE.csv"))
+
+            # REFERS_COLUMN 边
+            refers_col = metric_loader.refers_column_edges.copy()
+            if not refers_col.empty:
+                refers_col[":START_ID(Metric)"] += metric_offset
+                refers_col.to_csv(output / "REFERS_COLUMN.csv", index=False)
+                files.append("-r " + str(output / "REFERS_COLUMN.csv"))
+
+            # DERIVED_FROM 边
+            derived_from = metric_loader.derived_from_edges.copy()
+            if not derived_from.empty:
+                derived_from[":START_ID(Metric)"] += metric_offset
+                derived_from[":END_ID(Metric)"] += metric_offset
+                derived_from.to_csv(output / "DERIVED_FROM.csv", index=False)
+                files.append("-r " + str(output / "DERIVED_FROM.csv"))
+
+            # DIMENSION_USED 边
+            dim_used = metric_loader.dimension_used_edges.copy()
+            if not dim_used.empty:
+                dim_used[":START_ID(Metric)"] += metric_offset
+                dim_used[":END_ID(Dimension)"] += dim_offset
+                dim_used.to_csv(output / "DIMENSION_USED.csv", index=False)
+                files.append("-r " + str(output / "DIMENSION_USED.csv"))
+
+            # SUPERSEDES 边
+            supersedes = metric_loader.supersedes_edges
+            if not supersedes.empty:
+                supersedes.to_csv(output / "SUPERSEDES.csv", index=False)
+                files.append("-r " + str(output / "SUPERSEDES.csv"))
+
+            print(
+                f"成功生成指标数据：{len(df_metrics)} 个指标, "
+                f"{len(df_dimensions)} 个维度"
+            )
+            metric_count = len(df_metrics)
+        except Exception as e:
+            print(f"警告: 无法加载指标定义文件: {e}")
+
+    # --- Summary ---
+    print(f"成功导出: {len(df_tables)} 张表, {len(df_columns)} 个字段, "
+          f"{len(df_apps)} 个应用, {len(df_stds)} 个标准, {relations_count}个数据关系, {metric_count}个指标")
+    # print(f"ID 范围: 1 ~ {len(df_tables) + len(df_columns) + len(df_apps) + len(df_stds)}")
+    print(f"\nfalkordb-bulk-insert {{GRAPH}} {'  '.join(files)}")
+
+    if dry_run:
+        return
+
+    # --- Update graph and generate assets ---
+    from govio.cli.onboard import import_csv_to_falkordb
+    from govio.metadata.gen_networkx import build_graph
+
+    graph = config.get("graph") or {}
+    backend = graph.get("backend")
+    if not backend:
+        print("警告: 配置中未指定 backend，跳过图数据更新和 assets 生成")
+        return
+
+    # Update graph
+    if backend == "falkordb":
+        falkordb_cfg = graph.get("falkordb", {})
+        host = falkordb_cfg.get("host", "localhost")
+        port = falkordb_cfg.get("port", 6379)
+        graph_name = falkordb_cfg.get("graph", "ontology")
+        print(f"\n正在导入 CSV 到 FalkorDB ({host}:{port}/{graph_name})...")
+        try:
+            import_csv_to_falkordb(output, host, port, graph_name)
+            print("✓ FalkorDB 数据已更新")
+        except Exception as e:
+            print(f"❌ 导入 FalkorDB 失败: {e}")
+            return
+    elif backend == "networkx":
+        networkx_cfg = graph.get("networkx", {})
+        gml_path = networkx_cfg.get("gml_path", str(SKILLS_ASSETS_DIR / "ontology.gml"))
+        print(f"\n正在从 CSV 生成 GML 文件 ({gml_path})...")
+        try:
+            build_graph(str(output), gml_path)
+            print("✓ GML 文件已更新")
+        except Exception as e:
+            print(f"❌ 生成 GML 失败: {e}")
+            return
+
+    # Generate assets
+    print("\n正在生成 assets...")
+    try:
+        graph_obj = GraphFactory.create(graph)
+        generator = AssetsGenerator(graph_obj, SKILLS_ASSETS_DIR)
+        generator.generate_all()
+        print(f"✓ Assets 已生成到: {SKILLS_ASSETS_DIR}")
+    except Exception as e:
+        print(f"❌ 生成 assets 失败: {e}")
+        return
+
+    print("\n✅ meta-export 完成！")