kgmodule-utils 0.2.0__py3-none-any.whl

kg_utils/snapshots/models.py

@@ -0,0 +1,137 @@
+"""kg_utils/snapshots/models.py — Snapshot data models.
+
+Every snapshot is keyed by git tree hash and contains:
+  - Timestamp and branch metadata
+  - Metrics dict (domain-flexible: total_nodes, total_edges, node_counts, ...)
+  - Hotspots list and issues list
+  - Deltas vs. previous and baseline snapshots
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass
+class Snapshot:
+    """A temporal snapshot of KG metrics.
+
+    ``metrics`` is a free-form dict so that each domain can store whatever
+    fields it needs (docstring_coverage, total_files, etc.) without requiring
+    changes to this shared data model.  The only required keys are
+    ``total_nodes`` and ``total_edges`` -- the manager uses these for delta
+    computation.
+
+    ``vs_previous`` and ``vs_baseline`` are also free-form dicts so that
+    domain-specific delta fields (coverage_delta, files_delta, ...) can be
+    stored alongside the universal ``nodes`` and ``edges`` deltas.
+    """
+
+    branch: str
+    timestamp: str  # ISO 8601 UTC
+    metrics: dict[str, Any]
+    version: str = ""
+    hotspots: list[dict[str, Any]] = field(default_factory=list)
+    issues: list[str] = field(default_factory=list)
+    vs_previous: dict[str, Any] | None = None
+    vs_baseline: dict[str, Any] | None = None
+    tree_hash: str = ""
+
+    @property
+    def key(self) -> str:
+        """Stable file key: git tree hash."""
+        return self.tree_hash
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to a JSON-serializable dictionary."""
+        return {
+            "key": self.tree_hash,
+            "branch": self.branch,
+            "timestamp": self.timestamp,
+            "version": self.version,
+            "metrics": self.metrics,
+            "hotspots": self.hotspots,
+            "issues": self.issues,
+            "vs_previous": self.vs_previous,
+            "vs_baseline": self.vs_baseline,
+        }
+
+    @staticmethod
+    def from_dict(data: dict[str, Any]) -> Snapshot:
+        """Reconstruct a Snapshot from a dictionary loaded from JSON."""
+        raw = dict(data)  # shallow copy to avoid mutating caller's data
+
+        metrics = raw.pop("metrics", {})
+        vs_prev = raw.pop("vs_previous", None)
+        vs_base = raw.pop("vs_baseline", None)
+
+        # Normalise legacy 'tree_hash' field -> 'key'
+        if "key" not in raw and "tree_hash" in raw:
+            raw["key"] = raw.pop("tree_hash")
+        else:
+            raw.pop("tree_hash", None)
+
+        key = raw.pop("key", "")
+        raw.pop("commit", None)  # drop legacy field
+        raw.setdefault("version", "")
+
+        return Snapshot(
+            tree_hash=key,
+            metrics=metrics,
+            vs_previous=vs_prev,
+            vs_baseline=vs_base,
+            branch=raw.pop("branch", ""),
+            timestamp=raw.pop("timestamp", ""),
+            version=raw.pop("version", ""),
+            hotspots=raw.pop("hotspots", []),
+            issues=raw.pop("issues", []),
+        )
+
+
+@dataclass
+class PruneResult:
+    """Summary of a :meth:`SnapshotManager.prune_snapshots` operation.
+
+    :param removed: Keys of snapshots pruned as metric-duplicates.
+    :param orphaned_files: Filenames of JSON files deleted from disk because
+        they were not referenced by the manifest.
+    :param broken_entries: Keys of manifest entries whose JSON file was missing.
+    :param dry_run: ``True`` when the call was a dry run (nothing deleted).
+    """
+
+    removed: list[str]
+    orphaned_files: list[str]
+    broken_entries: list[str]
+    dry_run: bool
+
+    @property
+    def total_cleaned(self) -> int:
+        """Total number of items removed (or that *would* be removed in dry-run)."""
+        return len(self.removed) + len(self.orphaned_files) + len(self.broken_entries)
+
+
+@dataclass
+class SnapshotManifest:
+    """Index of all snapshots, with fast lookup by tree hash."""
+
+    format_version: str = "1.0"
+    last_update: str = ""
+    snapshots: list[dict[str, Any]] = field(default_factory=list)
+
+    def to_dict(self) -> dict[str, Any]:
+        """Serialize to dict."""
+        return {
+            "format": self.format_version,
+            "last_update": self.last_update,
+            "snapshots": self.snapshots,
+        }
+
+    @staticmethod
+    def from_dict(data: dict[str, Any]) -> SnapshotManifest:
+        """Reconstruct from dict."""
+        return SnapshotManifest(
+            format_version=data.get("format", "1.0"),
+            last_update=data.get("last_update", ""),
+            snapshots=data.get("snapshots", []),
+        )

kg_utils/types/__init__.py

@@ -0,0 +1,14 @@
+"""kg_utils.types — Core dataclasses and base classes for the KGModule SDK."""
+
+from kg_utils.types.specs import EdgeSpec, NodeSpec, QueryResult, SnippetPack
+from kg_utils.types.extractor import KGExtractor
+from kg_utils.types.module import KGModule
+
+__all__ = [
+    "EdgeSpec",
+    "KGExtractor",
+    "KGModule",
+    "NodeSpec",
+    "QueryResult",
+    "SnippetPack",
+]

kg_utils/types/extractor.py

@@ -0,0 +1,68 @@
+"""kg_utils/types/extractor.py — Abstract base class for KG extractors."""
+
+from __future__ import annotations
+
+from collections.abc import Iterator
+from pathlib import Path
+from typing import Any
+
+from kg_utils.types.specs import EdgeSpec, NodeSpec
+
+
+class KGExtractor:
+    """Base class for knowledge-graph extractors.
+
+    Subclasses must implement :meth:`node_kinds`, :meth:`edge_kinds`,
+    and :meth:`extract`.
+
+    :param repo_path: Absolute path to the repository or corpus root.
+    :param config: Optional domain-specific configuration dict.
+    """
+
+    def __init__(self, repo_path: Path, config: dict[str, Any] | None = None) -> None:
+        self.repo_path = repo_path
+        self.config = config or {}
+
+    def node_kinds(self) -> list[str]:
+        """Return canonical node kind names.
+
+        :return: List of node kind strings.
+        """
+        raise NotImplementedError
+
+    def edge_kinds(self) -> list[str]:
+        """Return canonical edge relation types.
+
+        :return: List of edge relation strings.
+        """
+        raise NotImplementedError
+
+    def meaningful_node_kinds(self) -> list[str]:
+        """Return node kinds included in the vector index and coverage metrics.
+
+        Override to exclude structural stubs from the default (all node_kinds).
+
+        :return: Subset of node_kinds() to index semantically.
+        """
+        return self.node_kinds()
+
+    def coverage_metric(self, nodes: list[NodeSpec]) -> float:
+        """Compute a domain coverage quality metric.
+
+        Default: fraction of meaningful nodes with a non-empty docstring.
+
+        :param nodes: All extracted NodeSpec objects.
+        :return: Coverage score in [0.0, 1.0].
+        """
+        meaningful = [n for n in nodes if n.kind in self.meaningful_node_kinds()]
+        if not meaningful:
+            return 0.0
+        covered = sum(1 for n in meaningful if n.docstring.strip())
+        return covered / len(meaningful)
+
+    def extract(self) -> Iterator[NodeSpec | EdgeSpec]:
+        """Traverse the source and yield NodeSpec / EdgeSpec objects.
+
+        :return: Iterator of NodeSpec and EdgeSpec objects.
+        """
+        raise NotImplementedError

kg_utils/types/module.py

@@ -0,0 +1,87 @@
+"""kg_utils/types/module.py — Abstract base class for KG modules."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+from kg_utils.types.extractor import KGExtractor
+from kg_utils.types.specs import QueryResult, SnippetPack
+
+
+class KGModule:
+    """Base class for knowledge-graph modules.
+
+    Subclasses must implement :meth:`make_extractor`, :meth:`kind`,
+    and should override :meth:`build`, :meth:`query`, :meth:`stats`,
+    :meth:`pack`, and :meth:`analyze` with domain-specific logic.
+
+    :param repo_root: Absolute path to the repository or corpus root.
+    :param db_path: Path for the SQLite graph database.
+    :param lancedb_dir: Path for the LanceDB vector index directory.
+    :param config: Optional domain-specific configuration dict.
+    """
+
+    def __init__(
+        self,
+        repo_root: Path,
+        db_path: Path | None = None,
+        lancedb_dir: Path | None = None,
+        config: dict[str, Any] | None = None,
+    ) -> None:
+        self.repo_root = repo_root
+        self.db_path = db_path
+        self.lancedb_dir = lancedb_dir
+        self.config = config or {}
+
+    def make_extractor(self) -> KGExtractor:
+        """Return the domain extractor for this module.
+
+        :return: KGExtractor subclass instance.
+        """
+        raise NotImplementedError
+
+    def kind(self) -> str:
+        """Return the KGKind string for this module.
+
+        :return: Kind string (e.g. "code", "meta", "doc").
+        """
+        raise NotImplementedError
+
+    def build(self, wipe: bool = False) -> None:
+        """Build the knowledge graph index.
+
+        :param wipe: If True, delete existing index before building.
+        """
+        raise NotImplementedError
+
+    def query(self, q: str, k: int = 8, **kwargs: Any) -> QueryResult:
+        """Query the knowledge graph.
+
+        :param q: Natural-language query string.
+        :param k: Number of results to return.
+        :return: QueryResult with matched nodes and edges.
+        """
+        raise NotImplementedError
+
+    def stats(self) -> dict[str, Any]:
+        """Return statistics about the knowledge graph.
+
+        :return: Dict with keys like total_nodes, total_edges, etc.
+        """
+        raise NotImplementedError
+
+    def pack(self, q: str, **kwargs: Any) -> SnippetPack:
+        """Pack query results with source context.
+
+        :param q: Natural-language query string.
+        :return: SnippetPack with nodes, edges, and snippets.
+        """
+        raise NotImplementedError
+
+    def analyze(self) -> str:
+        """Run full analysis and return a Markdown report.
+
+        :return: Markdown-formatted analysis report.
+        """
+        raise NotImplementedError

kg_utils/types/specs.py

@@ -0,0 +1,90 @@
+"""kg_utils/types/specs.py — Core dataclasses shared by all KG modules."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass
+class NodeSpec:
+    """Specification for a knowledge-graph node.
+
+    :param node_id: Unique identifier, typically ``<kind>:<path>:<qualname>``.
+    :param kind: Node kind (e.g. "file", "function", "class", "directory").
+    :param name: Short display name.
+    :param qualname: Fully-qualified name or relative path.
+    :param source_path: Path to the source file (relative to repo root).
+    :param docstring: Semantic content for vector indexing.
+    """
+
+    node_id: str
+    kind: str
+    name: str
+    qualname: str
+    source_path: str
+    docstring: str = ""
+
+
+@dataclass
+class EdgeSpec:
+    """Specification for a knowledge-graph edge.
+
+    :param source_id: Node ID of the edge source.
+    :param target_id: Node ID of the edge target.
+    :param relation: Relation type (e.g. "CONTAINS", "CALLS", "IMPORTS").
+    """
+
+    source_id: str
+    target_id: str
+    relation: str
+
+
+@dataclass
+class QueryResult:
+    """Result container returned by KGModule.query().
+
+    :param nodes: List of matched node dicts.
+    :param edges: List of matched edge dicts.
+    :param seeds: Number of seed nodes from vector search.
+    :param expanded_nodes: Number of nodes after graph expansion.
+    :param returned_nodes: Number of nodes actually returned.
+    :param hop: Number of hops used in graph expansion.
+    :param rels: Relation types used in expansion.
+    """
+
+    nodes: list[dict[str, Any]] = field(default_factory=list)
+    edges: list[dict[str, Any]] = field(default_factory=list)
+    seeds: int = 0
+    expanded_nodes: int = 0
+    returned_nodes: int = 0
+    hop: int = 0
+    rels: list[str] = field(default_factory=list)
+
+
+@dataclass
+class SnippetPack:
+    """Result container returned by KGModule.pack().
+
+    :param query: The original query string.
+    :param seeds: Number of seed nodes from vector search.
+    :param expanded_nodes: Number of nodes after graph expansion.
+    :param returned_nodes: Number of nodes actually returned.
+    :param hop: Number of hops used in expansion.
+    :param rels: Relation types used in expansion.
+    :param model: Embedding model identifier.
+    :param nodes: Node dicts included in the pack.
+    :param edges: Edge dicts included in the pack.
+    :param snippets: Source-code snippets (for code KGs).
+    """
+
+    query: str
+    seeds: int = 0
+    expanded_nodes: int = 0
+    returned_nodes: int = 0
+    hop: int = 0
+    rels: list[str] = field(default_factory=list)
+    model: str = ""
+    nodes: list[Any] = field(default_factory=list)
+    edges: list[Any] = field(default_factory=list)
+    snippets: list[Any] = field(default_factory=list)

kgmodule_utils-0.2.0.dist-info/METADATA

@@ -0,0 +1,210 @@
+Metadata-Version: 2.4
+Name: kgmodule-utils
+Version: 0.2.0
+Summary: Shared types and snapshot infrastructure for the KGModule SDK
+License: Elastic-2.0
+License-File: LICENSE
+Keywords: knowledge-graph,kgmodule,sdk,types,snapshots
+Author: Eric G. Suchanek, PhD
+Author-email: suchanek@flux-frontiers.com
+Requires-Python: >=3.12,<3.14
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Project-URL: Repository, https://github.com/Flux-Frontiers/kg_utils
+Description-Content-Type: text/markdown
+
+
+[![Python](https://img.shields.io/badge/python-3.12%20%7C%203.13-blue.svg)](https://www.python.org/)
+[![License: Elastic-2.0](https://img.shields.io/badge/License-Elastic%202.0-blue.svg)](https://www.elastic.co/licensing/elastic-license)
+[![Version](https://img.shields.io/badge/version-0.2.0-blue.svg)](https://github.com/Flux-Frontiers/KG_utils/releases)
+[![CI](https://github.com/Flux-Frontiers/KG_utils/actions/workflows/ci.yml/badge.svg)](https://github.com/Flux-Frontiers/KG_utils/actions/workflows/ci.yml)
+[![Poetry](https://img.shields.io/endpoint?url=https://python-poetry.org/badge/v0.json)](https://python-poetry.org/)
+
+# kgmodule-utils
+
+**kgmodule-utils** — Shared types and snapshot infrastructure for the KGModule SDK.
+
+*Author: Eric G. Suchanek, PhD*
+
+*Flux-Frontiers, Liberty TWP, OH*
+
+---
+
+## Overview
+
+kgmodule-utils is the **zero-dependency foundation package** for the Flux-Frontiers knowledge-graph ecosystem. It provides the canonical type abstractions and temporal snapshot infrastructure that all KGModule implementations — [PyCodeKG](https://github.com/Flux-Frontiers/pycode_kg), [FTreeKG](https://github.com/Flux-Frontiers/ftree_kg), [DocKG](https://github.com/Flux-Frontiers/doc_kg), [AgentKG](https://github.com/Flux-Frontiers/agent_kg) — depend on.
+
+Every KGModule shares the same `NodeSpec`, `EdgeSpec`, `KGExtractor`, and `KGModule` base classes defined here, ensuring consistent interfaces across the ecosystem. The snapshot subsystem enables temporal metric tracking, delta comparison, and pruning across git commits.
+
+---
+
+## Features
+
+- **Core type abstractions** — `NodeSpec`, `EdgeSpec`, `QueryResult`, `SnippetPack` dataclasses for knowledge-graph nodes, edges, and query results
+- **KGExtractor base class** — Abstract interface for domain-specific extractors with `extract()`, `node_kinds()`, `edge_kinds()`, and `coverage_metric()`
+- **KGModule base class** — Abstract interface for knowledge-graph modules with `build()`, `query()`, `pack()`, `stats()`, and `analyze()`
+- **Snapshot models** — `Snapshot` dataclass keyed by git tree hash with free-form metrics, hotspots, issues, and delta tracking
+- **SnapshotManager** — Capture, persist, load, list, diff, and prune snapshots with automatic deduplication and delta computation
+- **SnapshotManifest** — Fast-lookup index of all snapshots with format versioning
+- **Zero dependencies** — Stdlib-only; no external packages required at runtime
+
+---
+
+## Installation
+
+**Requirements:** Python ≥ 3.12, < 3.14
+
+### Standalone (pip)
+
+```bash
+pip install kgmodule-utils
+```
+
+### Existing Poetry project
+
+```bash
+poetry add kgmodule-utils
+```
+
+Or declare it directly in your `pyproject.toml`:
+
+```toml
+[tool.poetry.dependencies]
+kgmodule-utils = "^0.2.0"
+```
+
+---
+
+## Quick Start
+
+### Types — Define a KGModule
+
+```python
+from kg_utils.types import NodeSpec, EdgeSpec, KGExtractor, KGModule
+
+class MyExtractor(KGExtractor):
+    def node_kinds(self) -> list[str]:
+        return ["module", "function", "class"]
+
+    def edge_kinds(self) -> list[str]:
+        return ["CONTAINS", "CALLS", "IMPORTS"]
+
+    def extract(self, source_root: str):
+        # Yield NodeSpec and EdgeSpec objects from your domain
+        yield NodeSpec(
+            node_id="fn:main:hello",
+            kind="function",
+            name="hello",
+            qualname="main.hello",
+            source_path="main.py",
+            docstring="Greet the user.",
+        )
+        yield EdgeSpec(
+            source_id="mod:main",
+            target_id="fn:main:hello",
+            relation="CONTAINS",
+        )
+```
+
+### Snapshots — Track metrics over time
+
+```python
+from kg_utils.snapshots import SnapshotManager
+
+mgr = SnapshotManager(snapshots_dir=".my_kg/snapshots", package_name="my-kg")
+
+# Capture a snapshot from current metrics
+snapshot = mgr.capture(metrics={
+    "total_nodes": 142,
+    "total_edges": 387,
+    "coverage": 0.78,
+})
+
+# Save with automatic deduplication
+mgr.save_snapshot(snapshot)
+
+# List and compare
+snaps = mgr.list_snapshots(limit=5)
+delta = mgr.diff_snapshots(key_a=snaps[0].key, key_b=snaps[-1].key)
+```
+
+---
+
+## API Reference
+
+### `kg_utils.types`
+
+| Class | Description |
+|---|---|
+| `NodeSpec` | Dataclass for KG nodes: `node_id`, `kind`, `name`, `qualname`, `source_path`, `docstring` |
+| `EdgeSpec` | Dataclass for KG edges: `source_id`, `target_id`, `relation` |
+| `QueryResult` | Container for query responses with nodes, edges, and metadata |
+| `SnippetPack` | Extended result container with source-code snippets |
+| `KGExtractor` | Abstract base class for domain extractors |
+| `KGModule` | Abstract base class for knowledge-graph modules |
+
+### `kg_utils.snapshots`
+
+| Class | Description |
+|---|---|
+| `Snapshot` | Temporal snapshot keyed by git tree hash with free-form metrics and deltas |
+| `SnapshotManager` | Capture, persist, load, list, diff, and prune snapshots |
+| `SnapshotManifest` | Index of all snapshots with format versioning and fast lookup |
+| `PruneResult` | Summary of pruning operations: removed, orphaned, broken entries |
+
+---
+
+## Project Structure
+
+```
+KG_utils/
+├── LICENSE
+├── README.md
+├── pyproject.toml
+├── pytest.ini
+├── src/
+│   └── kg_utils/
+│       ├── __init__.py
+│       ├── py.typed                  # PEP 561 marker
+│       ├── types/
+│       │   ├── __init__.py           # Public re-exports
+│       │   ├── specs.py              # NodeSpec, EdgeSpec, QueryResult, SnippetPack
+│       │   ├── extractor.py          # KGExtractor ABC
+│       │   └── module.py             # KGModule ABC
+│       └── snapshots/
+│           ├── __init__.py           # Public re-exports
+│           ├── models.py             # Snapshot, SnapshotManifest, PruneResult
+│           └── manager.py            # SnapshotManager
+└── tests/
+    ├── __init__.py
+    ├── test_types.py
+    └── test_snapshots.py
+```
+
+---
+
+## Development
+
+```bash
+git clone https://github.com/Flux-Frontiers/KG_utils.git
+cd KG_utils
+poetry install --with dev
+```
+
+Run the test suite:
+
+```bash
+poetry run pytest
+```
+
+---
+
+## License
+
+[Elastic License 2.0](https://www.elastic.co/licensing/elastic-license) — see [LICENSE](LICENSE).
+
+Free to use, modify, and distribute. You may not offer the software as a hosted or managed service to third parties. Commercial use internally is permitted.
+

kgmodule_utils-0.2.0.dist-info/RECORD

@@ -0,0 +1,14 @@
+kg_utils/__init__.py,sha256=s-W_5K2Z_8H1mTpiCrWrNEHd5P3eAcCWUQdRNgoQ0H0,428
+kg_utils/embed.py,sha256=lIqUdOjnd2TU1-epa7RHoe0y2MYRC1xmhc-5y--wrV8,4969
+kg_utils/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+kg_utils/snapshots/__init__.py,sha256=IXgvXQ31AChUHR9fhrsMpMV_A_dSjoCly_Ad8rTdG9w,538
+kg_utils/snapshots/manager.py,sha256=DT0NyKLAumiXEUEIg3Igf4MH0BGK8bwm65fRFUFlgrc,19707
+kg_utils/snapshots/models.py,sha256=NxF2pqRU6LHyRujdDh4smyRZ8UbL3FIGANBgCmjPGSw,4576
+kg_utils/types/__init__.py,sha256=Q7q5Bb2F6F6zbjSTpCXErIKKu5QSHg-9YChhbr7AqDQ,372
+kg_utils/types/extractor.py,sha256=EozyNWlFfdzCaskUA6tK5iej_DfO7srAWtbEVPtffFU,2182
+kg_utils/types/module.py,sha256=_MRvVe1pbI_lAdT5VOhADgTbPhBHjdkaJzymigBaFsU,2713
+kg_utils/types/specs.py,sha256=QTXIe_P2TxVGB1LiFJ2-zovv1vBKQX68jcnDRcaQO30,2792
+kgmodule_utils-0.2.0.dist-info/METADATA,sha256=2wivJ8i0iIUCWMnpv3hRueTFHmN63XBgMfdsFgMHhvQ,7138
+kgmodule_utils-0.2.0.dist-info/WHEEL,sha256=kJCRJT_g0adfAJzTx2GUMmS80rTJIVHRCfG0DQgLq3o,88
+kgmodule_utils-0.2.0.dist-info/licenses/LICENSE,sha256=X-B9sT00-P4jkaspEftxUFB_KHtOa8qHiC1pe7yXBjA,3865
+kgmodule_utils-0.2.0.dist-info/RECORD,,

kgmodule_utils-0.2.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: poetry-core 2.3.1
+Root-Is-Purelib: true
+Tag: py3-none-any