kgmodule-utils 0.2.3__tar.gz → 0.3.0__tar.gz

{kgmodule_utils-0.2.3 → kgmodule_utils-0.3.0}/PKG-INFO

@@ -1,18 +1,24 @@
 Metadata-Version: 2.4
 Name: kgmodule-utils
-Version: 0.2.3
-Summary: Shared types and snapshot infrastructure for the KGModule SDK
+Version: 0.3.0
+Summary: Shared types, graph store, semantic index, and pipeline base for the KGModule SDK
 License: Elastic-2.0
 License-File: LICENSE
 Keywords: knowledge-graph,kgmodule,sdk,types,snapshots
 Author: Eric G. Suchanek, PhD
 Author-email: suchanek@flux-frontiers.com
 Requires-Python: >=3.12,<3.14
-Classifier: Development Status :: 3 - Alpha
+Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Provides-Extra: semantic
+Requires-Dist: lancedb (>=0.19.0) ; extra == "semantic"
+Requires-Dist: numpy (>=1.24.0) ; extra == "semantic"
+Requires-Dist: sentence-transformers (>=5.4.1) ; extra == "semantic"
+Requires-Dist: torch (>=2.5.1) ; extra == "semantic"
+Requires-Dist: transformers (>=4.40.0,<4.57) ; extra == "semantic"
 Project-URL: Repository, https://github.com/Flux-Frontiers/kg_utils
 Description-Content-Type: text/markdown
 

{kgmodule_utils-0.2.3 → kgmodule_utils-0.3.0}/pyproject.toml

@@ -10,8 +10,8 @@ build-backend = "poetry.core.masonry.api"
 
 [project]
 name = "kgmodule-utils"
-version = "0.2.3"
-description = "Shared types and snapshot infrastructure for the KGModule SDK"
+version = "0.3.0"
+description = "Shared types, graph store, semantic index, and pipeline base for the KGModule SDK"
 readme = "README.md"
 license = { text = "Elastic-2.0" }
 authors = [
@@ -19,7 +19,7 @@ authors = [
 ]
 keywords = ["knowledge-graph", "kgmodule", "sdk", "types", "snapshots"]
 classifiers = [
-    "Development Status :: 3 - Alpha",
+    "Development Status :: 4 - Beta",
     "Intended Audience :: Developers",
     "Programming Language :: Python :: 3",
     "Programming Language :: Python :: 3.12",
@@ -28,12 +28,28 @@ classifiers = [
 requires-python = ">=3.12,<3.14"
 dependencies = []
 
+[project.optional-dependencies]
+semantic = [
+    "lancedb>=0.19.0",
+    "numpy>=1.24.0",
+    "sentence-transformers>=5.4.1",
+    "torch>=2.5.1",
+    "transformers>=4.40.0,<4.57",
+]
+
 [project.urls]
 Repository = "https://github.com/Flux-Frontiers/kg_utils"
 
 [tool.poetry]
 packages = [{include = "kg_utils", from = "src"}]
 
+[tool.poetry.group.kgdeps]
+optional = true
+
+[tool.poetry.group.kgdeps.dependencies]
+pycode-kg = ">=0.18.1"
+doc-kg    = ">=0.15.2"
+
 [tool.poetry.group.dev]
 optional = true
 
@@ -72,6 +88,7 @@ module = [
     "sentence_transformers.*",
     "transformers.*",
     "numpy.*",
+    "lancedb",
 ]
 ignore_missing_imports = true
 

kgmodule_utils-0.3.0/src/kg_utils/__init__.py

@@ -0,0 +1,20 @@
+"""kg_utils — Shared types, store, semantic index, and pipeline base for the KGModule SDK.
+
+Sub-packages / modules:
+    kg_utils.types      — NodeSpec, EdgeSpec, BuildStats, QueryResult, SnippetPack,
+                          KGExtractor (abstract), KGModule (abstract interface).
+    kg_utils.store      — GraphStore: SQLite-backed authoritative node/edge store.
+    kg_utils.semantic   — Embedder, SentenceTransformerEmbedder, SemanticIndex, SeedHit.
+    kg_utils.pipeline   — KGModule: concrete base class with full build/query/pack pipeline.
+    kg_utils.snapshots  — Snapshot, SnapshotManager, SnapshotManifest, etc.
+    kg_utils.embed      — Embedder protocol, DEFAULT_MODEL, KNOWN_MODELS,
+                          kg_model_cache_dir(), resolve_model_path().
+    kg_utils.embedder   — Concrete SentenceTransformerEmbedder, get_embedder(),
+                          wrap_embedder(), load_sentence_transformer().
+
+Optional extras
+---------------
+    pip install 'kgmodule-utils[semantic]'   # lancedb + sentence-transformers
+"""
+
+__version__ = "0.3.0"

{kgmodule_utils-0.2.3 → kgmodule_utils-0.3.0}/src/kg_utils/embedder.py

@@ -99,29 +99,47 @@ def load_sentence_transformer(model_name: str = DEFAULT_MODEL) -> Any:
         from transformers import logging as hf_logging  # pylint: disable=import-outside-toplevel
 
         hf_logging.set_verbosity_error()
-    except ImportError:
+        # TQDM_DISABLE alone misses transformers' _tqdm_active gate
+        hf_logging.disable_progress_bar()
+    except (ImportError, ValueError):
         pass
 
     os.environ["TQDM_DISABLE"] = "1"
 
+    import torch  # pylint: disable=import-outside-toplevel
+
+    if torch.cuda.is_available():
+        device = "cuda"
+    else:
+        try:
+            device = "mps" if torch.backends.mps.is_available() else "cpu"
+        except AttributeError:
+            device = "cpu"
+
     resolved = KNOWN_MODELS.get(model_name, model_name)
     trust_remote = "nomic-ai/" in resolved
     local_path = resolve_model_path(resolved)
 
     if local_path.exists():
-        return SentenceTransformer(
+        model = SentenceTransformer(
             str(local_path),
             local_files_only=True,
             trust_remote_code=trust_remote,
+            device=device,
         )
-    try:
-        return SentenceTransformer(
-            resolved,
-            local_files_only=True,
-            trust_remote_code=trust_remote,
-        )
-    except OSError:
-        return SentenceTransformer(resolved, trust_remote_code=trust_remote)
+    else:
+        try:
+            model = SentenceTransformer(
+                resolved,
+                local_files_only=True,
+                trust_remote_code=trust_remote,
+                device=device,
+            )
+        except OSError:
+            model = SentenceTransformer(resolved, trust_remote_code=trust_remote, device=device)
+
+    model = model.to(device)
+    return model
 
 
 # ---------------------------------------------------------------------------
@@ -143,7 +161,8 @@ class SentenceTransformerEmbedder(Embedder):
             from transformers import logging as hf_logging  # pylint: disable=import-outside-toplevel
 
             hf_logging.set_verbosity_error()
-        except ImportError:
+            hf_logging.disable_progress_bar()
+        except (ImportError, ValueError):
             pass
 
         _prev = os.environ.get("TQDM_DISABLE")
@@ -157,7 +176,7 @@ class SentenceTransformerEmbedder(Embedder):
                 os.environ["TQDM_DISABLE"] = _prev
 
         self.model_name: str = KNOWN_MODELS.get(model_name, model_name)
-        # ST ≥5.4 renamed to get_embedding_dimension; ≤5.3 only had get_sentence_embedding_dimension.
+        # ST ≥5.4 renamed get_embedding_dimension; ≤5.3 had get_sentence_embedding_dimension.
         _dim_fn = getattr(self.model, "get_embedding_dimension", None) or getattr(
             self.model, "get_sentence_embedding_dimension", None
         )

kgmodule_utils-0.3.0/src/kg_utils/extractor.py

@@ -0,0 +1,76 @@
+"""kg_utils/types/extractor.py — Abstract base class for KG extractors."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from collections.abc import Iterator
+from pathlib import Path
+from typing import Any
+
+from kg_utils.specs import EdgeSpec, NodeSpec
+
+
+class KGExtractor(ABC):
+    """Abstract extraction protocol for any knowledge graph domain.
+
+    Subclass and implement :meth:`node_kinds`, :meth:`edge_kinds`, and
+    :meth:`extract`.  The concrete :class:`~kg_utils.pipeline.KGModule`
+    infrastructure (:class:`~kg_utils.store.GraphStore`,
+    :class:`~kg_utils.semantic.SemanticIndex`, snapshot management) is
+    provided by the framework — you only implement domain-specific parsing.
+
+    :param repo_path: Absolute path to the repository or corpus root.
+    :param config: Optional domain-specific configuration dict.
+    """
+
+    def __init__(self, repo_path: Path, config: dict[str, Any] | None = None) -> None:
+        self.repo_path = Path(repo_path).resolve()
+        self.config = config or {}
+
+    @abstractmethod
+    def node_kinds(self) -> list[str]:
+        """Return canonical node kind names emitted by this extractor.
+
+        :return: List of node kind strings (e.g. ``['module', 'class', 'function']``).
+        """
+
+    @abstractmethod
+    def edge_kinds(self) -> list[str]:
+        """Return canonical edge relation types emitted by this extractor.
+
+        :return: List of edge relation strings (e.g. ``['CONTAINS', 'CALLS']``).
+        """
+
+    @abstractmethod
+    def extract(self) -> Iterator[NodeSpec | EdgeSpec]:
+        """Traverse the source and yield NodeSpec / EdgeSpec objects.
+
+        Implementations must be deterministic: the same source should produce
+        the same stream on every call.
+
+        :return: Iterator of :class:`NodeSpec` and :class:`EdgeSpec` objects.
+        """
+
+    def meaningful_node_kinds(self) -> list[str]:
+        """Return node kinds included in the vector index and coverage metrics.
+
+        Default: all of :meth:`node_kinds`.  Override to exclude structural
+        stubs (e.g., unresolved import placeholders) from semantic indexing.
+
+        :return: Subset of node_kinds() to index semantically.
+        """
+        return self.node_kinds()
+
+    def coverage_metric(self, nodes: list[NodeSpec]) -> float:
+        """Compute a domain coverage quality metric.
+
+        Default: fraction of meaningful nodes with a non-empty docstring.
+
+        :param nodes: All extracted NodeSpec objects.
+        :return: Coverage score in [0.0, 1.0].
+        """
+        meaningful = [n for n in nodes if n.kind in self.meaningful_node_kinds()]
+        if not meaningful:
+            return 0.0
+        covered = sum(1 for n in meaningful if n.docstring.strip())
+        return covered / len(meaningful)

kgmodule_utils-0.3.0/src/kg_utils/module.py

@@ -0,0 +1,91 @@
+"""kg_utils/types/module.py — Minimal abstract interface for KG modules.
+
+For the concrete, production-grade base class with full build/query/pack
+infrastructure, use :class:`kg_utils.pipeline.KGModule` instead.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+from kg_utils.extractor import KGExtractor
+from kg_utils.specs import QueryResult, SnippetPack
+
+
+class KGModule:
+    """Base class for knowledge-graph modules.
+
+    Subclasses must implement :meth:`make_extractor`, :meth:`kind`,
+    and should override :meth:`build`, :meth:`query`, :meth:`stats`,
+    :meth:`pack`, and :meth:`analyze` with domain-specific logic.
+
+    :param repo_root: Absolute path to the repository or corpus root.
+    :param db_path: Path for the SQLite graph database.
+    :param lancedb_dir: Path for the LanceDB vector index directory.
+    :param config: Optional domain-specific configuration dict.
+    """
+
+    def __init__(
+        self,
+        repo_root: Path,
+        db_path: Path | None = None,
+        lancedb_dir: Path | None = None,
+        config: dict[str, Any] | None = None,
+    ) -> None:
+        self.repo_root = repo_root
+        self.db_path = db_path
+        self.lancedb_dir = lancedb_dir
+        self.config = config or {}
+
+    def make_extractor(self) -> KGExtractor:
+        """Return the domain extractor for this module.
+
+        :return: KGExtractor subclass instance.
+        """
+        raise NotImplementedError
+
+    def kind(self) -> str:
+        """Return the KGKind string for this module.
+
+        :return: Kind string (e.g. "code", "meta", "doc").
+        """
+        raise NotImplementedError
+
+    def build(self, wipe: bool = False) -> None:
+        """Build the knowledge graph index.
+
+        :param wipe: If True, delete existing index before building.
+        """
+        raise NotImplementedError
+
+    def query(self, q: str, k: int = 8, **kwargs: Any) -> QueryResult:
+        """Query the knowledge graph.
+
+        :param q: Natural-language query string.
+        :param k: Number of results to return.
+        :return: QueryResult with matched nodes and edges.
+        """
+        raise NotImplementedError
+
+    def stats(self) -> dict[str, Any]:
+        """Return statistics about the knowledge graph.
+
+        :return: Dict with keys like total_nodes, total_edges, etc.
+        """
+        raise NotImplementedError
+
+    def pack(self, q: str, **kwargs: Any) -> SnippetPack:
+        """Pack query results with source context.
+
+        :param q: Natural-language query string.
+        :return: SnippetPack with nodes, edges, and snippets.
+        """
+        raise NotImplementedError
+
+    def analyze(self) -> str:
+        """Run full analysis and return a Markdown report.
+
+        :return: Markdown-formatted analysis report.
+        """
+        raise NotImplementedError