hinge 0.1.0__py3-none-any.whl

hinge/kernel/projection/projection_spec.py

@@ -0,0 +1,35 @@
+from __future__ import annotations
+
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+
+class ProjectionParams(BaseModel):
+    """Free-form key/value parameters passed to a projection at run time.
+
+    Examples: window start/end dates, minimum interaction count, repo filter.
+    Each projection documents which keys it consumes in its SQL header.
+    """
+
+    values: dict[str, Any] = Field(default_factory=dict)
+
+
+class ProjectionSpec(BaseModel):
+    """Declarative description of one projection.
+
+    A spec is *metadata only*. It points the projection stage at a SQL model
+    file and tells callers what the output looks like. Specs are discovered
+    by the registry through the ``hinge.projection_specs`` entry-point group;
+    third-party packages register new projections the same way readers and
+    exporters are registered.
+
+    See ``hinge/dbt/models/networks/dev_interaction.sql`` for the
+    contract every model must satisfy.
+    """
+
+    name: str
+    description: str
+    model_name: str
+    output_node_types: list[str]
+    output_edge_types: list[str]

hinge/kernel/protocols/__init__.py

@@ -0,0 +1,24 @@
+from hinge.kernel.protocols.dataset_view import DatasetView
+from hinge.kernel.protocols.exporter_stage import ExporterStage, ExportReceipt
+from hinge.kernel.protocols.projection_stage import EngineFingerprint, ProjectionStage
+from hinge.kernel.protocols.reader_stage import ReaderDescriptor, ReaderStage
+from hinge.kernel.protocols.store_stage import (
+    DatasetMeta,
+    StoreCensus,
+    StoreFingerprint,
+    StoreStage,
+)
+
+__all__ = [
+    "DatasetMeta",
+    "DatasetView",
+    "EngineFingerprint",
+    "ExportReceipt",
+    "ExporterStage",
+    "ProjectionStage",
+    "ReaderDescriptor",
+    "ReaderStage",
+    "StoreCensus",
+    "StoreFingerprint",
+    "StoreStage",
+]

hinge/kernel/protocols/dataset_view.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Protocol
+
+
+class DatasetView(Protocol):
+    """Read-only handle to one dataset inside a store.
+
+    Stores produce this via :meth:`StoreStage.scope_to_dataset`; projections
+    consume it. Today every store backend ships a SQL view of the dataset so
+    the projection can read it with whatever query engine fits — the fields
+    are intentionally concrete (path + view names) rather than an abstract
+    "query handle", because every real projection backend needs either SQL
+    or row iteration anyway.
+
+    A future non-SQL store can implement this by writing its dataset out to
+    a temporary DuckDB file before returning — the contract stays one shape.
+    """
+
+    @property
+    def dataset_id(self) -> str: ...
+    @property
+    def db_path(self) -> Path: ...
+    @property
+    def nodes_view(self) -> str: ...
+    @property
+    def edges_view(self) -> str: ...
+
+    def close(self) -> None: ...

hinge/kernel/protocols/exporter_stage.py

@@ -0,0 +1,24 @@
+from __future__ import annotations
+
+from typing import BinaryIO, Protocol
+
+from pydantic import BaseModel
+
+from hinge.kernel.projection.projected_graph import ProjectedGraphHandle
+
+
+class ExportReceipt(BaseModel):
+    """Result of one export. ``snapshot_id`` is set by the runner when both
+    store and projection fingerprints are available; otherwise it stays None.
+    """
+
+    format: str
+    content_hash: str
+    schema_version: int
+    node_count: int
+    edge_count: int
+    snapshot_id: str | None = None
+
+
+class ExporterStage(Protocol):
+    def write(self, handle: ProjectedGraphHandle, sink: BinaryIO) -> ExportReceipt: ...

hinge/kernel/protocols/projection_stage.py

@@ -0,0 +1,36 @@
+from __future__ import annotations
+
+from typing import Any, Protocol
+
+from pydantic import BaseModel
+
+from hinge.kernel.projection.projected_graph import ProjectedGraphHandle
+from hinge.kernel.projection.projection_spec import ProjectionSpec
+from hinge.kernel.protocols.dataset_view import DatasetView
+
+
+class EngineFingerprint(BaseModel):
+    """Hash of the projection engine's own code (e.g. dbt project files).
+    Combined with the store fingerprint and schema version to make snapshot_id.
+    """
+
+    engine: str
+    project_hash: str
+
+
+class ProjectionStage(Protocol):
+    """Materialises a typed sub-graph from a dataset.
+
+    The projection takes a :class:`ProjectionSpec` (what to compute) and a
+    :class:`DatasetView` (where to read from). It never touches the store
+    directly — the store handed it everything it needs.
+    """
+
+    def run(
+        self,
+        spec: ProjectionSpec,
+        params: dict[str, Any],
+        view: DatasetView,
+    ) -> ProjectedGraphHandle: ...
+
+    def fingerprint(self) -> EngineFingerprint: ...

hinge/kernel/protocols/reader_stage.py

@@ -0,0 +1,52 @@
+from __future__ import annotations
+
+from collections.abc import Iterator
+from pathlib import Path
+from typing import Protocol, runtime_checkable
+
+from pydantic import BaseModel
+
+from hinge.kernel.protocols.store_stage import StoreStage
+from hinge.kernel.schema.schema_violation import SchemaViolation
+from hinge.kernel.schema.typed_edge import TypedEdge
+from hinge.kernel.schema.typed_node import TypedNode
+
+
+class ReaderDescriptor(BaseModel):
+    dataset: str
+    format: str
+    path: Path
+    byte_size: int
+
+
+class ReaderStage(Protocol):
+    """Reads a dataset file and yields typed graph elements directly.
+
+    A reader owns both the file-format parsing logic and the domain knowledge
+    needed to map raw records to ``TypedNode`` / ``TypedEdge`` objects. It may
+    also yield ``SchemaViolation`` instances for records it cannot map — the
+    runner counts and logs them rather than aborting the ingest.
+    """
+
+    def iter_elements(self) -> Iterator[TypedNode | TypedEdge | SchemaViolation]: ...
+    def describe(self) -> ReaderDescriptor: ...
+
+
+@runtime_checkable
+class BulkIngestReader(Protocol):
+    """Reader that loads a whole dataset in one bulk operation against the store.
+
+    Implementations push records directly into the store's native engine
+    (e.g. DuckDB SQL) and return summary counts. The runner detects this
+    shape and uses it in place of the element-by-element ``ReaderStage``
+    path — useful when the file format and the store backend let us avoid
+    a per-row Python round-trip.
+    """
+
+    def describe(self) -> ReaderDescriptor: ...
+    def bulk_ingest(self, store: StoreStage, dataset_id: str) -> tuple[int, int, int]:
+        """Load the dataset into ``store`` under ``dataset_id``.
+
+        Returns ``(records_read, nodes_written, edges_written)``.
+        """
+        ...

hinge/kernel/protocols/store_stage.py

@@ -0,0 +1,71 @@
+from __future__ import annotations
+
+from collections.abc import Iterable, Iterator
+from datetime import datetime
+from types import TracebackType
+from typing import Protocol
+
+from pydantic import BaseModel
+
+from hinge.kernel.protocols.dataset_view import DatasetView
+from hinge.kernel.schema.typed_edge import TypedEdge
+from hinge.kernel.schema.typed_node import TypedNode
+
+
+class StoreFingerprint(BaseModel):
+    """Content hash of one dataset inside a store. Used to compute snapshot_id."""
+
+    dataset_id: str
+    schema_version: int
+    node_count: int
+    edge_count: int
+    content_hash: str
+
+
+class StoreCensus(BaseModel):
+    """Per-type counts for one dataset."""
+
+    nodes_by_type: dict[str, int]
+    edges_by_type: dict[str, int]
+
+
+class DatasetMeta(BaseModel):
+    dataset_id: str
+    reader: str
+    path: str
+    ingested_at: datetime
+    node_count: int | None = None
+    edge_count: int | None = None
+
+
+class StoreStage(Protocol):
+    """Persists typed graph elements and surfaces them to projections.
+
+    The store is a context manager: it acquires whatever resources it needs
+    (file lock, connection, transaction) on ``__enter__`` and releases them on
+    ``__exit__``. This makes the DuckDB single-writer lock explicit at the
+    call site rather than buried inside method calls.
+    """
+
+    def __enter__(self) -> StoreStage: ...
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc: BaseException | None,
+        tb: TracebackType | None,
+    ) -> None: ...
+
+    # ---- ingest ----
+    def begin_dataset(self, dataset_id: str, reader: str, path: str) -> None: ...
+    def finalise_dataset(self, dataset_id: str, node_count: int, edge_count: int) -> None: ...
+    def upsert(
+        self, dataset_id: str, nodes: Iterable[TypedNode], edges: Iterable[TypedEdge]
+    ) -> int: ...
+    def discard_dataset(self, dataset_id: str) -> None: ...
+
+    # ---- read / project ----
+    def scope_to_dataset(self, dataset_id: str) -> DatasetView: ...
+    def query(self, dataset_id: str, node_types: list[str]) -> Iterator[TypedNode]: ...
+    def census(self, dataset_id: str) -> StoreCensus: ...
+    def fingerprint(self, dataset_id: str) -> StoreFingerprint: ...
+    def list_datasets(self) -> list[DatasetMeta]: ...

hinge/kernel/registry.py

@@ -0,0 +1,71 @@
+"""Entry-point based registry for stage implementations and projection specs.
+
+The kernel never imports a concrete stage class or a built-in spec. At startup
+the registry scans installed packages for the entry-point groups declared in
+``pyproject.toml`` and exposes them by name. Frontends call ``get_<role>``
+to obtain a stage instance and ``get_projection_spec(name)`` to look up a
+ProjectionSpec — symmetric: every extensible thing in the system plugs in the
+same way.
+
+Registered groups:
+
+  ``hinge.readers``           -> Reader classes
+  ``hinge.stores``            -> Store classes
+  ``hinge.projections``       -> Projection stage classes (engines)
+  ``hinge.projection_specs``  -> ProjectionSpec instances (one per task-specific subgraph)
+  ``hinge.exporters``         -> Exporter classes
+"""
+
+from __future__ import annotations
+
+from importlib.metadata import entry_points
+from typing import Any
+
+from hinge.kernel.projection.projection_spec import ProjectionSpec
+
+_GROUPS = {
+    "reader": "hinge.readers",
+    "store": "hinge.stores",
+    "projection": "hinge.projections",
+    "projection_spec": "hinge.projection_specs",
+    "exporter": "hinge.exporters",
+}
+
+
+def _load(role: str, name: str) -> Any:
+    group = _GROUPS[role]
+    matches = [ep for ep in entry_points(group=group) if ep.name == name]
+    if not matches:
+        available = sorted(ep.name for ep in entry_points(group=group))
+        raise LookupError(f"No {role} named {name!r} registered. Available: {available}")
+    return matches[0].load()
+
+
+def get_reader(name: str, **kwargs: Any) -> Any:
+    return _load("reader", name)(**kwargs)
+
+
+def get_store(name: str = "duckdb", **kwargs: Any) -> Any:
+    return _load("store", name)(**kwargs)
+
+
+def get_projection(name: str = "dbt", **kwargs: Any) -> Any:
+    return _load("projection", name)(**kwargs)
+
+
+def get_projection_spec(name: str) -> ProjectionSpec:
+    obj = _load("projection_spec", name)
+    if not isinstance(obj, ProjectionSpec):
+        raise TypeError(
+            f"Entry-point hinge.projection_specs:{name} must resolve to a "
+            f"ProjectionSpec instance, got {type(obj).__name__}"
+        )
+    return obj
+
+
+def get_exporter(name: str, **kwargs: Any) -> Any:
+    return _load("exporter", name)(**kwargs)
+
+
+def list_registered(role: str) -> list[str]:
+    return sorted(ep.name for ep in entry_points(group=_GROUPS[role]))

hinge/kernel/runner.py

@@ -0,0 +1,236 @@
+"""Linear pipeline runner.
+
+Pipeline shapes supported:
+
+    Reader → Store                          (ingest)
+    Store  → Projection                     (project)
+    Store  → Projection → Exporter          (export)
+
+The runner owns three responsibilities:
+
+  * batching and schema validation during ingest;
+  * transactional bookkeeping — a failed ingest leaves no half-written dataset;
+  * snapshot_id assembly during export, computed from store + projection
+    fingerprints + the HIN schema version + the tool version.
+
+Stage implementations are passed in; the runner never constructs them or
+imports concrete classes.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import logging
+import uuid
+from typing import Any, BinaryIO
+
+from pydantic import BaseModel
+
+from hinge.kernel.projection.projected_graph import ProjectedGraphHandle
+from hinge.kernel.projection.projection_spec import ProjectionSpec
+from hinge.kernel.protocols.dataset_view import DatasetView
+from hinge.kernel.protocols.exporter_stage import ExporterStage, ExportReceipt
+from hinge.kernel.protocols.projection_stage import ProjectionStage
+from hinge.kernel.protocols.reader_stage import BulkIngestReader, ReaderStage
+from hinge.kernel.protocols.store_stage import StoreStage
+from hinge.kernel.schema.hin_schema import HINSchema
+from hinge.kernel.schema.schema_violation import SchemaViolation
+from hinge.kernel.schema.typed_edge import TypedEdge
+from hinge.kernel.schema.typed_node import TypedNode
+
+logger = logging.getLogger(__name__)
+
+_TOOL_VERSION = "0.1.0"
+_BATCH_SIZE = 100_000
+
+
+class IngestReport(BaseModel):
+    dataset_id: str
+    elements_read: int
+    nodes_upserted: int
+    edges_upserted: int
+    violation_count: int
+
+
+def run_ingest(
+    reader: ReaderStage | BulkIngestReader, store: StoreStage, schema: HINSchema
+) -> IngestReport:
+    """Read a dataset and persist it. Transactional: on any exception the
+    half-written dataset is discarded so the store stays consistent.
+
+    Readers that implement ``bulk_ingest`` are dispatched through the bulk
+    path — schema validation is delegated to the reader because the records
+    never cross the Python boundary.
+    """
+    dataset_id = uuid.uuid4().hex
+    desc = reader.describe()
+    logger.info(
+        "ingest started — dataset_id=%s reader=%s format=%s path=%s size=%.1fMB",
+        dataset_id,
+        desc.dataset,
+        desc.format,
+        desc.path,
+        desc.byte_size / 1_048_576,
+    )
+
+    store.begin_dataset(dataset_id, desc.dataset, str(desc.path))
+    if isinstance(reader, BulkIngestReader):
+        try:
+            records, nodes_total, edges_total = reader.bulk_ingest(store, dataset_id)
+            store.finalise_dataset(dataset_id, nodes_total, edges_total)
+        except BaseException:
+            logger.exception("ingest failed — discarding dataset_id=%s", dataset_id)
+            store.discard_dataset(dataset_id)
+            raise
+        logger.info(
+            "ingest complete (bulk) — dataset_id=%s nodes=%d edges=%d",
+            dataset_id,
+            nodes_total,
+            edges_total,
+        )
+        return IngestReport(
+            dataset_id=dataset_id,
+            elements_read=records,
+            nodes_upserted=nodes_total,
+            edges_upserted=edges_total,
+            violation_count=0,
+        )
+
+    nodes_total = 0
+    edges_total = 0
+    violation_count = 0
+    elements_read = 0
+    node_buf: list[TypedNode] = []
+    edge_buf: list[TypedEdge] = []
+
+    try:
+        for element in reader.iter_elements():
+            elements_read += 1
+            if isinstance(element, SchemaViolation):
+                violation_count += 1
+                logger.warning(
+                    "violation [%s] %s:%d — %s",
+                    element.code,
+                    element.source,
+                    element.line,
+                    element.message,
+                )
+                continue
+            if isinstance(element, TypedNode):
+                if not schema.allows_node(element.type):
+                    violation_count += 1
+                    logger.warning(
+                        "violation [invalid_node_type] node id=%s — type %r not in schema",
+                        element.id,
+                        element.type,
+                    )
+                    continue
+                node_buf.append(element)
+            elif isinstance(element, TypedEdge):
+                if not schema.allows_edge(element.type):
+                    violation_count += 1
+                    logger.warning(
+                        "violation [invalid_edge_type] %s→%s — type %r not in schema",
+                        element.src_id,
+                        element.dst_id,
+                        element.type,
+                    )
+                    continue
+                edge_buf.append(element)
+
+            if len(node_buf) + len(edge_buf) >= _BATCH_SIZE:
+                store.upsert(dataset_id, node_buf, edge_buf)
+                nodes_total += len(node_buf)
+                edges_total += len(edge_buf)
+                logger.debug(
+                    "batch upserted — nodes=%d edges=%d (running total nodes=%d edges=%d)",
+                    len(node_buf),
+                    len(edge_buf),
+                    nodes_total,
+                    edges_total,
+                )
+                node_buf.clear()
+                edge_buf.clear()
+
+        if node_buf or edge_buf:
+            store.upsert(dataset_id, node_buf, edge_buf)
+            nodes_total += len(node_buf)
+            edges_total += len(edge_buf)
+
+        store.finalise_dataset(dataset_id, nodes_total, edges_total)
+    except BaseException:
+        logger.exception("ingest failed — discarding dataset_id=%s", dataset_id)
+        store.discard_dataset(dataset_id)
+        raise
+
+    logger.info(
+        "ingest complete — dataset_id=%s nodes=%d edges=%d violations=%d",
+        dataset_id,
+        nodes_total,
+        edges_total,
+        violation_count,
+    )
+    return IngestReport(
+        dataset_id=dataset_id,
+        elements_read=elements_read,
+        nodes_upserted=nodes_total,
+        edges_upserted=edges_total,
+        violation_count=violation_count,
+    )
+
+
+def run_projection(
+    projection: ProjectionStage,
+    spec: ProjectionSpec,
+    view: DatasetView,
+    params: dict[str, Any] | None = None,
+) -> ProjectedGraphHandle:
+    logger.info(
+        "projection started — spec=%s dataset_id=%s params=%s",
+        spec.name,
+        view.dataset_id,
+        params or {},
+    )
+    handle = projection.run(spec, params or {}, view)
+    logger.info("projection complete — spec=%s dataset_id=%s", spec.name, view.dataset_id)
+    return handle
+
+
+def run_export(
+    handle: ProjectedGraphHandle,
+    exporter: ExporterStage,
+    sink: BinaryIO,
+    *,
+    store: StoreStage | None = None,
+    projection: ProjectionStage | None = None,
+    schema: HINSchema | None = None,
+    dataset_id: str | None = None,
+) -> ExportReceipt:
+    """Write a projected graph. When store, projection, schema, and dataset_id
+    are all provided the receipt carries a snapshot_id; otherwise it doesn't.
+    """
+    logger.info("export started")
+    receipt = exporter.write(handle, sink)
+    if schema is not None:
+        receipt = receipt.model_copy(update={"schema_version": schema.schema_version})
+    if store and projection and schema and dataset_id:
+        receipt = receipt.model_copy(
+            update={"snapshot_id": _snapshot_id(store, projection, schema, dataset_id)}
+        )
+    logger.info(
+        "export complete — format=%s nodes=%d edges=%d snapshot=%s",
+        receipt.format,
+        receipt.node_count,
+        receipt.edge_count,
+        (receipt.snapshot_id or "")[:12] or "—",
+    )
+    return receipt
+
+
+def _snapshot_id(
+    store: StoreStage, projection: ProjectionStage, schema: HINSchema, dataset_id: str
+) -> str:
+    sf = store.fingerprint(dataset_id)
+    ef = projection.fingerprint()
+    parts = f"{sf.content_hash}|{ef.project_hash}|{schema.schema_version}|{_TOOL_VERSION}"
+    return hashlib.sha256(parts.encode("utf-8")).hexdigest()

hinge/kernel/schema/__init__.py

@@ -0,0 +1,12 @@
+from hinge.kernel.schema.hin_schema import HINSchema
+from hinge.kernel.schema.schema_violation import SchemaViolation, SchemaViolationError
+from hinge.kernel.schema.typed_edge import TypedEdge
+from hinge.kernel.schema.typed_node import TypedNode
+
+__all__ = [
+    "HINSchema",
+    "SchemaViolation",
+    "SchemaViolationError",
+    "TypedEdge",
+    "TypedNode",
+]

hinge/kernel/schema/hin_schema.py

@@ -0,0 +1,38 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+import yaml
+from pydantic import BaseModel, Field
+
+
+class HINSchema(BaseModel):
+    """The vocabulary of node and edge labels permitted at ingest.
+
+    Loaded from ``hinge/config/types.yaml`` — the single source of truth.
+    Labels are open strings; the schema simply enumerates which ones are
+    accepted. Projections may introduce labels not listed here; they only
+    bind ingest-time validation.
+    """
+
+    schema_version: int
+    node_types: set[str] = Field(default_factory=set)
+    edge_types: set[str] = Field(default_factory=set)
+    artifact_subtypes: set[str] = Field(default_factory=set)
+
+    @classmethod
+    def from_yaml(cls, path: Path) -> HINSchema:
+        raw: dict[str, Any] = yaml.safe_load(path.read_text(encoding="utf-8"))
+        return cls(
+            schema_version=int(raw["schema_version"]),
+            node_types=set(raw.get("node_types", [])),
+            edge_types=set(raw.get("edge_types", [])),
+            artifact_subtypes=set(raw.get("artifact_subtypes", [])),
+        )
+
+    def allows_node(self, label: str) -> bool:
+        return label in self.node_types
+
+    def allows_edge(self, label: str) -> bool:
+        return label in self.edge_types

hinge/kernel/schema/schema_violation.py

@@ -0,0 +1,21 @@
+from __future__ import annotations
+
+from pydantic import BaseModel
+
+
+class SchemaViolation(BaseModel):
+    code: str
+    message: str
+    source: str
+    line: int
+
+    def as_exception(self) -> SchemaViolationError:
+        return SchemaViolationError(self)
+
+
+class SchemaViolationError(Exception):
+    def __init__(self, violation: SchemaViolation) -> None:
+        super().__init__(
+            f"[{violation.code}] {violation.source}:{violation.line} — {violation.message}"
+        )
+        self.violation = violation

hinge/kernel/schema/typed_edge.py

@@ -0,0 +1,20 @@
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+
+class TypedEdge(BaseModel):
+    """A typed graph edge. ``type`` is an open string label — projections may
+    introduce new labels (e.g. ``"interacted_with"``) without touching any enum.
+    Validated against :class:`HINSchema` only at the ingest boundary; downstream
+    stages treat labels as opaque strings.
+    """
+
+    type: str
+    src_id: str
+    dst_id: str
+    timestamp: datetime | None = None
+    attrs: dict[str, Any] = Field(default_factory=dict)