langchain-okf 0.2.0__py3-none-any.whl

langchain_okf/__init__.py

@@ -0,0 +1,35 @@
+"""LangChain integration for the Open Knowledge Format (OKF).
+
+Provides document loaders, retrievers, graph utilities, and agent tools
+for working with OKF knowledge bundles in LangChain pipelines.
+"""
+
+from langchain_okf.document_loader import OKFBundleLoader
+from langchain_okf.generator import OKFBundleGenerator
+from langchain_okf.graph import OKFKnowledgeGraph
+from langchain_okf.retriever import OKFRetriever
+from langchain_okf.toolkit import (
+    OKFListConceptsTool,
+    OKFReadConceptTool,
+    OKFRelatedConceptsTool,
+    OKFSearchTool,
+    OKFToolkit,
+    OKFVerifyConceptTool,
+    OKFWriteConceptTool,
+)
+from langchain_okf.validate import validate_bundle
+
+__all__ = [
+    "OKFBundleGenerator",
+    "OKFBundleLoader",
+    "OKFKnowledgeGraph",
+    "OKFRetriever",
+    "OKFListConceptsTool",
+    "OKFReadConceptTool",
+    "OKFRelatedConceptsTool",
+    "OKFSearchTool",
+    "OKFVerifyConceptTool",
+    "OKFWriteConceptTool",
+    "OKFToolkit",
+    "validate_bundle",
+]

langchain_okf/_parser.py

@@ -0,0 +1,289 @@
+"""OKF concept document parser — OKF v0.2.
+
+Aligns with OKF v0.2 specification:
+https://github.com/GoogleCloudPlatform/knowledge-catalog/blob/main/okf/SPEC.md
+
+Key v0.2 additions over v0.1:
+- ``generated`` / ``verified`` trust families (§5.2)
+- ``sources`` provenance list (§5.1)
+- ``status`` and ``stale_after`` lifecycle fields (§5.4, §5.5)
+- Actor convention for ``generated.by`` / ``verified[].by`` (§7)
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+from datetime import date, datetime, timezone
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+_FRONTMATTER_DELIM = "---"
+_RESERVED_FILENAMES = {"index.md", "log.md"}
+_CROSS_LINK_RE = re.compile(
+    r"\[([^\]]*)\]\((/[^)]+\.md|\.\./[^)]+\.md|(?!\w+://)[^)]+\.md)\)"
+)
+
+
+def _to_iso8601_str(val: Any) -> str:
+    """Normalize datetime/date or string values to ISO 8601 UTC string."""
+    if val is None or val == "":
+        return ""
+    if isinstance(val, (datetime, date)):
+        if isinstance(val, datetime) and val.tzinfo is not None:
+            return val.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+        return val.strftime("%Y-%m-%dT%H:%M:%SZ")
+    return str(val)
+
+
+class OKFParseError(ValueError):
+    """Raised when an OKF document cannot be parsed."""
+
+
+@dataclass
+class OKFConcept:
+    """A parsed OKF v0.2 concept document.
+
+    Attributes:
+        concept_id: Slash-separated path identity (e.g. ``tables/orders``).
+        frontmatter: Parsed YAML frontmatter dict.
+        body: Markdown body after frontmatter.
+        source_path: Absolute path to the source ``.md`` file.
+    """
+
+    concept_id: str
+    frontmatter: dict[str, Any] = field(default_factory=dict)
+    body: str = ""
+    source_path: str = ""
+    _cached_links: list[str] | None = field(
+        default=None, init=False, repr=False, compare=False
+    )
+
+    # ── Core fields (§4.1) ────────────────────────────────────────────────────
+
+    @property
+    def type(self) -> str:
+        return str(self.frontmatter.get("type", ""))
+
+    @property
+    def title(self) -> str:
+        return str(self.frontmatter.get("title", ""))
+
+    @property
+    def description(self) -> str:
+        return str(self.frontmatter.get("description", ""))
+
+    @property
+    def resource(self) -> str:
+        return str(self.frontmatter.get("resource", ""))
+
+    @property
+    def tags(self) -> list[str]:
+        t = self.frontmatter.get("tags", [])
+        return t if isinstance(t, list) else [t]
+
+    # ── v0.1 compat ──────────────────────────────────────────────────────────
+
+    @property
+    def timestamp(self) -> str:
+        """Legacy v0.1 field. Prefer ``generated_at``."""
+        return str(self.frontmatter.get("timestamp", ""))
+
+    # ── Trust: generated (§5.2) ───────────────────────────────────────────────
+
+    @property
+    def generated(self) -> dict[str, str]:
+        """``generated: { by, at }`` block, or empty dict."""
+        g = self.frontmatter.get("generated", {})
+        return g if isinstance(g, dict) else {}
+
+    @property
+    def generated_by(self) -> str:
+        return self.generated.get("by", "")
+
+    @property
+    def generated_at(self) -> str:
+        """ISO 8601 datetime string of last meaningful content change."""
+        return _to_iso8601_str(self.generated.get("at", self.timestamp))
+
+    # ── Trust: verified (§5.2) ────────────────────────────────────────────────
+
+    @property
+    def verified(self) -> list[dict[str, str]]:
+        """Verification events as a list of ``{ by, at }`` dicts.
+
+        Per spec §5.2, a bare mapping is treated as a one-element list.
+        """
+        v = self.frontmatter.get("verified")
+        if v is None:
+            return []
+        raw = [v] if isinstance(v, dict) else (v if isinstance(v, list) else [])
+        out = []
+        for e in raw:
+            if isinstance(e, dict):
+                item = dict(e)
+                if "at" in item:
+                    item["at"] = _to_iso8601_str(item["at"])
+                out.append(item)
+        return out
+
+    @property
+    def trust_tier(self) -> str:
+        """Derived trust tier per §5.3.
+
+        Returns:
+            ``"human-reviewed"`` if any verifier uses ``human:`` prefix.
+            ``"machine-confirmed"`` if verified by non-human actors only.
+            ``"unverified"`` if ``verified`` is absent or empty.
+        """
+        events = self.verified
+        if not events:
+            return "unverified"
+        if any(e.get("by", "").startswith("human:") for e in events):
+            return "human-reviewed"
+        return "machine-confirmed"
+
+    # ── Lifecycle: status (§5.4) ──────────────────────────────────────────────
+
+    @property
+    def status(self) -> str:
+        """``draft`` | ``stable`` | ``deprecated``. Defaults to ``stable``."""
+        return str(self.frontmatter.get("status", "stable"))
+
+    # ── Lifecycle: stale_after (§5.5) ─────────────────────────────────────────
+
+    @property
+    def stale_after(self) -> str:
+        return _to_iso8601_str(self.frontmatter.get("stale_after", ""))
+
+    @property
+    def is_stale(self) -> bool:
+        """True when ``now >= stale_after`` (§5.5)."""
+        sa = self.stale_after
+        if not sa:
+            return False
+        try:
+            dt = datetime.fromisoformat(sa.replace("Z", "+00:00"))
+            return datetime.now(timezone.utc) >= dt
+        except (ValueError, TypeError):
+            return False
+
+    # ── Provenance: sources (§5.1) ────────────────────────────────────────────
+
+    @property
+    def sources(self) -> list[dict[str, Any]]:
+        """List of source provenance entries per §5.1."""
+        s = self.frontmatter.get("sources", [])
+        return s if isinstance(s, list) else []
+
+    # ── Attested Computation fields (§10) ─────────────────────────────────────
+
+    @property
+    def runtime(self) -> str:
+        """Computation runtime (e.g. ``bigquery``, ``python``). §10."""
+        return str(self.frontmatter.get("runtime", ""))
+
+    @property
+    def parameters(self) -> list[dict[str, Any]]:
+        """Declared parameters for an Attested Computation. §10."""
+        p = self.frontmatter.get("parameters", [])
+        return p if isinstance(p, list) else []
+
+    @property
+    def executor(self) -> dict[str, Any]:
+        """Executor block ``{ resource, receipt }``. §10."""
+        e = self.frontmatter.get("executor", {})
+        return e if isinstance(e, dict) else {}
+
+    @property
+    def attester(self) -> dict[str, Any]:
+        """Attester block ``{ resource }``. §10."""
+        a = self.frontmatter.get("attester", {})
+        return a if isinstance(a, dict) else {}
+
+    # ── Graph links ───────────────────────────────────────────────────────────
+
+    def cross_links(self) -> list[str]:
+        """Extract concept IDs referenced via markdown links in the body."""
+        if self._cached_links is not None:
+            return self._cached_links
+        links: list[str] = []
+        for _, href in _CROSS_LINK_RE.findall(self.body):
+            if href.endswith(".md"):
+                href = href[:-3]
+            href = href.lstrip("./")
+            if href.startswith("/"):
+                href = href[1:]
+            if href:
+                links.append(href)
+        self._cached_links = links
+        return links
+
+
+# ── Parsing helpers ───────────────────────────────────────────────────────────
+
+
+def parse_frontmatter(text: str) -> tuple[dict[str, Any], str]:
+    """Split an OKF markdown file into frontmatter dict and body string."""
+    lines = text.splitlines()
+    if not lines or lines[0].strip() != _FRONTMATTER_DELIM:
+        return {}, text
+
+    end_idx = None
+    for i in range(1, len(lines)):
+        if lines[i].strip() == _FRONTMATTER_DELIM:
+            end_idx = i
+            break
+    if end_idx is None:
+        raise OKFParseError("Unterminated YAML frontmatter block")
+
+    fm_text = "\n".join(lines[1:end_idx])
+    try:
+        fm = yaml.safe_load(fm_text) or {}
+    except yaml.YAMLError as e:
+        raise OKFParseError(f"Invalid YAML in frontmatter: {e}") from e
+    if not isinstance(fm, dict):
+        raise OKFParseError("Frontmatter must be a YAML mapping")
+
+    body = "\n".join(lines[end_idx + 1 :])
+    if body.startswith("\n"):
+        body = body[1:]
+    return fm, body
+
+
+def is_reserved(path: Path) -> bool:
+    """Check if a filename is an OKF reserved name (index.md, log.md)."""
+    return path.name in _RESERVED_FILENAMES
+
+
+def path_to_concept_id(bundle_root: Path, path: Path) -> str:
+    """Convert a file path to a slash-separated concept ID."""
+    rel = path.relative_to(bundle_root).with_suffix("")
+    return "/".join(rel.parts)
+
+
+def parse_bundle(bundle_root: Path) -> list[OKFConcept]:
+    """Walk an OKF bundle directory and parse all concept documents.
+
+    Skips reserved files (``index.md``, ``log.md``) per the spec.
+    """
+    root = Path(bundle_root)
+    concepts: list[OKFConcept] = []
+
+    for md_file in sorted(root.rglob("*.md")):
+        if is_reserved(md_file):
+            continue
+        text = md_file.read_text(encoding="utf-8")
+        fm, body = parse_frontmatter(text)
+        concept_id = path_to_concept_id(root, md_file)
+        concepts.append(
+            OKFConcept(
+                concept_id=concept_id,
+                frontmatter=fm,
+                body=body,
+                source_path=str(md_file),
+            )
+        )
+    return concepts

langchain_okf/document_loader.py

@@ -0,0 +1,189 @@
+"""OKF Bundle Document Loader for LangChain — OKF v0.2.
+
+Loads an OKF v0.2 knowledge bundle into LangChain ``Document`` objects,
+mapping frontmatter fields to metadata and preserving concept identity.
+
+v0.2 additions: ``generated``, ``verified``, ``sources``, ``status``,
+``stale_after``, and ``trust_tier`` are mapped to Document metadata so
+retrievers can filter on them.
+
+See the `OKF spec <https://github.com/GoogleCloudPlatform/knowledge-catalog/blob/main/okf/SPEC.md>`_
+for format details.
+
+Example:
+    .. code-block:: python
+
+        from langchain_okf import OKFBundleLoader
+
+        loader = OKFBundleLoader("path/to/bundle/")
+        docs = loader.load()
+
+        # Each doc has OKF v0.2 frontmatter as metadata
+        docs[0].metadata["type"]         # "BigQuery Table"
+        docs[0].metadata["concept_id"]   # "tables/orders"
+        docs[0].metadata["tags"]         # ["sales", "orders"]
+        docs[0].metadata["trust_tier"]   # "human-reviewed"
+        docs[0].metadata["status"]       # "stable"
+        docs[0].metadata["generated_at"] # "2026-06-20T22:53:05Z"
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Iterator
+from pathlib import Path
+
+from langchain_core.document_loaders import BaseLoader
+from langchain_core.documents import Document
+
+from langchain_okf._parser import OKFConcept, parse_bundle
+
+
+class OKFBundleLoader(BaseLoader):
+    """Load an OKF v0.2 knowledge bundle as LangChain Documents.
+
+    Each concept document in the bundle becomes one ``Document``, with
+    YAML frontmatter fields mapped to ``Document.metadata``. The
+    concept's markdown body becomes ``page_content``.
+
+    v0.2 metadata fields included in every document:
+    - ``trust_tier``: ``"human-reviewed"`` | ``"machine-confirmed"`` | ``"unverified"``
+    - ``status``: ``"draft"`` | ``"stable"`` | ``"deprecated"``
+    - ``stale_after``: ISO 8601 datetime (if set)
+    - ``generated_at`` / ``generated_by``: authorship info
+    - ``sources``: list of provenance entries
+
+    Args:
+        bundle_path: Path to the OKF bundle root directory.
+        include_types: If set, only load concepts matching these types.
+        exclude_types: If set, skip concepts matching these types.
+        include_tags: If set, only load concepts having at least one
+            of these tags.
+        include_status: If set, only load concepts with this status.
+            Common values: ``"stable"``, ``"draft"``, ``"deprecated"``.
+        exclude_stale: If True (default), exclude concepts whose
+            ``stale_after`` has passed.
+
+    Example:
+        .. code-block:: python
+
+            from langchain_okf import OKFBundleLoader
+
+            loader = OKFBundleLoader(
+                "./bundles/ga4/",
+                include_status="stable",
+                exclude_stale=True,
+            )
+            docs = loader.load()
+    """
+
+    def __init__(
+        self,
+        bundle_path: str | Path,
+        *,
+        include_types: list[str] | None = None,
+        exclude_types: list[str] | None = None,
+        include_tags: list[str] | None = None,
+        include_status: str | None = None,
+        exclude_stale: bool = True,
+    ) -> None:
+        self.bundle_path = Path(bundle_path)
+        self.include_types = set(include_types) if include_types else None
+        self.exclude_types = set(exclude_types) if exclude_types else None
+        self.include_tags = set(include_tags) if include_tags else None
+        self.include_status = include_status
+        self.exclude_stale = exclude_stale
+
+    def _should_include(self, concept: OKFConcept) -> bool:
+        if self.include_types and concept.type not in self.include_types:
+            return False
+        if self.exclude_types and concept.type in self.exclude_types:
+            return False
+        if self.include_tags and not (set(concept.tags) & self.include_tags):
+            return False
+        if self.include_status and concept.status != self.include_status:
+            return False
+        return not (self.exclude_stale and concept.is_stale)
+
+    def _concept_to_document(self, concept: OKFConcept) -> Document:
+        metadata: dict = {
+            "concept_id": concept.concept_id,
+            "source": concept.source_path,
+        }
+
+        # ── Core fields ───────────────────────────────────────────────────────
+        if concept.type:
+            metadata["type"] = concept.type
+        if concept.title:
+            metadata["title"] = concept.title
+        if concept.description:
+            metadata["description"] = concept.description
+        if concept.resource:
+            metadata["resource"] = concept.resource
+        if concept.tags:
+            metadata["tags"] = concept.tags
+
+        # ── v0.2 trust ────────────────────────────────────────────────────────
+        metadata["trust_tier"] = concept.trust_tier
+        metadata["status"] = concept.status
+        if concept.stale_after:
+            metadata["stale_after"] = concept.stale_after
+        if concept.generated_at:
+            metadata["generated_at"] = concept.generated_at
+        if concept.generated_by:
+            metadata["generated_by"] = concept.generated_by
+
+        # ── v0.2 provenance ───────────────────────────────────────────────────
+        if concept.sources:
+            # Store as JSON string so vector stores (e.g. Chroma) accept it
+            metadata["sources"] = json.dumps(concept.sources, default=str)
+            first_resource = concept.sources[0].get("resource", "")
+            if first_resource:
+                metadata["primary_source"] = first_resource
+
+        # ── v0.1 compat ───────────────────────────────────────────────────────
+        if concept.timestamp:
+            metadata["timestamp"] = concept.timestamp
+
+        # ── Cross-links ───────────────────────────────────────────────────────
+        cross_links = concept.cross_links()
+        if cross_links:
+            metadata["cross_links"] = cross_links
+
+        # ── Attested Computation ──────────────────────────────────────────────
+        if concept.runtime:
+            metadata["runtime"] = concept.runtime
+
+        # ── Remaining frontmatter (extension fields) ──────────────────────────
+        skip = {
+            "type",
+            "title",
+            "description",
+            "resource",
+            "tags",
+            "generated",
+            "verified",
+            "sources",
+            "status",
+            "stale_after",
+            "timestamp",
+            "runtime",
+            "parameters",
+            "executor",
+            "attester",
+        }
+        for key, value in concept.frontmatter.items():
+            if key not in skip and key not in metadata:
+                metadata[key] = value
+
+        return Document(page_content=concept.body, metadata=metadata)
+
+    def lazy_load(self) -> Iterator[Document]:
+        """Lazily load OKF concepts as Documents.
+
+        Yields:
+            One ``Document`` per concept, with frontmatter as metadata.
+        """
+        for concept in parse_bundle(self.bundle_path):
+            if self._should_include(concept):
+                yield self._concept_to_document(concept)