okf-parser 0.1.0__py3-none-any.whl

okf_parser/__init__.py

@@ -0,0 +1,13 @@
+"""Relational inspection and validation for Open Knowledge Format bundles."""
+
+from okf_parser.bundle import Bundle, load_bundle, validate_path
+from okf_parser.models import Severity, ValidationReport, Violation
+
+__all__ = [
+    "Bundle",
+    "Severity",
+    "ValidationReport",
+    "Violation",
+    "load_bundle",
+    "validate_path",
+]

okf_parser/bundle.py

@@ -0,0 +1,368 @@
+"""Load an OKF bundle into Ibis relations and validate its structure."""
+
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import asdict, dataclass
+from datetime import date
+from typing import TYPE_CHECKING, cast
+from urllib.parse import urlsplit
+
+import ibis
+import networkx as nx
+
+from okf_parser.discovery import discover_markdown
+from okf_parser.models import (
+    ConceptRecord,
+    LinkRecord,
+    ReservedRecord,
+    Severity,
+    ValidationReport,
+    Violation,
+)
+from okf_parser.parser import (
+    DocumentParseError,
+    concept_id,
+    is_reserved_document,
+    iter_markdown_links,
+    parse_document,
+    resolve_local_target,
+    split_optional_frontmatter,
+)
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+    from pathlib import Path
+
+    from ibis.expr.types import Table
+
+_CONCEPT_SCHEMA = ibis.schema(
+    {
+        "concept_id": "string",
+        "logical_key": "string",
+        "path": "string",
+        "concept_type": "string",
+        "title": "string",
+        "description": "string",
+        "frontmatter_json": "string",
+        "body": "string",
+    }
+)
+_RESERVED_SCHEMA = ibis.schema({"path": "string", "filename": "string", "body": "string"})
+_LINK_SCHEMA = ibis.schema(
+    {
+        "source_id": "string",
+        "raw_target": "string",
+        "target_id": "string",
+        "exists": "boolean",
+        "origin": "string",
+    }
+)
+_H1_RE = re.compile(r"^# [^#\n].*$", re.MULTILINE)
+_H2_RE = re.compile(r"^## ([^#\n].*)$", re.MULTILINE)
+_ISO_DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$")
+type TableRecord = ConceptRecord | ReservedRecord | LinkRecord
+
+
+def _table(records: Sequence[TableRecord], schema: ibis.Schema) -> Table:
+    rows = [asdict(record) for record in records]
+    return ibis.memtable(rows, schema=schema)
+
+
+@dataclass(frozen=True, slots=True)
+class Bundle:
+    """An immutable relational view of one OKF bundle."""
+
+    root: Path
+    concepts: Table
+    reserved: Table
+    links: Table
+    diagnostics: tuple[Violation, ...]
+    markdown_count: int
+
+    def validate(self) -> list[Violation]:
+        """Return deterministic diagnostics ordered by path, severity, and code."""
+        return sorted(
+            self.diagnostics,
+            key=lambda item: (item.path, item.severity.value, item.code, item.message),
+        )
+
+    @property
+    def is_conformant(self) -> bool:
+        """Whether the bundle has no normative errors."""
+        return not any(item.severity is Severity.ERROR for item in self.diagnostics)
+
+    def to_networkx(self) -> nx.MultiDiGraph:
+        """Project concepts and resolved Markdown links into a directed graph."""
+        graph = nx.MultiDiGraph(bundle_root=str(self.root))
+        for row in self.concepts.execute().to_dict(orient="records"):
+            graph.add_node(
+                row["concept_id"],
+                path=row["path"],
+                type=row["concept_type"],
+                title=row["title"],
+            )
+        for row in self.links.execute().to_dict(orient="records"):
+            if row["exists"] and isinstance(row["target_id"], str):
+                graph.add_edge(
+                    row["source_id"],
+                    row["target_id"],
+                    raw_target=row["raw_target"],
+                    origin=row["origin"],
+                )
+        return graph
+
+
+def _load_concept(
+    root: Path,
+    path: Path,
+    known_paths: set[Path],
+) -> tuple[ConceptRecord | None, list[LinkRecord], list[Violation]]:
+    relative = path.relative_to(root).as_posix()
+    try:
+        parsed = parse_document(path)
+    except (DocumentParseError, OSError) as exc:
+        return (
+            None,
+            [],
+            [Violation("OKF001", Severity.ERROR, relative, str(exc))],
+        )
+
+    raw_type = parsed.frontmatter.get("type")
+    concept_type = raw_type.strip() if isinstance(raw_type, str) else ""
+    diagnostics: list[Violation] = []
+    if not concept_type:
+        diagnostics.append(
+            Violation(
+                "OKF002",
+                Severity.ERROR,
+                relative,
+                "frontmatter must contain a non-empty string type",
+            )
+        )
+
+    doc_id = concept_id(root, path)
+    links: list[LinkRecord] = []
+    raw_links = [(target, "body") for target in iter_markdown_links(parsed.body)]
+    raw_links.extend(_iter_frontmatter_links(parsed.frontmatter))
+    for raw_target, origin in raw_links:
+        resolved = resolve_local_target(root, path, raw_target)
+        is_markdown = urlsplit_path(raw_target).lower().endswith(".md")
+        if resolved is None or not is_markdown:
+            continue
+        exists = resolved in known_paths
+        target_id = (
+            concept_id(root, resolved) if exists and not is_reserved_document(resolved) else None
+        )
+        links.append(LinkRecord(doc_id, raw_target, target_id, exists, origin))
+        if not exists:
+            diagnostics.append(
+                Violation(
+                    "OKF101",
+                    Severity.WARNING,
+                    relative,
+                    f"local Markdown link does not resolve: {raw_target}",
+                )
+            )
+
+    title = parsed.frontmatter.get("title")
+    description = parsed.frontmatter.get("description")
+    record = ConceptRecord(
+        concept_id=doc_id,
+        logical_key=doc_id,
+        path=relative,
+        concept_type=concept_type,
+        title=title if isinstance(title, str) else None,
+        description=description if isinstance(description, str) else None,
+        frontmatter_json=json.dumps(
+            parsed.frontmatter,
+            ensure_ascii=False,
+            sort_keys=True,
+            default=str,
+        ),
+        body=parsed.body,
+    )
+    return record, links, diagnostics
+
+
+def _iter_frontmatter_links(
+    value: object,
+    field_path: str = "frontmatter",
+) -> list[tuple[str, str]]:
+    """Find local Markdown references nested in producer-defined frontmatter."""
+    if isinstance(value, dict):
+        return [
+            item
+            for key, child in value.items()
+            for item in _iter_frontmatter_links(child, f"{field_path}.{key}")
+        ]
+    if isinstance(value, list):
+        return [
+            item
+            for index, child in enumerate(value)
+            for item in _iter_frontmatter_links(child, f"{field_path}[{index}]")
+        ]
+    if isinstance(value, str) and urlsplit_path(value).lower().endswith(".md"):
+        return [(value, field_path)]
+    return []
+
+
+def urlsplit_path(raw_target: str) -> str:
+    """Return only a link target's URL path."""
+    return urlsplit(raw_target).path
+
+
+def _validate_index(root: Path, path: Path, text: str) -> tuple[str, list[Violation]]:
+    relative = path.relative_to(root).as_posix()
+    diagnostics: list[Violation] = []
+    try:
+        frontmatter, body = split_optional_frontmatter(text)
+    except DocumentParseError as exc:
+        return text, [Violation("OKF004", Severity.ERROR, relative, str(exc))]
+
+    if frontmatter is not None:
+        if path.parent != root:
+            diagnostics.append(
+                Violation(
+                    "OKF004",
+                    Severity.ERROR,
+                    relative,
+                    "only the bundle-root index.md may contain frontmatter",
+                )
+            )
+        elif set(frontmatter) - {"okf_version"}:
+            diagnostics.append(
+                Violation(
+                    "OKF004",
+                    Severity.ERROR,
+                    relative,
+                    "root index.md frontmatter may contain only okf_version",
+                )
+            )
+    if _H1_RE.search(body) is None:
+        diagnostics.append(
+            Violation(
+                "OKF005",
+                Severity.ERROR,
+                relative,
+                "index.md must contain at least one level-one section",
+            )
+        )
+    return body, diagnostics
+
+
+def _validate_log(root: Path, path: Path, text: str) -> tuple[str, list[Violation]]:
+    relative = path.relative_to(root).as_posix()
+    diagnostics: list[Violation] = []
+    try:
+        frontmatter, body = split_optional_frontmatter(text)
+    except DocumentParseError as exc:
+        return text, [Violation("OKF006", Severity.ERROR, relative, str(exc))]
+    if frontmatter is not None:
+        diagnostics.append(
+            Violation("OKF006", Severity.ERROR, relative, "log.md must not contain frontmatter")
+        )
+    if _H1_RE.search(body) is None:
+        diagnostics.append(
+            Violation(
+                "OKF007",
+                Severity.ERROR,
+                relative,
+                "log.md must contain a level-one title",
+            )
+        )
+
+    headings = _H2_RE.findall(body)
+    parsed_dates: list[date] = []
+    for heading in headings:
+        if _ISO_DATE_RE.fullmatch(heading) is None:
+            diagnostics.append(
+                Violation(
+                    "OKF008",
+                    Severity.ERROR,
+                    relative,
+                    f"log date heading must use YYYY-MM-DD: {heading}",
+                )
+            )
+            continue
+        try:
+            parsed_dates.append(date.fromisoformat(heading))
+        except ValueError:
+            diagnostics.append(
+                Violation(
+                    "OKF008",
+                    Severity.ERROR,
+                    relative,
+                    f"log date heading is not a real date: {heading}",
+                )
+            )
+    if parsed_dates != sorted(parsed_dates, reverse=True):
+        diagnostics.append(
+            Violation(
+                "OKF009",
+                Severity.ERROR,
+                relative,
+                "log date groups must be ordered newest first",
+            )
+        )
+    return body, diagnostics
+
+
+def load_bundle(root: Path) -> Bundle:
+    """Scan a directory and compile its OKF documents into Ibis tables."""
+    root = root.resolve()
+    if not root.is_dir():
+        msg = f"bundle root is not a directory: {root}"
+        raise NotADirectoryError(msg)
+
+    paths = discover_markdown(root)
+    known_paths = {path.resolve() for path in paths}
+    concepts: list[ConceptRecord] = []
+    reserved: list[ReservedRecord] = []
+    links: list[LinkRecord] = []
+    diagnostics: list[Violation] = []
+
+    for path in paths:
+        relative = path.relative_to(root).as_posix()
+        if is_reserved_document(path):
+            try:
+                text = path.read_text(encoding="utf-8")
+            except (UnicodeDecodeError, OSError) as exc:
+                diagnostics.append(Violation("OKF003", Severity.ERROR, relative, str(exc)))
+                continue
+            if path.name == "index.md":
+                body, reserved_diagnostics = _validate_index(root, path, text)
+            else:
+                body, reserved_diagnostics = _validate_log(root, path, text)
+            diagnostics.extend(reserved_diagnostics)
+            reserved.append(ReservedRecord(relative, path.name, body))
+            continue
+
+        record, document_links, document_diagnostics = _load_concept(root, path, known_paths)
+        if record is not None:
+            concepts.append(record)
+        links.extend(document_links)
+        diagnostics.extend(document_diagnostics)
+
+    return Bundle(
+        root=root,
+        concepts=_table(concepts, _CONCEPT_SCHEMA),
+        reserved=_table(reserved, _RESERVED_SCHEMA),
+        links=_table(links, _LINK_SCHEMA),
+        diagnostics=tuple(diagnostics),
+        markdown_count=len(paths),
+    )
+
+
+def validate_path(path: Path) -> ValidationReport:
+    """Validate every Markdown file recursively below a path as OKF v0.2."""
+    bundle = load_bundle(path)
+    violations = tuple(bundle.validate())
+    return ValidationReport(
+        root=bundle.root,
+        markdown_count=bundle.markdown_count,
+        concept_count=cast("int", bundle.concepts.count().execute()),
+        reserved_count=cast("int", bundle.reserved.count().execute()),
+        violations=violations,
+    )

okf_parser/cli.py

@@ -0,0 +1,142 @@
+"""Expose okf-parser through Cyclopts and FastMCP."""
+
+from __future__ import annotations
+
+import json
+import sys
+from dataclasses import dataclass
+from typing import Literal
+
+from cyclopts import App
+from fastmcp import FastMCP
+
+from okf_parser.service import (
+    check_bundle,
+    check_format,
+    export_duckdb,
+    graph_bundle,
+    inventory_bundle,
+    write_format,
+)
+
+type McpTransport = Literal["stdio", "http", "sse"]
+
+
+@dataclass(frozen=True, slots=True)
+class CliResult:
+    """A JSON payload and its intended process exit code."""
+
+    payload: dict[str, object]
+    exit_code: int = 0
+
+
+def _render_cli_result(result: object) -> None:
+    """Render stable JSON and preserve command-specific exit status."""
+    if not isinstance(result, CliResult):
+        return
+    sys.stdout.write(
+        json.dumps(result.payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n"
+    )
+    if result.exit_code:
+        raise SystemExit(result.exit_code)
+
+
+app = App(
+    name="okf-parser",
+    help="Validate and inspect OKF bundles with Ibis and NetworkX.",
+    result_action=_render_cli_result,
+)
+mcp = FastMCP(
+    name="okf-parser",
+    instructions=(
+        "Deterministic tools for validating and inspecting Open Knowledge Format bundles. "
+        "Formatting checks are read-only; no tool exposed here rewrites files."
+    ),
+)
+
+
+@app.command
+def check(path: str) -> CliResult:
+    """Validate every Markdown file recursively as OKF v0.2."""
+    payload = check_bundle(path)
+    return CliResult(payload, 0 if payload["conformant"] else 1)
+
+
+@app.command
+def inventory(path: str) -> CliResult:
+    """Count concepts by type using an Ibis relation."""
+    return CliResult(inventory_bundle(path))
+
+
+@app.command
+def graph(path: str) -> CliResult:
+    """Summarize the resolved concept graph with NetworkX."""
+    return CliResult(graph_bundle(path))
+
+
+@app.command(name="format")
+def format_command(path: str, *, write: bool = False) -> CliResult:
+    """Check mdformat style, writing only when --write is explicit."""
+    payload = write_format(path) if write else check_format(path)
+    return CliResult(payload, 0 if payload["clean"] or write else 1)
+
+
+@app.command(name="duckdb")
+def duckdb_command(
+    path: str,
+    database: str = "okf.duckdb",
+    schema: str = "okf",
+) -> CliResult:
+    """Materialize bundle relations into a DuckDB database."""
+    return CliResult(export_duckdb(path, database, schema))
+
+
+@app.command
+def serve(
+    transport: McpTransport = "stdio",
+    host: str = "127.0.0.1",
+    port: int = 8000,
+) -> None:
+    """Serve read-only inspection tools through MCP."""
+    if transport == "stdio":
+        mcp.run()
+        return
+    mcp.run(transport=transport, host=host, port=port)
+
+
+@mcp.tool(name="check")
+def mcp_check(path: str) -> dict[str, object]:
+    """Validate every Markdown file recursively as OKF v0.2."""
+    return check_bundle(path)
+
+
+@mcp.tool(name="inventory")
+def mcp_inventory(path: str) -> dict[str, object]:
+    """Count concepts by type."""
+    return inventory_bundle(path)
+
+
+@mcp.tool(name="graph")
+def mcp_graph(path: str) -> dict[str, object]:
+    """Summarize resolved concept relationships."""
+    return graph_bundle(path)
+
+
+@mcp.tool(name="format_check")
+def mcp_format_check(path: str) -> dict[str, object]:
+    """Check mdformat style without modifying files."""
+    return check_format(path)
+
+
+def run_mcp_stdio() -> None:
+    """Run the MCP server over stdio."""
+    mcp.run()
+
+
+def main() -> None:
+    """Run the Cyclopts command-line application."""
+    app()
+
+
+if __name__ == "__main__":
+    main()

okf_parser/discovery.py

@@ -0,0 +1,34 @@
+"""Shared, symlink-safe Markdown discovery."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+IGNORED_DIRECTORIES = frozenset(
+    {".git", ".mypy_cache", ".pytest_cache", ".ruff_cache", ".ty_cache", ".venv"}
+)
+
+
+def discover_markdown(root: Path) -> tuple[Path, ...]:
+    """Return authored Markdown files without traversing environments or symlinks."""
+    resolved_root = root.resolve()
+    if not resolved_root.is_dir():
+        msg = f"Markdown root is not a directory: {resolved_root}"
+        raise NotADirectoryError(msg)
+
+    paths: list[Path] = []
+    for directory, directory_names, filenames in resolved_root.walk(follow_symlinks=False):
+        directory_names[:] = [
+            name
+            for name in directory_names
+            if name not in IGNORED_DIRECTORIES and not (directory / name).is_symlink()
+        ]
+        paths.extend(
+            candidate
+            for name in filenames
+            if name.endswith(".md") and not (candidate := directory / name).is_symlink()
+        )
+    return tuple(sorted(paths))

okf_parser/duckdb.py

@@ -0,0 +1,87 @@
+"""Materialize an OKF bundle as ordinary DuckDB tables."""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+from typing import TYPE_CHECKING, cast
+
+import ibis
+
+from okf_parser.bundle import Bundle, load_bundle
+
+if TYPE_CHECKING:
+    import duckdb
+
+_IDENTIFIER_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
+_DIAGNOSTIC_SCHEMA = ibis.schema(
+    {
+        "code": "string",
+        "severity": "string",
+        "path": "string",
+        "message": "string",
+    }
+)
+
+
+def _validate_schema_name(schema: str) -> None:
+    if _IDENTIFIER_RE.fullmatch(schema) is None:
+        msg = f"invalid DuckDB schema name: {schema!r}"
+        raise ValueError(msg)
+
+
+def _diagnostics_table(bundle: Bundle) -> ibis.Table:
+    rows = [
+        {
+            "code": item.code,
+            "severity": item.severity.value,
+            "path": item.path,
+            "message": item.message,
+        }
+        for item in bundle.validate()
+    ]
+    return ibis.memtable(rows, schema=_DIAGNOSTIC_SCHEMA)
+
+
+def attach_okf(
+    connection: duckdb.DuckDBPyConnection,
+    path: str | Path,
+    *,
+    schema: str = "okf",
+) -> dict[str, object]:
+    """Materialize one OKF bundle into a DuckDB schema.
+
+    The function creates four ordinary tables inside ``schema``:
+    ``concepts``, ``links``, ``reserved``, and ``diagnostics``. Once copied,
+    the tables are independent of Python and remain queryable from any
+    DuckDB client that opens the database.
+    """
+    _validate_schema_name(schema)
+    bundle = load_bundle(Path(path))
+    relations = {
+        "concepts": bundle.concepts,
+        "links": bundle.links,
+        "reserved": bundle.reserved,
+        "diagnostics": _diagnostics_table(bundle),
+    }
+
+    connection.execute("BEGIN TRANSACTION")
+    try:
+        connection.execute(f'CREATE SCHEMA IF NOT EXISTS "{schema}"')
+        for table_name, relation in relations.items():
+            qualified_name = f"{schema}.{table_name}"
+            connection.from_arrow(relation.to_pyarrow()).create(qualified_name)
+        connection.execute("COMMIT")
+    except Exception:
+        connection.execute("ROLLBACK")
+        raise
+
+    return {
+        "schema": schema,
+        "root": str(bundle.root),
+        "conformant": bundle.is_conformant,
+        "markdown_count": bundle.markdown_count,
+        "concept_count": cast("int", bundle.concepts.count().execute()),
+        "link_count": cast("int", bundle.links.count().execute()),
+        "diagnostic_count": len(bundle.diagnostics),
+    }

okf_parser/formatting.py

@@ -0,0 +1,46 @@
+"""Optional Markdown formatting checks, separate from OKF conformance."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+import mdformat
+
+from okf_parser.discovery import discover_markdown
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+
+@dataclass(frozen=True, slots=True)
+class FormatReport:
+    """Result of checking or formatting a Markdown tree."""
+
+    markdown_count: int
+    changed_paths: tuple[str, ...]
+    written: bool
+
+    @property
+    def clean(self) -> bool:
+        """Whether every file was already in canonical mdformat form."""
+        return not self.changed_paths
+
+
+def format_path(path: Path, *, write: bool = False) -> FormatReport:
+    """Check or explicitly rewrite every Markdown file below a path."""
+    root = path.resolve()
+    paths = discover_markdown(root)
+    changed: list[str] = []
+    for markdown_path in paths:
+        original = markdown_path.read_text(encoding="utf-8")
+        formatted = mdformat.text(
+            original,
+            extensions={"frontmatter", "gfm"},
+        )
+        if formatted == original:
+            continue
+        changed.append(markdown_path.relative_to(root).as_posix())
+        if write:
+            markdown_path.write_text(formatted, encoding="utf-8")
+    return FormatReport(len(paths), tuple(changed), write)

okf_parser/models.py

@@ -0,0 +1,86 @@
+"""Small immutable domain records used by the parser and validator."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import StrEnum
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+
+class Severity(StrEnum):
+    """A diagnostic's effect on conformance."""
+
+    ERROR = "error"
+    WARNING = "warning"
+
+
+@dataclass(frozen=True, slots=True)
+class Violation:
+    """One deterministic bundle diagnostic."""
+
+    code: str
+    severity: Severity
+    path: str
+    message: str
+
+
+@dataclass(frozen=True, slots=True)
+class ConceptRecord:
+    """One parsed OKF concept document."""
+
+    concept_id: str
+    logical_key: str
+    path: str
+    concept_type: str
+    title: str | None
+    description: str | None
+    frontmatter_json: str
+    body: str
+
+
+@dataclass(frozen=True, slots=True)
+class ReservedRecord:
+    """One reserved index.md or log.md document."""
+
+    path: str
+    filename: str
+    body: str
+
+
+@dataclass(frozen=True, slots=True)
+class LinkRecord:
+    """One local Markdown relationship originating in a concept."""
+
+    source_id: str
+    raw_target: str
+    target_id: str | None
+    exists: bool
+    origin: str
+
+
+@dataclass(frozen=True, slots=True)
+class ParsedDocument:
+    """Result of parsing one concept without applying semantic rules."""
+
+    path: Path
+    frontmatter: dict[str, object]
+    body: str
+
+
+@dataclass(frozen=True, slots=True)
+class ValidationReport:
+    """Aggregate result for every Markdown file below one path."""
+
+    root: Path
+    markdown_count: int
+    concept_count: int
+    reserved_count: int
+    violations: tuple[Violation, ...]
+
+    @property
+    def is_conformant(self) -> bool:
+        """Whether every Markdown file satisfies its OKF v0.2 rules."""
+        return not any(item.severity is Severity.ERROR for item in self.violations)

okf_parser/parser.py

@@ -0,0 +1,123 @@
+"""Filesystem-safe parsing primitives for OKF Markdown documents."""
+
+from __future__ import annotations
+
+import re
+from typing import TYPE_CHECKING
+from urllib.parse import unquote, urlsplit
+
+import yaml
+from markdown_it import MarkdownIt
+
+from okf_parser.models import ParsedDocument
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+_FRONTMATTER_RE = re.compile(
+    r"\A---[ \t]*\r?\n(.*?)\r?\n---[ \t]*(?:\r?\n(.*))?\Z",
+    re.DOTALL,
+)
+RESERVED_FILENAMES = frozenset({"index.md", "log.md"})
+_MARKDOWN = MarkdownIt("commonmark")
+
+
+def is_reserved_document(path: Path) -> bool:
+    """Return whether a Markdown file is metadata rather than an OKF concept."""
+    return path.name in RESERVED_FILENAMES
+
+
+class DocumentParseError(ValueError):
+    """Raised when one concept document cannot be structurally parsed."""
+
+
+def parse_document(path: Path) -> ParsedDocument:
+    """Parse YAML frontmatter and preserve the Markdown body."""
+    try:
+        text = path.read_text(encoding="utf-8")
+    except UnicodeDecodeError as exc:
+        msg = "document must be valid UTF-8"
+        raise DocumentParseError(msg) from exc
+
+    match = _FRONTMATTER_RE.match(text.removeprefix("\ufeff"))
+    if match is None:
+        msg = "concept must start with YAML frontmatter delimited by ---"
+        raise DocumentParseError(msg)
+
+    try:
+        value = yaml.safe_load(match.group(1))
+    except yaml.YAMLError as exc:
+        msg = f"invalid YAML frontmatter: {exc}"
+        raise DocumentParseError(msg) from exc
+
+    if value is None:
+        frontmatter: dict[str, object] = {}
+    elif isinstance(value, dict):
+        frontmatter = value
+    else:
+        msg = "frontmatter must be a YAML mapping"
+        raise DocumentParseError(msg)
+
+    return ParsedDocument(path=path, frontmatter=frontmatter, body=match.group(2) or "")
+
+
+def split_optional_frontmatter(text: str) -> tuple[dict[str, object] | None, str]:
+    """Split optional frontmatter for reserved documents."""
+    normalized = text.removeprefix("\ufeff")
+    if not normalized.startswith("---"):
+        return None, normalized
+    match = _FRONTMATTER_RE.match(normalized)
+    if match is None:
+        msg = "invalid YAML frontmatter delimiters"
+        raise DocumentParseError(msg)
+    try:
+        value = yaml.safe_load(match.group(1))
+    except yaml.YAMLError as exc:
+        msg = f"invalid YAML frontmatter: {exc}"
+        raise DocumentParseError(msg) from exc
+    if value is None:
+        return {}, match.group(2) or ""
+    if not isinstance(value, dict):
+        msg = "frontmatter must be a YAML mapping"
+        raise DocumentParseError(msg)
+    return value, match.group(2) or ""
+
+
+def concept_id(bundle_root: Path, path: Path) -> str:
+    """Derive the normative concept ID from a bundle-relative path."""
+    return path.relative_to(bundle_root).with_suffix("").as_posix()
+
+
+def iter_markdown_links(body: str) -> list[str]:
+    """Return non-image Markdown link targets in source order."""
+    links: list[str] = []
+    pending = list(reversed(_MARKDOWN.parse(body)))
+    while pending:
+        token = pending.pop()
+        if token.children:
+            pending.extend(reversed(token.children))
+        if token.type != "link_open":
+            continue
+        destination = token.attrGet("href")
+        if isinstance(destination, str):
+            links.append(destination)
+    return links
+
+
+def resolve_local_target(bundle_root: Path, source_path: Path, raw_target: str) -> Path | None:
+    """Resolve one local Markdown target while preventing bundle escape."""
+    split = urlsplit(raw_target)
+    if split.scheme or split.netloc or not split.path:
+        return None
+
+    decoded = unquote(split.path)
+    candidate = (
+        bundle_root / decoded.lstrip("/")
+        if decoded.startswith("/")
+        else source_path.parent / decoded
+    )
+    resolved = candidate.resolve()
+    root = bundle_root.resolve()
+    if not resolved.is_relative_to(root):
+        return None
+    return resolved

okf_parser/service.py

@@ -0,0 +1,92 @@
+"""JSON-ready application services shared by CLI and MCP."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import duckdb
+import networkx as nx
+
+from okf_parser.bundle import load_bundle, validate_path
+from okf_parser.duckdb import attach_okf
+from okf_parser.formatting import format_path
+
+
+def check_bundle(path: str) -> dict[str, object]:
+    """Validate every Markdown file below a path."""
+    report = validate_path(Path(path))
+    return {
+        "root": str(report.root),
+        "conformant": report.is_conformant,
+        "markdown_count": report.markdown_count,
+        "concept_count": report.concept_count,
+        "reserved_count": report.reserved_count,
+        "diagnostics": [
+            {
+                "code": item.code,
+                "severity": item.severity.value,
+                "path": item.path,
+                "message": item.message,
+            }
+            for item in report.violations
+        ],
+    }
+
+
+def inventory_bundle(path: str) -> dict[str, object]:
+    """Count concepts by their producer-defined type."""
+    bundle = load_bundle(Path(path))
+    rows = (
+        bundle.concepts.group_by("concept_type")
+        .aggregate(concept_count=lambda table: table.count())
+        .order_by("concept_type")
+        .execute()
+        .to_dict(orient="records")
+    )
+    return {"root": str(bundle.root), "types": rows}
+
+
+def graph_bundle(path: str) -> dict[str, object]:
+    """Summarize the resolved concept graph."""
+    bundle = load_bundle(Path(path))
+    graph = bundle.to_networkx()
+    return {
+        "root": str(bundle.root),
+        "nodes": graph.number_of_nodes(),
+        "edges": graph.number_of_edges(),
+        "weakly_connected_components": nx.number_weakly_connected_components(graph),
+        "strongly_connected_components": nx.number_strongly_connected_components(graph),
+        "directed_acyclic": nx.is_directed_acyclic_graph(graph),
+    }
+
+
+def check_format(path: str) -> dict[str, object]:
+    """Check mdformat canonical form without writing files."""
+    report = format_path(Path(path))
+    return {
+        "markdown_count": report.markdown_count,
+        "clean": report.clean,
+        "changed_paths": list(report.changed_paths),
+        "written": report.written,
+    }
+
+
+def write_format(path: str) -> dict[str, object]:
+    """Explicitly rewrite Markdown files into mdformat canonical form."""
+    report = format_path(Path(path), write=True)
+    return {
+        "markdown_count": report.markdown_count,
+        "clean": report.clean,
+        "changed_paths": list(report.changed_paths),
+        "written": report.written,
+    }
+
+
+def export_duckdb(path: str, database: str, schema: str = "okf") -> dict[str, object]:
+    """Materialize an OKF bundle into a DuckDB database file."""
+    connection = duckdb.connect(database)
+    try:
+        result = attach_okf(connection, path, schema=schema)
+    finally:
+        connection.close()
+    return {**result, "database": str(Path(database).resolve())}

okf_parser-0.1.0.dist-info/METADATA

@@ -0,0 +1,165 @@
+Metadata-Version: 2.4
+Name: okf-parser
+Version: 0.1.0
+Summary: Relational inspection and validation for Open Knowledge Format bundles
+Requires-Python: >=3.12
+Requires-Dist: cyclopts<5,>=4.22
+Requires-Dist: duckdb<2,>=1.4
+Requires-Dist: fastmcp<4,>=3.4
+Requires-Dist: ibis-framework[duckdb]<13,>=12
+Requires-Dist: markdown-it-py<5,>=4
+Requires-Dist: mdformat-frontmatter<3,>=2
+Requires-Dist: mdformat-gfm<2,>=1
+Requires-Dist: mdformat<2,>=1
+Requires-Dist: networkx<4,>=3.4
+Requires-Dist: pyyaml<7,>=6
+Description-Content-Type: text/markdown
+
+---
+type: Project
+title: okf-parser
+description: Relational inspection and validation for Open Knowledge Format bundles
+---
+
+# okf-parser
+
+Relational inspection and validation for
+[Open Knowledge Format (OKF) v0.2](https://github.com/GoogleCloudPlatform/knowledge-catalog/blob/main/okf/SPEC.md)
+bundles.
+
+`okf-parser` reads an OKF bundle without imposing a domain taxonomy, preserves
+unknown frontmatter fields, and exposes concepts and links as
+[Ibis](https://ibis-project.org/) tables. This makes bundle-wide rules—identity,
+lineage, cardinality, provenance, and profile-specific constraints—expressible
+as deterministic relational checks.
+
+## Why another OKF tool?
+
+The ecosystem already has good static linters and generators, including
+`okflint`, `okf-cli`, and `google-okf`. This project focuses on a different
+layer:
+
+- compile a bundle into queryable relational tables;
+- project those same relations into a NetworkX graph;
+- validate OKF v0.2 conformance without rejecting extensions;
+- distinguish normative errors from advisory diagnostics;
+- let projects add cross-concept rules as Ibis expressions;
+- produce stable human-readable and JSON reports for CI and agents.
+
+The parser and validation model are inspired by
+[`franklinbaldo/sisprev`](https://github.com/franklinbaldo/sisprev): parse
+documents independently from semantic validation, aggregate violations instead
+of failing at the first bad concept, preserve authored bodies, and test
+filesystem identity explicitly. No Sisprev-specific legal types are copied into
+the core.
+
+[`mrorigo/rust-okf`](https://github.com/mrorigo/rust-okf) inspired the stable
+logical key, conservative metadata preservation, BOM/CRLF handling, and clean
+separation between bundle parsing and downstream query surfaces. Its BM25,
+vector index, storage format, and HTTP server are intentionally outside this
+project's scope.
+
+## Quick start
+
+```bash
+uv sync
+uv run okf-parser check path/to/bundle
+uv run okf-parser inventory path/to/bundle
+uv run okf-parser graph path/to/bundle
+uv run okf-parser format path/to/bundle
+uv run okf-parser format path/to/bundle --write
+uv run okf-parser duckdb path/to/bundle knowledge.duckdb
+```
+
+The command exits with status `1` only when normative errors exist. Broken
+cross-links are warnings because OKF v0.2 explicitly says they do not make a
+bundle non-conformant.
+
+## GitHub Actions
+
+Add the repository as a CI check:
+
+```yaml
+steps:
+  - uses: actions/checkout@v4
+  - uses: franklinbaldo/okf-parser@v1
+    with:
+      path: knowledge
+```
+
+The composite action installs a pinned uv version and executes the same
+`validate_path()` function used by the Python API, CLI, and MCP server.
+
+Releases are published to PyPI from GitHub Releases through OIDC Trusted
+Publishing. No long-lived PyPI token is stored in the repository.
+
+Every pull request must increase the SemVer version in `pyproject.toml` and add
+exactly one matching `changelog/<version>.md` entry. CI compares both against
+the target branch before allowing merge.
+
+## MCP
+
+```bash
+uv run okf-parser serve
+uv run okf-parser-mcp
+uv run fastmcp run
+```
+
+Read-only tools: `check`, `inventory`, `graph`, and `format_check`.
+
+## DuckDB
+
+`okf-parser` is a regular uv-managed Python app; the DuckDB integration is part
+of the same package, not a native C++ subproject:
+
+```python
+import duckdb
+
+from okf_parser.duckdb import attach_okf
+
+connection = duckdb.connect("knowledge.duckdb")
+attach_okf(connection, "knowledge/")
+
+connection.sql("""
+    SELECT concept_type, count(*)
+    FROM okf.concepts
+    GROUP BY concept_type
+""").show()
+```
+
+The call creates `okf.concepts`, `okf.links`, `okf.reserved`, and
+`okf.diagnostics` as ordinary DuckDB tables. Use a new database or schema for
+each materialization.
+
+## Python API
+
+```python
+from pathlib import Path
+
+from okf_parser import load_bundle, validate_path
+
+bundle = load_bundle(Path("knowledge"))
+print(bundle.concepts.execute())
+print(bundle.links.execute())
+print(bundle.to_networkx())
+print(bundle.validate())
+
+report = validate_path(Path("knowledge"))
+assert report.markdown_count == report.concept_count + report.reserved_count
+assert report.is_conformant
+```
+
+## Current scope
+
+- UTF-8 Markdown discovery;
+- reserved `index.md` and `log.md` handling;
+- strict YAML-frontmatter parsing for concept documents;
+- required non-empty `type`;
+- stable concept IDs derived from paths;
+- Markdown-link extraction and resolution;
+- Ibis tables for concepts, reserved documents, and links;
+- NetworkX graph projection for traversal, cycles, components, and impact;
+- aggregated validation reports.
+
+Profiles, lifecycle/provenance family validation, external resources, and
+pluggable Ibis rules are the next milestones.

okf_parser-0.1.0.dist-info/RECORD

@@ -0,0 +1,13 @@
+okf_parser/__init__.py,sha256=VC0sd-vtVbE5WLJG047xr8CpWkGZy7zVbMCBPLoIl_w,338
+okf_parser/bundle.py,sha256=bnTWluxKEAT4Qq677TS6FbGqaTx8U65yjmkOmoOBfAk,12057
+okf_parser/cli.py,sha256=7uXdsBLVral2OZQo4FqpI4lth_oTFFXYOnKkZb5bA6Y,3591
+okf_parser/discovery.py,sha256=TB7dlc22YJCz-yklzpAZJfVVP85zdTcx90j4P35QHm8,1115
+okf_parser/duckdb.py,sha256=hi_fm0DG85PK_lw2Y-kmDXpJKWAkoIh6rpl3YbrY40k,2510
+okf_parser/formatting.py,sha256=M0elRV-mYA7ABnScqril56sr-L25NGYzYmwSQ9jkSLI,1351
+okf_parser/models.py,sha256=YmthROgmtzKskzolbbJW0XeXS5RBi64t92zoeQVKU1c,1851
+okf_parser/parser.py,sha256=SIN5jwljBv5BAnDMiFxBJsJkX_J2jtWXCwQVo085VSk,3989
+okf_parser/service.py,sha256=xhirXIw7Ysg5QJw0VwcIyyRJ7qCieRnAvjTWcGLSwMw,2996
+okf_parser-0.1.0.dist-info/METADATA,sha256=AFhbkJjTLq3iMWCDqMpWbkUP0ku5M2aED0EPKpxMwDU,5250
+okf_parser-0.1.0.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
+okf_parser-0.1.0.dist-info/entry_points.txt,sha256=4RSMWLWiBAVQ2Bf98e6unD_mL7sP0Y8da22RXbJW89A,97
+okf_parser-0.1.0.dist-info/RECORD,,

okf_parser-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.31.0
+Root-Is-Purelib: true
+Tag: py3-none-any

okf_parser-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+okf-parser = okf_parser.cli:main
+okf-parser-mcp = okf_parser.cli:run_mcp_stdio